00000800:00000200:25.0F:1619677445.119998:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.120001:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0F:1619677445.120001:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.120003:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.120005:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.120006:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.120007:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:25.0:1619677445.120011:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.120011:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.120012:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.120013:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:9.0:1619677445.120013:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000400:00000200:25.0:1619677445.120016:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.120018:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.120019:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.120021:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:25.0:1619677445.120025:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.120028:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:25.0:1619677445.120032:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780380] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.120033:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e3a00] (20)++
00000800:00000200:25.0:1619677445.120034:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e3a00] (21)++
00000800:00000200:25.0:1619677445.120036:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e3a00] (22)--
00000800:00000200:25.0:1619677445.120036:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.120037:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.120037:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.120051:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.120053:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.120054:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.120054:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.120058:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.120062:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.120403:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.120406:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.120407:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.120409:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.120412:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.120414:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.120415:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.120416:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.120417:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.120418:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.120422:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.120424:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.120427:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.120428:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.120429:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.120430:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.120431:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.120432:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.120434:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.120434:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.120447:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.120449:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.120450:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.120451:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.120474:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.120477:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.120478:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.120478:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.120480:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.120483:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.120651:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.120654:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.120655:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.120658:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.120661:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.120664:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:9.0:1619677445.120666:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96762566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.120668:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.120668:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.120670:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:9.0:1619677445.120680:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.120683:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:9.0:1619677445.120687:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.120687:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (21)++
00000800:00000200:9.0:1619677445.120688:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (22)++
00000800:00000200:9.0:1619677445.120690:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (23)--
00000800:00000200:9.0:1619677445.120691:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.120693:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.120693:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.121343:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.121346:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.121348:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.121349:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.121357:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.121359:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:25.0:1619677445.121360:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96762550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.121362:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.121363:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.121363:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:25.0:1619677445.121367:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.121369:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:25.0:1619677445.121372:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780380] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.121373:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e3a00] (20)++
00000800:00000200:25.0:1619677445.121373:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e3a00] (21)++
00000800:00000200:25.0:1619677445.121375:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e3a00] (22)--
00000800:00000200:25.0:1619677445.121375:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.121377:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.121378:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.121378:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.121391:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:9.0:1619677445.121393:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:9.0:1619677445.121394:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.121395:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.121445:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.121447:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.121448:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.121449:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.121451:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.121454:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.121463:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:9.0:1619677445.121465:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.121466:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.121466:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.121467:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.121468:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.122721:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.122723:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.122724:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.122726:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.122728:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.122730:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.122731:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96762529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.122732:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.122733:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.122739:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.122742:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.122743:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.122746:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.122747:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.122748:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.122749:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.122750:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.122751:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.122751:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.122881:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.122883:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.122884:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.122886:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.122890:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.122891:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:9.0:1619677445.122893:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.122894:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.122895:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.122896:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:9.0:1619677445.122899:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.122901:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:9.0:1619677445.122904:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c2502380] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.122904:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (20)++
00000800:00000200:9.0:1619677445.122905:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:9.0:1619677445.122906:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:9.0:1619677445.122907:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.122908:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.122908:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.123463:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.123466:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.123467:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.123469:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.123472:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.123473:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.123474:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.123476:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.123477:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.123478:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:25.0:1619677445.123482:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.123483:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.123486:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.123487:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.123487:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.123489:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.123490:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.123490:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.123491:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.124059:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.124061:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.124061:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000400:00000200:25.0:1619677445.124063:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:9.0:1619677445.124064:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:9.0:1619677445.124065:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.124066:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.124067:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:25.0:1619677445.124068:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.124069:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96762500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.124070:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.124071:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.124073:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:9.0:1619677445.124076:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:25.0:1619677445.124077:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.124077:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.124078:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.124079:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.124082:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.124083:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.124084:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.124085:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.124086:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.124086:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.124087:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.124121:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.124124:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.124125:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.124125:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.124128:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.124130:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.124150:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.124152:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.124153:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.124155:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.124157:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.124159:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:25.0:1619677445.124160:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.124162:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.124163:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.124165:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:25.0:1619677445.124169:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.124171:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:25.0:1619677445.124175:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780380] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.124176:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e3a00] (20)++
00000800:00000200:25.0:1619677445.124177:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e3a00] (21)++
00000800:00000200:25.0:1619677445.124178:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e3a00] (22)--
00000800:00000200:25.0:1619677445.124179:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.124180:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.124180:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.124209:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.124212:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.124212:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.124213:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.124216:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.124219:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.124532:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.124535:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.124536:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.124538:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.124541:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.124542:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:25.0:1619677445.124544:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96762485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.124546:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.124547:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.124548:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:25.0:1619677445.124552:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.124554:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:25.0:1619677445.124557:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ec00] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.124558:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91978412be00] (20)++
00000800:00000200:25.0:1619677445.124559:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91978412be00] (21)++
00000800:00000200:25.0:1619677445.124560:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91978412be00] (22)--
00000800:00000200:25.0:1619677445.124561:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.124562:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.124563:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.124570:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.124573:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.124574:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.124575:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.124578:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.124580:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:9.0:1619677445.124581:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96762484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.124582:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.124583:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.124584:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:9.0:1619677445.124588:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.124589:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:9.0:1619677445.124592:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.124593:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:9.0:1619677445.124593:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:9.0:1619677445.124595:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:9.0:1619677445.124596:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.124598:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.124599:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.124599:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.124612:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.124614:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.124615:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.124616:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.125006:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.125009:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.125010:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.125012:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.125014:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.125016:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:9.0:1619677445.125017:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.125019:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.125019:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.125020:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:9.0:1619677445.125023:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.125025:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:9.0:1619677445.125028:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.125028:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:9.0:1619677445.125029:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:9.0:1619677445.125031:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:9.0:1619677445.125031:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.125032:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:9.0:1619677445.125033:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:9.0:1619677445.125034:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.125046:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.125048:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.125049:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.125051:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.125536:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.125540:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.125540:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:9.0:1619677445.125542:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.125545:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.125547:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.125548:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.125549:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.125550:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.125551:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.125555:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.125557:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.125560:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.125560:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.125561:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.125563:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.125563:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.125564:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.125564:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.125784:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.125786:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:9.0:1619677445.125787:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:25.0:1619677445.125788:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.125790:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.125793:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.125805:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.125807:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.125808:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.125808:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.125811:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.125813:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.126030:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.126033:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.126033:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.126036:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.126039:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.126041:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.126042:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96762459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.126044:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.126045:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.126046:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.126050:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.126053:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.126060:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.126061:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.126061:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.126063:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.126064:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.126065:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.126065:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.126479:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.126482:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.126483:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.126483:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.126486:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.126490:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.126503:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.126506:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:9.0:1619677445.126506:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.126507:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.126508:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.126509:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.126723:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.126725:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.126726:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.126728:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.126731:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.126732:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:9.0:1619677445.126733:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96762447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.126734:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.126735:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.126736:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:9.0:1619677445.126739:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.126741:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:9.0:1619677445.126743:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.126744:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:9.0:1619677445.126745:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:9.0:1619677445.126747:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:9.0:1619677445.126747:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.126748:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.126748:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.126817:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.126819:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.126820:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.126821:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.126823:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.126826:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.127787:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:25.0:1619677445.127790:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:9.0:1619677445.127790:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000400:00000200:9.0:1619677445.127792:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.127795:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.127797:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.127798:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.127799:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.127800:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.127801:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.127804:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.127806:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.127808:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.127809:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.127810:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.127812:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.127813:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:9.0:1619677445.127813:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:9.0:1619677445.127814:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:25.0:1619677445.128154:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.128156:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.128157:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.128159:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.128162:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.128163:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.128165:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762426] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.128166:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.128167:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.128168:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.128172:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.128174:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.128176:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.128177:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.128178:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.128179:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.128180:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.128181:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.128182:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.128782:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.128786:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.128786:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:9.0:1619677445.128788:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.128791:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.128792:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.128793:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762416] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.128795:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.128796:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.128797:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.128800:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.128801:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.128804:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.128805:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.128805:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.128807:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.128807:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.128808:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.128808:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.129511:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.129513:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.129514:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.129516:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.129519:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.129520:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.129521:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.129523:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.129524:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.129525:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:25.0:1619677445.129528:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.129529:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.129532:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.129533:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.129533:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.129535:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.129535:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.129536:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.129537:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.130124:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.130127:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.130128:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.130130:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.130134:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.130136:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:9.0:1619677445.130138:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96762393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.130140:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.130142:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.130143:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:9.0:1619677445.130147:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.130149:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.0:1619677445.130152:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.130153:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:9.0:1619677445.130154:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:9.0:1619677445.130156:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:9.0:1619677445.130156:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.130157:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.130159:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.130159:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.130173:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.130176:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.130176:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.130177:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.130480:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.130484:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.130485:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.130486:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.130489:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.130491:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:25.0:1619677445.130492:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.130494:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.130496:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.130497:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:25.0:1619677445.130501:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.130503:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:25.0:1619677445.130506:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780380] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.130507:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e3a00] (20)++
00000800:00000200:25.0:1619677445.130507:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e3a00] (21)++
00000800:00000200:25.0:1619677445.130509:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e3a00] (22)--
00000800:00000200:25.0:1619677445.130510:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.130510:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.130512:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.130512:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.130526:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:9.0:1619677445.130528:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:9.0:1619677445.130528:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.130530:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.130560:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.130563:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.130563:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:9.0:1619677445.130565:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.130568:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.130569:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:9.0:1619677445.130571:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.130572:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.130573:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.130574:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:9.0:1619677445.130577:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.130579:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.0:1619677445.130582:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.130582:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:9.0:1619677445.130583:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:9.0:1619677445.130585:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:9.0:1619677445.130585:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.130586:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.130586:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.131122:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.131125:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.131125:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.131126:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.131128:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.131131:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.131376:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.131380:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.131381:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.131384:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.131387:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.131389:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.131390:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96762369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.0:1619677445.131396:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000400:00000200:9.0:1619677445.131396:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.0:1619677445.131397:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.131397:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.131398:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.131398:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.131403:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.131405:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.131406:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.131409:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.131409:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.131409:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.131410:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.131410:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.131412:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.131413:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.131414:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.131414:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.131583:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.131586:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.131587:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.131589:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.131592:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.131593:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:25.0:1619677445.131594:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.131596:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.131597:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.131598:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:25.0:1619677445.131603:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.131606:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:25.0:1619677445.131609:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780380] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.131610:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e3a00] (20)++
00000800:00000200:25.0:1619677445.131611:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e3a00] (21)++
00000800:00000200:25.0:1619677445.131612:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e3a00] (22)--
00000800:00000200:25.0:1619677445.131613:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.131613:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.131614:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.132087:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.132089:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.132090:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.132090:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.132093:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.132097:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.132108:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:9.0:1619677445.132115:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.132116:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.132117:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.132118:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.132119:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.132634:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.132637:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.132638:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.132638:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.132640:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.132644:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.132810:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.132813:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.132814:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.132814:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:25.0:1619677445.132816:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.132819:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.132831:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (21)++
00000800:00000200:25.0:1619677445.132834:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:9.0:1619677445.132834:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:9.0:1619677445.132835:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.132835:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983e00] (22)--
00000800:00000200:9.0:1619677445.132836:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.133453:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.133456:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.133457:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.133465:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.133468:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.133471:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.134808:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.134810:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:25.0:1619677445.134811:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:9.0:1619677445.134811:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.134815:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:9.0:1619677445.134823:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.135132:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.135134:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.135136:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.135137:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.135140:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.135142:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.135144:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.135145:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.135146:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.135147:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:25.0:1619677445.135150:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.135152:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.135154:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.135155:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.135156:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.135158:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.135158:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.135159:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.135159:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.135607:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:9.0:1619677445.135610:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:25.0:1619677445.135611:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:9.0:1619677445.135613:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.135616:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.135617:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:9.0:1619677445.135619:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.135620:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.135621:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.135622:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:9.0:1619677445.135625:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.135627:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.0:1619677445.135630:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.135630:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:9.0:1619677445.135631:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:9.0:1619677445.135633:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:9.0:1619677445.135633:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:9.0:1619677445.135634:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:9.0:1619677445.135635:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:25.0:1619677445.135833:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.135837:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.135837:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.135840:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.135844:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.135845:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:25.0:1619677445.135847:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:9.0:1619677445.135847:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.135848:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96762294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.135855:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.135856:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.135857:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:9.0:1619677445.135860:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.135862:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:9.0:1619677445.135863:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.0:1619677445.135865:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:9.0:1619677445.135865:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.135867:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96762292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.135868:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.135869:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.135870:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.135873:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780380] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.135874:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e3a00] (20)++
00000400:00000200:9.0:1619677445.135874:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.135875:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e3a00] (21)++
00000800:00000200:9.0:1619677445.135876:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.135877:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e3a00] (22)--
00000800:00000200:25.0:1619677445.135879:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.135880:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.135880:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.135881:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.135881:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.135881:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (22)++
00000800:00000200:9.0:1619677445.135884:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (23)--
00000800:00000200:9.0:1619677445.135885:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.135887:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.135887:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (23)++
00000800:00000200:9.0:1619677445.135888:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.135889:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.135901:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.135903:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.135904:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.135905:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.136643:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.136646:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.136647:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.136649:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.136652:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.136653:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.136655:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.136656:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.136657:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.136658:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.136662:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.136664:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.136668:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.136669:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.136670:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.136671:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.136672:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.136673:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.136673:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.136806:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.136808:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:25.0:1619677445.136809:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:9.0:1619677445.136810:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.136812:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.136815:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.137303:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.137305:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.137306:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.137306:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.137309:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.137312:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.137481:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.137483:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.137484:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.137486:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.137488:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.137490:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.137491:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.137492:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.137493:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.137494:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.137497:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.137499:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.137502:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.137503:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.137503:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.137505:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.137505:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.137506:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.137507:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.137703:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.137705:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.137707:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.137707:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.137709:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.137712:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.137719:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.137722:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.137722:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:9.0:1619677445.137723:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (21)++
00000800:00000200:9.0:1619677445.137724:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.137725:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.137744:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.137746:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.137747:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.137747:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.137750:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.137752:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.137862:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39fa00] (21)++
00000800:00000200:25.0:1619677445.137865:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39fa00] (22)--
00000800:00000200:9.0:1619677445.137865:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.46@o2ib3
00000400:00000200:9.0:1619677445.137867:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.46@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.137870:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.137871:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:9.0:1619677445.137872:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.137873:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.137874:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.137876:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:9.0:1619677445.137879:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.46@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.137881:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:9.0:1619677445.137882:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82900] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.137883:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983e00] (20)++
00000800:00000200:9.0:1619677445.137884:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983e00] (21)++
00000800:00000200:9.0:1619677445.137885:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983e00] (22)--
00000800:00000200:9.0:1619677445.137885:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39fa00] (21)++
00000800:00000200:9.0:1619677445.137886:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39fa00] (22)--
00000800:00000200:9.0:1619677445.137886:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39fa00] (21)--
00000800:00000200:25.0:1619677445.137899:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.137901:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.137901:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.137902:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:25.0:1619677445.137903:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.137904:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.138131:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.138134:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:9.0:1619677445.138134:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:9.0:1619677445.138135:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.138137:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:9.0:1619677445.138140:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.138149:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (21)++
00000800:00000200:25.0:1619677445.138152:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:25.0:1619677445.138153:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.138153:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.138154:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983e00] (23)--
00000800:00000200:25.0:1619677445.138154:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.138388:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.138391:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.138391:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.138392:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.138394:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.138401:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.138416:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.138419:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.138419:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.138420:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.138422:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.138425:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.138709:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.138712:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.138713:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.138715:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.138716:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.138718:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:9.0:1619677445.138719:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.138721:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.138722:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.138723:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:9.0:1619677445.138726:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.138727:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:9.0:1619677445.138730:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.138731:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:9.0:1619677445.138732:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:9.0:1619677445.138733:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:9.0:1619677445.138734:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.138734:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.138735:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.139111:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.139113:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.139114:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.139115:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.139117:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.139119:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.139133:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:9.0:1619677445.139136:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.139137:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.139137:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.139138:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.139139:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.139666:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.139669:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.139670:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.139672:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.139676:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.139677:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.139679:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.139680:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.139681:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.139682:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.139686:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.139687:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.139691:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.139691:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.139692:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.139694:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.139694:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.139696:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.139697:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.139697:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.139710:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:9.0:1619677445.139713:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:9.0:1619677445.139714:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.139715:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.140280:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.140283:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.140284:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.140287:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.140290:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.140291:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.140298:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96762217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.140300:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.140300:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.140301:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.140305:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.140306:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.140313:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.140314:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.140314:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.140316:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.140317:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.140318:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.140318:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.140502:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.140505:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.140508:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.140508:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.140512:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.140513:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.140515:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96762211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.140517:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.140518:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.140519:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.140522:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.140524:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.140527:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.140527:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.140528:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.140530:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.140530:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.140531:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.140532:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.141020:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.141022:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.141023:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.141026:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.141028:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.141031:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.141032:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96762203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.141034:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.141034:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.141036:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.141041:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.141043:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.141047:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.141048:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.141048:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.141050:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.141051:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.141051:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.141052:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.141361:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.141364:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.141365:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.141367:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.141370:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.141372:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:9.0:1619677445.141373:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.141374:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.141375:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.141376:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:9.0:1619677445.141380:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.141382:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.0:1619677445.141385:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.141386:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:9.0:1619677445.141387:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:9.0:1619677445.141388:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:9.0:1619677445.141389:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.141390:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.141391:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.141392:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.141405:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.141406:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.141407:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.141408:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.142000:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.142003:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.142004:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.142006:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.142009:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.142011:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.142013:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96762186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.142014:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.142019:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.0:1619677445.142020:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.142021:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.142021:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.142023:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.142024:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:9.0:1619677445.142025:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.142027:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.142030:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.142031:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.142032:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.142034:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.142034:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.142035:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.142035:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.142035:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.142036:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.142184:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.142187:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.142192:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.142194:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.142197:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.142198:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.142199:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96762182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.142200:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.142201:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.142202:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.142206:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.142208:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.142211:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.142212:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.142212:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.142214:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.142214:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.142215:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.142216:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.142511:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.142514:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.142515:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.142517:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.142520:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.142521:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:9.0:1619677445.142523:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.142524:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.142525:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.142526:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:9.0:1619677445.142529:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.142531:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:9.0:1619677445.142533:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c2502380] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.142534:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (20)++
00000800:00000200:9.0:1619677445.142534:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:9.0:1619677445.142536:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:9.0:1619677445.142536:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.142537:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:9.0:1619677445.142538:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:9.0:1619677445.142539:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.142551:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.142554:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.142554:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.142555:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.142737:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.142740:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:25.0:1619677445.142741:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:9.0:1619677445.142741:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.142744:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.142747:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.143147:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.143150:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.143151:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.143151:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:25.0:1619677445.143153:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.143156:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.143446:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.143448:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.143449:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.143450:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.143452:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.143454:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.143536:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.143538:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.143539:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.143540:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.143542:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.143545:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.144308:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.144312:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.144312:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:9.0:1619677445.144317:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.144320:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.144322:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.144324:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.144327:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.144328:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.144330:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.144334:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.144336:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.144339:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.144340:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.144341:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.144342:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.144348:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.144349:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.144349:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.144794:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.144797:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.144798:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.144800:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.144803:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.144804:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.144806:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.144807:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.144808:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.144809:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.144813:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.144816:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.144819:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.144820:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.144820:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.144822:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.144823:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.144824:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.144825:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.145656:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.145660:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:9.0:1619677445.145660:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:9.0:1619677445.145661:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.145663:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.145666:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.145777:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.145780:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.145781:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.145782:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.145785:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.145787:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:25.0:1619677445.145788:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.145789:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.145790:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.145791:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:25.0:1619677445.145794:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.145796:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:25.0:1619677445.145800:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ec00] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.145801:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91978412be00] (20)++
00000800:00000200:25.0:1619677445.145801:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91978412be00] (21)++
00000800:00000200:25.0:1619677445.145803:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91978412be00] (22)--
00000800:00000200:25.0:1619677445.145803:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.145804:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.145804:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.146737:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.146740:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.146740:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:9.0:1619677445.146742:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.146745:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.146746:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:9.0:1619677445.146748:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.146749:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.146750:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.146751:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:9.0:1619677445.146755:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.146756:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.0:1619677445.146759:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.146760:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:9.0:1619677445.146761:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:9.0:1619677445.146763:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:9.0:1619677445.146764:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.146764:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.146765:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.146885:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.146888:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.146889:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.146891:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.146894:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.146896:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.146899:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.146900:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.146902:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.146903:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.146907:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.146909:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.146912:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.146913:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.146914:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.146916:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.146918:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.146918:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.146919:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.147599:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.147602:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.147602:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:9.0:1619677445.147605:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.147607:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.147608:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.147610:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.147611:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.147612:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.147613:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.147616:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.147618:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.147621:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.147622:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.147622:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.147624:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.147625:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.147626:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.147627:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.147628:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.147640:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.147642:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.147643:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.147644:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.147802:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.147805:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.147806:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.147806:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.147809:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.147812:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.147822:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.147825:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.147826:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.147826:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.147827:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.147827:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.148241:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.148244:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.148245:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.148247:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.148250:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.148252:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:9.0:1619677445.148253:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.148254:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.148255:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.148256:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:9.0:1619677445.148259:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.148261:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:9.0:1619677445.148264:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.148265:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:9.0:1619677445.148265:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:9.0:1619677445.148267:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:9.0:1619677445.148267:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.148269:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:9.0:1619677445.148270:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:9.0:1619677445.148271:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.148284:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.148286:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.148286:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.148288:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.149394:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.149396:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.149397:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.149398:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.149400:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.149403:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.149446:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.149448:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.149449:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.149451:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.149454:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.149455:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.149456:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.149457:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.149459:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.149460:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.149463:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.149465:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.149467:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.149468:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.149469:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.149471:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.149471:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.149471:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.149472:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.149830:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.149830:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.149831:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:25.0:1619677445.149832:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.149833:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.149833:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.149835:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:9.0:1619677445.149835:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.149839:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.149839:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:9.0:1619677445.149840:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.149841:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.149847:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (21)++
00000800:00000200:9.0:1619677445.149850:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:25.0:1619677445.149851:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:9.0:1619677445.149851:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.149852:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983e00] (22)--
00000800:00000200:9.0:1619677445.149853:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.150042:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:25.0:1619677445.150045:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:9.0:1619677445.150046:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:25.0:1619677445.150047:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.150050:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.150051:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:25.0:1619677445.150052:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96762051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.150054:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.150055:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.150055:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:25.0:1619677445.150058:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.150059:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.150061:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f300] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.150062:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980000] (20)++
00000800:00000200:25.0:1619677445.150062:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.150063:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.150064:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:25.0:1619677445.150065:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:25.0:1619677445.150066:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:25.0:1619677445.150067:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:9.0:1619677445.150072:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.150075:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.150076:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.0:1619677445.150077:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.150077:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.150078:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.150079:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:25.0:1619677445.150080:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:25.0:1619677445.150080:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:25.0:1619677445.150081:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.0:1619677445.150265:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.150268:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.150269:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.150271:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.150273:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.150275:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.150277:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96762046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.150278:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.150279:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.150280:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.150284:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.150286:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.150290:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.150290:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.150291:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.150293:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.150300:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.150300:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.150301:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.150870:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.150873:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.150874:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.150875:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.150877:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.150880:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.150886:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.150889:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.150889:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:9.0:1619677445.150890:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.150891:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.150891:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.151409:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.151412:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.151413:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.151413:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.151415:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.151417:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.152174:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.152177:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.152178:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.152179:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.152181:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.152184:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.152208:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.152211:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.152212:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.152215:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.152218:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.152219:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.152221:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96762006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.152222:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.152227:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.152228:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.152231:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.152234:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.152237:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.152237:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.152238:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.152240:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.152240:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.152241:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.152242:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.152315:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.152318:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.152319:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.152321:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.152324:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.152325:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:9.0:1619677445.152327:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96762001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.152329:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.152330:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.152332:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:9.0:1619677445.152338:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.152341:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.0:1619677445.152344:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.152345:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:9.0:1619677445.152346:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:9.0:1619677445.152348:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:9.0:1619677445.152349:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.152350:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.152351:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.152441:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.152443:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.152444:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.152444:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:25.0:1619677445.152446:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.152449:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.153357:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.153360:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.153361:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.153363:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.153366:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.153368:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.153369:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.153370:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.153371:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.153373:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.153376:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.153378:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.153381:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.153381:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.153382:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.153384:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.153384:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.153385:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.153386:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.153639:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.153641:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.153642:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.153643:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.153645:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.153648:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.153882:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.153885:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:25.0:1619677445.153886:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:9.0:1619677445.153886:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.153888:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.153891:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.154970:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.154973:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.154974:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.154974:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:25.0:1619677445.154976:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.154978:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.155009:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.155011:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:25.0:1619677445.155012:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (23)--
00000400:00000200:9.0:1619677445.155012:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.155014:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.155017:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.155029:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (21)++
00000800:00000200:25.0:1619677445.155031:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:25.0:1619677445.155032:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.155032:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.155033:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.155034:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.155206:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.155209:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:9.0:1619677445.155212:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:25.0:1619677445.155214:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.155215:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.155217:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:9.0:1619677445.155218:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96761950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.155219:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.155220:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.155221:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:9.0:1619677445.155224:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.155227:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:9.0:1619677445.155229:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1200] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.155229:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38000] (20)++
00000800:00000200:9.0:1619677445.155230:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38000] (21)++
00000800:00000200:9.0:1619677445.155232:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38000] (22)--
00000800:00000200:9.0:1619677445.155232:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.155233:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.155234:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.155504:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.155508:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.155509:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.155510:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.155513:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.155514:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.155516:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.155517:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.155518:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.155519:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.155523:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.155524:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.155527:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.155528:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.155528:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.155530:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.155531:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.155532:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.155534:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.155535:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.155547:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.155549:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.155550:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.155551:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.155910:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.155913:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.155914:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.155914:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.155916:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.155919:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.155928:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.155932:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.155932:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:9.0:1619677445.155934:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (21)++
00000800:00000200:9.0:1619677445.155934:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.155935:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.155996:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.155999:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.156000:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.156000:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.156002:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.156004:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.156049:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.156051:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:25.0:1619677445.156052:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:9.0:1619677445.156052:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.156054:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.156057:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.156476:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.156479:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.156480:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.156481:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.156484:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.156486:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:25.0:1619677445.156487:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.156489:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.156490:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.156491:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:25.0:1619677445.156494:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.156496:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:25.0:1619677445.156498:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c2502380] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.156499:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (20)++
00000800:00000200:25.0:1619677445.156499:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:25.0:1619677445.156501:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.156502:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.156503:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.156504:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.156505:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.156517:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:9.0:1619677445.156519:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:9.0:1619677445.156520:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.156521:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.156855:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.156858:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.156859:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.156861:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.156863:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.156865:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.156866:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.156868:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.156869:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.156870:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.156873:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.156874:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.156877:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.156878:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.156878:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.156880:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.156881:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.156881:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.156882:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.156929:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.156932:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.156933:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.156933:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.156935:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.156939:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.157005:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.157008:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.157008:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:9.0:1619677445.157012:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.157015:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.157017:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:9.0:1619677445.157019:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761916] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.157021:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.157023:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.157024:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:9.0:1619677445.157029:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.157031:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:9.0:1619677445.157034:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c2502380] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.157035:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (20)++
00000800:00000200:9.0:1619677445.157035:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:9.0:1619677445.157037:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:9.0:1619677445.157037:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.157044:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.157044:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.157402:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.157406:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.157406:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.157407:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.157410:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.157414:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.157551:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.157554:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.157555:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.157558:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.157560:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.157562:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:9.0:1619677445.157563:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96761907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.157564:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.157566:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.157567:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:9.0:1619677445.157572:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.157573:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:9.0:1619677445.157576:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.157577:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:9.0:1619677445.157578:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:9.0:1619677445.157580:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:9.0:1619677445.157580:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.157581:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.157581:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.157692:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.157695:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.157696:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.157698:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.157700:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.157702:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.157703:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.157704:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.157705:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.157706:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:25.0:1619677445.157709:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.157710:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.157713:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.157714:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.157714:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.157716:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.157717:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.157717:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.157718:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.158288:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.158291:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.158292:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.158299:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.158303:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.158305:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:9.0:1619677445.158306:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96761896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.158308:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.158309:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.158310:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:9.0:1619677445.158314:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.158316:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:9.0:1619677445.158319:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.158320:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:9.0:1619677445.158320:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:9.0:1619677445.158322:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:9.0:1619677445.158323:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.158323:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.158324:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.158963:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.158966:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.158967:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.158967:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.158970:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.158973:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.158982:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.158986:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.158986:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:9.0:1619677445.158987:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (21)++
00000800:00000200:9.0:1619677445.158988:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.158988:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.159064:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.159067:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.159067:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.159069:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.159072:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.159073:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.159074:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.159075:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.159076:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.159077:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.159081:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.159082:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.159085:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.159086:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.159086:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.159088:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.159089:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.159090:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.159091:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.159092:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.159104:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:9.0:1619677445.159106:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:9.0:1619677445.159107:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.159108:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.159716:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.159719:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.159720:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.159720:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.159722:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.159725:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.159746:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.159748:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.159749:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.159751:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.159753:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.159755:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.159756:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.159757:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.159758:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.159759:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.159763:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.159765:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.159768:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.159768:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.159769:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.159771:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.159772:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.159772:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.159773:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.160474:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.160477:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.160478:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.160480:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.160483:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.160485:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.160486:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.160487:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.160488:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.160489:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:25.0:1619677445.160492:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.160494:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.160497:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.160498:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.160498:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.160500:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.160501:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.160501:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.160502:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.160706:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.160710:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.160710:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:9.0:1619677445.160716:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.160719:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.160721:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:9.0:1619677445.160723:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96761855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.160725:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.160729:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.0:1619677445.160731:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000400:00000200:9.0:1619677445.160731:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:25.0:1619677445.160733:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.160734:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.160736:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:9.0:1619677445.160740:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.160742:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.0:1619677445.160743:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.160744:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.160745:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.160746:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:9.0:1619677445.160747:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:9.0:1619677445.160749:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:9.0:1619677445.160750:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.160751:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.160752:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.160753:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.160765:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.160767:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.160768:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.160770:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.160993:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.160996:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.160997:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.160999:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.161002:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.161003:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:9.0:1619677445.161005:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.161006:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.161007:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.161008:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:9.0:1619677445.161012:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.161013:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.0:1619677445.161016:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.161017:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:9.0:1619677445.161018:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:9.0:1619677445.161019:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:9.0:1619677445.161020:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.161020:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.161021:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.161404:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.161408:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.161409:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.161409:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.161411:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.161415:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.161449:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.161452:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.161453:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.161455:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.161458:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.161459:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.161460:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.161461:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.161463:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.161464:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.161468:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.161470:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.161473:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.161474:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.161475:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.161477:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.161477:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.161478:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.161479:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.161828:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.161831:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.161831:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.161832:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.161834:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.161837:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.162394:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.162397:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.162397:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:9.0:1619677445.162399:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.162402:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.162404:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.162405:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.162407:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.162408:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.162409:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.162412:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.162414:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.162417:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.162418:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.162419:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.162420:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.162421:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.162422:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.162422:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.162493:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.162496:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.162496:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.162497:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.162499:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.162502:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.162509:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.162512:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.162512:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:9.0:1619677445.162512:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.162513:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.162514:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.162912:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.162914:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.162915:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.162919:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.162922:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.162924:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.162925:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96761815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.162928:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.162929:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.162930:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.162935:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.162938:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.162941:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.162941:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.162942:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.162950:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.162951:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.162953:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.162953:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.163328:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.163331:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:25.0:1619677445.163332:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:9.0:1619677445.163332:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.163335:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.163338:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.163463:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.163466:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.163466:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.163467:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.163470:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.163473:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.163504:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.163506:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.163507:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.163509:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.163510:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.163512:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.163512:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:25.0:1619677445.163516:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.163519:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.163521:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96761795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.163523:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.163524:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.163525:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:9.0:1619677445.163527:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.163529:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.163530:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000400:00000200:25.0:1619677445.163535:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.163538:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.163542:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.163543:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.163543:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.163545:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.163546:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.163546:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.163547:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.164753:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.164756:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.164757:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.164757:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.164759:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.164762:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.165166:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.165169:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.165169:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.165171:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.165174:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.165176:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.165177:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.165179:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.165179:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.165181:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.165184:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.165186:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.165189:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.165190:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.165191:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.165192:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.165193:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.165194:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.165195:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.165196:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.165208:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.165210:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.165211:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.165212:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.165381:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.165384:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.165385:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.165387:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.165390:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.165391:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:25.0:1619677445.165392:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.165394:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.165394:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.165395:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:25.0:1619677445.165399:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.165401:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:25.0:1619677445.165403:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c2502380] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.165404:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (20)++
00000800:00000200:25.0:1619677445.165404:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:25.0:1619677445.165406:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.165406:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.165407:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.165409:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.165409:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.165422:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:9.0:1619677445.165424:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:9.0:1619677445.165425:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.165426:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.166091:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.166094:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.166095:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.166097:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.166099:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.166101:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:9.0:1619677445.166102:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.166103:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.166104:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.166105:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:9.0:1619677445.166108:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.166110:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:9.0:1619677445.166113:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.166114:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:9.0:1619677445.166114:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:9.0:1619677445.166116:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:9.0:1619677445.166117:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.166117:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.166118:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.166354:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.166356:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.166358:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.166360:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.166360:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:9.0:1619677445.166361:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:25.0:1619677445.166362:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.166363:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:25.0:1619677445.166364:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:9.0:1619677445.166366:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.166369:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.0:1619677445.166374:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (21)++
00000400:00000200:9.0:1619677445.166375:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:25.0:1619677445.166376:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000400:00000200:9.0:1619677445.166376:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.0:1619677445.166377:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.166378:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983e00] (23)--
00000400:00000200:9.0:1619677445.166378:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.0:1619677445.166379:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.166380:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000400:00000200:9.0:1619677445.166380:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.166381:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:9.0:1619677445.166387:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.166391:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:9.0:1619677445.166394:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.166395:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:9.0:1619677445.166395:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:9.0:1619677445.166397:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:9.0:1619677445.166398:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:9.0:1619677445.166399:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:9.0:1619677445.166399:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:25.0:1619677445.166406:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.166408:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.166409:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.166409:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:25.0:1619677445.166411:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:25.0:1619677445.166414:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.166831:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.166834:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:9.0:1619677445.166837:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:25.0:1619677445.166840:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.166840:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.166842:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:9.0:1619677445.166843:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96761734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.166844:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.166845:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.166846:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:9.0:1619677445.166850:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.166852:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:9.0:1619677445.166855:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.166856:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:9.0:1619677445.166857:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:9.0:1619677445.166859:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:9.0:1619677445.166859:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.166860:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.166860:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.167091:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.167094:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.167103:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.167105:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.167107:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.167109:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.167110:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96761729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.167111:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.167112:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.167113:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.167122:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.167125:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.167128:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.167129:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.167129:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.167131:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.167132:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.167133:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.167134:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.167595:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.167598:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.167599:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.167601:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.167604:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.167605:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:9.0:1619677445.167606:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96761720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.167607:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.167608:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.167610:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:9.0:1619677445.167614:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.167616:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:9.0:1619677445.167619:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.167620:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:9.0:1619677445.167621:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:9.0:1619677445.167622:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:9.0:1619677445.167623:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.167624:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.167624:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.167768:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.167771:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.167772:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.167773:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.167781:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.167782:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:25.0:1619677445.167784:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96761717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.167786:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.167787:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.167788:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:25.0:1619677445.167791:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.167794:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.0:1619677445.167797:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.167798:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:25.0:1619677445.167799:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:25.0:1619677445.167801:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:25.0:1619677445.167801:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.167802:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.167803:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.168216:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.168220:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.168220:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:9.0:1619677445.168223:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.168226:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.168227:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:9.0:1619677445.168229:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96761707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.168231:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.168233:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.168234:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:9.0:1619677445.168238:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.168241:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.0:1619677445.168244:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.168245:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:9.0:1619677445.168246:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:9.0:1619677445.168255:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:9.0:1619677445.168255:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.168256:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.168257:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.168601:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:25.0:1619677445.168604:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:9.0:1619677445.168605:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:25.0:1619677445.168607:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.168610:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.168612:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:25.0:1619677445.168613:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.168615:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.168616:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.168617:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:25.0:1619677445.168623:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.168626:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:25.0:1619677445.168628:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8680] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.168629:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71ba00] (20)++
00000800:00000200:25.0:1619677445.168630:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71ba00] (21)++
00000800:00000200:25.0:1619677445.168632:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71ba00] (22)--
00000800:00000200:25.0:1619677445.168632:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:25.0:1619677445.168634:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed9d3400] (22)++
00000800:00000200:25.0:1619677445.168635:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (23)--
00000800:00000200:25.0:1619677445.168636:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:9.0:1619677445.168648:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)++
00000800:00000200:25.0:1619677445.168652:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (23)--
00000800:00000200:9.0:1619677445.168653:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed9d3400] (22)--
00000800:00000200:25.0:1619677445.168654:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.168655:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.168655:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:9.0:1619677445.168656:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.168657:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.168659:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.168662:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.168679:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.168682:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.168682:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:9.0:1619677445.168683:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.168684:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.168684:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.169060:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.169063:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.169064:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.169066:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.169069:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.169070:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.169072:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.169073:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.169074:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.169075:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.169079:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.169080:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.169083:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.169084:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.169084:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.169086:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.169087:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.169088:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.169089:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.169090:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.169102:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.169104:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.169105:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.169106:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.169189:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.169192:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.169192:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.169193:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.169195:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.169199:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.169443:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.169446:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.169447:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.169447:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.169449:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.169452:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.169754:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.169756:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.169757:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.169758:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.169760:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.169767:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.170127:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.170130:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:9.0:1619677445.170130:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:9.0:1619677445.170131:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.170133:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.170136:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.170348:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.170351:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.170352:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.170354:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.170356:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.170358:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.170359:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.170361:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.170362:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.170363:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.170366:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.170368:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.170370:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.170371:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.170372:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.170374:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.170374:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.170375:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.170375:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.170698:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.170701:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.170701:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:9.0:1619677445.170704:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.170707:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.170708:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.170709:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761667] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.170710:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.170712:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.170712:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.170716:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.170718:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.170720:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.170721:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.170722:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.170723:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.170724:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.170726:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:9.0:1619677445.170727:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:9.0:1619677445.170727:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.170740:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.170741:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.170742:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.170744:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.171229:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.171232:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.171233:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.171235:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.171238:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.171239:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:9.0:1619677445.171241:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.171242:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.171243:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.171244:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:9.0:1619677445.171247:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.171249:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:9.0:1619677445.171252:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.171252:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:9.0:1619677445.171253:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:9.0:1619677445.171255:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:9.0:1619677445.171255:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.171256:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.171256:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.171525:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.171528:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.171529:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.171529:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:25.0:1619677445.171531:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.171534:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.171782:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.171785:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.171786:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.171787:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.171790:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.171791:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.171793:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.171794:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.171796:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.171798:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.171801:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.171803:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.171807:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.171807:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.171808:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.171810:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.171810:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.171811:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.171811:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.171835:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.171837:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.171838:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.171841:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.171844:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.171846:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.171847:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.171849:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.171850:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.171851:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.171855:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.171858:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.171860:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.171861:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.171862:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.171863:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.171864:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.171865:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.171865:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.172161:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.172164:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:25.0:1619677445.172165:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:9.0:1619677445.172165:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.172167:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:9.0:1619677445.172170:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.172178:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (21)++
00000800:00000200:25.0:1619677445.172180:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:25.0:1619677445.172181:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.172181:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.172182:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.172183:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.172466:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.172470:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.172471:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.172472:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.172475:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.172476:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:9.0:1619677445.172477:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.172479:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.172480:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.172481:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:9.0:1619677445.172484:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.172485:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:9.0:1619677445.172488:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.172488:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:9.0:1619677445.172489:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:9.0:1619677445.172491:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:9.0:1619677445.172491:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.172493:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.172494:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.172494:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.172507:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.172509:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.172510:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.172511:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.173133:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.173135:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.173136:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.173139:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:9.0:1619677445.173139:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.173142:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.173143:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.173145:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:9.0:1619677445.173147:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96761623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.173149:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.173150:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.173152:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.173154:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.173155:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:9.0:1619677445.173156:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:25.0:1619677445.173157:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96761622] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.0:1619677445.173159:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.173164:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.173165:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.173166:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:9.0:1619677445.173170:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:25.0:1619677445.173171:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.173171:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:9.0:1619677445.173172:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.173173:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:9.0:1619677445.173174:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:9.0:1619677445.173175:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.173176:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.173176:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:9.0:1619677445.173176:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.173177:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:25.0:1619677445.173178:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:9.0:1619677445.173183:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.173184:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.173185:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:25.0:1619677445.173186:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.173187:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.173187:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.173728:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.173730:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.173731:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.173735:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.173739:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.173740:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:9.0:1619677445.173741:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96761614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.173743:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.173744:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.173744:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:9.0:1619677445.173749:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.173751:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.0:1619677445.173755:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.173756:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:9.0:1619677445.173756:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:9.0:1619677445.173758:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:9.0:1619677445.173759:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.173759:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.173760:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.174022:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.174025:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.174026:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.174029:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.174032:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.174033:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.174036:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.174037:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.174040:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.174042:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:25.0:1619677445.174046:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.174048:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.174051:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.174051:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.174052:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.174054:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.174055:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.174056:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.174056:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.174709:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.174713:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.174714:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.174716:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.174718:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.174720:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:9.0:1619677445.174721:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.174723:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.174724:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.174725:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:9.0:1619677445.174728:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.174729:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.0:1619677445.174732:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.174733:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:9.0:1619677445.174733:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:9.0:1619677445.174735:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:9.0:1619677445.174736:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.174737:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:9.0:1619677445.174738:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:9.0:1619677445.174739:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.174751:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.174754:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.174755:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.174755:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.174881:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (23)++
00000800:00000200:25.0:1619677445.174884:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (24)--
00000400:00000200:25.0:1619677445.174885:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.174885:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (23)++
00000400:00000200:25.0:1619677445.174888:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.174888:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (24)--
00000400:00000200:9.0:1619677445.174888:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.174890:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.174892:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.174893:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.174893:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.174904:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.174906:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.174907:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.174907:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.174908:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.174908:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.175282:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.175285:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.175286:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.175286:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.175289:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.175300:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.175629:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.175632:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.175633:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.175635:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.175638:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.175639:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:25.0:1619677445.175641:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.175642:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.175644:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.175646:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:25.0:1619677445.175649:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.175652:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:25.0:1619677445.175655:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ec00] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.175655:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91978412be00] (20)++
00000800:00000200:25.0:1619677445.175656:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91978412be00] (21)++
00000800:00000200:25.0:1619677445.175662:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91978412be00] (22)--
00000800:00000200:25.0:1619677445.175663:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.175664:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.175664:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.176056:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.176059:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.176060:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.176062:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.176065:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.176066:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.176068:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.176069:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.176070:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.176071:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.176075:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.176076:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.176079:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.176080:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.176080:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.176082:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.176083:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.176084:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.176084:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.176356:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.176359:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.176360:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.176361:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.176363:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.176366:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.176845:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.176848:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.176849:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.176851:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.176854:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.176856:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.176857:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96761559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.176858:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.176859:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.176860:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.176864:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.176866:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.176869:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.176870:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.176870:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.176872:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.176873:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.176874:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.176876:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.176876:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.176888:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.176890:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.176891:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.176892:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.177367:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.177371:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.177371:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:9.0:1619677445.177374:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.177377:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.177379:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:9.0:1619677445.177381:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96761551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.177382:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.177383:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.177384:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:9.0:1619677445.177388:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.177391:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:9.0:1619677445.177394:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.177395:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:9.0:1619677445.177396:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:9.0:1619677445.177398:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:9.0:1619677445.177398:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.177399:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.177400:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.177499:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.177502:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.177503:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.177505:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.177508:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.177509:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:25.0:1619677445.177510:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.177512:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.177513:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.177514:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:25.0:1619677445.177517:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.177519:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.0:1619677445.177522:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.177523:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:25.0:1619677445.177524:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:25.0:1619677445.177525:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:25.0:1619677445.177526:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.177527:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.177527:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.177903:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.177906:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.177906:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:9.0:1619677445.177910:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.177913:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.177915:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:9.0:1619677445.177917:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96761542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.177919:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.177920:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.177922:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:9.0:1619677445.177926:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.177928:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:9.0:1619677445.177931:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c2502380] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.177932:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:9.0:1619677445.177933:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:9.0:1619677445.177935:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (24)--
00000800:00000200:9.0:1619677445.177936:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.177937:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.177937:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.178115:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.178118:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.178119:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.178123:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.178126:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.178127:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.178128:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96761538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.178130:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.178131:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.178132:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.178138:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.178140:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.178143:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.178144:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.178145:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.178146:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.178147:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.178147:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.178148:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.178165:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.178167:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:9.0:1619677445.178168:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.0:1619677445.178169:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:9.0:1619677445.178170:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:9.0:1619677445.178174:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.178330:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.178333:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:9.0:1619677445.178333:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.178334:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.178336:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.178340:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.178785:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.178788:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.178789:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.178791:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.178794:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.178795:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:9.0:1619677445.178796:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.178798:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.178799:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.178800:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:9.0:1619677445.178803:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.178805:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:9.0:1619677445.178808:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.178809:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:9.0:1619677445.178809:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:9.0:1619677445.178811:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:9.0:1619677445.178812:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.178813:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.178813:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.179138:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.179141:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.179142:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.179142:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:25.0:1619677445.179144:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.179146:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.179417:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:9.0:1619677445.179420:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:25.0:1619677445.179421:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:9.0:1619677445.179423:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.179426:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.179428:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.179429:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.179430:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.179431:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.179432:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.179436:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.179438:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 504 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.179440:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.179441:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.179442:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.179443:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.179444:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:9.0:1619677445.179444:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:9.0:1619677445.179445:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:25.0:1619677445.179669:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.179672:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.179673:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.179673:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.179676:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.179679:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.179695:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:9.0:1619677445.179698:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.179699:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.179699:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.179700:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.179700:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.179940:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.179944:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.179944:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.179948:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.179951:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.179953:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.179954:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96761505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.179956:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.179957:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.179959:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.179963:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.179970:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.179977:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.179977:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.179978:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.179980:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.179981:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.179982:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.179983:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.179988:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.179996:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.179998:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.179999:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.180000:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.180693:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.180696:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.180696:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.180697:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.180700:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.180703:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.181107:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.181109:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.181110:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.181111:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.181114:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.181117:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.181652:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.181654:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.181656:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.181657:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.181660:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.181662:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.181663:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96761477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.181665:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.181666:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.181667:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.181670:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.181671:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.181675:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.181676:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.181676:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.181678:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.181678:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.181679:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.181680:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.181923:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.181926:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.181927:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.181927:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.181929:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.181931:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.182213:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.182216:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.182217:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.182219:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.182221:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.182223:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:9.0:1619677445.182225:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96761467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.182226:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.182227:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.182228:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:9.0:1619677445.182234:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.182236:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:9.0:1619677445.182239:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.182240:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:9.0:1619677445.182240:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:9.0:1619677445.182242:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:9.0:1619677445.182243:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.182244:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:9.0:1619677445.182245:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:9.0:1619677445.182246:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.182259:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.182261:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.182262:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.182262:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.182810:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.182813:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.182814:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.182816:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.182819:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.182821:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:25.0:1619677445.182822:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.182824:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.182825:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.182826:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:25.0:1619677445.182830:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.182832:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.0:1619677445.182834:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.182835:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:25.0:1619677445.182836:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:25.0:1619677445.182838:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:25.0:1619677445.182838:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.182846:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.182846:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.183208:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.183211:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.183212:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.183214:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.183217:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.183219:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:9.0:1619677445.183220:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.183222:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.183223:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.183224:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:9.0:1619677445.183227:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.183229:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.0:1619677445.183232:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.183232:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:9.0:1619677445.183233:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:9.0:1619677445.183235:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:9.0:1619677445.183235:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.183236:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.183236:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.183749:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.183752:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.183753:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.183755:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.183758:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.183759:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.183761:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.183762:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.183763:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.183764:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.183767:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.183769:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.183772:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.183772:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.183773:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.183775:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.183775:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.183776:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.183776:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.183938:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.183942:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.183945:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.183945:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.183948:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.183950:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.183951:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96761439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.183952:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.183953:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.183954:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.183958:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.183959:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.183964:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.183965:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.183965:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.183967:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.183968:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.183968:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.183969:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.184457:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.184459:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.184460:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.184462:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.184465:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.184466:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:25.0:1619677445.184467:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96761430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.184468:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.184469:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.184470:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:25.0:1619677445.184473:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.184476:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.0:1619677445.184479:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.184479:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:25.0:1619677445.184480:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:25.0:1619677445.184482:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:25.0:1619677445.184482:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.184483:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.184485:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.184485:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.184497:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.184499:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.184500:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.184501:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.184592:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.184595:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.184596:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.184596:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.184598:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.184601:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.184610:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:9.0:1619677445.184612:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:9.0:1619677445.184613:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.184614:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.184614:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.184615:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.184942:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.184945:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:9.0:1619677445.184945:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.184946:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.184948:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.184951:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.184960:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (21)++
00000800:00000200:9.0:1619677445.184963:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:25.0:1619677445.184964:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:9.0:1619677445.184964:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.184965:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983e00] (22)--
00000800:00000200:9.0:1619677445.184966:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.185125:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.185128:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.185129:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.185130:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.185133:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.185134:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:25.0:1619677445.185136:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.185137:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.185138:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.185139:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:25.0:1619677445.185144:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.185147:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.0:1619677445.185151:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.185151:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:25.0:1619677445.185152:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:25.0:1619677445.185154:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:25.0:1619677445.185154:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.185155:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.185155:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.185541:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.185544:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.185544:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.185545:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.185547:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.185550:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.186580:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.186583:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.186583:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.186585:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.186588:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.186590:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.186591:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.186592:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.186593:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.186594:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.186598:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.186600:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.186603:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.186604:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.186604:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.186606:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.186607:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.186607:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.186608:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.187586:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.187590:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.187591:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.187591:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.187593:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.187595:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.187744:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.187746:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.187747:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.187749:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.187752:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.187754:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.187755:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.187756:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.187758:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.187758:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:25.0:1619677445.187762:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.187763:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.187766:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.187767:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.187768:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.187769:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.187770:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.187770:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.187771:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.188282:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.188285:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.188286:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.188288:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.188291:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.188292:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:9.0:1619677445.188299:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.188300:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.188301:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.188302:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:9.0:1619677445.188305:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.188306:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:9.0:1619677445.188309:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.188310:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:9.0:1619677445.188310:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:9.0:1619677445.188312:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:9.0:1619677445.188313:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.188314:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:9.0:1619677445.188316:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:9.0:1619677445.188316:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.188330:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.188332:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.188333:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.188334:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.188502:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.188505:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.188506:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.188509:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.188512:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.188513:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:9.0:1619677445.188514:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.188516:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.188517:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.188518:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:9.0:1619677445.188521:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.188522:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:9.0:1619677445.188525:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.188526:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:9.0:1619677445.188527:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:9.0:1619677445.188528:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:9.0:1619677445.188529:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.188530:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.188530:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.189005:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.189008:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.189009:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.189009:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:25.0:1619677445.189016:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:25.0:1619677445.189019:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.189469:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.189471:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.189472:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.189474:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.189477:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.189478:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:9.0:1619677445.189479:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.189481:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.189482:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.189482:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:9.0:1619677445.189486:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.189487:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.0:1619677445.189491:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.189492:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.189493:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.189493:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.189494:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.189495:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:9.0:1619677445.189496:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:25.0:1619677445.189497:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:9.0:1619677445.189497:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.189498:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.189499:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.189500:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.189503:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.189505:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96761347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.0:1619677445.189506:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.189507:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.0:1619677445.189507:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:25.0:1619677445.189508:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.0:1619677445.189508:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.189508:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.189509:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:25.0:1619677445.189510:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.189514:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.189517:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.189520:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.189520:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.189521:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.189528:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.189529:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.189530:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.189531:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.189910:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.189913:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.189914:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.189916:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.189920:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.189922:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.189924:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.189925:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.189926:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.189927:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.189930:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.189932:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.189935:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.189936:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.189936:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.189938:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.189939:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.189940:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.189940:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.190072:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.190074:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:9.0:1619677445.190075:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.190076:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.190078:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.190081:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.190459:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.190462:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.190463:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.190463:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.190467:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.190470:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.190754:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.190757:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.190757:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.190758:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.190761:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.190764:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.190776:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:9.0:1619677445.190779:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:9.0:1619677445.190781:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.190781:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.190782:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.190783:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.191175:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.191178:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.191179:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.191179:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.191182:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.191184:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.191216:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.191219:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.191220:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.191222:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.191224:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.191225:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:25.0:1619677445.191227:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.191228:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.191229:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.191230:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:25.0:1619677445.191233:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.191235:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:25.0:1619677445.191237:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c2502380] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.191238:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (20)++
00000800:00000200:25.0:1619677445.191239:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:25.0:1619677445.191241:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.191241:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.191242:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.191242:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.191351:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.191353:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.191354:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.191355:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.191357:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.191360:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.191580:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.191582:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.191584:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.191584:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.191586:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.191588:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.192566:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.192569:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.192569:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:9.0:1619677445.192573:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.192575:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.192577:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:9.0:1619677445.192578:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.192580:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.192581:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.192583:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:9.0:1619677445.192589:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.192591:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.0:1619677445.192594:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.192595:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:9.0:1619677445.192597:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:9.0:1619677445.192599:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:9.0:1619677445.192600:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.192601:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.192602:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.193017:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.193020:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.193020:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.193022:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.193025:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.193027:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.193028:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.193029:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.193030:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.193031:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.193035:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.193037:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.193040:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.193041:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.193041:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.193043:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.193044:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.193044:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.193045:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.193828:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.193832:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.193837:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:9.0:1619677445.193839:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.193842:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.193844:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.193846:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.193847:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.193848:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.193849:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.193853:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.193855:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.193858:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.193859:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.193859:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.193861:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.193862:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.193862:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.193863:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.194561:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.194564:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.194564:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.194567:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.194570:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.194572:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:25.0:1619677445.194573:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.194575:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.194576:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.194578:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:25.0:1619677445.194583:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.194585:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.0:1619677445.194589:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.194589:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:25.0:1619677445.194590:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:25.0:1619677445.194592:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:25.0:1619677445.194593:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.194594:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.194595:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.194600:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.194608:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.194610:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.194611:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.194612:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.194698:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.194700:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.194701:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.194701:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.194704:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.194706:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.194715:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:9.0:1619677445.194718:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.194719:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.194719:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.194720:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.194721:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.195098:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.195100:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.195101:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.195101:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.195103:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.195106:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.195209:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.195211:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.195212:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.195214:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.195216:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.195218:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:9.0:1619677445.195219:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.195220:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.195221:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.195222:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:9.0:1619677445.195226:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.195227:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:9.0:1619677445.195230:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.195231:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:9.0:1619677445.195231:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:9.0:1619677445.195233:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:9.0:1619677445.195234:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.195234:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.195235:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.195356:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.195359:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.195359:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.195361:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.195362:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.195365:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.195530:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.195532:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.195533:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.195533:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.195535:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.195537:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.195701:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.195703:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.195704:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.195704:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:25.0:1619677445.195706:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.195709:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.196218:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.196221:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.196222:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.196223:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.196225:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.196228:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.196239:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.196241:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.196242:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.196242:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.196243:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.196244:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.196398:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.196401:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.196402:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.196404:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.196406:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.196408:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.196409:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.196411:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.196412:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.196413:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.196416:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.196418:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.196421:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.196421:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.196422:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.196424:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.196425:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.196425:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.196426:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.197536:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.197539:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.197539:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.197541:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.197545:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.197546:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.197547:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.197549:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.197550:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.197551:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:25.0:1619677445.197554:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.197556:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.197559:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.197559:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.197560:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.197562:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.197563:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.197563:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.197564:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.198585:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.198588:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.198589:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.198592:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.198597:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.198599:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.198600:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96761201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.198602:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.198603:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.198604:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.198609:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.198612:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.198615:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.198616:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.198616:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.198618:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.198619:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.198620:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.198621:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.198622:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.198634:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.198637:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.198637:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.198639:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.198814:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:9.0:1619677445.198816:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:25.0:1619677445.198817:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:9.0:1619677445.198819:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.198822:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.198824:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:9.0:1619677445.198825:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.198826:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.198827:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.198828:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:9.0:1619677445.198832:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.198834:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:9.0:1619677445.198836:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c2502380] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.198837:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (20)++
00000800:00000200:9.0:1619677445.198837:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:9.0:1619677445.198839:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:9.0:1619677445.198840:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:9.0:1619677445.198840:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:9.0:1619677445.198841:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:25.0:1619677445.199693:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.199696:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.199696:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.199700:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.199703:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.199704:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.199706:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96761180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.199707:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.199709:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.199710:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.199713:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.199715:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.199718:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.199719:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (22)++
00000800:00000200:25.0:1619677445.199720:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (23)++
00000800:00000200:25.0:1619677445.199722:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (24)--
00000800:00000200:25.0:1619677445.199722:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.199723:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.199724:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.199976:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.199979:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.199980:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.199980:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.199982:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.199985:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.200710:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.200712:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.200713:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.200714:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.200716:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.200718:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.200743:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.200746:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.200746:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.200747:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.200749:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.200752:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.200800:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.200803:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.200804:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.200806:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.200809:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.200810:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.200812:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.200813:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.200815:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.200816:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.200819:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.200821:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.200824:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.200825:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.200825:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.200827:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.200827:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.200828:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.200829:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.201152:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.201155:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.201155:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:9.0:1619677445.201158:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.201164:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.0:1619677445.201165:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000400:00000200:9.0:1619677445.201166:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:25.0:1619677445.201167:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.201168:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.201168:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96761156] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.201170:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:9.0:1619677445.201170:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.201171:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.201172:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:9.0:1619677445.201176:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.201178:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.201178:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.201180:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.201182:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.201183:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:9.0:1619677445.201186:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.201187:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.201188:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.201189:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.201189:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:9.0:1619677445.201189:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.201190:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.201190:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.201191:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.201191:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:9.0:1619677445.201192:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:9.0:1619677445.201193:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.201205:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.201207:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.201207:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.201208:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.201380:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.201382:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:25.0:1619677445.201383:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:9.0:1619677445.201383:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.201386:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:9.0:1619677445.201389:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.201403:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (21)++
00000800:00000200:25.0:1619677445.201406:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:9.0:1619677445.201406:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.201407:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.201407:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.201408:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.201627:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.201630:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.201631:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.201631:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.201634:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.201637:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.201648:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.201650:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.201651:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.201653:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.201653:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:9.0:1619677445.201655:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.201656:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.201656:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.201657:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.201659:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.201663:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.201665:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:9.0:1619677445.201667:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96761147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.201668:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.201669:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.201670:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:9.0:1619677445.201673:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.201675:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:9.0:1619677445.201678:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c2502380] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.201679:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (20)++
00000800:00000200:9.0:1619677445.201679:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:9.0:1619677445.201681:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:9.0:1619677445.201682:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.201683:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.201683:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.202636:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.202639:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.202640:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.202643:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.202646:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.202647:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.202649:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.202651:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.202652:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.202653:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.202659:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.202662:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.202666:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.202668:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.202669:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.202670:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.202671:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.202672:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.202673:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.202733:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.202735:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.202736:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.202736:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.202739:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.202742:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.202909:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.202912:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.202913:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.202915:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.202917:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.202919:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:25.0:1619677445.202920:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.202922:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.202923:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.202924:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:25.0:1619677445.202927:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.202929:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:25.0:1619677445.202931:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c2502380] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.202932:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (20)++
00000800:00000200:25.0:1619677445.202933:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:25.0:1619677445.202934:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.202935:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.202935:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.202935:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.203384:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.203386:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:25.0:1619677445.203387:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:9.0:1619677445.203388:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.203390:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:9.0:1619677445.203392:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.203538:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.203540:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.203541:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.203543:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.203546:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.203547:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.203548:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96761123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.203550:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.203551:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.203552:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.203557:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.203559:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.203563:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.203564:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.203565:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.203566:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.203567:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.203568:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.203570:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.203570:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.203584:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.203586:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.203587:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.203588:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.205122:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.205125:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.205125:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.205130:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.205133:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.205134:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.205136:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96761094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.205137:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.205138:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.205140:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.205144:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.205146:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.205150:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.205151:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (22)++
00000800:00000200:25.0:1619677445.205151:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (23)++
00000800:00000200:25.0:1619677445.205153:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (24)--
00000800:00000200:25.0:1619677445.205154:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.205154:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.205155:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.205990:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.205992:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.205992:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.205994:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.205995:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.205996:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.205997:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:25.0:1619677445.205999:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.206001:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.206003:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.206005:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.206007:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.206008:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:9.0:1619677445.206010:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.206011:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.206012:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000400:00000200:25.0:1619677445.206013:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.206015:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.206018:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.206019:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.206019:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.206025:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.206026:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.206027:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.206028:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.206453:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.206456:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.206457:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.206459:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.206462:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.206464:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.206465:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.206467:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.206469:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.206470:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.206475:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.206478:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.206481:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.206481:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.206482:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.206484:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.206484:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.206485:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.206486:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.206766:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.206769:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.206770:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.206770:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:25.0:1619677445.206773:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.206776:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.206907:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.206910:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.206910:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:9.0:1619677445.206912:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.206915:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.206917:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:9.0:1619677445.206918:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.206919:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.206920:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.206921:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:9.0:1619677445.206925:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.206926:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.0:1619677445.206929:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.206930:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:9.0:1619677445.206931:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:9.0:1619677445.206932:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:9.0:1619677445.206933:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.206934:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:9.0:1619677445.206935:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:9.0:1619677445.206936:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.206948:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.206950:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.206951:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.206952:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.207001:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.207004:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.207004:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.207005:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.207008:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.207011:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.207024:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.207026:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.207027:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.207027:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.207028:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.207029:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.207405:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.207408:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.207409:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.207409:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.207412:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.207415:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.207592:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.207594:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.207595:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.207595:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.207598:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.207605:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.207684:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.207686:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.207687:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.207688:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.207690:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.207692:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.208489:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.208492:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.208493:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.208495:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.208497:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.208499:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.208500:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.208502:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.208503:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.208504:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.208507:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.208509:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.208512:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.208513:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.208513:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.208515:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.208515:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.208516:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.208517:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.208765:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.208768:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.208769:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.208772:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.208774:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.208776:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:9.0:1619677445.208777:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.208779:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.208780:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.208781:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:9.0:1619677445.208784:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.208786:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:9.0:1619677445.208788:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ec00] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.208789:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91978412be00] (20)++
00000800:00000200:9.0:1619677445.208790:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91978412be00] (21)++
00000800:00000200:9.0:1619677445.208792:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91978412be00] (22)--
00000800:00000200:9.0:1619677445.208792:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.208793:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.208793:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.209152:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.209154:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:9.0:1619677445.209155:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:25.0:1619677445.209156:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.209158:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:25.0:1619677445.209161:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.209170:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (21)++
00000800:00000200:9.0:1619677445.209172:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:25.0:1619677445.209173:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:9.0:1619677445.209173:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.209174:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983e00] (22)--
00000800:00000200:9.0:1619677445.209175:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.209326:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.209329:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.209330:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.209334:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.209337:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.209339:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.209340:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96761030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.209343:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.209348:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.209351:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.209355:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.209357:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.209364:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.209365:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.209366:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.209367:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.209368:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.209369:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.209370:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.209371:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.209384:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.209386:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.209387:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.209388:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.210086:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.210088:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.210089:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.210091:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.210094:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.210095:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:25.0:1619677445.210097:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96761018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.210098:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.210100:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.210101:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:25.0:1619677445.210104:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.210106:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.0:1619677445.210109:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.210110:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:25.0:1619677445.210111:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:25.0:1619677445.210112:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:25.0:1619677445.210113:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.210114:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.210114:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.211024:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.211027:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.211028:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.211030:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.211032:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.211035:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.211045:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.211048:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.211049:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.211049:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.211050:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.211051:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.211194:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.211197:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.211198:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.211200:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.211202:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.211204:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:9.0:1619677445.211205:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96761002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.211206:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.211208:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.211209:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:9.0:1619677445.211212:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.211214:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:9.0:1619677445.211217:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.211217:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (21)++
00000800:00000200:9.0:1619677445.211218:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (22)++
00000800:00000200:9.0:1619677445.211219:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (23)--
00000800:00000200:9.0:1619677445.211220:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.211226:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.211226:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.211689:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.211692:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.211693:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.211695:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.211698:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.211699:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.211700:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.211702:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.211703:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.211704:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.211707:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.211709:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.211711:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.211712:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.211713:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.211714:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.211715:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.211716:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.211716:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.211796:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.211798:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:25.0:1619677445.211799:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:9.0:1619677445.211799:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.211803:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.211806:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.212170:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.212172:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.212173:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.212174:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.212176:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.212179:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.212548:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.212550:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:25.0:1619677445.212551:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (23)--
00000400:00000200:9.0:1619677445.212551:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.212554:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.212557:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.212616:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.212618:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.212619:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.212619:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.212622:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.212625:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.213120:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.213123:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.213123:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:9.0:1619677445.213125:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.213128:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.213130:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:9.0:1619677445.213131:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.213133:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.213134:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.213135:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:9.0:1619677445.213139:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.213141:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:9.0:1619677445.213145:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ec00] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.213146:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91978412be00] (20)++
00000800:00000200:9.0:1619677445.213146:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91978412be00] (21)++
00000800:00000200:9.0:1619677445.213148:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91978412be00] (22)--
00000800:00000200:9.0:1619677445.213149:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.213151:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:9.0:1619677445.213152:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:9.0:1619677445.213152:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.213166:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.213168:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.213168:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.213169:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.213189:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:9.0:1619677445.213192:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:25.0:1619677445.213193:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:9.0:1619677445.213195:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.213198:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.213200:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:9.0:1619677445.213201:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.213203:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.213203:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.213204:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:9.0:1619677445.213208:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.213210:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:9.0:1619677445.213212:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8680] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.213213:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71ba00] (20)++
00000800:00000200:9.0:1619677445.213213:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71ba00] (21)++
00000800:00000200:9.0:1619677445.213215:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71ba00] (22)--
00000800:00000200:9.0:1619677445.213215:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:9.0:1619677445.213217:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:9.0:1619677445.213219:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:9.0:1619677445.213219:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:25.0:1619677445.213232:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:25.0:1619677445.213233:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:25.0:1619677445.213234:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:25.0:1619677445.213235:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.0:1619677445.213393:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.213396:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.213397:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.213397:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.213400:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.213402:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.213666:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.213669:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.213670:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.213671:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.213674:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.213675:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.213677:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.213678:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.213679:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.213680:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.213683:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.213684:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.213687:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.213688:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.213688:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.213690:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.213691:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.213691:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.213692:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.214664:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.214667:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.214668:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.214670:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.214673:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.214674:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:9.0:1619677445.214675:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.214677:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.214678:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.214679:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:9.0:1619677445.214682:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.214684:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.0:1619677445.214687:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.214688:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:9.0:1619677445.214688:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:9.0:1619677445.214690:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:9.0:1619677445.214690:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.214691:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.214692:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.215492:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.215495:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.215496:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.215499:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.215504:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.215505:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.215507:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.215508:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.215509:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.215510:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:25.0:1619677445.215518:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.215520:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.215523:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.215524:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.215525:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.215526:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.215527:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.215528:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.215530:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.215530:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.215543:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.215545:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.215546:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.215547:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.216866:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:25.0:1619677445.216869:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:9.0:1619677445.216870:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:25.0:1619677445.216872:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.216875:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.216877:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:25.0:1619677445.216878:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.216879:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.216880:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.216882:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:25.0:1619677445.216885:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.216887:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:25.0:1619677445.216889:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1200] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.216890:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38000] (20)++
00000800:00000200:25.0:1619677445.216890:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38000] (21)++
00000800:00000200:25.0:1619677445.216892:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38000] (22)--
00000800:00000200:25.0:1619677445.216893:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:25.0:1619677445.216893:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:25.0:1619677445.216894:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:9.0:1619677445.218092:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.218094:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.218095:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.218097:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.218100:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.218102:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.218103:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.218104:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.218106:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.218107:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.218110:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.218111:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.218114:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.218115:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.218116:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.218118:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.218118:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.218119:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.218119:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.218201:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.218203:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.218204:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.218204:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.218208:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.218211:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.218217:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.218220:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.218221:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (23)++
00000800:00000200:9.0:1619677445.218221:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (23)++
00000800:00000200:25.0:1619677445.218222:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (25)--
00000800:00000200:9.0:1619677445.218222:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (25)--
00000800:00000200:9.0:1619677445.218222:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (24)--
00000400:00000200:25.0:1619677445.218223:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.218224:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.218225:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.218227:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.218552:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.218555:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.218556:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.218558:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.218560:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.218563:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:9.0:1619677445.218564:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.218567:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.218569:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.218570:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:9.0:1619677445.218576:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.218578:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:9.0:1619677445.218581:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.218582:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:9.0:1619677445.218583:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:9.0:1619677445.218585:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:9.0:1619677445.218585:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.218586:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.218586:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.218612:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.218615:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.218616:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.218616:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.218618:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.218622:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.219343:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.219346:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.219346:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:9.0:1619677445.219348:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.219351:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.219352:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:9.0:1619677445.219354:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.219355:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.219356:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.219357:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:9.0:1619677445.219361:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.219362:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:9.0:1619677445.219365:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.219366:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:9.0:1619677445.219367:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:9.0:1619677445.219368:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:9.0:1619677445.219369:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.219370:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.219371:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.219972:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.219975:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.219976:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.219976:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:25.0:1619677445.219978:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:25.0:1619677445.219981:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.219996:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.219999:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.220000:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.220003:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.220005:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.220007:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:9.0:1619677445.220009:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.220011:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.220013:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.220015:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:9.0:1619677445.220019:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.220022:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:9.0:1619677445.220024:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.220025:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:9.0:1619677445.220026:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:9.0:1619677445.220028:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:9.0:1619677445.220028:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.220029:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.220030:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.220560:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.220563:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.220564:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.220566:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.220569:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.220570:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.220572:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.220573:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.220574:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.220575:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:25.0:1619677445.220579:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.220580:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.220583:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.220584:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.220584:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.220586:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.220587:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.220587:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.220588:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.220751:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.220753:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.220754:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.220756:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.220759:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.220761:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.220762:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.220763:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.220764:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.220765:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.220769:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.220771:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.220774:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.220775:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.220775:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.220777:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.220777:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.220779:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.220780:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.220781:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.220794:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.220797:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.220798:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.220799:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.220828:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.220830:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.220831:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.220831:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.220833:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.220836:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.221487:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.221490:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.221491:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.221491:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.221497:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.221500:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.221507:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:9.0:1619677445.221510:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.221511:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.221511:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.221512:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.221512:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.222416:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.222419:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.222420:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.222420:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.222422:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.222425:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.222463:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.222465:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.222466:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.222468:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.222471:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.222472:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.222473:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.222474:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.222475:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.222476:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.222479:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.222481:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.222483:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.222485:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.222485:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.222487:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.222487:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.222488:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.222489:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.222736:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.222738:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.222739:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.222739:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:25.0:1619677445.222742:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.222745:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.222753:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (21)++
00000800:00000200:9.0:1619677445.222756:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:25.0:1619677445.222757:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:9.0:1619677445.222758:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983e00] (21)++
00000800:00000200:9.0:1619677445.222759:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983e00] (22)--
00000800:00000200:9.0:1619677445.222759:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.222990:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.222993:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.222994:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.222994:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.222997:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.223000:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.223311:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.223314:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.223315:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.223316:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.223319:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.223321:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:9.0:1619677445.223322:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.223324:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.223326:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.223327:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:9.0:1619677445.223330:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.223332:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:9.0:1619677445.223336:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.223337:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:9.0:1619677445.223337:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:9.0:1619677445.223339:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:9.0:1619677445.223342:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.223343:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.223344:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.223359:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.223362:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.223363:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.223363:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.223366:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.223369:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.223979:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.223982:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.223983:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.223983:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.223986:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.223988:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.224095:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.224098:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.224099:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.224101:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.224104:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.224105:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.224107:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.224108:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.224109:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.224110:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.224117:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.224119:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.224122:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.224123:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.224123:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.224125:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.224126:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.224127:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.224127:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.224202:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.224205:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:25.0:1619677445.224206:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:9.0:1619677445.224206:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.224208:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.224211:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.224300:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.224304:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.224305:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.224306:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.224309:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.224310:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:25.0:1619677445.224312:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.224313:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.224314:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.224315:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:25.0:1619677445.224318:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.224319:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:25.0:1619677445.224322:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ec00] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.224323:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91978412be00] (20)++
00000800:00000200:25.0:1619677445.224324:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91978412be00] (21)++
00000800:00000200:25.0:1619677445.224326:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91978412be00] (22)--
00000800:00000200:25.0:1619677445.224326:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.224327:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.224327:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.224393:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.224396:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.224397:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.224397:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.224400:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.224403:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.224411:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.224414:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.224415:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.224415:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.224416:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.224416:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.224638:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.224640:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.224641:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.224643:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.224645:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.224647:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:9.0:1619677445.224648:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.224649:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.224650:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.224651:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:9.0:1619677445.224654:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.224656:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:9.0:1619677445.224659:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.224659:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:9.0:1619677445.224660:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:9.0:1619677445.224661:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:9.0:1619677445.224662:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.224664:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:9.0:1619677445.224665:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:9.0:1619677445.224666:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.224678:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.224680:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.224681:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.224682:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.224799:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.224801:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.224802:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:25.0:1619677445.224803:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.224803:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.224805:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:9.0:1619677445.224807:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:25.0:1619677445.224808:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.224809:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000400:00000200:9.0:1619677445.224810:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.0:1619677445.224811:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.224812:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:9.0:1619677445.224815:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96760786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.224818:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.224819:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.224820:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:9.0:1619677445.224823:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.224826:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:9.0:1619677445.224829:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.224830:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:9.0:1619677445.224830:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:9.0:1619677445.224832:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:9.0:1619677445.224833:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.224835:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.224837:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.224838:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.224851:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.224853:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.224854:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.224855:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.225481:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.225483:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.225484:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.225486:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.225489:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.225491:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.225493:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.225495:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.225496:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.225497:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.225500:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.225502:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.225504:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.225505:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.225506:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.225508:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.225509:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.225509:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.225510:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.225937:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.225941:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.225943:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.225944:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.225947:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.225949:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:9.0:1619677445.225951:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.225953:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.225954:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.225955:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:9.0:1619677445.225961:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.225964:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:9.0:1619677445.225967:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.225968:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:9.0:1619677445.225969:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:9.0:1619677445.225971:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:9.0:1619677445.225972:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.225973:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.225974:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.226277:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.226280:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.226281:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.226284:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.226287:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.226301:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.226303:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.226306:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.226307:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.226308:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.226313:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.226315:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.226317:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.226318:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.226319:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.226320:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.226321:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.226322:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.226322:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.226837:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.226839:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.226840:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.226841:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.226843:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.226846:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.227043:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.227046:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.227047:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.227050:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.227053:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.227055:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.227056:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.227058:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.227059:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.227060:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.227064:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.227066:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.227069:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.227070:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.227071:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.227072:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.227073:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.227074:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.227074:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.227336:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.227339:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.227340:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.227340:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.227342:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.227345:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.228213:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39da00] (21)++
00000800:00000200:25.0:1619677445.228216:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.61@o2ib3
00000800:00000200:9.0:1619677445.228216:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39da00] (22)++
00000800:00000200:9.0:1619677445.228218:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.61@o2ib3
00000400:00000200:25.0:1619677445.228219:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.61@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.228220:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.61@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.228223:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.228223:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.228225:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:9.0:1619677445.228227:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.228229:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.228230:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.228231:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:9.0:1619677445.228237:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.61@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000400:00000200:25.0:1619677445.228240:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:9.0:1619677445.228240:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000400:00000200:25.0:1619677445.228242:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96760733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.228243:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.228245:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.228247:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:9.0:1619677445.228249:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679200] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.228250:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980600] (20)++
00000400:00000200:25.0:1619677445.228251:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.61@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.228251:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980600] (21)++
00000800:00000200:9.0:1619677445.228253:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980600] (22)--
00000800:00000200:9.0:1619677445.228254:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39da00] (23)++
00000800:00000200:9.0:1619677445.228255:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39da00] (24)--
00000800:00000200:9.0:1619677445.228255:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39da00] (23)--
00000800:00000200:9.0:1619677445.228257:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39da00] (22)++
00000800:00000200:25.0:1619677445.228259:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:9.0:1619677445.228259:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.61@o2ib3
00000800:00000200:24.2F:1619677445.228260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980600] (21)++
00000800:00000200:25.0:1619677445.228262:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3c00] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:9.0:1619677445.228262:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.61@o2ib3 : PUT - routed
00000800:00000200:25.0:1619677445.228264:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128400] (20)++
00000800:00000200:25.0:1619677445.228264:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128400] (21)++
00000400:00000200:9.0:1619677445.228264:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.0:1619677445.228266:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128400] (22)--
00000400:00000200:9.0:1619677445.228266:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:9.0:1619677445.228267:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96760732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.228270:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.228271:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.0:1619677445.228272:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39da00] (23)++
00000400:00000200:9.0:1619677445.228272:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:25.0:1619677445.228274:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39da00] (24)++
00000800:00000200:25.0:1619677445.228276:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39da00] (25)--
00000400:00000200:9.0:1619677445.228276:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.61@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.228277:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39da00] (24)--
00000800:00000200:25.0:1619677445.228278:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39da00] (23)++
00000800:00000200:9.0:1619677445.228278:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:25.0:1619677445.228279:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.61@o2ib3
00000800:00000200:9.0:1619677445.228281:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780380] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:25.0:1619677445.228282:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.61@o2ib3 : PUT - routed
00000800:00000200:9.0:1619677445.228282:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e3a00] (20)++
00000800:00000200:9.0:1619677445.228283:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e3a00] (21)++
00000400:00000200:25.0:1619677445.228284:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.228285:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e3a00] (22)--
00000400:00000200:25.0:1619677445.228286:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:25.0:1619677445.228288:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96760730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.228301:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.228302:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.228303:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:9.0:1619677445.228305:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39da00] (24)++
00000800:00000200:9.0:1619677445.228306:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39da00] (25)--
00000800:00000200:9.0:1619677445.228307:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39da00] (24)--
00000400:00000200:25.0:1619677445.228308:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.61@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.228309:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980600] (22)++
00000800:00000200:25.0:1619677445.228310:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:9.0:1619677445.228310:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980600] (23)--
00000400:00000200:9.0:1619677445.228311:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.228312:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.228315:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56400] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.228316:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea37e00] (20)++
00000800:00000200:25.0:1619677445.228316:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea37e00] (21)++
00000800:00000200:9.0:1619677445.228316:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980600] (22)--
00000800:00000200:9.0:1619677445.228316:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39da00] (23)++
00000800:00000200:25.0:1619677445.228318:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea37e00] (22)--
00000800:00000200:25.0:1619677445.228320:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39da00] (24)++
00000800:00000200:9.0:1619677445.228320:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.61@o2ib3
00000800:00000200:25.0:1619677445.228321:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39da00] (25)--
00000800:00000200:25.0:1619677445.228321:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39da00] (24)--
00000400:00000200:9.0:1619677445.228324:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.61@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.228325:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.0:1619677445.228328:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:9.0:1619677445.228329:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:25.0:1619677445.228330:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:9.0:1619677445.228330:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96760728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.228333:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.228334:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.228335:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:25.0:1619677445.228338:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.228340:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.228342:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.61@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:25.0:1619677445.228343:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:25.0:1619677445.228345:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96760726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.0:1619677445.228345:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:25.0:1619677445.228347:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.228348:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.228350:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:9.0:1619677445.228354:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780380] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:25.0:1619677445.228355:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.228355:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e3a00] (20)++
00000800:00000200:9.0:1619677445.228356:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e3a00] (21)++
00000800:00000200:25.0:1619677445.228358:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:9.0:1619677445.228358:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e3a00] (22)--
00000800:00000200:9.0:1619677445.228358:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39da00] (23)++
00000800:00000200:9.0:1619677445.228360:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39da00] (24)--
00000800:00000200:9.0:1619677445.228361:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39da00] (23)--
00000800:00000200:25.0:1619677445.228362:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ec00] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.228362:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980600] (21)--
00000800:00000200:25.0:1619677445.228363:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91978412be00] (20)++
00000800:00000200:9.0:1619677445.228364:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.228365:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:9.0:1619677445.228368:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:25.0:1619677445.228369:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91978412be00] (21)++
00000800:00000200:25.0:1619677445.228371:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91978412be00] (22)--
00000800:00000200:25.0:1619677445.228371:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (22)++
00000400:00000200:9.0:1619677445.228371:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.0:1619677445.228372:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.228372:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.228373:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39da00] (22)++
00000400:00000200:9.0:1619677445.228373:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:25.0:1619677445.228374:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39da00] (23)--
00000400:00000200:9.0:1619677445.228375:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96760724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.0:1619677445.228376:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39da00] (22)--
00000800:00000200:25.0:1619677445.228377:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:9.0:1619677445.228377:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.228378:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.0:1619677445.228379:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.228380:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39da00] (21)++
00000400:00000200:9.0:1619677445.228380:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:9.0:1619677445.228386:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.228387:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.61@o2ib3
00000800:00000200:9.0:1619677445.228388:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:25.0:1619677445.228391:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.61@o2ib3 : PUT - routed
00000800:00000200:9.0:1619677445.228392:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.228393:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:9.0:1619677445.228394:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000400:00000200:25.0:1619677445.228395:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.228396:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000400:00000200:25.0:1619677445.228397:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:25.0:1619677445.228398:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96760723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.228400:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.228401:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.228402:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:9.0:1619677445.228403:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.228404:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.228405:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:25.0:1619677445.228406:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.61@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.228406:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39da00] (22)++
00000800:00000200:9.0:1619677445.228407:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.61@o2ib3
00000800:00000200:25.0:1619677445.228408:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:9.0:1619677445.228409:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.61@o2ib3 : PUT - routed
00000800:00000200:25.0:1619677445.228410:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.228411:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (23)++
00000400:00000200:9.0:1619677445.228412:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.0:1619677445.228413:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (24)++
00000400:00000200:9.0:1619677445.228413:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:9.0:1619677445.228415:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96760722] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.0:1619677445.228416:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (23)--
00000400:00000200:9.0:1619677445.228417:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.228419:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.228420:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:25.0:1619677445.228421:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39da00] (23)++
00000800:00000200:25.0:1619677445.228422:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39da00] (24)++
00000400:00000200:9.0:1619677445.228423:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.61@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.228424:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39da00] (25)--
00000800:00000200:25.0:1619677445.228425:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39da00] (24)--
00000800:00000200:9.0:1619677445.228425:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:25.0:1619677445.228426:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39da00] (23)++
00000800:00000200:9.0:1619677445.228427:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56400] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.228428:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.61@o2ib3
00000800:00000200:9.0:1619677445.228428:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea37e00] (20)++
00000800:00000200:9.0:1619677445.228429:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea37e00] (21)++
00000400:00000200:25.0:1619677445.228430:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.61@o2ib3 : PUT - routed
00000800:00000200:9.0:1619677445.228431:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea37e00] (22)--
00000800:00000200:9.0:1619677445.228431:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39da00] (24)++
00000800:00000200:9.0:1619677445.228432:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39da00] (25)--
00000400:00000200:25.0:1619677445.228433:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.228433:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39da00] (24)--
00000800:00000200:9.0:1619677445.228434:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39da00] (23)++
00000800:00000200:9.0:1619677445.228435:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.61@o2ib3
00000400:00000200:25.0:1619677445.228440:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:9.0:1619677445.228440:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.61@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.228441:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.228443:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.228444:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.228444:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:9.0:1619677445.228447:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.228449:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.61@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:9.0:1619677445.228449:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:9.0:1619677445.228450:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.0:1619677445.228451:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:9.0:1619677445.228452:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.228453:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.228454:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:9.0:1619677445.228458:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.61@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.228459:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.228460:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:25.0:1619677445.228461:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.228461:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:9.0:1619677445.228462:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c2502380] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.228463:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:9.0:1619677445.228463:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (20)++
00000800:00000200:25.0:1619677445.228464:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39da00] (24)++
00000800:00000200:9.0:1619677445.228464:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:25.0:1619677445.228465:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39da00] (25)--
00000800:00000200:25.0:1619677445.228465:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39da00] (24)--
00000800:00000200:25.0:1619677445.228466:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39da00] (23)++
00000800:00000200:9.0:1619677445.228466:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:9.0:1619677445.228466:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39da00] (23)++
00000800:00000200:9.0:1619677445.228467:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39da00] (24)--
00000800:00000200:25.0:1619677445.228468:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.61@o2ib3
00000800:00000200:9.0:1619677445.228473:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39da00] (24)--
00000800:00000200:9.0:1619677445.228474:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39da00] (23)++
00000800:00000200:9.0:1619677445.228475:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39da00] (24)--
00000800:00000200:9.0:1619677445.228475:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39da00] (23)--
00000400:00000200:25.0:1619677445.228476:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.61@o2ib3 : PUT - routed
00000800:00000200:9.0:1619677445.228476:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39da00] (22)++
00000800:00000200:9.0:1619677445.228478:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.61@o2ib3
00000400:00000200:25.0:1619677445.228479:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.228480:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:9.0:1619677445.228481:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.61@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.228482:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96760718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.228483:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.228485:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.228487:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:25.0:1619677445.228492:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.61@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.228494:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:9.0:1619677445.228499:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.0:1619677445.228500:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f300] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.228500:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980000] (20)++
00000800:00000200:25.0:1619677445.228501:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980000] (21)++
00000400:00000200:9.0:1619677445.228501:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:25.0:1619677445.228503:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.228503:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96760717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.228505:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.228509:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.228510:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.228511:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39da00] (23)++
00000800:00000200:25.0:1619677445.228512:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39da00] (24)++
00000800:00000200:25.0:1619677445.228514:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39da00] (25)--
00000800:00000200:25.0:1619677445.228515:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39da00] (24)--
00000400:00000200:9.0:1619677445.228515:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.61@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.228516:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39da00] (23)++
00000800:00000200:9.0:1619677445.228516:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.228517:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.61@o2ib3
00000800:00000200:9.0:1619677445.228519:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:25.0:1619677445.228520:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.61@o2ib3 : PUT - routed
00000800:00000200:9.0:1619677445.228520:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:9.0:1619677445.228521:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000400:00000200:25.0:1619677445.228522:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.228523:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000400:00000200:25.0:1619677445.228524:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.228525:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96760716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.228526:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.228527:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.228528:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.228532:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39da00] (24)++
00000400:00000200:25.0:1619677445.228533:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.61@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.228533:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39da00] (25)--
00000800:00000200:9.0:1619677445.228533:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39da00] (24)--
00000800:00000200:25.0:1619677445.228534:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.228534:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.228535:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.228536:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.0:1619677445.228538:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:9.0:1619677445.228538:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.228539:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.228539:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.228539:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.228540:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39da00] (23)++
00000800:00000200:25.0:1619677445.228541:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.228541:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.61@o2ib3
00000800:00000200:25.0:1619677445.228542:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39da00] (24)++
00000800:00000200:25.0:1619677445.228543:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39da00] (25)--
00000800:00000200:25.0:1619677445.228543:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39da00] (24)--
00000800:00000200:25.0:1619677445.228544:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.228546:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000400:00000200:9.0:1619677445.228546:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.61@o2ib3 : PUT - routed
00000800:00000200:25.0:1619677445.228547:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.228548:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.228549:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.228550:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39da00] (23)++
00000400:00000200:9.0:1619677445.228550:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.0:1619677445.228552:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.61@o2ib3
00000400:00000200:9.0:1619677445.228552:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:9.0:1619677445.228554:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.228556:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.228559:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.228560:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:25.0:1619677445.228564:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.61@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.228566:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.61@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:25.0:1619677445.228567:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.228570:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:9.0:1619677445.228570:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:25.0:1619677445.228571:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96760713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.228572:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.228574:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.228576:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:9.0:1619677445.228579:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780e80] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.228579:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e3200] (20)++
00000800:00000200:9.0:1619677445.228580:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e3200] (21)++
00000800:00000200:9.0:1619677445.228583:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e3200] (22)--
00000800:00000200:9.0:1619677445.228583:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39da00] (24)++
00000800:00000200:9.0:1619677445.228584:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39da00] (25)--
00000800:00000200:9.0:1619677445.228585:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39da00] (24)--
00000800:00000200:9.0:1619677445.228586:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000400:00000200:25.0:1619677445.228587:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.61@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.228587:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39da00] (23)++
00000800:00000200:9.0:1619677445.228588:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39da00] (24)--
00000800:00000200:9.0:1619677445.228589:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39da00] (23)--
00000800:00000200:9.0:1619677445.228591:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39da00] (22)++
00000800:00000200:9.0:1619677445.228592:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.61@o2ib3
00000800:00000200:25.0:1619677445.228593:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:9.0:1619677445.228595:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.61@o2ib3 : PUT - routed
00000800:00000200:25.0:1619677445.228596:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1200] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.228597:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38000] (20)++
00000400:00000200:9.0:1619677445.228597:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.228599:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:9.0:1619677445.228600:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96760711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.0:1619677445.228601:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38000] (21)++
00000400:00000200:9.0:1619677445.228602:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.0:1619677445.228603:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38000] (22)--
00000400:00000200:9.0:1619677445.228604:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.228605:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:25.0:1619677445.228606:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39da00] (23)++
00000400:00000200:9.0:1619677445.228610:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.61@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.228612:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:25.0:1619677445.228614:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39da00] (24)++
00000800:00000200:25.0:1619677445.228615:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39da00] (25)--
00000800:00000200:25.0:1619677445.228616:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39da00] (24)--
00000800:00000200:9.0:1619677445.228616:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8680] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.228617:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39da00] (23)--
00000800:00000200:9.0:1619677445.228617:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71ba00] (20)++
00000800:00000200:9.0:1619677445.228617:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71ba00] (21)++
00000800:00000200:9.0:1619677445.228619:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71ba00] (22)--
00000800:00000200:9.0:1619677445.228620:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39da00] (22)++
00000800:00000200:9.0:1619677445.228621:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39da00] (23)--
00000800:00000200:9.0:1619677445.228625:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39da00] (23)--
00000800:00000200:9.0:1619677445.228626:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39da00] (22)++
00000800:00000200:9.0:1619677445.228628:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39da00] (23)--
00000800:00000200:9.0:1619677445.228629:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39da00] (22)--
00000800:00000200:9.0:1619677445.228630:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39da00] (21)--
00000800:00000200:9.0:1619677445.228788:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.228790:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.228791:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.228791:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.228793:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.228796:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.228815:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39da00] (21)++
00000800:00000200:25.0:1619677445.228817:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.61@o2ib3
00000800:00000200:9.0:1619677445.228818:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39da00] (22)--
00000400:00000200:25.0:1619677445.228820:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.61@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.228822:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.228824:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:25.0:1619677445.228825:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.228826:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.228827:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.228828:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:25.0:1619677445.228831:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.61@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.228833:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.0:1619677445.228835:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.228836:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:25.0:1619677445.228837:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:25.0:1619677445.228838:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:25.0:1619677445.228839:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39da00] (21)++
00000800:00000200:25.0:1619677445.228840:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39da00] (22)--
00000800:00000200:25.0:1619677445.228840:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39da00] (21)--
00000800:00000200:9.0:1619677445.228869:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.228873:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.228873:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:9.0:1619677445.228876:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.228879:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.228880:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:9.0:1619677445.228882:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.228884:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.228885:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.228886:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:9.0:1619677445.228890:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.228892:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:9.0:1619677445.228895:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ec00] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.228895:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91978412be00] (20)++
00000800:00000200:9.0:1619677445.228896:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91978412be00] (21)++
00000800:00000200:9.0:1619677445.228898:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91978412be00] (22)--
00000800:00000200:9.0:1619677445.228898:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.228900:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:9.0:1619677445.228901:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:9.0:1619677445.228902:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.228914:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.228916:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.228917:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.228919:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.229112:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.229115:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.229116:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.229118:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.229121:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.229123:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.229125:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.229126:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.229127:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.229128:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.229132:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.229134:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.229136:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.229137:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.229138:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.229139:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.229140:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.229141:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.229142:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.229143:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.229155:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.229157:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.229158:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.229159:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.229370:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.229373:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.229374:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.229376:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.229378:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.229380:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:9.0:1619677445.229381:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.229383:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.229384:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.229385:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:9.0:1619677445.229390:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.229392:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:9.0:1619677445.229395:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ec00] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.229396:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91978412be00] (20)++
00000800:00000200:9.0:1619677445.229397:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91978412be00] (21)++
00000800:00000200:9.0:1619677445.229399:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91978412be00] (22)--
00000800:00000200:9.0:1619677445.229399:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.229400:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.229401:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.229426:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.229429:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:9.0:1619677445.229429:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:25.0:1619677445.229430:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.229432:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.229435:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.229643:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.229646:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.229647:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.229647:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.229649:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.229651:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.229807:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.229810:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.229811:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.229811:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:25.0:1619677445.229814:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.229817:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.229835:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (21)++
00000800:00000200:25.0:1619677445.229838:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:9.0:1619677445.229838:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:9.0:1619677445.229839:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.229840:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983e00] (22)--
00000800:00000200:9.0:1619677445.229840:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.229857:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.229860:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.229860:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.229862:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.229865:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.229866:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:25.0:1619677445.229867:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.229869:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.229870:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.229872:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:25.0:1619677445.229876:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.229881:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:25.0:1619677445.229883:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ec00] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.229884:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91978412be00] (20)++
00000800:00000200:25.0:1619677445.229885:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91978412be00] (21)++
00000800:00000200:25.0:1619677445.229887:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91978412be00] (22)--
00000800:00000200:25.0:1619677445.229887:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.229888:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.229888:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.229940:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.229943:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:9.0:1619677445.229946:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:25.0:1619677445.229947:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.229949:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.229952:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:9.0:1619677445.229954:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96760678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.229956:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.229957:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.229959:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:9.0:1619677445.229964:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.229966:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:9.0:1619677445.229970:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.229971:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:9.0:1619677445.229972:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:9.0:1619677445.229974:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:9.0:1619677445.229974:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.229975:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.229976:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.230541:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.230544:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.230545:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.230547:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.230549:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.230551:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:25.0:1619677445.230552:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.230554:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.230554:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.230556:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:25.0:1619677445.230561:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.230564:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.0:1619677445.230567:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.230568:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:25.0:1619677445.230568:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:25.0:1619677445.230570:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:25.0:1619677445.230570:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.230571:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.230572:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.231092:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.231096:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:9.0:1619677445.231098:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:25.0:1619677445.231101:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.231101:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.231103:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.231105:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.231106:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.231107:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.231108:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.231112:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.231113:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.231116:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.231117:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.231117:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.231119:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.231120:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.231120:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.231121:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.231564:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.231567:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.231568:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.231570:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.231573:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.231574:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.231575:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.231576:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.231577:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.231578:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.231583:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.231586:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.231589:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.231590:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.231590:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.231592:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.231592:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.231594:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.231595:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.231596:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.231608:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.231610:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.231612:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.231612:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.232558:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.232561:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.232562:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.232564:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.232566:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.232568:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.232569:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.232570:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.232571:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.232573:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.232576:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.232577:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.232580:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.232581:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.232581:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.232583:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.232583:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.232584:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.232585:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.233365:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.233368:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.233369:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.233370:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.233372:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.233376:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.234084:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.234087:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:9.0:1619677445.234087:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:9.0:1619677445.234088:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.234091:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:9.0:1619677445.234093:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.234872:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.234874:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.234875:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.234876:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.234878:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.234881:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.234890:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:9.0:1619677445.234892:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.234893:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.234893:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.234894:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.234895:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.235011:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.235014:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.235015:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.235016:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.235019:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.235021:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:25.0:1619677445.235022:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.235024:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.235025:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.235026:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:25.0:1619677445.235029:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.235031:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.0:1619677445.235034:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.235035:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:25.0:1619677445.235036:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:25.0:1619677445.235038:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:25.0:1619677445.235038:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.235043:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.235044:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.235074:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.235078:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:9.0:1619677445.235078:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:9.0:1619677445.235079:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.235081:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.235085:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.235510:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.235513:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.235514:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.235516:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.235519:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.235521:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.235522:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.235523:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.235524:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.235525:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:25.0:1619677445.235528:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.235530:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.235533:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.235534:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.235534:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.235536:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.235537:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.235537:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.235538:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.235660:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.235663:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.235664:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.235665:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.235668:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.235670:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:9.0:1619677445.235671:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.235672:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.235673:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.235674:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:9.0:1619677445.235677:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.235678:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:9.0:1619677445.235682:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.235683:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:9.0:1619677445.235683:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:9.0:1619677445.235685:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:9.0:1619677445.235686:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.235686:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.235687:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.235840:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.235842:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.235843:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.235843:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.235846:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.235849:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.235928:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.235930:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.235931:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.235931:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.235933:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.235935:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.236024:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.236027:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.236028:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.236030:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.236032:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.236034:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.236035:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.236036:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.236037:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.236038:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.236042:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.236044:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.236047:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.236048:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.236049:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.236051:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.236051:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.236052:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.236054:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.236054:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.236067:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.236069:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.236070:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.236072:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.236401:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.236404:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.236405:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.236407:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.236410:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.236411:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.236413:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.236414:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.236415:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.236416:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.236420:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.236422:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.236425:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.236425:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.236426:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.236427:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.236428:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.236429:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.236429:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.236545:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.236547:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.236548:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.236551:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.236554:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.236556:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:25.0:1619677445.236557:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96760562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.236558:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.236559:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.236560:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:25.0:1619677445.236564:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.236566:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:25.0:1619677445.236569:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ec00] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.236570:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91978412be00] (20)++
00000800:00000200:25.0:1619677445.236570:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91978412be00] (21)++
00000800:00000200:25.0:1619677445.236572:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91978412be00] (22)--
00000800:00000200:25.0:1619677445.236573:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.236574:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.236575:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.236576:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.236588:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:9.0:1619677445.236590:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:9.0:1619677445.236591:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.236592:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.236869:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.236872:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.236873:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.236875:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.236878:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.236879:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.236881:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.236882:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.236883:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.236884:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.236888:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.236890:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.236892:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.236893:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.236894:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.236895:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.236896:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.236896:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.236897:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.236935:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.236937:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.236938:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.236938:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.236940:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.236943:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.237400:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.237403:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.237404:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.237406:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.237408:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.237410:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.237411:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.237413:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.237414:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.237415:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.237419:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.237421:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.237423:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.237424:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.237424:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.237426:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.237427:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.237428:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.237429:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.237540:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.237543:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.237544:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.237547:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.237549:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.237551:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.237559:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:9.0:1619677445.237563:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.237567:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.237568:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.237569:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.237570:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.237821:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.237824:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (23)--
00000800:00000200:9.0:1619677445.237824:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:9.0:1619677445.237825:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.237827:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:9.0:1619677445.237831:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.237878:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.237880:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.237881:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.237881:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:25.0:1619677445.237884:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.237886:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.237897:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (21)++
00000800:00000200:9.0:1619677445.237900:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:25.0:1619677445.237901:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:9.0:1619677445.237901:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.237902:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983e00] (22)--
00000800:00000200:9.0:1619677445.237902:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.238450:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.238453:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.238454:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.238455:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.238459:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.238460:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.238462:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.238463:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.238464:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.238465:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.238468:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.238470:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.238472:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.238473:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.238473:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.238475:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.238476:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.238477:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.238477:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.238525:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.238528:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.238529:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.238529:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.238531:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.238535:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.240221:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:25.0:1619677445.240224:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:9.0:1619677445.240224:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:25.0:1619677445.240226:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.240229:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.240231:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:25.0:1619677445.240232:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.240234:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.240234:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.240235:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:25.0:1619677445.240238:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.240240:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:25.0:1619677445.240243:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56400] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.240244:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea37e00] (20)++
00000800:00000200:25.0:1619677445.240245:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea37e00] (21)++
00000800:00000200:25.0:1619677445.240247:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea37e00] (22)--
00000800:00000200:25.0:1619677445.240248:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:25.0:1619677445.240249:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:25.0:1619677445.240249:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.0:1619677445.240427:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.240430:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.240431:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.240433:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.240436:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.240438:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:9.0:1619677445.240440:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96760500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.240441:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.240442:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.240443:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:9.0:1619677445.240448:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.240450:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:9.0:1619677445.240453:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.240455:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:9.0:1619677445.240455:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:9.0:1619677445.240457:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:9.0:1619677445.240458:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.240459:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.240461:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.240461:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.240479:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.240481:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.240482:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.240483:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.240597:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.240600:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:9.0:1619677445.240600:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:25.0:1619677445.240601:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.240604:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.240608:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.241247:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.241250:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.241251:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.241253:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.241256:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.241258:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:9.0:1619677445.241259:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.241261:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.241262:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.241263:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:9.0:1619677445.241266:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.241268:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.0:1619677445.241272:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.241273:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:9.0:1619677445.241273:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.241283:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.241284:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.241285:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (23)--
00000800:00000200:9.0:1619677445.241286:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:25.0:1619677445.241287:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:9.0:1619677445.241287:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.241287:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.241289:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000400:00000200:25.0:1619677445.241290:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.241290:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.241291:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.241292:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.241300:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96760480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.241301:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:25.0:1619677445.241302:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.241303:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.241305:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:9.0:1619677445.241309:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.241310:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.241310:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.241312:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.0:1619677445.241312:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.241315:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.241316:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.241317:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.241318:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.241319:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.241320:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.241321:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.241683:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.241687:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.241688:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.241689:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.241693:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.241694:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:9.0:1619677445.241695:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.241697:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.241697:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.241699:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:9.0:1619677445.241702:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.241704:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.0:1619677445.241707:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.241708:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:9.0:1619677445.241709:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:9.0:1619677445.241710:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:9.0:1619677445.241711:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.241712:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.241712:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.241963:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.241966:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.241967:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.241971:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.241974:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.241977:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:25.0:1619677445.241979:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.241981:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.241982:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.241983:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:25.0:1619677445.241990:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.241992:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:25.0:1619677445.241995:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ec00] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.241996:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91978412be00] (20)++
00000800:00000200:25.0:1619677445.241997:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91978412be00] (21)++
00000800:00000200:25.0:1619677445.241998:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91978412be00] (22)--
00000800:00000200:25.0:1619677445.241999:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.242000:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.242001:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.242127:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.242130:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.242131:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.242132:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.242140:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.242142:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.242143:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.242144:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.242145:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.242146:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.242150:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.242152:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.242158:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.242158:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.242159:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.242161:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.242161:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.242162:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.242162:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.242998:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.243000:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:9.0:1619677445.243001:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:25.0:1619677445.243002:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.243004:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:25.0:1619677445.243007:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.243174:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.243177:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.243177:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.243178:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.243180:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.243183:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.243599:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.243602:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.243603:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.243604:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.243607:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.243609:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.243610:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.243611:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.243612:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.243613:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.243616:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.243618:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.243621:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.243621:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.243622:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.243624:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.243624:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.243626:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.243627:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.243627:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.243640:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:9.0:1619677445.243642:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:9.0:1619677445.243642:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.243643:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.243976:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.243979:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.243980:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.243982:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.243985:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.243986:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.243988:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.243991:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.243992:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.243993:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.243996:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.243998:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.244000:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.244001:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.244002:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.244003:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.244004:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.244005:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.244005:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.244394:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.244397:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.244398:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.244398:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.244401:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.244404:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.244415:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.244418:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.244419:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.244419:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.244420:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.244420:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.244799:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.244802:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.244803:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.244803:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.244805:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.244808:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.245149:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.245152:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.245153:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.245156:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.245159:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.245161:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:25.0:1619677445.245163:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.245165:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.245166:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.245167:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:25.0:1619677445.245175:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.245177:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.0:1619677445.245180:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.245181:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:25.0:1619677445.245182:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:25.0:1619677445.245184:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:25.0:1619677445.245185:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.245186:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.245187:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.245188:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.245200:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.245202:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.245203:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.245205:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.246061:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.246064:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.246064:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.246065:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.246067:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.246070:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.246274:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.246277:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:25.0:1619677445.246278:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:9.0:1619677445.246278:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.0:1619677445.246284:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000400:00000200:9.0:1619677445.246284:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.246285:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.246286:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.246287:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000400:00000200:25.0:1619677445.246288:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.246288:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.246291:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.246710:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.246713:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.246714:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.246716:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.246718:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.246720:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.246721:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.246722:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.246723:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.246724:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.246727:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.246729:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.246732:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.246733:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.246733:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.246735:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.246736:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.246737:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.246737:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.247228:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.247231:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.247231:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.247232:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.247234:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.247237:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.247251:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.247252:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.247253:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:9.0:1619677445.247255:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000400:00000200:25.0:1619677445.247256:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:9.0:1619677445.247256:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.247257:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.247257:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.247258:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.247259:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.247259:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000400:00000200:25.0:1619677445.247260:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.247261:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96760378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.247263:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.247268:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.247269:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.247276:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.247278:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.247281:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.247282:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.247283:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.247288:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.247289:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.247290:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.247290:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.247667:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.247670:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.247671:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.247673:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.247675:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.247677:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.247678:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.247680:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.247681:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.247682:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.247684:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.247686:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.247689:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.247689:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.247690:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.247692:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.247692:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.247693:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.247693:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.248034:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.248037:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.248038:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.248040:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.248043:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.248044:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:25.0:1619677445.248046:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.248047:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.248048:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.248049:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:25.0:1619677445.248052:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.248054:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:25.0:1619677445.248057:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780380] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.248057:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e3a00] (20)++
00000800:00000200:25.0:1619677445.248058:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e3a00] (21)++
00000800:00000200:25.0:1619677445.248059:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e3a00] (22)--
00000800:00000200:25.0:1619677445.248060:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.248061:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.248061:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.248091:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.248094:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.248094:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:9.0:1619677445.248097:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.248099:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.248101:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:9.0:1619677445.248102:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.248103:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.248104:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.248105:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:9.0:1619677445.248108:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.248109:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.0:1619677445.248112:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.248113:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:9.0:1619677445.248113:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:9.0:1619677445.248115:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:9.0:1619677445.248116:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.248117:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.248118:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.248119:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.248131:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.248133:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.248134:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.248135:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.248388:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.248391:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.248392:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.248392:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.248394:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.248397:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.248549:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.248553:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.248554:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.248556:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.248559:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.248560:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:25.0:1619677445.248562:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.248563:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.248564:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.248565:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:25.0:1619677445.248568:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.248570:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:25.0:1619677445.248573:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780380] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.248574:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e3a00] (20)++
00000800:00000200:25.0:1619677445.248574:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e3a00] (21)++
00000800:00000200:25.0:1619677445.248576:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e3a00] (22)--
00000800:00000200:25.0:1619677445.248577:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.248577:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.248578:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.248640:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.248642:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:25.0:1619677445.248643:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:9.0:1619677445.248643:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.248646:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.248649:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.248661:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (21)++
00000800:00000200:25.0:1619677445.248663:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:25.0:1619677445.248664:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.248664:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.248665:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.248665:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.249002:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.249005:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.249006:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.249007:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.249010:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.249012:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:9.0:1619677445.249013:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.249015:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.249016:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.249017:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:9.0:1619677445.249020:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.249022:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:9.0:1619677445.249025:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ec00] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.249026:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91978412be00] (20)++
00000800:00000200:9.0:1619677445.249027:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91978412be00] (21)++
00000800:00000200:9.0:1619677445.249029:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91978412be00] (22)--
00000800:00000200:9.0:1619677445.249029:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.249031:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:9.0:1619677445.249032:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:9.0:1619677445.249032:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.249049:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.249051:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.249052:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.249053:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.249558:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.249561:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:9.0:1619677445.249561:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:9.0:1619677445.249562:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.249564:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.249567:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.250679:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.250681:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.250682:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.250682:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:25.0:1619677445.250684:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.250687:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.251664:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.251667:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.251667:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:9.0:1619677445.251671:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.251674:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.251677:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:9.0:1619677445.251680:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.251681:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.251683:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.251685:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:9.0:1619677445.251690:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.251693:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:9.0:1619677445.251696:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.251697:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:9.0:1619677445.251698:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:9.0:1619677445.251700:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:9.0:1619677445.251700:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.251702:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.251703:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.252599:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.252602:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.252603:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.252605:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.252608:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.252610:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:25.0:1619677445.252611:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.252612:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.252613:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.252614:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:25.0:1619677445.252617:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.252620:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.0:1619677445.252623:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.252624:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:25.0:1619677445.252624:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:25.0:1619677445.252626:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:25.0:1619677445.252627:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.252628:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.252628:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.252986:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.252988:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.252989:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.252990:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.252994:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.252998:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.253132:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.253134:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.253135:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.253137:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.253139:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.253141:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.253142:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.253144:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.253145:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.253146:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:25.0:1619677445.253149:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.253151:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.253154:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.253155:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.253156:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.253158:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.253158:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.253159:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.253159:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.253693:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.253695:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.253696:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.253698:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.253701:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.253704:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.253706:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.253707:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.253708:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.253709:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.253714:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.253716:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.253719:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.253720:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.253720:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.253722:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.253723:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.253724:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.253726:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.253726:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.253739:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.253742:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.253743:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.253744:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.254515:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.254518:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.254519:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.254521:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.254524:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.254525:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.254527:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.254528:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.254529:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.254530:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.254534:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.254536:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.254539:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.254540:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.254540:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.254542:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.254543:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.254543:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.254544:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.254597:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.254600:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.254603:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.254604:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.254605:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.254608:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.254781:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.254783:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:25.0:1619677445.254784:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:9.0:1619677445.254784:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.254786:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.254788:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.255053:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.255056:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.255057:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.255059:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.255062:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.255063:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:25.0:1619677445.255069:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.255070:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.255071:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.255072:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:25.0:1619677445.255075:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.255077:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:25.0:1619677445.255080:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780380] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.255081:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e3a00] (20)++
00000800:00000200:25.0:1619677445.255081:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e3a00] (21)++
00000800:00000200:25.0:1619677445.255083:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e3a00] (22)--
00000800:00000200:25.0:1619677445.255084:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.255085:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.255085:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.257111:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.257114:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.257115:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.257119:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.257122:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.257125:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:9.0:1619677445.257127:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96760210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.257129:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.257130:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.257132:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:9.0:1619677445.257136:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.257138:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:9.0:1619677445.257141:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.257142:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:9.0:1619677445.257143:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:9.0:1619677445.257144:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:9.0:1619677445.257145:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.257146:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.257148:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.257812:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.257815:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.257815:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.257817:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.257820:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.257822:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:25.0:1619677445.257823:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.257824:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.257825:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.257826:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:25.0:1619677445.257829:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.257831:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:25.0:1619677445.257834:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ec00] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.257835:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91978412be00] (20)++
00000800:00000200:25.0:1619677445.257836:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91978412be00] (21)++
00000800:00000200:25.0:1619677445.257837:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91978412be00] (22)--
00000800:00000200:25.0:1619677445.257838:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.257838:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.257839:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.257956:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.257958:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.257959:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.257959:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.257961:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.257963:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.257984:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.257987:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.257987:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.257987:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.257988:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.257989:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.258063:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.258065:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.258066:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.258068:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.258070:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.258072:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:9.0:1619677445.258073:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.258074:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.258075:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.258076:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:9.0:1619677445.258079:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.258081:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:9.0:1619677445.258084:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.258084:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:9.0:1619677445.258085:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:9.0:1619677445.258086:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:9.0:1619677445.258087:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.258088:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.258088:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.258287:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.258290:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.258291:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.258291:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.258298:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.258302:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.258460:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.258462:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.258463:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.258465:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.258468:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.258469:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:9.0:1619677445.258470:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.258472:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.258473:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.258474:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:9.0:1619677445.258477:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.258479:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:9.0:1619677445.258481:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ec00] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.258482:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91978412be00] (20)++
00000800:00000200:9.0:1619677445.258483:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91978412be00] (21)++
00000800:00000200:9.0:1619677445.258485:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91978412be00] (22)--
00000800:00000200:9.0:1619677445.258485:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.258486:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.258486:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.258569:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.258572:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.258572:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.258575:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.258578:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.258579:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.258581:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.258582:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.258583:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.258584:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.258587:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.258589:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.258591:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.258592:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.258593:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.258594:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.258595:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.258596:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.258597:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.258598:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.258611:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.258614:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.258615:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.258615:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.259073:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.259075:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.259076:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.259079:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.259081:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.259082:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:9.0:1619677445.259084:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.259085:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.259086:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.259087:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:25.0:1619677445.259093:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.259094:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.259095:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.259095:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:25.0:1619677445.259097:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.259097:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:9.0:1619677445.259100:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.259101:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.259101:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:9.0:1619677445.259101:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:25.0:1619677445.259102:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.259104:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:9.0:1619677445.259105:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.259106:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.259106:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.259247:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.259249:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.259250:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.259250:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.259253:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.259256:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.259626:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.259629:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.259629:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:9.0:1619677445.259631:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.259634:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.259635:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.259637:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.259638:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.259639:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.259640:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.259644:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.259645:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.259648:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.259649:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.259650:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.259651:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.259652:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.259653:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.259653:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.259874:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.259876:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.259877:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.259877:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.259879:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.259882:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.259899:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:9.0:1619677445.259902:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.259903:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.259903:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.259904:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.259905:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.260163:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.260166:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.260167:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.260167:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.260169:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.260172:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.260593:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.260596:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.260596:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:9.0:1619677445.260598:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.260601:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.260603:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:9.0:1619677445.260605:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.260607:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.260608:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.260609:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:9.0:1619677445.260613:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.260616:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:9.0:1619677445.260619:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.260620:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:9.0:1619677445.260621:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:9.0:1619677445.260623:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:9.0:1619677445.260623:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.260625:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:9.0:1619677445.260625:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:9.0:1619677445.260626:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.260639:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:9.0:1619677445.260643:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.260647:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.260649:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.260758:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.260760:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.260761:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.260764:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.260766:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.260768:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.260770:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.260771:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.260772:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.260773:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.260776:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.260778:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.260781:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.260782:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.260783:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.260785:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.260786:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.260787:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.260788:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.260932:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:25.0:1619677445.260935:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:9.0:1619677445.260936:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:25.0:1619677445.260938:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.260941:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.260942:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:25.0:1619677445.260943:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.260945:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.260946:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.260946:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:25.0:1619677445.260949:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.260950:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.260952:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f300] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.260952:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980000] (20)++
00000800:00000200:25.0:1619677445.260953:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.260954:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.260955:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:25.0:1619677445.260956:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:25.0:1619677445.260956:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.0:1619677445.260962:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.260964:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.260965:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.260965:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.260967:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.260968:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.261821:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.261824:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.261825:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.261827:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.261829:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.261830:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.261831:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.261833:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.261834:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.261835:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.261838:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.261840:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.261843:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.261843:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.261844:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.261846:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.261846:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.261847:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.261847:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.263283:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.263286:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.263286:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:9.0:1619677445.263289:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.263291:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.263300:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:9.0:1619677445.263302:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.263304:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.263305:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.263307:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:9.0:1619677445.263311:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.263312:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:9.0:1619677445.263315:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780380] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.263316:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e3a00] (20)++
00000800:00000200:9.0:1619677445.263317:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e3a00] (21)++
00000800:00000200:9.0:1619677445.263318:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e3a00] (22)--
00000800:00000200:9.0:1619677445.263319:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.263320:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.263321:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.263420:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.263422:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.263423:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.263425:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.263428:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.263430:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:25.0:1619677445.263431:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.263432:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.263433:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.263434:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:25.0:1619677445.263438:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.263439:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.0:1619677445.263442:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.263443:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:25.0:1619677445.263443:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:25.0:1619677445.263445:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:25.0:1619677445.263445:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.263446:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.263448:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.263448:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.263461:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.263463:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.263464:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.263464:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.263545:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.263548:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.263549:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.263549:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.263551:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.263555:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.264002:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.264004:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.264005:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.264007:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.264009:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.264011:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.264013:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.264014:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.264015:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.264016:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.264020:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.264022:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.264025:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.264025:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.264026:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.264028:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.264028:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.264029:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.264030:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.264183:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.264186:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.264187:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.264187:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.264190:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.264192:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.264202:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.264205:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:9.0:1619677445.264205:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.264206:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.264206:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.264207:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.264260:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.264262:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.264263:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.264265:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.264267:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.264269:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:9.0:1619677445.264270:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.264271:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.264272:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.264273:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:9.0:1619677445.264277:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.264278:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:9.0:1619677445.264281:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c2502380] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.264282:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (20)++
00000800:00000200:9.0:1619677445.264282:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:9.0:1619677445.264284:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:9.0:1619677445.264284:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.264285:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.264285:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.264502:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.264506:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.264506:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.264511:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.264513:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.264514:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:25.0:1619677445.264516:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.264518:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.264519:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.264520:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:25.0:1619677445.264524:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.264526:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.0:1619677445.264529:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.264530:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:25.0:1619677445.264530:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:25.0:1619677445.264532:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:25.0:1619677445.264533:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.264533:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.264534:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.265054:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.265056:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.265058:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.265062:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.265065:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.265067:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.265070:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.265072:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.265073:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.265075:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.265080:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.265082:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.265085:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.265086:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.265087:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.265089:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.265089:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.265090:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.265091:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.265193:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.265196:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.265196:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.265197:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.265199:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.265202:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.265447:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.265450:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.265451:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.265452:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.265455:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.265457:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:9.0:1619677445.265458:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.265459:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.265460:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.265461:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:9.0:1619677445.265465:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.265467:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:9.0:1619677445.265470:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.265471:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:9.0:1619677445.265471:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:9.0:1619677445.265473:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:9.0:1619677445.265473:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.265475:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:9.0:1619677445.265476:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:9.0:1619677445.265477:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.265489:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.265491:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.265492:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.265493:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.265902:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.265905:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.265906:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.265908:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.265911:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.265912:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.265914:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96760057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.265916:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.265917:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.265918:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.265922:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.265924:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.265927:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.265928:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.265928:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.265930:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.265931:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.265932:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.265933:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.265933:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.265946:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.265948:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.265949:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.265950:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.265983:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:25.0:1619677445.265986:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:9.0:1619677445.265987:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:25.0:1619677445.265988:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.265991:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.265993:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.265994:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.265995:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.266000:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.266001:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:25.0:1619677445.266004:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.266006:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.266009:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.266011:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.266011:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.266013:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.266014:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:25.0:1619677445.266014:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:25.0:1619677445.266015:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.0:1619677445.266124:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.266127:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.266128:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.266128:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.266132:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.266135:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.266261:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.266263:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.266264:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.266264:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.266267:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.266270:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.266285:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (21)++
00000800:00000200:25.0:1619677445.266288:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:9.0:1619677445.266288:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:9.0:1619677445.266289:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.266290:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983e00] (22)--
00000800:00000200:9.0:1619677445.266291:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.266411:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.266413:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:9.0:1619677445.266414:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.266415:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.266417:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.266420:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.266853:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.266857:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.266860:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.266883:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.266886:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[2] from 10.0.10.109@o2ib7
00000800:00000200:9.0:1619677445.266886:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.266889:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.266891:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.266891:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.266904:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.266907:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.266908:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.266908:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.266912:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: REPLY: OK
00000800:00000200:9.0:1619677445.266922:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.267777:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.267780:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.267781:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.267786:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.267789:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.267791:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.267793:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96760013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.267796:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.267797:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.267803:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:25.0:1619677445.267808:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.267810:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.267814:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.267815:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.267816:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.267826:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.267827:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.267828:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.267829:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.268357:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.268360:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.268361:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.268365:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.268369:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.268370:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:9.0:1619677445.268373:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:508, d:10, s:-96759992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.268376:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.268379:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.268380:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:9.0:1619677445.268386:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.268390:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.0:1619677445.268394:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.268395:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:9.0:1619677445.268396:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:9.0:1619677445.268397:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:9.0:1619677445.268398:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.268399:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.268400:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.268500:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.268503:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.268503:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.268506:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.268508:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.268510:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:25.0:1619677445.268511:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96759986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.268512:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.268513:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.268514:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:25.0:1619677445.268518:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.268519:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:25.0:1619677445.268522:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ec00] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.268523:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91978412be00] (20)++
00000800:00000200:25.0:1619677445.268524:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91978412be00] (21)++
00000800:00000200:25.0:1619677445.268525:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91978412be00] (22)--
00000800:00000200:25.0:1619677445.268526:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.268531:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.268532:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.269063:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.269065:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.269065:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.269067:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.269068:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.269069:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.269071:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:9.0:1619677445.269073:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.269075:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.269076:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.269078:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.269079:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.269080:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.269082:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.269083:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.269084:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000400:00000200:9.0:1619677445.269086:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.269088:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.269092:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.269092:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.269093:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.269095:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.269095:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.269096:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.269096:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.269773:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.269777:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.269778:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.269780:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.269783:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.269784:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.269785:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.269787:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.269787:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.269789:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:25.0:1619677445.269792:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.269794:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.269797:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.269798:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.269799:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.269800:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.269801:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.269802:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.269802:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.270485:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.270488:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.270489:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.270490:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.270492:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.270495:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.271609:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.271612:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.271613:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.271614:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.271617:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.271619:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.271620:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.271621:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.271622:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.271623:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.271626:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.271628:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.271631:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.271631:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.271632:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.271634:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.271634:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.271636:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.271637:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.271637:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.271650:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.271652:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.271653:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.271654:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.271957:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.271960:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.271961:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.271961:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.271964:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.271968:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.272715:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.272717:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.272719:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.272721:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.272722:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.272725:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.272734:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.272737:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.272737:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.272737:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.272738:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.272739:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.272773:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:25.0:1619677445.272776:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:9.0:1619677445.272776:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000400:00000200:9.0:1619677445.272778:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.272781:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.272782:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:9.0:1619677445.272783:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.272784:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.272785:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.272786:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:9.0:1619677445.272789:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.272791:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:9.0:1619677445.272792:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f300] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.272793:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980000] (20)++
00000800:00000200:9.0:1619677445.272793:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980000] (21)++
00000800:00000200:9.0:1619677445.272795:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.272795:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:9.0:1619677445.272797:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:9.0:1619677445.272798:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:9.0:1619677445.272798:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:25.0:1619677445.272803:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.272806:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.272807:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.272809:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.272809:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.272810:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.272810:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:9.0:1619677445.272810:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:25.0:1619677445.272811:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.0:1619677445.272811:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:9.0:1619677445.272819:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.272821:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.272822:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.272824:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.272825:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.272827:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:9.0:1619677445.272828:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.272829:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.272830:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.272831:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:9.0:1619677445.272834:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.272835:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:9.0:1619677445.272837:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c2502380] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.272838:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (20)++
00000800:00000200:9.0:1619677445.272838:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:9.0:1619677445.272840:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:9.0:1619677445.272841:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.272841:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.272841:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.272962:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.272964:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.272965:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.272967:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.272969:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.272971:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.272972:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.272974:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.272975:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.272976:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.272980:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.272982:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.272984:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.272985:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.272985:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.272987:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.272988:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.272988:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.272988:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.273457:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.273460:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.273460:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.273461:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.273463:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.273466:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.274068:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.274070:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.274071:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.274072:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.274074:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.274076:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.274407:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.274410:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.274411:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.274413:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.274416:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.274418:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:9.0:1619677445.274419:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96759883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.274421:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.274423:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.274430:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:9.0:1619677445.274433:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.274435:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.0:1619677445.274439:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.274439:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (23)++
00000800:00000200:9.0:1619677445.274440:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (24)++
00000800:00000200:9.0:1619677445.274443:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (26)--
00000800:00000200:9.0:1619677445.274443:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.274445:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.274445:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.275519:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.275522:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.275523:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.275525:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.275527:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.275529:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.275530:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.275531:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.275533:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.275533:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:25.0:1619677445.275537:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.275539:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.275542:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.275543:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.275544:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.275546:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.275546:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.275547:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.275549:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.275549:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.275562:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:9.0:1619677445.275564:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:9.0:1619677445.275565:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.275566:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.275718:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.275720:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.275721:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.275723:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.275726:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.275728:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.275729:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.275731:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.275731:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.275733:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.275736:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.275738:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.275741:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.275742:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.275742:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.275744:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.275744:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.275745:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.275745:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.276289:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.276292:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.276292:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.276300:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.276303:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.276305:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:25.0:1619677445.276306:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.276308:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.276309:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.276310:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:25.0:1619677445.276313:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.276315:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:25.0:1619677445.276318:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ec00] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.276318:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91978412be00] (20)++
00000800:00000200:25.0:1619677445.276319:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91978412be00] (21)++
00000800:00000200:25.0:1619677445.276321:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91978412be00] (22)--
00000800:00000200:25.0:1619677445.276322:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.276322:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.276323:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.276423:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.276426:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.276427:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.276427:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.276430:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.276433:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.276441:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.276444:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.276445:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.276445:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.276446:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.276447:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.277336:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.277339:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:25.0:1619677445.277340:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:9.0:1619677445.277340:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.277343:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.277346:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.277396:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.277402:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.277403:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.277403:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.277405:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.277408:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.277776:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.277779:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:25.0:1619677445.277780:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:9.0:1619677445.277780:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.277782:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.277785:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.277797:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (21)++
00000800:00000200:25.0:1619677445.277800:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:9.0:1619677445.277801:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.277802:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983e00] (21)++
00000800:00000200:25.0:1619677445.277803:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.277803:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.277983:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.277986:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.277987:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.277987:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.277989:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.277991:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.278667:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.278670:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.278671:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.278673:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.278676:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.278677:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.278679:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96759815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.278680:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.278681:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.278682:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.278686:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.278688:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.278692:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.278692:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.278693:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.278695:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.278696:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.278697:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.278699:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.278699:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.278712:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.278714:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.278715:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.278716:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.278844:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.278847:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.278848:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.278849:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.278851:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.278854:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.279131:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.279133:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.279134:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.279137:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.279139:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.279143:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.279151:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:9.0:1619677445.279154:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.279157:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.279157:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.279158:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.279159:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.279557:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.279560:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.279562:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.279563:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.279566:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.279567:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.279569:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.279570:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.279571:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.279572:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.279575:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.279576:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.279579:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.279580:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.279581:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.279582:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.279583:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.279584:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.279584:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.279870:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.279873:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.279874:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.279876:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.279879:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.279881:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:25.0:1619677445.279882:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.279883:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.279885:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.279885:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:25.0:1619677445.279889:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.279890:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:25.0:1619677445.279893:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780380] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.279894:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e3a00] (20)++
00000800:00000200:25.0:1619677445.279894:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e3a00] (21)++
00000800:00000200:25.0:1619677445.279896:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e3a00] (22)--
00000800:00000200:25.0:1619677445.279897:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.279898:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.279898:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.280320:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.280323:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.280324:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.280326:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.280329:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.280334:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.280335:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.280337:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.280338:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.280339:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.280343:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.280345:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.280348:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.280349:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.280350:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.280352:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.280352:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.280353:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.280354:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.281933:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.281936:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.281937:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.281937:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.281940:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.281943:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.281987:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.281989:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.281990:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.281992:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.281995:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.281996:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.282066:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.282069:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.282070:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.282071:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.282074:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.282076:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.282077:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.282078:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.282079:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.282080:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.282083:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.282085:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.282088:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.282089:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.282089:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.282091:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.282092:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.282092:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.282093:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.282189:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.282192:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:25.0:1619677445.282193:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:9.0:1619677445.282193:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.282195:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.282198:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.282666:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.282668:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.282669:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.282669:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.282671:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.282673:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.282879:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.282882:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:25.0:1619677445.282883:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (23)--
00000400:00000200:9.0:1619677445.282883:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.282885:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.282888:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.282934:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.282937:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.282938:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.282938:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.282941:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.282943:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.282959:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:9.0:1619677445.282961:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.282962:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.282962:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.282963:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.282964:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.283072:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.283074:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:9.0:1619677445.283075:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:25.0:1619677445.283076:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.283078:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:25.0:1619677445.283080:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.283106:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.283109:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.283109:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:9.0:1619677445.283111:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.283114:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.283116:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.283117:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.283118:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.283120:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.283120:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.283124:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.283125:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.283128:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.283129:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.283129:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.283131:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.283132:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.283132:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.283133:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.283226:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.283229:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.283230:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.283232:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.283235:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.283237:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.283239:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.283240:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.283241:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.283242:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.283247:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.283250:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.283253:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.283254:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.283254:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.283256:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.283257:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.283258:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.283259:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.283260:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.283274:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.283276:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.283277:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.283279:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.283516:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.283519:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.283520:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.283520:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.283522:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.283525:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.283785:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.283788:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.283789:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.283789:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:25.0:1619677445.283791:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.283794:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.283802:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (21)++
00000800:00000200:25.0:1619677445.283805:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:9.0:1619677445.283805:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:9.0:1619677445.283806:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.283807:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983e00] (22)--
00000800:00000200:9.0:1619677445.283807:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.283904:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.283906:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.283907:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.283907:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.283909:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.283910:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:9.0:1619677445.283911:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:25.0:1619677445.283912:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.283913:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.283914:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.283915:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.283917:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:9.0:1619677445.283918:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.283920:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.283920:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.283921:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:9.0:1619677445.283925:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.283926:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:9.0:1619677445.283929:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ec00] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.283930:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91978412be00] (20)++
00000800:00000200:9.0:1619677445.283931:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91978412be00] (21)++
00000800:00000200:9.0:1619677445.283932:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91978412be00] (22)--
00000800:00000200:9.0:1619677445.283933:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.283934:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:9.0:1619677445.283936:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:9.0:1619677445.283936:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.283948:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.283951:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.283951:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.283952:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.284756:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.284759:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.284760:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.284762:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.284765:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.284767:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.284768:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96759689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.284774:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.284775:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.284777:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:25.0:1619677445.284780:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.284782:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.284785:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.284786:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.284786:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.284788:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.284789:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.284789:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.284790:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.285376:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.285379:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.285380:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.285382:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.285385:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.285386:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:9.0:1619677445.285387:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.285389:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.285390:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.285391:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:9.0:1619677445.285395:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.285396:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.0:1619677445.285399:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.285400:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:9.0:1619677445.285401:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:9.0:1619677445.285402:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:9.0:1619677445.285403:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.285404:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.285404:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.285667:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.285669:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.285670:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.285673:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.285676:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.285677:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:25.0:1619677445.285679:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.285681:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.285682:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.285683:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:25.0:1619677445.285687:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.285689:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:25.0:1619677445.285691:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c2502380] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.285692:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (20)++
00000800:00000200:25.0:1619677445.285693:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:25.0:1619677445.285694:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.285695:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.285695:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.285696:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.286225:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.286228:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:9.0:1619677445.286231:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:25.0:1619677445.286234:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.286234:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.286236:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:9.0:1619677445.286238:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96759664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.286239:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.286241:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.286242:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:9.0:1619677445.286246:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.286247:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.0:1619677445.286250:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.286251:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:9.0:1619677445.286252:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:9.0:1619677445.286253:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:9.0:1619677445.286254:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.286255:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.286255:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.287365:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.287368:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.287370:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.287371:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.287380:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.287382:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.287383:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96759648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.287384:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.287385:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.287386:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.287390:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.287392:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.287395:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.287396:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.287397:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.287398:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.287399:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.287400:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.287400:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.287821:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.287824:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.287825:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.287826:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.287829:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.287832:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.288283:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.288285:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.288286:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.288288:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.288291:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.288292:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:25.0:1619677445.288299:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.288301:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.288301:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.288302:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:25.0:1619677445.288306:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.288307:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.0:1619677445.288310:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.288311:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:25.0:1619677445.288311:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:25.0:1619677445.288313:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:25.0:1619677445.288314:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.288315:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.288317:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.288317:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.288331:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.288333:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.288333:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.288334:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.288960:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.288962:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.288963:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.288966:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.288969:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.288970:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.289147:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.289150:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.289151:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.289153:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.289155:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.289157:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:9.0:1619677445.289158:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.289160:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.289161:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.289162:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:9.0:1619677445.289165:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.289167:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.0:1619677445.289170:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.289171:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:9.0:1619677445.289172:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:9.0:1619677445.289174:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:9.0:1619677445.289174:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.289176:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.289176:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.289210:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.289213:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.289214:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.289223:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.289226:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.289229:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.289230:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96759614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.289232:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.289233:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.289235:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.289239:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.289241:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.289244:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.289245:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.289250:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.289251:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.289252:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.289253:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.289253:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.289487:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.289490:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.289491:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.289491:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.289493:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.289495:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.289507:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.289510:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.289510:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.289510:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.289511:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.289512:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.289941:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.289945:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.289945:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:9.0:1619677445.289948:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.289950:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.289952:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:9.0:1619677445.289954:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96759600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.289955:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.289956:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.289957:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:9.0:1619677445.289961:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.289963:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:9.0:1619677445.289971:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.289971:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)++
00000800:00000200:9.0:1619677445.289972:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (23)++
00000800:00000200:9.0:1619677445.289975:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (24)--
00000800:00000200:9.0:1619677445.289976:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.289977:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:9.0:1619677445.289978:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:9.0:1619677445.289979:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.289991:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.289993:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.289995:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.289996:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.290471:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.290473:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.290474:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.290476:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.290479:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.290481:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.290482:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.290483:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.290484:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.290485:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.290488:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.290489:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.290492:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.290493:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.290494:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.290495:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.290496:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.290497:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.290497:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.291232:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.291236:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.291241:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:9.0:1619677445.291243:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.291245:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.291246:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.291247:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96759581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.291249:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.291249:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.291251:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.291259:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.291260:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.291263:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.291264:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.291264:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.291266:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.291267:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.291268:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.291268:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.291994:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.291997:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.291998:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.292000:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.292003:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.292004:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.292005:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.292007:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.292008:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.292009:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.292012:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.292013:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.292017:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.292017:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.292018:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.292020:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.292020:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.292021:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.292023:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.292023:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.292035:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.292038:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.292038:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.292039:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.292441:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.292444:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:9.0:1619677445.292447:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.292451:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.292452:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.292454:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96759564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.0:1619677445.292455:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.292456:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.292457:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.292459:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.292463:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.292465:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.292468:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.292469:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.292469:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.292471:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.292472:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.292473:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.292474:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.292765:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.292768:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:9.0:1619677445.292768:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:25.0:1619677445.292769:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.292771:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:25.0:1619677445.292774:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.293149:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.293152:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:9.0:1619677445.293154:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:25.0:1619677445.293158:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.293158:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.293161:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.293162:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96759554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.293164:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.293165:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.293167:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.293171:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.293175:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.293179:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.293180:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.293181:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.293183:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.293183:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.293184:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.293185:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.293195:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.293197:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.293198:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.293199:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.293201:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.293205:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.293601:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.293604:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.293605:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.293605:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.293607:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.293610:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.293787:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.293789:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.293790:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.293790:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.293792:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.293794:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.293827:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.293830:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.293830:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.293831:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.293833:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.293836:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.293847:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.293849:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.293850:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.293850:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.293851:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.293852:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.294442:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.294445:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:25.0:1619677445.294446:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:9.0:1619677445.294446:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.294448:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.294452:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.294594:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.294596:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.294598:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.294598:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.294601:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.294603:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.294813:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.294816:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.294817:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.294819:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.294821:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.294823:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.294824:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96759516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.294825:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.294826:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.294827:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.294830:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.294833:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.294836:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.294837:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.294837:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.294840:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.294840:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.294841:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.294842:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.295160:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.295163:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.295164:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.295164:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.295166:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.295169:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.295961:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.295964:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.295964:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:9.0:1619677445.295967:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.295969:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.295971:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:9.0:1619677445.295973:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.295974:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.295976:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.295977:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:9.0:1619677445.295981:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.295983:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:9.0:1619677445.295987:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ec00] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.295987:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91978412be00] (20)++
00000800:00000200:9.0:1619677445.295988:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91978412be00] (21)++
00000800:00000200:9.0:1619677445.295990:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91978412be00] (22)--
00000800:00000200:9.0:1619677445.295990:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.295991:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.295992:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.296166:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.296169:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.296170:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.296173:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.296180:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.296182:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.296184:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.296185:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.296186:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.296188:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.296191:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.296193:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.296196:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.296196:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.296197:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.296199:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.296199:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.296200:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.296201:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.297149:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.297151:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.297154:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.297155:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.297158:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.297160:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.297161:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96759467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.297163:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.297163:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.297164:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.297168:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.297169:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.297172:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.297173:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.297173:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.297175:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.297175:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.297177:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.297178:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.297179:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.297191:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.297194:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.297194:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.297195:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.297273:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.297275:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.297276:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.297279:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.297281:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.297283:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:25.0:1619677445.297284:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96759465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.297285:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.297286:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.297287:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:25.0:1619677445.297290:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.297292:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:25.0:1619677445.297301:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1200] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.297302:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38000] (20)++
00000800:00000200:25.0:1619677445.297303:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38000] (21)++
00000800:00000200:25.0:1619677445.297304:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38000] (22)--
00000800:00000200:25.0:1619677445.297305:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.297306:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.297306:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.297689:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.297692:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.297693:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.297693:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.297695:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.297698:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.297949:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.297952:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.297953:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.297955:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.297957:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.297959:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.297960:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.297961:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.297962:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.297963:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.297966:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.297967:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.297970:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.297971:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.297971:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.297973:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.297974:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.297975:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.297975:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.298330:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.298332:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.298333:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.298334:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.298336:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.298338:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.298351:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.298354:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.298355:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.298355:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.298356:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.298356:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.298488:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.298491:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.298491:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:9.0:1619677445.298494:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.298496:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.298498:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:9.0:1619677445.298499:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.298500:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.298502:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.298503:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:9.0:1619677445.298506:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.298508:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:9.0:1619677445.298511:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.298512:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:9.0:1619677445.298512:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:9.0:1619677445.298514:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:9.0:1619677445.298514:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.298516:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:9.0:1619677445.298517:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:9.0:1619677445.298518:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.298530:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.298532:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.298533:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.298534:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.298751:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.298753:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.298754:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.298755:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.298757:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.298760:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.299107:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.299110:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.299111:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.299115:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.299118:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.299119:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.299121:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96759434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.299122:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.299123:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.299125:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.299128:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.299130:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.299138:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.299139:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.299140:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.299141:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.299142:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.299143:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.299143:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.300186:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.300190:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.300190:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:9.0:1619677445.300194:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.300198:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.300200:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.300202:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96759412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.300204:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.300205:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.300206:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.300211:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.300214:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.300217:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.300218:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.300219:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.300220:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.300221:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.300222:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.300223:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.301741:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.301743:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.301744:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.301748:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.301751:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.301752:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.301753:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96759388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.301755:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.301756:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.301757:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.301761:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.301763:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.301771:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.301772:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.301772:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.301774:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.301775:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.301776:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.301777:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.301778:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.301790:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.301792:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.301793:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.301794:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.302713:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.302716:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.302717:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.302717:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.302719:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.302723:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.302817:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.302820:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.302820:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:9.0:1619677445.302823:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.302825:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.302828:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:9.0:1619677445.302829:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.302830:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.302831:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.302832:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:9.0:1619677445.302836:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.302837:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.0:1619677445.302840:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.302841:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:9.0:1619677445.302842:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:9.0:1619677445.302844:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:9.0:1619677445.302844:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.302845:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.302846:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.303569:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.303570:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.303571:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.303571:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.303573:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.303573:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.303574:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:9.0:1619677445.303576:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.303578:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.303579:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96759358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.303580:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.303582:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.303582:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.303584:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.303585:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.303586:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000400:00000200:9.0:1619677445.303586:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.303588:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.303591:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.303592:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.303593:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.303594:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.303595:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.303595:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.303596:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.303675:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.303678:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.303678:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.303681:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.303684:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.303687:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.303688:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96759354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.303689:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.303690:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.303691:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.303696:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.303698:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.303701:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.303702:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.303703:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.303705:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.303705:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.303706:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.303706:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.304227:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.304230:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.304231:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.304231:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.304233:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.304236:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.304249:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.304252:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:9.0:1619677445.304252:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.304253:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.304254:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.304254:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.304583:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.304585:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.304586:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.304589:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.304592:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.304593:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.304595:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.304597:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.304598:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.304600:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.304605:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.304608:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.304611:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.304611:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.304612:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.304614:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.304615:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.304616:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.304616:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.304640:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.304642:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.304643:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.304645:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.304648:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.304649:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:25.0:1619677445.304651:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.304652:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.304653:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.304654:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:25.0:1619677445.304658:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.304659:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:25.0:1619677445.304662:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ec00] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.304663:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91978412be00] (20)++
00000800:00000200:25.0:1619677445.304663:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91978412be00] (21)++
00000800:00000200:25.0:1619677445.304665:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91978412be00] (22)--
00000800:00000200:25.0:1619677445.304666:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.304667:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.304668:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.305454:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.305457:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (23)--
00000800:00000200:9.0:1619677445.305457:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:9.0:1619677445.305458:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.305460:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.305463:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.305473:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (21)++
00000800:00000200:25.0:1619677445.305476:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:25.0:1619677445.305477:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.305477:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.305478:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.305479:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.305774:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.305777:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.305778:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.305780:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.305783:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.305784:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:9.0:1619677445.305786:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.305788:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.305789:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.305790:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:9.0:1619677445.305797:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.305799:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:9.0:1619677445.305803:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.305803:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:9.0:1619677445.305804:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:9.0:1619677445.305806:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:9.0:1619677445.305807:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.305808:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.305808:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.305846:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.305849:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.305850:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.305853:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.305856:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.305857:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.305859:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.305860:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.305861:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.305862:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.305866:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.305868:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.305871:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.305871:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.305872:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.305874:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.305874:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.305876:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.305877:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.305878:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.305890:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.305892:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.305893:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.305894:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.306537:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.306540:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.306541:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.306541:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.306544:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.306547:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.306746:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.306749:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.306750:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.306751:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.306754:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.306755:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.306757:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.306758:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.306759:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.306760:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.306763:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.306765:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.306767:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.306768:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.306769:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.306770:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.306771:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.306772:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:9.0:1619677445.306773:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:9.0:1619677445.306774:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.306786:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.306788:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.306789:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.306791:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.307116:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.307118:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.307119:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.307119:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.307122:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.307124:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.307488:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.307492:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.307496:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.307497:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.307501:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.307505:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.307518:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.307521:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.307522:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.307522:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.307524:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.307526:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.307541:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:25.0:1619677445.307542:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:9.0:1619677445.307542:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:25.0:1619677445.307544:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.307545:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000400:00000200:9.0:1619677445.307545:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000800:00000200:25.0:1619677445.307546:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.307546:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.307547:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:25.0:1619677445.307548:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000400:00000200:9.0:1619677445.307548:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.307550:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:9.0:1619677445.307552:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.307553:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.307554:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.307555:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:9.0:1619677445.307559:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.307560:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:9.0:1619677445.307563:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8680] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.307564:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71ba00] (20)++
00000800:00000200:9.0:1619677445.307564:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71ba00] (21)++
00000800:00000200:9.0:1619677445.307566:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71ba00] (22)--
00000800:00000200:9.0:1619677445.307567:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:9.0:1619677445.307567:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:9.0:1619677445.307568:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:25.0:1619677445.307661:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.307663:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.307664:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.307666:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.307668:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.307670:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:25.0:1619677445.307671:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.307672:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.307673:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.307674:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:25.0:1619677445.307677:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.307678:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:25.0:1619677445.307681:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c2502380] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.307681:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (20)++
00000800:00000200:25.0:1619677445.307682:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:25.0:1619677445.307683:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.307684:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.307684:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.307685:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.308122:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.308125:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.308126:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.308128:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.308130:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.308131:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:9.0:1619677445.308133:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.308134:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.308135:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.308136:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:9.0:1619677445.308139:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.308140:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:9.0:1619677445.308143:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c2502380] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.308143:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (20)++
00000800:00000200:9.0:1619677445.308144:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:9.0:1619677445.308145:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:9.0:1619677445.308146:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.308147:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.308147:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.308234:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.308237:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.308238:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.308240:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.308242:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.308244:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.308245:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.308247:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.308248:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.308249:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:25.0:1619677445.308253:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.308254:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.308257:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.308258:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.308259:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.308260:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.308261:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.308261:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.308262:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.308633:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.308636:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.308637:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.308637:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.308639:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.308642:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.308656:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.308658:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.308659:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.308661:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.308664:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.308665:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:25.0:1619677445.308667:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.308668:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.308669:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.308670:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:25.0:1619677445.308673:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.308674:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:25.0:1619677445.308677:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c2502380] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.308678:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (20)++
00000800:00000200:25.0:1619677445.308678:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:25.0:1619677445.308680:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.308680:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.308681:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.308681:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.309236:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.309239:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.309241:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.0:1619677445.309242:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.309243:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.309246:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.309256:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.309259:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.309260:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.309262:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.309265:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.309266:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:25.0:1619677445.309267:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.309268:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.309270:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.309271:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:25.0:1619677445.309274:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.309276:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:25.0:1619677445.309279:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ec00] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.309280:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91978412be00] (20)++
00000800:00000200:25.0:1619677445.309280:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91978412be00] (21)++
00000800:00000200:25.0:1619677445.309282:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91978412be00] (22)--
00000800:00000200:25.0:1619677445.309283:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.309284:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.309285:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.309285:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.309300:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:9.0:1619677445.309303:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:9.0:1619677445.309304:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.309305:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.309474:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.309477:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.309478:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.309480:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.309484:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.309485:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.309487:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96759253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.309489:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.309490:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.309491:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.309499:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.309501:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.309504:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.309505:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.309506:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.309508:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.309509:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.309509:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.309510:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.309772:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.309775:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.309776:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.309776:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.309779:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.309781:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.310318:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.310321:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.310322:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.310323:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:25.0:1619677445.310324:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.310327:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.310502:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.310505:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.310506:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.310508:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.310511:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.310513:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.310513:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.310515:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.310517:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.310518:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.310522:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.310524:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.310527:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.310528:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.310528:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.310530:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.310531:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.310532:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.310533:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.311142:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.311145:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.311146:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.311146:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:25.0:1619677445.311149:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.311153:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.311478:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.311481:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.311482:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.311483:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.311486:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.311487:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.311488:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.311490:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.311491:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.311491:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.311495:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.311497:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.311500:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.311501:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.311502:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.311503:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.311504:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.311505:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.311507:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.311507:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.311524:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.311526:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.311527:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.311528:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.311551:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.311554:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:9.0:1619677445.311554:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:25.0:1619677445.311555:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.311557:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.311560:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.311851:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.311854:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.311855:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.311855:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.311857:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.311860:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.311869:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.311871:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.311872:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.311872:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.311873:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.311873:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.312146:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.312149:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000800:00000200:25.0:1619677445.312150:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (23)--
00000400:00000200:9.0:1619677445.312150:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.312153:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.312156:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.312165:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (21)++
00000800:00000200:25.0:1619677445.312168:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:25.0:1619677445.312169:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983e00] (22)++
00000800:00000200:9.0:1619677445.312169:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.312170:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.312170:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.312467:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.312470:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.312470:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.312471:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.312473:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.312475:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.312545:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.312548:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.0:1619677445.312548:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:25.0:1619677445.312551:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.312553:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.312555:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:25.0:1619677445.312556:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96759200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.312557:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.312558:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.312559:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:25.0:1619677445.312562:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.312564:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:25.0:1619677445.312572:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c2502380] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.312572:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (20)++
00000800:00000200:25.0:1619677445.312573:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:25.0:1619677445.312574:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.312575:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.312575:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.312576:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.0:1619677445.312673:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.312675:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.312676:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.312677:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.312679:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.312681:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.312887:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.312889:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.312890:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.312890:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.312893:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.312896:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.313060:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.313063:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.313064:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.313065:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.313067:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.313068:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:9.0:1619677445.313069:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96759190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.313071:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.313072:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.313073:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:9.0:1619677445.313079:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.313081:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.0:1619677445.313084:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.313084:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:9.0:1619677445.313085:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:9.0:1619677445.313087:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:9.0:1619677445.313087:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.313088:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.313089:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.313483:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.313485:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.313486:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.313487:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.313489:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.313492:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.313499:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:9.0:1619677445.313502:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.313503:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.313503:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.313504:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.313505:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.313566:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.313569:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.313570:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.313570:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.313572:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:25.0:1619677445.313575:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.313992:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.313995:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.313996:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.313998:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.314001:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.314002:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:9.0:1619677445.314004:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.314005:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.314006:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.314007:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:9.0:1619677445.314010:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.314012:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:9.0:1619677445.314014:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c2502380] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.314015:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (20)++
00000800:00000200:9.0:1619677445.314015:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:9.0:1619677445.314017:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:9.0:1619677445.314017:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.314018:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.314019:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.314076:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.314079:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.314079:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.314080:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.314082:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.314085:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.314371:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.314374:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.314376:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.314376:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.314379:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.314380:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.314382:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96759169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.314383:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.314385:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.314386:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.314390:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.314392:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.314394:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.314395:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.314396:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.314398:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.314399:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.314400:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.314400:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.314865:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.314868:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.314869:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.314869:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000400:00000200:25.0:1619677445.314871:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.314873:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:9.0:1619677445.315945:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.315948:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.315949:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.315952:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.315955:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.315958:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.315959:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96759142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.315961:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.315962:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.315963:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.315967:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.315969:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.315973:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.315974:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (23)++
00000800:00000200:9.0:1619677445.315976:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (26)++
00000800:00000200:9.0:1619677445.315980:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (24)--
00000800:00000200:9.0:1619677445.315981:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.315981:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.315982:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.316933:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.316935:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.316936:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.316936:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.316939:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.316942:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.317089:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.317093:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.0:1619677445.317094:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:9.0:1619677445.317096:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.317099:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.317100:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.317102:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.317103:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.317104:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.317105:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:9.0:1619677445.317108:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.317110:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.317113:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.317114:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:9.0:1619677445.317115:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:9.0:1619677445.317116:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:9.0:1619677445.317117:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.317118:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.317120:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.317120:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.317133:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.317135:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.317136:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.317137:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.318071:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.318074:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.318077:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:9.0:1619677445.318077:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.318078:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000400:00000200:25.0:1619677445.318080:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.318080:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.318081:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.318083:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:9.0:1619677445.318083:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:25.0:1619677445.318084:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96759106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.318086:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.318087:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.318088:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.318091:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.318093:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.0:1619677445.318094:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.318095:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.318097:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.318098:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.318098:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.318100:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.318101:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.318102:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.318102:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.318723:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.318725:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.318727:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.0:1619677445.318728:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000400:00000200:9.0:1619677445.318729:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.318730:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000400:00000200:25.0:1619677445.318732:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000800:00000200:9.0:1619677445.318733:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.318734:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:9.0:1619677445.318735:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000400:00000200:25.0:1619677445.318736:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.318736:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.318737:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (23)--
00000800:00000200:9.0:1619677445.318737:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.318738:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:25.0:1619677445.318739:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.0:1619677445.318739:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:9.0:1619677445.318740:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000400:00000200:25.0:1619677445.318741:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.318742:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.318743:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:25.0:1619677445.318747:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.318749:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:25.0:1619677445.318751:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c2502380] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.318753:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (20)++
00000800:00000200:25.0:1619677445.318753:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:25.0:1619677445.318755:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.318756:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:25.0:1619677445.318756:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:25.0:1619677445.318757:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.0:1619677445.318888:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.318891:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.318891:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:9.0:1619677445.318893:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.318896:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.318897:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:9.0:1619677445.318899:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.318900:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.318901:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.318902:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:9.0:1619677445.318905:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.318907:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.0:1619677445.318910:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.318911:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:9.0:1619677445.318911:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:9.0:1619677445.318913:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:9.0:1619677445.318913:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.0:1619677445.318914:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.318914:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.319718:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.319720:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.319721:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.319723:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.319726:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.319727:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.319729:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96759078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.319730:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.319731:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.319732:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:25.0:1619677445.319735:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.319736:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.319739:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.319740:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.319741:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.319742:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.319743:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.319743:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.319744:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.319786:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.319789:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:9.0:1619677445.319789:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:9.0:1619677445.319790:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.319792:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:9.0:1619677445.319795:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.320768:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.320771:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.320772:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.320773:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.320776:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.320778:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.320779:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.320781:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.320783:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.320783:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.320787:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.320789:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.320792:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.320792:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.320793:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.320795:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.320795:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.320797:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.320798:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.320798:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.320812:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.0:1619677445.320814:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.0:1619677445.320815:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.0:1619677445.320816:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.321939:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.321941:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.321942:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.0:1619677445.321942:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:25.0:1619677445.321944:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.321947:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:9.0:1619677445.323051:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:25.0:1619677445.323054:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:9.0:1619677445.323054:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000400:00000200:9.0:1619677445.323056:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.323059:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.323061:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:9.0:1619677445.323062:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.323063:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.323064:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.323065:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:9.0:1619677445.323068:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.323070:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:9.0:1619677445.323072:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56400] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.323073:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea37e00] (20)++
00000800:00000200:9.0:1619677445.323073:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea37e00] (21)++
00000800:00000200:9.0:1619677445.323075:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea37e00] (22)--
00000800:00000200:9.0:1619677445.323076:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:9.0:1619677445.323076:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:9.0:1619677445.323077:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:25.0:1619677445.323110:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.323113:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.323113:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.323115:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.323117:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.323119:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.323120:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.323121:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.323122:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.323123:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.323126:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.323127:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.323130:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.323131:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.323131:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.323133:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.323134:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.323134:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.323135:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.323399:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:9.0:1619677445.323402:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.323403:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000400:00000200:9.0:1619677445.323403:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.323405:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.0:1619677445.323409:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.324276:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.324279:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.324280:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.324281:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.324284:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.324286:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.324287:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.324288:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.324289:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.324290:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.324299:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.324301:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.324304:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.324305:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.324305:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.324307:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.324308:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.324309:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.324309:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.0:1619677445.324359:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.324362:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:25.0:1619677445.324363:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:9.0:1619677445.324366:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:9.0:1619677445.324369:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:9.0:1619677445.324371:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:9.0:1619677445.324372:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96759007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:9.0:1619677445.324374:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:9.0:1619677445.324375:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:9.0:1619677445.324376:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:9.0:1619677445.324380:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:9.0:1619677445.324382:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:9.0:1619677445.324385:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ec00] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:9.0:1619677445.324386:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91978412be00] (20)++
00000800:00000200:9.0:1619677445.324387:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91978412be00] (21)++
00000800:00000200:9.0:1619677445.324389:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91978412be00] (22)--
00000800:00000200:9.0:1619677445.324389:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.0:1619677445.324390:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.0:1619677445.324391:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.325179:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.325182:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.0:1619677445.325183:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:25.0:1619677445.325188:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.325196:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.325198:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.325200:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96758991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.325202:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.325208:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.325210:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.325218:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.325221:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.325224:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.325225:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.325225:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.325227:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.325227:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.325228:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.325229:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.2:1619677445.325250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980600] (21)++
00000800:00000200:25.0:1619677445.325265:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980600] (22)--
00000800:00000200:25.0:1619677445.325890:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.325893:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.325895:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.325898:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.325899:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.325900:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96758979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.325902:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.325903:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.325904:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.325907:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.325909:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.325911:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.325912:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.325913:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.325914:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.325915:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.325917:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.325918:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.325919:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.326679:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.327203:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:25.0:1619677445.327206:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000400:00000200:25.0:1619677445.327208:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.327211:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.327213:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:25.0:1619677445.327214:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96758958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.327216:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.327217:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.327218:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:25.0:1619677445.327221:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.327223:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.0:1619677445.327226:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.327227:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:25.0:1619677445.327227:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:25.0:1619677445.327229:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:25.0:1619677445.327230:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:25.0:1619677445.327230:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:25.0:1619677445.327231:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:25.0:1619677445.327417:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.327426:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.327429:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.327429:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.327430:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.327431:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.327432:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.328954:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.328957:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.328958:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.328960:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:25.0:1619677445.328967:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.328985:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.329027:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.329029:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.329031:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.329032:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.329046:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.329052:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.329053:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.329055:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.329057:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.329235:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.329238:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.329239:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.329240:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.329243:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.329316:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.329495:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.329498:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.329498:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.329500:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:25.0:1619677445.329502:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.329681:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.330321:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.330846:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.330848:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.330850:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.330853:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.330855:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.330856:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96758891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.330858:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.330858:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.330860:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.330863:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.330865:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.330867:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.330868:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.330869:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.330871:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.330872:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.330873:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.330875:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.330875:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.331182:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.331423:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.331426:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.331429:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.331431:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.331433:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.331435:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96758883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.331436:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.331438:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.331439:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:25.0:1619677445.331445:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.331447:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.331450:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.331451:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.331452:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.331453:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.331454:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.331455:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.331455:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.331999:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.332113:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.332115:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.332118:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.332120:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.332122:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.332123:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96758873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.332125:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.332128:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.332129:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.332132:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.332134:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.332136:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.332137:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.332138:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.332139:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.332140:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.332141:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.332141:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.332778:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.332890:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.332892:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.332894:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.332895:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.332898:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.333248:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.333261:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.333263:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.333266:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.333269:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.333271:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.333272:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96758853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.333274:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.333275:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.333276:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.333286:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.333288:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.333291:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.333292:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.333293:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.333307:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.333308:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.333309:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.333309:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.333634:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.333736:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.333738:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.333739:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.333741:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.333744:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.334328:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.334331:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.334332:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.334333:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.334336:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.334464:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.334745:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.334747:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.334748:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.334751:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.334753:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.335064:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.335100:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.335102:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.335103:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.335104:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.335415:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.335418:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.335419:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.335421:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.335424:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.336088:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.336334:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.336337:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.336338:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.336340:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.336344:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.336727:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.336730:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.336732:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.336735:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.336736:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.336738:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96758782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.336739:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.336740:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.336741:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:25.0:1619677445.336744:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.336746:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.336749:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.336750:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.336751:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.336753:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.336753:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.336754:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.336755:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.337323:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.337801:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.337804:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.337805:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.337807:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.337809:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.337859:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.337861:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.337863:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.337866:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.337868:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.337869:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96758765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.337871:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.337872:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.337873:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.337876:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.337878:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.337881:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.337882:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.337883:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.337885:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.337885:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.337887:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.337888:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.337889:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.338413:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.338470:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.338474:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.338478:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.338481:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.338483:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.338484:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96758753] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.338486:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.338487:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.338488:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.338491:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.338498:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.338501:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.338502:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.338502:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.338504:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.338505:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.338506:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.338508:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.338509:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.338657:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.338658:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.338659:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.338662:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.338666:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.338668:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.338950:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.339508:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.339510:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.339513:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.339516:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.339518:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.339519:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96758734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.339521:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.339522:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.339523:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.339526:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.339528:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.339531:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.339532:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.339532:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.339534:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.339534:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.339535:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.339536:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.340463:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.340716:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.340718:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.340719:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.340721:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.340724:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.340912:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.341511:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.341514:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.341515:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.341516:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.341519:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.341810:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.342555:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.342558:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.342561:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.342564:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.342566:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.342568:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96758683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.342569:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.342571:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.342572:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:25.0:1619677445.342576:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.342578:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.342580:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.342581:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.342582:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.342584:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.342584:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.342586:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.342588:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.342589:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.343392:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.343586:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.343588:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.343589:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.343591:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.343594:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.344073:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.344104:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.344107:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.344108:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.344109:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.344382:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.344385:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.344387:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.344389:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.344391:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:25.0:1619677445.344392:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96758653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.344393:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.344394:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.344395:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:25.0:1619677445.344398:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.344399:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.0:1619677445.344402:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.344403:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:25.0:1619677445.344403:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:25.0:1619677445.344405:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:25.0:1619677445.344406:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.344407:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.344407:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.344593:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.344603:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (21)++
00000800:00000200:25.0:1619677445.344605:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:25.0:1619677445.344606:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.344607:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983e00] (23)--
00000800:00000200:25.0:1619677445.344607:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.344609:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.345596:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.345762:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.345765:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.345767:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.345770:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.345772:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:25.0:1619677445.345774:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96758625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.345776:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.345777:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.345779:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:25.0:1619677445.345787:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.345789:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:25.0:1619677445.345792:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ec00] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.345793:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91978412be00] (21)++
00000800:00000200:25.0:1619677445.345793:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91978412be00] (22)++
00000800:00000200:25.0:1619677445.345795:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91978412be00] (23)--
00000800:00000200:25.0:1619677445.345796:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.345797:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.345797:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.345877:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.346352:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.346355:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.346357:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.346359:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.346361:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:25.0:1619677445.346362:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96758619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.346364:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.346366:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.346367:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:25.0:1619677445.346371:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.346373:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:25.0:1619677445.346376:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ec00] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.346377:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91978412be00] (20)++
00000800:00000200:25.0:1619677445.346378:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91978412be00] (21)++
00000800:00000200:25.0:1619677445.346379:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91978412be00] (22)--
00000800:00000200:25.0:1619677445.346380:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.346381:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.346381:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.347223:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.347261:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.347263:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.347264:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.347265:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.347469:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.347472:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.347474:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.347477:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.347479:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:25.0:1619677445.347480:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96758599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.347481:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.347482:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.347483:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:25.0:1619677445.347486:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.347488:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:25.0:1619677445.347491:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ec00] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.347491:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91978412be00] (20)++
00000800:00000200:25.0:1619677445.347492:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91978412be00] (21)++
00000800:00000200:25.0:1619677445.347494:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91978412be00] (22)--
00000800:00000200:25.0:1619677445.347494:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.347495:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.347495:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.347862:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:25.0:1619677445.348034:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.348037:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.348039:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.348041:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.348043:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.348044:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96758593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.348045:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.348047:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.348048:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.348051:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.348052:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.348055:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.348056:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.348056:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.348058:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.348058:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.348059:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.348060:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.348560:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.348590:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.348593:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.348595:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.348598:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.348600:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.348601:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96758580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.348603:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.348604:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.348605:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:25.0:1619677445.348608:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.348610:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.348613:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.348614:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.348614:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.348616:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.348617:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.348618:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.348618:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.349285:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.349456:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.349459:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.349461:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.349463:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.349466:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.349781:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.349820:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.349822:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.349823:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.349824:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.349844:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.349846:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.349848:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.349851:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.349852:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:25.0:1619677445.349853:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96758558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.349855:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.349856:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.349856:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:25.0:1619677445.349859:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.349861:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.0:1619677445.349864:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.349864:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:25.0:1619677445.349865:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:25.0:1619677445.349867:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:25.0:1619677445.349867:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.349868:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.349869:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.349870:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.350002:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.350260:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.350263:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.350265:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.350268:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.350270:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:25.0:1619677445.350271:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96758548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.350272:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.350273:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.350274:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:25.0:1619677445.350278:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.350280:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:25.0:1619677445.350282:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c2502380] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.350283:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (20)++
00000800:00000200:25.0:1619677445.350283:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:25.0:1619677445.350285:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.350285:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.350286:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.350287:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.350717:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.352118:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.352121:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.352124:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.352127:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.352128:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.352130:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96758515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.352131:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.352132:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.352133:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.352136:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.352137:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.352140:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.352141:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.352141:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.352143:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.352144:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.352145:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.352145:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.352411:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.352571:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.352574:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.352576:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.352578:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.352582:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.353134:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.353235:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.353238:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.353239:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.353241:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.353244:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.353457:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.353842:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.353845:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.353846:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.353848:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.353851:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.354193:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.354196:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.354198:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.354201:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.354202:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.354204:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96758484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.354205:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.354206:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.354207:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.354212:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.354215:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.354218:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.354220:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.354221:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.354222:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.354223:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.354225:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.354227:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.354228:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.354229:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8664c00] (22)--
00000800:00000200:25.0:1619677445.354240:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.354241:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.354242:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.354243:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.354257:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.354259:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.354260:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.354262:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.354264:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.354282:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (21)++
00000800:00000200:25.0:1619677445.354283:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:25.0:1619677445.354284:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.354285:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983e00] (23)--
00000800:00000200:25.0:1619677445.354285:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.354287:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8664c00] (22)--
00000800:00000200:25.0:1619677445.354288:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.354329:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8664c00] (21)++
00000800:00000200:25.0:1619677445.354332:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.54@o2ib3
00000400:00000200:25.0:1619677445.354335:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.54@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.354338:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.354339:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:25.0:1619677445.354341:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96758478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.354342:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.354344:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.354345:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:25.0:1619677445.354349:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.354351:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:25.0:1619677445.354355:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c2502380] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.354355:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (20)++
00000800:00000200:25.0:1619677445.354360:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:25.0:1619677445.354362:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.354363:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8664c00] (21)++
00000800:00000200:25.0:1619677445.354364:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fe8664c00] (22)++
00000800:00000200:25.0:1619677445.354365:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8664c00] (23)--
00000800:00000200:25.0:1619677445.354366:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8664c00] (22)--
00000800:00000200:25.0:1619677445.354386:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8664c00] (22)--
00000800:00000200:25.0:1619677445.354401:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.354402:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.354403:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.354405:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.354408:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.354408:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.354419:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.354421:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.354422:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.354424:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.354426:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.354459:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8664c00] (22)--
00000800:00000200:25.0:1619677445.354482:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8664c00] (21)++
00000800:00000200:25.0:1619677445.354485:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.54@o2ib3
00000400:00000200:25.0:1619677445.354488:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.54@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.354490:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.354491:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.354492:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96758471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.354494:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.354494:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.354495:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.354499:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.354506:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.354509:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.354510:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.354511:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.354512:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.354513:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8664c00] (21)++
00000800:00000200:25.0:1619677445.354514:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8664c00] (22)--
00000800:00000200:25.0:1619677445.354515:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8664c00] (21)--
00000800:00000200:25.0:1619677445.354593:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.354594:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.354597:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.354604:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.354605:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.354606:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96758469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.354608:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.354608:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.354610:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.354615:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.354617:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.354624:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.354625:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.354625:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.354627:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.354628:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.354628:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.354629:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.354630:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.354631:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8664c00] (23)++
00000800:00000200:25.0:1619677445.354632:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fe8664c00] (24)--
00000800:00000200:25.0:1619677445.354633:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8664c00] (23)--
00000800:00000200:25.0:1619677445.354635:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8664c00] (22)--
00000800:00000200:25.0:1619677445.354636:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.354637:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.354645:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.354648:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.354650:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.354651:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96758466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.354653:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.354654:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.354655:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.354667:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.354669:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.354672:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.354673:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.354674:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.354675:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.354679:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.354680:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.354680:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.354682:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8664c00] (22)--
00000800:00000200:25.0:1619677445.355208:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.355211:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.355213:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.355216:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.355218:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:25.0:1619677445.355219:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96758455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.355220:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.355221:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.355222:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:25.0:1619677445.355226:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.355227:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.0:1619677445.355230:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.355231:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:25.0:1619677445.355231:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:25.0:1619677445.355233:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:25.0:1619677445.355234:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.355235:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.355235:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.355501:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.355542:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.355544:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.355545:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.355546:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.355686:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.355689:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.355690:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.355691:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.355694:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.355766:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.356429:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.356432:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.356434:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.356437:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.356439:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.356440:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96758430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.356442:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.356444:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.356445:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.356449:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.356451:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.356455:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.356456:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.356456:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.356458:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.356458:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.356459:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.356460:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.356508:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.356547:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.356554:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.356555:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.356712:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.357580:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.357583:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.357584:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.357586:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.357590:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.358299:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.359282:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.359285:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.359289:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.359292:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.359302:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.359304:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96758381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.359306:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.359308:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.359309:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:25.0:1619677445.359315:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.359317:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.359321:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.359321:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.359322:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.359325:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.359326:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.359327:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.359328:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.360115:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.360283:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.360284:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.360285:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.360288:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:25.0:1619677445.360298:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (24)--
00000800:00000200:25.0:1619677445.360300:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.360302:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.360304:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.360305:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (23)++
00000800:00000200:25.0:1619677445.360306:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (24)--
00000800:00000200:25.0:1619677445.360306:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.360307:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.360314:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.360315:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.360316:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.360317:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.360320:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.360722:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.360767:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.360770:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.360771:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.360772:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.360996:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.360999:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.361000:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.361001:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.361013:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.361198:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.361444:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.361447:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.361448:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.361449:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.361452:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.361837:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.362752:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.362754:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.362755:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.362757:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.362761:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.362993:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.362996:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.362999:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.363002:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.363003:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.363005:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96758314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.363006:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.363007:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.363008:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.363011:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.363013:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.363016:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.363017:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.363018:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.363020:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.363020:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.363021:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.363022:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.363251:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.363334:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.363336:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.363338:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.363341:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.363343:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.363348:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96758311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.363350:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.363350:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.363351:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.363355:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.363356:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.363359:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.363360:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.363360:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.363362:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.363363:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.363364:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.363364:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.363639:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.363650:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (21)++
00000800:00000200:25.0:1619677445.363654:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:25.0:1619677445.363654:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.363655:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983e00] (23)--
00000800:00000200:25.0:1619677445.363657:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.364065:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.364100:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.364103:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.364103:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.364104:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.364348:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.364351:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.364352:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.364354:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.364357:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.364409:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.364445:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.364448:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.364449:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.364449:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.364881:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.364884:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.364886:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.364889:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.364895:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.364896:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96758282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.364897:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.364898:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.364899:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:25.0:1619677445.364903:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.364906:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.364909:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.364910:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.364910:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.364912:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.364913:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.364914:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.364914:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.365346:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.365373:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.365375:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.365376:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.365379:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.365382:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.365421:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.366073:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.366075:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.366077:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.366080:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.366082:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:25.0:1619677445.366083:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96758261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.366084:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.366085:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.366086:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:25.0:1619677445.366094:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.366095:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.0:1619677445.366099:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.366099:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:25.0:1619677445.366100:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:25.0:1619677445.366102:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:25.0:1619677445.366102:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.366103:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.366104:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.366152:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.366269:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.366273:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.366274:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.366276:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.366279:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.366296:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.366726:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.366728:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.366730:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.366733:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.366735:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:25.0:1619677445.366736:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96758250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.366738:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.366739:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.366740:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:25.0:1619677445.366743:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.366745:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.0:1619677445.366748:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.366749:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:25.0:1619677445.366750:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:25.0:1619677445.366751:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:25.0:1619677445.366752:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.366753:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.366753:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.367178:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.367213:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.367215:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.367216:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.367217:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.367435:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.367438:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.367439:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.367443:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.367447:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.367818:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.368011:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.368014:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.368017:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.368020:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.368022:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:25.0:1619677445.368024:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96758223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.368026:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.368026:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.368027:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:25.0:1619677445.368032:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.368035:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:25.0:1619677445.368037:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c2502380] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.368038:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (20)++
00000800:00000200:25.0:1619677445.368039:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:25.0:1619677445.368040:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.368041:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.368042:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.368042:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.368724:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.368941:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.368947:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.368949:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.368951:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.368954:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.369165:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.369464:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.369466:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.369467:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.369469:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:25.0:1619677445.369472:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.370126:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.370945:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.370948:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.370949:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.370951:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.370954:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.371015:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.371050:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.371053:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.371053:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.371054:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.371461:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.371464:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.371466:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.371469:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.371470:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:25.0:1619677445.371471:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96758167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.371473:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.371474:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.371475:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:25.0:1619677445.371481:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.371483:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:25.0:1619677445.371486:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780380] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.371487:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e3a00] (20)++
00000800:00000200:25.0:1619677445.371488:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e3a00] (21)++
00000800:00000200:25.0:1619677445.371490:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e3a00] (22)--
00000800:00000200:25.0:1619677445.371490:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.371492:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.371493:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.371499:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.372168:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.372175:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.372177:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.372181:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.372183:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.372185:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:25.0:1619677445.372187:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96758150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.372189:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.372190:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.372191:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:25.0:1619677445.372195:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.372197:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.0:1619677445.372200:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.372201:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:25.0:1619677445.372202:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:25.0:1619677445.372203:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:25.0:1619677445.372204:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.372205:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.372205:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.372545:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.372692:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.372695:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.372696:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.372698:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.372701:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.372898:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.373108:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.373111:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.373111:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.373113:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.373115:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.373152:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.374085:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.374087:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.374090:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.374093:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.374096:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.374098:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96758119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.374100:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.374101:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.374102:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.374108:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.374110:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.374114:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.374114:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.374115:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.374117:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.374117:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.374118:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.374118:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.374220:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.374410:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.374413:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.374414:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.374416:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.374419:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.374710:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.374718:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.374810:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.374812:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.374813:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.374815:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.374817:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.374838:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.374841:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.374843:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.374845:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.374846:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.374847:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96758105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.374849:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.374850:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.374851:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.374854:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.374856:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.374858:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.374859:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.374860:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.374861:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.374862:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.374864:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.374865:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.374865:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.375686:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.375735:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.375738:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.375740:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.375744:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.375745:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.375746:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96758090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.375748:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.375749:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.375750:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.375753:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.375755:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.375758:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.375759:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.375759:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.375761:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.375761:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.375763:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.375765:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.375766:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.375792:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.375845:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:25.0:1619677445.375848:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000400:00000200:25.0:1619677445.375849:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.375852:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.375854:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.375855:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96758088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.375856:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.375857:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.375859:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:25.0:1619677445.375862:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.375863:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.375866:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.375867:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.375868:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.375869:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.375870:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:25.0:1619677445.375871:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:25.0:1619677445.375872:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:25.0:1619677445.375873:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:25.0:1619677445.376204:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.376488:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.376495:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.376497:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.376500:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.376501:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.376502:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96758076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.376504:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.376505:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.376506:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.376510:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.376512:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.376515:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.376516:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.376516:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.376518:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.376519:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.376520:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.376520:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.376607:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.377068:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.377072:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.377075:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.377093:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.377095:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:25.0:1619677445.377096:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96758063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.377098:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.377099:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.377100:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:25.0:1619677445.377104:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.377106:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.0:1619677445.377109:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.377110:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:25.0:1619677445.377110:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:25.0:1619677445.377112:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:25.0:1619677445.377112:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.377113:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.377114:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.377363:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.377394:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.377397:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.377398:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.377400:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.377403:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.377607:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.377610:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.377613:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.377616:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.377618:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:25.0:1619677445.377620:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96758056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.377622:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.377623:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.377624:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:25.0:1619677445.377627:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.377629:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:25.0:1619677445.377632:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ec00] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.377633:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91978412be00] (21)++
00000800:00000200:25.0:1619677445.377633:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91978412be00] (22)++
00000800:00000200:25.0:1619677445.377635:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91978412be00] (23)--
00000800:00000200:25.0:1619677445.377636:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.377636:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.377637:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.377990:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.378291:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.378301:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.378303:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.378306:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.378308:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:25.0:1619677445.378309:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96758044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.378310:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.378316:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.378317:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:25.0:1619677445.378320:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.378323:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.0:1619677445.378326:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.378327:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:25.0:1619677445.378328:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:25.0:1619677445.378329:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:25.0:1619677445.378330:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.378332:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.378333:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.378334:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.378696:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.379872:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.379875:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.379877:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.379880:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.379887:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.379888:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96758021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.379889:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.379890:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.379891:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.379895:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.379896:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.379899:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.379900:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.379901:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.379902:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.379903:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.379904:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.379904:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.380351:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.381120:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.381123:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.381127:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.381131:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.381137:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.381138:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96757997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.381140:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.381145:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.381146:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.381150:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.381153:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.381156:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.381156:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.381157:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.381163:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.381163:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.381164:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.381165:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.381209:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.381497:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.381499:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.381500:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.381502:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:25.0:1619677445.381506:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.381682:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.382067:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.382070:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.382072:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.382075:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.382076:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.382078:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96757984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.382079:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.382080:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.382081:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.382084:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.382086:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.382089:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.382090:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.382090:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.382092:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.382093:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.382094:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.382095:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.382096:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.382168:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.382202:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.382204:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.382205:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.382206:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677445.382285:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.382292:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.382299:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.382300:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.382303:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.382305:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:13.0:1619677445.382334:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.382337:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.382338:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.382340:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.382343:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:25.0:1619677445.382460:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.382473:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.382476:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.382477:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.382478:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.382478:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.382480:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.382747:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677445.382892:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.382893:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.382893:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.382895:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.382898:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.382900:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.382912:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.382914:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.382917:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.382919:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.382921:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.382923:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96757964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.382924:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.382925:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.382926:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.382929:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.382931:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.382934:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.382935:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.382935:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.382937:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.382938:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.382939:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.382939:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.383013:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:13.0:1619677445.383556:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.383559:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.383560:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.383562:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.383564:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.383616:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.383619:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.383621:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.383624:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.383626:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.383627:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96757952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.383629:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.383630:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.383631:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.383634:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.383636:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.383639:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.383639:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.383640:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.383642:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.383642:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.383643:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.383643:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677445.384066:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.384227:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.384229:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.384230:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.384232:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.384235:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:13.0:1619677445.384354:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.384357:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.384357:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.384359:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.384361:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.384505:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.384508:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.384509:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.384511:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.384514:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.384529:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.384551:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.384942:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.384944:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.384945:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.384947:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.384949:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.385191:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.385198:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.385199:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.385201:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.385203:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.385332:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.385513:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:13.0:1619677445.385522:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:25.0:1619677445.385524:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (21)++
00000800:00000200:13.0:1619677445.385524:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.385525:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.0:1619677445.385527:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:25.0:1619677445.385528:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983e00] (22)++
00000400:00000200:13.0:1619677445.385528:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.385529:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983e00] (23)--
00000800:00000200:25.0:1619677445.385530:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:13.0:1619677445.385530:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:25.0:1619677445.385531:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:13.0:1619677445.385620:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.385782:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.385785:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.385786:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.385788:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.385791:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.385827:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.385841:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.385845:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.385846:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.385846:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.385847:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.386032:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.386110:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.386112:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.386113:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.386115:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.386117:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:13.0:1619677445.386535:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.386537:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.386538:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.386540:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.386543:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.386975:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.386992:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.386995:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.386996:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.386997:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.386997:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.387032:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.387034:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.387035:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.387037:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.387039:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.387127:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677445.387129:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:25.0:1619677445.387284:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.387287:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.387289:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.387292:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.387299:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:25.0:1619677445.387300:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96757890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.387302:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.387302:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.387304:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:25.0:1619677445.387307:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.387308:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:25.0:1619677445.387311:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c2502380] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.387311:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (20)++
00000800:00000200:25.0:1619677445.387312:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:25.0:1619677445.387313:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.387314:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.387315:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.387316:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677445.387327:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.387329:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.387330:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.387332:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.387336:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.387540:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.387711:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.387714:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.387716:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.387719:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.387721:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.387722:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96757881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.387724:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.387725:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.387726:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.387729:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.387730:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.387737:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.387738:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.387739:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.387740:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.387741:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.387742:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.387744:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.387744:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677445.387748:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.387764:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.387766:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677445.387798:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.387800:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.387801:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.0:1619677445.387802:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:13.0:1619677445.387803:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.387804:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.387805:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.387806:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677445.387806:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.387901:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.387903:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.387904:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.387906:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.387908:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:13.0:1619677445.388067:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.388070:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.388071:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.388073:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.388075:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.388112:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.388345:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.388348:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.388349:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.388356:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.388359:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:13.0:1619677445.388647:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.388939:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:13.0:1619677445.388961:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.388963:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.388964:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.388965:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.388968:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:25.0:1619677445.389131:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.389133:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.389136:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.389139:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.389141:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.389142:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96757857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.389144:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.389145:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.389145:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.389149:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.389151:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.389154:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.389154:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.389159:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.389161:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.389162:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.389162:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.389163:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677445.389169:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.389209:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677445.389211:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677445.389213:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677445.389216:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.389218:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:13.0:1619677445.389219:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96757856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677445.389220:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677445.389222:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677445.389223:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:13.0:1619677445.389226:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677445.389227:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677445.389230:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677445.389231:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:13.0:1619677445.389232:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:13.0:1619677445.389234:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:13.0:1619677445.389234:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677445.389235:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677445.389236:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:13.0:1619677445.389371:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.389615:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.389617:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.389618:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.389620:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.389623:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.390137:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.390148:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.390151:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.390152:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.390152:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.390153:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.390154:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:25.0:1619677445.390572:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.390575:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.390576:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.390577:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.390580:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:13.0:1619677445.390952:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.391107:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677445.391110:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.391112:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.391113:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.391115:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.391117:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.391468:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.391470:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.391473:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.391476:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.391477:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:25.0:1619677445.391479:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96757818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.391480:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.391481:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.391482:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:25.0:1619677445.391485:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.391486:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.0:1619677445.391489:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.391490:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:25.0:1619677445.391491:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:25.0:1619677445.391492:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:25.0:1619677445.391493:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.391494:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.391494:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.391613:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677445.391849:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:25.0:1619677445.391878:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.391880:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.391881:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.391883:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.391886:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.392360:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.392373:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.392375:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.392376:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.392377:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.392377:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.392379:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.392416:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.392427:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (21)++
00000800:00000200:25.0:1619677445.392429:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:25.0:1619677445.392430:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.392431:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983e00] (23)--
00000800:00000200:25.0:1619677445.392432:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.392433:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:13.0:1619677445.392604:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.392606:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.392607:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.392609:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.392612:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:25.0:1619677445.392717:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.392754:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.392756:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.392757:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.392758:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677445.392902:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.392981:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.392984:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.392985:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.392986:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.392989:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:25.0:1619677445.393355:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.393358:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.393359:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.393361:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.393364:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.393440:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.393487:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.393491:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.393495:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.393498:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.393500:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:25.0:1619677445.393502:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96757787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.393504:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.393505:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.393507:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:25.0:1619677445.393514:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.393517:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:25.0:1619677445.393521:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c2502380] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.393521:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (20)++
00000800:00000200:25.0:1619677445.393523:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:25.0:1619677445.393524:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.393525:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.393526:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.393527:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677445.393548:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.394026:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.394029:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.394029:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.394030:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.394031:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.394032:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.394303:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677445.394307:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.394335:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.394338:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.394338:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.0:1619677445.394339:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:13.0:1619677445.394340:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.394342:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:13.0:1619677445.394342:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.394343:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.394344:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677445.394364:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.394451:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.394454:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.394455:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.394457:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.394461:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.394536:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.394538:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.394540:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.394543:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.394545:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.394550:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96757766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.394552:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.394553:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.394554:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.394557:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.394559:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.394561:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.394562:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.394563:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.394564:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.394565:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.394566:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.394567:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677445.394681:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.395066:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.395068:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.395070:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.395071:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.395074:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:25.0:1619677445.395571:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.395972:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.395974:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.395976:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.395979:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.395981:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.395982:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96757739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.395983:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.395984:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.395985:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.395988:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.395990:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.395993:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.395994:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.395994:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.395996:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.395997:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.395997:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.395998:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677445.396544:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)++
00000800:00000200:13.0:1619677445.396547:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (24)--
00000400:00000200:13.0:1619677445.396548:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.396550:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.396553:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (24)--
00000800:00000200:13.0:1619677445.396554:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:25.0:1619677445.396797:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.396833:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.396835:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.396836:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.396837:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677445.396984:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.396987:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.396988:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.396990:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.396993:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:25.0:1619677445.397097:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.397099:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.397101:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.397102:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.397106:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:13.0:1619677445.397781:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.397792:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.397794:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.397795:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.397796:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.397796:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.397798:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.398159:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:25.0:1619677445.398387:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.398805:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.399198:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.399201:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.399202:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.399204:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.399206:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.399516:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:13.0:1619677445.399763:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.399765:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.399766:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.399767:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.399767:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.399768:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.400024:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.400025:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.400026:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.400028:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.400031:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.400032:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.400033:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:25.0:1619677445.400121:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.400124:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.400125:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.400128:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.400130:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.400131:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96757679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.400133:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.400134:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.400135:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.400138:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.400140:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.400143:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.400144:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.400144:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.400146:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.400147:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.400147:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.400148:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677445.400236:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.400238:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.400239:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.400240:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.400243:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.400283:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:25.0:1619677445.400447:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677445.400558:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.400560:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.400561:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.400563:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.400566:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.400616:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.400618:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.400620:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.400623:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.400625:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.400626:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96757668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.400627:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.400628:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.400630:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:25.0:1619677445.400636:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.400638:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.400641:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.400642:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.400643:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.400644:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.400645:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.400646:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.400646:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677445.400773:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.400776:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.400776:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.400780:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.400782:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.400798:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.400824:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.400855:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.400857:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.400858:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.400860:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.400862:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:25.0:1619677445.401376:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677445.401385:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.401407:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.401409:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.401410:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.401411:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677445.401660:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.401663:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.401663:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.401665:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.401667:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.402347:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.402349:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.402350:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.402352:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.402355:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.402441:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677445.402516:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:25.0:1619677445.402621:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.402624:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.402626:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.402629:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.402631:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.402632:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96757640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.402633:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.402634:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.402635:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.402638:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.402640:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.402643:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.402643:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.402644:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.402646:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.402646:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.402647:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.402647:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.403849:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.403913:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.403916:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.403918:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.403920:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.403931:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.404644:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.404689:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:13.0:1619677445.404689:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:25.0:1619677445.404691:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.404692:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677445.404692:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:25.0:1619677445.404693:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:13.0:1619677445.404693:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.404694:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.404697:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.404901:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.404904:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.404906:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.404909:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.404911:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.404912:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96757607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.404914:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.404915:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.404916:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.404919:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.404922:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.404925:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.404926:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.404926:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.404928:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.404928:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.404929:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.404930:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677445.404963:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.405041:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.405070:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.405073:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.405074:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.405075:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.405078:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.405308:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:25.0:1619677445.405335:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:13.0:1619677445.405335:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.405337:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.405338:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.405340:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.405342:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.405629:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.405803:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.405810:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.405812:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.405814:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.405816:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.405817:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96757586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.405819:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.405820:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.405820:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:25.0:1619677445.405824:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.405825:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.405828:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.405829:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.405829:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.405831:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.405832:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.405832:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.405833:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677445.405924:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.406015:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677445.406106:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.406350:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.406353:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.406354:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.406355:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.406357:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.406366:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.406369:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.406369:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.406370:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.406370:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.406371:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:25.0:1619677445.406395:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.406397:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.406404:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.406406:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.406408:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.406409:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96757575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.406411:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.406412:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.406414:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.406418:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.406420:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.406423:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.406424:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.406424:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.406426:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.406427:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.406428:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.406429:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.406430:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677445.406504:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.406507:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.406507:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.406510:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.406513:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.406857:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:25.0:1619677445.407426:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.407516:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.407518:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.407520:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.407522:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.407529:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.407713:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677445.407745:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.407747:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.407748:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.407749:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.407751:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.407763:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.407765:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.407766:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.407767:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.407769:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.407779:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.407782:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.407782:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.407783:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.407784:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.407786:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.407914:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.408065:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.408067:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.408068:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.408071:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.408074:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:25.0:1619677445.408093:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.408096:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.408097:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.408099:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.408101:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:13.0:1619677445.408272:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.408654:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.408655:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.408658:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.408660:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.408662:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:25.0:1619677445.408663:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96757539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.408665:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.408666:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.408667:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:25.0:1619677445.408670:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.408672:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.0:1619677445.408675:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.408676:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:25.0:1619677445.408677:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:25.0:1619677445.408678:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:25.0:1619677445.408679:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.408680:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.408681:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.408986:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677445.409019:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.409022:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.409022:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.409024:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.409026:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.409064:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.409066:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.409067:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.409069:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.409072:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.409318:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.409523:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.409526:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.409527:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.409533:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.409536:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:13.0:1619677445.410139:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.410142:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.410143:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.410145:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.410148:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.410325:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.410337:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.410340:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.410341:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.410341:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.410342:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.410343:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.410447:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.410462:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.410463:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.410465:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.410467:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.410469:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:25.0:1619677445.410470:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96757511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.410471:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.410472:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.410473:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:25.0:1619677445.410476:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.410477:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.0:1619677445.410480:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.410481:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:25.0:1619677445.410482:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:25.0:1619677445.410483:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:25.0:1619677445.410484:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.410485:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.410487:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.410487:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677445.410650:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:25.0:1619677445.410788:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.410912:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.410915:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.410915:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.410917:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.410920:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:13.0:1619677445.411142:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.411158:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.411161:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.411162:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.411163:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.411164:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.411178:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.411181:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.411182:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.411184:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.411186:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.411231:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.411278:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.411281:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.411285:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.411288:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.411290:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:25.0:1619677445.411291:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96757491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.411292:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.411299:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.411300:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:25.0:1619677445.411304:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.411306:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:25.0:1619677445.411310:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780380] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.411311:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e3a00] (20)++
00000800:00000200:25.0:1619677445.411311:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e3a00] (21)++
00000800:00000200:25.0:1619677445.411313:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e3a00] (22)--
00000800:00000200:25.0:1619677445.411314:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.411315:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.411315:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677445.411527:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.411530:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.411530:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.411532:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.411536:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.411598:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677445.411623:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.411722:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.411725:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.411726:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.411728:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.411731:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.412079:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.412090:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.412093:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.412095:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)++
00000800:00000200:13.0:1619677445.412096:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (24)--
00000800:00000200:13.0:1619677445.412097:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (24)--
00000800:00000200:13.0:1619677445.412111:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.412113:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.412114:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.412116:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.412118:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.412526:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.412542:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.412545:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.412550:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.412552:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.412554:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:25.0:1619677445.412555:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96757469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.412557:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.412558:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.412559:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:25.0:1619677445.412562:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.412564:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:25.0:1619677445.412566:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3987280] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.412567:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8400] (20)++
00000800:00000200:25.0:1619677445.412568:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:25.0:1619677445.412569:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:25.0:1619677445.412570:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.412571:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.412571:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677445.412582:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.412584:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.412585:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.412587:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.412589:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.412828:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.412838:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.412841:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.412841:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.412842:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.412843:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.412844:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:24.2:1619677445.413020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980600] (21)++
00000800:00000200:25.0:1619677445.413029:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980600] (22)--
00000800:00000200:13.0:1619677445.413062:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.413065:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.413066:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.413068:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.413071:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:25.0:1619677445.413149:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.413151:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.413152:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.413154:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.413160:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.413238:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.413337:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.413340:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.413342:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.413344:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.413346:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:25.0:1619677445.413347:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96757453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.413348:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.413349:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.413350:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:25.0:1619677445.413354:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.413355:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:25.0:1619677445.413357:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c2502380] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.413358:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (20)++
00000800:00000200:25.0:1619677445.413359:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:25.0:1619677445.413360:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.413361:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.413361:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.413362:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677445.413381:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.413518:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.413631:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.413633:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.413634:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.413637:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:25.0:1619677445.413640:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.414389:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.414392:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.414394:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.414395:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.414397:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.414398:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96757438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.414399:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.414400:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.414401:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.414404:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.414406:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.414409:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.414410:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.414410:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.414412:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.414413:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.414414:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.414414:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.414453:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.414493:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.414496:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.414496:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.414497:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677445.414497:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.414577:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.414580:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.414581:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.414584:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677445.414586:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.414607:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.414610:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.414611:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.414613:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.414615:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:13.0:1619677445.415127:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.415145:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.415148:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.415148:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.415149:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.415150:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.415151:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.415244:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.415368:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.415410:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.415412:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.415413:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677445.415413:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:25.0:1619677445.415414:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677445.415416:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.415416:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.415418:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.415421:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.415915:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.415917:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.415918:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.415920:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.415922:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.416079:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.416472:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.416475:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.416477:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.416481:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.416483:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.416484:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96757404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.416486:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.416489:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.416490:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.416494:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.416496:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.416500:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.416500:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.416501:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.416507:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.416508:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.416509:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.416510:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677445.416706:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.416708:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.416709:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.416710:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.416712:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.416719:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.416868:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:25.0:1619677445.417036:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677445.417076:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.417078:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.417079:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.417080:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.417084:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.417255:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.417256:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.417256:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.417258:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.417260:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.417304:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.417307:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.417309:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.417312:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.417314:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.417315:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96757390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.417317:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.417319:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.417320:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:25.0:1619677445.417324:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.417325:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.417329:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.417329:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.417330:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.417332:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.417332:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.417333:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.417333:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677445.417378:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.417381:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.417382:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.417384:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.417387:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.417416:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.417691:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.417694:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.417695:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.417697:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.417699:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:25.0:1619677445.417708:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.417722:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.417724:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.417725:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.417726:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.417727:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.417728:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:13.0:1619677445.417899:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.417955:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.417957:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.417958:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.417960:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.417962:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:25.0:1619677445.418034:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:13.0:1619677445.418312:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.418314:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.418315:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.418315:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.418316:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.418317:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.418318:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.418578:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.418581:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.418582:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.418583:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.418586:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:25.0:1619677445.418868:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.418870:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.418872:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.418873:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.418877:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.418934:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.419070:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.419072:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.419075:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.419078:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.419080:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:25.0:1619677445.419081:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96757358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.419083:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.419084:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.419085:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:25.0:1619677445.419088:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.419090:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:25.0:1619677445.419092:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3987280] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.419093:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8400] (20)++
00000800:00000200:25.0:1619677445.419094:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:25.0:1619677445.419096:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:25.0:1619677445.419096:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.419097:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.419098:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677445.419115:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.419175:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.419178:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.419179:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.419180:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.419183:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.419328:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.419329:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.419330:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.419331:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.419331:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.419440:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.419738:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.419740:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.419742:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.419743:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.419746:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:13.0:1619677445.419914:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.420268:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.420271:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.420272:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.420273:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.420276:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:25.0:1619677445.420301:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.420509:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.420511:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.420512:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.420515:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.420518:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.420535:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.420576:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.420578:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.420579:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.420580:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677445.420832:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.420868:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.420871:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.420873:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.420880:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.420882:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.420883:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96757329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.420884:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.420885:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.420886:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.420889:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.420892:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.420895:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.420896:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.420897:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.420898:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.420899:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.420900:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.420900:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.421617:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.421829:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.421831:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.421832:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.421833:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.421836:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:13.0:1619677445.422330:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.422333:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.422334:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.422336:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.422338:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.422355:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677445.422936:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.422949:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.422952:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.422953:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.422953:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.422954:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.422955:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.422996:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.422999:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.423001:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.423003:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.423005:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.423007:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96757297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.423010:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.423011:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.423013:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.423021:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.423025:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.423031:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.423032:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.423033:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.423039:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.423043:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677445.423043:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:25.0:1619677445.423044:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.423045:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677445.423046:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:25.0:1619677445.423047:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000400:00000200:13.0:1619677445.423047:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.0:1619677445.423048:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:13.0:1619677445.423048:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:25.0:1619677445.423049:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.423051:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:13.0:1619677445.423051:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:25.0:1619677445.423060:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.423061:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.423062:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.423063:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.423065:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.423067:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:13.0:1619677445.423655:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.423800:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.423804:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.423804:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.423806:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.423810:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.423845:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.424067:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.424070:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.424070:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.424072:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.424075:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.424193:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.424236:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677445.424239:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677445.424241:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677445.424244:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.424246:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:13.0:1619677445.424247:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96757274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677445.424248:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677445.424249:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677445.424250:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:13.0:1619677445.424254:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677445.424256:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.424258:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3987280] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677445.424258:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8400] (20)++
00000800:00000200:13.0:1619677445.424259:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.424261:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.424261:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677445.424263:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677445.424265:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677445.424265:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677445.424279:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677445.424280:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677445.424280:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677445.424282:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.424596:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.424599:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.424600:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.424602:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.424605:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.424694:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.424706:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (21)++
00000800:00000200:25.0:1619677445.424709:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.0:1619677445.424710:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.424710:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980000] (23)--
00000800:00000200:25.0:1619677445.424711:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.424712:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.424925:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:25.0:1619677445.425030:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.425032:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.425033:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.425035:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.425037:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.425078:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.425080:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.425082:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.425085:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.425087:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.425088:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96757261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.425089:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.425090:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.425091:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.425096:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.425098:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.425102:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.425102:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.425103:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.425104:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.425105:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.425106:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.425108:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.425108:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.425151:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:13.0:1619677445.425200:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.425203:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.425204:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.425208:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.425212:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.425272:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.425275:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.425277:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.425279:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.425282:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:25.0:1619677445.425283:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96757256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.425285:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.425285:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.425287:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:25.0:1619677445.425290:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.425292:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:25.0:1619677445.425300:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ec00] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.425301:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91978412be00] (20)++
00000800:00000200:25.0:1619677445.425302:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91978412be00] (21)++
00000800:00000200:25.0:1619677445.425304:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91978412be00] (22)--
00000800:00000200:25.0:1619677445.425304:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.425306:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.425307:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.425308:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677445.425330:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.425610:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.425733:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.425736:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.425737:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.425738:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.425742:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:13.0:1619677445.426201:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.426221:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677445.426263:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.426418:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.426420:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.426421:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.426427:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.426438:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:13.0:1619677445.426598:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.426601:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.426602:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.426603:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.426607:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.426644:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.426999:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.427002:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.427005:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.427007:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.427010:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.427011:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96757227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.427013:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.427014:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.427015:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.427018:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.427020:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.427023:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.427024:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.427029:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.427031:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.427032:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.427033:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.427033:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677445.427040:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.427080:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.427137:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.427139:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.427142:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.427145:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.427146:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:25.0:1619677445.427147:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96757224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.427153:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.427154:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.427155:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:25.0:1619677445.427158:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.427160:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:25.0:1619677445.427162:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780380] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.427163:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e3a00] (20)++
00000800:00000200:25.0:1619677445.427164:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e3a00] (21)++
00000800:00000200:25.0:1619677445.427165:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e3a00] (22)--
00000800:00000200:25.0:1619677445.427166:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.427168:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.427169:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.427174:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.427759:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677445.428165:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.428168:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.428169:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.428170:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.428173:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.428335:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.428338:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.428339:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.428341:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:25.0:1619677445.428344:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.428632:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677445.428707:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.428718:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.428721:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.428722:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.428723:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.428724:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.428793:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.428795:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.428796:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.428797:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.428800:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.428927:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.428944:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.428946:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.428947:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.428948:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.428948:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.428950:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.429399:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.429461:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.429468:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.429470:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677445.429471:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000400:00000200:25.0:1619677445.429473:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.429474:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.429474:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.429475:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:25.0:1619677445.429476:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96757193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677445.429476:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:25.0:1619677445.429478:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.429479:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.429479:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.429479:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000400:00000200:25.0:1619677445.429483:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.429484:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:25.0:1619677445.429487:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c2502380] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.429487:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (20)++
00000800:00000200:25.0:1619677445.429488:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:25.0:1619677445.429489:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.429490:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.429491:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.429491:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677445.429508:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.429654:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.429657:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.429658:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.429659:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.429662:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.429800:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.429810:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.429813:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.429814:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.429814:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.429815:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.429816:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:25.0:1619677445.430013:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:13.0:1619677445.430203:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.430205:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.430206:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.430208:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.430211:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.430230:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.430233:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.430235:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.430238:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.430239:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.430241:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96757181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.430242:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.430243:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.430244:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.430247:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.430248:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.430251:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.430252:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.430253:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.430254:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.430255:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.430256:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.430258:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.430258:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.430402:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:13.0:1619677445.430525:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.430535:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.430784:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.430787:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.430788:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.430790:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.430792:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.430793:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.430795:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.430796:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.430798:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.430801:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:13.0:1619677445.431027:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.431145:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.431148:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.431149:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.431150:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.431153:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.431556:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.431557:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.431558:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.431559:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.431564:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.431565:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.431776:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.431786:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.431788:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.431789:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.431790:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.431790:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.431792:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.431807:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:13.0:1619677445.431847:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.432277:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.432279:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.432280:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.432282:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.432284:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:25.0:1619677445.432335:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677445.432647:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.433171:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.433174:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.433176:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.433179:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.433181:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:25.0:1619677445.433182:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96757134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.433184:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.433186:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.433186:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:25.0:1619677445.433199:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.433203:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.0:1619677445.433206:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.433206:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:25.0:1619677445.433208:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:25.0:1619677445.433210:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:25.0:1619677445.433210:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.433211:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.433212:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.0:1619677445.433292:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677445.433351:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.433354:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.433355:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.433356:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.433359:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.433510:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.433512:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.433513:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.433515:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:25.0:1619677445.433518:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.433662:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.433663:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.433665:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.433668:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.433671:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:25.0:1619677445.433672:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96757124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.433673:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.433674:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.433675:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:25.0:1619677445.433679:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.433681:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:25.0:1619677445.433683:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ec00] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.433684:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91978412be00] (20)++
00000800:00000200:25.0:1619677445.433685:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91978412be00] (21)++
00000800:00000200:25.0:1619677445.433687:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91978412be00] (22)--
00000800:00000200:25.0:1619677445.433687:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.433689:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.433691:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.433691:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677445.433873:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.433969:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.433972:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.433973:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.433975:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.433978:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.434179:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:25.0:1619677445.434181:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.434184:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.434186:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.434189:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.434191:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:25.0:1619677445.434193:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96757117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.434195:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.434196:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.434197:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:25.0:1619677445.434200:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.434202:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:25.0:1619677445.434205:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ec00] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.434206:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91978412be00] (20)++
00000800:00000200:25.0:1619677445.434207:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91978412be00] (21)++
00000800:00000200:25.0:1619677445.434208:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91978412be00] (22)--
00000800:00000200:25.0:1619677445.434209:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.434210:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.434210:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.0:1619677445.434212:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.434214:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.434216:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.434218:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.434219:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.434220:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96757116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.434222:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.434222:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.434223:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.434227:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.434229:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.434232:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.434233:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.434233:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.434235:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.434236:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.434236:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.434237:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677445.434540:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.434543:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.434543:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.434546:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.434548:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.434584:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.434587:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.434587:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.434589:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677445.434592:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.434862:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.434946:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.434949:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.434949:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.434951:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.434954:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:25.0:1619677445.435007:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.435515:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983e00] (22)++
00000800:00000200:25.0:1619677445.435518:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983e00] (23)--
00000400:00000200:25.0:1619677445.435520:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.435523:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.435527:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (21)--
00000800:00000200:25.0:1619677445.435799:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (22)--
00000800:00000200:13.0:1619677445.435908:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.436021:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.436023:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:25.0:1619677445.436025:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.436028:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.436030:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:25.0:1619677445.436031:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96757090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.436033:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.436034:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.436035:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:25.0:1619677445.436039:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.436041:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.0:1619677445.436044:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.436045:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:25.0:1619677445.436046:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:25.0:1619677445.436047:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:25.0:1619677445.436048:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.0:1619677445.436049:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.436049:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677445.436123:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.436126:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.436126:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.436128:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.436131:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.436180:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.0:1619677445.436215:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980000] (22)++
00000800:00000200:25.0:1619677445.436217:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980000] (23)--
00000400:00000200:25.0:1619677445.436219:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.436220:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.0:1619677445.436223:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980000] (21)--
00000800:00000200:13.0:1619677445.436468:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:13.0:1619677445.436469:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (24)--
00000400:00000200:13.0:1619677445.436469:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.436471:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.436473:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.436676:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.436742:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.436745:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.436746:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.436747:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.436750:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.437146:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.437148:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.437149:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.437151:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.437153:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.437236:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.437488:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.437490:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.437491:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.437492:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.437494:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.437695:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983e00] (22)--
00000800:00000200:13.0:1619677445.437718:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.437720:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.437721:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.437723:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.437725:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.437762:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.437960:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:25.0:1619677445.437963:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000400:00000200:25.0:1619677445.437965:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.437968:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.437969:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:25.0:1619677445.437971:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96757057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.437972:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.437973:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.437974:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:25.0:1619677445.437978:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.437979:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:25.0:1619677445.437982:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8680] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.437983:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71ba00] (20)++
00000800:00000200:25.0:1619677445.437983:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71ba00] (21)++
00000800:00000200:25.0:1619677445.437985:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71ba00] (22)--
00000800:00000200:25.0:1619677445.437985:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:25.0:1619677445.437986:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:25.0:1619677445.437987:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:13.0:1619677445.438021:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.438024:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.438025:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.438026:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.438029:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:25.0:1619677445.438062:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:13.0:1619677445.438087:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.438097:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.438101:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.438101:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.438102:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.438103:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.438103:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:25.0:1619677445.438105:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)++
00000800:00000200:25.0:1619677445.438107:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed9d3400] (23)--
00000800:00000200:25.0:1619677445.438108:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:25.0:1619677445.438109:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:25.0:1619677445.438110:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.438145:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.0:1619677445.438148:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:25.0:1619677445.438148:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.0:1619677445.438149:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677445.438534:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.438948:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.438951:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:25.0:1619677445.438954:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:25.0:1619677445.438958:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:25.0:1619677445.438964:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:25.0:1619677445.438965:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96757038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:25.0:1619677445.438967:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:25.0:1619677445.438968:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:25.0:1619677445.438969:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:25.0:1619677445.438972:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.0:1619677445.438974:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:25.0:1619677445.438977:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c2502380] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:25.0:1619677445.438977:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (20)++
00000800:00000200:25.0:1619677445.438982:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:25.0:1619677445.438983:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:25.0:1619677445.438984:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.0:1619677445.438985:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:25.0:1619677445.438987:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:25.0:1619677445.438987:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677445.438996:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.438999:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.438999:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.439001:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.439004:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.439174:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:24.2:1619677445.439209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980600] (21)++
00000800:00000200:25.0:1619677445.439218:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980600] (22)--
00000800:00000200:24.2:1619677445.439228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980600] (20)++
00000800:00000200:13.0:1619677445.439533:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.439536:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.439537:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.439538:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.439541:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.439848:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.439851:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.439851:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.439853:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.439855:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.440833:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.440835:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.440836:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.440837:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.440840:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.440856:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.440858:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.440859:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.440859:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.440860:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.440861:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.441082:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.441085:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.441085:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.441087:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.441089:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.441266:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.441269:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.441270:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.441272:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.441275:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.441695:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.441722:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.441724:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.441725:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.441727:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.441730:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.443627:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.443720:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.443722:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.443723:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.443724:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.443727:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.444029:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.444185:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.444188:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.444189:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.444190:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.444193:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.444303:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.444306:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.444307:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.444309:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.444311:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.444386:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.445090:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.445093:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[5] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.445095:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:13.0:1619677445.445096:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (24)--
00000800:00000200:13.0:1619677445.445098:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.445219:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.445230:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.445232:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.445233:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:13.0:1619677445.445234:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (24)--
00000800:00000200:13.0:1619677445.445235:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.445236:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.445274:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.445481:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.445484:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.445484:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.445486:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.445489:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.445496:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.445601:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.445603:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.445604:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.445606:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.445609:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.445679:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.445689:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.445692:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.445693:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.445694:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.445695:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.445696:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.446121:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.446168:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.446170:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.446171:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.446173:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.446176:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.446315:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.446384:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.446747:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.446750:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.446751:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.446753:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.446756:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.446779:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.446933:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.447614:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.447616:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.447617:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.447619:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.447622:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.448768:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.448779:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.448782:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.448783:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.448783:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.448784:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.448785:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.448989:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.449403:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.449406:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.449407:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.449409:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.449411:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.449587:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.449600:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.449603:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.449603:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.449604:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.449604:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.449605:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.449915:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (24)++
00000800:00000200:13.0:1619677445.449919:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (24)--
00000400:00000200:13.0:1619677445.449920:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.449921:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.449924:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (24)--
00000800:00000200:13.0:1619677445.449925:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.450103:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.450105:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.450106:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.450108:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.450110:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.450234:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.450424:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.450426:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.450427:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.450429:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.450432:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.450529:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.450545:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.450548:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.450549:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.450549:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.450550:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.450551:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.450645:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.451113:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.451116:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.451117:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.451119:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.451122:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.451227:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.451399:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.451401:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.451402:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.451404:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.451407:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.451429:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.451635:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.451941:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.451951:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.451954:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.451955:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.451955:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.451956:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.451957:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.452753:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.452776:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.452778:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.452779:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.452782:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.452784:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.453226:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.453852:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.453855:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.453856:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.453857:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.453860:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.454308:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.454320:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.454324:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.454325:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.454326:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.454327:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.454634:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.454637:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.454637:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.454639:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.454642:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.454704:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.455178:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.455180:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.455181:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.455183:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.455186:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.455282:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.455647:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.455650:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.455651:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.455653:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.455655:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.455657:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.456074:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.456077:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.456078:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.456079:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.456082:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.456164:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.456167:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.456168:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.456171:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.456174:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.456489:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.456598:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.456600:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.456601:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.456603:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.456606:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.456822:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.457103:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.457106:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.457106:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.457108:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.457111:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.457667:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677445.457824:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.457826:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.457827:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.457829:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.457831:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.458728:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.458730:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.458731:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.458733:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.458735:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.458991:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.460217:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.460219:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.460220:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.460222:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.460225:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.460402:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.460482:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.460807:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.460827:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.460830:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.460830:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.460832:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.460835:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.461575:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.461577:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.461578:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.461580:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.461583:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.461776:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.461851:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.462693:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.462696:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.462697:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.462699:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.462702:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.463093:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.463238:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.463240:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.463241:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.463243:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.463245:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.463548:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.463551:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.463552:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.463554:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.463557:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.463618:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.463630:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.463633:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.463633:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.463634:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.463635:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.463636:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.464155:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.464531:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.464534:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.464535:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.464536:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.464540:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.464630:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.465395:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.465398:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.465399:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.465400:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.465403:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.465404:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.465825:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.465827:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.465828:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.465829:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.465832:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.465833:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.465834:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.466366:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.466369:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.466370:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.466371:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.466374:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.466417:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (24)--
00000800:00000200:13.0:1619677445.466429:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.466432:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.466433:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.466434:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.466437:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.466566:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.466568:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.466569:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.466570:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.466573:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.466861:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.467509:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.467511:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.467512:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.467515:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.467518:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.467925:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.467937:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.467939:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.467940:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.467941:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.467941:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.467942:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.468202:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.468204:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.468205:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.468207:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.468210:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.468302:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.468617:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.468619:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.468620:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.468622:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.468625:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.469051:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.469357:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.469634:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.469637:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.469638:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.469640:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.469642:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.471440:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.472293:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.472306:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.472346:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.472348:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.472349:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.472351:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.472354:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.473268:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.473278:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.473281:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.473281:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.473282:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.473283:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.473284:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.473439:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.473572:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.473574:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.473575:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.473577:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.473579:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.475218:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.475376:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.475930:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.476721:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.476723:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.476724:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.476726:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.476728:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.476730:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.477042:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.477045:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.477045:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.477047:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.477050:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.477208:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.478356:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.478359:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.478360:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.478361:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.478364:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.478571:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.478969:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.478972:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.478973:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.478975:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.478977:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.479632:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.479733:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.479735:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.479736:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.479738:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.479741:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.480468:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.480483:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.480485:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.480486:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.480487:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.480488:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.480489:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.480936:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.481578:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.481580:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.481581:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.481583:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.481586:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.481801:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.481850:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.481852:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.481853:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.481856:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.481859:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.481934:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.481937:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.481937:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.481939:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.481944:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.481945:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.481946:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.481947:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.481949:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.481950:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.482089:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.482279:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.482281:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.482282:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.482284:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.482287:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.482407:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.482477:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.482592:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.482594:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.482595:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.482596:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.482599:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.482756:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.482758:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.482761:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.482762:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.482763:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677445.482768:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.483251:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.483326:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.483328:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.483329:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.483331:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.483334:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.484063:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.484597:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.484599:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.484600:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.484602:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.484606:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.485517:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.485527:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.485530:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.485531:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.485531:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.485532:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.485533:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.485566:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.485574:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.486058:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.486060:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.486061:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.486062:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.486065:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.486213:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.486214:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.486216:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.486217:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.486219:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.486393:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.486396:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.486397:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.486399:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.486400:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.486402:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.486476:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.486479:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.486479:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.486481:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.486483:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.486484:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.486498:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.486502:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.486502:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.486503:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.486504:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.486505:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.486828:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.486830:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.486831:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.486834:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000400:00000200:12.0:1619677445.486835:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.486837:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.486971:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.486973:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.486974:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.486976:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.486977:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.486978:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.486999:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677445.487002:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:13.0:1619677445.487005:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677445.487006:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677445.487009:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.487011:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:12.0:1619677445.487013:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96756234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677445.487016:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677445.487018:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677445.487020:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:12.0:1619677445.487026:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677445.487029:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677445.487033:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677445.487033:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:12.0:1619677445.487034:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:12.0:1619677445.487036:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:12.0:1619677445.487036:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677445.487037:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677445.487038:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:13.0:1619677445.487284:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.487287:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.487287:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.487291:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.487291:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.487300:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.487601:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.487603:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.487605:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.487607:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.487608:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.487610:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.487632:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.487634:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.487635:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.487636:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.487636:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.487637:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.487962:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.487965:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.487966:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.487968:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.487969:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.487972:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.488000:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.488002:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.488003:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.488005:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.488006:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.488007:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.488118:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.488121:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.488121:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.488123:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.488125:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.488126:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.488139:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:12.0:1619677445.488141:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.488142:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.488143:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.488143:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.488144:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.488269:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.488272:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.488273:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.488275:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.488276:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.488277:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.488531:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.488534:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.488535:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.488537:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.488539:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.488541:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.488939:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)++
00000800:00000200:13.0:1619677445.488942:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (24)--
00000400:00000200:13.0:1619677445.488942:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.488945:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.488946:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)++
00000800:00000200:12.0:1619677445.488947:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (24)--
00000400:00000200:12.0:1619677445.488947:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.488948:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.488949:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000400:00000200:12.0:1619677445.488949:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.488952:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.488961:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.488964:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.488965:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.488966:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.488966:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.488968:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.488990:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.488992:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.488993:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.488995:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.488996:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.488998:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.489171:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677445.489173:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677445.489176:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677445.489177:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677445.489179:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.489181:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677445.489182:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96756185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677445.489184:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677445.489185:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677445.489186:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:13.0:1619677445.489190:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677445.489192:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677445.489195:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677445.489196:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:13.0:1619677445.489196:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:13.0:1619677445.489198:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:13.0:1619677445.489199:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677445.489200:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677445.489200:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677445.489534:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.489536:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.489537:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.489540:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.489541:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.489542:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.489579:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.489582:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.489582:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.489584:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.489585:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.489587:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.489964:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.489966:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.489967:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.489968:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.489970:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.489971:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.490685:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.490687:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.490688:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.490690:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.490692:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.490693:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.491388:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.491391:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.491393:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.491394:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.491396:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.491397:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.491398:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.491398:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.491399:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.491399:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.491400:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.491842:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.491844:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.491845:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.491848:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.491849:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.491850:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.491897:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.491900:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.491901:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.491903:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.491904:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.491906:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.491915:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:12.0:1619677445.491918:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.491919:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.491919:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.491920:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.491921:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.492289:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.492291:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.492292:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.492299:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.492299:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.492301:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.492377:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.492379:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.492380:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.492382:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.492383:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.492385:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.492567:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.492570:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.492571:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.492572:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.492573:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.492574:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.492588:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.492591:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.492592:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.492593:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.492593:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.492595:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.492636:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.492639:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.492640:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.492642:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.492643:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.492645:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.493206:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.493209:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.493210:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.493212:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.493213:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.493215:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.493230:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.493233:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.493233:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.493235:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.493236:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.493237:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.493289:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.493291:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.493292:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.493300:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000400:00000200:12.0:1619677445.493300:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.493302:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.493303:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:13.0:1619677445.493303:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.493303:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.493304:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.493305:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.493872:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.493875:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.493876:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.493877:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.493879:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.493879:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.494050:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.494053:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.494054:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.494055:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.494057:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.494058:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.494509:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.494512:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.494513:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.494515:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.494517:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.494518:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.494698:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.494701:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.494703:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.494705:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000400:00000200:12.0:1619677445.494705:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.494707:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.494715:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.494718:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.494719:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.494720:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.494721:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.494722:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.494792:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.494795:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.494795:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.494797:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.494798:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.494800:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.495241:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.495244:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.495245:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.495247:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.495248:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.495250:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.495601:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.495603:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.495604:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.495606:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.495608:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.495609:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.496566:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.496569:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.496570:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.496571:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.496573:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.496574:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.496752:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.496754:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.496755:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.496757:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.496758:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.496760:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.497142:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.497144:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.497145:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.497147:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677445.497148:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.497150:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.497409:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.497412:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.497412:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.497414:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.497416:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.497417:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.497426:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.497428:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.497429:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.497430:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.497431:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.497432:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.497719:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.497721:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.497722:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.497723:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.497725:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.497726:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.497727:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.497730:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.497730:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.497732:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.497734:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.497735:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.497750:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.497753:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.497754:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.497754:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.497755:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.497757:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.497775:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.497778:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.497779:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.497781:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.497782:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.497786:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.498056:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.498060:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.498061:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677445.498063:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000400:00000200:13.0:1619677445.498064:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.498068:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.498220:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.498222:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.498223:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.498226:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000400:00000200:12.0:1619677445.498226:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677445.498229:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.498609:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.498611:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.498612:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.498614:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.498616:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.498617:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.498625:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.498628:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.498629:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.498629:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.498630:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.498631:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.499325:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.499328:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.499329:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.499331:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.499333:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.499334:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.499491:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.499494:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.499495:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.499497:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.499498:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.499499:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.499877:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.499879:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.499880:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.499882:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.499883:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.499885:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.500350:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.500353:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.500354:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.500355:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.500357:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.500359:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.500657:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.500660:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.500661:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.500663:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000400:00000200:12.0:1619677445.500663:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.500665:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.500756:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.500758:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.500759:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.500761:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.500763:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.500764:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.500775:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:12.0:1619677445.500778:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.500779:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.500780:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.500780:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.500782:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.501136:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.501138:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.501139:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.501140:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.501142:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.501144:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.501202:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.501204:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.501205:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.501207:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.501208:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.501209:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.501221:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.501223:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.501224:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.501225:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.501226:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.501227:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.501471:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.501473:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.501474:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.501476:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.501478:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.501479:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.501753:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.501756:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.501756:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.501758:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.501760:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.501761:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.502435:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.502437:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.502438:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.502440:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.502442:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.502443:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.502653:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.502655:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.502656:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.502658:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.502660:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.502661:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.502671:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.502674:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.502675:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.502676:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.502677:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.502678:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.502702:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.502704:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.502705:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.502706:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.502708:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.502709:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.502799:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.502802:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.502803:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.502805:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.502806:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.502810:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.502929:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.502931:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.502931:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.502933:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.502935:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.502935:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.502946:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.502949:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.502950:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.502950:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.502951:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.502952:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.503436:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.503438:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.503439:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.503441:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.503443:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.503444:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.503497:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.503500:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.503501:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.503504:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000400:00000200:12.0:1619677445.503504:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677445.503507:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.504938:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.504941:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.504941:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.504943:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.504945:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.504946:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.505818:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.505820:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.505821:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.505822:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.505825:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.505826:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.505963:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.505965:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.505966:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.505969:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.505970:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (24)--
00000800:00000200:12.0:1619677445.505973:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.505986:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.505989:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.505990:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.505991:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.505993:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.505994:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.506002:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:12.0:1619677445.506005:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.506006:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.506007:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.506007:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.506008:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.507010:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:13.0:1619677445.507012:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (24)--
00000400:00000200:13.0:1619677445.507013:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.507015:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.507017:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:12.0:1619677445.507017:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (24)--
00000800:00000200:13.0:1619677445.507018:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (24)--
00000400:00000200:12.0:1619677445.507018:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.507019:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000400:00000200:12.0:1619677445.507021:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.507023:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.507035:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.507038:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.507039:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.507040:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.507040:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.507041:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.507927:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.507930:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.507932:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.507933:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.507935:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.507940:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.507956:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.507959:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.507960:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.507961:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.507963:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.507964:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.508350:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.508353:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.508354:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.508357:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000400:00000200:12.0:1619677445.508357:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.508360:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.508594:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.508596:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.508597:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.508598:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.508600:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.508601:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.509154:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.509156:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.509157:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.509159:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.509161:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.509162:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.509781:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677445.509784:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677445.509787:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677445.509788:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677445.509790:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.509793:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677445.509794:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96755828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677445.509796:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677445.509797:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677445.509798:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677445.509802:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677445.509804:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677445.509807:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677445.509808:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:13.0:1619677445.509808:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:13.0:1619677445.509810:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:13.0:1619677445.509811:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677445.509812:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677445.509813:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677445.509814:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677445.509827:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677445.509830:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677445.509831:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677445.509832:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677445.510454:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.510457:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.510458:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.510461:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.510462:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.510464:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.510694:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.510696:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.510697:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.510700:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.510701:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.510702:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.510715:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:12.0:1619677445.510718:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.510719:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.510719:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.510720:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.510721:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.510856:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.510859:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.510860:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.510861:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.510863:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.510864:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.510876:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.510879:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.510880:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.510880:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.510881:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.510883:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.510929:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.510932:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.510933:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.510935:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.510936:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.510937:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.511139:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.511141:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.511142:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.511143:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.511145:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.511146:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.511429:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.511431:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.511432:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.511433:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.511435:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.511436:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.511735:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.511738:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.511739:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.511740:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.511742:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.511743:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.512905:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.512908:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.512909:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.512911:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.512913:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.512914:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.513693:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.513695:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.513696:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.513698:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.513700:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.513701:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.513710:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.513712:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.513713:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.513714:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.513714:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.513715:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.513912:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.513915:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.513916:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.513917:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.513919:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.513921:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.514306:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.514309:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.514310:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.514312:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.514314:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.514315:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.514662:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.514664:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.514665:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.514666:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.514668:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.514669:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.515165:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.515168:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.515169:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.515172:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000400:00000200:12.0:1619677445.515172:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.515175:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.515183:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.515186:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.515187:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.515187:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.515188:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.515189:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.515479:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.515482:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.515483:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.515486:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000400:00000200:12.0:1619677445.515486:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.515489:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.515561:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.515563:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.515564:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.515567:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.515567:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.515570:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.515580:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.515582:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.515583:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.515584:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.515585:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.515586:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.515865:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.515867:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.515868:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.515869:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.515871:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.515872:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.515951:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.515954:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.515955:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.515958:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.515958:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.515961:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.516138:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.516141:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.516142:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.516144:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.516145:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.516146:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.516548:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.516550:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.516551:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.516552:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.516554:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.516555:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.516642:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.516644:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.516645:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.516646:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.516648:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.516649:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.516659:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.516662:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.516663:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.516663:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.516664:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.516665:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.516971:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.516974:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.516974:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.516976:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.516977:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.516979:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.517510:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.517512:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.517513:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.517515:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.517517:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.517518:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.518382:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.518384:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.518385:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.518387:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.518389:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.518390:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.518745:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.518748:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.518749:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.518751:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.518752:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.518753:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.518754:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.518754:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:13.0:1619677445.518755:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.518755:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.518756:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.518825:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.518828:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.518829:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.518830:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.518831:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.518833:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.518852:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677445.518855:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677445.518858:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677445.518859:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677445.518860:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.518862:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:12.0:1619677445.518863:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96755670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677445.518864:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677445.518865:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677445.518866:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:12.0:1619677445.518872:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677445.518875:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677445.518878:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677445.518879:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:12.0:1619677445.518879:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:12.0:1619677445.518881:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:12.0:1619677445.518882:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677445.518883:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677445.518883:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:13.0:1619677445.519097:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.519100:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.519101:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.519102:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.519103:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.519106:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.519263:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.519266:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.519267:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.519268:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.519270:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.519271:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.519449:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.519451:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.519452:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.519454:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.519456:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.519457:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:12.0:1619677445.519460:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.519461:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.519462:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.519462:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.519463:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.519680:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.519682:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.519683:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.519686:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.519686:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.519688:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.519702:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:12.0:1619677445.519705:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (24)--
00000400:00000200:12.0:1619677445.519706:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.519708:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.519709:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:13.0:1619677445.519710:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (24)--
00000800:00000200:12.0:1619677445.519710:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (24)--
00000400:00000200:13.0:1619677445.519711:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.519712:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.519712:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.519715:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.519731:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.519734:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.519735:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.519735:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.519736:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.519737:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.519922:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.519925:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.519926:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.519928:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.519929:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.519931:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.520227:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.520230:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.520231:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.520232:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.520234:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.520235:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.521662:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.521664:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.521665:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.521667:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.521669:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.521670:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.521742:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.521745:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.521746:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.521748:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.521749:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.521751:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.522235:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)++
00000800:00000200:13.0:1619677445.522238:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (24)--
00000400:00000200:13.0:1619677445.522239:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.522240:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.522242:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)++
00000800:00000200:13.0:1619677445.522243:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (24)--
00000800:00000200:12.0:1619677445.522243:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (24)--
00000800:00000200:13.0:1619677445.522244:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000400:00000200:12.0:1619677445.522244:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.522245:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.522247:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.522256:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.522259:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.522260:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.522260:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.522261:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.522262:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.522975:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.522978:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.522979:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.522980:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.522982:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.522983:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.523357:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.523360:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.523360:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.523363:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.523365:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.523365:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.523377:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.523380:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.523381:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.523382:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.523384:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.523385:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.523551:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.523554:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.523555:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.523556:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.523558:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.523559:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.523869:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.523871:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.523872:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.523874:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.523875:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.523876:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.523886:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:12.0:1619677445.523889:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.523890:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.523890:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.523891:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.523892:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.524093:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.524096:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.524097:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.524098:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.524099:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.524101:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.524111:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.524113:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.524114:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.524115:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.524115:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.524116:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.524228:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677445.524231:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677445.524233:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677445.524234:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677445.524236:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.524237:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:13.0:1619677445.524238:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96755571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677445.524240:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677445.524241:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677445.524242:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:13.0:1619677445.524246:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677445.524247:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:13.0:1619677445.524250:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1200] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677445.524251:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38000] (20)++
00000800:00000200:13.0:1619677445.524252:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38000] (21)++
00000800:00000200:13.0:1619677445.524254:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38000] (22)--
00000800:00000200:13.0:1619677445.524254:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677445.524255:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677445.524256:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677445.524494:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.524497:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.524498:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.524500:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.524502:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.524503:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.524683:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.524685:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.524686:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.524688:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.524689:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.524690:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.524899:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.524902:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.524903:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.524904:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.524906:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.524907:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.525132:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.525135:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.525136:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.525139:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.525139:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.525142:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.525230:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.525233:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.525234:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.525236:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.525237:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.525238:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.526732:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.526734:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.526735:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.526736:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.526739:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.526740:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.526847:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.526849:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.526850:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.526852:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.526853:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.526854:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.526865:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.526868:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.526869:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.526870:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.526871:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.526872:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.527330:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.527333:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.527333:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.527335:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.527337:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.527337:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.527346:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.527349:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.527350:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.527350:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.527351:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.527352:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.527922:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.527924:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.527925:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.527927:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.527928:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.527929:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.528110:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.528113:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.528114:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.528115:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.528117:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.528118:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.528503:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.528505:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.528506:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.528507:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.528509:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.528510:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.528579:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.528582:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.528583:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.528584:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.528585:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.528587:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.528671:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.528673:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.528674:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.528676:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.528677:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.528678:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.529317:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.529319:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.529320:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.529322:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.529324:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.529325:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.529334:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:12.0:1619677445.529337:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.529338:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.529339:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.529339:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.529341:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.529561:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.529564:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.529565:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.529567:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.529568:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.529569:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.529597:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.529599:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.529600:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.529602:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.529603:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.529605:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.529628:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.529631:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.529631:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.529633:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.529635:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.529636:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.529651:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.529654:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.529654:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.529655:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.529655:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.529656:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.530005:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.530007:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.530008:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.530010:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.530011:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.530013:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.530091:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.530093:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.530094:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.530095:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.530097:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.530098:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.530157:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.530160:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.530161:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.530163:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.530164:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.530166:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.530174:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.530176:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.530177:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.530178:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.530179:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.530180:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.531306:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.531309:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.531310:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.531311:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.531313:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.531314:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.533059:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.533062:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.533063:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.533065:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.533066:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.533068:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.533577:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.533580:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.533581:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.533583:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.533585:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.533587:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.533638:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.533641:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.533642:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.533643:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.533645:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.533646:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.533656:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.533659:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.533660:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.533661:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.533661:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.533663:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.534137:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.534140:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.534141:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.534142:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.534143:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.534145:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.534320:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.534323:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.534324:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.534325:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.534328:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.534329:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.534471:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.534473:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.534474:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.534476:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.534478:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.534479:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.534736:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.534738:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.534739:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.534742:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000400:00000200:12.0:1619677445.534742:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.534747:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.534765:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.534768:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.534769:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.534771:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.534772:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.534774:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.534800:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.534803:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.534804:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.534806:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.534807:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.534809:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.534818:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.534820:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.534821:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.534822:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.534822:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.534824:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.535073:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.535075:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.535076:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.535077:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.535079:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.535081:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.535140:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.535143:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.535143:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.535146:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.535147:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.535149:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.535348:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.535351:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.535352:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.535353:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.535356:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.535356:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.535843:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.535845:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.535846:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.535848:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.535849:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.535849:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.536141:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.536143:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.536144:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.536145:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.536147:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.536149:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.536159:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.536162:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.536166:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (24)--
00000800:00000200:13.0:1619677445.536167:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.536168:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.536168:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.536177:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.536179:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.536180:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.536182:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.536183:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.536184:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.537585:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.537587:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.537588:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.537590:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.537592:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.537593:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.537604:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.537607:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.537608:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.537609:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.537609:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.537611:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.538268:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.538270:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.538271:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.538273:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.538274:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.538275:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.538684:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.538686:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.538688:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.538690:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.538691:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.538692:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.538918:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.538921:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.538922:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.538924:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.538924:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.538927:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.539051:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.539054:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.539055:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.539056:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.539058:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.539059:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.539068:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.539071:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.539072:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.539073:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.539073:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.539075:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.539246:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.539249:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.539249:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.539251:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.539252:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.539253:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.539849:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.539852:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.539853:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.539855:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.539856:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.539858:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.540045:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.540047:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.540048:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.540050:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.540052:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.540053:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.540099:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.540101:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.540103:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.540105:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.540105:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.540107:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.540116:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.540119:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.540119:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.540121:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.540122:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.540123:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.540199:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677445.540202:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677445.540204:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677445.540206:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677445.540207:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.540209:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677445.540210:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96755302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677445.540212:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677445.540213:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677445.540214:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677445.540218:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677445.540220:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677445.540223:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677445.540224:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:13.0:1619677445.540225:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:13.0:1619677445.540227:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:13.0:1619677445.540228:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677445.540228:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677445.540229:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677445.540503:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.540505:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.540506:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.540508:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.540510:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.540511:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.540521:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.540524:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.540525:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.540525:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.540526:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.540527:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.540785:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.540787:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.540788:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.540790:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.540791:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.540792:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.540793:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.540796:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.540797:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.540798:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.540800:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.540801:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.540801:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.540804:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.540805:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.540806:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.540806:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.540807:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.540929:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.540932:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.540932:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.540934:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.540936:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.540936:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.541466:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.541468:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.541469:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.541471:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.541473:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.541480:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.541499:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.541502:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.541502:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.541504:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.541506:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.541507:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.541642:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.541645:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.541646:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.541648:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.541649:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.541650:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.541813:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.541816:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.541817:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.541819:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.541820:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.541821:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.541831:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.541834:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.541835:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.541835:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.541836:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.541837:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.541883:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.541885:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.541886:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.541888:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.541889:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.541890:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.542211:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.542213:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.542214:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.542216:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.542217:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.542218:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.542717:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.542720:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.542721:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.542724:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.542724:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.542727:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.543114:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.543117:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.543117:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.543119:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.543120:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.543121:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.543306:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.543309:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.543310:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.543312:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.543313:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.543314:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.543315:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.543315:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.543315:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.543316:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.543317:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.543696:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.543698:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.543699:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.543701:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.543702:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.543703:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.544639:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)++
00000800:00000200:12.0:1619677445.544642:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (24)--
00000400:00000200:12.0:1619677445.544643:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.544645:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.544647:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.544647:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.544656:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.544658:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.544659:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.544660:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.544662:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.544663:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.544663:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.544664:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.544664:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.544665:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.544665:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.544799:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.544801:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.544802:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.544804:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.544805:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.544807:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.545277:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.545280:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.545281:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.545282:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.545284:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.545286:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.545450:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.545452:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.545453:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.545456:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.545457:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.545459:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.545829:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.545831:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.545832:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.545833:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.545835:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.545836:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.545849:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.545852:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.545853:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.545854:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.545854:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.545855:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.546051:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)++
00000800:00000200:13.0:1619677445.546053:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (24)--
00000400:00000200:13.0:1619677445.546054:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.546056:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.546058:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)++
00000800:00000200:13.0:1619677445.546059:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (24)--
00000800:00000200:12.0:1619677445.546059:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (24)--
00000800:00000200:13.0:1619677445.546060:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000400:00000200:12.0:1619677445.546060:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.546061:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.546063:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.546075:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.546078:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.546079:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.546079:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.546080:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.546081:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.546279:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.546282:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.546283:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.546285:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.546286:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.546287:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.546775:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.546777:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.546778:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.546780:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.546782:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.546784:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.546822:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.546824:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.546824:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.546825:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.546826:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.546826:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:13.0:1619677445.546827:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.546829:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.546829:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.546830:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.546832:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.546833:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.547373:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.547376:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.547377:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.547379:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.547381:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.547382:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.547392:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.547395:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.547396:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.547396:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.547397:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.547398:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.547705:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.547708:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.547709:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.547711:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.547712:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.547713:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.547839:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.547841:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.547842:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.547843:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.547845:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.547846:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.548720:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.548722:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.548724:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.548725:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.548727:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.548728:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.549162:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.549165:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.549166:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.549168:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.549169:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.549171:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.549202:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.549205:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.549205:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.549207:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.549208:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.549211:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.549221:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.549224:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.549225:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.549225:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.549226:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.549228:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.549705:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.549707:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.549708:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.549710:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.549712:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.549713:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.549804:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.549807:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.549808:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.549809:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.549810:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.549812:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.550473:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.550476:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.550477:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.550478:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.550480:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.550481:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.550489:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.550492:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.550493:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.550493:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.550494:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.550496:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.550572:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.550575:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.550575:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.550577:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.550579:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.550580:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.550625:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.550628:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.550629:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.550631:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.550632:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.550633:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.550925:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.550928:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.550929:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.550930:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.550932:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.550933:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.551492:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.551494:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.551495:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.551497:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.551499:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.551500:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.551551:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.551554:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.551555:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.551557:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.551558:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.551559:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.551637:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.551640:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.551641:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.551642:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.551644:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.551644:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.551656:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.551659:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.551660:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.551660:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.551661:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.551662:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.551687:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.551689:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.551690:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.551692:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.551694:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.551695:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.552061:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.552063:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.552064:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.552066:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.552068:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.552069:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.552078:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:12.0:1619677445.552080:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.552081:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.552082:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.552082:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.552083:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.552777:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.552780:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.552781:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.552782:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.552784:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.552785:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.553973:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.553976:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.553976:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.553979:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.553981:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.553982:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.554150:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.554153:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.554154:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.554155:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.554157:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.554158:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.554168:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.554171:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.554172:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.554172:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.554173:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.554174:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.554346:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.554348:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.554349:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.554351:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.554353:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.554354:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.554480:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.554482:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.554483:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.554485:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.554486:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.554487:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.554755:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.554758:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.554759:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.554761:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.554762:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.554764:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.555888:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.555891:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.555892:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.555893:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.555895:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.555896:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.555968:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.555971:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.555972:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.555973:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.555975:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.555976:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.556890:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.556893:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.556893:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.556895:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.556897:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.556898:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.556942:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.556944:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.556946:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.556947:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.556949:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.556951:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.556985:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.556988:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.556989:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.556990:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.556990:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.556992:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.557135:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.557138:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.557138:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.557140:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.557142:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.557143:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.557158:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.557160:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.557161:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.557162:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.557162:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.557164:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.557355:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.557358:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.557358:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.557360:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.557362:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.557363:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.557625:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.557628:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.557629:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.557631:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.557632:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.557634:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.557679:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.557681:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.557682:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.557683:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.557685:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.557686:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.557876:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.557879:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.557880:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.557882:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.557882:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.557886:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.557973:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.557975:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.557976:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.557978:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.557979:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.557981:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.558789:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.558791:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.558792:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.558794:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.558796:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.558797:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.558810:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:12.0:1619677445.558812:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.558813:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.558814:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.558814:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.558815:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.559635:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.559638:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.559638:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.559640:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.559642:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.559643:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.559735:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.559737:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.559738:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.559740:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.559741:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.559742:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.560218:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.560221:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.560222:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.560224:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.560225:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.560227:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.560324:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.560326:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.560327:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.560328:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.560330:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.560331:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.561122:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.561124:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.561124:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.561126:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.561128:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.561129:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.561139:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.561142:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.561143:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.561143:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.561144:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.561145:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.561841:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.561844:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.561845:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.561847:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.561848:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.561855:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.561875:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.561878:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.561878:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.561880:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.561881:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.561883:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.562727:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.562730:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.562731:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.562732:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.562734:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.562735:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.562790:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.562792:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.562793:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.562795:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.562796:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.562797:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.563786:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.563789:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.563790:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.563792:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.563794:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.563795:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.563805:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.563808:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.563809:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.563809:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.563810:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.563811:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.563903:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.563905:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.563906:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.563908:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.563909:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.563910:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.563976:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.563978:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.563979:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.563982:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.563982:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.563985:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.564209:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.564212:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.564213:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.564215:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.564216:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.564217:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.564415:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.564418:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.564418:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.564419:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.564420:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.564420:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.564523:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.564526:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.564526:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.564528:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.564530:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.564538:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.564876:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.564879:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.564880:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.564881:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.564882:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (24)--
00000800:00000200:12.0:1619677445.564884:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.564895:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.564898:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.564899:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.564900:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.564902:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.564903:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.565045:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.565048:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.565048:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.565050:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.565051:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.565052:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.565067:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.565069:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.565070:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.565071:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.565071:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.565072:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.565166:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.565169:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.565170:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.565172:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.565173:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.565175:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.565588:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.565591:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.565592:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.565593:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.565595:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.565596:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.565796:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.565799:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.565800:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.565802:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.565803:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.565804:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.566011:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.566014:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.566015:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.566017:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.566018:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.566020:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.566029:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.566032:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.566032:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.566033:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.566033:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.566034:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.568346:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.568349:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.568350:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.568352:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.568354:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.568355:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.569154:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.569157:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.569158:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.569160:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.569162:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.569163:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.569175:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.569178:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.569179:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.569180:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (24)--
00000800:00000200:13.0:1619677445.569180:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.569181:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.569193:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.569196:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.569196:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.569198:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.569200:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.569201:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.569809:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.569812:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.569812:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.569814:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.569816:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.569817:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.570158:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.570161:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.570162:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.570164:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.570164:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.570166:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.570245:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.570248:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.570249:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.570251:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.570252:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.570254:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.570583:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.570585:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.570586:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.570588:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.570590:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.570591:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.570601:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.570602:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.570603:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000400:00000200:12.0:1619677445.570603:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.570606:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.570606:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000400:00000200:12.0:1619677445.570606:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.570607:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.570608:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.570609:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.570610:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.570611:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.570804:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.570807:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.570807:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.570809:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.570811:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.570812:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.571339:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.571341:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.571342:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.571344:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.571346:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.571347:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.572119:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.572122:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.572123:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.572124:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.572126:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.572127:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.572283:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.572286:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.572287:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.572289:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.572290:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.572291:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.572305:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.572308:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.572309:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.572309:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.572310:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.572311:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.572493:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.572495:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.572496:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.572498:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.572500:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.572501:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.573174:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.573176:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.573177:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.573180:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.573180:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.573183:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.573192:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:12.0:1619677445.573195:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.573196:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.573197:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.573198:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.573199:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.573228:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.573231:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.573232:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.573233:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.573234:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.573239:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.574725:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.574727:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.574728:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.574730:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.574732:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.574733:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.574823:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.574825:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.574826:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.574828:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.574829:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.574831:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.574841:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.574844:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.574845:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.574846:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.574846:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.574848:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.575572:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.575574:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.575575:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.575577:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.575579:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.575580:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.575632:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.575635:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.575635:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.575637:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.575639:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.575640:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.575890:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.575893:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.575894:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.575895:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.575896:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.575899:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.576025:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.576027:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.576028:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.576030:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.576032:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.576033:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.576034:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.576036:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.576037:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.576038:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.576040:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.576041:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.576308:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.576311:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.576312:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.576313:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.576315:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.576316:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.576324:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.576327:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.576328:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.576329:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.576329:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.576331:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.576418:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.576421:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.576421:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.576423:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.576425:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.576425:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.576683:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.576686:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.576686:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.576688:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.576690:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.576690:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.576790:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.576793:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.576794:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.576795:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.576797:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.576798:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.576808:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.576809:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.576809:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000400:00000200:13.0:1619677445.576810:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677445.576811:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:13.0:1619677445.576812:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.576812:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.576813:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.576814:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.576815:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.576816:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.576817:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.577081:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.577084:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.577084:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.577086:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.577088:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.577089:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.577333:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.577335:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.577336:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.577338:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.577340:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.577341:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.577375:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.577378:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.577378:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.577380:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.577381:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.577383:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.577541:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.577544:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.577545:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.577547:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000400:00000200:12.0:1619677445.577547:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.577550:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.577558:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.577560:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.577561:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.577562:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.577563:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.577564:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.577740:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.577742:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.577743:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.577745:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.577746:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.577747:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.577834:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.577837:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.577838:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.577840:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.577841:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.577842:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.577853:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.577856:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.577857:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.577858:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.577858:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.577860:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.578003:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.578006:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.578007:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.578009:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000400:00000200:12.0:1619677445.578009:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.578011:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.578121:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.578123:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.578124:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.578126:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.578127:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.578129:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.578313:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677445.578316:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677445.578318:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677445.578320:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677445.578320:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.578322:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:12.0:1619677445.578323:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96754691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677445.578325:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677445.578326:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677445.578327:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:12.0:1619677445.578331:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677445.578333:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:12.0:1619677445.578336:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f300] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677445.578337:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980000] (20)++
00000800:00000200:12.0:1619677445.578337:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980000] (21)++
00000800:00000200:12.0:1619677445.578339:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980000] (22)--
00000800:00000200:12.0:1619677445.578340:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677445.578341:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677445.578343:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677445.578343:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677445.578357:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677445.578359:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677445.578360:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677445.578361:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:13.0:1619677445.578455:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.578458:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.578459:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.578462:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.578462:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.578465:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.578536:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.578539:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.578539:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.578541:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.578543:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.578544:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.579609:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.579612:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.579613:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.579615:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.579616:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.579617:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.580396:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.580398:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.580399:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.580401:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.580403:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.580404:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.580415:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.580417:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.580418:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.580419:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.580419:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.580420:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.580805:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.580807:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.580808:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.580810:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.580812:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.580812:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.580943:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.580946:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.580946:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.580948:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.580950:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.580951:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.581060:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.581063:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.581063:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.581065:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.581067:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.581067:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.581165:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.581168:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.581168:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.581170:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.581172:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.581173:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.581498:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.581501:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.581502:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.581503:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.581505:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.581506:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.581516:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.581519:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.581521:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.581521:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.581522:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.581523:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.581720:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.581722:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.581723:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.581724:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.581726:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.581728:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.581757:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.581759:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.581760:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.581762:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.581763:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.581764:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.582448:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.582451:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.582452:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.582454:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.582455:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.582456:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.582471:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.582473:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.582474:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.582475:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.582475:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.582476:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.583356:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.583358:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.583359:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.583361:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.583363:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.583364:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.583714:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.583717:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.583717:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.583719:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.583720:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.583722:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.583730:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.583733:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.583734:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.583734:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.583735:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.583736:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.583974:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.583976:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.583977:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.583979:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.583980:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.583982:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.585131:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.585133:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.585135:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.585138:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.585138:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.585141:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.585151:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.585154:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.585155:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.585157:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.585158:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.585159:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.585772:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.585775:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.585776:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.585778:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.585780:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.585783:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.585792:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.585795:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.585796:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.585797:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.585797:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.585799:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.586390:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.586393:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.586394:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.586395:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.586397:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.586398:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.586488:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677445.586491:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677445.586493:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677445.586494:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677445.586495:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.586497:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:12.0:1619677445.586499:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96754553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677445.586500:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677445.586501:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677445.586502:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:12.0:1619677445.586505:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677445.586507:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677445.586510:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677445.586511:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:12.0:1619677445.586511:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:12.0:1619677445.586513:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:12.0:1619677445.586514:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677445.586515:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677445.586515:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:13.0:1619677445.586612:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.586615:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.586615:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.586617:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.586619:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.586623:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.586639:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.586641:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.586642:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.586643:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.586645:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.586646:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.587104:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.587106:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.587107:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.587108:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.587110:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.587111:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.587425:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.587428:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.587429:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.587431:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.587433:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.587435:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.587447:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.587450:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.587451:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.587452:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.587452:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.587454:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.587464:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.587466:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.587467:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.587468:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.587470:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.587471:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.587638:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.587640:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.587641:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.587643:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.587644:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.587646:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.588293:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.588300:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.588301:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.588302:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000400:00000200:12.0:1619677445.588303:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.588304:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.588305:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)++
00000800:00000200:13.0:1619677445.588305:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (24)--
00000800:00000200:13.0:1619677445.588306:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.588307:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.588308:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.589254:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677445.589257:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677445.589259:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677445.589261:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677445.589261:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.589263:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:12.0:1619677445.589264:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96754515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677445.589266:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677445.589267:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677445.589267:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:12.0:1619677445.589271:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677445.589273:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:12.0:1619677445.589275:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056be80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677445.589276:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718600] (20)++
00000800:00000200:12.0:1619677445.589276:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718600] (21)++
00000800:00000200:12.0:1619677445.589278:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718600] (22)--
00000800:00000200:12.0:1619677445.589279:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677445.589280:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677445.589280:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:13.0:1619677445.589595:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.589597:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.589598:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.589599:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.589602:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.589603:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.589920:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.589922:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.589923:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.589926:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.589926:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.589928:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.590328:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.590331:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.590332:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.590334:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.590336:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.590337:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.590721:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.590724:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.590725:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.590727:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.590728:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.590730:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.590757:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.590759:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.590760:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.590762:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.590763:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.590764:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.590772:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.590774:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.590775:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.590776:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.590776:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.590777:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.591080:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.591082:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.591083:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.591085:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.591086:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.591087:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.591116:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.591119:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.591120:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.591122:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.591123:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.591125:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.591811:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.591814:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.591814:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.591816:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.591818:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.591819:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.592013:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.592015:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.592016:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.592019:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.592019:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.592021:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.592030:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.592033:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.592034:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.592036:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.592037:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.592040:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.592058:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.592061:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.592062:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.592063:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.592065:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.592066:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.592078:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.592081:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.592082:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.592082:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.592083:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.592085:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.592760:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.592762:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.592763:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.592766:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.592767:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.592773:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.592774:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:12.0:1619677445.592776:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.592777:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.592778:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.592779:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.592780:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.592795:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.592798:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.592799:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.592801:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.592802:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.592803:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.595019:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.595022:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.595023:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.595025:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.595027:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.595028:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:7.0F:1619677445.595080:0:13334:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.52@o2ib7 Addr resolved: 0
00000800:00000200:7.0:1619677445.595091:0:13334:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.52@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:12.0:1619677445.595187:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.595189:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.595190:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.595192:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.595193:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.595194:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.595370:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.595373:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.595374:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.595377:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.595377:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.595380:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.595409:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.595411:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.595412:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.595414:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.595415:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.595416:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.595430:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.595433:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.595434:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.595435:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.595435:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.595436:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:7.0:1619677445.595467:0:13334:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.52@o2ib7 Route resolved: 0
00000800:00000200:7.0:1619677445.595477:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff9197842c0000 0xc042c0000(0xc042c0000)
00000800:00000200:7.0:1619677445.595479:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff9197842c1000 0xc042c1000(0xc042c1000)
00000800:00000200:7.0:1619677445.595480:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff9197842c2000 0xc042c2000(0xc042c2000)
00000800:00000200:7.0:1619677445.595480:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff9197842c3000 0xc042c3000(0xc042c3000)
00000800:00000200:7.0:1619677445.595481:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff9197842c4000 0xc042c4000(0xc042c4000)
00000800:00000200:7.0:1619677445.595481:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff9197842c5000 0xc042c5000(0xc042c5000)
00000800:00000200:7.0:1619677445.595482:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff9197842c6000 0xc042c6000(0xc042c6000)
00000800:00000200:7.0:1619677445.595482:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff9197842c7000 0xc042c7000(0xc042c7000)
00000800:00000200:7.0:1619677445.595483:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff9197842c8000 0xc042c8000(0xc042c8000)
00000800:00000200:7.0:1619677445.595483:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff9197842c9000 0xc042c9000(0xc042c9000)
00000800:00000200:7.0:1619677445.595484:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff9197842ca000 0xc042ca000(0xc042ca000)
00000800:00000200:7.0:1619677445.595485:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff9197842cb000 0xc042cb000(0xc042cb000)
00000800:00000200:7.0:1619677445.595485:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff9197842cc000 0xc042cc000(0xc042cc000)
00000800:00000200:7.0:1619677445.595486:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff9197842cd000 0xc042cd000(0xc042cd000)
00000800:00000200:7.0:1619677445.595486:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff9197842ce000 0xc042ce000(0xc042ce000)
00000800:00000200:7.0:1619677445.595487:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff9197842cf000 0xc042cf000(0xc042cf000)
00000800:00000200:7.0:1619677445.595487:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff9197842d0000 0xc042d0000(0xc042d0000)
00000800:00000200:7.0:1619677445.595488:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff9197842d1000 0xc042d1000(0xc042d1000)
00000800:00000200:12.0:1619677445.595748:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.595750:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.595751:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.595752:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.595754:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.595755:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.595847:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.595850:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.595850:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.595852:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.595853:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.595855:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.596333:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.596336:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.596336:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.596338:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.596341:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.596342:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.596370:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.596373:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.596374:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.596376:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.596377:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.596379:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.596392:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.596395:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.596396:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.596397:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.596398:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.596399:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.596460:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.596462:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.596463:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.596465:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.596467:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.596467:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:7.0:1619677445.596543:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (19)++
00000800:00000200:7.0:1619677445.596544:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (20)--
00000800:00000200:7.0:1619677445.596545:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (19)++
00000800:00000200:7.0:1619677445.596546:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (20)--
00000800:00000200:7.0:1619677445.596546:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (19)++
00000800:00000200:7.0:1619677445.596547:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (20)--
00000800:00000200:7.0:1619677445.596547:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (19)++
00000800:00000200:7.0:1619677445.596548:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (20)--
00000800:00000200:7.0:1619677445.596548:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (19)++
00000800:00000200:7.0:1619677445.596549:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (20)--
00000800:00000200:7.0:1619677445.596549:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (19)++
00000800:00000200:7.0:1619677445.596550:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (20)--
00000800:00000200:7.0:1619677445.596550:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (19)++
00000800:00000200:7.0:1619677445.596550:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (20)--
00000800:00000200:7.0:1619677445.596551:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (19)++
00000800:00000200:7.0:1619677445.596552:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (20)--
00000800:00000200:7.0:1619677445.596552:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (19)++
00000800:00000200:7.0:1619677445.596553:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (20)--
00000800:00000200:7.0:1619677445.596553:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (19)++
00000800:00000200:7.0:1619677445.596553:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (20)--
00000800:00000200:7.0:1619677445.596554:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (19)++
00000800:00000200:7.0:1619677445.596555:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (20)--
00000800:00000200:7.0:1619677445.596555:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (19)++
00000800:00000200:7.0:1619677445.596556:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (20)--
00000800:00000200:7.0:1619677445.596556:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (19)++
00000800:00000200:7.0:1619677445.596556:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (20)--
00000800:00000200:7.0:1619677445.596557:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (19)++
00000800:00000200:7.0:1619677445.596558:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (20)--
00000800:00000200:7.0:1619677445.596558:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (19)++
00000800:00000200:7.0:1619677445.596559:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (20)--
00000800:00000200:7.0:1619677445.596559:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (19)++
00000800:00000200:7.0:1619677445.596560:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (20)--
00000800:00000200:7.0:1619677445.596560:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (19)++
00000800:00000200:7.0:1619677445.596561:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (20)--
00000800:00000200:7.0:1619677445.596561:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (19)++
00000800:00000200:7.0:1619677445.596561:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (20)--
00000800:00000200:12.0:1619677445.596833:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.596835:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.596836:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.596838:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.596840:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.596841:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.596934:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.596937:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.596938:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.596939:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.596941:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.596942:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.596970:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.596972:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.596973:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.596975:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.596976:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.596978:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.597333:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.597335:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.597336:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.597338:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.597340:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.597341:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.597429:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.597432:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.597433:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.597435:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.597436:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.597437:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.597449:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.597452:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.597453:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:13.0:1619677445.597454:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (24)--
00000800:00000200:13.0:1619677445.597455:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.597456:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.597459:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.597460:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.597461:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.597463:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.597463:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.597521:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.597524:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.597525:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.597527:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.597528:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.597529:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.598045:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.598048:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.598049:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.598051:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.598052:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.598057:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.598104:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.598106:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.598107:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.598109:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.598110:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.598111:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.598125:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:12.0:1619677445.598128:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.598128:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.598129:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.598130:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.598131:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.598175:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.598178:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.598179:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.598180:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.598182:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.598182:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.598194:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.598197:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.598198:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.598198:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.598199:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.598200:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.599322:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.599324:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.599325:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.599327:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.599329:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.599330:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.599409:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.599412:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.599412:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.599414:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.599415:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.599416:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.600089:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.600091:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.600092:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.600094:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.600095:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.600097:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.600703:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.600706:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.600707:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.600708:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.600711:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.600711:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.601394:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.601397:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.601398:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.601400:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.601401:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.601403:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.601438:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.601441:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.601441:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.601443:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.601444:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.601445:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.601454:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.601456:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.601458:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.601459:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.601459:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.601460:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.601536:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:13.0:1619677445.601538:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (24)--
00000400:00000200:13.0:1619677445.601539:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.601541:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.601542:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:13.0:1619677445.601543:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (24)--
00000800:00000200:12.0:1619677445.601543:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (24)--
00000800:00000200:13.0:1619677445.601544:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000400:00000200:12.0:1619677445.601544:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.601545:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.601547:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.601940:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.601943:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.601944:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.601945:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.601947:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.601948:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.602012:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.602014:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.602015:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.602017:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.602018:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.602020:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.602033:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.602035:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.602036:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.602037:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.602038:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.602039:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.602244:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.602246:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.602247:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.602248:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.602250:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.602251:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.602383:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.602385:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.602386:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.602387:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.602389:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.602390:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.602400:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.602403:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.602403:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.602404:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.602405:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.602406:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.602741:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.602743:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.602744:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.602746:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.602747:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.602749:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.602796:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.602799:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.602800:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.602802:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.602803:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.602805:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.602908:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.602911:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.602911:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.602913:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.602915:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.602916:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.603141:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.603144:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.603145:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.603146:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.603147:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.603153:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.603170:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.603173:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.603173:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.603175:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.603176:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.603178:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.603465:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.603467:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.603468:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.603469:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.603471:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.603472:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.603485:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.603488:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.603489:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.603490:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.603490:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.603491:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.603918:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.603921:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.603922:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.603924:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.603925:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.603927:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:24.2:1619677445.604033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980600] (21)++
00000800:00000200:13.0:1619677445.604080:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.604083:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.604084:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.604086:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.604086:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.604089:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.604098:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.604101:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.604103:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.604104:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.604104:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.604105:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.604354:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.604357:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.604358:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.604360:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.604362:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.604364:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:24.2:1619677445.604831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980600] (21)++
00000800:00000200:13.0:1619677445.605755:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.605758:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.605759:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.605761:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.605763:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.605764:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.605812:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.605815:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.605816:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.605818:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.605819:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.605821:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.606117:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.606119:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.606120:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.606122:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.606123:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.606124:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.606513:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.606516:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.606516:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.606518:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.606521:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.606522:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:24.2:1619677445.606594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980600] (21)++
00000800:00000200:12.0:1619677445.606600:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.606602:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.606603:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.606605:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.606606:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.606608:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.606617:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.606621:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.606621:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.606622:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.606623:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.606624:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.606703:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.606706:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.606707:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.606709:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.606710:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.606711:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.606878:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.606881:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.606882:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.606883:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.606884:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.606885:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.607187:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.607189:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.607190:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.607192:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.607193:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.607195:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.607247:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.607250:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.607250:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.607252:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.607253:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.607255:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.607265:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.607268:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.607269:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.607270:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.607271:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.607272:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.607443:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.607446:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.607446:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.607448:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.607450:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.607451:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.608130:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.608133:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.608134:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.608135:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.608137:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.608138:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.608503:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.608506:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.608507:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.608509:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.608510:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.608511:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.608706:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.608709:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.608709:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.608712:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.608713:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.608715:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.608726:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.608729:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.608730:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)++
00000800:00000200:13.0:1619677445.608731:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (24)--
00000800:00000200:13.0:1619677445.608732:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.608733:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.608735:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.608736:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.608737:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.608739:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.608740:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.608818:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.608820:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.608821:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.608822:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.608824:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.608825:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.608965:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.608967:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.608968:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.608970:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.608971:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.608973:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.609126:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.609129:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.609129:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.609131:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.609133:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.609134:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.609666:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.609669:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.609669:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.609672:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.609674:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.609675:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.609676:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.609676:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.609677:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.609677:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.609677:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (24)--
00000800:00000200:12.0:1619677445.609678:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.609680:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.609714:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.609717:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.609718:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.609719:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.609721:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.609723:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.610527:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.610529:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.610530:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.610531:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.610533:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.610534:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.610547:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.610549:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.610550:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.610550:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.610551:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.610553:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.611058:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.611061:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.611062:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.611064:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.611065:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.611066:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.611105:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.611108:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.611108:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.611110:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.611111:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.611112:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.611557:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.611559:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.611560:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.611561:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.611564:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.611565:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.611678:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.611680:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.611681:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.611682:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.611684:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.611685:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.611701:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.611704:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.611705:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.611705:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.611706:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.611707:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.612051:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.612054:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.612054:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.612056:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.612058:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.612059:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.612101:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.612104:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.612105:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.612106:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.612108:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.612109:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.612873:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.612876:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.612877:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.612878:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.612880:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.612881:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.613100:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.613103:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.613103:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.613105:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.613107:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.613108:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.613485:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.613488:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.613489:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.613491:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.613493:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.613494:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.613804:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.613806:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.613807:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.613809:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.613810:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.613812:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.613824:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.613826:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.613826:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.613827:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.613828:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.613828:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.613828:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.613829:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.613829:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000400:00000200:12.0:1619677445.613830:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.613831:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.613833:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.613847:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.613850:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.613851:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.613852:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.613852:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.613854:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.614104:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.614107:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.614108:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.614110:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.614111:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.614113:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.614566:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.614568:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.614569:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.614572:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.614573:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.614575:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.614594:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.614596:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.614597:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.614599:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.614601:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.614610:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.614625:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.614626:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.614627:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.614629:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.614629:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.614630:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.614631:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000400:00000200:12.0:1619677445.614631:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.614632:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000400:00000200:12.0:1619677445.614632:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.614633:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.614635:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.614879:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.614881:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.614882:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.614884:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.614884:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.614886:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.614950:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.614952:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.614953:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.614955:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.614956:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.614958:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.614978:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.614981:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.614981:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.614982:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.614983:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.614984:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.615541:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.615543:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.615544:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.615547:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.615548:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.615549:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.615618:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.615621:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.615621:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.615623:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.615624:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.615625:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.615636:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:12.0:1619677445.615639:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.615640:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.615641:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.615641:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.615643:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.615711:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.615714:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.615714:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.615717:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.615717:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.615723:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.615741:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.615744:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.615745:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.615746:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.615748:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.615749:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.616300:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.616302:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.616303:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.616305:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.616306:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.616307:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.616318:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.616320:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.616321:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.616322:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.616322:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.616323:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.616718:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.616720:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.616721:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.616722:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.616724:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.616725:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.616843:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.616845:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.616846:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.616847:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.616849:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.616850:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.616954:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.616957:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.616958:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.616960:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.616961:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.616963:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.617718:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:13.0:1619677445.617721:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (24)--
00000400:00000200:13.0:1619677445.617722:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.617724:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.617725:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:13.0:1619677445.617727:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.617727:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (24)--
00000400:00000200:12.0:1619677445.617727:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.617728:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000400:00000200:12.0:1619677445.617729:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.617730:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.617731:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:13.0:1619677445.617731:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (24)--
00000800:00000200:12.0:1619677445.617732:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (24)--
00000800:00000200:13.0:1619677445.617733:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.617733:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.618616:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.618618:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.618619:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.618621:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.618623:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.618624:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.619153:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.619155:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.619156:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.619158:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.619159:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.619161:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.619325:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.619327:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.619328:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.619330:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.619332:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.619333:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.619485:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.619487:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.619488:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.619490:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.619491:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.619493:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.619518:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.619520:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.619521:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.619522:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.619524:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.619525:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.619653:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.619656:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.619656:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.619659:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000400:00000200:12.0:1619677445.619659:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.619662:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.619670:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.619673:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.619674:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.619675:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.619675:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.619676:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.619918:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677445.619921:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677445.619924:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677445.619925:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677445.619926:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.619927:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:12.0:1619677445.619929:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96753971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677445.619930:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677445.619931:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677445.619932:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:12.0:1619677445.619935:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677445.619937:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677445.619940:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778d80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677445.619941:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24000] (20)++
00000800:00000200:12.0:1619677445.619942:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24000] (21)++
00000800:00000200:12.0:1619677445.619944:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24000] (22)--
00000800:00000200:12.0:1619677445.619945:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677445.619946:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677445.619946:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:13.0:1619677445.621103:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.621106:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.621106:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.621108:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.621110:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.621111:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.621122:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.621125:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.621125:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.621126:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.621127:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.621128:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.621542:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.621544:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.621545:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.621547:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.621548:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.621550:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.621624:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.621627:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.621627:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.621629:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.621631:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.621632:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.621742:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.621745:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.621745:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.621747:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.621748:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.621750:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.622710:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.622712:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.622713:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.622714:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.622716:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.622717:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.622732:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.622735:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.622736:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.622737:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.622737:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.622738:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.623471:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.623474:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.623475:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.623478:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000400:00000200:12.0:1619677445.623478:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.623481:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.624089:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.624091:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.624092:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.624094:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.624095:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.624096:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.624279:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)++
00000800:00000200:12.0:1619677445.624282:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (24)--
00000400:00000200:12.0:1619677445.624283:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.624285:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.624286:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)++
00000800:00000200:13.0:1619677445.624287:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (24)--
00000400:00000200:13.0:1619677445.624288:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.624290:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.624292:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.624300:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.624301:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.624319:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.624321:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.624322:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.624325:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.624326:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.624327:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.624337:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.624340:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.624341:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.624341:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.624342:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.624343:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.624393:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.624395:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.624396:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.624398:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.624399:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.624401:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.624796:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:13.0:1619677445.624799:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (24)--
00000400:00000200:13.0:1619677445.624800:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.624801:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.624803:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:12.0:1619677445.624803:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (24)--
00000800:00000200:13.0:1619677445.624804:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.624804:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.624805:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000400:00000200:12.0:1619677445.624806:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.624808:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.624819:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.624822:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.624823:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.624823:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.624824:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.624825:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.625071:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.625074:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.625075:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.625077:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.625078:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.625080:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.625286:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.625288:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.625289:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.625292:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.625292:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.625306:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.625328:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.625330:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.625331:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.625333:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.625334:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.625336:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.625481:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.625484:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.625485:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.625487:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.625488:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.625489:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.625729:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.625731:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.625732:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.625733:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.625735:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.625736:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.625837:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.625839:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.625840:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.625842:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.625843:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.625844:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.626096:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.626098:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.626099:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.626101:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.626102:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.626103:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.626117:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.626120:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.626120:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.626121:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.626121:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.626123:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.626138:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.626141:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.626142:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.626144:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.626145:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.626146:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.627127:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.627129:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.627130:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.627132:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.627134:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.627135:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.627240:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.627243:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.627244:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.627246:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.627247:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.627249:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.627261:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:12.0:1619677445.627265:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.627265:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.627266:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.627267:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.627268:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.628379:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.628382:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.628382:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.628384:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.628386:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.628387:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.629050:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.629053:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.629054:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.629056:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.629058:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.629059:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.629060:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.629062:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.629063:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.629065:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.629067:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.629069:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.629069:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.629072:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.629073:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.629074:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.629074:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.629076:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.629451:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.629454:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.629455:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.629456:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.629458:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.629459:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.629611:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.629614:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.629615:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.629616:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.629618:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.629619:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.629942:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)++
00000800:00000200:13.0:1619677445.629945:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (24)--
00000400:00000200:13.0:1619677445.629946:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.629947:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.629948:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)++
00000800:00000200:12.0:1619677445.629949:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (24)--
00000800:00000200:13.0:1619677445.629950:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.629950:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.629951:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000400:00000200:12.0:1619677445.629952:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.629954:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.630005:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.630008:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.630009:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677445.630012:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000400:00000200:13.0:1619677445.630014:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.630017:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.630341:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.630343:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.630344:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.630345:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.630348:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.630349:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.630363:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:12.0:1619677445.630366:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.630367:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.630367:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.630368:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.630370:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.630581:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.630583:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.630584:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.630585:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.630587:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.630588:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.630603:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.630606:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.630607:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.630607:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.630608:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.630609:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.631223:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.631225:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.631226:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.631228:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.631230:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.631233:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.631251:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.631254:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.631255:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.631256:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.631257:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.631259:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.631589:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.631592:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.631593:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.631594:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.631596:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.631597:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.631728:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.631731:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.631732:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.631733:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.631734:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.631736:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.631928:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.631931:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.631931:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.631933:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.631935:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.631935:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.631950:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.631953:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.631954:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.631954:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.631955:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.631956:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.632910:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.632912:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.632913:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.632914:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.632916:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.632918:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.633808:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.633810:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.633811:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.633812:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.633815:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.633816:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.635895:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.635898:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.635899:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.635901:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.635902:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.635905:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.635970:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.635972:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.635973:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.635975:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.635976:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.635977:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.636129:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.636132:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.636132:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.636134:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.636136:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.636137:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.636150:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:12.0:1619677445.636153:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.636153:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.636154:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.636155:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.636156:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.636826:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.636828:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.636829:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.636831:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.636833:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.636834:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.637600:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.637603:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.637604:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.637606:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.637608:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.637609:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.637705:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.637708:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.637708:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.637710:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.637712:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.637714:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.637830:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.637832:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.637833:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.637835:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.637836:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.637839:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.639895:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)++
00000800:00000200:13.0:1619677445.639897:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (24)--
00000400:00000200:13.0:1619677445.639898:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.639900:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.639902:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)++
00000800:00000200:13.0:1619677445.639903:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (24)--
00000800:00000200:12.0:1619677445.639903:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (24)--
00000400:00000200:12.0:1619677445.639903:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.639904:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000400:00000200:12.0:1619677445.639905:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.639907:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.639957:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.639960:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.639961:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.639963:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.639963:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.639966:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.639973:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:12.0:1619677445.639976:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.639977:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.639978:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.639979:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.639980:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.640112:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.640115:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.640116:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.640117:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.640118:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.640119:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.640325:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.640328:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.640329:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.640330:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.640332:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.640333:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.640579:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.640581:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.640582:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.640583:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.640586:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.640587:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.640588:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.640591:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.640591:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.640593:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.640595:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.640595:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.640676:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:13.0:1619677445.640678:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (24)--
00000400:00000200:13.0:1619677445.640679:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.640681:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.640682:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:12.0:1619677445.640683:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (24)--
00000800:00000200:13.0:1619677445.640684:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.640684:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.640685:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000400:00000200:12.0:1619677445.640685:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.640688:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.640705:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.640708:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.640709:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.640709:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.640710:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.640712:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.640736:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.640738:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.640739:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.640741:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.640742:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.640743:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.640815:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.640818:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.640818:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.640820:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.640822:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.640823:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.640922:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.640924:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.640925:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.640927:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.640928:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (24)--
00000800:00000200:12.0:1619677445.640931:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.640941:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.640943:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.640944:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.640946:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.640947:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.640948:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.640959:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.640961:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.640962:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.640964:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.640965:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.640967:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.640975:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.640978:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.640979:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.640980:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.640981:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.640982:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.641195:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.641198:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.641199:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.641200:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.641202:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.641203:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.641407:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:13.0:1619677445.641409:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (24)--
00000400:00000200:13.0:1619677445.641410:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.641412:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.641414:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:13.0:1619677445.641415:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (24)--
00000800:00000200:12.0:1619677445.641415:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (24)--
00000800:00000200:13.0:1619677445.641416:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000400:00000200:12.0:1619677445.641416:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.641418:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.641420:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.641430:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.641433:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.641434:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.641435:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.641435:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.641437:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.641485:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.641488:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.641489:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.641491:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.641492:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.641493:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.641854:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.641856:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.641857:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.641859:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.641860:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.641862:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.641874:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.641877:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.641878:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.641880:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.641881:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.641882:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.643843:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677445.643846:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677445.643848:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677445.643850:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677445.643851:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.643853:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:13.0:1619677445.643854:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96753584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677445.643855:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677445.643856:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677445.643857:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:13.0:1619677445.643861:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677445.643862:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677445.643866:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f400] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677445.643866:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984000] (20)++
00000800:00000200:13.0:1619677445.643867:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984000] (21)++
00000800:00000200:13.0:1619677445.643869:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984000] (22)--
00000800:00000200:13.0:1619677445.643870:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677445.643871:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677445.643872:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677445.643873:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677445.643886:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677445.643889:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677445.643890:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677445.643891:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677445.644232:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.644234:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.644235:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.644237:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.644238:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.644239:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.644496:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.644499:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.644500:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.644501:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.644503:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.644504:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.644513:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.644516:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.644516:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.644517:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.644517:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.644518:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.644634:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.644637:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.644638:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.644640:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.644641:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.644642:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.644865:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.644868:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.644869:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.644870:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.644872:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.644873:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.644873:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.644876:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.644877:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.644878:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.644880:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.644881:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.644882:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.644885:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.644886:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.644887:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.644887:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.644888:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.645047:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.645049:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.645050:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.645052:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.645054:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.645055:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.645056:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.645058:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.645059:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.645061:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.645062:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.645063:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.645100:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.645103:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.645103:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.645105:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677445.645107:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.645107:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.645117:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.645119:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.645120:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.645121:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.645121:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.645122:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.645278:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.645280:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.645281:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.645282:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.645284:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.645285:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.645600:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.645603:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.645604:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.645606:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.645608:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.645609:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.645910:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.645912:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.645913:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.645915:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.645917:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.645918:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.645989:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.645992:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.645993:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.645994:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.645996:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.645997:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.646008:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.646012:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.646013:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.646013:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.646014:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.646016:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.646251:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.646254:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.646255:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.646256:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.646257:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.646258:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.646345:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.646347:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.646348:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.646350:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.646352:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.646352:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.646404:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.646407:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.646407:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.646409:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.646411:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.646412:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.646421:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.646424:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.646424:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.646425:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.646426:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.646427:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:24.2:1619677445.646719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980600] (19)++
00000800:00000200:7.0:1619677445.646778:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff919e46980600] (18)--
00000800:00000200:13.0:1619677445.646815:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.646817:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.646818:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.646820:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.646821:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.646827:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.646847:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.646849:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.646850:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.646853:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000400:00000200:12.0:1619677445.646853:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.646855:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.646957:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.646959:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.646960:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.646962:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.646963:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.646964:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.647108:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.647111:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.647111:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.647113:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.647115:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.647116:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.650258:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.650261:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.650262:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.650264:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.650266:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.650267:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.650553:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.650555:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.650556:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.650558:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.650560:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.650561:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.650573:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.650576:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.650576:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.650577:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.650578:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.650579:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.651042:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.651044:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.651045:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.651047:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.651048:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.651050:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.651505:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.651507:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.651508:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.651509:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.651512:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.651513:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.651754:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.651757:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.651758:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.651759:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.651760:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.651762:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.651892:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.651896:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (25)--
00000800:00000200:13.0:1619677445.651898:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.651898:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.651900:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.651903:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.651905:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.651907:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.651908:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.651909:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.651910:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:12.0:1619677445.651910:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.651911:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)++
00000800:00000200:13.0:1619677445.651911:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (24)--
00000800:00000200:12.0:1619677445.651912:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (24)--
00000800:00000200:13.0:1619677445.651913:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.651914:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.651914:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.651915:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.651915:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.651916:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000400:00000200:12.0:1619677445.651917:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.651918:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.652066:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.652068:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.652070:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.652071:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.652072:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.652074:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.652087:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.652090:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.652090:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.652091:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.652092:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.652093:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.652177:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.652179:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.652180:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.652182:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.652183:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.652185:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.652318:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.652321:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.652321:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.652323:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.652324:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.652326:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.652539:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.652542:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.652542:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.652546:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.652546:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.652553:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.652572:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.652575:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.652576:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.652578:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.652579:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.652580:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.652594:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.652597:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.652598:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.652598:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.652599:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.652600:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.652693:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.652695:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.652696:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.652698:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.652699:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.652705:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.652725:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.652727:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.652728:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.652730:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.652731:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.652733:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.652895:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.652898:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.652899:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.652900:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.652902:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.652903:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.652964:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.652967:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.652968:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.652970:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.652971:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.652972:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.653546:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677445.653548:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677445.653551:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677445.653553:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677445.653554:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.653556:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677445.653558:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96753428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677445.653559:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677445.653560:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677445.653561:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677445.653565:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677445.653567:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677445.653569:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9ff80] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677445.653570:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24600] (20)++
00000800:00000200:13.0:1619677445.653571:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24600] (21)++
00000800:00000200:13.0:1619677445.653573:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24600] (22)--
00000800:00000200:13.0:1619677445.653573:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677445.653574:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677445.653574:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677445.653844:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.653847:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.653848:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.653849:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.653850:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.653853:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.653864:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.653867:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.653868:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.653869:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.653870:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.653871:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.654437:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.654439:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.654440:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.654441:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.654443:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.654444:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.655096:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.655099:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.655100:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.655102:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677445.655103:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.655105:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.655823:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.655826:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.655827:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.655828:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.655831:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.655831:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.656054:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.656057:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.656058:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.656061:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.656061:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.656063:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.656355:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.656358:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.656359:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.656361:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.656363:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.656363:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.656364:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.656365:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.656365:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.656366:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (24)--
00000800:00000200:12.0:1619677445.656366:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (24)--
00000800:00000200:13.0:1619677445.656367:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.656368:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.656699:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.656701:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.656702:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.656704:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.656705:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.656707:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.656717:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.656720:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.656721:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.656721:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.656722:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.656724:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.656919:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.656921:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.656922:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.656923:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.656925:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.656926:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.657022:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.657024:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.657025:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.657027:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677445.657028:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.657029:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.657179:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.657182:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.657183:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.657184:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.657186:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.657188:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.657782:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.657785:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.657786:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.657787:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.657790:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.657791:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.657791:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.657792:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.657792:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.657793:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.657793:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (24)--
00000800:00000200:12.0:1619677445.657794:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.657795:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.658452:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.658454:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.658455:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.658457:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.658459:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.658460:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.658500:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.658502:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.658503:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.658505:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.658506:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.658507:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.658697:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.658700:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.658701:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.658703:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.658704:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.658706:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.659091:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.659093:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.659094:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.659096:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.659097:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.659099:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.659162:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.659165:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.659166:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.659167:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.659169:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.659170:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.659429:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.659432:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.659433:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.659435:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.659437:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.659437:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.659448:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.659450:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.659451:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.659452:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.659453:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.659454:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.659713:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.659716:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.659717:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.659718:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.659720:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.659725:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.659741:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.659744:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.659745:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.659746:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.659748:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.659749:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.659968:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.659970:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.659971:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.659973:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.659974:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.659976:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.660373:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.660376:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.660377:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.660379:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.660381:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.660381:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.660396:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:12.0:1619677445.660399:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.660400:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.660400:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.660401:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.660402:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.660433:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.660435:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.660436:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.660438:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.660439:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.660441:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.660611:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.660614:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.660615:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.660616:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.660618:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.660620:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.660636:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.660639:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.660639:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.660640:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.660641:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.660641:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.660721:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.660723:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.660724:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.660725:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.660727:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.660727:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.661347:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.661350:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.661351:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.661352:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.661354:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.661355:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.661818:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.661821:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.661822:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.661824:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.661825:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.661826:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.661835:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.661838:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.661839:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.661840:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.661842:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.661842:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.661925:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.661928:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.661929:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.661930:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.661932:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.661933:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.661948:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.661950:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.661951:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.661952:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.661953:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.661954:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.662787:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.662789:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.662790:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.662792:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.662794:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.662794:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.663117:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.663119:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.663120:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.663121:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.663123:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.663124:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.663253:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.663255:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.663256:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.663258:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.663259:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.663260:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.663524:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.663527:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.663528:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.663529:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.663531:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.663532:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.663656:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.663659:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.663660:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.663662:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.663663:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.663665:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.663732:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.663734:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.663735:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.663736:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.663738:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.663739:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.663749:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.663752:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.663752:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.663753:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.663753:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.663754:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.664074:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.664076:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.664077:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.664078:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.664080:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.664081:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.664097:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.664100:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.664101:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.664102:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.664102:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.664104:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.665206:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.665209:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.665209:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.665211:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.665213:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.665214:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.665378:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.665381:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.665381:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.665383:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.665385:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.665386:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.665773:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.665776:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.665777:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.665779:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.665780:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.665782:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.666483:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.666486:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.666487:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.666489:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.666490:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.666492:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.666919:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.666922:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.666923:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.666925:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.666926:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.666927:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.667099:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.667102:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.667103:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.667104:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.667105:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.667108:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.667115:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.667118:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.667119:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.667120:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.667120:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.667122:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.667300:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.667303:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.667304:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.667307:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.667308:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.667309:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.667340:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.667342:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.667343:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.667345:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.667347:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.667348:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.667349:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.667351:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.667352:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.667354:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.667355:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.667356:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.668054:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.668056:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.668057:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.668059:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.668061:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.668062:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.668092:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.668095:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.668095:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.668097:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.668098:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.668099:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.668110:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.668113:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.668114:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.668114:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.668115:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.668115:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.668512:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.668514:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.668515:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.668516:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.668517:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.668517:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:13.0:1619677445.668518:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.668519:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.668519:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.668520:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.668521:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.668522:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.668534:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.668537:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.668538:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.668538:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.668539:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.668540:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.668868:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.668871:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.668872:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.668874:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.668875:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.668877:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.669104:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.669106:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.669107:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.669108:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.669110:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.669112:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.669507:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.669509:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.669510:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.669511:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.669513:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.669515:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.669742:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.669744:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.669745:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.669746:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.669748:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.669750:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.669976:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.669979:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.669980:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.669982:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.669983:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.669985:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.669994:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:12.0:1619677445.669997:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.669997:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.669998:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.669999:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.670000:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.670336:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.670339:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.670339:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.670341:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.670343:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.670344:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.670495:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.670498:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.670499:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.670501:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.670502:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.670503:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.671080:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.671083:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.671084:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.671086:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.671087:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.671091:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.671109:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.671111:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.671112:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.671114:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.671115:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.671116:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.671635:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.671638:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.671639:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.671640:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.671642:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.671644:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.671654:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.671657:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.671658:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.671658:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.671660:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.671661:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.671906:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.671908:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.671909:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.671911:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.671912:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.671914:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.672463:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.672466:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.672467:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.672469:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.672471:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.672473:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.672728:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.672730:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.672731:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.672733:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.672734:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.672735:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.672747:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:12.0:1619677445.672750:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.672751:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.672752:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.672753:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.672753:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.673066:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.673069:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.673069:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.673072:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000400:00000200:12.0:1619677445.673072:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.673075:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.673346:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.673349:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.673349:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.673351:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.673353:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.673354:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.673362:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.673365:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.673365:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.673366:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.673366:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.673367:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.674433:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.674435:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.674436:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.674437:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.674440:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.674440:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.674929:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.674931:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.674932:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.674934:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.674935:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.674936:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.675157:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.675159:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.675160:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.675162:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.675163:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.675165:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.675660:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.675662:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.675664:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.675667:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.675667:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.675670:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.676043:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.676046:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.676046:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.676048:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.676050:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.676051:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.676082:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.676084:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.676085:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.676087:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.676088:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.676089:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.677061:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.677064:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.677065:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.677066:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.677068:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.677069:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.677072:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.677073:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.677073:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.677074:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.677075:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.677326:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.677328:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.677329:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.677332:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.677333:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.677335:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.677363:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.677366:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.677367:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.677368:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.677370:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.677372:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.677812:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.677814:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.677815:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.677817:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.677819:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.677820:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.678115:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.678118:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.678119:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.678121:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.678122:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.678123:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.678134:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.678137:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.678138:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.678139:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.678139:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.678140:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.678161:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.678164:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.678165:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.678166:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.678168:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.678169:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.678495:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.678498:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.678498:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.678500:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.678502:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.678503:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.678975:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.678978:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.678979:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.678981:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.678982:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.678984:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.679183:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.679185:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.679186:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.679187:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.679189:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.679190:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.679201:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.679204:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.679204:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.679205:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.679205:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.679206:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.679334:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:13.0:1619677445.679336:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (24)--
00000400:00000200:13.0:1619677445.679337:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.679338:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.679341:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:13.0:1619677445.679342:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (24)--
00000800:00000200:12.0:1619677445.679342:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (24)--
00000400:00000200:12.0:1619677445.679343:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.679344:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000400:00000200:12.0:1619677445.679344:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.679345:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.679346:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:12.0:1619677445.679346:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.679347:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.679348:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.679349:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.679892:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.679894:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.679895:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.679897:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.679898:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.679899:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.680092:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.680095:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.680096:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.680097:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.680098:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.680101:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.680525:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.680527:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.680528:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.680530:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.680532:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.680534:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.681102:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.681104:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.681105:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.681107:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.681109:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.681111:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.681486:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.681489:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.681490:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.681492:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.681494:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.681495:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.681559:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.681562:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.681563:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.681565:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.681566:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.681568:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.681579:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.681582:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.681583:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.681583:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.681584:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.681585:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.681823:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.681826:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.681827:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.681829:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.681830:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.681831:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.681898:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.681900:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.681900:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.681902:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.681904:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.681905:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.681914:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.681916:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.681917:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.681918:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.681918:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.681919:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.682103:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.682106:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.682107:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.682108:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.682110:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.682111:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.682162:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.682164:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.682165:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.682166:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.682168:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.682169:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.682621:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.682624:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.682625:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.682626:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.682628:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.682629:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.682753:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.682756:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.682757:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.682758:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.682760:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.682761:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.682777:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.682780:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.682780:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.682781:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.682782:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.682783:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.682862:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.682864:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.682865:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.682866:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.682868:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.682869:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.683045:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.683048:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.683049:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.683051:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.683052:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.683054:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.683810:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:13.0:1619677445.683812:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (24)--
00000400:00000200:13.0:1619677445.683813:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.683816:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.683817:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.683819:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.683828:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.683831:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.683832:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.683834:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.683835:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.683837:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.685064:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.685066:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.685067:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.685069:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.685071:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.685072:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.685081:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.685084:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.685085:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.685086:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.685086:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.685087:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.685390:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.685393:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.685394:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.685396:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.685398:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.685399:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.685573:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677445.685576:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677445.685578:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677445.685579:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677445.685581:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.685582:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677445.685583:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96752897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677445.685585:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677445.685587:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677445.685588:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:13.0:1619677445.685591:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677445.685593:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677445.685596:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c3f9ac00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677445.685597:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987e00] (20)++
00000800:00000200:13.0:1619677445.685597:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987e00] (21)++
00000800:00000200:13.0:1619677445.685600:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987e00] (22)--
00000800:00000200:13.0:1619677445.685600:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677445.685601:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677445.685602:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677445.685804:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.685806:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.685807:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.685808:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.685810:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.685811:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.686022:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.686025:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.686026:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.686028:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.686029:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.686031:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.686039:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:12.0:1619677445.686042:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.686043:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.686044:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.686044:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.686046:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.686230:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.686233:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.686234:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.686235:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.686236:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.686237:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.686382:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)++
00000800:00000200:12.0:1619677445.686384:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (24)--
00000400:00000200:12.0:1619677445.686385:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.686387:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.686389:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)++
00000800:00000200:13.0:1619677445.686390:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (24)--
00000800:00000200:12.0:1619677445.686390:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (24)--
00000400:00000200:13.0:1619677445.686391:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677445.686392:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000400:00000200:13.0:1619677445.686393:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.686395:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.686665:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.686667:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.686668:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.686670:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.686672:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.686673:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.686689:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.686692:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.686693:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.686694:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677445.686696:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.686697:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.686706:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.686710:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.686711:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.686711:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.686712:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.686713:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.686995:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.686998:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.686999:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.687001:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.687002:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.687004:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.687176:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.687179:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.687180:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.687182:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.687183:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.687184:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.687229:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.687232:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.687233:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.687235:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.687236:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.687237:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.687346:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.687349:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.687349:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.687351:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.687353:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.687355:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.687549:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.687552:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.687553:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.687554:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.687555:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.687557:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.687558:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)++
00000800:00000200:13.0:1619677445.687559:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (24)--
00000800:00000200:13.0:1619677445.687559:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.687560:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.687561:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.688309:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.688311:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.688312:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.688314:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.688316:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.688317:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.689136:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.689139:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.689140:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.689141:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.689142:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.689144:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.689157:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.689161:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.689162:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.689162:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.689163:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.689164:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.689225:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.689228:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.689229:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.689230:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.689232:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.689232:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.690051:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.690054:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.690055:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.690057:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.690058:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.690059:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.690209:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.690211:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.690212:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.690214:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.690215:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.690216:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.690510:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.690513:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.690513:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.690515:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.690517:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.690518:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.690526:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.690529:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.690530:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.690531:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.690531:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.690533:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.690898:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.690900:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.690901:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.690902:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.690904:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.690905:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.691114:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.691117:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.691118:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.691120:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.691121:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.691123:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.691159:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.691162:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (24)--
00000400:00000200:13.0:1619677445.691163:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.691165:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.691167:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (25)++
00000800:00000200:13.0:1619677445.691168:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (25)--
00000800:00000200:12.0:1619677445.691168:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (25)--
00000400:00000200:12.0:1619677445.691169:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.691170:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.691171:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:13.0:1619677445.691172:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (24)--
00000400:00000200:13.0:1619677445.691172:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677445.691173:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.691174:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.691174:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.691177:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.691195:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.691197:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.691198:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.691199:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.691199:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.691201:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.691353:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.691355:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.691356:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.691357:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.691359:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.691360:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.691555:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.691558:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.691559:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.691560:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.691561:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.691563:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.691690:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.691693:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.691694:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.691695:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.691697:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.691698:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.691710:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:12.0:1619677445.691714:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.691714:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.691715:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.691716:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.691717:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.691748:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.691750:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.691752:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.691754:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.691754:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.691757:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.692119:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.692122:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.692123:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.692124:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.692126:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.692128:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.692591:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.692594:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.692594:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.692596:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.692598:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.692599:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.693122:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.693125:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.693125:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.693127:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677445.693128:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.693129:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.693139:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.693142:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.693143:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.693143:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.693144:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.693145:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.693170:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.693173:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.693174:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.693175:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.693177:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.693178:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.693860:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.693863:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.693863:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.693865:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.693867:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.693868:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.694373:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.694375:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.694376:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.694378:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.694380:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.694381:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.695179:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.695181:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.695182:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.695184:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.695185:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.695186:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.695558:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.695560:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.695561:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.695563:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.695565:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.695566:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.696191:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.696193:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.696194:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.696195:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.696198:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.696199:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.696210:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:12.0:1619677445.696213:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.696213:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.696214:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.696214:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.696215:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.696572:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.696574:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.696575:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.696576:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.696578:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.696579:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.696616:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.696618:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.696619:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.696623:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000400:00000200:12.0:1619677445.696623:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.696628:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.696641:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.696644:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.696644:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.696644:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.696645:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.696645:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.696646:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000400:00000200:12.0:1619677445.696646:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.696647:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000400:00000200:12.0:1619677445.696648:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.696649:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.696653:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.696662:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.696665:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.696665:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.696667:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.696669:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.696670:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:23.0F:1619677445.696713:0:13382:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.54@o2ib7 Addr resolved: 0
00000800:00000200:23.0:1619677445.696724:0:13382:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.54@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:12.0:1619677445.696785:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.696787:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.696788:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.696790:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.696792:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.696793:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.696901:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:13.0:1619677445.696904:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (24)--
00000400:00000200:13.0:1619677445.696905:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.696907:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.696908:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:12.0:1619677445.696909:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (24)--
00000800:00000200:13.0:1619677445.696910:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.696910:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.696911:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000400:00000200:12.0:1619677445.696912:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.696914:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.696938:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.696940:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.696941:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.696942:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.696942:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.696944:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:23.0:1619677445.697071:0:13382:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.54@o2ib7 Route resolved: 0
00000800:00000200:23.0:1619677445.697083:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff919e46f1e000 0x12c6f1e000(0x12c6f1e000)
00000800:00000200:23.0:1619677445.697084:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff919e46f1f000 0x12c6f1f000(0x12c6f1f000)
00000800:00000200:23.0:1619677445.697085:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff919e46f20000 0x12c6f20000(0x12c6f20000)
00000800:00000200:23.0:1619677445.697085:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff919e46f21000 0x12c6f21000(0x12c6f21000)
00000800:00000200:23.0:1619677445.697086:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff919e46f22000 0x12c6f22000(0x12c6f22000)
00000800:00000200:23.0:1619677445.697086:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff919e46f23000 0x12c6f23000(0x12c6f23000)
00000800:00000200:23.0:1619677445.697087:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff919e46f24000 0x12c6f24000(0x12c6f24000)
00000800:00000200:23.0:1619677445.697087:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff919e46f25000 0x12c6f25000(0x12c6f25000)
00000800:00000200:23.0:1619677445.697088:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff919e46f26000 0x12c6f26000(0x12c6f26000)
00000800:00000200:23.0:1619677445.697088:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff919e46f27000 0x12c6f27000(0x12c6f27000)
00000800:00000200:23.0:1619677445.697089:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff919e46f28000 0x12c6f28000(0x12c6f28000)
00000800:00000200:23.0:1619677445.697089:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff919e46f29000 0x12c6f29000(0x12c6f29000)
00000800:00000200:23.0:1619677445.697090:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff919e46f2a000 0x12c6f2a000(0x12c6f2a000)
00000800:00000200:23.0:1619677445.697090:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff919e46f2b000 0x12c6f2b000(0x12c6f2b000)
00000800:00000200:23.0:1619677445.697091:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff919e46f2c000 0x12c6f2c000(0x12c6f2c000)
00000800:00000200:23.0:1619677445.697092:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff919e46f2d000 0x12c6f2d000(0x12c6f2d000)
00000800:00000200:23.0:1619677445.697092:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff919e46f2e000 0x12c6f2e000(0x12c6f2e000)
00000800:00000200:23.0:1619677445.697093:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff919e46f2f000 0x12c6f2f000(0x12c6f2f000)
00000800:00000200:12.0:1619677445.697234:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.697237:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.697238:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.697240:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.697241:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.697244:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.697261:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.697263:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.697264:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.697266:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.697267:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.697268:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.697873:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.697875:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.697876:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.697878:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.697880:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.697881:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:23.0:1619677445.697886:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e49d37c00] (19)++
00000800:00000200:23.0:1619677445.697887:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e49d37c00] (20)--
00000800:00000200:23.0:1619677445.697887:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e49d37c00] (19)++
00000800:00000200:23.0:1619677445.697888:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e49d37c00] (20)--
00000800:00000200:23.0:1619677445.697888:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e49d37c00] (19)++
00000800:00000200:23.0:1619677445.697888:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e49d37c00] (20)--
00000800:00000200:23.0:1619677445.697889:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e49d37c00] (19)++
00000800:00000200:23.0:1619677445.697889:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e49d37c00] (20)--
00000800:00000200:23.0:1619677445.697889:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e49d37c00] (19)++
00000800:00000200:23.0:1619677445.697890:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e49d37c00] (20)--
00000800:00000200:23.0:1619677445.697891:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e49d37c00] (19)++
00000800:00000200:23.0:1619677445.697891:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e49d37c00] (20)--
00000800:00000200:23.0:1619677445.697892:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e49d37c00] (19)++
00000800:00000200:23.0:1619677445.697893:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e49d37c00] (20)--
00000800:00000200:23.0:1619677445.697894:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e49d37c00] (19)++
00000800:00000200:23.0:1619677445.697895:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e49d37c00] (20)--
00000800:00000200:23.0:1619677445.697895:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e49d37c00] (19)++
00000800:00000200:23.0:1619677445.697896:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e49d37c00] (20)--
00000800:00000200:23.0:1619677445.697896:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e49d37c00] (19)++
00000800:00000200:23.0:1619677445.697896:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e49d37c00] (20)--
00000800:00000200:23.0:1619677445.697897:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e49d37c00] (19)++
00000800:00000200:23.0:1619677445.697897:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e49d37c00] (20)--
00000800:00000200:23.0:1619677445.697897:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e49d37c00] (19)++
00000800:00000200:23.0:1619677445.697898:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e49d37c00] (20)--
00000800:00000200:23.0:1619677445.697898:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e49d37c00] (19)++
00000800:00000200:23.0:1619677445.697899:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e49d37c00] (20)--
00000800:00000200:23.0:1619677445.697899:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e49d37c00] (19)++
00000800:00000200:23.0:1619677445.697899:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e49d37c00] (20)--
00000800:00000200:23.0:1619677445.697900:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e49d37c00] (19)++
00000800:00000200:23.0:1619677445.697900:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e49d37c00] (20)--
00000800:00000200:23.0:1619677445.697900:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e49d37c00] (19)++
00000800:00000200:23.0:1619677445.697901:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e49d37c00] (20)--
00000800:00000200:23.0:1619677445.697901:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e49d37c00] (19)++
00000800:00000200:23.0:1619677445.697902:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e49d37c00] (20)--
00000800:00000200:23.0:1619677445.697902:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e49d37c00] (19)++
00000800:00000200:23.0:1619677445.697902:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e49d37c00] (20)--
00000800:00000200:12.0:1619677445.697931:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.697934:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.697935:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.697936:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.697938:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.697939:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.697989:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.697992:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.697992:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.697994:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.697996:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.697997:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.698009:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.698013:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.698014:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.698014:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.698015:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.698016:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.698715:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.698718:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.698719:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.698720:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.698722:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.698723:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.699026:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677445.699030:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677445.699032:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677445.699033:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677445.699035:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.699036:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:13.0:1619677445.699038:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96752653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677445.699040:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677445.699041:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677445.699042:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:13.0:1619677445.699047:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677445.699050:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:13.0:1619677445.699053:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f300] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677445.699054:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980000] (20)++
00000800:00000200:13.0:1619677445.699055:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980000] (21)++
00000800:00000200:13.0:1619677445.699057:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980000] (22)--
00000800:00000200:13.0:1619677445.699057:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677445.699058:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677445.699059:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:24.2:1619677445.699092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (21)++
00000800:00000200:12.0:1619677445.699173:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:24.2:1619677445.699175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (20)++
00000800:00000200:12.0:1619677445.699175:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.699176:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.699177:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.699179:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.699180:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.699198:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.699200:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.699201:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.699202:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.699202:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.699204:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.699300:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.699303:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.699304:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.699306:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.699307:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.699309:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.699450:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.699452:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.699453:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.699455:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.699456:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.699457:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.700280:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.700282:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.700283:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.700285:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.700286:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.700287:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.701695:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.701698:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.701699:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.701701:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.701702:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.701703:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.701713:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:12.0:1619677445.701715:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.701716:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.701717:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.701717:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.701719:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.701822:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.701825:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.701825:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.701827:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.701829:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.701830:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.702062:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.702065:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.702066:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.702067:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.702069:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.702070:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.702081:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.702083:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.702084:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.702086:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.702087:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.702089:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.702427:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.702430:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.702430:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.702432:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.702434:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.702435:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.702490:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.702493:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.702494:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.702496:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.702496:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.702498:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.702507:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.702510:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.702511:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.702512:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.702512:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.702514:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.702581:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.702584:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.702584:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.702586:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.702588:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.702589:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.702869:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.702871:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.702872:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.702874:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.702875:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.702877:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.702889:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.702892:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.702893:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.702894:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.702894:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.702895:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.703839:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.703842:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.703843:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.703844:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.703846:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.703847:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.704057:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.704060:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.704061:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.704062:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.704063:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.704065:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.704139:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.704142:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.704143:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.704145:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.704146:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.704149:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.704509:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.704511:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.704512:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.704513:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.704516:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.704517:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.704596:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.704599:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.704600:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.704602:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.704602:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.704604:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.704678:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.704681:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.704682:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.704684:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.704685:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.704687:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.705029:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.705032:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.705032:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.705034:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.705036:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.705037:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.705049:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:12.0:1619677445.705052:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.705053:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.705054:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.705054:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.705056:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.705451:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.705453:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.705454:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.705456:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.705458:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.705459:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.706446:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677445.706449:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677445.706451:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677445.706454:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677445.706454:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.706456:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:12.0:1619677445.706457:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96752533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677445.706459:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677445.706460:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677445.706461:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:12.0:1619677445.706464:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677445.706466:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:12.0:1619677445.706468:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056be80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677445.706469:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718600] (20)++
00000800:00000200:12.0:1619677445.706470:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718600] (21)++
00000800:00000200:12.0:1619677445.706472:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718600] (22)--
00000800:00000200:12.0:1619677445.706472:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677445.706474:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677445.706475:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677445.706476:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677445.706489:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677445.706491:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677445.706492:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677445.706493:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:7.0:1619677445.706790:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff919b884e3a00] (18)--
00000800:00000200:23.0:1619677445.707104:0:13334:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.104@o2ib7 Addr resolved: 0
00000800:00000200:23.0:1619677445.707111:0:13334:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.104@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:13.0:1619677445.707284:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.707286:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.707287:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.707289:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.707291:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.707292:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.707307:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.707309:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.707310:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.707311:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.707311:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.707312:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:23.0:1619677445.707321:0:13334:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.104@o2ib7 Route resolved: 0
00000800:00000200:23.0:1619677445.707331:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff919b87899000 0x1007899000(0x1007899000)
00000800:00000200:23.0:1619677445.707331:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff919b8789a000 0x100789a000(0x100789a000)
00000800:00000200:23.0:1619677445.707332:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff919b8789b000 0x100789b000(0x100789b000)
00000800:00000200:23.0:1619677445.707332:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff919b8789c000 0x100789c000(0x100789c000)
00000800:00000200:23.0:1619677445.707333:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff919b8789d000 0x100789d000(0x100789d000)
00000800:00000200:23.0:1619677445.707333:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff919b8789e000 0x100789e000(0x100789e000)
00000800:00000200:23.0:1619677445.707334:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff919b8789f000 0x100789f000(0x100789f000)
00000800:00000200:23.0:1619677445.707335:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff919b878a0000 0x10078a0000(0x10078a0000)
00000800:00000200:23.0:1619677445.707335:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff919b878a1000 0x10078a1000(0x10078a1000)
00000800:00000200:23.0:1619677445.707336:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff919b878a2000 0x10078a2000(0x10078a2000)
00000800:00000200:23.0:1619677445.707337:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff919b878a3000 0x10078a3000(0x10078a3000)
00000800:00000200:23.0:1619677445.707337:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff919b878a4000 0x10078a4000(0x10078a4000)
00000800:00000200:23.0:1619677445.707338:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff919b878a5000 0x10078a5000(0x10078a5000)
00000800:00000200:23.0:1619677445.707338:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff919b878a6000 0x10078a6000(0x10078a6000)
00000800:00000200:23.0:1619677445.707339:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff919b878a7000 0x10078a7000(0x10078a7000)
00000800:00000200:23.0:1619677445.707339:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff919b878a8000 0x10078a8000(0x10078a8000)
00000800:00000200:23.0:1619677445.707340:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff919b878a9000 0x10078a9000(0x10078a9000)
00000800:00000200:23.0:1619677445.707340:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff919b878aa000 0x10078aa000(0x10078aa000)
00000800:00000200:12.0:1619677445.707426:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.707429:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.707430:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.707432:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.707433:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.707434:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.707521:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.707524:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.707524:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.707526:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.707527:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.707528:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:23.0:1619677445.707999:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (19)++
00000800:00000200:23.0:1619677445.708000:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (20)--
00000800:00000200:23.0:1619677445.708000:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (19)++
00000800:00000200:23.0:1619677445.708001:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (20)--
00000800:00000200:23.0:1619677445.708001:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (19)++
00000800:00000200:23.0:1619677445.708001:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (20)--
00000800:00000200:23.0:1619677445.708002:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (19)++
00000800:00000200:23.0:1619677445.708002:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (20)--
00000800:00000200:23.0:1619677445.708002:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (19)++
00000800:00000200:23.0:1619677445.708003:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (20)--
00000800:00000200:23.0:1619677445.708003:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (19)++
00000800:00000200:23.0:1619677445.708003:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (20)--
00000800:00000200:23.0:1619677445.708004:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (19)++
00000800:00000200:23.0:1619677445.708004:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (20)--
00000800:00000200:23.0:1619677445.708004:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (19)++
00000800:00000200:23.0:1619677445.708005:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (20)--
00000800:00000200:23.0:1619677445.708005:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (19)++
00000800:00000200:23.0:1619677445.708006:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (20)--
00000800:00000200:23.0:1619677445.708006:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (19)++
00000800:00000200:23.0:1619677445.708007:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (20)--
00000800:00000200:23.0:1619677445.708007:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (19)++
00000800:00000200:23.0:1619677445.708007:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (20)--
00000800:00000200:23.0:1619677445.708008:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (19)++
00000800:00000200:23.0:1619677445.708008:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (20)--
00000800:00000200:23.0:1619677445.708009:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (19)++
00000800:00000200:23.0:1619677445.708009:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (20)--
00000800:00000200:23.0:1619677445.708010:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (19)++
00000800:00000200:23.0:1619677445.708010:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (20)--
00000800:00000200:23.0:1619677445.708011:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (19)++
00000800:00000200:23.0:1619677445.708011:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (20)--
00000800:00000200:23.0:1619677445.708011:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (19)++
00000800:00000200:23.0:1619677445.708012:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (20)--
00000800:00000200:23.0:1619677445.708012:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (19)++
00000800:00000200:23.0:1619677445.708013:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (20)--
00000800:00000200:23.0:1619677445.708013:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (19)++
00000800:00000200:23.0:1619677445.708013:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (20)--
00000800:00000200:12.0:1619677445.708278:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:12.0:1619677445.708281:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (24)--
00000400:00000200:12.0:1619677445.708281:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.708283:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.708285:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.708285:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.708288:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.708289:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.708290:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.708292:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.708293:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.708725:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.708728:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.708729:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.708730:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.708732:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.708737:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.708752:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.708755:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.708755:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.708757:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.708759:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.708760:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.708770:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.708773:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.708774:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.708775:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.708775:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.708776:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.708823:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.708826:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.708827:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.708828:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.708830:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.708831:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.708845:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.708847:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.708848:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:13.0:1619677445.708849:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (24)--
00000800:00000200:13.0:1619677445.708849:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.708850:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.708853:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.708853:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.708855:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.708857:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.708858:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.709475:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.709478:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.709479:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.709481:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.709483:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.709485:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.709567:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.709570:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.709570:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.709572:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.709574:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.709575:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.709693:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.709695:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.709697:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.709698:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.709699:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.709701:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.709939:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.709942:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.709943:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.709944:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.709946:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.709947:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.710204:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.710206:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.710207:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.710209:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.710210:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.710212:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.710221:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.710224:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.710225:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.710225:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.710226:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.710227:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.710428:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.710431:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.710432:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.710434:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.710435:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.710436:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.710481:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.710483:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.710484:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.710485:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.710487:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.710488:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.710988:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.710990:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.710991:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.710993:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.710995:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.710996:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.711102:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.711105:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.711106:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.711108:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.711108:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.711111:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.711120:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:12.0:1619677445.711123:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.711124:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.711125:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:12.0:1619677445.711126:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.711127:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.711705:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.711708:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.711709:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.711710:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.711712:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.711713:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.712752:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.712754:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.712755:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.712756:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.712759:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.712759:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.713099:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.713102:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.713102:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.713104:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.713106:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.713107:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.713145:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.713148:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.713148:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.713150:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.713151:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.713153:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.713259:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.713261:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.713262:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.713264:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.713265:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.713266:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.713277:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.713279:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.713280:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.713281:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.713281:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.713283:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.713557:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.713559:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.713560:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.713562:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.713564:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.713565:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.713566:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.713568:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.713569:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.713570:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.713572:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.713573:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:12.0:1619677445.713573:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.713576:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.713577:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.713577:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.713578:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.713579:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.713718:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.713721:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.713722:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.713724:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.713725:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.713727:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.713931:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.713933:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.713934:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.713936:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.713938:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.713938:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.713967:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.713969:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.713970:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.713971:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.713973:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:12.0:1619677445.713974:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.714422:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.714425:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.714426:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.714427:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.714429:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.714431:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.714605:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.714608:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:13.0:1619677445.714609:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.714611:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.714612:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.714613:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.715426:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.715428:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.715429:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.715431:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.715433:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.715433:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.715445:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.715447:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.715448:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.715449:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.715449:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.715450:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.716179:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.716181:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.716182:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.716184:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.716185:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.716187:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.716711:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.716714:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.716714:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.716717:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.716718:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.716720:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.716777:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.716780:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.716781:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.716783:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.716783:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.716785:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.716895:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:12.0:1619677445.716898:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8400] (23)--
00000400:00000200:12.0:1619677445.716899:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.716901:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000400:00000200:12.0:1619677445.716901:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.716904:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:13.0:1619677445.716911:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)++
00000800:00000200:13.0:1619677445.716914:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.716915:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8400] (22)++
00000800:00000200:13.0:1619677445.716916:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8400] (23)--
00000800:00000200:13.0:1619677445.716916:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (22)--
00000800:00000200:13.0:1619677445.716917:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.717079:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.717082:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.717082:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.717084:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.717085:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.717088:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.717099:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.717102:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.717102:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.717103:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.717104:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.717106:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.718463:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.718465:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.718466:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.718468:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.718470:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.718471:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.718530:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (20)++
00000800:00000200:12.0:1619677445.718532:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17d8400] (21)--
00000800:00000200:12.0:1619677445.718533:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (20)--
00000800:00000200:12.0:1619677445.718534:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (19)--
00000800:00000200:12.0:1619677445.718570:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.718572:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.718573:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.718575:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.718577:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:7.0:1619677445.718577:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff91a7c17d8400] (18)--
00000800:00000200:13.0:1619677445.718578:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:7.0:1619677445.719104:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff919e46983e00] (18)--
00000800:00000200:12.0:1619677445.719107:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.719109:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.719110:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.719111:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.719113:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.719114:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:23.0:1619677445.719212:0:13382:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.101@o2ib7 Addr resolved: 0
00000800:00000200:23.0:1619677445.719217:0:13382:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.101@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:12.0:1619677445.719254:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.719256:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.719257:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.719258:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.719260:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.719261:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.719271:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.719274:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.719275:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.719276:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.719276:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.719277:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:23.0:1619677445.719474:0:13382:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.101@o2ib7 Route resolved: 0
00000800:00000200:23.0:1619677445.719484:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff919e45d37000 0x12c5d37000(0x12c5d37000)
00000800:00000200:23.0:1619677445.719485:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff919e45d38000 0x12c5d38000(0x12c5d38000)
00000800:00000200:23.0:1619677445.719485:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff919e45d39000 0x12c5d39000(0x12c5d39000)
00000800:00000200:23.0:1619677445.719486:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff919e45d3a000 0x12c5d3a000(0x12c5d3a000)
00000800:00000200:23.0:1619677445.719486:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff919e45d3b000 0x12c5d3b000(0x12c5d3b000)
00000800:00000200:23.0:1619677445.719487:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff919e45d3c000 0x12c5d3c000(0x12c5d3c000)
00000800:00000200:23.0:1619677445.719488:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff919e45d3d000 0x12c5d3d000(0x12c5d3d000)
00000800:00000200:23.0:1619677445.719488:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff919e45d3e000 0x12c5d3e000(0x12c5d3e000)
00000800:00000200:23.0:1619677445.719489:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff919e45d3f000 0x12c5d3f000(0x12c5d3f000)
00000800:00000200:23.0:1619677445.719489:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff919e45d40000 0x12c5d40000(0x12c5d40000)
00000800:00000200:23.0:1619677445.719490:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff919e45d41000 0x12c5d41000(0x12c5d41000)
00000800:00000200:23.0:1619677445.719490:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff919e45d42000 0x12c5d42000(0x12c5d42000)
00000800:00000200:23.0:1619677445.719491:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff919e45d43000 0x12c5d43000(0x12c5d43000)
00000800:00000200:23.0:1619677445.719491:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff919e45d44000 0x12c5d44000(0x12c5d44000)
00000800:00000200:23.0:1619677445.719492:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff919e45d45000 0x12c5d45000(0x12c5d45000)
00000800:00000200:23.0:1619677445.719492:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff919e45d46000 0x12c5d46000(0x12c5d46000)
00000800:00000200:23.0:1619677445.719493:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff919e45d47000 0x12c5d47000(0x12c5d47000)
00000800:00000200:23.0:1619677445.719493:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff919e45d48000 0x12c5d48000(0x12c5d48000)
00000800:00000200:23.0:1619677445.719603:0:13334:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.105@o2ib7 Addr resolved: 0
00000800:00000200:23.0:1619677445.719609:0:13334:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.105@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:7.0:1619677445.719627:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff918febd24000] (18)--
00000800:00000200:12.0:1619677445.719792:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.719795:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.719796:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.719798:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.719800:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.719800:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.719848:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.719851:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.719852:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.719853:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.719855:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.719856:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:7.0:1619677445.719908:0:13334:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.103@o2ib7 Addr resolved: 0
00000800:00000200:7.0:1619677445.719915:0:13334:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.103@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:7.0:1619677445.720043:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff9193c2984000] (18)--
00000800:00000200:12.0:1619677445.720061:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.720063:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.720064:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.720066:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.720067:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.720068:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:23.0:1619677445.720130:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (19)++
00000800:00000200:23.0:1619677445.720131:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (20)--
00000800:00000200:23.0:1619677445.720132:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (19)++
00000800:00000200:23.0:1619677445.720132:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (20)--
00000800:00000200:23.0:1619677445.720132:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (19)++
00000800:00000200:23.0:1619677445.720133:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (20)--
00000800:00000200:23.0:1619677445.720133:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (19)++
00000800:00000200:23.0:1619677445.720133:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (20)--
00000800:00000200:23.0:1619677445.720134:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (19)++
00000800:00000200:23.0:1619677445.720134:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (20)--
00000800:00000200:23.0:1619677445.720134:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (19)++
00000800:00000200:23.0:1619677445.720135:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (20)--
00000800:00000200:23.0:1619677445.720135:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (19)++
00000800:00000200:23.0:1619677445.720136:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (20)--
00000800:00000200:23.0:1619677445.720136:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (19)++
00000800:00000200:23.0:1619677445.720136:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (20)--
00000800:00000200:23.0:1619677445.720137:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (19)++
00000800:00000200:23.0:1619677445.720137:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (20)--
00000800:00000200:23.0:1619677445.720137:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (19)++
00000800:00000200:23.0:1619677445.720138:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (20)--
00000800:00000200:23.0:1619677445.720138:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (19)++
00000800:00000200:23.0:1619677445.720138:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (20)--
00000800:00000200:23.0:1619677445.720139:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (19)++
00000800:00000200:23.0:1619677445.720139:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (20)--
00000800:00000200:23.0:1619677445.720139:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (19)++
00000800:00000200:23.0:1619677445.720140:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (20)--
00000800:00000200:23.0:1619677445.720140:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (19)++
00000800:00000200:23.0:1619677445.720141:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (20)--
00000800:00000200:23.0:1619677445.720141:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (19)++
00000800:00000200:23.0:1619677445.720141:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (20)--
00000800:00000200:23.0:1619677445.720142:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (19)++
00000800:00000200:23.0:1619677445.720142:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (20)--
00000800:00000200:23.0:1619677445.720142:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (19)++
00000800:00000200:23.0:1619677445.720143:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (20)--
00000800:00000200:23.0:1619677445.720143:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (19)++
00000800:00000200:23.0:1619677445.720143:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (20)--
00000800:00000200:23.0:1619677445.720153:0:13382:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.105@o2ib7 Route resolved: 0
00000800:00000200:23.0:1619677445.720163:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff918febcec000 0x46bcec000(0x46bcec000)
00000800:00000200:23.0:1619677445.720163:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff918febced000 0x46bced000(0x46bced000)
00000800:00000200:23.0:1619677445.720164:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff918febcee000 0x46bcee000(0x46bcee000)
00000800:00000200:23.0:1619677445.720164:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff918febcef000 0x46bcef000(0x46bcef000)
00000800:00000200:23.0:1619677445.720165:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff918febcf0000 0x46bcf0000(0x46bcf0000)
00000800:00000200:23.0:1619677445.720165:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff918febcf1000 0x46bcf1000(0x46bcf1000)
00000800:00000200:23.0:1619677445.720166:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff918febcf2000 0x46bcf2000(0x46bcf2000)
00000800:00000200:23.0:1619677445.720167:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff918febcf3000 0x46bcf3000(0x46bcf3000)
00000800:00000200:23.0:1619677445.720167:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff918febcf4000 0x46bcf4000(0x46bcf4000)
00000800:00000200:23.0:1619677445.720168:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff918febcf5000 0x46bcf5000(0x46bcf5000)
00000800:00000200:23.0:1619677445.720168:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff918febcf6000 0x46bcf6000(0x46bcf6000)
00000800:00000200:23.0:1619677445.720169:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff918febcf7000 0x46bcf7000(0x46bcf7000)
00000800:00000200:23.0:1619677445.720169:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff918febcf8000 0x46bcf8000(0x46bcf8000)
00000800:00000200:23.0:1619677445.720170:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff918febcf9000 0x46bcf9000(0x46bcf9000)
00000800:00000200:23.0:1619677445.720170:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff918febcfa000 0x46bcfa000(0x46bcfa000)
00000800:00000200:23.0:1619677445.720171:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff918febcfb000 0x46bcfb000(0x46bcfb000)
00000800:00000200:23.0:1619677445.720171:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff918febcfc000 0x46bcfc000(0x46bcfc000)
00000800:00000200:23.0:1619677445.720172:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff918febcfd000 0x46bcfd000(0x46bcfd000)
00000800:00000200:23.0:1619677445.720796:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (19)++
00000800:00000200:23.0:1619677445.720797:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (20)--
00000800:00000200:23.0:1619677445.720798:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (19)++
00000800:00000200:23.0:1619677445.720798:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (20)--
00000800:00000200:23.0:1619677445.720798:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (19)++
00000800:00000200:23.0:1619677445.720799:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (20)--
00000800:00000200:23.0:1619677445.720799:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (19)++
00000800:00000200:23.0:1619677445.720799:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (20)--
00000800:00000200:23.0:1619677445.720800:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (19)++
00000800:00000200:23.0:1619677445.720800:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (20)--
00000800:00000200:23.0:1619677445.720800:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (19)++
00000800:00000200:23.0:1619677445.720801:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (20)--
00000800:00000200:23.0:1619677445.720801:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (19)++
00000800:00000200:23.0:1619677445.720801:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (20)--
00000800:00000200:23.0:1619677445.720802:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (19)++
00000800:00000200:23.0:1619677445.720802:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (20)--
00000800:00000200:23.0:1619677445.720805:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (19)++
00000800:00000200:23.0:1619677445.720806:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (20)--
00000800:00000200:23.0:1619677445.720806:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (19)++
00000800:00000200:23.0:1619677445.720806:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (20)--
00000800:00000200:23.0:1619677445.720807:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (19)++
00000800:00000200:23.0:1619677445.720807:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (20)--
00000800:00000200:23.0:1619677445.720808:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (19)++
00000800:00000200:23.0:1619677445.720808:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (20)--
00000800:00000200:23.0:1619677445.720808:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (19)++
00000800:00000200:23.0:1619677445.720809:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (20)--
00000800:00000200:23.0:1619677445.720809:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (19)++
00000800:00000200:23.0:1619677445.720809:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (20)--
00000800:00000200:23.0:1619677445.720810:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (19)++
00000800:00000200:23.0:1619677445.720810:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (20)--
00000800:00000200:23.0:1619677445.720810:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (19)++
00000800:00000200:23.0:1619677445.720811:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (20)--
00000800:00000200:23.0:1619677445.720811:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (19)++
00000800:00000200:23.0:1619677445.720812:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (20)--
00000800:00000200:23.0:1619677445.720812:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (19)++
00000800:00000200:23.0:1619677445.720812:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (20)--
00000800:00000200:23.0:1619677445.720819:0:13382:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.103@o2ib7 Route resolved: 0
00000800:00000200:23.0:1619677445.720836:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff91a7c101e000 0x1c4101e000(0x1c4101e000)
00000800:00000200:23.0:1619677445.720837:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff91a7c101f000 0x1c4101f000(0x1c4101f000)
00000800:00000200:23.0:1619677445.720837:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff91a7c1020000 0x1c41020000(0x1c41020000)
00000800:00000200:23.0:1619677445.720838:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff91a7c1021000 0x1c41021000(0x1c41021000)
00000800:00000200:23.0:1619677445.720838:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff91a7c1022000 0x1c41022000(0x1c41022000)
00000800:00000200:23.0:1619677445.720839:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff91a7c1023000 0x1c41023000(0x1c41023000)
00000800:00000200:23.0:1619677445.720839:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff91a7c1024000 0x1c41024000(0x1c41024000)
00000800:00000200:23.0:1619677445.720840:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff91a7c1025000 0x1c41025000(0x1c41025000)
00000800:00000200:23.0:1619677445.720840:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff91a7c11f7000 0x1c411f7000(0x1c411f7000)
00000800:00000200:23.0:1619677445.720841:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff91a7c1200000 0x1c41200000(0x1c41200000)
00000800:00000200:23.0:1619677445.720841:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff91a7c1201000 0x1c41201000(0x1c41201000)
00000800:00000200:23.0:1619677445.720842:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff91a7c1202000 0x1c41202000(0x1c41202000)
00000800:00000200:23.0:1619677445.720842:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff91a7c1203000 0x1c41203000(0x1c41203000)
00000800:00000200:23.0:1619677445.720843:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff91a7c1204000 0x1c41204000(0x1c41204000)
00000800:00000200:23.0:1619677445.720844:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff91a7c1205000 0x1c41205000(0x1c41205000)
00000800:00000200:23.0:1619677445.720844:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff91a7c1206000 0x1c41206000(0x1c41206000)
00000800:00000200:23.0:1619677445.720845:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff91a7c1207000 0x1c41207000(0x1c41207000)
00000800:00000200:23.0:1619677445.720845:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff91a7c1208000 0x1c41208000(0x1c41208000)
00000800:00000200:13.0:1619677445.721256:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.721258:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.721259:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.721261:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.721263:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.721264:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.721273:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.721276:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.721277:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.721277:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.721278:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.721279:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.721327:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.721330:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.721330:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.721332:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.721334:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.721335:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:23.0:1619677445.721454:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (19)++
00000800:00000200:23.0:1619677445.721455:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (20)--
00000800:00000200:23.0:1619677445.721455:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (19)++
00000800:00000200:23.0:1619677445.721455:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (20)--
00000800:00000200:23.0:1619677445.721456:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (19)++
00000800:00000200:23.0:1619677445.721456:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (20)--
00000800:00000200:23.0:1619677445.721456:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (19)++
00000800:00000200:23.0:1619677445.721457:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (20)--
00000800:00000200:23.0:1619677445.721457:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (19)++
00000800:00000200:23.0:1619677445.721458:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (20)--
00000800:00000200:23.0:1619677445.721458:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (19)++
00000800:00000200:23.0:1619677445.721458:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (20)--
00000800:00000200:23.0:1619677445.721459:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (19)++
00000800:00000200:23.0:1619677445.721459:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (20)--
00000800:00000200:23.0:1619677445.721459:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (19)++
00000800:00000200:23.0:1619677445.721460:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (20)--
00000800:00000200:23.0:1619677445.721460:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (19)++
00000800:00000200:23.0:1619677445.721460:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (20)--
00000800:00000200:23.0:1619677445.721461:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (19)++
00000800:00000200:23.0:1619677445.721461:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (20)--
00000800:00000200:23.0:1619677445.721461:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (19)++
00000800:00000200:23.0:1619677445.721462:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (20)--
00000800:00000200:23.0:1619677445.721462:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (19)++
00000800:00000200:23.0:1619677445.721463:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (20)--
00000800:00000200:23.0:1619677445.721463:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (19)++
00000800:00000200:23.0:1619677445.721463:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (20)--
00000800:00000200:23.0:1619677445.721464:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (19)++
00000800:00000200:23.0:1619677445.721464:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (20)--
00000800:00000200:23.0:1619677445.721464:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (19)++
00000800:00000200:23.0:1619677445.721465:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (20)--
00000800:00000200:23.0:1619677445.721465:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (19)++
00000800:00000200:23.0:1619677445.721465:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (20)--
00000800:00000200:23.0:1619677445.721466:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (19)++
00000800:00000200:23.0:1619677445.721467:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (20)--
00000800:00000200:23.0:1619677445.721467:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (19)++
00000800:00000200:23.0:1619677445.721467:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (20)--
00000800:00000200:13.0:1619677445.721496:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.721498:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.721499:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.721500:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677445.721503:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.721504:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.721617:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.721620:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.721621:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.721623:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.721624:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.721626:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.721661:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.721664:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.721665:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.721667:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.721668:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.721670:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.721678:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.721681:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.721681:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.721682:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.721683:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.721684:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.721998:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.722000:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.722001:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.722003:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.722004:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.722006:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.722170:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.722173:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.722174:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.722176:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.722177:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.722179:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.722278:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.722280:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.722281:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.722283:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.722285:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.722287:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.722327:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.722330:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.722330:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.722332:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.722334:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.722335:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.722345:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.722348:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.722349:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.722350:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.722350:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.722351:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:23.0:1619677445.722478:0:13382:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.102@o2ib7 Addr resolved: 0
00000800:00000200:23.0:1619677445.722485:0:13382:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.102@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:13.0:1619677445.722629:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.722631:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.722632:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.722634:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.722636:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.722637:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:23.0:1619677445.722689:0:13382:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.102@o2ib7 Route resolved: 0
00000800:00000200:23.0:1619677445.722707:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff9193c2977000 0x842977000(0x842977000)
00000800:00000200:23.0:1619677445.722708:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff9193c2978000 0x842978000(0x842978000)
00000800:00000200:23.0:1619677445.722708:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff9193c2979000 0x842979000(0x842979000)
00000800:00000200:23.0:1619677445.722709:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff9193c297a000 0x84297a000(0x84297a000)
00000800:00000200:23.0:1619677445.722709:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff9193c297b000 0x84297b000(0x84297b000)
00000800:00000200:23.0:1619677445.722710:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff9193c297c000 0x84297c000(0x84297c000)
00000800:00000200:23.0:1619677445.722710:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff9193c297d000 0x84297d000(0x84297d000)
00000800:00000200:23.0:1619677445.722711:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff9193c26de000 0x8426de000(0x8426de000)
00000800:00000200:23.0:1619677445.722711:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff9193c26df000 0x8426df000(0x8426df000)
00000800:00000200:23.0:1619677445.722712:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff9193c26e0000 0x8426e0000(0x8426e0000)
00000800:00000200:23.0:1619677445.722712:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff9193c26e1000 0x8426e1000(0x8426e1000)
00000800:00000200:23.0:1619677445.722713:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff9193c26e2000 0x8426e2000(0x8426e2000)
00000800:00000200:23.0:1619677445.722714:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff9193c26e3000 0x8426e3000(0x8426e3000)
00000800:00000200:23.0:1619677445.722714:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff9193c26e4000 0x8426e4000(0x8426e4000)
00000800:00000200:23.0:1619677445.722715:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff9193c26e5000 0x8426e5000(0x8426e5000)
00000800:00000200:23.0:1619677445.722715:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff9193c26e6000 0x8426e6000(0x8426e6000)
00000800:00000200:23.0:1619677445.722716:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff9193c26e7000 0x8426e7000(0x8426e7000)
00000800:00000200:23.0:1619677445.722716:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff9193c26e8000 0x8426e8000(0x8426e8000)
00000800:00000200:12.0:1619677445.722753:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)++
00000800:00000200:12.0:1619677445.722756:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (25)--
00000400:00000200:12.0:1619677445.722757:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.722758:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.722760:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)++
00000800:00000200:13.0:1619677445.722761:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (25)--
00000800:00000200:12.0:1619677445.722761:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (25)--
00000400:00000200:13.0:1619677445.722762:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677445.722762:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000400:00000200:13.0:1619677445.722763:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.722764:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677445.722765:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.722765:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.722766:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.722767:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.722769:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.722937:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.722940:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.722941:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.722943:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.722943:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.722945:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.723108:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.723111:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.723112:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.723114:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.723115:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.723116:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:23.0:1619677445.723353:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (19)++
00000800:00000200:23.0:1619677445.723353:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (20)--
00000800:00000200:23.0:1619677445.723353:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (19)++
00000800:00000200:23.0:1619677445.723354:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (20)--
00000800:00000200:23.0:1619677445.723354:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (19)++
00000800:00000200:23.0:1619677445.723355:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (20)--
00000800:00000200:23.0:1619677445.723355:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (19)++
00000800:00000200:23.0:1619677445.723355:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (20)--
00000800:00000200:23.0:1619677445.723356:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (19)++
00000800:00000200:23.0:1619677445.723356:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (20)--
00000800:00000200:23.0:1619677445.723357:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (19)++
00000800:00000200:23.0:1619677445.723357:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (20)--
00000800:00000200:23.0:1619677445.723358:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (19)++
00000800:00000200:23.0:1619677445.723358:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (20)--
00000800:00000200:23.0:1619677445.723358:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (19)++
00000800:00000200:23.0:1619677445.723359:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (20)--
00000800:00000200:23.0:1619677445.723359:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (19)++
00000800:00000200:23.0:1619677445.723359:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (20)--
00000800:00000200:23.0:1619677445.723360:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (19)++
00000800:00000200:23.0:1619677445.723360:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (20)--
00000800:00000200:23.0:1619677445.723360:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (19)++
00000800:00000200:23.0:1619677445.723361:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (20)--
00000800:00000200:23.0:1619677445.723361:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (19)++
00000800:00000200:23.0:1619677445.723362:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (20)--
00000800:00000200:23.0:1619677445.723362:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (19)++
00000800:00000200:23.0:1619677445.723362:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (20)--
00000800:00000200:23.0:1619677445.723363:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (19)++
00000800:00000200:23.0:1619677445.723363:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (20)--
00000800:00000200:23.0:1619677445.723363:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (19)++
00000800:00000200:23.0:1619677445.723364:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (20)--
00000800:00000200:23.0:1619677445.723364:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (19)++
00000800:00000200:23.0:1619677445.723364:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (20)--
00000800:00000200:23.0:1619677445.723365:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (19)++
00000800:00000200:23.0:1619677445.723365:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (20)--
00000800:00000200:23.0:1619677445.723365:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (19)++
00000800:00000200:23.0:1619677445.723366:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (20)--
00000800:00000200:13.0:1619677445.723621:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.723623:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.723624:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.723626:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.723628:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.723629:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.723638:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.723641:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.723642:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.723642:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.723643:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.723644:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:24.2:1619677445.723722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (21)++
00000800:00000200:24.2:1619677445.724324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (20)++
00000800:00000200:12.0:1619677445.724427:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.724430:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.724431:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.724432:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.724435:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.724436:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.724670:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.724673:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.724674:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.724676:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.724677:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.724678:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.725497:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.725500:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.725501:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.725502:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.725505:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.725513:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:7.0:1619677445.725703:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff91abeb718600] (18)--
00000800:00000200:12.0:1619677445.726138:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.726140:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.726141:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.726143:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.726145:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.726145:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.726159:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.726162:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.726163:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.726163:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.726164:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.726165:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.726192:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.726195:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.726195:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.726197:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.726199:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.726200:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.727348:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.727351:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.727352:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.727354:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.727355:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.727357:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:23.0:1619677445.728991:0:13382:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.108@o2ib7 Addr resolved: 0
00000800:00000200:23.0:1619677445.728997:0:13382:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.108@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:23.0:1619677445.729303:0:13382:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.108@o2ib7 Route resolved: 0
00000800:00000200:23.0:1619677445.729317:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff91abeb775000 0x206b775000(0x206b775000)
00000800:00000200:23.0:1619677445.729318:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff91abeb776000 0x206b776000(0x206b776000)
00000800:00000200:23.0:1619677445.729318:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff91abeb777000 0x206b777000(0x206b777000)
00000800:00000200:23.0:1619677445.729319:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff91abeb778000 0x206b778000(0x206b778000)
00000800:00000200:23.0:1619677445.729320:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff91abeb779000 0x206b779000(0x206b779000)
00000800:00000200:23.0:1619677445.729320:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff91abeb77a000 0x206b77a000(0x206b77a000)
00000800:00000200:23.0:1619677445.729321:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff91abeb77b000 0x206b77b000(0x206b77b000)
00000800:00000200:23.0:1619677445.729322:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff91abeb77c000 0x206b77c000(0x206b77c000)
00000800:00000200:23.0:1619677445.729322:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff91abeb77d000 0x206b77d000(0x206b77d000)
00000800:00000200:23.0:1619677445.729323:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff91abeb77e000 0x206b77e000(0x206b77e000)
00000800:00000200:23.0:1619677445.729323:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff91abeb77f000 0x206b77f000(0x206b77f000)
00000800:00000200:23.0:1619677445.729324:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff91abeb780000 0x206b780000(0x206b780000)
00000800:00000200:23.0:1619677445.729324:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff91abeb781000 0x206b781000(0x206b781000)
00000800:00000200:23.0:1619677445.729325:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff91abeb782000 0x206b782000(0x206b782000)
00000800:00000200:23.0:1619677445.729325:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff91abd9199000 0x2059199000(0x2059199000)
00000800:00000200:23.0:1619677445.729326:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff91abd919a000 0x205919a000(0x205919a000)
00000800:00000200:23.0:1619677445.729326:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff91abd919b000 0x205919b000(0x205919b000)
00000800:00000200:23.0:1619677445.729327:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff91abd919c000 0x205919c000(0x205919c000)
00000800:00000200:12.0:1619677445.729462:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.729465:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.729466:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.729468:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.729470:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.729470:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:23.0:1619677445.729986:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (19)++
00000800:00000200:23.0:1619677445.729987:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (20)--
00000800:00000200:23.0:1619677445.729987:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (19)++
00000800:00000200:23.0:1619677445.729988:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (20)--
00000800:00000200:23.0:1619677445.729988:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (19)++
00000800:00000200:23.0:1619677445.729988:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (20)--
00000800:00000200:23.0:1619677445.729989:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (19)++
00000800:00000200:23.0:1619677445.729989:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (20)--
00000800:00000200:23.0:1619677445.729989:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (19)++
00000800:00000200:23.0:1619677445.729990:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (20)--
00000800:00000200:23.0:1619677445.729990:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (19)++
00000800:00000200:23.0:1619677445.729990:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (20)--
00000800:00000200:23.0:1619677445.729991:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (19)++
00000800:00000200:23.0:1619677445.729991:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (20)--
00000800:00000200:23.0:1619677445.729991:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (19)++
00000800:00000200:23.0:1619677445.729992:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (20)--
00000800:00000200:23.0:1619677445.729992:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (19)++
00000800:00000200:23.0:1619677445.729993:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (20)--
00000800:00000200:23.0:1619677445.729993:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (19)++
00000800:00000200:23.0:1619677445.729993:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (20)--
00000800:00000200:23.0:1619677445.729994:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (19)++
00000800:00000200:23.0:1619677445.729994:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (20)--
00000800:00000200:23.0:1619677445.729995:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (19)++
00000800:00000200:23.0:1619677445.729995:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (20)--
00000800:00000200:23.0:1619677445.729996:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (19)++
00000800:00000200:23.0:1619677445.729996:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (20)--
00000800:00000200:23.0:1619677445.729997:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (19)++
00000800:00000200:23.0:1619677445.729997:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (20)--
00000800:00000200:23.0:1619677445.729997:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (19)++
00000800:00000200:23.0:1619677445.729998:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (20)--
00000800:00000200:23.0:1619677445.729998:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (19)++
00000800:00000200:23.0:1619677445.729999:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (20)--
00000800:00000200:23.0:1619677445.729999:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (19)++
00000800:00000200:23.0:1619677445.729999:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (20)--
00000800:00000200:23.0:1619677445.730000:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (19)++
00000800:00000200:23.0:1619677445.730000:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (20)--
00000800:00000200:13.0:1619677445.730126:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.730128:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.730129:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.730130:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.730132:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.730134:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.731128:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.731130:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.731131:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.731132:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.731135:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.731136:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.731151:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.731154:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.731154:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.731155:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.731156:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.731157:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.732372:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.732374:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.732375:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.732378:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.732379:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.732384:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.732401:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.732403:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.732404:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.732406:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.732407:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.732409:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.732576:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.732579:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.732580:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.732581:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.732583:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.732584:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.732654:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.732656:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.732657:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.732658:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.732660:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.732661:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.732673:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:13.0:1619677445.732675:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.732676:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.732677:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:13.0:1619677445.732677:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.732679:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.735577:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.735579:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.735580:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.735582:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.735584:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.735585:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.735890:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.735892:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.735893:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.735895:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.735897:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.735898:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.735994:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.735996:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.735998:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.736000:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.736001:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.736002:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.736031:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.736034:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.736034:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.736036:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.736037:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.736038:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.736046:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.736049:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.736050:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.736051:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.736051:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.736053:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.736618:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.736620:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.736621:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.736622:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.736624:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.736625:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.737038:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:13.0:1619677445.737040:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (24)--
00000400:00000200:13.0:1619677445.737041:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.737043:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.737044:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:13.0:1619677445.737045:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (24)--
00000800:00000200:12.0:1619677445.737045:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (24)--
00000800:00000200:13.0:1619677445.737046:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000400:00000200:12.0:1619677445.737046:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.737047:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.737050:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.737161:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.737164:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.737165:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.737167:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.737168:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.737170:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.737182:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.737184:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.737185:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.737185:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.737186:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.737187:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.737735:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:12.0:1619677445.737738:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (24)--
00000400:00000200:12.0:1619677445.737739:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.737741:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.737742:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)++
00000800:00000200:13.0:1619677445.737744:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (24)--
00000400:00000200:13.0:1619677445.737744:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677445.737744:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.737745:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000400:00000200:13.0:1619677445.737746:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.737748:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.737818:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.737820:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.737821:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.737823:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.737824:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.737826:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.738119:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.738121:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.738122:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.738124:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.738125:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.738127:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.738141:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.738144:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.738145:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.738145:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.738146:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.738148:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.738849:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.738851:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.738852:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.738853:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.738855:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.738856:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:23.0:1619677445.739179:0:13382:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.115@o2ib7 Addr resolved: 0
00000800:00000200:23.0:1619677445.739186:0:13382:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.115@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:7.0:1619677445.739342:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff9197927b6e00] (18)--
00000800:00000200:23.0:1619677445.739419:0:13382:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.115@o2ib7 Route resolved: 0
00000800:00000200:23.0:1619677445.739447:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff919784291000 0xc04291000(0xc04291000)
00000800:00000200:23.0:1619677445.739448:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff919784292000 0xc04292000(0xc04292000)
00000800:00000200:23.0:1619677445.739450:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff919784293000 0xc04293000(0xc04293000)
00000800:00000200:23.0:1619677445.739451:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff919784294000 0xc04294000(0xc04294000)
00000800:00000200:23.0:1619677445.739452:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff919784295000 0xc04295000(0xc04295000)
00000800:00000200:23.0:1619677445.739454:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff919784296000 0xc04296000(0xc04296000)
00000800:00000200:23.0:1619677445.739455:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff919784297000 0xc04297000(0xc04297000)
00000800:00000200:23.0:1619677445.739456:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff919784298000 0xc04298000(0xc04298000)
00000800:00000200:23.0:1619677445.739458:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff919784299000 0xc04299000(0xc04299000)
00000800:00000200:23.0:1619677445.739459:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff91978429a000 0xc0429a000(0xc0429a000)
00000800:00000200:23.0:1619677445.739460:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff91978390f000 0xc0390f000(0xc0390f000)
00000800:00000200:23.0:1619677445.739461:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff919783912000 0xc03912000(0xc03912000)
00000800:00000200:23.0:1619677445.739462:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff919783913000 0xc03913000(0xc03913000)
00000800:00000200:23.0:1619677445.739463:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff919783914000 0xc03914000(0xc03914000)
00000800:00000200:23.0:1619677445.739464:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff919783915000 0xc03915000(0xc03915000)
00000800:00000200:23.0:1619677445.739465:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff919783916000 0xc03916000(0xc03916000)
00000800:00000200:23.0:1619677445.739466:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff919783917000 0xc03917000(0xc03917000)
00000800:00000200:23.0:1619677445.739467:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff919783918000 0xc03918000(0xc03918000)
00000800:00000200:13.0:1619677445.739771:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.739774:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.739775:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.739776:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.739779:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.739780:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:7.0:1619677445.739907:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff9193c2987e00] (18)--
00000800:00000200:23.0:1619677445.740231:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (19)++
00000800:00000200:23.0:1619677445.740232:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (20)--
00000800:00000200:23.0:1619677445.740233:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (19)++
00000800:00000200:23.0:1619677445.740233:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (20)--
00000800:00000200:23.0:1619677445.740233:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (19)++
00000800:00000200:23.0:1619677445.740234:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (20)--
00000800:00000200:23.0:1619677445.740234:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (19)++
00000800:00000200:23.0:1619677445.740235:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (20)--
00000800:00000200:23.0:1619677445.740235:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (19)++
00000800:00000200:23.0:1619677445.740235:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (20)--
00000800:00000200:23.0:1619677445.740236:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (19)++
00000800:00000200:23.0:1619677445.740236:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (20)--
00000800:00000200:23.0:1619677445.740236:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (19)++
00000800:00000200:23.0:1619677445.740237:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (20)--
00000800:00000200:23.0:1619677445.740237:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (19)++
00000800:00000200:23.0:1619677445.740237:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (20)--
00000800:00000200:23.0:1619677445.740238:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (19)++
00000800:00000200:23.0:1619677445.740238:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (20)--
00000800:00000200:23.0:1619677445.740238:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (19)++
00000800:00000200:23.0:1619677445.740239:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (20)--
00000800:00000200:23.0:1619677445.740239:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (19)++
00000800:00000200:23.0:1619677445.740240:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (20)--
00000800:00000200:23.0:1619677445.740240:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (19)++
00000800:00000200:23.0:1619677445.740240:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (20)--
00000800:00000200:23.0:1619677445.740241:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (19)++
00000800:00000200:23.0:1619677445.740241:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (20)--
00000800:00000200:23.0:1619677445.740241:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (19)++
00000800:00000200:23.0:1619677445.740242:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (20)--
00000800:00000200:23.0:1619677445.740242:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (19)++
00000800:00000200:23.0:1619677445.740242:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (20)--
00000800:00000200:23.0:1619677445.740243:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (19)++
00000800:00000200:23.0:1619677445.740243:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (20)--
00000800:00000200:23.0:1619677445.740244:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (19)++
00000800:00000200:23.0:1619677445.740244:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (20)--
00000800:00000200:23.0:1619677445.740244:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (19)++
00000800:00000200:23.0:1619677445.740245:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (20)--
00000800:00000200:23.0:1619677445.740253:0:13382:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.110@o2ib7 Route resolved: 0
00000800:00000200:23.0:1619677445.740266:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff9193c26e9000 0x8426e9000(0x8426e9000)
00000800:00000200:23.0:1619677445.740266:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff9193c26ea000 0x8426ea000(0x8426ea000)
00000800:00000200:23.0:1619677445.740267:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff9193c26eb000 0x8426eb000(0x8426eb000)
00000800:00000200:23.0:1619677445.740268:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff9193c26ec000 0x8426ec000(0x8426ec000)
00000800:00000200:23.0:1619677445.740268:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff9193c26ed000 0x8426ed000(0x8426ed000)
00000800:00000200:23.0:1619677445.740269:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff9193c26ee000 0x8426ee000(0x8426ee000)
00000800:00000200:23.0:1619677445.740269:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff9193c26ef000 0x8426ef000(0x8426ef000)
00000800:00000200:23.0:1619677445.740270:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff9193c26f0000 0x8426f0000(0x8426f0000)
00000800:00000200:23.0:1619677445.740270:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff9193c26f1000 0x8426f1000(0x8426f1000)
00000800:00000200:23.0:1619677445.740271:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff9193c26f2000 0x8426f2000(0x8426f2000)
00000800:00000200:23.0:1619677445.740271:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff9193c26f3000 0x8426f3000(0x8426f3000)
00000800:00000200:23.0:1619677445.740272:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff9193c26f4000 0x8426f4000(0x8426f4000)
00000800:00000200:23.0:1619677445.740272:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff9193c26f5000 0x8426f5000(0x8426f5000)
00000800:00000200:13.0:1619677445.740272:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:23.0:1619677445.740273:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff9193c26f6000 0x8426f6000(0x8426f6000)
00000800:00000200:23.0:1619677445.740273:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff9193c26f7000 0x8426f7000(0x8426f7000)
00000800:00000200:23.0:1619677445.740274:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff9193c26f8000 0x8426f8000(0x8426f8000)
00000800:00000200:23.0:1619677445.740274:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff9193c26f9000 0x8426f9000(0x8426f9000)
00000800:00000200:23.0:1619677445.740275:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff9193c26fa000 0x8426fa000(0x8426fa000)
00000800:00000200:13.0:1619677445.740275:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.740276:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.740278:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.740278:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.740279:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.740343:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.740345:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.0.10.103@o2ib7
00000400:00000200:13.0:1619677445.740348:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.103@o2ib7 : PUT - routed
00000800:00000200:12.0:1619677445.740350:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.740351:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.740431:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.740434:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.740435:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.740436:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.740438:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.740439:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.740572:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.740575:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.740576:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.740577:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.740578:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.740580:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.740590:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.740593:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.740594:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.740595:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.740595:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.740597:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.740650:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.740652:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.740653:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.740654:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.740654:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.740655:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.740655:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.740656:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.740657:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:12.0:1619677445.740657:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.740658:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.740659:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.740865:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.740867:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.740868:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.740870:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.740870:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.740872:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:23.0:1619677445.740896:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (19)++
00000800:00000200:23.0:1619677445.740897:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (20)--
00000800:00000200:23.0:1619677445.740897:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (19)++
00000800:00000200:23.0:1619677445.740897:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (20)--
00000800:00000200:23.0:1619677445.740898:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (19)++
00000800:00000200:23.0:1619677445.740898:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (20)--
00000800:00000200:23.0:1619677445.740899:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (19)++
00000800:00000200:23.0:1619677445.740899:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (20)--
00000800:00000200:23.0:1619677445.740899:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (19)++
00000800:00000200:23.0:1619677445.740900:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (20)--
00000800:00000200:23.0:1619677445.740900:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (19)++
00000800:00000200:23.0:1619677445.740900:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (20)--
00000800:00000200:23.0:1619677445.740901:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (19)++
00000800:00000200:23.0:1619677445.740901:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (20)--
00000800:00000200:23.0:1619677445.740901:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (19)++
00000800:00000200:23.0:1619677445.740902:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (20)--
00000800:00000200:23.0:1619677445.740902:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (19)++
00000800:00000200:23.0:1619677445.740903:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (20)--
00000800:00000200:23.0:1619677445.740903:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (19)++
00000800:00000200:23.0:1619677445.740904:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (20)--
00000800:00000200:23.0:1619677445.740904:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (19)++
00000800:00000200:23.0:1619677445.740905:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (20)--
00000800:00000200:23.0:1619677445.740905:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (19)++
00000800:00000200:23.0:1619677445.740905:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (20)--
00000800:00000200:23.0:1619677445.740906:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (19)++
00000800:00000200:23.0:1619677445.740906:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (20)--
00000800:00000200:23.0:1619677445.740906:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (19)++
00000800:00000200:23.0:1619677445.740907:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (20)--
00000800:00000200:23.0:1619677445.740907:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (19)++
00000800:00000200:23.0:1619677445.740907:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (20)--
00000800:00000200:23.0:1619677445.740908:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (19)++
00000800:00000200:23.0:1619677445.740908:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (20)--
00000800:00000200:23.0:1619677445.740908:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (19)++
00000800:00000200:23.0:1619677445.740909:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (20)--
00000800:00000200:23.0:1619677445.740909:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (19)++
00000800:00000200:23.0:1619677445.740910:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (20)--
00000800:00000200:12.0:1619677445.741345:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.741347:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.741348:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.741350:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.741352:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.741352:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.741386:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.741389:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.741389:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.741392:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.741393:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.741394:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.741554:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.741557:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.741558:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:7.0:1619677445.741558:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff91978412be00] (18)--
00000400:00000200:13.0:1619677445.741560:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.741561:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.741562:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.741824:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.741826:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:12.0:1619677445.741827:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.741828:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.741830:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.741831:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.741842:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)++
00000800:00000200:12.0:1619677445.741845:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.741846:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:12.0:1619677445.741846:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8c00] (23)--
00000800:00000200:12.0:1619677445.741847:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:12.0:1619677445.741848:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:13.0:1619677445.741891:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)++
00000800:00000200:13.0:1619677445.741894:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8c00] (23)--
00000400:00000200:13.0:1619677445.741895:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.741896:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.741897:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (22)--
00000800:00000200:13.0:1619677445.741899:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:7.0:1619677445.742613:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff91a3aea37e00] (18)--
00000800:00000200:12.0:1619677445.743097:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (20)++
00000800:00000200:12.0:1619677445.743099:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17d8c00] (21)--
00000800:00000200:12.0:1619677445.743100:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (20)--
00000800:00000200:12.0:1619677445.743101:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (19)--
00000800:00000200:7.0:1619677445.743142:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff91a7c17d8c00] (18)--
00000800:00000200:23.0:1619677445.743313:0:13382:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.107@o2ib7 Addr resolved: 0
00000800:00000200:23.0:1619677445.743320:0:13382:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.107@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:12.0:1619677445.744887:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.744890:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.744891:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.744892:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.744895:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.744896:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.744905:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.744908:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.744909:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.744909:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.744910:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.744911:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.745170:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.745173:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.745174:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.745176:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677445.745176:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.745179:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.745279:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677445.745282:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677445.745283:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.745285:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.745285:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677445.745287:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.745287:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677445.745287:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.745289:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677445.745289:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.745291:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.745738:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.745740:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.745741:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.745743:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.745745:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.745746:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.745758:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.745761:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.745762:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.745762:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.745763:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.745764:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.747010:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.747013:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.747014:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.747017:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677445.747017:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.747021:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.747168:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.747171:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.747172:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.747174:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.747175:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.747177:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.747761:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.747764:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.747765:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.747766:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.747768:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.747769:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:7.0:1619677445.748756:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff919e46980000] (18)--
00000800:00000200:13.0:1619677445.749416:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.749419:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.749420:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.749422:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.749423:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.749425:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.749435:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.749438:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.749439:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.749440:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.749441:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.749442:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:7.0:1619677445.749783:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff91a3aea38000] (18)--
00000800:00000200:7.0:1619677445.750730:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff918febd24600] (18)--
00000800:00000200:12.0:1619677445.751145:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.751148:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.751148:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.751150:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.751152:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.751153:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:23.0:1619677445.751186:0:13382:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.109@o2ib7 Route resolved: 0
00000800:00000200:23.0:1619677445.751204:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff919e45acf000 0x12c5acf000(0x12c5acf000)
00000800:00000200:23.0:1619677445.751205:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff919e45ad0000 0x12c5ad0000(0x12c5ad0000)
00000800:00000200:23.0:1619677445.751205:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff919e45ad1000 0x12c5ad1000(0x12c5ad1000)
00000800:00000200:23.0:1619677445.751206:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff919e45ad2000 0x12c5ad2000(0x12c5ad2000)
00000800:00000200:23.0:1619677445.751206:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff919e45ad3000 0x12c5ad3000(0x12c5ad3000)
00000800:00000200:23.0:1619677445.751207:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff919e45ad4000 0x12c5ad4000(0x12c5ad4000)
00000800:00000200:23.0:1619677445.751208:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff919e45ad5000 0x12c5ad5000(0x12c5ad5000)
00000800:00000200:23.0:1619677445.751208:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff919e45ad6000 0x12c5ad6000(0x12c5ad6000)
00000800:00000200:23.0:1619677445.751209:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff919e45ad7000 0x12c5ad7000(0x12c5ad7000)
00000800:00000200:23.0:1619677445.751209:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff919e45ad8000 0x12c5ad8000(0x12c5ad8000)
00000800:00000200:23.0:1619677445.751210:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff919e45ad9000 0x12c5ad9000(0x12c5ad9000)
00000800:00000200:23.0:1619677445.751210:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff919e45ada000 0x12c5ada000(0x12c5ada000)
00000800:00000200:23.0:1619677445.751211:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff919e45adb000 0x12c5adb000(0x12c5adb000)
00000800:00000200:23.0:1619677445.751211:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff919e45663000 0x12c5663000(0x12c5663000)
00000800:00000200:23.0:1619677445.751212:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff919e45664000 0x12c5664000(0x12c5664000)
00000800:00000200:23.0:1619677445.751213:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff919e45665000 0x12c5665000(0x12c5665000)
00000800:00000200:23.0:1619677445.751213:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff919e45666000 0x12c5666000(0x12c5666000)
00000800:00000200:23.0:1619677445.751214:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff919e45667000 0x12c5667000(0x12c5667000)
00000800:00000200:13.0:1619677445.751496:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)++
00000800:00000200:13.0:1619677445.751499:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677445.751499:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.751501:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.751503:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677445.751504:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677445.751505:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.751505:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.751505:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.751507:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.751509:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.751721:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.751723:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.751724:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.751726:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.751728:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.751729:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.751742:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.751745:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.751746:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.751747:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.751747:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.751749:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:23.0:1619677445.751869:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (19)++
00000800:00000200:23.0:1619677445.751870:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (20)--
00000800:00000200:23.0:1619677445.751870:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (19)++
00000800:00000200:23.0:1619677445.751870:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (20)--
00000800:00000200:23.0:1619677445.751871:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (19)++
00000800:00000200:23.0:1619677445.751871:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (20)--
00000800:00000200:23.0:1619677445.751871:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (19)++
00000800:00000200:23.0:1619677445.751872:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (20)--
00000800:00000200:23.0:1619677445.751872:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (19)++
00000800:00000200:23.0:1619677445.751872:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (20)--
00000800:00000200:23.0:1619677445.751873:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (19)++
00000800:00000200:23.0:1619677445.751873:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (20)--
00000800:00000200:23.0:1619677445.751874:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (19)++
00000800:00000200:23.0:1619677445.751874:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (20)--
00000800:00000200:23.0:1619677445.751874:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (19)++
00000800:00000200:23.0:1619677445.751875:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (20)--
00000800:00000200:23.0:1619677445.751875:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (19)++
00000800:00000200:23.0:1619677445.751876:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (20)--
00000800:00000200:23.0:1619677445.751876:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (19)++
00000800:00000200:23.0:1619677445.751876:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (20)--
00000800:00000200:23.0:1619677445.751877:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (19)++
00000800:00000200:23.0:1619677445.751877:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (20)--
00000800:00000200:23.0:1619677445.751877:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (19)++
00000800:00000200:23.0:1619677445.751878:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (20)--
00000800:00000200:23.0:1619677445.751878:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (19)++
00000800:00000200:23.0:1619677445.751878:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (20)--
00000800:00000200:23.0:1619677445.751879:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (19)++
00000800:00000200:23.0:1619677445.751879:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (20)--
00000800:00000200:23.0:1619677445.751879:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (19)++
00000800:00000200:23.0:1619677445.751880:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (20)--
00000800:00000200:23.0:1619677445.751880:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (19)++
00000800:00000200:23.0:1619677445.751881:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (20)--
00000800:00000200:23.0:1619677445.751881:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (19)++
00000800:00000200:23.0:1619677445.751882:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (20)--
00000800:00000200:23.0:1619677445.751882:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (19)++
00000800:00000200:23.0:1619677445.751883:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (20)--
00000800:00000200:7.0:1619677445.752023:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff919b884e3200] (18)--
00000800:00000200:23.0:1619677445.752854:0:13382:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.113@o2ib7 Addr resolved: 0
00000800:00000200:23.0:1619677445.752859:0:13382:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.113@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:23.0:1619677445.753091:0:13382:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.113@o2ib7 Route resolved: 0
00000800:00000200:23.0:1619677445.753107:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff918febcfe000 0x46bcfe000(0x46bcfe000)
00000800:00000200:23.0:1619677445.753107:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff918febcff000 0x46bcff000(0x46bcff000)
00000800:00000200:23.0:1619677445.753108:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff918febd00000 0x46bd00000(0x46bd00000)
00000800:00000200:23.0:1619677445.753108:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff918febd01000 0x46bd01000(0x46bd01000)
00000800:00000200:23.0:1619677445.753109:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff918febd02000 0x46bd02000(0x46bd02000)
00000800:00000200:23.0:1619677445.753110:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff918febd03000 0x46bd03000(0x46bd03000)
00000800:00000200:23.0:1619677445.753110:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff918fe95e2000 0x4695e2000(0x4695e2000)
00000800:00000200:23.0:1619677445.753111:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff918fe95e3000 0x4695e3000(0x4695e3000)
00000800:00000200:23.0:1619677445.753111:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff918fe95e4000 0x4695e4000(0x4695e4000)
00000800:00000200:23.0:1619677445.753112:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff918fe95e5000 0x4695e5000(0x4695e5000)
00000800:00000200:23.0:1619677445.753112:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff918fe95e6000 0x4695e6000(0x4695e6000)
00000800:00000200:23.0:1619677445.753113:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff918fe95e7000 0x4695e7000(0x4695e7000)
00000800:00000200:23.0:1619677445.753113:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff918fe95e8000 0x4695e8000(0x4695e8000)
00000800:00000200:23.0:1619677445.753114:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff918fe95e9000 0x4695e9000(0x4695e9000)
00000800:00000200:23.0:1619677445.753114:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff918fe95ea000 0x4695ea000(0x4695ea000)
00000800:00000200:23.0:1619677445.753115:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff918fe95eb000 0x4695eb000(0x4695eb000)
00000800:00000200:23.0:1619677445.753115:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff918fe95ec000 0x4695ec000(0x4695ec000)
00000800:00000200:23.0:1619677445.753116:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff918fe95ed000 0x4695ed000(0x4695ed000)
00000800:00000200:13.0:1619677445.753370:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.753373:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.753374:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.753375:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.753377:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.753379:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:23.0:1619677445.753750:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (19)++
00000800:00000200:23.0:1619677445.753751:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (20)--
00000800:00000200:23.0:1619677445.753751:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (19)++
00000800:00000200:23.0:1619677445.753752:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (20)--
00000800:00000200:23.0:1619677445.753752:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (19)++
00000800:00000200:23.0:1619677445.753752:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (20)--
00000800:00000200:23.0:1619677445.753753:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (19)++
00000800:00000200:23.0:1619677445.753753:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (20)--
00000800:00000200:23.0:1619677445.753754:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (19)++
00000800:00000200:23.0:1619677445.753754:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (20)--
00000800:00000200:23.0:1619677445.753755:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (19)++
00000800:00000200:23.0:1619677445.753755:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (20)--
00000800:00000200:23.0:1619677445.753756:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (19)++
00000800:00000200:23.0:1619677445.753756:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (20)--
00000800:00000200:23.0:1619677445.753756:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (19)++
00000800:00000200:23.0:1619677445.753757:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (20)--
00000800:00000200:23.0:1619677445.753757:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (19)++
00000800:00000200:23.0:1619677445.753757:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (20)--
00000800:00000200:23.0:1619677445.753758:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (19)++
00000800:00000200:23.0:1619677445.753758:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (20)--
00000800:00000200:23.0:1619677445.753758:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (19)++
00000800:00000200:23.0:1619677445.753759:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (20)--
00000800:00000200:23.0:1619677445.753759:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (19)++
00000800:00000200:23.0:1619677445.753760:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (20)--
00000800:00000200:23.0:1619677445.753760:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (19)++
00000800:00000200:23.0:1619677445.753760:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (20)--
00000800:00000200:23.0:1619677445.753761:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (19)++
00000800:00000200:23.0:1619677445.753761:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (20)--
00000800:00000200:23.0:1619677445.753761:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (19)++
00000800:00000200:23.0:1619677445.753762:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (20)--
00000800:00000200:23.0:1619677445.753762:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (19)++
00000800:00000200:23.0:1619677445.753762:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (20)--
00000800:00000200:23.0:1619677445.753763:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (19)++
00000800:00000200:23.0:1619677445.753763:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (20)--
00000800:00000200:23.0:1619677445.753764:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (19)++
00000800:00000200:23.0:1619677445.753764:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (20)--
00000800:00000200:12.0:1619677445.753926:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.753928:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.753929:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.753931:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.753933:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.753934:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.754408:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.754410:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.754411:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.754413:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.754415:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.754416:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.755173:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677445.755176:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677445.755177:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.755179:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.755180:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677445.755181:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677445.755182:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677445.755182:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.755183:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677445.755184:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.755186:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.755193:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.755196:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.755197:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.755198:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.755198:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.755199:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.755429:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.755431:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.755431:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.755433:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.755435:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.755436:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.755492:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677445.755496:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677445.755498:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677445.755499:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677445.755500:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.755502:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:12.0:1619677445.755503:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:505, d:12, s:-96751899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677445.755504:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677445.755505:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677445.755507:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:12.0:1619677445.755510:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677445.755512:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677445.755516:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677445.755517:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:12.0:1619677445.755518:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:12.0:1619677445.755520:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:12.0:1619677445.755521:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677445.755522:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677445.755523:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:13.0:1619677445.756028:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.756030:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.756031:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.756032:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.756034:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.756035:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.756478:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.756481:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.756482:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.756484:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.756486:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.756486:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.756498:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.756501:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.756502:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.756502:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.756503:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.756505:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.757281:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.757283:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.757284:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.757285:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.757287:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.757288:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.760238:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.760241:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.760241:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.760243:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.760245:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.760246:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.760514:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.760516:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.760517:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.760519:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.760521:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.760522:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.760778:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.760780:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.760781:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.760783:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.760785:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.760785:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.760797:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.760799:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.760800:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.760801:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.760801:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.760802:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.761537:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.761540:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.761540:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.761542:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.761544:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.761545:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.762381:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.762383:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.762384:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.762385:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.762388:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.762389:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.763612:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.763614:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.763615:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.763617:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.763618:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.763623:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:7.0:1619677445.763693:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff91abeb71ba00] (18)--
00000800:00000200:23.0:1619677445.763976:0:13382:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.116@o2ib7 Addr resolved: 0
00000800:00000200:23.0:1619677445.763982:0:13382:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.116@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:23.0:1619677445.764320:0:13382:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.116@o2ib7 Route resolved: 0
00000800:00000200:23.0:1619677445.764329:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff91abd919d000 0x205919d000(0x205919d000)
00000800:00000200:23.0:1619677445.764329:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff91abd919e000 0x205919e000(0x205919e000)
00000800:00000200:23.0:1619677445.764330:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff91abd919f000 0x205919f000(0x205919f000)
00000800:00000200:23.0:1619677445.764330:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff91abd91a0000 0x20591a0000(0x20591a0000)
00000800:00000200:23.0:1619677445.764331:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff91abd91a1000 0x20591a1000(0x20591a1000)
00000800:00000200:23.0:1619677445.764331:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff91abd91a2000 0x20591a2000(0x20591a2000)
00000800:00000200:23.0:1619677445.764332:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff91abd91a3000 0x20591a3000(0x20591a3000)
00000800:00000200:23.0:1619677445.764333:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff91abd91a4000 0x20591a4000(0x20591a4000)
00000800:00000200:23.0:1619677445.764333:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff91abd91a5000 0x20591a5000(0x20591a5000)
00000800:00000200:23.0:1619677445.764334:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff91abd91a6000 0x20591a6000(0x20591a6000)
00000800:00000200:23.0:1619677445.764335:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff91abd91a7000 0x20591a7000(0x20591a7000)
00000800:00000200:23.0:1619677445.764335:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff91abd91a8000 0x20591a8000(0x20591a8000)
00000800:00000200:23.0:1619677445.764336:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff91abd91a9000 0x20591a9000(0x20591a9000)
00000800:00000200:23.0:1619677445.764336:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff91abd91aa000 0x20591aa000(0x20591aa000)
00000800:00000200:23.0:1619677445.764337:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff91abd91ab000 0x20591ab000(0x20591ab000)
00000800:00000200:23.0:1619677445.764337:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff91abd91ac000 0x20591ac000(0x20591ac000)
00000800:00000200:23.0:1619677445.764338:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff91abd91ad000 0x20591ad000(0x20591ad000)
00000800:00000200:23.0:1619677445.764338:0:13382:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff91abd91ae000 0x20591ae000(0x20591ae000)
00000800:00000200:12.0:1619677445.764851:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.764854:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.764855:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.764856:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.764858:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.764859:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.764860:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.764860:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.764861:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.764862:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.764862:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.764899:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.764901:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.764902:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.764903:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.764905:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.764906:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:23.0:1619677445.764996:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (19)++
00000800:00000200:23.0:1619677445.764996:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (20)--
00000800:00000200:23.0:1619677445.764997:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (19)++
00000800:00000200:23.0:1619677445.764997:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (20)--
00000800:00000200:23.0:1619677445.764997:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (19)++
00000800:00000200:23.0:1619677445.764998:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (20)--
00000800:00000200:23.0:1619677445.764998:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (19)++
00000800:00000200:23.0:1619677445.764999:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (20)--
00000800:00000200:23.0:1619677445.764999:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (19)++
00000800:00000200:23.0:1619677445.764999:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (20)--
00000800:00000200:23.0:1619677445.765000:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (19)++
00000800:00000200:23.0:1619677445.765000:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (20)--
00000800:00000200:23.0:1619677445.765000:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (19)++
00000800:00000200:23.0:1619677445.765001:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (20)--
00000800:00000200:23.0:1619677445.765001:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (19)++
00000800:00000200:23.0:1619677445.765001:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (20)--
00000800:00000200:23.0:1619677445.765002:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (19)++
00000800:00000200:23.0:1619677445.765002:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (20)--
00000800:00000200:23.0:1619677445.765002:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (19)++
00000800:00000200:23.0:1619677445.765003:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (20)--
00000800:00000200:23.0:1619677445.765003:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (19)++
00000800:00000200:23.0:1619677445.765004:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (20)--
00000800:00000200:23.0:1619677445.765004:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (19)++
00000800:00000200:23.0:1619677445.765004:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (20)--
00000800:00000200:23.0:1619677445.765005:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (19)++
00000800:00000200:23.0:1619677445.765006:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (20)--
00000800:00000200:23.0:1619677445.765006:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (19)++
00000800:00000200:23.0:1619677445.765006:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (20)--
00000800:00000200:23.0:1619677445.765007:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (19)++
00000800:00000200:23.0:1619677445.765007:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (20)--
00000800:00000200:23.0:1619677445.765007:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (19)++
00000800:00000200:23.0:1619677445.765008:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (20)--
00000800:00000200:23.0:1619677445.765008:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (19)++
00000800:00000200:23.0:1619677445.765008:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (20)--
00000800:00000200:23.0:1619677445.765009:0:13382:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (19)++
00000800:00000200:23.0:1619677445.765009:0:13382:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (20)--
00000800:00000200:13.0:1619677445.765387:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (29)++
00000800:00000200:13.0:1619677445.765389:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (30)--
00000400:00000200:13.0:1619677445.765390:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.765392:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.765394:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.765395:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (29)++
00000800:00000200:13.0:1619677445.765396:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677445.765396:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (30)--
00000800:00000200:13.0:1619677445.765397:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (29)++
00000400:00000200:12.0:1619677445.765397:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.765398:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (30)++
00000400:00000200:12.0:1619677445.765398:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.765399:0:19942:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.0.10.111@o2ib7: no credits
00000800:00000200:13.0:1619677445.765400:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (31)--
00000800:00000200:12.0:1619677445.765401:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (30)--
00000800:00000200:13.0:1619677445.765402:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (29)--
00000800:00000200:12.0:1619677445.765403:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.765404:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (28)++
00000800:00000200:12.0:1619677445.765404:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.765405:0:19942:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.0.10.111@o2ib7: no credits
00000400:00000200:12.0:1619677445.765405:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.765406:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (29)--
00000400:00000200:12.0:1619677445.765406:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:13.0:1619677445.765407:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.765408:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.765408:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.765409:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.765410:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (28)++
00000800:00000200:13.0:1619677445.765411:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (29)--
00000800:00000200:12.0:1619677445.765411:0:19943:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.0.10.111@o2ib7: no credits
00000800:00000200:13.0:1619677445.765412:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (28)++
00000800:00000200:12.0:1619677445.765412:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (28)--
00000400:00000200:12.0:1619677445.765413:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.765414:0:19942:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.0.10.111@o2ib7: no credits
00000800:00000200:13.0:1619677445.765414:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (28)--
00000400:00000200:12.0:1619677445.765414:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:13.0:1619677445.765415:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.765417:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.765417:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (27)--
00000800:00000200:12.0:1619677445.765418:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (26)++
00000800:00000200:12.0:1619677445.765419:0:19943:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.0.10.111@o2ib7: no credits
00000800:00000200:13.0:1619677445.765420:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (27)--
00000800:00000200:13.0:1619677445.765420:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (26)++
00000800:00000200:12.0:1619677445.765420:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (26)--
00000400:00000200:12.0:1619677445.765421:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.765423:0:19942:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.0.10.111@o2ib7: no credits
00000800:00000200:13.0:1619677445.765423:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (26)--
00000400:00000200:12.0:1619677445.765423:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:13.0:1619677445.765424:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.765425:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.765425:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (25)--
00000800:00000200:12.0:1619677445.765426:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)++
00000800:00000200:13.0:1619677445.765428:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (25)--
00000800:00000200:12.0:1619677445.765428:0:19943:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.0.10.111@o2ib7: no credits
00000800:00000200:12.0:1619677445.765428:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (25)--
00000800:00000200:13.0:1619677445.765429:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.765429:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.765431:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.765434:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.765450:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.765453:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.765454:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677445.765455:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677445.765456:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.765457:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.765466:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.765469:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.765470:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.765471:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.765473:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.765473:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.765475:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.765475:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[4] from 10.0.10.111@o2ib7
00000400:00000200:12.0:1619677445.765478:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.111@o2ib7 : PUT - routed
00000800:00000200:12.0:1619677445.765481:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.765692:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677445.765695:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677445.765696:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.765697:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.765699:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677445.765700:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677445.765700:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677445.765701:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677445.765701:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.765703:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.765706:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.765796:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.765798:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.765799:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.765801:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.765802:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.765804:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.765885:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.765887:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.765888:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.765889:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.765891:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.765892:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.765904:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.765907:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.765908:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.765908:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.765909:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.765910:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.766070:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.766073:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.766074:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.766076:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.766077:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.766079:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.766460:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.766462:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.766464:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.766466:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.766467:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.766469:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.766765:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.766767:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.766768:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.766770:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.766772:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.766773:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.766787:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.766791:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.766791:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.766792:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.766793:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.766794:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.767089:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.767092:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.767093:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.767094:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.767095:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.767097:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.767176:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.767178:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.767179:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.767180:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.767182:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.767183:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.767468:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.767471:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.767472:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.767473:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.767475:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.767476:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.768102:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.768105:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.768106:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.768108:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.768109:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.768111:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.768123:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.768126:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.768127:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.768128:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.768128:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.768130:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.768329:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.768332:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.768332:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.768335:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.768336:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.768342:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.768362:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.768365:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.768366:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.768367:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.768368:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.768371:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.768379:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.768382:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.768383:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.768384:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.768385:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.768386:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.769118:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.769121:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.769122:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.769124:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.769124:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.769127:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.769987:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.769989:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.769990:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.769992:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.769994:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.769996:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.770029:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.770031:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.770032:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.770035:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.770035:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.770039:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.770119:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.770121:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.770122:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.770125:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677445.770125:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.770127:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.770627:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.770630:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.770631:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.770633:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.770634:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.770635:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.770644:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.770647:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.770648:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.770649:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.770649:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.770651:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.770760:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.770762:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.770763:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.770764:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.770766:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.770767:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.770907:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.770910:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.770910:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.770912:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.770914:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.770915:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.771068:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.771070:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.771071:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.771073:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.771074:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.771075:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.771322:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.771325:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.771326:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.771328:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.771329:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.771331:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.771705:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677445.771707:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677445.771708:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.771710:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.771711:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677445.771712:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677445.771712:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677445.771713:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677445.771714:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677445.771716:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.771717:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.771726:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.771729:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.771729:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.771730:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.771731:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.771732:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.772080:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.772082:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.772083:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.772085:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.772086:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.772088:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.772273:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.772276:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.772277:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.772278:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.772280:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.772281:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.772637:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.772639:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.772640:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.772642:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.772644:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.772645:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.772645:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.772646:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.772647:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.772647:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677445.772647:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.772648:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.772650:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.772859:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.772862:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.772863:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.772865:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.772866:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.772867:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.773778:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.773781:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.773781:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.773783:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.773785:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.773786:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.2:1619677445.774214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980600] (17)++
00000800:00000200:13.0:1619677445.774775:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.774777:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.774778:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.774781:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.774782:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.774784:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.774794:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.774797:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.774798:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.774798:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.774799:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.774800:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.775497:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.775499:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.775500:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.775502:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.775504:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.775505:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.775552:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.775555:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.775555:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.775557:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.775559:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.775560:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.776051:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.776053:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.776054:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.776056:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.776057:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.776059:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.777045:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.777048:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.777048:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.777051:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.777052:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.777055:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.777387:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.777390:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.777390:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.777392:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.777394:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.777395:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.777969:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.777971:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.777973:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.777974:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.777975:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.777977:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.777990:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.777993:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.777993:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.777994:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.777995:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.777996:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.778098:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.778101:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.778101:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.778103:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.778105:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.778105:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.778118:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.778120:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.778121:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.778122:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.778122:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.778124:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.778370:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.778373:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.778373:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.778375:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.778377:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.778378:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.778768:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.778771:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.778772:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.778773:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.778775:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.778776:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.778921:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.778923:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.778924:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.778926:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.778927:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.778929:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.779535:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.779538:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.779539:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.779541:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.779542:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.779543:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.780140:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.780143:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.780143:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.780145:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.780146:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.780148:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.780160:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.780162:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.780163:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.780164:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.780165:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.780166:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.780740:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.780742:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.780743:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.780745:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.780747:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.780748:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.781487:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.781490:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.781491:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.781492:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.781495:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.781496:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.781932:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.781934:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.781935:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.781936:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.781938:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.781939:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.781953:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.781956:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.781957:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.781957:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.781958:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.781959:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.782209:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.782211:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.782212:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.782214:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.782216:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.782216:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.782570:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.782572:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.782573:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.782575:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.782577:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.782578:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.782999:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.783002:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.783003:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.783005:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.783006:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.783008:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.783438:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.783441:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.783442:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.783444:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.783446:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.783446:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.783513:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.783516:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.783516:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.783518:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.783519:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.783521:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.784460:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.784463:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.784464:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.784466:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.784467:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.784468:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.785397:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.785400:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.785401:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.785402:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.785404:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.785405:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.785417:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.785420:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.785421:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.785421:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.785422:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.785424:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.785752:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.785754:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.785755:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.785757:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.785758:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.785760:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.785771:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.785774:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.785775:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (24)++
00000800:00000200:13.0:1619677445.785776:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677445.785777:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.785778:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.785781:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.785782:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.785784:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.785785:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.785786:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.786545:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.786548:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.786549:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.786550:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.786552:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.786553:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.786641:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.786643:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.786644:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.786646:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.786647:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.786649:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.786757:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.786764:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.786764:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.786766:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.786767:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.786770:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.787736:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.787738:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.787739:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.787740:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.787742:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.787743:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.787839:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.787841:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.787842:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677445.787845:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677445.787846:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.787849:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.787874:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.787877:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.787877:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.787879:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.787881:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.787882:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.787892:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.787895:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.787896:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.787896:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.787897:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.787898:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.788275:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.788277:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.788278:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.788280:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.788281:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.788283:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.788496:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.788499:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.788500:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.788501:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677445.788503:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.788504:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.788650:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.788653:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.788654:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.788655:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.788657:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.788658:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.788669:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.788672:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.788673:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.788673:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.788674:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.788675:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.789833:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.789835:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.789836:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.789838:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.789840:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.789841:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.790001:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677445.790003:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677445.790006:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677445.790007:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677445.790008:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.790011:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677445.790013:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96751449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677445.790014:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677445.790015:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677445.790017:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677445.790020:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677445.790023:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677445.790027:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677445.790028:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:13.0:1619677445.790029:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677445.790031:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:13.0:1619677445.790032:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677445.790033:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677445.790033:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677445.790632:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.790635:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.790635:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.790637:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.790639:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.790640:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.790652:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.790655:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.790657:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.790657:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.790658:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.790660:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.790941:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.790944:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.790945:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.790946:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.790947:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.790949:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.791267:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.791270:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.791271:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.791272:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.791273:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.791275:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.791694:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.791697:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.791698:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.791700:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.791701:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.791702:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.791761:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.791764:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.791765:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.791767:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.791768:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.791769:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.792974:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.792977:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.792977:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.792979:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.792981:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.792982:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.793096:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.793098:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.793099:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.793100:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.793102:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.793103:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.793113:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.793116:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.793117:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.793118:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.793118:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.793120:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.793548:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.793551:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.793552:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.793554:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.793555:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.793557:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.793586:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.793589:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.793590:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.793592:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.793593:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.793595:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.793666:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.793668:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.793669:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.793670:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.793672:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.793673:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.794184:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.794186:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.794187:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.794190:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677445.794192:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.794194:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.794813:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.794815:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.794816:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.794817:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.794820:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.794820:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.794828:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.794831:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.794832:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.794832:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.794833:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.794834:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.795160:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677445.795162:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:12.0:1619677445.795167:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677445.795171:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677445.795173:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.795175:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:13.0:1619677445.795176:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96751374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677445.795178:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677445.795179:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677445.795180:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:13.0:1619677445.795184:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677445.795187:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677445.795189:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.795190:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677445.795190:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.795191:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677445.795191:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:12.0:1619677445.795191:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.795193:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.795194:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677445.795195:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677445.795196:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677445.795196:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677445.795196:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.795197:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.795210:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.795213:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.795213:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.795214:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.795215:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.795216:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.795242:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.795245:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.795246:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.795248:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677445.795248:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.795251:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.795404:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.795406:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.795407:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.795408:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.795410:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.795411:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.796026:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.796029:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.796030:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.796032:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.796032:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.796034:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.796047:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.796050:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.796051:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.796052:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.796054:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.796055:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.796235:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.796238:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.796239:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.796240:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.796242:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.796243:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.796316:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.796319:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.796320:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.796322:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.796324:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.796324:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.796337:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.796339:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.796340:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.796341:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.796341:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.796342:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.797302:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.797304:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.797305:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.797308:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.797309:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.797314:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.798158:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.798161:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.798162:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.798165:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677445.798165:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.798168:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.798205:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.798208:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.798209:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.798210:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.798212:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.798213:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.798380:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.798382:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.798383:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.798385:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.798387:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.798387:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.798401:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.798404:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.798405:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.798405:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.798405:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.798407:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.798667:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.798670:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.798671:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.798672:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.798674:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.798675:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.798725:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.798728:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.798729:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.798731:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.798732:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.798733:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.798744:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.798747:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.798748:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.798748:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.798749:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.798750:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.798977:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.798980:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.798980:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.798982:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.798983:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.798984:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.799167:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.799170:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.799171:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.799173:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.799173:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.799177:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.799563:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.799565:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.799566:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.799568:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.799570:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.799571:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.800236:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.800239:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.800240:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.800241:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.800243:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.800243:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.800669:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.800671:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.800672:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.800673:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.800675:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.800676:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.800864:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.800866:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.800867:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.800869:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.800870:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.800872:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.801372:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.801375:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.801376:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.801377:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.801379:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.801380:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.801395:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.801398:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.801399:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.801400:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.801401:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.801402:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.801610:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.801612:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.801613:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.801615:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.801616:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.801617:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.801627:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.801630:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.801631:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.801631:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.801632:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.801634:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.802301:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.802304:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.802305:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.802306:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.802308:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.802309:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.802434:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.802437:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.802438:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.802440:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.802441:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.802442:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.802694:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.802697:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.802698:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.802700:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.802701:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.802703:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.802784:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.802787:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.802788:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.802790:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.802791:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.802793:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.803176:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.803179:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.803179:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.803182:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.803182:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.803185:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.803209:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.803212:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.803213:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.803215:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.803216:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.803218:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.803235:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.803237:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.803238:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.803239:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.803239:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.803241:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.803280:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.803284:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.803284:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.803286:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.803287:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.803288:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.803699:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677445.803702:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677445.803703:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.803705:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.803706:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677445.803707:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677445.803708:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.803708:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.803709:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:12.0:1619677445.803710:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.803712:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.803713:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677445.803713:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.803714:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.803714:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.803715:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.803959:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.803961:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.803962:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.803964:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.803965:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.803966:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.804155:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.804158:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.804158:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.804160:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.804162:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.804163:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.804291:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.804299:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.804299:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.804299:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.804301:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.804303:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.804398:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.804400:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.804401:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.804402:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.804404:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.804405:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.804655:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.804658:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.804659:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.804661:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.804662:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.804664:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.806445:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.806447:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.806448:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.806450:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.806452:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.806453:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.806464:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.806467:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.806467:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.806468:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.806469:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.806470:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.807010:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677445.807013:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677445.807015:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677445.807017:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677445.807017:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.807020:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:12.0:1619677445.807021:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96751203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677445.807023:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677445.807024:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677445.807025:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:12.0:1619677445.807028:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677445.807030:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677445.807033:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677445.807034:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:12.0:1619677445.807035:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:12.0:1619677445.807037:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:12.0:1619677445.807038:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677445.807039:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677445.807041:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677445.807041:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677445.807055:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677445.807058:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677445.807059:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677445.807060:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:13.0:1619677445.807237:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.807239:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.807240:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.807242:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.807243:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.807245:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.807738:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.807741:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.807742:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.807744:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.807745:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.807747:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.807747:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.807747:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.807748:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.807748:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.807749:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.807796:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.807798:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.807800:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.807801:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.807802:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.807804:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.807904:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.807907:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.807908:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.807909:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.807910:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.807912:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.808088:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.808091:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.808092:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.808094:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.808095:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.808097:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.808269:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.808272:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.808273:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.808275:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.808276:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.808277:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.808851:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.808854:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.808855:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.808857:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.808859:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.808860:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.808868:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.808870:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.808871:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.808871:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.808872:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.808873:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.809271:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.809273:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.809274:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.809275:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.809278:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.809278:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.809423:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.809425:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.809427:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.809428:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.809430:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.809432:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.809611:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.809614:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.809615:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.809616:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.809618:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.809619:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.809756:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.809758:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.809759:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.809760:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.809762:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.809763:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.809872:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.809874:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.809875:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.809877:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.809878:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.809880:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.809890:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.809893:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.809894:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.809894:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.809895:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.809896:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.810400:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.810402:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.810403:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.810405:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.810407:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.810408:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.810512:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.810514:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.810514:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.810516:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.810518:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.810519:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.810530:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.810533:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.810533:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.810534:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.810535:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.810535:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.811661:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.811664:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.811665:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.811666:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.811668:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.811669:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.811993:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.811997:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.811998:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.812000:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677445.812000:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.812003:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.812101:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.812104:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.812105:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.812106:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.812107:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.812109:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.812117:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.812120:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.812121:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.812121:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.812122:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.812123:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.812205:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.812208:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.812209:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.812210:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.812212:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.812213:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.812982:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.812984:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.812985:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.812987:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.812988:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.812989:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.813616:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.813619:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.813620:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.813623:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.813623:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.813626:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.814198:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.814201:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.814202:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.814204:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677445.814204:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.814207:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.814578:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677445.814580:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677445.814581:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.814583:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.814585:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677445.814585:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677445.814586:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677445.814587:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677445.814587:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.814588:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.814590:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.814669:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.814672:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.814672:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.814674:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.814675:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.814677:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.814685:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.814688:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.814689:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.814689:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.814690:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.814691:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.815714:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.815717:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.815718:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.815719:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.815721:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.815722:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.816373:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.816375:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.816376:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.816378:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.816380:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.816381:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.816392:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.816395:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.816396:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.816396:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.816397:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.816398:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.816894:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.816897:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.816897:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.816899:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.816901:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.816901:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.817176:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.817178:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.817179:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.817181:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.817182:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.817183:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.817307:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.817310:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.817310:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.817312:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.817314:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.817315:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.817459:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.817461:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.817462:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.817463:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.817465:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.817466:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.818534:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.818536:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.818537:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.818539:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.818541:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.818542:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.818839:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.818841:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.818842:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.818844:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.818845:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.818846:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.818863:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.818866:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.818867:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.818867:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.818868:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.818869:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.819139:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.819142:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.819143:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.819145:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.819146:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.819148:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.819222:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.819225:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.819226:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.819228:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.819228:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.819231:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.820432:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.820435:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.820436:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.820438:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.820440:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.820441:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.820452:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.820455:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.820456:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.820457:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.820457:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.820459:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.820736:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.820739:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.820739:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.820741:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.820743:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.820743:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.821104:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.821106:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.821107:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.821109:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.821110:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.821112:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.821380:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.821383:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.821385:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.821386:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.821388:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.821389:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.821815:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.821818:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.821819:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.821821:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.821822:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.821824:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.821831:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.821834:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.821834:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.821835:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.821836:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.821836:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.822257:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.822259:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.822260:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.822261:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677445.822263:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.822265:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.822372:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.822375:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.822376:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.822379:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677445.822380:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.822383:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.823211:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.823214:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.823215:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.823217:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.823217:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.823220:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.823739:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677445.823742:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677445.823743:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.823745:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.823746:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677445.823747:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677445.823748:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677445.823748:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677445.823749:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:13.0:1619677445.823750:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.823751:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.823752:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.823752:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677445.823752:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677445.823753:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.823754:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.823843:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.823845:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.823846:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.823848:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.823850:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.823851:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.824657:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.824660:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.824661:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.824663:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.824664:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.824667:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.824674:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.824677:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.824678:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.824679:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.824680:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.824681:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.824698:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.824701:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.824702:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.824704:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.824705:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.824706:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.824997:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.825000:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.825000:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.825002:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.825003:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.825005:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.825082:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.825084:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.825085:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.825086:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.825088:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.825090:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.825101:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.825104:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.825105:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.825105:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.825106:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.825107:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.825590:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.825592:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.825593:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.825595:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.825597:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.825597:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.826272:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677445.826274:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677445.826275:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.826277:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.826278:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677445.826279:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.826279:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677445.826280:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677445.826280:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.826281:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.826283:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.826434:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.826436:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.826437:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.826439:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.826441:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.826442:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.826687:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.826690:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.826691:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.826693:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.826694:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.826695:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.826708:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.826711:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.826712:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.826712:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.826713:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.826714:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.827042:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.827044:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.827045:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.827047:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.827048:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.827049:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.827350:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.827353:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.827353:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.827355:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.827357:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.827358:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.827622:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.827625:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.827625:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.827627:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.827629:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.827630:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.827688:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.827691:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.827692:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.827693:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.827695:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.827696:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.829028:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.829032:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.829033:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.829034:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.829037:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.829038:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.829039:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.829039:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.829040:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.829040:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677445.829041:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.829041:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.829041:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.829381:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.829384:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.829384:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.829386:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.829388:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.829388:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.829481:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.829483:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.829484:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.829486:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.829487:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.829489:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.829498:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.829501:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.829502:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.829503:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.829503:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.829505:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.830063:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.830066:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.830067:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.830069:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.830069:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.830071:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.830463:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.830466:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.830467:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.830469:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.830470:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.830472:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.830482:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.830486:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (25)--
00000400:00000200:13.0:1619677445.830487:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.830488:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.830488:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.830493:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.830495:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.830496:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.830497:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677445.830497:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677445.830498:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.830500:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.831014:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.831017:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.831018:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.831019:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.831021:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.831028:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.831037:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.831040:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.831041:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.831042:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.831043:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.831044:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.831135:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.831138:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.831139:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.831140:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.831142:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.831144:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.831144:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.831147:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.831147:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.831150:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.831151:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.831152:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.831336:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.831339:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.831340:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.831342:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.831344:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.831359:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.831773:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.831776:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.831777:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.831778:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.831780:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.831781:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.831878:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677445.831880:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677445.831881:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.831883:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.831884:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677445.831885:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677445.831886:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677445.831886:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.831887:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.831887:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.831890:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.831897:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.831900:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.831901:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.831901:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.831902:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.831903:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.832501:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.832503:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.832504:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.832506:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.832508:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.832509:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.832870:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.832873:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.832874:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.832875:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.832877:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.832878:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.833201:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.833204:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.833205:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.833207:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.833207:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.833210:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.833220:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.833223:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.833224:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.833224:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.833225:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.833226:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.833511:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.833514:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.833515:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.833517:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.833518:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.833520:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.834108:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.834110:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.834111:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.834113:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.834114:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.834116:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.835530:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.835532:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.835533:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.835534:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.835537:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.835538:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.835565:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.835567:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.835568:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.835570:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.835571:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.835573:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.835620:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.835622:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.835623:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.835624:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.835624:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.835625:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.835675:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.835677:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.835678:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.835680:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.835681:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.835683:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.836081:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.836083:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.836084:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.836086:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.836087:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.836088:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.836241:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.836244:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.836245:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.836247:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.836248:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.836250:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.836298:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.836300:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.836301:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.836303:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.836305:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.836306:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.836389:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.836392:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.836393:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.836394:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.836395:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.836397:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.836406:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.836409:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.836410:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.836411:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.836411:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.836412:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.836435:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.836437:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.836438:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.836440:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.836441:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.836442:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.836453:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.836455:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.836456:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.836456:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.836457:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.836458:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.837184:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.837186:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.837187:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.837188:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.837190:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.837191:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.837573:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.837576:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.837576:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.837578:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.837580:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.837581:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.837956:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.837958:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.837959:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.837961:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.837962:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.837964:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.838815:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.838818:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.838819:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.838820:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.838823:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.838824:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.839443:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.839446:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.839447:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.839449:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.839450:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.839452:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.839462:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.839465:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.839465:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.839466:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.839466:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.839468:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.839930:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.839933:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.839933:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.839935:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.839937:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.839938:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.840317:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.840320:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.840321:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.840323:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.840325:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.840326:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.840466:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.840468:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.840469:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.840470:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.840472:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.840473:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.840576:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.840579:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.840579:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.840581:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.840583:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.840584:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.840629:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.840632:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.840633:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.840635:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.840636:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.840637:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.840646:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.840649:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.840650:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.840650:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.840651:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.840653:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.840727:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.840729:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.840730:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.840732:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.840733:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.840734:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.841159:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.841162:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.841163:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.841164:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.841165:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.841167:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.841824:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.841826:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.841827:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.841829:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.841831:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.841832:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.842048:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.842050:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.842051:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.842053:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.842054:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.842056:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.842270:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.842273:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.842273:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.842275:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.842277:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.842277:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.842290:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.842300:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.842301:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.842302:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.842303:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.842303:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.842454:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.842456:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.842457:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.842459:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.842460:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.842462:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.842472:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.842475:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.842475:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.842476:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.842477:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.842478:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.842851:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.842854:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.842854:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.842856:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.842858:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.842858:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.843027:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.843030:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.843031:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.843032:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.843034:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.843035:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.843271:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.843274:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.843274:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.843276:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.843278:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.843279:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.843322:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.843324:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.843325:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.843327:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.843329:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.843330:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.843426:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.843429:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.843429:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.843431:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.843433:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.843433:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.843779:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.843782:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.843783:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.843784:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.843786:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.843787:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.843803:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.843805:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.843806:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.843807:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.843808:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.843809:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.844206:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.844209:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.844210:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.844211:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.844213:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.844214:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.844989:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.844992:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.844993:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.844994:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.844996:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.844997:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.845421:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.845423:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.845424:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.845425:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.845428:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.845429:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.845753:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.845755:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.845756:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.845758:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.845759:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.845761:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.845776:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (18)++
00000800:00000200:13.0:1619677445.845778:0:19942:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17d8400] (19)--
00000800:00000200:13.0:1619677445.845779:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (18)--
00000800:00000200:13.0:1619677445.845779:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (17)++
00000800:00000200:13.0:1619677445.845780:0:19942:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17d8400] (18)--
00000800:00000200:13.0:1619677445.845781:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (17)--
00000800:00000200:13.0:1619677445.845781:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (16)++
00000800:00000200:13.0:1619677445.845782:0:19942:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17d8400] (17)--
00000800:00000200:13.0:1619677445.845782:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (16)--
00000800:00000200:12.0:1619677445.845783:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (15)++
00000800:00000200:12.0:1619677445.845786:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17d8400] (16)--
00000800:00000200:12.0:1619677445.845786:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (15)--
00000800:00000200:12.0:1619677445.845787:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (14)++
00000800:00000200:12.0:1619677445.845787:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17d8400] (15)--
00000800:00000200:12.0:1619677445.845788:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (14)--
00000800:00000200:12.0:1619677445.845788:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (13)++
00000800:00000200:12.0:1619677445.845789:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17d8400] (14)--
00000800:00000200:13.0:1619677445.845790:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (12)++
00000800:00000200:12.0:1619677445.845790:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (13)--
00000800:00000200:13.0:1619677445.845793:0:19942:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17d8400] (13)--
00000800:00000200:13.0:1619677445.845793:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (12)--
00000800:00000200:13.0:1619677445.845794:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (11)++
00000800:00000200:13.0:1619677445.845795:0:19942:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17d8400] (12)--
00000800:00000200:13.0:1619677445.845795:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (11)--
00000800:00000200:13.0:1619677445.845795:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (10)++
00000800:00000200:13.0:1619677445.845797:0:19942:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17d8400] (11)--
00000800:00000200:13.0:1619677445.845797:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (10)--
00000800:00000200:12.0:1619677445.845798:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (9)++
00000800:00000200:12.0:1619677445.845801:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17d8400] (10)--
00000800:00000200:12.0:1619677445.845801:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (9)--
00000800:00000200:12.0:1619677445.845802:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (8)++
00000800:00000200:12.0:1619677445.845802:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17d8400] (9)--
00000800:00000200:12.0:1619677445.845803:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (8)--
00000800:00000200:12.0:1619677445.845803:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (7)++
00000800:00000200:12.0:1619677445.845804:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17d8400] (8)--
00000800:00000200:12.0:1619677445.845805:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (7)--
00000800:00000200:13.0:1619677445.845806:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (6)++
00000800:00000200:13.0:1619677445.845809:0:19942:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17d8400] (7)--
00000800:00000200:13.0:1619677445.845809:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (6)--
00000800:00000200:13.0:1619677445.845810:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8400] (5)++
00000800:00000200:13.0:1619677445.845810:0:19942:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17d8400] (6)--
00000800:00000200:13.0:1619677445.845811:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (5)--
00000800:00000200:13.0:1619677445.845811:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8400] (4)--
00000800:00000200:13.0:1619677445.846049:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.846051:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.846052:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.846053:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.846054:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.846055:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.846067:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.846070:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.846071:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.846071:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.846072:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.846074:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.2:1619677445.846162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (20)++
00000800:00000200:13.0:1619677445.846490:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.846492:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.846493:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.846495:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.846496:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.846497:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.846508:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.846511:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.846511:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.846512:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.846513:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.846514:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.846651:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.846654:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.846654:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.846657:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.846657:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.846660:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.847625:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.847628:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.847628:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.847630:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.847632:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.847633:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.847725:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.847728:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.847728:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.847730:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.847732:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.847733:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.848419:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.848422:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.848423:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.848424:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.848426:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.848427:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.848442:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.848444:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.848446:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.848446:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.848447:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.848448:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.848781:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.848784:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.848785:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.848786:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.848788:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.848789:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.849701:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677445.849704:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677445.849705:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.849706:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.849708:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677445.849709:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677445.849709:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677445.849710:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677445.849710:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.849711:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.849714:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.849944:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677445.849947:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677445.849950:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677445.849951:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677445.849952:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.849954:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:13.0:1619677445.849956:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96750600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677445.849957:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677445.849958:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677445.849959:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:13.0:1619677445.849963:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677445.849965:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:13.0:1619677445.849968:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677445.849968:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:13.0:1619677445.849969:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:13.0:1619677445.849972:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:13.0:1619677445.849972:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677445.849973:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677445.849974:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677445.850542:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.850544:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.850545:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.850547:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.850549:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.850549:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.850579:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.850582:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.850583:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.850583:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.850584:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.850585:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.851095:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.851097:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.851098:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.851099:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.851101:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.851102:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.851214:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.851217:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.851218:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.851220:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.851221:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.851222:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.851282:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.851285:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.851285:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.851287:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.851289:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.851290:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.851690:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.851692:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.851693:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.851694:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.851696:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.851697:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.852332:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.852334:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.852335:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.852337:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.852339:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.852340:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.852630:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.852633:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.852633:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.852635:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.852637:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.852638:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.852649:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.852652:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.852652:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.852653:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.852654:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.852655:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.854271:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.854273:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.854274:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.854275:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.854278:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.854279:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.854418:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.854420:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.854421:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.854423:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.854425:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.854426:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.854427:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.854428:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.854428:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.854428:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.854429:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677445.854429:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.854430:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.854847:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.854849:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.854850:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.854852:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.854853:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.854855:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.854987:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.854989:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.854990:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.854991:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.854993:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.854994:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.855415:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.855417:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.855418:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.855420:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.855422:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.855424:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.855447:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.855450:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.855451:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.855453:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.855454:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.855456:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.856492:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.856495:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.856496:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.856497:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.856499:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.856500:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.856509:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.856511:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.856512:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.856513:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.856513:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.856514:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.856948:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.856950:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.856951:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.856953:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.856954:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.856957:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.857344:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.857347:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.857348:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.857349:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.857351:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.857352:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.857360:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.857362:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.857363:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.857365:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.857366:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.857367:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.857535:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.857537:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.857538:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.857539:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.857541:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.857543:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.858052:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.858054:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.858055:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.858057:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.858059:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.858060:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.858074:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.858077:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.858078:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.858079:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.858080:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.858081:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.858109:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.858111:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.858115:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677445.858115:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677445.858117:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.858120:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.858427:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.858430:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.858431:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.858433:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.858435:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.858436:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.858670:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.858672:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.858673:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.858675:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.858676:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.858677:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.859192:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.859195:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.859196:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.859197:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.859198:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.859201:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.859215:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.859218:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.859218:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.859219:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.859220:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.859221:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.859329:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.859332:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.859333:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.859336:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.859336:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.859339:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.859756:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.859759:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.859759:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.859761:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.859763:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.859764:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.860569:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.860571:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.860572:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.860574:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.860576:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.860577:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.860730:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.860733:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.860734:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.860736:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.860737:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.860738:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.861020:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.861022:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.861023:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.861025:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.861026:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.861027:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.861044:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.861047:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.861048:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.861048:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.861049:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.861050:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.861625:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.861627:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.861628:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.861629:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.861630:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.861631:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.861631:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.861632:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:12.0:1619677445.861632:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.861633:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677445.861635:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.861638:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.861646:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.861649:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.861649:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.861650:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.861651:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.861652:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.861787:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.861789:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.861790:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.861792:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.861793:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.861795:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.863538:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.863541:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.863541:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.863543:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.863545:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.863546:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.863582:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.863585:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.863586:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.863588:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.863589:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.863592:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.864203:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.864205:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.864206:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.864208:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.864209:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.864211:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.864219:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.864222:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.864222:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.864224:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.864225:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.864226:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.864234:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.864237:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.864238:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.864239:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.864240:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.864241:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.865383:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.865385:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.865386:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.865388:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.865390:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.865391:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.865406:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.865409:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.865410:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.865411:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.865413:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.865414:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.865443:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.865446:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.865447:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.865448:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.865449:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.865451:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.865974:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.865976:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.865977:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.865979:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.865980:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.865981:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.865993:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.865996:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.865997:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.865998:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.865998:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.866000:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.866855:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.866857:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.866858:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.866860:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.866862:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.866863:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.867164:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.867167:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.867168:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.867169:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.867170:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.867172:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.867423:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.867425:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.867426:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.867427:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.867429:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.867431:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.867441:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.867444:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.867445:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.867445:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.867446:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.867447:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.867848:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.867851:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.867852:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.867854:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677445.867854:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.867857:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.868325:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.868327:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.868328:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.868330:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.868332:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.868333:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.868525:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.868527:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.868528:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.868529:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.868531:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.868532:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.869323:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (18)++
00000800:00000200:13.0:1619677445.869325:0:19942:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17d8c00] (19)--
00000800:00000200:13.0:1619677445.869326:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (18)--
00000800:00000200:13.0:1619677445.869326:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (17)++
00000800:00000200:13.0:1619677445.869327:0:19942:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17d8c00] (18)--
00000800:00000200:13.0:1619677445.869327:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (17)--
00000800:00000200:13.0:1619677445.869328:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (16)++
00000800:00000200:13.0:1619677445.869328:0:19942:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17d8c00] (17)--
00000800:00000200:13.0:1619677445.869329:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (16)--
00000800:00000200:13.0:1619677445.869330:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (15)++
00000800:00000200:13.0:1619677445.869332:0:19942:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17d8c00] (16)--
00000800:00000200:13.0:1619677445.869332:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (15)--
00000800:00000200:13.0:1619677445.869332:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (14)++
00000800:00000200:13.0:1619677445.869333:0:19942:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17d8c00] (15)--
00000800:00000200:13.0:1619677445.869333:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (14)--
00000800:00000200:13.0:1619677445.869334:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (13)++
00000800:00000200:13.0:1619677445.869336:0:19942:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17d8c00] (14)--
00000800:00000200:12.0:1619677445.869336:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (14)++
00000800:00000200:13.0:1619677445.869337:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (14)--
00000800:00000200:13.0:1619677445.869338:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (13)++
00000800:00000200:12.0:1619677445.869338:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17d8c00] (13)--
00000800:00000200:13.0:1619677445.869339:0:19942:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17d8c00] (13)--
00000800:00000200:12.0:1619677445.869339:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (13)--
00000800:00000200:13.0:1619677445.869340:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (11)--
00000800:00000200:12.0:1619677445.869341:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (10)++
00000800:00000200:12.0:1619677445.869343:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17d8c00] (11)--
00000800:00000200:12.0:1619677445.869343:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (10)--
00000800:00000200:12.0:1619677445.869344:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (9)++
00000800:00000200:12.0:1619677445.869345:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17d8c00] (10)--
00000800:00000200:12.0:1619677445.869346:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (9)--
00000800:00000200:12.0:1619677445.869346:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (8)++
00000800:00000200:12.0:1619677445.869347:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17d8c00] (9)--
00000800:00000200:13.0:1619677445.869348:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (9)++
00000800:00000200:12.0:1619677445.869348:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (9)--
00000800:00000200:13.0:1619677445.869349:0:19942:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17d8c00] (8)--
00000800:00000200:12.0:1619677445.869350:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (8)++
00000800:00000200:13.0:1619677445.869351:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (8)--
00000800:00000200:13.0:1619677445.869352:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (7)++
00000800:00000200:12.0:1619677445.869352:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17d8c00] (7)--
00000800:00000200:12.0:1619677445.869353:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (7)--
00000800:00000200:13.0:1619677445.869354:0:19942:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17d8c00] (6)--
00000800:00000200:12.0:1619677445.869354:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (6)--
00000800:00000200:13.0:1619677445.869355:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8c00] (4)--
00000800:00000200:12.0:1619677445.869738:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.869740:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.869740:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.869742:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.869744:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.869745:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.870339:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.870342:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.870343:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.870344:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.870347:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.870348:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.870356:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.870358:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.870359:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.870360:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.870361:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.870362:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.870459:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.870462:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.870462:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.870464:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.870466:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.870466:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.870535:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.870538:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.870538:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.870540:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.870542:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.870542:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.871192:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.871195:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.871195:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.871197:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.871199:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.871199:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.871995:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677445.871998:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677445.872001:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677445.872003:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677445.872003:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.872005:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:12.0:1619677445.872007:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96750311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677445.872008:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677445.872009:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677445.872010:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:12.0:1619677445.872014:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677445.872016:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677445.872019:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677445.872020:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:12.0:1619677445.872021:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:12.0:1619677445.872023:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:12.0:1619677445.872023:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677445.872024:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677445.872025:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:13.0:1619677445.872319:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.872322:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.872323:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.872324:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.872326:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.872327:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.872847:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.872850:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.872851:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.872853:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.872855:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.872855:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.872866:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.872868:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.872869:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.872870:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.872870:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.872871:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.873120:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.873123:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.873124:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.873126:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.873127:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.873128:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.873528:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.873532:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.873533:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.873534:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.873536:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.873537:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.873546:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.873549:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.873550:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.873551:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.873552:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.873553:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.873773:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.873775:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.873776:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.873778:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.873779:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.873781:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.874004:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.874007:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.874008:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.874009:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.874011:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.874012:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.874253:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.874256:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.874257:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.874259:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.874260:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.874261:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.874803:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.874805:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.874806:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.874808:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.874810:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.874811:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.875007:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.875009:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.875010:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.875011:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.875013:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.875014:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.875028:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.875031:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.875032:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.875033:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.875033:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.875035:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.875289:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.875291:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.875292:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677445.875295:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677445.875299:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.875301:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.875383:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.875385:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.875386:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.875388:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.875390:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.875391:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.875482:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.875485:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.875485:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.875487:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.875489:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.875490:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.875869:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.875871:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.875872:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.875874:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.875875:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.875876:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.875892:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.875895:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.875896:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.875897:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.875898:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.875899:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.876454:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.876457:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.876457:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.876459:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.876461:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.876462:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.876867:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.876869:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.876870:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.876871:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.876873:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.876873:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.876964:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.876966:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.876967:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.876969:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.876970:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.876972:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.877455:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.877457:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.877458:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.877460:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.877462:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.877462:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.877472:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.877475:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.877475:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.877476:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.877477:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.877478:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.877816:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.877819:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.877820:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.877821:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.877822:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.877824:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.878102:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.878104:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.878105:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.878107:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.878108:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.878114:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.878525:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.878528:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.878528:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.878530:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.878532:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.878533:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.878846:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.878849:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.878850:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.878852:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.878852:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.878855:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.878863:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.878865:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.878866:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.878867:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.878868:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.878869:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.878963:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.878965:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.878966:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.878967:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677445.878969:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.878970:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.879115:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.879118:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.879119:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.879121:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.879122:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.879124:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.879540:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.879542:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.879543:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.879545:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.879547:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.879554:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.879573:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.879576:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.879576:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.879578:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.879579:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.879581:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.879590:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.879593:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.879594:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.879594:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.879595:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.879597:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.879971:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.879974:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.879975:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.879977:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.879978:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.879980:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.880340:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.880342:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.880343:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.880345:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.880347:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.880348:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.880745:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.880748:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.880749:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.880752:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677445.880753:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.880758:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.880910:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.880912:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.880913:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.880915:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.880916:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.880917:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.880935:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.880938:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.880938:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.880939:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.880939:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.880940:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.881113:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.881115:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.881116:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.881117:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.881119:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.881120:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.881321:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.881324:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.881324:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.881326:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.881328:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.881329:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.881492:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.881494:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.881495:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.881496:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.881498:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.881499:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.881507:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.881510:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.881510:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.881511:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.881512:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.881513:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.882127:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.882130:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.882130:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.882132:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.882133:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.882134:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.883095:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.883098:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.883098:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.883100:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.883102:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.883103:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.883598:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.883600:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.883601:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.883602:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.883604:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.883605:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.883847:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.883850:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.883851:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.883853:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.883853:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.883856:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.883885:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.883888:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.883889:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.883891:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.883892:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.883893:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.884365:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.884367:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.884368:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.884370:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.884372:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.884373:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.884381:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.884384:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.884385:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.884385:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.884386:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.884387:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.884803:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.884805:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.884806:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.884808:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.884809:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.884811:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.884822:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.884826:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.884827:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.884828:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.884829:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.884830:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.884851:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.884854:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.884855:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.884856:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.884858:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.884859:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.885694:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.885697:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.885698:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.885699:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.885702:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.885703:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.885975:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.885979:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.885979:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.885982:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.885982:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.885984:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.886496:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.886499:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.886500:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.886502:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.886504:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.886506:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.886519:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677445.886522:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677445.886522:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.886524:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.886525:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677445.886526:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677445.886526:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677445.886527:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.886528:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677445.886529:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.886532:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.886539:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.886542:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.886543:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.886543:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.886544:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.886545:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.886590:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.886593:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.886594:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.886597:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677445.886597:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.886600:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.886709:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.886712:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.886712:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.886714:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.886716:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.886717:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.887478:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.887481:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.887482:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.887484:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.887486:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.887487:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.887495:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.887497:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.887498:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.887499:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.887499:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.887501:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.887530:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.887532:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.887533:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.887535:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.887536:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.887537:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.888524:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.888527:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.888528:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.888530:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.888531:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.888533:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.888565:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677445.888567:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677445.888568:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.888570:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.888571:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677445.888573:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677445.888573:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677445.888573:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677445.888574:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677445.888575:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.888576:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.889203:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.889205:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.889206:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.889208:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.889209:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.889211:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.889264:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.889267:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.889268:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.889269:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.889271:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.889272:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.889281:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.889284:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.889285:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.889285:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.889286:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.889287:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.889626:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.889632:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677445.889633:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.889634:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.889634:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.889639:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.889640:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.889642:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.889643:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677445.889643:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677445.889644:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (25)--
00000800:00000200:12.0:1619677445.889644:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (25)--
00000800:00000200:12.0:1619677445.889644:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677445.889645:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677445.889646:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677445.889647:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.889649:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.890008:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.890011:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.890011:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.890013:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.890014:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.890016:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.890630:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.890632:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.890633:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.890635:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.890637:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.890638:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.890659:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.890662:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.890662:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.890664:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.890665:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.890666:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.890819:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.890822:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.890823:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.890825:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.890826:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.890827:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.890839:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.890840:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.890841:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.890842:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.890842:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.890843:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.890844:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.890844:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:13.0:1619677445.890845:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677445.890845:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677445.890847:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.890849:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.891460:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.891463:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.891463:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.891465:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.891468:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.891470:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.891510:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.891513:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.891513:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.891515:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.891516:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.891520:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.891537:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.891540:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.891541:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.891542:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.891544:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.891544:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.891554:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.891556:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.891557:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.891558:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.891558:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.891560:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.891790:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.891793:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.891793:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.891795:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.891797:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.891798:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.892307:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.892309:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.892310:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.892312:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.892314:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.892315:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.892604:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.892606:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.892607:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.892609:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.892610:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.892611:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.893369:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.893371:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.893372:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.893374:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.893376:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.893377:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.893394:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.893396:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.893397:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.893398:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.893398:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.893399:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.893546:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.893548:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.893549:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.893551:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.893552:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.893553:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.893770:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.893773:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.893773:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.893775:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.893776:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.893778:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.893996:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.893999:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.894000:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.894002:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.894003:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.894004:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.894020:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.894023:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.894024:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.894026:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.894026:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.894029:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.894762:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.894764:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.894765:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.894767:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.894768:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.894770:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.894998:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.895001:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.895002:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.895003:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.895005:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.895006:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.895018:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.895021:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.895022:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.895022:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.895023:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.895025:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.895390:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.895393:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.895394:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.895396:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.895397:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.895399:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.895466:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.895469:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.895470:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.895471:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.895473:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.895475:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.895794:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.895796:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.895797:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.895798:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.895800:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.895800:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.896099:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.896101:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.896102:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.896104:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.896106:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.896107:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.896120:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.896124:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.896124:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.896125:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.896126:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.896127:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.896543:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.896546:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.896546:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.896548:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.896550:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.896551:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.896859:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.896862:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.896862:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.896864:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.896866:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.896867:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.897120:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.897123:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.897124:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.897126:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.897127:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.897129:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.897338:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.897341:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.897341:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.897343:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.897345:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.897346:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.897356:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.897358:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.897359:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.897360:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.897361:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.897362:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.897742:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.897745:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.897746:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.897748:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.897749:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.897750:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.897810:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.897813:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.897814:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.897816:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.897817:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.897818:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.897831:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.897833:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.897834:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.897835:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.897836:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.897837:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.898375:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.898378:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.898379:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.898381:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.898382:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.898384:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.898416:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.898419:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.898420:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.898421:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.898422:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.898425:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.899745:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.899747:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.899748:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.899750:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.899752:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.899754:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.899804:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.899806:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.899808:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.899810:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677445.899811:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.899817:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.900809:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.900811:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.900812:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.900814:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.900816:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.900817:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.900824:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.900827:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.900828:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.900828:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.900829:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.900830:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.900924:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.900926:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.900927:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.900929:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.900930:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.900931:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.901417:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.901419:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.901420:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.901421:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.901423:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.901424:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.902151:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.902153:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.902154:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.902155:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.902158:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.902159:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.902625:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.902628:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.902629:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.902631:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.902632:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.902634:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.903089:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.903091:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.903092:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.903094:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.903095:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.903097:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.903111:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.903114:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.903114:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.903115:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.903116:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.903118:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.903619:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.903621:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.903622:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.903623:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.903625:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.903626:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.903635:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.903638:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.903638:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.903639:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.903639:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.903640:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.904169:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.904172:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.904173:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.904174:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.904175:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.904177:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.904488:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.904491:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.904491:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.904493:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.904495:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.904496:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.904664:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.904667:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.904668:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.904669:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.904670:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.904672:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.905144:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.905146:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.905147:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.905149:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.905151:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.905151:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.905286:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.905289:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.905290:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.905291:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.905293:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.905301:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.905302:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.905304:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.905305:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.905306:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.905306:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.905307:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.906267:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.906270:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.906271:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.906273:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.906274:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.906276:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.907003:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.907005:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.907006:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.907008:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.907010:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.907011:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.907135:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.907137:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.907138:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.907140:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.907141:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.907142:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.907278:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.907281:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.907282:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.907283:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677445.907285:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.907286:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.907510:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.907513:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.907514:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.907516:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.907517:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.907519:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.907531:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.907534:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.907535:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.907535:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.907536:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.907537:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.907728:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.907730:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.907731:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.907732:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.907734:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.907735:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.907874:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.907877:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.907878:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.907880:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.907881:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.907883:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.907892:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.907895:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.907896:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.907897:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.907897:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.907899:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.908248:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.908251:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.908251:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.908253:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.908255:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.908256:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.908497:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.908500:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.908501:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.908503:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.908504:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.908505:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.908888:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.908890:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.908891:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.908893:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.908894:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.908895:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.909530:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.909532:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.909533:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.909534:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.909536:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.909537:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.909549:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.909552:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.909553:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.909553:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.909554:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.909555:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.909716:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.909719:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.909720:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.909721:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.909723:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.909724:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.910127:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.910130:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.910131:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.910134:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.910134:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.910137:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.910441:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.910443:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.910444:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.910446:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.910448:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.910450:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.910669:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.910672:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.910672:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.910674:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.910676:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.910677:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.910689:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.910691:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.910692:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.910693:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.910693:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.910694:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.910853:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.910855:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.910855:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.910857:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.910859:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.910860:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.911451:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.911453:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.911454:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.911455:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.911457:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.911458:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.911863:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.911865:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.911866:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.911867:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.911869:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.911871:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.912528:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.912531:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.912532:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.912534:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.912535:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.912537:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.912544:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.912547:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.912548:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.912548:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.912549:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.912551:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.912731:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.912733:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.912734:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.912736:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.912737:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.912739:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.912812:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.912815:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.912816:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.912818:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.912819:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.912820:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.912965:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.912968:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.912969:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.912971:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677445.912972:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.912974:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.913113:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.913116:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.913116:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.913118:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.913120:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.913121:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.913506:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.913508:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.913509:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.913512:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.913513:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.913515:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.914056:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.914058:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.914059:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.914060:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.914062:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.914063:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.914331:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.914334:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.914335:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.914337:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.914338:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.914345:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.914346:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.914348:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.914349:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.914350:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.914350:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.914350:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.914765:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.914767:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.914768:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.914769:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.914771:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.914772:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.914786:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.914788:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.914789:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.914790:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.914790:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.914791:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.915125:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.915128:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.915130:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.915131:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.915132:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.915134:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.915146:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.915149:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.915149:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.915151:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.915152:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.915154:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.915490:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.915492:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.915493:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.915495:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.915497:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.915498:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.916535:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.916537:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.916538:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.916540:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.916542:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.916543:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.916641:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.916643:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.916644:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.916646:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.916647:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.916649:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.916657:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.916660:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.916661:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.916661:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.916662:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.916663:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.917287:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.917289:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.917291:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.917292:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.917293:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.917301:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.917652:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.917655:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.917656:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.917658:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.917659:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.917661:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.917980:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.917983:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.917984:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.917987:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.917987:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.917991:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.918078:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.918080:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.918081:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.918082:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.918084:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.918085:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.918259:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.918262:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.918263:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.918264:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.918266:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.918267:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.918876:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.918878:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.918879:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.918881:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.918882:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.918883:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.918895:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.918898:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.918899:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.918899:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.918900:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.918901:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.919100:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.919102:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.919103:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.919106:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677445.919106:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.919109:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.919125:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.919127:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.919128:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.919130:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.919132:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.919133:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.919143:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.919146:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.919146:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.919147:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.919148:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.919150:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.919164:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.919166:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.919167:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.919169:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.919170:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.919171:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.919272:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.919274:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.919275:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.919277:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.919278:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.919280:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.919658:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.919660:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.919661:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.919663:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.919665:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.919666:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.920822:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.920825:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.920826:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.920829:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.920830:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.920832:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.921109:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.921112:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.921113:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.921114:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.921115:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.921117:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.921127:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.921130:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.921131:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.921132:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.921132:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.921133:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.921251:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.921253:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.921254:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.921257:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.921258:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.921260:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.921778:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.921781:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.921782:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.921785:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.921786:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.921787:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.922095:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.922098:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.922099:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.922101:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.922102:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.922104:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.922328:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677445.922330:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677445.922331:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.922333:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.922335:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677445.922336:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677445.922336:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677445.922336:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.922337:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677445.922338:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.922341:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.922350:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.922352:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.922353:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.922354:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.922354:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.922356:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.922586:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.922589:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.922590:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.922591:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.922593:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.922594:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.922991:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.922994:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.922994:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.922996:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.922998:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.922998:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.923127:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.923130:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.923131:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.923133:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.923134:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.923136:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.923351:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.923354:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.923355:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.923358:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.923359:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.923361:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.923361:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.923362:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.923362:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.923363:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.923363:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.923462:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.923464:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.923465:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.923467:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.923468:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.923469:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.923481:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.923484:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.923484:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.923485:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.923486:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.923487:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.923764:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.923766:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.923767:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.923769:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.923770:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.923771:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.924028:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.924031:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.924032:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.924034:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.924035:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.924042:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.924062:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.924064:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.924065:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.924066:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.924068:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.924069:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.924141:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.924144:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.924144:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.924146:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.924148:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.924148:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.924779:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.924782:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.924784:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677445.924786:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677445.924787:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.924790:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.925221:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.925223:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.925224:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.925225:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.925227:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.925228:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.925332:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.925335:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.925336:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.925337:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.925340:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.925341:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.925342:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.925342:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.925342:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.925343:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.925344:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.925599:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.925601:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.925602:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.925604:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.925605:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.925606:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.925619:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.925622:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.925623:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.925624:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.925624:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.925626:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.925955:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.925957:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.925958:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.925960:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.925961:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.925962:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.926518:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.926521:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.926522:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.926524:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.926525:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.926527:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.926908:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.926910:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.926911:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.926913:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.926914:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.926916:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.927006:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.927008:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.927009:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.927011:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.927012:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.927014:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.927059:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.927061:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.927062:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.927064:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.927065:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.927066:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.927468:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.927471:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.927472:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.927473:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.927475:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.927477:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.928102:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.928104:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.928105:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.928107:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.928109:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.928110:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.928121:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.928123:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.928124:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.928125:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.928125:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.928126:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.928417:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.928420:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.928420:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.928422:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.928424:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.928425:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.928436:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.928438:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.928439:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.928440:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.928441:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.928442:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.928704:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.928706:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.928707:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.928708:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.928710:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.928711:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.928861:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.928864:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.928865:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.928868:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.928868:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.928871:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.928889:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.928891:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.928892:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.928894:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.928895:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.928896:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.929300:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.929302:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.929303:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.929305:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.929307:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.929308:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.929469:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.929472:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.929473:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.929475:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677445.929476:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.929478:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.929683:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.929686:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.929686:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.929689:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.929690:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.929691:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.929702:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.929705:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.929706:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.929706:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.929707:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.929709:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.929889:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.929892:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.929892:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.929895:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.929896:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.929897:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.930142:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.930145:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.930146:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.930147:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.930148:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.930150:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.930301:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.930303:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.930304:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.930305:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.930307:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.930308:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.930550:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.930552:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.930553:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.930556:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.930556:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.930559:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.930699:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.930701:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.930702:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.930704:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.930705:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.930706:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.930716:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.930718:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.930719:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.930720:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.930720:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.930721:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.930758:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.930760:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.930761:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.930762:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.930764:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.930765:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.930777:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.930779:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.930780:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.930781:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.930781:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.930783:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.931387:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.931389:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.931390:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.931392:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.931394:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.931395:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.932478:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.932481:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.932482:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.932483:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.932485:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.932486:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.933451:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677445.933454:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677445.933455:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.933457:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.933458:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677445.933459:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677445.933460:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677445.933460:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.933461:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677445.933462:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.933464:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.933475:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.933477:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.933478:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.933479:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.933479:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.933480:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.933630:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.933633:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.933634:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.933635:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.933636:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.933638:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.933910:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.933912:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.933914:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.933916:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.933917:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677445.933918:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.933929:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.933931:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.933932:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.933933:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.933935:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.933936:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.934242:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.934244:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.934245:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.934247:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.934248:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.934250:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.934260:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.934263:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.934264:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.934264:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.934265:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.934267:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.935087:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.935090:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.935091:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.935094:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.935095:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.935097:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.935589:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.935591:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.935592:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.935595:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.935596:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.935600:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.935618:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.935621:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.935622:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.935623:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.935624:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.935626:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.935779:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.935782:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.935783:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.935784:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.935786:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.935787:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.935970:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.935973:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.935973:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.935975:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.935976:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.935978:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.935988:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.935990:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.935990:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.935992:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.935994:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.935995:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.936005:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.936008:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.936009:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.936009:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.936010:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.936010:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.936298:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.936301:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.936301:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.936304:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.936306:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.936307:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.937160:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677445.937163:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677445.937166:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677445.937167:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677445.937170:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.937172:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677445.937174:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96749390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677445.937176:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677445.937178:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677445.937181:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677445.937187:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677445.937191:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677445.937194:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677445.937195:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:13.0:1619677445.937196:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677445.937197:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:13.0:1619677445.937198:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677445.937199:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677445.937199:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677445.937581:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.937584:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.937585:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.937587:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.937589:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.937590:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.938304:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.938306:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.938307:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.938309:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.938311:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.938311:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.938320:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.938322:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.938323:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.938324:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.938325:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.938326:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.939009:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.939012:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.939013:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.939015:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.939016:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.939018:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.939473:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.939476:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.939477:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.939478:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.939481:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.939481:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.939795:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.939798:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.939798:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.939800:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.939802:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.939803:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.939906:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.939909:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.939910:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.939912:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.939913:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.939914:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.940000:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.940003:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.940004:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.940006:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.940007:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.940008:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.940581:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.940584:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.940585:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.940587:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.940588:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.940589:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.940590:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.940590:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.940591:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.940592:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.940592:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.940890:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.940892:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.940893:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.940894:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.940896:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.940897:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.940940:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.940942:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.940943:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.940946:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677445.940946:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.940949:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.940958:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.940961:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.940962:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.940962:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.940963:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.940964:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.941349:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.941352:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.941353:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.941354:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.941356:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.941358:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.942017:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.942020:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.942020:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.942022:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.942024:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.942025:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.942408:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.942410:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.942411:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.942413:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.942415:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.942416:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.942623:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.942625:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.942626:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.942627:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.942629:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.942630:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.943041:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.943043:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.943044:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.943046:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.943047:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.943048:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.943967:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.943969:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.943970:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.943971:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.943974:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.943974:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.943985:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.943988:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.943989:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.943989:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.943990:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.943990:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.945119:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.945122:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.945122:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.945124:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.945126:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.945127:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.945142:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.945145:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.945145:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.945146:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.945147:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.945148:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.945289:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.945291:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.945292:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677445.945298:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677445.945300:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.945303:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.945357:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677445.945359:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677445.945360:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.945362:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.945363:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677445.945364:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677445.945364:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677445.945365:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677445.945366:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677445.945367:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.945369:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.945615:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.945618:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.945618:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.945620:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677445.945622:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.945623:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.945720:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.945722:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.945723:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.945725:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.945726:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.945727:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.945860:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.945862:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.945863:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.945865:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.945866:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.945867:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.945878:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.945880:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.945881:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.945882:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.945882:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.945883:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.946125:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.946128:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.946130:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.946131:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.946132:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.946134:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.946147:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.946150:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.946150:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.946152:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.946154:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.946154:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.946179:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.946181:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.946182:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.946184:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.946185:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.946186:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.946373:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.946375:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.946376:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677445.946376:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.946377:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.946378:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:12.0:1619677445.946378:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.946379:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.946381:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.946382:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.946383:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.946384:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.946385:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.946386:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.946386:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.946386:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.946387:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.946387:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.946389:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.946476:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.946478:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.946479:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.946480:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.946481:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.946482:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.946492:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.946495:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.946495:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.946496:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.946497:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.946498:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.946571:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.946573:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.946574:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.946576:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.946577:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.946579:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.946579:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.946582:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.946583:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.946584:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.946586:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.946587:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.946872:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.946874:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.946875:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.946877:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.946878:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.946880:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.947545:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.947547:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.947548:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.947550:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.947552:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.947553:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.948198:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.948201:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.948202:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.948203:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.948204:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.948206:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.948490:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.948493:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.948494:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.948496:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.948497:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.948499:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.948507:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.948510:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.948511:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.948511:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.948512:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.948513:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.948813:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.948816:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.948817:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.948819:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.948820:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.948821:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.949176:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.949178:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.949179:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.949181:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.949182:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.949184:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.949573:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.949576:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.949577:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.949579:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.949581:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.949581:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.949704:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.949706:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.949707:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.949709:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.949710:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.949712:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.949719:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.949722:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.949724:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.949725:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.949725:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.949726:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.950120:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.950122:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.950123:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.950125:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.950127:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.950128:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.950339:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.950342:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.950342:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.950345:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.950346:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.950347:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.950689:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.950692:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.950693:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.950694:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.950696:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.950697:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.950708:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.950711:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.950711:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.950712:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.950712:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.950714:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.950836:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.950839:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.950840:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.950842:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.950842:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.950844:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.950922:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.950924:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.950925:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:12.0:1619677445.950925:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.950926:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.950926:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.950927:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:13.0:1619677445.950928:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.950930:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.950931:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.950932:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.950933:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.951610:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677445.951612:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677445.951613:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.951615:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.951616:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677445.951617:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677445.951617:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677445.951618:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677445.951618:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677445.951620:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.951622:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.951632:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.951635:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.951636:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.951637:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.951637:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.951639:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.952136:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.952138:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.952139:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.952141:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.952142:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.952143:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.952462:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.952465:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.952465:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.952468:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.952470:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.952471:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.952479:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.952482:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.952483:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.952484:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.952485:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.952486:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.952653:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.952656:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.952657:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.952658:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.952660:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.952661:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.954150:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.954153:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.954154:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.954156:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.954158:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.954159:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.954608:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.954611:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.954611:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.954613:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.954615:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.954617:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.954728:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.954730:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.954731:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.954732:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.954734:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.954735:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.954960:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.954962:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.954963:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.954965:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.954966:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.954967:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.955135:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.955138:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.955138:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.955140:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.955142:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.955142:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.955208:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.955211:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.955212:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.955213:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.955214:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.955216:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.955225:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.955228:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.955229:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.955229:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.955230:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.955232:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.955644:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.955646:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.955647:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.955649:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.955651:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.955652:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.955709:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.955712:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.955713:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677445.955715:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677445.955717:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.955719:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.955728:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.955731:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.955731:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.955732:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.955732:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.955733:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.956254:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.956256:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.956257:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.956258:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.956261:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.956262:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.956917:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.956920:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.956921:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.956922:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.956924:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.956925:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.957224:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.957226:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.957227:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.957228:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.957230:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.957231:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.957241:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.957243:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.957244:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.957245:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.957245:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.957246:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.957550:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.957552:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.957553:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.957554:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.957557:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.957558:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.957705:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.957708:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.957709:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.957710:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.957712:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.957714:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.958000:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.958002:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.958003:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.958005:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.958006:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.958008:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.958179:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.958182:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.958182:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.958184:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.958185:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.958186:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.958333:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.958335:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.958336:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.958338:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.958340:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.958341:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.958355:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.958357:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.958358:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.958358:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.958359:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.958360:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.958687:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.958688:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.958689:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.958691:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.958692:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.958693:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.958693:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.958693:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.958694:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.958695:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677445.958695:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.958696:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.958780:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.958783:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.958783:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.958785:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.958786:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.958788:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.959962:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.959965:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.959966:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.959967:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.959969:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.959971:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.960141:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.960143:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.960145:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677445.960147:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677445.960148:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.960151:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.960841:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)++
00000800:00000200:12.0:1619677445.960844:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (25)--
00000400:00000200:12.0:1619677445.960845:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.960847:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.960849:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)++
00000800:00000200:12.0:1619677445.960849:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (25)--
00000800:00000200:13.0:1619677445.960850:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (25)--
00000400:00000200:13.0:1619677445.960851:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677445.960851:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677445.960852:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677445.960853:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:12.0:1619677445.960853:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.960855:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.960855:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.960856:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.960858:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.960858:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.960859:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677445.960859:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.960860:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.960861:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.961301:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.961303:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.961304:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.961306:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.961308:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.961309:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.961320:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.961323:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.961324:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.961325:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.961325:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.961327:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.961654:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.961656:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.961657:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.961659:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.961660:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.961661:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.961894:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.961897:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.961897:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.961899:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.961901:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.961901:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.961913:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.961915:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.961916:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.961917:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.961918:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.961919:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.963430:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.963433:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.963434:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.963435:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.963437:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.963438:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.963535:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.963538:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.963539:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.963541:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.963541:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.963544:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.963740:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.963742:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.963743:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.963744:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.963746:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.963747:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.963931:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.963933:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.963934:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.963936:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.963938:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.963938:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.964440:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.964444:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.964445:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.964446:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.964448:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.964456:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.964470:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.964472:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.964473:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:12.0:1619677445.964473:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.964474:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.964475:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.964475:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:13.0:1619677445.964477:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.964477:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.964478:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.964479:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.964480:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.964488:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.964490:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.964492:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.964492:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.964493:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.964494:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.964510:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.964512:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.964513:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.964515:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.964516:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.964517:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.964528:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.964530:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.964531:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.964532:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.964532:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.964533:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.964952:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.964954:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.964955:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.964956:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.964959:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.964959:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.965483:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.965486:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.965487:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.965489:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.965490:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.965492:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.965723:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.965726:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.965726:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.965728:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.965729:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.965730:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.967405:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.967407:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.967408:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.967409:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.967411:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.967412:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.967614:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.967617:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.967618:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.967620:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.967621:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.967623:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.968443:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.968446:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.968447:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.968459:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.968462:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[3] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.968464:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677445.968466:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677445.968467:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.968467:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.968483:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.968486:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.968487:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.968489:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: REPLY: OK
00000800:00000200:13.0:1619677445.968490:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.968491:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.968961:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.968964:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.968965:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.968967:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.968968:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.968970:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.969533:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.969536:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.969537:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.969539:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.969541:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.969542:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.969575:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.969577:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.969578:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.969579:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.969581:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.969581:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.969593:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.969595:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.969596:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.969596:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.969597:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.969598:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.970201:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.970204:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.970204:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.970206:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677445.970209:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.970210:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.970523:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.970525:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.970526:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.970528:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.970530:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.970531:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.971753:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.971756:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.971756:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.971758:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.971760:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.971762:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.971770:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.971772:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.971772:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.971774:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.971775:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.971777:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.971946:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.971949:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.971950:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.971953:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.971953:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.971956:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.971967:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.971970:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.971972:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.971972:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.971973:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.971974:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.972151:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.972154:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.972155:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.972158:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677445.972158:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.972160:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.972531:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.972533:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.972534:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.972536:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.972538:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.972540:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.973045:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.973048:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.973049:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.973051:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.973052:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.973054:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.973064:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.973067:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.973067:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.973068:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.973069:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.973070:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.974039:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.974041:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.974042:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.974043:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.974046:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.974047:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.974636:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.974638:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.974639:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.974641:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.974643:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.974644:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.974724:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.974727:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.974727:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.974729:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.974731:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.974732:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.975055:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.975058:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.975058:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.975060:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.975062:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.975063:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.975227:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.975229:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.975230:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.975232:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.975233:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.975235:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.975246:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.975249:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.975250:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.975251:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.975251:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.975253:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.975535:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.975538:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.975538:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.975540:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.975542:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.975543:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.975552:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.975555:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.975555:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.975556:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.975556:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.975557:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.975647:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.975649:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.975650:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.975651:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.975653:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.975654:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.975727:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.975730:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.975731:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.975732:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.975734:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.975735:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.976012:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.976014:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.976016:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.976017:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.976018:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.976020:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.976359:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.976362:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.976363:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.976365:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.976367:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.976368:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.976397:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.976399:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.976401:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.976402:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.976404:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.976405:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.976444:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.976447:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.976448:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.976449:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.976450:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.976452:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.976482:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.976485:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.976486:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.976487:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.976489:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.976490:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.976500:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.976502:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.976503:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.976504:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.976504:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.976506:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.976826:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.976829:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.976830:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.976831:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.976832:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.976834:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.976943:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.976946:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.976947:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.976950:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677445.976950:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.976953:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.976966:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.976968:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.976969:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.976971:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.976972:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.976973:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.976987:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.976990:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.976991:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.976991:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.976992:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.976993:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.977211:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.977213:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.977214:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.977217:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.977217:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.977219:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.977508:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.977511:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.977512:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.977514:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.977515:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.977517:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.977525:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.977527:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.977528:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.977529:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.977529:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.977531:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.977879:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.977881:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.977882:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.977884:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.977885:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.977888:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.978391:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.978393:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.978394:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.978396:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.978398:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.978399:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.978914:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.978917:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.978917:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.978919:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.978920:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.978922:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.979677:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.979680:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.979681:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.979683:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.979685:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.979685:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.980428:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.980430:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.980431:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.980433:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.980435:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.980436:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.980743:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.980746:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.980747:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.980748:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.980750:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.980751:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.980762:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.980765:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.980766:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.980766:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.980767:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.980769:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.981533:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.981536:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.981537:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.981539:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.981540:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.981541:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.981629:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.981632:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.981633:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.981636:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.981636:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.981638:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.981768:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.981770:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.981771:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.981773:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.981774:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.981775:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.981845:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.981846:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.981847:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677445.981847:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:13.0:1619677445.981849:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.981849:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.981850:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.981852:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.981853:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:12.0:1619677445.981853:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.981854:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.981858:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.981859:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.981861:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.981862:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.981862:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.981863:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.981864:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.982233:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677445.982235:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677445.982236:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.982238:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.982239:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677445.982240:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677445.982240:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677445.982241:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677445.982242:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677445.982243:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.982246:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.982419:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.982422:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.982422:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.982424:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.982426:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.982428:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.982695:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.982698:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.982698:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.982700:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.982702:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.982703:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.982712:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.982715:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.982716:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.982717:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.982717:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.982719:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.982837:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.982839:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.982840:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.982841:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.982843:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.982844:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.983801:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.983804:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.983805:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.983808:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.983809:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.983812:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.984028:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.984031:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.984032:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.984033:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.984035:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.984036:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.984044:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.984047:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.984048:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.984048:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.984049:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.984051:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.985428:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.985430:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.985431:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.985432:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.985435:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.985435:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.985704:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.985707:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.985707:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.985709:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.985711:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.985711:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.985819:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.985821:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.985822:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.985823:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.985825:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.985826:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.985987:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.985989:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.985990:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.985992:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.985993:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.985995:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.986010:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.986012:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.986013:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.986014:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.986015:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.986019:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.986377:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.986379:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.986380:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.986381:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.986383:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.986384:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.986464:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.986467:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.986468:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.986470:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.986471:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.986473:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.986939:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.986942:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.986943:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.986945:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.986946:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.986951:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.986953:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677445.986955:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.986956:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.986956:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677445.986957:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.986958:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.986966:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.986968:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.986969:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.986971:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.986972:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.986973:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.987068:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.987071:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.987072:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.987073:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.987074:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.987077:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.987085:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.987088:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.987088:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.987091:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.987091:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.987094:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.987414:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.987416:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.987417:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.987419:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.987421:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.987422:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.988161:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.988164:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.988165:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.988167:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677445.988168:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.988170:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.988180:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.988183:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.988184:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.988184:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.988185:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.988186:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.988283:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.988285:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.988286:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.988288:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.988289:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.988291:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.988336:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.988338:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.988339:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.988341:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.988343:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.988344:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.989702:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.989704:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.989705:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.989706:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.989708:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.989710:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.990679:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.990682:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.990683:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.990684:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.990686:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.990687:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.991003:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677445.991006:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677445.991008:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677445.991010:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677445.991011:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.991013:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:13.0:1619677445.991014:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96748605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677445.991015:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677445.991017:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677445.991018:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:13.0:1619677445.991020:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677445.991022:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:13.0:1619677445.991025:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677445.991026:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:13.0:1619677445.991026:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:13.0:1619677445.991028:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:13.0:1619677445.991029:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677445.991030:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677445.991032:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677445.991032:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677445.991046:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677445.991048:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677445.991049:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677445.991050:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677445.991419:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.991422:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.991423:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.991424:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.991427:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.991428:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.991439:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677445.991442:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.991443:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.991443:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.991444:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.991446:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.991972:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.991974:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.991975:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.991977:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.991978:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.991979:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.992421:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.992423:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.992424:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.992425:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.992427:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.992428:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.992439:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.992442:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.992443:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.992444:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.992444:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.992446:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.992637:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.992640:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.992641:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.992642:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.992644:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.992645:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.992973:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.992976:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.992977:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.992979:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.992980:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.992981:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.993010:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)++
00000800:00000200:12.0:1619677445.993013:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677445.993014:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.993016:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.993017:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677445.993018:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677445.993019:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677445.993020:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.993021:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.993021:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.993024:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.993324:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.993327:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.993328:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.993330:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.993331:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.993333:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.993333:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.993334:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.993334:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.993334:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.993335:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.994057:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.994060:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.994060:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.994062:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.994064:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.994065:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.994123:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.994125:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.994126:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.994127:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.994129:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.994129:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.994182:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.994184:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.994185:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.994186:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.994188:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.994189:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.994639:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.994641:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.994642:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.994644:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.994645:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.994647:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.994773:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.994775:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.994776:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.994777:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.994779:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.994780:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.994789:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677445.994791:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677445.994792:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.994793:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677445.994793:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.994794:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.994882:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.994885:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.994886:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.994888:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.994889:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.994890:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.995444:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.995447:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.995447:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.995449:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.995451:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.995452:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.995529:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.995532:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.995533:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.995534:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.995536:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.995536:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.995554:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.995557:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.995558:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.995558:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.995559:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.995560:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.995707:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.995710:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.995711:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.995712:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.995713:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.995715:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.995763:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.995765:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.995766:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.995768:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.995769:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.995770:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.996159:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.996161:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.996162:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.996164:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.996165:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.996166:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.996602:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.996604:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.996605:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.996607:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.996609:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.996611:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677445.996612:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677445.996612:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.996613:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.996613:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677445.996614:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.996980:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.996982:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.996983:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.996985:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.996986:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.996987:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.2:1619677445.997013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (20)++
00000800:00000200:12.0:1619677445.997019:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.997022:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.997023:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.997024:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.997026:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.997035:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.997051:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.997053:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.997054:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.997055:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.997057:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.997058:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.997127:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.997130:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.997131:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.997133:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.997134:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.997135:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.997149:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677445.997152:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677445.997153:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.997154:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677445.997154:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.997156:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.997441:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677445.997444:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677445.997445:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.997446:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.997449:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677445.997451:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677445.998335:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.998338:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.998339:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.998341:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.998343:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.998344:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.998833:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.998835:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.998836:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.998838:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.998840:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.998841:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.999199:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677445.999202:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677445.999203:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.999205:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677445.999205:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677445.999208:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677445.999219:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677445.999222:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677445.999222:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.999223:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677445.999224:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.999225:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677445.999606:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677445.999608:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677445.999609:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677445.999611:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677445.999613:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677445.999614:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677445.999847:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677445.999849:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677445.999850:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677445.999852:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677445.999853:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677445.999855:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.000078:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.000081:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.000082:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.000083:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.000085:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.000086:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.000096:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.000098:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.000099:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.000099:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.000100:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.000102:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.000514:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.000517:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.000518:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.000519:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.000521:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.000522:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.000571:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.000573:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.000574:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.000575:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.000577:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.000578:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.000852:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.000855:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.000856:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.000857:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.000859:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.000860:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.001290:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.001292:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.001293:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.001299:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.001300:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.001302:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.001309:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.001312:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.001313:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.001314:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.001314:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.001315:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.001687:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.001690:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.001690:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.001692:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.001694:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.001694:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.001780:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.001783:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.001784:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.001786:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.001787:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.001789:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.002341:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.002344:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.002345:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.002347:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.002348:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.002350:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.002380:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.002382:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.002383:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.002385:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.002386:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.002388:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.002504:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.002507:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.002508:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.002510:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.002511:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.002513:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.002522:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.002524:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.002525:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.002526:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.002526:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.002528:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.002557:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.002560:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.002561:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.002563:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.002563:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.002565:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.002771:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.002773:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.002774:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.002776:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.002777:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.002779:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.003147:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.003150:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.003151:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.003153:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.003154:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.003156:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.003167:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.003169:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.003170:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.003171:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.003171:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.003173:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.003585:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.003587:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.003588:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.003590:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.003592:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.003593:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.004516:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.004519:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.004519:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.004521:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.004524:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.004526:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.004545:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.004548:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.004549:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.004550:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.004551:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.004553:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.005302:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.005305:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.005306:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.005308:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.005310:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.005311:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.005515:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.005517:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.005518:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.005520:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.005521:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.005522:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.005532:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.005535:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.005536:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.005536:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.005537:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.005537:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.007373:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.007375:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.007376:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.007378:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.007380:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.007381:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.007850:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.007853:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.007854:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.007856:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.007856:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.007859:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.007944:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.007947:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.007948:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.007949:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.007951:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.007952:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.007952:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.007955:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.007956:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.007958:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.007960:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.007960:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.007961:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.007964:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.007965:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.007965:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.007966:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.007968:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.008000:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.008003:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.008003:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.008005:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.008007:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.008008:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.008118:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.008120:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.008121:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.008123:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.008124:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.008126:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.008518:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.008519:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.008519:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:13.0:1619677446.008520:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.008521:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.008522:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:12.0:1619677446.008522:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.008525:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.008525:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.008526:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.008527:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.008528:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.008850:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.008853:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.008854:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.008856:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.008857:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.008858:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.008868:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.008871:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.008871:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.008872:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.008872:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.008873:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.009068:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.009071:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.009072:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.009074:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.009075:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.009077:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.009087:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.009090:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.009091:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.009092:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.009093:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.009094:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.009575:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.009578:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.009579:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.009581:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.009583:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.009584:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.009585:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.009588:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.009589:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.009590:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.009591:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.009593:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.010249:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.010251:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.010252:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.010254:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.010255:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.010256:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.010601:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.010603:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.010604:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.010605:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.010607:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.010608:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.010925:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.010927:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.010928:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.010930:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.010931:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.010932:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.010943:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.010946:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.010947:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.010947:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.010948:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.010949:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.011638:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.011640:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.011641:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.011643:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.011645:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.011646:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.012285:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.012288:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.012289:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.012290:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.012292:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.012299:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.012871:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.012873:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.012874:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.012876:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.012877:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.012879:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.012890:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.012893:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.012894:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.012895:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.012895:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.012897:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.013081:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.013083:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.013084:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.013086:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.013087:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.013091:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.013244:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.013247:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.013247:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.013249:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.013251:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.013252:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.013406:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.013409:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.013410:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.013412:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.013414:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.013416:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.013514:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.013517:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.013518:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.013519:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.013520:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.013522:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.013914:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.013917:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.013918:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.013920:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.013921:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.013922:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.014241:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.014243:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.014244:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.014246:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.014247:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.014248:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.014326:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.014329:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.014329:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.014331:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.014334:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.014334:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.014348:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.014352:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.014353:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.014353:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.014354:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.014355:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.014412:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.014415:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.014417:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.014419:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.014419:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.014422:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.014430:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.014433:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.014434:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.014435:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.014435:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.014437:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.014510:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.014513:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.014514:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.014516:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.014516:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.014518:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.014890:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.014892:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.014893:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.014894:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.014896:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.014897:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.014981:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.014983:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.014985:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.014986:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.014987:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.014989:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.015342:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.015345:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.015346:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.015347:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.015349:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.015350:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.015590:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.015592:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.015593:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.015595:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.015596:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.015597:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.015901:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.015904:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.015904:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.015906:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.015908:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.015909:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.015910:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.015912:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.015913:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.015915:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.015916:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.015918:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.015918:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.015921:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.015922:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.015922:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.015923:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.015924:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.016226:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.016228:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.016229:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.016230:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.016232:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.016233:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.016241:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.016244:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.016245:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.016245:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.016246:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.016247:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.017034:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.017037:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.017038:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.017040:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.017042:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.017043:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.017255:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.017257:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.017258:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.017260:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677446.017261:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.017263:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.017710:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.017713:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.017714:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.017715:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.017717:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.017718:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.017865:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.017868:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.017869:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.017870:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.017872:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.017874:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.018111:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.018114:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.018115:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.018117:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.018118:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.018119:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.018422:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.018424:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.018425:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.018427:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.018429:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.018430:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.018438:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.018441:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.018441:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.018442:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.018442:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.018443:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.018610:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.018612:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.018613:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.018615:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.018616:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.018617:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.018800:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.018803:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.018804:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.018805:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.018807:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.018808:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.019046:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.019048:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.019049:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.019051:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.019052:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.019054:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.019336:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.019338:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.019339:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.019341:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.019343:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.019344:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.019515:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.019517:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.019518:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.019520:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.019521:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.019522:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.019536:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.019539:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.019540:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.019540:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.019541:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.019543:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.019746:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.019748:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.019749:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.019751:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.019752:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.019753:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.019763:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.019766:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.019767:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.019767:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.019768:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.019769:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.019935:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.019937:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.019938:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.019939:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.019941:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.019942:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.020194:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.020197:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.020198:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.020200:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.020201:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.020202:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.020545:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.020548:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.020549:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.020550:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.020552:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.020553:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.020588:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.020590:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.020591:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.020593:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.020595:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.020595:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.021144:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.021146:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.021147:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.021149:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.021151:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.021152:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.021165:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.021168:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.021168:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.021169:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.021170:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.021170:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.021179:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.021181:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.021182:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.021184:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.021185:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.021186:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.021342:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.021345:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.021346:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.021347:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.021349:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.021350:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.021863:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.021865:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.021866:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.021868:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.021869:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.021873:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.021890:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.021892:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.021893:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.021896:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.021896:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.021898:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.022416:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.022418:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.022419:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.022420:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.022423:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.022424:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.022466:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.022469:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.022470:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.022472:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.022473:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.022474:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.022486:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.022489:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.022490:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.022491:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.022492:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.022493:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.023460:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.023462:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.023463:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.023465:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.023467:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.023468:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.023820:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.023822:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.023823:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.023825:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.023826:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.023828:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.023838:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.023841:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.023842:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.023842:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.023843:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.023844:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.023987:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.023989:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.023990:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.023992:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.023993:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.023995:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.024038:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.024040:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.024041:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.024043:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.024044:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.024046:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.024292:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.024301:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.024301:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.024302:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.024304:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.024307:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.024339:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.024341:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.024342:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.024344:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.024345:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.024348:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.024710:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.024713:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.024714:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.024715:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.024717:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.024718:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.024730:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.024733:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.024734:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.024735:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.024735:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.024737:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.024831:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.024833:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.024834:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.024836:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.024837:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.024838:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.025016:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.025019:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.025020:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.025022:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.025023:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.025025:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.025108:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.025111:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.025112:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.025114:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.025115:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.025116:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.025518:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.025521:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677446.025522:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.025523:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.025525:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.025526:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677446.025526:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677446.025527:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.025527:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.025529:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.025531:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.025540:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.025543:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.025543:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.025544:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.025545:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.025546:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.025929:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.025932:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.025932:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.025934:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.025936:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.025937:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.026509:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.026511:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.026512:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.026513:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.026516:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.026517:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.026531:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.026534:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.026535:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.026535:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.026536:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.026537:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.026742:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677446.026745:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677446.026745:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.026747:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.026749:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677446.026750:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677446.026750:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677446.026751:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.026752:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677446.026753:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.026754:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.026917:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.026920:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.026921:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.026924:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.026924:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.026928:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.027248:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.027250:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.027251:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.027252:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.027254:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.027255:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.027302:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.027305:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.027306:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.027307:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.027309:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.027310:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.027318:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.027321:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.027322:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.027323:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.027323:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.027325:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.027968:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.027970:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.027971:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.027973:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.027974:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.027975:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.028064:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677446.028067:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677446.028069:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677446.028071:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677446.028072:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.028073:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:13.0:1619677446.028075:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96748069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677446.028076:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677446.028077:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677446.028078:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:13.0:1619677446.028082:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677446.028084:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677446.028087:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677446.028088:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:13.0:1619677446.028089:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:13.0:1619677446.028092:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:13.0:1619677446.028092:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677446.028093:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677446.028094:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677446.029178:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.029181:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.029182:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.029183:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.029185:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.029186:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.029196:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.029199:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.029200:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.029200:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.029201:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.029202:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.029632:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.029634:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.029635:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.029637:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.029639:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.029640:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.029807:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.029810:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.029811:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.029813:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.029814:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.029815:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.030036:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.030038:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.030039:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.030040:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.030042:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.030043:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.030245:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.030246:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.030247:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.030247:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:13.0:1619677446.030249:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.030249:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.030250:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.030252:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.030253:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.030254:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.030255:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.030256:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.030256:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.030257:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.030257:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.030258:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.030259:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.030616:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.030618:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.030619:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.030621:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.030622:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.030623:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.030748:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.030750:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.030751:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.030753:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.030754:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.030756:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.030973:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.030976:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.030977:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.030978:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.030980:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.030980:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.031509:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.031511:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.031512:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.031514:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.031516:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.031517:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.031747:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.031750:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.031751:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.031752:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677446.031754:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.031756:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.031875:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.031877:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.031878:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.031880:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.031881:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.031883:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.031893:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.031896:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.031897:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.031897:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.031898:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.031900:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.032074:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.032076:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.032077:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.032079:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.032080:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.032082:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.032094:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.032097:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.032098:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.032098:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.032099:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.032100:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.032795:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.032798:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.032798:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.032800:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.032802:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.032804:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.033451:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.033454:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.033455:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.033457:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.033459:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.033460:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.034445:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.034448:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.034449:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.034451:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.034453:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.034454:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.034825:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.034828:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.034829:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.034831:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.034832:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.034833:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.035235:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.035237:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.035239:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.035241:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.035241:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.035243:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.035290:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.035292:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.035299:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.035300:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.035301:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.035304:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.036695:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.036698:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.036699:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.036701:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.036702:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.036703:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.036714:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.036717:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.036718:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.036718:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.036719:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.036720:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.037077:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.037079:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.037080:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.037081:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.037083:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.037084:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.037094:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.037096:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.037097:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.037098:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.037098:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.037099:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.037288:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.037290:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (25)--
00000400:00000200:12.0:1619677446.037291:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.037293:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.037297:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.037302:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677446.037303:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.037303:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.037304:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.037304:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.037306:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.037465:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.037467:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.037468:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.037469:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.037471:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.037473:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.037602:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.037604:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.037605:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.037606:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.037608:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.037609:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.038061:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.038063:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.038064:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.038067:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.038067:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.038070:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.038079:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.038081:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.038082:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.038083:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.038083:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.038085:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.038215:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.038218:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.038219:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.038220:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.038222:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.038223:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.038454:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.038457:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.038458:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.038459:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.038461:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.038463:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.038961:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.038964:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.038965:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.038967:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.038968:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.038970:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.039365:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.039367:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.039368:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.039369:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.039371:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.039373:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.039684:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.039687:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.039688:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.039690:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.039691:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.039693:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.039763:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.039765:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.039766:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.039768:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.039769:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.039772:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.039782:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.039786:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.039787:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.039787:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.039788:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.039789:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.039977:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.039979:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.039980:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.039982:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.039982:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.039983:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.039984:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.039984:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.039985:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:12.0:1619677446.039986:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.039987:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.039991:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.039993:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.039995:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.039995:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.039996:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.039997:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.039997:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.040170:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.040172:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.040173:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.040174:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.040176:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.040177:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.040877:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.040879:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.040880:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.040882:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.040884:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.040885:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.040917:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.040920:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.040921:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.040923:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.040923:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.040925:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.040958:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.040961:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.040961:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.040963:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.040965:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.040966:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.041215:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.041218:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.041218:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.041220:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.041221:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.041223:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.041235:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.041237:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.041238:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.041239:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.041239:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.041241:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.041879:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.041881:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.041882:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.041884:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.041886:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.041886:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.042512:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.042515:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.042516:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.042517:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.042519:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.042520:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.042870:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.042873:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.042874:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.042876:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.042877:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.042879:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.042956:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.042959:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.042960:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.042963:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.042963:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.042965:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.042979:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)++
00000800:00000200:13.0:1619677446.042982:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677446.042983:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.042984:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.042985:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677446.042987:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.042987:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677446.042987:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.042988:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.042989:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.042991:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.042999:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.043002:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.043003:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.043004:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.043004:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.043006:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.043273:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.043275:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.043276:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.043279:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.043279:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.043282:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.043289:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.043292:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.043297:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.043299:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.043300:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.043301:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.043388:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.043391:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.043392:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.043393:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.043395:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.043396:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.043663:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.043666:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.043667:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.043668:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.043670:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.043671:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.044504:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.044507:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.044508:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.044509:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.044511:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.044512:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.044658:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.044660:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.044661:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.044663:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.044664:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.044665:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.045322:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.045325:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.045328:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.045329:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.045330:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.045332:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.045344:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.045346:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.045347:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.045349:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.045350:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.045351:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.045363:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.045365:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.045366:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.045367:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.045367:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.045368:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.045553:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.045556:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.045557:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.045559:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.045560:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.045562:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.045570:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.045573:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.045574:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.045574:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.045575:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.045576:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.045890:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.045892:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.045893:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.045895:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.045896:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.045897:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.045930:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.045932:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.045933:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.045934:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.045936:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.045937:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.2:1619677446.046077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (20)++
00000800:00000200:12.0:1619677446.046083:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.046086:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.046087:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.046088:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.046090:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.046091:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.046131:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.046134:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.046135:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.046136:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.046137:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.046139:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.046199:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.046201:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.046202:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.046204:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.046206:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.046207:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.046208:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.046209:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.046209:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.046210:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.046211:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.046214:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.046569:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.046571:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.046572:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.046575:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.046576:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.046578:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.046586:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.046590:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.046591:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.046592:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.046592:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.046593:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.046613:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.046615:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.046617:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.046619:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.046619:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.046622:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.046634:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.046637:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.046638:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.046639:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.046640:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.046641:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.047055:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.047057:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.047058:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.047060:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.047061:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.047063:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.047138:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.047141:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.047141:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.047143:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.047145:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.047146:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.047384:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.047386:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.047387:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.047389:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.047391:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.047392:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.047875:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.047877:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.047878:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.047880:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.047881:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.047882:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.048742:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.048744:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.048745:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.048746:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.048749:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.048750:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.049117:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.049119:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.049120:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.049122:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.049123:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.049124:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.049260:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.049263:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.049263:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.049265:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.049267:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.049267:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.049278:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.049281:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.049282:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.049283:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.049283:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.049285:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.049974:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.049977:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.049978:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.049979:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.049981:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.049982:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.050225:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.050228:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.050229:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.050231:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.050232:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.050234:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.050273:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.050274:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.050275:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.050276:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:13.0:1619677446.050277:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.050277:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.050278:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.050279:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.050279:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.050280:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.050281:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.050282:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.050300:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.050302:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.050303:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.050304:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.050304:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.050305:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.050411:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.050414:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.050414:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.050416:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.050418:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.050419:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.050533:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.050536:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.050537:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.050539:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.050540:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.050544:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.051020:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.051023:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.051024:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.051026:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.051027:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.051029:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.051038:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.051041:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.051042:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.051043:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.051043:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.051045:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.051652:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.051655:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.051656:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.051658:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.051659:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.051661:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.052176:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.052178:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.052179:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.052181:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.052182:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.052184:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.052505:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.052507:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.052508:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.052510:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.052512:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.052513:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.052521:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.052524:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.052524:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.052525:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.052526:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.052527:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.052734:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.052737:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.052738:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.052739:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.052741:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.052742:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.052788:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.052791:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.052792:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.052793:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.052795:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.052797:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.052932:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.052935:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.052936:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.052937:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.052938:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.052940:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.053741:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.053744:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.053745:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.053747:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.053749:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.053750:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.054029:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.054032:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.054032:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.054034:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.054036:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.054037:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.054552:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.054555:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.054556:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.054557:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.054559:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.054560:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.054572:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.054575:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.054576:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.054576:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.054577:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.054578:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.054633:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.054636:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.054637:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.054638:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.054640:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.054642:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.054958:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.054960:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.054961:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.054963:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.054964:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.054965:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.055111:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.055113:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.055114:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.055116:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.055118:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.055119:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.055434:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.055437:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.055438:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.055439:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.055442:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.055443:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.055451:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.055455:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.055455:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.055456:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.055457:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.055458:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.055815:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.055818:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.055818:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.055820:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.055822:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.055823:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.055839:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.055842:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.055843:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.055843:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.055844:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.055845:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.055992:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.055995:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.055996:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.055997:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.055998:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.056000:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.056029:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.056032:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.056034:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.056035:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.056036:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.056038:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.056288:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.056291:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.056292:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.056296:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677446.056299:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.056302:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.056376:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.056378:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.056379:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.056380:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.056382:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.056383:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.056633:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.056636:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.056637:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.056638:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.056640:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.056641:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.057263:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.057266:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.057267:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.057269:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.057270:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.057272:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.057285:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.057287:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.057288:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.057289:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.057290:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.057290:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.057491:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.057493:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.057494:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.057496:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.057498:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.057499:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.057611:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.057614:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.057614:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.057616:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.057617:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.057619:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.057678:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.057681:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.057682:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.057684:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.057685:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.057686:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.057696:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.057699:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.057700:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.057701:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.057701:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.057703:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.058205:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.058207:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.058208:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.058209:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.058211:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.058212:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.058817:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.058819:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.058821:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.058823:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.058824:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.058826:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.058946:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.058948:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.058949:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.058950:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.058952:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.058954:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.059158:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.059160:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.059161:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.059163:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.059165:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.059165:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.059179:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.059182:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.059182:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.059183:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.059183:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.059185:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.059462:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.059465:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.059466:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.059468:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.059470:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.059471:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.059862:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.059865:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.059865:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.059867:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.059868:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.059869:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.059999:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.060002:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.060002:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.060004:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.060006:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.060006:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.060572:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.060574:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.060575:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.060576:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.060578:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.060579:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.060858:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.060861:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.060861:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.060863:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.060864:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.060866:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.061122:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.061125:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.061126:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.061128:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.061129:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.061131:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.061142:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.061144:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.061145:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.061146:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.061147:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.061148:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.061221:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.061223:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.061224:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.061226:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.061227:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.061229:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.061495:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.061498:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.061498:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.061500:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.061502:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.061503:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.061955:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.061958:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.061959:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.061961:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.061962:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.061965:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.062162:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.062165:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.062166:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.062168:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.062169:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.062175:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.062175:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.062177:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.062178:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.062181:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.062182:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.062183:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.062192:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.062194:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.062195:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.062197:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.062198:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.062199:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.062548:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.062550:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.062551:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.062553:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.062555:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.062557:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.062806:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.062809:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.062810:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.062812:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.062813:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.062815:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.062826:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.062829:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.062830:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.062830:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.062831:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.062832:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.063161:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.063164:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.063165:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.063166:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.063168:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.063169:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.064266:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.064268:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.064268:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.064270:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.064271:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.064272:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.064273:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.064275:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.064277:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.064278:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.064279:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.064279:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.064851:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.064853:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.064854:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.064856:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.064858:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.064859:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.065323:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.065325:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.065326:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.065328:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.065330:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.065330:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.065342:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.065343:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.065344:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.065344:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:13.0:1619677446.065345:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.065346:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.065346:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.065347:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.065348:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.065349:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.065350:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.065350:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.065351:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.065351:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.065352:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677446.065352:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.065353:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.065355:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.065356:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.065748:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.065750:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.065751:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.065752:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.065754:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.065755:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.065796:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.065799:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.065800:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.065801:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.065802:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.065804:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.065915:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.065917:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.065918:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.065920:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.065921:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.065923:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.066671:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.066674:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.066675:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.066676:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.066678:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.066679:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.066767:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.066769:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.066770:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.066772:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.066774:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.066775:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.066842:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.066844:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.066845:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.066847:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.066848:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.066850:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.067683:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.067685:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.067686:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.067688:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.067690:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.067692:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.067701:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.067704:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.067705:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.067706:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.067706:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.067708:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.067862:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.067865:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.067866:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.067867:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.067868:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.067869:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.068350:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.068353:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.068354:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.068356:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.068358:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.068359:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.068488:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.068491:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.068492:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.068494:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.068495:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.068496:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.069281:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.069286:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (25)--
00000400:00000200:13.0:1619677446.069287:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.069289:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.069289:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.069293:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.069301:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)++
00000800:00000200:13.0:1619677446.069303:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677446.069304:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.069306:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.069307:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.069308:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677446.069309:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.069310:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677446.069310:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.069311:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (25)--
00000400:00000200:13.0:1619677446.069311:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.069311:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (25)--
00000800:00000200:12.0:1619677446.069311:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677446.069313:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677446.069314:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.069315:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.069587:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.069589:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.069590:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.069591:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.069593:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.069594:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.069604:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.069607:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.069608:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.069608:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.069609:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.069610:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.069948:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.069950:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.069951:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.069952:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.069954:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.069955:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.070846:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.070848:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.070849:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.070850:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.070852:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.070853:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.070920:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.070923:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.070924:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.070926:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.070927:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.070928:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.072330:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.072332:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.072333:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.072334:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.072336:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.072337:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.072341:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.072341:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.072342:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.072343:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.072344:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.073269:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.073272:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.073273:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.073275:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.073276:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.073278:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.074114:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.074116:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.074117:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.074119:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.074120:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.074122:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.074247:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.074250:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.074251:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.074252:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.074254:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.074255:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.075109:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.075112:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.075113:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.075114:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.075116:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.075117:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.075128:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.075130:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.075131:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.075132:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.075132:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.075133:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.075658:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.075661:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.075661:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.075663:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.075664:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.075665:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.075776:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.075780:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.075781:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.075783:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.075784:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.075785:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.075860:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.075863:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.075865:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.075867:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.075867:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.075869:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.075878:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.075881:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.075882:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.075883:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.075883:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.075885:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.075902:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.075905:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.075906:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.075907:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.075909:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.075909:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.076260:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.076262:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.076263:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.076264:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.076266:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.076267:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.077469:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.077472:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.077472:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.077476:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.077476:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.077478:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.079679:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.079682:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.079683:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.079685:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.079686:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.079687:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.079697:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.079700:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.079701:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.079702:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.079702:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.079703:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.079762:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.079765:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.079766:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.079768:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.079768:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.079771:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.080206:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.080209:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.080210:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.080212:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.080213:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.080214:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.080378:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.080380:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.080382:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.080383:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.080385:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.080386:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.080386:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.080387:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.080388:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.080388:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677446.080388:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.080389:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.080390:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.080602:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.080604:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.080605:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.080606:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.080608:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.080609:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.080665:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.080667:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.080668:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.080669:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.080671:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.080672:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.081329:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.081332:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.081332:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.081334:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.081336:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.081337:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.081665:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.081668:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.081669:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.081670:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.081672:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.081673:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.081788:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.081791:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.081792:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.081793:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.081795:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.081796:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.081813:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.081815:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.081816:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.081816:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.081817:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.081818:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.082221:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.082223:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.082224:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.082226:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.082227:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.082228:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.082296:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.082303:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.082304:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.082304:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677446.082306:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.082308:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.082549:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.082553:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.082554:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.082556:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.082557:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.082559:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.084300:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.084304:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.084305:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.084306:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.084307:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.084309:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.084309:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.084310:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677446.084311:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.084311:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.084311:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.084530:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.084532:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.084533:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.084535:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.084536:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.084538:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.085011:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.085013:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.085014:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.085016:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.085017:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.085019:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.085443:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.085446:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.085447:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.085450:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.085450:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.085452:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.085465:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.085468:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.085469:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.085469:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.085470:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.085471:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.085967:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.085970:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.085970:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.085972:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.085973:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.085975:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.086093:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.086095:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.086096:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.086097:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.086099:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.086104:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.086120:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.086123:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.086123:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.086125:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.086126:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.086127:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.086361:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.086363:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.086364:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.086365:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.086368:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.086369:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.086382:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.086386:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.086387:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.086387:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.086388:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.086389:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.086491:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.086494:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.086495:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.086497:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.086498:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.086500:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.086556:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.086558:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.086559:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.086560:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.086562:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.086563:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.086940:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.086943:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.086944:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.086946:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.086947:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.086949:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.086965:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.086967:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.086968:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.086969:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.086971:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.086972:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.086985:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.086987:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.086988:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.086989:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.086989:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.086990:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.087074:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.087076:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.087077:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.087080:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.087081:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.087082:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.088075:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.088077:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.088078:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.088080:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.088082:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.088083:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.088285:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.088288:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.088288:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.088290:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.088292:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.088293:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.088758:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.088761:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.088762:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.088764:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.088765:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.088767:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.089316:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.089318:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.089319:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.089321:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.089324:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.089325:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.089326:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.089328:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.089329:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.089331:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.089333:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.089334:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.089346:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.089349:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.089350:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.089350:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.089351:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.089352:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.089689:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.089692:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.089693:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.089695:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.089695:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.089697:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.089973:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.089976:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.089977:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.089979:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.089980:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.089982:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.090253:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.090255:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.090256:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.090258:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.090259:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.090261:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.090368:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.090371:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.090372:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.090374:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.090375:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.090377:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.091114:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.091116:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.091117:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.091118:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.091120:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.091121:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.091132:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.091134:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.091135:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.091136:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.091137:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.091137:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.091341:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.091344:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.091345:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.091346:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.091349:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.091349:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.091361:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.091364:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.091365:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.091365:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.091366:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.091367:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.091392:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.091394:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.091395:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.091397:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.091398:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.091400:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.091756:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.091759:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.091759:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.091761:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.091763:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.091764:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.092166:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.092168:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.092169:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.092170:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.092172:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.092173:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.092427:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.092429:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.092430:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.092431:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.092433:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.092434:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.092815:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.092818:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.092818:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.092820:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.092821:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.092823:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.093196:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.093199:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.093200:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.093201:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.093203:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.093204:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.093894:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.093897:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.093898:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.093899:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.093901:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.093902:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.093917:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.093920:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.093921:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.093922:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.093922:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.093924:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.094463:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.094465:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.094466:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.094468:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.094470:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.094470:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.094680:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.094683:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.094684:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.094686:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.094687:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.094689:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.094912:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.094914:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.094916:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.094917:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.094918:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.094920:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.094937:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.094940:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.094941:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.094941:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.094942:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.094943:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.095288:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.095290:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.095291:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.095293:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.095295:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.095300:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.096253:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.096256:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.096256:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.096258:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.096260:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.096261:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.096977:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.096979:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.096980:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.096982:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.096984:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.096985:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.097188:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.097190:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.097191:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.097193:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.097194:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.097195:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.097204:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.097206:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.097207:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.097208:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.097208:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.097209:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.097879:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.097881:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.097882:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.097884:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.097886:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.097887:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.097888:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.097890:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.097891:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.097892:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.097893:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.097894:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.097996:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.097999:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.098000:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.098002:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.098003:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.098004:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.098294:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.098301:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.098302:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.098303:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.098304:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.098306:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.098659:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.098661:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.098662:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.098664:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.098665:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.098667:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.098677:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.098680:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.098681:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.098681:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.098682:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.098683:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.098801:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.098803:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.098804:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.098805:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.098807:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.098808:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.098825:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.098828:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.098829:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.098830:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.098830:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.098832:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.098924:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.098927:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.098928:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.098930:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.098931:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.098932:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.099629:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.099632:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.099633:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.099634:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.099637:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.099638:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.099771:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.099773:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.099774:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.099776:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.099777:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.099779:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.099981:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.099984:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.099985:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.099987:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.099988:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.099991:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.100022:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.100025:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.100026:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.100028:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.100029:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.100031:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.100040:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.100043:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.100044:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.100045:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.100045:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.100046:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.100554:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.100557:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.100558:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.100561:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.100561:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.100565:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.101334:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.101337:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.101338:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.101340:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.101342:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.101342:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.101674:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.101677:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.101678:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.101679:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.101680:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.101682:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.101690:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.101692:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.101693:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.101694:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.101694:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.101695:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.102115:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.102118:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.102119:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.102120:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.102122:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.102123:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.102238:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.102240:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.102241:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.102243:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.102244:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.102245:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.102269:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.102271:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.102272:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.102274:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.102275:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.102277:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.102343:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.102345:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.102346:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.102348:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.102350:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.102350:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.102826:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.102828:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.102829:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.102831:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.102832:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.102834:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.102844:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.102847:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.102848:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.102848:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.102849:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.102851:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.103323:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.103326:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.103326:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.103328:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.103331:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.103332:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.103712:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.103715:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.103715:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.103717:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.103719:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.103719:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.104433:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.104436:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.104437:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.104439:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.104440:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.104442:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.104453:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.104456:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.104457:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.104458:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.104458:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.104460:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.104913:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.104915:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.104916:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.104918:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.104919:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.104921:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.104996:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.104999:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.105000:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.105002:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.105003:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.105005:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.105247:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.105249:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.105250:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.105252:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.105253:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.105255:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.105604:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.105607:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.105608:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.105609:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.105611:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.105612:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.106055:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.106058:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.106059:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.106060:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.106062:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.106063:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.106164:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.106167:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.106167:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.106169:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.106171:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.106172:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.106415:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.106418:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.106418:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.106419:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.106420:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.106421:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.106532:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.106535:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.106536:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.106537:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.106539:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.106540:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.106995:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.106998:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.106999:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.107000:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.107002:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.107003:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.107016:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.107019:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.107019:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.107020:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.107021:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.107021:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.107086:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.107088:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.107089:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.107091:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.107093:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.107094:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.107288:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.107291:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.107292:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.107296:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677446.107299:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.107302:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.107596:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.107599:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.107599:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.107601:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.107603:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.107604:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.107825:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.107828:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.107829:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.107831:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.107832:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.107833:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.107930:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.107933:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.107934:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.107935:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.107937:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.107945:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.108007:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.108010:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.108011:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.108012:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.108014:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.108015:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.108030:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.108032:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.108033:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.108034:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.108035:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.108036:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.108459:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.108461:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.108462:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.108464:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.108466:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.108467:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.108889:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.108891:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.108892:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.108894:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.108895:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.108896:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.109318:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:24.2:1619677446.109321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (21)++
00000800:00000200:13.0:1619677446.109321:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.109322:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.109325:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.109326:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.109328:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.109328:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.109329:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677446.109330:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.109331:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.109332:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.109332:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.109333:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.109362:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.109364:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.109365:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.109366:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:12.0:1619677446.109367:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.109368:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.109369:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.109372:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.109375:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.109376:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.109377:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.109381:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.109785:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.109787:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.109788:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.109791:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.109791:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.109793:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.109967:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.109970:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.109970:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.109973:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.109973:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.109976:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.110207:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.110210:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.110211:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.110212:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.110213:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.110215:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.110609:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.110611:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.110612:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.110614:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.110617:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.110618:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.110619:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.110622:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.110622:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.110624:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.110626:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.110627:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.110628:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.110631:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.110632:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.110633:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.110634:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.110635:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.110635:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.110639:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.110639:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.110640:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.110641:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.110642:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.110983:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.110985:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.110986:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.110989:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.110989:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.110992:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.111847:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.111850:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.111851:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.111852:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.111854:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.111855:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.111910:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.111912:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.111913:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.111914:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.111916:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.111916:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.111944:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.111946:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.111947:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.111948:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.111950:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.111951:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.112217:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.112220:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.112221:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.112223:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.112223:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.112226:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.112437:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.112439:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.112441:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.112443:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.112444:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.112446:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.112636:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.112638:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.112639:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.112640:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.112642:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.112643:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.112653:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.112656:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.112656:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.112657:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.112658:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.112659:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.112708:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.112711:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.112712:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.112714:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.112714:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.112716:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.112725:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.112728:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.112729:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.112730:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.112730:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.112731:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.112938:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.112940:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.112941:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.112943:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.112944:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.112946:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.113029:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.113032:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.113032:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.113034:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.113036:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.113037:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.113099:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.113102:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.113103:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.113105:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.113105:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.113107:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.113409:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.113412:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.113412:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.113414:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.113416:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.113417:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.113426:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.113429:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.113430:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.113430:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.113431:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.113431:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.113469:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.113472:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.113473:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.113474:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.113476:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.113477:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.113539:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.113542:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.113543:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.113544:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.113546:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.113547:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.114243:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.114246:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.114246:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.114248:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.114250:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.114251:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.114623:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.114626:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.114626:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.114628:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.114630:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.114631:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.114889:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.114891:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.114892:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.114894:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.114895:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.114897:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.115792:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.115794:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.115795:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.115797:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.115799:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.115800:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.115810:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.115813:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.115814:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.115815:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.115815:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.115817:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.115850:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.115852:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.115853:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.115855:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.115856:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.115858:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.116158:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.116161:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.116161:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.116164:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.116164:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.116170:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.116172:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.116174:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.116175:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.116175:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.116176:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.116176:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.116729:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.116732:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.116732:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.116734:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677446.116736:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.116737:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.117399:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.117402:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.117402:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.117404:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.117406:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.117407:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.117931:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.117934:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.117935:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.117937:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.117939:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.117940:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.118016:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.118019:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.118020:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.118022:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.118023:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.118024:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.118035:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.118038:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.118039:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.118039:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.118040:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.118041:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.118212:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677446.118214:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677446.118215:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.118217:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.118218:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.118219:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677446.118219:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677446.118220:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.118220:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677446.118222:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.118224:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.118569:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.118571:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.118572:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.118574:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.118576:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.118577:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.119218:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.119221:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.119221:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.119223:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.119224:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.119225:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.119643:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.119645:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.119646:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.119647:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.119650:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.119651:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.119776:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.119778:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.119779:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.119780:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.119783:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.119784:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.119917:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.119919:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.119920:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.119922:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.119923:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.119924:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.119925:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.119928:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.119929:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.119932:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.119933:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:12.0:1619677446.119933:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.119935:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.119935:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.119936:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.119937:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.119938:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.119939:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.119940:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.119943:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.119944:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.119944:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.119945:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.119946:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.120190:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.120193:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.120194:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.120196:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.120197:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.120199:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.120564:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.120566:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.120567:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.120569:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.120571:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.120573:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.120581:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.120583:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.120584:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.120586:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.120587:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.120588:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.120686:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.120688:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.120689:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.120691:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.120692:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.120693:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.120829:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.120831:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.120832:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.120835:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.120836:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.120838:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.121438:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.121440:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.121441:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.121443:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.121444:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.121446:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.121610:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.121612:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.121613:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.121615:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.121616:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.121618:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.121629:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.121632:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.121633:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.121633:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.121634:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.121635:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.121723:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.121726:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.121727:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.121729:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.121730:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.121732:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.121925:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.121927:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.121928:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.121929:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.121931:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.121931:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.121946:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.121949:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.121950:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.121950:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.121951:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.121952:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.123689:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.123692:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.123693:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.123695:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.123697:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.123699:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.124494:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.124496:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.124497:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.124498:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.124500:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.124502:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.124607:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677446.124610:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677446.124612:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677446.124613:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677446.124615:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.124616:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:13.0:1619677446.124618:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96746653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677446.124619:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677446.124621:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677446.124622:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:13.0:1619677446.124625:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677446.124627:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677446.124629:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677446.124630:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677446.124631:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677446.124633:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677446.124634:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677446.124635:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677446.124635:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677446.125369:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.125371:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.125372:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.125373:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.125375:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.125376:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.125447:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.125450:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.125450:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.125452:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.125453:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.125454:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.125469:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.125472:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.125473:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.125473:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.125474:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.125475:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.126116:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.126118:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.126119:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.126120:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.126122:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.126124:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.126400:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.126403:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.126404:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.126406:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.126408:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.126410:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.126467:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.126469:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.126470:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.126471:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.126473:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.126473:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.126851:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.126853:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.126854:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.126856:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.126857:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.126859:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.126962:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.126964:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.126965:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.126966:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.126968:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.126969:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.126989:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.126991:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.126992:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.126993:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.126994:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.126995:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.127341:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.127344:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.127345:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.127347:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.127348:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.127351:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.127526:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.127529:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.127530:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.127531:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.127534:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.127534:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.127535:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.127538:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.127539:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.127541:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.127542:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.127543:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.128149:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.128151:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.128152:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.128154:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.128155:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.128158:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.128174:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.128176:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.128177:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.128177:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:13.0:1619677446.128179:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.128179:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.128180:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.128180:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.128181:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.128182:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.128183:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.128184:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.129542:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.129544:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.129545:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.129548:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.129549:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.129554:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.129573:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.129575:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.129576:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.129577:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.129579:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.129580:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.129593:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.129596:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.129597:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.129597:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.129598:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.129599:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.130084:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.130087:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.130088:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.130089:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.130091:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.130092:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.130274:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.130277:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.130278:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.130279:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.130281:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.130282:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.130448:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.130451:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.130451:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.130453:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.130455:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.130456:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.130862:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.130864:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.130865:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.130867:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.130868:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.130869:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.130886:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.130889:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.130889:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.130890:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.130890:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.130891:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.131163:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.131165:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.131166:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.131167:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.131169:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.131170:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.131230:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.131233:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.131234:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.131236:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.131237:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.131238:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.131275:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.131277:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.131278:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.131279:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.131281:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.131282:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.131424:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.131427:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.131427:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.131429:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.131431:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.131432:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.131643:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.131646:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.131647:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.131649:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.131650:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.131651:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.131851:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.131854:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.131855:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.131857:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.131858:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.131863:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.131872:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.131875:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.131876:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.131877:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677446.131877:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.131878:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.131881:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.131882:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.131883:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.131885:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.131886:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.132247:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.132250:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677446.132250:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.132252:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.132254:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.132255:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677446.132255:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677446.132256:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.132257:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.132258:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.132258:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:13.0:1619677446.132259:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.132261:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.132261:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.132262:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.132263:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.132264:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.132274:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.132277:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.132278:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.132279:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.132279:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.132281:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.132506:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.132509:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.132510:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.132511:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.132513:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.132514:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.132525:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.132527:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.132529:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.132529:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.132530:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.132531:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.133007:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.133009:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.133010:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.133012:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.133014:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.133019:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.133079:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.133081:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.133082:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.133085:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.133085:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.133089:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.133143:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.133145:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.133146:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.133149:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.133149:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.133155:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.133173:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.133175:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.133176:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.133179:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.133179:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.133183:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.133322:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.133325:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.133325:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.133327:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.133329:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.133330:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.133504:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.133506:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.133507:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.133509:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.133510:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.133511:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.133524:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.133527:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.133528:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.133528:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.133529:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.133530:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.133877:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.133880:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.133881:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.133882:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.133884:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.133885:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.134269:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.134272:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.134273:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.134274:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.134276:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.134277:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.135029:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.135031:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.135032:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.135033:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.135036:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.135036:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.135048:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.135050:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.135051:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.135052:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.135052:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.135053:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.135762:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.135765:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.135765:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.135767:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.135769:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.135771:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.136016:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.136019:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.136019:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.136021:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.136022:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.136024:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.136066:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.136069:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.136070:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.136072:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.136073:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.136075:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.136598:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.136601:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.136602:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.136604:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.136605:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.136606:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.136803:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.136806:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.136806:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.136809:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.136809:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.136812:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.137305:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.137308:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.137309:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.137310:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.137312:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.137313:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.137320:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.137323:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.137324:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.137325:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.137325:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.137327:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.137645:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.137647:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.137648:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.137650:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.137651:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.137652:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.137813:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.137816:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.137816:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.137818:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.137819:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.137821:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.138238:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.138241:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.138242:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.138244:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.138245:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.138247:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.138345:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.138347:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.138348:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.138350:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.138352:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.138353:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.138356:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.138357:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.138358:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.138358:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.138360:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.138381:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.138384:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.138385:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.138387:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.138388:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.138389:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.138400:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.138402:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.138403:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.138404:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.138405:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.138406:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.138978:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677446.138980:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677446.138981:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.138982:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.138984:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.138985:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677446.138985:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677446.138986:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.138986:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677446.138987:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.138989:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.139171:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.139174:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.139175:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.139177:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.139178:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.139179:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.139384:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.139387:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.139388:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.139390:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.139392:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.139393:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.139512:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.139514:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.139515:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.139517:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.139518:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.139519:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.139603:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.139606:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.139607:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.139609:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.139610:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.139612:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.139624:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.139627:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.139628:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.139628:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.139629:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.139630:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.140515:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.140517:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.140518:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.140519:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.140521:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.140522:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.141092:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.141094:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.141095:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.141096:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.141098:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.141099:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.141198:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.141201:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.141201:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.141203:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.141205:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.141206:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.141219:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.141223:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.141224:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.141225:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.141226:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.141227:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.141659:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.141661:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.141663:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.141664:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.141667:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.141668:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.142137:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.142139:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.142140:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.142142:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.142144:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.142145:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.142146:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677446.142149:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677446.142150:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.142151:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.142153:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.142154:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677446.142154:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677446.142155:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.142155:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677446.142157:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.142159:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.142705:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.142707:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.142708:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.142710:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.142712:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.142713:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.142817:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.142819:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.142820:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.142822:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.142823:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.142824:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.142844:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.142847:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.142848:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.142849:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.142849:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.142851:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.143318:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.143321:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.143322:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.143323:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.143326:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.143327:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.143654:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.143656:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.143656:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:13.0:1619677446.143657:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.143657:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.143658:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:12.0:1619677446.143658:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.143659:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.143661:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.143662:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.143663:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.143664:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.143676:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.143679:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.143680:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.143680:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.143681:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.143683:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.144179:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.144182:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.144182:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.144184:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.144186:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.144187:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.144264:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.144267:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.144267:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.144269:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.144271:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.144272:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.144805:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.144808:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.144808:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.144810:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.144811:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.144813:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.2:1619677446.144975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (21)++
00000800:00000200:24.2:1619677446.145117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (20)++
00000800:00000200:13.0:1619677446.145207:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.145210:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.145210:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.145212:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.145214:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.145215:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.145228:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.145230:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.145231:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.145232:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.145232:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.145233:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.145843:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.145845:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.145846:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.145847:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.145849:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.145850:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.145924:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.145926:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.145926:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.145928:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.145930:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.145931:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.146266:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.146269:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.146270:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.146272:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.146272:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.146276:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.146285:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.146287:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.146288:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.146289:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.146289:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.146290:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.146481:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.146483:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.146484:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.146485:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.146487:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.146488:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.146659:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.146661:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.146662:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.146663:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.146665:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.146666:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.147012:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.147015:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.147016:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.147018:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.147019:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.147021:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.147256:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.147259:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.147260:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.147262:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.147262:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.147265:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.147276:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.147279:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.147280:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.147281:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.147281:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.147283:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.147612:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.147615:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.147616:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.147617:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.147619:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.147620:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.148516:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.148518:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.148519:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.148521:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.148523:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.148524:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.148696:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.148698:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.148699:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.148700:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.148702:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.148704:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.149121:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.149128:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677446.149129:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.149129:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.149131:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.149135:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.149136:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.149138:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.149139:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.149139:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.149140:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.149142:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.149157:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.149160:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.149161:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.149161:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.149162:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.149163:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.149856:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.149859:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.149861:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.149863:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.149864:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.149866:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.150036:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.150038:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.150039:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.150041:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.150042:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.150044:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.150119:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.150121:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.150122:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.150124:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.150126:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.150126:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.150775:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.150778:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.150778:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.150780:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.150782:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.150783:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.150940:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.150943:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.150944:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.150946:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.150947:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.150949:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.150965:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.150968:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.150969:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.150969:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.150970:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.150971:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.151689:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.151692:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.151693:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.151694:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.151696:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.151698:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.151707:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.151710:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.151710:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.151711:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.151712:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.151712:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.152441:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.152444:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.152445:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.152447:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.152449:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.152450:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.153712:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.153714:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.153715:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.153716:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.153718:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.153719:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.153839:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.153841:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.153842:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.153844:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.153845:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.153846:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.154137:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.154140:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.154141:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.154143:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.154144:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.154147:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.154570:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.154572:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.154573:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.154575:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.154576:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.154577:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.154854:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.154856:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.154857:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.154858:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.154860:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.154861:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.155207:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.155210:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.155211:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.155213:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.155213:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.155216:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.155229:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.155232:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.155233:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.155233:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.155234:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.155236:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.156194:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.156197:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.156198:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.156199:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.156201:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.156202:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.156212:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.156215:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.156216:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.156217:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.156217:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.156218:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.157057:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.157060:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.157061:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.157062:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.157064:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.157065:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.157490:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.157492:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.157493:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.157495:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.157497:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.157498:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.157902:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.157905:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.157905:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.157907:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.157908:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.157909:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.158133:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.158135:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.158136:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.158137:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.158139:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.158140:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.158181:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.158187:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.158188:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.158190:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.158191:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.158192:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.158803:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.158805:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.158806:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.158807:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.158811:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.158812:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.159210:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.159213:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.159214:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.159216:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.159217:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.159219:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.159227:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.159231:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.159231:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.159232:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.159233:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.159234:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.159339:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677446.159341:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677446.159342:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.159344:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.159345:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.159346:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677446.159346:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677446.159347:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.159348:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677446.159349:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.159352:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.159353:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.159355:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.159356:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.159356:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.159357:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.159357:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.159576:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.159579:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.159580:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.159582:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.159583:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.159585:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.159721:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.159723:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.159724:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.159726:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.159728:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.159729:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.159850:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.159853:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.159854:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.159856:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.159857:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.159859:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.160386:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.160389:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.160390:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.160391:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.160393:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.160394:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.160403:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.160405:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.160406:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.160407:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.160408:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.160409:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.160528:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.160531:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.160532:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.160533:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.160535:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.160536:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.160965:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.160967:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.160968:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.160970:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.160971:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.160973:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.162106:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.162108:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.162109:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.162110:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.162113:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.162114:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.162187:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.162190:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.162191:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.162192:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.162194:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.162195:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.162592:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.162594:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.162595:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.162596:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.162598:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.162599:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.162610:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.162612:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.162613:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.162614:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.162614:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.162615:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.162748:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.162751:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.162751:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.162753:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.162754:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.162756:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.162801:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.162803:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.162804:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.162805:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.162807:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.162808:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.162860:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.162863:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.162863:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.162865:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.162867:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.162868:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.163016:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.163019:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.163019:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.163022:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.163023:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.163024:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.163035:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.163039:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.163040:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.163040:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.163041:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.163042:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.163097:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.163100:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.163100:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.163102:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.163104:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.163106:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.163650:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.163653:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.163654:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.163656:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.163658:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.163662:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.163751:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.163754:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.163755:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.163756:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.163758:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.163759:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.164206:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.164208:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.164209:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.164209:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:13.0:1619677446.164210:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.164210:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.164211:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.164212:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.164214:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.164215:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.164216:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.164217:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.164264:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.164267:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.164268:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.164268:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.164269:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.164270:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.164405:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.164408:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.164409:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.164410:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.164412:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.164413:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.164425:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.164428:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.164429:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.164429:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.164430:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.164431:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.164795:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.164798:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.164799:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.164800:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.164801:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.164803:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.165106:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.165109:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.165110:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.165111:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.165113:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.165114:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.165192:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.165195:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.165196:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.165197:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.165199:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.165200:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.165320:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.165323:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.165324:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.165325:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.165327:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.165328:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.165755:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.165758:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.165758:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.165760:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.165761:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.165763:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.165772:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.165775:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.165777:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.165777:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.165777:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.165778:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.166042:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.166045:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677446.166046:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.166048:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.166049:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677446.166051:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.166061:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.166064:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.166065:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.166067:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.166068:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.166069:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.166365:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.166368:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.166369:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.166370:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.166372:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.166373:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.168007:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.168010:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.168011:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.168013:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.168015:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.168015:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.168398:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.168400:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.168401:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.168403:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.168405:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.168411:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.168605:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.168608:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.168608:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.168610:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.168612:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.168613:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.168623:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.168626:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.168627:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.168627:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.168628:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.168629:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.168813:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.168815:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.168816:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.168817:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.168820:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.168820:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.168840:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.168842:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.168843:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.168846:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.168846:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.168847:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.169126:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.169129:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.169130:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.169132:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.169132:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.169134:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.169198:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.169201:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.169201:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.169203:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.169205:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.169206:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.169389:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.169392:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.169393:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.169394:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.169396:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.169397:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.169430:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.169432:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.169433:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.169435:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677446.169436:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.169437:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.169652:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.169654:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.169655:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.169656:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.169658:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.169660:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.169706:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.169708:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.169709:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.169711:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.169712:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.169714:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.169723:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.169725:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.169726:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.169727:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.169727:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.169729:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.169801:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.169804:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.169805:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.169806:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.169808:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.169808:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.170069:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.170072:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.170073:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.170074:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.170076:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.170077:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.170283:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.170286:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.170286:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.170289:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.170289:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.170303:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.170325:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.170328:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.170329:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.170332:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.170332:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.170336:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.170748:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.170751:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.170751:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.170753:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.170754:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.170755:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.170765:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.170768:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.170769:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.170770:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.170771:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.170772:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.171134:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.171137:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.171137:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.171139:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.171141:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.171142:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.172279:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.172282:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.172283:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.172284:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.172286:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.172287:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.172299:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.172302:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.172302:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.172303:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.172304:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.172304:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.173234:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.173236:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.173237:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.173238:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.173240:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.173240:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.173363:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.173366:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.173367:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.173369:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.173370:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.173372:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.173828:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.173830:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.173831:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.173833:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.173834:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.173836:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.174138:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.174141:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.174142:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.174144:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.174145:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.174151:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.174170:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.174172:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.174173:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.174175:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.174176:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.174178:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.174863:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.174865:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.174866:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.174868:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.174870:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.174871:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.174880:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.174883:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.174884:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.174885:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.174885:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.174886:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.175030:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.175034:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (25)--
00000400:00000200:12.0:1619677446.175035:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.175036:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.175037:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.175041:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.175042:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.175044:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.175044:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.175046:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:12.0:1619677446.175046:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.175047:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.175047:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.175048:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.175049:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.175049:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.175049:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.175414:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677446.175416:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677446.175419:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677446.175421:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677446.175422:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.175424:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:12.0:1619677446.175425:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96745911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677446.175426:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677446.175427:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677446.175429:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:12.0:1619677446.175432:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677446.175434:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677446.175438:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677446.175438:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:12.0:1619677446.175439:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:12.0:1619677446.175441:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:12.0:1619677446.175442:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677446.175442:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677446.175443:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:13.0:1619677446.175469:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.175471:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.175472:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.175473:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.175475:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.175476:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.175668:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.175670:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.175671:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.175673:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.175674:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.175675:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.176356:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.176359:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.176360:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.176361:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.176363:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.176364:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.176488:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677446.176491:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677446.176493:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677446.176495:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677446.176495:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.176497:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:12.0:1619677446.176497:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:13.0:1619677446.176498:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.176498:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96745891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677446.176499:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.176499:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677446.176501:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:12.0:1619677446.176501:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677446.176502:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:13.0:1619677446.176505:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.176506:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:12.0:1619677446.176506:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677446.176508:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677446.176511:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677446.176512:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:12.0:1619677446.176512:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:12.0:1619677446.176514:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:12.0:1619677446.176515:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677446.176516:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677446.176518:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677446.176519:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677446.176531:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677446.176534:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677446.176534:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677446.176535:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:13.0:1619677446.176581:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.176584:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.176585:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.176587:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.176588:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.176590:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.176598:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.176601:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.176603:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.176603:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.176604:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.176605:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.176637:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.176640:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.176640:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.176642:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.176644:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.176645:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.177351:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.177354:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.177355:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.177357:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.177359:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.177360:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.177716:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.177718:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.177719:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.177720:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.177722:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.177723:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.178166:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.178168:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.178169:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.178171:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.178172:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.178174:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.178183:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.178186:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.178187:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.178187:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.178188:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.178189:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.178548:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.178551:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.178551:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.178553:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.178555:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.178556:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.179056:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.179059:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.179060:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.179062:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.179063:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.179064:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.179143:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.179146:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.179147:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.179149:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.179150:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.179152:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.179353:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.179355:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.179356:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.179357:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.179360:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.179360:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.179605:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.179608:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.179609:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.179611:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.179612:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.179614:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.179622:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.179625:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.179626:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.179627:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.179627:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.179629:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.179738:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.179741:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.179741:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.179743:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.179744:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.179745:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.179756:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.179759:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.179760:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.179761:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.179761:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.179763:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.180346:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.180348:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.180349:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.180350:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.180353:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.180354:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.180602:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.180603:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.180604:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.180604:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:13.0:1619677446.180605:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.180605:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.180606:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.180607:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.180607:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.180608:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.180609:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.180610:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.180939:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.180941:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.180943:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.180944:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.180945:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.180946:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.181126:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.181128:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.181129:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.181131:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.181132:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.181134:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.181742:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.181744:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.181745:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.181747:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.181749:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.181750:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.181759:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.181762:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.181763:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.181764:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.181764:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.181765:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.181829:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.181832:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.181833:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.181836:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.181837:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.181839:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.183181:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.183184:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.183185:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.183187:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.183188:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.183190:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.183519:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.183522:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.183523:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.183525:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.183527:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.183528:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.183698:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.183700:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.183701:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.183703:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.183704:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.183706:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.184025:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.184027:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.184028:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.184030:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.184031:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.184034:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.184043:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.184045:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.184046:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.184047:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.184047:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.184049:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.184916:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.184918:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.184919:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.184920:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.184922:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.184923:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.184933:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.184936:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.184937:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.184937:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.184938:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.184939:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.184968:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.184970:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.184971:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.184973:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.184975:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.184976:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.185198:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.185201:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.185202:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.185205:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.185205:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.185208:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.185301:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.185303:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.185304:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.185306:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.185308:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.185314:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.185332:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.185334:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.185335:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.185336:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.185338:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.185339:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.185561:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.185564:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.185565:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.185567:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.185568:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.185570:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.185927:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.185930:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.185930:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.185933:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.185934:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.185935:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.186078:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.186080:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.186081:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.186082:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.186084:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.186086:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.186106:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.186109:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.186109:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.186110:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.186111:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.186112:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.186354:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.186356:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.186357:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.186359:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.186360:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.186362:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.186372:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.186375:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.186376:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.186377:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.186377:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.186379:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.186400:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.186402:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.186403:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.186405:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.186406:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.186408:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.188032:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.188034:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.188035:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.188037:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.188039:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.188040:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.188041:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.188044:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.188044:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.188046:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.188048:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.188048:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.188857:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.188860:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.188861:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.188863:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.188865:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.188865:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.189333:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.189336:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.189337:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.189339:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.189340:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.189342:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.189342:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.189343:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.189343:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.189344:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.189344:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.189542:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.189544:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.189545:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.189547:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.189548:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.189550:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.189591:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.189593:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.189594:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.189596:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.189598:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.189600:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.189609:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.189612:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.189612:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.189614:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.189616:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.189617:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.189862:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.189865:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.189866:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.189868:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.189869:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.189871:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.189882:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.189885:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.189886:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.189886:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.189887:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.189888:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.190123:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.190125:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.190126:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.190127:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.190129:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.190129:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.190195:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.190197:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.190198:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.190200:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.190202:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.190203:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.191124:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.191127:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.191128:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.191130:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.191132:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.191134:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.191398:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.191401:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.191401:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.191403:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.191405:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.191406:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.191454:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.191457:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.191457:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.191459:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.191461:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.191461:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.191597:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.191599:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.191600:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.191601:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.191603:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.191604:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.191615:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.191618:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.191619:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.191619:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.191620:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.191621:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.192164:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.192167:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.192168:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.192170:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.192171:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.192172:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.192321:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.192324:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.192326:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.192328:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.192329:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.192330:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.193233:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.193236:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.193236:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.193238:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.193240:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.193241:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.193539:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.193542:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.193543:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.193545:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.193546:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.193547:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.193559:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.193562:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.193563:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.193563:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.193564:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.193565:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.194115:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.194117:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.194118:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.194120:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.194121:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.194123:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.194161:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.194164:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.194165:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.194166:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.194168:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.194169:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.194566:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.194568:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.194569:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.194571:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.194573:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.194574:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.194773:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.194776:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.194776:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.194778:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.194780:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.194781:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.194794:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.194797:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.194798:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.194798:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.194799:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.194800:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.195066:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.195068:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.195069:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.195071:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.195072:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.195074:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.195083:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.195086:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.195087:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.195087:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.195088:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.195089:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.195496:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.195498:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.195499:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.195501:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.195502:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.195503:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.195893:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.195896:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.195897:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.195898:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.195900:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.195900:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.196088:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.196091:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.196092:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.196094:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.196095:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.196097:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.196193:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.196195:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.196196:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.196198:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.196199:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.196201:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.196490:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.196493:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.196494:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.196496:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.196498:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.196499:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.196616:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.196618:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.196619:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.196621:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.196622:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.196624:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.196634:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.196637:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.196638:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.196639:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.196639:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.196641:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.197269:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.197271:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.197272:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.197274:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.197275:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.197276:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.197678:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.197681:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.197682:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.197684:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.197685:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.197687:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.198223:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.198226:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.198226:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.198228:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.198229:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.198230:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.198314:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.198316:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.198317:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.198319:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.198321:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.198322:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.198610:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.198612:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.198613:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.198614:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.198616:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.198616:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.198626:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.198629:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.198630:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.198631:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.198631:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.198633:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.198942:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.198944:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.198945:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.198947:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.198948:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.198950:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.199007:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.199010:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.199011:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.199012:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.199014:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.199015:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.199595:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.199598:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.199599:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.199601:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.199602:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.199604:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.199681:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.199684:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.199685:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.199687:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.199688:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.199690:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.199700:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.199704:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.199705:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.199705:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.199706:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.199707:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.199800:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.199803:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.199804:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.199805:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.199807:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.199808:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.199817:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.199819:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.199820:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.199821:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.199821:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.199823:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.200275:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.200277:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.200278:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.200280:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.200281:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.200282:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.200504:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.200506:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.200507:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.200509:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.200511:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.200512:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.200554:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.200556:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.200557:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.200559:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.200560:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.200562:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.201082:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.201085:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.201085:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.201087:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.201089:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.201090:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.201136:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.201139:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.201140:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.201142:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.201142:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.201144:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.201754:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.201757:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.201758:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.201761:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.201761:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.201763:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.202276:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.202279:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.202280:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.202282:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.202283:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.202285:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.202297:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.202300:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.202301:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.202302:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.202302:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.202303:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.202759:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.202761:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.202762:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.202764:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.202766:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.202767:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.203131:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.203133:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.203134:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.203136:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.203137:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.203139:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.203149:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.203151:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.203152:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.203153:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.203153:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.203155:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.203231:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677446.203234:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677446.203236:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677446.203238:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677446.203239:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.203241:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677446.203242:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96745492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677446.203243:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677446.203244:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677446.203245:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677446.203248:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677446.203250:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677446.203253:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677446.203254:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:12.0:1619677446.203254:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677446.203257:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677446.203258:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677446.203259:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677446.203259:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:13.0:1619677446.204226:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.204228:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.204229:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.204230:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.204232:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.204234:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.204488:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.204490:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.204491:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.204492:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.204495:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.204495:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.204746:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.204749:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.204750:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.204752:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.204753:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.204754:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.205027:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.205029:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.205030:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.205032:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.205033:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.205034:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.205046:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.205048:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.205050:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.205050:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.205051:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.205052:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.205264:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.205266:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.205268:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.205270:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.205270:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.205273:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.205829:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.205832:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.205833:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.205834:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.205836:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.205837:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.206555:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.206557:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.206558:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.206560:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.206562:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.206563:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.206573:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.206575:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.206576:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.206578:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.206579:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.206580:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.206590:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.206592:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.206593:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.206594:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.206595:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.206596:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.206931:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.206934:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.206934:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.206936:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.206938:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.206939:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.207148:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.207150:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.207151:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.207154:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.207154:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.207157:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.207168:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.207170:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.207171:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.207173:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.207174:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.207175:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.207446:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.207449:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.207450:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.207451:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.207453:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.207454:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.207836:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.207839:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.207840:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.207842:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.207843:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.207845:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.208142:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.208145:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.208146:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.208148:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677446.208149:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.208151:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.208219:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.208221:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.208222:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.208224:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.208226:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.208227:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.208270:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.208272:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.208273:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.208275:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.208276:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.208278:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.208401:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.208404:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.208405:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.208405:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.208406:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.208407:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.208905:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.208908:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.208909:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.208911:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.208912:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.208914:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.208959:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.208962:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.208963:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.208964:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.208966:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.208966:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.208978:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.208981:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.208982:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.208982:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.208983:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.208984:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.209323:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.209325:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.209326:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.209328:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.209330:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.209331:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.209740:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.209743:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.209743:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.209745:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.209747:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.209748:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.210300:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.210303:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.210303:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.210305:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.210307:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.210308:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.210585:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.210587:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.210588:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.210590:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.210591:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.210593:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.210602:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.210605:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.210606:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.210606:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.210607:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.210608:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.210813:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.210815:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.210816:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.210817:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.210819:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.210820:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.210905:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.210908:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.210908:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.210910:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.210912:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.210913:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.210925:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.210928:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.210929:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.210929:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.210930:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.210932:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.211015:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.211017:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.211018:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.211019:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.211021:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.211022:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.211386:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.211388:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.211389:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.211391:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.211392:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.211393:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.211728:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.211731:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.211731:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.211733:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.211734:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.211736:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.211834:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.211837:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.211838:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.211840:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.211840:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.211843:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.212025:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.212027:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.212028:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.212029:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.212031:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.212032:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.212043:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.212046:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.212047:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.212047:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.212048:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.212049:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.212212:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.212215:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.212216:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.212217:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.212219:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.212220:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.212328:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677446.212330:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677446.212331:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.212333:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.212335:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677446.212336:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677446.212336:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677446.212337:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.212338:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677446.212339:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.212341:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.213816:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.213818:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.213819:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.213821:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.213823:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.213824:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.214129:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.214132:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.214133:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.214134:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.214136:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.214137:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.214149:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.214152:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.214153:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.214153:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.214154:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.214155:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.214753:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.214755:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.214756:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.214758:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.214760:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.214761:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.214770:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.214773:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.214773:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.214774:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.214775:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.214775:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.214948:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.214950:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.214952:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.214953:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.214954:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.214956:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.215202:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.215204:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.215205:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.215207:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.215208:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.215209:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.215421:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.215423:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.215424:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.215425:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.215427:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.215428:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.215775:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.215778:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.215779:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.215781:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.215781:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.215790:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.215808:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.215811:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.215812:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.215813:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.215814:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.215815:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.215839:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.215842:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.215842:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.215843:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.215844:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.215845:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.216235:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.216238:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.216239:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.216240:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.216242:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.216243:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.216313:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.216316:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.216316:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.216318:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.216320:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.216321:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.216502:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.216505:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.216505:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.216507:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.216508:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.216509:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.216726:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.216729:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.216730:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.216732:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.216734:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.216735:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.217472:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.217474:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.217476:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.217477:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677446.217479:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.217480:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.217489:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.217491:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.217492:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.217493:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.217493:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.217495:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.217527:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.217530:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.217531:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.217532:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.217533:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.217534:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.218647:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.218649:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.218650:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.218652:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.218654:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.218655:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.218665:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.218668:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.218669:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.218669:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.218670:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.218671:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.219387:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.219390:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.219391:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.219393:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.219395:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.219395:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.219421:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.219423:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.219424:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.219426:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.219427:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.219429:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.219559:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.219562:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.219563:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.219565:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.219566:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.219568:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.220407:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.220409:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.220410:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.220412:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.220414:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.220415:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.220522:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.220524:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.220525:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.220527:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.220528:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.220530:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.220673:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.220676:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.220676:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.220678:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.220680:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.220681:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.220692:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.220695:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.220696:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.220696:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.220697:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.220698:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.221085:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.221088:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.221089:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.221091:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.221091:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.221094:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.221180:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.221182:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.221183:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.221185:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.221186:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.221188:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.221727:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.221730:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.221730:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.221732:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.221734:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.221735:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.221864:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.221866:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.221867:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.221869:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.221870:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.221872:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.221881:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.221884:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.221885:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.221885:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.221886:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.221887:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.222208:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.222211:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.222212:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.222213:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.222216:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.222217:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.222764:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.222767:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.222767:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.222769:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.222771:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.222772:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.222782:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.222786:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.222787:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.222788:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.222789:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.222790:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.223433:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.223436:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.223437:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.223439:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.223441:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.223441:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.223673:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.223675:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.223676:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.223677:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.223679:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.223680:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.223822:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.223824:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.223826:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.223828:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.223828:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.223831:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.224213:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.224216:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.224217:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.224219:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.224220:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.224223:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.224231:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.224234:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.224234:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.224235:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.224236:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.224238:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.224755:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.224757:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.224758:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.224759:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.224761:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.224762:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.224889:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.224892:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.224893:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.224894:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.224896:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.224897:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.225388:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.225390:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.225391:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.225394:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.225395:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.225397:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.225424:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.225426:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.225427:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.225429:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.225431:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.225432:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.225533:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.225535:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.225536:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.225538:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.225540:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.225541:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.225775:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.225777:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.225778:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.225780:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.225781:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.225783:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.225791:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.225794:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.225795:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.225795:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.225796:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.225797:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.226549:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.226551:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.226552:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.226554:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.226556:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.226557:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.226930:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.226933:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.226933:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.226935:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.226936:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.226937:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.227160:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.227162:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.227163:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.227165:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.227166:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.227168:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.227836:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.227839:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.227839:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.227841:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.227843:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.227844:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.228726:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.228729:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.228729:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.228731:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.228733:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.228735:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.228743:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.228746:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.228747:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.228748:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.228748:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.228750:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.228931:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.228934:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.228935:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.228937:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.228938:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.228939:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.228950:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.228953:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.228954:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.228954:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.228955:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.228956:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.229099:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.229102:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.229103:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.229104:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.229106:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.229107:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.229251:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.229253:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.229254:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.229256:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.229257:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.229259:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.2:1619677446.229284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (21)++
00000800:00000200:12.0:1619677446.229356:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.229359:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.229360:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.229362:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.229364:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.229365:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.229468:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.229471:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.229472:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.229474:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.229475:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.229488:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.229506:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.229509:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.229510:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.229512:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.229513:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.229522:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.229523:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.229526:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.229530:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)++
00000800:00000200:12.0:1619677446.229531:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677446.229532:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.229533:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.229542:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.229544:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.229545:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.229548:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.229548:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.229550:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.229668:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.229671:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.229671:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.229673:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.229674:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.229677:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.229900:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.229903:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.229903:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.229905:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.229907:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.229908:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.230108:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.230110:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.230111:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.230113:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.230114:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.230115:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.230134:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.230137:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.230138:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.230138:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.230139:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.230140:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.230619:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.230622:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.230622:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.230624:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.230626:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.230627:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.230657:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.230660:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.230661:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.230662:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.230664:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.230665:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.230676:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.230678:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.230679:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.230681:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.230682:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.230683:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.230759:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.230762:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.230763:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.230764:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.230766:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.230767:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.231026:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.231028:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.231029:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.231030:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.231032:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.231033:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.231044:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.231046:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.231047:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.231048:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.231048:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.231049:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.231389:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.231391:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.231392:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.231394:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.231396:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.231397:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.232249:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.232252:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.232252:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.232254:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.232255:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.232256:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.232350:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.232352:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.232353:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.232354:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.232357:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.232357:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.232436:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.232438:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.232440:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.232441:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.232442:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.232445:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.232454:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.232457:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.232458:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.232458:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.232459:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.232461:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.233094:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.233096:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.233097:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.233099:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.233100:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.233102:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.233891:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.233893:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.233894:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.233896:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.233898:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.233899:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.233971:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.233974:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.233974:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.233976:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.233978:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.233979:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.234108:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.234110:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.234111:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.234113:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.234114:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.234116:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.234129:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.234132:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.234133:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.234134:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.234134:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.234136:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.2:1619677446.234255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (20)++
00000800:00000200:12.0:1619677446.234601:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.234604:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.234605:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.234607:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.234609:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.234610:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.234747:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.234749:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.234750:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.234752:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.234753:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.234754:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.234755:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.234759:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.234760:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.234762:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.234763:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.234764:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.234765:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.234766:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.234766:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.234767:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.234768:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.234768:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.235380:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.235383:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.235383:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.235385:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.235387:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.235388:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.236541:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.236544:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.236545:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.236547:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.236549:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.236549:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.237623:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.237626:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.237627:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.237629:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.237630:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.237632:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.237639:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.237643:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.237644:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.237644:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.237645:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.237646:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.238385:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.238388:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.238388:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.238390:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.238392:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.238393:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.238715:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.238717:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.238718:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.238720:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.238721:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.238722:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.238926:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.238929:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.238930:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.238931:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.238933:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.238934:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.239062:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.239064:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.239065:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.239067:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.239068:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.239069:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.239498:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.239501:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.239502:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.239504:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.239505:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.239507:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.239587:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.239590:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.239591:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.239592:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.239594:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.239595:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.239606:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.239608:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.239608:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.239608:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.239609:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.239609:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.239609:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677446.239610:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.239611:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:13.0:1619677446.239612:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.239612:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.239613:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.239850:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.239852:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.239853:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.239855:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.239856:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.239858:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.239872:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.239875:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.239876:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.239877:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.239878:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.239879:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.240165:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.240167:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.240168:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.240170:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.240172:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.240173:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.240297:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.240300:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.240301:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.240302:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.240304:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.240305:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.240421:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.240424:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.240425:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.240426:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.240428:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.240429:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.240694:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.240696:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677446.240697:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.240699:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.240700:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677446.240701:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677446.240702:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.240702:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.240703:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.240704:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.240706:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.240716:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.240718:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.240719:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.240720:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.240721:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.240722:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.240801:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.240803:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.240804:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.240807:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.240807:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.240810:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.241298:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.241301:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.241301:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.241304:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.241305:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.241307:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.241334:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.241336:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.241337:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.241339:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.241340:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.241341:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.241771:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.241773:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.241774:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.241776:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.241777:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.241779:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.241868:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.241870:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.241871:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.241872:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.241874:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.241875:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.242188:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.242191:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.242192:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.242194:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.242195:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.242197:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.242207:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.242210:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.242211:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.242211:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.242212:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.242213:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.242775:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.242778:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.242779:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.242781:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.242783:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.242784:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.243151:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.243154:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.243154:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.243156:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.243157:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.243159:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.243756:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.243758:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.243759:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.243760:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:12.0:1619677446.243760:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.243762:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.243762:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.243763:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677446.243764:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.243764:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.243765:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.243766:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.243767:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.243767:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.243768:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.243769:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.243770:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.244863:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.244866:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.244867:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.244869:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.244870:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.244874:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.244995:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.244997:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.244998:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.245001:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.245001:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.245003:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.245101:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.245103:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.245104:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.245105:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.245107:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.245108:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.245121:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.245124:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.245125:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.245125:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.245126:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.245127:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.245512:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.245514:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.245515:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.245516:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.245519:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.245520:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.245653:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.245656:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.245656:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.245658:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.245660:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.245661:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.245709:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.245711:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.245712:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.245714:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.245716:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.245717:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.245718:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677446.245720:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677446.245721:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.245723:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.245724:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677446.245725:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677446.245725:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677446.245726:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.245726:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.245727:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.245733:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.245735:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.245736:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.245737:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.245738:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.245738:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.245739:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.245872:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.245874:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.245875:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.245877:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.245878:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.245879:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.2:1619677446.245892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (21)++
00000800:00000200:12.0:1619677446.246235:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.246237:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.246238:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.246239:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.246241:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.246242:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.246253:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.246255:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.246256:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.246257:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.246257:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.246258:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.246729:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.246731:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.246732:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.246734:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.246736:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.246737:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.246800:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.246802:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.246803:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.246805:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.246806:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.246807:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.247289:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.247292:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.247300:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.247300:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.247302:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.247306:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.247308:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.247310:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.247311:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.247311:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.247312:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.247312:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.247456:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.247459:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.247459:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.247461:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.247462:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.247464:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.2:1619677446.247681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (20)++
00000800:00000200:12.0:1619677446.247865:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.247868:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.247869:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.247870:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.247872:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.247873:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.248644:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.248647:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.248647:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.248649:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.248651:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.248652:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.248882:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.248884:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.248885:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.248887:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.248889:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.248890:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.248961:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.248963:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.248963:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:13.0:1619677446.248964:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.248965:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.248965:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.248966:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:12.0:1619677446.248967:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.248969:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.248970:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.248971:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.248972:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.249641:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.249643:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.249643:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:13.0:1619677446.249644:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.249644:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.249645:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.249646:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:12.0:1619677446.249647:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.249648:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.249649:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.249651:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.249651:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.249661:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.249663:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.249663:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.249664:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.249664:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.249665:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.249665:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.249665:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.249665:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.249666:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.249668:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.249668:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.249718:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.249721:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.249722:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.249723:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.249724:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.249726:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.250646:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.250649:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.250650:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.250652:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.250653:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.250654:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.251125:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.251128:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.251128:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.251130:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.251132:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.251133:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.251235:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.251238:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.251239:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.251241:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.251242:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.251244:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.251401:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.251403:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.251404:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.251406:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.251408:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.251411:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.251499:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.251501:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.251502:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.251504:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.251506:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.251507:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.251515:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.251518:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.251519:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.251520:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.251520:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.251521:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.251661:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.251664:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.251665:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.251666:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.251668:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.251669:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.252393:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.252396:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.252397:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.252398:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.252400:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.252401:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.252409:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.252412:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.252413:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.252413:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.252414:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.252416:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.253268:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.253271:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.253273:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.253275:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677446.253275:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.253278:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.253409:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.253412:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.253413:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.253415:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.253417:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.253418:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.253773:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.253776:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.253777:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.253779:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.253780:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.253782:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.254095:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.254098:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[1] from 10.0.10.103@o2ib7
00000400:00000200:12.0:1619677446.254100:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.103@o2ib7 : PUT - routed
00000800:00000200:13.0:1619677446.254101:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.254103:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.254118:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.254121:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.254124:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677446.254125:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.103@o2ib7 : PUT - routed
00000800:00000200:13.0:1619677446.254128:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.254343:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.254346:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.0.10.103@o2ib7
00000400:00000200:12.0:1619677446.254348:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.103@o2ib7 : PUT - routed
00000800:00000200:12.0:1619677446.254350:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.254351:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.254756:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.254758:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.254759:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.254761:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.254762:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.254763:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.254964:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.254966:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.254967:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.254970:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.254970:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.254973:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.254982:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.254985:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.254986:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.254986:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.254987:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.254989:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.255145:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.255147:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.0.10.103@o2ib7
00000400:00000200:13.0:1619677446.255149:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.103@o2ib7 : PUT - routed
00000800:00000200:12.0:1619677446.255151:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.255152:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.255164:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.255166:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.255167:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.255168:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.255170:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.255170:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.255210:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.255213:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.255213:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.255215:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.255217:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.255218:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.255560:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.255563:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.255564:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.255566:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.255568:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.255568:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.255663:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.255666:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.255666:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.255668:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.255669:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.255671:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.255731:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.255733:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.255734:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.255736:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.255737:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.255738:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.255747:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.255749:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.255749:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.255749:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.255750:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:13.0:1619677446.255751:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: ACK: OK
00000800:00000200:12.0:1619677446.255751:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.255752:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.255752:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.255753:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.255754:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.255755:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.256159:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.256162:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.256163:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.256165:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.256165:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.256167:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.256183:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.256185:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.256186:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.256188:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.256189:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.256191:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.256328:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.256330:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.256331:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.256333:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.256334:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.256336:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.256566:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.256569:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.256570:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.256572:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.256573:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.256575:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.256583:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.256586:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.256586:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.256587:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.256588:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.256588:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.256654:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.256657:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.256657:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.256659:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.256661:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.256662:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.256833:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.256836:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.256837:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.256838:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: ACK: OK
00000800:00000200:12.0:1619677446.256840:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.256841:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.257204:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.257206:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.257207:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.257209:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.257210:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.257212:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.257307:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.257310:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.257311:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.257313:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.257314:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.257316:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.257478:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.257480:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.257481:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.257483:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.257484:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.257485:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.257498:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.257500:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.257501:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.257502:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.257502:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.257504:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.257694:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.257696:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.257697:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.257699:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.257700:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.257701:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.257898:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.257901:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.257902:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.257903:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.257905:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.257906:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.258021:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.258023:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.258024:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.258026:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.258027:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.258028:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.258042:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.258045:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.258046:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.258046:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.258047:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.258049:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.258082:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.258084:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.258085:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.258087:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.258088:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.258090:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.258327:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.258330:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.258331:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.258333:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.258334:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.258335:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.259092:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.259094:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.259095:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.259097:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.259099:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.259100:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.259940:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.259942:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.259943:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.259945:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.259947:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.259948:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.260232:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.260234:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.260235:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.260238:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.260239:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.260241:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.260252:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.260255:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.260256:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.260256:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.260257:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.260258:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.260592:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.260594:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.260595:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.260597:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.260599:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.260600:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.260680:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.260682:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.260683:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.260685:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.260686:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.260687:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.260698:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.260701:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.260702:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.260702:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.260703:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.260704:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.261031:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.261034:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.261034:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.261036:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.261037:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.261038:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.261166:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.261169:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.261169:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.261171:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.261172:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.261174:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.261366:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.261369:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.261369:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.261371:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.261373:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.261381:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.261400:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.261402:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.261403:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.261405:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.261406:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.261408:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.261800:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.261803:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.261804:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.261805:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.261807:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.261808:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.261820:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.261824:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.261824:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.261825:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.261825:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.261827:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.262113:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.262115:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.262116:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.262117:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.262119:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.262120:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.262302:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.262304:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.262305:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.262307:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.262309:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.262310:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.262312:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.262313:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.262314:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.262315:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.262316:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.262394:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.262397:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.262398:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.262399:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.262401:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.262403:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.263031:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.263034:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.263034:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.263036:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.263038:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.263039:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.263815:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.263818:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.263819:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.263820:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.263822:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.263823:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.264805:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.264808:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.264809:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.264811:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.264812:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.264813:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.265165:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.265167:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.265168:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.265169:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.265171:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.265172:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.265945:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.265948:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.265949:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.265950:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.265952:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.265954:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.266036:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.266038:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.266039:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.266040:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.266042:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.266043:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.266069:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.266071:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.266072:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.266073:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.266073:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.266075:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.266138:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.266140:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.266141:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.266143:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.266144:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.266145:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.266157:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.266159:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.266160:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.266161:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.266162:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.266163:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.266774:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.266777:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.266778:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.266780:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.266782:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.266782:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.267403:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.267406:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.267407:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.267410:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.267410:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.267413:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.267442:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.267445:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.267446:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.267448:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.267449:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.267451:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.267644:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.267646:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.267647:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.267648:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.267650:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.267651:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.268262:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.268264:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.268265:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.268267:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.268268:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.268269:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.268655:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.268657:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.268658:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.268660:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.268662:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.268663:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.268777:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677446.268779:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677446.268780:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.268782:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.268783:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.268784:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677446.268784:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677446.268785:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.268785:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677446.268787:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.268791:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.268793:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.268794:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.268795:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.268796:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.268796:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.268797:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.268855:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.268858:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.268859:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.268861:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.268861:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.268863:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.269794:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.269797:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.269798:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.269799:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.269801:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.269802:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.270538:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.270541:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.270541:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.270543:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.270545:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.270546:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.270563:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.270565:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.270566:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.270567:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.270567:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.270569:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.271127:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.271129:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.271130:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.271133:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.271133:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.271140:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.272617:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.272620:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.272620:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.272622:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.272624:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.272626:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.272633:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.272636:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.272637:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.272638:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.272639:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.272640:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.272986:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.272988:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.272989:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.272990:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:12.0:1619677446.272990:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.272991:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.272991:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.272993:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.272993:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.272995:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.272996:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.272997:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.273589:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.273592:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.273593:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.273595:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.273596:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.273598:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.273915:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.273917:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.273918:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.273920:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.273921:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.273923:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.275253:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.275255:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.275256:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.275258:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.275260:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.275262:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.275271:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.275273:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.275274:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677446.275275:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677446.275275:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.275276:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.275277:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.275277:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.275278:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.275280:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.275283:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.275523:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.275525:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.275526:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.275528:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.275530:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.275531:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.275785:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.275788:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.275789:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.275791:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.275792:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.275793:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.277816:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.277819:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.277819:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.277821:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.277824:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.277825:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.278037:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.278040:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.278041:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.278042:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.278044:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.278045:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.278055:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.278057:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.278058:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.278059:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.278059:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.278060:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.278244:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.278246:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.278247:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.278249:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.278251:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.278251:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.278683:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.278686:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.278687:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.278689:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.278690:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.278692:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.280267:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.280269:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.280270:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.280272:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.280274:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.280275:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.280291:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.280292:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.280293:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.280293:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.280300:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.280301:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:13.0:1619677446.280302:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.280302:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.280302:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.280304:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.280305:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.280306:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.280507:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.280509:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.280511:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.280513:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.280513:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.280515:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.280548:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.280551:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.280552:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.280553:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.280555:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.280556:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.280748:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.280751:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.280751:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.280753:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.280754:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.280756:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.280784:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.280786:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.280787:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.280788:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.280790:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.280791:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.280803:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.280805:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.280806:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.280807:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.280807:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.280808:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.281050:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.281053:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.281054:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.281055:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.281056:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.281058:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.281103:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.281106:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.281107:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.281110:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.281110:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.281112:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.281377:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.281379:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.281380:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.281381:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.281383:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.281384:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.281398:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.281400:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.281401:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.281402:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.281402:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.281403:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.281530:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677446.281533:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677446.281535:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677446.281536:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677446.281537:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.281539:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:13.0:1619677446.281540:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96744342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677446.281541:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677446.281542:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677446.281543:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:13.0:1619677446.281546:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677446.281548:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:13.0:1619677446.281550:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677446.281551:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:13.0:1619677446.281551:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:13.0:1619677446.281553:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:13.0:1619677446.281554:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677446.281554:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677446.281555:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677446.281623:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.281626:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.281627:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.281629:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.281630:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.281632:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.283155:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.283157:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.283158:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.283159:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.283161:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.283162:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.283278:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.283281:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.283282:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.283284:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.283285:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.283286:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.283932:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.283935:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.283937:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.283940:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.283940:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.283943:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.283951:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.283954:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.283955:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.283956:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.283956:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.283958:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.283993:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.283996:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.283997:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.283998:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.283999:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.284001:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.284361:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.284364:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.284365:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.284366:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.284368:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.284369:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.284666:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.284669:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.284670:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.284672:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.284673:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.284675:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.285063:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.285065:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.285066:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.285067:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.285069:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.285070:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.285280:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.285282:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.285283:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.285284:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.285286:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.285287:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.285298:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.285301:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.285302:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.285303:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.285303:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.285304:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.285392:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.285394:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.285395:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.285397:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.285398:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.285400:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.285499:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.285502:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.285503:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.285505:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.285506:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.285508:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.286089:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.286092:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.286092:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.286094:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.286095:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.286097:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.286123:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.286125:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.286126:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.286127:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.286129:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.286130:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.286143:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.286147:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.286148:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.286149:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.286149:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.286150:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.286269:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.286271:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.286272:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.286274:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.286275:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.286277:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.286289:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.286292:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.286298:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.286299:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.286299:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.286300:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.286431:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.286434:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.286435:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.286437:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.286438:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.286440:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.286578:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.286580:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.286581:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.286582:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.286584:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.286585:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.286651:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.286653:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.286654:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.286656:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.286657:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.286660:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.286706:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.286708:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.286710:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.286712:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.286713:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.286714:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.286715:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.286716:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.286716:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.286718:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677446.286719:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.286722:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.287345:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.287348:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.287349:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.287351:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.287353:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.287354:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.287361:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.287364:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.287365:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.287366:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.287366:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.287368:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.288123:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.288125:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.288126:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.288128:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.288129:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.288131:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.288413:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.288415:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.288417:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.288418:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.288420:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.288421:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.288891:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.288894:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.288895:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.288897:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.288898:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.288899:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.289130:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.289133:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.289134:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.289136:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.289137:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.289139:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.289147:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.289150:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.289151:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.289151:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.289152:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.289153:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.289213:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.289216:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.289217:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.289219:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.289220:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.289221:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.289733:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.289736:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.289737:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.289739:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.289740:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.289741:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.289818:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.289821:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.289822:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.289823:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.289825:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.289826:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.290257:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.290260:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.290261:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.290262:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.290264:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.290265:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.290283:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.290285:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.290287:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.290287:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.290289:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.290289:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.290755:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.290757:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.290758:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.290760:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.290762:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.290763:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.291342:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.291344:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.291345:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.291348:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.291349:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.291351:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.291721:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.291724:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.291725:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.291727:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.291728:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.291730:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.291821:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.291824:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.291825:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.291826:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.291828:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.291829:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.291840:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.291843:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.291843:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.291844:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.291845:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.291846:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.292059:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.292061:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.292062:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.292064:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.292066:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.292067:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.292593:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.292595:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.292596:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.292598:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.292600:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.292601:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.293340:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.293343:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.293344:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.293346:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.293348:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.293349:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.293992:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.293995:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.293997:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.293999:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.294000:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.294004:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.294076:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.294078:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.294079:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.294080:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.294082:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.294083:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.294093:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.294095:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.294096:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.294097:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.294097:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.294099:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.294276:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.294278:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.294279:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.294281:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.294282:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.294284:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.294300:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.294303:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.294304:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.294304:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.294305:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.294305:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.294478:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.294481:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.294481:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.294484:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.294485:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.294489:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.294507:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.294510:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.294511:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.294513:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.294514:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.294516:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.294660:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.294663:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.294663:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.294665:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.294667:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.294667:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.294915:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.294917:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.294918:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.294919:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.294921:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.294922:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.295152:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.295155:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.295156:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.295158:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.295159:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.295161:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.295508:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.295510:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.295511:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.295513:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.295515:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.295516:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.296753:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.296756:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.296757:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.296759:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.296761:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.296762:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.296770:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.296773:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.296773:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.296774:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.296775:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.296776:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.297146:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.297149:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.297150:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.297151:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.297152:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.297154:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.297700:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.297702:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.297703:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.297705:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.297707:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.297708:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.298365:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.298367:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.298368:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.298370:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.298372:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.298373:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.298941:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.298944:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.298945:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.298946:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.298947:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.298949:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.298958:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.298961:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.298961:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.298962:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.298963:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.298964:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.299404:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.299406:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.299407:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.299409:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.299411:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.299412:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.299421:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.299424:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.299425:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.299425:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.299426:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.299427:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.299893:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.299896:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.299897:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.299898:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.299900:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.299901:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.299967:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.299970:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.299971:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.299973:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.299973:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.299975:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.300163:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.300165:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.300166:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.300167:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.300169:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.300170:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.300359:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.300362:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.300362:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.300364:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.300366:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.300367:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.300517:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.300519:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.300520:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.300521:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.300523:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.300524:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.300674:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.300677:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.300678:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.300679:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.300681:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.300683:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.300692:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.300695:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.300696:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.300697:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.300697:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.300699:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.301371:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.301373:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.301374:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.301376:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.301377:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.301379:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.302002:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.302005:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.302006:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.302008:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.302009:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.302011:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.302109:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.302111:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.302112:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.302113:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:13.0:1619677446.302114:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.302114:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.302115:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.302117:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.302117:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.302118:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.302119:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.302120:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.302120:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.302120:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.302121:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.302123:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.302124:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.302254:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.302257:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.302258:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.302259:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.302260:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.302262:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.302474:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.302477:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.302478:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.302480:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.302481:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.302483:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.302840:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.302843:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.302843:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.302845:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.302846:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.302848:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.303048:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.303051:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.303052:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.303055:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.303055:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.303057:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.303085:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.303088:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.303089:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.303092:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.303093:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.303098:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.303100:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.303102:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.303102:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.303103:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.303103:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.303104:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.304066:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.304069:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.304069:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.304071:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.304073:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.304074:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.304160:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.304162:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.304163:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.304164:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.304166:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.304166:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.304177:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.304180:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.304180:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.304181:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.304182:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.304183:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.304290:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.304292:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.304300:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.304300:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677446.304302:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.304304:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.305209:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.305211:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.305212:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.305214:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.305216:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.305217:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.305386:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.305389:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.305390:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.305392:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.305393:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.305395:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.305617:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.305619:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.305620:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.305621:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.305624:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.305625:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.305994:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.305996:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.305997:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.305999:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.306001:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.306002:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.306003:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.306005:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.306006:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.306007:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.306009:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.306011:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.306011:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.306014:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.306015:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.306015:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.306016:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.306017:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.306680:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.306682:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.306683:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.306685:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.306687:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.306688:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.306695:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.306698:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.306699:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.306700:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.306701:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.306702:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.306722:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.306724:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.306725:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.306727:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.306729:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.306730:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.307229:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.307232:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.307233:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.307235:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.307236:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.307237:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.307462:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.307465:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.307466:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.307468:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.307470:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.307471:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.307583:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.307586:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.307588:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.307590:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.307590:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.307593:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.307748:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.307751:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.307752:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.307754:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.307755:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.307756:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.307929:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.307932:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.307933:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.307934:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.307935:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.307937:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.307950:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.307953:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.307953:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.307954:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.307955:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.307956:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.308385:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.308388:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.308389:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.308391:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.308392:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.308393:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.308426:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.308428:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.308429:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.308430:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.308432:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.308432:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.309022:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.309025:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.309025:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.309027:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.309028:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.309030:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.309586:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.309588:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.309589:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.309591:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.309593:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.309594:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.310014:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.310017:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.310018:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.310019:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.310021:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.310021:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.310035:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.310037:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.310038:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.310038:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.310039:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.310040:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.310198:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.310201:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.310202:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.310204:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.310205:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.310207:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.310217:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.310219:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.310220:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.310221:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.310221:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.310223:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.310237:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.310240:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.310241:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.310242:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.310243:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.310244:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.310511:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.310512:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.310513:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.310514:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:13.0:1619677446.310515:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.310515:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.310516:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.310517:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.310518:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.310519:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.310520:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.310521:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.310940:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.310942:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.310943:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.310945:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.310946:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.310948:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.310974:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677446.310977:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677446.310978:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.310980:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000400:00000200:12.0:1619677446.310980:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.310982:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677446.310982:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.310983:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.310984:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.310984:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.310986:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.310998:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.311000:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.311001:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.311002:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.311003:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.311004:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.311446:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.311449:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.311450:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.311452:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.311453:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.311454:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.312091:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.312094:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.312094:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.312096:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.312097:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.312099:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.312493:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.312496:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.312496:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.312498:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.312500:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.312501:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.312585:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.312587:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.312588:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.312590:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.312591:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.312593:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.314187:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.314190:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.314190:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.314192:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.314194:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.314195:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.314204:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.314206:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.314207:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.314208:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.314208:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.314210:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.314281:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.314284:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.314285:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.314286:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.314288:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.314290:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.314301:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.314304:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.314305:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.314306:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.314306:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.314307:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.315108:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.315110:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.315111:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.315112:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.315114:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.315115:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.315953:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.315956:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.315957:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.315958:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.315960:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.315962:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.316211:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.316214:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.316215:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.316216:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.316218:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.316218:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.316521:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.316524:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.316525:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.316527:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.316528:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.316530:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.316543:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.316546:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.316547:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.316549:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.316549:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.316551:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.317062:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.317065:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.317066:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.317068:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.317069:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.317070:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.317081:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.317084:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.317085:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.317085:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.317086:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.317087:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.319070:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.319072:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.319073:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.319075:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.319077:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.319078:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.320683:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.320685:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.320686:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.320688:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.320690:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.320691:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.320700:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.320702:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.320703:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.320704:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.320704:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.320705:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.321077:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.321079:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.321080:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.321082:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.321084:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.321085:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.321346:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.321348:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.321349:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.321351:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.321353:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.321354:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.321595:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.321597:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.321598:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.321599:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.321601:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.321602:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.321921:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.321924:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.321925:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.321926:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.321928:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.321929:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.322747:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.322750:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.322751:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.322753:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.322754:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.322756:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.322756:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.322757:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.322757:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.322757:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.322758:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.322969:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.322972:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[1] from 10.0.10.111@o2ib7
00000400:00000200:13.0:1619677446.322975:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.111@o2ib7 : PUT - routed
00000800:00000200:12.0:1619677446.322975:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.322977:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.323118:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.323121:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.323121:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.323123:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.323125:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.323126:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.323510:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.323513:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.323514:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.323516:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.323517:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.323519:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.323612:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.323615:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.323616:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.323619:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.323620:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.323625:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.323626:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.323628:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.323628:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.323629:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.323631:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.323633:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.323648:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.323651:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.323652:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.323653:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.323653:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.323655:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.324011:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.324013:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.324014:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.324016:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.324017:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.324018:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.324162:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.324165:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.324166:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.324167:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.324169:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.324170:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.324229:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.324231:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.324232:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.324234:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.324235:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.324236:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.324306:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.324308:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.324309:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.324310:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.324312:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.324314:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.324398:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.324401:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.324402:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.324403:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.324405:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.324406:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.325162:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.325165:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.325166:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.325168:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.325169:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.325170:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.325432:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.325435:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.325436:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.325437:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.325440:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.325441:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.325451:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.325453:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.325454:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.325455:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.325455:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.325457:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.325534:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.325536:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.325537:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.325539:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.325541:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.325541:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.326498:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.326500:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.326501:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.326503:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.326505:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.326506:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.326545:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.326548:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.326549:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.326551:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.326552:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.326553:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.326567:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.326570:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.326571:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.326572:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.326572:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.326574:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.327144:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.327146:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.327147:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.327149:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.327150:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.327152:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.327867:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.327869:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.327870:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.327872:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.327874:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.327876:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.328081:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.328084:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.328085:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.328087:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.328088:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.328091:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.328375:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.328378:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.328379:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.328380:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.328382:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.328383:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.328596:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.328599:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.328600:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.328602:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.328603:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.328604:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.328614:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.328616:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.328617:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.328617:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.328618:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.328619:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.329364:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.329367:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.329368:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.329369:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.329371:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.329372:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.329417:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.329420:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.329420:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.329422:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.329424:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.329425:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.329436:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.329439:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.329440:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.329441:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.329441:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.329443:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.329519:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.329521:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.329522:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.329523:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.329525:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.329526:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.329602:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.329605:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.329606:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.329608:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.329608:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.329611:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.329649:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.329651:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.329652:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.329655:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.329655:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677446.329658:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.329674:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.329677:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.329678:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.329678:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.329679:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.329680:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.329816:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.329819:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.329820:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.329821:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.329822:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.329824:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.330135:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.330137:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.330138:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.330139:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.330141:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.330142:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.330196:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.330198:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.330199:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.330201:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.330202:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.330205:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.330260:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.330263:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.330263:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.330265:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.330267:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.330268:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.330880:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.330882:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.330883:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.330885:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.330887:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.330888:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.330898:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.330900:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.330901:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.330902:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.330904:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.330905:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.330917:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.330920:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.330921:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.330921:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.330922:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.330923:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.331258:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.331261:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.331261:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.331263:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.331265:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.331266:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.331498:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.331501:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.331502:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.331504:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.331505:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.331507:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.331670:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.331672:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.331673:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.331674:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.331676:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.331677:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.331845:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.331848:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.331849:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.331850:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.331852:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.331853:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.331863:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.331866:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.331867:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.331868:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.331868:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.331870:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.331926:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.331929:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.331929:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.331931:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.331933:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.331934:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.332417:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.332420:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.332421:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.332423:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.332424:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.332426:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.333322:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.333325:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.333326:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.333330:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.333330:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.333335:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.333337:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.333340:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.333341:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.333341:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.333342:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.333343:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.333955:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.333957:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.333958:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.333960:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.333961:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.333963:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.334018:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.334020:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.334021:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.334023:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.334024:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.334025:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.334880:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.334883:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.334884:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.334885:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.334887:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.334888:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.334918:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.334920:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.334921:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.334923:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.334924:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.334925:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.334939:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.334942:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.334943:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.334944:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.334945:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.334946:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.335213:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.335215:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.335217:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.335219:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.335219:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.335221:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.335234:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.335236:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.335237:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.335239:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.335240:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.335242:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.335326:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677446.335329:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677446.335332:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677446.335334:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677446.335334:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.335336:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677446.335338:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96743559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677446.335339:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677446.335340:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677446.335341:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677446.335345:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677446.335346:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677446.335349:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677446.335350:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:12.0:1619677446.335351:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677446.335353:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677446.335354:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677446.335354:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677446.335355:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:13.0:1619677446.336049:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.336052:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.336053:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.336054:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.336056:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.336056:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.336636:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.336639:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.336640:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.336641:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.336643:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.336644:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.336659:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.336662:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.336663:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.336663:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.336664:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.336665:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.336726:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.336728:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.336729:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.336731:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.336732:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.336734:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.337202:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.337204:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.337205:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.337207:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.337208:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.337209:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.337675:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.337678:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.337679:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.337681:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.337682:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.337683:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.337749:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.337751:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.337752:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.337754:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.337755:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.337757:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.338064:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.338066:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.338067:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.338068:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.338070:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.338070:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.338084:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.338087:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.338088:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.338088:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.338089:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.338090:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.338370:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.338372:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.338373:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.338375:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.338377:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.338378:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.338873:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.338875:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.338876:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.338878:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.338879:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.338881:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.338941:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.338943:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.338944:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.338945:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.338947:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.338948:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.339770:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.339773:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.339773:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.339775:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.339777:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.339778:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.339787:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.339789:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.339790:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.339790:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.339791:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.339792:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.340021:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.340023:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.340024:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.340026:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.340028:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.340029:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.340165:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.340168:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.340169:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.340171:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.340172:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.340174:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.340429:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.340431:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.340432:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.340434:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.340436:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.340436:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.340519:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.340522:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.340523:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.340525:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.340526:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.340528:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.341095:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.341098:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.341099:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.341100:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.341101:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.341103:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.341115:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.341118:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.341119:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.341119:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.341120:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.341121:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.341482:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.341485:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.341485:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.341487:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.341489:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.341490:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.341592:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.341594:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.341595:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.341596:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.341598:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.341599:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.341612:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.341615:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.341615:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.341616:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.341617:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.341618:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.342028:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.342030:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.342031:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.342033:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.342034:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.342036:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.343011:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.343014:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.343014:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.343016:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.343018:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.343019:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.343139:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.343142:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.343143:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.343145:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.343145:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.343148:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.343159:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.343162:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.343163:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.343164:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.343165:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.343167:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.343531:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.343534:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.343535:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.343539:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.343540:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.343542:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.343797:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.343800:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.343801:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.343803:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.343804:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.343805:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.344037:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.344039:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.344040:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.344041:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.344043:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.344044:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.344541:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.344543:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.344544:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.344546:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.344548:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.344549:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.344607:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.344610:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.344611:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.344613:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.344615:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.344616:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.345417:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.345419:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.345420:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.345421:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.345423:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.345424:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.345434:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.345437:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.345437:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.345438:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.345439:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.345440:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.346637:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.346639:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.346640:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.346642:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:13.0:1619677446.346643:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.346643:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.346644:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.346646:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.346646:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.346647:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.346648:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.346649:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.346818:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.346821:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.346823:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.346824:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.346824:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.346827:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.346838:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.346841:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.346842:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.346842:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.346843:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.346844:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.347042:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.347045:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.347046:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.347048:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.347049:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.347052:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.347243:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.347245:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.347246:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.347248:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.347249:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.347250:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.347535:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.347538:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.347539:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.347541:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.347543:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.347544:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.347695:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.347697:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.347698:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.347700:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.347701:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.347702:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.348434:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.348437:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.348438:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.348440:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.348442:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.348443:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.348536:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.348538:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.348539:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.348541:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.348542:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.348544:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.348556:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.348559:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.348559:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.348560:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.348561:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.348561:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.349244:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.349247:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.349248:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.349250:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.349251:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.349252:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.349262:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.349265:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.349266:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.349267:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.349267:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.349269:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.349920:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.349923:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.349924:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.349925:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.349927:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.349928:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.349969:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.349971:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.349972:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.349974:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.349975:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.349976:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.350272:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.350275:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.350276:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.350277:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.350279:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.350280:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.350765:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.350767:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.350768:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.350770:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.350772:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.350776:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.350793:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.350796:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.350797:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.350798:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.350799:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.350800:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.350854:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.350856:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.350857:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.350858:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.350860:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.350861:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.350938:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677446.350940:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677446.350941:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.350943:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.350944:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.350945:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677446.350946:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.350946:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.350948:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.350948:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.350950:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.350959:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.350962:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.350963:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.350963:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.350964:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.350965:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.351534:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.351537:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.351538:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.351540:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.351541:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.351543:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.351644:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.351646:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.351647:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.351649:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.351650:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.351651:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.351710:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.351712:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.351713:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.351714:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.351716:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.351717:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.351728:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.351731:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.351732:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.351733:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.351733:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.351735:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.351821:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.351823:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.351824:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.351825:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.351826:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677446.351828:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:12.0:1619677446.351828:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.351829:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:13.0:1619677446.351830:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677446.351831:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677446.351832:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.351834:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677446.351836:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96743323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677446.351840:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677446.351841:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677446.351842:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677446.351846:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677446.351848:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677446.351851:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677446.351852:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:13.0:1619677446.351853:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677446.351855:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:13.0:1619677446.351855:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677446.351857:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677446.351858:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677446.351859:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677446.351873:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677446.351875:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677446.351876:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677446.351877:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677446.352480:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.352483:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.352484:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.352486:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.352487:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.352489:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.352498:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.352501:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.352502:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.352502:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.352504:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.352505:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.353031:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.353034:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.353035:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.353036:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.353037:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.353039:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.353484:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.353486:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.353487:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.353488:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.353491:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.353491:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.355331:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.355334:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.355335:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.355338:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.355339:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.355342:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.355646:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.355649:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.355649:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.355651:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.355653:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.355654:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.355662:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.355664:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.355665:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.355666:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.355667:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.355668:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.355735:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.355737:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.355738:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.355740:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.355741:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.355743:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.356206:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.356209:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.356210:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.356211:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.356212:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.356213:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.357126:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.357129:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.357130:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.357132:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.357134:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.357135:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.357172:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.357174:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.357175:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.357178:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.357178:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.357180:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.357191:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.357194:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.357195:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.357196:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.357197:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.357199:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.357520:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.357523:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.357524:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.357526:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.357528:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.357529:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.357649:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.357651:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.357652:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.357653:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.357655:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.357656:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.358237:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.358240:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.358240:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.358242:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.358244:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.358245:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.358856:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.358859:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.0.10.103@o2ib7
00000400:00000200:13.0:1619677446.358861:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.103@o2ib7 : GET - for me
00000400:00000200:13.0:1619677446.358863:0:19942:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.103@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000800:00000200:12.0:1619677446.358863:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677446.358867:0:19942:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.103@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:13.0:1619677446.358868:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.358870:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677446.358872:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:13.0:1619677446.358878:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : REPLY try# 0
00000800:00000200:13.0:1619677446.358881:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.358884:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677446.358885:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.358886:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.358887:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.358888:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.358899:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.358902:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.358903:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.358904:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: REPLY: OK
00000800:00000200:13.0:1619677446.358906:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.358907:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.359812:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.359815:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.359816:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.359818:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.359819:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.359821:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.359866:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.359869:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.359870:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.359872:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.359873:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.359875:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.360191:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.360194:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.360195:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.360196:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.360198:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.360199:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.360207:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.360210:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.360211:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.360211:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.360212:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.360213:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.361215:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.361218:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.361218:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.361220:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.361222:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.361223:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.361593:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.361595:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.361596:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.361598:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.361600:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.361601:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.361610:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.361612:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.361613:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.361614:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.361614:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.361615:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.361909:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.361912:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.361912:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.361914:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.361916:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.361917:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.362025:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.362027:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.362028:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.362030:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.362031:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.362032:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.362464:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.362466:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.362467:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.362469:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.362471:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.362472:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.362484:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.362486:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.362487:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.362488:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.362490:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.362490:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.362529:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.362531:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.362532:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.362534:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.362535:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.362537:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.362923:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.362926:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.362927:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.362928:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.362929:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.362931:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.362943:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.362946:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.362947:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.362947:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.362948:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.362950:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.363014:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.363016:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.363017:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.363019:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.363020:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.363021:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.363338:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.363340:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.363341:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.363343:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.363345:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.363346:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.363414:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.363417:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.363418:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.363420:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.363421:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.363423:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.364107:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.364109:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.364110:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.364112:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.364114:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.364115:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.365626:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.365629:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.365629:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.365631:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.365633:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.365634:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.365646:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.365649:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.365650:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.365650:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.365651:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.365652:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.366140:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.366142:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.366143:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.366144:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.366146:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.366147:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.366159:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.366162:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.366163:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.366164:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.366164:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.366166:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.366224:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.366227:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.366228:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.366230:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.366231:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.366233:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.366233:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.366234:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.366235:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.366235:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.366237:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.366239:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.366846:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.366848:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.366849:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.366850:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.366853:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.366854:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.367085:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.367088:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.367089:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.367091:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.367091:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.367094:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.367491:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.367493:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.367494:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.367496:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.367498:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.367499:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.367708:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.367711:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.367711:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.367713:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.367715:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.367716:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.367725:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.367728:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.367729:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.367730:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.367730:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.367732:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.368446:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.368449:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.368449:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.368451:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.368453:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.368454:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.368681:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.368684:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.368684:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.368686:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.368688:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.368689:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.368984:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.368986:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.368987:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.368989:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.368990:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.368992:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.369229:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.369232:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.369233:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.369235:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.369236:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.369237:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.369250:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.369253:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.369254:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.369254:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.369256:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.369257:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.369278:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.369280:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.369281:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.369283:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.369284:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.369286:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.369976:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.369978:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.369979:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.369981:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.369983:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.369984:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.370325:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.370328:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.370329:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.370330:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.370332:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.370333:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.370342:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.370345:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.370345:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.370346:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.370347:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.370348:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.370847:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.370850:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.370850:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.370852:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.370854:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.370855:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.370887:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.370889:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.370890:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.370892:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.370893:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.370895:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.371070:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.371073:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.371074:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.371076:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.371077:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.371079:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.373246:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.373249:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.373250:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.373251:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.373254:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.373254:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.373613:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.373616:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.373617:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.373618:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.373620:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.373621:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.373630:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.373633:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.373633:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.373634:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.373635:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.373636:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.373671:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.373673:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.373674:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.373676:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.373677:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.373679:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.374153:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.374156:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.374157:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.374158:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.374160:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.374161:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.374429:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.374431:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.374432:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.374434:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.374435:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.374436:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.375057:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.375060:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.375061:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.375062:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.375064:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.375065:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.375867:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677446.375870:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677446.375870:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.375872:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.375874:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.375875:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.375877:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.375878:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.375879:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.375881:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.375882:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.375882:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.375883:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.375883:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.375883:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677446.375884:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677446.375884:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.375885:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.375915:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.375917:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.375918:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.375919:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.375921:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.375922:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.375938:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.375940:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.375941:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.375942:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.375942:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.375943:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.376647:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.376649:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.376650:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.376652:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.376653:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.376655:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.376855:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.376857:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.376858:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.376860:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.376861:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.376863:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.376902:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.376905:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.376906:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.376908:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.376909:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.376911:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.377063:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.377065:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.377066:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.377068:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.377069:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.377070:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.377634:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.377636:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.377637:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.377639:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.377641:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.377642:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.377654:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.377656:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.377657:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.377658:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.377658:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.377659:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.377883:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.377886:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.377887:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.377888:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.377891:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.377892:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.377901:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.377904:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.377905:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.377907:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.377908:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.377909:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.377967:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.377969:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.377970:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.377972:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.377973:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.377974:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.378262:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.378264:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.378265:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.378266:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.378268:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.378269:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.378672:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.378675:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.378676:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.378678:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.378679:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.378681:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.378681:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.378682:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.378682:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.378683:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.378684:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.378901:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.378903:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.378904:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.378905:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.378907:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.378908:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.379305:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.379308:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.379308:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.379310:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.379312:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.379313:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.379314:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.379315:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.379315:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.379316:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.379316:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.381084:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.381086:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.381087:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.381088:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.381090:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.381092:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.381316:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.381319:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.381320:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.381322:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.381324:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.381325:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.382012:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.382014:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.382015:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.382016:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.382018:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.382019:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.382138:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.382141:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.382142:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.382143:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.382145:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.382147:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.382321:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.382324:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.382325:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.382326:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.382328:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.382329:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.384095:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.384098:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.384099:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.384101:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.384102:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.384104:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.384112:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.384114:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.384115:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.384116:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.384116:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.384117:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.384871:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.384874:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.384875:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.384876:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.384879:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.384879:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.385353:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.385355:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.385356:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.385358:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.385360:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.385361:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.385370:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.385373:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.385373:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.385374:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.385375:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.385376:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.385399:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.385402:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.385402:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.385404:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.385405:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.385407:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.385733:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.385736:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.385737:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.385738:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.385740:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.385741:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.386010:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.386013:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.386014:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.386015:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.386017:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.386018:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.386439:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.386442:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.386443:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.386444:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.386446:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.386447:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.386991:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.386994:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.386995:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.386997:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.386998:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.386999:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.387973:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.387976:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.387976:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.387978:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.387980:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.387981:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.387990:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.387992:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.387993:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.387993:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.387994:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.387995:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.388099:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.388102:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.388103:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.388104:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.388106:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.388107:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.388672:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.388675:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.388675:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.388677:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.388679:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.388679:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.389712:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.389714:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.389714:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.389716:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.389718:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.389719:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.389857:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.389860:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.389861:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.389863:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.389864:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.389866:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.389874:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.389877:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.389878:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.389879:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.389879:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.389881:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.390034:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.390036:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.390037:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.390038:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.390040:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.390041:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.390338:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.390341:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.390342:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.390343:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.390345:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.390347:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.390596:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)++
00000800:00000200:13.0:1619677446.390598:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677446.390599:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.390601:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.390602:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677446.390604:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677446.390604:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677446.390605:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:12.0:1619677446.390605:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.390607:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.390607:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000400:00000200:12.0:1619677446.390607:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.390608:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677446.390609:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.390609:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.390616:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.390637:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.390639:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.390640:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.390642:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.390643:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.390645:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.390863:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.390865:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.390867:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.390869:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.390869:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.390872:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.390882:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.390885:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.390886:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.390886:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.390887:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.390889:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.390991:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.390993:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.390994:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.390996:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.390997:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.390998:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.391124:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.391127:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.391128:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.391129:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.391131:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.391132:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.391407:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.391409:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.391410:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.391412:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.391414:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.391415:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.391442:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)++
00000800:00000200:13.0:1619677446.391445:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677446.391446:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.391447:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.391449:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677446.391449:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677446.391450:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.391451:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.391451:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.391452:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.391455:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.391456:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.391458:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.391459:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.391459:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.391460:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.391461:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.391851:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.391853:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.391854:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.391856:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.391857:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.391859:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.392073:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.392075:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.392076:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.392078:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.392079:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.392080:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.392546:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.392548:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.392550:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.392551:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.392553:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.392554:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.392564:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.392566:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.392567:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.392568:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.392569:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.392570:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.392920:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.392922:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.392923:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.392925:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.392926:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.392927:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.393046:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.393049:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.393050:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.393052:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.393053:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.393055:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.393456:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.393458:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.393459:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.393461:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.393463:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.393464:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.393472:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.393475:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.393476:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.393477:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.393477:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.393479:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.394447:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.394450:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.394450:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.394452:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677446.394454:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.394455:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.394928:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.394930:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.394931:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.394934:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.394934:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.394937:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.394968:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.394971:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.394972:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.394974:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.394975:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.394977:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.395146:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.395149:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.395150:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.395152:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.395153:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.395154:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.395505:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.395508:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.395508:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.395510:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.395512:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.395513:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.395525:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.395527:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.395528:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.395529:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.395530:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.395530:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.395758:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.395761:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.395762:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.395764:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.395765:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.395768:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.395789:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.395791:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.395792:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.395794:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.395795:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.395796:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.396080:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.396082:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.396083:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.396084:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.396086:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.396088:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.397206:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.397209:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.397210:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.397212:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.397214:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.397215:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.398050:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.398053:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.398054:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.398056:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.398057:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.398059:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.398072:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.398075:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.398076:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.398076:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.398077:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.398079:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.400032:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.400035:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.400036:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.400037:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.400039:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.400040:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.400636:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.400639:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (25)--
00000400:00000200:12.0:1619677446.400640:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.400641:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.400643:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.400644:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.400646:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.400647:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.400648:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.400649:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.400650:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.400666:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.400669:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.400669:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.400670:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.400671:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.400671:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.401390:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.401393:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.401394:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.401397:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.401398:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.401399:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.401430:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.401433:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.401434:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.401436:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677446.401437:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.401440:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.401737:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.401740:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.401740:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.401742:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.401743:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.401745:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.402104:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.402107:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.402108:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.402110:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.402110:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.402112:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.402209:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.402212:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.402213:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.402215:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.402216:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.402217:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.403174:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.403176:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.403177:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.403179:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.403181:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.403181:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.403190:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.403193:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.403194:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.403195:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.403195:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.403197:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.403369:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.403371:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.403372:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.403374:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.403376:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.403377:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.403388:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.403390:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.403391:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.403391:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.403392:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.403393:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.403469:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.403472:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.403472:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.403474:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.403476:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.403476:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.403548:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.403551:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.403552:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.403553:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.403555:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.403555:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.404057:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.404060:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.404061:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.404062:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.404064:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.404071:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.404121:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.404123:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.404124:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.404125:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.404127:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.404128:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.404520:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.404522:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.404523:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.404525:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.404527:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.404527:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.404528:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.404531:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.404531:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.404533:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.404533:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.404535:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.404627:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.404629:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.404630:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.404631:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.404633:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.404633:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.404646:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.404649:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.404649:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.404650:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.404651:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.404652:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.404806:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677446.404808:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677446.404809:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.404811:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.404812:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677446.404813:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.404813:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677446.404813:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.404815:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.404815:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.404820:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.404821:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.404823:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.404824:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.404824:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.404824:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.404825:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.405015:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.405017:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.405018:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.405020:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.405021:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.405023:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.405438:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.405440:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.405441:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.405443:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.405445:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.405446:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.406222:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.406225:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.406225:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.406227:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.406229:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.406230:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.407131:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.407133:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.407134:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.407136:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.407138:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.407139:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.407161:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.407164:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.407164:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.407166:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.407167:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.407169:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.407180:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.407183:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.407184:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.407185:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.407186:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.407187:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.407489:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.407491:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.407492:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.407494:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.407496:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.407497:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.408723:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.408725:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.408726:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.408727:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.408730:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.408731:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.408982:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.408985:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.408986:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.408987:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.408990:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.408990:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.409659:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.409662:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.409662:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.409664:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.409666:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.409669:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.409685:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.409687:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.409688:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.409689:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.409691:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.409692:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.409712:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.409715:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.409715:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.409716:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.409716:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.409717:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.410325:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.410328:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.410329:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.410331:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.410333:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.410333:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.410343:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.410347:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.410348:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.410349:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.410349:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.410350:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.410375:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.410378:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.410379:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.410381:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.410382:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.410384:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.410449:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.410451:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.410452:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.410453:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.410455:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.410456:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.410782:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.410785:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.410786:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.410788:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.410789:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.410791:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.411006:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.411008:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.411009:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.411010:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.411012:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.411013:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.411077:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.411079:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.411080:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.411081:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.411083:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.411084:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.411259:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.411261:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.411262:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.411264:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.411265:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.411266:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.411278:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.411281:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.411282:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.411283:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.411283:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.411284:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.411768:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.411771:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.411772:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.411774:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.411775:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.411777:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.411860:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.411862:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.411863:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.411865:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.411866:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.411867:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.412903:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.412905:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.412906:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.412907:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.412909:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.412910:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.413275:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.413278:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.413279:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.413281:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.413281:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.413284:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.413293:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.413300:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.413302:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.413303:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.413304:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.413304:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.413477:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.413479:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.413480:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.413481:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.413483:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.413484:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.414068:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.414070:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.414071:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.414073:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.414074:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.414075:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.414319:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.414322:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.414323:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.414324:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.414327:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.414327:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.414621:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.414624:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.414624:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.414626:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.414627:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.414629:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.414738:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.414741:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.414742:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.414744:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.414745:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.414747:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.414757:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.414760:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.414760:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.414761:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.414761:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.414762:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.414921:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.414924:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.414925:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.414926:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.414928:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.414929:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.414942:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.414944:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.414945:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.414946:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.414946:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.414947:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.414997:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.414999:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.415000:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.415001:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.415003:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.415004:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.415552:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.415555:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.415556:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.415557:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.415559:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.415560:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.415803:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.415806:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.415807:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.415808:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.415809:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.415811:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.416009:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.416011:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.416012:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.416013:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.416015:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.416016:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.416281:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.416283:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.416284:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.416286:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.416287:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.416289:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.416306:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.416308:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.416309:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.416311:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.416313:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.416314:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.416315:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.416315:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.416316:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.416316:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.416317:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.416656:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.416658:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.416659:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.416661:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.416662:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.416663:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.417396:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.417399:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.417400:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.417402:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.417404:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.417405:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.417415:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.417417:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.417418:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.417419:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.417419:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.417421:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.417810:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.417813:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.417814:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.417816:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.417817:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.417819:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.417849:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.417851:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.417853:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.417854:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.417855:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.417857:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.417963:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.417965:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.417966:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.417968:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.417969:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.417971:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.419305:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.419308:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.419309:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.419311:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.419312:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.419314:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.419803:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.419806:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.419807:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.419809:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.419810:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.419811:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.420230:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.420233:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.420234:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.420235:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.420237:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.420237:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.420277:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.420280:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.420281:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.420281:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.420282:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.420284:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.420313:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.420315:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.420316:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.420318:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.420319:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.420321:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.420634:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.420637:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.420638:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.420640:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.420641:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.420643:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.420800:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.420802:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.420803:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.420804:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.420806:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.420807:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.420816:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.420819:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.420820:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.420821:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.420821:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.420822:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.421259:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.421262:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.421262:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.421264:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.421265:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.421266:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.421375:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.421378:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.421378:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.421380:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.421382:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.421384:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.422250:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.422252:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.422253:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.422254:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.422256:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.422257:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.422464:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.422467:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.422467:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.422469:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.422472:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.422472:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.422482:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.422484:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.422485:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.422487:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.422488:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.422490:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.422894:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.422897:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.422898:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.422899:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.422900:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.422902:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.422912:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.422915:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.422916:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.422916:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.422917:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.422918:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.423601:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.423604:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.423605:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.423607:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.423609:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.423610:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.423611:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.423612:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.423614:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.423615:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.423616:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.423617:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.423618:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.423619:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.423619:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.423620:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.423621:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.423622:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.424907:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.424909:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.424910:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.424911:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.424914:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.424915:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.425325:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.425328:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.425329:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.425330:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.425332:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.425334:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.425812:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.425815:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.425815:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.425817:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.425819:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.425820:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.425856:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.425858:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.425859:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.425860:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.425862:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.425863:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.427319:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.427322:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.427323:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.427324:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.427327:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.427328:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.427338:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.427341:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.427342:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.427342:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.427343:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.427344:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.427886:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.427888:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.427889:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.427891:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.427892:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.427894:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.427908:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.427911:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.427912:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.427914:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677446.427915:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.427918:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.428531:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677446.428534:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677446.428535:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.428536:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.428538:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677446.428539:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677446.428540:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677446.428540:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.428540:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677446.428542:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.428544:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.428556:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.428558:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.428559:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.428560:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.428560:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.428561:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.428873:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.428875:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.428877:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.428878:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.428879:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.428881:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.429106:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.429109:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.429110:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.429111:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.429112:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.429115:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.429124:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.429127:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.429127:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.429128:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.429129:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.429130:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.429631:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.429634:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.429635:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.429637:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.429638:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.429640:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.429978:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.429981:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.429981:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.429983:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.429984:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.429985:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.430018:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.430020:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.430021:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.430022:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.430024:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.430025:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.430092:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677446.430095:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677446.430099:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677446.430099:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677446.430102:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.430104:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:13.0:1619677446.430105:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96742242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677446.430108:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677446.430110:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677446.430113:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:13.0:1619677446.430119:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677446.430122:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677446.430125:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677446.430126:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677446.430127:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677446.430129:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677446.430130:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677446.430130:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677446.430131:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677446.430439:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.430441:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.430442:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.430443:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.430445:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.430446:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.430742:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.430744:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.430745:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.430747:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.430748:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.430749:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.430850:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.430853:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.430853:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.430855:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.430857:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.430858:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.430869:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.430872:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.430872:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.430874:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.430874:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.430875:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.431070:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.431072:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.431074:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.431076:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.431077:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.431078:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.431879:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.431882:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.431882:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.431884:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.431886:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.431887:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.431895:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.431898:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.431899:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.431899:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.431900:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.431901:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.432338:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.432340:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.432341:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.432342:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.432345:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.432346:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.432452:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.432455:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.432456:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.432457:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.432458:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.432460:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.432578:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.432580:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.432581:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.432583:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.432584:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.432586:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.433198:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.433201:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.433202:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.433204:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.433205:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.433206:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.433217:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.433220:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.433221:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.433221:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.433222:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.433223:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.433356:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.433358:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.433359:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.433360:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.433362:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.433363:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.433478:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.433480:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.433481:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.433483:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.433484:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.433485:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.433745:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.433747:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.433748:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.433749:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.433751:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.433753:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.433906:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.433908:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.433909:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.433911:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.433912:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.433914:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.434335:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.434338:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.434339:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.434340:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.434343:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.434344:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.434418:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.434421:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.434422:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.434424:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.434424:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.434426:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.434486:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.434489:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.434490:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.434493:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.434493:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.434496:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.434505:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.434508:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.434509:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.434510:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.434510:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.434511:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.434899:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.434901:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.434902:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.434904:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.434905:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.434906:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.434934:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.434937:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.434938:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.434938:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.434940:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.434941:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.435008:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.435010:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.435011:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.435012:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.435014:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.435015:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.435336:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.435339:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.435339:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.435341:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.435343:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.435345:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.435549:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.435552:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.435553:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.435554:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.435556:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.435557:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.435850:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.435853:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.435854:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.435856:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.435857:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.435857:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.435980:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.435983:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.435984:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.435986:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.435987:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.435989:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.436131:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.436134:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.436135:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.436137:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.436138:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.436140:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.436392:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.436394:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.436395:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.436396:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.436398:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.436399:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.436410:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.436412:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.436413:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.436414:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.436414:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.436415:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.436797:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.436800:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.436801:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.436802:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.436804:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.436804:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.436817:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.436820:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.436821:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.436822:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.436822:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.436823:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.437554:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.437557:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.437557:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.437559:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677446.437561:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.437562:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.437618:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.437621:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.437622:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.437623:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.437625:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.437626:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.438913:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.438916:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.438917:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.438919:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.438920:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.438922:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.438990:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.438992:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.438993:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.438994:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.438996:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.438997:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.439385:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.439388:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.439389:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.439390:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.439392:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.439393:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.439401:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.439404:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.439405:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.439405:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.439406:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.439407:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.439569:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.439572:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.439573:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.439575:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.439577:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.439577:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.439638:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.439640:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.439641:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.439644:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.439644:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.439646:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.440017:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.440019:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.440020:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.440021:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.440024:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.440025:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.440138:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.440141:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.440142:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.440144:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.440145:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.440147:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.440353:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.440356:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.440356:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.440358:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.440360:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.440361:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.440372:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.440375:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.440375:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.440376:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.440377:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.440377:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.440419:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677446.440423:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:12.0:1619677446.440425:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677446.440426:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677446.440430:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.440431:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:13.0:1619677446.440434:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96742102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677446.440436:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677446.440437:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677446.440439:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:13.0:1619677446.440445:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677446.440448:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:13.0:1619677446.440450:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677446.440451:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:13.0:1619677446.440452:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:13.0:1619677446.440454:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:13.0:1619677446.440454:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677446.440455:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677446.440456:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677446.440762:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.440764:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.440764:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.440766:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:12.0:1619677446.440766:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.440767:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.440768:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.440768:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677446.440769:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.440769:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.440770:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.440772:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.440937:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.440940:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.440941:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.440942:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.440944:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.440944:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.441326:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.441329:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.441330:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.441331:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.441333:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.441334:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.441393:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.441395:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.441396:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.441398:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.441399:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.441401:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.441411:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.441414:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.441415:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.441416:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.441416:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.441417:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.441641:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.441643:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.441644:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.441645:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.441647:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.441648:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.441662:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.441665:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.441666:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.441666:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.441667:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.441668:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.441695:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.441701:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677446.441702:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.441703:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.441704:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.441708:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.441710:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.441712:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.441712:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.441713:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.441714:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.441716:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.444356:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.444358:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.444359:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.444361:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.444363:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.444365:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.445033:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.445036:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.445036:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.445039:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.445039:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.445042:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.445201:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.445204:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.445205:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.445206:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.445208:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.445209:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.445430:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.445433:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.445434:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.445437:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.445437:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.445440:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.445872:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.445874:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.445875:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.445877:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.445878:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.445880:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.445890:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.445893:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.445894:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.445894:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.445895:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.445897:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.446068:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.446070:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.446071:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.446073:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.446074:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.446075:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.446204:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.446206:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.446207:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.446209:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.446210:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.446212:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.446225:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.446227:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.446228:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.446229:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.446230:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.446231:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.446270:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.446273:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.446274:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.446277:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.446277:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.446283:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.446715:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.446718:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.446718:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.446720:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.446722:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.446723:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.446758:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.446760:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.446761:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.446762:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.446764:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.446765:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.446778:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.446781:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.446782:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.446783:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.446783:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.446785:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.446811:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.446813:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.446814:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.446815:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.446817:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.446818:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.447106:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.447108:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.447109:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.447111:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.447112:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.447113:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.448482:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.448485:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.448486:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.448488:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.448489:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.448491:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.448563:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.448565:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.448566:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.448568:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.448569:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.448571:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.449213:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.449216:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.449217:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.449218:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.449220:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.449221:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.450600:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.450602:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.450603:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.450605:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.450607:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.450607:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.450617:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.450620:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.450620:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.450621:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.450622:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.450623:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.451027:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.451030:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.451031:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.451032:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.451034:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.451035:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.451133:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.451136:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.451137:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.451139:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.451140:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.451141:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.451171:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.451173:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.451174:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.451176:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.451177:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.451178:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.451190:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.451192:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.451193:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.451194:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.451194:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.451195:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.451816:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.451818:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.451819:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.451821:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.451823:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.451824:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.451876:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.451878:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.451879:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.451881:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.451882:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.451883:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.452216:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.452219:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.452220:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.452221:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.452223:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.452224:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.453692:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.453695:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.453696:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.453697:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.453699:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.453700:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.453869:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.453872:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.453873:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.453875:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.453876:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.453878:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.453887:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.453890:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.453891:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.453891:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.453892:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.453893:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.454560:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.454562:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.454563:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.454565:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.454567:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.454567:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.454616:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.454618:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.454619:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.454621:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.454622:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.454624:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.454675:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.454677:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.454679:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.454681:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.454681:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.454691:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.454703:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.454705:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.454706:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.454708:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.454709:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.454710:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.454720:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.454723:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.454724:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.454724:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.454725:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.454727:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.454805:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.454807:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.454808:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.454810:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.454811:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.454813:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.455065:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.455067:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.455068:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.455069:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.455071:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.455072:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.455468:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.455471:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.455472:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.455474:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.455476:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.455476:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.455579:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.455582:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.455583:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.455585:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.455586:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.455588:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.455600:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.455603:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.455604:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.455605:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.455605:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.455606:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.455938:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.455940:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.455941:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.455942:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.455944:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.455945:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.456077:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.456080:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.456081:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.456083:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.456084:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.456086:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.456144:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.456146:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.456147:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.456148:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.456150:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.456151:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.456161:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.456163:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.456164:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.456165:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.456165:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.456166:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.456326:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.456329:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.456329:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.456331:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.456333:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.456335:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.456622:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.456624:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.456625:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.456627:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.456628:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.456631:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.456647:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.456649:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.456650:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.456651:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.456653:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.456654:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.457057:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.457060:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.457061:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.457063:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.457065:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677446.457066:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.457067:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.457069:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.457070:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.457071:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.457074:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.457075:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.457076:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.457079:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.457079:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.457081:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.457083:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.457084:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.457124:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.457127:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.457128:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.457129:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.457129:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.457131:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.457279:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.457282:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.457282:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.457284:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.457286:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.457286:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.457298:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.457301:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.457302:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.457302:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.457303:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.457304:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.458566:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.458568:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.458569:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.458570:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.458572:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.458574:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.459152:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.459154:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.459155:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.459157:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.459158:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.459160:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.459487:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.459490:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.459491:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.459493:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.459494:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.459496:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.459786:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.459789:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.459790:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.459792:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.459792:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.459795:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.459837:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.459839:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.459839:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.459839:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.459840:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.459841:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.459843:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677446.459844:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.459844:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:13.0:1619677446.459845:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.459845:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.459848:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.460161:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677446.460164:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677446.460165:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.460166:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.460168:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.460169:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677446.460169:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677446.460170:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.460170:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677446.460171:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.460173:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.460181:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.460184:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.460184:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.460185:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.460186:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.460187:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.460346:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.460349:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.460350:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.460351:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.460353:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.460354:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.460408:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.460410:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.460411:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.460412:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.460414:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.460415:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.460967:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.460970:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.460971:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.460973:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.460974:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.460975:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.460985:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.460988:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.460989:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.460989:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.460990:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.460991:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.461076:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.461078:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.461079:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.461080:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.461082:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.461083:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.461369:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.461372:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.461372:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.461374:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.461376:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.461378:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.461388:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.461390:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.461391:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.461392:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.461392:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.461393:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.461490:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.461492:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.461493:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.461495:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.461496:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.461498:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.461568:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.461570:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.461571:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.461572:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.461574:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.461575:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.462346:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.462349:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.462350:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.462351:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.462353:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.462363:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.464125:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.464128:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.464129:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.464131:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.464132:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.464134:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.464144:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.464147:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.464148:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.464148:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.464149:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.464150:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.464314:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.464317:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.464317:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.464319:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.464321:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.464323:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.464355:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.464358:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.464359:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.464360:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.464361:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.464363:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.464859:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.464862:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.464863:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.464865:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.464866:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.464868:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.465205:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.465208:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.465209:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.465210:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.465212:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.465212:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.465425:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.465428:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.465429:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.465431:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.465433:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.465434:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.465442:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.465444:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.465445:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.465446:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.465446:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.465447:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.465464:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.465466:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.465467:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.465469:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.465470:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.465471:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.465796:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.465799:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.465799:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.465801:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.465803:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.465804:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.465958:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.465960:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.465961:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.465962:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.465964:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.465965:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.465978:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.465980:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.465981:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.465982:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.465983:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.465984:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.466013:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.466015:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.466016:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.466018:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.466019:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.466021:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.466264:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.466267:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.466267:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.466269:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.466271:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.466271:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.466750:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.466752:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.466753:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.466755:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.466757:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.466758:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.466766:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.466769:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.466770:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.466770:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.466771:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.466772:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.466857:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.466860:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.466860:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.466862:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.466863:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.466864:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.467232:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.467234:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677446.467235:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.467236:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.467238:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677446.467238:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677446.467239:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677446.467239:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.467240:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.467241:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.467243:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.467428:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.467430:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.467431:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.467433:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.467435:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.467437:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.468472:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.468475:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.468475:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.468477:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.468479:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.468480:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.468608:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.468611:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.468612:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.468613:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.468615:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.468616:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.468745:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.468747:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.468748:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.468750:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.468751:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.468752:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.468760:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.468763:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.468764:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.468765:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.468766:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.468767:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.469012:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.469014:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.469015:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.469017:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.469018:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.469019:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.469031:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.469034:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.469035:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.469036:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.469037:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.469038:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.469162:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.469165:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.469166:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.469168:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.469169:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.469171:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.469307:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.469310:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.469311:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.469312:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.469314:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.469316:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.469764:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.469767:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.469768:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.469769:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.469770:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.469772:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.470008:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.470010:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.470011:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.470014:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.470015:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677446.470017:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.470548:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.470550:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.470551:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.470552:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.470555:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.470556:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.470566:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.470569:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.470571:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.470571:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.470572:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.470573:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.470926:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.470929:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.470929:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.470931:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.470933:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.470934:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.471658:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677446.471661:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677446.471663:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677446.471665:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.471665:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677446.471666:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:13.0:1619677446.471667:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96741649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677446.471669:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677446.471669:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677446.471671:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:13.0:1619677446.471675:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677446.471677:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677446.471679:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.471680:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677446.471680:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.471680:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.471681:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:13.0:1619677446.471681:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:12.0:1619677446.471682:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.471683:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:13.0:1619677446.471684:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677446.471684:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677446.471685:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:13.0:1619677446.471686:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.471689:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.471706:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.471709:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.471710:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.471711:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.471712:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.471714:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.471845:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.471847:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.471848:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.471849:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.471851:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.471852:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.472507:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.472510:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.472511:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.472513:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.472514:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.472516:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.472525:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.472527:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.472528:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.472529:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.472530:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.472531:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.472700:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.472703:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.472704:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.472706:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.472707:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.472708:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.472791:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.472794:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.472794:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.472796:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.472798:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.472799:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.472842:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.472845:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.472846:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.472847:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.472849:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.472849:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.472867:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.472869:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.472870:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.472870:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.472871:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.472872:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.473912:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.473915:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.473915:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.473917:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.473919:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.473920:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.474008:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.474011:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.474012:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.474013:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.474015:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.474016:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.474017:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.474019:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.474020:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.474021:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.474023:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.474024:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.474233:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.474236:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.474237:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.474238:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.474239:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.474241:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.474576:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.474579:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.474580:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.474582:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.474583:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.474585:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.475991:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.475993:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.475994:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.475996:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.475998:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.475999:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.476010:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.476012:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.476013:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.476014:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.476014:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.476015:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.477247:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.477250:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.477251:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.477253:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.477255:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.477256:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.477266:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.477269:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.477269:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.477270:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.477271:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.477272:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.477465:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.477468:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.477468:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.477470:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.477472:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.477473:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.477691:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.477694:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.477695:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.477696:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.477698:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.477699:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.478135:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.478138:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.478138:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.478140:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.478141:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.478142:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.478514:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.478516:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.478517:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.478519:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.478522:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.478523:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.478879:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.478882:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.478882:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.478884:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.478885:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.478887:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.478987:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.478989:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.478990:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.478992:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.478993:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.478994:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.480017:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.480020:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.480021:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.480022:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.480024:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.480025:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.480036:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.480039:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.480040:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.480041:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.480041:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.480043:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.480225:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.480227:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.480228:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.480229:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.480231:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.480232:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.480305:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.480307:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.480308:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.480310:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.480312:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.480313:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.480595:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.480598:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.480599:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.480600:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.480602:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.480603:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.481239:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.481242:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.481242:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.481244:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.481246:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.481247:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.481259:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.481262:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.481263:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.481264:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.481264:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.481265:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.482260:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.482263:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.482264:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.482266:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.482268:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.482269:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.482721:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.482723:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.482724:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.482726:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.482728:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.482729:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.482845:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.482847:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.482849:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.482850:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.482852:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.482853:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.482865:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.482867:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.482868:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.482868:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.482869:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.482870:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.483187:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.483190:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.483192:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.483194:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.483194:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.483196:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.483208:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.483211:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.483212:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.483213:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.483215:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.483216:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.483399:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.483402:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.483402:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.483404:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.483406:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.483407:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.483415:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.483418:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.483419:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.483420:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.483421:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.483422:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.483672:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.483675:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.483676:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.483677:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.483679:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.483680:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.484056:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.484058:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.484059:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.484062:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.484062:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.484065:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.484091:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.484094:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.484095:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.484097:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.484098:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.484099:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.484110:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.484112:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.484113:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.484113:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.484113:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.484114:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.484114:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.484115:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:12.0:1619677446.484115:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.484116:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.484117:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.484119:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.484167:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.484170:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.484171:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.484172:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.484173:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.484175:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.484293:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677446.484301:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:13.0:1619677446.484302:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677446.484303:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677446.484306:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.484308:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677446.484310:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96741462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677446.484311:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677446.484312:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677446.484313:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677446.484316:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677446.484318:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677446.484321:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677446.484321:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:12.0:1619677446.484322:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677446.484324:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677446.484325:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677446.484326:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677446.484327:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677446.484328:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677446.484342:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677446.484345:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677446.484345:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677446.484346:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:13.0:1619677446.484597:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.484600:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.484601:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.484603:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.484604:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.484606:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.485116:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.485118:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.485119:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.485121:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.485122:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.485123:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.485229:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.485232:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.485233:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.485235:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.485236:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.485238:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.485722:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.485725:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.485726:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.485727:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.485729:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.485730:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.485741:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.485744:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.485745:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.485745:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.485746:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.485747:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.486748:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.486751:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.486752:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.486754:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.486756:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.486756:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.488409:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.488411:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.488412:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.488414:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.488416:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.488417:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.488425:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.488428:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.488429:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.488430:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.488430:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.488432:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.488787:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.488790:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.488791:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.488793:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.488793:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.488796:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.488976:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.488979:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.488980:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.488981:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.488982:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.488984:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.489381:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.489384:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.489385:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.489386:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.489388:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.489390:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.489452:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.489455:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.489456:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.489458:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.489459:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.489460:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.489537:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677446.489540:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677446.489543:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677446.489543:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677446.489546:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.489548:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677446.489549:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96741381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677446.489550:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677446.489551:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677446.489553:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677446.489556:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677446.489558:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677446.489561:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677446.489562:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:13.0:1619677446.489562:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677446.489564:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:13.0:1619677446.489565:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677446.489566:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677446.489567:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677446.489669:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.489672:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.489673:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.489675:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.489676:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.489678:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.490059:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.490061:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.490062:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.490064:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.490065:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.490067:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.490079:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.490082:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.490082:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.490083:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.490083:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.490084:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.490318:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.490320:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.490321:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.490323:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.490326:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.490327:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.490744:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.490746:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.490747:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.490748:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.490750:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.490751:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.490764:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.490767:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.490768:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.490769:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.490770:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.490771:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.490791:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.490794:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.490795:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.490796:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.490798:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.490799:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.491293:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.491299:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.491300:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.491301:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.491302:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.491304:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.491413:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.491416:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.491417:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.491419:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.491420:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.491422:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.491520:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.491523:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.491524:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.491525:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.491527:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.491528:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.493761:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.493763:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.493764:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.493766:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.493768:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.493769:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.493780:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.493783:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.493784:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.493784:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.493785:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.493786:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.494157:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.494160:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.494160:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.494162:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.494164:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.494165:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.494200:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.494202:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.494202:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.494204:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.494206:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.494207:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.494322:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.494324:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.494325:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.494327:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.494329:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.494330:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.494529:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.494531:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.494532:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.494534:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.494535:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.494537:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.494648:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.494651:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.494651:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.494653:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.494655:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.494656:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.494665:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.494667:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.494668:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.494669:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.494669:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.494670:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.494753:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.494755:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.494756:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.494757:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.494759:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.494760:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.494853:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.494857:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.494858:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.494860:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.494861:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.494862:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.495039:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.495042:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.495042:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.495045:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.495046:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.495049:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.495589:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.495591:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.495592:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.495594:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.495596:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.495597:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.495605:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.495607:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.495608:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.495609:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.495609:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.495610:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.495747:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.495750:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.495750:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.495752:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.495753:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.495757:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.496065:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.496068:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.496069:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.496070:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.496071:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.496073:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.496131:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.496134:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.496134:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.496136:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.496137:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.496139:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.496174:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.496177:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.496178:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.496179:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.496181:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.496182:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.496684:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.496687:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.496688:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.496690:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.496692:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.496693:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.496702:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.496704:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.496705:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.496705:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.496706:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.496707:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.496908:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.496910:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.496911:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.496913:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.496915:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.496915:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.497292:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677446.497301:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677446.497303:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677446.497307:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677446.497310:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.497312:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:13.0:1619677446.497314:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96741268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677446.497316:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677446.497317:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677446.497318:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:13.0:1619677446.497325:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677446.497328:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:13.0:1619677446.497331:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677446.497332:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:13.0:1619677446.497333:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:13.0:1619677446.497335:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:13.0:1619677446.497336:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677446.497337:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677446.497338:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677446.497432:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.497434:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.497435:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.497437:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.497439:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.497440:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.498801:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.498803:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.498803:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.498805:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.498805:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.498807:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.498807:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.498807:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.498808:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.498809:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:12.0:1619677446.498809:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.498810:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.498811:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.498811:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.498812:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.498813:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.498814:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.499098:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.499100:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.499101:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.499103:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.499104:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.499105:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.499206:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.499208:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.499209:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.499211:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.499212:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.499213:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.499248:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.499250:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.499251:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.499252:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.499254:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.499254:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.499646:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.499649:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.499650:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.499652:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.499653:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.499654:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.499655:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.499655:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.499656:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.499657:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.499658:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.499683:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.499686:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.499687:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.499689:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.499690:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.499692:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.499776:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.499778:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.499779:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.499781:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.499782:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.499783:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.499795:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.499798:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.499799:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.499799:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.499800:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.499801:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.499978:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.499981:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.499982:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.499985:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677446.499986:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.499989:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.500034:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.500036:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.500037:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.500039:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.500040:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.500041:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.500420:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.500423:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.500423:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.500425:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.500427:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.500428:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.500820:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.500823:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.500824:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.500826:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.500827:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.500829:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.500948:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.500951:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.500952:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.500954:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.500955:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.500957:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.500966:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.500969:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.500970:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.500971:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.500971:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.500973:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.500988:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.500991:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.500992:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.500994:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.500994:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.500996:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.501610:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.501612:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.501613:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.501615:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.501617:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.501618:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.501797:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.501799:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.501800:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.501802:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.501803:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.501805:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.502213:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.502216:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.502217:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.502218:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.502220:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.502221:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.502231:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.502234:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.502234:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.502235:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.502236:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.502237:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.502468:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.502471:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.502471:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.502473:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.502475:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.502476:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.503067:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677446.503070:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677446.503072:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677446.503074:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677446.503075:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.503077:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:12.0:1619677446.503078:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96741189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677446.503079:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677446.503080:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677446.503081:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:12.0:1619677446.503084:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677446.503085:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:12.0:1619677446.503088:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677446.503089:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:12.0:1619677446.503089:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:12.0:1619677446.503092:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:12.0:1619677446.503092:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677446.503093:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677446.503093:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:13.0:1619677446.503382:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.503385:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.503386:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.503387:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.503389:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.503390:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.503399:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.503402:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.503403:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.503403:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.503404:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.503405:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.503674:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.503677:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.503677:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.503679:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.503681:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.503682:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.503907:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.503910:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.503910:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.503912:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.503914:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.503915:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.504665:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.504667:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.504668:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.504670:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.504672:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.504672:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.504729:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.504732:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.504733:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.504734:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.504735:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.504736:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.504817:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.504820:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.504821:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.504823:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.504824:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.504826:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.505047:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.505050:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.505051:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.505052:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.505054:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.505055:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.505065:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.505068:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.505069:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.505069:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.505070:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.505071:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.505100:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.505102:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.505103:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.505105:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.505106:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.505107:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.505371:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.505373:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.505374:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.505376:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.505378:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.505379:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.505781:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.505783:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.505784:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.505784:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:13.0:1619677446.505785:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.505786:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.505786:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.505788:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.505788:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.505789:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.505790:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.505792:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.506357:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677446.506359:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677446.506360:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.506362:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.506364:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677446.506365:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677446.506365:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677446.506366:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.506366:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:13.0:1619677446.506368:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.506368:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.506369:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677446.506370:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677446.506370:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677446.506371:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.506372:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.506584:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.506586:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.506587:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.506588:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.506590:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.506591:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.506601:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.506603:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.506604:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.506605:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.506605:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.506607:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.507341:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.507344:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.507345:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.507346:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.507348:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.507349:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.507631:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.507634:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.507635:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.507636:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.507638:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.507639:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.508089:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.508092:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.508093:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.508094:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.508095:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.508098:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.508106:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.508109:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.508110:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.508111:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.508111:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.508113:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.508763:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.508765:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.508766:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.508768:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.508770:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.508771:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.509127:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.509130:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.509130:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.509132:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.509133:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.509135:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.509173:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.509175:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.509176:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.509178:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.509179:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.509180:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.509230:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.509233:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.509233:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.509235:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.509237:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.509237:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.509250:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.509253:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.509254:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.509254:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.509255:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.509256:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.509737:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.509740:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.509741:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.509742:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.509744:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.509745:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.509807:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.509809:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.509810:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.509812:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.509813:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.509814:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.510301:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.510304:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.510305:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.510307:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.510308:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.510309:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.510482:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.510485:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.510486:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.510488:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.510489:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.510491:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.510510:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.510512:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.510513:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.510515:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.510516:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.510517:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.510529:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.510532:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.510533:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.510533:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.510534:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.510535:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.510890:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.510892:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.510893:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.510895:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.510897:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.510899:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.511660:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.511662:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.511663:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.511665:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.511667:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.511668:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.511769:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.511772:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.511773:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.511775:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.511776:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.511777:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.511865:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.511867:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.511868:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.511870:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.511871:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.511873:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.511883:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.511885:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.511886:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.511887:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.511887:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.511888:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.512756:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.512758:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.512759:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.512760:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.512763:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.512763:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.513012:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677446.513014:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677446.513015:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.513017:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.513019:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677446.513020:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677446.513020:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677446.513021:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.513021:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.513023:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.513025:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.513808:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.513811:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.513812:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.513814:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.513815:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.513816:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.514423:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677446.514426:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677446.514427:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.514428:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.514430:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677446.514431:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677446.514432:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677446.514432:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.514432:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677446.514434:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.514436:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.514443:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.514446:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.514447:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.514447:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.514448:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.514448:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.514596:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.514598:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.514599:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.514601:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.514603:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.514604:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.514893:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.514895:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.514896:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.514898:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.514899:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.514901:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.515008:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.515010:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.515011:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.515012:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.515014:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.515015:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.515167:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.515169:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.515170:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.515172:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.515173:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.515174:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.515183:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.515187:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.515187:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.515188:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.515189:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.515190:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.515473:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.515475:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.515476:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.515478:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.515480:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.515482:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.515615:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.515618:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.515619:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.515620:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.515622:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.515623:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.515635:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.515638:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.515639:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.515640:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.515640:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.515642:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.516019:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.516022:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.516024:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.516025:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.516026:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.516035:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.516096:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.516098:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.516099:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.516100:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.516102:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.516103:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.516138:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.516140:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.516141:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.516143:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.516144:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.516146:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.516421:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.516423:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.516424:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.516426:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.516428:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.516429:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.516708:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.516711:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.516711:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.516713:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.516714:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.516716:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.516857:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.516859:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.516860:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.516861:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.516863:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.516864:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.516875:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.516877:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.516878:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.516879:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.516879:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.516880:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.517589:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.517592:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.517592:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.517595:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.517596:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.517597:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.517702:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.517704:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.517705:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.517707:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.517708:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.517710:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.517721:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.517724:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.517725:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.517725:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.517726:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.517728:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.518160:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.518162:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.518163:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.518164:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.518166:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.518167:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.518459:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.518461:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.518462:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.518463:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.518466:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.518467:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.518937:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.518940:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.518940:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.518942:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.518943:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.518944:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.519221:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.519224:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.519224:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.519226:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.519228:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.519228:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.519240:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.519243:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.519244:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.519244:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.519245:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.519246:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.519791:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.519794:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.519795:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.519796:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.519798:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.519799:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.519925:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.519928:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.519929:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.519931:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677446.519932:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.519935:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.519972:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.519975:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.519975:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.519977:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.519979:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.519979:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.520163:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.520166:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.520166:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.520168:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.520169:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.520170:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.521481:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.521484:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.521484:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.521487:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.521488:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.521490:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.521490:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.521491:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.521491:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.521492:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.521492:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.521934:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.521936:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.521937:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.521939:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.521940:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.521941:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.522390:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.522393:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.522394:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.522396:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.522397:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.522398:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.522415:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.522418:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.522419:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.522420:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.522420:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.522421:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.522505:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.522508:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.522509:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.522510:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.522511:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.522513:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.522661:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.522663:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.522664:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.522666:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.522667:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.522669:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.522794:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.522797:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.522798:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.522800:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.522801:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.522803:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.523351:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.523353:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.523354:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.523355:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.523357:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.523358:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.523732:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.523734:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.523735:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.523737:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.523738:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.523739:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.523867:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.523870:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.523871:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.523873:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.523874:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.523876:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.524289:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.524291:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.524292:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.524296:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677446.524298:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.524302:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.524303:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.524306:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.524308:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.524308:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.524309:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.524310:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.524520:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.524522:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.524523:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.524525:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.524526:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.524527:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.524538:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.524540:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.524542:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.524542:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.524543:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.524544:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.524691:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.524693:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.524694:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.524695:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.524697:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.524698:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.2:1619677446.524863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (21)++
00000800:00000200:13.0:1619677446.524867:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.524870:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.524871:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.524874:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.524874:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.524876:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.525300:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.525303:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.525304:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.525306:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.525308:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.525308:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.525505:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.525508:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.525508:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.525510:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.525511:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.525513:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.526697:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.526699:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.526700:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.526702:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.526704:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.526705:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.526715:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.526718:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.526719:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.526719:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.526720:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.526721:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.527027:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.527030:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.527031:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.527033:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.527033:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.527036:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.527438:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.527441:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.527442:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.527444:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.527445:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.527447:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.527852:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.527854:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.527855:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.527857:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.527858:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.527859:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.528109:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.528111:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.528112:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.528114:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.528116:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.528117:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.528177:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.528179:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.528180:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.528182:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.528183:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.528184:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.528208:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.528210:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.528211:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.528213:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.528214:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.528215:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.528228:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.528231:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.528231:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.528232:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.528232:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.528233:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.528428:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.528431:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.528432:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.528433:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.528435:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.528436:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.528446:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.528449:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.528450:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.528451:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.528451:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.528453:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.529479:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.529481:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.529482:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.529484:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.529486:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.529486:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.529598:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.529601:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.529602:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.529604:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.529605:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.529607:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.530340:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.530342:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.530343:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.530344:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.530347:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.530348:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.530398:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.530400:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.530401:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.530403:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.530404:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.530405:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.530860:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.530863:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.530863:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.530865:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.530866:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.530868:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.531723:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.531726:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.531727:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.531729:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.531731:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.531732:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.531741:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.531743:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.531744:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.531745:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.531745:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.531747:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.532135:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.532137:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.532138:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.532140:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.532141:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.532142:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.532479:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.532482:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.532483:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.532484:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.532486:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.532487:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.532761:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.532763:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.532764:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.532765:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.532767:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.532768:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.532779:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.532782:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.532783:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.532783:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.532784:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.532786:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.532811:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.532814:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.532815:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.532816:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.532817:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.532819:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.533132:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.533134:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.533135:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.533137:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.533139:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.533140:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.533735:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.533738:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.533739:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.533742:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.533743:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.533744:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.533745:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.533746:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.533747:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.533747:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.533748:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.534043:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.534045:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.534046:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.534048:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.534049:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.534051:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.534366:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.534368:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.534369:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.534371:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.534373:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.534374:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.534701:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.534703:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.534704:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.534706:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.534707:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.534708:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.535304:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.535306:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.535307:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.535309:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.535311:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.535312:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.535663:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.535666:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.535666:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.535668:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.535669:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.535671:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.535683:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.535685:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.535686:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.535686:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.535687:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.535689:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.535758:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.535760:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.535761:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.535763:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.535764:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.535765:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.536375:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.536378:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.536379:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.536382:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.536383:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.536385:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.536551:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.536553:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.536554:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.536556:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.536557:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.536559:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.537669:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.537671:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.537672:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.537674:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.537676:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.537677:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.537687:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.537690:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.537691:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.537691:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.537692:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.537693:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.538121:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.538123:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.538124:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.538126:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.538127:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.538129:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.538172:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.538174:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.538175:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.538177:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.538178:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.538179:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.538515:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.538518:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.538518:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.538520:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.538522:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.538523:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.538666:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.538668:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.538669:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.538672:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.538673:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.538674:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.539266:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.539269:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.539270:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.539271:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.539273:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.539274:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.540513:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.540515:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.540516:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.540517:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.540520:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.540521:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.540532:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.540534:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.540535:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.540536:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.540536:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.540537:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.540673:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.540675:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.540676:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.540677:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.540679:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.540680:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.541251:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.541253:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.541254:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.541256:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.541258:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.541258:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.542856:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.542859:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.542860:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.542861:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.542863:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.542864:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.542880:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.542883:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.542883:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.542884:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.542884:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.542886:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.543382:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.543385:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.543386:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.543387:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.543389:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.543390:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.543748:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.543751:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.543752:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.543753:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.543755:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.543756:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.544245:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.544248:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.544249:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.544250:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.544252:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.544253:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.544492:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.544494:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.544495:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.544496:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.544498:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.544503:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.544505:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.544506:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.544507:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.544507:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.544508:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.545147:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.545149:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.545150:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.545151:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.545153:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.545154:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.545710:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.545713:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.545714:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.545715:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.545717:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.545719:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.546351:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.546354:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.546355:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.546357:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.546358:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.546361:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.546370:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.546373:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.546374:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.546375:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.546375:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.546377:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.546460:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.546462:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.546463:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.546465:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.546466:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.546467:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.546637:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.546640:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.546641:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.546643:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.546643:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.546646:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.546992:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.546995:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.546996:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.546997:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.546999:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.547000:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.547553:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.547556:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.547556:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.547558:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.547560:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.547561:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.547569:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.547572:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.547573:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.547573:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.547574:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.547576:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.548450:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.548453:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.548453:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.548456:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.548457:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.548460:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.549205:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.549207:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.549208:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.549210:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.549211:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.549213:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.549353:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.549355:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.549356:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.549358:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.549360:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.549361:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.549372:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.549375:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.549376:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.549377:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.549378:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.549378:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.549416:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.549419:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.549419:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.549421:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.549423:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.549424:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.549868:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.549870:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.549871:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.549873:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.549874:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.549875:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.550202:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.550205:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.550205:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.550207:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.550208:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.550210:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.550744:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.550746:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.550747:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.550749:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.550751:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.550752:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.550821:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.550824:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.550825:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.550828:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.550828:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.550831:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.551128:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.551130:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.551131:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.551133:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.551134:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.551135:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.551147:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.551151:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.551153:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.551153:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)++
00000800:00000200:12.0:1619677446.551154:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677446.551156:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.551163:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.551166:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.551167:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.551168:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.551169:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.551171:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.551231:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.551233:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.551234:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.551236:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.551237:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.551239:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.551417:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.551419:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.551420:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.551422:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.551424:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.551425:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.551839:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.551842:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.551843:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.551845:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.551846:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.551848:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.551862:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.551865:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.551866:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.551867:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.551868:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.551869:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.551991:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.551993:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.551994:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.551995:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.551997:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.551998:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.552071:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.552074:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.552075:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.552076:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.552077:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.552079:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.554563:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.554565:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.554566:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.554568:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.554570:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.554571:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.554747:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.554749:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.554750:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.554751:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.554753:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.554754:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.554997:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.554999:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.555000:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.555001:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.555003:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.555004:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.555016:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.555019:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.555019:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.555020:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.555021:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.555022:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.555161:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.555163:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.555164:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.555165:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.555167:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.555168:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.555180:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.555183:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.555183:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.555185:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.555185:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.555186:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.555253:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.555256:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.555257:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.555259:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.555260:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.555261:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.555350:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.555352:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.555353:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.555355:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.555357:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.555358:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.555791:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677446.555794:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677446.555795:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.555797:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.555798:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677446.555799:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677446.555800:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.555800:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677446.555801:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677446.555802:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.555804:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.555888:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.555891:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.555892:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.555894:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.555895:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.555896:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.556036:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.556038:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.556039:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.556040:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.556042:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.556043:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.556189:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.556191:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.556192:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.556194:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.556195:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.556196:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.556208:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.556211:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.556212:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.556212:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.556213:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.556214:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.556555:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.556558:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.556559:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.556562:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.556562:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.556565:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.556574:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.556577:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.556578:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.556579:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.556579:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.556581:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.557008:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.557010:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.557011:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.557014:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.557014:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.557016:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.557164:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.557167:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.557168:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.557169:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.557171:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.557172:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.557235:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.557241:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.557241:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.557243:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.557245:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.557246:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.557735:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.557738:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.557738:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.557740:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.557742:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.557743:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.559365:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.559367:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.559368:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.559370:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.559372:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.559373:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.560104:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.560107:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.560107:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.560109:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.560111:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.560112:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.560121:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.560124:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.560124:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.560125:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.560126:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.560127:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.560155:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.560158:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.560159:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.560160:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.560163:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.560164:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.560165:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.560167:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.560168:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.560169:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.560171:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.560172:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.560518:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.560521:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.560522:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.560524:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.560526:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.560530:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.560537:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.560540:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.560541:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.560541:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.560542:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.560543:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.560544:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.560547:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.560547:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.560549:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.560551:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.560552:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.560923:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.560925:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.560926:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.560927:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.560929:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.560930:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.561130:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.561133:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.561133:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.561135:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.561136:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.561138:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.561723:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.561725:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.561726:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.561728:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.561730:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.561734:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.561748:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677446.561750:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677446.561751:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.561753:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.561754:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.561755:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677446.561756:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.561756:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.561757:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.561757:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.561758:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.561759:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.561760:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677446.561760:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677446.561761:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.561762:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.562160:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.562163:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.562164:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.562166:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.562167:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.562168:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.562251:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.562254:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.562255:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.562257:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677446.562257:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.562260:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.562273:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.562276:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.562277:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.562279:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.562280:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.562281:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.562295:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.562302:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.562303:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.562304:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.562305:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.562305:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.562618:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.562621:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.562622:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.562623:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.562625:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.562626:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.564781:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.564783:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.564784:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.564785:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.564787:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.564788:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.565009:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.565011:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.565012:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.565013:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.565015:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.565016:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.565775:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.565777:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.565778:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.565779:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.565781:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.565783:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.566276:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.566279:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.566280:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.566282:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.566283:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.566284:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.566294:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.566301:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.566301:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.566302:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.566303:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.566303:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.566534:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.566537:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.566538:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.566540:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.566541:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.566543:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.566558:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.566561:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.566561:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.566562:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.566563:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.566565:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.567237:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.567240:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.567240:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.567243:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.567243:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.567247:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.567280:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.567283:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.567284:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.567286:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.567287:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.567288:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.567347:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.567349:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.567350:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.567351:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.567353:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.567354:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.567616:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.567618:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.567619:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.567621:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.567622:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.567626:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.567951:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.567954:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.567955:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.567956:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.567958:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.567959:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.568138:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.568140:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.568141:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.568143:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.568144:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.568145:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.568157:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.568160:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.568161:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.568162:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.568162:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.568164:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.568211:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.568214:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.568215:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.568216:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.568217:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.568219:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.569043:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.569045:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.569046:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.569047:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.569049:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.569050:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.569107:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.569110:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.569111:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.569113:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.569114:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.569115:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.569612:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.569615:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.569616:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.569618:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.569620:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.569621:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.569648:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.569650:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.569651:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.569652:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.569654:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.569655:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.569664:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.569667:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.569668:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.569669:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.569669:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.569671:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.569692:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.569695:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.569695:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.569698:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.569698:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.569700:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.570222:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.570224:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.570225:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.570227:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.570228:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.570229:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.570243:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.570245:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.570246:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.570247:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.570247:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.570248:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.570953:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.570955:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.570956:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.570958:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.570960:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.570961:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.571322:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.571325:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.571326:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.571328:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.571330:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.571330:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.571675:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.571678:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.571679:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.571680:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.571682:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.571683:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.571750:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.571753:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.571753:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.571755:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.571757:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.571758:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.572102:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.572105:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.572106:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.572108:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.572108:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.572111:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.572121:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.572124:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.572125:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.572126:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.572127:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.572128:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.572331:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.572333:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.572334:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.572336:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.572338:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.572339:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.572524:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.572527:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.572528:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.572529:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.572531:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.572532:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.572628:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.572631:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.572631:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.572633:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.572635:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.572636:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.572647:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.572650:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.572651:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.572651:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.572652:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.572654:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.572692:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.572695:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.572696:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.572698:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.572698:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.572700:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.573301:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.573304:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.573305:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.573307:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.573309:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.573315:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.573333:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.573335:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (25)--
00000400:00000200:13.0:1619677446.573336:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.573338:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.573339:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677446.573340:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677446.573341:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.573341:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.573342:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.573342:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.573344:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.573356:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.573359:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.573359:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.573360:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.573361:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.573362:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.573953:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.573956:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.573956:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.573958:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.573960:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.573961:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.574606:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.574609:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.574609:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.574611:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.574613:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.574614:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.575202:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.575205:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.575205:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.575207:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.575209:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.575210:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.575945:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.575948:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.575948:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.575950:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.575952:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.575953:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.576067:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677446.576070:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677446.576071:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677446.576073:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677446.576074:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.576076:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:13.0:1619677446.576077:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96740174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677446.576079:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677446.576081:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677446.576083:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:13.0:1619677446.576088:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677446.576090:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677446.576094:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677446.576095:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677446.576095:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677446.576097:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677446.576098:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677446.576100:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677446.576102:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677446.576103:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677446.576118:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677446.576120:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677446.576121:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677446.576122:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677446.576524:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.576527:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.576527:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.576529:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.576531:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.576532:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.576639:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.576641:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.576642:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.576644:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.576645:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.576647:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.576657:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.576660:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.576660:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.576662:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.576662:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.576663:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.577287:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.577290:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.577291:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.577293:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.577293:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.577302:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.577303:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.577305:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.577305:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.577306:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.577306:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.577307:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.577777:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.577779:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.577780:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.577781:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.577783:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.577784:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.577991:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.577993:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.577994:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.577996:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.577997:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.577998:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.578362:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.578365:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.578365:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.578367:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.578369:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.578370:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.578522:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.578525:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.578525:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.578527:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.578529:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.578530:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.578541:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.578544:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.578545:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.578545:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.578546:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.578547:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.578812:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.578814:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.578815:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.578816:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.578818:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.578819:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.579202:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.579204:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.579205:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.579207:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.579208:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.579210:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.581304:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.581307:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.581308:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.581310:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.581312:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.581313:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.582142:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.582145:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.582146:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.582148:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.582149:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.582156:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.582157:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.582161:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.582162:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.582162:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.582163:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.582164:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.582172:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.582174:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.582175:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.582177:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.582178:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.582180:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.582408:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.582411:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.582412:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.582414:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.582415:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.582416:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.582858:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.582860:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.582861:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.582864:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.582864:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.582867:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.583263:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.583265:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.583266:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.583268:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.583269:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.583270:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.583622:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.583624:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.583625:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.583627:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.583629:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.583630:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.583631:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.583632:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.583632:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.583632:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.583633:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.583633:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.583634:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.583668:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.583670:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.583671:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.583673:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.583674:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.583675:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.584238:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.584241:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.584242:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.584243:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.584245:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.584246:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.584781:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.584784:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.584785:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.584787:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.584788:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.584790:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.584797:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.584800:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.584801:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.584801:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.584802:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.584804:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.585610:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.585613:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.585614:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.585616:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.585617:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.585619:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.585925:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.585928:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.585929:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.585930:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.585932:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.585933:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.586047:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.586050:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.586051:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.586052:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677446.586054:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.586055:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.586794:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.586796:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.586797:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.586799:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.586801:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.586802:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.587140:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.587142:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.587143:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.587144:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.587146:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.587147:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.587548:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.587550:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.587551:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.587553:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.587555:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.587556:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.587567:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.587570:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.587570:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.587571:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.587571:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.587572:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.587642:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.587645:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.587645:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.587647:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.587649:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.587649:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.587882:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.587884:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.587885:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.587887:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.587888:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.587890:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.587902:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.587905:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.587905:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.587906:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.587907:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.587909:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.588327:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.588329:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.588330:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.588332:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.588334:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.588338:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.588356:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.588359:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.588360:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.588362:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.588363:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.588364:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.588510:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.588513:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.588514:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.588515:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.588516:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.588520:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.588536:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.588538:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.588539:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.588540:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.588542:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.588543:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.589420:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.589422:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.589423:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.589425:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.589426:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.589427:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.589531:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.589534:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.589534:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.589536:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.589538:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.589538:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.590685:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.590688:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.590689:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.590690:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.590692:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.590693:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.590705:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.590708:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.590709:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.590709:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.590710:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.590711:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.590971:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.590974:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.590974:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.590976:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.590978:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.590979:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.591446:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.591448:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.591449:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.591451:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.591453:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.591454:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.591692:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.591695:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.591696:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.591699:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.591699:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.591702:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.591931:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.591933:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.591934:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.591935:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.591937:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.591938:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.591950:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.591953:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.591953:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.591954:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.591954:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.591955:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.592007:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.592010:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.592010:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.592012:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.592014:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.592015:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.592025:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.592028:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.592028:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.592029:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.592030:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.592030:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.592159:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.592162:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.592163:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.592165:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.592166:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.592172:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.592188:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.592191:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677446.592192:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.592194:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.592195:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.592196:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677446.592196:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677446.592197:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.592198:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.592198:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.592199:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.592199:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.592201:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.592443:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.592445:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.592446:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.592447:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.592449:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.592450:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.592536:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.592538:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.592539:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.592541:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.592542:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.592544:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.592640:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.592643:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.592644:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.592645:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.592647:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.592648:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.592658:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.592661:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.592662:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.592662:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.592663:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.592663:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.592827:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.592829:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.592830:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.592832:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.592833:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.592834:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.593313:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.593316:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.593317:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.593319:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.593321:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.593322:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.593909:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.593911:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.593912:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.593914:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.593915:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.593918:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.593928:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.593931:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.593932:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.593932:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.593933:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.593935:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.594319:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.594321:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.594322:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.594323:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.594325:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.594326:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.594692:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.594695:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.594696:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.594698:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.594699:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.594700:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.595235:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.595238:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.595239:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.595241:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.595242:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.595243:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.596384:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.596387:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.596387:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.596389:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.596391:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.596393:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.596724:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.596727:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.596727:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.596729:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.596731:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.596731:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.596962:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.596965:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.596966:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.596968:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.596969:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.596971:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.596971:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.596972:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.596973:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.596973:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.596974:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.597129:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.597131:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.597132:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.597133:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.597135:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.597136:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.597149:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.597152:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.597153:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.597154:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.597154:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.597156:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.597340:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.597343:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.597344:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.597345:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.597347:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.597348:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.597602:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.597604:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.597605:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.597607:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.597608:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.597610:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.597709:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.597711:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.597712:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.597714:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.597716:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.597717:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.597718:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.597720:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.597721:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.597723:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.597724:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.597726:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.597899:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.597901:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.597902:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.597903:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.597905:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.597906:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.598580:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.598582:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.598583:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.598586:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.598587:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.598588:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.598600:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.598603:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.598604:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.598605:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.598606:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.598607:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.599027:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.599030:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.599030:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.599032:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.599033:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.599034:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.599232:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.599235:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.599236:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.599237:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.599239:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.599240:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.599747:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.599749:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.599750:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.599751:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.599754:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.599755:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.600344:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.600345:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.600346:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.600347:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:13.0:1619677446.600348:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.600348:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.600349:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.600350:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.600351:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:12.0:1619677446.600351:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.600353:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.600354:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.600354:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.600354:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.600355:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.600356:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.600357:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.600978:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.600981:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.600982:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.600984:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.600986:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.600987:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.600995:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.600998:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.600999:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.601000:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.601000:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.601002:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.601364:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.601366:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.601367:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.601368:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.601371:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.601372:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.601459:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677446.601462:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677446.601464:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677446.601467:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677446.601467:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.601469:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:12.0:1619677446.601470:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96739830] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677446.601472:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677446.601473:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677446.601474:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:12.0:1619677446.601477:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677446.601480:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677446.601483:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677446.601484:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:12.0:1619677446.601484:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:12.0:1619677446.601486:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:12.0:1619677446.601487:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677446.601488:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677446.601488:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:13.0:1619677446.601632:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.601635:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.601636:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.601638:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.601638:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.601642:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.601791:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.601794:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.601795:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.601797:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.601798:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.601799:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.602027:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.602029:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.602030:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.602032:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.602034:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.602035:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.602247:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.602249:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.602250:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.602252:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.602253:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.602254:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.602332:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.602334:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.602335:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.602337:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.602338:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.602340:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.602349:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.602352:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.602353:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.602353:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.602354:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.602355:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.602552:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.602553:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.602554:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.602555:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:12.0:1619677446.602556:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.602557:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.602557:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.602559:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.602559:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.602560:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.602561:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.602563:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.602806:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.602809:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.602810:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.602812:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.602813:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.602815:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.603107:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.603109:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.603110:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.603112:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.603113:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.603118:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.603119:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.603121:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.603122:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.603122:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.603123:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.603124:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.603460:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.603462:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.603463:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.603465:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.603467:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.603468:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.603893:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.603895:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.603896:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.603898:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.603899:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.603901:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.604859:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.604862:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.604862:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.604864:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.604866:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.604867:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.605786:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.605789:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.605790:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.605792:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.605793:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.605795:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.606358:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.606361:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.606362:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.606364:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.606365:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.606367:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.606376:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.606379:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.606381:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.606382:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677446.606382:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677446.606383:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.606392:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.606394:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.606395:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.606396:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.606398:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.606399:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.606408:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.606410:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.606411:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.606413:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.606414:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.606415:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.606425:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.606428:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.606429:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.606429:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.606430:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.606431:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.606577:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.606579:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.606580:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.606582:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.606584:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.606585:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.606644:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.606647:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.606648:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.606650:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.606651:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.606653:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.606850:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.606853:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.606854:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.606855:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.606856:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.606859:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.607035:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.607038:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.607038:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.607040:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.607042:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.607043:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.607099:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.607101:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.607102:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.607104:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.607105:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.607106:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.607116:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.607119:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.607120:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.607120:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.607121:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.607122:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.607428:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.607431:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.607432:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.607433:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.607435:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.607436:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.607635:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.607637:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.607638:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.607640:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.607642:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.607643:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.608003:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.608005:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.608006:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.608007:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.608009:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.608010:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.608308:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.608310:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.608311:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.608313:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.608315:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.608316:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.608317:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.608318:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.608318:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.608319:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677446.608319:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.608319:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.608320:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.608702:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.608705:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.608706:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.608707:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.608709:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.608710:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.609838:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.609840:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.609841:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.609843:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.609845:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.609846:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.610461:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.610463:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.610464:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.610465:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.610467:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.610469:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.610477:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.610480:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.610481:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.610481:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.610482:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.610483:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.610550:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.610552:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.610553:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.610556:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.610556:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.610558:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.610629:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.610632:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.610632:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.610634:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.610636:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.610636:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.611169:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.611171:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.611172:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.611174:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.611175:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.611176:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.611177:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.611179:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.611180:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.611181:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.611183:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.611184:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.611398:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.611401:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.611402:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.611404:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.611405:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.611407:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.611415:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.611418:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.611419:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.611420:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.611420:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.611422:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.611687:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.611689:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.611690:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.611693:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.611693:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.611695:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.611940:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.611942:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.611943:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.611944:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.611946:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.611947:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.611962:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.611964:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.611965:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.611965:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.611966:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.611968:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.612521:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.612523:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.612524:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.612526:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.612528:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.612529:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.612630:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.612632:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.612633:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.612635:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.612637:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.612638:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.612795:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.612798:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.612799:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.612800:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.612802:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.612803:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.612846:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.612849:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.612850:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.612852:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.612853:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.612855:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.613255:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.613257:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.613258:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.613259:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.613261:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.613262:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.613273:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.613276:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.613276:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.613277:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.613277:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.613278:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.613392:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.613394:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.613395:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.613396:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.613398:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.613399:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.615343:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.615346:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.615346:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.615348:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.615350:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.615351:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.615674:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.615677:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.615678:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.615680:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.615681:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.615682:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.615692:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.615695:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.615696:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.615697:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.615697:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.615698:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.616071:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.616073:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.616074:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.616076:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.616078:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.616078:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.616397:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.616400:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.616401:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.616402:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.616404:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.616405:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.616620:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.616622:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.616622:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.616624:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.616626:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.616627:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.616791:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.616793:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.616794:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.616796:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.616797:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.616799:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.616884:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.616887:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.616888:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.616889:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.616891:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.616892:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.617091:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.617093:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.617094:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.617096:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.617097:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.617098:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.617109:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.617112:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.617113:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.617114:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.617114:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.617116:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.617378:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.617381:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.617382:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.617384:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.617385:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.617386:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.617608:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.617611:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.617612:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.617613:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.617615:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.617615:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.617969:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.617971:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.617972:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.617974:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.617975:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.617976:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.617999:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.618000:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.618001:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:12.0:1619677446.618001:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.618002:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.618002:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:13.0:1619677446.618003:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.618004:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677446.618005:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.618006:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.618007:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.618007:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.618008:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.618008:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.618009:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.618009:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.618010:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.618529:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.618531:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.618532:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.618534:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.618536:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.618537:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.618546:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.618548:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.618549:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.618550:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.618550:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.618552:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.618861:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.618864:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.618864:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.618866:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.618867:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.618868:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.619084:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.619087:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.619087:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.619089:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.619091:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.619092:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.619108:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.619110:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.619111:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.619113:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.619114:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.619115:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.619956:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.619958:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.619959:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.619961:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.619963:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.619964:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.620005:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.620008:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.620008:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.620010:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.620011:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.620012:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.620788:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.620791:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.620792:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.620794:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.620795:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.620797:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.620806:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.620809:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.620810:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.620810:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.620812:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.620813:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.621224:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.621227:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.621228:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.621230:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.621231:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.621233:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.621651:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.621653:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.621654:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.621657:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.621658:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.621659:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.621660:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.621661:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.621661:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.621662:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.621662:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.621662:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.621663:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:12.0:1619677446.621663:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.621664:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.621665:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.621668:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.622649:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.622652:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.622653:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.622654:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.622656:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.622657:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.623038:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.623040:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.623041:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.623042:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.623044:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.623046:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.623784:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.623786:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.623787:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.623789:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.623791:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.623792:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.623894:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.623897:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.623897:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.623899:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677446.623900:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.623902:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.623912:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.623914:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.623915:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:12.0:1619677446.623915:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.623916:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.623917:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:12.0:1619677446.623917:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.623918:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.623918:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.623919:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.623921:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.623922:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.624084:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.624086:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.624087:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.624089:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.624090:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.624092:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.624614:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.624617:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.624618:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.624620:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.624621:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.624623:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.624871:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.624873:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.624874:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.624876:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.624877:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.624878:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.626892:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.626895:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.626896:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.626897:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.626899:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.626900:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.627007:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.627010:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.627010:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.627012:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.627013:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.627014:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.627028:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.627031:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.627031:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.627032:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.627032:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.627034:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.627241:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.627244:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.627245:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.627246:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.627248:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.627249:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.627260:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.627264:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.627264:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.627265:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.627266:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.627267:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.627354:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.627357:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.627358:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.627359:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.627361:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.627362:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.627832:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.627835:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.627836:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.627838:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.627839:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.627841:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.627876:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.627878:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.627879:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.627880:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.627882:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.627883:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.628020:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.628022:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.628023:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.628025:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.628026:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.628027:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.628126:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.628129:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.628130:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.628131:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.628133:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.628134:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.628249:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.628252:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.628252:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.628254:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.628255:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.628256:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.628272:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.628274:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.628275:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.628276:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.628276:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.628278:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.628783:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.628786:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.628787:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.628788:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.628790:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.628791:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.629027:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.629030:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.629030:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.629032:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.629034:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.629035:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.629503:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677446.629506:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677446.629507:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.629509:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.629511:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677446.629512:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677446.629512:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677446.629513:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.629513:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677446.629514:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.629517:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.629946:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.629948:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.629949:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.629951:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.629952:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.629954:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.629954:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677446.629954:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.629955:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.629955:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.629956:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.629999:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.630002:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.630003:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.630004:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.630005:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.630007:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.630018:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.630021:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.630022:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.630022:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.630023:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.630024:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677446.630090:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.630092:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.630093:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.630097:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.630099:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.630102:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.630103:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.630103:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.630105:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.630106:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.630106:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.630106:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.630183:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.630198:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677446.630201:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677446.630202:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.630203:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.630204:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.630406:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:12.0:1619677446.630552:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.630555:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.630556:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.630557:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.630559:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.630560:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.630684:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.630687:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677446.630692:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.630695:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.630697:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.630699:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96739418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.630701:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.630702:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.630703:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.630708:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.630711:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.630714:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.630715:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.630715:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.630718:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.630718:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.630719:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.630721:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.630721:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.630969:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.630972:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677446.630973:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.630975:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.630978:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.631291:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.631373:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.631376:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677446.631377:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.631379:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.631382:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.631639:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.631694:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.631696:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.631699:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.631701:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.631703:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677446.631704:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96739403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.631705:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.631706:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.631707:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677446.631712:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.631714:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677446.631717:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.631718:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677446.631718:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677446.631720:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.631721:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.631722:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.631722:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.631733:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.631736:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.631737:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.631740:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.631741:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.631743:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.631831:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.632078:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.632080:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.632083:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.632086:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.632088:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.632089:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96739398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.632090:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.632091:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.632092:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.632095:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.632097:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.632100:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.632100:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.632101:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.632103:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.632104:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.632104:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.632105:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.632153:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.632156:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.632157:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.632158:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.632160:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.632161:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.632166:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:13.0:1619677446.632412:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.632414:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.632415:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.632416:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.632419:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.632420:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.632567:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.632570:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.632571:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.632572:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.632573:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.632575:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.632621:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.632624:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677446.632627:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.632630:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.632631:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.632632:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96739389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.632634:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.632635:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.632636:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.632639:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.632641:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.632644:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.632645:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.632646:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.632647:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.632647:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.632648:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.632649:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.632982:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.632985:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.632986:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.632987:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.632989:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.632990:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.633003:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.633006:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.633007:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.633007:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.633008:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.633009:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.633041:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.633044:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.633045:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.633047:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.633047:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.633050:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.633239:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.633253:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677446.633256:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677446.633257:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.633258:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677446.633258:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.633360:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:13.0:1619677446.633484:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.633487:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.633487:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.633490:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.633491:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.633493:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.633501:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.633504:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.633505:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.633505:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.633506:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.633507:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.633679:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.633681:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.633682:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.633684:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.633685:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.633686:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.634172:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.634175:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677446.634176:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.634178:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.634181:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677446.634315:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.634317:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.634318:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.634320:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.634322:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.634323:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.634416:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.634456:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.634459:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.634459:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.634461:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.634580:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.634593:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677446.634595:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677446.634597:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.634598:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.634598:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677446.634627:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.634629:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.634630:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.634631:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.634633:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.634634:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.634710:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.634754:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.634756:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.634757:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.634758:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.635416:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.635419:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.635420:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.635421:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.635423:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.635424:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.635527:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.635724:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.635727:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.635731:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.635734:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.635736:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677446.635737:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96739346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.635739:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.635740:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.635742:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677446.635746:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.635748:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677446.635752:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.635752:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677446.635753:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677446.635755:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677446.635755:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.635756:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.635757:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.635790:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.636138:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.636141:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677446.636144:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.636147:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.636148:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677446.636149:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96739340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.636150:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.636151:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.636152:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677446.636155:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.636157:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677446.636161:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.636161:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677446.636162:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677446.636164:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677446.636164:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.636165:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.636165:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.636390:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677446.636684:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.636687:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677446.636688:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.636691:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.636693:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.636780:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.636941:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.636944:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.636947:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.636950:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.636952:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677446.636953:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96739325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.636955:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.636955:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.636956:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677446.636965:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.636967:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677446.636969:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.636970:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677446.636970:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677446.636972:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677446.636972:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.636973:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.636974:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.636986:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.636988:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.636989:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.636991:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.636993:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.636994:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.637099:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.637101:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.637102:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:12.0:1619677446.637102:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.637103:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.637103:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.637105:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:12.0:1619677446.637105:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.637108:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.637109:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.637110:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.637111:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.637111:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.637112:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.637112:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.637113:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.637114:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.637420:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.637671:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.637674:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.637676:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.637679:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.637680:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677446.637682:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96739315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.637683:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.637684:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.637685:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677446.637687:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.637689:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677446.637691:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.637692:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677446.637692:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677446.637693:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.637694:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.637695:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.637695:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.637706:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.637709:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.637709:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.637711:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.637713:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.637714:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.637731:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.637734:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.637735:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.637735:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.637736:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.637737:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.637789:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.637791:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.637792:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.637794:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.637795:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.637797:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.637898:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.637901:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.637901:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.637903:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.637905:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.637905:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.638049:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.638089:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.638090:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.638091:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.638092:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.638134:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.638169:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.638171:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.638172:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.638173:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.638395:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.638397:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.638398:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.638400:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.638402:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.638403:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.638720:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:12.0:1619677446.638909:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.638911:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.638912:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.638914:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.638915:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.638916:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.638918:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.638921:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677446.638922:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.638924:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677446.638927:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.638942:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:13.0:1619677446.639099:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.639102:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.639103:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.639104:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.639106:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.639107:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.639288:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.639291:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.639292:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.639297:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.639299:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.639302:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.639596:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.639599:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.639600:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.639602:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.639603:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.639605:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.639617:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.639620:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.639620:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.639621:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.639622:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.639623:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.639957:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.639958:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.639959:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.639960:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:13.0:1619677446.639961:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.639964:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.639964:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.639965:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:12.0:1619677446.639965:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.639967:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.639968:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:12.0:1619677446.639968:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.639969:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.639969:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.639971:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.639971:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.639973:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.640006:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.640008:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.640009:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.640011:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.640012:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.640014:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.640221:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.640224:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677446.640227:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.640230:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.640232:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677446.640233:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96739273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.640235:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.640236:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.640237:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677446.640241:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.640243:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677446.640246:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.640246:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677446.640247:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677446.640249:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677446.640249:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.640250:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.640251:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.640379:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677446.640391:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677446.640546:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677446.640549:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677446.640551:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677446.640553:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677446.640554:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.640556:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:12.0:1619677446.640557:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96739269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677446.640559:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677446.640560:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677446.640561:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:12.0:1619677446.640564:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677446.640566:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:12.0:1619677446.640568:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677446.640569:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:12.0:1619677446.640569:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:12.0:1619677446.640571:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:12.0:1619677446.640571:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677446.640572:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677446.640573:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.0:1619677446.640897:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677446.640900:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000400:00000200:8.0:1619677446.640904:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.640907:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.640908:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:8.0:1619677446.640910:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96739265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.640912:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.640913:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.640914:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:8.0:1619677446.640918:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.640921:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677446.640923:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.640924:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:8.0:1619677446.640924:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677446.640926:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.640926:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677446.640927:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:8.0:1619677446.640929:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:8.0:1619677446.640929:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677446.640945:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:8.0:1619677446.640946:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:8.0:1619677446.640947:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677446.640948:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:8.0:1619677446.640985:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.641107:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.641109:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677446.641111:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.641113:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.641116:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.641129:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:13.0:1619677446.641796:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.641798:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.641799:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.641800:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.641802:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.641804:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.641814:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.641817:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677446.641820:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.641821:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.641822:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677446.641824:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96739255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.641825:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.641826:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.641827:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677446.641830:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.641832:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677446.641834:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.641835:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677446.641836:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677446.641838:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677446.641838:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.641839:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.641839:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.642380:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.642382:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.642383:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.642384:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.642386:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.642387:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.642581:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.642723:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.642725:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677446.642728:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.642731:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.642733:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.642734:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96739238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.642735:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.642736:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.642737:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.642741:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.642743:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.642746:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.642747:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.642747:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.642749:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.642749:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.642751:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.642752:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.642752:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:12.0:1619677446.642770:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.642772:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.642773:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.642774:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.642776:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.642777:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.642934:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.642937:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.642938:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.642939:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.642941:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.642943:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.642952:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.642955:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.642956:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.642957:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.642958:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.642959:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.643258:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.643261:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.643264:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.643267:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.643268:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.643270:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96739232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.643271:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.643272:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.643273:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.643277:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.643279:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.643282:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.643283:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.643283:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.643285:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.643286:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.643286:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.643287:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.643474:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.643784:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.643786:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.643790:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.643793:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.643794:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.643796:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96739225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.643797:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.643798:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.643799:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.643802:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.643804:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.643806:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.643807:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.643808:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.643810:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.643810:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.643811:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.643811:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.644052:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677446.644106:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.644109:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.644109:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.644111:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.644113:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.644117:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:13.0:1619677446.644119:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.644119:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677446.644121:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.644123:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.644127:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677446.644137:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.644140:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.644140:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.644142:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.644144:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.644145:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.644335:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.644338:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.644339:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.644340:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.644342:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.644343:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.644658:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:12.0:1619677446.644701:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.644704:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.644705:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.644707:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.644708:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.644710:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.644718:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.644721:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.644722:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.644723:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.644723:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.644725:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.644778:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.644780:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.644781:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.644783:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.644784:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.644785:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.644912:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.644914:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.644915:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.644916:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.644919:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.644920:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.644921:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.644924:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.644924:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.644926:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.644927:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.644931:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.644932:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.644935:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.644935:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.644935:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.644936:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.644937:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.645015:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.645018:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.645018:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.645021:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.645021:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.645023:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.645036:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.645039:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677446.645041:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.645043:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.645046:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.645267:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:12.0:1619677446.645351:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.645354:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.645355:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.645358:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.645358:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.645361:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.645490:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.645492:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.645494:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.645496:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.645496:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.645498:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.645518:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.645521:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.645524:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.645526:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.645528:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677446.645529:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96739196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.645530:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.645531:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.645532:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677446.645536:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.645537:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677446.645541:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.645542:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677446.645543:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677446.645545:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677446.645546:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.645548:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.645555:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.645558:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.645560:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.645561:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.645562:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.645563:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.645564:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.645589:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.645591:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.645591:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:12.0:1619677446.645756:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.645759:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.645760:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.645761:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.645763:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.645764:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.646754:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.646756:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.646757:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.646759:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.646761:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.646762:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.647289:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.647292:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.647303:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.647306:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.647307:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677446.647309:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96739176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.647310:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.647311:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.647312:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677446.647316:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.647318:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677446.647321:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.647322:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677446.647323:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677446.647325:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677446.647325:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.647326:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.647326:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.647425:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.647434:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.647435:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677446.647437:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677446.647438:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.647439:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677446.647440:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.647441:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.647747:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.647750:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.647753:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.647757:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.647758:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.647760:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96739169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.647761:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.647762:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.647763:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.647766:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.647768:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.647771:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.647772:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.647773:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.647775:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.647775:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.647776:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.647777:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.647958:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.647960:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.647961:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.647963:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.647965:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.647966:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.647979:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.647982:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.647983:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.647984:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.647984:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.647986:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.648120:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.648123:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.648124:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.648126:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.648127:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677446.648129:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.648138:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.648141:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.648142:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.648144:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.648144:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.648146:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.648797:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.648854:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.648856:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677446.648857:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.648860:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.648863:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.649132:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:13.0:1619677446.649157:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.649160:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.649160:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.649162:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.649164:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.649165:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.649177:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.649179:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.649180:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.649181:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.649182:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.649183:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.649286:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.649289:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.649289:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.649292:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.649293:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.649299:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.649505:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.649507:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.649508:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.649509:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.649511:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.649512:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.649678:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.649681:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677446.649682:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.649684:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.649687:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.649689:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:12.0:1619677446.650039:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.650042:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.650043:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.650044:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.650046:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.650047:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.650240:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.650243:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.650244:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.650245:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.650247:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.650248:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.650258:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.650261:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.650263:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.650263:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.650264:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.650265:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.650402:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.650405:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677446.650407:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.650409:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.650412:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.650424:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.650459:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.650461:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677446.650464:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.650466:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.650468:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.650470:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96739131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.650471:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.650472:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.650473:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.650478:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.650482:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.650485:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.650486:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.650487:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.650489:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.650489:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.650491:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.650492:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.650492:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:12.0:1619677446.650532:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.650534:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.650535:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.650537:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.650539:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.650540:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.651008:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.651011:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.651014:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.651017:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.651018:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.651019:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96739123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.651021:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.651021:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.651022:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.651025:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.651027:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.651029:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.651031:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.651032:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.651033:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.651034:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.651035:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.651035:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.651147:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.651377:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.651380:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677446.651381:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.651384:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.651387:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.651753:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677446.651760:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.651762:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.651763:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.651765:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.651767:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.651768:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.651776:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.651779:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.651780:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.651780:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.651780:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.651781:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.651797:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.651800:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.651801:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.651802:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.651804:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.651805:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.651846:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.651849:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.651850:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.651851:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.651853:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.651854:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.652282:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.652285:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.652288:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.652291:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.652292:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.652300:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96739107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.652301:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.652302:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.652303:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.652307:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.652308:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.652311:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.652312:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.652313:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.652315:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.652316:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.652317:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.652318:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.652319:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677446.652428:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.652431:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.652431:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.652433:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.652435:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.652437:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.652467:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.652469:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.652470:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.652472:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.652473:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.652475:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.652486:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.652489:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.652490:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.652491:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.652491:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.652493:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.652915:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.652918:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.652923:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.652926:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.652928:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677446.652929:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96739099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.652932:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.652934:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.652935:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677446.652940:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.652942:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677446.652945:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.652946:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677446.652947:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677446.652949:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677446.652949:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.652950:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.652951:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.653201:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.653203:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677446.653205:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.653208:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.653217:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.653218:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96739090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.653220:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.653221:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.653222:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.653225:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.653227:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.653230:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.653231:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.653231:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.653241:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.653241:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.653243:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.653244:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.653244:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.653667:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677446.653715:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.653717:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.653718:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.653720:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.653722:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.653723:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.653820:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.653822:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.653823:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.653825:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.653826:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.653827:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.654041:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.654044:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.654046:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.654047:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.654048:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.654050:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.654111:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.654113:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677446.654115:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.654117:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.654120:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677446.654129:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.654132:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.654133:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.654134:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.654136:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.654137:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.654340:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:12.0:1619677446.654698:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.654701:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.654702:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.654704:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.654706:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.654707:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.654715:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.654717:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.654718:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.654719:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.654720:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.654721:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.655299:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.655302:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.655302:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.655304:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.655306:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.655307:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.655400:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.655403:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677446.655404:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.655407:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.655415:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677446.655436:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.655439:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.655440:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.655442:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.655443:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.655445:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.655474:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.655476:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.655477:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.655479:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.655480:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.655482:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.655644:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.655646:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.655647:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.655648:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.655650:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.655651:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.655670:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.655672:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.655673:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.655674:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.655675:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.655676:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.655817:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.655965:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.655968:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.655971:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.655974:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.655976:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.655977:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96739053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.655979:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.655979:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.655981:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.655984:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.655986:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.655989:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.655989:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.655990:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.655992:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.655994:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.655995:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.655996:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.656434:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.656497:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.656499:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.656502:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.656505:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.656506:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.656508:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96739047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.656509:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.656510:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.656511:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.656514:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.656516:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.656518:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.656519:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.656524:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.656527:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.656527:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.656529:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.656531:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.656531:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.656947:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.656950:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677446.656951:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.656954:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.656957:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.656968:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:12.0:1619677446.657001:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.657004:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.657004:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.657006:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677446.657008:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.657009:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.657053:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.657056:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677446.657058:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.657062:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.657064:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.657065:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96739038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.657067:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.657067:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.657069:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.657072:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.657074:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.657078:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.657079:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.657079:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.657081:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.657082:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.657082:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.657083:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.657624:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.657662:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.657664:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.657665:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.657666:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.657952:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.657955:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.657955:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.657957:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.657959:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.657960:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.658086:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677446.658116:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.658118:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.658119:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.658121:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.658122:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.658124:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.658132:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:12.0:1619677446.658133:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.658136:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.658136:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.658137:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.658138:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.658139:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.658140:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.658142:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.658145:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.658147:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677446.658148:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96739024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.658149:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.658151:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.658151:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677446.658155:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.658156:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677446.658159:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.658160:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677446.658161:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677446.658162:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.658163:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.658164:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.658165:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.658175:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.658177:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.658178:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.658180:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.658181:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.658182:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.658246:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.658249:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.658249:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.658251:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.658252:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.658254:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.658356:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.658359:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.658360:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.658362:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.658364:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.658365:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.658506:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.658509:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.658509:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.658511:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.658513:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.658514:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.658523:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.658526:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.658527:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.658527:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.658528:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.658529:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.658885:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.658888:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.658889:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.658890:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.658892:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.658893:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.658908:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.659495:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.659498:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.659501:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.659504:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.659506:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677446.659507:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96739005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.659508:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.659509:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.659510:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677446.659513:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.659515:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677446.659518:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.659519:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677446.659520:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677446.659522:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677446.659522:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.659522:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.659523:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.659615:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:13.0:1619677446.660261:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.660264:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.660265:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.660266:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.660268:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.660269:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.660269:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.660271:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.660274:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.660277:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.660278:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677446.660279:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.660282:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.660283:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.660284:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677446.660287:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.660289:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677446.660292:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.660292:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677446.660299:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677446.660300:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677446.660301:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.660302:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.660302:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.660334:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.660336:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.660337:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.660338:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.660340:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.660342:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.660570:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677446.660711:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.660714:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677446.660715:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.660717:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.660720:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.660733:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:12.0:1619677446.660824:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.660827:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.660828:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.660830:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.660831:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.660832:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.660886:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.660888:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.660889:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.660890:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.660892:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.660893:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.661071:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.661073:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.661074:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.661076:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.661077:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.661081:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.661089:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.661091:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.661092:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.661093:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.661093:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.661094:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.661408:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.661410:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.661413:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.661416:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.661417:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677446.661419:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.661420:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.661421:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.661422:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677446.661425:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.661426:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677446.661429:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.661430:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677446.661431:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677446.661433:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677446.661434:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.661435:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.661437:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.661437:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.661613:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.661616:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677446.661618:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.661621:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.661623:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677446.661624:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.661625:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.661626:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.661627:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677446.661631:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.661632:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677446.661635:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.661636:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677446.661636:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677446.661638:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677446.661639:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.661639:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.661640:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.661813:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.661816:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.661817:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.661818:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.661820:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.661822:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.661880:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677446.661977:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.661979:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.661980:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.661981:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.661984:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.661985:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.661996:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.661999:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.661999:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.662000:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.662001:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.662001:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.662150:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.662153:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677446.662154:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.662156:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.662159:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677446.662379:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.662381:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.662382:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.662384:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.662386:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.662387:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.662405:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.662407:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.662408:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.662410:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.662411:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.662412:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.662705:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.662707:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.662708:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.662710:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.662711:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.662713:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.662782:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.662784:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.662785:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.662786:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.662788:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.662789:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.662799:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.662801:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.662802:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.662803:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.662803:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.662804:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.662871:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.662909:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.662912:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.662913:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677446.663337:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.663339:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.663340:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.663342:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.663344:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.663345:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.663591:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.663593:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.663594:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.663596:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.663597:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.663599:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.663851:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.663854:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677446.663858:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.663861:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.663863:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.663864:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96738943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.663865:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.663866:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.663867:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.663870:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.663871:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.663874:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.663875:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.663876:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.663877:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.663878:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.663878:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.663879:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.663972:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.663975:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.663976:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.663978:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.663979:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.663981:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.664461:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677446.664504:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.664507:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.664508:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.664509:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.664511:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.664512:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.664522:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.664525:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.664526:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.664527:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.664527:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.664529:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.665263:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.665266:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677446.665268:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.665271:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.665273:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.665274:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.665275:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.665276:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.665277:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.665280:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.665281:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.665284:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.665285:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.665286:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.665287:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.665288:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.665288:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.665289:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.665563:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.665565:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.665566:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.665567:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.665569:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.665570:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.665647:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.665649:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.665650:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.665652:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.665653:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.665655:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.666030:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.666033:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.666033:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.666035:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.666037:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.666038:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.666198:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.666201:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.666201:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.666203:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.666204:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.666205:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.666273:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.666301:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.666301:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.666453:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.666455:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.666456:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.666458:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.666460:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.666461:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.666558:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.666560:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.666562:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.666563:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.666564:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.666566:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.666575:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.666577:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.666578:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.666579:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.666580:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.666581:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.666782:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.666784:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.666785:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.666786:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.666788:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.666788:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.666831:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.666834:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.666835:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.666835:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.666837:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.666840:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.666960:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.666963:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.666963:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.666966:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.666970:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.666971:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.666972:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96738905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.666974:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.666975:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.666976:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.666980:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.666982:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.666985:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.666985:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.666986:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.666989:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.666989:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.666990:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.666991:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.667521:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.667524:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.667525:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.667529:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.667532:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.667533:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.667535:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96738897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.667537:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.667538:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.667539:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.667543:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.667545:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.667547:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.667549:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.667550:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.667550:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.667551:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:24.0:1619677446.667552:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.667552:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.667554:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.667555:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.667555:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.667556:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.667556:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.667558:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.667559:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.667571:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.667573:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.667574:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.667575:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.667985:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.667987:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.667988:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.667989:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.667991:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.667992:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.668004:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.668007:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.668008:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.668009:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.668009:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.668010:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.668033:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.668035:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.668036:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.668037:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.668039:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.668042:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.668096:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.668100:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.668100:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.668103:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.668105:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.668107:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677446.668108:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.668109:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.668110:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.668111:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677446.668115:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.668117:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677446.668120:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.668120:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677446.668121:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677446.668123:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677446.668123:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.668124:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.668125:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.668357:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.668360:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.668361:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.668363:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.668366:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.668367:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.668368:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.668370:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.668371:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.668372:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.668375:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.668376:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.668379:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.668380:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.668381:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.668382:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.668383:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.668385:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.668386:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.668386:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.668399:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.668401:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.668402:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.668403:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.668469:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.668472:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.668473:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.668474:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.668477:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.668477:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.669113:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.669116:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.669117:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.669119:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.669122:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.669123:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677446.669125:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.669126:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.669127:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.669128:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677446.669131:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.669133:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677446.669135:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.669136:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677446.669137:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677446.669138:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677446.669139:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.669140:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.669140:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.669349:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.669352:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.669353:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.669355:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.669356:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.669358:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.669480:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.669483:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.669484:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.669486:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.669489:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.669490:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.669491:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.669493:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.669494:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.669495:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.669500:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.669502:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.669505:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.669506:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.669507:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.669509:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.669509:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.669510:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.669511:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.669835:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.669837:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.669838:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.669840:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.669843:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.669845:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.669846:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.669848:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.669849:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.669850:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.669853:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.669855:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.669858:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.669859:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.669859:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.669861:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.669861:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.669862:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.669863:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.669987:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.669989:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.669990:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.669992:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.669993:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.669995:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.670388:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.670391:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.670392:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.670394:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.670397:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.670398:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677446.670399:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.670401:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.670402:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.670402:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677446.670408:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.670410:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677446.670413:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.670414:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677446.670415:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677446.670417:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677446.670417:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.670418:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.670418:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.670430:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.670433:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.670434:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.670434:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.670436:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.670439:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.670450:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677446.670453:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677446.670453:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677446.670454:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.670455:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677446.670456:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.670497:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.670500:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.670501:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.670502:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.670506:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.670507:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.670508:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.670510:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.670511:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.670512:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.670515:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.670517:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.670519:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.670520:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.670520:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.670522:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.670523:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.670523:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.670524:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.670846:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677446.670849:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677446.670850:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677446.670851:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.670854:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.670856:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:24.0:1619677446.670857:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.670859:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.670860:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.670861:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:24.0:1619677446.670864:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.670867:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:24.0:1619677446.670869:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.670869:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:24.0:1619677446.670870:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:24.0:1619677446.670871:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:24.0:1619677446.670872:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677446.670873:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677446.670873:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:13.0:1619677446.671023:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.671025:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.671026:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.671027:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.671030:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.671030:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.671119:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.671122:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.671123:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.671125:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.671127:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.671129:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677446.671130:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.671132:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.671133:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.671134:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677446.671138:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.671140:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677446.671143:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.671144:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677446.671145:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677446.671146:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677446.671147:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.671148:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.671150:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.671150:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.671162:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.671164:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.671165:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.671166:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.671216:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.671218:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.671219:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.671223:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.671226:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.671228:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.671229:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96738853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.671231:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.671233:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.671234:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.671239:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.671242:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.671245:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.671246:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.671247:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.671248:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.671249:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.671250:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.671251:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.671251:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677446.671252:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.671254:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.671255:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.671257:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.671258:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.671260:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.671265:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.671267:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.671268:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.671269:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.671603:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.671605:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.671606:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.671607:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.671610:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.671610:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.671620:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.671622:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.671623:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.671623:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.671624:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.671625:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.671659:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.671661:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.671662:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.671665:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.671665:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.671673:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.671692:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.671694:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.671695:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.671697:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.671698:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.671699:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.671707:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.671709:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.671710:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.671711:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.671713:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.671714:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.671723:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.671726:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.671727:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.671728:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.671728:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.671730:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.671960:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.671963:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.671963:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.671965:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.671967:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.671970:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.671986:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.671988:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.671989:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.671990:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.671992:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.671993:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.672173:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.672176:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.672177:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.672179:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.672180:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.672181:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.672188:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.672191:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.672192:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:13.0:1619677446.672192:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:8.0:1619677446.672193:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.672195:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677446.672196:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.672197:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.672197:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.672198:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.672199:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.672199:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677446.672416:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.672419:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.672420:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.672421:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.672423:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.672424:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.672547:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.672550:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.672551:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.672553:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.672554:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.672556:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.673043:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.673046:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.673047:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.673049:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.673050:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.673051:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.673164:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:12.0:1619677446.673164:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.673166:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:24.0:1619677446.673167:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:12.0:1619677446.673167:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.673168:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.673168:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.673170:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677446.673170:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.673171:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677446.673173:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.673174:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677446.673175:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.673176:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.673177:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.673178:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677446.673181:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.673183:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677446.673186:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.673187:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677446.673187:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677446.673189:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677446.673189:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.673190:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.673191:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.673491:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.673493:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.673494:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.673496:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.673497:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:13.0:1619677446.673498:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.673499:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.673500:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677446.673501:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677446.673503:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.673506:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.673508:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677446.673509:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.673512:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.673513:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.673514:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677446.673518:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.673520:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677446.673524:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.673524:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:8.0:1619677446.673525:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:8.0:1619677446.673527:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:8.0:1619677446.673527:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677446.673528:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677446.673528:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:24.0:1619677446.673803:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.673806:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.673807:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.673809:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.673811:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.673813:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677446.673814:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.673815:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.673816:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.673817:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677446.673822:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.673825:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677446.673827:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.673828:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677446.673829:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677446.673831:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.673831:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.673832:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.673832:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.673843:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.673846:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.673846:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.673848:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.673849:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.673851:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.673868:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.673872:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.673872:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.673873:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.673874:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.673876:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.673902:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.673905:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.673906:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.673908:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.673908:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.673910:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.674903:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.674906:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.674907:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.674907:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.674909:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.674913:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.674993:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.674996:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.674997:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.674998:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.675000:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.675002:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:12.0:1619677446.675002:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:24.0:1619677446.675003:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96738798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.675004:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677446.675004:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:24.0:1619677446.675005:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677446.675005:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.675006:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677446.675006:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.675008:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.675009:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:24.0:1619677446.675010:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.675011:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.675014:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.675015:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.675015:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.675017:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.675017:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.675018:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.675018:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.675500:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.675503:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.675504:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.675506:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.675507:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.675509:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.675768:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.675770:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.675771:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.675773:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.675776:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.675778:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.675780:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.675781:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.675782:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.675783:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.675786:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.675788:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.675791:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.675792:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.675792:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.675794:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.675794:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.675795:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.675795:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.675940:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.675942:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.675943:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.675946:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.675946:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.675950:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.675959:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.675962:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.675962:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.675963:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.675963:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.675964:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.675964:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.675964:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.675966:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:12.0:1619677446.675966:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.675967:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.675969:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.676118:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677446.676121:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677446.676122:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677446.676124:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.676127:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.676128:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677446.676129:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.676131:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.676132:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.676133:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677446.676137:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.676139:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677446.676141:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.676142:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677446.676143:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677446.676144:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677446.676145:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677446.676146:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:24.0:1619677446.676147:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:24.0:1619677446.676148:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677446.676161:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:8.0:1619677446.676163:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:8.0:1619677446.676163:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677446.676164:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:24.0:1619677446.676271:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.676274:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.676275:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.676277:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.676279:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.676281:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677446.676282:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.676283:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.676284:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.676285:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677446.676288:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.676289:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677446.676292:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.676293:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677446.676299:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677446.676301:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677446.676301:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.676302:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.676303:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.676304:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.676317:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.676319:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.676320:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.676321:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.676359:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.676361:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.676362:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.676364:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.676365:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.676367:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.676839:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.676841:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.676843:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.676844:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.676846:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.676847:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.677009:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.677011:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.677012:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.677014:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.677015:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.677017:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.677063:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.677066:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.677067:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.677069:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.677072:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.677073:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.677074:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96738766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.677075:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.677076:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.677077:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.677080:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.677082:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.677085:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.677085:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.677086:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.677088:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.677088:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.677089:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.677089:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.677128:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.677130:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.677131:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.677133:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.677134:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.677135:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.677146:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.677149:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.677149:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.677150:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.677151:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.677151:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.677382:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.677385:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.677386:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.677387:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.677389:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.677390:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.677390:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.677393:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.677394:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.677394:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.677396:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.677398:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677446.677400:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677446.677402:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.677403:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.677404:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.677405:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.677406:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.677563:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.677566:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.677567:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.677567:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.677570:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.677573:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.677583:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677446.677586:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677446.677587:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.677587:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.677589:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.677590:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.677767:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.677769:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.677770:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.677772:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.677775:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.677777:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.677778:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.677779:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.677780:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.677781:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.677785:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.677787:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.677790:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.677790:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.677791:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.677793:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.677793:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.677794:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.677794:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.677833:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda10800] (21)++
00000800:00000200:8.0:1619677446.677836:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.17@o2ib3
00000800:00000200:24.0:1619677446.677837:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda10800] (22)--
00000400:00000200:8.0:1619677446.677839:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.17@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.677842:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.677844:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:8.0:1619677446.677845:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96738755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.677846:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.677847:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.677848:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677446.677852:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.17@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.677855:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:8.0:1619677446.677858:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.677858:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (21)++
00000800:00000200:8.0:1619677446.677859:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:8.0:1619677446.677861:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:8.0:1619677446.677862:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda10800] (21)++
00000800:00000200:8.0:1619677446.677862:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda10800] (22)--
00000800:00000200:8.0:1619677446.677863:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda10800] (21)--
00000800:00000200:12.0:1619677446.677876:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.677879:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.677879:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.677881:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.677883:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.677884:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.678091:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.678093:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.678094:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.678096:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.678097:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.678099:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.678334:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.678337:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.678338:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.678340:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.678342:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.678344:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677446.678345:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.678347:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.678348:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.678349:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677446.678352:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.678353:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677446.678356:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.678357:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677446.678358:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677446.678360:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677446.678360:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.678360:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.678361:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.678613:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.678615:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.678616:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.678618:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.678620:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.678621:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.678817:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.678819:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.678820:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.678821:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.678823:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.678824:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.678865:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.678868:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.678869:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.678870:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.678873:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.678875:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677446.678876:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.678877:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.678878:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.678879:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677446.678882:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.678884:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677446.678886:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.678887:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677446.678888:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677446.678889:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.678890:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.678890:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.678891:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.678901:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.678903:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.678904:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.678905:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.678907:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.678908:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.678920:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.678923:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.678924:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.678924:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.678925:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.678926:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.679086:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.679089:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.679090:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.679092:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.679094:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.679096:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677446.679097:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.679098:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.679099:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.679100:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677446.679104:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.679105:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677446.679108:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.679109:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677446.679109:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677446.679111:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677446.679111:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.679112:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.679113:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.679344:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.679348:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.679350:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.679350:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.679354:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.679356:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.679357:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738737] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.679358:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.679359:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.679360:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.679364:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.679366:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.679369:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.679369:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.679370:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.679372:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.679372:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.679374:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.679375:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.679375:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.679389:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.679391:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.679392:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.679393:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.679837:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.679840:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.679841:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.679843:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.679846:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.679848:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677446.679850:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.679851:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.679852:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.679853:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677446.679859:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.679862:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677446.679865:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.679866:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677446.679867:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677446.679869:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677446.679869:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.679870:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.679871:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677446.680160:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.680163:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.680164:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.680164:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.680166:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.680169:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.680917:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.680919:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.680920:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.680922:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.680925:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.680926:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677446.680927:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.680928:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.680929:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.680930:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677446.680933:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.680935:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677446.680938:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.680939:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677446.680939:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677446.680941:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677446.680941:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.680943:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.680944:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.680945:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.680958:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.680960:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.680960:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.680961:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.681245:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.681248:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.681249:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.681250:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.681253:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.681254:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.681445:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.681447:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.681448:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.681450:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.681452:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.681453:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.681591:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.681593:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.681594:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.681595:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.681597:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.681600:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.681830:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.681833:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.681834:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.681835:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.681838:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.681840:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.681841:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.681842:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.681843:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.681844:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.681847:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.681849:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.681851:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.681852:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.681853:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.681854:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.681855:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.681855:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.681856:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.681893:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.681897:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677446.681898:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.681899:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.681899:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.681901:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.681909:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.681912:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.681912:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.681914:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.681916:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.681917:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.681926:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.681929:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.681929:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.681930:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.681930:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.681931:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.681968:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.681970:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.681971:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.681973:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.681974:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.681976:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.682012:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.682015:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.682016:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677446.682016:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.682019:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.682020:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.682020:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677446.682023:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.682023:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000400:00000200:8.0:1619677446.682023:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.682025:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:12.0:1619677446.682026:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677446.682027:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.682029:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.682030:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.682031:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:8.0:1619677446.682036:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.682039:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.0:1619677446.682041:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.682042:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:8.0:1619677446.682043:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:8.0:1619677446.682044:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:8.0:1619677446.682045:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.682046:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.682047:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.682056:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.682059:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.682059:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.682061:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.682062:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.682064:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.682197:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.682199:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.682201:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.682201:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.682203:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.682206:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.682543:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.682545:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.682546:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.682548:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.682550:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.682552:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.682554:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96738693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.682555:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.682557:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.682558:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:12.0:1619677446.682564:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677446.682566:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677446.682567:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.682567:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.682568:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:12.0:1619677446.682569:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.682571:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.682572:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.682572:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.682572:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.682573:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.682575:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.682576:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.682576:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.682577:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.682581:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.682584:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.682585:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.682586:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677446.682586:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.682588:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.682591:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.682591:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.682593:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.682594:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.682595:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.682917:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.682919:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.682920:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.682922:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.682923:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.682926:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.682942:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.682944:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.682945:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.682947:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.682948:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.682949:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.683009:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.683011:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.683012:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.683013:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.683015:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.683015:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.683028:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.683031:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.683032:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.683033:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.683033:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.683034:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.683079:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.683082:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.683083:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.683085:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.683088:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.683089:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.683090:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.683091:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.683092:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.683093:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.683097:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.683099:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.683102:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.683102:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.683103:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.683104:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.683105:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.683105:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.683106:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.683149:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.683152:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.683153:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.683155:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.683156:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.683157:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.683206:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.683209:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.683210:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.683211:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.683213:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.683214:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.683304:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.683307:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.683308:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.683310:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.683312:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.683313:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.683525:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.683528:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.683529:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.683529:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.683532:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.683535:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.683797:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.683799:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.683800:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.683801:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.683804:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.683806:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.683807:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.683808:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.683809:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.683810:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.683813:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.683814:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.683817:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.683818:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.683819:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.683820:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.683821:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.683822:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.683823:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.683824:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.683837:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.683838:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.683839:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.683840:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.684113:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.684116:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.684117:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.684119:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.684120:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.684121:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.684792:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.684795:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.684795:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.684797:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.684799:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.684800:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.684944:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.684947:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.684948:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.684950:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.684953:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.684955:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677446.684956:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.684957:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.684959:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.684960:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677446.684963:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.684964:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677446.684967:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.684968:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677446.684969:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677446.684970:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677446.684971:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.684972:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.684972:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.685049:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.685052:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.685053:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.685055:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.685063:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.685064:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677446.685065:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96738660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.685067:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.685068:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.685069:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677446.685073:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.685075:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677446.685079:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.685079:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677446.685080:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677446.685082:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677446.685082:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.685083:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.685084:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.685229:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.685232:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.685233:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.685233:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.685237:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.685242:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.685249:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677446.685253:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.685253:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677446.685254:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677446.685255:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.685256:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.685572:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.685575:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.685576:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.685577:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.685580:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.685582:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677446.685583:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.685584:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.685585:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.685586:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677446.685590:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.685592:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677446.685595:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.685595:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677446.685596:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677446.685598:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.685598:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.685600:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.685601:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.685601:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677446.685611:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.685613:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.685614:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.685614:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:13.0:1619677446.685615:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.685616:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.685617:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677446.685618:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.685618:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.685618:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.685630:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.685632:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.685633:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.685634:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.685634:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.685636:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.686570:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.686573:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.686574:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.686576:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.686578:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.686579:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.686822:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.686825:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.686826:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.686828:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.686831:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.686832:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.686833:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.686835:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.686836:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.686837:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.686840:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.686841:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.686844:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.686845:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.686846:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.686848:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.686848:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.686849:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.686850:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677446.686852:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.686854:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.686855:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.686856:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.686858:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.686859:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.686862:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96738636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.686864:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.686864:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.686865:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.686870:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.686872:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.686875:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.686875:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.686876:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.686877:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.686878:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.686878:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.686879:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.687001:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.687003:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.687004:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.687006:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.687008:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.687009:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.687261:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.687264:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.687265:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.687267:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.687268:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.687270:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.687280:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.687282:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.687283:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.687284:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.687284:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.687285:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.687320:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.687322:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.687323:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.687324:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.687326:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.687327:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.687452:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.687455:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.687455:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.687457:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.687459:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.687460:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.687665:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.687668:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.687668:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677446.687671:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.687673:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.687675:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.687677:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.687678:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.687679:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.687680:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.687684:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.687686:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.687689:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.687689:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.687690:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.687692:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.687692:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.687693:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.687693:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.687813:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.687815:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677446.687816:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.687817:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.687819:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.687823:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.687831:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.687833:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677446.687833:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.687834:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677446.687835:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.687836:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677446.687836:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677446.687837:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677446.687838:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.687839:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:8.0:1619677446.687839:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.687841:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.687842:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.687844:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.687845:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.687846:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.687851:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.687854:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.687857:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.687858:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.687859:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.687861:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.687861:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.687862:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.687862:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.688167:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.688169:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.688170:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.688172:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.688173:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.688175:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.688187:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.688190:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.688191:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.688191:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.688192:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.688193:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.688471:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.688474:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.688474:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.688476:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.688478:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.688479:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.688521:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.688524:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.688525:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.688527:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.688530:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.688532:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677446.688533:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.688534:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.688536:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.688537:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677446.688540:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.688541:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677446.688544:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.688545:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677446.688545:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677446.688547:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677446.688548:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.688549:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.688550:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.688550:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.688564:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.688566:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.688567:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.688567:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677446.688731:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.688734:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.688734:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.688736:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.688737:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.688739:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.689173:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.689176:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.689177:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.689180:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.689180:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.689183:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.689818:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.689821:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.689822:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.689824:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.689827:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.689828:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.689829:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.689831:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.689832:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.689833:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.689836:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.689837:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.689841:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.689841:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.689842:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.689843:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.689844:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.689845:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.689845:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.690030:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.690033:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.690034:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.690035:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.690037:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.690038:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.690287:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.690290:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.690291:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.690291:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.690297:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.690300:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677446.690364:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.690367:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.690367:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.690369:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.690371:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.690372:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.690380:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.690383:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.690384:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.690384:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.690385:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.690386:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.690488:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.690490:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.690491:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.690493:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.690495:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.690495:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.690589:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.690592:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.690592:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677446.690594:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.690597:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.690599:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.690600:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.690601:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.690602:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.690603:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.690608:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.690610:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.690614:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.690615:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.690616:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.690617:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.690618:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.690618:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.690619:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.690737:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.690740:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.690741:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.690742:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.690744:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.690744:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677446.691090:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.691093:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.691094:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.691094:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.691097:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.691100:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677446.691164:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.691167:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.691167:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.691169:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.691171:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.691172:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.691653:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.691655:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.691657:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.691658:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.691661:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.691662:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677446.691663:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.691665:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.691666:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.691666:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677446.691669:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.691671:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677446.691674:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.691674:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677446.691675:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677446.691677:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677446.691677:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.691678:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.691678:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.691898:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.691900:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677446.691901:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.691902:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.691904:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.691908:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.692203:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.692206:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.692206:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.692208:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.692211:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.692213:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677446.692214:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.692215:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.692217:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.692217:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677446.692220:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.692222:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677446.692225:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.692226:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677446.692227:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677446.692228:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677446.692229:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.692229:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.692230:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677446.692264:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.692266:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677446.692267:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.692268:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.692270:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.692272:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677446.692368:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.692371:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.692372:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.692375:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.692376:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.692378:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.692519:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.692522:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.692524:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.692530:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.692533:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.692534:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.692536:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96738563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.692537:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.692538:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.692540:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.692543:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.692545:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.692548:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.692549:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.692550:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.692552:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.692552:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.692554:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.692555:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.692556:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.692568:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.692570:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.692571:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.692572:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.692826:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.692829:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.692830:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.692831:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.692833:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.692834:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.692848:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.692850:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.692851:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.692852:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.692853:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.692854:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.692882:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.692885:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.692886:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.692887:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.692888:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.692890:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.692957:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.692960:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.692961:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.692963:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.692967:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.692969:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677446.692970:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96738556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.692971:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.692972:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.692973:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677446.692977:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.692979:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677446.692982:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.692982:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677446.692987:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677446.692989:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677446.692990:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.692991:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.692992:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.692993:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.693006:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.693008:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.693009:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.693010:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.693229:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.693232:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.693233:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.693235:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.693238:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.693239:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677446.693240:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.693241:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.693242:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.693244:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677446.693247:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.693249:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677446.693252:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.693253:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677446.693254:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677446.693256:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677446.693256:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.693257:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.693257:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677446.693306:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.693309:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.693310:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.693310:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.693313:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.693317:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677446.693523:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.693525:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.693526:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.693528:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.693530:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.693531:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677446.693531:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.693534:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677446.693535:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.693536:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.693538:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677446.693539:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677446.693539:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677446.693540:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.693541:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.693542:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:12.0:1619677446.693542:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.693543:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.693544:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.693544:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.693545:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.693546:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.693547:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.693576:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.693578:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.693579:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.693581:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.693582:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.693584:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.693850:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.693853:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.693854:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.693855:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.693858:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.693860:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677446.693861:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.693863:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.693864:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.693865:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677446.693870:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.693872:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677446.693875:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.693876:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677446.693877:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677446.693878:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677446.693879:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.693880:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.693880:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.694075:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.694077:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677446.694078:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.694079:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.694081:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.694084:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677446.694340:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.694342:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.694343:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.694344:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.694346:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.694347:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.694815:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.694817:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.694819:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.694820:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.694823:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.694824:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.694826:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.694827:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.694828:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.694829:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.694832:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.694834:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.694837:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.694838:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.694838:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.694840:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.694840:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.694841:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.694841:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.694898:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.694900:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.694901:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.694903:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.694904:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.694906:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.694931:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.694934:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.694935:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.694937:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.694938:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.694939:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.694954:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.694957:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.694958:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.694958:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.694959:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.694960:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.695323:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.695325:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.695326:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.695328:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.695330:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.695331:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.695339:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.695342:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.695343:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.695343:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.695344:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.695345:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.695418:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.695420:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.695421:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.695422:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.695424:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.695425:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.695664:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.695667:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.695668:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.695670:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.695673:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.695675:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.695676:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.695677:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.695678:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.695680:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.695684:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.695686:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.695688:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.695689:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.695690:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.695691:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.695692:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.695693:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.695693:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.696184:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.696187:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.696187:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.696189:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.696191:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.696192:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.696339:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.696341:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.696343:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.696344:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.696348:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.696349:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677446.696350:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.696351:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.696352:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.696353:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677446.696357:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.696358:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677446.696362:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.696363:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677446.696363:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677446.696365:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677446.696366:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.696366:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.696367:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.696440:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.696442:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.696443:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.696444:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.696446:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.696447:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.696509:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.696512:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.696513:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.696514:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.696517:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.696519:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677446.696520:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.696521:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.696522:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.696523:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677446.696526:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.696528:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677446.696531:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.696531:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677446.696532:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677446.696533:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677446.696534:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.696535:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.696537:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.696537:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.696550:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.696552:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.696553:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.696554:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.696619:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.696622:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.696623:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.696625:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.696626:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.696627:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.697217:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.697219:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.697220:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.697222:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.697224:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.697225:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.697234:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.697237:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.697238:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.697238:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.697240:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.697243:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.697254:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677446.697256:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677446.697257:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.697258:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.697258:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677446.697258:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.697301:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.697304:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.697305:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.697305:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.697308:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.697311:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.697321:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677446.697324:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677446.697325:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.697325:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.697326:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677446.697326:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677446.697558:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.697561:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.697561:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.697563:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.697565:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.697566:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.697751:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.697753:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677446.697754:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.697755:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.697757:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.697760:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.697761:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.697763:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.697765:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.697766:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.697769:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.697770:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677446.697772:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.697773:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.697774:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.697775:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677446.697779:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.697781:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677446.697785:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.697785:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677446.697786:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677446.697788:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677446.697789:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.697790:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.697791:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.697895:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.697898:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.697899:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.697900:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.697903:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.697905:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677446.697906:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.697907:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.697908:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.697909:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677446.697913:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.697914:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677446.697917:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.697918:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677446.697918:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677446.697920:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677446.697921:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.697921:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.697922:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677446.698288:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.698291:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.698292:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.698292:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.698302:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.698304:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677446.698351:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.698353:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.698354:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.698355:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.698357:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.698358:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.698368:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.698370:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.698371:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.698372:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.698372:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.698373:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.698510:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.698512:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.698513:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.698515:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.698516:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.698518:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.698528:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.698531:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.698532:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.698532:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.698533:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.698534:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.698538:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.698540:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.698542:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.698542:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.698544:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.698547:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.698576:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.698579:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.698580:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.698582:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.698584:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.698586:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.698587:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.698588:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.698589:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.698590:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.698594:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.698595:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.698598:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.698599:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.698600:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.698602:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.698602:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.698602:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.698603:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.698958:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.698960:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.698962:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.698963:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.698966:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.698968:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.698969:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.698971:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.698972:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.698973:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.698976:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.698978:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.698981:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.698982:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.698983:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.698985:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.698985:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.698987:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.698988:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.698988:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.699001:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.699003:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.699004:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.699005:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.699207:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.699209:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.699211:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.699212:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.699213:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.699216:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.699706:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.699709:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.699710:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.699712:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.699713:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.699715:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.699911:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.699914:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.699915:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.699917:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.699920:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.699922:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.699923:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.699925:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.699926:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.699927:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.699930:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.699931:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.699934:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.699935:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.699935:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.699937:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.699938:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.699938:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.699938:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.700538:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.700541:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.700542:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.700544:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.700545:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.700547:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.700734:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677446.700737:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677446.700738:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.700740:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.700741:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.700742:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677446.700743:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.700743:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.700744:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.700744:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.700747:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.700842:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.700845:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.700846:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.700848:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.700852:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.700853:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677446.700855:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96738443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.700856:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.700858:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.700859:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677446.700862:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.700864:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677446.700867:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.700868:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677446.700869:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677446.700871:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677446.700872:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.700873:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.700873:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.700882:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.700884:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.700886:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.700887:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.700888:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.700890:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.700929:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.700932:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.700933:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.700936:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.700938:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.700940:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.700942:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.700944:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.700946:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.700947:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.700954:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.700957:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.700961:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.700961:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.700962:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.700964:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.700964:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.700966:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.700967:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.700968:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677446.700975:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.700978:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.700979:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.700980:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:13.0:1619677446.700981:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.700982:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:12.0:1619677446.700982:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677446.700983:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677446.700983:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.700984:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.700995:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.700997:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.700998:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.700999:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.700999:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.701000:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.701046:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.701048:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.701049:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.701051:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.701052:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.701053:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.701064:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.701067:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.701068:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.701069:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677446.701069:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.701071:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.701073:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.701074:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.701075:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.701077:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.701078:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.701290:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.701292:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.701293:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.701300:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.701302:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.701305:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.701380:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.701382:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.701383:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.701384:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.701388:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.701389:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.701390:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96738434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.701392:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.701393:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.701393:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.701397:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.701398:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677446.701401:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:24.0:1619677446.701402:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.701402:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.701403:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677446.701403:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.701404:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.701405:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:13.0:1619677446.701405:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.701406:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.701406:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.701407:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.701407:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.701408:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.701642:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.701645:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.701646:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.701648:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.701651:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.701652:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.701654:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.701655:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.701656:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.701657:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.701660:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.701662:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.701665:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.701666:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.701666:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.701668:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.701668:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.701669:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.701669:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.701885:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.701888:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.701888:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.701890:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.701892:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.701893:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.701993:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.701995:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.701996:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.701997:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.701999:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.701999:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.702236:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.702239:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.702239:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.702241:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.702244:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.702246:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677446.702247:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.702249:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.702249:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.702251:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677446.702254:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.702256:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677446.702259:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.702260:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677446.702261:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677446.702263:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677446.702263:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.702264:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.702264:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.702619:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.702622:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.702623:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.702624:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.702628:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.702629:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677446.702630:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.702632:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.702633:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.702634:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677446.702637:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.702638:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677446.702641:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.702642:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677446.702642:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677446.702644:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677446.702644:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.702645:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.702645:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.702689:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.702691:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.702692:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.702694:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.702697:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.702699:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.702701:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96738418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.702703:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.702704:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.702705:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.702705:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.702706:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677446.702708:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.702709:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:24.0:1619677446.702710:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.702712:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677446.702712:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677446.702713:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:12.0:1619677446.702715:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677446.702715:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:12.0:1619677446.702716:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.702717:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.702718:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.702718:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:12.0:1619677446.702718:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.702720:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.702721:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.702721:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:12.0:1619677446.702721:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.702723:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.702725:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.702725:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677446.702726:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.702727:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.702727:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677446.702727:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677446.702728:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.702729:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.702738:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.702740:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.702741:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.702742:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.703476:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.703479:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.703481:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.703482:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.703485:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.703487:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677446.703488:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96738413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677446.703489:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:8.0:1619677446.703489:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.703490:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677446.703491:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:8.0:1619677446.703491:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:13.0:1619677446.703492:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.703493:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:8.0:1619677446.703495:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677446.703496:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677446.703498:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.703499:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.703501:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.703501:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677446.703502:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677446.703503:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.703504:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.703504:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.703505:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.703515:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.703518:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.703518:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.703520:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.703521:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.703522:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.703539:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.703541:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.703542:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.703543:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.703543:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.703544:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.703859:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.703861:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677446.703862:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:24.0:1619677446.703863:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.703864:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677446.703865:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:8.0:1619677446.703867:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677446.703868:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677446.703869:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:8.0:1619677446.703869:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.703870:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.703871:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677446.703872:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.703874:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.703876:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.703877:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677446.703880:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.703883:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677446.703887:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.703888:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677446.703888:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677446.703890:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677446.703890:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.703891:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.703891:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.704167:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.704170:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.704171:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.704173:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.704174:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.704175:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.704268:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.704270:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.704272:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.704274:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.704276:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.704278:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.704279:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96738404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.704281:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.704282:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.704283:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.704286:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.704288:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.704291:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.704292:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.704292:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.704301:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:12.0:1619677446.704301:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:24.0:1619677446.704302:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.704302:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.704303:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.704304:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.704305:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.704307:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.704309:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.704310:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.704320:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.704323:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.704324:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.704325:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.704326:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.704327:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.704736:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.704739:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.704740:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.704740:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.704742:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.704745:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.704781:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.704783:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.704784:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.704786:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.704788:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.704790:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.704791:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.704793:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.704793:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.704795:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.704798:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.704799:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.704802:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.704803:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.704804:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.704805:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.704806:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.704807:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.704808:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.704809:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.704822:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.704824:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.704824:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.704826:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.704957:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.704959:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.704960:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.704962:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.704963:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.704964:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.705150:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.705152:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.705153:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.705154:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.705156:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.705158:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.705204:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.705206:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.705207:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.705209:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.705210:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.705212:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.705903:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.705906:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.705907:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.705909:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.705911:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677446.705912:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.705923:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.705925:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.705926:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.705927:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677446.705927:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.705930:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.705932:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.705933:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.705934:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.705936:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.705937:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.706244:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.706247:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.706249:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.706251:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.706254:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.706255:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.706257:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.706258:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.706259:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.706260:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.706263:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.706265:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.706268:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.706269:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.706269:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.706271:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.706271:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.706272:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.706273:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.706385:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.706387:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.706388:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.706389:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.706391:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.706394:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677446.706484:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.706487:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.706487:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.706489:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.706491:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.706493:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.706501:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.706503:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.706504:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.706506:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.706508:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.706509:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.706662:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.706665:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.706666:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.706667:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.706669:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.706670:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.706910:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.706913:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.706913:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.706915:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.706917:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.706918:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.706985:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.706988:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.706989:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.706991:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.706994:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.706995:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.706997:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96738357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.706998:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.706999:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.707000:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.707003:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.707005:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.707008:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.707009:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.707010:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.707011:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.707012:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.707012:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.707013:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.707682:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.707685:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.707686:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.707688:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.707690:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.707691:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.707693:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.707694:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.707695:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.707697:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.707701:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.707704:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.707707:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.707708:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.707709:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.707711:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.707711:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.707712:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.707713:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.708111:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.708114:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.708115:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.708115:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.708117:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.708120:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.708341:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.708344:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.708345:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.708347:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.708351:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.708352:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.708354:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.708355:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.708356:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.708357:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.708360:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.708362:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.708365:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.708366:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.708366:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.708368:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.708368:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.708370:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.708371:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.708371:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.708385:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.708387:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.708388:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.708389:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.708782:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.708785:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.708785:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.708787:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.708789:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.708790:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.708793:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.708794:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.708795:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.708795:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.708796:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.708883:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.708885:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.708886:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.708888:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.708891:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.708893:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.708895:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.708896:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.708897:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.708898:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.708901:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.708902:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.708905:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.708906:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.708906:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.708908:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.708909:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.708910:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.708910:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.709150:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.709153:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.709153:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.709155:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.709157:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.709157:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.709170:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.709173:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.709173:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.709174:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.709175:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.709176:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.709232:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.709235:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.709236:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.709237:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.709239:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.709240:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.709395:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.709398:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.709398:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.709400:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.709402:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.709403:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.709729:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.709732:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.709734:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.709736:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.709739:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.709740:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677446.709741:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96738329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.709743:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.709744:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.709745:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677446.709749:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.709751:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677446.709754:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.709754:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677446.709755:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677446.709767:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (23)--
00000800:00000200:8.0:1619677446.709767:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.709768:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.709768:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.710069:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.710072:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.710073:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.710074:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.710076:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.710077:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.710147:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.710150:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.710151:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.710153:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.710155:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.710157:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.710158:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.710160:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.710160:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.710161:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.710165:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.710167:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.710170:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.710170:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.710171:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.710173:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.710173:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.710175:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.710176:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.710176:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.710189:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.710191:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.710192:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.710193:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.710701:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677446.710703:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677446.710704:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.710706:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.710708:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677446.710709:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677446.710709:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677446.710710:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.710710:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677446.710712:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.710714:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.710736:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.710739:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.710740:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.710742:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.710745:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.710747:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.710748:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.710749:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.710750:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.710751:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.710755:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.710756:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.710759:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.710760:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.710761:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.710763:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.710763:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.710764:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.710765:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677446.711146:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.711148:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.711149:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.711151:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.711154:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.711156:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.711157:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738309] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.711158:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.711159:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.711160:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.711163:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.711165:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.711167:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.711168:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.711169:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.711170:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.711171:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.711171:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.711172:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.711195:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.711199:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.711199:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.711202:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.711205:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.711207:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677446.711208:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96738306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.711222:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.711223:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.711225:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677446.711230:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.711232:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677446.711236:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.711236:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677446.711237:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677446.711239:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677446.711239:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.711240:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.711241:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.711592:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.711595:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.711596:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.711598:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.711599:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.711601:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.711609:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.711612:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.711613:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.711614:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.711615:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.711616:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.711753:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.711756:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.711757:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.711759:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.711762:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.711763:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.711765:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.711767:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.711768:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.711770:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.711788:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.711791:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.711794:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.711795:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677446.711796:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000800:00000200:24.0:1619677446.711799:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (23)--
00000800:00000200:24.0:1619677446.711799:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.711800:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.711801:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.711908:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.711910:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.711911:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.711914:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.711914:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.711917:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.712406:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.712409:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.712410:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.712411:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.712415:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.712416:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.712417:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.712419:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.712420:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.712421:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.712425:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.712427:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.712430:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.712430:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.712431:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.712432:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.712433:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.712433:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.712434:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.712462:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.712465:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677446.712466:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.712467:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.712469:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.712472:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677446.712681:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.712683:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.712684:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.712686:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.712688:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.712689:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.712804:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.712807:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.712808:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.712810:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.712811:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.712812:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.712844:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.712847:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.712848:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.712848:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.712852:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.712856:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.712857:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677446.712860:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.712860:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677446.712861:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677446.712862:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.712863:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.712880:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.712883:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.712884:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.712884:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.712886:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.712889:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.712902:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677446.712904:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677446.712905:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.712905:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.712906:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.712907:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.712969:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.712971:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.712972:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.712974:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.712976:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.712978:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.712979:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.712981:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.712981:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.712983:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.712986:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.712988:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.712991:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.712992:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.712992:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.712994:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.712994:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.712996:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.712997:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.712997:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.713010:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.713012:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.713013:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.713014:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.713445:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.713448:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.713449:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.713451:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.713452:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.713454:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.713462:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.713465:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.713466:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.713466:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.713467:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.713468:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.713535:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.713539:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.713539:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.713544:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.713547:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.713550:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.713551:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96738275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.713553:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.713554:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.713555:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.713560:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.713563:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.713566:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.713567:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.713567:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.713569:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.713570:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.713571:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.713571:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.713580:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.713583:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.713583:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.713585:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.713586:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.713588:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.714050:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.714052:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.714053:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.714055:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.714056:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.714057:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.714196:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.714199:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.714200:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.714200:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.714203:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677446.714206:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.714227:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.714229:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.714230:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.714234:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.714236:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.714238:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.714240:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.714242:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.714243:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.714244:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.714253:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.714257:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.714260:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.714261:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.714262:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.714263:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.714264:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.714265:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.714267:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.714269:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.714271:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.714274:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.714275:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.714276:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677446.714276:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677446.714278:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677446.714278:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.714279:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.714281:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.714282:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.714284:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.714999:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.715002:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.715003:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.715006:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.715009:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.715010:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.715012:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96738255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.715014:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.715015:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.715017:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.715021:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.715022:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.715025:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.715026:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.715027:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.715029:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.715030:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.715031:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.715032:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.715033:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.715046:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.715048:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.715049:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.715051:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.715352:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.715355:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.715356:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.715358:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.715361:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.715363:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677446.715364:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.715366:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.715367:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.715368:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677446.715371:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.715372:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677446.715375:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.715376:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677446.715376:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677446.715378:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677446.715378:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.715379:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.715379:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.715513:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.715516:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.715517:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.715519:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.715522:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.715523:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.715525:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96738249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.715526:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.715527:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.715528:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.715531:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.715533:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.715536:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.715537:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.715537:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.715539:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.715540:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.715540:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.715541:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.715842:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.715844:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.715846:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.715847:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.715849:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.715850:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.715858:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.715861:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.715862:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.715862:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.715863:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.715864:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.715962:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.715965:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.715966:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.715968:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.715970:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.715972:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.715973:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.715974:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.715975:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.715976:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.715979:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.715981:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.715984:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.715984:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.715985:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.715987:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.715988:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.715988:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.715989:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.716155:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.716158:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.716158:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.716160:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.716161:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.716162:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.716364:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.716366:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677446.716367:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.716368:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.716370:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.716374:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677446.716490:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.716493:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.716494:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.716496:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.716497:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.716498:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.716759:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.716761:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.716762:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.716763:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.716765:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.716766:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.717011:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.717013:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.717014:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.717016:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.717018:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.717019:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.717457:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.717459:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.717460:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.717462:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.717465:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.717467:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.717468:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.717469:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.717470:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.717471:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.717475:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.717477:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.717479:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.717481:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.717482:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.717484:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.717484:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.717485:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.717485:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677446.717541:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.717543:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677446.717544:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.717545:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.717547:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.717550:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677446.717587:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.717589:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.717590:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.717591:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.717594:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.717595:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.717606:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.717608:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.717609:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.717610:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.717611:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.717612:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.717734:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.717736:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.717737:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.717739:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.717742:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.717744:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.717745:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96738218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.717746:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.717747:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.717748:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.717751:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.717752:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.717755:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.717756:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.717756:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.717758:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.717758:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.717759:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.717761:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.717761:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.717774:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.717777:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.717778:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.717779:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.717823:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.717826:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.717827:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.717827:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.717830:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.717833:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.718477:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.718480:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.718481:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.718482:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.718485:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.718487:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677446.718489:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96738211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.718490:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.718491:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.718492:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677446.718496:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.718499:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677446.718500:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.718502:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677446.718502:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677446.718503:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.718503:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:24.0:1619677446.718504:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.718504:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677446.718506:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677446.718507:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.718507:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.718508:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.718508:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.718509:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.718557:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.718560:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.718561:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.718562:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.718564:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.718565:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.719062:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.719064:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.719065:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.719067:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.719068:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.719070:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.719107:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.719110:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.719111:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.719112:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.719116:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.719117:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677446.719118:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96738200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.719119:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.719120:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.719121:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677446.719125:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.719127:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677446.719129:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.719130:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677446.719131:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677446.719133:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677446.719133:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.719134:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.719134:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.719167:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.719170:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.719170:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.719172:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.719173:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.719175:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.719716:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.719719:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.719720:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.719722:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.719725:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.719727:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.719728:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.719730:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.719731:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.719732:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.719735:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.719737:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.719740:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.719741:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.719741:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.719743:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.719744:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.719744:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.719745:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.720292:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.720299:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.720299:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.720302:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.720305:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.720306:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677446.720308:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.720310:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.720310:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.720311:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677446.720315:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.720317:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677446.720320:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.720321:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677446.720321:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677446.720323:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677446.720323:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.720324:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.720326:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.720326:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.720339:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.720341:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.720342:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.720343:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.720546:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.720548:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.720549:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.720551:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.720553:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.720554:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.720567:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.720570:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.720571:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.720571:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.720572:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.720573:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.720849:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.720852:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.720853:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.720854:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.720855:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.720857:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.720871:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.720874:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.720875:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.720876:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.720878:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.720879:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.720889:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.720892:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.720892:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.720893:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.720894:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.720894:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.721013:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.721016:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.721017:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.721017:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.721020:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677446.721023:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.721032:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677446.721034:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677446.721035:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.721035:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.721036:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.721036:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.721049:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39b600] (21)++
00000800:00000200:24.0:1619677446.721052:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.65@o2ib3
00000800:00000200:8.0:1619677446.721052:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39b600] (22)--
00000400:00000200:24.0:1619677446.721054:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.2.65@o2ib3 : GET - for me
00000400:00000200:24.0:1619677446.721057:0:19938:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.2.65@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:24.0:1619677446.721059:0:19938:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.2.65@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:24.0:1619677446.721060:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39b600] (21)++
00000800:00000200:24.0:1619677446.721062:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39b600] (22)--
00000400:00000200:24.0:1619677446.721063:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.2.65@o2ib3 local destination
00000400:00000200:24.0:1619677446.721067:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.2.65@o2ib3(10.51.2.65@o2ib3:10.51.2.65@o2ib3) : REPLY try# 0
00000800:00000200:24.0:1619677446.721069:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.2.65@o2ib3
00000800:00000200:24.0:1619677446.721071:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919fce371e80] -> 10.51.2.65@o2ib3 (2) version: 12
00000800:00000200:24.0:1619677446.721071:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919fee39b600] (21)++
00000800:00000200:24.0:1619677446.721072:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39b600] (22)++
00000800:00000200:24.0:1619677446.721074:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919fee39b600] (23)--
00000800:00000200:24.0:1619677446.721074:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39b600] (22)--
00000800:00000200:8.0:1619677446.721087:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39b600] (22)++
00000800:00000200:8.0:1619677446.721088:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39b600] (23)--
00000400:00000200:8.0:1619677446.721089:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.721090:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39b600] (22)--
00000400:00000200:8.0:1619677446.721090:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.2.65@o2ib3: REPLY: OK
00000800:00000200:8.0:1619677446.721093:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39b600] (21)--
00000800:00000200:13.0:1619677446.721155:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.721158:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.721159:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.721160:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.721162:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.721163:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.721173:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.721175:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.721176:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.721176:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.721179:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.721181:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677446.721340:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.721342:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.721343:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.721345:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.721347:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.721348:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.721380:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.721383:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.721384:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.721385:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.721389:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.721390:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677446.721391:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.721392:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.721393:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.721394:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677446.721398:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.721399:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677446.721402:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.721403:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677446.721404:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677446.721406:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677446.721406:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.721407:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.721407:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.721552:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.721555:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.721556:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.721558:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.721559:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.721560:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.721569:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.721571:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.721572:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.721573:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.721574:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.721575:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.721765:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.721766:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.721767:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.721767:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.721768:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.721768:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.721769:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:12.0:1619677446.721770:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.721771:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.721772:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.721773:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.721774:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.721971:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.721974:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.721975:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.721977:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.721980:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.721981:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677446.721982:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.721984:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.721985:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.721986:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677446.721989:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.721992:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677446.721994:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.721995:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677446.721995:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677446.721997:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.721997:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.721999:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.722000:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.722000:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677446.722009:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.722011:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.722012:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.722013:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:12.0:1619677446.722014:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.722015:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.722015:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.722015:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677446.722018:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.722018:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.722345:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.722346:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.722347:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.722348:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:13.0:1619677446.722349:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.722350:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.722350:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.722351:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.722352:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:12.0:1619677446.722352:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.722354:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.722355:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.722475:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.722478:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.722479:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.722480:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.722483:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.722484:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677446.722486:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.722487:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.722488:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.722489:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677446.722492:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.722493:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677446.722496:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.722496:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677446.722497:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677446.722499:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.722499:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.722500:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.722500:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.722511:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.722513:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.722514:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.722516:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.722517:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.722518:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.722532:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.722535:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.722535:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.722536:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.722537:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.722537:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.723091:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677446.723094:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677446.723095:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.723096:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.723098:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.723099:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677446.723099:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677446.723100:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.723100:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677446.723101:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.723103:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.723147:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.723150:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.723151:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.723151:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.723154:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.723157:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.723168:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677446.723171:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677446.723172:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.723172:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677446.723173:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677446.723174:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.723540:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.723543:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.723543:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677446.723544:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.723545:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.723546:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:24.0:1619677446.723548:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677446.723549:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.723550:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.723550:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:24.0:1619677446.723552:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:8.0:1619677446.723552:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.723553:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.723555:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.723556:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.723557:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677446.723559:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.723561:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677446.723564:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.723564:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677446.723565:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677446.723566:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.723567:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.723568:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.723568:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.723579:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.723582:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.723582:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.723584:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.723586:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.723588:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.723710:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.723712:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.723714:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.723715:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.723717:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.723719:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677446.723720:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96738137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.723721:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.723722:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.723723:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677446.723726:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.723728:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677446.723731:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.723731:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677446.723732:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677446.723734:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677446.723734:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.723735:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.723735:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.723921:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.723923:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.723925:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.723925:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.723927:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.723930:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677446.723966:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.723968:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.723969:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.723971:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.723972:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.723974:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.723985:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.723988:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.723988:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.723989:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.723990:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.723991:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.724086:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.724088:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.724089:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.724091:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.724092:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.724093:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.724389:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.724392:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.724392:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.724394:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.724396:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.724398:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.724411:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.724413:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.724414:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.724415:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.724417:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.724418:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.724438:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.724441:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.724442:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.724443:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.724446:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.724448:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677446.724449:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.724451:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.724451:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.724452:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677446.724457:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.724459:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677446.724462:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.724463:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677446.724464:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677446.724466:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677446.724466:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.724467:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.724467:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677446.724562:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.724565:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.724566:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.724568:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.724570:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.724573:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.724574:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.724575:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.724576:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.724577:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.724581:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.724582:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.724586:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.724586:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.724587:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.724589:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.724589:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.724590:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.724590:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.724649:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.724651:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.724652:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.724654:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.724655:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.724657:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.724667:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.724670:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.724671:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.724671:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.724672:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.724673:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.724774:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.724776:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.724777:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.724779:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.724780:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.724783:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.725105:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.725108:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.725109:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.725109:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.725111:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:8.0:1619677446.725114:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677446.725196:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.725198:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.725199:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.725202:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.725202:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.725204:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677446.725204:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.725204:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.725205:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.725206:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.725206:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.725207:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:12.0:1619677446.725208:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.725208:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.725210:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677446.725212:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:24.0:1619677446.725213:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.725213:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677446.725214:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.725215:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96738114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677446.725215:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.725216:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:24.0:1619677446.725217:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677446.725217:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.725217:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677446.725218:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.725218:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:12.0:1619677446.725218:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677446.725221:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.725223:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.725225:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.725226:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.725227:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.725228:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.725229:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.725230:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.725232:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.725232:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.725245:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.725248:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.725248:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.725250:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.725603:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.725606:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.725607:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.725607:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.725608:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.725611:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.725662:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.725665:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.725666:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.725667:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.725675:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.725676:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.725677:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96738106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.725679:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.725680:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.725682:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.725686:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.725688:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.725691:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.725692:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.725693:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.725695:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.725695:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.725696:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.725697:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.726226:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.726228:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.726230:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.726230:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.726232:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.726235:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.726510:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.726513:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.726514:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.726515:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.726519:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.726520:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677446.726521:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.726522:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.726523:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.726525:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677446.726528:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.726530:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677446.726533:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.726534:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677446.726535:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677446.726536:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677446.726537:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.726537:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.726538:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.726656:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.726658:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.726659:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.726661:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.726664:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.726666:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.726667:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.726668:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.726669:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.726670:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.726673:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.726675:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.726677:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.726678:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.726679:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.726681:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.726681:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.726682:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.726684:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.726685:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.726697:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.726698:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.726699:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.726700:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.727107:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.727110:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.727110:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.727113:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.727114:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.727116:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.727402:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.727404:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.727405:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.727407:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.727409:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.727410:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.727683:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.727686:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.727687:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.727689:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.727692:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.727694:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.727695:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.727696:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.727697:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.727699:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.727702:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.727704:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.727707:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.727707:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.727708:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.727710:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.727710:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.727711:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.727711:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.728069:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.728072:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.728073:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.728073:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.728075:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.728078:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677446.728084:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.728086:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.728087:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.728089:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.728091:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.728092:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:13.0:1619677446.728092:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.728095:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677446.728096:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.728096:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677446.728097:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677446.728097:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.728187:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.728190:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.728191:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.728192:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.728195:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.728197:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.728198:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.728199:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.728200:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.728201:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.728204:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.728205:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.728208:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.728210:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.728210:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.728212:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.728213:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.728213:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.728214:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.728573:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.728576:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.728576:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.728578:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.728580:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.728581:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.728591:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.728595:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.728596:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.728596:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.728597:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.728599:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.728644:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.728646:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.728647:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.728647:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.728649:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.728651:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.728670:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.728672:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.728673:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677446.728674:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677446.728675:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000400:00000200:8.0:1619677446.728675:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677446.728676:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.728677:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677446.728678:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.728678:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.728679:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:8.0:1619677446.728680:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677446.728681:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.728683:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.728684:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.728684:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677446.728688:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.728689:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677446.728691:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.728692:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677446.728693:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677446.728694:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.728695:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.728695:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.728696:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.728707:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.728709:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.728710:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.728712:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.728713:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.728714:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.729337:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.729339:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.729341:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.729342:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.729344:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.729345:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.729372:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.729375:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.729376:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.729378:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.729380:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.729381:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677446.729383:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.729384:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.729385:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.729386:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677446.729390:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.729392:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677446.729394:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.729395:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677446.729396:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677446.729397:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677446.729398:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.729399:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.729400:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.729401:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.729414:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.729416:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.729417:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.729418:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.729445:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.729447:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.729448:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.729450:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.729451:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.729453:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.729548:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.729551:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.729552:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.729554:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.729557:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.729559:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.729560:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.729562:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.729563:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.729565:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.729569:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.729570:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.729573:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.729574:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.729575:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.729577:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.729577:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.729578:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.729579:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.729668:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.729671:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.729672:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.729674:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.729675:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.729677:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.729689:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.729692:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.729693:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.729694:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.729694:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.729696:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.729908:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.729911:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.729912:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.729912:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.729914:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.729917:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.730175:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677446.730178:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677446.730179:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.730180:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.4.38@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.101@o2ib7 : PUT - routed
00000800:00000200:24.0:1619677446.730182:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.730283:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.730286:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.730287:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.730287:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.730289:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.730292:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.730313:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.730316:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.730317:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.730318:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.730321:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.730323:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677446.730324:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.730325:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.730326:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.730327:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677446.730331:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.730333:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677446.730336:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.730336:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677446.730337:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677446.730339:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677446.730340:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.730341:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.730342:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.730343:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.730357:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.730359:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.730360:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.730361:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677446.730438:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:12.0:1619677446.730438:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677446.730441:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:12.0:1619677446.730441:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.730441:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.730442:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:12.0:1619677446.730444:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:24.0:1619677446.730445:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677446.730445:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.730446:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677446.730448:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.730450:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.730451:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96738037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.730452:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.730453:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.730454:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.730457:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.730459:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.730462:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.730462:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.730463:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.730465:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.730465:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.730466:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.730467:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.730714:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.730716:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.730717:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.730719:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.730720:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.730722:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.731079:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.731081:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.731082:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.731082:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.731085:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677446.731088:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.731108:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.731111:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.731112:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.731113:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.731116:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.731117:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.731119:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96738031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.731120:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.731121:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.731122:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.731125:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.731128:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.731131:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.731132:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.731132:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.731134:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.731134:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.731135:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.731136:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.731163:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.731166:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.731167:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.731167:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.731169:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.731171:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.731254:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.731256:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.731257:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.731258:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.731261:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.731263:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677446.731264:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96738026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.731265:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.731266:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.731267:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677446.731270:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.731272:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677446.731275:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.731275:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677446.731276:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677446.731278:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677446.731278:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.731279:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.731280:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.731405:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.731407:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.731408:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.731410:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.731411:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.731413:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.731906:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.731909:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.731910:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.731910:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.731912:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677446.731915:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677446.731962:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.731964:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.731965:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.731966:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.731968:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.731969:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.731970:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.731970:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.731970:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.731971:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.731972:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.732117:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.732120:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.732121:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.732123:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.732123:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677446.732124:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.732126:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:13.0:1619677446.732126:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.732127:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.732128:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.732130:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.732133:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.732147:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.732150:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677446.732151:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.732152:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677446.732153:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.732154:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.732154:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.732154:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677446.732155:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.732156:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:8.0:1619677446.732157:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.732159:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.732160:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.732162:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.732162:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.732164:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.732167:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.732169:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.732172:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.732173:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.732173:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.732175:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.732175:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.732176:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.732176:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.732804:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.732807:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.732809:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.732810:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.732813:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.732815:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.732816:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96738003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.732818:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.732819:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.732820:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.732824:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.732826:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.732829:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.732830:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.732831:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.732832:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.732833:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.732834:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.732836:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.732836:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.732849:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.732851:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.732852:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.732853:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.732980:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.732983:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.732984:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.732987:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.732989:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.732991:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677446.732992:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.732994:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.732995:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.732996:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677446.732999:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.733001:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677446.733004:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.733005:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677446.733006:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677446.733007:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677446.733008:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.733008:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.733009:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.733218:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.733221:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.733221:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.733223:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.733225:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.733226:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.733872:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.733874:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.733875:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.733876:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:12.0:1619677446.733877:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.733878:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:13.0:1619677446.733879:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.733880:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677446.733880:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677446.733882:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.733885:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.733887:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.733888:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.733889:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.733890:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.733891:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.733895:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.733897:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.733900:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.733901:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.733902:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.733904:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.733904:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.733905:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.733906:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677446.734134:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.734137:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.734138:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.734140:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.734143:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.734144:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677446.734146:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.734148:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.734149:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.734150:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677446.734157:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.734160:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677446.734164:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.734164:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677446.734165:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677446.734167:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677446.734167:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.734168:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.734168:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.734191:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.734194:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.734195:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.734198:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.734198:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.734202:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.734214:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.734217:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.734218:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.734219:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.734219:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.734221:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.734457:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.734460:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.734461:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.734463:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.734464:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.734466:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.734491:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.734493:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.734494:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.734495:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.734496:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.734500:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677446.734834:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.734837:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.734838:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.734840:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.734842:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.734843:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.734894:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.734897:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.734898:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.734900:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.734900:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.734903:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.735106:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.735109:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.735110:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.735112:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.735114:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.735116:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.735117:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.735118:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.735119:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.735120:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.735124:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.735126:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.735128:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.735129:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.735130:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.735131:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.735132:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.735132:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.735133:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.735314:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.735317:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.735318:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.735318:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.735321:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.735324:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.735473:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.735475:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.735477:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.735477:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.735480:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.735483:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677446.735489:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.735492:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.735492:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.735492:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000400:00000200:12.0:1619677446.735494:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.735495:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677446.735496:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:13.0:1619677446.735496:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.735496:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.735497:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:12.0:1619677446.735498:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.735498:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677446.735679:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.735682:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.735683:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.735685:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.735685:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.735687:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.736228:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.736230:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.736231:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.736233:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.736235:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.736236:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.736244:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.736247:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.736248:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.736248:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.736249:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.736250:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.736282:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.736285:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.736286:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.736289:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.736292:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.736300:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677446.736302:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.736304:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.736305:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.736306:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677446.736309:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.736311:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677446.736314:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.736315:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677446.736316:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677446.736317:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677446.736318:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.736318:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.736319:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.736320:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.736322:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.736323:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.736325:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.736327:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.736328:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.736710:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.736713:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.736713:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.736715:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.736717:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.736718:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.736920:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.736924:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.736925:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.736926:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.736928:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.736929:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.737251:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.737254:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.737255:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.737257:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.737259:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.737261:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.737262:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.737264:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.737265:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.737266:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.737269:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.737271:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.737274:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.737275:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.737275:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.737277:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.737278:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.737279:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.737280:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.737281:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.737294:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.737301:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.737302:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.737304:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.737518:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.737521:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.737522:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.737524:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.737526:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.737527:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.737535:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.737538:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.737539:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.737540:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.737540:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.737542:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.738074:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.738077:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.738077:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.738079:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.738081:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.738082:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.738093:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.738095:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.738096:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.738097:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.738097:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.738098:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.738300:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.738305:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.738306:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.738309:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.738312:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.738315:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.738317:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.738319:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.738320:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.738322:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.738328:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.738330:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.738334:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.738335:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.738336:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.738338:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.738338:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.738340:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.738342:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.738512:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.738514:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.738515:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.738517:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.738521:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.738522:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677446.738524:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.738525:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.738526:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.738527:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677446.738530:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.738532:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677446.738535:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.738536:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677446.738537:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677446.738538:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677446.738539:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.738540:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.738542:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.738542:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.738556:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.738558:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.738558:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.738560:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.738622:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.738624:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.738625:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.738626:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.738628:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.738630:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.738939:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.738942:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.738943:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.738944:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.738946:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.738947:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.739027:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.739029:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.739030:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.739033:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.739033:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.739035:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.739148:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.739151:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.739152:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.739154:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.739155:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.739156:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.739166:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.739169:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.739169:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.739170:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.739171:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.739172:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.739225:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.739228:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.739229:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.739229:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.739232:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.739235:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.739274:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.739277:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.739278:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.739280:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.739283:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.739285:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.739286:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737908] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.739287:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.739288:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.739290:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.739299:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.739301:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.739304:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.739304:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.739305:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.739307:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.739307:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.739308:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.739309:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.739484:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.739487:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.739488:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677446.739489:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.739491:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.739494:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.739591:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.739594:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.739594:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.739597:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.739600:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.739602:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677446.739603:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96737902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.739606:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.739607:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.739608:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677446.739614:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.739617:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677446.739621:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677446.739621:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677446.739622:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677446.739622:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677446.739624:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:13.0:1619677446.739624:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:24.0:1619677446.739625:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.739625:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:13.0:1619677446.739625:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.739626:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:13.0:1619677446.739626:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.739628:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.739629:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.739720:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.739723:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.739723:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.739725:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.739726:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.739728:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.740042:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.740044:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.740045:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.740045:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.740048:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.740050:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.740232:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.740235:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.740236:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.740237:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.740240:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.740242:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.740243:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.740244:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.740246:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.740247:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.740251:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.740253:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.740256:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.740257:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.740258:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.740259:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.740260:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.740261:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.740261:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.740495:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.740497:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.740499:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.740502:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.740506:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.740508:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.740509:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.740511:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.740513:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.740514:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.740519:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.740523:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.740526:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.740527:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.740528:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.740529:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.740530:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.740531:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.740532:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.740543:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.740546:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.740547:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.740549:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.740550:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.740552:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.741023:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:13.0:1619677446.741024:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677446.741026:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:13.0:1619677446.741026:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.741027:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.741027:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.741029:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677446.741029:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.741030:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677446.741032:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.741033:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677446.741034:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677446.741035:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.741036:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.741038:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.741038:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677446.741042:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677446.741043:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677446.741044:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:12.0:1619677446.741046:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677446.741047:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677446.741047:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677446.741048:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677446.741048:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:12.0:1619677446.741048:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.741048:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.741050:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677446.741050:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:12.0:1619677446.741050:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.741051:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.741052:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.741110:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.741112:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.741113:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.741116:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.741119:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.741125:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.741126:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96737881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.741128:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.741129:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.741130:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.741133:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.741134:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.741137:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.741138:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.741139:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.741141:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.741142:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.741143:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.741144:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.741145:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677446.741145:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000400:00000200:24.0:1619677446.741146:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.741148:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.741153:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.741154:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677446.741155:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.741155:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.741157:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.741157:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.741158:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.741318:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.741321:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.741322:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.741322:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.741324:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.741328:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677446.741441:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.741443:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.741444:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.741446:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.741448:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.741449:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.741531:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.741533:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.741535:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.741536:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.741539:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.741540:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.741544:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.741545:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.741546:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.741547:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.741551:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.741554:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.741557:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.741558:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.741558:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.741560:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.741561:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.741562:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.741563:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.741564:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.741576:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.741579:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.741579:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.741580:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.741581:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.741583:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.741584:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.741586:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.741588:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.741589:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.742004:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.742006:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.742007:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.742009:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.742010:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.742011:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.742039:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.742042:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.742043:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.742047:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.742050:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.742052:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.742053:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.742055:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.742056:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.742057:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.742061:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.742063:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.742066:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.742067:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.742067:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.742070:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.742070:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.742071:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.742071:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677446.742419:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.742421:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.742422:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.742424:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.742427:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.742429:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.742430:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.742432:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.742433:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.742434:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.742437:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.742439:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.742441:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.742442:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.742443:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.742444:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.742445:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.742446:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.742447:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.742539:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.742541:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.742542:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.742544:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.742546:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.742547:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.742590:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.742593:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.742594:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.742596:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.742599:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.742601:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.742602:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.742603:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.742604:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.742605:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.742608:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.742610:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.742613:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.742614:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.742614:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.742616:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.742617:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.742618:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.742618:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.743224:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.743226:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.743227:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.743230:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.743230:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.743233:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.743377:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.743379:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.743380:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.743382:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.743385:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.743386:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.743387:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96737853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.743389:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.743390:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.743390:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.743394:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.743395:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.743398:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.743399:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.743399:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.743401:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.743401:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.743402:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.743403:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.743556:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.743558:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.743559:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677446.743560:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.743562:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.743565:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.743574:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677446.743576:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677446.743577:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.743577:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677446.743578:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677446.743579:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677446.743999:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.744002:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.744003:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.744004:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.744006:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.744007:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.744078:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.744081:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.744082:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.744082:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.744084:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677446.744087:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.744098:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677446.744101:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677446.744102:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.744102:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677446.744103:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677446.744104:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677446.744112:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.744114:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.744116:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.744117:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.744118:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.744120:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.744130:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.744133:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.744134:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.744135:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.744135:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.744137:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.744254:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.744257:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.744258:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.744262:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.744264:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.744266:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677446.744268:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.744269:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.744270:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.744271:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677446.744275:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.744277:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677446.744280:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.744281:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677446.744281:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677446.744283:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677446.744283:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.744284:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.744285:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.744294:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.744302:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.744302:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.744303:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.744305:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.744308:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.744495:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.744498:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.744499:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.744499:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.744501:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.744504:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.744797:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.744800:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.744801:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.744801:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.744803:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.744806:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.745147:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.745149:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.745150:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.745152:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.745155:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.745157:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.745159:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.745160:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.745161:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.745162:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.745165:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.745166:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.745169:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.745170:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.745170:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.745172:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.745173:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.745174:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.745175:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.745176:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.745189:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.745191:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.745192:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.745193:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.745220:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.745222:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.745223:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.745225:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.745226:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.745229:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.745751:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.745754:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.745755:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.745757:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.745759:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.745760:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.745769:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.745772:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.745773:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.745773:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.745774:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.745775:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.746077:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.746080:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.746081:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.746083:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.746086:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.746087:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677446.746089:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96737813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.746091:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.746092:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.746093:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677446.746097:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.746099:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677446.746102:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.746103:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677446.746104:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677446.746106:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677446.746108:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.746108:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.746109:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.746439:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.746441:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.746443:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.746444:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.746446:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.746447:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.746856:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.746859:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.746859:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.746861:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.746862:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.746864:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.747083:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.747086:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.747087:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.747087:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.747090:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.747092:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677446.747232:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.747234:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.747235:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.747236:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.747238:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.747239:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.747365:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.747367:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:24.0:1619677446.747369:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677446.747369:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.747370:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677446.747372:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.747374:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677446.747374:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.747375:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.747377:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.747378:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.747379:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677446.747382:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677446.747382:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.747383:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:24.0:1619677446.747384:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:8.0:1619677446.747385:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.747386:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.747387:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.747388:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.747390:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.747391:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:8.0:1619677446.747391:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.747392:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677446.747393:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.747394:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.747395:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.747396:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.747397:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.747397:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.747397:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.747398:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.747398:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.747399:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.747400:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.747401:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.747401:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.747402:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.747407:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.747409:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.747410:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.747412:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.747414:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.747415:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.747426:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.747428:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.747428:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.747428:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.747429:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.747429:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.747430:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.747430:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.747431:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.747432:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.747432:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.747435:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.747494:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.747497:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.747498:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.747498:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.747500:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.747503:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.748107:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.748110:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.748111:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.748116:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.748123:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.748125:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677446.748127:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96737788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677446.748128:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (24)++
00000400:00000200:8.0:1619677446.748129:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677446.748130:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (24)--
00000400:00000200:24.0:1619677446.748131:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.748131:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.748132:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677446.748133:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.748136:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677446.748137:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.748138:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.748139:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.748140:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.748140:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:24.0:1619677446.748142:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.748143:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.748144:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677446.748145:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677446.748147:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677446.748149:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677446.748150:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677446.748151:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.748151:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.748152:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.748152:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.748152:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.748153:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.748154:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.748154:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.748155:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677446.748163:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.748166:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.748167:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.748168:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:13.0:1619677446.748169:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.748170:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.748170:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677446.748170:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.748171:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.748172:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.748639:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.748644:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.748648:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677446.748651:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.748654:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.748657:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.748659:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96737781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.748660:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.748661:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.748662:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.748670:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.748672:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.748675:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.748676:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)++
00000800:00000200:8.0:1619677446.748676:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (24)++
00000800:00000200:8.0:1619677446.748679:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (25)--
00000800:00000200:8.0:1619677446.748679:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.748680:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.748681:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.749097:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.749100:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.749101:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.749103:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.749104:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.749105:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.749278:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.749281:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.749282:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.749284:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.749287:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.749288:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.749290:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.749292:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.749300:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.749301:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.749305:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.749306:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.749309:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.749310:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.749311:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.749312:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.749313:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.749314:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.749315:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.749316:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.749329:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.749331:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.749332:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.749333:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.749379:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.749382:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.749382:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.749384:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.749386:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.749387:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.749452:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.749455:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.749456:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.749457:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.749458:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.749460:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.749470:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.749473:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.749474:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.749475:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.749475:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.749477:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.749544:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.749546:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.749547:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.749547:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.749549:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.749552:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677446.749891:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.749893:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.749895:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.749896:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.749897:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.749899:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.749909:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.749911:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.749912:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.749912:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.749913:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.749914:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.750000:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.750003:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.750004:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.750005:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.750006:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.750008:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.750191:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.750193:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677446.750194:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.750195:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.750196:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.750198:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.750407:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.750410:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.750411:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.750413:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.750415:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.750417:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677446.750418:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.750419:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.750420:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.750421:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677446.750425:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.750427:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677446.750431:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.750431:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.750432:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677446.750433:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677446.750433:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677446.750434:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.750435:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677446.750436:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:8.0:1619677446.750436:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.750437:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.750438:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.750439:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.750439:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:12.0:1619677446.750535:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.750538:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.750538:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.750540:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.750542:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.750543:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.750843:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.750845:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.750846:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.750848:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.750850:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.750851:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.751033:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.751036:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.751037:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.751040:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.751043:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.751045:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.751047:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.751049:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.751050:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.751051:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.751056:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.751058:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.751061:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.751062:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.751063:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.751064:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.751065:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.751066:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.751066:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.751976:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.751978:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.751980:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.751980:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.751982:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.751985:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.752008:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.752011:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.752012:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.752014:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.752017:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.752019:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677446.752021:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.752023:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.752024:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.752025:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677446.752028:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.752030:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677446.752033:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.752033:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677446.752034:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677446.752036:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677446.752037:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.752037:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.752038:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.752043:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.752045:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.752046:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.752047:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.752049:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.752050:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.752436:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.752439:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.752439:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.752441:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.752443:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.752444:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.752469:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.752472:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.752474:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.752475:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.752476:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677446.752478:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.752485:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.752488:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.752489:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677446.752490:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677446.752490:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.752491:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.752494:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.752495:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.752496:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.752498:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.752499:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.752636:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.752638:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677446.752639:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.752640:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.752642:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.752646:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.752652:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677446.752655:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677446.752656:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.752656:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.752657:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.752657:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.753019:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677446.753022:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677446.753023:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677446.753025:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.753027:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.753028:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677446.753030:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.753031:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.753032:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.753033:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677446.753036:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.753038:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677446.753041:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.753041:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677446.753042:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677446.753044:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677446.753045:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677446.753045:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677446.753046:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:8.0:1619677446.753119:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.753122:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.753122:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.753124:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.753127:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.753129:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677446.753130:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.753132:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.753133:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.753133:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677446.753137:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.753138:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677446.753141:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.753142:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677446.753142:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677446.753144:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677446.753145:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.753145:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.753146:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.753259:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.753261:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.753262:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.753265:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.753265:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.753272:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.753290:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.753299:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.753300:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.753300:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.753302:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.753305:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.753391:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.753394:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677446.753394:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.753396:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.753397:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.753399:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677446.753399:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677446.753400:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.753401:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.753402:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677446.753408:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.753409:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.753412:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.753413:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.753414:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.753414:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.753415:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.753758:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.753760:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.753761:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.753763:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.753766:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.753767:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.753769:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.753770:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.753771:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.753772:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.753776:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.753778:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.753781:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.753781:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.753782:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.753784:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.753784:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.753786:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.753787:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.753787:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.753800:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.753802:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.753803:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.753804:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.753869:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.753871:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.753872:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.753874:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.753875:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.753877:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.754099:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.754102:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.754103:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.754103:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.754105:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.754108:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.754119:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677446.754122:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677446.754123:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.754123:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.754124:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.754124:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.754243:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.754245:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.754246:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.754248:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.754251:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.754253:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677446.754254:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.754255:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.754256:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.754257:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677446.754261:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.754262:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677446.754265:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.754266:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677446.754267:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677446.754269:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677446.754269:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.754270:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.754270:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.754435:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.754437:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.754438:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.754439:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.754441:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.754444:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677446.754578:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.754581:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.754582:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.754584:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.754586:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.754587:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.754828:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.754831:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.754832:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.754832:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.754834:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.754837:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677446.754844:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.754847:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.754848:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.754849:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.754850:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.754852:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.754863:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.754865:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.754866:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.754867:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.754867:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.754868:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.754903:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.754906:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.754907:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.754908:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.754911:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.754918:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.754919:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.754921:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.754921:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.754922:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.754927:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.754929:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.754932:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.754933:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.754933:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.754935:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.754935:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.754936:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.754937:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.755050:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.755053:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.755054:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.755055:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677446.755056:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.755058:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.755254:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.755257:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.755258:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.755258:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.755260:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.755263:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.755358:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.755361:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.755362:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.755366:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.755369:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.755371:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677446.755372:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737682] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.755373:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.755374:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.755375:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677446.755379:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.755380:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677446.755384:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.755385:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677446.755385:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677446.755387:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677446.755387:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.755388:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.755388:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677446.755780:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.755783:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.755784:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.755787:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.755790:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.755791:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.755792:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.755794:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.755796:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.755797:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.755800:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.755803:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.755806:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.755807:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.755808:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.755810:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.755810:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.755811:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.755812:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.756080:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.756083:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.756084:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.756085:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.756087:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.756088:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.756098:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.756100:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.756101:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.756102:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.756102:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.756103:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.756443:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.756446:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.756447:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.756449:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.756452:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.756453:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.756455:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.756456:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.756457:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.756458:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.756462:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.756463:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.756466:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.756467:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.756468:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.756470:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.756470:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.756471:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.756472:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.756533:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.756536:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.756537:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.756538:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.756541:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.756542:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.757242:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.757245:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.757246:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.757248:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.757250:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.757252:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677446.757253:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.757254:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.757255:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.757256:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677446.757260:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.757262:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677446.757265:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.757265:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677446.757266:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677446.757268:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677446.757268:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.757270:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.757271:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.757271:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.757284:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.757286:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.757287:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.757288:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.757475:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.757477:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.757478:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.757480:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.757482:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.757484:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.757514:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677446.757517:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677446.757519:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677446.757520:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677446.757522:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.757524:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677446.757525:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96737651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677446.757526:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677446.757527:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677446.757528:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677446.757532:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677446.757532:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.757533:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.757534:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.757535:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:12.0:1619677446.757536:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.757538:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.757539:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677446.757539:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.757540:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:13.0:1619677446.757540:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677446.757542:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:13.0:1619677446.757543:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677446.757543:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677446.757544:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677446.757639:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.757641:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.757642:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.757644:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.757645:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.757647:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.758189:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.758192:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.758192:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.758194:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.758197:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.758199:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677446.758200:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.758202:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.758203:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.758204:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677446.758208:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.758210:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677446.758213:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.758214:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677446.758214:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677446.758216:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677446.758216:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.758217:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.758219:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.758219:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.758232:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.758234:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.758235:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.758236:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677446.758476:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.758479:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.758480:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.758480:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.758482:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.758485:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.758495:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677446.758498:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.758498:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677446.758499:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.758500:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.758500:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677446.758568:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.758570:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.758571:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.758572:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.758574:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.758575:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.759236:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.759238:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.759240:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.759241:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.759244:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.759245:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677446.759247:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.759248:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.759250:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.759251:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677446.759254:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.759256:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677446.759259:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.759259:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677446.759260:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677446.759262:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677446.759262:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.759263:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.759264:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.759457:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.759459:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.759460:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.759462:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.759465:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.759467:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677446.759469:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96737630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.759471:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.759472:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.759473:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677446.759477:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.759478:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677446.759481:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.759482:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677446.759483:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677446.759485:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677446.759486:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.759486:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.759487:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.759649:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.759651:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677446.759652:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.759653:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.759655:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.759659:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677446.760308:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.760310:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.760311:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.760313:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.760315:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.760320:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.760816:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.760820:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.760821:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.760822:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.760825:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.760827:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.760828:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.760829:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.760830:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.760831:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.760835:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.760838:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.760841:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.760842:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.760842:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.760844:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.760845:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.760845:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.760846:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.760872:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.760874:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.760875:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.760877:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.760879:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.760880:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.760890:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.760894:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.760895:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.760895:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.760896:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.760897:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.761488:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.761490:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.761491:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.761493:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.761497:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.761498:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.761500:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.761501:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.761502:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.761503:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.761506:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.761508:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.761511:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.761512:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.761513:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.761515:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.761516:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.761517:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.761517:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.761922:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.761925:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.761926:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.761928:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.761931:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.761933:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.761934:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.761935:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.761937:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.761937:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.761940:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.761942:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.761945:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.761945:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.761946:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.761947:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.761948:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.761948:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.761949:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.762123:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.762126:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.762127:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.762128:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.762130:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.762131:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.762141:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.762144:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.762144:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.762145:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.762146:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.762146:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.762206:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.762209:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.762211:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.762212:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.762215:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.762217:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677446.762218:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.762219:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.762220:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.762221:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677446.762224:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.762226:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677446.762228:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.762229:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677446.762230:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677446.762231:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.762232:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.762232:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.762233:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.762243:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.762245:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.762246:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.762247:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.762249:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.762250:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.762271:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.762273:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.762274:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.762275:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.762277:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.762278:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.762854:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.762857:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.762857:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.762859:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.762861:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.762862:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.762872:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.762875:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.762876:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.762877:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.762880:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.762882:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.762883:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.762884:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.762885:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.762886:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.762889:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.762891:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.762893:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.762894:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.762895:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.762896:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.762897:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.762898:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.762899:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.762900:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.762913:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.762915:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.762916:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.762917:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.763147:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.763150:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.763151:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.763152:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.763153:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.763155:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.763605:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.763608:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.763609:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.763611:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.763613:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.763614:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.763678:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.763681:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.763682:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.763682:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.763685:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.763688:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677446.764043:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.764046:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.764047:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.764048:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.764050:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.764051:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.764062:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.764065:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.764065:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.764066:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.764067:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.764068:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.764259:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.764262:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.764264:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.764265:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.764266:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677446.764267:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:13.0:1619677446.764268:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.764270:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.764271:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.764273:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.764276:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.764278:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.764280:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.764282:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.764283:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.764284:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.764289:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.764292:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.764301:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.764302:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.764303:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.764305:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.764305:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.764307:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.764308:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.764309:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.764322:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.764324:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.764325:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.764326:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677446.764710:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.764713:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.764714:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.764716:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.764719:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.764720:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.764723:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.764724:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.764726:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.764727:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.764731:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.764734:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.764737:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.764738:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.764738:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.764740:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.764741:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.764742:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.764743:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.765625:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.765628:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.765629:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.765629:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.765631:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.765634:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677446.765818:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.765821:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.765822:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.765823:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.765825:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.765826:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.765842:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.765845:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.765846:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.765846:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.765847:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.765849:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.765994:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.765997:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.765998:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.765998:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.766000:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.766002:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677446.766222:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.766225:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.766226:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.766228:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.766229:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.766230:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.766925:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.766928:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.766929:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.766930:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.766933:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.766935:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.766936:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.766937:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.766938:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.766939:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.766942:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.766944:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.766946:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.766947:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.766948:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.766949:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.766950:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.766951:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.766952:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.767059:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.767062:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.767062:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.767064:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.767066:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.767068:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.767480:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677446.767483:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677446.767484:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677446.767486:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.767491:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.767493:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:24.0:1619677446.767495:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.767496:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.767497:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.767498:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:24.0:1619677446.767502:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.767504:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:24.0:1619677446.767508:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.767508:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677446.767509:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:24.0:1619677446.767509:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:8.0:1619677446.767509:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000400:00000200:8.0:1619677446.767511:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677446.767512:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:24.0:1619677446.767512:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (23)++
00000800:00000200:24.0:1619677446.767514:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (24)--
00000800:00000200:24.0:1619677446.767514:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (23)--
00000400:00000200:8.0:1619677446.767514:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.767515:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677446.767515:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677446.767522:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.767523:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.767524:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.767525:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677446.767528:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.767530:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677446.767532:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.767533:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677446.767533:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677446.767534:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.767535:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677446.767536:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677446.767536:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:24.0:1619677446.767547:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.767549:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.767551:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.767552:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.767553:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.767554:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677446.767555:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.767555:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677446.767556:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677446.767558:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.767560:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.767562:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677446.767563:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.767564:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.767565:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.767566:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677446.767571:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.767574:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677446.767578:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.767579:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677446.767581:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677446.767581:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677446.767582:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677446.767583:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:24.0:1619677446.767584:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677446.767584:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (23)++
00000800:00000200:8.0:1619677446.767585:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (24)--
00000400:00000200:24.0:1619677446.767586:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.767586:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677446.767587:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000400:00000200:24.0:1619677446.767588:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:8.0:1619677446.767588:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000400:00000200:24.0:1619677446.767589:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.767590:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.767591:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.767592:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677446.767595:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.767597:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677446.767597:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.767598:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:8.0:1619677446.767598:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:8.0:1619677446.767600:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96737524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.767601:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.767603:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.767604:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677446.767608:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.767609:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677446.767609:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:8.0:1619677446.767609:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.767611:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677446.767611:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:24.0:1619677446.767612:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (23)++
00000800:00000200:8.0:1619677446.767613:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.767614:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (24)++
00000800:00000200:8.0:1619677446.767614:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:24.0:1619677446.767615:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (25)--
00000800:00000200:8.0:1619677446.767615:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:24.0:1619677446.767616:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (24)--
00000800:00000200:24.0:1619677446.767617:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (23)++
00000800:00000200:8.0:1619677446.767617:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:24.0:1619677446.767618:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000400:00000200:24.0:1619677446.767621:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.767623:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.767623:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (24)++
00000400:00000200:24.0:1619677446.767625:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:8.0:1619677446.767625:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (25)--
00000400:00000200:24.0:1619677446.767626:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96737523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677446.767626:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (24)--
00000800:00000200:8.0:1619677446.767627:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (23)++
00000400:00000200:24.0:1619677446.767628:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677446.767628:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000400:00000200:24.0:1619677446.767630:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.767631:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:8.0:1619677446.767633:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.767636:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.767637:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.767639:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:8.0:1619677446.767639:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:8.0:1619677446.767641:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.767642:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.767643:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.767644:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677446.767648:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677446.767648:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.767649:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.0:1619677446.767649:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:24.0:1619677446.767650:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:8.0:1619677446.767651:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.767652:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:8.0:1619677446.767652:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:24.0:1619677446.767653:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (24)++
00000800:00000200:8.0:1619677446.767653:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:24.0:1619677446.767654:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (25)--
00000800:00000200:8.0:1619677446.767654:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:8.0:1619677446.767654:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (25)++
00000800:00000200:24.0:1619677446.767655:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (24)--
00000800:00000200:8.0:1619677446.767655:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (25)--
00000800:00000200:8.0:1619677446.767655:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677446.767656:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677446.767658:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677446.767659:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677446.767660:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677446.767661:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677446.767662:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000400:00000200:24.0:1619677446.767663:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677446.767664:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000400:00000200:8.0:1619677446.767665:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.767666:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.767667:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677446.767669:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96737521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.767670:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.767671:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.767672:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677446.767674:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.767676:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677446.767677:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677446.767677:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677446.767678:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:8.0:1619677446.767679:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.767680:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.767681:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677446.767684:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.767685:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.767686:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:24.0:1619677446.767686:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677446.767686:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677446.767688:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:24.0:1619677446.767689:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (23)++
00000800:00000200:8.0:1619677446.767689:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.767690:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (24)++
00000800:00000200:8.0:1619677446.767690:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677446.767690:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677446.767691:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (25)--
00000800:00000200:24.0:1619677446.767692:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (24)--
00000800:00000200:8.0:1619677446.767692:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.767692:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (24)++
00000800:00000200:24.0:1619677446.767693:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (24)++
00000800:00000200:8.0:1619677446.767693:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (24)--
00000800:00000200:8.0:1619677446.767694:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (24)--
00000800:00000200:24.0:1619677446.767695:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677446.767695:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (23)++
00000800:00000200:8.0:1619677446.767696:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000400:00000200:24.0:1619677446.767698:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.767699:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.767700:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.767701:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.767703:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.767704:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96737519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677446.767706:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677446.767706:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.767707:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677446.767708:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:8.0:1619677446.767708:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677446.767709:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.767710:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677446.767711:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:24.0:1619677446.767712:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96737518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677446.767713:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677446.767714:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.767714:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677446.767715:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.767716:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.767716:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677446.767718:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677446.767720:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.767722:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.767722:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.767723:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.767724:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.767724:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.767725:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677446.767727:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (23)++
00000800:00000200:8.0:1619677446.767727:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677446.767728:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677446.767728:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (24)++
00000800:00000200:24.0:1619677446.767729:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (25)++
00000800:00000200:8.0:1619677446.767729:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (25)--
00000800:00000200:24.0:1619677446.767730:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (25)--
00000800:00000200:24.0:1619677446.767730:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677446.767730:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (25)--
00000800:00000200:8.0:1619677446.767736:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677446.767739:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (23)++
00000800:00000200:8.0:1619677446.767739:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677446.767740:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (24)--
00000800:00000200:24.0:1619677446.767741:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (23)--
00000400:00000200:8.0:1619677446.767741:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677446.767742:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677446.767744:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000400:00000200:8.0:1619677446.767744:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.767746:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677446.767747:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.767748:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96737517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.767750:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.767751:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.767752:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677446.767756:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.767757:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677446.767758:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677446.767759:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677446.767759:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:24.0:1619677446.767761:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.767762:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.767763:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:8.0:1619677446.767765:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.767766:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:8.0:1619677446.767767:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:24.0:1619677446.767768:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.767769:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:8.0:1619677446.767769:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (23)++
00000800:00000200:24.0:1619677446.767770:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677446.767771:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (24)++
00000800:00000200:24.0:1619677446.767772:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.767772:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (25)--
00000800:00000200:24.0:1619677446.767773:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (21)++
00000800:00000200:8.0:1619677446.767773:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (24)--
00000800:00000200:24.0:1619677446.767774:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (22)++
00000800:00000200:8.0:1619677446.767774:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677446.767776:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (24)--
00000800:00000200:24.0:1619677446.767776:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677446.767777:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677446.767778:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677446.767787:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677446.767789:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677446.767790:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677446.767792:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:24.0:1619677446.767943:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677446.767947:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677446.767947:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677446.767951:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.767953:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.767954:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677446.767955:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96737511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.767958:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.767959:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.767960:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677446.767964:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.767966:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677446.767970:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.767970:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677446.767971:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677446.767973:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.767974:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677446.767975:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677446.767975:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:13.0:1619677446.767985:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.767988:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.767989:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.767990:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.767992:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.767993:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.768047:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.768050:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.768051:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.768052:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.768055:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.768057:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.768058:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.768060:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.768061:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.768062:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.768065:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.768067:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.768070:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.768071:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.768072:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.768073:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.768074:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.768074:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.768075:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.768247:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677446.768249:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677446.768251:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.768252:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.768253:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.768254:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.768255:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.768256:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:24.0:1619677446.768257:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677446.768258:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.768259:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.768260:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.768261:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677446.768265:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.768267:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677446.768267:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.768269:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677446.768270:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.768270:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677446.768271:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677446.768271:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677446.768272:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677446.768272:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.768273:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.768274:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677446.768274:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677446.768275:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677446.768275:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677446.768275:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.768276:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677446.768277:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677446.768279:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677446.768279:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000400:00000200:8.0:1619677446.768281:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.768282:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.768284:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:8.0:1619677446.768285:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.768286:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.768287:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.768288:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:8.0:1619677446.768300:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.768301:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.0:1619677446.768305:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.768306:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:8.0:1619677446.768306:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.0:1619677446.768308:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:8.0:1619677446.768309:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677446.768310:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677446.768311:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677446.768312:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677446.768324:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677446.768327:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677446.768328:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677446.768329:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:8.0:1619677446.768857:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe9286a00] (21)++
00000800:00000200:8.0:1619677446.768860:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.60@o2ib3
00000800:00000200:24.0:1619677446.768861:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe9286a00] (22)--
00000400:00000200:8.0:1619677446.768863:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.60@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677446.768865:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000400:00000200:8.0:1619677446.768866:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.768868:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:8.0:1619677446.768868:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:8.0:1619677446.768869:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737497] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677446.768871:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.768871:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677446.768872:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:8.0:1619677446.768872:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.768873:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:12.0:1619677446.768874:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.768876:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:12.0:1619677446.768877:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96737496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.768877:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.60@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677446.768879:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677446.768879:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 584 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:12.0:1619677446.768880:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677446.768881:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677446.768882:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.768883:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:8.0:1619677446.768883:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:12.0:1619677446.768884:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.768885:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:8.0:1619677446.768885:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe9286a00] (21)++
00000800:00000200:12.0:1619677446.768886:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677446.768886:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe9286a00] (22)--
00000800:00000200:8.0:1619677446.768887:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe9286a00] (21)--
00000800:00000200:12.0:1619677446.768889:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677446.768889:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:12.0:1619677446.768890:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:12.0:1619677446.768892:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:12.0:1619677446.768893:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677446.768894:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677446.768896:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677446.768896:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:24.0:1619677446.768904:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.768907:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.768908:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.768908:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.768910:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.768912:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:24.0:1619677446.768913:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677446.768914:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677446.768915:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677446.768916:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.0:1619677446.768938:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.768941:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.768942:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.768943:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.768946:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.768948:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.768949:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.768950:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.768951:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.768952:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.768955:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.768957:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.768959:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.768960:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.768961:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.768962:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.768963:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.768964:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.768965:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.768966:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.768978:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.768980:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.768981:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.768982:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.769104:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.769107:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.769108:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.769110:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.769111:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.769113:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.769587:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.769589:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.769590:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.769592:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.769594:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.769595:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.769892:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.769894:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.769895:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.769896:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.769898:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.769899:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.769914:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.769916:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.769917:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.769918:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.769918:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.769920:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.769974:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.769977:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.769978:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.769979:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.769980:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.769982:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.769993:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.769995:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.769996:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.769997:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.769997:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.769999:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.770026:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.770028:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.770029:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.770030:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.770032:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.770033:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.770429:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.770432:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.770433:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.770435:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.770439:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.770440:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:8.0:1619677446.770442:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.770443:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.770444:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.770445:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:8.0:1619677446.770450:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.770453:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.0:1619677446.770455:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.770456:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:8.0:1619677446.770457:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:8.0:1619677446.770458:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:8.0:1619677446.770459:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.770459:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.770460:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.770484:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.770486:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.770487:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.770489:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.770491:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.770491:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.770603:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677446.770606:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677446.770608:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677446.770609:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677446.770610:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.770612:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:13.0:1619677446.770613:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96737468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677446.770614:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677446.770615:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677446.770616:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:13.0:1619677446.770620:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677446.770622:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677446.770625:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677446.770626:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677446.770627:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677446.770629:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677446.770629:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677446.770630:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677446.770630:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:24.0:1619677446.770703:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.770706:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.770707:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.770710:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.770716:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.770718:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.770720:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.770722:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.770723:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.770724:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.770728:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.770729:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.770733:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.770734:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.770735:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.770737:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.770738:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.770738:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.770739:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.770739:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.770740:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:8.0:1619677446.770740:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.770741:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.770742:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.770750:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.770972:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe9286a00] (21)++
00000800:00000200:24.0:1619677446.770975:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.60@o2ib3
00000800:00000200:8.0:1619677446.770977:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe9286a00] (22)--
00000400:00000200:24.0:1619677446.770978:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.60@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.770981:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.770982:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.770984:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.770985:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.770986:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.770987:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.770990:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.60@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.770992:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.770994:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.770995:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.770996:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.770997:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.770998:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe9286a00] (21)++
00000800:00000200:24.0:1619677446.770999:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe9286a00] (22)--
00000800:00000200:24.0:1619677446.770999:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe9286a00] (21)--
00000800:00000200:8.0:1619677446.771118:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.771120:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.771121:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.771121:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.771123:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.771127:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.772350:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.772353:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.772354:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.772356:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.772358:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.772360:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.772362:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.772363:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.772364:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.772365:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.772369:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.772370:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.772373:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.772374:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.772375:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.772377:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.772377:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.772378:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.772379:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.773085:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.773088:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.773089:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.773091:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.773094:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.773095:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.773097:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.773098:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.773099:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.773100:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.773103:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.773105:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.773108:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.773108:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.773109:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.773111:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.773111:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.773112:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.773115:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.773121:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.773123:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.773125:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.773125:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.773128:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.773129:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.773130:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.773133:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.773134:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.773134:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.773139:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.773140:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.773143:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.773144:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.773145:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.773147:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.773147:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.773148:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.773149:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.773400:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.773402:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.773403:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.773405:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.773407:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.773407:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.773559:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.773562:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.773563:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.773565:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677446.773567:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.773575:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.774222:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.774225:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.774226:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.774229:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.774229:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.774232:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.774242:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.774245:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.774245:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.774246:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.774246:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.774248:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.774277:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.774280:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.774281:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.774282:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.774284:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.774284:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.774464:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.774467:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.774468:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.774470:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.774473:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.774474:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.774475:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.774477:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.774478:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.774478:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.774481:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.774483:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.774486:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.774487:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.774488:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.774489:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.774490:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.774490:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.774491:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.774654:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.774657:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.774658:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.774658:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.774660:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.774663:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.774677:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677446.774679:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677446.774680:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.774680:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677446.774681:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677446.774682:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677446.774925:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.774928:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.774928:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.774930:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.774932:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.774933:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.774995:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.774998:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.774999:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.775001:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.775002:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.775004:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.775049:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.775052:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.775053:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.775054:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.775056:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.775057:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.775304:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.775307:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.775308:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.775308:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.775311:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.775314:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.775450:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.775453:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.775454:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.775454:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.775457:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.775461:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.775508:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.775511:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.775512:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.775514:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.775517:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.775519:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.775520:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.775521:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.775522:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.775523:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.775526:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.775528:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.775531:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.775532:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.775532:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.775534:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.775534:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.775536:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.775537:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.775538:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.775550:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.775552:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.775553:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.775554:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.775631:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.775634:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.775635:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.775637:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.775639:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.775640:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.775648:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.775650:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.775651:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.775652:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.775652:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.775654:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.775897:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.775900:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.775902:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.775903:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.775904:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.775906:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.775918:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.775921:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.775921:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.775923:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.775924:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.775926:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.776217:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.776220:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.776221:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.776223:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.776224:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.776225:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.776235:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.776238:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.776239:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.776240:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.776241:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.776242:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.776421:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.776424:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.776425:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.776428:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.776432:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.776433:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.776435:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.776436:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.776437:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.776438:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.776442:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.776443:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.776446:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.776447:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.776448:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.776449:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677446.776450:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.0:1619677446.776450:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.776451:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:13.0:1619677446.776452:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:8.0:1619677446.776452:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000400:00000200:13.0:1619677446.776453:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.776453:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:13.0:1619677446.776455:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.776457:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.776457:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677446.776466:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.776468:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.776469:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.776470:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.776506:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.776508:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.776509:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.776511:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.776512:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.776514:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.776676:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.776679:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.776680:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.776682:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.776685:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.776687:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677446.776687:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.776688:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677446.776689:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:8.0:1619677446.776689:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677446.776690:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.776690:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.776691:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677446.776692:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.776693:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.776695:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677446.776695:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.776697:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.776700:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.776701:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.776702:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.776703:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.776704:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.776704:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.776705:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.777110:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.777113:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.777114:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.777116:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.777119:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.777120:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677446.777122:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.777123:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.777124:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.777125:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677446.777129:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.777130:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677446.777133:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.777133:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677446.777134:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677446.777136:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.777136:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.777137:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.777137:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.777148:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.777150:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.777151:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.777153:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.777155:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.777156:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.777167:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.777169:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.777170:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.777171:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.777171:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.777172:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.777570:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.777573:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.777573:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.777575:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.777577:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.777578:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.778076:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.778079:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.778080:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.778081:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.778084:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.778085:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677446.778087:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.778088:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.778089:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.778090:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677446.778093:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.778094:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677446.778097:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.778097:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677446.778098:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677446.778100:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.778100:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.778101:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.778101:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.778112:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.778114:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.778115:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.778116:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.778118:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.778118:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.778278:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.778281:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677446.778281:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.778282:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.778284:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.778287:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.778298:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677446.778301:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677446.778302:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.778303:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.778304:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677446.778304:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.778305:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.778307:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.778308:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.778309:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.778312:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.778313:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.778314:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.778316:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.778317:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.778318:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.778321:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.778322:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.778325:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.778326:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.778326:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.778328:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.778329:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.778329:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.778330:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.778593:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.778596:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.778597:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.778599:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.778601:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.778602:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.779221:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.779223:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.779224:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.779226:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.779227:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.779228:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.779242:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.779245:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.779246:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.779246:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.779247:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.779248:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.779582:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.779585:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.779586:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.779589:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.779590:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.779591:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.779817:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.779819:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.779820:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.779821:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.779823:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.779824:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.780289:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.780291:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.780292:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.780296:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.780298:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.780300:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.780377:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.780379:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.780380:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.780381:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.780383:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.780384:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.780599:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.780602:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.780603:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.780606:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.780609:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.780610:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.780611:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96737335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.780617:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.780618:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.780619:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.780626:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.780630:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.780633:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.780634:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.780635:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.780641:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.780642:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.780643:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.780644:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.780646:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.780648:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.780648:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677446.780649:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.780651:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.780654:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.780748:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.780751:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.780752:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.780755:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.780759:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.780761:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.780763:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:508, d:10, s:-96737330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.780765:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.780766:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.780767:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.780771:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:24.0:1619677446.780772:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.780773:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.780774:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.780774:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.780777:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677446.780778:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.780779:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.780780:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.780781:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677446.780782:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.780783:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.780783:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.780784:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.780785:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.780896:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.780898:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.780899:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.780900:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.780902:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.780902:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.781035:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.781038:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.781038:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677446.781039:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.781042:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.781044:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.781345:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.781348:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.781349:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.781353:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.781356:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.781358:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677446.781360:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.781361:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.781363:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.781364:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677446.781368:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.781370:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677446.781373:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.781374:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677446.781375:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677446.781377:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677446.781378:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.781380:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.781381:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.781382:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.781395:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.781397:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.781398:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.781398:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677446.781401:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.781403:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.781404:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.781406:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.781408:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.781409:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.781423:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.781425:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.781426:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.781427:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.781428:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.781429:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.781511:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.781513:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.781514:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.781515:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.781517:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.781518:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.781529:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.781531:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.781532:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.781533:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.781534:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.781535:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.782154:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.782157:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.782158:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.782160:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.782162:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.782164:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.782165:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.782167:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.782168:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.782169:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.782172:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.782174:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.782176:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.782177:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.782178:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.782179:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.782180:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.782181:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.782182:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.782183:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.782195:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.782197:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.782198:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.782199:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.782423:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.782426:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.782427:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.782428:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.782430:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.782431:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.782525:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.782528:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.782528:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.782530:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.782531:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.782533:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.782807:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.782810:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.782811:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.782814:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.782814:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.782817:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.783178:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.783180:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.783182:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.783183:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.783186:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.783188:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.783189:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.783190:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.783191:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.783192:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.783196:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.783197:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.783200:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.783201:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.783201:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.783203:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.783203:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.783204:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.783204:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.783233:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.783235:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.783236:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.783238:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.783241:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.783243:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677446.783244:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.783245:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.783246:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.783247:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677446.783250:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.783251:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677446.783253:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.783254:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677446.783255:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677446.783256:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.783257:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.783257:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.783258:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.783268:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.783270:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.783271:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.783272:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.783274:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.783275:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.783368:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.783370:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.783371:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.783372:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.783374:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.783375:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.783571:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.783574:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.783575:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.783576:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.783577:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.783579:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.783777:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.783779:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.783780:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.783781:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.783783:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.783784:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.783803:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.783805:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.783806:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.783807:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.783807:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.783808:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.784142:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.784145:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.784146:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.784148:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.784149:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.784152:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.784491:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.784494:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.784495:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.784498:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.784501:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.784502:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.784504:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.784506:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.784507:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.784508:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.784511:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.784513:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.784516:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.784517:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.784517:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.784519:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.784520:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:12.0:1619677446.784524:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.784527:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.784528:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.784529:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.784529:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.784530:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.784531:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.784532:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.784759:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.784762:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.784763:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.784765:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.784767:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.784769:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677446.784770:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.784772:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.784773:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.784774:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677446.784777:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.784779:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677446.784782:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.784783:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677446.784783:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677446.784785:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677446.784786:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.784786:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.784786:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.784893:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.784895:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.784896:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.784898:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.784899:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.784901:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.785265:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.785268:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.785269:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.785271:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.785274:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.785275:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.785276:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737276] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.785278:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.785279:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.785279:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.785283:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.785285:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.785288:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.785289:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.785289:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.785291:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.785292:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.785292:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.785293:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.785331:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.785334:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.785335:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.785337:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.785338:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.785340:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.785347:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.785350:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.785351:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.785352:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.785352:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.785354:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.785429:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.785431:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.785432:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.785433:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.785435:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.785436:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.785448:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.785450:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.785451:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.785452:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.785452:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.785454:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.785780:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.785783:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.785784:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.785786:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.785788:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.785789:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.785791:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.785792:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.785794:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.785794:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.785798:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.785799:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.785802:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.785803:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.785804:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.785806:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.785806:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.785807:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.785808:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.785809:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.785822:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.785825:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.785826:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.785827:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.785848:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.785850:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.785851:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.785854:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.785854:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.785856:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.785986:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.785988:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.785989:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.785991:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.785992:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.785993:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.786280:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.786282:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.786283:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.786284:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.786286:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.786287:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.786530:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.786533:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.786534:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.786536:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.786538:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.786539:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.786867:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.786870:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.786871:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.786871:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.786873:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.786876:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.786892:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677446.786894:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677446.786895:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.786895:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.786896:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.786897:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677446.787602:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.787605:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.787606:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.787607:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.787609:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.787610:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.787621:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.787624:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.787625:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.787628:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.787631:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.787633:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.787634:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.787636:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.787638:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.787639:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.787642:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.787644:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.787647:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.787647:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.787648:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.787650:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.787651:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.787652:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.787652:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.787697:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.787700:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.787701:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.787702:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.787704:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.787705:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.787948:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.787951:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.787952:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.787953:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.787953:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.787955:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.788070:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.788073:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.788074:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.788076:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.788077:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.788078:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.788426:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.788429:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.788429:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677446.788431:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.788434:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.788436:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.788437:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.788439:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.788439:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.788441:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.788444:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.788446:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.788449:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.788449:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.788450:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.788452:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.788452:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.788453:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.788454:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.788455:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.788469:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.788471:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.788472:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.788473:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.788473:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.788475:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.788475:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.788479:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.788482:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.788484:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677446.788485:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.788487:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.788488:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.788489:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677446.788492:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.788494:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677446.788497:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.788498:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677446.788498:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677446.788500:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.788500:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.788501:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.788501:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.788512:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.788514:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.788515:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.788517:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.788519:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.788520:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.788603:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.788606:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.788606:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.788609:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.788609:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.788611:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.788620:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.788624:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.788627:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.788627:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)++
00000800:00000200:12.0:1619677446.788628:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.788628:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.788637:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.788640:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.788640:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.788642:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.788643:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.788645:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.788960:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.788965:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.788969:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.788971:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.788974:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.788977:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677446.788978:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96737227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.788980:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.788986:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.788988:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677446.788992:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.788995:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677446.788998:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.788999:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677446.789000:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677446.789002:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.789002:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.789003:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.789004:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.789014:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.789016:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.789017:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.789019:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.789021:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.789022:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.789096:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.789099:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.789099:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.789101:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.789103:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.789103:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677446.789218:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.789221:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.789222:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.789225:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.789227:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.789228:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.789230:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.789231:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.789232:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.789233:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.789236:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.789238:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.789241:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.789242:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.789242:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.789244:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.789245:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.789245:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.789246:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.789484:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.789487:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.789487:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.789490:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.789492:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.789494:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677446.789495:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.789496:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.789497:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.789498:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677446.789502:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.789504:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677446.789506:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.789507:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677446.789508:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677446.789509:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.789510:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.789510:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.789511:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.789521:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.789524:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.789524:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.789526:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.789528:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.789529:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.790146:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.790149:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.790150:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.790151:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.790154:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.790155:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677446.790156:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.790158:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.790159:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.790160:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677446.790163:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.790165:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677446.790168:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.790169:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677446.790169:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677446.790171:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677446.790171:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.790172:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.790172:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.790523:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.790526:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.790527:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.790528:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.790531:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.790532:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.790761:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.790764:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.790764:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.790767:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.790769:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.790771:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.790772:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.790773:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.790775:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.790776:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.790780:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.790781:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.790785:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.790786:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.790786:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.790788:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.790789:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.790790:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.790791:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.790792:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.790804:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.790807:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.790808:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677446.790808:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:24.0:1619677446.790809:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.790811:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.790812:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.790814:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.790816:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.790817:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.790828:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.790830:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.790831:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.790832:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.790832:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.790834:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.791037:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.791040:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.791041:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.791042:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.791044:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.791045:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.791144:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.791146:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.791148:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.791148:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.791150:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.791153:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.791895:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677446.791898:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677446.791899:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677446.791900:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.791903:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.791905:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.791906:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.791907:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.791908:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.791909:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.791913:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.791914:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.791917:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.791918:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.791919:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.791920:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.791921:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677446.791922:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed9d3400] (22)++
00000800:00000200:24.0:1619677446.791923:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (23)--
00000800:00000200:24.0:1619677446.791924:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677446.791937:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)++
00000800:00000200:8.0:1619677446.791938:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed9d3400] (23)--
00000800:00000200:8.0:1619677446.791939:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677446.791940:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:24.0:1619677446.791986:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39ea00] (21)++
00000800:00000200:24.0:1619677446.791989:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.17@o2ib3
00000800:00000200:8.0:1619677446.791990:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39ea00] (22)--
00000400:00000200:24.0:1619677446.791991:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.17@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.791994:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.791996:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:24.0:1619677446.791997:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.791999:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.792000:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.792003:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:24.0:1619677446.792007:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.17@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.792008:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 976 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:24.0:1619677446.792011:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.792012:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:24.0:1619677446.792013:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:24.0:1619677446.792014:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:24.0:1619677446.792015:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39ea00] (21)++
00000800:00000200:24.0:1619677446.792016:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39ea00] (22)--
00000800:00000200:24.0:1619677446.792017:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39ea00] (21)--
00000800:00000200:8.0:1619677446.792189:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.792192:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.792192:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.792195:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.792197:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.792199:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677446.792200:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.792201:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.792202:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.792203:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677446.792207:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.792208:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677446.792212:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.792212:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677446.792213:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677446.792215:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677446.792215:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.792216:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.792216:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677446.792334:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.792336:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.792337:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.792337:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.792340:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.792342:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677446.792506:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.792508:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.792509:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.792510:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.792512:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.792513:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.792522:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.792525:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.792526:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.792526:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.792527:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.792528:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.792743:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.792746:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.792746:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.792748:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.792750:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.792751:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.792786:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.792789:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.792790:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.792790:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.792792:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677446.792795:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.792899:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.792901:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.792902:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.792902:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.792905:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.792907:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677446.793097:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.793099:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.793100:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.793101:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.793103:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.793105:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.793130:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.793133:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.793134:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.793136:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.793139:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.793140:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.793142:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.793144:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.793146:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.793147:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.793152:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.793154:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.793162:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.793162:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677446.793163:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (23)++
00000800:00000200:8.0:1619677446.793166:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (25)--
00000800:00000200:8.0:1619677446.793166:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.793167:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.793168:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.793503:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.793505:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.793506:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.793507:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.793510:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.793510:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.793743:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.793746:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.793747:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.793749:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.793752:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.793753:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677446.793755:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.793756:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.793757:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.793758:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677446.793762:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.793764:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677446.793767:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.793767:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677446.793768:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677446.793770:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.793770:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.793771:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.793771:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.793782:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.793784:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.793785:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.793786:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.793788:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.793789:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.793802:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.793805:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.793805:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.793806:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.793807:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.793808:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.793859:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.793861:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.793862:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.793865:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.793865:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.793871:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.793890:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.793892:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.793894:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.793895:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.793896:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.793898:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.793954:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.793957:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.793958:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.793960:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.793960:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.793962:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.794150:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.794152:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.794153:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.794154:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.794156:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.794157:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.794441:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.794444:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.794445:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.794445:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:13.0:1619677446.794447:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.794448:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.794448:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.794449:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677446.794450:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677446.794451:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.794453:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.794455:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677446.794456:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96737150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.794457:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.794458:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.794459:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677446.794463:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.794465:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677446.794468:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.794469:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677446.794469:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677446.794471:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.794471:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.794472:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.794473:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.794482:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.794485:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.794486:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.794487:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.794489:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.794490:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.794502:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677446.794504:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677446.794505:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.794506:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677446.794507:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.794508:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:24.0:1619677446.794509:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677446.794509:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.794509:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.794511:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677446.794512:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.794515:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.794516:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737149] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.794518:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.794520:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.794522:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.794527:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.794530:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.794533:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.794534:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.794534:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.794536:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.794536:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.794538:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.794540:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.794541:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.794552:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.794555:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.794556:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.794557:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.794876:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.794878:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.794879:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.794880:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.794882:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.794883:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.794892:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.794894:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.794895:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.794896:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.794896:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.794897:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.794956:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.794959:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.794960:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.794962:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.794963:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.794964:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.794972:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.794974:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.794975:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.794978:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.794981:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.794982:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677446.794984:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.794985:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.794986:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.794987:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677446.794990:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.794992:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677446.794994:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.794995:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677446.794996:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677446.794997:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.794997:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.794999:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.795000:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.795000:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677446.795009:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.795011:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.795012:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.795013:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:12.0:1619677446.795014:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.795015:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.795016:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677446.795016:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.795017:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.795017:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.795342:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677446.795344:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677446.795345:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.795346:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677446.795347:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.795348:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.795348:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677446.795350:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:8.0:1619677446.795350:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.795351:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:8.0:1619677446.795351:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:13.0:1619677446.795352:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.795353:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:8.0:1619677446.795353:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.795354:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.795355:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.795356:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:12.0:1619677446.795357:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.795358:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.795359:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.795360:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:8.0:1619677446.795360:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677446.795362:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.795362:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.795365:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.795365:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.795366:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.795368:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.795368:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.795369:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.795369:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.796270:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.796273:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.796274:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.796276:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.796278:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.796280:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.796281:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.796282:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.796283:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.796284:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.796287:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.796289:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.796292:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.796292:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.796298:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.796300:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.796300:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.796301:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.796302:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.796529:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.796531:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.796532:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.796534:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.796536:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.796537:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.796604:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.796607:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.796608:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.796608:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.796610:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.796614:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.796998:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.797000:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677446.797001:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.797002:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.797004:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.797006:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.797021:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677446.797024:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677446.797025:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.797025:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.797026:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.797026:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677446.797555:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.797558:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.797559:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.797560:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.797563:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.797564:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.797742:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677446.797745:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677446.797747:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677446.797748:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677446.797750:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.797752:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:13.0:1619677446.797754:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96737106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677446.797755:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677446.797756:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677446.797757:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:13.0:1619677446.797761:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677446.797764:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:13.0:1619677446.797766:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677446.797767:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:13.0:1619677446.797768:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:13.0:1619677446.797770:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:13.0:1619677446.797770:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677446.797771:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677446.797772:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677446.797787:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.797789:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.797790:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.797791:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.797793:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.797794:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.797807:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.797809:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.797810:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.797810:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.797811:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.797812:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.798013:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.798015:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.798016:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.798017:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.798019:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.798020:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.798102:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.798105:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.798106:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.798109:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.798109:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.798112:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.798123:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.798126:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.798126:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.798127:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.798128:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.798129:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.798245:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.798247:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677446.798248:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.798249:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.798251:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.798253:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.798272:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677446.798274:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677446.798276:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.798276:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.798276:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.798277:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677446.798515:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.798518:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.798519:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.798520:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.798522:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.798527:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.798543:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.798545:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.798546:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.798548:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.798549:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.798550:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.798603:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.798605:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.798606:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.798607:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.798609:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.798610:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.798953:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.798955:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.798956:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.798958:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.798959:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.798961:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.798973:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.798976:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.798977:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.798978:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.798978:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.798980:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.799170:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.799172:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.799173:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.799175:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.799176:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.799178:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.799502:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.799505:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.799506:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.799508:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.799510:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.799512:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.799514:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.799515:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.799516:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.799517:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.799520:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.799522:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.799525:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.799526:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.799526:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.799528:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.799529:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.799529:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.799530:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.799545:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.799548:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.799549:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.799557:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.799560:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.799562:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677446.799563:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.799565:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.799566:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.799566:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677446.799569:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.799571:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677446.799574:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.799575:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677446.799575:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677446.799577:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677446.799577:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.799578:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.799579:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677446.800410:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.800413:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.800414:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.800414:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.800417:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:24.0:1619677446.800420:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.800667:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.800670:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.800671:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.800672:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.800675:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.800676:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.800677:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.800679:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.800680:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.800680:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.800684:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.800685:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.800688:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.800689:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.800690:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.800691:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.800692:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.800693:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.800694:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.800695:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.800707:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.800709:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.800710:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.800711:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.800760:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.800762:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.800763:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.800766:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.800767:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.800769:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.800903:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.800905:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.800906:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.800908:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.800910:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.800911:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.801219:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.801221:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.801222:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.801224:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.801227:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.801229:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677446.801230:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.801232:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.801233:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.801234:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677446.801237:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.801239:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677446.801242:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.801243:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677446.801244:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677446.801245:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677446.801246:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.801246:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.801247:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677446.801402:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677446.801405:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677446.801405:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677446.801410:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.801414:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.801416:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677446.801417:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.801418:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.801420:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.801421:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677446.801426:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.801428:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677446.801431:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.801432:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:24.0:1619677446.801433:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:24.0:1619677446.801435:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:24.0:1619677446.801435:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677446.801437:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677446.801437:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:24.0:1619677446.801439:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.801441:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.801442:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.801443:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.801446:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.801448:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677446.801449:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96737056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.801451:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.801453:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.801455:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.0:1619677446.801455:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.801457:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677446.801458:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.801459:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677446.801460:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.801461:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677446.801464:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.801464:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.801464:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.801465:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677446.801466:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677446.801468:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677446.801469:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.801469:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.801470:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.801644:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.801646:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.801647:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.801649:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.801651:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.801652:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.801873:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.801875:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.801878:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.801879:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.801879:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.801882:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.801894:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.801897:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.801897:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.801899:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.801901:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.801901:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.801902:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.801903:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.801903:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.801904:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.801904:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.801967:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.801970:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.801971:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.801972:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.801975:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.801977:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677446.801978:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.801979:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.801980:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.801981:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677446.801984:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.801986:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677446.801989:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.801990:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677446.801990:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677446.801992:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677446.801992:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.801993:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.801994:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677446.802531:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.802533:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.802534:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.802537:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.802540:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.802541:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.802543:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.802544:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.802546:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.802547:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.802552:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.802554:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.802557:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.802558:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.802559:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.802561:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.802562:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.802563:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.802563:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.802576:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.802578:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.802579:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.802581:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.802584:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.802587:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.802588:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96737039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.802594:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.802595:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.802596:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.802599:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.802601:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.802604:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.802605:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.802606:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.802607:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.802608:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.802610:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.802611:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.802611:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.802624:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.802626:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.802627:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.802628:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:6.0:1619677446.802769:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.802771:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.802774:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.803143:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:12.0:1619677446.803233:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.803235:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.803236:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.803238:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.803239:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.803240:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.803280:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.803282:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.803283:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000400:00000200:12.0:1619677446.803283:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.803285:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677446.803285:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:13.0:1619677446.803288:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677446.803288:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.803289:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.803290:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677446.803291:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.803292:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:13.0:1619677446.803298:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96737031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677446.803300:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677446.803301:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677446.803302:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:13.0:1619677446.803305:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677446.803307:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:13.0:1619677446.803309:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677446.803310:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:13.0:1619677446.803311:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:13.0:1619677446.803312:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:13.0:1619677446.803313:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677446.803313:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677446.803314:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:6.0:1619677446.803553:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.803556:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.803558:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.803560:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.803562:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.803620:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.803622:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.803623:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.803624:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.803627:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.803628:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.803638:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.803640:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.803641:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.803642:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.803642:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.803643:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.803788:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.803790:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.803792:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.803792:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.803794:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.803797:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677446.804341:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.804447:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.804450:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.804451:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.804454:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.804457:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.804533:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.804535:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.804536:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.804538:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.804540:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.804541:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.804747:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.804750:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.804751:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.804752:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677446.804755:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:24.0:1619677446.804756:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.804758:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.804759:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96737012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.804762:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.804762:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.804763:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.804767:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.804769:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.804772:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.804772:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.804773:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.804775:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.804775:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.804776:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.804776:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677446.804784:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677446.804786:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677446.804788:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.804788:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.804789:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.804814:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.804849:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.804851:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.804852:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.804853:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677446.805054:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:12.0:1619677446.805292:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677446.805300:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:13.0:1619677446.805302:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677446.805302:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677446.805304:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.805306:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677446.805307:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96737004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677446.805309:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677446.805310:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677446.805311:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677446.805315:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677446.805317:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677446.805320:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677446.805320:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:12.0:1619677446.805321:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677446.805323:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677446.805323:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677446.805325:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677446.805327:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677446.805327:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677446.805341:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677446.805343:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677446.805344:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677446.805345:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:6.0:1619677446.805701:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.805704:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.805705:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.805707:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.805710:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.805768:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:13.0:1619677446.805998:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.806000:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.806001:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.806004:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.806004:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.806007:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.806070:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.806072:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.806074:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.806074:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.806076:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.806079:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677446.806229:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.806232:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677446.806236:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.806239:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.806240:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677446.806242:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96736994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.806243:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.806245:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.806246:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677446.806251:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.806253:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677446.806256:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.806257:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677446.806258:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677446.806261:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677446.806261:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.806262:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.806263:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677446.806470:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.806473:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.806474:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.806475:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.806476:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.806479:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.806487:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677446.806490:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677446.806491:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.806493:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677446.806494:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.806494:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.806614:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.806617:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.806618:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.806618:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.806620:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.806623:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677446.806656:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.806659:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.806660:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.806662:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.806663:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.806665:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.806781:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:13.0:1619677446.806828:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.806831:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.806832:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.806833:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.806835:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.806836:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.806848:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.806850:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.806851:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.806852:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.806852:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.806854:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.807001:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.807005:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.807005:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677446.807008:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.807011:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.807012:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677446.807013:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:13.0:1619677446.807015:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:8.0:1619677446.807015:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96736982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677446.807016:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.807017:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:8.0:1619677446.807017:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.807018:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677446.807019:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:8.0:1619677446.807019:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677446.807020:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677446.807023:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.807025:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.807028:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.807029:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.807029:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.807031:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.807032:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.807033:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.807034:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.807035:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.807048:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.807050:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.807051:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.807052:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.807336:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.807339:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.807340:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.807342:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.807344:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.807344:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.807407:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.807409:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.807410:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.807411:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.807413:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.807413:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677446.807417:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.807420:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.807422:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.807424:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:6.0:1619677446.807424:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.807427:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677446.807427:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.807428:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.807428:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.807429:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.807431:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.807483:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.807497:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677446.807499:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677446.807500:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.807501:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.807502:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.807550:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.807553:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.807554:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.807555:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.807557:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.807557:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.807726:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.807729:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.807730:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.807731:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.807733:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.807734:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.807751:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.807754:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.807755:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.807757:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.807760:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.807761:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.807763:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.807764:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.807765:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.807766:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.807770:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.807772:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.807775:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.807776:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.807776:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.807778:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.807778:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.807779:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.807779:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677446.807856:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:24.0:1619677446.807923:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.807925:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.807926:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.807930:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.807932:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.807934:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677446.807936:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.807937:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.807938:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.807939:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677446.807943:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.807945:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677446.807949:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.807950:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677446.807951:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677446.807953:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677446.807953:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.807955:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.807955:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677446.807966:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.807969:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.807970:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.807972:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.807975:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.808008:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.808010:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.808011:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.808012:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.808014:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.808014:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.808131:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.808134:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.808135:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.808137:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.808138:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.808139:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.808149:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.808151:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.808152:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.808153:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.808154:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.808155:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.808245:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.808280:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.808282:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.808283:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.808284:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677446.808446:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.808449:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.808450:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.808450:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.808452:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.808456:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677446.808659:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.808661:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.808662:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.808664:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.808666:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.808667:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.808715:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.808718:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:13.0:1619677446.808719:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677446.808719:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.808721:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677446.808721:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.808722:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.808723:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.808724:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:24.0:1619677446.808725:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:12.0:1619677446.808725:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.808726:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677446.808727:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96736960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.808728:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.808729:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.808730:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.808733:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.808735:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.808738:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.808739:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.808739:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.808742:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.808743:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.808744:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.808745:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677446.808753:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677446.808839:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.808842:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.808843:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.808843:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.808845:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.808848:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677446.808856:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.808858:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.808859:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.808860:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.808862:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.808863:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.809345:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.809347:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.809348:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.809350:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.809352:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.809353:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.809361:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.809364:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.809364:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.809365:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.809366:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.809367:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.809461:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.809464:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.809464:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.809466:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.809469:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.809470:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.809472:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.809473:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.809474:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.809475:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.809478:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.809480:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.809483:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.809484:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.809484:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.809486:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.809487:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.809487:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.809488:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677446.809573:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.809575:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677446.809576:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.809579:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.809581:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.809594:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.809617:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.809620:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.809621:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.809623:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.809626:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.809640:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:13.0:1619677446.809953:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.809956:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.809957:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.809958:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.809959:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.809961:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.810165:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.810168:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677446.810171:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.810173:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.810174:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677446.810175:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.810177:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.810178:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.810179:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677446.810182:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.810184:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677446.810187:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.810188:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677446.810189:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677446.810190:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677446.810191:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.810192:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.810192:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677446.810360:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.810518:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.810521:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.810523:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.810524:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.810527:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.810910:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.810913:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.810914:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.810916:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.810918:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.810920:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.810980:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:13.0:1619677446.811074:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.811076:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.811077:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.811078:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.811080:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.811081:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.811344:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.811347:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.811347:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.811349:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.811351:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.811352:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.811361:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.811363:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.811364:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.811365:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.811365:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.811367:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.811415:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.811418:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.811419:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.811421:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.811423:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.811447:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677446.811792:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.811794:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.811795:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.811797:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.811798:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.811799:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.811884:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.811887:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.811889:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.811890:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.811891:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.811893:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.812069:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.812072:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677446.812075:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.812078:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.812080:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677446.812082:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96736919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.812083:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.812085:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.812086:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677446.812090:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.812092:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677446.812096:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.812097:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677446.812097:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677446.812099:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677446.812100:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.812101:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.812102:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.812103:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.812140:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677446.812282:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.812285:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.812286:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.812288:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.812291:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.812292:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677446.812299:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.812300:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.812301:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.812302:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677446.812306:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.812308:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677446.812312:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.812313:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677446.812313:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677446.812315:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677446.812315:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.812316:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.812317:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677446.812334:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (25)--
00000800:00000200:6.0:1619677446.812347:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.812349:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677446.812350:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (23)++
00000800:00000200:6.0:1619677446.812351:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.812352:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.812365:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.812367:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.812368:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.812370:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.812372:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.812373:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.812382:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.812384:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.812385:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.812386:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.812386:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.812387:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677446.812388:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.812390:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.812391:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.812393:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.812395:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.812397:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.812398:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.812399:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.812400:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.812401:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.812404:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.812406:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.812408:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.812409:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.812410:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.812411:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.812412:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.812413:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.812414:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.812415:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.812428:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.812430:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.812431:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.812432:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.812467:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.812469:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.812470:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.812472:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.812473:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.812475:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.812685:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677446.812687:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677446.812688:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.812689:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.812691:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677446.812692:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677446.812692:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677446.812693:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.812693:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.812694:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.812696:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.812877:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.812879:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.812880:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.812882:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677446.812883:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.812885:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.812919:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.812922:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.812922:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.812924:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.812925:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.812927:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.813063:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:24.0:1619677446.813070:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.813072:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.813074:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.813076:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.813079:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.813080:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677446.813081:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.813082:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.813083:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.813085:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677446.813088:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.813090:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677446.813092:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.813093:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677446.813094:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677446.813095:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.813096:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.813096:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.813097:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.813107:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.813109:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.813110:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.813112:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.813113:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.813114:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.813191:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.813194:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.813195:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.813197:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.813200:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677446.813541:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.813544:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.813545:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.813546:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.813549:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.813551:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.813552:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.813553:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.813554:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.813555:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.813559:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.813560:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.813563:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.813564:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.813564:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.813566:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.813566:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.813567:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.813567:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.813677:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.813680:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.813681:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.813682:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.813684:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.813685:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.813689:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.813692:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.813693:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677446.813694:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:24.0:1619677446.813695:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.813697:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.813697:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.813698:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.813698:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:24.0:1619677446.813699:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:13.0:1619677446.813699:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:24.0:1619677446.813700:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677446.813700:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:24.0:1619677446.813702:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.813703:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.813704:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.813707:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.813709:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.813712:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.813712:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.813713:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.813715:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.813716:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.813717:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.813718:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.813719:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.813731:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.813733:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.813734:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.813735:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677446.813804:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.813809:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.813843:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.813846:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.813846:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677446.813849:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.813851:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.813853:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677446.813855:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.813857:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.813858:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.813859:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677446.813865:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.813868:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677446.813871:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.813872:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677446.813873:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677446.813874:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677446.813875:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.813876:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.813877:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.813887:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.813889:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.813890:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.813892:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.813893:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.813894:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.813906:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.813910:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.813911:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.813911:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.813912:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.813913:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.814230:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.814233:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.814233:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.814234:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.814236:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.814239:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.814252:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677446.814254:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.814255:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.814256:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.814257:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.814257:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677446.814311:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.814314:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.814315:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.814317:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.814320:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.814321:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677446.814322:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.814324:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.814325:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.814326:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677446.814329:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.814330:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677446.814333:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.814334:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677446.814334:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677446.814336:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.814336:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.814337:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.814338:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.814350:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.814352:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.814353:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.814354:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.814355:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677446.814356:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.814357:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.814357:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.814358:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.814360:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.814362:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.814364:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.814365:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.814366:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.814367:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.814368:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.814370:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.814372:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.814374:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.814375:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.814376:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.814377:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.814378:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.814378:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.814379:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677446.814572:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.814575:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.814576:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.814576:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.814578:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.814580:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.814626:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677446.814628:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677446.814629:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:6.0:1619677446.814631:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.814634:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.814636:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:6.0:1619677446.814637:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96736887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.814639:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.814640:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.814641:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:6.0:1619677446.814646:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.814649:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677446.814652:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.814652:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677446.814653:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677446.814656:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677446.814656:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677446.814657:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677446.814658:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:22.0:1619677446.814671:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.814674:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.814675:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.814675:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.814678:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.814682:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.814894:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.814897:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.814898:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.814898:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.814900:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677446.814903:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677446.815008:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.815011:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.815012:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.815015:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.815018:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.815019:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.815020:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.815022:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.815023:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.815024:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.815029:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.815031:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.815034:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.815035:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.815036:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.815038:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.815039:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.815040:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.815044:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677446.815045:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.815047:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.815048:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.815050:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677446.815051:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:24.0:1619677446.815052:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.815053:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.815054:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96736881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677446.815054:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.815055:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.815056:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.815057:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677446.815057:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:24.0:1619677446.815058:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:12.0:1619677446.815058:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.815059:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677446.815062:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.815064:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.815067:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.815068:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.815069:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.815070:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.815071:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.815072:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.815072:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.815103:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.815106:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.815107:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.815107:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.815109:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.815113:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.815122:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677446.815125:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677446.815126:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.815126:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677446.815127:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677446.815127:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677446.815197:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.815200:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677446.815200:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.815201:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.815203:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.815206:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677446.815596:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.815598:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.815599:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.815601:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.815603:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.815604:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.815939:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.815941:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.815942:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.815944:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.815947:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.815948:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677446.815949:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96736871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.815951:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.815952:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.815953:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677446.815955:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.815956:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:6.0:1619677446.815957:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.0:1619677446.815959:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.815960:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.815960:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.815960:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.815961:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677446.815962:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:8.0:1619677446.815963:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.815964:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677446.815964:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.815965:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677446.815966:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677446.815966:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.816007:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.816009:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.816010:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.816012:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.816013:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.816015:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.816026:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.816029:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.816030:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.816031:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.816031:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.816033:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.816097:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.816100:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.816101:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.816102:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.816104:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.816106:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:22.0:1619677446.816107:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.816108:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.816109:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.816111:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.816112:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.816114:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.816398:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.816401:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.816402:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.816405:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.816405:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.816408:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.816520:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c603a9000] (21)++
00000800:00000200:24.0:1619677446.816523:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.21@o2ib3
00000800:00000200:8.0:1619677446.816524:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c603a9000] (22)--
00000400:00000200:24.0:1619677446.816526:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.21@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.816529:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.816531:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:8.0:1619677446.816532:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:24.0:1619677446.816533:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677446.816533:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677446.816534:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.816535:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.816536:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.816536:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:24.0:1619677446.816537:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:8.0:1619677446.816540:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.816541:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:24.0:1619677446.816542:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.816544:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:24.0:1619677446.816547:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.816548:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:24.0:1619677446.816548:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:24.0:1619677446.816550:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:24.0:1619677446.816551:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c603a9000] (21)++
00000800:00000200:24.0:1619677446.816551:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c603a9000] (22)--
00000800:00000200:24.0:1619677446.816552:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c603a9000] (21)--
00000800:00000200:6.0:1619677446.816788:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.816791:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.816792:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.816794:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.816796:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.816797:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677446.816799:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96736857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.816800:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.816801:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.816802:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677446.816806:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.816809:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677446.816811:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.816812:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677446.816813:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677446.816814:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677446.816815:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.816815:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.816816:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677446.816937:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.816940:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.816941:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.816941:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.816944:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.816947:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.816954:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677446.816957:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677446.816958:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.816958:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677446.816960:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677446.816960:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677446.816970:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.816973:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.816974:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.816975:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.816976:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.816978:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.817155:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.817158:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.817159:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.817161:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.817162:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.817164:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.817322:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.817324:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.817325:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.817327:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.817330:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.817331:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677446.817333:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.817334:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.817335:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.817336:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677446.817340:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.817342:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677446.817344:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.817345:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677446.817346:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677446.817347:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677446.817348:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.817349:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.817350:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.817350:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.817361:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.817363:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.817364:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.817365:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677446.817460:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.817462:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.817463:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.817463:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.817466:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.817468:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.817477:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677446.817480:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.817480:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677446.817481:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.817482:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.817482:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.817495:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.817497:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.817498:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.817499:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.817501:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.817504:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.817513:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677446.817515:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.817516:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.817516:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.817517:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.817518:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.817713:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.817715:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.817716:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.817718:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.817720:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.817721:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.817731:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.817733:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.817734:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.817735:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.817735:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.817736:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:12.0:1619677446.817737:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.817739:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.817740:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.817742:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.817744:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.817746:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.817747:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.817749:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.817750:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.817751:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.817754:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.817756:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.817758:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.817759:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.817760:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.817761:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.817762:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.817763:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.817765:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.817765:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.817778:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.817780:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.817781:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.817782:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.817930:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.817931:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.817932:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.817932:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:13.0:1619677446.817934:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.817934:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.817934:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.817936:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.817938:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.817939:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.817940:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.817941:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.818026:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.818029:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.818030:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.818030:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.818032:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.818034:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.818092:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.818094:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.818095:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.818095:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.818098:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.818100:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677446.818555:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.818558:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.818559:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.818560:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.818564:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.818565:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677446.818567:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.818568:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.818569:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.818570:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677446.818573:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.818574:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677446.818577:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.818577:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677446.818578:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677446.818580:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677446.818580:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.818581:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.818581:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677446.818594:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.818597:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.818598:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.818598:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.818600:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.818603:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.818691:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.818694:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.818695:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.818697:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.818699:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.818701:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677446.818702:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96736834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.818704:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.818705:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.818706:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677446.818710:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.818712:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677446.818714:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.818715:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677446.818715:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677446.818717:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677446.818718:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.818718:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.818719:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.818900:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.818903:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.818904:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.818905:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.818907:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.818908:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.819077:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.819080:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.819081:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.819082:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.819084:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.819093:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.819094:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.819096:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.819097:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.819098:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.819099:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.819099:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677446.819292:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.819299:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.819299:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.819302:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.819305:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.819306:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677446.819307:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.819309:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.819310:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.819310:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677446.819313:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.819315:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677446.819317:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.819318:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677446.819318:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677446.819320:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.819320:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.819321:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.819323:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.819324:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677446.819334:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.819336:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.819337:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.819337:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.819339:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:13.0:1619677446.819340:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.819340:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677446.819341:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.819341:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.819342:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.819373:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.819375:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.819376:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.819376:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.819378:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.819381:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677446.819583:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.819585:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.819587:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.819588:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.819591:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.819593:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.819594:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.819596:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.819597:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.819598:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677446.819600:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:24.0:1619677446.819601:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.819603:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:24.0:1619677446.819604:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677446.819604:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.819606:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677446.819607:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.819608:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.819608:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:22.0:1619677446.819609:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.819610:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.819610:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:22.0:1619677446.819610:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:24.0:1619677446.819611:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677446.819611:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96736825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677446.819612:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677446.819613:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.819614:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.819615:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677446.819618:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.819620:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677446.819623:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.819623:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677446.819624:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677446.819626:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677446.819626:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.819627:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.819628:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677446.820060:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.820062:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.820063:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.820065:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.820068:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.820069:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677446.820070:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.820072:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.820073:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.820074:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677446.820077:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.820078:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677446.820080:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.820081:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677446.820082:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677446.820083:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.820083:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.820084:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.820085:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.820095:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.820098:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.820099:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.820101:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.820103:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.820103:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.820112:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.820115:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.820116:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.820117:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.820117:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.820119:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.820246:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.820249:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.820250:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.820253:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.820253:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.820256:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.820443:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.820446:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.820446:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.820448:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.820450:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.820451:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.820455:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.820457:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.820458:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.820460:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.820462:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.820464:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.820465:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.820466:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.820467:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.820468:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.820471:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.820472:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.820475:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.820476:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.820476:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.820478:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.820478:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.820480:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.820480:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677446.820738:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.820741:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.820742:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.820742:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.820744:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.820746:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.820923:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.820925:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.820926:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.820928:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.820929:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.820930:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.820944:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.820946:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.820947:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.820948:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.820948:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.820950:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.821047:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.821050:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.821051:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.821053:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.821056:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.821057:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.821058:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.821060:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.821061:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.821062:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.821065:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.821067:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.821070:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.821071:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.821071:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.821073:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.821074:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.821074:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.821075:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677446.821101:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.821104:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.821105:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.821107:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.821109:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.821111:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677446.821112:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.821114:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.821115:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.821116:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677446.821119:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.821121:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677446.821123:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.821124:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677446.821124:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677446.821126:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677446.821126:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.821127:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.821128:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677446.821322:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677446.821322:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677446.821325:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:13.0:1619677446.821325:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.821326:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.821326:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.821328:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677446.821328:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.821329:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677446.821331:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.821331:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677446.821332:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.821333:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.821335:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.821336:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.821337:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.821340:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.821342:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.821344:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.821345:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.821346:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.821347:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.821348:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.821348:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.821349:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677446.821764:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.821767:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.821768:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.821769:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.821771:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.821774:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.821890:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.821892:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.821893:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.821895:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.821896:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.821898:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.821975:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.821977:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.821978:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.821980:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.821983:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.821984:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677446.821986:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96736789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.821987:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.821988:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.821989:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677446.821992:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.821994:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677446.821997:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.821997:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677446.821998:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677446.822000:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677446.822000:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.822001:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.822002:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.822003:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.822014:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.822017:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.822018:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.822018:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.822454:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.822456:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.822457:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.822459:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.822460:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.822461:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.822540:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.822543:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.822544:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.822546:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.822549:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.822550:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.822552:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.822553:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.822554:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.822555:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.822559:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.822561:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.822564:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.822565:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.822565:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.822568:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.822568:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.822569:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.822569:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677446.822672:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.822675:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.822676:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.822678:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.822680:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.822682:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677446.822683:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.822684:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.822686:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.822687:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677446.822691:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.822694:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677446.822696:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.822697:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677446.822698:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677446.822699:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677446.822700:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.822700:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.822701:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677446.822998:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.823000:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.823002:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.823002:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.823004:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.823006:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677446.823096:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:22.0:1619677446.823098:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:24.0:1619677446.823099:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.823100:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.823100:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:8.0:1619677446.823100:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:6.0:1619677446.823101:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.823102:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.823103:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:22.0:1619677446.823104:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.823106:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677446.823108:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677446.823446:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.823448:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.823449:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677446.823450:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.823452:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.823455:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677446.823610:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.823613:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.823613:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.823615:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.823617:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.823618:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.823653:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.823655:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.823656:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.823658:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.823660:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.823661:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.823663:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.823664:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.823665:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.823666:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.823669:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.823671:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.823674:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.823674:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.823675:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.823676:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.823677:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.823678:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.823679:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.823680:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.823693:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.823695:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.823696:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.823697:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.823746:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677446.823747:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677446.823748:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.823749:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.823750:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:13.0:1619677446.823750:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.823750:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:13.0:1619677446.823752:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.823752:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:6.0:1619677446.823753:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.823755:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.823757:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677446.823758:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.823759:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.823760:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.823761:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:12.0:1619677446.823765:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:6.0:1619677446.823765:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677446.823767:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677446.823767:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677446.823769:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.823769:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.823770:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.823771:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.823771:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.823771:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677446.823772:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677446.823774:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677446.823774:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.823775:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.823776:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677446.823895:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.823898:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.823899:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.823899:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.823902:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.823905:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.823993:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.823996:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.823997:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.823998:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.824000:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.824003:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.824014:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.824017:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.824018:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.824018:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.824019:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.824019:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.824749:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.824752:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.824752:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677446.824754:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.824757:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.824758:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677446.824760:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.824761:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.824762:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.824763:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677446.824766:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.824768:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677446.824770:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.824771:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677446.824772:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677446.824773:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677446.824774:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.824774:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.824775:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677446.824846:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.824849:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.824850:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.824851:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.824853:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:24.0:1619677446.824856:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.824929:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.824932:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.824933:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.824934:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.824937:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.824938:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677446.824940:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736753] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.824941:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.824942:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.824943:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:13.0:1619677446.824944:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.824947:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.824947:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.824947:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677446.824949:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.824949:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:12.0:1619677446.824951:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.824951:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677446.824952:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.824952:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677446.824953:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677446.824955:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677446.824955:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.824956:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.824957:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.824958:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677446.824968:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:24.0:1619677446.824971:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:22.0:1619677446.824971:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.824972:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.824972:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:24.0:1619677446.824973:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.824973:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.824974:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:22.0:1619677446.824975:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.824978:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.825048:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.825051:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.825052:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.825052:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.825055:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.825057:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677446.825264:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.825266:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.825267:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.825269:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.825272:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.825273:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.825275:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.825276:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.825277:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.825278:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.825281:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.825283:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.825286:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.825287:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.825287:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.825289:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.825289:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.825290:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.825290:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.825398:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.825401:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.825402:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.825403:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.825405:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.825406:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.825462:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.825465:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.825466:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.825467:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.825469:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.825470:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.825656:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.825659:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.825660:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.825662:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.825664:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.825666:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677446.825667:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.825668:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.825669:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.825670:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677446.825673:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.825675:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677446.825678:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.825679:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677446.825679:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677446.825681:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677446.825682:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.825682:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.825683:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677446.825695:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.825698:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.825699:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.825700:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.825702:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.825704:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.825711:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.825714:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.825715:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.825716:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.825718:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.825719:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.826147:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.826149:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.826150:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.826152:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.826155:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.826156:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677446.826158:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.826159:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.826160:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.826161:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677446.826164:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.826165:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677446.826168:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.826169:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677446.826169:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677446.826171:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677446.826172:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.826172:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.826173:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.826241:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.826243:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.826244:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.826246:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.826247:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.826249:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.826252:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.826255:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.826255:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.826256:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.826258:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:6.0:1619677446.826258:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.826261:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677446.826261:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.826262:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.826262:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.826263:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.826264:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.826270:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.826273:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.826274:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.826274:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.826275:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.826276:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677446.826365:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.826369:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.826371:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.826372:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.826375:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.826376:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677446.826378:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.826379:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.826380:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.826381:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677446.826384:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.826386:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677446.826389:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.826389:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677446.826390:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677446.826392:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677446.826392:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.826393:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.826394:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677446.826406:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.826410:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.826410:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.826411:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.826413:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.826416:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.826432:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.826435:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.826435:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.826437:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.826439:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.826441:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677446.826442:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.826443:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.826444:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.826445:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677446.826449:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.826451:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677446.826454:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.826454:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677446.826455:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677446.826457:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677446.826457:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.826459:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.826460:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.826460:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.826470:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.826472:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.826473:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.826474:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677446.826579:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.826581:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677446.826582:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.826583:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.826584:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.826587:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677446.826807:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.826810:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.826810:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.826812:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:12.0:1619677446.826812:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.826814:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.826815:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:12.0:1619677446.826815:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.826816:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.826818:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.826821:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.826822:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.826823:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.826825:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.826826:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.826827:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.826830:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.826832:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.826835:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.826836:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.826836:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.826838:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.826838:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.826839:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.826840:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.826960:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.826963:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.826964:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.826965:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.826967:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.826968:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.827002:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.827005:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.827006:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.827008:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.827009:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.827010:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.827021:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.827024:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.827025:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.827026:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.827027:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.827028:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.827054:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.827056:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.827057:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.827058:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.827060:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.827062:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.827162:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.827165:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.827166:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677446.827166:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.827168:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.827171:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.827181:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677446.827184:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677446.827184:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.827185:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.827186:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.827186:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.827269:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.827272:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.827272:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677446.827276:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.827278:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.827280:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677446.827281:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96736721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.827282:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.827284:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.827284:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677446.827288:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.827290:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677446.827299:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.827300:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.827301:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.827301:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:22.0:1619677446.827302:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.827302:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:22.0:1619677446.827303:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.827305:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677446.827305:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.827306:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.827307:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.827307:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677446.827308:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.827408:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.827410:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.827411:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.827413:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.827415:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.827416:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.827623:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.827626:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.827628:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.827629:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.827632:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.827633:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677446.827635:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.827636:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.827637:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.827638:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677446.827641:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.827643:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677446.827645:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.827646:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677446.827646:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677446.827647:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677446.827648:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.827648:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.827649:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.827649:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677446.827651:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.827652:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.827652:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.827654:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.827657:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.827659:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.827662:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.827663:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.827665:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.827666:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:13.0:1619677446.827667:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.827668:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.827669:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.827670:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.827670:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.827672:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.827672:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677446.827675:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.827678:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.827679:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.827681:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.827683:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.827684:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.827686:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.827687:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.827688:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.827690:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.827701:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.827704:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.827707:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.827708:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.827709:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.827711:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.827711:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.827713:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.827714:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.827715:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.827727:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.827729:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.827730:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.827731:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677446.827935:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.827937:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.827938:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.827938:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.827941:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677446.827944:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.827977:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.827980:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.827981:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.827981:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.827983:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.827987:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.828030:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.828033:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.828034:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.828037:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677446.828039:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.828050:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.828205:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.828208:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.828209:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.828209:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.828211:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.828213:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.828345:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.828348:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.828349:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.828349:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.828351:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.828354:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.828402:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.828405:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.828406:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.828409:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.828412:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.828414:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677446.828416:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96736701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.828417:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.828418:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.828420:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677446.828423:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.828425:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677446.828428:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.828429:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:6.0:1619677446.828430:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (23)++
00000800:00000200:6.0:1619677446.828433:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (25)--
00000800:00000200:6.0:1619677446.828433:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.828434:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.828435:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677446.828507:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677446.828511:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[3] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677446.828511:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.828515:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.109@o2ib7 : PUT - routed
00000800:00000200:24.0:1619677446.828517:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677446.828526:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.828528:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.828528:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.828530:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.828532:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.828533:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.828535:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677446.828538:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677446.828539:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.828540:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.13.21@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.109@o2ib7 : PUT - routed
00000800:00000200:22.0:1619677446.828542:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677446.828542:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677446.828544:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.828545:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.828546:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.828546:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677446.828548:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.828549:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.828550:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.828551:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.828551:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677446.828553:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.828554:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:12.0:1619677446.828554:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.828557:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.828557:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.828558:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.828558:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.828559:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.828625:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.828627:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.828628:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.828629:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.828631:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.828632:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.828923:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.828925:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.828926:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.828928:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.828929:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.828930:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.829065:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.829068:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.829069:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.829070:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.829073:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.829074:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:13.0:1619677446.829074:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:24.0:1619677446.829076:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677446.829076:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:24.0:1619677446.829077:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677446.829077:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.829078:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.829079:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:13.0:1619677446.829079:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.829080:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:24.0:1619677446.829082:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.829083:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.829084:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.829086:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.829087:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677446.829087:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:24.0:1619677446.829089:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:24.0:1619677446.829089:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.829091:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.829092:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.829092:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677446.829093:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.829095:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.829096:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677446.829097:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677446.829097:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.829098:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.829101:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.829102:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.829103:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.829105:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.829105:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677446.829105:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.829107:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.829108:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.829109:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677446.829165:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.829168:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.829169:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.829171:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.829174:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.829175:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677446.829177:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.829178:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.829179:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.829180:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677446.829183:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.829185:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677446.829188:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.829189:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677446.829189:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677446.829191:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677446.829191:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.829192:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.829192:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677446.829306:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.829308:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.829309:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.829309:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.829312:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.829314:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677446.830144:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.830147:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.830148:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.830150:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.830153:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.830155:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.830156:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.830157:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.830158:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.830159:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.830163:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.830165:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.830168:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.830168:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.830169:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.830170:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.830171:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.830172:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.830172:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.830369:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.830372:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.830373:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.830375:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.830377:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.830378:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.830758:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.830761:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.830762:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.830764:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.830765:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.830766:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.831022:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.831025:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.831026:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.831028:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.831031:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.831032:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.831034:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.831036:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.831037:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.831038:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.831041:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.831043:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.831046:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.831047:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.831048:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.831050:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.831050:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.831051:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.831052:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677446.831080:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.831083:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.831085:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.831085:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.831087:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.831091:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677446.831128:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.831130:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.831131:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.831133:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.831134:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.831136:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.831147:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.831150:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.831150:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.831151:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.831152:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.831152:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.831274:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.831277:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.831277:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.831278:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.831280:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.831282:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677446.831346:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:6.0:1619677446.831347:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:24.0:1619677446.831349:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.831350:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.831350:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:6.0:1619677446.831351:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.831352:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.831352:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.831353:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:6.0:1619677446.831355:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.831356:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677446.831358:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677446.831461:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.831465:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.831469:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677446.831471:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.831474:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.831476:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.831478:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736667] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.831479:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.831480:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.831481:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.831485:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.831486:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.831490:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.831490:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.831491:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.831493:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.831493:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.831494:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.831495:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677446.831742:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.831744:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.831745:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.831747:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677446.831749:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:22.0:1619677446.831749:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.831751:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:24.0:1619677446.831752:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:22.0:1619677446.831752:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96736663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.831753:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.831753:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:22.0:1619677446.831754:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.831755:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:22.0:1619677446.831755:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.831756:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:24.0:1619677446.831758:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:22.0:1619677446.831759:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.831761:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677446.831764:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.831764:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677446.831765:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677446.831767:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677446.831768:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.831769:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.831770:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.831771:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.831781:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.831782:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.831783:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.831784:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677446.832167:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.832170:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.832171:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.832171:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.832173:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.832175:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677446.832377:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.832379:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.832380:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.832381:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.832383:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.832386:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.832393:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677446.832395:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677446.832396:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.832396:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677446.832397:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677446.832398:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.832441:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.832443:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.832445:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.832446:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.832449:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.832451:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677446.832452:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.832453:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.832454:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.832455:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677446.832459:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.832460:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677446.832463:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.832464:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677446.832465:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677446.832466:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677446.832467:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.832467:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.832468:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677446.832481:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.832484:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.832484:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.832485:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.832487:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.832490:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.832498:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.832500:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.832501:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.832501:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.832502:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.832503:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.832542:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.832545:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.832546:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.832548:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.832550:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.832550:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.832630:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.832632:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.832633:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.832635:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.832639:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.832640:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.832642:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.832644:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.832645:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.832647:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:6.0:1619677446.832648:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.832651:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.832651:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677446.832655:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.832657:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.832659:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677446.832661:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96736647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.832662:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677446.832663:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677446.832664:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:6.0:1619677446.832665:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.832666:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677446.832667:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.832668:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.832669:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.832670:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:6.0:1619677446.832670:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.832671:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.832672:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677446.832672:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:24.0:1619677446.832673:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677446.832675:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.832676:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677446.832677:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677446.832679:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677446.832679:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.832680:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.832681:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677446.832937:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.832939:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.832940:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.832942:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.832943:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.832945:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.833194:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.833197:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.833197:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.833198:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.833200:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.833203:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.833243:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.833245:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.833246:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.833247:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.833249:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.833251:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.833262:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677446.833264:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677446.833265:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.833266:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.833266:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.833267:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677446.833391:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.833393:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.833394:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.833396:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.833399:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.833401:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677446.833402:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.833403:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.833404:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.833412:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677446.833415:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.833417:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677446.833420:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.833421:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677446.833421:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677446.833423:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677446.833424:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.833424:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.833425:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677446.833438:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.833441:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.833442:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.833442:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.833444:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.833447:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.833507:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.833510:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.833511:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.833512:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.833514:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677446.833515:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:13.0:1619677446.833515:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.833517:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.833518:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.833519:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.833521:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.833523:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.833562:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.833564:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.833565:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.833567:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.833570:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.833571:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677446.833572:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.833574:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.833575:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.833576:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677446.833579:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.833581:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677446.833583:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.833584:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677446.833584:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677446.833586:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677446.833587:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.833587:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.833588:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677446.833910:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:12.0:1619677446.833910:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677446.833913:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:12.0:1619677446.833913:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.833913:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.833914:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:12.0:1619677446.833915:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:24.0:1619677446.833917:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677446.833917:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.833918:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677446.833919:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.833920:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677446.833922:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.833923:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.833924:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.833925:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677446.833928:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.833930:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677446.833934:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.833934:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677446.833934:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:24.0:1619677446.833935:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.833936:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:24.0:1619677446.833937:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677446.833937:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:6.0:1619677446.833937:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.833938:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.833939:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.833940:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000400:00000200:22.0:1619677446.833940:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.833941:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677446.833942:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.833951:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677446.833953:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:6.0:1619677446.833953:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.833954:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.833954:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.833955:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.833956:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:6.0:1619677446.833956:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.833957:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677446.833959:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677446.833984:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.833986:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.833987:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.833989:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.833991:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.833993:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.833994:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.833995:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.833996:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.833997:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.834000:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.834001:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.834004:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.834005:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.834005:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.834007:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.834007:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.834008:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.834009:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.834010:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.834022:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.834025:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.834025:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.834027:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677446.834065:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.834068:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.834069:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.834069:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.834071:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.834078:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.834079:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.834081:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.834082:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.834082:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.834084:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.834084:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.834143:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.834146:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.834147:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.834149:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.834151:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.834152:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677446.834154:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.834155:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.834156:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.834157:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677446.834160:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.834162:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677446.834164:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.834165:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677446.834165:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677446.834167:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677446.834168:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.834168:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.834169:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677446.834447:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.834450:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.834451:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.834452:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.834455:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.834457:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.834459:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.834460:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.834461:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.834461:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.834464:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.834466:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.834469:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.834469:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.834470:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.834472:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.834472:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.834473:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.834473:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.834650:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.834653:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.834654:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.834655:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.834658:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.834659:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.834660:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.834663:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.834663:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.834665:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.834667:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.834668:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.834682:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.834685:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.834686:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.834687:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.834687:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.834689:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.834754:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.834757:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.834758:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.834760:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.834763:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.834764:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677446.834765:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.834767:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.834768:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.834769:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677446.834772:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.834774:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677446.834777:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.834778:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677446.834778:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677446.834780:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677446.834781:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.834783:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.834784:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.834784:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.834794:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.834796:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.834797:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.834798:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.834867:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.834869:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.834870:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.834871:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.834873:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.834874:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.834886:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.834888:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.834889:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.834890:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.834890:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.834891:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.835092:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.835094:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.835095:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.835100:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.835103:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.835105:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.835107:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.835108:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.835110:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.835111:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.835122:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.835124:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.835127:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.835128:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.835128:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.835130:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.835130:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.835131:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.835132:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.835136:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.835138:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.835139:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.835141:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.835143:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.835144:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.835482:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.835484:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.835485:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.835487:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.835489:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.835489:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.835700:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.835702:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.835703:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.835705:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.835706:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.835707:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.835714:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.835717:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.835718:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.835719:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.835720:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.835722:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.835847:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.835849:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.835850:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.835850:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.835852:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677446.835855:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677446.836033:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.836036:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.836038:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.836039:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.836042:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.836044:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677446.836046:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.836048:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.836049:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.836050:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677446.836053:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.836055:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677446.836058:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.836059:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677446.836059:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677446.836061:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677446.836061:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.836062:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.836063:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.836240:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.836243:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.836244:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.836246:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.836249:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.836251:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.836252:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96736603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.836254:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.836255:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.836255:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.836259:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.836261:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.836263:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.836264:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.836273:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.836275:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.836275:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.836276:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.836277:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677446.836385:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.836388:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.836389:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.836390:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.836392:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.836394:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.836752:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.836754:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.836755:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.836756:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.836758:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.836761:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.836855:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.836858:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.836859:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.836861:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.836863:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.836864:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.836951:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.836955:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:24.0:1619677446.836957:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677446.836957:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.836961:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.836962:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.836964:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.836965:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.836966:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.836967:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.836970:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.836971:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.836974:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.836975:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.836976:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.836978:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.836979:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.836979:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.836980:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677446.836982:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.836983:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.836983:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.836985:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.836987:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.836989:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.836989:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.836991:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.836991:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.836992:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.836995:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.836996:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.837000:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.837001:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.837001:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.837003:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.837004:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.837004:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.837005:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677446.837312:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.837315:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.837315:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677446.837318:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.837325:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.837327:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677446.837329:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96736589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.837330:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.837331:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.837332:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677446.837335:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.837337:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677446.837340:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.837341:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677446.837342:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677446.837344:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677446.837345:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.837346:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.837346:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.837347:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.837347:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:22.0:1619677446.837348:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.837348:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.837350:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.837353:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.837366:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.837369:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.837369:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677446.837370:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.837371:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.837371:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.837438:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.837440:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.837441:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.837441:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.837443:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.837446:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677446.837553:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.837557:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.837557:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.837561:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.837564:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.837566:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677446.837567:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.837569:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.837570:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.837571:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677446.837581:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.837584:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677446.837588:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.837589:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677446.837589:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677446.837591:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677446.837592:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.837593:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.837595:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.837595:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677446.837605:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.837607:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.837608:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.837609:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:13.0:1619677446.837609:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.837611:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.837612:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677446.837612:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.837613:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.837613:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.837622:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.837625:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.837626:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.837626:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.837627:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.837628:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.837846:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.837849:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.837850:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.837850:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.837852:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.837854:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.837863:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677446.837866:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677446.837866:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.837867:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.837868:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.837868:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.837969:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.837971:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.837972:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.837973:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.837975:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.837979:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.838005:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.838008:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.838009:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.838009:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.838012:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.838014:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.838231:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.838233:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677446.838234:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.838235:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.838237:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.838239:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677446.838308:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.838311:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.838312:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.838314:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.838316:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.838318:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.838319:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.838321:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.838322:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.838323:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.838326:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.838327:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.838330:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.838331:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.838331:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.838333:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.838334:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.838335:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.838337:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.838337:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.838350:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.838352:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.838353:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.838354:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677446.838380:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.838381:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.838382:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.838383:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.838384:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.838385:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.838387:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.838387:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677446.838390:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677446.838391:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.838393:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.838394:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.838395:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:6.0:1619677446.838396:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.838397:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.838399:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:22.0:1619677446.838400:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.838402:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677446.838405:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.838406:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677446.838407:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677446.838409:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677446.838410:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.838410:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.838411:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677446.838490:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:6.0:1619677446.838492:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:24.0:1619677446.838493:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.838493:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:6.0:1619677446.838495:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.838496:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:8.0:1619677446.838496:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.838496:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.838499:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.838499:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:8.0:1619677446.838501:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677446.838502:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96736571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677446.838503:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:8.0:1619677446.838504:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.838505:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.838506:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677446.838512:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.838514:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677446.838518:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.838519:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677446.838519:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (23)++
00000800:00000200:8.0:1619677446.838521:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (24)--
00000800:00000200:22.0:1619677446.838522:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:8.0:1619677446.838522:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.838522:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.838523:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677446.838525:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:6.0:1619677446.838526:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (24)++
00000400:00000200:22.0:1619677446.838527:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.838527:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000400:00000200:6.0:1619677446.838528:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.838529:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677446.838530:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.838532:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.838532:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.838533:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677446.838538:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677446.838541:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677446.838542:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677446.838544:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.838547:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.838549:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677446.838550:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.838551:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.838552:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.838553:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677446.838557:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.838560:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677446.838563:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.838564:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:24.0:1619677446.838564:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:24.0:1619677446.838566:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:24.0:1619677446.838567:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677446.838567:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:6.0:1619677446.838567:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677446.838568:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:22.0:1619677446.838570:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.838570:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677446.838571:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.838572:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.838572:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677446.838616:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.838619:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.838620:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.838620:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.838623:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.838626:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677446.838771:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.838774:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.838775:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.838775:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.838777:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.838780:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.838789:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677446.838791:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677446.838792:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.838792:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.838793:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.838794:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.838816:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.838818:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.838819:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.838820:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.838822:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.838825:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.839048:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.839051:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:6.0:1619677446.839051:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:13.0:1619677446.839052:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.839054:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:13.0:1619677446.839054:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.839054:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:12.0:1619677446.839055:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.839057:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:6.0:1619677446.839057:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.839059:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.839061:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677446.839062:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.839064:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.839065:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.839066:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677446.839069:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.839071:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677446.839075:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.839076:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677446.839076:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677446.839078:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677446.839079:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.839080:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.839080:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677446.839130:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.839133:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.839134:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.839135:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.839138:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.839139:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677446.839141:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.839142:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.839143:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.839144:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677446.839147:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.839149:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677446.839152:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.839152:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677446.839153:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677446.839154:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.839155:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.839156:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.839156:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.839166:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.839169:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.839169:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.839171:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.839173:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.839173:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.839445:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.839448:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.839448:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.839450:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.839451:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.839454:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.839502:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.839504:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.839506:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.839507:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.839509:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.839510:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.839529:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.839531:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.839532:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.839532:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.839535:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.839537:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677446.839560:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.839563:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.839563:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677446.839565:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.839568:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.839570:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677446.839571:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.839572:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.839574:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.839575:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677446.839578:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.839580:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677446.839582:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.839583:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677446.839584:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677446.839585:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677446.839586:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.839588:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.839589:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.839589:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.839599:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.839601:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.839602:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.839603:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677446.839878:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.839881:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.839882:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.839883:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.839885:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.839886:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.839950:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.839952:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.839953:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.839954:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.839955:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.839958:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677446.840059:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.840061:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.840062:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.840062:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.840065:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.840067:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.840186:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.840189:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.840189:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.840191:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.840194:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.840195:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677446.840196:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96736546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.840198:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.840199:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.840200:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:13.0:1619677446.840202:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:22.0:1619677446.840203:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.840204:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677446.840205:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.840205:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.840208:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677446.840208:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.840209:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677446.840209:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:12.0:1619677446.840209:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677446.840211:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:13.0:1619677446.840211:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.840212:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.840212:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.840213:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677446.840222:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.840225:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.840226:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.840226:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.840227:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.840228:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.840651:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.840653:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.840654:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.840654:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.840656:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:6.0:1619677446.840659:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.840716:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.840718:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.840719:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.840721:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.840723:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.840724:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.840734:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.840737:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.840738:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.840739:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.840739:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.840740:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.840777:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.840779:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.840779:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.840780:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.840780:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.840781:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.840782:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:13.0:1619677446.840783:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.840785:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.840786:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.840787:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.840788:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.840814:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.840816:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.840819:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.840819:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677446.840820:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.840821:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677446.840822:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.840824:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.840824:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.840825:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.840827:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.840828:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.840829:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.840831:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.840833:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677446.840834:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.840835:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677446.840835:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.840837:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.840838:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.840839:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.840842:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.840843:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:24.0:1619677446.840844:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.840844:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.840846:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.840847:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.840848:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.840849:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.840849:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.840850:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677446.840851:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.840851:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.840851:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.840854:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.840854:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.840855:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.840855:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.840990:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.840993:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.840994:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.840994:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.840996:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.840999:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677446.841291:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.841300:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.841300:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.841301:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.841303:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.841306:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.841541:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.841544:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677446.841544:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.841546:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.841547:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677446.841548:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677446.841549:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.841549:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.841550:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.841550:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.841552:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.841563:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.841565:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.841567:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.841567:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.841568:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.841569:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.841649:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.841651:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.841652:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.841654:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.841655:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.841656:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.841996:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.841998:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.841999:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.841999:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.842001:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.842004:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677446.842148:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.842151:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.842152:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.842153:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.842156:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.842157:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.842159:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.842160:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.842162:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.842163:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.842166:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:24.0:1619677446.842168:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.842168:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.842170:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.842170:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677446.842173:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677446.842173:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.842174:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.842175:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:8.0:1619677446.842175:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.842176:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677446.842177:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:8.0:1619677446.842178:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.842179:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.842180:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.842181:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.842183:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.842183:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.842184:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.842191:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.842193:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.842196:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.842197:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.842197:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.842199:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.842199:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.842201:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.842202:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.842203:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.842217:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.842219:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.842220:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.842221:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.842564:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.842566:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.842567:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.842569:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.842571:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.842572:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.842642:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.842644:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.842645:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.842646:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.842648:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.842649:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.842803:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.842806:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.842807:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.842809:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.842811:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.842813:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677446.842814:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96736518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.842816:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.842817:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.842818:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677446.842819:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.842820:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.842821:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.842823:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677446.842824:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.842825:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677446.842827:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.842828:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.842828:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.842829:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677446.842829:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677446.842831:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677446.842832:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.842833:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.842833:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677446.842930:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.842933:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.842933:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.842936:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.842939:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.842940:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.842941:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.842942:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.842944:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.842945:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.842948:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.842949:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.842952:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.842953:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.842954:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.842956:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.842956:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.842957:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.842957:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677446.843043:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.843046:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.843047:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.843049:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.843052:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.843053:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.843054:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.843057:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.843058:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.843059:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.843064:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.843066:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.843069:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.843070:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.843071:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.843072:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.843073:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.843073:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.843074:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677446.843083:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.843085:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.843086:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.843086:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.843088:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.843090:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.843103:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.843107:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.843107:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677446.843108:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677446.843109:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.843110:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.843127:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.843129:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.843130:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.843133:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.843133:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.843136:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.843427:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.843430:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.843431:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.843431:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.843434:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.843437:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677446.843448:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.843451:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.843452:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.843454:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.843457:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.843458:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.843460:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.843461:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.843462:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.843463:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.843467:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.843468:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.843471:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.843472:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.843473:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.843475:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.843476:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.843476:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.843477:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.843621:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.843624:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.843625:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.843627:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.843628:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.843630:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.843638:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.843640:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.843641:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.843642:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.843642:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.843644:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.843728:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.843731:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.843731:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.843732:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.843734:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.843737:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.844017:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.844020:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.844021:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.844022:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.844023:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:22.0:1619677446.844024:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.844025:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.844027:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.844028:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.844028:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.844031:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.844033:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677446.844034:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96736501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.844036:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.844037:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.844038:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677446.844044:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.844046:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677446.844050:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.844051:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677446.844052:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677446.844054:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677446.844055:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.844056:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.844056:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677446.844071:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.844074:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.844075:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.844076:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.844078:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.844081:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.844090:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677446.844092:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.844093:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.844093:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.844094:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.844095:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.844167:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.844169:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.844171:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.844171:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.844173:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.844176:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.844196:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677446.844199:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677446.844200:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.844200:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.844201:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.844201:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.844333:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.844336:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.844337:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.844337:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.844339:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.844342:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677446.844514:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.844517:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.844518:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.844518:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677446.844518:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.844520:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.844520:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.844521:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.844523:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.844524:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677446.844525:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.844526:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.844536:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677446.844538:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677446.844539:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.844539:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.844541:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.844541:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.844933:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:6.0:1619677446.844935:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677446.844936:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.844937:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677446.844938:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.844938:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:24.0:1619677446.844941:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.844942:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.844943:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.844945:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:6.0:1619677446.844945:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.844946:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.844947:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:24.0:1619677446.844949:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.844949:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96736489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.844951:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.844951:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.844953:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.844954:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:6.0:1619677446.844954:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:24.0:1619677446.844960:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677446.844960:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.844963:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:6.0:1619677446.844963:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:24.0:1619677446.844966:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.844967:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:6.0:1619677446.844967:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.844968:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:6.0:1619677446.844968:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677446.844968:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:24.0:1619677446.844969:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677446.844970:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:6.0:1619677446.844970:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:24.0:1619677446.844971:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:6.0:1619677446.844971:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677446.844972:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677446.844972:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.844973:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.844974:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.844984:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.844986:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.844987:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.844988:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.845091:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.845094:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.845094:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.845096:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.845098:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.845099:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.845277:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.845280:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.845281:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677446.845282:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.845284:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.845286:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677446.845394:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.845397:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.845398:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.845400:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.845403:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.845404:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677446.845406:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.845407:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.845408:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.845409:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677446.845413:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.845414:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677446.845417:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.845418:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677446.845419:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677446.845420:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677446.845421:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.845421:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.845422:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677446.845595:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.845598:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.845599:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.845600:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.845602:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.845603:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.845611:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.845614:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.845615:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.845616:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.845616:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.845618:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.845671:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.845674:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.845674:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.845676:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.845677:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.845678:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.845697:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.845700:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.845701:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.845703:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.845705:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.845707:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.845708:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.845709:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.845710:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.845711:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.845715:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.845716:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.845719:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.845720:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.845721:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.845722:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.845723:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.845724:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.845725:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.845726:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.845739:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.845741:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.845741:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.845743:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.845807:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.845810:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.845810:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.845812:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.845813:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.845814:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.846219:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.846221:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.846222:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.846224:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.846227:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.846229:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677446.846230:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.846232:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.846232:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.846234:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677446.846237:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.846239:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677446.846241:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.846242:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677446.846243:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677446.846244:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677446.846245:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.846246:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.846246:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677446.846347:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.846350:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.846351:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.846352:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.846354:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.846355:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.846923:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.846926:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.846927:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.846929:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.846929:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.846932:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.846944:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.846947:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.846948:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.846949:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.846949:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.846950:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.847046:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.847048:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.847049:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.847051:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.847052:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.847053:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.847333:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.847336:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.847337:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.847337:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.847340:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.847343:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.847375:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.847378:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.847379:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.847379:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.847381:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.847384:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677446.847590:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.847593:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.847594:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.847594:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.847596:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677446.847600:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.847725:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.847727:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.847728:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.847729:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.847732:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.847734:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.847736:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.847737:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.847738:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.847739:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.847744:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.847747:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.847750:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.847750:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.847751:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.847753:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.847753:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.847754:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.847755:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677446.847772:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.847774:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.847775:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.847776:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.847777:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.847780:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.847878:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.847880:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.847882:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.847883:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.847886:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.847887:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.847972:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.847975:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.847976:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.847976:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.847978:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.847980:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:22.0:1619677446.847982:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.847983:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:22.0:1619677446.847984:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:13.0:1619677446.847984:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.847985:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.847986:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.847987:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.847987:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677446.847987:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:13.0:1619677446.847988:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:22.0:1619677446.847989:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.847993:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.847995:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.847997:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.847998:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.847999:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.847999:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.847999:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.848366:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.848368:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.848369:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.848370:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.848371:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.848373:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.848544:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.848546:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.848547:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.848549:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.848551:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.848552:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.848608:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.848611:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.848612:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.848612:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.848614:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.848617:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.848739:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.848741:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.848742:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.848745:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.848747:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.848750:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677446.848752:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.848754:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.848755:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.848757:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677446.848763:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.848766:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677446.848770:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.848771:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677446.848772:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677446.848774:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677446.848774:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.848775:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.848776:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677446.848786:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.848788:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.848790:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.848790:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.848792:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.848794:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.848910:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.848912:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.848913:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.848914:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.848916:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.848917:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.848933:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.848936:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.848937:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.848938:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.848939:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.848940:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.849110:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677446.849113:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677446.849114:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677446.849116:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.849119:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.849120:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677446.849122:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.849123:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.849124:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.849125:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677446.849128:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.849130:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677446.849133:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.849134:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677446.849134:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677446.849136:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677446.849137:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677446.849137:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677446.849138:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:6.0:1619677446.849150:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.849153:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.849154:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.849155:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.849157:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.849161:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.849168:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.849172:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677446.849172:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.849173:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.849174:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.849175:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.849191:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.849193:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.849194:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.849195:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.849197:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.849199:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.849287:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.849290:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.849291:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.849293:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.849294:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.849301:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.849501:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.849503:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.849504:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.849506:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.849509:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.849510:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.849511:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.849512:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.849513:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.849514:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.849517:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.849518:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677446.849521:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.0:1619677446.849522:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.849523:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.849523:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677446.849524:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677446.849524:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.849525:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.849525:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.849525:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.849526:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.849526:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677446.849527:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.849530:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.849572:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.849575:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.849576:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.849578:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.849580:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.849582:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677446.849583:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.849585:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.849586:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.849587:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677446.849591:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.849592:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677446.849595:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.849596:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677446.849597:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677446.849598:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677446.849599:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.849600:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.849601:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.849601:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.849612:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.849614:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.849615:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.849615:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677446.849663:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.849665:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.849666:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.849671:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.849674:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.849676:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677446.849677:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.849678:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.849679:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.849680:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677446.849684:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.849686:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677446.849688:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.849689:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677446.849690:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677446.849692:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677446.849692:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.849693:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.849695:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.849695:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:6.0:1619677446.849705:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677446.849708:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:6.0:1619677446.849708:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.849709:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.849709:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.849710:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.849711:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.849711:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:6.0:1619677446.849711:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.849714:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.849992:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677446.849995:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677446.849997:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677446.849998:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677446.850000:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.850002:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:12.0:1619677446.850003:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96736427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677446.850005:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677446.850006:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677446.850007:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:12.0:1619677446.850011:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677446.850013:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:12.0:1619677446.850016:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677446.850017:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:12.0:1619677446.850017:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:12.0:1619677446.850019:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:12.0:1619677446.850020:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677446.850021:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677446.850021:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:22.0:1619677446.850036:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.850038:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.850039:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.850040:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.850042:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.850045:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.850086:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.850089:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.850090:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.850090:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.850092:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.850095:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.850104:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.850106:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.850107:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.850107:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677446.850108:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677446.850109:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.850109:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.850110:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.850111:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.850111:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.850112:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.850112:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.850121:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677446.850124:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677446.850125:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.850125:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.850126:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.850126:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.850241:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.850244:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.850245:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.850246:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.850247:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.850250:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.850266:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.850268:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.850269:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.850270:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.850272:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.850273:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.850274:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.850274:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.850275:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.850275:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.850276:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.850362:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.850365:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677446.850365:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.850366:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.850368:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.850372:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.850374:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.850375:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.850376:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:22.0:1619677446.850379:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.850382:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.850383:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677446.850384:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96736418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.850386:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.850387:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.850388:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677446.850392:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.850393:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677446.850396:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.850397:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677446.850397:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677446.850399:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677446.850400:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.850401:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.850401:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677446.850403:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.850407:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677446.850413:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.850414:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.850415:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.850419:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677446.850654:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.850657:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.850657:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.850659:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.850661:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.850662:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.850761:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.850764:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.850765:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.850766:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.850769:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.850771:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677446.850772:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.850773:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.850774:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.850775:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677446.850778:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.850780:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677446.850782:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.850783:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677446.850783:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677446.850785:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.850786:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.850786:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.850787:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.850796:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.850799:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.850799:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.850801:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.850803:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.850804:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.850867:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.850870:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.850871:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.850871:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.850873:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.850876:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677446.851046:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.851049:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.851050:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.851051:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.851053:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.851055:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.851059:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.851062:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.851062:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.851064:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.851065:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.851066:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.851301:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.851303:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677446.851304:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.851305:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.851307:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.851310:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677446.851329:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.851332:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.851333:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.851335:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.851337:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.851338:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677446.851340:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736405] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.851341:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.851342:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.851343:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677446.851347:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.851348:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677446.851351:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.851352:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677446.851353:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677446.851354:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677446.851355:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.851355:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.851356:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677446.851427:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.851430:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.851431:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.851433:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.851435:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.851436:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677446.851438:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96736403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.851440:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.851441:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.851442:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677446.851446:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.851448:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677446.851451:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.851452:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677446.851452:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677446.851454:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677446.851455:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.851455:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.851456:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677446.851602:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.851605:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.851606:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.851606:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.851608:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.851610:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677446.851704:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.851707:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.851707:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677446.851710:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.851713:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.851715:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.851716:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.851718:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.851720:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.851721:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.851726:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.851728:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.851731:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.851732:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.851733:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.851734:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.851735:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.851736:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.851736:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.852041:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.852044:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.852044:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.852046:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.852048:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.852049:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.852341:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:12.0:1619677446.852344:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677446.852344:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.852345:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.852346:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.852347:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.852348:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.852348:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:12.0:1619677446.852350:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.852351:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677446.852352:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.852353:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.852418:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.852421:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.852422:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.852422:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.852424:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677446.852427:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677446.852432:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.852435:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.852436:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.852436:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.852438:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.852441:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.852451:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677446.852453:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677446.852454:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.852454:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.852455:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.852456:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.852633:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.852636:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.852636:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.852638:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.852640:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.852642:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677446.852643:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.852645:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.852645:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.852647:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677446.852650:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.852652:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677446.852654:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.852655:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677446.852656:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677446.852657:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677446.852658:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.852658:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.852659:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677446.852849:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (23)++
00000800:00000200:6.0:1619677446.852852:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000800:00000200:22.0:1619677446.852853:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (24)++
00000400:00000200:6.0:1619677446.852853:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.852854:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000400:00000200:22.0:1619677446.852854:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.852856:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677446.852856:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.852859:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.852860:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.852860:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:13.0:1619677446.852922:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.852925:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.852926:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.852928:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.852929:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.852930:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.852945:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.852948:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.852949:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.852950:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.852950:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.852952:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.853389:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.853392:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.853393:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.853394:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.853396:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.853397:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.853407:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.853410:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.853411:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.853411:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.853413:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.853416:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.853966:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.853969:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.853970:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.853972:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.853979:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.853980:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677446.853982:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96736371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.853983:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.853985:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.853986:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677446.853990:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.853990:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677446.853992:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.0:1619677446.853993:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.853994:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677446.853996:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.853997:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:8.0:1619677446.853997:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677446.853998:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:8.0:1619677446.853999:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.853999:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.854000:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677446.854000:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.854000:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:8.0:1619677446.854001:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677446.854001:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.854002:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:8.0:1619677446.854002:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96736370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677446.854003:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.854004:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.854004:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.854005:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.854006:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:8.0:1619677446.854009:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.854010:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.854011:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677446.854014:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:8.0:1619677446.854015:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.854017:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:8.0:1619677446.854017:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677446.854018:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.854020:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677446.854020:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.854020:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.854021:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677446.854021:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677446.854023:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.854023:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677446.854024:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.854024:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677446.854025:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677446.854025:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677446.854026:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:6.0:1619677446.854026:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.854027:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677446.854032:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.854041:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.854044:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.854044:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.854045:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.854179:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.854181:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.854182:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.854184:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.854187:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.854188:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.854190:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.854191:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.854192:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.854193:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.854196:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.854197:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.854200:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.854201:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.854201:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.854203:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.854204:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.854205:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.854206:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677446.854409:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.854412:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.854413:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.854414:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.854417:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.854419:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677446.854420:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.854421:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.854422:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.854423:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677446.854427:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.854429:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677446.854432:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.854433:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677446.854434:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677446.854436:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677446.854436:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.854437:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.854438:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677446.854597:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.854600:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.854601:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.854601:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.854603:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.854605:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.854618:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.854620:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.854622:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.854623:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.854625:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.854626:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.854633:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.854636:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.854637:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.854638:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.854639:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.854640:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.854649:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.854652:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.854653:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.854655:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.854658:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.854659:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.854661:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.854662:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.854663:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.854664:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.854667:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.854668:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.854671:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.854672:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.854673:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.854675:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.854675:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.854676:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.854678:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.854678:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.854691:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.854693:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.854694:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.854695:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677446.854714:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.854717:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.854718:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.854718:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.854720:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.854724:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.854889:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.854892:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.854893:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.854894:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.854895:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.854897:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.855099:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.855102:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.855103:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.855105:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.855107:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.855109:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.855110:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.855111:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.855112:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.855113:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.855117:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.855118:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.855121:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.855122:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.855123:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.855124:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.855125:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.855125:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.855126:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677446.855301:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.855304:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.855305:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.855309:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.855312:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.855314:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677446.855316:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96736352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.855317:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.855318:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.855319:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677446.855324:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.855326:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677446.855329:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.855330:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677446.855330:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677446.855333:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677446.855334:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.855335:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.855338:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677446.855340:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.855342:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.855343:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.855343:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.855345:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.855348:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677446.855477:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.855481:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.855484:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.855485:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000400:00000200:8.0:1619677446.855485:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677446.855487:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000400:00000200:8.0:1619677446.855489:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.855491:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677446.855491:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677446.855493:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.855494:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.855500:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.855502:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677446.855504:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.855506:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677446.855507:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96736348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.855509:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.855510:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.855510:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677446.855511:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677446.855513:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:24.0:1619677446.855515:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677446.855517:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677446.855522:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.855523:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677446.855523:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.855524:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677446.855524:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677446.855524:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (23)++
00000800:00000200:24.0:1619677446.855525:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677446.855526:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677446.855527:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677446.855527:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:22.0:1619677446.855527:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (24)++
00000800:00000200:8.0:1619677446.855528:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (25)--
00000800:00000200:24.0:1619677446.855529:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:8.0:1619677446.855529:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:22.0:1619677446.855530:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:8.0:1619677446.855530:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:22.0:1619677446.855531:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.855531:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.855534:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.855535:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.855536:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:22.0:1619677446.855538:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:8.0:1619677446.855538:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677446.855538:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.855539:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:22.0:1619677446.855540:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:8.0:1619677446.855541:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:22.0:1619677446.855542:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.855543:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.855543:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677446.855543:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:6.0:1619677446.855545:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677446.855545:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (23)++
00000400:00000200:22.0:1619677446.855546:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.855546:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (24)--
00000800:00000200:6.0:1619677446.855547:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.855548:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.855549:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677446.855554:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677446.855557:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677446.855558:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.855558:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677446.855559:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677446.855560:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677446.855571:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.855574:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.855575:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.855576:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.855578:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.855580:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.855772:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.855775:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.855776:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.855777:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.855779:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.855782:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677446.855864:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.855867:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.855868:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.855870:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.855873:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.855874:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.855875:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.855877:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.855879:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.855880:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.855886:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.855889:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.855892:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.855893:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.855894:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.855895:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.855896:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.855896:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.855897:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.855914:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.855917:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.855917:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.855919:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.855920:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.855921:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.855990:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.855992:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.855993:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.855995:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.855996:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.855998:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.856287:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.856289:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.856290:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.856292:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.856293:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.856300:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.856331:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.856333:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.856334:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.856334:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.856337:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.856339:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677446.856405:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.856407:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.856408:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.856410:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.856411:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.856412:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.856422:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.856424:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.856425:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.856426:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.856426:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.856427:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.856429:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.856431:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.856432:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.856434:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.856437:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.856439:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.856440:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.856441:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.856442:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.856443:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677446.856444:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:24.0:1619677446.856446:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677446.856446:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.856446:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.856448:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.856449:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677446.856450:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.856451:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.856452:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.856452:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.856453:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.856455:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.856455:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.856456:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.856457:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.856468:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.856471:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.856472:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.856473:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.856474:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.856475:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.856580:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.856583:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.856583:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677446.856585:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.856588:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.856589:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677446.856591:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.856592:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.856593:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.856594:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677446.856598:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.856600:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677446.856602:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.856603:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677446.856604:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677446.856606:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677446.856606:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.856607:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.856608:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677446.856634:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.856636:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.856637:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.856639:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.856641:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.856642:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.856783:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677446.856786:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677446.856787:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677446.856789:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.856791:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.856793:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677446.856794:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96736330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.856796:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.856797:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.856798:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677446.856802:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.856804:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677446.856806:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.856807:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677446.856808:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677446.856809:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677446.856810:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677446.856811:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677446.856812:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:13.0:1619677446.856831:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.856834:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.856835:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.856836:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.856838:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.856839:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.856908:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.856910:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.856911:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.856913:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.856914:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.856916:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.857029:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.857032:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.857033:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.857034:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.857035:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.857037:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.857199:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.857202:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.857203:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.857203:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.857205:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.857208:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677446.857212:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.857214:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.857215:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.857217:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.857218:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:13.0:1619677446.857218:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.857219:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.857220:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677446.857221:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.857221:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.857223:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.857223:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.857233:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.857236:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.857237:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.857237:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.857238:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.857239:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.857302:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.857305:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.857306:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.857307:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.857309:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.857310:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.857397:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.857400:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.857401:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.857401:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.857403:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.857407:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677446.857473:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.857475:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.857476:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.857478:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.857480:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.857481:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677446.857483:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.857484:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.857485:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.857486:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677446.857489:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.857491:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677446.857494:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.857494:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677446.857495:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677446.857496:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677446.857497:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.857498:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.857499:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.857500:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.857510:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.857512:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.857513:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.857514:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677446.857560:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.857562:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.857564:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.857565:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.857568:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.857569:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677446.857571:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.857572:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.857573:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.857574:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677446.857577:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.857578:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677446.857582:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.857583:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677446.857584:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677446.857585:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677446.857586:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.857587:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.857589:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.857589:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.857602:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.857604:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.857605:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.857607:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677446.857773:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.857776:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.857777:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.857777:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.857779:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.857782:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.858111:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.858114:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.858114:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.858115:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.858117:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.858119:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.858128:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677446.858130:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.858131:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.858131:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.858132:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.858133:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.858209:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.858212:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.858212:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.858213:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.858215:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.858218:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.858254:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.858257:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.858258:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.858259:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.858261:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.858262:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.858326:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.858329:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.858330:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.858332:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.858334:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.858335:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.858352:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.858354:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.858356:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.858356:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.858359:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.858362:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677446.858895:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.858898:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.858898:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677446.858901:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.858903:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.858904:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677446.858905:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.858907:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.858908:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.858909:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677446.858913:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.858916:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677446.858918:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.858919:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677446.858920:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677446.858922:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677446.858922:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.858923:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.858923:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.858948:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.858951:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.858952:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.858953:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.858954:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.858956:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.859282:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677446.859284:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677446.859285:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677446.859287:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677446.859290:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.859291:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677446.859293:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.859299:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.859300:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.859301:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677446.859304:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677446.859306:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677446.859307:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.859308:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677446.859308:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677446.859309:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.859310:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677446.859311:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677446.859311:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:24.0:1619677446.859323:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.859325:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.859326:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.859326:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.859327:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677446.859329:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677446.859763:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.859766:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.859767:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.859768:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.859769:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.859769:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.859770:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.859773:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:6.0:1619677446.859773:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677446.859774:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.859775:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.859777:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677446.859779:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96736292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.859780:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.859781:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.859782:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677446.859786:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.859788:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677446.859791:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.859792:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677446.859792:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677446.859794:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677446.859795:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.859795:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.859796:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677446.859976:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.859979:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.859980:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.859981:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.859983:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.859985:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.859997:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.860000:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.860001:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.860001:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.860002:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.860003:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.860046:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.860049:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.860050:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.860051:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.860053:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.860055:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.860056:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.860058:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.860059:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.860060:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.860063:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.860065:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.860068:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.860069:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.860069:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.860071:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.860071:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.860072:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.860073:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677446.860232:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.860235:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.860236:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.860236:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.860238:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.860240:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.860385:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.860387:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.860388:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.860390:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.860392:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.860394:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677446.860395:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.860396:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.860397:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.860398:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677446.860403:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.860406:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:24.0:1619677446.860407:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:6.0:1619677446.860409:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.860410:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:6.0:1619677446.860410:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:8.0:1619677446.860411:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:6.0:1619677446.860411:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677446.860413:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:24.0:1619677446.860414:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677446.860414:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.860414:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.860415:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:24.0:1619677446.860416:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.860418:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677446.860419:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96736285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.860421:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.860422:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.860423:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677446.860426:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.860428:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677446.860431:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.860432:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677446.860432:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677446.860434:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677446.860434:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.860435:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.860435:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677446.860447:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.860450:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.860451:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.860451:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.860453:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.860456:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.860469:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.860471:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.860472:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.860473:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.860474:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.860476:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.860498:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.860500:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.860501:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.860501:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.860502:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.860503:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.860713:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.860715:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.860716:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.860718:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.860720:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.860721:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.860915:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.860918:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.860918:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.860920:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.860923:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.860925:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677446.860926:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.860928:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.860929:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.860930:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677446.860933:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.860935:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677446.860937:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.860938:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677446.860938:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677446.860940:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.860941:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.860941:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.860941:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.860952:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.860954:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.860955:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.860956:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.860958:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.860959:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.861023:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.861025:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.861026:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.861027:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.861029:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.861032:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.861050:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.861053:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.861054:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.861054:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.861057:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.861063:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677446.861064:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.861067:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.861068:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.861070:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.861072:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.861074:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.861075:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.861076:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.861077:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.861078:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.861082:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.861082:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:24.0:1619677446.861084:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677446.861084:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.861085:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.861085:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.861087:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.861088:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.861088:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:6.0:1619677446.861088:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.861090:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677446.861090:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677446.861091:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.861091:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.861092:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.861153:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.861155:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.861156:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.861158:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.861160:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.861161:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.861185:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.861188:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.861188:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.861189:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.861190:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.861190:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.861194:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.861197:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677446.861197:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.861198:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.861200:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.861203:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.861212:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677446.861214:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677446.861215:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.861215:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.861217:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.861217:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677446.861272:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.861275:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.861276:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.861279:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.861281:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.861283:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.861285:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.861287:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.861288:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.861288:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.861292:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.861299:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.861302:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.861303:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.861304:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.861306:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.861306:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.861307:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.861307:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677446.861315:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.861318:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.861318:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.861319:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.861321:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.861324:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.861517:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.861519:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.861520:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.861522:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.861524:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.861525:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.861551:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.861553:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.861554:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.861557:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.861558:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.861560:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.861627:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.861630:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.861631:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.861633:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.861636:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.861637:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.861639:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.861640:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.861641:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.861642:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.861646:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.861647:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.861651:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.861651:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.861652:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.861654:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.861655:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.861656:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.861658:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.861658:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.861671:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.861673:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.861674:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.861675:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.861705:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.861707:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.861708:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.861709:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.861711:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.861712:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.861723:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.861725:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.861726:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.861727:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.861729:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.861732:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.861839:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.861842:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.861843:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.861844:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.861845:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.861847:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.861955:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.861958:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.861959:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.861961:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.861964:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.861966:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677446.861968:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96736262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.861969:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677446.861969:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:24.0:1619677446.861970:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.861971:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.861971:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.861972:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.861974:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677446.861974:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.861975:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.861976:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.861978:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.861979:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.861979:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677446.861980:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677446.861982:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.861982:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.861983:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.861983:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.861990:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.861993:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.861994:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677446.861995:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677446.861995:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.861997:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.861999:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.862000:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.862001:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.862003:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.862004:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.862154:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.862157:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.862157:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.862160:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.862162:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.862164:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677446.862165:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.862167:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.862168:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.862169:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677446.862172:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.862174:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677446.862176:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.862177:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677446.862178:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677446.862179:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677446.862180:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.862181:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.862182:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.862183:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.862193:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.862195:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.862196:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.862197:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677446.862260:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677446.862261:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:22.0:1619677446.862263:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.862263:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:8.0:1619677446.862264:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:6.0:1619677446.862264:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.862265:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:6.0:1619677446.862266:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:8.0:1619677446.862268:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677446.862270:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:8.0:1619677446.862271:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.862272:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677446.862274:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.862275:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.862276:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.862277:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:22.0:1619677446.862280:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:8.0:1619677446.862281:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.862283:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:22.0:1619677446.862284:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677446.862284:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.862285:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.862286:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.862286:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677446.862286:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.862287:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677446.862287:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677446.862289:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677446.862289:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.862290:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.862291:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.862304:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.862307:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.862308:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.862310:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.862312:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.862313:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.862318:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.862321:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.862322:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.862322:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.862324:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.862327:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677446.862436:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.862440:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.862441:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.862443:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.862445:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.862447:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.862449:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.862450:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.862451:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.862452:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.862455:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.862456:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.862459:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.862460:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.862460:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.862462:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.862463:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.862464:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.862465:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.862466:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677446.862466:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.862468:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.862469:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.862470:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.862472:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.862474:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.862478:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.862480:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.862481:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.862482:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.862665:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.862667:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.862668:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.862669:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.862671:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.862672:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.862689:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.862692:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.862693:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.862694:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.862695:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.862696:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.862897:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.862900:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.862901:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.862902:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.862904:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.862905:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.863029:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.863032:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.863033:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.863033:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.863036:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.863040:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677446.863050:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.863054:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.863054:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.863056:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677446.863058:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000400:00000200:22.0:1619677446.863059:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.863060:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677446.863061:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96736246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677446.863061:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677446.863062:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.863062:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000400:00000200:22.0:1619677446.863063:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677446.863063:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000400:00000200:22.0:1619677446.863064:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677446.863064:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:22.0:1619677446.863065:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:12.0:1619677446.863065:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.863067:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.863068:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.863069:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677446.863070:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.863071:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677446.863071:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677446.863073:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.863074:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677446.863075:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:12.0:1619677446.863076:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.863077:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677446.863077:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.863078:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.863078:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677446.863465:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677446.863468:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677446.863468:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000400:00000200:6.0:1619677446.863472:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.863475:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.863477:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:6.0:1619677446.863478:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.863480:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.863482:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.863483:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:6.0:1619677446.863488:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.863491:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:6.0:1619677446.863494:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.863495:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:6.0:1619677446.863495:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677446.863497:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677446.863498:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677446.863499:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:6.0:1619677446.863500:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:6.0:1619677446.863500:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677446.863511:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677446.863513:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:13.0:1619677446.863513:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:22.0:1619677446.863514:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677446.863515:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:13.0:1619677446.863516:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.863516:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.863518:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.863520:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.863521:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.863552:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.863554:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.863555:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.863557:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.863558:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.863560:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.863572:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.863574:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.863575:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.863576:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.863576:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.863578:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.864051:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.864054:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677446.864055:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.864056:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.864058:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.864061:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677446.864152:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.864155:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.864156:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.864158:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.864161:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.864163:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677446.864164:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.864165:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.864166:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.864167:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677446.864171:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.864173:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677446.864175:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.864176:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677446.864177:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677446.864178:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677446.864179:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.864179:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.864180:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677446.864473:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.864476:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.864477:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.864477:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.864480:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.864482:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677446.864558:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.864561:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.864562:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.864562:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.864564:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677446.864568:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.864582:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677446.864585:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677446.864586:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.864586:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677446.864587:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677446.864588:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677446.864698:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.864701:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.864702:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.864704:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.864706:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.864706:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.864887:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.864890:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.864891:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.864893:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.864895:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.864897:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677446.864898:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.864899:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.864900:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.864901:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677446.864905:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.864907:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677446.864910:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.864911:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677446.864912:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677446.864914:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677446.864915:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.864915:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.864916:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677446.865165:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.865168:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.865168:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677446.865170:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.865172:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.865174:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.865175:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.865177:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.865178:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.865179:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.865182:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.865184:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.865186:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.865187:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.865188:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.865190:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.865190:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.865191:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.865191:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.865245:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.865248:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.865249:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.865250:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.865252:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.865253:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.865266:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.865269:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.865270:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.865270:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.865271:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.865272:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.865479:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.865482:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.865483:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.865484:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.865486:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.865489:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.865540:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.865543:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.865544:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.865544:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.865546:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.865548:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677446.865576:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.865578:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.865580:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.865582:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.865584:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.865586:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677446.865587:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.865589:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.865590:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.865591:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677446.865594:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.865595:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677446.865599:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.865599:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:22.0:1619677446.865599:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677446.865600:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677446.865602:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:22.0:1619677446.865602:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.865602:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677446.865603:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.865603:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.865604:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:22.0:1619677446.865604:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.865607:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.865609:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677446.865610:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96736215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.865611:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.865613:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.865614:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677446.865617:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.865619:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677446.865623:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.865623:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677446.865624:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677446.865626:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677446.865627:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.865628:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.865629:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.865630:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.865640:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.865642:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.865643:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.865644:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677446.865887:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.865890:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.865891:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.865891:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.865893:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.865896:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.866119:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.866121:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.866122:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.866122:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:13.0:1619677446.866124:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.866126:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:13.0:1619677446.866126:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.866126:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677446.866126:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.866127:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.866129:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.866132:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.866141:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.866144:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.866145:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.866145:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.866146:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.866147:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.866299:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.866302:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.866303:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.866303:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.866305:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.866308:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.866570:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.866573:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.866573:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.866574:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.866576:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.866579:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677446.866633:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.866636:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.866637:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.866639:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.866642:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.866643:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677446.866645:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.866646:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677446.866646:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:8.0:1619677446.866647:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.866648:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677446.866648:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.866649:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.866649:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.866651:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677446.866652:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.866654:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677446.866654:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677446.866657:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.866658:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677446.866659:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677446.866660:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677446.866661:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:22.0:1619677446.866662:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:8.0:1619677446.866663:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.866664:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:22.0:1619677446.866665:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:8.0:1619677446.866665:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677446.866666:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.866666:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.866667:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.866670:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.866672:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.866674:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.866675:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.866675:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.866677:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.866678:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.866680:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:22.0:1619677446.866680:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677446.866681:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.866682:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.866863:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.866867:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.866868:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.866870:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.866872:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.866874:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.866875:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.866876:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.866877:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.866878:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.866881:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.866883:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.866886:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.866886:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.866887:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.866889:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.866889:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.866890:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.866890:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677446.867071:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.867074:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.867075:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.867075:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.867078:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.867081:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.867290:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.867293:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.867298:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.867299:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.867299:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.867301:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.867417:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.867419:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.867421:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.867422:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.867423:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.867424:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.867428:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.867431:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.867432:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.867434:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.867437:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.867439:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677446.867440:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.867442:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.867443:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.867444:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677446.867447:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.867449:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677446.867451:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.867452:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677446.867453:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677446.867454:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677446.867455:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.867456:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.867456:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677446.867493:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.867496:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.867497:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.867497:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.867500:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.867502:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.867517:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.867520:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677446.867520:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.867521:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.867522:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.867522:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.867552:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.867554:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.867555:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.867557:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.867558:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.867560:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.867906:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.867909:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.867910:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.867910:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.867912:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.867915:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677446.867958:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.867960:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.867961:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.867964:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.867966:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.867968:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.867969:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.867971:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.867972:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.867972:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.867975:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.867977:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.867980:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.867981:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.867982:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.867983:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.867984:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.867985:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.867985:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.867988:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.867990:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.867991:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.867993:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.867994:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.867995:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.868010:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.868013:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.868013:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.868015:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.868016:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.868017:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.868096:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.868099:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.868100:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.868102:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.868105:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.868106:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.868108:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.868109:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.868110:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.868111:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.868114:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.868116:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.868119:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.868120:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.868122:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.868123:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.868124:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.868125:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.868125:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.868166:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.868168:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.868170:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.868170:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.868171:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.868174:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.868224:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.868226:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.868227:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.868228:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.868229:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677446.868230:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.868232:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677446.868232:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:12.0:1619677446.868233:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.868235:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.868236:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.868238:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.868383:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.868386:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.868387:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.868387:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.868389:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.868391:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677446.868426:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.868429:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.868430:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.868430:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.868432:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.868435:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.868529:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.868532:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.868532:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.868534:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.868536:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.868537:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.868686:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.868688:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.868689:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.868691:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.868692:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.868694:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.868786:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.868789:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.868790:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.868791:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.868793:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.868793:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.868827:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.868830:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.868831:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.868833:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.868835:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.868837:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677446.868838:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.868839:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.868840:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.868841:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677446.868844:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.868846:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677446.868848:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.868849:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677446.868850:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677446.868852:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677446.868853:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.868853:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.868854:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677446.868916:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.868919:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.868920:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.868924:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.868927:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.868929:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.868931:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.868932:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.868933:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.868934:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.868939:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.868942:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.868945:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.868946:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.868946:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.868948:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.868949:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.868950:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.868951:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:13.0:1619677446.868952:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.0:1619677446.868952:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677446.868955:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.868956:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.868958:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.868959:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.868960:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.868966:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.868968:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.868969:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.868969:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:12.0:1619677446.868970:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677446.868970:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:24.0:1619677446.868971:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.868971:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.868972:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:12.0:1619677446.868973:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:8.0:1619677446.868973:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.868974:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.868974:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:24.0:1619677446.868975:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677446.868975:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.868976:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.869224:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.869226:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.869227:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.869227:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.869229:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.869231:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.869247:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677446.869250:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677446.869251:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.869251:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677446.869252:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.869252:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677446.869354:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.869357:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.869358:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.869360:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.869362:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.869364:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677446.869365:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96736163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.869367:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.869368:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677446.869369:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:22.0:1619677446.869370:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:13.0:1619677446.869371:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.869372:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.869373:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677446.869374:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.869375:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:12.0:1619677446.869376:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.869377:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.869378:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.869379:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677446.869380:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677446.869382:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677446.869383:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.869383:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.869384:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677446.869614:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.869617:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.869618:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.869620:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.869622:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.869624:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677446.869625:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.869626:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.869627:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.869628:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677446.869631:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.869633:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677446.869636:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.869637:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677446.869637:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677446.869639:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.869639:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.869640:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.869640:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.869650:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.869653:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.869654:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.869655:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.869657:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.869658:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.869670:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.869672:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677446.869673:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.869674:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.869676:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677446.869678:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677446.870040:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.870042:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.870043:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.870043:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.870045:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.870047:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.870165:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.870168:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.870168:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.870170:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.870172:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.870174:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677446.870175:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.870177:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.870178:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.870179:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677446.870182:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.870184:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677446.870187:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.870188:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677446.870189:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677446.870190:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677446.870191:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.870192:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.870193:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.870194:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.870204:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.870206:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.870207:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.870208:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.870586:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.870589:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.870590:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.870592:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.870593:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.870594:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.870628:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.870630:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.870631:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.870632:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.870634:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.870635:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.870648:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.870651:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.870652:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.870652:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.870653:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.870654:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.870945:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.870948:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.870949:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.870949:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.870951:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.870953:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.870963:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677446.870966:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.870967:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.870967:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.870968:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.870968:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677446.871092:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:22.0:1619677446.871095:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.0:1619677446.871095:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.871096:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.871097:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677446.871098:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.871099:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.871099:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:8.0:1619677446.871100:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.871101:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:22.0:1619677446.871102:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:8.0:1619677446.871104:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677446.871105:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:8.0:1619677446.871105:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.871106:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.871107:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677446.871110:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.871112:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677446.871115:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.871115:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677446.871116:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677446.871118:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677446.871118:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.871119:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.871119:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.871130:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.871132:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.871134:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.871135:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.871136:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.871137:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.871201:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.871204:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.871205:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.871207:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.871210:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.871211:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677446.871213:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.871214:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.871216:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.871217:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677446.871221:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.871224:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677446.871228:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.871229:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677446.871230:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677446.871232:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677446.871233:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.871233:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.871234:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677446.871266:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.871268:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.871270:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.871271:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.871274:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.871276:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.871277:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.871279:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.871280:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.871281:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.871284:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.871286:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.871289:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.871289:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.871290:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.871291:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.871292:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.871293:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.871303:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.871396:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.871399:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.871400:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.871401:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.871403:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.871404:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.871413:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.871416:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.871416:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.871417:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.871418:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.871418:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.871579:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.871581:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.871583:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.871584:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.871585:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.871587:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.871660:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.871664:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.871664:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.871665:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.871667:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.871670:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.871710:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.871713:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.871714:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.871716:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.871717:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.871719:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.872175:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.872178:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.872179:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.872181:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.872183:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.872185:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677446.872186:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.872187:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.872188:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.872189:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677446.872192:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.872193:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677446.872197:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.872197:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677446.872198:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677446.872200:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677446.872200:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.872201:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.872203:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.872203:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:6.0:1619677446.872213:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:24.0:1619677446.872216:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:6.0:1619677446.872216:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.872217:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.872217:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.872218:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.872219:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:6.0:1619677446.872219:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.872220:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677446.872222:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.872319:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.872321:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.872322:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.872323:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:13.0:1619677446.872324:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.872326:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:13.0:1619677446.872326:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:22.0:1619677446.872327:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.872327:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.872327:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.872329:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.872333:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.872795:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.872798:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.872799:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.872799:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.872800:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677446.872801:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.872803:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.872803:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.872804:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:12.0:1619677446.872805:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.872807:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.872808:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.872809:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.872811:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.872813:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.872814:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.872815:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.872816:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.872930:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.872933:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.872935:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.872937:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.872940:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.872941:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677446.872942:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.872944:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.872945:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.872946:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677446.872949:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.872951:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677446.872954:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.872955:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677446.872955:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677446.872957:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677446.872957:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.872958:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.872959:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677446.872970:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.872973:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.872974:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.872974:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.872977:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.872979:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.872987:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677446.872990:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.872991:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.872992:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.872993:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.872993:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.873017:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.873020:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.873021:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.873021:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.873023:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.873025:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.873159:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.873161:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.873162:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.873164:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.873167:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.873168:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677446.873169:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.873171:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.873172:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.873173:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677446.873177:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.873179:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677446.873181:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.873182:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677446.873182:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677446.873184:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677446.873185:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.873185:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.873186:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677446.873263:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.873265:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.873266:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.873267:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.873269:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.873272:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.873322:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.873325:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.873325:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.873327:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.873329:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.873331:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.873344:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.873347:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.873348:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.873349:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.873349:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.873350:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.873371:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.873373:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.873374:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.873376:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.873377:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.873379:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.873519:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.873521:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.873522:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.873524:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.873525:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.873526:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.873586:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.873590:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.873590:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.873592:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.873595:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.873597:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.873598:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.873599:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.873600:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.873602:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.873605:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.873606:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.873609:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.873610:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.873611:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.873612:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.873613:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.873613:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.873614:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677446.873694:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.873696:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.873697:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.873697:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.873700:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.873703:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677446.873916:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:22.0:1619677446.873917:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.873919:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:8.0:1619677446.873919:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.873920:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:6.0:1619677446.873920:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.873921:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.873921:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:22.0:1619677446.873923:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:12.0:1619677446.873924:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:8.0:1619677446.873924:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677446.873925:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.873927:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.873928:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:8.0:1619677446.873928:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.873930:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677446.873931:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677446.873932:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.873933:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677446.873933:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.873934:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.873936:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677446.873939:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.873940:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677446.873944:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.873945:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677446.873946:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677446.873947:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677446.873948:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.873948:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.873949:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677446.873961:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.873964:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.873965:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.873968:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.873970:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.873972:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677446.873973:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.873975:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.873976:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.873977:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677446.873980:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.873982:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677446.873985:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.873986:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677446.873987:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677446.873989:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677446.873989:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.873990:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.873991:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.874352:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.874354:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.874355:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.874357:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.874359:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.874360:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.874369:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.874372:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.874373:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.874373:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.874374:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.874375:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.874647:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.874649:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.874651:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.874652:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.874655:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.874657:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.874658:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.874660:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.874661:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.874662:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.874665:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.874667:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.874670:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.874671:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.874671:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.874673:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.874673:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.874674:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.874675:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677446.874698:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.874701:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.874701:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.874703:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.874706:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.874708:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677446.874709:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.874710:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.874711:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.874712:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677446.874716:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.874718:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677446.874720:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.874721:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677446.874721:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677446.874723:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677446.874723:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.874725:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.874726:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.874726:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.874737:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.874739:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.874740:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.874741:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677446.875218:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.875221:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.875222:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.875222:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.875224:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.875227:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677446.875378:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.875381:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.875382:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.875382:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.875384:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.875386:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.875397:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.875400:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.875400:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677446.875400:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.875401:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.875402:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.875480:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.875483:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.875483:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.875485:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.875488:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.875489:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677446.875491:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96736081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.875492:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.875494:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.875495:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:12.0:1619677446.875498:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:22.0:1619677446.875499:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.875501:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:12.0:1619677446.875501:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.875502:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.875503:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.875504:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.875505:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677446.875506:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:13.0:1619677446.875506:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.875506:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677446.875507:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677446.875508:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.875509:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.875509:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677446.875967:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677446.875970:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677446.875970:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000400:00000200:6.0:1619677446.875972:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.875975:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.875976:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677446.875978:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.875979:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.875980:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.875981:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677446.875985:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.875986:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677446.875989:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.875990:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:6.0:1619677446.875990:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677446.875992:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:6.0:1619677446.875992:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677446.875993:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:6.0:1619677446.875994:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:6.0:1619677446.875995:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677446.876006:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000800:00000200:22.0:1619677446.876008:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:22.0:1619677446.876009:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677446.876010:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:24.0:1619677446.876065:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.876067:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.876069:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.876070:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.876073:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.876074:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677446.876076:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.876077:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.876078:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.876079:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677446.876082:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.876083:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677446.876086:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.876086:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677446.876087:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677446.876088:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677446.876089:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.876089:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.876090:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.876101:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.876103:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.876104:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.876105:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.876107:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.876108:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.876145:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.876148:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.876149:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.876149:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.876151:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.876154:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.876237:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.876240:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.876241:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.876242:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.876243:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.876245:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.876472:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.876475:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.876476:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.876478:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.876481:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.876483:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.876484:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.876485:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.876486:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.876487:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.876490:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.876492:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.876495:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.876496:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.876496:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.876498:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.876498:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.876500:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.876501:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.876502:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.876516:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.876518:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.876518:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.876519:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677446.876622:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.876625:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.876626:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.876626:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.876628:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.876632:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.876768:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.876770:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.876771:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.876774:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.876776:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.876778:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677446.876779:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96736065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.876781:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.876782:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.876783:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677446.876787:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.876789:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677446.876792:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.876792:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:6.0:1619677446.876793:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:6.0:1619677446.876796:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:6.0:1619677446.876796:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.876797:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.876797:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.876867:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.876870:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.876871:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.876872:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.876874:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.876877:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.876885:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.876888:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.876889:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.876890:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.876890:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.876892:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.877061:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.877063:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.877064:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.877066:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.877068:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.877069:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.877340:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.877343:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.877344:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.877346:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.877349:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.877351:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.877352:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.877353:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.877354:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.877355:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.877358:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.877359:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.877362:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.877363:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.877363:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.877365:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.877366:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.877367:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.877368:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.877369:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.877381:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.877384:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.877385:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.877386:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677446.877465:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.877468:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.877469:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.877469:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.877471:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.877474:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.877829:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.877831:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.877832:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.877834:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.877836:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.877837:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.877838:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677446.877841:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677446.877842:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677446.877844:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.877847:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.877849:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677446.877850:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.877851:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.877852:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.877854:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677446.877857:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.877859:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677446.877862:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.877862:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677446.877863:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677446.877865:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677446.877865:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677446.877866:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677446.877866:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:13.0:1619677446.878030:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.878032:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.878033:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.878035:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.878036:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.878037:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.878042:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.878044:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.878045:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.878047:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677446.878047:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677446.878050:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.878050:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677446.878051:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:12.0:1619677446.878051:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.878051:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:24.0:1619677446.878052:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677446.878052:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677446.878054:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.878054:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677446.878054:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677446.878055:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.878058:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.878060:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.878063:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.878064:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.878065:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.878066:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.878067:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.878068:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.878068:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.878289:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.878292:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.878292:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.878299:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:13.0:1619677446.878299:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.878299:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.878301:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.878302:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.878303:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.878305:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.878308:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.878309:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677446.878311:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.878312:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.878313:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.878314:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677446.878317:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.878319:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677446.878322:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.878322:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677446.878323:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677446.878325:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677446.878325:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.878326:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.878326:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677446.878337:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.878340:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.878341:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.878344:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.878347:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.878349:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.878350:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96736044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.878352:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.878353:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.878354:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.878357:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.878364:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.878367:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.878368:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)++
00000800:00000200:8.0:1619677446.878369:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (24)++
00000800:00000200:8.0:1619677446.878372:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (25)--
00000800:00000200:8.0:1619677446.878373:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.878374:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.878374:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677446.878925:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:12.0:1619677446.878927:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677446.878927:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.878928:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.878929:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.878930:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.878931:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.878932:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:12.0:1619677446.878933:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.878934:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.878936:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.878939:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.878947:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.878950:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677446.878950:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (23)++
00000800:00000200:22.0:1619677446.878952:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (24)++
00000800:00000200:6.0:1619677446.878952:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000800:00000200:22.0:1619677446.878953:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (24)--
00000800:00000200:22.0:1619677446.878953:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677446.878953:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.878955:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.878955:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.878958:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.879111:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.879114:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.879115:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.879115:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.879118:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.879122:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.879130:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677446.879132:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.879133:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.879134:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677446.879135:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.879136:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.879150:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.879153:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.879154:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.879154:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.879156:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.879159:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.879177:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.879179:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.879180:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.879182:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.879184:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.879185:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.879389:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.879392:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.879392:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.879394:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.879396:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.879397:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.879452:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.879455:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.879456:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.879457:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.879459:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.879462:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.879505:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.879508:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.879509:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.879511:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677446.879512:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:22.0:1619677446.879513:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.879514:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:22.0:1619677446.879515:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677446.879515:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.879517:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96736028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677446.879517:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:22.0:1619677446.879518:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677446.879518:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:22.0:1619677446.879520:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677446.879520:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:22.0:1619677446.879521:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677446.879525:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.879527:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677446.879530:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677446.879530:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677446.879531:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677446.879532:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677446.879533:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:13.0:1619677446.879533:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677446.879534:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677446.879534:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.879534:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:22.0:1619677446.879535:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:13.0:1619677446.879535:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677446.879536:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:13.0:1619677446.879536:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.879537:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.879593:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.879595:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.879596:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677446.879596:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677446.879781:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.879784:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.879785:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.879785:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.879787:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.879790:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.879808:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677446.879810:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677446.879811:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.879811:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.879812:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.879813:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.879904:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.879907:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.879908:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.879908:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.879910:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.879912:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677446.880285:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.880286:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.880287:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.880288:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677446.880288:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.880290:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677446.880291:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.880300:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.880301:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.880302:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:22.0:1619677446.880303:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:6.0:1619677446.880303:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.880304:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.880306:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.880307:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.880308:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677446.880312:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.880314:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677446.880316:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.880317:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677446.880318:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677446.880320:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677446.880320:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.880321:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.880321:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677446.880364:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.880367:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.880369:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.880370:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.880373:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.880375:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.880376:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.880377:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.880378:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.880379:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.880383:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.880385:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.880388:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.880389:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.880389:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.880391:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.880391:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.880392:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.880393:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677446.880423:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.880426:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.880427:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.880427:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.880429:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.880432:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677446.880821:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.880823:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.880825:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.880826:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.880830:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.880832:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677446.880833:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736014] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.880834:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.880835:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.880836:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677446.880840:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.880841:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677446.880844:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.880845:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677446.880846:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677446.880847:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677446.880848:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.880849:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.880849:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677446.880861:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.880864:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.880865:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.880865:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.880867:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.880870:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.880954:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.880957:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.880957:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677446.880959:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.880962:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.880964:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677446.880965:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96736013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.880966:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.880967:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.880968:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677446.880972:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.880973:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677446.880976:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.880977:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677446.880977:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677446.880979:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677446.880980:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.880980:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.880981:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677446.881040:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.881043:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.881044:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.881044:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.881046:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.881049:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677446.881340:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.881343:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.881344:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.881345:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.881347:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.881349:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.881360:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677446.881362:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.881363:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.881363:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.881364:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.881365:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677446.881632:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.881635:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.881635:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.881637:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.881640:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.881641:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677446.881643:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.881644:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.881645:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.881646:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677446.881649:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.881651:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677446.881654:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.881655:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677446.881656:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677446.881658:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677446.881658:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.881659:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.881659:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677446.881671:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.881674:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.881674:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.881675:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.881677:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.881680:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.881777:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.881779:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.881780:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.881782:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.881784:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.881786:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677446.881787:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96736003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.881788:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.881789:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.881790:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677446.881794:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.881797:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677446.881800:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.881801:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677446.881802:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677446.881804:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677446.881804:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.881804:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.881805:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677446.881815:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.881818:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.881819:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.881822:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.881825:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.881827:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.881829:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96736000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.881831:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.881832:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.881834:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.881838:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.881840:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.881843:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.881843:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.881844:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.881846:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.881846:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.881847:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.881848:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677446.881859:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.881861:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.881862:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.881863:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.881866:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.881874:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.882155:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.882157:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.882158:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.882158:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.882160:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.882163:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.882185:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.882188:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.882189:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.882191:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.882192:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.882193:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.882521:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.882524:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.882525:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.882525:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.882528:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.882530:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.882539:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677446.882542:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.882543:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.882543:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.882544:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.882544:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.882867:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.882869:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.882870:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.882873:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.882874:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.882881:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.882890:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.882892:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.882893:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.882894:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.882896:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:13.0:1619677446.882896:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.882898:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.882899:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.882900:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.882900:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.882902:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.882906:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.882908:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.882910:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.882911:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.882912:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.882913:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.882914:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.882923:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677446.882926:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677446.882927:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.882927:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.882928:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.882928:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.882962:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.882965:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.882966:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.882966:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.882968:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.882970:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.882999:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.883002:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.883003:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.883004:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.883005:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.883007:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.883031:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.883034:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.883035:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.883037:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.883039:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.883041:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.883042:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.883044:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.883045:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.883046:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.883049:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.883051:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.883054:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.883055:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.883056:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.883057:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.883058:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.883059:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.883060:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.883060:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.883073:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.883075:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.883076:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.883077:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677446.883270:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.883273:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.883274:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.883274:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.883277:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.883280:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.883292:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.883299:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.883300:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.883301:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677446.883302:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.883304:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.883483:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.883486:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.883487:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.883489:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.883491:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.883493:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677446.883494:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.883496:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.883497:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.883498:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677446.883502:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.883504:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677446.883506:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.883507:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677446.883507:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677446.883509:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677446.883510:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.883512:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.883513:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.883513:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.883523:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.883525:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.883525:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.883526:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677446.883576:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.883579:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.883580:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.883582:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.883585:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.883586:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677446.883587:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.883589:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.883590:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.883591:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677446.883595:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.883597:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677446.883599:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.883600:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677446.883600:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677446.883602:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677446.883602:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.883603:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.883603:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.883614:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.883617:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.883618:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.883619:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.883621:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.883622:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.883708:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.883711:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.883711:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.883713:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.883714:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.883715:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.883759:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.883761:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.883762:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.883762:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.883765:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.883768:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.883783:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.883786:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.883786:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.883789:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.883789:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.883792:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.884049:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.884052:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.884053:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.884054:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.884057:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.884059:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.884061:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.884062:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.884063:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.884064:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.884067:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.884068:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.884071:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.884072:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.884072:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.884074:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.884074:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.884075:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.884076:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.884077:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.884089:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.884091:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.884092:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.884093:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.884368:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.884371:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.884372:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.884374:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.884375:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.884376:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.884377:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.884378:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.884379:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.884379:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677446.884379:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.884380:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.884381:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.884462:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.884464:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.884465:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.884467:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.884468:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.884470:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.884478:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.884481:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.884482:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.884483:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.884484:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.884485:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.884521:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.884524:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.884525:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000400:00000200:6.0:1619677446.884525:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.884526:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.884527:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (23)++
00000400:00000200:6.0:1619677446.884527:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.884528:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (24)--
00000800:00000200:22.0:1619677446.884529:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.884530:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.884531:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.884687:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.884690:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.884691:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.884691:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.884694:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.884696:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.884717:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.884719:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.884720:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.884722:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.884723:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.884725:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.884862:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.884864:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.884865:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.884867:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.884868:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.884869:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.884888:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.884891:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.884891:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677446.884893:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.884895:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.884897:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677446.884898:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.884900:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.884901:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.884902:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677446.884905:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.884906:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677446.884909:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.884910:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677446.884911:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677446.884912:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677446.884913:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.884913:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.884914:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.884975:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.884977:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.884978:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.884979:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.884981:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.884981:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677446.885029:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677446.885031:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677446.885032:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677446.885034:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.885037:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.885038:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:22.0:1619677446.885040:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.885041:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.885042:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.885043:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677446.885046:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.885048:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677446.885051:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.885052:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:22.0:1619677446.885053:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677446.885055:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677446.885056:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677446.885056:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677446.885057:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:12.0:1619677446.885067:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.885070:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.885071:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.885073:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.885074:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.885076:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.885219:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.885221:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.885222:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.885222:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.885224:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.885227:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.885346:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.885349:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.885350:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.885352:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.885353:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.885354:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.885363:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.885365:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.885367:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.885367:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.885368:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.885369:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.885523:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.885526:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.885527:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.885529:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.885532:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.885533:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677446.885535:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.885536:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.885537:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.885538:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677446.885541:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.885543:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677446.885545:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.885546:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677446.885547:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677446.885549:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677446.885549:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.885550:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.885550:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677446.885869:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.885872:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.885873:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.885873:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.885875:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.885877:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.885879:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.885881:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.885882:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.885884:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.885885:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.885886:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.885956:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.885958:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.885959:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.885960:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.885962:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.885965:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.886134:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.886136:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.886137:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.886139:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.886140:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.886142:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.886236:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.886239:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.886240:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.886242:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.886244:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.886245:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677446.886246:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96735947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.886248:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.886249:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.886250:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677446.886253:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.886256:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677446.886259:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.886260:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677446.886261:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677446.886263:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677446.886263:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.886264:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.886265:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677446.886288:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.886291:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.886292:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.886292:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.886299:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.886302:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.886395:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.886399:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.886399:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.886400:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.886402:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.886404:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.886757:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.886759:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.886760:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.886762:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.886764:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.886765:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.886802:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.886805:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.886807:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.886807:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.886809:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.886812:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677446.886883:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.886886:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.886887:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.886888:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.886891:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.886892:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677446.886893:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.886895:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.886896:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.886897:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677446.886901:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.886903:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677446.886905:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.886906:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677446.886907:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677446.886909:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677446.886909:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.886910:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.886911:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.886912:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.886922:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.886924:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.886925:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.886926:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677446.886928:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.886931:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.886932:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.886934:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.886937:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.886938:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677446.886939:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:24.0:1619677446.886940:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96735936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.886941:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677446.886941:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:24.0:1619677446.886942:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677446.886942:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:24.0:1619677446.886943:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677446.886943:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.886945:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:24.0:1619677446.886948:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.886950:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.886952:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677446.886953:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.886954:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677446.886955:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:13.0:1619677446.886956:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677446.886957:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677446.886958:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.886958:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.886959:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.886959:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.886959:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.886960:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:13.0:1619677446.886961:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.886962:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677446.886963:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677446.886964:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:13.0:1619677446.886964:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.886964:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.886965:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.886969:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.886970:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.886973:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.886974:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.886974:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.886976:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.886978:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.886990:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.886992:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.886993:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.886994:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.886994:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.886995:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.887228:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.887231:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.887232:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:8.0:1619677446.887232:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:6.0:1619677446.887232:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.887234:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:6.0:1619677446.887234:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.887235:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.887236:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.887237:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:8.0:1619677446.887238:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.887241:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677446.887734:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.887736:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.887737:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.887737:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.887739:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.887742:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.887833:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677446.887836:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677446.887836:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.887838:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:12.0:1619677446.887838:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.887840:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:24.0:1619677446.887841:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:12.0:1619677446.887841:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677446.887842:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:8.0:1619677446.887842:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.887843:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677446.887843:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.887843:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677446.887844:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:24.0:1619677446.887846:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.887848:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.887849:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677446.887849:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.887850:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677446.887851:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.887852:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677446.887852:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677446.887853:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677446.887853:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.887853:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.887853:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.887854:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677446.887856:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.887857:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.887860:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.887861:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.887862:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.887863:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.887864:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.887865:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.887865:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.887916:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:6.0:1619677446.887918:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:24.0:1619677446.887919:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.887919:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:6.0:1619677446.887920:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.887921:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:8.0:1619677446.887922:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.887922:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.887923:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677446.887924:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.887924:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.887926:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:8.0:1619677446.887926:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:6.0:1619677446.887926:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:12.0:1619677446.887927:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.887928:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96735921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677446.887929:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.887930:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:8.0:1619677446.887930:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.887931:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677446.887932:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677446.887932:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.887935:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.887937:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.887941:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.887941:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.887942:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.887944:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.887945:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.887946:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.887946:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677446.888000:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.888002:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677446.888003:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.888004:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.888006:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.888009:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.888085:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.888088:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.888089:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.888089:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.888091:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.888094:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.888107:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677446.888109:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677446.888110:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.888111:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.888112:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.888112:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.888194:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.888196:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.888197:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.888198:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.888200:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.888201:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.888215:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.888217:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.888218:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.888219:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.888219:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.888221:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.888523:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.888525:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.888527:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.888528:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.888531:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.888532:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:6.0:1619677446.888532:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:24.0:1619677446.888533:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96735914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677446.888534:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:24.0:1619677446.888535:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677446.888535:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.888535:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.888536:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.888537:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:6.0:1619677446.888538:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:24.0:1619677446.888541:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.888541:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677446.888544:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.888547:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.888547:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.888548:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.888550:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.888551:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.888552:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.888553:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.888554:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.888566:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.888568:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.888569:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.888571:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677446.888790:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.888793:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.888794:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.888794:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.888797:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.888800:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.888834:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.888837:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.888837:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677446.888839:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.888842:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.888844:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677446.888845:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96735910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.888847:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.888847:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.888849:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677446.888852:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.888855:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677446.888858:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.888858:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677446.888859:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677446.888861:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677446.888862:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.888862:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.888863:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677446.888903:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.888905:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.888906:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.888907:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.888908:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.888910:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.889145:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.889147:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.889148:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.889150:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.889152:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.889153:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.889224:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.889226:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.889227:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.889229:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.889232:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.889233:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.889234:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.889235:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.889236:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.889237:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.889240:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.889242:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.889244:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.889245:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.889246:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.889248:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.889248:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.889249:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.889249:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677446.889428:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.889430:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.889432:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.889432:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.889435:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.889438:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.889693:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.889695:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.889696:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.889697:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.889699:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.889700:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.889720:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.889722:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.889723:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.889725:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.889727:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.889729:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677446.889730:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.889731:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.889732:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.889733:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677446.889736:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.889737:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677446.889740:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.889741:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677446.889741:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677446.889743:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677446.889743:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.889744:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.889744:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677446.889957:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.889960:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.889960:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677446.889962:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.889965:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.889967:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.889968:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.889970:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.889971:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.889972:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.889975:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.889977:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.889980:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.889980:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.889981:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.889983:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.889984:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.889985:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.889985:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677446.890064:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.890067:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.890068:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.890068:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.890071:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.890074:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.890093:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.890095:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.890096:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.890097:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.890099:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.890100:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.890527:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.890530:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.890531:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.890531:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.890533:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.890538:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.890539:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.890542:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677446.890542:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.890543:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.890544:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.890544:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.890561:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.890563:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.890565:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.890566:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.890568:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.890569:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.890622:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.890625:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.890626:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.890626:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.890628:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.890630:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.890642:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677446.890645:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677446.890645:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.890646:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.890647:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.890648:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.890749:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.890751:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.890752:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.890754:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.890755:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.890757:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.890766:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.890769:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.890770:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.890771:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.890771:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.890773:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.890810:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.890813:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.890814:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.890814:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.890816:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.890818:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.890998:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.891001:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.891001:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.891003:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.891006:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.891008:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677446.891009:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.891010:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.891012:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.891012:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677446.891016:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.891017:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677446.891020:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.891021:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677446.891021:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677446.891023:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677446.891024:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.891024:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.891025:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.891339:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.891341:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.891342:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.891344:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.891346:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.891346:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677446.891353:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.891356:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.891357:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.891358:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.891360:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.891362:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.891966:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.891969:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.891969:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.891971:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.891973:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.891976:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677446.891977:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.891978:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.891979:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.891980:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677446.891984:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.891986:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677446.891989:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.891990:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677446.891990:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677446.891992:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677446.891993:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.891994:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.891995:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.891996:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.892006:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.892008:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.892008:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.892009:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677446.892348:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.892350:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.892351:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.892352:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.892354:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.892355:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.892468:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.892470:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.892471:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.892473:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.892475:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.892475:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.892486:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.892489:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.892489:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.892490:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.892491:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.892492:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.892547:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.892550:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.892551:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.892554:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.892557:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.892559:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677446.892560:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96735865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.892562:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.892563:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.892564:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677446.892567:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.892569:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677446.892572:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.892573:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677446.892573:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677446.892575:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:24.0:1619677446.892575:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.892576:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.892577:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.892580:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677446.892582:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677446.892583:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.892585:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.892586:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.892587:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677446.892588:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.892588:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.892589:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677446.892590:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.892592:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.892798:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.892801:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.892801:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.892804:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.892807:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.892808:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677446.892809:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96735861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.892811:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.892811:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.892813:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677446.892816:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677446.892817:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:22.0:1619677446.892818:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677446.892820:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677446.892820:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.892820:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.892821:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677446.892821:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:12.0:1619677446.892822:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.892823:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:13.0:1619677446.892823:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677446.892824:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.892824:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.892825:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677446.892825:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.892841:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.892844:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.892845:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.892845:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.892846:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.892847:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.892884:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.892887:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.892888:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.892890:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.892890:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.892892:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.892958:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.892960:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.892961:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.892963:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.892963:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:12.0:1619677446.892964:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.892965:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.892965:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.892967:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.892967:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.892969:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.892972:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.892986:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677446.892989:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677446.892990:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.892990:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677446.892991:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677446.892991:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677446.893289:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.893292:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.893299:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.893300:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677446.893301:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.893303:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.893321:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.893323:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.893324:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.893326:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.893327:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.893328:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.893397:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.893400:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.893401:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.893402:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.893405:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.893406:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677446.893407:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.893409:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.893410:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.893411:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677446.893414:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.893415:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677446.893418:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.893419:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677446.893419:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677446.893421:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677446.893421:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.893422:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.893423:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677446.893435:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.893439:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.893440:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.893440:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.893443:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.893445:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.893576:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.893578:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.893579:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.893580:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.893582:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.893583:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.893596:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.893598:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.893599:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.893600:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.893600:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.893601:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.893789:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.893792:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.893792:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.893794:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.893795:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.893797:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.894067:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.894070:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.894071:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.894072:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.894075:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.894076:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:24.0:1619677446.894078:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:22.0:1619677446.894078:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.894080:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677446.894081:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:22.0:1619677446.894081:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.894082:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.894082:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.894086:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.894088:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677446.894089:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.894091:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:22.0:1619677446.894091:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.894093:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96735846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677446.894094:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.894094:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:24.0:1619677446.894095:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677446.894095:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:24.0:1619677446.894096:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677446.894097:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677446.894097:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:24.0:1619677446.894098:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677446.894098:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.894099:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:24.0:1619677446.894104:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.894107:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.894110:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.894111:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.894112:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.894114:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.894114:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.894115:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.894117:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.894117:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.894133:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.894135:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.894136:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.894137:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.894220:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.894222:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.894223:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.894224:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.894227:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.894228:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.894536:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677446.894539:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677446.894540:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677446.894542:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.894544:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.894546:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677446.894547:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.894548:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.894549:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.894550:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677446.894553:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.894555:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677446.894558:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.894559:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677446.894559:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677446.894561:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677446.894561:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677446.894562:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677446.894562:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:13.0:1619677446.894573:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.894576:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.894577:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.894578:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.894580:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.894581:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.894823:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.894826:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.894827:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.894827:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.894829:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.894832:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.894906:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677446.894908:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:13.0:1619677446.894909:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.894910:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.894911:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677446.894911:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:13.0:1619677446.894912:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.894912:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:8.0:1619677446.894912:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.894914:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.894915:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.894917:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677446.894930:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.894933:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.894934:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.894935:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.894935:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.894937:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.895142:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.895145:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:13.0:1619677446.895146:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:6.0:1619677446.895147:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.895148:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677446.895149:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.895150:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.895151:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.895152:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:22.0:1619677446.895153:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:12.0:1619677446.895153:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:22.0:1619677446.895154:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96735831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.895155:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677446.895155:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:22.0:1619677446.895156:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.895157:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677446.895161:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.895162:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677446.895165:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.895165:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677446.895166:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677446.895168:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677446.895168:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.895169:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.895170:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677446.895687:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.895690:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.895690:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.895692:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.895694:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.895698:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.895727:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.895729:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.895730:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.895732:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.895733:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.895735:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.896091:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.896094:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.896095:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.896095:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.896097:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.896100:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677446.896227:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.896230:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.896231:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.896232:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.896236:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.896237:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.896239:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.896240:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.896242:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.896243:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.896247:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.896249:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.896252:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.896253:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.896254:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.896256:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.896256:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.896257:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.896258:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.896337:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.896339:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.896340:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.896341:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.896343:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.896344:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.896540:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.896543:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.896544:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.896546:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.896548:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.896549:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677446.896550:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.896552:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.896553:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.896554:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677446.896557:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.896558:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677446.896561:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.896561:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677446.896562:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677446.896564:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677446.896564:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.896565:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.896566:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.896567:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.896577:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.896578:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.896579:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.896580:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677446.896599:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.896602:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.896603:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.896603:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.896605:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.896609:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.896617:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677446.896619:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.896620:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.896621:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.896622:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.896622:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.897146:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.897148:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.897149:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.897149:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.897152:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.897154:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.897184:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.897187:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.897188:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.897190:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.897192:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.897194:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677446.897195:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.897196:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.897197:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.897199:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677446.897204:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.897207:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677446.897211:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.897212:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677446.897212:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677446.897215:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677446.897216:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.897217:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.897218:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.897301:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.897304:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.897304:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.897306:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.897308:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.897309:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.897321:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.897323:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.897324:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.897325:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.897325:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.897326:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.897509:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.897512:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.897512:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.897515:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.897518:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.897519:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677446.897520:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.897522:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.897522:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.897524:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677446.897527:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.897529:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677446.897531:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.897532:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677446.897533:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677446.897534:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.897535:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.897535:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.897536:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.897546:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.897548:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.897549:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.897551:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.897553:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.897554:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.897603:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.897606:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.897607:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.897607:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.897609:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.897614:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.897616:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677446.897618:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677446.897618:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677446.897620:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.897623:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.897624:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:22.0:1619677446.897625:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.897627:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.897628:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.897629:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677446.897632:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.897633:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.897635:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.897636:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677446.897636:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.897638:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.897638:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677446.897639:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677446.897639:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:6.0:1619677446.897650:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.897652:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.897653:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.897654:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.897654:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.897656:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.897731:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.897734:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.897734:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.897736:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.897737:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.897739:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.897749:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.897751:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.897752:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.897753:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.897754:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.897755:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.897792:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.897794:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.897795:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.897795:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.897798:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.897800:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.897807:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677446.897810:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.897811:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.897811:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.897812:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.897813:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.897851:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.897854:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.897854:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.897856:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.897858:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.897858:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.897941:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.897944:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.897945:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.897947:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.897949:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.897950:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677446.897951:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.897953:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.897954:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.897955:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677446.897959:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.897960:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677446.897963:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.897964:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677446.897964:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677446.897966:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677446.897967:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.897967:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.897968:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677446.898010:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.898013:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.898014:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.898017:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.898019:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.898021:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.898022:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.898023:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.898024:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.898025:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.898028:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.898029:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.898032:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.898033:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.898034:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.898036:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.898037:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.898037:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.898038:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.898082:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.898084:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.898085:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.898085:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.898086:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.898086:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.898087:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:12.0:1619677446.898088:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.898090:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.898091:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.898092:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.898093:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.898406:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.0:1619677446.898406:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.898408:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.898409:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677446.898409:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.898410:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.898411:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677446.898412:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.898414:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:8.0:1619677446.898414:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.898415:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:13.0:1619677446.898416:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.898416:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677446.898416:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96735787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.898418:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.898419:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677446.898420:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.898420:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.898420:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677446.898422:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.898423:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:8.0:1619677446.898423:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677446.898424:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.898426:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.898428:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.898429:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.898430:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.898431:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.898432:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.898433:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.898435:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.898435:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.898447:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.898450:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.898450:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.898451:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.898502:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.898505:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.898506:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.898508:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.898509:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.898510:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.898721:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.898723:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.898724:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.898726:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.898729:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.898730:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.898732:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.898733:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.898734:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.898735:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.898738:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.898740:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.898743:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.898744:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.898745:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.898747:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.898747:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.898748:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.898748:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.898888:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.898891:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.898891:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.898893:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.898894:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.898896:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.898905:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.898908:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.898909:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.898909:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.898910:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.898911:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.899051:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.899054:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.899054:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.899056:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.899057:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.899059:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.899172:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.899175:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.899176:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.899176:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.899180:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.899183:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.899192:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677446.899195:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.899196:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677446.899197:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677446.899198:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.899198:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677446.899224:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.899226:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.899228:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.899229:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.899230:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.899232:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.899233:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.899235:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.899236:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.899238:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.899240:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.899242:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.899243:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.899244:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.899245:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.899246:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.899250:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.899252:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.899254:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.899255:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.899256:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.899258:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.899259:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.899260:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.899261:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.899262:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.899274:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.899276:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.899277:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.899277:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:6.0:1619677446.899569:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.899572:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.899573:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.899573:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.899575:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.899578:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.899743:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.899745:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.899746:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.899748:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.899751:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.899752:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677446.899753:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.899755:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.899756:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.899757:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677446.899760:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.899762:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677446.899765:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.899766:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677446.899766:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677446.899768:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677446.899769:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.899769:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.899770:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677446.899964:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.899967:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.899968:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.899968:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.899970:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.899973:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.899978:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.899981:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.899982:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.899983:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.899985:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.899986:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.900164:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.900166:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.900167:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.900169:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.900170:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.900172:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.900181:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.900184:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.900184:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.900185:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.900186:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.900187:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.900257:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.900259:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.900260:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.900262:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.900263:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.900265:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.900275:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.900277:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.900278:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.900279:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.900279:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.900281:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.900340:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.900342:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.900343:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.900343:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.900346:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.900348:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.900360:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677446.900362:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677446.900363:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.900363:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.900364:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.900365:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.900509:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.900512:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.900513:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.900513:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.900515:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.900517:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.900706:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.900708:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.900709:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.900712:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.900715:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.900717:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677446.900718:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96735757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.900719:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.900721:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.900722:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677446.900726:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.900727:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677446.900730:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.900731:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677446.900731:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677446.900733:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677446.900734:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.900735:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.900736:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.900737:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.900748:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.900750:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.900751:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.900752:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.900782:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.900785:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.900785:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.900787:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.900789:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.900790:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.900898:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.900901:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.900901:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677446.900905:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.900908:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.900910:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.900911:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96735754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.900912:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.900913:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.900914:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.900918:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.900920:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.900922:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.900923:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.900924:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.900926:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.900926:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.900927:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.900927:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677446.901022:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.901025:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.901026:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.901026:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.901028:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.901032:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.901043:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.901045:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.901046:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.901048:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.901049:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.901051:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.901152:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.901155:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.901155:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.901156:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.901158:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.901161:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.901169:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677446.901172:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.901173:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.901173:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.901174:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.901175:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.901255:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.901258:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.901258:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.901260:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.901262:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.901262:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.901267:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.901270:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.901271:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.901272:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.901274:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.901276:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.901388:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.901390:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.901391:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.901393:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.901395:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.901396:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.901599:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.901602:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.901603:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.901603:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.901605:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.901609:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677446.901660:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.901663:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.901663:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677446.901665:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.901667:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.901669:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677446.901670:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.901671:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.901673:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.901673:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677446.901677:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.901679:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677446.901681:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.901682:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677446.901683:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677446.901685:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677446.901685:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.901686:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.901686:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677446.901842:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.901844:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.901845:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.901847:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.901848:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.901850:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.901873:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.901876:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.901877:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.901878:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.901880:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.901883:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.902201:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.902203:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.902204:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.902205:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.902207:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.902209:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.902217:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.902220:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.902221:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.902221:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.902222:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.902223:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.902344:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.902347:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.902347:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.902348:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.902350:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.902353:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.902378:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.902381:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.902381:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.902384:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.902385:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.902387:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677446.902388:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.902389:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.902390:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.902391:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677446.902395:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.902397:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677446.902400:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.902401:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677446.902401:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677446.902403:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677446.902404:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.902404:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.902405:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677446.902814:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.902817:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.902817:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.902818:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.902820:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.902823:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.902831:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.902834:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677446.902834:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.902835:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.902836:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.902836:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.902918:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.902921:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.902922:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.902923:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.902925:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.902926:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.903133:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.903136:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.903137:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.903139:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.903142:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.903143:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.903145:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.903146:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.903147:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.903148:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.903151:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.903153:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.903156:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.903157:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.903158:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.903159:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.903160:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.903161:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.903161:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677446.903607:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.903610:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.903611:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.903613:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.903615:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.903617:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.903618:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96735718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.903619:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.903620:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677446.903620:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:24.0:1619677446.903621:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677446.903623:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.903624:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.903624:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.903625:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677446.903626:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.903627:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.903629:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.903629:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677446.903630:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.903631:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.903633:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.903633:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.903634:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.903635:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.903694:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.903697:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.903698:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.903699:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.903702:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.903704:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.903705:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.903707:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.903707:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.903708:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.903712:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.903713:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.903715:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.903716:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.903717:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.903718:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.903719:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.903719:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.903720:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677446.903829:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.903832:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.903833:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.903835:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.903838:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.903840:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677446.903842:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96735714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.903844:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.903845:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.903846:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677446.903850:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.903851:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677446.903854:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.903855:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677446.903856:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677446.903858:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677446.903858:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.903858:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677446.903859:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:22.0:1619677446.903859:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677446.903862:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.903863:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.903863:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.903866:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.903869:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677446.904168:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.904171:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.904171:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.904173:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.904175:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.904176:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.904186:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.904189:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.904190:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.904190:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.904190:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.904191:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.904248:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.904251:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.904252:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.904254:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.904255:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.904257:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.904330:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.904333:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.904334:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.904336:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.904337:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.904339:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.904347:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.904350:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677446.904351:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.904353:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.904353:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.904354:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.904356:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.904356:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:24.0:1619677446.904358:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677446.904358:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.904362:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.904363:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.904364:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.904365:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.904366:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.904367:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.904371:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.904372:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.904376:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.904377:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.904377:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.904379:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.904380:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.904380:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.904381:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677446.904587:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.904590:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.904591:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.904591:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.904593:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.904596:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677446.904744:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677446.904747:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677446.904749:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677446.904750:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.904754:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.904755:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677446.904757:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.904758:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.904759:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.904760:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677446.904763:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.904765:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677446.904768:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.904768:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677446.904769:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677446.904771:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677446.904771:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677446.904772:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677446.904772:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:22.0:1619677446.904800:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.904803:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.904804:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.904804:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.904806:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.904808:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677446.904814:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.904817:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.904818:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.904818:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.904820:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.904822:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677446.904948:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.904951:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.904952:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.904954:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.904956:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.904958:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677446.904959:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.904961:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.904962:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.904963:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677446.904967:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.904969:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677446.904972:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.904972:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677446.904973:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677446.904975:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677446.904976:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.904977:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.904978:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.904978:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.904990:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.904992:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.904993:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.904994:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677446.905055:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.905058:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.905059:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.905061:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.905064:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.905065:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.905067:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96735699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.905069:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.905070:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.905071:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.905075:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.905077:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.905080:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.905080:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.905081:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.905082:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.905083:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.905083:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.905084:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.905123:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.905126:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.905127:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.905127:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.905129:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.905132:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677446.905314:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.905316:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.905317:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.905317:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.905319:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.905322:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.905384:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.905387:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.905388:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.905390:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.905392:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.905393:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.905396:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.905399:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.905400:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.905400:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.905402:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.905404:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.905729:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.905732:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.905733:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.905733:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.905735:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.905739:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.905833:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.905836:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.905837:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.905838:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.905841:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.905843:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677446.905845:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96735686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.905847:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.905848:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.905849:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677446.905852:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.905854:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677446.905857:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.905858:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677446.905859:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677446.905861:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677446.905861:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.905862:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.905863:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677446.906017:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.906020:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.906021:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.906022:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.906025:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.906026:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.906027:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.906029:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.906030:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.906031:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.906034:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.906036:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.906038:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.906039:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.906040:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.906042:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.906043:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.906044:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.906045:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.906046:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.906058:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.906061:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.906062:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.906063:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677446.906175:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.906178:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.906179:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.906179:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.906181:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.906184:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.906192:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.906194:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.906195:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.906195:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.906196:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.906197:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.906584:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.906587:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.906588:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.906589:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.906591:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.906594:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.906643:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.906646:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.906646:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.906647:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.906649:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.906653:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.906673:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.906675:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.906676:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.906679:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.906680:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.906682:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.906713:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.906716:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.906717:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.906717:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.906719:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.906721:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.906736:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.906740:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.906740:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.906744:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677446.906746:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:22.0:1619677446.906747:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.906748:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:12.0:1619677446.906748:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.906749:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.906750:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96735674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677446.906752:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.906752:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:22.0:1619677446.906753:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.906754:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677446.906754:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:22.0:1619677446.906756:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677446.906763:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.906767:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677446.906771:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.906772:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677446.906773:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677446.906774:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677446.906775:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.906776:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.906777:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.906788:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.906790:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:8.0:1619677446.906790:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:13.0:1619677446.906792:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.906794:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677446.906795:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:13.0:1619677446.906795:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.906795:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:13.0:1619677446.906797:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:8.0:1619677446.906798:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.906801:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.906803:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.906804:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.906806:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.906807:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.906808:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.906812:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.906814:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.906817:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.906817:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.906818:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.906820:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.906820:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.906821:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.906822:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.906823:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.906835:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.906838:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.906838:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.906839:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677446.906998:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.907001:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.907002:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.907002:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.907005:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.907007:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.907017:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.907019:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.907020:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.907020:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.907021:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.907022:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.907147:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.907150:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677446.907151:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.907165:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.907168:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[3] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677446.907168:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.907171:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.907175:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.907175:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.907197:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.907200:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677446.907200:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.907201:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.907203:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: REPLY: OK
00000800:00000200:6.0:1619677446.907205:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677446.907684:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.907686:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.907687:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.907688:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.907690:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.907691:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.907701:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.907704:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.907704:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.907705:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.907706:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.907706:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.907961:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.907963:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.907964:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.907965:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.907967:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.907968:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.907977:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.907980:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.907981:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.907981:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.907982:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.907984:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.908145:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.908148:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.908149:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.908150:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.908151:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.908153:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.908166:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.908169:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677446.908170:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.908171:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.908173:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.908176:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677446.908222:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.908224:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.908225:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.908227:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.908228:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.908229:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.908424:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.908427:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.908427:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.908429:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.908431:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.908432:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.908594:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.908596:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.908598:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.908598:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.908601:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.908603:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677446.908638:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.908640:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.908641:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.908643:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.908643:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:12.0:1619677446.908644:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.908645:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.908646:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.908648:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.908649:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.908650:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677446.908653:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677446.908709:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.908712:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.908713:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.908715:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.908717:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.908718:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.908720:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.908721:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.908722:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.908723:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.908727:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.908728:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.908731:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.908732:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.908732:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.908734:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.908734:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.908735:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.908735:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677446.908801:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.908803:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.908804:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.908806:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.908809:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.908811:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677446.908812:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.908813:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.908814:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.908815:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677446.908818:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.908820:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677446.908823:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.908824:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677446.908825:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677446.908827:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677446.908828:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.908828:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.908829:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677446.909146:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.909148:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.909149:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.909150:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.909151:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.909155:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677446.909202:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.909204:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.909205:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.909207:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:12.0:1619677446.909207:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.909208:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.909209:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.909209:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.909210:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:13.0:1619677446.909211:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.909211:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.909214:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.909215:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.909217:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.909218:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.909219:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.909221:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.909223:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677446.909230:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.909233:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.909234:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.909235:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.909235:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677446.909235:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:12.0:1619677446.909237:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.909238:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677446.909239:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.909239:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.909240:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.909241:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.909344:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677446.909347:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677446.909348:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677446.909349:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.909352:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.909354:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677446.909355:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.909357:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.909358:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.909359:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677446.909363:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.909366:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677446.909369:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.909370:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:22.0:1619677446.909371:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677446.909373:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:22.0:1619677446.909373:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677446.909374:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677446.909375:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:6.0:1619677446.909441:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.909444:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.909445:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.909445:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.909448:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.909451:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.909589:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.909592:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.909593:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.909594:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.909596:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.909597:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.909680:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.909683:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.909684:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.909686:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.909688:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.909690:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677446.909691:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.909693:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.909693:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.909695:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677446.909698:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.909700:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677446.909702:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.909703:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677446.909704:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677446.909705:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677446.909706:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.909707:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.909708:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.909709:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.909719:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.909721:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.909721:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.909722:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677446.909784:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.909787:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.909788:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.909790:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.909791:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.909793:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.909872:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.909875:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.909876:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.909876:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.909878:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.909880:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677446.909926:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.909929:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.909930:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.909932:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.909935:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.909937:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.909938:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.909939:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.909941:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.909942:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.909946:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.909947:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.909950:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.909951:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.909952:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.909953:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.909954:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.909954:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.909955:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677446.910305:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.910308:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.910309:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.910311:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.910313:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.910314:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677446.910316:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96735627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.910317:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.910319:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.910320:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677446.910324:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:6.0:1619677446.910324:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.910326:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.910326:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:22.0:1619677446.910327:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.910329:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.910329:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.910330:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677446.910331:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677446.910332:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.910333:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677446.910334:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.910334:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.910335:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.910336:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.910442:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.910444:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.910445:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.910447:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.910449:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.910450:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.910483:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.910485:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.910486:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.910488:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.910489:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.910491:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.910500:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.910502:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.910503:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.910504:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.910504:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.910506:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.911013:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.911016:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.911017:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.911021:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.911024:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.911026:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677446.911028:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96735619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.911030:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.911031:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.911033:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677446.911037:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.911039:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677446.911042:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.911043:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677446.911044:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677446.911046:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677446.911047:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.911048:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.911049:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677446.911484:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.911487:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.911488:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.911489:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.911492:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.911494:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.911496:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96735613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.911497:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.911498:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.911498:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.911502:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.911503:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.911506:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.911506:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.911507:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.911509:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.911509:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.911510:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.911510:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677446.911837:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.911839:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.911840:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.911842:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.911844:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.911846:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677446.911847:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.911849:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.911849:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.911851:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677446.911854:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.911855:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677446.911858:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.911858:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677446.911859:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677446.911860:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677446.911861:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.911862:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.911862:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677446.911931:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.911934:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.911935:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.911935:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.911937:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.911940:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.911950:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677446.911952:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:6.0:1619677446.911953:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.911954:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.911954:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:24.0:1619677446.911955:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.911955:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:6.0:1619677446.911955:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.911956:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:8.0:1619677446.911959:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.911962:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.911965:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.911966:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96735608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677446.911968:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000400:00000200:8.0:1619677446.911968:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677446.911969:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000400:00000200:8.0:1619677446.911970:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.911971:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.911973:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.911974:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677446.911975:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.911977:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:24.0:1619677446.911978:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96735607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.911980:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.911981:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.911982:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677446.911982:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.911985:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.911987:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:24.0:1619677446.911989:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.911989:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.911990:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:8.0:1619677446.911990:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.911991:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677446.911991:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.911993:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677446.911993:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (22)++
00000800:00000200:8.0:1619677446.911993:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.911994:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.911995:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (23)++
00000800:00000200:8.0:1619677446.911995:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.911996:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677446.911997:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (24)--
00000800:00000200:8.0:1619677446.911997:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (24)--
00000800:00000200:24.0:1619677446.911998:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677446.912008:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.912010:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677446.912011:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.912012:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:24.0:1619677446.912013:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000400:00000200:8.0:1619677446.912013:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.912014:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677446.912015:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677446.912016:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.912016:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:24.0:1619677446.912024:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677446.912027:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677446.912027:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.912027:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677446.912028:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677446.912029:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677446.912527:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.912530:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.912531:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.912531:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.912533:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.912537:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.912606:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.912609:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.912610:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.912611:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.912613:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.912614:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.912749:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.912751:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.912752:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.912752:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.912755:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.912759:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.912894:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.912897:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.912898:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.912900:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.912900:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.912903:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.912914:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.912916:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.912917:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.912917:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.912920:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.912923:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.912946:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.912949:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.912949:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.912951:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.912953:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.912953:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.912964:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.912967:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.912967:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.912968:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677446.912969:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.912969:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.912971:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.912971:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.912971:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.912973:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.912975:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.913025:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.913028:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.913029:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.913030:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.913033:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.913035:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.913036:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.913037:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.913038:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.913039:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.913042:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.913044:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.913047:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.913047:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.913048:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.913050:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.913050:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.913051:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.913052:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.913053:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.913065:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.913067:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.913068:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.913069:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.913231:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.913233:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.913234:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.913235:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.913237:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.913238:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.913420:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.913423:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.913424:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.913425:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.913427:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.913428:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.913490:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.913493:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.913494:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.913496:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.913497:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.913499:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.913705:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:6.0:1619677446.913706:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.0:1619677446.913708:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:6.0:1619677446.913708:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:24.0:1619677446.913709:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:6.0:1619677446.913709:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.913710:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:8.0:1619677446.913711:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.913712:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:8.0:1619677446.913714:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.913714:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:8.0:1619677446.913716:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.913717:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.913719:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.913720:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.913721:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.913725:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.913726:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.913730:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.913730:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.913731:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.913733:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.913734:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.913734:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.913735:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.914146:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.914149:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.914150:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.914151:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.914153:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.914154:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.914217:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.914220:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.914221:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.914222:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.914224:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.914225:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.914237:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.914240:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.914241:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.914242:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.914242:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.914244:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.914280:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.914282:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.914283:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.914285:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.914286:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.914288:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.914361:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.914363:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.914364:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.914366:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.914368:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.914369:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.914767:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.914769:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.914770:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.914770:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.914773:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.914776:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.914777:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677446.914779:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677446.914779:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677446.914781:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.914783:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.914785:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:22.0:1619677446.914786:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.914788:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.914789:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.914790:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:22.0:1619677446.914793:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.914795:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677446.914799:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.914799:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:22.0:1619677446.914800:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:22.0:1619677446.914802:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:22.0:1619677446.914803:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677446.914803:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677446.914804:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:24.0:1619677446.914850:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.914853:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.914854:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.914856:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.914859:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.914860:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.914862:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.914863:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.914864:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.914865:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.914868:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.914868:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677446.914871:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677446.914871:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.914871:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:24.0:1619677446.914874:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677446.914874:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677446.914875:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.914875:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.914877:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.914877:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:6.0:1619677446.914877:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.914878:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.914879:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677446.914879:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677446.914880:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96735573] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.914881:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.914882:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.914883:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677446.914887:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.914889:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677446.914892:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.914892:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677446.914893:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677446.914895:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677446.914895:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.914897:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.914898:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.914898:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.914909:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.914911:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.914912:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.914913:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677446.915395:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.915398:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.915401:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.915401:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677446.915403:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.915404:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677446.915404:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.915405:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.915406:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677446.915407:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677446.915408:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96735568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.915410:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.915411:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.915412:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677446.915414:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.915415:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677446.915416:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677446.915417:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000400:00000200:6.0:1619677446.915417:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.915418:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.915419:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.915419:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677446.915420:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.915422:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.915423:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677446.915424:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677446.915426:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677446.915427:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.915427:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.915427:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677446.915662:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.915665:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.915666:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.915666:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.915668:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.915671:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.915681:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.915684:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.915684:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677446.915685:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.915687:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.915687:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.915701:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.915704:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.915704:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.915705:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.915706:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.915709:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677446.915879:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.915882:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.915883:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.915883:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.915885:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.915889:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677446.916067:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.916070:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.916071:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.916073:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.916075:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.916077:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677446.916078:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.916080:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.916081:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.916082:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677446.916085:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.916087:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677446.916090:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.916091:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677446.916091:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677446.916093:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677446.916093:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.916094:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.916095:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677446.916307:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.916310:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.916311:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.916311:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.916313:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:24.0:1619677446.916317:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.916331:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.916334:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.916335:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.916335:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.916337:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.916340:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.916436:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.916439:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.916440:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.916441:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.916445:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.916446:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.916447:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.916449:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.916450:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.916451:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.916454:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.916455:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.916458:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.916459:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.916459:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.916461:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.916462:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.916462:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.916463:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.916715:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.916718:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.916719:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.916721:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.916724:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.916726:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.916727:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.916728:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.916729:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.916730:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.916734:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.916736:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.916738:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.916739:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.916740:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.916742:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.916743:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.916743:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.916744:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677446.917025:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.917028:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.917029:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.917031:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.917034:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.917035:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:12.0:1619677446.917036:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:22.0:1619677446.917037:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96735548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.917038:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677446.917038:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:22.0:1619677446.917039:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677446.917039:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.917040:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:12.0:1619677446.917041:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.917043:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:22.0:1619677446.917044:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677446.917044:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.917046:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677446.917049:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.917050:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677446.917051:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677446.917052:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677446.917053:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677446.917053:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677446.917054:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.917054:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.917055:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.917056:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.917057:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.917058:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.917059:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.917315:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.917318:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.917319:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.917321:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.917324:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.917325:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.917326:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.917332:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.917333:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.917334:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.917338:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.917342:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.917345:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.917346:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.917347:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.917348:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.917349:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.917349:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.917350:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.917560:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.917563:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.917564:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.917566:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.917569:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.917571:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.917572:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.917573:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.917574:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.917575:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.917579:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.917580:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.917583:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.917584:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.917584:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.917586:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.917587:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.917588:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.917589:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.917590:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.917602:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.917604:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.917605:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.917606:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677446.917624:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.917627:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.917627:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.917628:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.917630:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.917633:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.917745:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.917748:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.917749:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.917751:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.917753:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.917754:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.917856:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)++
00000800:00000200:13.0:1619677446.917858:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677446.917859:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.917861:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.917862:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677446.917863:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677446.917863:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:22.0:1619677446.917864:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:12.0:1619677446.917864:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.917865:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.917866:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.917867:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:12.0:1619677446.917867:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.917868:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.917870:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.917873:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.917875:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:13.0:1619677446.917876:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677446.917877:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96735533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.917878:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.917879:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677446.917879:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677446.917880:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:13.0:1619677446.917880:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.917881:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.917881:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.917883:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:22.0:1619677446.917884:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.917886:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677446.917889:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.917889:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677446.917890:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677446.917892:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677446.917892:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.917893:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.917894:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.917895:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.917905:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.917907:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.917908:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.917909:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677446.918258:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.918260:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.918261:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.918262:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.918264:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.918265:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.918380:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.918383:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.918385:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.918387:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.918388:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.918390:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.918403:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.918406:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.918406:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.918408:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.918409:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.918411:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.918541:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.918544:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.918545:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.918553:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.918557:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.918559:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.918560:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96735524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.918562:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.918563:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.918565:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.918569:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.918572:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.918575:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.918575:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.918576:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.918578:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.918579:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.918580:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.918581:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:12.0:1619677446.918582:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.918584:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.918585:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.918586:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.918588:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.918589:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.918592:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.918594:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.918595:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.918596:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.918596:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677446.918728:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.918731:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.918732:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.918732:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.918734:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.918738:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.918748:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677446.918751:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677446.918753:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677446.918754:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677446.918755:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.918757:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:13.0:1619677446.918759:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96735520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677446.918760:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677446.918761:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677446.918762:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:13.0:1619677446.918765:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677446.918767:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:13.0:1619677446.918769:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677446.918770:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:13.0:1619677446.918771:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:13.0:1619677446.918772:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:13.0:1619677446.918773:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677446.918774:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677446.918774:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.0:1619677446.918784:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.918786:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677446.918787:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.918788:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.918789:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.918792:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.918805:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677446.918807:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677446.918808:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.918808:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677446.918809:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677446.918809:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.918824:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677446.918827:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677446.918828:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677446.918830:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677446.918833:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.918835:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677446.918837:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.918839:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.918841:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.918842:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677446.918848:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677446.918851:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677446.918855:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.918855:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677446.918856:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677446.918858:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677446.918859:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677446.918860:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677446.918861:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677446.918862:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:12.0:1619677446.918867:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.918870:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.918871:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.918873:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:12.0:1619677446.918873:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.918874:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677446.918875:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.918876:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.918876:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.918876:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.918878:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677446.918882:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677446.918887:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677446.918890:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:13.0:1619677446.918890:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677446.918891:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677446.918892:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:13.0:1619677446.918893:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.918894:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.918894:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.918895:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.918896:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.918898:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677446.918901:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.918902:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.918902:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.918903:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.918904:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.918960:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.918962:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.918963:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.918964:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.918965:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.918967:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.919113:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.919116:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.919117:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.919118:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.919120:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.919121:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.919377:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.919380:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.919381:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.919381:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.919383:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677446.919385:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.919474:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.919477:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.919478:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.919480:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.919482:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.919483:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.919591:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.919594:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.919594:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.919596:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.919597:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.919598:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.919898:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.919900:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.919901:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.919903:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.919904:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.919905:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.920092:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677446.920095:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677446.920095:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677446.920097:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.920100:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.920102:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:22.0:1619677446.920103:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.920104:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.920105:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.920107:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:22.0:1619677446.920110:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.920112:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:22.0:1619677446.920115:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.920116:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:22.0:1619677446.920117:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:22.0:1619677446.920119:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:22.0:1619677446.920119:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677446.920121:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677446.920122:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677446.920122:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677446.920133:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:6.0:1619677446.920135:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:6.0:1619677446.920136:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677446.920137:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:22.0:1619677446.920339:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.920341:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.920343:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.920343:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.920345:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.920348:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.920473:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.920475:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.920476:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.920478:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.920480:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.920482:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677446.920483:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.920484:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.920485:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.920486:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677446.920490:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.920491:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677446.920494:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.920495:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677446.920496:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677446.920498:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677446.920498:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.920499:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.920499:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.920739:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.920742:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.920743:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.920744:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.920746:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.920747:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.920757:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.920759:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.920760:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.920761:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.920761:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.920763:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.920770:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.920774:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.920774:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677446.920778:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.920782:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.920783:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.920789:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96735499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.920791:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.920792:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.920793:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.920796:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.920798:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.920801:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.920802:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.920807:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.920809:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.920809:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677446.920810:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.0:1619677446.920810:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.920811:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677446.920812:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.920813:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.920814:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.920816:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.920819:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.921004:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.921007:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.921008:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.921010:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.921013:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.921019:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677446.921020:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96735494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.921022:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.921023:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.921024:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677446.921027:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.921029:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677446.921032:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.921032:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677446.921034:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677446.921036:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677446.921036:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.921037:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.921038:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677446.921040:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.921043:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.921043:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.921044:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.921046:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.921049:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.921604:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.921607:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.921607:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.921610:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.921612:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.921614:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677446.921615:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.921617:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.921618:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.921619:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677446.921623:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.921626:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677446.921628:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.921629:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677446.921630:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677446.921632:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677446.921632:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.921633:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.921634:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677446.921771:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.921774:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.921775:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.921777:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.921779:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.921780:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.921789:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677446.921790:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:12.0:1619677446.921792:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:24.0:1619677446.921793:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:12.0:1619677446.921793:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:24.0:1619677446.921794:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.921794:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.921794:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677446.921794:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:12.0:1619677446.921795:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:24.0:1619677446.921796:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.921799:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677446.922103:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.922106:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.922107:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.922109:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.922110:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.922111:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.922170:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.922173:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.922173:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.922176:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.922176:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.922178:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.922225:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677446.922228:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677446.922229:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677446.922231:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677446.922234:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.922235:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677446.922236:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.922238:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.922239:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.922241:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677446.922245:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677446.922248:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677446.922249:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.922250:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677446.922251:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677446.922252:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.922253:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677446.922254:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677446.922254:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:24.0:1619677446.922275:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.922277:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.922278:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.922279:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.922282:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677446.922284:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.922300:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677446.922303:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677446.922305:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.922305:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.922306:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.922307:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677446.922326:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.922329:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.922330:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.922332:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.922334:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.922338:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.922342:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.922345:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.922346:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.922347:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.922349:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.922350:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677446.922352:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.922353:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677446.922354:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.922355:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.922356:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677446.922356:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.922357:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.922359:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.922360:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677446.922360:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.922360:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677446.922362:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96735474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.922363:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.922364:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677446.922364:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677446.922365:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.922369:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.922370:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.922373:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.922374:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.922375:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.922376:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.922377:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.922378:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.922378:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677446.922444:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.922447:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.922448:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.922450:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.922453:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.922455:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677446.922456:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.922458:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.922459:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.922461:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677446.922468:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.922471:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677446.922474:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.922475:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677446.922476:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677446.922477:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677446.922478:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.922479:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.922480:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.922481:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.922492:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.922494:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.922495:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.922495:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677446.922510:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.922512:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.922513:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.922516:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.922518:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.922520:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.922521:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.922522:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.922524:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.922525:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.922528:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.922530:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.922533:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.922534:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.922535:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.922537:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.922538:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.922538:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.922539:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677446.922590:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.922592:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.922593:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.922594:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.922596:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.922598:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.922608:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677446.922610:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.922611:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.922611:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.922612:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.922613:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.922647:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.922650:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.922650:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.922652:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.922654:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.922654:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.922725:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.922727:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.922728:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.922729:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.922731:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.922732:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.922827:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.922829:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.922831:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.922831:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.922833:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.922836:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677446.922929:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.922931:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.922932:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.922934:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.922935:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.922937:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.922946:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.922948:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.922949:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.922950:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.922950:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.922952:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.923068:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.923071:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.923071:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.923072:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.923075:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.923078:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.923240:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.923242:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.923243:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.923245:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.923246:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.923247:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.923257:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.923260:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.923261:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.923261:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.923262:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.923263:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.923417:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.923420:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.923420:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677446.923421:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.923423:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.923430:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.923432:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677446.923434:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677446.923435:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.923435:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.923436:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.923436:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677446.923441:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.923444:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.923444:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.923447:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.923448:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.923451:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.923468:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.923471:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.923472:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.923474:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.923474:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.923476:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.923547:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.923549:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.923551:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.923551:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.923553:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.923555:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677446.923593:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.923596:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.923597:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.923599:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.923601:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.923603:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677446.923604:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677446.923605:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:22.0:1619677446.923605:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.923606:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677446.923608:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:22.0:1619677446.923608:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.0:1619677446.923609:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.923611:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:24.0:1619677446.923613:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.923614:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677446.923615:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.923616:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.923617:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:24.0:1619677446.923618:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.923620:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96735453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677446.923620:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.923621:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:24.0:1619677446.923622:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677446.923622:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:24.0:1619677446.923623:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.923624:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677446.923624:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677446.923624:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.923625:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.923625:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:8.0:1619677446.923625:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.923627:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677446.923628:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677446.923628:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96735452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677446.923630:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.923630:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.923631:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.923632:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677446.923635:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.923636:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:8.0:1619677446.923636:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.923637:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.923638:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677446.923639:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.923639:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.923640:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.923641:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.923641:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.923642:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.923642:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677446.923642:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677446.923644:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.923644:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.923645:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.923646:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.923647:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.923656:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.923659:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.923659:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.923661:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.923663:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.923663:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.923823:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.923825:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.923826:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.923827:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.923829:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.923829:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677446.923855:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677446.923858:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677446.923859:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677446.923860:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677446.923863:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.923864:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677446.923865:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96735447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.923866:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.923867:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.923868:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677446.923870:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677446.923872:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677446.923873:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.923875:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677446.923875:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677446.923877:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677446.923877:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677446.923878:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677446.923878:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:8.0:1619677446.923899:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.923901:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677446.923902:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.923903:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.923904:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677446.923905:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677446.923985:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.923988:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.923989:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.923991:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.923992:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.923994:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.924153:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.924155:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.924157:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.924158:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.924160:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.924162:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677446.924163:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.924164:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.924165:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.924166:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677446.924170:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.924171:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677446.924174:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.924175:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677446.924176:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677446.924178:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677446.924178:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.924179:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.924179:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677446.924191:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.924194:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.924195:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.924195:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.924197:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.924200:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677446.924429:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.924431:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.924432:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.924432:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.924434:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.924437:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.924447:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677446.924449:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677446.924450:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.924450:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677446.924451:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677446.924451:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677446.924666:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.924669:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.924669:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.924671:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.924673:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.924674:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.924956:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.924958:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.924959:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.924959:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.924961:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.924964:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.925234:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.925237:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.925237:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.925239:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.925240:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.925241:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.925251:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.925254:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.925255:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.925255:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.925256:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.925257:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.925351:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.925354:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.925355:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.925355:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.925358:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.925360:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.925856:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.925859:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.925859:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.925861:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.925864:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.925866:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677446.925867:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.925870:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.925871:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.925872:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677446.925877:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.925880:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677446.925883:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.925884:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677446.925884:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677446.925886:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677446.925887:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.925887:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.925888:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677446.926441:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.926444:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.926445:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.926445:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.926447:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.926449:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.926714:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.926716:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.926717:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.926717:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.926719:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677446.926722:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.926898:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.926900:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.926901:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677446.926903:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:6.0:1619677446.926903:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677446.926905:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:6.0:1619677446.926905:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.926906:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.926907:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677446.926909:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:6.0:1619677446.926909:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96735418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677446.926910:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:6.0:1619677446.926911:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677446.926912:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:6.0:1619677446.926912:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.926913:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677446.926917:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.926920:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677446.926921:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677446.926922:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.926923:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677446.926924:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677446.926925:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:6.0:1619677446.926925:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:13.0:1619677446.926926:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:6.0:1619677446.926926:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.926927:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:13.0:1619677446.926928:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.926928:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:6.0:1619677446.926928:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.926929:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.926969:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.926971:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.926972:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.926972:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.926975:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:8.0:1619677446.926978:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677446.926984:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.926987:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.926988:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.926990:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.926991:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.926993:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.927211:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677446.927211:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.927213:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.927214:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.927215:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:13.0:1619677446.927216:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.927217:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.927217:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.927218:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677446.927218:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.927219:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.927220:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.927221:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.927222:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.927223:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.927224:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96735410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.927224:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:24.0:1619677446.927226:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.927227:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.927229:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.927233:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.927235:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.927236:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.927238:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.927239:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.927240:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.927240:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.927242:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.927243:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.927244:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.927245:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.927246:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.927259:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.927261:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.927261:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.927263:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.927507:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.927510:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.927510:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.927512:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.927514:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.927515:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.927563:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.927567:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.927567:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.927568:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.927571:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.927577:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.927580:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.927582:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677446.927583:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.927584:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.927585:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.927585:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.927585:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.927587:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.927588:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.927588:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.927590:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.927592:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677446.927593:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.927595:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.927596:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.927597:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677446.927600:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.927602:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677446.927604:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.927605:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677446.927606:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677446.927607:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677446.927608:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.927609:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.927610:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.927611:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.927621:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.927623:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.927624:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.927625:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677446.927745:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.927747:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.927748:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.927750:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.927751:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.927753:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.927796:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.927799:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.927800:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.927800:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.927802:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.927805:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.927816:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677446.927819:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677446.927820:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.927820:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.927821:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.927822:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.927969:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.927972:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.927972:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.927973:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.927975:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.927978:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.928033:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.928035:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.928036:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.928037:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.928039:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.928041:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.928095:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.928098:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.928099:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.928101:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.928103:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.928105:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677446.928106:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.928107:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.928108:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.928109:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677446.928114:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.928116:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677446.928118:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.928119:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677446.928120:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677446.928121:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677446.928122:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.928122:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.928123:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677446.928149:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.928152:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.928153:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.928153:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.928156:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.928159:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.928325:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.928328:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.928329:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.928331:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.928333:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.928334:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.928342:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.928345:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.928346:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.928347:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.928347:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.928349:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.928635:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.928637:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.928638:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.928641:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.928644:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.928646:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.928647:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96735387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677446.928648:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:24.0:1619677446.928649:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677446.928649:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:24.0:1619677446.928650:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.928651:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.928654:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.928655:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677446.928657:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.928658:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.928659:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677446.928660:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96735386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.928661:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.928662:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.928663:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677446.928666:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.928667:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:8.0:1619677446.928667:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.928668:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.928669:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677446.928670:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.928671:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.928672:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.928672:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.928673:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.928673:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677446.928674:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.928674:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677446.928680:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.928681:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.928682:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.928683:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.928684:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677446.928692:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.928694:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.928695:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.928696:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:12.0:1619677446.928697:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.928698:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:13.0:1619677446.928698:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.928699:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677446.928699:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.928701:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677446.928784:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.928786:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.928787:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.928789:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.928792:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.928793:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677446.928795:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96735382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.928796:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.928797:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.928798:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677446.928804:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.928807:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677446.928808:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.928810:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.928811:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677446.928811:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.928812:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:22.0:1619677446.928813:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (23)++
00000400:00000200:6.0:1619677446.928814:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.928815:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:22.0:1619677446.928816:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.928817:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.928817:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.928818:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677446.928818:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.928985:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.928988:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.928989:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.928989:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.928991:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.928993:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.929002:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677446.929005:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.929006:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.929006:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.929007:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.929007:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677446.929068:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.929071:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.929072:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.929072:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.929074:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.929077:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.929150:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.929152:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.929153:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.929153:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.929155:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.929158:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677446.929222:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.929224:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.929225:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.929226:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.929228:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.929229:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.929428:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.929431:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.929432:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.929433:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.929436:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.929437:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.929438:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.929440:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.929441:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.929441:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.929445:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.929447:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.929449:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.929450:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.929451:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.929453:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.929454:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.929454:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.929455:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.929569:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.929571:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.929573:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.929574:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.929576:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.929578:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.929586:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.929589:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.929590:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.929591:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.929591:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.929592:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.930130:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.930133:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.930134:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.930135:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.930136:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677446.930139:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677446.930141:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.930144:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.930145:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.930148:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.930151:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.930153:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.930155:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.930158:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.930159:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.930161:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.930167:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.930171:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.930174:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.930174:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.930175:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.930177:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.930178:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.930179:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.930180:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.930191:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.930193:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.930195:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.930195:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.930199:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.930203:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677446.930271:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.930274:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.930274:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.930276:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.930277:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.930278:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.930640:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.930643:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.930644:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.930646:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.930648:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.930649:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677446.930651:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96735364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.930652:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.930653:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.930655:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677446.930658:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.930660:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677446.930663:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.930664:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677446.930665:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677446.930666:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677446.930667:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.930668:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.930668:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677446.931399:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.931402:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.931403:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.931403:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.931404:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.931407:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.931418:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677446.931420:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677446.931421:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.931422:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677446.931422:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677446.931423:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.931504:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677446.931507:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677446.931507:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677446.931509:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.931511:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.931513:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:24.0:1619677446.931514:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.931515:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.931516:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.931517:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:24.0:1619677446.931520:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.931521:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:24.0:1619677446.931523:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.931524:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:24.0:1619677446.931525:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:24.0:1619677446.931526:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:24.0:1619677446.931527:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677446.931528:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677446.931528:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:8.0:1619677446.932275:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.932278:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677446.932279:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.932280:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.932282:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.932285:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.932299:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677446.932302:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677446.932303:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.932303:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677446.932304:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677446.932305:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677446.932305:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.932308:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.932309:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.932311:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.932313:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.932314:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.932564:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.932567:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.932568:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.932569:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.932573:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.932574:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677446.932575:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.932576:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.932577:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.932579:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677446.932582:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.932584:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677446.932587:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.932587:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677446.932588:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677446.932589:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677446.932590:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.932591:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.932592:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.932593:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.932603:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.932605:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.932606:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.932607:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677446.932625:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.932627:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.932628:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.932628:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.932630:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.932633:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677446.932680:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.932683:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.932684:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.932684:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.932686:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.932689:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677446.932928:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.932931:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.932932:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.932932:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.932934:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.932937:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.932979:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.932982:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.932982:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.932984:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.932986:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.932986:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677446.933064:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.933067:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.933068:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.933070:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.933072:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.933074:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677446.933075:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96735339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.933076:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.933077:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.933078:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:24.0:1619677446.933081:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:22.0:1619677446.933083:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.933084:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.933085:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.933085:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677446.933085:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.933087:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.933088:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.933089:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677446.933089:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:24.0:1619677446.933090:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677446.933091:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677446.933092:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.933092:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.933093:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677446.933121:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.933124:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.933125:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.933128:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.933130:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.933132:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.933133:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.933134:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.933136:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.933136:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.933140:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.933142:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.933145:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.933146:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.933146:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.933148:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.933149:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.933150:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.933150:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677446.933179:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.933182:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.933183:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.933185:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.933187:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.933189:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.933191:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.933192:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.933193:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.933194:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.933197:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.933199:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.933201:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.933202:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.933202:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.933204:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.933205:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.933205:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.933206:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677446.933328:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.933331:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.933332:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:12.0:1619677446.933332:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677446.933332:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.933334:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.933335:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.933335:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:12.0:1619677446.933337:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.933337:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.933339:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.933339:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.933348:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.933351:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677446.933351:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.933352:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.933353:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.933354:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.933400:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.933403:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.933404:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.933404:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.933406:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.933410:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677446.933491:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:22.0:1619677446.933494:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677446.933494:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.933495:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.933495:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.933497:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:8.0:1619677446.933497:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.933498:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.933500:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677446.933501:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:22.0:1619677446.933503:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.933504:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677446.933505:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96735330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.933507:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.933508:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.933509:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677446.933513:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.933516:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677446.933518:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.933519:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677446.933520:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677446.933521:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677446.933522:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.933522:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.933523:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677446.933615:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.933618:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.933619:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.933621:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.933623:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.933625:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677446.933626:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.933629:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.933631:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.933632:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677446.933637:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.933641:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677446.933643:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.933644:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677446.933645:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677446.933646:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.933647:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.933648:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.933648:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.933659:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.933662:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.933662:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.933664:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.933665:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.933666:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.933676:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.933679:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.933680:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.933681:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.933682:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.933683:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.933913:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.933916:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.933917:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.933917:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.933920:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.933923:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.933932:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.933934:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.933935:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.933937:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.933938:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.933939:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.934078:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.934081:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.934082:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.934084:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.934085:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.934087:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.934101:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.934104:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.934104:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.934107:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.934110:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.934112:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677446.934113:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.934115:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.934117:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.934118:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677446.934123:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.934126:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677446.934128:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.934129:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677446.934130:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677446.934131:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677446.934132:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.934133:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.934133:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.934145:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.934148:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.934149:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.934151:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.934152:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.934154:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.934165:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.934167:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.934167:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.934168:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.934168:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.934169:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.934169:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677446.934170:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.934170:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.934171:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677446.934172:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.934174:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.934180:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.934183:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.934184:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.934185:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.934187:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.934189:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677446.934305:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.934307:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.934309:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.934310:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.934313:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.934315:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.934316:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.934317:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.934318:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.934320:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.934322:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.934324:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.934327:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.934328:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.934328:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.934330:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.934330:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.934332:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.934333:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.934333:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.934347:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.934349:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.934350:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.934351:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.934529:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.934531:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.934532:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.934533:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.934535:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.934536:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.934660:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.934662:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.934663:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.934665:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.934666:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.934668:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.934885:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.934888:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.934889:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.934889:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.934891:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677446.934893:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.934996:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.934999:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.935000:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.935002:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.935002:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.935004:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.935026:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.935029:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.935030:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.935030:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.935032:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.935035:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.935069:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.935071:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.935072:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.935074:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.935075:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.935077:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.935085:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.935087:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.935088:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.935089:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.935090:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.935091:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.935115:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.935117:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.935118:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.935121:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.935123:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.935124:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.935126:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.935127:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.935128:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.935129:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.935133:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.935134:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.935137:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.935138:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.935139:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.935140:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.935141:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.935141:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.935142:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677446.935778:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.935781:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.935782:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.935784:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.935786:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.935788:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677446.935789:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.935791:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.935791:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.935793:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677446.935796:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.935798:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677446.935801:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.935802:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677446.935803:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677446.935805:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677446.935805:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.935806:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.935807:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.935807:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.935818:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.935820:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.935821:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.935822:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677446.935996:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.935999:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.936000:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.936001:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.936003:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.936004:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.936397:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.936400:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.936401:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.936404:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.936406:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.936408:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.936409:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96735293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.936411:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.936412:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.936413:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.936417:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.936419:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.936423:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.936424:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.936424:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.936426:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.936427:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.936428:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.936428:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.936937:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.936940:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.936940:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.936942:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.936944:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.936945:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.936954:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.936957:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.936958:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.936959:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.936960:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.936961:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.937105:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.937108:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.937108:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.937109:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.937111:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.937114:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.937123:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.937125:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677446.937126:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.937127:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.937128:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.937128:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.937205:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.937208:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.937208:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.937210:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.937212:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.937213:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.937637:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.937640:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.937640:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.937642:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.937644:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.937645:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.937710:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.937713:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.937713:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677446.937714:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.937716:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.937719:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.937802:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.937805:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.937806:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.937808:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.937811:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.937814:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677446.937816:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.937818:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.937819:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.937821:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677446.937828:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.937832:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677446.937834:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.937835:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677446.937836:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677446.937838:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677446.937839:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.937839:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.937840:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.937851:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.937854:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.937856:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.937858:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677446.937860:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.937864:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.938053:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.938056:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.938057:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.938058:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.938060:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.938060:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.938424:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.938427:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.938427:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.938429:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.938431:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.938432:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.938444:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.938447:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.938447:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.938448:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.938448:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.938449:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.938493:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.938496:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.938497:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.938497:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.938500:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.938502:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.938514:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677446.938517:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677446.938518:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.938518:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.938519:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.938519:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.938520:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.938521:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.938521:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677446.938525:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.938527:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.938529:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677446.938530:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96735271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.938532:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.938533:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.938535:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677446.938538:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.938541:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677446.938544:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.938545:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677446.938546:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677446.938548:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677446.938548:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.938549:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.938550:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677446.938693:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.938696:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.938697:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.938698:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.938700:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.938703:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.938726:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.938729:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.938730:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.938732:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.938733:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.938735:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.938839:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.938842:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.938844:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.938846:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:8.0:1619677446.938847:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677446.938848:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.938849:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.938850:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.938851:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:8.0:1619677446.938852:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677446.938853:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96735266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.938855:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.938856:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.938857:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677446.938860:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677446.938861:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000400:00000200:8.0:1619677446.938861:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.938862:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677446.938863:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.938863:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677446.938864:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.938865:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677446.938866:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.938867:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.938867:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677446.938868:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677446.938870:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.938870:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.938871:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.938873:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.938873:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677446.938883:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.938885:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:24.0:1619677446.938886:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:13.0:1619677446.938886:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.938887:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.938888:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.938888:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677446.938889:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.938890:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.938890:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.939077:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.939080:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.939080:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677446.939082:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.939085:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.939086:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677446.939088:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.939089:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.939090:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.939091:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677446.939094:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.939096:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677446.939099:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.939099:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677446.939100:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677446.939102:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677446.939102:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.939103:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.939103:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677446.939179:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.939182:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.939183:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.939185:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.939185:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.939187:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.939200:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.939202:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.939203:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.939205:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.939206:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.939207:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.939314:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677446.939317:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677446.939319:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677446.939322:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677446.939322:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.939324:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677446.939325:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96735260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677446.939326:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677446.939328:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677446.939328:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677446.939332:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677446.939334:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.939335:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:12.0:1619677446.939337:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.939337:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.939338:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677446.939338:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:12.0:1619677446.939339:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677446.939340:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:8.0:1619677446.939340:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677446.939341:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677446.939342:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677446.939342:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000400:00000200:8.0:1619677446.939343:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.939344:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677446.939350:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96735259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.939351:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.939352:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.939353:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677446.939356:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.939358:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677446.939361:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.939362:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677446.939362:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677446.939364:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677446.939364:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.939365:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.939366:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.939377:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.939379:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.939380:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.939382:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.939383:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.939384:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.939440:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.939442:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677446.939443:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.939444:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.939446:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.939449:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677446.939504:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.939507:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.939508:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.939510:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.939512:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.939514:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.939515:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.939516:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.939517:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.939518:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.939522:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.939523:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.939526:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.939527:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.939528:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.939529:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.939530:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.939531:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.939531:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677446.939635:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.939638:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.939639:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.939642:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.939648:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.939649:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677446.939651:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96735253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.939652:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.939654:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.939655:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677446.939659:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.939661:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677446.939664:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.939665:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677446.939666:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677446.939668:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.939668:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677446.939669:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.939670:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.939671:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.939671:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.939672:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000400:00000200:22.0:1619677446.939673:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.939673:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.939674:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.939676:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.939679:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.939681:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.939682:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.939684:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.939684:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:13.0:1619677446.939685:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677446.939686:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:12.0:1619677446.939687:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.939687:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.939688:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.939697:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.939700:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.939701:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.939702:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.939703:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.939704:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.939841:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677446.939844:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677446.939845:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677446.939847:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.939849:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.939851:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:22.0:1619677446.939852:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.939853:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.939854:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.939855:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677446.939858:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.939860:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677446.939862:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.939863:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:22.0:1619677446.939864:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677446.939866:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677446.939866:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677446.939867:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677446.939867:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:13.0:1619677446.940119:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.940122:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.940123:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.940124:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.940126:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.940127:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.940136:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.940140:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.940140:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.940142:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.940143:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.940144:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.940163:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.940166:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.940167:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.940168:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.940170:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.940171:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.940429:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.940432:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.940434:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.940435:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.940436:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.940438:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.940559:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.940561:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.940563:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.940563:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.940565:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.940568:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677446.940796:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.940799:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.940799:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677446.940802:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.940804:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.940805:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677446.940806:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.940808:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.940809:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.940810:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677446.940813:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.940815:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677446.940818:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.940819:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677446.940819:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677446.940821:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677446.940822:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.940822:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.940823:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677446.940881:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.940883:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677446.940885:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677446.940885:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.940888:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.940890:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.940891:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96735238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.940892:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.940893:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.940894:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677446.940894:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.940896:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:24.0:1619677446.940897:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.940897:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677446.940898:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:22.0:1619677446.940898:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.940900:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.940901:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.940902:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.940902:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677446.940903:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677446.940904:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.940905:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.940906:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.940907:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.940908:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.940920:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.940922:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.940923:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.940924:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.941155:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.941157:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.941158:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.941158:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.941160:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.941163:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677446.941332:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.941334:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.941335:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.941336:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.941338:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677446.941343:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.941344:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677446.941346:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677446.941347:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.941348:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.941348:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.941349:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.941925:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.941928:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.941929:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.941930:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.941932:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.941933:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.942039:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.942041:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677446.942042:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.942043:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.942044:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.942047:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.942361:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.942364:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.942365:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.942367:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.942370:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.942371:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.942372:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.942374:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.942375:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.942376:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.942380:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.942381:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.942384:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.942385:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.942386:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.942387:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.942388:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.942389:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.942389:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.942755:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.942757:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.942758:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.942759:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.942761:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.942763:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.942840:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.942843:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.942844:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.942844:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.942846:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.942848:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.943003:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.943006:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.943006:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677446.943008:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.943011:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.943012:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677446.943013:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.943015:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.943016:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.943017:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677446.943019:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.943021:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677446.943024:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.943024:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677446.943025:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677446.943026:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677446.943027:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.943028:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.943028:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.943523:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.943525:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.943526:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.943527:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.943529:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.943530:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.943540:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.943543:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.943544:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.943545:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.943545:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.943547:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.943572:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.943574:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.943575:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.943575:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.943578:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677446.943580:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677446.943660:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.943663:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.943663:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.943665:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.943667:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.943668:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.943706:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.943709:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.943710:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.943712:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.943715:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.943716:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677446.943717:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.943719:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.943720:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.943721:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677446.943724:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.943726:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677446.943729:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.943729:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677446.943730:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677446.943732:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677446.943732:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.943733:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.943733:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.943971:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.943974:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.943975:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.943976:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.943978:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.943979:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.944144:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.944146:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.944147:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.944148:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.944150:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.944154:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.944169:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.944172:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.944173:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.944174:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.944175:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.944177:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.944374:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677446.944377:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677446.944379:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677446.944381:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.944381:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677446.944383:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:13.0:1619677446.944384:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96735200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677446.944386:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677446.944387:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677446.944388:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:13.0:1619677446.944392:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677446.944394:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677446.944397:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677446.944398:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:13.0:1619677446.944398:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:13.0:1619677446.944400:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:13.0:1619677446.944401:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677446.944402:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677446.944404:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677446.944404:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:8.0:1619677446.944404:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.944409:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.944412:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677446.944416:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677446.944419:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000400:00000200:8.0:1619677446.944419:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.944421:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000400:00000200:8.0:1619677446.944421:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:12.0:1619677446.944422:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677446.944423:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000400:00000200:8.0:1619677446.944424:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96735198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.944427:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.944428:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.944429:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.944431:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.944433:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.944434:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677446.944435:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677446.944436:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.944438:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.944439:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677446.944440:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677446.944441:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96735197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677446.944441:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.944442:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:24.0:1619677446.944443:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.944444:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677446.944444:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:24.0:1619677446.944445:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677446.944446:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.944447:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.944447:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:24.0:1619677446.944449:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.944452:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677446.944455:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.944455:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.944456:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677446.944457:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677446.944459:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677446.944460:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.944460:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.944461:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677446.944475:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.944479:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.944481:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.944482:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677446.944483:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:6.0:1619677446.944484:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.944485:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.944486:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.944486:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677446.944488:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677446.944488:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96735195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.944490:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.944491:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.944492:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677446.944495:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.944497:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:6.0:1619677446.944498:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.944500:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677446.944504:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.944505:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677446.944506:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677446.944507:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677446.944508:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.944510:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.944511:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.944511:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.944522:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.944524:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.944525:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.944526:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677446.944529:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.944531:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.944533:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.944533:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.944535:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.944538:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677446.945004:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.945006:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.945007:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.945010:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.945011:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.945016:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.945033:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.945036:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.945037:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.945038:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.945040:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.945041:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.945055:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.945058:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.945059:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.945060:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.945062:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.945063:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.945073:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.945076:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.945076:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.945077:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.945078:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.945078:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.945232:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.945235:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.945236:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.945238:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.945241:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.945243:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677446.945245:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.945246:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.945247:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.945248:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677446.945251:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.945253:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677446.945256:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.945257:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677446.945257:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677446.945259:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677446.945260:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.945260:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.945261:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677446.945340:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.945343:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.945344:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.945344:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.945346:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.945350:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677446.945438:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.945440:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677446.945441:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.945442:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.945444:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.945446:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677446.945504:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677446.945507:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677446.945508:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.945508:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.945509:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.945510:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677446.945532:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.945535:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.945536:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.945537:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.945540:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.945541:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.945543:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.945544:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.945545:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.945547:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.945554:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.945557:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.945560:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.945561:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.945562:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.945563:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.945564:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.945565:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.945565:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.945646:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.945648:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.945649:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.945651:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.945653:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.945654:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.945663:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.945665:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.945666:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.945667:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.945667:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.945669:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.945763:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.945766:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.945767:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.945767:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.945769:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.945771:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.945848:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.945850:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.945851:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.945851:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.945853:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.945856:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.945869:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.945872:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677446.945872:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.945873:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.945874:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.945874:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677446.946063:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.946066:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.946067:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.946067:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.946070:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.946071:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:24.0:1619677446.946072:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677446.946074:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677446.946077:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677446.946078:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677446.946079:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.946081:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:8.0:1619677446.946082:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000400:00000200:12.0:1619677446.946083:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96735174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677446.946084:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677446.946085:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677446.946085:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677446.946086:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:12.0:1619677446.946087:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.0:1619677446.946087:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.946088:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.946088:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677446.946089:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:12.0:1619677446.946090:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.946091:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.946092:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677446.946092:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:6.0:1619677446.946094:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677446.946095:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677446.946096:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:12.0:1619677446.946096:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:6.0:1619677446.946097:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.946098:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:6.0:1619677446.946098:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:12.0:1619677446.946099:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677446.946099:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000400:00000200:6.0:1619677446.946099:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96735173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677446.946100:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000400:00000200:6.0:1619677446.946101:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.946102:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.946104:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677446.946108:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.946110:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677446.946112:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.946113:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677446.946113:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677446.946115:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677446.946115:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.946116:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.946117:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.946214:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.946217:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.946218:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.946221:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.946221:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.946229:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.946247:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.946250:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.946251:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.946253:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.946254:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.946257:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.946287:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.946289:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.946290:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.946291:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:22.0:1619677446.946292:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.946292:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.946299:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.946301:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.946304:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.946304:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.946305:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.946305:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677446.946633:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.946637:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.946638:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.946640:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.946642:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.946644:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677446.946645:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.946646:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.946647:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.946648:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677446.946652:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.946653:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677446.946656:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.946657:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677446.946657:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677446.946659:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.946659:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.946660:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.946661:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.946672:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.946674:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.946675:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.946677:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.946679:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.946680:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.946723:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.946726:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.946728:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.946728:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:13.0:1619677446.946730:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:22.0:1619677446.946731:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.946733:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.946733:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.946734:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.946735:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.946736:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.946738:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.946813:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.946815:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.946816:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.946818:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.946820:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.946821:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.946896:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.946898:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.946900:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.946902:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.946904:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.946905:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.946907:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.946908:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.946909:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.946910:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.946913:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.946915:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.946918:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.946918:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.946919:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.946921:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.946921:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.946922:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.946923:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.946924:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.946937:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.946938:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.946939:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.946940:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.947047:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.947050:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.947050:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.947053:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.947054:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.947055:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.947081:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.947084:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.947084:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.947086:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.947087:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.947088:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.947102:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.947105:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.947106:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.947107:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.947107:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.947108:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.947395:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.947398:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.947399:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.947401:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.947402:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.947403:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.947413:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.947416:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.947417:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.947418:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.947418:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.947420:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.947455:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.947458:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.947459:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.947459:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.947461:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.947464:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677446.947605:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.947607:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677446.947608:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.947609:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.947610:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.947613:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677446.948281:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.948284:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.948285:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.948285:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.948288:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.948290:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.948313:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.948315:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.948316:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.948318:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.948320:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.948321:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677446.948322:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.948323:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.948324:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.948325:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677446.948328:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.948330:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677446.948332:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.948333:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677446.948334:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677446.948336:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677446.948336:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.948337:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.948337:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677446.948509:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.948512:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.948513:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.948514:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.948517:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.948518:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.948520:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.948521:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.948522:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.948523:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.948527:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.948529:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.948531:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.948532:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.948533:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.948535:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.948535:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.948536:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.948536:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.948882:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.948885:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.948885:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.948887:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.948889:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.948890:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.949143:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.949146:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.949146:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.949148:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.949149:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.949150:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.949159:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.949162:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.949163:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.949163:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.949166:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.949168:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.949197:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.949200:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.949201:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.949202:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.949205:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.949206:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677446.949207:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96735133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.949209:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.949210:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.949211:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:13.0:1619677446.949213:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677446.949214:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677446.949215:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.949216:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.949216:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:13.0:1619677446.949218:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.949219:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677446.949219:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.949220:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677446.949220:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:13.0:1619677446.949221:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.949222:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677446.949222:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.949223:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.949224:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.949225:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.949235:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.949237:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.949237:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.949238:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677446.949298:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.949301:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.949301:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.949303:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.949305:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.949306:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.949317:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.949319:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.949320:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.949321:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.949321:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.949323:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.949495:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.949498:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.949499:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677446.949500:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.949503:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.949511:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677446.949783:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.949786:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.949787:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.949787:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.949789:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.949791:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.949802:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677446.949805:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677446.949805:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.949806:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.949807:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.949807:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.949882:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.949885:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.949886:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.949886:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.949888:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677446.949891:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.949901:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.949904:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.949905:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.949905:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.949906:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:13.0:1619677446.949906:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:22.0:1619677446.949907:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.949909:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.949910:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.949911:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.949913:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.949914:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:13.0:1619677446.949914:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.949917:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.949917:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.949920:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.949922:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.949924:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.949925:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96735122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.949927:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.949928:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.949929:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.949933:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.949935:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.949938:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.949939:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.949940:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.949941:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.949942:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.949943:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.949944:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.949945:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677446.949945:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.949948:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.949948:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.949950:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.949952:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.949953:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.949957:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.949960:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.949960:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.949961:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677446.950244:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.950246:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.950247:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.950249:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677446.950250:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677446.950251:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.950252:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.950253:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.950253:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:13.0:1619677446.950255:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:6.0:1619677446.950255:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96735119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677446.950257:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:6.0:1619677446.950257:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.950257:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677446.950258:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.950259:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:6.0:1619677446.950259:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:12.0:1619677446.950261:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.950262:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.950262:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677446.950264:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.950265:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677446.950265:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:12.0:1619677446.950266:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.950267:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.950268:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677446.950268:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677446.950270:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677446.950271:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.950271:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.950272:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677446.950327:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.950329:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.950331:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.950331:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.950333:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.950336:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.950620:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.950623:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.950623:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677446.950625:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.950628:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.950630:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.950631:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96735114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.950633:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.950634:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.950635:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677446.950638:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:8.0:1619677446.950638:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.950640:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677446.950641:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.950642:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.950642:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:8.0:1619677446.950643:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677446.950644:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.950644:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.950644:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.950646:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.950647:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.950647:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677446.950648:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677446.950648:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.950766:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.950769:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.950770:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.950772:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.950773:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.950779:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.950802:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.950805:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.950806:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.950806:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.950808:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.950812:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.950978:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.950981:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.950982:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.950982:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.950984:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:8.0:1619677446.950987:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677446.950998:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677446.951001:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677446.951002:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.951002:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677446.951003:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677446.951004:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677446.951029:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.951032:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.951033:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.951036:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.951037:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.951040:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.951049:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.951051:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.951052:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.951054:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.951055:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.951056:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.951362:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.951365:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.951366:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.951367:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.951370:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.951372:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.951373:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.951374:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.951375:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.951376:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.951379:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.951381:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.951384:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.951384:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.951385:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.951387:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.951388:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.951389:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.951389:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677446.951443:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.951446:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.951446:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677446.951447:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.951449:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.951452:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.951490:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.951493:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.951494:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.951495:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.951496:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.951498:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.951627:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.951629:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.951631:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.951631:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.951634:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677446.951636:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.951679:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.951682:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.951682:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.951684:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.951686:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.951688:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677446.951689:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.951691:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.951692:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.951693:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677446.951697:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.951700:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677446.951703:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.951703:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677446.951704:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677446.951706:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677446.951706:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.951707:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.951708:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677446.951722:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.951726:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.951726:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.951729:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.951732:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.951733:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:13.0:1619677446.951733:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:24.0:1619677446.951734:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96735096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677446.951735:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:24.0:1619677446.951736:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677446.951736:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.951737:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.951738:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677446.951738:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.951740:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:24.0:1619677446.951741:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.951743:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677446.951744:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.951746:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.951747:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.951748:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.951749:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.951750:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.951751:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.951751:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.952079:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.952081:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.952082:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.952084:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.952087:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.952089:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.952090:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96735093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.952092:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.952093:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.952094:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.952097:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677446.952098:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677446.952099:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677446.952101:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.952102:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.952102:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677446.952104:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.952104:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.952104:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677446.952105:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.952106:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:12.0:1619677446.952107:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.952107:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.952107:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.952108:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677446.952929:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.952932:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.952933:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.952935:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.952937:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.952939:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.952940:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.952941:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.952942:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.952943:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.952946:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.952948:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.952951:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.952951:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.952952:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.952954:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.952955:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.952956:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.952957:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.952958:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.952971:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.952973:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.952974:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.952975:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.953163:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.953166:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.953167:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.953169:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.953171:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.953173:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677446.953174:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.953176:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.953177:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.953177:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677446.953180:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.953182:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677446.953184:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.953185:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677446.953185:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677446.953187:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.953187:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.953188:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.953188:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.953211:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.953213:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.953214:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.953215:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.953218:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.953219:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.953294:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.953302:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677446.953303:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.953304:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.953306:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.953308:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.953316:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677446.953319:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677446.953320:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.953320:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677446.953321:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677446.953322:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677446.953333:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.953336:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.953337:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.953339:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.953341:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.953343:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677446.953344:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96735077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.953346:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.953346:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.953348:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677446.953351:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.953353:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677446.953356:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.953356:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677446.953357:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677446.953359:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677446.953359:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.953360:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.953361:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677446.953451:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.953454:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.953455:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.953455:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.953458:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.953461:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.953847:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.953849:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.953850:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.953851:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.953854:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.953854:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677446.953854:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.953857:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.953858:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.953858:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.953860:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.953862:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677446.953863:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677446.953866:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.953866:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.953867:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.953868:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.953869:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.953872:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677446.953875:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677446.953875:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.953876:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.953877:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.953877:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677446.953916:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.953918:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677446.953919:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.953920:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.953921:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.953924:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677446.954225:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.954227:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.954228:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.954230:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.954232:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.954234:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677446.954235:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.954236:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.954238:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.954238:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677446.954242:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.954243:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677446.954245:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.954246:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677446.954247:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677446.954249:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677446.954249:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.954250:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.954251:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.954252:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.954262:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.954264:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.954264:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.954265:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.954373:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.954375:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.954376:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.954378:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.954379:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.954380:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.954686:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.954689:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.954690:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.954692:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.954694:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.954695:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677446.954697:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.954698:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.954699:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.954700:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677446.954704:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.954706:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677446.954708:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.954709:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677446.954710:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677446.954711:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677446.954712:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.954713:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.954713:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.954855:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.954857:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.954858:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.954860:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.954861:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.954863:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.954883:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.954885:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.954887:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.954888:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.954890:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.954892:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.954893:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96735056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.954895:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.954896:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.954896:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:12.0:1619677446.954899:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:24.0:1619677446.954900:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.954901:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677446.954902:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.954903:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.954904:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677446.954904:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.954905:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.954905:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677446.954906:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.954906:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.954907:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.954908:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.954908:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.954909:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.954990:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.954993:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.954994:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.954995:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.954996:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.954998:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.955013:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.955016:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.955017:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.955017:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.955018:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.955020:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.955132:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.955135:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.955136:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.955138:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.955139:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.955141:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.955261:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.955264:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.955265:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.955266:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.955268:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.955271:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677446.955357:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.955360:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677446.955360:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.955361:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.955363:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.955366:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.955505:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.955507:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.955508:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.955510:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.955512:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.955513:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677446.955514:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.955515:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.955516:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.955517:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677446.955521:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.955522:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677446.955524:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.955525:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677446.955526:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677446.955528:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677446.955528:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.955528:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.955529:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677446.955745:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.955748:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.955749:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.955750:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.955753:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.955755:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677446.955756:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.955757:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.955758:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.955759:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677446.955762:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.955764:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677446.955767:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.955768:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677446.955768:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677446.955770:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677446.955770:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.955772:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.955773:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.955774:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.955787:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.955789:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.955790:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.955791:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.956012:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.956014:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.956015:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.956017:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.956019:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.956019:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.956215:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.956217:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677446.956218:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.956219:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.956221:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.956224:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.956424:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.956427:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.956428:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.956429:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.956431:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.956432:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.956573:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.956575:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.956576:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.956577:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.956579:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.956580:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.956591:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.956594:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.956595:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.956595:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.956596:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.956597:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.956661:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.956663:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.956664:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.956665:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.956667:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.956668:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.956770:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.956773:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.956774:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.956776:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.956780:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.956781:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.956783:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.956784:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.956785:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.956786:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.956789:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.956791:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.956794:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.956795:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.956795:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.956797:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.956797:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.956798:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.956798:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.956828:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.956831:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.956832:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.956833:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.956836:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.956838:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.956839:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.956840:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.956841:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.956842:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.956845:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.956846:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.956849:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.956850:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.956850:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.956852:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.956853:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.956853:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.956854:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677446.957064:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.957067:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.957067:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.957068:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.957070:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.957074:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.957107:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.957110:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.957110:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.957112:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.957115:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.957116:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677446.957117:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.957119:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.957120:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.957121:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677446.957124:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.957126:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677446.957129:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.957129:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677446.957130:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677446.957131:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677446.957132:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.957133:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.957133:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.957190:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.957192:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.957193:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.957195:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.957196:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.957198:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.957264:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.957267:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.957267:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.957269:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.957271:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.957272:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.957497:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:24.0:1619677446.957500:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:6.0:1619677446.957500:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.957501:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.957501:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.957503:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:6.0:1619677446.957503:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.957504:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.957506:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677446.957506:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:24.0:1619677446.957508:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.957510:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.957511:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.957512:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.957513:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.957514:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677446.957514:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.957517:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677446.957517:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:24.0:1619677446.957518:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.957518:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.957519:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.957519:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677446.957521:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.957524:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.957525:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.957526:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.957528:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.957528:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.957529:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.957529:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.957572:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.957575:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.957575:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.957577:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.957579:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.957580:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.957593:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.957596:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.957596:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.957597:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.957598:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.957598:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.957786:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.957789:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.957789:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.957790:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.957792:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.957795:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.957997:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.958000:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.958001:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.958003:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.958004:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.958005:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.958056:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.958058:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.958059:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.958060:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.958062:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677446.958065:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.958163:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.958166:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.958167:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.958169:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.958171:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.958173:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.958175:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96735010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.958176:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.958177:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.958179:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.958183:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.958184:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.958188:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.958189:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.958189:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.958191:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.958191:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.958192:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.958193:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677446.958203:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.958206:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.958207:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.958207:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.958209:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.958212:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.958215:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.958218:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.958219:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.958220:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.958222:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.958223:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.958364:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.958366:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.958367:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.958368:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.958370:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.958371:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.958380:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.958383:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.958383:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.958384:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.958385:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.958385:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.958574:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.958577:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.958578:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.958580:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.958582:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.958584:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677446.958585:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96735006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.958586:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.958587:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.958588:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677446.958591:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.958593:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677446.958596:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.958597:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677446.958597:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677446.958599:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677446.958600:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.958601:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.958602:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.958603:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.958613:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.958615:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.958615:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.958616:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.958882:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.958884:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.958886:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.958888:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.958888:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.958891:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.959249:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.959251:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.959252:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.959252:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.959254:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.959257:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.959351:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.959354:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.959355:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.959357:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.959359:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.959361:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677446.959362:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.959363:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.959364:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.959365:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677446.959368:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.959370:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677446.959373:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.959373:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677446.959374:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677446.959376:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677446.959376:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.959377:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.959377:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677446.959427:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.959430:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.959431:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.959432:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.959434:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.959435:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.959746:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.959749:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.959750:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.959750:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:13.0:1619677446.959751:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.959752:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:12.0:1619677446.959753:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:8.0:1619677446.959753:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.959754:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:13.0:1619677446.959754:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677446.959755:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.959759:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677446.959992:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.959995:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.959995:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677446.959999:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.960001:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.960002:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677446.960004:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96734989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.960006:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.960007:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.960008:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677446.960013:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.960015:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677446.960018:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.960018:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677446.960019:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677446.960021:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677446.960021:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.960022:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.960023:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677446.960031:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.960033:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.960034:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.960036:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.960037:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.960039:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.960179:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:24.0:1619677446.960180:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677446.960181:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.960182:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.960182:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:24.0:1619677446.960183:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.960184:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677446.960185:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:24.0:1619677446.960187:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677446.960188:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:24.0:1619677446.960190:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.960192:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.960193:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.960194:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.960195:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.960196:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:6.0:1619677446.960196:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:24.0:1619677446.960200:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.960200:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (23)++
00000800:00000200:6.0:1619677446.960200:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677446.960201:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (25)--
00000800:00000200:6.0:1619677446.960201:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (23)++
00000800:00000200:24.0:1619677446.960202:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677446.960202:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (25)--
00000800:00000200:6.0:1619677446.960202:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.960203:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.960203:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.960204:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.960205:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.960206:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.960207:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677446.960207:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677446.960208:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.960209:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.960210:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.960212:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.960212:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.960225:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.960227:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.960228:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.960229:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677446.960448:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.960450:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.960451:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.960451:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.960453:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.960456:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.960844:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.960847:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.960848:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.960850:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.960852:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.960854:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677446.960855:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.960857:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.960858:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.960859:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677446.960863:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.960865:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677446.960867:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.960868:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677446.960869:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677446.960871:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677446.960871:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.960872:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.960873:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.961048:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.961050:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.961051:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.961055:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.961055:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.961060:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.961061:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.961063:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.961064:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.961064:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.961065:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.961065:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.961200:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.961202:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.961203:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.961205:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.961206:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.961208:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.961447:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.961450:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.961451:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.961452:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.961454:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.961455:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.961464:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.961466:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.961467:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.961468:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.961468:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.961469:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.961806:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.961808:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677446.961809:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.961810:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.961812:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.961815:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.961822:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677446.961824:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677446.961825:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.961825:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.961826:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.961827:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677446.962181:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.962184:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.962184:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.962185:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.962187:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.962189:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.962191:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:6.0:1619677446.962191:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677446.962192:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.962194:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.962197:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.962198:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:22.0:1619677446.962199:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:24.0:1619677446.962200:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.962201:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.962202:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677446.962202:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677446.962202:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:24.0:1619677446.962203:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677446.962203:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.962204:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.962204:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:24.0:1619677446.962207:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.962208:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.962212:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.962213:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677446.962214:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677446.962216:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.962216:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.962217:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.962218:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677446.962320:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.962323:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.962324:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677446.962325:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.962327:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.962330:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677446.962638:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.962641:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.962641:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677446.962644:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.962646:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.962648:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677446.962649:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.962650:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.962651:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.962652:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677446.962656:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.962658:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677446.962660:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.962661:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677446.962661:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677446.962663:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677446.962664:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.962665:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.962666:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.962666:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.962676:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.962678:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.962678:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.962679:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677446.962724:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.962727:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.962727:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.962731:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.962733:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.962735:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.962736:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96734953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.962738:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.962739:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.962740:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:6.0:1619677446.962742:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000400:00000200:24.0:1619677446.962744:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.962745:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:24.0:1619677446.962746:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677446.962746:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000400:00000200:6.0:1619677446.962746:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.962747:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000400:00000200:22.0:1619677446.962748:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.962748:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.962750:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677446.962750:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:24.0:1619677446.962751:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.962752:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677446.962752:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.962752:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:24.0:1619677446.962753:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677446.962753:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677446.962754:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.962754:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.962755:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.962760:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.962762:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.962763:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.962765:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.962767:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.962768:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.962812:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.962814:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.962815:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.962816:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.962817:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.962820:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.962854:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.962857:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.962858:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.962860:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.962861:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.962863:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.963059:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.963061:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.963062:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.963064:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.963065:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.963067:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.963145:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.963148:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.963149:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.963150:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.963152:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.963154:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.963187:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.963190:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.963191:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.963193:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.963193:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.963196:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.963314:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.963318:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.963318:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.963319:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.963321:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.963326:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.963327:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677446.963329:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.963330:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.963330:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.963330:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.963331:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.963449:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.963451:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.963452:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.963454:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.963456:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.963457:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.963476:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.963479:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.963480:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.963480:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.963482:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.963486:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.963631:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.963634:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.963635:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.963637:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.963638:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.963639:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.963639:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.963641:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.963643:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.963644:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677446.963648:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:8.0:1619677446.963648:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.963649:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677446.963650:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677446.963651:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.963652:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677446.963652:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677446.963653:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.963653:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:8.0:1619677446.963653:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677446.963654:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677446.963654:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677446.963657:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.963657:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677446.963659:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677446.963660:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.963661:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:8.0:1619677446.963661:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677446.963661:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.963662:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677446.963662:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:6.0:1619677446.963663:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.963664:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.963664:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.963665:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:6.0:1619677446.963666:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677446.963667:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.963668:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677446.963675:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.963678:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.963679:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.963680:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:13.0:1619677446.963680:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.963681:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:12.0:1619677446.963681:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.963682:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677446.963683:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.963684:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.963719:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.963721:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.963722:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.963723:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.963725:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.963726:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.963829:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.963831:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.963832:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.963834:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.963835:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.963836:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.963880:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.963883:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.963884:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.963885:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:13.0:1619677446.963886:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.963887:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677446.963888:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.963888:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.963889:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.963889:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:22.0:1619677446.963891:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.963894:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.963902:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.963905:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.963906:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.963907:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.963908:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.963909:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.963953:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.963956:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.963956:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677446.963958:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.963960:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.963962:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677446.963963:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.963964:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.963965:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.963966:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677446.963969:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.963971:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677446.963973:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.963974:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677446.963975:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677446.963976:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677446.963977:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.963977:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.963978:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677446.964132:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.964135:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.964136:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.964138:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.964140:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.964142:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677446.964143:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.964145:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.964146:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.964148:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677446.964151:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.964153:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677446.964155:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.964156:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677446.964157:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677446.964158:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.964159:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.964159:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.964160:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.964170:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.964172:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.964173:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.964175:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.964176:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.964177:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.964180:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.964183:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.964184:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.964184:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.964187:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.964189:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677446.964190:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.964192:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.964193:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.964194:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.964195:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.964196:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.964226:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.964228:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.964229:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.964231:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.964232:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:13.0:1619677446.964233:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.964234:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.964235:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.964236:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.964236:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.964238:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.964241:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.964454:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.964456:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.964457:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.964459:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.964461:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.964462:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.964465:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.964467:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.964469:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.964470:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677446.964472:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:24.0:1619677446.964473:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.964474:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:12.0:1619677446.964474:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:24.0:1619677446.964475:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677446.964475:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.964476:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:24.0:1619677446.964477:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.964478:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677446.964478:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:24.0:1619677446.964479:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:12.0:1619677446.964479:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:24.0:1619677446.964482:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.964483:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.964486:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.964487:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.964487:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.964489:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.964490:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.964490:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.964491:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.964584:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.964588:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.964588:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677446.964592:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.964595:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.964597:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:8.0:1619677446.964599:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.964600:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.964601:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.964603:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:8.0:1619677446.964606:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.964608:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.0:1619677446.964611:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.964612:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:8.0:1619677446.964613:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:8.0:1619677446.964614:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:8.0:1619677446.964615:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.964615:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.964616:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677446.964838:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.964841:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.964842:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.964844:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.964846:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.964848:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677446.964850:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.964851:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.964852:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.964853:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677446.964856:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.964858:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677446.964860:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.964861:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677446.964862:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677446.964863:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677446.964864:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.964865:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.964865:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677446.964978:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.964980:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.964981:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677446.964982:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.964984:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.964987:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.964995:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677446.964997:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677446.964998:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.964999:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.965000:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.965000:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.965073:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.965076:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.965077:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.965077:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.965079:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.965081:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.965156:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.965159:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.965159:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677446.965162:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677446.965163:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.965166:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.965274:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.965277:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.965278:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.965280:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.965282:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.965283:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.965284:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.965286:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.965287:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.965287:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.965291:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.965300:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.965304:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.965305:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.965306:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.965308:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.965309:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.965309:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.965310:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677446.965350:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.965352:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.965353:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.965353:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.965356:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.965358:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.965585:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.965587:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.965588:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.965590:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.965592:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.965594:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677446.965595:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.965596:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.965597:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.965598:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677446.965602:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.965604:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677446.965607:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.965608:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677446.965609:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677446.965611:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677446.965612:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.965612:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.965612:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677446.965672:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.965675:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.965676:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.965678:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.965679:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.965681:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.965691:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.965693:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.965694:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.965695:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.965695:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.965697:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.965982:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.965985:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.965986:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.965988:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.965991:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.965992:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677446.965993:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.965995:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.965996:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.965997:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677446.966001:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.966003:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677446.966006:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.966007:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677446.966008:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677446.966009:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677446.966010:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.966011:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.966012:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.966013:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677446.966023:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:24.0:1619677446.966026:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:22.0:1619677446.966026:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.966027:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.966027:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677446.966028:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.966029:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.966030:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:22.0:1619677446.966030:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.966033:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.966043:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677446.966045:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.966046:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.966047:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.966048:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.966048:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.966289:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.966292:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.966293:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.966298:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.966300:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.966302:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.966449:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.966452:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.966453:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.966454:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.966457:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.966459:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677446.966461:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.966462:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.966464:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.966465:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677446.966470:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.966472:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677446.966475:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.966476:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677446.966477:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677446.966479:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677446.966480:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.966481:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.966482:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.966483:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.966493:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.966495:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.966496:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.966497:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.966502:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.966504:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.966505:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.966507:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.966508:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.966509:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.966609:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.966612:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.966613:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.966615:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.966619:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.966620:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.966623:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.966625:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.966627:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.966628:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.966631:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.966634:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.966637:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.966638:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.966638:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.966640:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.966640:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.966641:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.966642:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677446.966886:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.966889:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.966890:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.966890:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.966892:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.966896:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677446.966917:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.966920:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.966921:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.966922:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.966924:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.966927:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.967506:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.967509:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.967510:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.967510:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.967513:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.967516:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.967541:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.967544:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.967545:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.967545:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.967547:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.967549:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.967602:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.967604:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.967605:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.967607:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.967609:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.967613:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.967799:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.967801:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.967802:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.967804:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.967805:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.967807:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.967871:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.967873:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677446.967874:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.967875:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.967877:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.967880:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677446.967923:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.967926:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.967927:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:12.0:1619677446.967928:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:6.0:1619677446.967928:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.967930:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.967931:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.967932:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.967932:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:12.0:1619677446.967934:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.967935:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.967937:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.967941:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.967944:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.967945:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.967945:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.967946:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.967947:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.967949:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.967952:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.967953:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.967954:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.967955:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.967956:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.967987:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.967990:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.967991:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.967991:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.967993:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.967996:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.968235:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.968238:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.968238:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.968240:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.968243:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.968245:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677446.968246:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.968248:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.968249:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.968250:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677446.968253:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.968255:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677446.968258:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.968259:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677446.968259:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677446.968261:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677446.968262:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.968262:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.968263:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677446.968314:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.968318:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.968321:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.968322:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:24.0:1619677446.968323:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677446.968323:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677446.968327:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.968327:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.968330:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677446.968332:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.968334:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.968335:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.968336:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677446.968342:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.968345:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:8.0:1619677446.968345:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.968348:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.968349:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96734875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677446.968350:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.968351:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:8.0:1619677446.968351:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677446.968352:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:8.0:1619677446.968352:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.968353:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.968354:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677446.968355:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.968356:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.968357:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:8.0:1619677446.968357:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.968358:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.968364:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.968365:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.968366:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.968367:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.968368:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677446.968369:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.968369:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.968370:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.968372:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.968373:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677446.968373:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.968374:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677446.968374:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.968375:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.968376:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.968389:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.968391:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.968392:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.968393:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677446.968498:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.968501:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.968502:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.968503:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.968505:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.968508:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677446.968769:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.968771:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.968772:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.968773:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.968775:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.968777:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677446.968944:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.968947:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.968948:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.968950:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.968953:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.968954:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677446.968955:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.968957:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.968958:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.968959:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677446.968962:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.968964:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677446.968966:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.968967:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677446.968968:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677446.968969:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677446.968970:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.968971:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.968971:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677446.969006:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.969009:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.969010:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.969011:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.969013:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.969014:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.969171:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.969173:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.969174:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.969175:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:24.0:1619677446.969176:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.969177:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.969180:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677446.969181:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677446.969183:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:12.0:1619677446.969183:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677446.969184:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.969185:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.969186:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:12.0:1619677446.969186:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.969186:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.969186:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677446.969187:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.969188:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:12.0:1619677446.969188:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:8.0:1619677446.969189:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.969190:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.969191:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677446.969192:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.969193:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.969195:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.969196:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.969197:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.969201:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.969203:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677446.969204:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.969206:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677446.969206:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677446.969207:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677446.969207:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.969207:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677446.969208:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.969208:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.969209:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.969209:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.969210:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.969210:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.969211:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.969247:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.969250:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.969250:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.969253:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.969253:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.969255:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.969499:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.969502:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.969503:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.969503:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.969505:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.969508:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.969517:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677446.969520:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677446.969520:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.969521:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.969522:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.969522:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.969634:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.969636:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.969637:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.969640:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.969642:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.969644:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677446.969646:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96734854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.969648:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.969650:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.969652:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677446.969659:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.969661:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677446.969664:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.969665:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677446.969667:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (23)++
00000800:00000200:6.0:1619677446.969669:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:6.0:1619677446.969670:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.969671:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.969672:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677446.969891:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.969893:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.969894:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.969896:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.969898:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.969899:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.970035:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.970037:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.970038:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.970039:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.970041:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.970042:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.970260:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.970263:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.970264:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.970266:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.970269:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.970270:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.970271:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.970273:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.970274:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.970274:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.970278:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.970279:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.970282:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.970283:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.970283:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.970285:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.970285:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.970286:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.970287:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677446.970362:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.970364:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.970365:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.970365:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.970368:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677446.970370:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.970386:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.970388:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.970389:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.970389:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.970392:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.970394:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677446.970482:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.970485:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.970486:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.970487:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.970489:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.970490:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.970541:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.970543:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.970544:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.970546:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.970547:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.970549:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.970710:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.970713:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.970714:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.970716:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.970718:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.970720:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677446.970721:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.970722:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.970723:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.970724:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677446.970728:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.970730:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677446.970733:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.970734:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677446.970734:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677446.970736:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677446.970736:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.970738:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.970739:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.970739:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.970751:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.970753:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.970754:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.970755:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677446.970782:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.970784:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.970786:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.970786:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.970788:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.970792:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677446.970794:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.970796:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.970797:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.970799:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.970800:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.970801:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.970813:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.970816:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.970817:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.970817:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.970818:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.970819:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.971132:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.971134:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.971135:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.971137:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677446.971138:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.971139:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.971295:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.971303:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677446.971303:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:13.0:1619677446.971304:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.971305:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.971306:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.971307:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:13.0:1619677446.971307:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.971309:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677446.971309:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.971314:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.971316:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.971317:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.971319:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.971321:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.971322:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.971327:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.971331:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.971338:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.971339:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.971339:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.971341:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.971342:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.971343:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.971344:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677446.971359:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.971362:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.971363:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.971363:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.971366:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.971371:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677446.971372:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677446.971374:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677446.971375:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677446.971376:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677446.971376:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677446.971377:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677446.971763:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.971767:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.971768:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.971769:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.971772:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.971774:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677446.971775:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.971776:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.971777:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.971778:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677446.971782:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.971783:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677446.971786:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.971787:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677446.971787:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677446.971789:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677446.971790:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.971790:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.971791:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677446.971810:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.971813:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.971814:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.971821:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.971823:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.971825:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.971826:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.971828:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.971829:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.971830:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.971834:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.971835:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.971839:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.971839:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.971840:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.971843:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.971843:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.971844:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.971845:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677446.972137:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.972140:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.972141:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.972143:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.972146:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.972147:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677446.972149:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.972150:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.972151:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.972152:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677446.972156:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.972158:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677446.972160:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.972162:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677446.972162:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677446.972164:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677446.972164:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.972165:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.972166:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677446.972255:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.972258:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.972259:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.972259:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.972261:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.972263:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.972303:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.972305:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.972306:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.972307:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.972310:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.972311:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.972312:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.972314:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.972314:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.972315:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.972318:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.972320:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.972323:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.972324:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.972324:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.972326:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.972327:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.972328:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.972329:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.972329:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.972342:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.972344:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.972345:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.972346:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.972649:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.972651:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.972652:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.972654:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.972656:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.972657:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.972918:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.972921:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.972922:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.972924:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.972926:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.972928:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677446.972929:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.972931:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.972932:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.972933:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677446.972935:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.972937:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677446.972940:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.972941:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677446.972941:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677446.972943:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677446.972943:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.972944:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.972944:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677446.972957:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.972960:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.972961:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.972961:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.972964:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.972969:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.972987:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.972990:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.972990:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.972991:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.972993:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.972996:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.973077:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.973079:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.973080:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.973082:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.973083:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.973085:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.973211:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.973214:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.973215:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.973215:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.973217:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.973219:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.973231:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677446.973233:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.973235:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.973235:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677446.973235:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000400:00000200:22.0:1619677446.973236:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.973236:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.973237:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.973238:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.973238:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.973239:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.973241:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.973257:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677446.973259:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677446.973260:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.973260:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.973261:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.973262:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.973301:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.973303:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.973304:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.973305:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.973306:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677446.973309:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677446.973370:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe9286a00] (21)++
00000800:00000200:24.0:1619677446.973375:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe9286a00] (22)--
00000800:00000200:8.0:1619677446.973378:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.60@o2ib3
00000400:00000200:8.0:1619677446.973381:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.60@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.973384:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.973385:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:8.0:1619677446.973387:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.973389:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.973389:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.973391:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:8.0:1619677446.973395:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.60@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.973396:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:8.0:1619677446.973400:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.973400:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:8.0:1619677446.973401:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:8.0:1619677446.973403:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:8.0:1619677446.973403:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe9286a00] (21)++
00000800:00000200:8.0:1619677446.973405:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fe9286a00] (22)++
00000800:00000200:8.0:1619677446.973406:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe9286a00] (23)--
00000800:00000200:8.0:1619677446.973407:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe9286a00] (22)--
00000800:00000200:24.0:1619677446.973424:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe9286a00] (22)++
00000800:00000200:24.0:1619677446.973426:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fe9286a00] (23)--
00000800:00000200:24.0:1619677446.973427:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe9286a00] (22)--
00000800:00000200:24.0:1619677446.973428:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.973429:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe9286a00] (21)--
00000800:00000200:24.0:1619677446.973430:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.973430:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.973432:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.973435:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.973437:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.973438:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.973439:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.973440:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.973441:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.973444:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.973446:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.973448:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.973449:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.973449:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.973451:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.973452:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.973452:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.973454:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.973454:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.973472:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.973474:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.973474:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.973475:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.973510:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.973512:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.973513:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.973515:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.973517:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.973518:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.973530:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.973533:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.973535:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (24)++
00000800:00000200:13.0:1619677446.973536:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677446.973536:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677446.973538:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.973546:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.973548:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.973549:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.973550:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.973552:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.973552:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677446.973554:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.973557:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.973558:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.973560:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.973562:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.973564:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677446.973566:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96734798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.973568:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.973570:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.973571:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677446.973575:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.973578:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677446.973582:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.973582:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677446.973583:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677446.973585:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677446.973586:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.973587:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.973587:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677446.973643:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.973646:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.973647:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.973648:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.973650:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.973651:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.973718:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.973721:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.973722:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.973722:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.973725:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.973728:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.973981:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.973984:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.973984:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.973985:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.973987:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.973989:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.974094:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.974097:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.974098:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.974100:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.974101:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.974103:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.974115:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.974118:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.974119:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.974119:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.974120:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.974122:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.974139:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.974142:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.974143:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.974145:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.974148:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.974149:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677446.974150:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.974152:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.974152:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.974153:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677446.974157:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.974158:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677446.974161:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.974162:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677446.974162:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677446.974164:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677446.974164:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:13.0:1619677446.974164:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:24.0:1619677446.974165:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.974166:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.974166:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.974167:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.974168:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.974170:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.974170:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677446.974177:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.974180:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.974181:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.974182:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.974183:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.974186:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.974195:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677446.974198:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.974199:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.974199:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.974200:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.974201:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.974264:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.974267:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:12.0:1619677446.974267:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:22.0:1619677446.974268:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.974268:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.974270:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.974270:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.974270:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.974273:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.974274:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.974274:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.974276:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.974464:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.974466:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.974467:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.974469:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.974472:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.974473:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677446.974475:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.974476:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.974477:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.974478:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677446.974481:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.974483:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677446.974486:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.974486:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677446.974487:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677446.974489:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677446.974489:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.974490:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.974491:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.974751:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.974753:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.974754:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.974756:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.974758:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.974759:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.974799:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.974802:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.974803:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.974803:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.974805:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.974808:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677446.974903:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.974906:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.974907:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.974908:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.974911:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.974913:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.974915:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.974916:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.974917:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.974918:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.974922:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.974924:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.974928:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.974929:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.974929:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.974931:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.974931:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.974932:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.974932:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677446.974956:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.974960:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.974960:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.974961:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.974963:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.974970:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.975232:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.975234:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.975235:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.975237:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.975238:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.975240:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.975256:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.975258:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.975259:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.975260:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.975262:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.975264:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.975384:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.975387:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.975388:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.975389:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.975391:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.975392:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.975392:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.975393:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.975393:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.975394:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.975394:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.975524:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.975526:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.975527:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.975527:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.975530:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.975532:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.975567:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.975570:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.975571:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.975573:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.975575:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.975576:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677446.975577:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.975579:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.975580:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.975581:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677446.975584:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.975586:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677446.975589:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.975589:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677446.975590:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677446.975592:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677446.975593:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.975594:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.975595:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.975595:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.975606:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.975607:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.975608:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.975609:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.975622:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.975625:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.975626:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.975629:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.975629:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.975632:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.975663:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.975667:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.975667:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.975668:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.975670:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.975672:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.975681:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677446.975684:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.975685:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.975685:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.975687:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.975687:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677446.975729:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.975732:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.975733:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.975735:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.975738:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.975740:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.975741:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96734764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.975742:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.975743:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.975744:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.975748:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.975750:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.975753:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.975754:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.975754:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.975756:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.975756:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.975757:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.975758:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.976009:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.976011:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.976012:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.976013:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.976015:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.976019:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.976082:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.976085:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.976086:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.976087:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.976089:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.976090:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.976228:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.976231:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.976231:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.976232:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.976234:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.976235:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:22.0:1619677446.976237:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677446.976238:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.976239:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.976239:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.976241:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.976244:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677446.976337:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.976340:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.976341:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.976342:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.976344:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.976345:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.976357:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.976359:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.976360:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.976361:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.976361:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.976363:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.976398:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.976400:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.976401:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.976403:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.976405:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.976407:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677446.976408:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.976410:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.976411:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.976412:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677446.976415:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.976417:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677446.976419:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.976420:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677446.976421:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677446.976423:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677446.976424:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.976424:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.976425:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677446.976555:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.976558:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.976558:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.976560:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.976561:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.976562:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.977110:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.977113:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.977113:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.977114:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.977116:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.977119:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.977251:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.977254:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.977255:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.977256:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.977259:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.977260:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677446.977261:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.977263:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.977264:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.977265:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677446.977269:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.977270:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677446.977273:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.977274:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677446.977274:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677446.977276:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677446.977276:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.977277:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.977277:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677446.977315:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.977318:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.977319:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.977321:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.977324:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.977325:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.977327:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.977328:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.977329:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.977330:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.977334:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.977335:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.977338:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.977339:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.977340:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.977341:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.977342:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.977342:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.977343:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.978071:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.978074:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.978075:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.978077:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.978079:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.978080:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.978088:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.978091:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.978092:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.978093:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.978093:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.978094:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.978397:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.978400:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.978400:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.978402:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.978404:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.978405:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677446.978425:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.978428:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.978429:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.978430:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.978433:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.978435:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.978436:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.978437:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.978438:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.978439:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.978442:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.978444:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.978447:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.978447:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.978448:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.978450:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.978451:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.978451:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.978452:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677446.978610:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.978613:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.978614:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.978614:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.978616:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.978619:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677446.978632:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.978635:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.978636:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.978638:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.978639:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.978641:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.978903:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.978905:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.978906:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.978906:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.978908:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.978911:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.978922:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677446.978925:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677446.978926:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.978926:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.978927:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.978928:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.978979:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.978982:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.978983:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.978985:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.978986:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.978988:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.978998:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.979001:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:24.0:1619677446.979002:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:22.0:1619677446.979002:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677446.979004:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:6.0:1619677446.979004:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.979006:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.979006:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:6.0:1619677446.979006:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.979008:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:24.0:1619677446.979009:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:6.0:1619677446.979010:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96734726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.979011:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677446.979012:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:6.0:1619677446.979012:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.979013:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677446.979017:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.979018:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677446.979022:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.979022:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677446.979023:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677446.979025:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677446.979025:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.979026:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.979027:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677446.979252:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.979255:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.979256:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.979256:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.979258:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.979261:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.979322:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.979324:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.979325:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.979326:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.979328:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.979333:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.979363:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.979366:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.979367:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.979369:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.979371:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.979372:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.979379:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.979383:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.979384:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.979384:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.979385:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.979386:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.979462:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.979465:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.979466:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.979467:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.979468:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.979471:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.979481:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677446.979483:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.979484:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.979485:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.979486:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.979486:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677446.979595:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.979598:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.979599:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.979601:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.979604:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.979605:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.979606:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.979608:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.979609:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.979610:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.979613:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.979614:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.979618:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.979618:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.979619:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.979621:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.979621:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.979623:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.979624:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.979624:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.979637:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.979639:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.979640:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.979641:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677446.979874:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.979877:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.979878:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.979880:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.979883:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.979884:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677446.979885:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96734715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.979887:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.979888:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.979889:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677446.979892:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.979894:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677446.979897:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.979897:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677446.979898:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677446.979899:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677446.979900:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.979901:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.979902:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.979903:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.979913:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.979916:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.979917:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677446.979917:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.980209:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.980211:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.980212:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.980213:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.980215:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.980218:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.980250:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.980252:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.980253:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.980255:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.980256:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.980257:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.980320:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.980322:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.980323:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.980325:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.980327:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.980328:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.980428:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.980431:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.980432:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.980434:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.980436:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.980437:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.980438:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.980439:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.980439:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677446.980439:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.980439:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.980440:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.980441:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.980644:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.980647:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.980648:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.980649:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.980652:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.980654:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677446.980655:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.980656:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.980657:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.980658:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677446.980662:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.980663:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677446.980666:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.980666:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677446.980667:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677446.980669:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677446.980669:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.980670:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.980670:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677446.980814:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.980817:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.980818:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.980818:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.980821:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.980824:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677446.980925:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.980928:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.980929:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.980931:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.980934:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.980935:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.980937:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.980938:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.980939:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.980940:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.980943:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.980945:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.980948:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.980949:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.980949:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.980951:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.980951:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.980952:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.980952:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677446.981381:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.981383:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.981384:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.981385:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.981387:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.981391:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.981472:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.981475:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.981476:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.981479:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.981479:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.981482:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.981561:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.981564:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.981565:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.981565:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.981567:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.981571:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.981626:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.981628:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.981629:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.981630:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.981632:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.981633:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.981671:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.981673:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.981674:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.981676:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.981678:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.981680:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677446.981681:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.981683:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.981683:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.981685:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677446.981688:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.981689:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677446.981692:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.981693:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677446.981693:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677446.981695:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677446.981695:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.981696:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.981697:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677446.981810:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.981812:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.981813:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.981815:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.981818:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.981820:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.981821:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734688] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.981823:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.981824:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.981825:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.981828:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.981830:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.981833:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.981833:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.981834:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.981836:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.981836:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.981837:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.981837:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.982069:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.982072:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.982072:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.982075:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.982075:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.982078:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.982110:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.982113:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677446.982114:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.982115:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.982117:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.982123:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.982125:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677446.982127:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677446.982128:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.982129:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.982130:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.982130:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677446.982344:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.982346:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.982347:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.982348:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.982351:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.982352:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.982360:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.982363:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.982364:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.982364:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.982365:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.982366:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.982506:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.982509:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.982509:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677446.982511:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.982514:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.982515:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677446.982516:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.982518:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.982519:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.982520:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677446.982523:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.982525:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677446.982528:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.982529:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677446.982529:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677446.982531:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677446.982531:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.982532:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.982532:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677446.982560:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.982563:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.982564:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.982566:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.982569:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.982570:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677446.982572:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.982574:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.982578:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.982580:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677446.982587:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.982591:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677446.982594:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.982595:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677446.982596:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677446.982598:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.982599:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.982600:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.982602:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.982603:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677446.982610:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.982612:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.982613:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.982614:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.982615:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:22.0:1619677446.982616:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:12.0:1619677446.982617:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677446.982617:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:22.0:1619677446.982618:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:13.0:1619677446.982618:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.982619:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:22.0:1619677446.982620:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.982620:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.982622:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.982625:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677446.982626:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.983039:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.983042:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.983042:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.983044:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.983046:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.983047:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.983125:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.983128:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.983129:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.983130:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.983132:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.983134:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.983135:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.983136:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.983137:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.983138:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.983142:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.983143:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.983146:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.983147:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.983147:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.983149:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.983150:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.983151:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.983151:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.983223:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.983226:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.983227:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.983228:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.983230:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.983231:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.983367:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.983369:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.983370:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.983370:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.983373:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.983376:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677446.983433:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.983436:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.983437:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.983439:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.983442:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.983444:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.983445:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.983447:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.983448:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.983449:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.983452:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.983454:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.983457:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.983457:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.983458:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.983460:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.983460:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.983461:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.983461:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.983630:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.983632:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.983633:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.983634:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.983636:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.983637:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.983746:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.983749:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.983750:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.983750:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.983753:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.983755:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677446.983889:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.983891:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.983892:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.983894:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.983895:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.983897:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677446.984019:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.984022:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.984023:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.984023:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.984025:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.984028:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677446.984277:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.984279:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.984280:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.984281:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.984283:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.984284:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.984296:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.984303:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.984304:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.984305:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.984306:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.984307:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.984318:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.984321:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.984322:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.984326:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.984328:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.984330:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677446.984331:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96734652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.984334:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677446.984335:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000400:00000200:6.0:1619677446.984336:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677446.984337:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000400:00000200:6.0:1619677446.984337:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677446.984340:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.984343:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.984344:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677446.984345:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677446.984346:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96734650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677446.984347:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:22.0:1619677446.984348:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.984349:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.984350:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677446.984353:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.984353:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.984355:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677446.984355:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677446.984355:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677446.984358:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.984358:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677446.984359:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677446.984359:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.984360:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677446.984360:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.984361:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.984362:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677446.984362:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.984363:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677446.984364:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677446.984364:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:6.0:1619677446.984364:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (23)--
00000800:00000200:6.0:1619677446.984365:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.984367:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.984368:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.984369:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.984370:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677446.984370:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.984371:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (23)++
00000800:00000200:22.0:1619677446.984371:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (24)--
00000800:00000200:22.0:1619677446.984372:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.984373:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.984373:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.984374:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.984374:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.984375:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677446.984375:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677446.984475:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.984478:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.984479:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.984480:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.984484:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.984486:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677446.984487:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.984488:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.984489:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.984491:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677446.984494:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.984495:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677446.984498:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.984499:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677446.984499:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677446.984501:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677446.984501:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.984502:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.984503:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677446.984514:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.984517:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.984518:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.984518:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.984520:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.984523:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.984534:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.984536:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.984537:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.984538:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.984539:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.984541:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.984622:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.984625:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.984625:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.984627:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.984629:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.984630:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.984661:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.984664:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.984665:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.984667:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.984668:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.984669:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.984706:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.984708:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.984709:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.984710:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.984712:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.984715:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.984921:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.984924:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677446.984924:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.984925:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.984927:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.984930:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677446.985359:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.985362:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.985363:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.985363:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:6.0:1619677446.985363:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:24.0:1619677446.985366:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:6.0:1619677446.985366:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.985367:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:24.0:1619677446.985369:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677446.985369:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.985369:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.985370:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.985371:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.985372:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677446.985373:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677446.985374:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96734637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.985376:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.985377:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677446.985378:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000400:00000200:6.0:1619677446.985378:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:22.0:1619677446.985381:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677446.985382:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:22.0:1619677446.985382:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677446.985382:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:22.0:1619677446.985383:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677446.985383:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (21)++
00000400:00000200:6.0:1619677446.985383:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.985384:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:22.0:1619677446.985385:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677446.985385:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677446.985386:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.985386:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677446.985387:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.985388:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.985389:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.985389:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.985390:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677446.985390:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677446.985392:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677446.985393:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.985393:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.985394:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677446.985484:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.985487:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.985488:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.985489:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.985491:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.985492:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.985744:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.985747:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.985748:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.985750:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.985751:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.985753:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.985762:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.985765:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.985766:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.985766:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.985767:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.985768:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.985912:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.985914:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.985915:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.985917:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.985918:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.985919:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.985954:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.985956:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.985957:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.985958:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.985960:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.985961:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.986032:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.986035:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.986036:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.986038:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.986040:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.986042:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677446.986043:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96734628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.986044:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.986045:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.986046:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677446.986050:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.986051:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677446.986054:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.986055:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677446.986055:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677446.986057:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677446.986058:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.986058:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.986059:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677446.986122:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.986124:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.986126:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.986127:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.986130:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.986131:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.986133:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.986134:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.986135:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.986136:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.986139:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.986141:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.986144:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.986145:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.986146:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.986148:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.986148:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.986149:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.986151:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.986151:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.986164:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.986166:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.986166:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.986168:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.986264:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.986266:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.986267:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.986268:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677446.986270:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.986271:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.986282:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.986285:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.986286:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.986287:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.986287:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.986289:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.986474:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)++
00000800:00000200:13.0:1619677446.986477:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677446.986478:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.986480:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.986481:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677446.986482:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677446.986483:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677446.986483:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677446.986484:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677446.986485:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.986487:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677446.986524:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.986527:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.986528:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.986528:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.986530:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677446.986532:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677446.986768:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677446.986771:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677446.986772:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.986772:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677446.986774:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.986777:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677446.986828:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.986831:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.986832:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.986835:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.986838:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.986840:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677446.986841:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96734615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.986843:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.986844:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.986845:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677446.986849:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.986850:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677446.986853:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.986854:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677446.986855:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677446.986856:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677446.986856:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.986858:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.986858:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.986859:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.986862:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.986863:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.986865:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.986865:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.986867:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.987007:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.987010:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.987011:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.987013:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.987014:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.987015:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.987108:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.987111:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.987111:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.987113:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.987114:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.987116:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.987127:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.987129:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.987130:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.987131:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.987131:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.987133:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.987210:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.987212:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.987213:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.987215:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.987218:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.987219:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.987220:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.987222:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.987223:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.987224:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.987228:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.987230:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.987234:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.987234:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.987235:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.987237:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.987237:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.987238:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.987239:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677446.987262:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.987265:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.987266:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.987266:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.987268:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.987270:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.987325:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.987328:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.987329:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.987329:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.987331:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.987337:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.987338:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677446.987341:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677446.987342:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.987342:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.987343:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.987344:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677446.987721:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677446.987723:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:6.0:1619677446.987723:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677446.987724:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677446.987726:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677446.987726:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.987727:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.987727:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.987728:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.987730:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:24.0:1619677446.987731:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677446.987732:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677446.987733:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:24.0:1619677446.987734:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.987735:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.987736:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677446.987739:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.987739:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677446.987741:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677446.987742:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.987742:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677446.987744:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.987745:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:24.0:1619677446.987745:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:22.0:1619677446.987745:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677446.987747:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:24.0:1619677446.987747:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000400:00000200:22.0:1619677446.987747:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677446.987748:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677446.987749:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000400:00000200:22.0:1619677446.987749:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677446.987750:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96734603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.987751:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.987753:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.987753:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677446.987757:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.987759:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677446.987761:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.987762:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677446.987762:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677446.987764:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677446.987765:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.987766:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.987767:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.987768:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.987780:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.987782:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.987783:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.987784:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.987868:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.987870:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.987871:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.987873:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.987875:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.987875:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.987886:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.987888:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.987888:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.987890:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.987892:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.987892:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.987983:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.987985:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.987987:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.987988:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.987991:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.987992:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.987994:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.987995:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.987996:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.987997:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.988000:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.988002:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.988005:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.988006:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.988006:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.988008:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.988008:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.988009:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.988010:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.988128:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.988131:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.988132:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.988134:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.988135:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.988136:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.988387:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.988390:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.988391:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.988391:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.988393:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.988396:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.988408:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.988410:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.988412:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.988412:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.988414:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.988416:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677446.988646:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.988649:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.988649:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.988651:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.988653:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.988654:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.988851:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.988854:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677446.988854:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.988855:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.988857:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.988860:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.988994:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.988996:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.988997:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.988997:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.988999:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.989001:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677446.989013:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677446.989016:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677446.989017:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.989017:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.989018:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.989018:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677446.989220:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677446.989223:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000400:00000200:24.0:1619677446.989226:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.989229:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.989229:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:6.0:1619677446.989230:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:24.0:1619677446.989231:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.989232:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96734582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677446.989232:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:24.0:1619677446.989233:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677446.989233:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:24.0:1619677446.989234:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.989234:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.989236:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677446.989237:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:24.0:1619677446.989240:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.989240:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677446.989243:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.989246:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.989247:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.989247:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.989249:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.989250:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677446.989250:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677446.989251:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:22.0:1619677446.989328:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.989330:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.989332:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.989332:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.989334:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.989336:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677446.989346:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677446.989348:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.989349:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.989349:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.989350:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.989350:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.989370:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.989373:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.989373:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.989375:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.989377:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.989379:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677446.989380:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.989382:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.989383:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.989384:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677446.989387:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.989389:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677446.989392:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.989393:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677446.989393:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677446.989395:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677446.989396:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.989396:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.989397:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677446.989500:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.989503:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.989504:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677446.989506:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.989509:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.989510:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.989511:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.989513:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.989514:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.989515:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.989519:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.989520:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.989523:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.989524:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.989524:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.989526:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.989526:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.989527:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.989527:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677446.989579:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.989581:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.989582:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.989584:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.989586:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.989587:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.989596:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.989599:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.989600:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.989601:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.989601:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.989603:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.989776:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.989779:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.989779:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.989780:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.989782:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.989785:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677446.989840:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.989842:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.989843:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.989845:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.989847:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.989847:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677446.990164:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.990167:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.990168:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.990168:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.990170:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.990173:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677446.990197:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.990200:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.990201:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.990203:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.990203:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:13.0:1619677446.990204:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.990205:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.990206:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.990207:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.990209:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.990211:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.990213:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.990214:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.990215:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.990216:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677446.990217:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:8.0:1619677446.990217:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:13.0:1619677446.990220:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677446.990220:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.990221:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:8.0:1619677446.990221:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677446.990222:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.990223:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.990223:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.990225:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.990226:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.990227:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.990229:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.990229:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.990230:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.990231:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677446.990749:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.990752:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.990753:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.990753:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.990755:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.990758:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.990770:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677446.990773:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.990773:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677446.990774:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.990775:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677446.990775:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677446.990814:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.990817:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.990818:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.990820:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.990821:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.990823:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.990825:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.990828:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677446.990829:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.990830:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.990832:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.990836:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677446.990848:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.990850:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.990851:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677446.990851:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.990853:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.990855:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.990884:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.990886:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.990887:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.990889:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.990891:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.990893:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677446.990894:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.990895:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.990896:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.990898:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677446.990902:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.990904:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677446.990907:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.990908:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677446.990908:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677446.990910:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677446.990911:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.990911:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.990912:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.990932:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.990935:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.990936:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.990938:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.990939:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.990940:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.991138:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.991141:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677446.991141:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.991142:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.991144:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.991146:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677446.991245:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.991248:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.991249:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.991251:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.991253:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.991255:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.991256:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.991257:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.991259:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.991260:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.991263:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.991264:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.991267:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.991268:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.991269:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.991271:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.991271:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.991272:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.991274:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.991274:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.991287:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.991289:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.991290:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.991291:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677446.991552:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.991554:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.991555:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.991557:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.991558:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677446.991559:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.991560:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.991561:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.991562:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.991564:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.991567:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.991568:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677446.991569:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96734552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.991570:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.991571:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.991572:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677446.991576:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.991578:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677446.991581:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.991581:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677446.991582:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677446.991584:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677446.991584:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.991585:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.991586:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677446.991639:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.991641:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677446.991642:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.991643:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.991645:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.991648:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.991673:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.991676:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.991677:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.991678:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.991679:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.991682:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.991947:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.991950:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.991950:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.991952:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.991954:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.991955:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.991968:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.991970:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.991971:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.991972:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.991974:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.991975:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.991976:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.991976:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.991976:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.991977:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.991978:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.992057:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.992061:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.992066:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677446.992068:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.992071:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.992073:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677446.992075:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.992077:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.992078:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.992079:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677446.992083:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.992085:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677446.992088:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.992088:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677446.992089:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677446.992091:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677446.992091:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677446.992091:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.0:1619677446.992092:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.992094:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.992094:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677446.992094:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.992095:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.992095:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.992097:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:6.0:1619677446.992099:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677446.992107:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.992109:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.992110:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.992110:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677446.992129:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.992132:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.992132:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.992134:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.992137:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.992139:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677446.992140:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.992142:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.992143:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.992144:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677446.992148:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.992149:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677446.992153:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.992153:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677446.992154:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677446.992156:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677446.992157:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.992158:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.992159:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.992160:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.992170:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.992172:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.992172:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.992173:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677446.992215:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.992217:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.992218:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.992220:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.992223:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.992225:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677446.992226:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.992227:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.992228:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.992229:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.992233:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.992235:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.992237:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.992238:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677446.992239:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.992241:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677446.992242:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.992242:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.992243:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.992421:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.992423:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.992424:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.992426:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.992428:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.992429:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.992494:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.992497:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.992498:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.992498:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.992501:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.992504:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.992823:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.992826:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.992826:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.992828:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.992830:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.992831:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677446.993286:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.993289:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.993290:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.993292:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.993293:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.993299:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.993354:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.993357:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677446.993358:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.993360:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.993362:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.993364:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677446.993366:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.993367:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.993368:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.993370:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677446.993375:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.993377:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677446.993380:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.993381:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677446.993381:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677446.993383:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677446.993383:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.993384:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.993385:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677446.993491:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.993494:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.993495:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.993497:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.993498:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.993500:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.993504:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.993507:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677446.993508:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677446.993508:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677446.993511:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677446.993511:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677446.993512:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.993513:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677446.993514:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.993515:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677446.993515:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.993516:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677446.993518:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.993520:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.993521:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.993522:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677446.993526:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.993529:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677446.993533:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.993533:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677446.993534:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677446.993536:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677446.993537:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677446.993537:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.993538:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.993547:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.993549:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.993550:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.993551:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.993554:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.993554:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.993755:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.993758:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.993759:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677446.993761:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.993764:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.993765:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677446.993767:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.993768:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.993769:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.993770:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677446.993773:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.993775:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677446.993777:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.993778:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677446.993778:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677446.993780:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677446.993780:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.993781:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677446.993781:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677446.993795:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.993797:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.993798:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.993799:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.993801:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.993802:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.993940:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.993943:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.993944:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.993944:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.993946:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.993949:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677446.993958:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677446.993961:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677446.993962:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.993962:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.993963:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677446.993963:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.994030:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.994033:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.994033:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.994035:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.994036:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.994037:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.994050:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677446.994052:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677446.994053:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.994054:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677446.994054:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.994055:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677446.994112:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.994115:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677446.994115:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.994116:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.994118:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.994120:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677446.994397:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677446.994400:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677446.994401:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677446.994401:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.994403:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.994406:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677446.994716:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.994718:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.994719:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.994719:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.994722:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.994724:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677446.995101:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.995104:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.995105:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.995106:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.995109:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.995110:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.995112:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.995114:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.995115:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.995116:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.995121:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.995122:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.995125:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.995126:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.995127:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.995129:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.995129:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.995130:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.995130:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.995399:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.995401:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.995402:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.995403:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.995405:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.995407:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677446.995552:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.995555:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677446.995556:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677446.995559:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677446.995561:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677446.995563:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677446.995564:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677446.995565:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677446.995567:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677446.995567:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677446.995571:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677446.995573:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677446.995576:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677446.995577:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677446.995577:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677446.995579:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677446.995580:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.995581:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.995581:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677446.995917:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.995919:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.995920:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.995922:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.995925:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.995927:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:8.0:1619677446.995928:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.995929:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.995930:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.995931:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:8.0:1619677446.995935:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.995936:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.0:1619677446.995939:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.995940:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:8.0:1619677446.995940:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:8.0:1619677446.995941:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:8.0:1619677446.995942:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.995943:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.995943:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677446.996094:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677446.996097:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677446.996098:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677446.996098:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677446.996101:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677446.996104:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677446.996206:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677446.996209:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677446.996210:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677446.996211:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.996213:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.996216:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677446.996224:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677446.996227:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677446.996228:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677446.996228:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.996229:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677446.996230:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677446.996498:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677446.996501:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677446.996502:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677446.996502:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.996504:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677446.996507:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677446.996585:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.996588:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.996588:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677446.996592:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.996595:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.996597:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677446.996599:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.996601:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.996602:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.996604:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677446.996609:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.996612:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677446.996615:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.996616:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677446.996617:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677446.996619:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677446.996620:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.996621:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.996621:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677446.996663:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.996666:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677446.996667:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.996667:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677446.996669:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.996672:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677446.996749:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.996751:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.996752:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.996754:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.996756:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.996757:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677446.996905:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.996908:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.996909:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.996911:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677446.996911:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.996914:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.996932:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677446.996935:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677446.996936:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.996936:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677446.996937:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.996938:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677446.996957:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677446.996960:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677446.996961:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:6.0:1619677446.996961:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:24.0:1619677446.996963:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677446.996963:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:24.0:1619677446.996965:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.996965:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.996965:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.996967:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:6.0:1619677446.996968:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:24.0:1619677446.996969:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.996970:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.996971:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677446.996971:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:24.0:1619677446.996972:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677446.996976:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.996978:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677446.996981:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.996981:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:22.0:1619677446.996981:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:24.0:1619677446.996982:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677446.996984:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677446.996984:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677446.996984:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:24.0:1619677446.996985:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:22.0:1619677446.996985:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.996986:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.996986:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677446.996987:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677446.996988:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677446.996989:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.997001:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677446.997004:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677446.997005:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.997006:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677446.997254:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.997258:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.997258:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677446.997261:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.997263:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.997265:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677446.997267:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.997269:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.997270:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.997271:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677446.997276:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.997278:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677446.997281:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.997282:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677446.997283:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677446.997284:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677446.997285:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.997286:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677446.997287:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677446.997287:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.997299:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677446.997301:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677446.997302:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677446.997303:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677446.997331:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.997334:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.997335:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.997337:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.997340:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.997341:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.997342:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.997344:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.997345:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.997346:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.997351:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.997353:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.997356:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.997358:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.997358:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.997360:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.997361:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.997362:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677446.997362:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677446.997557:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677446.997559:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677446.997560:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.997562:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.997563:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677446.997564:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.997695:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.997697:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.997698:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.997700:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.997701:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.997703:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677446.997899:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677446.997902:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.997902:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677446.997905:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677446.997907:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.997908:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677446.997910:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96734463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.997912:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.997913:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.997914:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677446.997918:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.997919:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677446.997922:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.997923:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677446.997923:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677446.997925:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677446.997925:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.997927:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.997928:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677446.998090:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.998092:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.998093:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.998094:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.998096:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.998097:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.998120:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677446.998122:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677446.998123:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.998124:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.998126:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677446.998127:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677446.998377:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.998380:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677446.998381:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677446.998383:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677446.998385:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677446.998387:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677446.998388:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677446.998389:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677446.998390:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677446.998391:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677446.998394:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677446.998396:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677446.998399:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677446.998400:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677446.998400:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677446.998403:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677446.998403:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677446.998404:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677446.998404:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677446.998779:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677446.998781:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677446.998781:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:22.0:1619677446.998782:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.998783:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:22.0:1619677446.998784:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677446.998785:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677446.998787:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677446.998788:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:6.0:1619677446.998788:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677446.998789:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677446.998790:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677446.998792:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677446.998793:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677446.998795:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677446.998795:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677446.998800:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677446.998803:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677446.998805:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677446.998806:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677446.998807:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677446.998808:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677446.998809:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677446.998810:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677446.998810:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677446.998824:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677446.998826:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677446.998827:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677446.998827:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677446.998829:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677446.998833:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677446.999131:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.999134:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677446.999135:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677446.999137:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677446.999139:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677446.999141:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677446.999142:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677446.999144:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677446.999145:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677446.999146:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677446.999150:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677446.999152:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677446.999155:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677446.999155:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677446.999156:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677446.999158:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677446.999158:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677446.999159:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677446.999160:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677446.999161:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.999178:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677446.999180:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677446.999180:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677446.999182:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677446.999225:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677446.999228:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677446.999229:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677446.999230:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677446.999232:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677446.999235:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677446.999299:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677446.999302:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677446.999303:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677446.999305:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677446.999306:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677446.999307:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677446.999609:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.999611:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677446.999612:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677446.999613:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677446.999616:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677446.999616:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.999632:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677446.999635:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677446.999636:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677446.999637:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677446.999637:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677446.999638:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.000006:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.000008:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.000009:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.000011:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.000012:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.000013:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.000027:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.000030:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.000031:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.000032:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.000032:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.000033:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.000039:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.000043:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.000043:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.000045:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.000048:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.000049:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.000050:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96734439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.000052:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.000053:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.000054:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.000057:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.000059:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.000062:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.000063:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.000064:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.000066:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.000066:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.000067:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.000068:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.000536:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.000538:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.000539:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.000540:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.000541:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677447.000543:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.000543:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.000544:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.000545:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.000547:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.000550:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.000551:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.000552:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.000555:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.000555:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.000556:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.000560:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.000562:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.000564:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.000565:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.000566:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.000567:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.000568:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.000569:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.000569:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.000717:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.000719:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.000720:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.000723:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.000726:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.000728:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.000730:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.000731:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.000732:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.000733:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.000737:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.000739:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.000741:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.000742:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.000743:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.000744:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.000744:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.000745:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.000746:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.000752:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.000755:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.000756:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:13.0:1619677447.000756:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677447.000756:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.000759:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.000759:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.000759:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:13.0:1619677447.000761:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.000762:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677447.000762:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.000764:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.000915:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.000917:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.000918:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.000919:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.000921:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.000926:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.000927:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.000929:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.000930:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.000931:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.000931:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.000932:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.001127:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.001130:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.001131:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.001131:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.001133:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.001135:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.001194:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.001197:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.001197:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.001199:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.001201:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.001201:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.001391:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.001394:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.001395:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.001396:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.001398:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.001404:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.001405:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.001407:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.001408:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.001408:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.001410:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.001410:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677447.001425:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.001428:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.001428:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.001429:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.001431:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.001435:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.001472:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.001475:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.001476:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.001477:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.001480:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.001481:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677447.001483:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.001484:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.001485:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.001486:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677447.001489:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.001490:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.001493:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.001494:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677447.001494:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677447.001496:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.001496:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.001497:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.001498:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.001509:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.001512:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.001513:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.001514:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.001516:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.001519:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.001532:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.001535:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.001535:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.001536:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.001537:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.001537:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.001690:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.001693:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.001694:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.001694:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.001696:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.001699:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.001715:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.001718:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.001718:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.001720:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.001723:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.001725:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.001726:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.001727:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.001728:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.001730:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.001733:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.001735:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.001738:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.001739:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.001740:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.001742:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.001742:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.001744:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.001745:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.001745:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.001755:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.001757:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.001758:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.001759:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.001844:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.001847:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.001848:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.001848:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.001850:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.001852:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.001886:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.001889:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.001889:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.001890:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.001892:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.001894:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.001953:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.001955:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.001957:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.001957:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.001959:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.001962:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677447.002145:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.002148:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.002149:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.002149:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.002152:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.002155:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677447.002171:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.002174:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.002174:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.002175:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.002177:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.002180:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.002253:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.002255:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.002256:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.002256:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.002259:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.002261:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.002461:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.002463:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.002465:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.002465:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.002467:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.002470:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677447.002506:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.002509:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.002509:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.002511:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.002513:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.002514:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.002562:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.002565:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.002566:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.002568:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.002571:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.002573:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.002574:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96734402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.002576:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.002578:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.002579:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.002584:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.002587:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.002590:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.002591:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.002591:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.002593:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.002594:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.002595:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.002595:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.002659:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:6.0:1619677447.002661:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677447.002662:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.002663:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.002663:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.002663:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.002664:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.002664:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.002666:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.002667:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:8.0:1619677447.002669:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677447.002670:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.002685:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.002688:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.002689:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.002690:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.002693:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.002694:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.002695:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.002697:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.002697:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.002698:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.002702:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.002703:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.002706:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.002707:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.002707:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.002709:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.002709:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.002710:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.002711:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.003087:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.003090:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.003091:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.003091:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.003093:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.003097:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.003106:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.003109:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.003110:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.003111:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.003112:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.003112:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.003378:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.003381:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.003381:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.003383:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.003385:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.003386:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.003395:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.003397:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.003398:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.003399:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.003399:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.003400:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.003586:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.003589:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.003590:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.003593:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.003596:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.003597:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.003599:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.003600:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.003601:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.003602:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.003605:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.003607:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.003610:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.003611:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.003611:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.003613:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.003614:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.003614:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.003615:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.003676:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.003678:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.003679:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.003682:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.003682:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.003684:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.003707:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.003710:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.003711:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.003713:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.003716:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.003717:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.003719:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.003720:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.003721:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.003722:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.003725:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.003727:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.003730:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.003731:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.003731:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.003733:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.003733:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.003734:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.003734:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.004194:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.004197:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.004198:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.004198:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.004200:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.004202:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.004232:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.004234:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.004235:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.004237:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.004239:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.004241:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.004242:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.004243:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.004244:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.004245:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.004248:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.004250:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677447.004252:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:22.0:1619677447.004253:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.004254:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.004254:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:24.0:1619677447.004255:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:22.0:1619677447.004256:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.004256:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677447.004256:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677447.004257:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:24.0:1619677447.004258:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.004258:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:24.0:1619677447.004261:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.004262:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.004264:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.004265:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.004266:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.004268:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.004271:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.004273:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.004277:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.004278:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.004279:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.004281:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.004281:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.004283:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.004284:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.004285:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.004305:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.004307:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.004308:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.004310:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.004641:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.004643:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.004644:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.004646:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.004648:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.004649:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.004787:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.004789:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.004790:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.004790:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.004793:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.004795:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.004856:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.004856:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:24.0:1619677447.004860:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:22.0:1619677447.004860:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:8.0:1619677447.004860:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677447.004860:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:24.0:1619677447.004864:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.004864:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.004866:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.004867:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.004868:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.004869:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.004870:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.004871:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.004872:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.004873:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.004874:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.004875:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.004875:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.004876:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.004879:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.004881:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.004883:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.004884:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.004885:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:22.0:1619677447.004885:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.004886:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.004887:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.004888:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.004888:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:24.0:1619677447.004889:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.004889:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:24.0:1619677447.004890:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.004890:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:24.0:1619677447.004891:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.004891:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:24.0:1619677447.004892:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.004893:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.004895:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.004896:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677447.004901:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000800:00000200:6.0:1619677447.004903:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:6.0:1619677447.004904:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677447.004905:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:8.0:1619677447.004908:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.004910:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.004911:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.004913:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.005253:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.005256:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.005257:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.005259:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.005261:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.005263:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.005264:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.005265:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.005266:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.005268:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.005273:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.005276:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.005279:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.005280:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.005281:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.005283:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.005283:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.005284:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.005285:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.005286:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.005299:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.005302:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.005302:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.005304:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.005411:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.005414:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.005414:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.005416:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677447.005418:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.005419:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.005667:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.005670:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.005671:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.005672:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.005674:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.005677:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.005687:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.005690:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.005691:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.005691:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.005692:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.005692:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.006068:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.006070:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.006071:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.006073:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.006074:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.006076:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.006146:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.006149:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.006150:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.006152:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.006154:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.006156:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677447.006158:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.006160:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.006160:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.006161:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677447.006165:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.006167:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677447.006170:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.006170:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677447.006171:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677447.006173:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677447.006173:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.006174:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.006174:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.006187:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.006190:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.006191:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.006192:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.006194:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.006197:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.006224:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677447.006225:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.0:1619677447.006227:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.006228:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677447.006228:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.006229:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:8.0:1619677447.006229:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.006229:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.006232:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.006232:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.006234:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.006235:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.006235:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:8.0:1619677447.006237:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.006238:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.006239:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.006242:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.006244:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.006246:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.006247:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.006248:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.006249:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.006250:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.006250:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.006251:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.006398:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.006401:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.006402:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.006405:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.006406:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.006410:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.006451:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.006454:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.006455:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.006455:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.006457:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.006460:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.006782:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.006785:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.006786:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.006787:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.006790:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.006791:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.006793:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.006794:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.006795:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.006796:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.006799:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.006801:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.006803:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.006804:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.006804:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.006806:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.006806:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.006807:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.006808:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.006818:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.006820:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.006821:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.006823:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.006824:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.006829:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.006838:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.006840:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.006841:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.006842:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.006842:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.006843:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.006845:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.006847:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.006848:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.006849:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.006852:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.006853:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.006866:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.006869:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.006870:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.006870:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.006871:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.006871:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.007017:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.007020:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.007021:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.007021:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.007024:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.007027:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677447.007039:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.007042:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677447.007042:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.007043:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.007044:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.007045:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677447.007082:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.007084:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.007085:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.007085:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.007088:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.007091:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677447.007098:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.007100:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.007101:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.007102:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.007103:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.007106:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.007228:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.007231:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.007232:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.007233:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.007235:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.007238:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.007248:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.007251:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.007252:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000400:00000200:6.0:1619677447.007252:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.007253:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.007254:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (23)++
00000400:00000200:6.0:1619677447.007254:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.007256:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (24)--
00000800:00000200:22.0:1619677447.007256:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.007257:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.007258:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.007258:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.007260:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.007260:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.007262:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.007264:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.007266:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.007267:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.007268:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.007269:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.007270:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.007273:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.007275:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.007277:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.007278:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.007279:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.007281:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.007281:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.007282:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.007282:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.007416:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.007418:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.007419:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.007420:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.007422:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.007423:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.007479:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.007481:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.007483:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.007484:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.007486:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.007487:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.007488:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.007489:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.007490:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.007491:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.007494:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.007495:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.007498:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.007498:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.007499:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.007500:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.007501:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.007501:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.007502:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.007512:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.007514:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.007515:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.007517:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.007518:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.007520:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.007595:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677447.007598:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677447.007600:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.007601:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677447.007603:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.007605:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:13.0:1619677447.007607:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96734334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.007609:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677447.007610:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677447.007612:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:13.0:1619677447.007616:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.007619:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:13.0:1619677447.007621:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.007622:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:13.0:1619677447.007622:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:13.0:1619677447.007624:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:13.0:1619677447.007625:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677447.007626:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677447.007626:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677447.007816:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.007819:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.007820:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.007821:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.007823:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.007824:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.007876:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.007879:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.007880:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.007880:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.007882:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.007885:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677447.007893:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.007895:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.007896:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.007896:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.007898:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.007900:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.007962:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.007965:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.007966:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.007968:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.007969:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.007971:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.007999:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.008001:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.008002:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.008003:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.008005:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.008011:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.008012:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.008014:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.008015:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.008015:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.008016:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.008017:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.008058:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.008060:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.008062:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.008063:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.008066:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.008068:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.008070:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.008072:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.008073:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.008074:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.008077:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.008079:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.008082:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.008083:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.008083:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.008085:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.008086:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.008086:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.008087:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.008104:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.008107:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.008108:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.008109:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.008112:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.008113:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.008114:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.008115:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.008117:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.008117:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.008121:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.008122:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.008125:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.008126:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.008127:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.008128:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.008129:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.008129:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.008130:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.008367:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.008369:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.008370:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.008371:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.008372:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677447.008373:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.008375:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.008375:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.008375:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:12.0:1619677447.008377:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.008379:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.008380:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.008388:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.008391:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.008392:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.008393:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.008393:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.008395:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.008481:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.008484:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.008485:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.008485:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.008487:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.008490:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.008682:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.008685:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.008686:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.008687:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.008689:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.008692:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.008911:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.008913:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.008915:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.008916:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.008919:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.008921:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.008922:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.008923:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.008925:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.008926:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.008929:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.008931:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.008934:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.008935:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.008936:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.008938:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.008938:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.008939:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.008940:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677447.008964:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.008967:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.008968:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.008969:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.008971:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.008974:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677447.008984:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.008987:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.008988:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.008988:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.008989:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.008989:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677447.009043:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.009045:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.009046:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.009048:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.009050:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.009051:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.009052:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.009054:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.009055:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.009056:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.009060:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.009061:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.009064:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.009064:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.009065:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.009067:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.009067:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.009068:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.009069:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.009334:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.009337:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.009338:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.009340:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.009342:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.009345:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.009564:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677447.009567:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677447.009568:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677447.009570:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677447.009572:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.009573:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677447.009575:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.009576:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.009577:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.009578:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677447.009581:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677447.009583:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677447.009586:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.009587:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677447.009588:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677447.009590:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677447.009591:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677447.009591:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677447.009592:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:6.0:1619677447.009604:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.009607:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.009608:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.009608:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.009610:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.009613:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.009623:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.009625:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.009627:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:13.0:1619677447.009627:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:6.0:1619677447.009627:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.009628:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.009628:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.009630:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.009630:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.009633:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.009634:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.009635:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.009659:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.009662:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.009663:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.009663:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.009665:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.009668:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.009723:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.009725:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.009726:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.009728:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.009730:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.009732:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.009733:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.009734:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.009735:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.009736:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.009740:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.009741:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.009744:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.009745:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.009745:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.009747:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.009748:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.009749:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.009750:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.009750:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677447.009800:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.009802:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.009803:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.009806:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.009808:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.009810:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.009811:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.009813:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.009814:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.009815:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.009818:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.009820:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.009823:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.009824:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.009824:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.009826:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.009827:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.009828:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.009829:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.009830:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.009843:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.009844:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.009845:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.009846:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.010097:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.010100:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.010101:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.010101:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677447.010337:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.010339:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.010340:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.010342:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.010343:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.010344:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.010918:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.010920:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.010921:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.010923:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.010931:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.010933:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.010934:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.010935:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.010937:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.010938:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.010942:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.010944:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.010947:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.010948:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.010948:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.010950:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.010950:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.010951:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.010952:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.010962:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.010964:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.010965:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.010967:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.010970:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.010971:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.010972:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96734289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.010974:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.010975:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.010976:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.010979:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.010981:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.010984:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.010984:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.010985:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.010987:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.010988:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.010988:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.010989:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.011155:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.011158:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.011159:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.011159:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.011162:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.011166:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.011272:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.011274:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.011275:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.011277:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.011278:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.011279:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.011555:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.011558:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.011559:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.011559:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.011563:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.011565:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.011600:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.011603:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.011604:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.011606:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.011607:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.011609:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.011757:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.011759:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.011760:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.011761:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.011763:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.011764:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.011775:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.011778:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.011779:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.011779:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.011780:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.011781:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.011979:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.011981:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.011982:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.011982:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.011985:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.011991:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.012009:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.012012:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.012013:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.012013:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.012016:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.012019:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.012029:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.012032:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.012033:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:12.0:1619677447.012033:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677447.012033:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.012034:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.012035:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.012036:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.012038:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.012039:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.012040:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.012043:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.012052:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.012054:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.012055:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.012056:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.012057:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.012058:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.012110:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.012113:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.012113:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.012114:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.012116:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.012119:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.012265:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.012268:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.012269:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.012270:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.012272:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.012275:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.012424:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.012427:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.012428:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.012431:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.012433:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.012435:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.012437:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.012438:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.012440:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.012440:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.012446:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.012450:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.012453:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.012454:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.012454:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.012456:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.012457:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.012458:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.012459:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.012472:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.012474:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.012476:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.012476:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677447.012476:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:12.0:1619677447.012478:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:22.0:1619677447.012479:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:12.0:1619677447.012479:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.012482:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.012483:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.012484:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.012486:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.012494:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.012496:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.012497:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.012497:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.012498:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.012499:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.012515:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.012518:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.012519:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.012521:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.012522:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.012524:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.012758:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677447.012761:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677447.012762:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677447.012764:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.012767:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.012768:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677447.012769:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.012771:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.012772:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.012773:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677447.012775:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.012777:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.012779:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.012779:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.012780:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.012781:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.012781:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677447.012783:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677447.012784:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677447.012784:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677447.012795:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.012796:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.012797:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.012798:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677447.012799:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677447.012800:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677447.012800:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.012801:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.012802:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677447.012803:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:8.0:1619677447.012888:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.012891:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.012892:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.012895:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.012897:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.012898:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:8.0:1619677447.012899:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.012901:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.012902:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.012903:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:8.0:1619677447.012908:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.012910:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.0:1619677447.012912:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.012913:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:8.0:1619677447.012914:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (23)++
00000800:00000200:8.0:1619677447.012915:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:22.0:1619677447.012916:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677447.012916:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.012917:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.012918:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.012918:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677447.012919:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.012920:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.012920:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.012922:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.012925:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.012931:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.012933:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.012933:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.012935:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677447.012984:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.012986:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.012987:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.012987:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.012989:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677447.012992:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.013013:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.013015:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.013016:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.013017:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.013018:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.013021:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.013033:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.013035:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.013036:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.013037:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.013038:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.013038:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.013151:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.013154:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.013155:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.013155:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.013157:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.013160:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677447.013222:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.013225:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.013226:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.013227:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.013229:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.013230:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.013271:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677447.013274:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677447.013275:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677447.013277:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.013279:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.013281:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677447.013282:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.013284:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.013285:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.013285:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677447.013289:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.013291:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677447.013294:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.013296:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:8.0:1619677447.013302:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:8.0:1619677447.013303:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:8.0:1619677447.013304:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677447.013304:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677447.013305:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:6.0:1619677447.013309:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.013312:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.013313:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.013315:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.013318:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.013320:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.013321:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96734253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.013323:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.013324:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:6.0:1619677447.013325:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.013326:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.013326:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.013327:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.013329:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.013330:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.013332:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.013333:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.013334:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.013335:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.013336:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.013337:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.013338:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.013339:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.013339:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.013340:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.013428:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.013431:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.013432:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.013434:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.013436:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.013437:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.013439:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.013440:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.013441:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.013442:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.013446:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.013447:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.013450:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.013451:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.013452:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.013454:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.013454:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.013455:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.013455:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.013589:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.013592:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.013592:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.013594:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.013596:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.013597:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.013752:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.013754:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.013755:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.013755:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.013758:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.013761:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677447.013783:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.013786:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.013787:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.013789:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.013791:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.013792:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677447.013793:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.013795:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.013796:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.013797:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677447.013800:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.013802:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677447.013805:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.013806:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677447.013806:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677447.013808:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677447.013808:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.013809:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.013810:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.013821:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.013825:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.013825:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.013826:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.013828:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.013831:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.014253:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.014256:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.014257:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.014258:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.014259:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.014261:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.014300:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.014303:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.014304:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.014306:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.014307:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.014308:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.014309:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.014310:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.014310:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.014310:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.014311:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.014459:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.014462:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.014463:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.014465:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.014468:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.014469:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.014471:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.014472:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.014474:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.014475:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.014478:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.014479:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.014481:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.014483:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.014483:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.014486:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.014487:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.014488:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:24.0:1619677447.014489:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:8.0:1619677447.014489:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:24.0:1619677447.014490:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96734239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.014491:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.014492:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.014492:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:24.0:1619677447.014493:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.014494:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.014495:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.014496:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677447.014497:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.014499:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.014501:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.014504:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.014505:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.014505:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:12.0:1619677447.014506:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:24.0:1619677447.014508:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.014508:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:12.0:1619677447.014508:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:24.0:1619677447.014509:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.014509:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:12.0:1619677447.014509:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.014510:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.014512:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.014512:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.014530:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.014532:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.014533:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.014535:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.014536:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.014537:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.014652:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.014654:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.014655:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.014657:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.014659:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.014660:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.014667:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.014670:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.014671:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.014671:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.014672:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.014673:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.014707:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.014710:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.014711:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.014713:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.014716:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.014718:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.014719:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.014721:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.014722:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.014724:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.014728:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.014730:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.014733:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.014734:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.014734:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.014736:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.014736:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.014738:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.014739:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.014739:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677447.014744:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.014747:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.014748:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.014749:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:12.0:1619677447.014750:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.014751:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:13.0:1619677447.014751:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677447.014752:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677447.014752:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.014753:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.015220:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.015223:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.015224:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.015225:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.015226:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.015227:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.015300:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.015303:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.015304:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.015305:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.015308:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.015309:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:6.0:1619677447.015309:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.015310:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.015311:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:6.0:1619677447.015312:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.015313:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677447.015313:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.015315:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:8.0:1619677447.015317:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.015320:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.015320:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.015322:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:6.0:1619677447.015322:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:8.0:1619677447.015324:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.015325:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677447.015326:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.015326:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:8.0:1619677447.015327:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.015327:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.015328:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:8.0:1619677447.015329:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.015329:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.015330:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.015331:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:8.0:1619677447.015341:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.015343:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.015346:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.015347:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677447.015348:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (23)++
00000800:00000200:8.0:1619677447.015350:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677447.015350:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.015356:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.015362:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.015362:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.015375:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.015378:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.015379:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.015379:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677447.015543:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.015546:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.015547:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.015549:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.015551:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.015553:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.015554:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.015555:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.015556:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.015557:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.015560:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.015562:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.015566:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.015567:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.015567:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.015569:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.015570:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.015571:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.015571:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.015799:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.015801:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.015802:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.015804:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.015806:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.015807:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.015816:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.015819:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.015820:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.015821:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.015821:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.015822:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.015844:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.015847:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.015847:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.015851:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.015857:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.015858:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.015859:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96734221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.015861:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.015862:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.015863:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.015868:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.015871:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.015874:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.015875:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.015876:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.015876:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.015877:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.015878:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.015879:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.015879:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.015879:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.015880:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.015881:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.015882:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.015885:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.015902:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.015904:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.015905:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.015907:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.015909:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.015910:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.016090:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.016093:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.016094:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.016096:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.016099:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.016101:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.016102:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.016103:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.016105:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.016105:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.016109:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.016110:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.016113:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.016114:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.016115:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.016117:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.016117:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.016118:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.016118:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.016258:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.016260:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.016261:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.016262:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.016264:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.016265:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.016366:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.016369:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.016370:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.016370:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.016372:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.016375:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.016438:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.016441:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.016442:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.016443:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.016445:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.016448:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.016456:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.016458:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.016459:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.016460:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.016461:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.016461:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.016589:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.016592:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.016593:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.016594:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.016596:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.016597:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.016914:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.016916:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.016918:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.016918:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:24.0:1619677447.016920:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677447.016920:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.016922:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.016924:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.016925:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.016926:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.016927:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677447.016927:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:24.0:1619677447.016928:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.016929:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.016929:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.016930:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.016931:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.016931:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.016932:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:8.0:1619677447.016932:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.016933:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.016937:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.016938:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.016942:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.016942:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.016943:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.016945:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.016945:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.016946:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.016946:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.017022:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677447.017024:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677447.017026:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:6.0:1619677447.017026:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.017028:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.017029:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.017029:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.017031:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.017031:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:6.0:1619677447.017032:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96734202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.017034:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.017036:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.017037:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.017041:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.017041:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.017042:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677447.017043:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.017044:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677447.017047:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.017048:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677447.017048:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.017050:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677447.017051:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677447.017052:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677447.017052:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:13.0:1619677447.017211:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.017214:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.017215:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.017216:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.017218:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.017219:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.017229:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.017232:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.017232:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.017234:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.017236:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.017237:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.017779:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.017782:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.017783:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.017783:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.017785:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.017787:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.017858:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.017861:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.017861:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.017862:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.017864:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.017867:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.017908:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.017911:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.017912:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.017914:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.017915:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.017916:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.018049:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.018052:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.018053:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.018055:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.018057:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.018059:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.018060:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96734190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.018061:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.018062:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.018063:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.018068:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.018070:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.018073:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.018074:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.018075:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.018077:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:24.0:1619677447.018078:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:22.0:1619677447.018078:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.018078:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.018079:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.018081:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.018082:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.018084:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.018086:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.018088:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.018089:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.018091:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.018092:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.018093:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.018098:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.018100:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.018103:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.018104:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.018105:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.018106:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.018107:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.018108:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.018109:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.018110:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.018118:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.018120:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.018121:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.018123:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.018123:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:6.0:1619677447.018123:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:13.0:1619677447.018124:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.018125:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:12.0:1619677447.018126:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.018126:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677447.018127:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.018127:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:8.0:1619677447.018128:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:6.0:1619677447.018128:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.018130:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.018135:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.018136:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.018138:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.018138:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.018139:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.018140:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.018140:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.018141:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.018144:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.018145:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.018145:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.018146:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.018147:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.018178:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.018180:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.018181:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.018182:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.018184:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.018186:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.018215:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.018217:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.018218:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.018220:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.018221:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.018223:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.018385:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677447.018388:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677447.018389:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677447.018392:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677447.018394:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.018396:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677447.018397:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.018399:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.018400:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.018402:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677447.018406:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677447.018408:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.018409:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.018410:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.018410:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.018412:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.018412:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677447.018413:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677447.018413:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:6.0:1619677447.018420:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.018423:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.018424:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:8.0:1619677447.018424:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:6.0:1619677447.018424:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.018426:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.018427:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.018428:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.018428:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.018429:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:8.0:1619677447.018430:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677447.018432:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677447.018833:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.018837:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.018837:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.018839:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.018841:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.018842:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.018921:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.018924:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.018925:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.018926:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.018929:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.018931:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.018932:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.018934:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.018935:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.018937:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.018940:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.018943:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.018946:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.018947:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.018947:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.018949:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.018950:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.018951:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.018952:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.018952:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.018963:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.018965:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.018966:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.018966:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677447.019065:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.019067:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.019068:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.019068:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.019071:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.019073:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677447.019150:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.019152:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.019153:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.019153:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.019155:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.019159:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677447.019169:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.019172:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.019173:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.019173:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.019174:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.019174:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677447.019527:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.019530:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.019531:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.019533:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.019536:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.019537:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.019539:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.019540:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.019541:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.019542:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.019546:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.019547:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.019550:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.019551:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.019552:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.019554:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.019554:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.019555:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.019556:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.019662:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.019666:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.019667:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.019669:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.019670:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.019671:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.019683:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.019686:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.019687:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.019688:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.019688:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.019690:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.019790:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.019793:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.019794:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.019797:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.019799:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.019800:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.019802:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.019803:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.019804:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.019805:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.019809:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.019810:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.019813:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.019814:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.019815:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.019817:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.019817:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.019818:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.019819:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.019983:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.019986:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.019987:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.019989:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.019992:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.019993:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.019995:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.019996:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.019997:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.019998:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.020002:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.020003:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.020006:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.020007:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.020007:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.020009:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.020010:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.020010:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.020011:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.020204:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.020207:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.020208:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.020208:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.020210:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.020213:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677447.020264:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.020266:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677447.020267:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.020269:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.020270:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677447.020271:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677447.020272:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.020272:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.020273:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.020274:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.020279:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.020288:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.020290:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.020291:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.020292:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.020292:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.020301:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.020388:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.020391:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.020391:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.020392:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.020394:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.020397:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.020556:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.020559:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.020561:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.020562:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.020564:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.020566:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.020567:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.020569:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.020570:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.020571:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.020573:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:24.0:1619677447.020575:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.020575:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.020577:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.020577:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.020580:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.020581:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.020581:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:8.0:1619677447.020581:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.020582:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.020583:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.020584:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:8.0:1619677447.020585:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.020585:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.020586:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.020587:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.020588:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.020590:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.020590:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.020591:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.020596:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.020599:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.020603:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.020604:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.020604:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.020605:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.020605:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.020606:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.020607:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.020607:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.020608:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.020609:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.020609:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.020617:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.020620:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.020621:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.020623:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.020625:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.020627:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.020629:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96734151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.020630:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.020631:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.020632:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:24.0:1619677447.020634:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:6.0:1619677447.020636:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.020637:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.020638:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.020638:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:6.0:1619677447.020638:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.020640:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.020641:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.020641:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.020642:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:24.0:1619677447.020644:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.020644:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.020644:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.020645:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.020646:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.020657:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.020660:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.020661:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.020661:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.020662:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.020662:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677447.020704:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.020706:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.020707:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.020708:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.020710:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.020711:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.021075:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.021077:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.021079:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.021080:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.021082:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.021084:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.021085:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.021086:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.021087:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.021088:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.021091:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.021093:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677447.021094:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677447.021096:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.021096:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:12.0:1619677447.021096:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:24.0:1619677447.021097:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:12.0:1619677447.021097:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.021098:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.021099:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.021099:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.021100:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:12.0:1619677447.021100:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.021101:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.021101:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.021181:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.021184:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:12.0:1619677447.021184:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:22.0:1619677447.021185:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.021185:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:12.0:1619677447.021187:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:22.0:1619677447.021188:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:12.0:1619677447.021188:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.021190:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.021191:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.021193:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.021195:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.021196:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.021199:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.021199:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.021200:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.021201:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.021201:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.021221:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.021224:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.021225:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.021227:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.021230:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.021231:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.021233:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.021234:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.021235:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.021236:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.021239:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.021241:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.021244:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.021245:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.021246:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.021248:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.021248:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.021249:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.021250:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.021298:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.021301:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.021302:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.021304:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.021305:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.021306:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.021407:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.021410:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.021411:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.021413:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.021415:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.021416:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.021418:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.021419:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.021420:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.021421:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.021424:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.021426:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.021428:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.021429:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.021429:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.021431:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.021431:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.021432:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.021432:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.021488:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.021491:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.021491:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.021493:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.021495:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.021496:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.021510:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.021513:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.021513:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.021514:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.021514:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.021515:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.021902:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.021905:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.021905:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.021906:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.021908:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.021912:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.022080:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.022084:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.022085:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.022087:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.022089:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.022091:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.022092:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.022094:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.022095:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.022096:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.022099:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.022101:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.022104:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.022105:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.022105:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.022107:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.022107:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.022108:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.022109:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.022246:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.022249:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.022250:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.022251:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.022253:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.022254:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.022481:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.022484:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.022485:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.022487:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.022489:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.022491:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677447.022492:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.022494:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.022495:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.022496:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677447.022500:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.022502:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.022505:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.022506:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677447.022506:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677447.022508:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.022508:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.022514:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.022515:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.022516:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:6.0:1619677447.022522:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.022525:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.022526:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.022526:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.022528:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:6.0:1619677447.022528:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.022531:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.022532:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:6.0:1619677447.022532:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.022534:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.022631:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.022634:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.022635:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.022635:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.022638:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.022642:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.022848:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.022851:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.022852:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.022853:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.022855:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.022856:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.022937:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.022939:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:8.0:1619677447.022939:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:12.0:1619677447.022940:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.022942:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.022942:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.022943:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677447.022944:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.022945:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:8.0:1619677447.022945:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.022948:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.022949:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.022951:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.022952:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.022953:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.022954:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.022958:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.022959:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.022963:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.022964:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.022964:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.022966:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.022966:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.022967:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.022968:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.023167:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.023170:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.023171:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.023173:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.023174:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.023176:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.023201:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.023204:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.023204:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.023206:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.023208:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.023210:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.023212:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.023213:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.023214:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.023215:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.023219:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.023220:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.023222:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.023223:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.023224:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.023226:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.023226:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.023228:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.023229:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.023229:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.023240:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.023241:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.023242:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.023243:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.023654:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.023657:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.023657:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.023659:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.023661:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.023662:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.023820:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.023823:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.023824:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.023825:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.023826:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.023828:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.023828:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.023832:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.023832:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.023833:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.023835:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.023838:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.023840:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.023843:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.023844:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.023844:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.023845:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.023845:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.023847:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.023850:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.023851:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.023851:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.023853:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.023853:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.023932:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.023934:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.023936:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.023937:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.023939:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.023939:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.023956:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.023959:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.023960:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.023960:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.023961:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.023963:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.024076:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.024079:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.024080:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.024083:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.024085:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.024086:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.024088:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96734104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.024089:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.024091:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.024092:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.024095:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.024097:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.024100:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.024100:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.024101:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.024103:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.024103:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.024104:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.024104:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.024107:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.024110:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.024111:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.024113:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.024115:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.024117:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677447.024381:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.024384:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.024384:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.024386:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.024388:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.024389:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.024486:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.024489:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.024490:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.024490:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.024493:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.024495:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.024554:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.024557:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.024557:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.024559:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.024561:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.024562:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.024919:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.024921:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.024923:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.024923:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.024925:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.024927:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677447.025060:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.025062:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.025063:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.025065:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.025067:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.025068:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.025141:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.025144:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.025145:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.025146:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.025149:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.025150:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.025151:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.025152:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.025153:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.025154:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.025158:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.025159:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.025162:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.025163:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.025163:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.025165:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.025166:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.025167:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.025168:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.025168:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.025181:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.025184:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.025185:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.025186:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.025298:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.025301:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.025302:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.025302:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.025304:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.025306:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.025471:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.025473:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.025474:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.025476:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.025478:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.025479:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.025532:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.025535:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.025535:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.025536:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.025538:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.025541:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.025739:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.025742:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.025742:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.025744:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.025747:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.025749:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.025750:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.025751:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.025752:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.025753:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.025757:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.025759:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.025762:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.025763:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.025764:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.025765:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.025766:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.025766:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.025767:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.025875:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.025878:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.025878:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.025880:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.025881:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.025882:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.025894:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.025897:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.025897:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.025898:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.025898:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.025899:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.026362:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:24.0:1619677447.026365:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:22.0:1619677447.026365:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.026366:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.026367:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.026368:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.026369:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.026369:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.026369:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.026371:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.026374:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677447.026375:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.026377:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.026379:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.026380:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.026380:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.026381:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.026381:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.026382:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.026382:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.026383:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.026385:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.026388:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.026389:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.026390:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.026391:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.026392:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.026393:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.026396:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.026398:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.026401:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.026402:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.026403:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.026404:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.026405:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.026406:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.026406:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.026459:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.026462:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.026463:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.026463:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.026465:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.026467:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677447.026559:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.026562:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.026563:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.026564:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.026566:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.026569:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.026570:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.026572:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.026573:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.026574:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.026577:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.026579:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.026582:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.026582:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.026583:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.026585:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.026585:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.026586:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.026587:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.026610:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.026613:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.026614:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.026616:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.026618:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.026620:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.026708:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.026711:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.026712:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.026712:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.026714:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.026717:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.026827:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.026829:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.026830:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.026832:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.026833:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.026834:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.027079:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.027082:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.027083:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.027083:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.027085:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.027088:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.027270:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.027273:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.027273:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.027276:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.027278:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.027280:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.027281:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.027283:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.027284:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.027284:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.027287:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.027289:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.027291:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.027292:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.027299:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.027301:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.027301:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.027302:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.027303:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.027304:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.027315:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.027317:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.027318:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.027319:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.027338:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.027341:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.027342:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.027344:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.027347:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.027348:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.027349:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.027351:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.027352:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:8.0:1619677447.027352:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.027352:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.027355:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.027356:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.027356:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.027356:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:8.0:1619677447.027358:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.027359:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.027361:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.027362:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.027362:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.027363:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.027364:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.027365:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.027366:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.027366:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677447.027390:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:13.0:1619677447.027392:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677447.027394:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.027394:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677447.027395:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.027396:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.027398:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677447.027398:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.027400:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.027400:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677447.027403:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.027404:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96734058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.027406:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.027406:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677447.027407:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.027409:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.027414:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.027416:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.027419:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.027420:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.027421:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.027423:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.027424:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.027424:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.027424:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:24.0:1619677447.027425:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.027426:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.027427:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.027429:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.027430:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.027432:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.027448:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.027451:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.027451:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.027453:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.027454:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.027456:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.027906:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.027909:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.027909:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677447.027912:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.027914:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.027916:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.027918:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.027919:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.027920:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.027921:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.027925:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.027926:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.027929:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.027930:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.027930:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.027933:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.027933:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.027934:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.027934:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677447.028199:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.028203:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.028204:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.028208:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.028210:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.028216:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.028218:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.028220:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.028221:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.028223:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.028228:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.028231:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.028235:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.028236:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.028237:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.028239:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.028239:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.028240:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.028241:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.028241:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.028242:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:8.0:1619677447.028242:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.028243:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.028244:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.028252:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677447.028274:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.028277:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.028278:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.028280:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.028281:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.028283:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.028302:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.028303:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.028304:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.028304:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.028305:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.028306:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.028307:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.028307:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.028308:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.028308:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.028310:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.028315:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.028316:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.028318:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.028319:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.028319:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.028319:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.028320:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.028517:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.028520:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.028521:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.028524:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.028526:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.028527:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:6.0:1619677447.028528:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:13.0:1619677447.028530:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:6.0:1619677447.028530:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96734045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.028531:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.028532:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677447.028533:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.028533:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.028534:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:6.0:1619677447.028534:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:13.0:1619677447.028537:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:6.0:1619677447.028539:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.028542:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.028545:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.028546:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.028547:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.028548:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.028549:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.028550:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.028550:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.028662:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.028664:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.028665:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.028666:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.028669:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.028669:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.028730:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.028732:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.028733:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.028735:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.028738:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.028739:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.028740:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.028742:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.028743:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.028744:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.028747:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.028749:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.028752:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.028752:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.028753:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.028755:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.028756:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.028756:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.028757:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677447.029102:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.029105:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.029106:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.029107:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.029111:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.029113:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.029114:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.029115:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.029116:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.029117:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.029122:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.029124:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.029128:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.029129:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.029129:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.029131:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.029132:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.029133:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.029134:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.029135:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.029149:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.029151:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.029151:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.029152:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.029191:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.029194:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.029195:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.029197:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:22.0:1619677447.029198:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.029198:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.029199:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.029201:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.029205:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.029206:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.029207:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.029208:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.029208:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (23)++
00000800:00000200:22.0:1619677447.029209:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.029209:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.029209:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.029210:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.029432:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.029435:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.029435:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.029437:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.029439:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.029440:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.029492:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.029495:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.029496:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.029498:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.029501:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.029503:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.029505:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96734033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.029506:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.029508:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.029509:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677447.029509:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.029512:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.029513:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:8.0:1619677447.029514:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.029515:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.029517:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:6.0:1619677447.029518:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.029519:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:8.0:1619677447.029521:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677447.029521:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96734031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.029522:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:6.0:1619677447.029523:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.029524:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.029525:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:8.0:1619677447.029528:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.029529:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:6.0:1619677447.029529:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.029530:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.029531:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:6.0:1619677447.029531:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:8.0:1619677447.029533:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.029533:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:6.0:1619677447.029534:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.029535:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.029535:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.029537:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.029537:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.029538:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.029539:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.029546:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.029549:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.029549:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.029551:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.029848:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.029850:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.029851:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.029853:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.029854:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.029856:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.029865:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.029867:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.029868:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.029869:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.029869:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.029870:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.029896:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.029898:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.029899:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.029900:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.029902:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.029904:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.030490:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.030493:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.030493:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.030494:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.030496:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.030499:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677447.030509:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.030512:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.030513:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.030513:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.030514:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.030515:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677447.030773:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.030775:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.030776:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.030778:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.030779:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.030781:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.030809:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.030811:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.030812:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.030814:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.030817:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.030818:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.030819:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.030821:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.030822:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.030823:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.030826:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.030828:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.030831:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.030832:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.030832:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.030834:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.030834:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.030835:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.030836:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.030875:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.030878:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.030879:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.030879:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.030881:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.030884:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.030884:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.030887:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.030888:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.030888:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.030890:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.030893:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677447.031064:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.031067:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.031068:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.031068:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.031072:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.031075:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677447.031164:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.031167:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.031168:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.031169:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.031172:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.031174:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.031175:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96734010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.031177:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.031178:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.031179:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.031182:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.031184:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.031188:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.031188:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:6.0:1619677447.031188:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677447.031189:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.031191:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.031191:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677447.031191:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.031192:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.031192:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677447.031192:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.031193:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677447.031195:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.031204:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.031206:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.031209:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.031209:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.031210:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.031211:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.031211:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.031220:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.031223:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.031224:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.031224:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.031226:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.031229:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.031466:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.031469:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.031470:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.031470:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.031473:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.031475:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.031502:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.031506:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.031506:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.031507:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.031509:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.031512:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.031521:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.031523:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.031524:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.031526:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.031528:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.031529:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.031872:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.031875:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.031876:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.031877:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.031879:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.031882:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.031921:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677447.031921:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677447.031924:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:6.0:1619677447.031924:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.031925:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:13.0:1619677447.031925:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.031927:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.031927:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.031928:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:6.0:1619677447.031930:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.031931:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:6.0:1619677447.031932:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.031934:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96734000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.031935:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.031936:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.031937:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.031940:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.031942:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.031945:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.031946:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.031947:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.031949:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.031949:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.031950:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.031951:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.031952:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.031963:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.031965:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.031966:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.031967:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.032279:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.032282:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.032283:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.032283:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.032285:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.032287:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.032431:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.032433:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.032435:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.032436:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.032439:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.032441:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.032442:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.032443:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.032444:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.032445:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.032449:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.032452:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.032454:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.032455:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.032456:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.032457:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.032458:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.032458:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.032459:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.032470:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.032472:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.032473:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.032475:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.032477:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.032478:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.032495:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.032498:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.032500:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)++
00000800:00000200:12.0:1619677447.032501:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677447.032501:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.032503:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.032511:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.032513:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.032513:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.032515:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.032516:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.032517:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.032544:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677447.032546:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677447.032547:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677447.032549:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.032551:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.032553:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677447.032554:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.032555:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.032556:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.032557:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677447.032560:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.032562:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677447.032564:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.032565:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:24.0:1619677447.032566:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:24.0:1619677447.032567:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:24.0:1619677447.032568:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677447.032568:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677447.032569:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:12.0:1619677447.032623:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.032626:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.032627:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.032628:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.032630:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.032631:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.032896:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.032899:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.032900:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.032901:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.032902:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.032902:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.032904:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.032906:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:6.0:1619677447.032907:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.032909:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.032909:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.032911:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.032912:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000400:00000200:6.0:1619677447.032912:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:22.0:1619677447.032913:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000400:00000200:6.0:1619677447.032913:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.032914:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.032915:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.032915:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.032916:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:6.0:1619677447.032916:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.032917:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.032922:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.032924:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.032927:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.032928:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.032929:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.032931:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.032931:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.032932:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.032932:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.033330:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.033333:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.033333:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.033334:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.033336:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.033339:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.033425:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.033428:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.033429:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.033431:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.033434:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.033436:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.033438:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.033440:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.033441:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.033442:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.033448:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.033452:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.033457:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.033458:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.033458:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.033460:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.033461:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.033462:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.033462:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677447.033474:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.033476:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.033478:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.033478:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.033480:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677447.033484:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677447.033493:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.033495:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.033496:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.033497:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.033498:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.033498:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677447.033546:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.033548:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.033549:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.033551:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.033553:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.033554:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.033582:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.033586:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.033587:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.033589:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.033590:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.033591:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.033646:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.033649:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.033650:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.033652:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.033653:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.033654:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.033665:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.033667:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.033667:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.033667:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.033668:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.033668:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.033669:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.033669:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677447.033670:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.033670:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.033671:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.033672:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.034188:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.034191:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.034192:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.034194:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.034197:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.034198:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.034199:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.034201:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.034202:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.034203:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.034207:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.034209:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.034211:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.034212:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.034213:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.034214:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.034215:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.034216:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.034216:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.034261:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.034264:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:12.0:1619677447.034265:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:6.0:1619677447.034265:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.034266:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.034268:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.034268:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.034269:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.034270:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.034272:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.034272:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.034273:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.034428:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.034430:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.034432:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.034434:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.034437:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.034438:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.034440:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.034441:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.034442:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.034443:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.034446:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.034447:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.034450:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.034451:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.034452:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.034453:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.034454:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.034454:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.034456:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.034721:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.034724:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.034725:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.034727:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.034730:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.034731:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677447.034733:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.034734:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.034735:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.034736:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677447.034739:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.034740:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.034743:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.034744:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677447.034744:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677447.034746:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.034746:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.034747:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.034748:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.034760:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.034763:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.034763:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.034764:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.034766:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.034769:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.034777:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.034779:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.034780:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.034781:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.034782:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.034782:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.034860:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.034863:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.034864:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.034866:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.034869:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.034871:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.034872:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96733961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.034874:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.034875:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.034876:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:6.0:1619677447.034876:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.034878:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.034878:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.034880:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.034880:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.034882:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.034883:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.034884:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.034886:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.034887:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.034888:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.034890:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.034890:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.034891:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.034891:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.034956:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.034958:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.034959:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.034961:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.034962:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.034964:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.035223:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.035226:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.035227:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.035228:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.035231:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.035233:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677447.035234:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.035235:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.035236:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.035237:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677447.035240:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.035241:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677447.035244:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.035246:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677447.035246:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677447.035249:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677447.035249:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.035249:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.035250:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.035250:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.035251:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677447.035252:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.035252:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.035255:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.035256:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.035257:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96733956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.035258:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.035259:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.035259:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.035262:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:8.0:1619677447.035262:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.035264:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.035265:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.035265:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.035266:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.035266:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.035267:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.035267:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:22.0:1619677447.035268:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.035269:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.035269:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.035270:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.035271:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.035272:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.035273:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.035285:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.035287:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.035288:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.035290:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.035491:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.035494:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.035495:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.035495:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.035497:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.035500:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.035555:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.035557:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.035558:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.035559:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.035562:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.035566:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677447.035598:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.035601:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.035602:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.035602:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:6.0:1619677447.035602:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.035604:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:12.0:1619677447.035604:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.035605:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.035607:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:12.0:1619677447.035607:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.035609:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.035610:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.035890:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.035894:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.035894:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.035896:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.035899:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.035900:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.035901:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96733947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.035903:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.035904:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.035906:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.035909:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:6.0:1619677447.035910:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.035911:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.035912:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.035912:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.035914:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.035916:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.035917:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.035917:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.035918:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.035919:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.035920:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.035920:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.035922:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.035923:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.035924:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.035934:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.035937:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.035937:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.035938:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.035972:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.035976:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.035976:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.035979:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.035982:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.035984:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677447.035986:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96733943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.035987:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.035989:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.035990:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677447.035994:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.035996:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677447.035999:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.036000:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677447.036000:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:13.0:1619677447.036001:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677447.036002:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677447.036002:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.036004:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:13.0:1619677447.036004:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:24.0:1619677447.036005:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.036005:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:13.0:1619677447.036005:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.036006:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:13.0:1619677447.036007:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.036008:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677447.036009:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:13.0:1619677447.036010:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:6.0:1619677447.036010:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.036011:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.036012:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.036013:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.036015:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.036016:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.036017:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.036017:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:8.0:1619677447.036018:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:6.0:1619677447.036018:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.036020:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.036020:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:6.0:1619677447.036020:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.036021:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:24.0:1619677447.036022:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677447.036022:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.036022:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.0:1619677447.036023:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.036023:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.036024:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.036025:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.036027:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.036028:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.036029:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.036029:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.036030:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.036033:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.036036:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.036037:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.036037:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.036039:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.036039:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.036276:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.036278:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.036279:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.036280:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.036282:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.036285:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.036324:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.036327:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.036327:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.036328:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.036330:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.036333:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.036443:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.036446:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.036447:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.036448:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.036450:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.036453:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.036747:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.036750:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.036751:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.036751:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.036753:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.036755:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.036767:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.036769:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.036770:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.036770:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.036773:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.036779:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.036849:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.036852:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.036853:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.036853:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.036855:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.036858:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677447.036946:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.036949:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.036949:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.036951:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.036953:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.036955:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.037178:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.037180:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.037181:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.037181:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.037183:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.037186:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.037367:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.037369:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.037370:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.037370:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.037372:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.037374:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.037457:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.037460:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.037460:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.037461:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.037464:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.037471:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.037473:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:24.0:1619677447.037474:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:6.0:1619677447.037475:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:24.0:1619677447.037476:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:22.0:1619677447.037476:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.037476:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.0:1619677447.037477:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.037478:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.037478:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.037479:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:24.0:1619677447.037480:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.037483:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677447.037560:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.037562:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.037563:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.037564:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.037567:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.037568:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.037751:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:22.0:1619677447.037754:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000400:00000200:22.0:1619677447.037755:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.037755:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (24)++
00000800:00000200:6.0:1619677447.037756:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000400:00000200:6.0:1619677447.037756:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.037758:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.037759:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.037760:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.037761:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.037761:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.037764:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.037764:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.037765:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.037766:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.037766:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.037788:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.037790:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.037791:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.037792:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.037793:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.037795:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.037810:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.037813:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.037814:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.037816:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.037819:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.037820:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.037822:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.037823:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.037824:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.037825:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.037828:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.037830:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.037834:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.037835:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.037835:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.037837:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.037838:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.037839:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.037839:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.038157:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.038160:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.038161:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.038163:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.038164:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.038166:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.038443:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.038445:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.038446:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.038448:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.038451:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.038453:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.038454:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.038455:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.038457:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.038458:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.038461:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.038463:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.038466:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.038467:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.038467:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.038470:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.038470:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.038471:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.038471:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.038519:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.038522:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.038522:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.038524:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.038526:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.038527:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.038933:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.038936:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.038936:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.038938:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.038939:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.038941:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.038950:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.038952:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.038953:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.038954:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.038955:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.038956:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.039029:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.039031:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677447.039034:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.039036:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.039037:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.039039:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.039041:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.039043:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.039044:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.039046:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.039049:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.039051:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.039054:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.039055:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.039055:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.039057:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.039057:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.039058:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.039058:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.039346:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.039349:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.039350:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.039351:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.039353:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.039354:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.039363:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.039366:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.039366:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.039367:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.039368:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.039370:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.039409:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.039411:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.039412:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.039413:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.039413:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.039414:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:12.0:1619677447.039414:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.039417:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.039422:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.039424:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.039425:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.039426:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.039440:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.039443:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.039443:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.039445:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.039446:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.039447:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.039453:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.039456:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.039456:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.039459:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.039462:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.039465:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.039466:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.039467:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.039469:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.039469:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.039475:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.039477:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.039480:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.039481:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.039482:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.039484:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.039484:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.039485:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.039486:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.039500:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.039502:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.039504:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.039504:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.039506:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.039509:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.039783:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.039786:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.039787:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.039788:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.039789:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:8.0:1619677447.039792:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677447.039802:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.039805:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.039806:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.039806:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.039807:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.039808:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677447.039932:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.039935:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.039936:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.039937:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.039939:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.039941:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.039942:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.039943:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.039943:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.039943:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.039945:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.039948:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.040008:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.040011:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.040012:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.040014:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.040016:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.040018:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.040019:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.040020:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.040021:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.040022:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.040025:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.040026:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.040029:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.040030:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.040030:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.040032:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.040032:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.040033:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.040033:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.040150:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.040153:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.040153:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.040155:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.040157:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.040157:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.040220:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.040223:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.040224:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.040226:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.040229:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.040230:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.040231:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96733886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.040233:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.040234:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.040234:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.040239:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:8.0:1619677447.040239:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.040241:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.040242:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.040243:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.040243:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:8.0:1619677447.040244:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.040244:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:22.0:1619677447.040245:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.040245:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.040247:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.040247:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.040248:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.040248:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.040249:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.040269:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.040272:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.040273:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.040275:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.040277:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.040279:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.040280:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.040281:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.040282:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.040283:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.040287:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.040288:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.040291:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.040292:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.040292:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.040299:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.040299:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.040301:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.040302:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.040302:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.040314:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.040316:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.040316:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.040318:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.040510:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.040513:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.040513:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.040514:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.040516:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.040519:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.040811:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.040813:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.040814:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.040815:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.040817:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.040820:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677447.041048:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.041051:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.041052:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.041052:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.041054:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.041057:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.041066:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.041069:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.041069:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.041070:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.041071:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.041072:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.041126:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.041129:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.041129:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:22.0:1619677447.041131:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:8.0:1619677447.041131:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.041134:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:8.0:1619677447.041134:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.041135:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.041137:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.041137:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.041139:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96733876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.041140:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.041140:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.041141:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.041142:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:8.0:1619677447.041142:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.041143:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.041145:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.041147:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.041148:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:8.0:1619677447.041148:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.041151:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.041152:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.041154:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.041155:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.0:1619677447.041156:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.041157:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.041157:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.041158:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:8.0:1619677447.041158:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.041159:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.0:1619677447.041159:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.041160:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:8.0:1619677447.041160:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677447.041161:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677447.041161:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.041162:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.041163:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.041279:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.041281:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.041283:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.041283:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.041285:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.041288:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677447.041517:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.041520:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.041520:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.041521:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.041523:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677447.041526:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677447.041534:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.041536:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.041537:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.041538:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.041538:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.041539:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677447.041759:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.041762:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.041762:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.041763:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.041765:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.041768:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.041788:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.041792:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.041793:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.041793:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.041796:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.041799:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.041911:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.041914:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.041914:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.041915:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.041918:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.041920:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.041937:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.041940:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.041941:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.041941:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.041943:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.041945:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.042145:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.042148:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.042149:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.042149:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.042151:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.042154:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.042158:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.042160:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.042161:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.042162:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.042164:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:13.0:1619677447.042165:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.042166:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.042168:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.042168:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.042169:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.042170:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.042171:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.042178:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.042181:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.042182:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.042182:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.042183:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.042184:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.042202:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.042205:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.042206:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.042206:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.042208:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.042210:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.042420:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.042422:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.042423:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.042424:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.042427:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.042427:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.042556:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.042559:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.042560:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.042562:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.042563:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.042565:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.042569:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.042571:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.042572:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.042573:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.042574:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.042577:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.042602:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.042604:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.042606:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.042607:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.042610:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.042611:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.042612:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.042614:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.042615:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.042616:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.042619:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.042621:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.042623:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.042624:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.042625:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.042626:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.042627:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.042628:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.042630:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.042630:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.042643:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.042645:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.042646:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.042646:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.042804:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.042807:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.042808:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.042810:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.042813:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.042814:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.042815:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.042816:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.042817:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.042818:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.042821:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.042822:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.042825:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.042826:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.042826:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.042828:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.042828:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.042829:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.042830:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.042839:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.042842:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.042843:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.042844:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.042846:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.042847:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.042920:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.042923:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.042924:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.042924:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.042926:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.042929:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677447.043108:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.043111:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.043112:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.043113:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.043115:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.043115:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.043128:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.043130:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.043131:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.043132:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.043132:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.043133:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.043243:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.043246:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.043246:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.043249:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.043250:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.043252:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.043260:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.043263:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.043264:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.043265:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.043266:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.043267:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.043287:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.043289:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.043290:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.043292:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.043293:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677447.043299:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.043302:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:13.0:1619677447.043302:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:22.0:1619677447.043304:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.043304:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:22.0:1619677447.043306:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.043306:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.043307:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (23)++
00000800:00000200:6.0:1619677447.043308:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (24)--
00000800:00000200:6.0:1619677447.043309:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.043310:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.043310:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.043403:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.043405:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.043406:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.043408:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.043410:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.043411:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.043412:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.043413:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.043414:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.043415:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.043418:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.043420:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.043423:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.043423:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.043424:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.043426:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.043426:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.043427:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.043428:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.043517:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.043520:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.043521:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.043523:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.043524:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.043526:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.043641:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.043643:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.043644:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.043646:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.043647:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.043649:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.043788:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.043791:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.043791:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.043793:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.043795:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.043797:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.043798:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733842] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.043800:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.043801:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.043802:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.043805:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.043807:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.043811:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.043811:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.043812:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.043814:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.043815:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.043816:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.043816:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.043874:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.043876:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.043877:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.043879:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.043880:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.043882:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.044001:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.044003:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.044004:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.044004:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.044008:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.044015:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.044036:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.044039:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.044040:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.044040:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.044042:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.044045:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.044052:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.044054:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.044055:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.044056:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.044058:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.044059:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.044071:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.044073:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.044074:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.044075:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.044075:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.044076:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.044085:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.044087:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.044088:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.044089:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.044091:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.044092:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.044305:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.044308:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.044309:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.044310:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.044312:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.044313:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.044460:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.044464:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.044466:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.044466:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.044468:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.044471:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.044618:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.044620:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.044622:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.044623:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.044625:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.044626:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.044887:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.044889:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.044890:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.044891:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.044893:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.044894:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.044906:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.044908:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.044909:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.044910:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.044910:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.044911:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.044915:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:13.0:1619677447.044916:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677447.044918:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:13.0:1619677447.044918:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.044919:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.044919:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.044920:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677447.044921:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677447.044921:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.044923:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677447.044924:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.044926:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.044927:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.044928:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.044929:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.044930:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.044934:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.044936:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.044938:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.044939:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.044940:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.044942:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.044942:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.044944:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.044945:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.044946:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.044958:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.044960:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.044961:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.044962:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.045050:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.045053:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.045054:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.045054:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.045056:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.045058:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.045064:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.045067:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.045067:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.045069:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.045071:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.045073:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.045074:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.045076:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.045077:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.045078:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.045080:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.045082:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.045084:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.045085:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.045086:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.045088:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.045088:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.045089:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.045089:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.045194:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.045197:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.045198:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.045199:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.045202:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.045204:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.045205:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96733823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.045206:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.045207:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.045208:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.045212:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.045215:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.045217:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.045218:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.045219:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.045220:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.045221:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.045221:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.045222:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.045675:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.045678:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.045679:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.045681:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.045684:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.045685:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.045686:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96733819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.045688:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.045689:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.045690:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.045693:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.045695:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677447.045696:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:22.0:1619677447.045697:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.045698:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.045699:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:12.0:1619677447.045699:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.045699:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.045700:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.045701:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:12.0:1619677447.045701:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.045702:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.045703:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:12.0:1619677447.045703:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677447.045704:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:13.0:1619677447.045704:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.045714:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.045716:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.045718:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.045719:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.045867:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.045870:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.045871:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.045871:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.045874:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.045879:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.045896:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.045899:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.045900:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.045900:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.045902:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.045904:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.045915:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.045917:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.045918:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.045919:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.045919:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.045920:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.046100:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.046103:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.046103:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.046106:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.046108:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.046109:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.046110:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96733813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.046112:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.046113:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.046114:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.046117:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.046119:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.0:1619677447.046120:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:6.0:1619677447.046122:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.046122:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:24.0:1619677447.046123:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.046123:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:6.0:1619677447.046123:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:8.0:1619677447.046124:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.046125:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:8.0:1619677447.046126:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.046126:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.046126:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.046127:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.046129:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677447.046387:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.046390:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.046390:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.046391:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.046393:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.046395:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.046405:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.046408:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.046408:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.046409:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.046411:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.046411:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.046486:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.046489:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.046490:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.046492:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.046495:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.046496:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.046498:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.046499:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.046500:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.046501:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.046504:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.046506:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.046508:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.046509:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.046510:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.046511:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.046512:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.046513:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.046513:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.046566:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.046569:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.046570:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.046571:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.046572:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.046572:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.046573:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.046575:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.046576:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.046576:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.046578:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.046580:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.046581:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.046582:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.046583:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.046584:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.046587:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.046589:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.046591:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.046592:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.046593:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.046595:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.046595:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.046596:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.046596:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.046718:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.046720:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.046721:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.046723:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.046725:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.046726:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.046841:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.046844:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.046845:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.046846:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.046848:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.046849:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.046857:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.046860:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.046860:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.046861:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.046862:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.046863:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.046929:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.046931:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.046931:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.046933:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.046935:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.046936:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.047135:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.047137:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.047138:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.047138:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.047140:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.047142:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.047278:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.047281:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.047282:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.047282:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.047284:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.047287:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.047306:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.047308:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.047309:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.047311:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.047314:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.047315:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.047316:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96733800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.047318:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.047319:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.047320:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.047323:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.047325:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.047328:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.047328:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.047329:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.047331:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.047331:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.047332:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.047332:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.047907:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.047910:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.047910:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.047912:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.047914:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.047915:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.048010:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.048012:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.048013:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.048013:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.048015:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.048018:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.048021:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.048024:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.048025:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.048027:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.048030:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.048031:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.048033:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.048034:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.048035:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.048036:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.048040:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.048041:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.048045:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.048046:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.048047:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.048049:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.048049:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.048050:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.048051:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677447.048267:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.048270:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.048270:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.048274:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.048277:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.048278:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.048279:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96733785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.048280:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.048281:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.048283:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.048290:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:8.0:1619677447.048291:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.048292:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.048293:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.048301:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.048302:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.048304:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677447.048304:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.048305:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.048306:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.048308:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.048308:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.048309:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.048310:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.048312:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.048312:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.048325:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.048327:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.048327:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.048328:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.048412:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.048414:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.048415:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.048417:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.048419:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.048421:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.048440:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.048440:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.048441:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.048442:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.048442:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:22.0:1619677447.048444:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.048445:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.048447:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.048447:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.048448:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.048449:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.048450:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000400:00000200:6.0:1619677447.048450:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:22.0:1619677447.048451:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.048451:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.048452:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.048453:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.048453:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.048454:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:6.0:1619677447.048454:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.048455:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.048459:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.048461:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.048464:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.048465:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.048465:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.048467:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.048468:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.048469:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.048469:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.048627:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.048630:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.048631:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.048632:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.048633:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.048636:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.048745:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.048748:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.048748:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677447.048751:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.048754:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:8.0:1619677447.048755:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.048757:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.048758:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.048758:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:6.0:1619677447.048758:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:8.0:1619677447.048760:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96733773] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.048761:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000400:00000200:8.0:1619677447.048761:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.048763:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:8.0:1619677447.048763:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.048763:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:12.0:1619677447.048767:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:8.0:1619677447.048767:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.048769:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:12.0:1619677447.048771:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.048772:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.048773:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.048774:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.048774:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.048774:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.048775:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:12.0:1619677447.048776:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.048776:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.048777:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.048777:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.048778:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.048784:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.048787:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.048787:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.048788:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.048789:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.048790:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.048926:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.048928:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.048929:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.048931:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.048932:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.048933:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.049190:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.049193:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.049193:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.049196:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.049196:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.049199:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.049421:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.049423:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.049424:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.049426:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.049428:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.049429:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.049437:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.049440:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.049441:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.049441:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.049442:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.049443:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.049504:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.049507:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.049508:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.049509:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.049511:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.049514:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.049696:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.049698:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.049699:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.049701:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.049702:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.049704:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.049946:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.049949:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.049950:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.049952:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.049955:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.049956:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.049958:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.049959:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.049960:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.049961:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.049964:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.049966:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.049969:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.049970:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.049971:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.049973:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.049973:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.049974:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.049974:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.050035:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.050038:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.050039:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.050041:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.050043:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.050045:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.050046:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.050047:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.050049:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.050049:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.050053:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.050055:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.050058:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.050059:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.050059:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.050061:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.050062:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.050063:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.050064:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.050065:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.050075:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.050077:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.050078:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.050079:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.050343:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.050346:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.050347:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.050349:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.050351:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.050352:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.050949:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.050952:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.050953:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.050955:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.050958:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.050960:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:8.0:1619677447.050960:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:22.0:1619677447.050961:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.050963:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.050964:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:22.0:1619677447.050964:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.050964:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:22.0:1619677447.050965:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677447.050969:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.050970:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.050972:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:8.0:1619677447.050972:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.050974:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:22.0:1619677447.050975:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.050976:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000400:00000200:8.0:1619677447.050976:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96733747] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.050977:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:8.0:1619677447.050977:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.050978:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.050979:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.050979:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:8.0:1619677447.050979:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.050980:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.050980:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:8.0:1619677447.050983:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.050985:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.050988:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.050989:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.050990:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.050992:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.050992:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.050994:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.050995:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.050995:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.051008:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.051010:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.051011:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.051012:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677447.051092:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.051095:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.051096:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.051100:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.051103:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.051104:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.051106:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96733745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.051107:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.051108:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.051109:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.051113:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.051115:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.051118:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.051118:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.051119:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.051121:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.051122:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.051122:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.051123:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.051266:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.051268:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.051269:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.051269:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.051271:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.051274:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.051358:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.051360:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.051361:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.051363:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.051365:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.051366:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.051458:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.051460:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.051461:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.051463:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.051466:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.051468:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.051469:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.051470:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.051472:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.051472:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.051476:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.051477:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.051480:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.051481:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.051481:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.051483:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.051484:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.051484:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.051485:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.051592:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.051594:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.051595:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.051596:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.051598:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.051601:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.051611:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.051613:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.051614:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.051614:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.051615:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.051615:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.051693:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.051696:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.051696:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.051698:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.051700:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.051701:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.051710:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.051713:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.051713:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.051714:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.051714:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.051715:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.052025:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.052027:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.052028:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:13.0:1619677447.052028:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:6.0:1619677447.052030:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.052031:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.052031:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.052032:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.052033:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.052033:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:12.0:1619677447.052034:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:6.0:1619677447.052035:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96733736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.052036:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:6.0:1619677447.052036:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.052037:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.052038:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.052044:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.052046:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.052049:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.052050:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.052050:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.052052:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.052053:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.052053:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.052054:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.052077:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.052080:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.052081:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.052081:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.052083:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.052087:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677447.052101:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.052104:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.052105:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.052105:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.052107:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.052110:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.052110:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.052113:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.052113:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677447.052114:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.052115:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.052115:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677447.052418:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.052421:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.052422:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.052422:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.052424:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.052428:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.052434:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.052436:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.052438:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.052439:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.052442:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.052443:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.052444:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96733729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.052445:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.052446:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.052447:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.052450:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.052452:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.052455:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.052456:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.052457:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.052458:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.052459:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.052459:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.052460:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.052739:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.052742:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.052743:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.052745:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.052747:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.052748:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.053071:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.053074:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.053075:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.053075:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.053077:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.053080:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677447.053341:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.053344:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.053345:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.053345:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.053347:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.053350:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677447.053403:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.053405:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.053407:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.053407:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.053411:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.053412:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.053414:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.053415:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.053416:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.053417:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.053421:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.053423:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.053426:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.053427:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.053428:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.053429:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.053430:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.053431:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.053432:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677447.053443:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.053445:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.053447:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.053447:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.053450:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.053452:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677447.053459:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.053462:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.053463:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.053463:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.053464:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.053465:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677447.053470:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.053472:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.053473:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.053475:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.053476:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.053478:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.053623:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.053626:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.053627:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.053629:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.053631:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.053633:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.053635:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96733707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.053637:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.053638:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.053639:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.053643:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.053647:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.053651:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.053651:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.053652:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.053654:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.053654:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.053656:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.053657:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.053658:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.053668:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.053670:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.053671:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.053672:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.053819:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.053821:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.053822:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.053825:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.053825:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.053832:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.053853:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.053856:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.053857:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.053859:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.053860:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.053862:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.054230:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.054232:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.054233:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.054235:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.054236:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.054237:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.054251:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.054252:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.054254:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:13.0:1619677447.054254:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.054255:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.054255:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:24.0:1619677447.054256:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.054256:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.054256:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.054257:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.054257:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.054260:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677447.054424:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.054427:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.054427:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.054429:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.054431:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.054433:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.054440:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.054443:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.054444:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.054445:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.054445:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.054446:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.054446:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.054449:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.054450:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.054451:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.054453:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.054456:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.054546:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.054549:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.054550:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.054553:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.054556:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.054558:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.054560:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96733694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.054561:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.054562:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.054564:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.054568:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.054570:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.054573:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.054574:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.054574:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.054576:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.054577:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.054578:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.054579:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.054580:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.054593:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.054596:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.054597:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.054598:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.054626:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.054628:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.054628:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.054630:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.054632:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.054633:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.054732:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.054735:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.054736:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.054737:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.054739:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.054742:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.054750:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.054753:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.054753:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.054754:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.054755:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.054756:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.054859:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.054862:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.054863:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.054866:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.054866:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.054869:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.055032:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.055034:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.055035:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.055037:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.055039:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.055039:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677447.055051:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.055054:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.055055:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.055057:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.055058:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.055059:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.055075:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.055078:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.055079:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.055079:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.055080:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.055081:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.055097:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.055099:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.055100:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.055102:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.055103:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.055105:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.055411:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.055414:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.055415:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.055417:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.055420:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.055422:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.055423:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96733683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.055425:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.055426:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.055427:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.055431:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.055433:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.055436:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.055436:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.055437:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.055439:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.055439:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.055440:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.055441:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.055518:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.055521:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.055522:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677447.055522:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:8.0:1619677447.055524:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.055525:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.055526:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.055526:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.055527:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.055529:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677447.055529:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:8.0:1619677447.055531:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96733679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.055532:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:8.0:1619677447.055533:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.055534:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.055535:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.055539:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.055541:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.055545:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.055545:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.055546:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.055548:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.055548:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.055549:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.055550:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677447.055913:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.055916:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.055917:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.055918:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.055921:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.055923:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.055924:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.055926:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.055927:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.055928:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.055932:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.055934:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.055937:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.055938:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.055939:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.055941:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.055941:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.055942:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.055943:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.055966:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.055969:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.055970:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.055971:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.055973:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.055974:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.056388:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.056390:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.056391:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.056391:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.056394:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.056398:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.056399:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.056401:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.056402:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.056403:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.056403:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.056404:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.056616:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.056619:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.056620:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.056622:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.056625:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.056627:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.056628:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96733671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.056630:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.056631:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.056632:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.056636:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.056638:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.056641:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.056641:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.056642:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.056644:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.056644:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.056645:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.056646:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.056708:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.056710:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.056711:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.056713:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.056715:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.056716:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.056892:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.056895:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.056896:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.056896:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.056899:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.056901:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.057104:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.057107:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.057108:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.057110:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.057113:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.057114:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.057116:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.057117:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.057118:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.057119:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.057122:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.057124:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.057126:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.057127:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.057128:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.057129:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.057130:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.057130:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.057131:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.057201:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.057204:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.057205:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.057207:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.057210:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.057211:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:13.0:1619677447.057212:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000400:00000200:8.0:1619677447.057212:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.057214:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.057215:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:8.0:1619677447.057215:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.057217:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677447.057218:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.057219:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677447.057220:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.057220:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677447.057221:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:8.0:1619677447.057222:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:13.0:1619677447.057223:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96733661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.057224:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677447.057225:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.057225:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677447.057226:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:8.0:1619677447.057226:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.057227:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.057228:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:13.0:1619677447.057229:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.057229:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.057230:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677447.057231:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677447.057231:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.057233:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.057234:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:13.0:1619677447.057235:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:13.0:1619677447.057236:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:13.0:1619677447.057237:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677447.057237:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677447.057238:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677447.057318:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.057321:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.057321:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.057323:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.057325:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.057326:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.057336:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.057339:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.057339:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.057340:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.057342:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.057345:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.057393:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.057396:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.057396:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.057398:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.057400:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.057401:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.057432:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.057434:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.057436:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.057436:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.057437:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.057440:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.057718:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.057721:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.057722:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.057724:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.057726:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.057727:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.057728:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.057730:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.057731:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.057732:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.057735:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.057737:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.057739:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.057740:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.057741:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.057743:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.057743:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.057744:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.057745:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.057746:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.057756:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.057757:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.057758:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.057759:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.057840:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.057843:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.057843:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.057845:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.057846:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.057848:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.057858:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.057861:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.057862:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.057862:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.057863:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.057864:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.057985:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.057988:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.057989:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.057989:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.057991:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.057995:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.058041:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.058043:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.058044:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.058045:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.058047:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.058047:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.058106:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.058108:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.058109:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.058110:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.058113:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.058114:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.058201:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.058204:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.058205:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.058207:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.058210:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.058211:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.058214:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96733647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.058215:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.058216:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.058217:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.058222:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.058224:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.058227:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.058228:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.058229:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.058231:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.058232:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.058232:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.058233:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.058236:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.058238:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.058239:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.058241:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.058242:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.058243:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.058511:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.058513:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.058514:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.058515:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.058517:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.058518:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.058527:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.058530:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.058531:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.058531:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.058532:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.058533:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.058711:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:24.0:1619677447.058714:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.058714:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.058715:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.058715:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:24.0:1619677447.058717:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:22.0:1619677447.058718:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.058718:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677447.058720:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:24.0:1619677447.058721:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.058724:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.058726:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.058728:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.058730:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.058732:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.058733:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.058739:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.058742:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.058745:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.058746:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.058747:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.058749:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.058749:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.058750:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.058751:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.058762:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.058765:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.058766:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.058766:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.058768:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.058771:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.058940:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.058942:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.058943:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.058944:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.058945:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.058951:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.058961:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.058964:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.058964:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.058965:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.058966:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.058967:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.058973:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.058975:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.058976:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.058978:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.058979:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.058980:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.058981:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.058984:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.058985:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.058987:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.058987:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.058999:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.059055:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.059057:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.059058:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.059058:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.059061:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.059062:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.059207:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.059210:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.059211:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.059211:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.059214:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.059217:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677447.059227:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.059230:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.059230:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.059231:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.059231:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.059232:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677447.059243:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.059246:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.059247:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.059248:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.059251:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.059253:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.059253:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.059255:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.059256:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.059257:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.059261:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.059262:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677447.059263:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.059265:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.059265:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677447.059267:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.059268:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:22.0:1619677447.059268:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.059269:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.059271:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.059271:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:22.0:1619677447.059271:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.059272:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.059272:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:22.0:1619677447.059273:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.059275:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96733624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.059276:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.059278:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.059279:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.059284:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.059287:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.059291:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.059292:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.059293:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.059301:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.059303:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.059303:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.059304:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.059306:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.059309:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.059310:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.059310:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.059312:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.059315:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.059394:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.059397:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.059398:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.059399:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:13.0:1619677447.059401:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677447.059401:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.059404:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:6.0:1619677447.059404:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:13.0:1619677447.059405:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.059407:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.059408:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.059410:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.059420:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.059423:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.059424:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.059424:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.059425:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677447.059426:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:12.0:1619677447.059426:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.059428:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.059429:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.059430:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.059432:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.059434:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.059473:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.059476:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.059477:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.059477:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.059479:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.059482:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.059492:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.059495:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.059495:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.059496:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.059497:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.059497:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.059688:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.059691:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.059691:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.059693:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.059696:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.059699:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.059700:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96733615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.059702:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.059703:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.059704:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.059710:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.059713:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.059717:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.059718:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.059718:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.059720:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.059720:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.059722:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.059724:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.059725:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677447.059737:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.0:1619677447.059737:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:13.0:1619677447.059739:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:8.0:1619677447.059739:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:13.0:1619677447.059741:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.059741:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:13.0:1619677447.059743:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.059743:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.059743:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677447.059744:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.059746:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:13.0:1619677447.059746:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.059746:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.059749:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.059751:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.059753:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.059754:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.059755:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.059756:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.059757:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.059761:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.059763:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.059766:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.059767:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.059767:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.059769:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.059769:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.059770:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.059771:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.059772:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.059785:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.059787:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.059788:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.059794:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677447.060059:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.060062:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.060063:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.060065:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.060067:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.060069:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.060070:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96733609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.060072:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.060072:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.060073:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.060077:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.060079:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.060082:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.060083:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.060084:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.060086:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.060086:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.060087:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.060087:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.060127:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.060130:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.060130:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.060132:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.060134:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.060135:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.060333:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.060336:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677447.060337:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677447.060339:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.060342:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.060344:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:6.0:1619677447.060345:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.060346:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.060347:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.060348:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:6.0:1619677447.060352:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.060354:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:6.0:1619677447.060357:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.060358:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:6.0:1619677447.060359:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677447.060361:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677447.060361:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.060362:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677447.060362:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:13.0:1619677447.060375:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.060378:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.060379:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.060380:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.060382:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.060383:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.060392:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.060394:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.060395:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.060396:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.060397:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.060398:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.060423:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.060425:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.060426:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.060428:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.060431:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.060432:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.060434:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.060435:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.060436:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.060437:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.060440:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.060442:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.060445:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.060445:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.060446:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.060448:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.060448:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.060449:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.060449:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677447.060492:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.060494:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.060495:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.060497:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.060499:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.060501:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.060502:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.060503:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.060504:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.060505:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.060508:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.060509:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.060512:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.060512:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.060513:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.060514:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.060515:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.060515:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.060516:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.060526:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.060529:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.060529:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.060531:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.060532:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.060533:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.060667:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.060670:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.060671:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.060674:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.060674:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.060677:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.060775:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.060778:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.060778:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.060779:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.060781:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.060784:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.060827:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.060829:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.060830:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.060833:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.060835:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.060836:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.060838:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.060840:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.060842:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.060843:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.060848:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.060851:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.060854:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.060855:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.060856:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.060858:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.060858:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.060859:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.060860:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.060865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677447.061212:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.061214:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.061215:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.061217:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.061220:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.061226:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.061228:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96733595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.061229:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.061230:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.061231:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.061236:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.061240:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.061243:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.061244:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.061245:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.061247:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.061248:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.061249:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.061250:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.061261:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.061263:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.061264:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.061265:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.061267:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.061270:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.061306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.061326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677447.061379:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.061382:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.061383:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.061385:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.061386:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.061388:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.061433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.061451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:24.0:1619677447.061460:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.061463:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.061464:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.061467:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.061470:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.061472:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.061473:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96733589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.061475:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.061475:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677447.061475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:24.0:1619677447.061477:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.061480:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.061482:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.061484:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.061485:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.061486:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.061487:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.061488:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.061489:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.061489:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.061499:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.061502:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.061503:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.061504:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.061506:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.061507:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.061518:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.061521:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.061522:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.061522:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.061523:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.061523:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.061688:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.061691:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.061691:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.061692:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.061694:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.061697:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.061778:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.061780:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.061782:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.061782:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.061784:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.061786:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.061854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.062106:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.062109:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.062110:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.062110:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.062113:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.062115:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.062125:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.062127:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.062128:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.062128:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.062129:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.062130:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.062227:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.062230:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.062231:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.062232:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.062235:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.062236:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.062237:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.062239:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.062240:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.062241:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.062244:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.062245:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.062248:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.062249:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.062250:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.062251:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.062252:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.062252:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.062253:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.062370:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.062372:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.062373:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.062375:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.062377:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.062378:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.062646:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.062648:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.062649:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.062650:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.062651:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.062654:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.062679:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.062683:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.062687:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.2:1619677447.062688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677447.062689:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.062691:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.062693:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.062694:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96733574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.062696:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.062697:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.062698:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.062702:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.062705:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.062708:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.062709:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.062709:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.062711:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.062712:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.062713:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.062714:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.062715:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.062728:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.062730:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.062730:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.062731:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.063042:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.063045:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.063046:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.063048:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.063049:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.063051:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.063110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.063172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.063178:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.063181:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.063182:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.063182:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.063184:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.063186:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.063335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.063380:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.063383:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.063384:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.063387:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.063389:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.063391:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.063393:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96733568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.063394:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.063395:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.063396:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.063400:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.063402:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.063405:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.063406:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.063406:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677447.063408:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:6.0:1619677447.063408:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.063408:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.063410:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:13.0:1619677447.063411:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:6.0:1619677447.063411:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000400:00000200:13.0:1619677447.063412:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.063412:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:13.0:1619677447.063415:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.063415:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.063417:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.063422:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.063424:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.063425:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.063426:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.063561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.063567:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.063570:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.063571:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.063571:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.063573:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.063576:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677447.063866:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.063869:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.063870:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.063873:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.063876:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.063878:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.063879:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96733560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677447.063880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:8.0:1619677447.063882:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.063884:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.063886:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.063887:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.063889:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.063890:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.063892:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.063892:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.063894:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.063895:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.063896:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677447.063898:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.063899:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.063900:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.0:1619677447.063906:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (24)--
00000800:00000200:8.0:1619677447.063906:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.063907:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.063908:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.063954:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.063957:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.063958:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.063958:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.063960:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.063963:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.064077:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.064080:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.064081:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.064083:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.064087:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.064088:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.064089:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.064091:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.064092:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.064093:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.064097:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.064098:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.064101:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.064102:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.064103:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.064105:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.064105:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.064106:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.064106:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.064266:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.064269:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.064270:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.064270:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.064272:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.064275:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.064286:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.064289:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.064290:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.064291:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.064291:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:13.0:1619677447.064293:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677447.064298:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:12.0:1619677447.064301:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.064301:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.064302:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.064303:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.064304:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.064384:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.064386:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.064387:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.064387:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.064390:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.064393:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.064595:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.064598:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.064599:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.064601:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.064601:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.064608:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.064618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.064637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:9.2:1619677447.064642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.064652:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.064655:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.064656:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.064658:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.064660:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.064662:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.064663:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.064664:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.064665:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.064667:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.064670:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.064672:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.064675:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.064676:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.064676:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.064678:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.064679:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.064679:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.064680:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.064834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.064841:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.064844:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.064845:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.064845:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.064847:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.064851:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.064852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.064858:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.064861:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.064861:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.064862:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.064863:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.064863:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.064937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.064994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.065011:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.065014:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.065015:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.065017:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.065020:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.065021:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.065023:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96733542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.065024:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.065025:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.065026:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.065029:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.065031:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.065034:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.065034:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.065035:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.065037:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.065037:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.065039:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.065040:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.065041:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.065053:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.065055:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.065056:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.065058:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.065149:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.065152:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.065153:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.065153:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.065155:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.065158:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.065218:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.065221:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.065222:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.065224:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.065225:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.065226:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.065239:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.065241:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.065242:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.065243:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.065244:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.065245:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.065519:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.065522:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.065523:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.065523:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.065525:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.065527:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.065607:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.065610:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.065611:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.065613:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.065615:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.065617:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.065618:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.065620:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.065621:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.065622:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.065625:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.065627:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.065631:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.065632:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.065632:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.065634:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.065635:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.065635:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.065637:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.065763:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.065766:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.065766:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.065768:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.065770:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.065771:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.065946:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.065949:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.065950:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.065952:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.065953:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.065955:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.066035:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4ce00] (21)++
00000800:00000200:22.0:1619677447.066038:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4ce00] (22)--
00000800:00000200:6.0:1619677447.066038:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.14.20@o2ib3
00000400:00000200:6.0:1619677447.066041:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.20@o2ib3 : REPLY - routed
00000800:00000200:6.0:1619677447.066044:0:19937:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000800:00000200:6.0:1619677447.066049:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4ce00] (21)++
00000800:00000200:6.0:1619677447.066050:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4ce00] (22)++
00000800:00000200:6.0:1619677447.066051:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4ce00] (23)--
00000800:00000200:6.0:1619677447.066051:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4ce00] (22)--
00000800:00000200:22.0:1619677447.066066:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4ce00] (22)++
00000800:00000200:22.0:1619677447.066067:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4ce00] (23)--
00000800:00000200:22.0:1619677447.066069:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4ce00] (22)--
00000800:00000200:8.0:1619677447.066102:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.066105:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.066106:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.066108:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.066111:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.066112:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.066114:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.066115:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.066118:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.066118:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.066122:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.066123:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.066126:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.066127:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.066127:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.066129:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.066129:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.066130:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.066130:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.066190:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.066193:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.066193:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.066194:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.066196:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.066199:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.066210:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.066212:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.066213:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.066213:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.066215:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.066215:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.066377:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.066380:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.066381:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.066384:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.066386:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.066387:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.066389:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.066390:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.066391:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.066392:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.066398:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.066400:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.066404:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.066405:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.066406:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.066408:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.066408:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.066409:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.066410:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.066415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.066425:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.066428:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.066429:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.066429:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.066433:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.066437:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.066439:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.066440:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.066441:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.066442:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.066444:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.066446:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.066453:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4ce00] (22)++
00000800:00000200:6.0:1619677447.066455:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.14.20@o2ib3
00000800:00000200:22.0:1619677447.066456:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4ce00] (23)--
00000800:00000200:6.0:1619677447.066458:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4ce00] (22)--
00000400:00000200:6.0:1619677447.066459:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.066460:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:6.0:1619677447.066461:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96733524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.066463:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.066464:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.066465:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:6.0:1619677447.066469:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : REPLY try# 0
00000800:00000200:6.0:1619677447.066471:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 1048576 bytes in 256 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.066472:0:19937:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000800:00000200:6.0:1619677447.066477:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.066478:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:6.0:1619677447.066479:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.066480:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.066481:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4ce00] (21)++
00000800:00000200:6.0:1619677447.066481:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4ce00] (22)--
00000800:00000200:6.0:1619677447.066482:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4ce00] (21)--
00000800:00000200:22.0:1619677447.066493:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.066495:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.066496:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.066528:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.066531:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[2] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.066532:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.066536:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.066550:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.066550:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:9.2:1619677447.066648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.066845:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.066848:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.066849:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.066851:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.066854:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.066855:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:24.0:1619677447.066857:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96733521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.066858:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.066859:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.066860:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:24.0:1619677447.066863:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.066866:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:24.0:1619677447.066869:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.066869:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:24.0:1619677447.066870:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:24.0:1619677447.066872:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:24.0:1619677447.066872:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.066873:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.066873:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.066975:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.066978:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.066979:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.066981:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.066983:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.066984:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.067175:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.067179:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.067179:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.067180:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.067182:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: REPLY: OK
00000800:00000200:6.0:1619677447.067183:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.067253:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.067256:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.067257:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.067258:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.067260:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.067263:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.067635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.067636:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.067639:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.067640:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.067640:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.067642:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.067645:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.067655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:9.2:1619677447.067768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.067945:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.067948:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.067949:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.067951:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.067953:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.067954:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.067956:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.067957:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.067958:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.067959:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.067963:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.067965:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.067967:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.067968:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.067968:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.067970:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.067971:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.067972:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.067973:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.067974:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.067984:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.067986:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.067987:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.067988:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.068186:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.068189:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.068190:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.068191:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.068193:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.068195:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.068239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.068305:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.068308:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.068309:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.068310:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.068312:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.068313:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.068327:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.068330:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.068331:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.068332:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.068333:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.068334:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.068347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.068355:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.068358:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.068359:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.068359:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.068361:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.068364:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677447.068659:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.068662:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.068663:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.068664:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.068666:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.068668:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677447.068669:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.068670:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.068671:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.068672:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677447.068675:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.068677:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677447.068680:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.068681:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677447.068682:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677447.068683:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677447.068684:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.068685:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.068685:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.068698:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.068702:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.068703:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.068703:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.068705:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.068707:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.068716:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.068718:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.068719:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.068720:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.068721:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.068721:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.068750:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.068752:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.068753:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.068755:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.068756:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.068757:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.068764:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.068766:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.068767:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.068769:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.068772:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.068773:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.068775:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.068777:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.068778:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.068779:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.068782:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.068784:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.068787:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.068788:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.068788:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.068790:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.068790:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.068791:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.068791:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.068842:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.068845:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.068846:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.068847:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.068849:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.068851:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.068987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.069046:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.069049:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.069050:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.069052:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.069054:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.069055:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.069057:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.069058:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.069059:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.069060:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.069063:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.069065:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.069067:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.069068:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.069069:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.069070:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.069071:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.069071:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.069072:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.069160:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.069162:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.069164:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.069166:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.069168:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.069169:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.069171:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96733493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.069172:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.069172:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:24.0:1619677447.069173:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.069174:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:13.0:1619677447.069175:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.069176:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.069177:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677447.069177:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.069178:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.069179:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:13.0:1619677447.069181:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.069183:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.069183:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.069184:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.069186:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.069186:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.069188:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.069189:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.069189:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.069202:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.069204:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.069205:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.069206:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.069219:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.069222:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.069223:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.069223:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.069226:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.069229:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.069237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.069511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.069517:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.069520:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.069521:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.069521:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.069523:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.069526:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677447.069553:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.069555:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.069556:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.069558:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.069560:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.069561:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.069804:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.069806:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.069807:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.069809:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.069810:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.069811:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.069839:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.069842:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.069842:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677447.069844:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.069847:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.069849:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.069850:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.069851:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.069852:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.069853:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.069856:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.069858:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.069861:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.069861:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.069862:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.069864:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.069865:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.069866:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.069866:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.069871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.069925:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.069928:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.069929:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.069930:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.069931:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.069933:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.069945:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.069948:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.069948:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.069949:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.069950:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.069951:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.070011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.070017:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.070020:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.070021:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.070021:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.070024:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.070027:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677447.070079:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:22.0:1619677447.070080:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:12.0:1619677447.070082:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:22.0:1619677447.070083:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:12.0:1619677447.070083:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.070084:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:13.0:1619677447.070085:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.070085:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.070086:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.070088:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:22.0:1619677447.070089:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.070090:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.070092:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.070093:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.070094:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.070095:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.070099:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.070101:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.070103:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.070104:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.070105:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.070107:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.070107:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.070108:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.070108:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.070157:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.070159:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.070160:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.070161:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.070163:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.070164:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.070285:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.070288:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.070288:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.070291:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.070300:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.070303:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.070429:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.070432:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.070432:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677447.070434:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.070437:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.070438:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.070440:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.070441:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.070442:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.070443:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.070445:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.070447:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.070449:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.070450:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.070450:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.070452:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.070452:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.070453:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.070454:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.070465:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.070467:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.070468:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.070471:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.070471:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.070473:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.070481:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.070484:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.070485:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.070486:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.070486:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.070487:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.070490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.070536:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.070539:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.070540:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.070541:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.070543:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.070544:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.070596:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.070599:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.070600:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.070602:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677447.070603:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.070605:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.070620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.070808:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.070810:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.070811:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.070813:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.070814:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.070815:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.070824:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.070827:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.070828:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.070829:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.070832:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.070834:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.070835:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.070836:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.070837:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.070838:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.070841:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.070843:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.070845:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.070846:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.070847:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.070848:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.070849:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.070850:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.070850:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.070927:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.070929:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.070930:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.070932:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.070935:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.070936:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.070938:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.070939:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.070940:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.070941:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.070944:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.070945:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.070947:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.070948:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.070949:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.070950:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.070951:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.070952:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.070952:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.070962:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.070964:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.070965:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.070966:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.070968:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.070969:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.070990:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.070993:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.070994:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.070994:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.070995:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.070995:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.071035:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.071038:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.071039:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.071039:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.071041:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.071044:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.071148:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.071151:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.071151:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.071153:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.071155:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.071157:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.071158:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.071159:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.071160:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.071161:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.071166:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.071168:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.071174:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.071175:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.071176:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.071178:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.071178:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.071179:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.071180:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.071194:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.071197:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.071198:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.071198:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.071200:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.071202:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.071397:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.071399:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.071400:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.071400:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.071402:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.071405:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.071415:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.071418:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.071419:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.071420:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.071421:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.071421:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.071756:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.071759:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.071759:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.071760:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.071762:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.071765:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.071773:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.071775:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.071776:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.071776:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.071777:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.071778:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.071920:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.071922:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.071923:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.071925:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.071927:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.071927:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.071931:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.071934:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.071935:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.071936:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.071939:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.071940:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.071942:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.071943:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.071944:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.071945:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.071948:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.071949:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.071952:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.071953:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.071954:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.071955:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.071956:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.071957:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.071958:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.071959:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.071971:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.071973:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.071974:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.071975:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.072099:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.072102:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.072103:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.072103:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.072105:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.072108:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.072215:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.072218:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.072219:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.072219:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.072221:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.072223:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.072378:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.072381:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.072382:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.072382:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.072384:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.072387:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.072432:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.072434:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.072435:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.072435:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.072438:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.072440:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.072465:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.072468:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.072469:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.072469:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.072471:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.072474:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.072553:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.072555:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.072556:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.072558:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.072560:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.072561:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.072934:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.072937:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.072937:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.072939:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.072941:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.072941:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.072965:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.072968:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.072969:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.072970:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.072971:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.072972:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.073083:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.073085:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.073086:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.073087:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.073089:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.073091:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.073214:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.073218:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.073218:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.073221:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.073224:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.073225:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.073227:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96733442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.073228:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.073229:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.073230:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.073233:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.073235:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.073238:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.073239:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.073239:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.073241:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.073241:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.073242:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.073243:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.073451:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.073453:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.073454:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.073455:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.073457:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.073458:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.073471:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.073474:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.073475:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.073477:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.073479:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.073481:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.073482:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.073484:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.073484:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.073486:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.073489:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.073492:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.073494:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.073495:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.073496:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.073498:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.073498:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.073500:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.073501:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.073501:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.073512:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.073514:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.073514:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.073515:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.073667:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.073670:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.073671:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.073673:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.073676:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.073677:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.073679:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.073680:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.073681:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.073682:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.073685:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.073687:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.073690:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.073690:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.073691:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.073693:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.073693:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.073695:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.073696:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.073697:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677447.073699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.073709:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.073711:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.073712:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.073714:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.073974:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.073977:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.073978:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.073980:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.073981:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.073983:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.074261:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.074263:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.074264:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.074266:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.074267:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.074268:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.074315:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.074318:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.074319:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.074320:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.074322:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.074323:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.074324:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.074326:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.074327:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.074329:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.074332:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.074333:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:12.0:1619677447.074333:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677447.074334:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.074335:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.074336:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.074337:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.074337:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:12.0:1619677447.074337:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.074338:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.074338:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.074340:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677447.074341:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.074343:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.074345:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.074346:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.074347:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.074349:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.074349:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.074350:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.074351:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.074355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.074375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677447.074391:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.074394:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.074395:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.074397:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.074397:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.074399:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.074615:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.074618:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.074619:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.074621:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.074623:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.074625:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.074626:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96733424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.074627:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.074628:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.074629:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.074633:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.074635:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.074638:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.074638:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.074639:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.074641:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.074641:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.074642:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.074642:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.074696:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.074699:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.074700:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.074700:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.074703:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.074706:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.074714:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.074717:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.074717:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.074718:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.074719:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.074720:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.074822:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.074824:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.074825:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.074826:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.074828:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.074829:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.074896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.074902:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.074905:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.074906:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.074906:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.074908:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.074912:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677447.075001:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.075004:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.075005:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.075006:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.075009:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.075010:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.075011:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733416] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.075013:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.075014:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.075015:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.075018:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.075020:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.075023:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.075024:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.075025:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.075027:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.075027:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.075028:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.075028:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.075033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.075113:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.075116:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.075117:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.075118:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.075121:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.075123:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.075124:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.075126:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.075127:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.075127:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.075131:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.075132:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.075135:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.075136:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.075136:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.075138:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.075138:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.075139:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.075139:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.075226:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:9.2:1619677447.075228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:6.0:1619677447.075229:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.075230:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.075232:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.075234:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.075235:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:6.0:1619677447.075235:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.075237:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96733413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.075238:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:6.0:1619677447.075238:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.075239:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.075239:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:6.0:1619677447.075239:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.075241:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:24.0:1619677447.075242:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.075246:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:6.0:1619677447.075247:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677447.075248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:6.0:1619677447.075251:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.0:1619677447.075254:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:6.0:1619677447.075255:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.075256:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.075256:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:24.0:1619677447.075257:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.075257:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:6.0:1619677447.075258:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:8.0:1619677447.075259:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (21)++
00000800:00000200:6.0:1619677447.075259:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677447.075260:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.075260:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.075260:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.075261:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.075288:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.075290:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.075291:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.075297:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.075300:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.075304:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.075437:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.075440:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.075440:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.075442:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.075444:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.075445:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.075485:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.075488:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.075489:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.075490:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.075493:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.075494:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.075495:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.075497:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.075498:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.075498:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.075502:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.075503:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.075506:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.075507:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.075507:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.075509:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.075510:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.075510:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.075511:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.075516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.075720:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677447.075723:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677447.075725:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677447.075726:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.075730:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.075731:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.075732:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.075734:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.075735:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.075736:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.075739:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.075740:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.075743:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.075744:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.075744:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.075746:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.075746:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677447.075747:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677447.075747:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:6.0:1619677447.075881:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.075884:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.075885:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.075886:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.075888:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.075891:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.075899:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.075901:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.075902:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.075902:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.075903:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.075904:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.075910:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.075913:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.075913:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.075915:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.075917:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.075918:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.075937:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.075940:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.075940:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.075941:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.075942:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.075943:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.076003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.076010:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.076013:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.076014:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.076014:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.076016:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.076019:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.076085:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.076088:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.076089:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.076091:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.076093:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.076095:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.076096:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.076097:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.076098:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.076099:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.076102:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.076104:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.076107:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.076108:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.076108:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.076110:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.076111:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.076111:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.076112:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.076175:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.076178:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.076179:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.076179:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.076182:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.076184:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.076209:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.076212:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.076213:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.076214:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.076217:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.076219:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.076220:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.076222:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.076223:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.076223:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.076226:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.076228:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.076231:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.076232:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.076232:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.076234:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.076234:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.076236:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.076237:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.076238:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.076251:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.076253:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.076254:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677447.076254:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.076326:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.076329:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.076330:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.076332:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.076334:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.076334:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.076345:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.076348:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.076349:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.076349:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.076350:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.076351:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.076405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.076411:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.076414:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.076414:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.076415:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.076417:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.076420:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.076456:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:6.0:1619677447.076459:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:22.0:1619677447.076460:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (24)++
00000400:00000200:6.0:1619677447.076460:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.076461:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000400:00000200:22.0:1619677447.076461:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.076462:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.076463:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.076465:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.076466:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.076466:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.076830:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.076832:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.076833:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.076833:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.076835:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.076838:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.076873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.076926:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.076929:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.076930:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.076932:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.076934:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.076936:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.076937:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.076938:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.076939:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.076940:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.076943:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.076945:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.076948:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.076949:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.076949:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.076951:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.076951:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.076952:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.076953:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.077066:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.077068:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.077069:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.077069:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.077071:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.077074:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.077189:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.077192:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.077193:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.077194:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.077196:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.077199:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.077207:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.077210:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.077210:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.077211:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.077212:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.077212:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.077214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.077231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677447.077354:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.077357:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.077358:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.077358:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.077360:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.077363:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.077508:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.077511:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.077511:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.077513:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.077515:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.077516:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.077573:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.077576:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.077577:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.077577:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.077579:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.077582:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.077941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.077947:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.077949:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.077949:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.077951:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.077952:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.077953:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.077955:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.077956:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.077957:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:8.0:1619677447.077958:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.077959:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96733374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.077960:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.077961:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.077961:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677447.077963:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.077963:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.077964:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:8.0:1619677447.077964:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.077965:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.077966:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:12.0:1619677447.077966:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.077967:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:8.0:1619677447.077968:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.077970:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.2:1619677447.077971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:8.0:1619677447.077973:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.077974:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.077975:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.077977:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.077977:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.077978:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.077979:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.077980:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.077980:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.077980:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.077981:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.077982:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677447.077986:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.077986:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.077994:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.077997:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.077997:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.077998:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.078043:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.078046:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.078046:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.078047:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.078049:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.078051:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.078380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.078387:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.078390:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.078390:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.078391:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.078393:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.078396:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677447.078712:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.078715:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.078715:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.078717:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.078719:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.078720:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.078746:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.078748:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.078749:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.078751:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.078752:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.078753:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.078850:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.078853:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.078854:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.078857:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.078860:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.078862:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.078863:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.078865:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.078867:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.078869:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.078875:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.078878:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.078882:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.078883:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.078884:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.078886:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.078886:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.078887:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.078888:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.2:1619677447.078964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (21)++
00000800:00000200:24.0:1619677447.078968:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)++
00000800:00000200:24.0:1619677447.078971:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e49d37c00] (23)--
00000400:00000200:24.0:1619677447.078972:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.078972:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)--
00000400:00000200:24.0:1619677447.078974:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.078977:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)--
00000800:00000200:22.0:1619677447.079055:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.079058:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.079059:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.079060:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.079063:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.079065:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.079066:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.079068:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.079069:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.079070:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.079073:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.079075:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.079078:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.079079:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.079079:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.079081:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.079082:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.079083:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.079084:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.079085:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.079096:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.079098:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.079099:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.079100:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.079650:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.079653:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.079653:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.079655:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.079657:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.079658:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.079779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.079828:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.079831:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.079832:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.079834:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677447.079835:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.079838:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.079847:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.079850:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.079851:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.079852:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.079853:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.079854:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.079883:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.079885:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.079886:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.079887:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.079889:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.079890:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.079944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.079951:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.079954:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.079954:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.079955:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.079957:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.079960:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677447.080042:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.080045:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.080046:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.080047:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.080049:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.080051:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.080178:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.080181:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.080182:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.080183:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.080185:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.080187:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.080188:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.080189:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.080190:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.080191:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.080194:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.080195:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.080198:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.080198:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.080199:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.080201:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.080201:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.080202:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.080202:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.080243:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.080246:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.080247:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.080249:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.080251:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.080253:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.080255:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.080256:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.080258:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.080259:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.080263:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.080265:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.080269:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.080269:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.080270:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.080272:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.080273:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.080274:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.080275:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.080279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.080297:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.080300:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.080304:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.080306:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.080309:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.080311:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.080313:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.080314:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.080315:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.080316:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.080320:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.080322:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.080325:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.080326:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.080327:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.080329:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.080329:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.080330:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.080331:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.080510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.080516:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.080519:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.080520:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.080520:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.080522:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.080525:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677447.080637:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.080639:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.080640:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.080641:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.080643:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.080645:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.080655:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.080658:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.080658:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.080659:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.080660:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.080661:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.080716:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.080719:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.080720:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.080722:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.080724:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.080726:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.080727:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.080729:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.080730:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.080731:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.080736:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.080739:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.080742:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.080744:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.080744:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.080746:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.080747:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.080747:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.080748:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.080771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.080909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.080916:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.080918:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.080919:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.080920:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.080922:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.080925:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677447.080988:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.080990:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.080991:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.080993:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.080995:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.080996:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.081240:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.081243:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.081244:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.081244:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.081246:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.081248:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.081259:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.081262:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.081263:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.081265:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.081267:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.081269:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677447.081270:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.081271:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.081272:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.081273:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677447.081276:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.081278:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677447.081281:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.081282:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677447.081283:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677447.081285:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677447.081285:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.081286:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.081286:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.081301:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.081303:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.081304:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.081305:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.081307:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.081311:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.081312:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.081313:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677447.081314:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677447.081316:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.081318:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.081320:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:22.0:1619677447.081321:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.081322:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.081323:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.081324:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677447.081327:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.081328:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.081330:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.081331:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677447.081332:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.081333:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.081333:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.081334:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677447.081334:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:6.0:1619677447.081347:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.081348:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.081349:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.081350:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.081351:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.081353:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.081452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.081483:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.081485:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.081486:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.081488:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.081490:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.081492:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.081493:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.081494:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.081495:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.081496:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.081500:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.081502:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.081505:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.081506:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.081507:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.081509:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.081509:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.081510:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.081510:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.081593:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.081596:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.081596:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.081599:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.081600:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.081602:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.081603:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.081606:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.081607:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.081608:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.081611:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.081614:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.081625:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.081627:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.081628:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.081628:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.081629:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.081630:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.081739:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.081742:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.081742:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.081744:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.081747:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.081749:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.081750:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.081752:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.081752:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.081753:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.081757:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.081759:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.081761:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.081762:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.081763:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.081765:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.081765:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.081766:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.081766:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.081826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.081832:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.081834:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.081835:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.081835:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.081837:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.081840:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.081844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677447.081850:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.081852:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.081853:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.081853:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.081854:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.081854:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.081961:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.081963:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.081964:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.081964:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.081966:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.081968:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.082267:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.082269:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.082270:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.082271:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.082273:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.082276:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.082385:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.082388:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.082389:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.082391:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.082392:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.082393:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.082394:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.082394:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677447.082395:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.082395:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.082396:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.082818:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.082821:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677447.082822:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677447.082823:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.082826:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.082828:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:6.0:1619677447.082829:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.082830:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.082831:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.082832:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:6.0:1619677447.082835:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.082837:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677447.082840:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.082841:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:6.0:1619677447.082842:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:6.0:1619677447.082844:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:6.0:1619677447.082845:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.082846:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:6.0:1619677447.082847:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:6.0:1619677447.082847:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677447.082858:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677447.082860:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677447.082861:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677447.082862:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677447.083026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.083047:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:9.2:1619677447.083048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677447.083049:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.083050:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.083051:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.083053:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.083055:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.083070:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.083073:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.083074:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.083075:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.083076:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.083076:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.083152:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.083155:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.083156:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.083158:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.083160:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.083162:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.083163:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.083164:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.083165:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.083166:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.083170:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.083171:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.083174:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.083175:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.083176:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.083177:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.083178:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.083179:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.083179:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677447.083239:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.083242:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.083242:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.083245:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.083248:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.083249:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.083251:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96733305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.083252:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.083253:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.083254:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.083260:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.083262:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677447.083263:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.083265:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:8.0:1619677447.083265:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677447.083266:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.083266:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.083267:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:13.0:1619677447.083268:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.083268:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677447.083269:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.083269:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.083270:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.083272:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.083272:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677447.083274:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.083285:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.083287:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.083288:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.083288:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.083324:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.083326:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.083327:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.083329:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.083331:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.083332:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.083354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.083379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.083386:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.083388:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.083390:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.083390:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.083392:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677447.083395:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.083471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.083642:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.083645:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.083645:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.083647:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.083649:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.083649:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.083662:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.083664:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.083664:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.083665:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.083666:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:12.0:1619677447.083666:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.083667:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.083668:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.083668:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.083669:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.083670:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.083671:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.084012:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.084015:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.084016:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.084016:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.084018:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.084020:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.084063:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.084065:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.084066:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.084067:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.084069:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.084070:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.084076:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.084079:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.084080:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.084080:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.084082:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.084085:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.084134:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.084137:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.084138:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.084139:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.084141:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.084142:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.084250:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.084253:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.084254:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.084256:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.084259:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.084260:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.084261:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.084262:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.084263:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.084264:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.084267:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.084269:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.084272:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.084272:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.084273:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.084275:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.084275:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.084276:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.084276:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.084470:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.084473:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.084474:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.084476:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.084477:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.084478:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.084488:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.084491:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.084492:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.084494:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.084495:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.084496:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.084591:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.084594:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.084595:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.084597:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.084600:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.084601:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677447.084603:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.084604:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.084605:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.084606:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677447.084610:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.084612:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.084615:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.084616:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677447.084617:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677447.084618:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.084619:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.084620:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.084621:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.084622:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.084630:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.084633:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.084633:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.084634:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:12.0:1619677447.084635:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.084636:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.084636:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677447.084637:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.084638:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677447.084638:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.084639:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.084639:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:13.0:1619677447.084640:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.084640:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677447.084642:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.084644:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.084656:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.084658:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.084659:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.084661:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.084662:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.084664:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.084864:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.084867:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.084868:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.084868:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.084870:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.084873:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.085068:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.085070:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.085071:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.085073:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.085076:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.085077:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.085079:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.085080:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.085082:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.085083:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.085087:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.085091:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.085094:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.085094:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.085095:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.085097:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.085098:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.085099:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.085100:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.085101:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.085188:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.085190:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.085191:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.085192:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.085206:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.085210:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.085210:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.085214:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.085216:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.085218:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.085219:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.085222:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.085223:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.085225:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.085229:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.085231:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.085234:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.085235:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.085236:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.085237:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.085238:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.085239:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.085240:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.085361:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677447.085363:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677447.085365:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677447.085366:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.085369:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.085371:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.085372:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.085373:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.085374:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.085375:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.085378:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.085380:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.085383:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.085383:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.085384:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.085386:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.085386:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677447.085387:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:8.0:1619677447.085388:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:8.0:1619677447.085389:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677447.085402:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:24.0:1619677447.085405:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:24.0:1619677447.085405:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677447.085406:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:13.0:1619677447.085439:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.085442:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.085443:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.085445:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.085446:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.085448:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.085538:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.085541:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.085542:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.085544:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.085547:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.085548:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.085549:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96733273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.085551:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.085552:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.085553:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.085557:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.085559:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.085561:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.085562:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.085562:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.085564:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:9.2:1619677447.085565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:6.0:1619677447.085565:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.085565:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.085566:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.085576:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.085578:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.085580:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.085580:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.085582:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.085585:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677447.086018:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.086021:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.086022:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.086024:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.086026:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.086028:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.086029:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.086031:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.086033:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.086034:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.086038:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.086041:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.086044:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.086045:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.086046:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.086047:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.086048:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.086049:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.086049:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.086200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:12.0:1619677447.086205:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:24.0:1619677447.086207:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:12.0:1619677447.086208:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.086209:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.086210:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.086211:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.086211:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.086211:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.086211:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.086213:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.086214:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.086216:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677447.086221:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.086224:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.086225:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.086225:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.086226:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.086227:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.086324:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.086326:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.086327:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.086329:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.086332:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.086334:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.086335:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.086337:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.086338:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.086339:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.086342:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.086344:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.086347:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.086348:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.086349:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.086350:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.086351:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.086352:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.086352:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.086713:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.086716:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.086717:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.086717:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.086720:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.086727:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.086744:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.086746:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.086747:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.086748:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.086750:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.086752:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.086762:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.086764:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.086765:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.086765:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.086766:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:12.0:1619677447.086767:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:6.0:1619677447.086767:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.086769:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.086770:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.086772:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.086774:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.086774:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.086930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.086936:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.086939:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.086940:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.086940:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.086941:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.086944:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.086950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677447.086957:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.086959:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.086960:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.086960:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.086961:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.086962:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677447.086991:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.086994:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.2:1619677447.086995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.086996:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.086997:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.087000:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.087002:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677447.087004:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96733254] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.087006:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.087007:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.087009:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677447.087012:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.087014:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677447.087017:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.087018:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677447.087019:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677447.087021:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677447.087022:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.087023:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.087024:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.087035:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:9.2:1619677447.087036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.087038:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.087039:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.087039:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.087042:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.087044:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.087059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677447.087182:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.087185:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.087186:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.087186:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.087188:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.087191:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.087200:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.087203:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.087203:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.087203:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.087204:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.087205:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.087305:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.087308:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.087308:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.087309:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.087311:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.087313:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.087594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.087689:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.087691:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.087692:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.087694:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.087696:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.087697:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.087728:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.087730:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.087732:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.087733:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.087733:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.087735:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.087736:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.087736:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.087737:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.087737:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.087738:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.087740:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.087776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.087867:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.087870:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.087871:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.087873:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.087874:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.087875:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.087886:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.087889:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.087890:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.087890:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.087891:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.087892:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.087927:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.087930:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.087931:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.087933:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.087936:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.087937:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.087938:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.087939:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.087940:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.087941:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.087945:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.087947:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.087950:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.087951:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.087952:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.087953:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.087954:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.087955:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.087956:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.087956:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.087971:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.087973:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.087974:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.087975:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.087998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.088217:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.088219:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.088220:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.088222:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.088225:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.088227:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.088228:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.088229:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.088230:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.088231:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.088234:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.088235:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.088238:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.088239:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.088240:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.088242:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.088242:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.088243:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.088243:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.088247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.088266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:9.2:1619677447.088396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:22.0:1619677447.088400:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.088403:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:24.0:1619677447.088404:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:22.0:1619677447.088404:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.088404:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:24.0:1619677447.088407:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:22.0:1619677447.088407:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.088408:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.088408:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:22.0:1619677447.088409:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:24.0:1619677447.088410:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.088413:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.088415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:9.2:1619677447.088420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.088422:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.088424:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.088425:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.088425:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.088426:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.088427:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677447.088530:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.088533:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.088534:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.088534:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.088536:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.088538:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.088624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.088679:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.088682:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.088683:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.088685:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.088686:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.088688:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.088786:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.088788:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.088789:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.088791:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.088794:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.088796:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.088797:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96733229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.088799:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.088799:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.088801:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.088804:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.088806:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.088809:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.088810:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.088810:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.088812:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.088813:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:12.0:1619677447.088813:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:22.0:1619677447.088814:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.088814:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.088815:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.088816:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.088818:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677447.088819:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.088820:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.088988:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.088990:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.088991:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.088992:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.088994:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.089000:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:24.0:1619677447.089001:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677447.089002:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:24.0:1619677447.089005:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:22.0:1619677447.089005:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:8.0:1619677447.089005:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677447.089005:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.089006:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.089007:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000400:00000200:24.0:1619677447.089008:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.089008:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:24.0:1619677447.089011:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.089012:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.089014:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96733225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.089015:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.089016:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.089017:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.089021:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.089022:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.089025:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.089026:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.089026:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.089028:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.089028:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.089029:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.089030:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.089066:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.089069:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.089069:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.089072:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.089072:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.089074:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.089093:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.089095:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.089096:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.089098:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.089099:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.089101:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.089529:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.089531:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.089532:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.089534:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.089536:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.089537:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.089546:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.089548:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.089549:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.089549:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.089550:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.089551:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.089937:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.089940:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.089940:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.089942:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.089944:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.089945:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.090060:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.090063:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.090064:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.090066:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.090067:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.090068:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.090169:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.090171:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.090173:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.090174:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.090177:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.090179:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.090180:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.090181:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.090182:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.090183:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.090186:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.090188:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.090191:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.090191:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.090192:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.090194:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.090194:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.090195:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.090195:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.090323:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.090326:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.090327:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.090329:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.090331:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.090333:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.090334:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.090336:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.090337:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.090338:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.090343:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.090345:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.090348:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.090349:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.090350:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.090352:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.090352:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.090354:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.090355:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.090355:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.090365:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.090367:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.090368:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.090369:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.090373:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.090375:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.090376:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.090377:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.090380:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.090380:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.090389:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.090392:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.090392:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.090393:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.090394:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.090395:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.090812:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.090815:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.090815:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.090817:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.090818:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.090820:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.090828:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.090830:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:22.0:1619677447.090833:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.090833:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.090835:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.090836:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.090837:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.090837:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.090839:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:22.0:1619677447.090839:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.090839:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.090841:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96733204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.090842:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:22.0:1619677447.090842:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.090843:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:22.0:1619677447.090844:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.090846:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:24.0:1619677447.090847:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.090849:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.090850:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:24.0:1619677447.090851:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:22.0:1619677447.090851:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.090851:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:24.0:1619677447.090852:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96733203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.090852:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.090853:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.090854:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.090854:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.090854:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:24.0:1619677447.090855:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.090855:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:24.0:1619677447.090856:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.090857:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.090859:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.090860:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.090860:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:24.0:1619677447.090862:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.090862:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.090863:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.090863:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677447.090864:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.090864:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.090867:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.090868:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.090868:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.090870:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.090870:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.090871:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.090872:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.090882:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.090884:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.090885:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.090886:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.090888:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.090889:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.091252:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.091255:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.091255:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.091256:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.091258:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.091261:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.091279:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.091281:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:8.0:1619677447.091281:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677447.091282:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677447.091283:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:22.0:1619677447.091284:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.091285:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.091287:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.091288:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.091289:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.091289:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.091291:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.091291:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.091298:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.091299:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.091300:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.091301:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.091302:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.091302:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.091303:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.091306:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.091308:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.091311:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677447.091311:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.091312:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.091313:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.091314:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.091315:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.091315:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.091315:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.091316:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.091317:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.091318:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.091318:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677447.091324:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (24)--
00000800:00000200:8.0:1619677447.091325:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.091326:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.091326:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.091639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:6.0:1619677447.091642:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.091645:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:24.0:1619677447.091646:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:22.0:1619677447.091646:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.091647:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.091649:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:6.0:1619677447.091649:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.091650:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.091650:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:6.0:1619677447.091652:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:24.0:1619677447.091653:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.091656:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677447.091669:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.091671:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.091672:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.091674:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.091677:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.091678:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677447.091679:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.091681:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.091682:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.091683:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677447.091686:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.091688:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.091691:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.091692:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677447.091692:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677447.091694:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.091695:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.091696:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.091698:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.091698:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677447.091708:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:24.0:1619677447.091711:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:22.0:1619677447.091711:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.091711:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.091712:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.091713:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.091713:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.091714:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:22.0:1619677447.091714:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.091717:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.091885:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.091888:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.091889:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.091891:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.091893:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.091895:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.091896:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.091897:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.091899:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.091900:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.091904:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.091907:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.091910:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.091911:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.091912:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.091914:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.091917:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.091918:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.091919:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.091937:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.091940:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.091941:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.091942:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.091944:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.091945:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:22.0:1619677447.091946:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.091947:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.091948:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.091950:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.091952:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.091953:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.092026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.092033:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.092035:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.092036:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.092037:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.092039:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.092042:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.092044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.092051:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.092053:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.092054:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.092055:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.092056:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:24.0:1619677447.092058:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677447.092119:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.092122:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.092122:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.092123:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.092125:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.092128:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.092139:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.092141:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.092142:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.092143:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.092144:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.092144:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.092169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.092189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.092198:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.092200:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.092201:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.092203:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.092206:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.092207:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677447.092209:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.092210:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.092211:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.092212:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677447.092215:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.092217:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.092220:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.092221:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677447.092221:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677447.092223:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.092224:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.092224:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.092225:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.092237:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.092239:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.092240:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.092241:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.092243:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.092245:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.092251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:9.2:1619677447.092385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.092429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.092440:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.092442:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.092443:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.092444:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:24.0:1619677447.092445:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.092445:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.092448:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.092449:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.092450:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677447.092451:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.092451:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.092453:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.092455:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96733178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.092456:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.092457:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.092458:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:9.2:1619677447.092459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:8.0:1619677447.092462:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.092463:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.092467:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.092467:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.092468:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.092470:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.092471:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.092472:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.092473:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.092474:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.092487:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.092490:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.092490:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.092492:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.092662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.092668:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.092671:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.092671:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.092672:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.092673:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.092676:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.092682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.092687:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.092690:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.092691:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.092691:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.092692:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.092693:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677447.092705:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.092708:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.092709:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.092710:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.092711:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.092714:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.092747:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.092749:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.092751:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.092751:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.092753:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.092755:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.092904:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.092906:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.092907:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.092909:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.092911:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.092912:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.093024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.093101:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.0:1619677447.093102:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.093103:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.093104:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.093104:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.093105:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677447.093105:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677447.093105:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.093107:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.093108:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:8.0:1619677447.093108:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.093109:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:13.0:1619677447.093109:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.093110:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.093111:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:8.0:1619677447.093111:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.093112:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:13.0:1619677447.093113:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677447.093114:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.093115:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.093116:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.093117:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.093120:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.093122:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677447.093124:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.093125:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.093126:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:12.0:1619677447.093127:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.093127:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677447.093128:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677447.093128:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677447.093129:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.093129:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.093129:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.093130:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.093130:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.093131:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.093180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.093236:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.093239:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.093240:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:9.2:1619677447.093240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:6.0:1619677447.093240:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.093243:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.093245:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.093246:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.093249:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.093250:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.093250:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.093252:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.093254:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.093604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.093631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.093694:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.093696:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.093697:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.093697:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.093700:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.093702:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.093711:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.093714:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.093715:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:13.0:1619677447.093715:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677447.093715:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.093716:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.093717:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.093718:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.093718:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.093720:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.093722:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.093723:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.093892:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.093895:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.093896:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.093898:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.093900:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.093902:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.093903:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.093904:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.093905:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.093906:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.093909:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.093911:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.093914:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.093915:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.093915:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.093917:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.093917:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.093918:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.093918:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.094155:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.094158:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.094159:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.094159:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.094162:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.094164:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.094173:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.094176:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.094177:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.094177:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.094179:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.094179:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.094185:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.094188:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:24.0:1619677447.094189:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:12.0:1619677447.094189:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.094190:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.094192:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.094193:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:12.0:1619677447.094193:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.094193:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.094197:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.094200:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.094203:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.094204:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.094207:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.094208:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.094210:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.094214:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.094218:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.094220:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.094221:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.094222:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.094224:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.094224:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.094225:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.094226:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.094232:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.094235:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.094236:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.094237:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.094239:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.094240:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.094286:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.094289:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.094290:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.094292:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.094299:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.094301:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.094302:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.094303:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.094304:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.094305:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.094309:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.094311:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.094314:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.094315:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.094316:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.094318:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.094318:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.094319:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.094320:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.094321:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.094333:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.094335:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.094336:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.094336:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677447.094380:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.094383:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.094383:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.094385:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.094387:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.094389:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.094402:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.094405:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.094406:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.094407:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.094408:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.094409:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.094531:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.094534:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.094534:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.094536:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.094538:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.094539:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.094963:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.094966:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.094967:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.094967:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.094970:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.094973:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.095024:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.095026:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.095027:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.095028:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.095030:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.095033:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.095120:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.095123:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.095124:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.095127:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.095130:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.095132:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.095134:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96733139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.095136:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.095137:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.095138:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.095141:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.095143:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.095146:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.095147:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.095148:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.095150:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.095150:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.095151:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.095151:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.095264:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.095267:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.095268:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.095269:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.095271:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.095272:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.095386:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.095388:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.095389:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.095390:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.095393:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.095394:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.095655:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.095658:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.095658:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.095660:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.095663:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.095665:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.095666:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.095668:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.095669:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.095670:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.095673:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.095675:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.095678:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.095679:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.095680:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.095682:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.095683:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.095683:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.095684:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.095735:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.095738:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.095738:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.095740:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.095741:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.095743:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.095777:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.095780:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.095780:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.095781:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.095783:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.095786:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.096189:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.096192:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.096193:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.096193:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.096195:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.096198:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.096260:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.096263:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677447.096263:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677447.096266:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.096269:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.096271:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:22.0:1619677447.096272:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96733130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.096274:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.096276:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.096276:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677447.096281:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.096283:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.096286:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.096287:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:22.0:1619677447.096287:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.096289:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677447.096290:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.096291:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677447.096291:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677447.096297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.096310:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.096313:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:9.2:1619677447.096314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.096314:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.096316:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.096318:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.096320:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.096321:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96733128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.096323:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.096324:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.096325:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.096329:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.096331:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.096334:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.096335:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.096335:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.096338:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.096338:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.096339:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.096339:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.096344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.096363:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.096366:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.096367:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.096369:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.096372:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.096373:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.096375:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96733126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.096376:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.096377:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.096378:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.096382:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.096384:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.096387:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.096388:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.096388:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.096390:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.096391:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.096391:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.096392:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.096513:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.096516:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.096517:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.096517:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.096520:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.096522:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.096657:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.096660:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.096661:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.096663:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.096666:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.096667:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.096669:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.096670:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.096671:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.096672:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.096675:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.096677:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.096680:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.096681:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.096681:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.096682:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.096683:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.096684:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.096686:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.096686:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.096699:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.096701:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.096702:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.096703:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.096725:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.096728:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.096729:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.096729:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.096732:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.096738:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.096739:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.096741:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.096742:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.096742:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.096743:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.096743:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.096788:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.096791:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.096792:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.096794:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.096795:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.096797:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.096805:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.096808:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.096809:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.096809:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.096810:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.096811:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.096945:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.096947:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.096948:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.096949:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.096951:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.096952:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.096955:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:22.0:1619677447.096956:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:12.0:1619677447.096956:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.096957:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.096957:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677447.096959:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:6.0:1619677447.096959:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.096960:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677447.096960:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:6.0:1619677447.096962:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.096964:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.096966:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.096967:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96733116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.096968:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.096970:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:6.0:1619677447.096970:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.096971:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677447.096972:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.096973:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.096975:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.096975:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.096976:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677447.096977:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677447.096978:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.096980:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.096981:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.096982:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.096983:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.096984:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.096984:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.096985:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.097022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.097105:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.097108:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.097108:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677447.097110:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.097113:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.097115:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.097116:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.097118:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.097119:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.097120:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.097123:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.097125:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.097127:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.097128:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.097128:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.097130:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.097130:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.097131:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.097132:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.097133:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.097141:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.097144:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.097145:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.097146:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.097147:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.097148:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677447.097148:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.097148:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.097149:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.097150:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.097277:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.097279:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.097280:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.097280:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.097283:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.097286:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.097294:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.097299:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.097302:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.097303:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.097304:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.097304:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.097326:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.097329:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.097329:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.097331:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.097333:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.097334:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.097363:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.097366:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.097367:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.097368:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.097369:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.097370:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.097370:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.097372:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.097374:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.097375:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.097376:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.097376:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.097385:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.097388:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.097389:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.097391:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.097392:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.097393:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.097402:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.097404:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.097405:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.097406:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.097406:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.097407:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.097615:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.097618:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.097619:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.097621:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.097623:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.097625:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.097626:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96733104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.097629:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.097630:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.097631:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.097642:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.097644:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.097646:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.097647:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677447.097648:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (23)++
00000800:00000200:22.0:1619677447.097650:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:22.0:1619677447.097651:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.097652:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.097654:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.097654:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.097665:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.097667:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.097667:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.097668:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.097934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:22.0:1619677447.097938:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.097941:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:8.0:1619677447.097941:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:22.0:1619677447.097942:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.097942:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:8.0:1619677447.097943:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:22.0:1619677447.097944:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.097945:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.097945:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.097946:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:8.0:1619677447.097947:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.097949:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677447.097977:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.097980:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.097981:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.097983:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677447.097983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:24.0:1619677447.097985:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.097987:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.097988:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96733097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.097989:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.097990:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.097991:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.097994:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.097996:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.097999:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.098000:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.098001:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.098002:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.098003:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.098003:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.098004:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.098200:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.098202:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.098203:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.098206:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.098206:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.098209:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.098227:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.098229:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.098230:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.098230:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.098233:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.098235:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.098457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.098466:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.098469:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.098470:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.098470:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.098472:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.098475:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677447.098704:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.098706:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.098707:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.098709:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.098711:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.098713:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.098715:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733089] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.098716:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.098717:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.098718:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.098721:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.098723:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.098726:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.098727:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.098727:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.098729:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.098729:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.098730:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.098730:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.099358:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.099361:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.099362:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.099364:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.099366:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.099368:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.099369:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.099370:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.099371:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.099372:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.099375:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.099377:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.099379:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.099380:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.099381:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.099383:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.099383:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.099384:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.099384:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.099390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677447.099416:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.099419:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.099420:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.099422:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.099425:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.099427:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.099428:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.099429:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.099430:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.099431:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.099435:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.099437:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.099440:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.099440:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.099441:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.099443:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.099443:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.099444:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.099445:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.099513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.099520:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.099522:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.099523:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.099523:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.099526:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.099529:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.099530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677447.099536:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.099539:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.099540:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.099540:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.099547:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.099547:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.099564:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.099567:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.099568:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.099568:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.099570:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.099572:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.099634:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.099637:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.099638:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.099638:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.099640:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.099642:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.099653:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.099655:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.099656:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.099657:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.099657:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.099658:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.099768:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.099771:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.099771:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.099772:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.099774:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.099777:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.100394:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677447.100396:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677447.100397:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:6.0:1619677447.100400:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.100403:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.100405:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677447.100406:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.100408:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.100409:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.100410:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677447.100413:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.100415:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677447.100417:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.100418:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:6.0:1619677447.100419:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677447.100421:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:6.0:1619677447.100421:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677447.100422:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:6.0:1619677447.100423:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:6.0:1619677447.100424:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677447.100434:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000800:00000200:22.0:1619677447.100436:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:22.0:1619677447.100437:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677447.100438:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:24.0:1619677447.100450:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.100453:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.100454:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.100455:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.100458:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.100459:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.100461:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.100462:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.100463:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.100464:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.100468:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.100470:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.100475:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.100476:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.100476:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.100478:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.100478:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.100480:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.100481:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.100481:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.100494:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.100496:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.100497:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.100498:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.100629:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.100632:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.100632:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.100633:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.100633:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.100634:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.100767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.100787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:12.0:1619677447.100868:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677447.100871:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677447.100871:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.100873:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.100875:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.100876:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677447.100876:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.100876:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.100878:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.100878:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.100880:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.100888:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.100892:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.100893:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.100895:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.100897:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.100899:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.100900:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.100902:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.100903:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.100904:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.100908:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.100909:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.100912:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.100913:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.100914:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.100916:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.100917:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.100917:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.100918:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.100948:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.100951:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.100952:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.100952:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.100955:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.100955:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:22.0:1619677447.100958:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.100958:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.100959:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.100960:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.100962:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.100963:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.101062:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.101064:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.101065:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.101067:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.101068:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.101069:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.101080:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.101082:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.101083:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.101083:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.101084:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.101085:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.101383:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.101385:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.101386:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.101387:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.101389:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.101393:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.101411:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.101414:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.101415:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.101417:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.101418:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.101419:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.101454:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.101457:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.101458:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.101459:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.101460:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.101462:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.101517:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.101519:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.101520:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.101522:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.101523:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.101524:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.101603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.101735:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.101738:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.101739:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.101741:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.101743:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.101745:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.101746:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.101748:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.101748:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.101750:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.101753:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.101755:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.101758:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.101759:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.101760:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.101761:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.101762:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.101763:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.101764:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.101866:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.101869:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.101869:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.101871:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.101873:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.101873:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677447.101984:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.101987:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.101988:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.101988:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.101991:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.101994:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.102026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.102168:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.102171:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.102172:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.102174:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.102177:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.102178:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677447.102180:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.102181:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.102182:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.102183:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677447.102186:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.102187:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677447.102190:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.102191:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677447.102191:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677447.102193:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677447.102193:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.102194:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.102195:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.102206:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.102209:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.102210:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.102211:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.102213:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.102216:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.102225:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.102227:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.102228:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.102228:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.102229:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.102230:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.102260:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.102263:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.102264:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.102266:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.102269:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.102270:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.102271:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96733050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.102273:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.102274:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.102275:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.102278:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.102280:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.102283:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.102283:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.102284:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.102286:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.102286:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.102287:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.102288:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.102289:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.102303:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.102306:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.102306:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.102307:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.102397:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.102400:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.102401:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.102401:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.102404:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.102407:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.102417:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.102420:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.102420:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.102421:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.102422:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.102422:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.102454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.102467:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.102470:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.102471:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.102472:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.102476:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.102477:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.102478:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.102480:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.102481:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.102482:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.102485:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.102488:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.102490:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.102491:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.102492:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.102493:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.102494:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.102495:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.102495:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677447.102619:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677447.102622:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677447.102623:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677447.102625:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.102628:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.102630:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677447.102631:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.102633:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.102635:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.102636:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677447.102642:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.102644:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677447.102647:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.102647:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677447.102648:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (22)++
00000800:00000200:24.0:1619677447.102650:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (23)--
00000800:00000200:24.0:1619677447.102650:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677447.102651:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677447.102651:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:22.0:1619677447.102660:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:22.0:1619677447.102663:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:6.0:1619677447.102663:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (24)++
00000400:00000200:22.0:1619677447.102664:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.102664:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000400:00000200:6.0:1619677447.102665:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.102666:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.102667:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.102670:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.102670:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.102671:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.102826:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.102829:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.102830:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.102832:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.102833:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.102835:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.102879:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.102882:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.102883:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.102884:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.102885:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.102887:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.102899:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.102902:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.102903:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.102903:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.102904:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.102905:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.103051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.103058:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.103060:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.103061:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.103062:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.103064:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.103067:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.103180:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.103182:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.103183:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.103183:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.103185:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.103188:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.103242:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.103245:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.103246:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.103247:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.103249:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.103252:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.103260:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.103262:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.103263:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.103263:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.103264:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.103265:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.103341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.103363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677447.103431:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.103434:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.103434:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.103436:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.103438:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.103439:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.103611:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.103613:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.103615:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.103616:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.103619:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.103620:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.103622:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96733031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.103624:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.103625:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.103626:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.103629:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.103631:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.103634:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.103634:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.103635:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677447.103637:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (23)--
00000800:00000200:24.0:1619677447.103638:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.103638:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.103639:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.103703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.103710:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.103712:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.103713:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.103714:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.103716:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.103719:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677447.103893:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.103896:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.103896:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.103898:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.103900:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.103900:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.103959:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.103962:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.103962:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.103966:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.103969:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.103970:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.103972:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96733025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.103973:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.103974:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.103975:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.103979:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.103981:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.103983:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.103984:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.103984:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.103986:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.103987:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.103987:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.103988:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.103992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.104095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.104101:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.104104:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.104105:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.104105:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.104107:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.104110:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677447.104134:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.104137:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.104137:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.104138:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.104140:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.104143:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.104252:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.104255:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.104255:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677447.104257:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.104260:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.104262:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.104263:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.104264:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.104265:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.104266:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.104269:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.104271:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.104274:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.104275:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.104275:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.104277:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.104278:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.104278:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.104279:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.104472:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.104474:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.104475:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.104475:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.104477:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.104479:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.104676:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.104679:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.104679:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.104682:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.104684:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.104685:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.104687:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96733017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.104688:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677447.104688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:24.0:1619677447.104689:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.104691:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.104695:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.104696:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.104697:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677447.104697:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677447.104697:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.104698:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.104699:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.104700:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.104700:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.104701:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.104702:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.104702:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:13.0:1619677447.104702:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.104703:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.104704:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:12.0:1619677447.104704:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.104705:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677447.104705:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.104705:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.104706:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.104706:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.104707:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.104708:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.104708:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.104708:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.104709:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (24)--
00000800:00000200:24.0:1619677447.104710:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.104714:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677447.104715:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.104718:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.104718:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.104719:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.104719:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.104720:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.104796:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.104799:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.104800:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.104800:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.104803:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.104805:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.104859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.105221:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.105224:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.105225:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.105225:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.105227:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.105230:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.105250:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.105252:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.105253:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.105255:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.105257:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.105258:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.105260:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96733011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.105261:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.105262:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.105263:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.105266:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.105268:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.105271:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.105271:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.105272:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.105274:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.105274:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.105275:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.105275:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.105280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:24.0:1619677447.105353:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.105356:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.105357:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.105359:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.105362:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.105363:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.105364:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96733010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.105366:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.105367:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.105368:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.105371:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.105373:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.105376:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.105377:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.105377:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.105379:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.105380:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.105381:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.105382:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.105383:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.105400:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.105402:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.105403:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.105404:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.105431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.105437:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.105440:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.105441:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.105441:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.105443:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:24.0:1619677447.105446:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677447.105461:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.105464:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.105464:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.105466:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.105468:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.105469:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.105561:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.105564:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.105565:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.105567:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.105570:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.105571:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.105573:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96733006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.105574:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.105575:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.105576:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.105580:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.105582:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.105585:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.105586:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.105587:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.105589:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.105589:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.105590:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.105591:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.105699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.105713:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.105716:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.105717:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.105717:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.105719:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.105722:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.105730:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.105733:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.105734:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.105734:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.105735:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.105735:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.105984:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.105987:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.105988:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.105989:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.105991:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.105992:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.106214:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)++
00000800:00000200:13.0:1619677447.106217:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677447.106218:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.106220:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.106221:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.106222:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.106222:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677447.106223:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.106224:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.106224:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.106227:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.106244:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.106247:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.106248:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.106248:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.106249:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.106251:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.106298:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.106301:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.106302:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.106304:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.106305:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.106307:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.106308:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.106310:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.106311:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.106312:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.106316:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.106318:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.106322:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.106323:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.106323:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919861b98400] (21)++
00000800:00000200:22.0:1619677447.106324:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.106325:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.56@o2ib3
00000800:00000200:22.0:1619677447.106326:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.106327:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:6.0:1619677447.106327:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.4.56@o2ib3 : GET - for me
00000800:00000200:22.0:1619677447.106328:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.106328:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.106329:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b98400] (22)--
00000400:00000200:6.0:1619677447.106330:0:19937:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.4.56@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:6.0:1619677447.106333:0:19937:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.4.56@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:6.0:1619677447.106334:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919861b98400] (21)++
00000800:00000200:6.0:1619677447.106335:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919861b98400] (22)--
00000400:00000200:6.0:1619677447.106337:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.4.56@o2ib3 local destination
00000400:00000200:6.0:1619677447.106341:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.4.56@o2ib3(10.51.4.56@o2ib3:10.51.4.56@o2ib3) : REPLY try# 0
00000800:00000200:6.0:1619677447.106342:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.4.56@o2ib3
00000800:00000200:6.0:1619677447.106344:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919876b8ff80] -> 10.51.4.56@o2ib3 (2) version: 12
00000800:00000200:6.0:1619677447.106345:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919861b98400] (21)++
00000800:00000200:6.0:1619677447.106346:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919861b98400] (22)++
00000800:00000200:6.0:1619677447.106347:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919861b98400] (23)--
00000800:00000200:6.0:1619677447.106347:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b98400] (22)--
00000800:00000200:22.0:1619677447.106359:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919861b98400] (22)++
00000800:00000200:22.0:1619677447.106361:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919861b98400] (23)--
00000400:00000200:22.0:1619677447.106362:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.106362:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b98400] (22)--
00000400:00000200:22.0:1619677447.106364:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.4.56@o2ib3: REPLY: OK
00000800:00000200:22.0:1619677447.106367:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b98400] (21)--
00000800:00000200:24.0:1619677447.106544:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.106547:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.106548:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.106550:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.106553:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.106554:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.106555:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.106557:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.106558:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.106559:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.106562:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.106563:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.106566:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.106567:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.106568:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.106569:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.106570:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.106571:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.106571:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.106604:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.106607:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.106608:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.106610:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.106613:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.106614:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677447.106615:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.106617:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.106617:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.106618:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677447.106621:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.106623:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.106626:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.106626:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677447.106627:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (23)++
00000800:00000200:6.0:1619677447.106627:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:8.0:1619677447.106630:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (25)--
00000800:00000200:8.0:1619677447.106630:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:6.0:1619677447.106631:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:22.0:1619677447.106632:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:8.0:1619677447.106632:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000400:00000200:6.0:1619677447.106632:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.106633:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.106633:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:6.0:1619677447.106634:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.106638:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.106640:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.106642:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.106643:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.106643:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.106645:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.106646:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:6.0:1619677447.106647:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.106648:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.106649:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.106649:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.106710:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.106712:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.106713:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.106715:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.106717:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.106718:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.106733:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.106735:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.106736:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.106738:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.106739:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.106740:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.106774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.106813:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.106816:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.106817:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.106819:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.106820:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.106824:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.106840:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:9.2:1619677447.106841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:12.0:1619677447.106842:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.106843:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.106844:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.106846:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.106846:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.106848:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.106850:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.106851:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.106851:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.106853:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.106856:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677447.106859:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.106862:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.106863:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.106863:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.106864:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.106865:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.107051:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.107054:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.107055:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.107055:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.107058:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.107061:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.107075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.107081:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.107084:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.107085:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.107086:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.107088:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.107091:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677447.107164:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.107167:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.107168:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.107170:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.107172:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.107173:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.107174:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.107176:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.107177:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.107178:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.107181:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.107183:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.107186:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.107187:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.107187:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.107189:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.107190:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.107191:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.107192:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.107192:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.107202:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.107204:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.107205:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.107206:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.107338:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677447.107341:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677447.107342:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677447.107344:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677447.107346:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.107347:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677447.107349:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.107350:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.107351:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.107352:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677447.107355:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677447.107357:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.107358:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.107359:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.107360:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.107361:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.107361:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677447.107362:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677447.107363:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677447.107367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.107373:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.107375:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.107376:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.107377:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.107378:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677447.107380:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.107392:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.107395:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.107396:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.107396:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.107399:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.107401:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.107409:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.107412:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.107413:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.107413:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.107414:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.107415:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.107609:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.107611:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.107612:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.107614:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.107616:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.107616:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.107618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.107637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677447.107717:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677447.107720:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677447.107720:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000400:00000200:6.0:1619677447.107722:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.107725:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.107727:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:6.0:1619677447.107729:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.107730:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.107731:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.107732:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:6.0:1619677447.107736:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.107738:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:6.0:1619677447.107741:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.107742:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:6.0:1619677447.107742:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:6.0:1619677447.107744:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:6.0:1619677447.107745:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.107746:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677447.107746:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677447.107750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.107757:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.107760:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.107761:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.107761:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.107763:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.107766:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.108058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.108071:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.108074:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.108075:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.108076:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.108077:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.108081:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.108133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.108139:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.108142:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.108143:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.108143:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.108145:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.108148:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.108151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.108157:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.108160:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.108161:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.108161:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.108162:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.108163:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677447.108188:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.108190:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.108191:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677447.108192:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677447.108193:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.108195:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.108196:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.108196:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.108198:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.108198:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:12.0:1619677447.108199:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:8.0:1619677447.108199:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.108200:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.108201:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677447.108202:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.108203:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.108206:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.108208:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677447.108211:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.108211:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.108212:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.108212:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677447.108214:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.108214:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677447.108214:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677447.108215:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677447.108215:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.108215:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:12.0:1619677447.108216:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.108217:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.108217:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.108327:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.108330:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.108330:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.108331:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.108333:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.108336:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.108389:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.108391:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.108392:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.108392:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.108394:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.108397:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.108530:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.108532:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.108533:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.108535:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.108538:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.108539:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.108541:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.108542:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.108543:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.108544:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.108547:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.108549:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.108551:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.108552:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.108553:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.108554:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.108555:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.108556:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.108556:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.108728:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.108731:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.108732:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.108732:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.108734:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.108736:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.108797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.108918:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.108920:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.108921:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.108922:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.108924:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.108931:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.108933:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.108935:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.108935:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.108936:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.108937:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.108938:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.108959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:13.0:1619677447.108978:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.108981:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.108981:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.108982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000400:00000200:13.0:1619677447.108983:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.108985:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.108986:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.109310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.109317:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.109320:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.109321:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.109321:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.109323:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.109326:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.109329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:8.0:1619677447.109335:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.109337:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.109338:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.109338:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.109339:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.109340:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677447.109688:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.109690:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.109691:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.109692:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.109694:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.109695:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.109778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.109787:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.109789:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.109790:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.109790:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:24.0:1619677447.109792:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:8.0:1619677447.109792:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.109795:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.109795:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.109796:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677447.109797:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.109799:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.109801:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.109802:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732945] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.109804:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.109804:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.109806:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.109809:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.109811:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.109814:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.109814:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.109815:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.109817:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.109817:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.109818:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.109818:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.109867:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.109869:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.109871:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.109872:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.109874:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.109874:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.109900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.109921:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.109923:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.109923:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.109924:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:22.0:1619677447.109925:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.109927:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.109927:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.109930:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.109930:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.109931:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.109932:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.109932:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.109933:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.109934:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.109936:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.109937:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.109940:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.109942:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.109945:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.109946:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.109946:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.109948:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.109949:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.109949:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.109950:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.109996:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.109998:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.109999:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.109999:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.110002:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.110004:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.110014:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.110016:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.110017:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.110017:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.110018:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.110018:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.110117:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.110120:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.110121:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.110123:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.110126:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.110128:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.110129:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.110130:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.110131:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.110133:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.110136:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.110138:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.110141:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.110142:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.110143:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.110145:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.110145:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.110146:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.110146:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.110150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.110169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:22.0:1619677447.110309:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.110312:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.110312:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.110316:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.112@o2ib7 : GET - for me
00000400:00000200:22.0:1619677447.110320:0:19936:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.112@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:22.0:1619677447.110322:0:19936:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.112@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:22.0:1619677447.110324:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.110324:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.110326:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:22.0:1619677447.110330:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : REPLY try# 0
00000800:00000200:22.0:1619677447.110333:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.110334:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.110335:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.110336:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.110338:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.110338:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:9.2:1619677447.110339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.110347:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.110350:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.110351:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.110351:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:13.0:1619677447.110352:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.110353:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:12.0:1619677447.110354:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:6.0:1619677447.110354:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.110355:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:13.0:1619677447.110355:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:6.0:1619677447.110357:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: REPLY: OK
00000800:00000200:6.0:1619677447.110359:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.110579:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.110582:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.110584:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.110585:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.110588:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.110589:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.110591:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.110593:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.110594:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.110595:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.110599:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.110602:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.110605:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.110605:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.110606:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.110608:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.110609:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.110609:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.110610:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.110614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677447.110619:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.110621:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.110622:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.110624:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.110627:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677447.110627:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.110629:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:8.0:1619677447.110629:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:12.0:1619677447.110630:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.110631:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677447.110632:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.110633:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:8.0:1619677447.110633:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.110634:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677447.110634:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.110635:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.110638:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.110640:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.110643:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.110643:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.110644:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.110646:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.110646:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.110647:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.110648:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.110649:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.110661:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.110663:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.110664:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.110665:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.110714:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.110716:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.110717:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.110719:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.110720:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.110721:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.110730:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.110733:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.110734:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.110734:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.110735:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.110735:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.110770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:12.0:1619677447.111120:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.111122:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.111123:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.111125:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.111126:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.111128:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.111254:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.111257:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.111258:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.111260:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.111261:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.111263:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.111421:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.111424:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.111424:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.111426:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.111429:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.111431:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.111432:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.111433:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.111435:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.111436:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.111439:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.111441:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.111444:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.111445:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.111445:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.111447:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.111447:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.111448:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.111449:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.111492:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.111495:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.111496:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.111497:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.111499:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.111500:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.111518:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.111521:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.111522:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.111524:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.111526:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.111528:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.111529:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.111531:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.111531:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.111533:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.111536:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.111538:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.111541:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.111542:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.111543:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.111545:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.111545:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.111547:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.111548:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.111548:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.111558:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.111560:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.111561:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.111562:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.111730:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.111733:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.111734:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.111736:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.111737:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.111738:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.111759:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.111762:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.111763:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.111763:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.111764:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.111765:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.111774:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.111777:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.111778:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.111779:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.111781:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.111784:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.111910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.112008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.112015:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.112017:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.112018:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.112019:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.112021:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.112024:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677447.112157:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.112161:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.112161:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.112163:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.112164:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.112166:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.112226:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.112229:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.112229:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.112231:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.112234:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.112235:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.112236:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.112238:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.112239:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.112240:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.112243:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.112245:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.112247:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.112248:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.112249:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.112250:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.112251:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.112251:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.112252:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.112274:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.112276:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.112278:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.112278:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.112280:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.112283:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.112497:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.112500:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.112501:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.112502:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.112504:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.112506:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.112678:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.112681:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.112681:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.112683:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.112685:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.112686:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.112694:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.112696:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.112697:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.112698:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.112698:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.112700:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.112728:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.112730:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.112731:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.112732:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.112734:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.112737:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.112758:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.112760:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.112761:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.112763:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.112764:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.112766:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.112780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.113086:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.113089:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.113090:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.113091:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.113093:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.113096:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.113098:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.113101:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.113102:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.113104:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.113105:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.113106:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.113394:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.113397:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.113398:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.113400:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.113402:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.113404:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.113405:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.113407:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.113408:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.113410:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.113411:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.113414:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:22.0:1619677447.113415:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.113415:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677447.113417:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.113418:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.113420:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.113421:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.113421:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.113421:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:24.0:1619677447.113423:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:22.0:1619677447.113423:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:24.0:1619677447.113425:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96732892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.113426:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.113428:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.113429:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.113429:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:24.0:1619677447.113430:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.113430:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:24.0:1619677447.113434:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.113437:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.113440:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.113441:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.113441:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.113443:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.113444:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.113444:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.113445:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.113452:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:13.0:1619677447.113455:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677447.113455:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.113456:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.113456:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.113458:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:6.0:1619677447.113458:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:13.0:1619677447.113459:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.113460:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.113460:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.113463:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.113463:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677447.113471:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.113474:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.113474:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.113475:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.113476:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.113477:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.113547:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.113550:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.113551:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.113553:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.113554:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.113556:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.113566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.113586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677447.113709:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.113711:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.113712:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.113714:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.113715:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.113717:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.113808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.114089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.114090:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.114093:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.114093:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.114096:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.114098:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.114100:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:12.0:1619677447.114101:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:8.0:1619677447.114101:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96732883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.114103:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:8.0:1619677447.114103:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677447.114104:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.114104:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.114105:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677447.114106:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.114107:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:8.0:1619677447.114109:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.114111:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.114111:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.114114:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.114114:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.114115:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.114117:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.114118:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.114118:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.114119:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.114206:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.114208:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.114209:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.114209:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.114212:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.114215:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.114218:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.114221:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.114222:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.114223:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.114224:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:13.0:1619677447.114225:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.114225:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677447.114227:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.114227:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.114228:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.114230:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.114230:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.114234:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:9.2:1619677447.114236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:12.0:1619677447.114237:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.114238:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.114239:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.114239:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.114241:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.114243:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.114245:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.114246:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.114247:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.114249:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.114252:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.114382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.114562:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.114566:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.114566:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.114567:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.114569:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.114572:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.114605:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.114608:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.114609:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.114611:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.114613:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.114615:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.114616:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96732875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.114617:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.114618:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.114619:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.114622:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.114623:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.114626:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.114626:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.114627:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.114629:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.114629:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.114630:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.114631:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.114707:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.114710:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.114710:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.114712:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.114714:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.114715:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.114986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.115009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677447.115071:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.115074:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.115075:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.115075:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.115077:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.115080:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.115331:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.115334:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.115335:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.115337:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.115340:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.115341:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.115343:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.115344:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.115346:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.115347:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.115352:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.115354:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.115357:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.115358:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.115359:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.115361:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.115362:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.115363:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.115364:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.115368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.115371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:12.0:1619677447.115420:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.115422:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.115423:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.115425:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.115427:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.115428:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.115430:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.115431:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.115432:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.115432:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.115433:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.115474:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.115477:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.115477:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.115480:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.115482:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.115483:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.115485:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.115486:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.115487:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.115488:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.115491:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.115493:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.115496:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.115497:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.115498:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.115500:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.115500:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.115501:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.115501:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.115538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.115680:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.115683:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.115684:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.115685:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.115688:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.115689:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.115691:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.115692:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.115693:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.115694:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.115697:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.115699:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.115702:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.115702:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.115703:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.115704:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.115705:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.115706:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.115707:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.115708:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.115721:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.115723:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.115724:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.115724:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677447.115848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.115854:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.115857:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:9.2:1619677447.115862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.115862:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.115863:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.115865:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.115868:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677447.115891:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.115894:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.115895:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.115897:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.115898:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.115899:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.115933:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.115936:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.115936:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.115936:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.115939:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.115941:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.116005:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.116007:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.116008:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.116010:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.116011:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.116013:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.116099:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.116102:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.116102:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.116103:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.116105:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.116108:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.116310:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.116313:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.116314:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.116316:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.116319:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.116320:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.116322:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.116323:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.116324:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.116325:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.116328:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.116330:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.116333:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.116334:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.116334:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.116336:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.116336:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.116337:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.116337:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.116377:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.116380:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.116381:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.116383:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.116384:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.116386:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.116516:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.116519:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.116520:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.116523:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.116526:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.116528:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.116529:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96732849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.116531:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.116532:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.116533:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.116536:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.116538:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.116541:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.116541:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.116542:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.116544:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.116544:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.116546:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.116547:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.116547:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.116550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.116557:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.116559:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.116560:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.116561:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.116568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.116585:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677447.116588:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677447.116589:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677447.116591:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.116593:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.116594:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.116596:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96732847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.116597:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.116598:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.116599:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:12.0:1619677447.116600:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:22.0:1619677447.116602:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.116602:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.116603:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.116604:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:12.0:1619677447.116605:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.116606:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677447.116607:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.116607:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:12.0:1619677447.116607:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.116608:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.116610:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.116610:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677447.116611:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677447.116612:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:13.0:1619677447.116622:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.116625:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.116625:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.116626:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.116627:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.116628:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.116701:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.116704:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.116704:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677447.116707:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.116709:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.116711:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.116712:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.116714:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.116715:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.116717:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.116722:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.116724:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.116728:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.116728:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.116729:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.116731:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.116732:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.116733:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.116734:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.116735:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677447.116737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.116747:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.116749:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.116750:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.116751:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.116755:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.116758:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.116759:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.116761:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.116761:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.116764:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.116779:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.116782:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.116783:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.116785:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.116786:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.116787:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.116944:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.116946:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.116947:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.116949:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.116950:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.116951:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.116973:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.116975:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.116976:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.116978:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.116981:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.116982:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.116983:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.116984:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.116986:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.116987:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.116989:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.116991:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.116994:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.116995:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.116995:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.116997:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.116998:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.116998:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.116999:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.117128:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.117131:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.117132:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.117134:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.117134:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.117136:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.117449:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.117452:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.117452:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.117455:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.117457:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.117459:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.117460:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.117462:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.117463:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.117464:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.117468:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.117468:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677447.117470:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.117471:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.117472:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677447.117473:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.117475:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.117475:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:6.0:1619677447.117476:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.117477:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.117477:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.117478:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677447.117478:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.117479:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677447.117480:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.117481:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96732832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.117483:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:6.0:1619677447.117483:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.117484:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.117485:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:6.0:1619677447.117485:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:12.0:1619677447.117486:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.117489:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.117489:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.117490:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.117491:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.117492:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.117494:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.117495:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.117496:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.117498:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.117498:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.117499:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.117499:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.117502:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.117505:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.117506:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.117506:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.117507:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.117508:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.117651:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.117653:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.117654:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.117656:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.117657:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.117658:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.117683:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.117686:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.117687:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.117687:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.117689:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.117696:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.117697:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.117699:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.117700:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.117701:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.117701:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.117702:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.117931:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.117934:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.117935:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.117937:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.117937:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.117939:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.117954:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.117957:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.117958:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.117958:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.117959:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.117960:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.117968:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.117971:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.117972:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.117974:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.117977:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.117979:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.117980:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.117981:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.117982:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.117983:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.117986:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.117988:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.117991:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.117991:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.117992:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.117994:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.117994:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.117995:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.117996:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677447.118016:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.118018:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.118019:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.118021:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.118024:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.118025:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.118027:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.118028:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.118029:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.118030:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.118033:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.118034:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.118037:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.118037:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.118038:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.118039:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.118040:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.118041:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.118042:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.118043:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.118055:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.118058:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.118058:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.118059:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.118098:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.118101:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.118102:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.118102:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.118105:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.118108:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.118151:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.118153:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.118154:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.118155:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.118157:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.118158:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.118415:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.118418:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.118419:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.118420:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.118422:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.118423:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.118569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.118576:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.118579:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.118579:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.118580:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.118582:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.118585:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677447.118620:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.118622:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.118623:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.118624:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.118626:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.118629:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.118641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.118714:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.118717:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.118717:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.118719:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.118722:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.118723:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.118725:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.118727:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.118728:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.118730:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.118734:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.118737:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.118739:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.118740:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.118741:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.118743:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.118743:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.118744:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.118744:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.118969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.118975:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (23)++
00000800:00000200:24.0:1619677447.118977:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (24)--
00000400:00000200:24.0:1619677447.118978:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.118978:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (23)++
00000400:00000200:24.0:1619677447.118980:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.118980:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (24)--
00000400:00000200:8.0:1619677447.118981:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.118982:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.118983:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.118984:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.118985:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.118994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677447.119000:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.119003:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.119003:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.119004:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.119005:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.119005:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677447.119092:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.119094:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.119095:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.119096:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.119098:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.119098:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.119138:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.119141:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.119142:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.119143:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.119145:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.119146:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.119156:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.119159:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.119160:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.119161:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.119161:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.119163:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.119301:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.119303:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.119304:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.119304:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.119307:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.119309:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.119398:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.119400:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.119401:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.119402:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.119404:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.119407:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.119446:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.119448:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.119449:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.119451:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.119453:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.119454:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.119511:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.119514:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.119514:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.119515:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.119517:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.119520:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.119583:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.119586:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.119587:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.119588:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.119591:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.119592:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.119594:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.119595:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.119596:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.119597:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.119600:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.119601:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.119604:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.119604:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.119605:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.119606:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.119607:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.119607:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.119608:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.119618:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.119621:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.119621:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.119623:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.119624:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.119626:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.119701:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.119704:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.119706:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.119709:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.119711:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.119712:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.119714:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.119716:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.119717:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.119718:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.119726:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.119730:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.119733:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.119734:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.119735:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.119736:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.119737:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.119738:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.119738:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.119742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.119750:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.119753:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.119753:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.119755:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.119758:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.119766:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677447.120168:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.120170:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.120171:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.120173:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.120175:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.120177:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.120178:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.120179:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.120180:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.120181:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.120184:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.120186:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.120189:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.120189:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.120190:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.120191:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.120192:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.120193:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.120193:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.120223:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.120225:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.120226:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.120227:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.120229:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.120232:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.120239:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.120242:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.120243:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.120243:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.120244:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.120245:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.120449:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.120451:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.120452:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.120453:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.120454:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.120457:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.120464:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.120467:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.120468:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.120469:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.120469:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.120470:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.120601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.120607:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.120609:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.120610:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.120611:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.120612:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.120615:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.120657:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.120659:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.120660:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.120662:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.120664:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.120665:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.120753:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.120756:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.120756:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.120758:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.120761:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.120762:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.120764:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.120765:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.120766:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.120768:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.120773:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.120775:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.120778:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.120778:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.120779:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.120781:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.120781:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.120783:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.120784:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.120784:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.120787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.120794:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.120796:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.120797:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.120798:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.120852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.121174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.121175:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.121178:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.121179:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.121181:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677447.121182:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.121183:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.121191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:12.0:1619677447.121194:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.121197:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.121198:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.121198:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.121199:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.121201:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.121384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677447.121621:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.121624:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.121625:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.121627:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.121630:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.121631:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.121632:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.121634:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.121634:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.121635:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.121638:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.121640:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677447.121640:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677447.121642:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677447.121642:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677447.121643:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.121643:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.121644:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:13.0:1619677447.121645:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.121646:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.121646:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677447.121647:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.121647:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.121647:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.121649:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.121720:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.121723:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.121724:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.121725:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.121726:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.121728:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.121934:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.121937:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.121938:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.121940:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.121941:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.121942:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.121950:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.121953:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.121954:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.121956:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.121959:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.121960:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.121961:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.121963:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.121964:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.121965:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.121969:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.121970:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.121974:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.121974:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.121975:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.121977:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.121978:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.121979:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.121979:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.122056:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.122059:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.122060:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.122060:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.122062:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.122065:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.122215:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.122218:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.122219:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.122221:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.122224:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.122225:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677447.122226:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.122228:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.122229:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.122230:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677447.122233:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.122235:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677447.122238:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.122239:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677447.122239:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677447.122241:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677447.122242:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.122243:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.122243:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.122254:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.122257:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.122258:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.122258:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.122261:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.122264:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.122394:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.122397:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.122398:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.122400:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.122403:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:8.0:1619677447.122403:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.122404:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.122405:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.122407:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:8.0:1619677447.122407:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.122408:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.122408:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.122408:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:8.0:1619677447.122409:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.122410:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.122412:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:8.0:1619677447.122413:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.122415:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.122418:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.122419:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.122419:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.122421:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.122421:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.122422:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.122423:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.122853:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.122854:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.122855:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.122856:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.122856:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.122858:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.122859:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.122861:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.122862:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.122862:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.122863:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.122864:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.122866:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732753] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.122867:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.122868:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.122869:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.122873:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.122875:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.122878:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.122879:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.122879:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.122881:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.122882:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.122883:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.122883:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.122914:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.122917:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.122918:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.122920:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.122923:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.122924:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.122925:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.122927:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.122928:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.122929:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.122932:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.122934:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.122937:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.122938:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.122938:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.122940:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.122941:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.122942:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.122943:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.122944:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.122956:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.122958:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.122959:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.122959:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677447.122987:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.122990:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.122991:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:13.0:1619677447.122991:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:6.0:1619677447.122991:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.122993:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:6.0:1619677447.122993:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:13.0:1619677447.122994:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.122996:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.122996:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.122998:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.122998:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677447.123010:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.123013:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.123014:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.123014:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.123015:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.123015:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.123132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.123139:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.123141:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.123142:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.123143:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.123145:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.123148:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.123152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677447.123158:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.123161:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.123162:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.123162:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.123163:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.123163:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.123314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.123352:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.123355:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.123356:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.123358:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.123360:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.123361:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.123390:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.123392:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.123393:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.123394:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.123396:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.123397:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.123493:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.123497:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.123498:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.123499:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.123502:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.123504:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.123505:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.123507:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.123507:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.123508:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.123511:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.123513:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.123516:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.123516:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.123517:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.123518:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.123519:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.123520:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.123520:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.123560:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.123564:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.123565:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.123565:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.123567:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.123570:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.123729:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.123732:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.123733:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.123735:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.123736:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.123737:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.123748:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.123750:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.123751:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.123752:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.123752:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.123753:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.123840:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.123843:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.123844:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.123845:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.123846:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.123848:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.124097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.124104:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.124107:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.124108:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.124108:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.124110:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.124114:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677447.124146:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.124149:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.124149:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.124151:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.124154:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.124156:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.124157:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.124158:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.124159:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.124160:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.124163:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.124165:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.124168:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.124168:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.124169:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.124171:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.124171:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.124172:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.124172:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.124306:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.124309:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.124310:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.124310:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.124312:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.124315:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.124403:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.124406:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.124407:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.124409:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.124412:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.124413:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.124415:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.124416:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.124417:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.124418:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.124422:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.124423:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.124426:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.124427:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.124428:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.124430:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.124430:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.124433:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.124437:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.124439:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.124439:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.124439:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.124442:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.124443:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.124443:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.124444:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.124445:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.124446:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:24.0:1619677447.124447:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.124449:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.124451:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.124453:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.124454:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.124456:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.124462:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.124464:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.124467:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.124468:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.124469:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.124471:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.124471:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.124472:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.124473:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.124481:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.124484:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.124485:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.124485:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.124487:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.124489:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.124499:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.124502:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.124502:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.124503:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.124504:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.124504:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.124629:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.124632:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.124633:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.124633:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.124635:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.124638:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.124951:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.124954:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.124955:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.124957:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.124960:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.124961:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:8.0:1619677447.124962:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.124963:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96732721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.124964:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.124964:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.124965:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.124965:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.124966:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.124968:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:6.0:1619677447.124968:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.124971:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:8.0:1619677447.124972:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.124977:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.124978:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.124980:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677447.124980:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677447.124981:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.124981:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:8.0:1619677447.124982:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000400:00000200:12.0:1619677447.124983:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.124983:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:8.0:1619677447.124983:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.124984:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677447.124985:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677447.124985:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.124987:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.124995:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.124997:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.124998:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.124999:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:13.0:1619677447.125000:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.125001:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:22.0:1619677447.125002:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:13.0:1619677447.125002:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:22.0:1619677447.125003:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.125003:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.125003:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677447.125003:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:12.0:1619677447.125004:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.125005:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.125005:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:22.0:1619677447.125006:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.125008:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.125092:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.125094:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.125095:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.125096:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.125099:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.125102:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.125273:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.125276:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.125277:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.125277:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.125280:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.125282:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.125293:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.125299:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.125300:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.125302:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.125302:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.125303:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.125304:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.125304:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:22.0:1619677447.125305:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.125305:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.125308:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.125310:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.125311:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96732713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.125312:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.125313:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.125314:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.125319:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.125321:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.125322:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.125324:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.125324:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.125325:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:6.0:1619677447.125325:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.125326:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:6.0:1619677447.125327:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.125328:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.125328:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.125330:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.125330:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.125331:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.125331:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.125331:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.125342:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.125343:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.125344:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.125345:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.125572:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.125574:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.125575:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.125577:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.125579:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.125580:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.125715:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677447.125717:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677447.125718:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.125719:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.125721:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.125722:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677447.125722:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677447.125723:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.125724:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.125724:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.125726:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.125743:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.125746:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.125747:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.125747:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.125748:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.125749:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.125839:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.125842:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677447.125845:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.125846:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.125848:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.125849:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.125851:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.125852:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.125854:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.125856:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.125860:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.125863:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.125867:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.125868:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.125869:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (23)++
00000800:00000200:24.0:1619677447.125871:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (24)--
00000800:00000200:24.0:1619677447.125872:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.125873:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.125873:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.125905:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.125908:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.125909:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.125910:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.125912:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.125913:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.126064:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.126066:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.126068:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.126068:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.126070:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.126072:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.126082:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.126084:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.126085:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.126086:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.126087:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.126087:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.126510:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.126513:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.126513:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.126515:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.126517:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.126518:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.126589:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.126592:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.126593:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.126595:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.126598:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.126599:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.126600:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.126602:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.126603:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.126604:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.126607:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.126609:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.126611:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.126612:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.126613:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.126614:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.126615:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.126616:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.126616:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.126621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.126707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.126764:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.126767:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.126768:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.126768:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.126770:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.126774:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.126905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.126973:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.126976:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.126977:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.126978:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.126981:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.126983:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.126984:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.126986:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.126987:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.126989:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.126992:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.126994:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.126996:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.126997:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.126998:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.127000:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.127001:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.127001:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.127002:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.127006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.127025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:22.0:1619677447.127058:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.127061:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.127062:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.127062:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.127065:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.127068:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.127092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.127288:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.127291:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.127292:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.127293:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.127304:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.127307:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.127421:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.127423:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.127424:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.127426:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.127429:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.127430:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.127431:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.127432:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.127434:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.127434:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.127438:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.127439:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.127442:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.127442:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.127443:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.127445:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.127445:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.127446:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.127446:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.127451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.127471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:24.0:1619677447.127486:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.127489:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.127490:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.127492:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.127495:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.127496:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.127498:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.127499:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.127500:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.127501:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.127504:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.127506:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.127509:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.127510:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.127510:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.127512:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.127512:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.127514:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.127515:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.127515:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.127528:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.127530:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.127531:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.127532:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.127605:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.127607:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.127608:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.127610:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.127612:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.127613:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.127684:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.127687:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.127688:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.127689:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.127690:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.127692:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.127783:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.127785:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.127786:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.127788:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.127789:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.127790:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.127922:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.127925:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.127926:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.127928:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.127931:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.127933:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.127934:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.127936:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.127936:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.127938:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.127941:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.127942:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.127946:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.127947:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.127948:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.127949:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.127950:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.127951:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.127951:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.127956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.128301:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.128304:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.128305:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.128306:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.128310:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.128311:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.128312:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.128314:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.128315:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.128316:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.128318:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.128320:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.128323:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.128324:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.128324:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.128327:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.128327:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.128328:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.128328:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.128412:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.128415:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.128415:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.128417:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.128419:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.128420:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.128430:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.128432:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.128433:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.128434:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.128434:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.128435:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.128490:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.128493:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.128494:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.128497:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.128500:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.128502:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.128503:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96732670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.128505:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.128506:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.128507:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.128512:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.128513:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677447.128514:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:13.0:1619677447.128515:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.128516:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.128517:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.128517:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.128518:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:12.0:1619677447.128519:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:6.0:1619677447.128519:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:13.0:1619677447.128520:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677447.128520:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.128521:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.128522:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.128523:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.128529:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.128531:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.128532:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.128534:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.128535:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.128536:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.128537:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.128537:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.128538:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.128539:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.128539:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.128782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.128789:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:22.0:1619677447.128790:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:24.0:1619677447.128792:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.128793:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.128793:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:8.0:1619677447.128793:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:6.0:1619677447.128794:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.128795:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.128796:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.128797:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.128799:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677447.128800:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.129018:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.129020:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.129021:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.129023:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.129024:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.129026:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.129171:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.129174:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.129175:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.129177:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.129180:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.129181:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677447.129183:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.129184:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.129185:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.129186:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677447.129189:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.129191:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.129194:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.129195:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677447.129195:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677447.129197:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.129198:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.129199:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.129200:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.129201:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:6.0:1619677447.129211:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.129214:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:24.0:1619677447.129215:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:22.0:1619677447.129215:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.129215:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.129217:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.129218:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:6.0:1619677447.129218:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.129219:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.129220:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.129220:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.129220:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.129222:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.129223:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.129227:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.129228:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.129229:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.129231:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.129232:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.129233:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.129236:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.129238:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.129241:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.129242:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.129242:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.129244:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.129244:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.129245:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.129245:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.129343:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.129346:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.129347:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.129347:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.129349:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.129352:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.129377:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.129379:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.129380:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.129382:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.129384:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.129385:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.129421:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.129424:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.129424:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.129425:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.129427:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.129430:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.129441:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.129443:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.129444:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.129445:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.129446:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.129446:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.129493:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.129496:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.129497:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.129498:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.129500:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.129501:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.129620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.129627:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.129630:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.129631:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.129631:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.129633:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.129637:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.129769:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.129772:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.129772:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.129775:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.129777:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.129779:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.129781:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96732651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.129782:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.129784:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.129785:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.129789:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.129791:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.129793:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.129794:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.129795:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.129797:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.129797:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.129798:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.129799:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.129800:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.129810:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.129813:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.129814:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.129814:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.129857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.129994:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.129997:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.129998:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.130002:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.130005:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.130007:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.130009:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.130011:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.130012:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.130013:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.130020:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.130023:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.130027:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.130027:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.130028:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.130029:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.130030:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.130031:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.130031:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.130053:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.130055:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:9.2:1619677447.130055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:12.0:1619677447.130056:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.130058:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.130059:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.130061:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.130062:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.130065:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.130066:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.130066:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.130069:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.130073:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677447.130112:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.130115:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.130116:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.130116:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.130119:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.130122:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.130420:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.130422:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677447.130423:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.130425:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.130427:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677447.130427:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677447.130428:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677447.130428:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.130429:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.130431:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.130434:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.130435:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.130437:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.130438:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.130438:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.130439:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.130439:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.130548:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.130551:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.130552:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.130555:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.130558:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.130560:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.130562:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.130564:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.130565:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.130567:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.130573:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.130576:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.130580:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.130581:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.130582:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.130583:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.130584:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.130585:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.130586:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.130610:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.130613:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.130614:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.130616:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.130617:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.130618:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.130831:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.130834:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.130834:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.130835:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.130837:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.130841:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.130903:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.130906:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.130907:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.130909:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.130910:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.130911:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.130915:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.130918:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.130919:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.130921:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.130924:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.130925:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.130926:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.130928:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.130929:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.130930:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.130933:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.130935:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.130937:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.130938:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.130939:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.130940:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.130941:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.130942:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.130944:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.130944:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.130957:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.130959:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.130960:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.130961:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.131209:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.131212:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.131212:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.131214:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.131216:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.131217:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.131319:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.131322:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.131323:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.131323:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.131326:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.131329:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.131338:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.131341:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.131342:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.131342:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.131343:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.131344:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.131360:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.131363:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.131364:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.131364:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.131366:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.131369:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.131561:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.131563:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.131564:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.131566:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.131568:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.131569:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.131577:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.131580:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.131581:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.131581:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.131582:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.131583:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.131682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.131701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:22.0:1619677447.131748:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.131750:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.131751:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.131752:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.131753:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:22.0:1619677447.131754:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.131756:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.131756:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.131757:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.131758:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.131758:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.131759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677447.131760:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.131762:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96732621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.131763:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.131764:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.131765:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.131768:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.131770:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.131773:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.131774:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.131776:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.131778:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.131778:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.131779:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.131779:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.131982:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:12.0:1619677447.131984:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677447.131985:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.131986:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.131987:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:24.0:1619677447.131988:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677447.131988:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.131989:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.131991:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.131991:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677447.131992:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:12.0:1619677447.131992:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677447.131993:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.131995:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.131996:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.131997:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.132000:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.132002:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.132005:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.132006:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.132007:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.132009:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.132010:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.132010:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.132011:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.132152:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.132155:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.132156:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.132156:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.132159:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.132161:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.132172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.132373:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.132376:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.132377:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.132377:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.132380:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.132383:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.132403:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.132406:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.132407:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.132409:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.132411:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.132412:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.132487:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.132490:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.132491:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.132493:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.132496:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.132497:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.132499:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.132500:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.132501:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.132502:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.132505:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.132507:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.132510:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.132511:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.132511:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.132513:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.132514:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.132514:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.132515:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.132576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.132619:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.132621:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:22.0:1619677447.132624:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.132625:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.132626:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:22.0:1619677447.132627:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.132627:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.132628:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.132629:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.132631:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96732608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.132631:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.132632:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.132634:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.132635:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.132640:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.132643:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.132643:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.132644:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.132645:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.132646:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.132647:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.132647:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.132648:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.132648:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.132649:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.132649:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.132651:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.132651:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.132652:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.132653:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.132657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.132663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.132669:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.132672:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.132673:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.132674:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.132676:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.132679:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677447.132952:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.132955:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.132956:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.132956:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.132958:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.132961:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.132973:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.132976:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.132977:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.132978:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.132979:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.132981:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.133155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.133167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.133174:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.133176:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.133177:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.133178:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.133180:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.133182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.133182:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.133189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.133196:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.133199:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.133199:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.133200:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.133201:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.133201:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.133233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.133252:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.133255:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.133256:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.133257:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.133258:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.133260:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.133269:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.133271:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.133272:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.133273:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.133273:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.133274:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.133402:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.133405:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.133406:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.133408:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.133414:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.133416:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.133417:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.133419:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.133420:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.133421:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.133424:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.133426:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.133429:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.133430:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.133430:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.133432:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.133432:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.133433:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.133434:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.133691:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.133694:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.133694:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.133696:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.133699:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.133700:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.133701:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.133703:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.133704:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.133705:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.133709:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.133710:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.133713:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.133713:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.133714:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.133716:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.133716:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.133717:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.133718:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.133719:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.133730:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.133731:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.133732:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.133733:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.133768:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.133771:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.133772:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.133773:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.133775:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.133777:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.133851:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.133854:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.133854:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.133856:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.133858:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.133859:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.133942:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677447.133946:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677447.133946:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677447.133948:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.133950:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.133952:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:22.0:1619677447.133953:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.133954:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.133956:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.133957:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677447.133960:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.133962:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.133964:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.133965:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:22.0:1619677447.133966:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.133968:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677447.133968:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677447.133969:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677447.133969:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677447.133984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.134129:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.134133:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.134133:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.134136:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.134138:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.134140:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.134141:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.134142:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.134144:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.134144:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.134148:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.134150:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.134153:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.134154:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.134154:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.134156:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.134157:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.134158:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.134158:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.134162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.134275:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.134277:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.134279:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.134280:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677447.134281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:8.0:1619677447.134284:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.134286:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.134287:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.134289:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:8.0:1619677447.134289:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.134290:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.134290:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.134291:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.134292:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.134299:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.134303:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.134303:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.134304:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677447.134305:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.134309:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.134310:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.134310:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.134312:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.134312:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.134313:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.134314:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.134341:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.134343:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.134344:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.134346:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.134348:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.134349:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.134385:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.134389:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.134389:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.134390:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.134392:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.134395:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.134454:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.134457:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.134457:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.134459:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.134461:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.134462:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.134474:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.134477:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.134478:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.134479:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.134479:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.134481:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.134546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.134564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.134661:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.134664:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.134665:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.134665:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.134667:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.134670:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.134925:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.134928:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.134930:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.134931:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.134934:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.134936:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.134937:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.134938:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.134939:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.134940:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.134944:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.134945:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.134948:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.134949:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.134949:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.134951:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.134951:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.134952:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.134953:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.134993:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.134996:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.134996:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.134999:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.135001:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.135002:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.135004:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.135005:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.135007:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.135008:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.135012:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.135014:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.135017:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.135018:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.135018:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.135020:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.135021:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.135021:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.135022:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.135041:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.135043:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.135044:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.135046:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.135047:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.135048:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.135062:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.135065:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.135066:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.135066:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.135067:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.135068:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.2:1619677447.135163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (21)++
00000800:00000200:24.0:1619677447.135167:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)++
00000800:00000200:24.0:1619677447.135171:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e49d37c00] (23)--
00000400:00000200:24.0:1619677447.135172:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.135172:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)--
00000400:00000200:24.0:1619677447.135175:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.135178:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)--
00000800:00000200:22.0:1619677447.135201:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.135204:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.135204:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.135205:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.135207:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.135210:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.135221:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.135224:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.135225:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.135225:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.135226:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.135226:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.135449:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.135452:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.135452:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.135454:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.135456:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.135457:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.135530:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.135532:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.135533:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.135535:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.135538:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.135540:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.135541:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.135542:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.135543:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.135544:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.135547:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.135549:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.135552:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.135553:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.135554:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.135555:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.135555:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.135557:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.135558:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.135558:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.135571:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.135573:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.135574:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.135574:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.135582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.135589:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.135591:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.135592:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.135592:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.135594:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.135597:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677447.135601:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.135603:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.135604:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.135605:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.135607:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.135608:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.135839:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.135842:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.135842:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.135845:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.135848:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.135850:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.135851:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96732562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.135853:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.135853:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.135855:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.135858:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.135860:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.135862:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.135863:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.135864:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.135866:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.135866:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.135867:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.135867:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.135867:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.135870:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.135871:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.135873:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.135874:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.135875:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.135955:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.135958:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.135958:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.135961:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.135961:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.135964:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.135976:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.135978:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.135979:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.135980:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.135981:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.135982:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.135989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.136093:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.136096:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.136097:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.136099:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.136102:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.136103:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.136105:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.136106:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.136107:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.136108:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.136111:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.136113:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.136116:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.136117:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.136118:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.136119:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.136120:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.136121:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.136121:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.136282:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.136284:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.136285:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.136285:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.136288:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.136290:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.136301:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.136304:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.136305:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.136305:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.136306:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.136307:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.136413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.136500:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.136504:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.136504:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.136509:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.136511:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.136514:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.136515:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.136517:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.136518:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.136520:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.136525:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.136527:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.136531:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.136532:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.136533:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.136535:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.136536:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.136537:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.136538:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.136539:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:6.0:1619677447.136547:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.136549:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.136550:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:13.0:1619677447.136551:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677447.136551:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:6.0:1619677447.136551:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.136552:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.136553:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677447.136553:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:13.0:1619677447.136554:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.136554:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:6.0:1619677447.136555:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.136556:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:13.0:1619677447.136557:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.136558:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.136560:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.136629:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.136631:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.136632:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.136633:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.136635:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.136637:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.136726:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.136729:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.136730:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.136731:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.136733:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.136734:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.136754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.136872:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.136875:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:22.0:1619677447.136876:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.0:1619677447.136876:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.136878:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.136878:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.136879:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.136879:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:24.0:1619677447.136881:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.136882:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.136882:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.136884:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.136884:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:24.0:1619677447.136886:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.136887:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.136888:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.136892:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.136893:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.136896:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.136897:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.136897:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.136899:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.136899:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.136900:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.136901:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.136905:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.136908:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.136908:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.136910:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.136912:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.136914:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.136915:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96732541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.136917:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.136918:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.136919:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.136923:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.136925:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.136927:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.136928:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.136929:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.136930:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.136931:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.136932:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.136932:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.136937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.137186:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.137189:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.137190:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.137190:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.137192:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.137195:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.137287:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.137289:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.137290:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.137292:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.137294:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.137303:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.137304:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.137306:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.137306:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.137307:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.137307:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.137308:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.137395:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.137398:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.137399:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.137401:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.137402:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.137404:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.137545:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (23)++
00000800:00000200:6.0:1619677447.137548:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000800:00000200:22.0:1619677447.137549:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (24)++
00000400:00000200:6.0:1619677447.137549:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.137550:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000400:00000200:22.0:1619677447.137550:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.137551:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.137552:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.137555:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.137555:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.137556:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.137571:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.137574:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.137574:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.137575:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.137576:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.137576:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.137690:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:9.2:1619677447.137693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.137693:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.137694:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.137694:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.137696:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.137698:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.137714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:12.0:1619677447.137758:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.137760:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.137761:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.137762:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.137764:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.137765:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.137789:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.137792:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.137793:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677447.137794:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000400:00000200:22.0:1619677447.137795:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.137797:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.137797:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:22.0:1619677447.137799:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.137800:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96732527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677447.137800:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.137801:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:22.0:1619677447.137802:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.137803:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677447.137803:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.137804:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:12.0:1619677447.137805:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:12.0:1619677447.137806:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96732526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677447.137808:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.137809:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677447.137809:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677447.137810:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.137811:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.137814:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.137815:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.137815:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:12.0:1619677447.137815:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.137817:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.137817:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:12.0:1619677447.137818:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.137819:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:12.0:1619677447.137819:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.137820:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.137820:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677447.137820:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677447.137821:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.137823:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.137823:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:9.2:1619677447.137823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:12.0:1619677447.137825:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677447.137826:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677447.137827:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:6.0:1619677447.137830:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.137832:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.137833:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.137834:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.137837:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.137838:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.137839:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.137840:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000400:00000200:13.0:1619677447.137841:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.137842:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677447.137842:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677447.137843:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.137844:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.137845:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677447.138001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.138009:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.138011:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.138012:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.138013:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.138015:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.138018:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.138036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677447.138042:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.138045:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.138045:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.138045:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.138047:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.138047:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677447.138136:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.138139:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.138139:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.138140:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.138142:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.138145:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.138233:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.138236:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.138237:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.138239:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.138240:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.138242:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.138252:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.138254:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.138255:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.138256:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.138256:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.138258:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.138438:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.138441:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.138441:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.138442:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.138445:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.138447:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.138617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.138655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.138662:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.138664:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.138665:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.138665:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.138668:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.138671:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677447.138695:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.138698:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.138699:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.138701:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.138702:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.138704:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.138759:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677447.138762:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677447.138763:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.138764:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.138766:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677447.138767:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.138767:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677447.138767:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.138769:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.138769:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.138771:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.138812:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.138814:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.138815:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.138817:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.138819:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.138821:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.138822:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.138823:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.138824:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.138825:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.138829:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.138831:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.138833:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.138834:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.138835:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.138837:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.138837:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.138838:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.138838:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.138974:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.138977:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.138978:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.138979:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.138980:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.138981:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.139029:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.139031:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.139032:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.139033:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.139035:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.139038:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.139049:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.139051:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.139052:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.139052:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.139053:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.139054:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.139095:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.139098:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.139099:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.139099:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.139101:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.139103:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677447.139104:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.139106:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.139107:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.139108:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.139111:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.139112:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.139113:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.139115:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.139116:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.139117:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.139120:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.139121:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.139124:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.139125:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.139125:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.139127:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.139128:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.139128:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.139129:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.139179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.139186:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.139189:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.139189:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.139190:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.139192:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.139196:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677447.139668:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.139671:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.139672:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.139673:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.139675:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.139676:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.139791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.139800:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.139802:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.139803:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.139804:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.139806:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.139807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.139809:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.140093:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.140096:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.140097:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.140099:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.140100:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.140102:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.140113:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.140116:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.140117:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.140117:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.140118:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.140120:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.140121:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.140124:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.140124:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.140127:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.140129:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.140131:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.140132:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.140133:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.140134:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.140135:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.140140:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.140142:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.140145:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.140145:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.140146:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.140148:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.140148:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.140149:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.140150:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.140170:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.140173:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.140174:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.140176:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.140178:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.140179:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.140181:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.140182:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.140183:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.140184:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.140187:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.140189:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.140192:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.140193:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.140194:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.140195:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.140196:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.140197:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.140198:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.140199:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.140212:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.140214:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.140214:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.140215:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.140336:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.140338:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.140339:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.140340:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.140342:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.140343:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.140421:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.140423:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.140424:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.140426:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.140427:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.140428:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.140439:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.140442:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.140442:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.140443:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.140444:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.140445:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.140758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.140790:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.140790:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:24.0:1619677447.140792:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:22.0:1619677447.140793:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.140794:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.140794:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:6.0:1619677447.140794:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:24.0:1619677447.140795:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.140795:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:22.0:1619677447.140796:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.140798:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.140799:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.140800:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:22.0:1619677447.140800:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:24.0:1619677447.140801:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.140804:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.140805:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.140806:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.140807:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.140809:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.140809:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.140810:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.140811:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.140812:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.140813:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.140814:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.140817:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.140818:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:8.0:1619677447.140818:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.140819:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.140819:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.140821:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.140822:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.140823:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.140823:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.140824:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.140824:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.140825:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.140825:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.140826:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.140832:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.140832:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.140833:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.140833:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.140852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.140961:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.140964:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.140965:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.140967:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.140969:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.140970:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.140972:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.140973:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.140974:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.140975:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.140978:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.140980:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.140983:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.140983:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.140984:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.140986:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.140986:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.140987:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.140988:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.141200:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.141203:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.141204:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.141204:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.141206:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.141209:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.141265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.141314:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.141317:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.141317:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.141318:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.141320:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.141322:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.141417:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.141420:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.141420:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.141422:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.141424:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.141425:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.141513:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.141516:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.141517:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.141519:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.141522:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.141523:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.141524:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.141526:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.141527:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.141528:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.141531:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.141532:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.141535:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.141536:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.141537:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.141538:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.141539:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.141539:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.141540:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.141557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.141581:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.141584:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.141585:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.141586:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.141588:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.141589:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.141666:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.141669:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.141670:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.141670:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.141672:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.141675:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.141684:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.141687:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.141688:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.141688:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.141689:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.141689:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.141711:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.141713:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.141714:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.141716:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.141720:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.141722:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.141723:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.141724:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.141725:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.141726:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.141729:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.141731:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.141734:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.141735:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.141735:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.141737:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.141738:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.141738:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.141738:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.141923:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.141926:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.141927:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.141927:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.141929:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.141933:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.141940:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.141943:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.141944:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.141945:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.141946:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.141946:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.141979:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.141981:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.141982:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.141983:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.141984:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.141986:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.141999:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.142002:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.142002:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.142004:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.142006:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.142008:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.142009:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.142011:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.142012:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.142013:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.142016:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.142018:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.142021:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.142022:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.142024:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.142026:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.142026:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.142028:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.142029:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.142029:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.142039:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.142041:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.142042:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.142043:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.142088:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.142090:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.142091:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.142093:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.142096:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.142097:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:24.0:1619677447.142099:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.142100:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96732468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.142100:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.142101:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.142101:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:24.0:1619677447.142102:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.142104:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.142104:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.142106:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.142107:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:24.0:1619677447.142110:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.142112:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.142115:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.142116:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.142117:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.142119:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.142120:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.142120:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.142121:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.142138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.142328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.142345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.142347:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.142350:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.142351:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.142351:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.142354:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.142357:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.142607:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.142610:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.142611:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.142612:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.142614:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.142615:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.142734:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.142737:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.142738:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.142739:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.142741:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.142742:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.142753:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.142756:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.142757:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.142758:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.142758:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.142760:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.142774:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.142777:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.142778:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.142779:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.142782:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.142783:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.142785:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.142786:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.142787:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.142788:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.142791:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.142793:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.142795:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677447.142797:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677447.142797:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.142798:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.142799:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:24.0:1619677447.142800:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.142801:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:24.0:1619677447.142802:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.142802:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:24.0:1619677447.142805:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.142807:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.142809:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.142810:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.142812:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.142813:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.142815:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.142815:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.142816:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677447.142817:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.142819:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.142823:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.142824:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.142825:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.142827:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.142828:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677447.142829:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed9d3400] (22)++
00000800:00000200:24.0:1619677447.142831:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (23)--
00000800:00000200:8.0:1619677447.142831:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.142832:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677447.142833:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (23)--
00000800:00000200:13.0:1619677447.142833:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.142836:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677447.142836:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (22)--
00000400:00000200:13.0:1619677447.142837:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.142837:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:13.0:1619677447.142838:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.142840:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.142840:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.142845:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)++
00000800:00000200:24.0:1619677447.142848:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed9d3400] (23)--
00000800:00000200:24.0:1619677447.142848:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677447.142849:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:22.0:1619677447.142914:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.142917:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.142918:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.142918:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.142920:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.142922:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.142995:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.142998:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.142998:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.143000:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.143003:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.143004:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.143005:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.143007:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.143008:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.143009:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.143012:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.143014:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.143017:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.143018:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.143019:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.143021:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.143022:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.143022:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.143023:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.143038:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.143041:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.143042:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.143044:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.143047:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.143048:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.143049:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.143051:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.143052:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.143053:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.143058:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.143060:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.143063:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.143064:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.143065:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.143066:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.143067:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.143067:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.143068:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.143224:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.143226:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.143227:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.143229:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.143230:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.143232:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.143282:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.143285:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.143286:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.143289:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.143291:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.143305:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.143308:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.143309:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.143311:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.143313:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.143314:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677447.143325:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.143326:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.143328:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.143329:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.143330:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.143336:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.143339:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.143342:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.143343:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.143343:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.143345:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.143345:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.143346:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.143347:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.143355:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.143358:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.143358:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.143359:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.143362:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.143365:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.143370:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.143372:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.143373:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.143374:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:12.0:1619677447.143375:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.143377:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.143377:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.143377:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.143378:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.143378:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.143379:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.143380:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.143387:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.143390:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.143390:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.143391:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.143392:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.143393:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.143461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.143467:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.143470:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.143471:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.143472:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.143474:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.143477:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.143871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.143890:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677447.143893:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677447.143894:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677447.143896:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.143898:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.143900:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677447.143901:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.143903:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.143903:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:22.0:1619677447.143904:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.143905:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.143906:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.143906:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677447.143909:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.143910:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.143912:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.143912:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.143914:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:22.0:1619677447.143915:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.143915:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:8.0:1619677447.143915:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.143916:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:8.0:1619677447.143917:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.143918:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:22.0:1619677447.143918:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000400:00000200:8.0:1619677447.143918:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.143919:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:22.0:1619677447.143920:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677447.143921:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677447.143921:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:8.0:1619677447.143927:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.143930:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:6.0:1619677447.143932:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:8.0:1619677447.143933:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.143934:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:6.0:1619677447.143934:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:8.0:1619677447.143935:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:6.0:1619677447.143935:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:8.0:1619677447.143936:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:6.0:1619677447.143936:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:8.0:1619677447.143937:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.143938:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.143938:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.143969:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.143972:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.143972:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.143974:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.143977:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.143978:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.143979:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.143981:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.143982:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.143983:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.143986:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.143988:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.143991:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.143991:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.143992:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.143994:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.143994:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.143995:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.143995:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.144000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.144164:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.144167:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.144168:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.144168:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.144171:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.144174:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.144463:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.144465:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.144466:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.144468:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.144470:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.144471:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.144518:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.144521:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.144522:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.144523:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.144525:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.144526:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.144591:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.144594:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.144595:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.144598:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.144600:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.144602:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.144603:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.144604:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.144605:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.144606:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.144610:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.144611:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.144614:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.144615:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.144615:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.144618:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.144618:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.144619:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.144620:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677447.144621:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.144623:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677447.144626:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.144628:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.144629:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.144630:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.144631:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.144633:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.144633:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.144634:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.144638:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.144641:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.144644:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.144645:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.144646:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.144647:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.144648:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.144649:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.144651:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.144652:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.144664:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.144666:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.144667:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.144668:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.144968:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.144970:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.144971:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.144973:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677447.144974:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.144976:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.145141:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.145144:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.145144:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.145146:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.145148:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.145149:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.145182:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.145185:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.145186:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.145188:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.145190:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.145191:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.145193:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.145194:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.145195:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.145196:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.145199:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.145201:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.145203:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.145204:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.145205:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.145206:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.145207:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.145208:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.145208:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.145255:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.145258:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.145259:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.145259:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.145261:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.145264:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.145337:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.145340:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.145341:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.145341:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.145343:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.145346:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.145729:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.145732:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.145733:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.145735:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.145738:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.145740:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.145741:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.145742:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.145743:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.145744:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.145747:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.145749:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.145752:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.145753:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.145753:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.145756:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.145756:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.145757:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.145758:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.145759:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677447.145762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.145771:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.145773:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.145774:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.145776:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.145827:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.145830:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.145831:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.145833:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.145835:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.145836:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.145836:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.145839:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.145840:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.145841:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.145843:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.145844:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.146019:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.146021:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.146022:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.146024:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.146027:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.146029:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.146031:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.146032:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.146033:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.146034:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.146038:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.146040:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.146043:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.146043:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.146044:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.146046:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.146046:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.146047:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.146047:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.146100:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.146103:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.146104:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.146106:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.146109:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.146110:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.146112:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.146113:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.146114:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.146115:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.146118:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.146120:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.146124:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.146125:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.146125:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.146127:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.146128:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.146129:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.146130:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.146130:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.146141:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.146143:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.146143:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.146144:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.146213:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.146215:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.146216:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.146218:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.146220:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.146221:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.146232:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.146235:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.146235:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.146236:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.146237:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.146237:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.146401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.146446:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.146449:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.146450:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.146454:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.146457:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.146458:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.146460:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96732407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.146462:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.146464:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.146465:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.146469:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.146471:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.146474:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.146475:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.146476:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.146478:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.146483:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.146484:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677447.146485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.146485:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.146504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677447.146572:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.146575:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.146576:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.146576:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.146579:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.146581:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.146584:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.146587:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.146588:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.146589:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.146592:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.146593:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.146763:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.146765:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.146767:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.146768:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.146771:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.146772:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.146774:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.146775:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.146776:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.146777:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.146780:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.146782:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.146785:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.146786:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.146787:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.146788:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.146789:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.146790:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.146790:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.146863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.146941:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.146944:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.146945:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.146946:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.146948:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.146948:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.146973:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.146975:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.146976:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.146978:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.146980:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.146982:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.146983:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.146984:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.146985:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.146986:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.146989:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.146990:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.146993:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.146994:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.146995:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.146997:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.146997:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.146998:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.146998:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.147003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.147094:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.147097:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677447.147098:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677447.147100:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.147102:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.147104:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:9.2:1619677447.147104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:22.0:1619677447.147105:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96732396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.147106:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.147107:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.147108:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:24.0:1619677447.147111:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:22.0:1619677447.147112:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.147114:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:22.0:1619677447.147114:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677447.147114:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.147115:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.147117:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.147118:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.147118:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.147118:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:24.0:1619677447.147120:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677447.147120:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.147120:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.147121:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677447.147122:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677447.147126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.147199:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.147202:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.147202:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.147206:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.147209:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.147211:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.147212:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.147214:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.147215:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.147216:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.147221:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.147223:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.147227:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.147227:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.147228:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.147230:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.147231:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.147231:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.147232:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.147556:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.147558:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.147559:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.147560:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677447.147562:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.147563:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.147633:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.147635:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.147635:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.147636:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677447.147639:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.147639:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.147642:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.147642:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.147643:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.147644:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.147646:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.147647:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.147648:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.147649:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:8.0:1619677447.147649:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.147651:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96732388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.147652:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.147652:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.147652:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.147653:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.147653:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.147654:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.147654:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.147656:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.147656:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.147658:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.147659:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.147660:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.147660:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:8.0:1619677447.147660:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.147662:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.147662:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.147663:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.147664:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.147665:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.147666:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.147666:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.147667:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.147667:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.147673:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.147675:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.147675:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.147676:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.147676:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.147681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.147705:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.147707:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.147708:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.147710:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.147711:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.147713:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.147724:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.147727:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.147728:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.147728:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.147729:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.147730:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.147886:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.147888:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.147889:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.147890:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.147892:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.147893:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.147997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.148004:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.148007:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.148008:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.148008:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.148010:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.148010:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.148013:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:24.0:1619677447.148014:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677447.148014:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.148015:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.148017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000400:00000200:6.0:1619677447.148017:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.148020:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.148023:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.148026:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.148026:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677447.148027:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.148028:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.148028:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677447.148030:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.148033:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.148033:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.148034:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.148035:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.148036:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.148228:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.148230:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.148231:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.148232:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.148234:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.148236:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.148342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.148347:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.148350:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.148350:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.148352:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.148354:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.148355:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.148359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677447.148366:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.148368:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.148369:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.148370:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.148371:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.148373:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.148389:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.148392:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.148393:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.148395:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.148397:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.148399:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.148401:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96732376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.148402:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.148403:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.148404:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.148408:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.148411:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.148413:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.148414:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.148415:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.148417:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.148417:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.148418:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.148419:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.148423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.148641:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.148644:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.148645:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.148645:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.148648:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.148650:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.148663:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.148665:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.148666:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.148667:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.148667:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.148668:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.148678:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.148680:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.148681:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.148683:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.148684:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.148685:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.148774:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.148776:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.148777:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.148779:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.148780:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.148782:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.149070:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.149072:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.149073:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.149074:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.149076:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.149079:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.149164:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.149167:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.149168:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.149169:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:24.0:1619677447.149170:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.149170:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677447.149172:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.149174:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.149174:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.149176:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.149178:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.149180:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.149181:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.149184:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.149185:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.149186:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.149187:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96732365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.149189:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.149189:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.149190:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.149190:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.149195:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.149196:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.149197:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.149200:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.149200:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:8.0:1619677447.149200:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.149202:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.149202:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.149203:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.149204:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.149205:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.149211:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.149212:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.149213:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.149215:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.149216:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.149217:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.149218:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.149218:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.149219:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.149220:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.149220:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.149228:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677447.149229:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.149231:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.149232:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.149233:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.149249:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.149251:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.149252:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.149254:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.149256:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.149257:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.149269:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.149272:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.149273:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.149274:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.149274:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.149276:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.149495:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.149497:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.149498:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.149500:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.149503:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.149504:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.149505:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.149507:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.149508:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.149509:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.149512:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.149514:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.149516:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.149517:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.149517:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.149519:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.149519:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.149520:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.149520:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.149525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.149544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:12.0:1619677447.149672:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.149674:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.149676:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.149678:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.149679:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.149681:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.149706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.149797:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.149800:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.149801:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.149802:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.149803:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.149807:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.150060:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.150063:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.150064:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.150065:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.150069:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.150070:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677447.150072:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.150073:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.150074:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.150075:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677447.150079:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.150080:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.150083:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.150084:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677447.150085:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677447.150086:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.150087:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.150087:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.150088:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.150100:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.150103:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.150104:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.150104:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.150106:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.150110:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.150117:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.150120:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.150121:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.150121:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.150122:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.150123:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.150173:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.150176:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.150176:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.150178:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.150180:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.150181:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.150191:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.150193:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.150194:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.150195:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.150197:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.150198:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.150268:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.150270:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.150271:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.150272:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.150274:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.150274:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677447.150285:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.150288:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.150289:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.150291:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.150298:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.150300:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.150301:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.150302:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.150304:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.150305:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.150308:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.150309:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.150312:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.150313:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.150313:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.150315:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.150316:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.150317:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.150318:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.150319:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.150330:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.150332:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.150332:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.150333:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.150420:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.150422:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.150423:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.150425:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.150428:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.150429:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.150430:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.150432:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.150435:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.150436:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.150439:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.150441:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.150443:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.150444:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.150445:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.150447:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.150447:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.150448:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.150448:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.150529:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.150531:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.150532:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.150533:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.150535:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.150537:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.150662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.150745:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.150748:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.150749:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.150749:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.150751:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.150754:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.150973:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.150975:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.150976:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.150978:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.150980:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.150981:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.150996:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.150998:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.150999:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.150999:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.151001:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.151004:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.151109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.151116:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.151118:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.151119:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.151120:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.151122:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677447.151125:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.151130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.151137:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.151139:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.151140:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.151140:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.151141:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.151142:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.151274:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.151276:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.151277:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.151279:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.151281:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.151283:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.151284:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.151285:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.151287:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.151287:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.151290:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.151292:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.151299:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.151300:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.151301:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.151303:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.151303:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.151304:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.151304:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.151351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.151371:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.151375:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.151375:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677447.151377:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.151380:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.151381:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.151383:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.151384:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.151385:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.151386:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.151389:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.151390:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677447.151390:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.151393:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677447.151393:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677447.151394:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.151394:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677447.151394:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000400:00000200:12.0:1619677447.151395:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.151396:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:8.0:1619677447.151396:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:12.0:1619677447.151397:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677447.151397:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.151397:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.151398:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.151408:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.151410:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.151411:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.151412:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.151414:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.151415:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.151428:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.151431:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.151432:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.151432:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.151433:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.151434:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.151600:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.151603:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.151604:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.151604:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.151606:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.151609:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.151665:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.151668:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.151669:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.151669:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.151671:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.151678:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.151679:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.151681:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.151682:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.151682:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.151682:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.151683:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.151763:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.151766:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.151766:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.151768:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.151769:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.151770:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.151796:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.151798:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:24.0:1619677447.151801:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.151801:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.151802:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:24.0:1619677447.151803:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.151803:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677447.151805:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.151806:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.151808:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.151809:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.151810:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.151812:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.151814:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.151815:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.151816:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.151817:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.151818:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732326] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.151821:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.151821:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.151822:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.151826:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.151827:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:8.0:1619677447.151827:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.151828:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.151829:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.151830:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.151831:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.151831:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.151832:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677447.151832:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.151833:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.151833:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.151834:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.151841:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.151841:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.151842:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.151842:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.152002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.152318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.152325:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.152328:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.152329:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.152329:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.152331:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.152335:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677447.152343:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.152346:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.152346:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677447.152348:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.152351:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.152352:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.152353:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.152354:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.152355:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.152356:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.152359:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.152361:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.152364:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.152365:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.152366:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.152367:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.152368:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.152369:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.152371:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.152371:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677447.152373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.152384:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.152386:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.152386:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.152387:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.152391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:9.2:1619677447.152428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.152522:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.152525:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.152525:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.152527:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.152529:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.152530:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.152574:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.152576:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.152577:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.152579:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.152582:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.152584:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.152585:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.152587:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.152588:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.152589:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.152594:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.152597:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.152599:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.152600:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.152600:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.152602:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.152603:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.152604:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.152605:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.152642:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.152645:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.152645:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.152647:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.152648:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.152649:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.152718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.152725:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.152728:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.152729:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.152730:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.152732:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.152739:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677447.152817:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.152819:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.152820:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.152822:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.152825:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.152826:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.152828:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.152830:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.152831:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.152832:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.152835:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.152837:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.152840:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.152841:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.152842:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.152843:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.152844:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.152845:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.152846:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.153060:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.153062:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.153064:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.153064:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.153067:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.153071:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.153298:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.153300:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.153301:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.153302:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.153304:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.153307:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.153499:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.153502:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.153503:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.153505:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.153507:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.153509:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.153510:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.153512:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.153513:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.153514:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.153519:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.153522:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.153525:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.153527:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.153527:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.153529:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.153530:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.153531:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.153531:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.153547:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.153549:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.153550:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.153550:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.153552:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.153555:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.153665:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.153668:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.153669:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.153671:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.153672:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.153674:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.153678:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.153681:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.153682:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677447.153682:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.153684:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677447.153684:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.153685:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.153686:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.153687:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:8.0:1619677447.153687:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.153689:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:8.0:1619677447.153689:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.153690:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.153691:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.153692:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.153693:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.153697:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.153699:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.153702:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.153703:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.153704:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.153706:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.153706:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.153707:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.153709:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.153709:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.153722:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.153724:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.153724:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.153725:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.154089:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.154091:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.154092:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.154094:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.154095:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.154097:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.154107:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.154110:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.154111:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.154112:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.154113:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.154114:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.154177:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.154180:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.154181:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.154181:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.154183:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.154186:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.154189:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.154192:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.154193:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.154194:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.154196:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.154197:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.154548:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.154551:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.154551:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.154554:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.154557:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.154558:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.154559:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.154561:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.154562:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.154563:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.154566:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.154568:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.154571:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.154571:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.154572:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.154574:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.154574:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.154575:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.154577:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.154577:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.154587:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.154589:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.154589:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.154590:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.154599:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.154602:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.154603:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.154606:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.154609:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.154612:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677447.154613:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.154615:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.154618:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.154619:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677447.154624:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.154627:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.154630:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.154631:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677447.154632:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677447.154633:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.154634:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.154635:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.154635:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.154645:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.154648:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:22.0:1619677447.154649:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:12.0:1619677447.154649:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.154650:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.154652:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:13.0:1619677447.154652:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677447.154652:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.154653:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.154654:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:22.0:1619677447.154655:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.154658:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.154667:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.154669:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.154670:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.154670:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.154672:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.154672:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.154731:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.154734:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.154735:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.154737:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.154740:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.154742:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.154743:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.154744:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.154745:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.154747:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.154751:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.154753:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.154755:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.154756:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677447.154757:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (23)++
00000800:00000200:24.0:1619677447.154759:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (25)--
00000800:00000200:24.0:1619677447.154760:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.154761:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.154761:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.155061:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.155064:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.155064:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.155066:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.155067:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.155069:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.155096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.155126:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.155129:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.155130:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.155131:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.155133:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.155136:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.155237:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.155240:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.155240:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.155243:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.155245:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.155247:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.155250:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.155252:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.155254:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.155256:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.155261:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.155264:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.155266:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.155267:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.155268:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.155270:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.155270:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.155271:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.155271:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.155284:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.155287:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.155287:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.155290:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.155290:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.155293:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.155335:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.155337:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.155338:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.155340:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.155343:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.155345:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.155346:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.155348:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:8.0:1619677447.155348:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.155348:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.155350:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:13.0:1619677447.155351:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.155351:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.155353:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677447.155354:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.155355:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.155355:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677447.155357:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.155358:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.155359:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.155360:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.155362:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.155362:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.155363:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.155364:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.155368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.155732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.155739:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.155741:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.155742:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.155743:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.155745:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677447.155747:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677447.155880:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.155882:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.155883:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.155885:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.155887:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.155888:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.155898:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.155900:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.155901:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.155903:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.155905:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.155905:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.155918:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.155921:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.155921:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.155922:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.155922:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.155923:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.155962:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.155965:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.155966:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.155968:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.155971:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.155972:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.155974:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.155975:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.155976:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677447.155977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:8.0:1619677447.155977:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.155980:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.155982:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.155985:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.155985:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.155986:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.155988:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.155988:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.155989:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.155989:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.156080:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.156082:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.156083:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.156084:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.156086:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.156089:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.156097:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.156100:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.156101:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.156102:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.156105:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.156106:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.156157:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.156159:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.156160:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.156162:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.156163:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.156164:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.156177:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.156180:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.156180:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.156181:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.156182:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.156183:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.156192:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.156194:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.156195:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.156196:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.156198:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.156201:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.156214:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.156217:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.156218:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.156218:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.156220:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.156223:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.156231:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.156234:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.156235:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.156235:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.156236:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.156236:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.156456:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.156459:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.156460:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.156463:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.156466:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.156468:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.156469:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.156471:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.156472:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.156473:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.156477:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.156479:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.156482:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.156483:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.156484:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.156486:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.156486:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.156487:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.156488:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.156502:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.156505:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.156506:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.156506:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.156508:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.156511:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.156583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.156601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:9.2:1619677447.156686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:6.0:1619677447.156687:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.156690:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.156691:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677447.156693:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:6.0:1619677447.156693:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.156695:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:6.0:1619677447.156695:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.156696:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:24.0:1619677447.156697:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.156697:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.156697:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96732256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.156699:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.156700:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.156700:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.156702:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.156705:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.156707:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.156708:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677447.156709:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.156710:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.156711:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.156713:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.156713:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.156714:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.156715:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.156836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.156857:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.156860:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.156861:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.156862:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.156864:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.156865:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.157003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.157019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:12.0:1619677447.157038:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.157040:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.157041:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.157043:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.157045:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.157046:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.157160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.157268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.157274:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.157277:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.157278:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.157278:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.157280:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.157282:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.157286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:8.0:1619677447.157292:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.157298:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.157299:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677447.157300:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.157301:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.157301:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.157636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.157715:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.157718:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.157718:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.157719:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.157721:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.157724:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.157753:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.157755:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.157756:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.157758:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.157760:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.157761:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.157762:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.157764:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.157765:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.157766:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.157769:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.157771:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.157773:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677447.157773:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.157774:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.157774:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.157775:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:6.0:1619677447.157776:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.157776:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677447.157777:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677447.157777:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.157777:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:24.0:1619677447.157778:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.157782:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.157784:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.157786:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.157788:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.157789:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.157790:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.157797:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.157799:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.157802:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.157803:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.157804:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.157805:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.157806:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.157807:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.157807:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.157843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.157849:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.157852:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.157853:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.157853:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.157856:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.157859:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.158124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.158130:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.158133:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.158134:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.158134:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.158136:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.158139:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677447.158216:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.158219:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677447.158219:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.158221:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.158223:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.158224:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677447.158224:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677447.158225:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.158225:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.158227:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.158229:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.158316:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.158318:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.158319:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.158320:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.158322:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.158324:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.158335:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.158338:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.158338:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.158339:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.158340:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.158340:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.158507:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.158510:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.158511:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.158512:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.158514:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.158516:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.158696:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.158699:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.158700:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.158702:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.158705:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.158706:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.158708:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.158709:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.158710:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.158712:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.158715:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.158716:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.158719:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.158720:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.158721:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.158722:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.158723:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.158724:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.158726:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.158726:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.158740:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.158742:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.158743:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.158744:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.158917:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.158920:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.158921:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.158921:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.158924:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.158926:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.158946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.158952:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.158955:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.158956:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.158956:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.158958:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.158961:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.159029:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.159031:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.159032:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.159033:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.159035:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.159037:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.159087:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.159090:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.159091:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.159092:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.159093:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.159094:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.159099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.159106:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.159109:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:13.0:1619677447.159109:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677447.159109:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:13.0:1619677447.159111:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677447.159112:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.159112:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.159113:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.159113:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:22.0:1619677447.159114:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.159114:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:22.0:1619677447.159116:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.159117:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.159118:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.159119:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.159120:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.159124:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.159126:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.159128:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.159129:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.159130:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.159132:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.159132:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.159133:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.159134:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.159135:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.159138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.159149:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.159151:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.159151:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.159153:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.159356:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.159358:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.159359:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.159362:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.159364:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.159366:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.159367:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677447.159367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:24.0:1619677447.159369:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.159371:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.159372:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.159376:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.159378:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.159381:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.159383:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.159384:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.159385:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.159386:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.159387:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.159387:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.159388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.159405:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:13.0:1619677447.159407:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677447.159408:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.159409:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677447.159410:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.159411:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.159413:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.159414:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.159416:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677447.159416:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.159418:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.159420:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677447.159422:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.159423:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.159425:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.159426:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677447.159430:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.159431:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.159434:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.159435:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677447.159436:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677447.159437:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (23)--
00000800:00000200:8.0:1619677447.159438:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.159439:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.159440:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.159441:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677447.159445:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:22.0:1619677447.159448:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:6.0:1619677447.159448:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (24)++
00000400:00000200:22.0:1619677447.159449:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.159449:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000400:00000200:6.0:1619677447.159450:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.159451:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.159452:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.159454:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:22.0:1619677447.159455:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.159455:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.159456:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.159457:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.159458:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.159458:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.159467:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.159470:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.159470:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.159471:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.159472:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.159472:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.159907:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.159910:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.159911:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.159913:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.159915:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.159917:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.159918:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.159920:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.159921:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.159922:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.159925:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.159927:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.159930:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.159930:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.159931:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.159933:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.159934:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.159935:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.159935:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.159939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.160220:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.160223:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.160224:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.160225:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.160228:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.160230:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.160231:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.160232:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.160233:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.160234:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.160237:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.160239:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.160242:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.160243:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.160243:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.160245:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.160246:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.160246:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.160247:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.160248:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.160251:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.160252:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.160254:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.160257:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.160258:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.160259:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96732205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.160261:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.160261:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.160263:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.160267:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.160269:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.160272:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.160273:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.160273:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.160275:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.160276:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.160276:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.160277:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.160281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.160537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.160543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.160550:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.160553:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.160554:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.160555:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.160558:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.160558:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.160666:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.160668:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.160669:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.160670:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.160672:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.160674:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.160851:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.160854:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.160855:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.160857:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.160859:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.160861:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.160862:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96732199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.160863:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:6.0:1619677447.160863:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.160864:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.160865:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.160866:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.160866:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:13.0:1619677447.160869:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.160869:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.160870:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.160872:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.160873:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.160876:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.160876:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.160877:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.160879:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.160880:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.160880:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.160881:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.160885:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.160889:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.160889:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.160890:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.160891:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.160893:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.160986:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.160988:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.160989:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.160989:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.160992:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.160994:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.161034:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.161036:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.161037:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.161038:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.161040:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.161041:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.161055:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.161057:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.161058:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.161059:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.161059:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.161061:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.161138:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677447.161140:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677447.161142:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677447.161143:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.161146:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.161148:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677447.161149:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.161150:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.161151:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.161152:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677447.161155:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.161157:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677447.161160:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.161160:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:24.0:1619677447.161161:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:24.0:1619677447.161163:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:24.0:1619677447.161163:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677447.161164:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677447.161164:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.2:1619677447.161169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677447.161176:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.161179:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.161180:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.161182:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.161184:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.161186:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.161188:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677447.161190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:8.0:1619677447.161190:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.161191:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.161192:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.161196:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.161198:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.161201:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.161202:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.161203:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.161205:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.161205:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.161206:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.161207:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.161213:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.161215:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.161216:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.161217:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.161219:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.161220:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.161516:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.161518:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.161519:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.161521:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.161523:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.161524:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.161526:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.161529:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.161530:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.161530:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.161532:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.161535:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.161560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.161615:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.161617:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.161618:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.161620:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.161621:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.161623:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.161830:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.161833:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.161835:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.161837:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.161840:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.161841:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.161843:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96732186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.161844:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.161845:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.161846:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.161850:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.161852:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.161854:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.161855:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.161856:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.161858:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.161858:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.161859:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.161860:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.161863:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.161866:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.161867:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.161869:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.161869:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.161874:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.161884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.161901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:9.2:1619677447.161981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.161986:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.161989:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.161990:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.161991:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.161993:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.161995:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.162039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.162045:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.162048:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.162049:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.162049:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.162051:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.162054:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.162196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.162221:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.162224:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.162224:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.162226:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.162229:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.162231:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.162232:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.162233:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.162234:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.162235:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.162239:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.162241:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.162244:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.162245:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.162246:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.162248:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.162248:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.162249:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.162249:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.162400:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.162402:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.162404:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.162405:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.162408:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.162409:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.162410:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.162412:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.162413:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.162415:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.162418:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.162420:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.162423:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.162423:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.162424:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.162426:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.162426:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.162427:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.162428:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.162575:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.162577:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.162578:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.162580:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.162582:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.162583:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.163144:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.163147:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.163147:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.163149:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.163152:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.163154:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.163155:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96732172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.163158:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.163159:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.163161:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.163165:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.163168:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.163173:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.163174:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.163175:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.163177:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.163178:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.163180:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.163181:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.163181:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.163192:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.163193:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.163194:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.163194:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:22.0:1619677447.163195:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.163195:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.163196:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.163197:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.163197:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.163200:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.163366:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677447.163369:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677447.163370:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677447.163372:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.163375:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.163377:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.163379:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96732166] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.163380:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.163380:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:8.0:1619677447.163381:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.163382:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.163383:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677447.163383:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.163384:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.163384:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.163385:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.163386:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.163387:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.163387:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:13.0:1619677447.163388:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.163389:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677447.163389:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.163391:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.163391:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:6.0:1619677447.163391:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.163392:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.163392:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677447.163392:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.163393:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.163393:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:8.0:1619677447.163393:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.163394:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677447.163395:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.163396:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677447.163396:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677447.163397:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:12.0:1619677447.163406:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.163408:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:24.0:1619677447.163409:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000400:00000200:12.0:1619677447.163409:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.163411:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.163412:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:13.0:1619677447.163413:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.163413:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:12.0:1619677447.163414:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677447.163419:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.163421:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.163424:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:24.0:1619677447.163426:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.163427:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.163428:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.163429:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:24.0:1619677447.163435:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.163436:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:24.0:1619677447.163438:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:24.0:1619677447.163441:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.163442:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:24.0:1619677447.163443:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.0:1619677447.163444:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677447.163445:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:24.0:1619677447.163446:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (23)++
00000800:00000200:24.0:1619677447.163447:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (24)--
00000800:00000200:24.0:1619677447.163448:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (23)--
00000800:00000200:24.0:1619677447.163449:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677447.163449:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.163452:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.163454:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677447.163456:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96732162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.163457:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.163458:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.163459:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677447.163468:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.163470:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677447.163472:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:24.0:1619677447.163473:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677447.163473:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.163474:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:8.0:1619677447.163475:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:24.0:1619677447.163476:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.163477:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:8.0:1619677447.163478:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (23)++
00000400:00000200:24.0:1619677447.163479:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.163480:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (24)++
00000400:00000200:24.0:1619677447.163481:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:8.0:1619677447.163481:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (25)--
00000800:00000200:8.0:1619677447.163482:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (24)--
00000400:00000200:24.0:1619677447.163483:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.163483:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (23)--
00000400:00000200:24.0:1619677447.163484:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.163486:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.163487:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677447.163492:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.163495:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.163496:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (23)++
00000800:00000200:8.0:1619677447.163497:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (24)--
00000800:00000200:8.0:1619677447.163497:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (23)--
00000800:00000200:24.0:1619677447.163498:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.163498:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.163499:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.163499:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677447.163501:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.163501:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677447.163502:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677447.163502:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.2:1619677447.163506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.163517:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.163519:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.163520:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.163521:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.163522:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.163524:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.163526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.163533:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.163536:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.163537:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.163537:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.163537:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.163539:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677447.163542:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.163545:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.163545:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.163547:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.163548:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.163549:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.163563:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.163566:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.163567:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.163567:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.163568:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.163569:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.163861:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.163863:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.163864:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.163866:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.163868:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.163870:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.163871:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.163872:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.163873:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.163874:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.163878:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.163880:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.163883:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.163883:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.163884:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.163886:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.163886:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.163887:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.163888:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.163888:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.163901:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.163903:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.163904:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.163905:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.164049:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.164052:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.164053:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.164053:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.164055:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.164059:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.164241:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.164244:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.164245:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.164247:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.164248:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.164250:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.164399:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.164402:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.164403:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.164403:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.164405:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.164408:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.164421:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.164424:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.164425:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.164425:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.164426:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.164427:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.164563:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.164566:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.164566:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.164567:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.164569:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.164571:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.164586:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.164589:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.164590:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.164592:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.164595:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.164597:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.164598:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.164600:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.164602:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.164603:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.164613:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.164615:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.164618:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.164619:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.164620:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.164622:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.164622:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.164623:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.164624:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.164721:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.164724:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.164725:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.164727:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.164730:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.164731:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.164733:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.164734:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.164735:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.164736:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.164739:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.164741:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.164744:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.164745:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.164745:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.164747:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.164748:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.164749:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.164750:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.164750:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.164763:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.164765:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.164765:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677447.164765:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677447.164766:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.164767:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.164768:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.164770:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.164773:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.164775:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.164776:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96732142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.164777:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.164778:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.164779:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.164783:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.164784:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.164787:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.164788:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.164789:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.164791:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.164791:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.164792:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.164792:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.165246:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.165249:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.165249:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.165250:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.165252:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.165255:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.165331:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.165333:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.165334:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.165336:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.165339:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.165340:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.165341:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.165343:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.165344:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.165345:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.165348:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.165349:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.165352:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.165353:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.165353:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.165355:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.165355:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677447.165355:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.165356:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.165357:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.165358:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.165359:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.165361:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677447.165362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:8.0:1619677447.165364:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.165366:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.165367:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.165368:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.165369:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.165370:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.165374:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.165376:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.165380:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.165380:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.165381:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.165383:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.165383:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.165384:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.165384:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.165425:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.165428:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.165429:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.165431:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.165433:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.165434:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.165716:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.165719:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.165719:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.165722:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.165722:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.165726:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.165744:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.165746:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.165747:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.165749:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.165750:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.165751:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.165760:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.165763:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.165764:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.165765:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.165765:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.165766:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.165993:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.165996:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.165996:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.165998:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.166001:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.166002:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.166003:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.166005:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.166006:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.166007:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.166010:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.166011:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.166014:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.166015:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.166015:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.166017:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.166017:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.166018:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.166018:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.166023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:24.0:1619677447.166088:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.166091:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.166091:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.166093:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.166096:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.166098:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.166099:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.166101:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.166102:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.166103:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.166106:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.166108:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.166111:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.166112:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.166113:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.166114:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.166115:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.166116:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.166116:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.166201:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.166203:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.166204:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.166206:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.166209:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.166210:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677447.166211:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.166212:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.166213:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.166214:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677447.166217:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.166218:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.166219:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.0:1619677447.166221:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.166222:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:8.0:1619677447.166222:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:22.0:1619677447.166223:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.166223:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.166223:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.166225:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.166225:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.166225:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.166226:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.166226:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.166228:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.166239:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.166241:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.166242:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.166243:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.166243:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000400:00000200:22.0:1619677447.166244:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.166244:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.166244:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.166245:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.166246:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.166247:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.166250:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.166279:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.166282:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.166282:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.166284:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.166286:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.166287:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.166550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.166558:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.166560:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.166561:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.166561:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.166563:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:24.0:1619677447.166567:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677447.166709:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.166712:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.166713:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.166715:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.166718:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.166720:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.166721:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677447.166722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:8.0:1619677447.166723:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.166723:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.166725:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.166728:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.166730:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.166730:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.166731:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.166732:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.166734:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.166734:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.166735:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.166736:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.166737:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.166738:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.166739:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677447.166739:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.166740:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.166740:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.166745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677447.166751:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.166753:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.166754:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.166754:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.166755:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.166755:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.166815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.166889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.166901:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.166903:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.166904:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677447.166905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677447.166906:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.166909:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.166910:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.166913:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732112] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.166914:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.166915:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.166916:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.166920:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.166922:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.166924:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.166925:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.166926:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.166927:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.166928:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.166929:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.166930:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.166931:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.166944:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.166946:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.166947:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.166948:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.167134:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.167137:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.167138:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.167139:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.167140:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.167143:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.167153:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.167155:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.167156:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.167157:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.167157:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.167158:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.167210:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.167213:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.167214:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.167215:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.167217:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.167218:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.167398:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.167401:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.167402:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.167404:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.167406:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.167406:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.167465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.167472:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.167475:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.167476:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:9.2:1619677447.167476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677447.167476:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.167478:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.167482:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677447.167629:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.167631:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.167632:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.167634:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.167635:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.167636:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.167740:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.167743:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.167743:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.167745:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.167748:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.167749:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.167751:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.167753:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.167754:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.167755:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.167758:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.167761:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.167764:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.167765:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.167766:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.167768:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.167770:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.167771:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.167772:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.167773:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.167787:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.167788:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.167789:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.167791:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.167791:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.167793:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.167793:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.167794:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.167796:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.167799:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.167864:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.167866:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.167867:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.167867:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.167869:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.167872:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.168156:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.168159:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.168160:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.168161:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.168163:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.168164:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.168175:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.168177:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.168178:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.168179:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.168179:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.168180:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.168735:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677447.168738:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677447.168741:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677447.168743:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.168743:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677447.168744:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677447.168746:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96732093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677447.168746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.168747:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:13.0:1619677447.168747:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677447.168748:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.168749:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:13.0:1619677447.168749:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.168750:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.168751:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.168753:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:13.0:1619677447.168754:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.168755:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.168756:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677447.168760:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.168761:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (21)++
00000800:00000200:13.0:1619677447.168762:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (22)++
00000800:00000200:13.0:1619677447.168765:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:13.0:1619677447.168766:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677447.168767:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677447.168767:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677447.168771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:24.0:1619677447.168786:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.168789:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.168790:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677447.168791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000400:00000200:24.0:1619677447.168792:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.168794:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.168796:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.168797:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.168799:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.168800:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.168801:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.168806:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.168808:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.168811:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.168811:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.168812:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.168812:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.168813:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:8.0:1619677447.168814:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.168815:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.168816:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.168817:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.168817:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.168818:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677447.168819:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.168819:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.168825:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.168826:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.168827:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.168828:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.168831:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.168832:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677447.168832:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.168835:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.168835:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:8.0:1619677447.168836:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.168837:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.168838:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677447.168838:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.168840:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.168840:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:6.0:1619677447.168840:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.168841:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:6.0:1619677447.168841:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:8.0:1619677447.168842:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:6.0:1619677447.168842:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96732090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.168843:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.168845:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677447.168846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677447.168846:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.168849:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.168850:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.168853:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.168854:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.168854:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.168856:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.168856:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.168857:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.168858:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.168919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.168925:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.168928:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.168929:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.168929:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.168931:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.168934:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677447.169004:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.169007:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.169008:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.169010:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.169011:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.169012:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.169365:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.169367:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.169368:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.169369:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.169371:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.169373:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.169444:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.169447:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.169447:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.169448:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.169450:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.169453:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.169578:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.169580:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.169581:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.169583:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.169585:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.169586:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.169678:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.169681:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.169681:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.169683:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.169684:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.169685:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.169781:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.169784:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.169785:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.169786:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.169788:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.169789:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.170041:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.170044:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.170045:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.170047:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.170049:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.170051:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.170052:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.170053:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.170054:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.170055:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.170059:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.170060:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.0:1619677447.170060:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.170063:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677447.170063:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.170063:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:22.0:1619677447.170064:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.170065:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.170066:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:8.0:1619677447.170066:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.170067:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.170067:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.170068:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:8.0:1619677447.170068:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.170070:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.170076:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.170078:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.170079:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.170080:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.170085:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.170087:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.170090:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.170091:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.170092:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.170093:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.170094:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.170095:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.170095:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.170141:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.170144:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.170145:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.170146:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.170148:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.170151:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.170361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.170648:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.170650:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.170651:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.170653:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.170655:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.170656:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.170667:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.170670:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.170671:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.170672:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.170672:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.170674:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.170675:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.170678:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.170679:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.170680:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.170683:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.170685:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.170686:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.170688:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.170688:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.170689:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.170692:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.170694:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.170697:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.170697:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.170698:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.170700:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.170700:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.170701:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.170701:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.170770:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.170773:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.170774:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.170774:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.170777:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.170780:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.170788:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.170791:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.170792:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.170792:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.170793:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.170794:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.170802:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.170805:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.170806:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.170808:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.170809:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.170811:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.170821:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.170824:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.170825:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.170825:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.170826:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.170828:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.170941:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.170943:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.170944:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.170946:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.170947:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.170949:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.170985:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.170988:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.170989:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.170990:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.170992:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.170994:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.170995:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96732053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.170996:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.170997:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.170998:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.171002:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.171003:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:22.0:1619677447.171004:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.171006:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.171006:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:22.0:1619677447.171007:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.171007:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:13.0:1619677447.171007:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.171009:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.171009:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:13.0:1619677447.171009:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.171010:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677447.171010:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677447.171011:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.171011:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.171392:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.171395:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.171396:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.171396:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.171398:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.171401:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677447.171401:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.171403:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.0:1619677447.171404:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.171405:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677447.171405:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.171406:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.171407:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.171408:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:6.0:1619677447.171408:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.171410:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.171411:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.171411:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.171412:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000400:00000200:8.0:1619677447.171412:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.171412:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.171413:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:8.0:1619677447.171414:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.171416:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.171417:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.171419:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.171420:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.171421:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:9.2:1619677447.171422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:8.0:1619677447.171425:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.171427:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.171430:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.171431:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.171431:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.171431:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:24.0:1619677447.171432:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.171432:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:24.0:1619677447.171434:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.171434:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.171435:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.171436:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.171437:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.171438:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677447.171438:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.171444:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.171452:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.171454:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.171455:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.171456:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.171493:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.171497:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.171498:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.171500:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.171503:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.171505:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677447.171506:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.171507:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.171508:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.171509:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677447.171512:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.171513:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.171516:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.171517:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677447.171517:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677447.171519:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.171520:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.171521:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.171522:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.171522:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:6.0:1619677447.171533:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:24.0:1619677447.171535:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:22.0:1619677447.171536:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.171536:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:24.0:1619677447.171537:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:6.0:1619677447.171537:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.171538:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.171539:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:6.0:1619677447.171539:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.171542:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.171571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.171593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:22.0:1619677447.171749:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.171752:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.171753:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.171755:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.171757:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.171759:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.171760:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.171761:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.171762:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.171763:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.171766:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.171768:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.171771:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.171772:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.171772:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.171774:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.171775:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.171776:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.171776:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.171777:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.171787:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.171789:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.171790:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.171792:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.171910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.172014:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.172016:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.172017:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.172020:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.172021:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.172022:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.172094:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.172097:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.172097:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:8.0:1619677447.172098:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:6.0:1619677447.172098:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.172100:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.172101:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.172102:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.172104:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.172104:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:8.0:1619677447.172106:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.172108:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.172109:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.172111:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.172112:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.172113:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.172116:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.172117:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.172120:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.172121:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.172123:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.172125:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.172125:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.172126:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.172127:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.172127:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677447.172128:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.172128:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.172130:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.172133:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.172135:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.172136:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96732038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.172137:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.172138:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.172139:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.172142:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.172143:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.172146:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.172147:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.172148:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.172150:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.172150:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.172151:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.172152:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.172191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.172198:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.172200:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.172201:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.172201:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.172203:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.172206:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677447.172770:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.172772:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.172773:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.172775:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.172777:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.172778:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.172802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.172810:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.172813:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677447.172815:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.172815:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.172819:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.172820:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.172822:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.172823:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.172824:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.172825:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.172829:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.172830:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.172833:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.172834:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.172835:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.172836:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.172837:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.172838:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.172839:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.173228:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.173231:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.173232:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.173236:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.173239:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.173240:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.173242:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96732028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.173244:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.173245:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.173246:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.173249:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.173251:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.173253:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.173254:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.173255:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.173256:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.173257:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.173257:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.173258:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.173258:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.173261:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.173262:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.173265:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.173265:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.173267:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.173519:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.173522:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.173523:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.173524:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.173526:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.173530:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.173657:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.173660:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.173661:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.173662:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.173663:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.173665:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.173674:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.173676:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.173677:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.173678:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.173678:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.173680:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.173702:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.173705:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.173706:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.173707:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.173708:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.173710:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.173761:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.173763:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:6.0:1619677447.173764:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:24.0:1619677447.173765:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677447.173767:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:8.0:1619677447.173767:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.173767:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.173768:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.173770:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.173770:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.173772:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.173773:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.173774:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:8.0:1619677447.173775:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.173776:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.173776:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.173779:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.173781:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.173784:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.173785:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.173785:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.173787:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.173788:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.173789:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.173789:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.173794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.173870:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.173873:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.173874:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.173875:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.173877:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.173880:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.174096:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.174099:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.174100:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.174100:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.174102:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.174105:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.174268:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.174271:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.174272:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.174274:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.174276:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.174278:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.174279:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96732015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.174281:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.174283:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.174284:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.174289:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.174292:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.174302:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.174303:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.174304:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.174306:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.174306:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.174307:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.174307:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.174427:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.174429:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.174431:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.174431:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.174433:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.174435:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.174440:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677447.174443:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677447.174443:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.174444:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:12.0:1619677447.174445:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.174447:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:13.0:1619677447.174447:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:22.0:1619677447.174448:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:13.0:1619677447.174448:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677447.174448:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:6.0:1619677447.174448:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.174449:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000400:00000200:13.0:1619677447.174449:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.174450:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.174450:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677447.174451:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.174453:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.174460:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.174463:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.174464:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.174464:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.174465:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.174466:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.174594:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.174596:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.174597:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.174599:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.174600:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.174601:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.174663:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.174666:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.174666:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.174668:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.174669:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.174671:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.174691:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.174694:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.174695:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.174697:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.174699:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.174701:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.174702:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96732007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.174704:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.174705:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.174706:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.174709:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.174711:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.174714:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.174715:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.174715:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.174717:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.174717:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.174718:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.174719:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.174933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:13.0:1619677447.175037:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.175040:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.175040:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.175042:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677447.175043:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.175044:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.175062:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.175064:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.175066:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.175068:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.175071:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.175072:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.175073:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.175075:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.175075:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.175076:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.175080:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.175081:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.175084:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.175086:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.175086:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.175088:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.175089:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.175089:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.175090:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.175094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.175112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.175119:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.175122:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.175123:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.175125:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.175127:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.175128:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.175128:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.175129:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.175130:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.175130:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.175130:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677447.175132:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.175133:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.175446:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.175450:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.175450:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.175452:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.175455:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.175456:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.175458:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.175460:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.175461:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.175462:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.175465:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.175466:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.175470:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.175470:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.175471:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.175473:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.175474:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.175474:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.175475:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.175529:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.175532:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.175532:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.175534:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.175536:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.175537:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.175625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.175633:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.175635:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.175636:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.175637:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.175639:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.175642:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.175690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.175787:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.175790:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.175791:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.175791:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.175794:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.175797:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.175805:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.175807:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.175808:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.175808:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.175809:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.175810:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.175855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.175862:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.175864:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.175865:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.175866:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.175868:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:8.0:1619677447.175871:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.175875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677447.175882:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.175885:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.175886:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.175886:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.175887:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.175889:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677447.175911:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.175914:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.175914:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.175916:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.175919:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.175921:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.175922:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.175923:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.175924:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.175925:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.175929:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.175930:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.175933:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.175934:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.175934:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.175936:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.175936:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.175938:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.175939:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.2:1619677447.175940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.175940:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.175951:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.175952:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677447.175953:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.175954:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.175954:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000400:00000200:24.0:1619677447.175957:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.175958:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.175961:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.175961:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.175962:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.175963:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:8.0:1619677447.175963:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:24.0:1619677447.175964:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.175965:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:8.0:1619677447.175965:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.175966:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677447.175966:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.175967:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:8.0:1619677447.175967:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.175968:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677447.175972:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.175974:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677447.175977:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.175978:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:8.0:1619677447.175978:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:8.0:1619677447.175980:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:8.0:1619677447.175980:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677447.175981:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677447.175982:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:13.0:1619677447.176144:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.176147:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.176148:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.176149:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.176151:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.176152:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.176169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.176175:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.176178:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.176179:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.176179:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.176181:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.176183:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.176190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677447.176196:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.176199:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.176199:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.176200:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.176201:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.176202:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.176215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.176221:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.176223:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.176225:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.176225:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.176227:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.176229:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677447.176264:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.176267:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.176268:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.176270:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.176273:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.176274:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.176276:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96731980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.176277:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.176278:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.176279:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.176283:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.176285:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.176287:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.176288:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.176289:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.176290:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.176291:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.176292:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.176293:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.176298:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.176298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.176304:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.176306:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.176307:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.176308:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.176428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.176773:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.176778:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.176778:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.176782:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.176784:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.176789:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.176791:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.176792:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.176793:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.176794:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.176798:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.176800:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:9.2:1619677447.176801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.176802:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.176804:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.176804:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.176806:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.176807:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.176808:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.176808:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.176809:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.176809:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:24.0:1619677447.176810:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.176810:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.176812:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.176815:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677447.176844:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677447.176847:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677447.176848:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677447.176851:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.176854:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.176855:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:8.0:1619677447.176856:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.176858:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.176859:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.176859:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:8.0:1619677447.176862:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.176864:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677447.176866:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.176867:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:8.0:1619677447.176867:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.176868:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.176869:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677447.176870:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677447.176870:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:9.2:1619677447.176874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.176880:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.176881:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.176882:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.176883:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.176884:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.176886:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677447.176911:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.176914:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.176914:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.176917:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.176919:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.176920:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.176922:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96731973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.176923:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.176924:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.176925:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.176928:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.176930:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.176933:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.176934:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.176934:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.176936:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.176936:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.176937:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.176938:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.176942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.177147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.177323:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.177326:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.177327:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.177327:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.177329:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.177332:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.177347:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.177350:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.177351:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.177351:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.177353:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.177356:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.177398:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.177401:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.177402:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.177403:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.177406:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.177407:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.177408:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.177410:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.177411:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.177412:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.177415:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.177416:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.177419:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.177420:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.177421:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.177422:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.177423:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.177424:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.177424:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.177465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.177485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677447.177588:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.177591:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.177592:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.177592:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.177594:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.177597:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.177775:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.177778:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.177779:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.177781:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.177784:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.177785:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.177787:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.177788:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.177789:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.177790:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.177793:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.177795:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.177798:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.177799:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.177800:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.177802:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.177802:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.177803:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.177803:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.177969:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:9.2:1619677447.177971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.177972:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.177972:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.177973:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.177976:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.177979:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.177992:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.177995:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.177995:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.177997:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.177999:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.178000:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.178213:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.178216:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.178217:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.178219:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.178220:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.178221:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.178232:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.178235:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.178236:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.178236:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.178237:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.178239:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.178563:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.178566:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.178567:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.178568:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.178570:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.178571:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.178787:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.178790:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.178791:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.178793:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.178796:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.178798:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.178799:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.178800:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.178801:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.178802:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.178806:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.178809:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.178812:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.178812:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.178813:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.178815:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.178815:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.178816:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.178817:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.178818:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.178831:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.178833:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.178834:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.178835:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.178840:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.178843:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.178844:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.178845:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.178847:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.178849:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.178858:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.178861:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.178861:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.178862:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.178863:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.178864:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.178923:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.178925:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.178926:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.178928:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.178930:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.178932:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.178933:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96731950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.178934:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.178935:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.178936:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.178941:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.178943:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.178946:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.178946:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.178947:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.178949:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.178950:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.178950:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.178951:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.179627:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.179630:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.179631:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.179632:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.179634:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.179638:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.179685:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.179687:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.179688:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.179690:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.179692:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.179693:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.179833:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.179836:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.179837:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.179839:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.179841:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.179842:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.179844:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.179845:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.179846:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.179847:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.179850:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.179852:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.179855:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.179856:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.179856:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.179858:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.179858:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.179859:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.179859:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.179968:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.179971:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.179971:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.179973:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.179975:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.179976:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.180143:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.180148:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.180153:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677447.180155:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.180158:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.180160:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.180161:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.180162:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.180163:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.180164:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.180167:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.180169:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.180172:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.180173:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.180173:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.180175:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.180175:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.180177:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.180178:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.180179:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.180191:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.180193:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.180194:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.180195:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.180213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.180413:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.180415:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.180416:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.180418:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.180421:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.180423:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.180424:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.180425:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.180426:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.180427:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.180431:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.180433:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.180436:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.180437:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.180437:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.180439:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.180440:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.180440:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.180441:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.180446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.180566:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.180569:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.180569:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.180571:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.180573:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.180574:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.180603:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.180605:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.180606:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.180608:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.180610:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.180610:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.180611:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.180614:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.180615:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.180616:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.180616:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.180618:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.180638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.180645:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.180647:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.180648:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.180649:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.180651:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.180655:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.180658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:8.0:1619677447.180665:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.180667:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677447.180668:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.180669:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.180669:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.180670:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677447.180870:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.180872:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.180874:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.180875:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.180877:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.180879:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.180880:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.180881:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.180882:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.180883:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.180886:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.180888:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.180890:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.180891:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.180892:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.180894:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.180894:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.180895:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.180895:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677447.181035:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.181038:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.181039:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.181043:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.181046:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.181049:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:9.2:1619677447.181051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:8.0:1619677447.181051:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.181054:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.181055:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.181057:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.181058:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.181059:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.181061:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.181063:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.181064:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.181067:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.181067:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.181069:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677447.181072:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.181073:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.181073:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.181075:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.181076:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.181077:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.181077:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677447.181204:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.181207:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.181208:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.181210:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.181213:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.181214:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677447.181216:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.181217:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.181218:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.181219:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677447.181222:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.181223:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677447.181226:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.181227:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677447.181227:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677447.181229:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677447.181229:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.181230:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.181230:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.181244:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.181247:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.181248:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.181248:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.181251:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.181254:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.181255:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.181256:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.181256:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.181260:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.181263:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.181264:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.181266:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96731918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.181267:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.181268:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.181269:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.181273:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.181274:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:9.2:1619677447.181275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.181277:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.181278:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.181278:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.181280:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.181281:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.181282:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.181283:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.181284:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.181295:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:9.2:1619677447.181300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:22.0:1619677447.181302:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.181302:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.181303:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.181355:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (21)++
00000800:00000200:24.0:1619677447.181358:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (22)--
00000800:00000200:8.0:1619677447.181358:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000400:00000200:8.0:1619677447.181361:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.181364:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.181365:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677447.181366:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.181368:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.181369:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.181370:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677447.181373:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.181375:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.181377:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.181378:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677447.181378:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.181380:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.181380:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (21)++
00000800:00000200:8.0:1619677447.181381:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (22)--
00000800:00000200:8.0:1619677447.181382:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (21)--
00000800:00000200:9.2:1619677447.181387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.181396:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.181398:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.181398:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.181399:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.181400:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (21)++
00000800:00000200:24.0:1619677447.181401:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.181401:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000800:00000200:24.0:1619677447.181402:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677447.181403:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (22)--
00000400:00000200:8.0:1619677447.181403:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.181405:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.181406:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.181407:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731916] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.181409:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.181410:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.181411:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.181414:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.181416:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.181419:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.181420:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.181421:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.181423:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.181423:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (21)++
00000800:00000200:8.0:1619677447.181424:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fe8667200] (22)++
00000800:00000200:8.0:1619677447.181426:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (23)--
00000800:00000200:8.0:1619677447.181427:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (22)--
00000800:00000200:9.2:1619677447.181428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.181439:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (22)++
00000800:00000200:24.0:1619677447.181442:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fe8667200] (23)--
00000800:00000200:24.0:1619677447.181443:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (22)--
00000800:00000200:8.0:1619677447.181443:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (21)++
00000800:00000200:9.2:1619677447.181445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.181446:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000800:00000200:24.0:1619677447.181448:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (22)--
00000400:00000200:8.0:1619677447.181449:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.181450:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.181451:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.181452:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.181452:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.181454:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.181455:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677447.181456:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.181457:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.181458:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.181460:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.181462:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.181464:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:8.0:1619677447.181464:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.181465:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (22)++
00000800:00000200:24.0:1619677447.181466:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000800:00000200:8.0:1619677447.181466:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:24.0:1619677447.181469:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.181469:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.181470:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:24.0:1619677447.181471:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.181471:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:24.0:1619677447.181473:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:8.0:1619677447.181473:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:24.0:1619677447.181474:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.181476:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.181476:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.181477:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.181478:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (23)++
00000800:00000200:8.0:1619677447.181479:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (24)--
00000800:00000200:8.0:1619677447.181480:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (23)--
00000800:00000200:8.0:1619677447.181481:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (22)++
00000400:00000200:24.0:1619677447.181482:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.181482:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000800:00000200:24.0:1619677447.181484:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:8.0:1619677447.181484:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.181486:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:8.0:1619677447.181486:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.181487:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.181488:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677447.181488:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:8.0:1619677447.181488:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:13.0:1619677447.181489:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.181489:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.181489:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96731911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.181490:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:13.0:1619677447.181491:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.181491:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.181492:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.181493:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:8.0:1619677447.181493:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.181494:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (23)++
00000800:00000200:24.0:1619677447.181495:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (24)--
00000800:00000200:13.0:1619677447.181495:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.181496:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (23)--
00000800:00000200:24.0:1619677447.181497:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (22)++
00000400:00000200:8.0:1619677447.181497:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.181498:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000800:00000200:8.0:1619677447.181499:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.181500:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.181502:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.181503:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.181503:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:6.0:1619677447.181503:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677447.181504:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:24.0:1619677447.181505:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:8.0:1619677447.181506:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:6.0:1619677447.181506:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:24.0:1619677447.181507:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96731910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.181507:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:12.0:1619677447.181507:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:6.0:1619677447.181507:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.181508:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.181510:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.181510:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:6.0:1619677447.181510:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.181511:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.181511:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:24.0:1619677447.181512:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:12.0:1619677447.181512:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.181514:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:24.0:1619677447.181516:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.181516:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (23)++
00000800:00000200:8.0:1619677447.181517:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (24)--
00000800:00000200:6.0:1619677447.181517:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.181518:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:8.0:1619677447.181518:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (23)--
00000800:00000200:8.0:1619677447.181519:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (22)++
00000800:00000200:24.0:1619677447.181520:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.181521:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:8.0:1619677447.181521:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000800:00000200:24.0:1619677447.181522:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:8.0:1619677447.181523:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.181524:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:24.0:1619677447.181525:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (23)++
00000400:00000200:8.0:1619677447.181525:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.181527:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677447.181529:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.181530:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.181531:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.181532:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.181533:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fe8667200] (24)++
00000800:00000200:24.0:1619677447.181534:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (25)--
00000800:00000200:24.0:1619677447.181535:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (24)--
00000800:00000200:24.0:1619677447.181536:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (23)++
00000400:00000200:8.0:1619677447.181536:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.181538:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000800:00000200:8.0:1619677447.181538:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:24.0:1619677447.181541:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.181541:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.181542:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.181542:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:24.0:1619677447.181544:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.181544:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.181545:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (24)++
00000800:00000200:8.0:1619677447.181545:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (25)--
00000400:00000200:24.0:1619677447.181546:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:8.0:1619677447.181546:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (24)--
00000400:00000200:24.0:1619677447.181547:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731908] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.181547:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (23)++
00000800:00000200:8.0:1619677447.181548:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fe8667200] (24)--
00000800:00000200:8.0:1619677447.181548:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (23)--
00000400:00000200:24.0:1619677447.181549:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.181549:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (22)++
00000400:00000200:24.0:1619677447.181550:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.181550:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000400:00000200:24.0:1619677447.181551:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.181553:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.181555:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.181555:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.181556:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677447.181556:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:8.0:1619677447.181556:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:13.0:1619677447.181558:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:8.0:1619677447.181558:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96731907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.181559:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.181559:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.181560:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677447.181561:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.181561:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:12.0:1619677447.181562:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.181563:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.181564:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.181565:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:8.0:1619677447.181565:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.181566:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.181567:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.181567:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:24.0:1619677447.181568:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (23)++
00000800:00000200:24.0:1619677447.181569:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (24)--
00000800:00000200:8.0:1619677447.181569:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.181570:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (23)--
00000800:00000200:8.0:1619677447.181570:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:24.0:1619677447.181571:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (22)++
00000800:00000200:8.0:1619677447.181571:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:24.0:1619677447.181572:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000400:00000200:24.0:1619677447.181574:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.181577:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.181577:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:24.0:1619677447.181578:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:24.0:1619677447.181579:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96731906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.181581:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.181581:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.181582:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:8.0:1619677447.181583:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (23)++
00000800:00000200:8.0:1619677447.181584:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (24)--
00000800:00000200:8.0:1619677447.181584:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (23)--
00000800:00000200:8.0:1619677447.181585:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (22)++
00000400:00000200:24.0:1619677447.181586:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.181588:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.0:1619677447.181588:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000800:00000200:24.0:1619677447.181592:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.181593:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:24.0:1619677447.181594:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:24.0:1619677447.181596:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:8.0:1619677447.181596:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.181597:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (23)++
00000400:00000200:8.0:1619677447.181598:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.181599:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fe8667200] (24)++
00000800:00000200:24.0:1619677447.181600:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (25)--
00000800:00000200:24.0:1619677447.181601:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (24)--
00000800:00000200:24.0:1619677447.181602:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (23)++
00000800:00000200:24.0:1619677447.181606:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000400:00000200:8.0:1619677447.181606:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.181607:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96731904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.181609:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.181610:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.181611:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.181615:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.181616:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.181619:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.181620:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.181622:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:8.0:1619677447.181622:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.181623:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:24.0:1619677447.181624:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96731903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.181624:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:24.0:1619677447.181625:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.181626:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.181626:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:24.0:1619677447.181627:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:8.0:1619677447.181628:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (24)++
00000800:00000200:8.0:1619677447.181628:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (25)--
00000800:00000200:22.0:1619677447.181629:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677447.181629:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (24)--
00000400:00000200:24.0:1619677447.181631:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.181632:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:24.0:1619677447.181633:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:22.0:1619677447.181633:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.181633:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.181635:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.181636:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (23)++
00000800:00000200:24.0:1619677447.181637:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.181637:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:22.0:1619677447.181637:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.181637:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fe8667200] (24)--
00000800:00000200:24.0:1619677447.181638:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:8.0:1619677447.181638:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (23)--
00000800:00000200:8.0:1619677447.181639:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (22)--
00000800:00000200:24.0:1619677447.181640:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:24.0:1619677447.181640:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (21)++
00000800:00000200:24.0:1619677447.181641:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (22)--
00000800:00000200:24.0:1619677447.181642:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (21)--
00000800:00000200:6.0:1619677447.181647:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.181650:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.181651:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.181651:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.181652:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.181653:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.181777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.181781:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (21)++
00000800:00000200:8.0:1619677447.181783:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000800:00000200:24.0:1619677447.181784:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.181786:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.181786:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.181787:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.181788:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.181789:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.181790:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677447.181791:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96731899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.181793:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.181795:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.181797:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:24.0:1619677447.181799:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.181800:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (22)--
00000800:00000200:24.0:1619677447.181801:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:8.0:1619677447.181801:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.181804:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.181807:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.181808:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677447.181809:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677447.181811:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.181811:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (22)++
00000800:00000200:8.0:1619677447.181812:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (23)--
00000800:00000200:8.0:1619677447.181814:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (22)--
00000800:00000200:8.0:1619677447.181815:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (21)++
00000800:00000200:8.0:1619677447.181817:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000800:00000200:24.0:1619677447.181819:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (22)--
00000400:00000200:8.0:1619677447.181819:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.181821:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.181823:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:22.0:1619677447.181824:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:8.0:1619677447.181824:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96731897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.181825:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.181826:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.181827:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:12.0:1619677447.181827:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:8.0:1619677447.181827:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.181827:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.181828:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.181829:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.181830:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.181831:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.181831:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677447.181832:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.181833:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677447.181834:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.181834:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677447.181835:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.181837:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.181838:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:8.0:1619677447.181838:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677447.181840:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:8.0:1619677447.181841:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (21)++
00000800:00000200:8.0:1619677447.181842:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fe8667200] (22)++
00000800:00000200:8.0:1619677447.181843:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (23)--
00000800:00000200:8.0:1619677447.181844:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (22)--
00000800:00000200:9.2:1619677447.181846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:24.0:1619677447.181855:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (22)++
00000800:00000200:24.0:1619677447.181857:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000800:00000200:8.0:1619677447.181858:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (23)++
00000800:00000200:8.0:1619677447.181859:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fe8667200] (24)--
00000800:00000200:8.0:1619677447.181860:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (23)--
00000400:00000200:24.0:1619677447.181861:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.181861:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (22)--
00000400:00000200:24.0:1619677447.181866:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.181868:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:9.2:1619677447.181868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000400:00000200:24.0:1619677447.181869:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.181870:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.181871:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.181872:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:24.0:1619677447.181875:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.181876:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:24.0:1619677447.181879:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.181880:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:24.0:1619677447.181880:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:24.0:1619677447.181882:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:24.0:1619677447.181882:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (21)++
00000800:00000200:24.0:1619677447.181883:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (22)--
00000800:00000200:24.0:1619677447.181884:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (21)--
00000800:00000200:8.0:1619677447.181901:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (21)++
00000800:00000200:8.0:1619677447.181904:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000800:00000200:24.0:1619677447.181905:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (22)--
00000400:00000200:8.0:1619677447.181910:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.181912:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.181913:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:8.0:1619677447.181915:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.181916:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.181917:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.181918:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:9.2:1619677447.181920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:8.0:1619677447.181921:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.181924:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:8.0:1619677447.181927:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.181928:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.181928:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:8.0:1619677447.181929:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:24.0:1619677447.181930:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.181931:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.181931:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:8.0:1619677447.181932:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (21)++
00000400:00000200:24.0:1619677447.181933:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.181933:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (22)--
00000800:00000200:8.0:1619677447.181934:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (21)--
00000800:00000200:8.0:1619677447.181935:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.181936:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.181945:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.181948:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.181948:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.181949:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.181951:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.181954:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.181956:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (21)++
00000800:00000200:8.0:1619677447.181958:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000800:00000200:24.0:1619677447.181959:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (22)--
00000400:00000200:8.0:1619677447.181961:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.181964:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.181965:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677447.181967:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.181968:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.181969:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.181970:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677447.181973:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.181975:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677447.181977:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.181978:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:8.0:1619677447.181978:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:8.0:1619677447.181980:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:8.0:1619677447.181980:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (21)++
00000800:00000200:8.0:1619677447.181981:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (22)--
00000800:00000200:8.0:1619677447.181982:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (21)--
00000800:00000200:24.0:1619677447.181995:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (21)++
00000800:00000200:24.0:1619677447.181998:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000800:00000200:8.0:1619677447.181999:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (22)--
00000400:00000200:24.0:1619677447.182000:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.182003:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.182004:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.182006:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.182008:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.182009:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.182009:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.182012:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.182015:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.182018:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.182019:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.182019:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.182021:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.182021:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (21)++
00000800:00000200:24.0:1619677447.182022:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fe8667200] (22)++
00000800:00000200:24.0:1619677447.182025:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (24)--
00000800:00000200:24.0:1619677447.182026:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (23)--
00000800:00000200:24.0:1619677447.182028:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (22)++
00000800:00000200:24.0:1619677447.182030:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000400:00000200:24.0:1619677447.182033:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.182034:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (23)++
00000400:00000200:24.0:1619677447.182035:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.182036:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000400:00000200:24.0:1619677447.182037:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:24.0:1619677447.182039:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96731889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.182040:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.182042:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.182043:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:9.2:1619677447.182043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:24.0:1619677447.182048:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.182050:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:8.0:1619677447.182051:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.182054:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677447.182054:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.182056:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:24.0:1619677447.182056:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:8.0:1619677447.182056:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.182057:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96731888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.182058:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:8.0:1619677447.182059:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.182060:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.182061:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.182062:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (24)++
00000800:00000200:22.0:1619677447.182062:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:24.0:1619677447.182063:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (25)--
00000800:00000200:24.0:1619677447.182064:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (24)--
00000800:00000200:24.0:1619677447.182065:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (23)++
00000800:00000200:22.0:1619677447.182066:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.182066:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:24.0:1619677447.182067:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fe8667200] (24)--
00000400:00000200:22.0:1619677447.182067:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.182068:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (23)--
00000800:00000200:24.0:1619677447.182069:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:22.0:1619677447.182069:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.182071:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.182072:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.182073:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.182073:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.182074:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.182076:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.182076:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.182077:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:8.0:1619677447.182077:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.182078:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677447.182079:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96731887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.182080:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677447.182081:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.182082:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.182083:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.182085:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (22)++
00000800:00000200:8.0:1619677447.182086:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (23)--
00000400:00000200:24.0:1619677447.182087:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.182087:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (22)--
00000800:00000200:8.0:1619677447.182088:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (21)--
00000800:00000200:24.0:1619677447.182089:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.182089:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.182092:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.182092:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677447.182093:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.182093:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677447.182094:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:24.0:1619677447.182095:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.182095:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:13.0:1619677447.182095:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.182096:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.182096:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:13.0:1619677447.182096:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.182098:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.182099:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.182346:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.182349:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.182349:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.182351:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.182353:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.182355:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.182356:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.182358:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.182359:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.182360:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.182363:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.182364:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.182367:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.182368:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.182368:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.182370:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.182370:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.182371:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.182372:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.182377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677447.182519:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.182521:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.182523:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.182524:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.182528:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.182529:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.182531:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.182532:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.182534:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.182535:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.182538:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.182540:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.182543:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.182544:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.182544:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.182546:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.182547:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.182548:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.182549:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.182550:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677447.182552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.182563:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.182565:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.182566:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.182567:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.182601:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.182604:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.182605:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.182605:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.182607:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.182609:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.182628:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.182631:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.182631:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.182632:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.182633:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.182633:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.182732:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.182735:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.182735:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.182737:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.182740:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.182741:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.182788:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.182791:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.182792:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.182794:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.182797:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.182798:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.182800:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.182801:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.182802:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.182803:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.182806:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.182807:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.182810:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.182811:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.182812:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.182813:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.182814:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.182815:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.182815:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.183375:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.183378:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.183379:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.183379:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.183382:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.183385:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.183387:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.183390:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.183391:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.183393:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.183396:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.183397:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.183398:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.183400:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.183401:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.183403:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.183407:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.183410:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.183413:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.183414:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.183415:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.183416:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.183417:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.183418:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.183419:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.183420:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.183432:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.183434:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.183435:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.183436:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.183472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.183492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677447.183642:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.183645:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.183646:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.183646:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.183648:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.183650:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.183778:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.183781:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.183781:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.183782:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.183784:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.183786:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.183956:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.183959:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.183960:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.183961:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.183964:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.183965:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.183967:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.183968:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.183969:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.183970:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.183973:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.183974:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.183977:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.183978:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.183978:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.183980:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.183980:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.183981:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.183982:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.184068:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.184070:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.184071:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.184073:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.184075:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.184077:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.184078:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.184079:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.184080:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.184081:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.184084:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.184085:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.184089:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.184090:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.184090:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.184093:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.184093:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.184094:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.184094:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.184180:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.184182:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:13.0:1619677447.184182:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:6.0:1619677447.184183:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:13.0:1619677447.184184:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:22.0:1619677447.184185:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677447.184185:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.184187:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.184188:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.184189:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:22.0:1619677447.184190:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:13.0:1619677447.184190:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:22.0:1619677447.184192:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96731863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.184193:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.184195:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.184195:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677447.184199:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.184201:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.184203:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.184204:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:22.0:1619677447.184205:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.184205:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677447.184207:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677447.184208:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.184208:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:12.0:1619677447.184208:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:22.0:1619677447.184209:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:12.0:1619677447.184209:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.184210:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.184211:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.184213:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.184213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.184222:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe9286a00] (21)++
00000800:00000200:8.0:1619677447.184225:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.60@o2ib3
00000800:00000200:24.0:1619677447.184226:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe9286a00] (22)--
00000400:00000200:8.0:1619677447.184230:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.60@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.184233:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.184234:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:8.0:1619677447.184236:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.184237:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.184238:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.184239:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:8.0:1619677447.184242:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.60@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.184244:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 976 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:8.0:1619677447.184247:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.184248:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:8.0:1619677447.184249:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:8.0:1619677447.184250:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:8.0:1619677447.184251:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe9286a00] (21)++
00000800:00000200:8.0:1619677447.184252:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe9286a00] (22)--
00000800:00000200:8.0:1619677447.184252:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe9286a00] (21)--
00000800:00000200:9.2:1619677447.184301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677447.184657:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.184660:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.184660:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.184661:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.184663:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.184666:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.184869:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.184872:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.184873:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.184875:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.184878:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.184879:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.184880:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.184882:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.184883:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.184884:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.184886:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:24.0:1619677447.184887:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.184888:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.184889:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.184890:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.184890:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:24.0:1619677447.184891:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.184892:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.184892:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:22.0:1619677447.184892:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.184894:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.184894:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.184895:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677447.184895:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.184896:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.184956:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.184959:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.184960:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.184963:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.184965:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.184967:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.184969:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96731852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.184970:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.184972:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.184973:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.184978:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.184980:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.184983:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.184984:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.184985:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.184987:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.184987:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.184988:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.184989:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.185094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.185325:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.185327:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.185328:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.185330:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.185332:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.185333:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.185377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.185384:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.185387:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.185388:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.185388:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.185390:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.185394:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677447.185589:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.185592:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.185592:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.185595:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.185597:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.185599:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.185601:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.185602:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.185604:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.185605:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.185609:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.185611:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.185614:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.185615:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.185616:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.185618:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.185618:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.185619:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.185619:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.185689:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.185692:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.185693:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.185694:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.185696:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.185697:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.185738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.185759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:9.2:1619677447.185782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.185788:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.185791:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.185792:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.185792:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.185794:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.185798:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.185802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677447.185808:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.185810:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.185811:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.185811:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.185813:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.185813:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.185953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.186139:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.186142:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.186143:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.186143:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.186146:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.186149:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.186160:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.186162:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.186163:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.186163:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.186164:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.186165:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.186210:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.186213:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.186214:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.186214:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.186216:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.186218:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.186226:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.186228:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.186229:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.186231:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.186232:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.186234:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.186495:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.186498:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.186499:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.186501:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.186504:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.186505:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.186506:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96731832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.186507:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.186508:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677447.186509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:6.0:1619677447.186509:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.186512:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.186514:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.186517:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.186518:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.186519:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.186520:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.186521:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.186522:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.186523:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.186524:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.186533:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.186536:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.186537:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.186537:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.186616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.186663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.186670:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.186673:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.186674:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.186674:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.186676:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.186678:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.186683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.186689:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.186691:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.186692:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.186692:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.186693:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.186694:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.186706:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.186709:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.186710:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.186710:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.186712:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.186715:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.186844:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.186847:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.186848:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.186848:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.186850:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.186853:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.186942:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.186944:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.186945:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.186947:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.186949:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.186950:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.186991:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.186993:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.186994:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.186996:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.186997:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.187004:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.187005:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.187007:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.187008:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.187009:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.187009:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.187010:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.187022:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.187025:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.187026:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.187028:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.187029:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.187030:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.187041:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.187044:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.187045:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.187047:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.187050:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.187051:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.187053:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.187054:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.187055:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.187056:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.187059:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.187061:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.187064:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.187065:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.187066:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.187067:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.187068:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.187068:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.187069:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.187134:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.187137:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.187138:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.187140:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.187143:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.187144:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.187146:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.187147:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.187148:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.187149:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.187152:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.187154:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.187156:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.187157:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.187158:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.187160:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.187160:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.187161:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.187161:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.187166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.187450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.187514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.187521:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.187524:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.187525:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.187525:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.187527:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.187528:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677447.187530:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677447.187530:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.187531:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.187533:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.187535:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.187536:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.187547:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.187550:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.187551:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.187551:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.187552:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.187554:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.187668:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.187670:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.187671:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.187672:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.187674:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.187675:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.187775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.187794:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.187797:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:8.0:1619677447.187797:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677447.187798:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.187799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.187800:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677447.187800:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.187800:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:22.0:1619677447.187803:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.187803:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.187805:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:8.0:1619677447.187805:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.187806:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.187807:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.187808:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.187808:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.187809:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.187810:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677447.187811:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.187812:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.187814:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.187815:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.187818:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:8.0:1619677447.187819:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.187821:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.187822:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:8.0:1619677447.187822:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.187823:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.187825:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.187825:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677447.187825:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.187826:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677447.187826:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.187827:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.187827:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.187828:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.187829:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.187831:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.187832:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.187836:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.187838:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.187838:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.187840:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.187841:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.187842:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:24.0:1619677447.187843:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.187843:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.187844:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.187845:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:8.0:1619677447.187845:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:24.0:1619677447.187846:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.187848:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.187849:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.187850:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.187854:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.187857:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.187860:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.187860:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.187861:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.187863:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.187863:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.187864:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.187865:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.187869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.187913:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.187915:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.187916:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.187918:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.187920:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.187921:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.188014:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.188017:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.188017:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.188018:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.188021:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.188027:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.188029:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.188031:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.188032:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.188032:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.188033:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.188034:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.188108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.188114:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.188117:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.188118:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.188118:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.188120:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.188123:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.188201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.188333:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.188336:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.188337:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.188339:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.188342:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.188343:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.188345:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.188346:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.188347:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.188348:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.188352:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.188353:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.188356:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.188357:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.188357:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.188359:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.188360:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.188360:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.188361:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.188366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.188535:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.188538:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.188539:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.188541:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.188544:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.188546:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.188551:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.188553:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.188554:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.188555:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.188559:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.188561:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.188564:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.188565:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.188566:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.188567:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.188568:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.188569:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.188569:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.188592:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.188595:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.188596:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.188597:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.188598:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.188602:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.188612:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.188615:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.188616:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.188616:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.188617:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.188618:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.188683:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677447.188686:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677447.188687:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677447.188689:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.188692:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.188694:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677447.188695:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.188697:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.188698:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.188699:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677447.188704:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.188707:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677447.188710:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.188711:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677447.188711:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677447.188713:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677447.188714:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677447.188715:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677447.188716:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677447.188717:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:22.0:1619677447.188727:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.188729:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:8.0:1619677447.188729:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000400:00000200:22.0:1619677447.188731:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.188731:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.188732:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677447.188732:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:22.0:1619677447.188733:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.188734:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:22.0:1619677447.188736:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.188785:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.188787:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.188788:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.188788:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.188790:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.188793:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.189046:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.189049:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.189050:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.189050:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.189052:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.189055:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.189213:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.189216:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.189217:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.189217:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.189219:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.189221:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.189285:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.189287:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.189288:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.189290:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.189299:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.189301:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.189302:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.189304:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.189305:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.189306:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.189309:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.189311:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.189314:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.189314:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.189315:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.189317:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.189317:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.189322:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.189323:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.189399:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.189402:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.189402:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.189405:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.189407:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.189409:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.189410:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96731791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.189412:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.189413:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677447.189413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:22.0:1619677447.189415:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.189418:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.189420:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.189423:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.189424:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.189425:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.189427:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.189427:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.189428:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.189429:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.189436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677447.189615:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.189618:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.189619:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.189620:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.189622:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.189623:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.189696:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.189699:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.189700:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.189700:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.189703:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.189705:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.189716:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.189719:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.189720:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.189720:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.189721:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.189721:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.189802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.189809:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.189812:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.189813:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.189813:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.189815:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.189818:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.189847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.189888:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.189891:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.189891:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.189893:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.189896:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.189897:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.189899:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.189900:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.189901:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.189902:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.189906:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.189907:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.189910:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.189911:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.189912:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.189913:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.189914:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.189914:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.189914:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.190015:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.190017:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.190018:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.190020:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.190022:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.190023:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.190048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.190055:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.190057:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.190058:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.190059:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.190061:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677447.190064:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677447.190159:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.190163:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.190163:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.190167:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.190172:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.190174:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.190176:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.190177:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.190178:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.190179:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.190183:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677447.190183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.190186:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.190190:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.190190:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.190191:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.190191:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.190192:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.190194:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.190194:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:8.0:1619677447.190194:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.190196:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000400:00000200:8.0:1619677447.190196:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.190197:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.190198:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.190199:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.190208:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677447.190209:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.190210:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.190212:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677447.190212:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677447.190218:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.190221:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.190222:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.190224:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.190225:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.190226:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.190446:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.190449:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.190450:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.190452:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.190455:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.190456:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.190458:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.190459:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.190460:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.190461:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.190465:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.190467:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.190470:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.190470:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.190471:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.190473:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.190474:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.190475:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.190476:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.190477:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.190487:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.190489:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.190490:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.190491:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.190564:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.190567:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.190567:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.190569:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.190571:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.190572:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.190835:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.190838:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.190838:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.190839:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.190841:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.190845:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.190853:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.190856:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.190857:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.190859:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.190860:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.190861:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.190875:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.190878:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.190879:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.190879:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.190880:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.190882:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.191156:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.191160:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.191161:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.191164:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.191167:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.191168:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.191169:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.191171:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.191173:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.191174:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.191181:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.191184:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.191187:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.191188:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.191189:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.191191:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.191192:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.191193:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.191194:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.191208:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.191211:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.191212:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.191213:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.191215:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.191216:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.191408:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.191411:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.191413:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:22.0:1619677447.191414:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.191414:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.191416:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.191417:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.191418:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.191419:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.191420:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96731765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.191422:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.191423:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.191424:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.191428:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.191431:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.191431:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.191432:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.191433:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.191434:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.191435:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.191436:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.191438:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.191438:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.191439:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.191439:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.191548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.191758:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.191761:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.191762:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.191763:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.191766:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.191768:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.191769:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731760] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.191770:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.191771:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.191772:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.191775:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.191777:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.191779:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.191780:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.191780:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.191782:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.191783:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.191784:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.191785:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.191786:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.191798:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.191800:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.191801:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.191802:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.192103:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.192105:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.192106:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.192108:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.192110:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.192111:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.192122:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.192124:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.192125:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.192126:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.192126:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.192127:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.192392:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.192395:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.192396:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.192398:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.192401:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.192402:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.192403:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96731751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.192405:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.192406:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.192407:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.192410:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.192412:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.192415:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.192416:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.192416:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.192418:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.192418:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.192419:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.192420:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.192425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.192442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:9.2:1619677447.192821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.192929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.192958:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.192961:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.192962:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.192964:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.192967:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.192969:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.192971:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96731746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.192972:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.192973:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.192974:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.192978:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.192981:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.192983:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.192984:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.192984:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.192986:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.192987:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.192987:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.192988:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.192997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.193020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.193157:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.193160:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.193161:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.193164:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.193166:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.193168:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.193169:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96731741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.193171:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.193171:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.193172:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.193177:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.193180:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.193182:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.193183:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.193184:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.193185:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.193186:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.193186:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.193187:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.193192:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.193196:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.193196:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.193197:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.193199:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.193202:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.193589:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.193592:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.193593:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.193595:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.193598:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.193599:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.193601:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731737] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.193602:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.193603:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.193605:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.193608:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.193609:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.193612:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.193613:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.193614:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.193616:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.193616:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.193617:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.193617:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.193622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.193697:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.193700:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.193701:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.193702:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.193703:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.193705:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.193793:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.193795:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.193796:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.193798:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.193801:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.193803:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.193804:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.193806:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.193807:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.193808:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.193812:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.193814:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.193816:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.193817:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.193818:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.193820:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.193821:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.193821:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.193822:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.193839:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.193842:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.193843:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.193843:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.193845:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.193848:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.193933:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.193935:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.193936:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.193937:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.193940:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.193942:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.193959:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677447.193962:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677447.193963:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677447.193965:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.193968:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.193970:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:6.0:1619677447.193970:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:8.0:1619677447.193971:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96731730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.193973:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.193973:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.193974:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:8.0:1619677447.193974:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.193974:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.193975:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677447.193977:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.193978:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.193981:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.193982:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.193984:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.193984:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677447.193985:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677447.193987:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.193987:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677447.193988:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677447.193988:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:22.0:1619677447.194000:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.194003:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.194003:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.194004:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.194006:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.194009:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.194285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.194291:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.194297:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.194298:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.194299:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.194301:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.194304:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.194346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.194447:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.194450:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.194451:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.194452:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.194456:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.194457:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.194458:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.194460:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.194461:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.194462:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.194465:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.194467:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.194470:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.194471:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.194471:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.194473:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.194473:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.194474:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.194474:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.194605:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.194608:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.194608:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.194609:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.194610:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.194611:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.194748:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.194750:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.194751:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.194753:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677447.194755:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.194756:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.194758:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.194761:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.194762:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.194762:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.194765:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.194768:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.194783:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.194784:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.194784:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.194785:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.194786:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.194787:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.194787:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.194788:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.194788:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.194789:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.194790:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.194792:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.194793:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96731720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.194795:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.194796:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.194797:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:13.0:1619677447.194797:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.194800:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:22.0:1619677447.194801:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677447.194801:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.194802:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:13.0:1619677447.194803:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.194804:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.194805:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.194806:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.194806:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.194807:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.194809:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.194810:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.194812:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.194813:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.194813:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.194823:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.194825:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.194826:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.194826:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677447.194896:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.194900:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677447.194901:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.194903:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.194904:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.194905:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677447.194906:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.194906:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.194907:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.194907:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.194910:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.194945:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.194948:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.194949:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.194950:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.194951:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.194952:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.195052:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.195054:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677447.195054:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.195055:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000400:00000200:6.0:1619677447.195056:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.195058:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.195058:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.195062:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.195062:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.195063:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:22.0:1619677447.195064:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:6.0:1619677447.195064:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677447.195065:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96731714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.195065:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000400:00000200:22.0:1619677447.195067:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.195068:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.195068:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:22.0:1619677447.195069:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:13.0:1619677447.195069:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.195070:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.195071:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677447.195072:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:22.0:1619677447.195073:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.195073:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.195073:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.195075:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677447.195075:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.195075:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.195078:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.195078:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:12.0:1619677447.195078:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.195079:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677447.195081:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:22.0:1619677447.195081:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.195082:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677447.195083:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:24.0:1619677447.195114:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.195117:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.195118:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.195120:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.195123:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.195124:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.195125:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.195127:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.195128:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.195129:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.195131:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.195133:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.195136:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.195137:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.195137:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.195139:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.195139:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.195141:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.195142:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.195142:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.195156:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.195158:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.195158:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.195160:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.195185:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.195187:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.195189:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.195189:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.195190:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677447.195191:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.195193:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.195194:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.195194:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:13.0:1619677447.195195:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.195197:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.195198:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.195210:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.195213:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.195214:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.195215:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.195215:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.195217:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.195404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.195482:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.195485:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.195485:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.195486:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.195488:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.195491:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.195499:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.195502:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.195503:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.195503:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.195504:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.195505:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.195576:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.195579:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.195580:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.195581:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.195583:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.195584:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.195653:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.195656:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.195657:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.195659:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.195662:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.195663:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.195664:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.195666:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.195667:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.195668:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.195671:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.195673:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:22.0:1619677447.195675:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677447.195676:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.195677:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.195677:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677447.195678:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.195678:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677447.195679:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.195680:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.195680:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.195681:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:22.0:1619677447.195682:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.195684:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.195686:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.195687:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96731703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.195689:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.195690:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.195691:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.195695:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.195697:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.195700:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.195701:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.195701:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.195703:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:13.0:1619677447.195703:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:22.0:1619677447.195704:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.195704:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.195705:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.195705:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.195706:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.195708:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.195709:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.195710:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.195714:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.195716:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.195717:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.195718:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.195719:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.195721:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.195828:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.195830:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.195831:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.195832:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.195834:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.195835:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.195960:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.195962:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.195963:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.195965:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.195966:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.195967:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.196232:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.196235:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.196235:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.196237:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.196238:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.196239:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.196255:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.196258:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.196259:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.196259:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.196261:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.196265:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.196266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.196283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.196391:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.196393:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.196395:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.196397:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.196399:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.196401:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.196402:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.196403:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.196404:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.196405:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.196410:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.196412:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.196415:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.196416:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.196417:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.196419:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.196419:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.196420:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.196420:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.196623:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.196625:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.196626:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.196628:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.196630:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.196631:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.196639:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.196642:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.196643:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.196643:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.196644:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.196645:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.196702:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.196704:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.196705:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.196707:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.196710:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.196711:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.196712:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.196714:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.196715:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.196716:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.196719:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.196721:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.196724:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.196725:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.196726:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.196728:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.196728:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.196729:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.196729:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.196816:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.196818:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.196819:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.196821:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.196824:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.196825:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677447.196827:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.196828:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.196829:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.196830:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677447.196834:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.196835:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677447.196838:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.196839:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677447.196839:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677447.196841:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677447.196842:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.196843:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.196844:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.196845:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:6.0:1619677447.196855:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677447.196857:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:22.0:1619677447.196858:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.196858:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.196859:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.196860:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.196860:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:6.0:1619677447.196861:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.196862:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.196864:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.196938:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.196941:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.196942:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.196943:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.196945:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.196946:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.197692:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.197696:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.197696:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.197699:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.197702:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.197704:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.197705:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.197706:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.197707:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.197708:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.197712:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.197714:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.197717:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.197717:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.197718:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.197720:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.197720:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.197721:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.197721:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.197899:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.197901:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.197903:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.197904:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.197907:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.197909:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.197910:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.197911:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.197912:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.197913:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.197916:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.197918:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.197921:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.197921:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.197922:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.197923:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.197924:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.197925:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.197925:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.198101:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.198104:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.198105:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.198105:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.198107:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.198110:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.198278:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677447.198281:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677447.198282:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677447.198284:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677447.198286:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.198288:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677447.198289:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.198291:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.198292:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.198297:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677447.198302:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677447.198305:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.198307:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.198308:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677447.198309:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.198311:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.198311:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677447.198312:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677447.198313:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677447.198317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:22.0:1619677447.198323:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677447.198325:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.198326:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:22.0:1619677447.198326:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.198326:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:24.0:1619677447.198327:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.198328:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:22.0:1619677447.198329:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.198331:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677447.198333:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:22.0:1619677447.198333:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.198334:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.198336:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96731672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.198338:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.198339:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677447.198339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000400:00000200:22.0:1619677447.198340:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.198345:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.198346:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:22.0:1619677447.198348:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677447.198349:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.198349:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.198350:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:22.0:1619677447.198351:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.198351:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:8.0:1619677447.198351:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:22.0:1619677447.198352:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677447.198352:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677447.198354:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.198354:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.198356:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.198357:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.198358:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.198360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.198368:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.198370:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.198371:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.198372:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.198381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:13.0:1619677447.198681:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.198684:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.198685:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.198686:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.198688:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.198689:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.198759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.198761:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.198764:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.198765:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.198766:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.198767:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.198769:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.198784:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.198787:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.198788:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.198789:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.198790:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.198791:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.198818:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.198820:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.198821:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.198822:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.198824:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.198825:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.198960:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.198963:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.198964:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.198965:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677447.198967:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.198968:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.199031:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.199034:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.199035:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.199036:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.199039:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.199041:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.199042:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.199043:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.199044:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.199045:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.199049:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.199051:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.199054:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.199055:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.199055:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.199057:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.199057:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.199058:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.199059:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.199179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.199199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:13.0:1619677447.199374:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.199376:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.199377:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.199379:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.199381:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.199382:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.199528:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.199530:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.199531:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.199533:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.199534:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.199536:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.199545:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.199547:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.199548:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.199549:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.199549:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.199550:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.199692:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.199695:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.199696:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.199698:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.199701:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.199702:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.199704:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.199705:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.199706:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.199707:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.199711:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.199713:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.199716:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.199717:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.199717:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.199719:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.199719:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.199721:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.199722:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.199722:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.199735:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.199737:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.199738:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.199739:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.199740:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.199742:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.199743:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.199745:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.199746:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.199754:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.199774:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.199777:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.199777:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.199779:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.199780:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.199782:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.199876:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.199879:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.199879:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.199881:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.199882:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.199884:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.199966:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.199969:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.199970:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.199975:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.199977:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.199979:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.199980:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96731648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.199981:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.199982:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.199983:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.199987:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.199989:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.199994:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.199994:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.199995:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.199995:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.199996:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.199997:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:6.0:1619677447.199997:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.199998:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.199999:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.199999:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.200000:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.200001:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.200002:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.200015:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.200018:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.200019:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.200019:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.200020:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.200021:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.200051:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.200054:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.200055:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.200057:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.200058:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.200059:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.200118:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.200120:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.200121:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.200123:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.200124:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.200125:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.200138:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.200140:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.200141:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.200142:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.200143:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.200144:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.200188:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.200190:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.200191:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.200193:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.200194:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.200196:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.200241:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.200243:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.200244:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.200246:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.200247:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.200249:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.200258:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.200261:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.200262:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.200262:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.200264:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.200265:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.200270:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.200273:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.200273:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.200274:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.200276:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.200279:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.200319:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.200322:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.200323:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.200324:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.200327:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.200328:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.200330:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.200331:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.200332:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.200333:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.200336:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.200338:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.200341:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.200341:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.200342:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.200344:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.200344:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.200345:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.200345:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.200550:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.200552:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.200553:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.200555:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.200557:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.200558:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.200695:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.200697:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.200698:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.200699:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.200701:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.200702:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.200731:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.200734:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.200735:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.200735:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.200738:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.200746:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.200765:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.200767:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.200768:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.200768:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.200771:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.200773:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.200777:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.200779:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.200780:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.200782:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.200783:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.200784:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.200785:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.200787:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.200788:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.200788:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.200789:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.200790:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.200828:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.200831:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.200832:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.200834:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.200836:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.200838:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.200839:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.200841:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.200842:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.200843:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.200847:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.200848:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.200851:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.200852:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.200853:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.200855:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.200855:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.200856:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.200857:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.200861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.200953:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.200956:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.200956:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.200957:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.200959:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.200961:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.201039:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.201041:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.201042:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.201044:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.201047:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.201048:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.201049:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.201051:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.201052:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.201053:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.201056:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.201058:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.201061:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.201061:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.201061:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.201062:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.201063:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.201064:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677447.201064:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.201064:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.201065:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.201066:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:8.0:1619677447.201066:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.201074:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.201075:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.201077:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.201078:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.201079:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.201080:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.201084:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.201087:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.201090:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.201090:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.201091:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.201093:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.201093:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.201094:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.201095:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.201174:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.201177:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.201178:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.201179:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.201181:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.201183:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.201191:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.201193:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.201194:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.201195:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.201195:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.201197:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.201232:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.201234:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.201235:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.201237:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.201238:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.201240:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.201251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.201281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.201287:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.201290:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.201291:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.201291:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.201300:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.201304:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677447.201536:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe9286a00] (21)++
00000800:00000200:8.0:1619677447.201539:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.60@o2ib3
00000800:00000200:24.0:1619677447.201541:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe9286a00] (22)--
00000400:00000200:8.0:1619677447.201541:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.60@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.201544:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.201547:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:8.0:1619677447.201548:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.201550:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.201551:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.201552:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:8.0:1619677447.201555:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.60@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.201556:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:8.0:1619677447.201559:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.201560:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:8.0:1619677447.201561:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:8.0:1619677447.201563:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:8.0:1619677447.201563:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe9286a00] (21)++
00000800:00000200:8.0:1619677447.201563:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe9286a00] (22)--
00000800:00000200:8.0:1619677447.201564:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe9286a00] (21)--
00000800:00000200:24.0:1619677447.201607:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.201610:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.201611:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.201613:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.201616:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.201617:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.201619:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.201620:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.201621:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.201622:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.201625:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.201625:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:24.0:1619677447.201626:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677447.201628:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:24.0:1619677447.201629:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.201630:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.201630:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.201630:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.201630:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.201632:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.201633:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.201633:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.201634:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.201634:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.201635:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.201639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.201659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677447.201823:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.201826:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.201826:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.201828:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.201830:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.201831:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.201952:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.201955:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.201955:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.201957:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.201960:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.201961:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.201963:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.201964:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.201965:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.201966:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.201970:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.201971:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.201974:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.201975:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.201975:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.201977:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.201977:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.201978:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.201978:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.202033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.202156:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.202159:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.202160:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677447.202162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677447.202162:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.202165:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.202167:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.202168:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.202170:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.202171:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.202172:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.202177:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.202179:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.202182:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.202183:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.202184:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.202186:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.202186:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.202187:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.202187:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677447.202578:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.202581:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.202582:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.202584:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.202587:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.202588:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.202590:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.202591:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.202592:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.202593:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.202596:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.202598:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.202600:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.202600:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.202601:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.202603:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.202603:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.202604:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.202604:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.202616:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.202618:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.202619:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.202620:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.202623:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.202623:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.202698:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.202701:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.202702:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.202705:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.202709:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.202710:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.202712:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96731604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.202713:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.202714:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.202715:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.202719:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.202721:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.202723:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.202724:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.202725:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.202726:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.202727:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.202728:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.202729:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.202730:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.202740:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.202742:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.202743:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.202744:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.202786:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.202789:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.202789:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.202790:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.202792:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.202795:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.203149:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.203151:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.203152:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.203152:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.203154:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.203157:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.203291:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.203301:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.203301:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.203302:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.203304:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677447.203307:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.203318:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.203320:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.203322:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.203323:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.203326:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.203327:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.203328:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.203330:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.203330:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.203331:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.203334:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.203335:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.203338:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.203338:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.203339:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.203340:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.203341:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.203342:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.203343:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.203344:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.203352:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.203355:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.203355:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.203356:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:12.0:1619677447.203357:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.203358:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.203359:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677447.203359:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.203360:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.203360:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.203363:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.203366:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.203367:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.203369:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.203371:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.203373:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.203374:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.203375:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.203376:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.203378:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.203381:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.203383:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.203385:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.203386:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.203387:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.203388:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.203389:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.203390:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.203390:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.203451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.203631:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.203634:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.203634:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.203636:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.203637:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.203638:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.203651:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.203653:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.203654:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.203655:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.203656:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.203657:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.203770:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.203773:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.203774:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.203775:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.203777:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.203778:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.203791:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.203794:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.203794:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.203795:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.203796:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.203796:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.204070:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.204073:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.204074:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.204075:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.204078:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.204080:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.204081:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.204085:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.204086:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.204087:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.204095:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.204098:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.204101:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.204102:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.204103:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.204104:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.204105:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.204107:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.204108:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.204108:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677447.204109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.204119:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.204122:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.204123:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:24.0:1619677447.204124:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.204125:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.204125:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.204127:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.204127:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.204128:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.204130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.204133:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.204134:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.204136:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677447.204136:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.204137:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.204137:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.204138:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677447.204151:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.204153:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.204154:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.204156:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.204159:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.204161:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.204162:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.204164:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.204165:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.204166:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.204170:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.204172:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.204176:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.204176:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.204179:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.204180:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.204181:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.204181:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.204182:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.204185:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.204188:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.204188:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.204190:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.204192:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.204193:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.204345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.204415:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.204418:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.204419:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.204419:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.204421:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.204424:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.204542:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.204545:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.204546:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.204548:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.204549:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.204550:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.204591:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.204593:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.204594:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.204595:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.204597:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.204598:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.204688:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.204691:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.204692:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.204692:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.204695:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.204698:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.204706:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.204709:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.204709:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.204710:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.204711:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.204711:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.204896:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.204898:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.204899:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.204901:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.204902:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.204904:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.205226:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.205229:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.205230:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.205231:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.205233:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.205234:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.205249:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.205252:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.205253:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.205253:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.205254:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.205255:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.205320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.205339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.205353:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.205356:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.205357:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.205359:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.205362:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.205364:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.205365:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.205366:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.205367:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.205368:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.205372:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.205373:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.205376:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.205377:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.205377:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.205379:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.205379:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.205380:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.205380:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.205428:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.205430:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.205431:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.205433:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.205435:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.205435:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.205461:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.205463:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.205465:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.205466:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.205467:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.205469:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.205533:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.205535:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.205536:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.205536:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.205538:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.205541:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.205573:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.205576:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.205576:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.205579:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.205581:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.205582:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.205583:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.205585:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.205586:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.205587:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.205590:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.205592:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.205595:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.205596:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.205596:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.205599:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.205599:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.205600:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.205600:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.205952:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.205955:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.205956:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.205957:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.205959:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.205960:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.206105:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.206108:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.206109:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.206110:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.206112:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.206114:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.206115:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.206116:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.206117:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.206118:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.206121:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.206123:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.206126:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.206126:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.206127:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.206128:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.206129:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.206130:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.206130:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.206241:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.206244:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.206245:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.206245:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.206248:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.206251:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.206271:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.206273:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.206274:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.206276:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.206278:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.206279:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.206349:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.206352:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.206353:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.206354:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.206357:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.206359:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.206366:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.206369:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.206370:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.206371:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.206371:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.206373:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.206406:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.206409:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.206410:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.206411:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.206413:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.206414:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.206461:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.206464:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.206465:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.206465:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.206467:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.206469:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.206729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.206797:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.206800:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.206801:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.206804:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.206807:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.206808:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.206810:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.206812:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.206813:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.206814:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.206820:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.206823:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.206826:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.206827:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.206828:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.206830:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.206830:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.206831:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.206832:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.206851:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.206854:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.206855:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.206856:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.206858:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.206872:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.206874:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.206875:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.206876:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.206877:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.206878:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.206879:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.206949:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.206952:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.206953:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.206954:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.206957:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.206959:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.206960:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.206961:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.206962:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.206963:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.206966:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.206967:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.206970:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.206971:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.206972:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.206973:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.206974:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.206975:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.206975:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.207110:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.207113:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.207114:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.207116:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.207117:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.207118:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.207305:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.207308:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.207309:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.207309:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.207311:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.207313:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.207352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.207359:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.207362:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.207363:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.207363:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.207365:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.207368:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677447.207454:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.207457:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.207458:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.207461:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.207463:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.207465:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.207466:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.207468:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.207469:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.207470:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.207476:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.207478:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.207481:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.207482:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.207483:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.207485:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.207486:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.207487:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.207488:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.207488:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.207499:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.207501:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.207501:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:13.0:1619677447.207502:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:6.0:1619677447.207502:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.207505:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.207505:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.207507:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.207509:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.207510:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.207520:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.207523:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.207523:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.207524:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.207525:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.207526:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.207689:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.207691:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.207692:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.207693:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.207694:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:13.0:1619677447.207695:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.207696:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.207697:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.207698:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.207699:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.207701:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.207703:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.207756:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.207759:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.207760:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.207762:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.207765:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.207767:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.207769:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96731536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.207771:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.207773:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.207774:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.207779:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.207781:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.207786:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.207787:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.207788:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.207789:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.207790:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.207791:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.207793:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.207794:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677447.207799:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.207802:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.207803:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.207805:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.207806:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.207810:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.207811:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.0:1619677447.207811:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:13.0:1619677447.207813:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:8.0:1619677447.207813:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:13.0:1619677447.207814:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.207814:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:24.0:1619677447.207815:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.207815:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:13.0:1619677447.207816:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.207818:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.207857:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.207860:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.207861:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.207863:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.207866:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.207867:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.207869:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.207870:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.207871:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.207872:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.207876:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.207877:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.207880:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.207880:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.207881:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.207883:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.207883:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.207884:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.207884:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.207905:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.207907:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.207908:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.207909:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.207912:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.207912:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677447.207998:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.208001:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.208002:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.208004:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.208007:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.208009:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.208010:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.208011:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.208013:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.208014:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.208019:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.208022:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.208025:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.208026:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.208027:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.208028:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.208029:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.208030:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.208030:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.208035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:12.0:1619677447.208045:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.208048:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.208049:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.208051:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.208052:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.208054:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.208082:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.208085:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.208085:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.208086:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.208088:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.208092:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.208315:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.208318:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.208319:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.208321:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.208322:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.208324:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.208331:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.208334:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.208335:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.208336:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.208336:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.208338:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.208724:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.208727:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:13.0:1619677447.208728:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:22.0:1619677447.208729:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:13.0:1619677447.208730:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:6.0:1619677447.208730:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.208731:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:13.0:1619677447.208731:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.208732:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:13.0:1619677447.208732:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.208733:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.208733:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.208734:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.208734:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:22.0:1619677447.208735:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.208735:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.208736:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96731518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.208738:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.208739:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.208740:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.208743:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.208745:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:6.0:1619677447.208745:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.208746:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.208747:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.208748:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.208748:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.208749:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.208750:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.208751:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.208752:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.208753:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.208755:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.208755:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.208756:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.208756:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.208931:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.208934:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.208935:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.208937:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.208939:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.208941:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.208942:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.208943:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.208944:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.208945:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.208948:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.208950:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.208952:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.208953:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.208954:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.208955:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.208956:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.208956:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.208957:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.209035:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.209037:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.209038:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.209040:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.209042:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.209042:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.209043:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.0:1619677447.209044:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:13.0:1619677447.209046:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.209047:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.209047:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.209048:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:13.0:1619677447.209049:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.209050:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:8.0:1619677447.209050:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.209051:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.209052:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.209052:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.209053:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.209053:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677447.209053:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.209054:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:8.0:1619677447.209054:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:12.0:1619677447.209055:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677447.209055:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.209057:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.209058:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.209059:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.209062:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.209064:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.209068:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.209068:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.209069:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.209071:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.209072:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.209072:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.209073:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.209077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.209428:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.209430:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.209431:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.209432:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.209434:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.209435:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.209498:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.209500:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.209501:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.209503:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.209504:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.209505:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.209506:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.209509:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.209510:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.209510:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.209512:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.209515:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.209520:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.209523:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.209524:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.209526:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.209527:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.209528:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.209904:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.209906:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.209907:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.209908:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.209910:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.209911:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.209934:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.209937:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.209938:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.209940:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.209943:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.209944:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.209945:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.209947:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.209947:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.209948:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.209952:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.209953:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.209957:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.209957:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.209958:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.209959:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.209960:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.209961:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.209962:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.209963:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.209976:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.209978:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.209978:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.209979:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.210292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:12.0:1619677447.210334:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.210337:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.210338:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.210340:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.210342:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.210343:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.210357:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.210360:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.210361:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.210362:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.210362:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.210363:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.210418:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.210420:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.210421:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.210423:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.210424:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.210426:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.210439:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.210441:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.210442:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.210443:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.210443:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.210445:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.210484:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.210487:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.210488:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.210488:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.210490:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.210493:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.210503:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.210505:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.210506:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.210507:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.210508:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.210508:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.210539:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.210541:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.210542:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.210544:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.210545:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.210547:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.210652:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.210655:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.210656:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.210657:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.210660:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.210662:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.210663:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.210665:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.210666:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.210667:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.210674:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.210676:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.210680:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.210681:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.210681:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.210683:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.210683:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.210684:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.210685:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.210735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.210742:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.210745:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.210746:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.210746:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.210748:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.210752:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677447.210974:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.210977:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.210978:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.210979:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.210981:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.210982:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.211022:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.211025:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.211025:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.211027:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.211030:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.211031:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.211033:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.211034:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.211035:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.211036:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.211040:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.211042:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.211045:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.211046:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.211046:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.211049:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.211049:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.211050:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.211050:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.211141:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.211144:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.211145:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.211145:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.211147:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.211150:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.211434:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.211436:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.211437:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.211438:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.211440:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.211445:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.211447:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.211449:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.211450:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.211450:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.211452:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.211455:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.211726:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.211729:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.211730:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.211733:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.211736:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.211738:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.211740:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.211741:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.211743:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.211744:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.211748:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.211751:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.211754:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.211755:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.211755:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.211757:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.211757:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.211758:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.211759:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.211762:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.211765:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.211766:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.211766:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.211768:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.211770:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.211818:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.211820:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.211821:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.211823:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.211826:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.211827:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.211829:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96731477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.211830:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.211831:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.211832:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.211835:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.211836:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.211839:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.211840:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.211841:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.211842:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.211843:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.211844:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.211845:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.211845:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.211856:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.211858:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.211859:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.211860:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.211875:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.211878:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.211879:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.211879:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.211882:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.211885:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.212173:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.2:1619677447.212174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.212176:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677447.212178:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.212179:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.212180:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.212182:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.212182:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.212183:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.212184:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.212185:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.212185:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.212187:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.212188:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.212189:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.212192:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.212193:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:8.0:1619677447.212193:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.212195:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.212198:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.212199:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.212200:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.212202:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.212203:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.212204:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.212204:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.212209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.212282:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.212285:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.212286:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.212286:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.212289:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.212301:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.212324:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.212326:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.212327:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.212328:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.212331:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.212332:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.212347:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.212350:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.212350:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.212352:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.212355:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.212357:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.212358:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.212359:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.212360:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.212361:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.212365:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.212367:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.212370:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.212370:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.212371:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.212373:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.212373:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.212374:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.212374:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.212678:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.212681:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.212682:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.212684:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.212687:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.212689:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.212690:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.212691:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.212692:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.212693:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.212696:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.212698:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.212701:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.212702:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.212702:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.212704:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.212704:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.212706:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.212707:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.212708:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.212720:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.212722:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.212723:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.212724:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.212796:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.212799:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.212800:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.212801:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.212803:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.212803:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677447.212902:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.212905:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.212905:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.212906:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.212908:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.212911:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.212919:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.212921:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.212922:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.212922:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.212923:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.212924:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.213072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.213079:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.213082:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.213083:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.213084:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.213086:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.213089:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677447.213223:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.213226:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.213227:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.213228:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.213230:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.213231:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.213326:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.213329:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.213330:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.213331:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.213335:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.213337:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.213338:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.213339:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.213340:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.213341:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.213343:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.213346:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:8.0:1619677447.213346:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.213347:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677447.213348:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.213350:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.213351:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.213352:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.213352:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:22.0:1619677447.213353:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.213354:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:8.0:1619677447.213354:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.213354:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.213355:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677447.213356:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96731457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.213356:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677447.213358:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.213359:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.213361:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.213365:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.213367:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.213369:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.213370:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)++
00000800:00000200:22.0:1619677447.213371:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (24)++
00000800:00000200:22.0:1619677447.213374:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (25)--
00000800:00000200:22.0:1619677447.213374:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.213375:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.213376:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.213382:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.213385:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.213386:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.213388:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.213389:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.213390:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.213398:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.213401:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.213402:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.213402:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.213403:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.213405:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.213532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.213556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677447.213624:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.213627:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.213628:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.213628:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.213630:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.213632:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.213644:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.213646:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.213647:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.213649:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.213650:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.213651:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.213914:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.213917:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.213918:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.213919:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.213921:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.213922:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.214181:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.214183:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.214184:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.214186:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.214188:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.214190:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.214192:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.214193:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.214194:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.214195:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.214198:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.214200:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.214203:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.214204:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.214204:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.214206:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.214206:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.214207:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.214207:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.214299:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.214302:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.214303:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.214306:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.214307:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.214308:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.214346:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.214350:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (25)--
00000400:00000200:12.0:1619677447.214351:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.214353:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.214354:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677447.214355:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677447.214355:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677447.214356:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.214356:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677447.214357:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.214360:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.214369:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.214372:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.214373:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.214374:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.214374:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.214376:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.214392:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.214395:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.214396:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.214398:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.214400:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.214402:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.214403:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.214405:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.214406:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.214407:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.214410:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.214412:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.214414:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.214415:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.214416:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.214418:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.214418:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.214419:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.214420:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.214424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.214445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.214600:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.214603:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.214604:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.214604:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.214606:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.214610:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.214617:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.214620:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.214621:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.214621:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.214622:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.214623:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.214680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.214959:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.214961:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.214962:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.214964:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.214965:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.214967:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.215062:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.215065:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.215066:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.215068:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.215071:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.215073:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.215074:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.215075:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.215076:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.215077:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.215080:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.215082:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.215085:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.215086:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.215086:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.215088:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.215089:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.215090:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.215090:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677447.215294:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.215300:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.215302:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:12.0:1619677447.215302:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:24.0:1619677447.215305:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.215305:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.215306:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.215307:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.215309:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.215310:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.215310:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:24.0:1619677447.215311:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:13.0:1619677447.215311:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.215312:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.215312:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.215312:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677447.215313:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.215313:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677447.215315:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.215315:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.215316:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.215320:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.215322:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.215325:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.215326:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.215326:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.215328:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.215328:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.215329:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.215330:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.215457:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.215461:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.215461:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.215462:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.215464:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.215467:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.215619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.215667:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.215670:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.215671:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.215671:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.215673:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.215676:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.215894:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.215897:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.215897:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.215898:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.215900:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.215903:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.216028:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:9.2:1619677447.216029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:12.0:1619677447.216031:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.216034:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.216035:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.216035:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.216036:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:12.0:1619677447.216038:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.216039:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.216040:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.216040:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.216042:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.216045:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677447.216050:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.216052:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.216053:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.216055:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.216056:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.216058:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.216178:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.216181:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.216181:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.216185:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.216187:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.216189:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.216191:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96731413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.216193:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.216194:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.216195:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.216199:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.216202:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.216205:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.216206:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.216206:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.216208:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.216209:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.216210:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.216211:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.216212:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677447.216212:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.216215:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.216216:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.216218:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.216219:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.216220:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.216222:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.216225:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.216226:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.216226:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.216344:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.216346:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.216347:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.216348:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.216350:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.216352:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.216355:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.216357:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.216359:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.216360:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.216363:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.216365:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.216366:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.216367:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.216369:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.216369:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.216373:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.216375:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.216377:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.216378:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.216379:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.216381:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.216381:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.216383:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.216384:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.216384:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.216397:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.216399:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.216400:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.216401:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.216413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.216462:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.216464:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.216465:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.216466:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.216468:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.216470:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.216480:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.216482:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.216483:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.216483:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.216484:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.216485:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.216732:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.216735:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.216735:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.216736:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.216738:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.216740:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.216776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.216875:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.216878:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.216879:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.216879:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.216881:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.216884:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.216928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.216931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.217053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.217072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.217199:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.217202:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.217202:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.217203:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.217205:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.217208:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.217253:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.217256:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.217258:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.217259:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000400:00000200:6.0:1619677447.217259:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.217260:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000400:00000200:22.0:1619677447.217262:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.217262:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.217264:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.217265:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.217267:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.217268:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.217269:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.217271:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.217273:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:6.0:1619677447.217273:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.217274:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96731397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.217275:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.217276:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.217277:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.217278:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:22.0:1619677447.217281:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.217281:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.217282:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677447.217282:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.217283:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.217285:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.217286:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:6.0:1619677447.217286:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.217286:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.217287:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:6.0:1619677447.217287:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.217288:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.217289:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:6.0:1619677447.217289:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677447.217290:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677447.217290:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677447.217291:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:13.0:1619677447.217346:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.217348:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.217349:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.217350:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677447.217352:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.217354:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.217564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.217579:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:9.2:1619677447.217581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.217582:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.217583:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.217583:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.217586:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.217591:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.217659:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.217662:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.217663:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.217665:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.217665:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.217668:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.217670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.217678:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:13.0:1619677447.217679:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677447.217680:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.217681:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.217682:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.217682:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.217683:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:24.0:1619677447.217684:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.217684:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.217684:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.217686:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.217691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.217693:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.217695:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.217698:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.217699:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.217700:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.217701:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.217701:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677447.217714:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.217717:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.217718:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.217718:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.217721:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.217723:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.217734:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.217737:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.217738:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.217738:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.217739:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.217740:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.217915:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.217918:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.217919:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.217919:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.217922:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.217925:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.217938:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.217941:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.217941:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.217942:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.217943:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.217944:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.218029:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:9.2:1619677447.218030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:22.0:1619677447.218032:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.218033:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.218034:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.218037:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.218037:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:22.0:1619677447.218039:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.218040:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96731375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.218040:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.218041:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.218041:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.218042:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.218043:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677447.218044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677447.218044:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.218045:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.218049:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677447.218049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:22.0:1619677447.218052:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.218053:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.218054:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.218055:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:22.0:1619677447.218055:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.218055:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.218056:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.218057:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:22.0:1619677447.218058:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.218059:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677447.218059:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (21)++
00000800:00000200:22.0:1619677447.218060:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677447.218060:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:22.0:1619677447.218061:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.218061:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.218076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:12.0:1619677447.218123:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.218125:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.218126:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.218128:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.218129:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.218130:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.218136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.218209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.218286:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.218289:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.218290:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.218292:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.218300:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.218301:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.218303:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.218304:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.218305:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.218306:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.218309:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.218311:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.218314:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.218315:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.218315:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.218317:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.218317:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.218318:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.218318:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.218416:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.218419:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.218420:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.218421:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.218423:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.218424:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.218633:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.218636:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.218637:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.218637:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.218639:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.218642:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.218896:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.218898:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.218899:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.218901:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.218903:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.218903:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.218914:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.218917:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.218917:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.218919:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.218921:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.218922:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.218941:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.218943:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.218944:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.218945:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.218945:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.218947:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.219073:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.219077:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.219077:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.219082:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.219084:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.219086:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.219088:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.219089:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.219090:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.219091:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:9.2:1619677447.219093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:8.0:1619677447.219096:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.219098:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.219100:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.219101:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.219102:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.219102:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:24.0:1619677447.219103:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.219103:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:24.0:1619677447.219105:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.219105:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.219106:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.219107:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.219108:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.219108:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.219109:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677447.219213:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.219216:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.219216:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.219218:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.219219:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.219221:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.219298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.219305:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.219308:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.219309:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.219309:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.219312:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.219315:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677447.219334:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.219337:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.219337:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.219339:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.219341:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.219342:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.219389:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.219392:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.219393:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.219395:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.219397:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.219399:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.219400:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.219402:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.219403:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.219404:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.219409:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.219411:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.219414:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.219415:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.219415:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.219417:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.219418:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.219418:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.219419:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.219423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.219635:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.219637:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.219638:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.219639:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.219641:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.219643:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.219689:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.219691:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.219693:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.219694:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.219696:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.219696:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.219712:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.219714:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.219716:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.219717:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.219720:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.219721:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.219723:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.219725:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.219727:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.219728:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.219733:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.219735:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.219737:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.219738:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.219739:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.219740:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.219741:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.219741:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.219742:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.219768:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.219771:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.219772:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.219774:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.219775:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.219777:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.219839:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.219842:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.219842:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.219845:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.219847:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.219849:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.219850:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.219851:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.219852:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.219853:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.219856:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.219858:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.219861:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.219861:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.219862:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.219864:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.219864:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.219865:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.219865:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.219912:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.219915:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.219916:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.219916:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.219918:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.219920:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.219999:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.220002:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.220002:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.220004:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.220005:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.220006:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.220023:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.220025:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.220026:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.220026:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.220027:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.220028:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.220274:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.220277:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.220277:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.220279:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.220281:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.220282:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.220289:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.220291:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.220292:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.220300:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.220302:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.220304:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.220306:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.220307:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.220308:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.220309:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.220322:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.220324:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.220327:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.220328:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.220329:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.220331:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.220332:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.220343:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.220344:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.220345:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.220358:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.220360:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.220361:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.220362:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.220522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.220544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.220632:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.220634:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.220635:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.220636:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.220638:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.220641:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.220742:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.220745:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.220746:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.220748:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.220750:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.220751:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.220772:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.220775:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.220776:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.220778:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.220781:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.220783:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677447.220784:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.220785:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.220786:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.220787:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677447.220790:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.220792:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677447.220795:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.220795:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677447.220796:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677447.220798:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677447.220798:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.220799:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.220801:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.220801:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677447.220801:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.220804:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:22.0:1619677447.220807:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.220807:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.220808:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:22.0:1619677447.220810:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.220810:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.220811:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.220812:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.220813:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96731332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.220814:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:6.0:1619677447.220814:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.220815:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.220816:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.220816:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:22.0:1619677447.220817:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.220818:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:8.0:1619677447.220818:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.220821:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.220822:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.220822:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.220825:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.220828:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.220828:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.220829:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.220831:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.220832:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.220833:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.220834:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.220835:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.220845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.220845:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.220847:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.220848:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.220848:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677447.221019:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.221022:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.221023:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.221025:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.221028:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.221029:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.221031:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.221032:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.221033:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.221034:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.221037:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.221039:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.221042:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.221043:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.221043:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.221045:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.221046:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.221046:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.221047:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.221151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.221261:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.221264:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.221265:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.221266:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.221268:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.221271:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.221280:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.221283:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.221283:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.221284:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.221285:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.221286:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.221604:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.221607:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.221607:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.221609:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.221611:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.221613:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.221614:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96731321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.221615:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.221617:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.221618:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:12.0:1619677447.221621:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677447.221621:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.221623:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:12.0:1619677447.221624:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.221624:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.221626:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677447.221627:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.221627:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:13.0:1619677447.221628:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677447.221628:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:12.0:1619677447.221630:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.221630:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.221631:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.221631:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.221632:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.221664:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:22.0:1619677447.221666:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677447.221666:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.221668:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677447.221669:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:8.0:1619677447.221669:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.221670:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.221670:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.221672:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.221672:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.221674:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:22.0:1619677447.221675:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:8.0:1619677447.221676:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.221677:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.221678:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.221679:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.221682:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.221684:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.221687:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.221688:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.221689:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.221690:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.221691:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.221691:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.221692:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.222013:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.222015:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.222016:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.222019:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677447.222020:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.222023:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.222032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:12.0:1619677447.222036:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.222038:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.222040:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.222040:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.222041:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.222042:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.222112:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.222115:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.222116:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.222116:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.222118:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.222121:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.222225:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.222227:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.222228:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.222230:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.222232:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.222234:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.222236:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.222238:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.222239:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.222240:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.222244:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.222247:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.222250:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.222251:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.222252:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.222253:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.222254:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.222255:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.222255:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.222281:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.222283:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.222284:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.222286:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.222287:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677447.222287:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:13.0:1619677447.222289:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.222290:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.222290:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.222291:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.222300:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.222302:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.222305:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.222309:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.222309:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.222310:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.222311:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.222312:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.222378:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.222381:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.222382:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.222385:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.222388:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.222389:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.222391:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.222392:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.222393:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.222394:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.222398:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.222400:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.222403:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.222403:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.222404:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.222406:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.222406:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.222407:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.222407:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.222582:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.222585:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.222586:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.222586:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:22.0:1619677447.222588:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.222588:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.222589:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (23)++
00000800:00000200:6.0:1619677447.222590:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (24)--
00000800:00000200:22.0:1619677447.222591:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.222591:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.222592:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.222620:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.222623:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.222624:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.222625:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.222626:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.222628:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.222801:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.222804:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.222805:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.222805:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.222807:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.222810:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.222850:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.0:1619677447.222852:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:13.0:1619677447.222853:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.222854:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.222855:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677447.222856:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:13.0:1619677447.222856:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.222856:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.222858:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:8.0:1619677447.222858:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.222861:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.222862:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:8.0:1619677447.222864:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.222865:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.222866:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.222867:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677447.222871:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.222873:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:8.0:1619677447.222875:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.222876:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:8.0:1619677447.222877:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:8.0:1619677447.222878:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:8.0:1619677447.222879:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677447.222879:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677447.222880:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:6.0:1619677447.223185:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.223187:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.223188:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.223188:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.223191:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.223193:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.223439:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.223441:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.223442:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.223444:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.223446:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.223447:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.223504:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.223506:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.223507:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.223509:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.223512:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.223513:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.223514:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.223516:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.223517:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.223518:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.223520:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.223522:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.223525:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.223526:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.223526:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.223528:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.223529:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.223529:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.223530:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.223827:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.223830:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.223831:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.223833:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.223835:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.223837:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.223838:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.223839:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.223840:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.223841:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.223845:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.223847:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.223850:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.223850:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.223851:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.223853:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.223853:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.223854:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.223854:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.223899:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.223902:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.223903:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.223905:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.223906:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.223908:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.224211:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.224214:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.224215:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.224217:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.224218:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.224220:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.224491:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:12.0:1619677447.224494:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677447.224494:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.224495:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.224495:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.224496:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.224497:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.224498:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:12.0:1619677447.224499:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.224501:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677447.224501:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.224503:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.224609:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.224612:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.224613:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.224614:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.224617:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.224619:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.224620:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.224622:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.224623:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.224624:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.224627:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.224629:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.224631:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.224632:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.224633:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.224634:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.224635:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.224636:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.224636:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.224747:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.224750:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.224751:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.224752:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.224755:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.224756:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.224757:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.224759:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.224760:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.224761:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.224764:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.224766:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.224769:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.224769:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.224770:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.224772:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.224772:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.224773:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.224774:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.224775:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.224785:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.224787:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.224787:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.224788:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.224825:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.224828:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.224829:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.224831:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.224832:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.224834:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.224845:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.224849:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.224850:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.224851:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.224851:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.224853:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.224861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.224907:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.224910:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.224911:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.224911:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.224913:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.224916:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.224925:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.224928:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.224929:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.224929:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.224930:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.224931:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.225042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.225085:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.225087:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.225088:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.225090:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.225091:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.225092:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:12.0:1619677447.225092:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.225095:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.225096:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.225099:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.225101:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.225102:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:24.0:1619677447.225103:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.225104:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.225105:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677447.225106:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.225106:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:24.0:1619677447.225107:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.225107:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.225108:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:24.0:1619677447.225109:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677447.225109:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:24.0:1619677447.225112:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.225114:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.225117:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.225117:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.225118:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.225120:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.225120:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.225121:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.225122:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.225123:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677447.225123:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.225126:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.225127:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.225127:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.225129:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.225131:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.225136:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.225138:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.225139:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.225140:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.225425:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.225428:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.225429:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.225431:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.225434:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.225436:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.225437:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.225438:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.225439:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.225440:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.225443:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.225445:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.225448:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.225449:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.225449:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.225451:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.225452:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.225452:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.225452:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.225511:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.225514:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.225515:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.225515:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.225517:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.225520:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.225663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.225671:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.225674:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.225675:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.225677:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.225679:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.225681:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.225682:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.225683:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.225683:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000400:00000200:24.0:1619677447.225684:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.225685:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:22.0:1619677447.225686:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677447.225686:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677447.225690:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.225691:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.225693:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:22.0:1619677447.225693:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.225694:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:24.0:1619677447.225696:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677447.225696:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96731265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.225697:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:22.0:1619677447.225697:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.225698:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:22.0:1619677447.225698:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.225699:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677447.225700:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.225700:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.225701:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.225701:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677447.225703:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.225705:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.225708:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.225709:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.225710:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.225711:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.225712:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677447.225713:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677447.225713:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677447.225756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.225777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677447.225913:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.225916:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.225916:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.225918:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.225919:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.225920:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.225922:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.225923:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.225924:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.225926:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.225927:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.225928:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.225929:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.225931:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.225933:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96731260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.225935:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.225935:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.225935:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.225936:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.225938:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.225941:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677447.225942:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.225942:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.225943:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.225945:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.225945:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.225946:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)++
00000800:00000200:6.0:1619677447.225947:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (24)--
00000800:00000200:22.0:1619677447.225948:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.225948:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.225948:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.225949:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.225949:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.225951:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.225952:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.225952:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.225953:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.225958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.226000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.226063:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.226065:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.226066:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.226067:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.226070:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.226070:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:4.2:1619677447.226113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.226128:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.226130:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.226131:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.226133:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.226134:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.226136:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.226144:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.226147:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.226148:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.226149:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.226153:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.226154:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.226155:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.226157:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.226157:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.226158:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.226161:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.226163:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.226167:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.226168:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.226168:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.226170:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.226170:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.226172:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.226173:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.226173:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677447.226176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.226186:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.226188:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.226189:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.226190:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.226194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:4.2:1619677447.226244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.226252:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.226254:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.226256:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.226257:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.226260:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.226262:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.226263:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.226265:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.226266:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.226266:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.226270:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.226271:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.226274:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.226275:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.226275:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.226277:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.226277:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.226278:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.226279:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.226349:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.226352:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.226352:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.226354:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.226356:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.226357:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.226571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.226617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.226618:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.226620:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.226621:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.226622:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.226623:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.226626:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.226724:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.226726:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.226727:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.226728:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.226730:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.226731:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.226747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.226755:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.226758:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.226759:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.226760:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.226763:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.226765:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.226766:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.226767:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.226768:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.226769:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677447.226772:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:24.0:1619677447.226773:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.226774:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:24.0:1619677447.226775:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:13.0:1619677447.226775:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.226777:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.226778:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.226778:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.226779:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.226779:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677447.226779:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.226781:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.226781:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.226782:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.226783:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.226848:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.226851:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.226852:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.226854:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.226856:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.226858:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.226859:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.226860:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.226862:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.226863:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.226866:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.226868:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.226871:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.226872:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.226873:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.226875:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.226876:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.226876:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.226877:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.227042:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.227045:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.227045:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677447.227047:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.227050:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.227052:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677447.227053:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.227055:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.227056:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.227057:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677447.227060:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.227062:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.227065:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.227065:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677447.227066:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677447.227068:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.227068:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.227069:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.227069:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.227081:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.227084:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.227085:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.227085:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.227087:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.227090:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.227159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.227177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:4.2:1619677447.227250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.227298:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.227302:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.227303:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.227303:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.227306:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.227310:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.227312:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.227314:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.227315:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.227315:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.227317:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.227317:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.227334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.227341:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.227344:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.227345:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.227346:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.227348:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.227351:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677447.227559:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.227562:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.227562:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.227565:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.227567:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.227569:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.227570:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.227572:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.227573:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.227574:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.227577:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.227579:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.227582:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.227583:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.227584:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.227586:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.227586:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.227588:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.227589:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.227589:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.227599:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.227601:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.227602:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.227603:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.227855:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.227857:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.227858:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.227858:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.227861:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.227863:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.227875:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.227877:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.227878:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.227878:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.227879:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.227880:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.227900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.227923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.227947:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.227950:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.227951:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.227952:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.227954:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.227957:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.228030:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.228033:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.228035:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.228036:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.228039:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.228040:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677447.228042:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.228043:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.228044:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.228045:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677447.228048:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.228049:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.228052:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.228053:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677447.228053:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677447.228055:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.228055:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.228056:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.228057:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.228068:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.228071:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.228072:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.228072:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.228074:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.228077:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.228141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.228148:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.228151:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.228152:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.228154:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.228154:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677447.228157:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000400:00000200:24.0:1619677447.228158:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.228158:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:24.0:1619677447.228159:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.228161:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.228161:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.228162:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.228163:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.228164:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.228164:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.228165:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:6.0:1619677447.228168:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96731228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.228170:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.228171:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.228172:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.228173:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:24.0:1619677447.228175:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.228178:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.228179:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.228180:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.228181:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.228181:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677447.228182:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.228183:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.228184:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677447.228184:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.228185:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677447.228186:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.228186:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677447.228186:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.228188:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.228189:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677447.228190:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:6.0:1619677447.228191:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:6.0:1619677447.228192:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:4.2:1619677447.228192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.228200:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:12.0:1619677447.228201:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677447.228202:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:12.0:1619677447.228203:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677447.228203:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:12.0:1619677447.228204:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.228204:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:12.0:1619677447.228206:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.228207:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.228208:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:13.0:1619677447.228208:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.228208:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.228208:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000400:00000200:22.0:1619677447.228209:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.228210:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:6.0:1619677447.228210:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677447.228211:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.228211:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.228212:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:22.0:1619677447.228213:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.228256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.228462:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.228465:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.228465:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.228466:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.228468:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.228470:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.228504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.228549:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.228552:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.228553:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.228554:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.228555:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.228558:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.228568:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.228570:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.228571:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.228572:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.228573:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.228573:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.228670:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.228672:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.228673:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.228675:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.228677:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.228678:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.228689:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.228691:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.228692:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.228692:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.228693:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.228694:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.228830:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.228833:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.228834:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.228836:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.228839:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.228840:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.228842:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.228843:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.228845:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.228846:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.228850:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.228852:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.228854:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.228855:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.228855:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.228857:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.228858:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.228859:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.228859:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.228893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.228937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.228941:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.228944:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.228945:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.228946:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.228948:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.228951:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.229141:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.229144:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.229144:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.229146:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.229148:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.229149:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.229184:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.229186:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.229187:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.229189:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.229192:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.229194:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.229195:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.229196:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.229197:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.229198:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.229201:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.229203:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.229206:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.229206:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.229207:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.229209:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.229209:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.229210:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.229210:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.229390:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.229393:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.229393:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.229395:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.229397:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.229398:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.229405:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.229408:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.229409:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.229410:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.229410:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.229412:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.229532:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:12.0:1619677447.229534:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:22.0:1619677447.229535:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.229536:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.229536:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:12.0:1619677447.229537:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.229538:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.229539:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:12.0:1619677447.229540:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.229541:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677447.229542:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.229543:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.229549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.229596:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.229599:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.229600:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.229600:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.229603:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.229606:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.229807:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.229810:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.229811:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.229812:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.229814:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.229815:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.230142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.230287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.230346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.230379:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.230382:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.230382:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677447.230384:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:22.0:1619677447.230385:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.230387:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.230387:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.230388:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.230389:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.230390:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96731199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.230391:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.230391:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.230392:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.230393:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.230394:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:12.0:1619677447.230394:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:22.0:1619677447.230398:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.230401:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677447.230401:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677447.230403:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.230404:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:13.0:1619677447.230404:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.230405:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:13.0:1619677447.230405:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.230406:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.230406:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677447.230407:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.230407:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.230408:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:13.0:1619677447.230408:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.230409:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.230409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.230412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:12.0:1619677447.230432:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.230435:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.230436:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.230437:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.230439:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.230440:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.230588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.230596:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.230598:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.230599:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.230601:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.230604:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.230606:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.230607:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.230608:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.230609:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.230610:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.230613:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.230615:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.230618:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.230619:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.230619:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.230620:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.230621:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.230622:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.230622:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677447.230664:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.230667:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.230668:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.230670:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.230672:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.230674:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.230675:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.230676:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.230677:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.230678:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.230681:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.230682:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.230685:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.230686:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.230686:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.230688:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.230689:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.230690:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.230691:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.230692:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.230705:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.230707:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.230708:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.230709:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.230794:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:4.2:1619677447.230796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.230797:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.230798:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.230799:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.230800:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.230802:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.230814:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.230817:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.230818:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.230818:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.230821:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.230823:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.230968:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.230970:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.230971:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.230974:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.230974:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.230977:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.230996:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.230999:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.231000:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.231000:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.231002:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.231006:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.231008:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.231009:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.231010:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.231011:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.231011:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.231012:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000400:00000200:22.0:1619677447.231013:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.231013:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.231014:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.231015:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.231016:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.231016:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.231097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.231104:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.231106:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.231107:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.231108:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.231110:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.231111:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.231118:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:4.2:1619677447.231120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.231121:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.231122:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.231123:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.231123:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.231125:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.231145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.231152:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.231155:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.231156:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.231158:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.231160:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.231161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:24.0:1619677447.231162:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.231163:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.231164:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.231165:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.231166:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.231166:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:24.0:1619677447.231169:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.231169:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.231170:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.231170:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.231171:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.231172:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.231173:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.231174:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.231175:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.231175:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.231177:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.231177:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.231178:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.231178:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.231198:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.231200:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.231201:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.231203:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.231206:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.231208:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.231209:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.231211:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.231212:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.231212:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.231216:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.231217:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.231220:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.231221:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.231221:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.231223:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.231224:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.231225:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.231225:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.231229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.231230:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.231232:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.231233:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.231235:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.231238:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.231239:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.231240:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96731184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.231242:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.231243:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.231244:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.231249:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.231251:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.231254:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.231254:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.231255:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.231257:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.231257:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.231258:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.231259:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.231263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.231283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:13.0:1619677447.231437:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.231440:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.231441:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.231443:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.231444:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.231446:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.231489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.231588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.231595:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.231598:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.231599:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.231601:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.231603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fe9287000] (20)++
00000400:00000200:24.0:1619677447.231604:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.231606:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.231607:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.231609:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.231610:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.231612:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.231612:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe9287000] (21)++
00000800:00000200:8.0:1619677447.231614:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.10@o2ib3
00000400:00000200:8.0:1619677447.231616:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.10@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.231617:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.231619:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.231620:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.231621:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:8.0:1619677447.231623:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96731179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.231625:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.231626:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.231627:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:24.0:1619677447.231631:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.231632:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.231632:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.231634:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.231635:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.231636:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.231636:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:8.0:1619677447.231636:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.231638:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe9287000] (22)--
00000800:00000200:8.0:1619677447.231638:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:8.0:1619677447.231647:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.231648:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:8.0:1619677447.231648:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:8.0:1619677447.231650:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:8.0:1619677447.231651:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe9287000] (21)++
00000800:00000200:8.0:1619677447.231652:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe9287000] (22)--
00000800:00000200:8.0:1619677447.231653:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe9287000] (21)--
00000800:00000200:9.2:1619677447.231667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.231670:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.231673:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:24.0:1619677447.231675:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.231679:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.231680:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.231681:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.231683:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.231684:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.231686:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.231688:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.231692:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.231694:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.231697:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.231698:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.231698:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.231700:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.231701:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.231702:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.231703:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.231706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.231728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:4.2:1619677447.231797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.231857:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.231860:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.231860:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.231863:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.231865:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.231867:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.231868:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.231869:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.231870:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.231871:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.231875:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.231877:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.231880:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.231881:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.231881:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.231883:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.231884:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.231885:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.231886:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.231887:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.231897:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.231899:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.231899:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.231900:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.231997:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.232000:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.232001:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.232002:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.232004:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.232005:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.232044:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.232046:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.232047:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.232049:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.232050:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.232052:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.232210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.232217:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.232219:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.232220:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.232222:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.232225:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.232227:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.232228:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.232229:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.232230:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.232231:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.232235:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.232236:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.232239:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.232240:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.232240:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.232242:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.232242:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.232244:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.232245:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.232245:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.232251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.232258:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.232260:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.232261:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.232262:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.232306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.232313:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.232315:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.232316:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.232317:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.232319:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.232322:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.232384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.232428:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.232431:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.232432:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.232432:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.232434:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.232438:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.232447:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.232449:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.232450:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.232450:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.232452:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.232452:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.232576:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.232579:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.232580:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.232580:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.232582:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.232585:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.232789:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.232792:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.232793:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.232793:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:4.2:1619677447.232793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:22.0:1619677447.232795:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.232798:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.232801:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.232804:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.232805:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.232806:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.232809:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.232811:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.232812:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.232813:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.232814:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.232815:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.232819:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.232821:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.232824:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.232824:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.232825:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.232827:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.232827:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.232828:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.232828:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.232860:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.232862:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.232863:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.232865:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.232868:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.232869:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.232871:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.232872:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.232873:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.232874:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.232878:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.232880:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.232883:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.232883:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.232884:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.232886:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.232886:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.232887:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.232888:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.232979:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.232982:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.232983:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.232984:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.232987:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.232989:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.232990:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.232992:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.232993:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.232994:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.232997:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.232999:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.233002:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.233003:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.233004:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.233006:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.233007:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.233007:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.233008:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.233012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.233223:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.233225:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.233226:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.233227:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.233229:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.233230:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.233242:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.233245:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.233246:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.233246:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.233247:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.233247:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.233454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.233460:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.233463:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.233464:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.233465:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.233467:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.233470:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.233472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:8.0:1619677447.233478:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.233481:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.233482:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.233482:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.233483:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.233484:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.233551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.233610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.233612:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:4.2:1619677447.233613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:12.0:1619677447.233615:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.233616:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.233617:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.233620:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.233621:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.233907:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.233910:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.233911:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.233911:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.233913:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.233915:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.234006:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.234009:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677447.234010:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.234011:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.234013:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.234014:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.234014:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677447.234014:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.234015:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.234016:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.234018:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.234047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.234093:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.234095:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.234096:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.234097:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.234099:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.234102:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.234111:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.234114:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.234115:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.234117:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.234117:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.234119:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.234355:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.234357:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.234358:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.234359:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.234362:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.234363:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.234373:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.234375:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.234376:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.234377:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.234377:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.234378:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.234441:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.234443:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.234444:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.234445:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.234447:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.234448:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.234572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.234616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.234619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:12.0:1619677447.234630:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.234632:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.234633:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.234635:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.234636:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.234637:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.234662:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.234664:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.234665:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.234667:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.234668:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.234669:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.234738:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.234740:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.234741:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.234743:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.234746:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.234747:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.234749:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.234750:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.234751:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.234752:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.234755:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.234757:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.234760:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.234761:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.234762:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.234763:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.234764:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.234764:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.234765:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.234895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.234915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:4.2:1619677447.234948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.235085:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.235088:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.235088:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.235090:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.235092:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.235093:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.235250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.235396:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.235398:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.235399:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.235401:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.235403:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.235404:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.235513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.235555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:12.0:1619677447.235636:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.235639:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.235640:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.235642:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.235643:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.235645:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.235660:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.235663:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.235664:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.235664:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.235665:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.235666:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.235744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.235751:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.235754:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.235755:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.235757:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.235760:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.235762:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.235763:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.235764:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.235765:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.235766:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.235770:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.235771:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.235774:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.235775:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.235776:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.235777:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.235778:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.235778:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.235779:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.235806:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.235809:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.235811:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.235812:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.235814:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677447.235815:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.235816:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:12.0:1619677447.235817:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:6.0:1619677447.235817:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96731124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677447.235818:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.235819:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.235821:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.235821:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.235821:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.235822:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:12.0:1619677447.235823:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:6.0:1619677447.235826:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.235829:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.235832:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.235832:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.235833:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.235836:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.235836:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.235837:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.235837:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.235842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.235867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.235868:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.235871:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.235872:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.235875:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.235875:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.235877:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.235888:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.235890:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.235891:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.235892:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.235892:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:4.2:1619677447.235892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.235893:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.235966:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.235968:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.235969:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.235971:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.235973:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.235974:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.236248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.236264:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.236266:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.236267:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.236269:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.236270:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.236271:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.236301:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.236304:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:12.0:1619677447.236304:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:22.0:1619677447.236305:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.236305:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:12.0:1619677447.236307:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:22.0:1619677447.236308:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:12.0:1619677447.236308:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.236310:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:12.0:1619677447.236310:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677447.236312:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.236313:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.236472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.236514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.236522:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.236525:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.236526:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.236528:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.236531:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.236532:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.236534:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.236535:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.236536:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.236537:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.236540:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.236542:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.236545:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.236546:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.236546:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.236548:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.236548:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.236549:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.236550:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.236656:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.236659:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.236659:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.236661:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.236662:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.236664:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.236671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.236829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.236836:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.236839:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.236840:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.236840:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.236842:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.236845:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.236879:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.236882:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.236882:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.236884:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.236886:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.236888:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.236889:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.236890:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.236891:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.236892:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.236896:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.236898:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.236900:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.236901:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.236902:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.236903:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.236904:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.236905:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.236906:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.236906:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.236909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.236917:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.236919:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.236920:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.236921:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.237019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.237027:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.237029:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.237030:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.237032:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.237034:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.237036:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.237037:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.237038:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.237039:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.237040:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:12.0:1619677447.237042:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677447.237043:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.237044:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677447.237045:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.237046:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.237047:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.237048:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.237049:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.237049:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.237049:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:12.0:1619677447.237050:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.237051:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.237051:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.237053:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.237054:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.237054:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.237060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677447.237061:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.237063:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.237064:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.237064:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.237065:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.237066:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.237067:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.237069:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.237069:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.237071:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.237246:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.237249:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.237250:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.237250:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.237250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:6.0:1619677447.237252:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.237255:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.237264:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.237267:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.237267:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.237268:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.237269:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.237270:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.237291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.237298:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.237300:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.237302:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.237302:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.237304:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.237307:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.237316:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.237318:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.237319:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.237320:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.237321:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.237321:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.237528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.237536:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.237539:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.237540:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.237542:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.237545:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.237546:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.237547:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.237550:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.237551:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.237552:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.237555:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.237557:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.237559:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.237560:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.237561:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.237562:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.237563:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.237564:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.237564:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.237616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.237623:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.237626:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.237626:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.237627:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.237629:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.237632:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.237652:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.237655:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.237656:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.237658:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.237660:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.237662:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.237663:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.237664:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.237666:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.237667:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.237670:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.237672:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.237675:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.237676:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.237677:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.237679:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.237679:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.237680:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.237680:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.237796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.237803:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.237805:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.237807:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.237807:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.237809:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:8.0:1619677447.237812:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677447.237969:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.237971:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.237972:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.237974:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.237975:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:4.2:1619677447.237975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.237977:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.238026:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.238029:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.238030:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.238031:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:12.0:1619677447.238032:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.238033:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677447.238034:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:12.0:1619677447.238035:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.238035:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.238036:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.238038:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.238041:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.238237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.238244:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.238247:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.238248:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.238252:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.238255:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.238256:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.238258:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.238259:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.238260:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.238262:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:9.2:1619677447.238263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:24.0:1619677447.238266:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.238268:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.238271:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.238272:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.238272:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.238273:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:8.0:1619677447.238273:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.238274:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:8.0:1619677447.238275:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.238276:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.238277:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.238278:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.238278:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.238279:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.238279:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677447.238463:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.238465:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.238466:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.238467:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.238470:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.238470:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.238554:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.238557:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.238557:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.238559:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.238560:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.238562:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.238622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.238634:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d2400] (21)++
00000800:00000200:6.0:1619677447.238637:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.22@o2ib3
00000800:00000200:22.0:1619677447.238638:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d2400] (22)--
00000800:00000200:9.2:1619677447.238639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:6.0:1619677447.238641:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.22@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.238646:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.238648:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:6.0:1619677447.238650:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.238652:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.238652:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.238654:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:6.0:1619677447.238660:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.238664:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 456 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677447.238667:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.238668:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:6.0:1619677447.238669:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:6.0:1619677447.238672:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:6.0:1619677447.238673:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d2400] (21)++
00000800:00000200:6.0:1619677447.238674:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d2400] (22)++
00000800:00000200:6.0:1619677447.238676:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d2400] (23)--
00000800:00000200:6.0:1619677447.238677:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d2400] (22)--
00000800:00000200:22.0:1619677447.238690:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d2400] (22)++
00000800:00000200:22.0:1619677447.238692:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d2400] (23)--
00000800:00000200:22.0:1619677447.238693:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d2400] (22)--
00000800:00000200:22.0:1619677447.238694:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d2400] (21)--
00000800:00000200:4.2:1619677447.238708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.238753:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.238756:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.238757:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.238758:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.238760:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.238763:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.238861:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.238864:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.238864:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.238866:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.238868:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.238869:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.238877:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.238880:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.238881:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.238881:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.238882:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.238883:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.239214:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.239217:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.239218:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.239220:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.239221:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.239223:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.239233:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.239235:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.239236:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.239237:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.239237:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.239239:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.239314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.239322:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.239325:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.239326:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.239329:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.239333:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.239334:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.239336:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.239338:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.239339:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.239340:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.239345:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.239348:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.239351:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.239353:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.239353:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.239356:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.239356:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.239357:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.239357:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.239358:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.239358:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.239359:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.239366:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.239368:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.239369:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.239371:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96731072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.239372:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.239374:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.239376:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.239380:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.239386:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.239389:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.239390:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.239390:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.239392:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.239393:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.239394:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.239396:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.239396:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.239409:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.239411:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.239412:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.239413:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.239433:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.239436:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.239437:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.239438:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.239440:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.239441:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.239445:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.239447:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.239448:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.239450:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.239453:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.239454:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.239455:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.239457:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.239458:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.239459:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.239462:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.239464:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.239467:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.239467:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.239468:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.239470:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.239470:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.239471:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.239471:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.239520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.239671:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.239674:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.239675:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.239675:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.239677:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.239681:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.239784:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.239787:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.239788:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.239790:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.239791:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.239793:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.239938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.239962:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.239965:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.239966:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.239967:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.239968:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.239970:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.239976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.239979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:9.2:1619677447.240083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.240102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.240306:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.240309:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.240310:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.240312:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.240314:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.240316:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.240317:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96731060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.240318:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.240319:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.240320:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.240324:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.240326:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.240329:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.240330:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.240330:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.240332:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.240332:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.240333:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.240334:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.240424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:4.2:1619677447.240430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.240535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.240558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.240565:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.240568:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.240569:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.240569:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.240572:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.240574:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677447.240602:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.240605:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.240606:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.240608:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.240610:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.240611:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.240668:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.240670:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.240671:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.240672:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.240675:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.240678:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.240707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.240751:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.240753:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.240754:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.240755:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.240757:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.240760:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.240917:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.240920:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.240921:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.240923:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.240923:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.240925:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.241035:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.241038:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.241039:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.241041:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.241043:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.241045:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.241046:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.241047:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.241048:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.241049:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:4.2:1619677447.241050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:6.0:1619677447.241052:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.241054:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.241056:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.241057:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.241057:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.241059:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.241059:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.241061:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.241062:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.241062:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.241065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.241072:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.241073:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.241074:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.241075:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.241172:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.241174:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.241175:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.241177:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.241178:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.241180:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.241352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.241371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.241403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.241408:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.241411:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.241412:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.241413:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.241415:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.241418:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.241450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.241458:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.241461:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.241464:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.241465:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.241467:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.241470:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.241471:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.241473:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.241475:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.241476:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.241480:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.241482:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.241486:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.241487:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.241488:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.241490:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.241490:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.241492:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.241493:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.241499:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.241500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677447.241503:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.241506:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.241507:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.241508:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:13.0:1619677447.241509:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.241510:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.241511:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:12.0:1619677447.241511:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:24.0:1619677447.241512:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.241512:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.241513:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.241513:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.241514:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.241514:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.241514:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677447.241516:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.241516:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:9.2:1619677447.241657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.241664:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.241667:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.241668:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.241668:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.241671:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.241674:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.241677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677447.241683:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.241685:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.241686:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.241686:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.241687:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.241688:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.241708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.241714:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677447.241717:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677447.241719:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.241720:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677447.241722:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.241724:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677447.241726:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96731034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677447.241727:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677447.241728:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677447.241729:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677447.241734:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.241736:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677447.241739:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.241740:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:12.0:1619677447.241741:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677447.241743:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677447.241743:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677447.241744:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677447.241745:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:13.0:1619677447.241895:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.241898:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.241899:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.241901:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.241901:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.241904:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.241914:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.241917:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.241918:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.241918:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.241919:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.241920:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.241995:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.241998:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.241998:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.241999:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.242001:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.242004:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.242040:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.242043:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.242043:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.242044:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.242045:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.242046:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.242200:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.242203:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.242204:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.242204:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.242207:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.242210:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.242242:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.242244:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.242245:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.242246:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.242248:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.242250:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.242262:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.242264:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.242265:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.242265:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.242266:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.242267:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.242280:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.242283:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.242283:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.242285:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.242287:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.242289:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.242290:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96731026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.242292:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.242293:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.242299:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.242302:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.242303:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.242306:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.242307:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.242307:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.242309:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.242309:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.242310:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.242310:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.242316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:4.2:1619677447.242401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.242448:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.242451:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.242452:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.242452:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.242455:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.242458:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.242526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.242533:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.242536:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.242538:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.242539:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.242542:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.242544:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.242545:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.242547:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.242548:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.242548:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.242552:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.242554:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.242557:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.242557:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.242558:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.242560:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.242560:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.242561:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.242561:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.242764:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.242767:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:13.0:1619677447.242767:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:6.0:1619677447.242767:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.242768:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.242770:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:6.0:1619677447.242770:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:13.0:1619677447.242771:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.242772:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:13.0:1619677447.242773:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.242775:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.242775:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677447.242859:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.242861:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.242862:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.242863:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.242865:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.242867:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.242903:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.242906:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.242907:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.242907:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.242909:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.242911:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.242922:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.242925:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.242925:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.242926:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.242927:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.242927:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.243208:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.243210:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.243211:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.243212:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.243214:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.243215:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.243272:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.243275:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.243276:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.243276:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.243278:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.243280:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.243323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.243332:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.243334:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.243335:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.243337:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.243340:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.243341:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.243342:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.243343:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.243344:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.243345:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:9.2:1619677447.243347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:24.0:1619677447.243348:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.243351:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.243354:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.243355:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.243355:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.243356:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.243356:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.243357:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.243358:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.243358:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:8.0:1619677447.243358:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.243359:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677447.243359:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677447.243360:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677447.243361:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.243361:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677447.243362:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.243363:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.243365:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.243367:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.243367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000400:00000200:22.0:1619677447.243369:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.243370:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96731012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.243371:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.243372:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.243373:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000400:00000200:22.0:1619677447.243373:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:24.0:1619677447.243377:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.243377:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.243378:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000400:00000200:22.0:1619677447.243378:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.243379:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.243379:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677447.243381:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.243384:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.243385:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.243386:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.243388:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.243388:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.243389:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.243390:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.243597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.243639:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.243642:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.243643:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.243643:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.243645:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.243648:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.243886:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.243889:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.243890:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.243892:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.243896:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.243897:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.243899:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96731005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.243900:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.243901:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.243902:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.243906:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.243907:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.243911:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.243911:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.243912:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.243914:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.243915:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.243915:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.243916:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.243920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.243942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677447.243943:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.243945:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.243946:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.243949:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.243950:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.243951:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.243978:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.243981:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:4.2:1619677447.243981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:22.0:1619677447.243982:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.243982:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.243984:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.243988:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.243988:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.243991:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.243992:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.243994:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.243997:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.243998:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.244000:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96731001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.244001:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.244002:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.244003:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.244006:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.244008:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.244011:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.244012:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.244012:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.244014:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.244015:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.244015:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.244016:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.244197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.244237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.244242:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.244245:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.244245:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.244246:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.244248:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.244251:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.244356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:9.2:1619677447.244362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677447.244364:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.244366:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.244367:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.244367:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.244370:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:8.0:1619677447.244374:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.244382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:12.0:1619677447.244621:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.244623:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.244624:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.244625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:12.0:1619677447.244626:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.244627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:13.0:1619677447.244628:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.244629:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.244631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.244633:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.244635:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.244635:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.244636:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.244637:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.244638:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.244639:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.244641:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.244643:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.244644:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.244646:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.244647:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.244648:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.244649:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.244650:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.244651:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:24.0:1619677447.244652:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.244655:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.244658:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.244659:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.244659:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.244661:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.244661:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.244662:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.244664:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.244664:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.244670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.244677:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.244679:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.244680:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.244682:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.244945:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.244948:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.244949:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.244951:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.244951:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.244954:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.245064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.245079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.245109:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.245112:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.245112:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.245115:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.245118:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.245120:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.245122:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96730988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.245123:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.245125:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.245126:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.245131:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.245134:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.245136:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.245137:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.245138:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.245140:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.245140:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.245141:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.245142:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.245154:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.245157:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.245159:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.245161:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.245163:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.245166:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.245173:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.245176:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.245177:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.245178:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.245178:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.245179:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.245356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:4.2:1619677447.245502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.245519:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.245521:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.245522:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.245524:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.245526:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.245526:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.245541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.245548:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.245551:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.245552:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.245555:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.245558:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.245559:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.245561:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.245562:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.245563:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.245564:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.245567:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.245569:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.245572:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.245573:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.245573:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.245575:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.245575:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.245576:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.245577:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.245611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.245847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.245862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.245869:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.245871:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.245872:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.245872:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.245875:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.245878:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677447.245898:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.245901:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.245902:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.245903:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.245905:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:4.2:1619677447.245905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.245907:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.245946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.245954:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.245956:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.245957:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.245958:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.245958:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.245960:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.245961:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.245964:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.245964:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.245965:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:6.0:1619677447.245965:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.245966:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.245967:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.245969:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.245970:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.245972:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.245975:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.245977:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.245979:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.245980:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.245981:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.245983:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.245983:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.245984:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.245985:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.245986:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.245997:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.246000:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.246001:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.246001:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677447.246095:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.246097:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.246099:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.246100:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.246101:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.246103:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.246206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.246422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.246440:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:9.2:1619677447.246441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677447.246442:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.246443:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.246445:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.246446:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.246447:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.246552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.246593:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.246596:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.246596:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.246598:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.246600:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.246601:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.246613:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.246616:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.246617:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.246617:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.246618:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.246619:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.246891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.246898:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.246901:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.246902:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.246904:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.246904:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.246908:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677447.246946:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.246948:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.246949:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.246951:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.246953:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.246955:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.246956:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96730961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.246958:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.246959:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.246961:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.246965:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.246968:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.246970:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.246971:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.246972:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.246974:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.246976:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.246977:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.246978:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.246980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.246997:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.247000:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.247001:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.247001:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.247004:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.247007:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.247017:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.247019:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.247020:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.247021:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.247022:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.247023:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.247254:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.247257:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.247258:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.247260:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.247260:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.247262:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.247272:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.247275:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.247276:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.247277:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.247278:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.247279:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.247315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:4.2:1619677447.247334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.247380:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.247382:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.247383:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.247384:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.247386:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.247389:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.247392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.247654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.247661:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.247663:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.247664:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.247666:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.247669:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.247671:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.247672:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96730950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.247673:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.247674:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.247675:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.247678:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.247680:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.247683:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.247684:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.247684:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677447.247685:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677447.247686:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.247687:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.247687:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:12.0:1619677447.247688:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.247688:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.247688:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:12.0:1619677447.247690:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.247692:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.247693:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.247886:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.247890:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.247890:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.247893:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.247896:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.247898:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.247899:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96730947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.247900:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.247901:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.247902:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.247906:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.247908:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.247908:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677447.247911:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.247911:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.247911:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:24.0:1619677447.247912:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000400:00000200:12.0:1619677447.247912:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.247913:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.247914:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:24.0:1619677447.247914:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.247915:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677447.247915:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:24.0:1619677447.247916:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.247916:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.247925:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.247928:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.247928:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.247930:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.247931:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.247932:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.247990:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.247992:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.247993:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.247994:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.247996:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.247996:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.248239:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.248242:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.248242:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.248244:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.248246:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.248248:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.248249:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.248251:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.248253:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.248254:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.248258:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.248261:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.248263:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.248264:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.248265:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.248267:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.248267:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.248268:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.248269:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.248275:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.248277:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.248278:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.248279:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.248281:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.248282:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.248297:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.248300:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.248301:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.248302:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.248303:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.248305:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.248437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.248475:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.248477:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.248478:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.248480:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.248481:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:4.2:1619677447.248481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677447.248483:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.248486:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.248488:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.248490:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.248490:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.248492:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.248494:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.248725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.248733:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.248736:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.248737:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.248738:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.248741:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.248742:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.248744:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.248746:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.248747:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.248748:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.248751:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.248753:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.248755:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.248756:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.248757:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.248758:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.248759:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.248759:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.248760:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.248833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.248839:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.248842:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.248843:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.248843:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.248846:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.248849:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.249009:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.249012:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.249013:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.249013:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.249016:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.249019:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.249031:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.249034:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.249035:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.249035:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.249036:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.249037:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.249062:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.249065:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.249066:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.249067:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.249068:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.249070:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.249120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677447.249527:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.249530:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.249531:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.249533:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.249536:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.249537:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.249538:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.249540:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.249541:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.249542:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.249545:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.249547:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.249550:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.249551:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.249552:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.249554:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.249554:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.249555:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.249556:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.249560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.249564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.249571:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.249575:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.249575:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.249577:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.249581:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.249582:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.249584:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.249585:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.249586:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.249587:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.249590:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.249592:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.249594:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.249595:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.249595:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.249597:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.249598:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.249599:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.249600:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.249601:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.249606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.249613:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.249615:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.249616:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.249617:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.249695:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.249698:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.249699:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.249699:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.249701:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.249703:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.249733:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.249736:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.249737:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.249739:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.249741:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.249743:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.249744:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.249745:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.249746:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.249747:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.249750:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.249752:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.249755:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.249755:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.249756:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.249758:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.249758:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.249759:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.249759:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.249773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.250110:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.250113:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.250114:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.250114:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.250116:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.250119:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.250143:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.250145:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.250147:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.250148:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.250151:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.250153:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.250155:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.250156:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.250157:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.250158:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.250162:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.250164:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.250167:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.250168:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.250169:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.250170:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.250171:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.250172:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.250173:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.250173:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.250187:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.250188:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.250189:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.250191:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:4.2:1619677447.250215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.250222:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.250224:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:4.2:1619677447.250224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677447.250225:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.250227:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.250230:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.250231:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.250233:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.250234:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.250236:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.250237:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.250240:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.250242:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.250244:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.250245:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.250246:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.250247:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.250248:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.250249:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.250249:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.250263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.250507:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.250510:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.250511:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.250511:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.250513:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.250515:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.250607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.250626:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.250628:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.250629:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.250631:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.250633:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.250634:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.250646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:4.2:1619677447.250822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.250829:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.250832:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.250833:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.250835:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.250838:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.250839:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.250840:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.250842:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.250842:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.250843:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.250846:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.250848:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.250851:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.250852:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.250853:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.250854:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.250855:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.250856:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.250856:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.251034:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.251037:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.251037:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.251039:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.251042:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.251044:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.251045:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.251047:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.251048:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.251049:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.251053:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.251056:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.251059:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.251060:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.251060:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.251062:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.251063:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.251064:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.251065:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.251066:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.251074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.251076:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.251078:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.251078:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.251079:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.251090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:4.2:1619677447.251206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.251214:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.251217:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.251218:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.251218:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.251220:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.251222:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.251305:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.251307:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.251308:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.251309:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.251312:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.251313:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.251361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.251368:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.251371:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.251372:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.251374:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.251377:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.251378:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.251380:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.251381:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.251382:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.251383:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.251386:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.251388:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.251390:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.251391:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.251392:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.251393:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.251394:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.251394:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.251395:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.251485:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.251488:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.251489:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.251490:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.251492:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.251493:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.251600:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.251603:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.251604:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.251606:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.251608:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.251609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:22.0:1619677447.251610:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.251611:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.251612:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.251613:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.251615:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.251618:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.251620:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.251623:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.251624:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.251624:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.251626:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.251626:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.251627:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.251628:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.251660:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.251663:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.251663:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.251664:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.251667:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.251674:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.251676:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:9.2:1619677447.251678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.251678:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.251679:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.251680:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.251680:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.251681:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.252003:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.252005:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.252006:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.252007:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.252009:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.252011:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.252133:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.252136:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.252137:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.252139:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.252140:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.252142:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.252151:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.252153:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.252154:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.252155:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.252155:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.252156:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.252193:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.252195:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.252196:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.252198:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.252199:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.252200:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.252213:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.252215:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.252216:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.252217:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.252217:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.252218:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.252329:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.252332:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.252333:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.252335:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677447.252336:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.252338:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.252339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.252575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.252625:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.252628:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.252630:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.252631:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.252633:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.252634:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.252686:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.252689:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.252689:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.252692:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.252695:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.252697:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.252698:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.252700:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.252702:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.252703:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.252707:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.252710:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.252713:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.252714:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.252714:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.252717:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.252717:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.252718:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.252719:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.252785:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.252788:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.252789:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.252791:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.252792:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.252794:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.252862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.253052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.253059:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.253062:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.253063:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.253064:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.253067:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.253070:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.253072:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.253073:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.253074:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.253076:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.253088:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.253090:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.253093:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.253094:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677447.253095:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (23)++
00000800:00000200:8.0:1619677447.253096:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677447.253097:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.253103:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.253104:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.253105:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.253110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.253117:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.253120:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.253121:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.253121:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.253188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.253229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.253234:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.253237:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.253238:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.253238:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.253240:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.253244:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.253252:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.253254:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.253255:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:24.0:1619677447.253256:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:6.0:1619677447.253256:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.253256:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.253257:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.253258:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.253259:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.253261:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.253264:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.253265:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.253267:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.253268:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.253269:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.253270:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.253273:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.253274:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.253277:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.253278:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.253278:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.253280:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.253281:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.253282:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.253282:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.253286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.253306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.253368:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.253371:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.253372:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.253374:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.253376:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.253376:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.253385:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.253387:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.253388:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.253389:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.253389:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.253391:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.253517:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:22.0:1619677447.253520:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:6.0:1619677447.253520:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (24)++
00000400:00000200:22.0:1619677447.253521:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.253521:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000400:00000200:6.0:1619677447.253522:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.253523:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.253524:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.253527:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.253528:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.253529:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.253555:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.253558:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.253559:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.253561:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.253563:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.253565:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.253566:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730869] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.253567:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.253568:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.253569:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.253573:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.253575:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.253578:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.253579:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.253580:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.253582:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.253582:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.253583:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.253583:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.253698:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.253701:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.253702:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.253702:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.253704:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.253706:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.253765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.253768:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.253770:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.253771:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.253771:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.253775:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.253777:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.253787:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.253791:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.253792:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.253794:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.253795:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.253796:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.253806:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.253808:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.253809:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.253810:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.253812:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.253814:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.253902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.253913:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.253916:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.253916:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.253918:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.253920:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.253921:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.253947:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.253950:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.253950:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.253951:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.253953:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.253956:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.254254:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.254257:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.254258:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.254260:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.254263:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.254265:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677447.254266:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.254267:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.254268:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.254269:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677447.254272:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.254274:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.254277:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.254278:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677447.254278:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677447.254280:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.254281:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.254281:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.254282:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.254298:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.254301:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.254302:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.254302:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.254304:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.254307:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.254332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.254380:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.254383:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.254384:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:9.2:1619677447.254384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677447.254384:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.254387:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.254390:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.254464:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.254467:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.254468:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.254468:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.254471:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.254473:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.254484:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.254487:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.254488:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.254488:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.254489:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.254490:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.254753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.254773:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.254775:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.254776:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.254778:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.254780:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.254781:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.254798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:13.0:1619677447.254842:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.254845:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.254846:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.254848:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.254849:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.254850:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.254932:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.254935:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.254935:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.254936:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.254938:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.254940:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.255107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.255150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.255155:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.255157:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.255158:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.255159:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.255161:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.255164:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.255237:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.255240:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.255241:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.255241:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.255243:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.255245:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.255344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.255352:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.255355:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.255356:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.255358:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.255361:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.255362:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.255364:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96730843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.255365:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.255366:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.255367:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:4.2:1619677447.255370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677447.255371:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.255371:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677447.255372:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677447.255373:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.255374:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.255376:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677447.255376:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.255377:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.255377:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677447.255378:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.255379:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677447.255379:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.255380:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.255380:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.255381:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.255388:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.255391:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.255392:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.255393:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.255393:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.255395:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.255434:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.255436:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.255438:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.255440:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.255443:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.255445:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.255446:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.255447:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.255448:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.255449:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.255452:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.255454:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.255457:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.255457:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.255458:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.255460:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.255460:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.255461:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.255462:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.255466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.255578:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.255581:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.255581:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.255583:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.255584:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.255586:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.255701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.255789:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.255791:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.255793:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.255794:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.255795:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.255797:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.255901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.255908:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.255911:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.255912:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.255914:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.255917:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.255918:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.255919:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.255921:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.255921:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.255922:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.255926:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.255927:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.255930:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.255931:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.255931:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.255933:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.255933:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.255934:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.255935:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.255997:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.256000:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.256000:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.256002:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.256005:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.256007:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.256008:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.256009:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.256010:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.256012:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.256015:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.256017:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.256019:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.256020:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.256021:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.256023:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.256023:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.256025:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.256026:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.256027:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.256029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.256056:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.256059:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.256060:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.256060:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.256063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.256195:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.256198:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.256199:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.256201:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.256202:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.256203:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.256344:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.256347:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.256347:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.256349:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.256351:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.256352:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.256387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.256395:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.256398:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.256398:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.256401:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.256404:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.256406:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.256407:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96730827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.256408:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.256409:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.256411:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.256414:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.256415:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.256423:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.256423:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.256424:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.256426:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.256426:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.256427:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.256427:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.256473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.256624:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.256627:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.256628:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.256630:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.256632:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.256634:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.256635:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.256636:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.256637:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.256638:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.256642:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.256644:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.256647:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.256647:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.256648:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.256650:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.256650:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.256651:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.256651:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.256723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.256743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:4.2:1619677447.256846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.256853:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.256856:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.256857:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.256859:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.256862:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.256863:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.256865:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.256866:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.256867:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.256868:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.256871:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.256873:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.256876:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.256876:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.256877:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.256879:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.256879:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.256880:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.256882:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.256882:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.256888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.256895:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.256897:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.256897:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.256899:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.257005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.257029:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.257032:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.257033:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.257033:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.257035:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.257038:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.257039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.257045:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.257048:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.257049:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.257050:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.257052:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.257055:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677447.257136:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.257138:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.257139:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.257141:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.257144:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.257145:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.257147:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.257148:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.257149:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.257150:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.257154:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.257155:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.257158:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.257159:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.257159:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:4.2:1619677447.257160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.257161:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.257162:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.257162:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.257163:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.257204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.257208:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.257211:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.257212:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.257213:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.257215:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.257218:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.257228:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.257231:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.257232:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.257232:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.257233:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.257234:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.257275:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.257277:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.257278:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.257280:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.257281:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.257283:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.257291:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.257299:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.257300:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.257301:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.257301:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.257302:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.257343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.257350:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.257353:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.257354:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.257357:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.257360:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.257361:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.257363:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96730812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.257364:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.257365:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.257366:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.257369:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.257375:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.257378:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.257379:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.257379:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.257381:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.257382:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.257383:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.257383:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.257747:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.257750:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.257750:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.257752:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.257754:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.257755:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.257816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.257859:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.257862:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.257863:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.257863:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.257866:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.257868:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.257912:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.257915:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.257915:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.257917:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.257919:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.257921:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.257922:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.257923:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.257924:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.257925:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.257929:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.257930:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.257934:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.257934:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.257935:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.257937:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.257938:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.257938:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.257939:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.257994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.258000:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.258003:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.258004:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.258004:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.258006:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.258006:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.258009:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.258009:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.258010:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:12.0:1619677447.258011:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.258013:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.258014:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.258018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677447.258024:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.258027:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.258028:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.258028:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.258029:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.258030:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677447.258181:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.258184:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.258185:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.258186:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.258189:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.258191:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.258192:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.258193:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.258194:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.258195:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.258199:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.258201:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.258203:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.258204:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.258204:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.258206:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.258206:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.258208:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.258209:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.258209:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677447.258218:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.258221:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.258222:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.258222:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:13.0:1619677447.258223:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.258224:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.258224:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.258225:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.258226:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.258226:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.258269:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.258272:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.258273:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.258273:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.258275:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.258277:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.258318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.258396:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.258398:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.258399:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.258399:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.258401:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.258404:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.258414:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.258416:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.258417:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.258417:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.258418:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.258419:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.258536:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.258539:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.258540:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.258540:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.258542:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.258545:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.258587:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.258589:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.258590:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.258591:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.258593:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.258594:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.258698:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc6800] (21)++
00000800:00000200:22.0:1619677447.258701:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc6800] (22)--
00000800:00000200:6.0:1619677447.258701:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.26@o2ib3
00000400:00000200:6.0:1619677447.258703:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.26@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.258707:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.258708:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:6.0:1619677447.258709:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.258711:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.258712:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.258712:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677447.258715:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.26@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.258717:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.258719:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.258720:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677447.258720:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.258722:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.258723:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919874bc6800] (21)++
00000800:00000200:6.0:1619677447.258723:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919874bc6800] (22)--
00000800:00000200:6.0:1619677447.258724:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc6800] (21)--
00000800:00000200:22.0:1619677447.258732:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc6800] (21)++
00000800:00000200:22.0:1619677447.258734:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.26@o2ib3
00000800:00000200:4.2:1619677447.258734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.258736:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:22.0:1619677447.258737:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.26@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.258737:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.258738:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.258739:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.258739:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.258740:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.258741:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:6.0:1619677447.258741:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc6800] (22)--
00000400:00000200:22.0:1619677447.258742:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.258742:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:22.0:1619677447.258744:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.258745:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.258747:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.258752:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.26@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.258754:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.258754:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.258756:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.258758:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.258759:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.258759:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:6.0:1619677447.258760:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.258761:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.258766:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919874bc6800] (22)++
00000400:00000200:6.0:1619677447.258766:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.258767:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919874bc6800] (23)++
00000800:00000200:22.0:1619677447.258768:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919874bc6800] (24)--
00000400:00000200:6.0:1619677447.258768:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:22.0:1619677447.258769:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc6800] (23)--
00000400:00000200:6.0:1619677447.258769:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96730794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.258771:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:6.0:1619677447.258771:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.258772:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.258774:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677447.258774:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.258775:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (23)++
00000800:00000200:22.0:1619677447.258776:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (24)--
00000800:00000200:22.0:1619677447.258777:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.258778:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc6800] (22)++
00000400:00000200:6.0:1619677447.258778:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.258780:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.26@o2ib3
00000800:00000200:6.0:1619677447.258781:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.258782:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.26@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.258784:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.258785:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:22.0:1619677447.258786:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.258786:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:22.0:1619677447.258788:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:22.0:1619677447.258789:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96730793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.258789:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:22.0:1619677447.258790:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.258791:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.258792:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677447.258793:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.258794:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (23)++
00000800:00000200:6.0:1619677447.258795:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (24)--
00000400:00000200:22.0:1619677447.258796:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.26@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.258796:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.258797:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.258798:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677447.258799:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.258801:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.258801:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.258802:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:6.0:1619677447.258802:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.258803:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:6.0:1619677447.258803:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.258806:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677447.258807:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919874bc6800] (23)++
00000800:00000200:22.0:1619677447.258807:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919874bc6800] (24)--
00000800:00000200:6.0:1619677447.258807:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.258808:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc6800] (24)++
00000800:00000200:22.0:1619677447.258809:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc6800] (24)--
00000800:00000200:6.0:1619677447.258809:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919874bc6800] (24)--
00000800:00000200:22.0:1619677447.258810:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.258810:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc6800] (22)--
00000800:00000200:22.0:1619677447.258811:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.258811:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc6800] (21)++
00000800:00000200:22.0:1619677447.258812:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.258812:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.258813:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.258813:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.26@o2ib3
00000800:00000200:22.0:1619677447.258814:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc6800] (22)++
00000800:00000200:22.0:1619677447.258816:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.26@o2ib3
00000400:00000200:6.0:1619677447.258816:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.26@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.258818:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.26@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.258818:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677447.258819:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.258820:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.258821:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:6.0:1619677447.258821:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:12.0:1619677447.258822:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.258822:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96730792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677447.258823:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.258824:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.258825:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:6.0:1619677447.258825:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.258826:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.258829:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:12.0:1619677447.258829:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:6.0:1619677447.258829:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.26@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.258830:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677447.258831:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96730791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.258832:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.258832:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:13.0:1619677447.258833:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.258833:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677447.258834:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.258834:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.258834:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:22.0:1619677447.258835:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:12.0:1619677447.258835:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.258837:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.258838:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:22.0:1619677447.258839:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.26@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.258839:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.258840:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677447.258841:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.258842:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919874bc6800] (23)++
00000800:00000200:22.0:1619677447.258843:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.258843:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919874bc6800] (24)--
00000800:00000200:22.0:1619677447.258844:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:6.0:1619677447.258844:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc6800] (23)--
00000800:00000200:22.0:1619677447.258845:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:6.0:1619677447.258845:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc6800] (22)--
00000800:00000200:22.0:1619677447.258848:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:22.0:1619677447.258848:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919874bc6800] (21)++
00000800:00000200:22.0:1619677447.258849:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919874bc6800] (22)--
00000800:00000200:22.0:1619677447.258849:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc6800] (21)--
00000800:00000200:13.0:1619677447.258939:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.258942:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.258942:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.258944:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.258945:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.258946:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.259018:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.259021:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.259022:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.259023:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.259024:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.259026:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.259224:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.259227:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.259228:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.259228:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.259230:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.259232:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.259381:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.259384:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.259385:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.259386:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.259388:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.259391:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.259433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.259542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.259549:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.259552:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.259553:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.259554:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.259557:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.259559:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.259560:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.259561:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.259562:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.259563:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.259566:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.259568:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.259571:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.259571:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.259572:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.259574:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.259574:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.259575:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.259575:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.259727:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.259729:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.259730:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.259731:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.259733:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.259735:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.259822:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.259825:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.259826:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.259826:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.259828:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.259831:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.259869:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.259872:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.259873:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.259875:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.259878:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.259879:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.259880:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96730778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.259882:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.259883:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.259884:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.259887:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.259888:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:12.0:1619677447.259891:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677447.259891:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.259892:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.259893:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:12.0:1619677447.259894:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677447.259894:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:12.0:1619677447.259895:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.259895:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:12.0:1619677447.259896:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.259896:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.259896:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.259898:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.259899:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.259939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.259979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.259984:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.259987:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.259987:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.259988:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.259991:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.259994:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.260005:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.260008:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.260009:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.260009:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.260010:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.260011:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.260201:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.260203:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.260204:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.260205:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677447.260207:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.260208:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.260335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.260372:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.260375:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.260376:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.260378:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.260379:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.260381:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.260385:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.260388:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.260389:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.260390:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.260392:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.260397:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.260398:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.260400:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.260401:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.260401:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.260402:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.260403:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.260507:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.260510:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677447.260511:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677447.260513:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.260516:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.260517:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:6.0:1619677447.260519:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.260520:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.260521:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.260522:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:6.0:1619677447.260526:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.260528:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677447.260531:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.260532:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:6.0:1619677447.260532:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:6.0:1619677447.260534:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:6.0:1619677447.260535:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.260536:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677447.260537:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:22.0:1619677447.260560:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.260562:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.260563:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.260563:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.260566:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.260570:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.260580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.260633:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.260635:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.260636:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.260637:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.260639:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.260641:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.260754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.260865:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.260868:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.260869:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.260871:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.260871:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.260878:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.260885:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.260888:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.260889:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.260889:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677447.260890:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.260892:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.260894:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.260895:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.260897:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.260898:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.260899:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.260993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.261010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:9.2:1619677447.261064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.261095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:4.2:1619677447.261122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:24.0:1619677447.261130:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677447.261133:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677447.261133:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677447.261136:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.261139:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.261140:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:24.0:1619677447.261141:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.261142:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.261143:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.261144:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:24.0:1619677447.261147:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.261149:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.261151:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.261152:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677447.261152:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.261154:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.261154:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677447.261155:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:24.0:1619677447.261157:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:24.0:1619677447.261157:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:9.2:1619677447.261159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:4.2:1619677447.261163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677447.261168:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.261169:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.261170:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.261171:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:24.0:1619677447.261172:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000400:00000200:8.0:1619677447.261172:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.261173:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677447.261174:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.261175:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677447.261175:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.2:1619677447.261178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677447.261184:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.261187:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.261188:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.261188:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.261189:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.261190:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.261219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.261228:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.261232:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.261232:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.261235:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.261237:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.261238:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.261239:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.261241:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.261242:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.261243:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.261246:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.261248:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.261251:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.261252:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.261252:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.261254:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.261254:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.261255:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.261256:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.261289:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.261292:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.261292:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:4.2:1619677447.261296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:22.0:1619677447.261299:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.261302:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.261303:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.261304:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96730754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.261306:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.261308:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.261309:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:9.2:1619677447.261310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:22.0:1619677447.261313:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677447.261315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:22.0:1619677447.261316:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.261318:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.261319:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:24.0:1619677447.261320:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:22.0:1619677447.261320:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:24.0:1619677447.261322:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:22.0:1619677447.261322:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:24.0:1619677447.261323:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.261324:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:24.0:1619677447.261325:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.261325:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677447.261325:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:22.0:1619677447.261326:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.261327:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.261329:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.261331:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677447.261332:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.261332:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.261334:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677447.261337:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.261346:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.261349:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.261351:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.261351:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.261353:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.261355:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.261537:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.261540:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.261541:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.261542:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.261544:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.261545:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.261548:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.261549:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.261550:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.261550:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.261552:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.261603:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.261606:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.261607:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.261608:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.261610:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.261612:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.261658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.261753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.261801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.261816:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.261819:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.261820:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.261820:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.261823:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.261825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:6.0:1619677447.261826:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.261833:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.261837:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.261838:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.261839:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.261843:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.261844:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.261846:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96730746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.261847:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.261848:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.261849:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.261852:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:24.0:1619677447.261853:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.261854:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.261855:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.261856:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.261856:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:24.0:1619677447.261857:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.261858:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:22.0:1619677447.261858:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.261859:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.261860:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.261861:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.261862:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.261863:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:22.0:1619677447.261863:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:24.0:1619677447.261864:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677447.261864:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.261866:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.261868:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.261868:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.261869:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:4.2:1619677447.261869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.261870:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.261876:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.261878:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.261879:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.261881:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.262083:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.262086:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.262087:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.262089:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.262091:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.262092:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.262094:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.262095:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.262096:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.262097:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.262100:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.262102:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.262105:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.262105:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.262106:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.262108:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.262108:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.262109:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.262110:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.262126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.262152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:4.2:1619677447.262320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.262328:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.262331:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.262332:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.262334:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.262336:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.262338:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:22.0:1619677447.262338:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:24.0:1619677447.262339:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96730739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.262340:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.262340:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:24.0:1619677447.262341:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.262341:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.262342:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:24.0:1619677447.262343:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677447.262344:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.262346:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:24.0:1619677447.262347:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.262349:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.262352:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.262353:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.262354:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.262355:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.262356:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.262356:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.262357:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.262454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.262502:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.262504:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.262505:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.262506:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.262508:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.262511:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.262520:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.262523:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.262523:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.262524:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.262525:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.262526:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.262700:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.262702:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.262703:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.262705:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.262707:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.262709:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.262710:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.262711:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.262713:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.262714:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.262717:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.262719:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.262721:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.262722:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.262723:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.262724:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.262725:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.262725:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.262726:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.262760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:4.2:1619677447.262764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677447.262767:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.262770:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.262771:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.262773:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.262776:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.262778:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.262779:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.262781:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.262782:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.262783:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.262787:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.262789:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.262792:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.262793:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.262794:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.262795:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.262796:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.262797:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.262797:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.262860:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.262863:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.262864:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.262866:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.262867:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.262869:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.262885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:13.0:1619677447.263044:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.263047:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.263048:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.263049:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.263051:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.263052:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.263089:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.263092:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.263093:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.263094:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.263096:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.263097:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.263111:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.263114:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.263115:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.263116:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.263116:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.263118:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.263256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:4.2:1619677447.263386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.263400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.263493:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.263496:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.263496:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.263498:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.263501:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.263503:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.263504:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.263505:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.263506:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.263507:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.263510:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.263512:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.263514:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.263515:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.263516:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.263517:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.263518:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.263519:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.263520:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.263521:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:4.2:1619677447.263530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.263537:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677447.263539:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:24.0:1619677447.263540:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.263541:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677447.263541:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.263542:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.263542:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:24.0:1619677447.263543:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.263546:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.263547:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.263549:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730722] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.263550:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.263551:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.263552:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.263556:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.263557:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.263560:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.263561:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.263562:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.263563:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.263564:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.263564:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.263565:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.263647:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.263650:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.263651:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.263653:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.263654:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.263655:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.263837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.263856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677447.264174:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.264177:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.264178:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.264180:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.264182:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.264184:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.264185:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.264186:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.264187:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.264188:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.264192:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.264194:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.264197:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.264198:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.264199:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.264201:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.264201:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.264202:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.264202:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.264397:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.264400:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.264401:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.264403:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.264405:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.264407:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.264415:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.264417:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.264418:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.264420:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.264421:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.264422:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.264458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.264653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.264693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.264699:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.264702:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.264702:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.264703:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.264705:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.264708:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.264781:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.264783:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.264784:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.264786:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.264787:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.264789:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.265073:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.265076:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.265076:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.265078:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.265080:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.265081:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.265181:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.265183:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.265184:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.265185:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.265187:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.265189:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.265263:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.265266:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.265266:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.265268:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.265270:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.265270:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.265284:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.265287:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.265288:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.265288:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.265289:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.265290:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.265576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.265617:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:4.2:1619677447.265619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.265620:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.265621:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.265623:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.265624:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.265625:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.265850:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.265852:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.265853:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.265854:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.265856:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.265857:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.266040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.266048:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.266051:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.266052:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.266052:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.266054:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.266058:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.266143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.266166:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.266169:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.266170:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.266172:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.266173:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.266175:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.266185:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.266187:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.266188:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.266189:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.266190:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.266191:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.266469:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.266472:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.266472:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.266474:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.266476:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.266477:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.266492:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.266495:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.266497:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.266498:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.266502:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.266503:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.266505:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96730684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.266507:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.266508:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.266510:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:12.0:1619677447.266512:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:22.0:1619677447.266515:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.266515:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.266516:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.266518:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:12.0:1619677447.266518:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.266520:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.266520:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677447.266521:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:12.0:1619677447.266521:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.266522:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.266524:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.266524:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.266525:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.266526:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.266531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.266621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.266628:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.266631:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.266632:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.266632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677447.266633:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.266636:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.266637:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.266639:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730682] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.266640:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.266642:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.266643:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.266647:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.266649:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.266652:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.266653:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.266653:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.266655:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.266656:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.266657:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.266658:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.266659:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.266664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.266672:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.266674:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.266675:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.266676:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677447.266758:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.266761:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.266762:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.266764:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.266767:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.266768:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.266769:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.266771:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.266772:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.266772:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.266776:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.266777:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.266780:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.266781:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.266781:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.266783:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.266784:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.266784:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.266785:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.266789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.266821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:4.2:1619677447.266889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.266927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.266933:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.266936:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.266936:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.266937:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.266939:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.266942:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.267120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.267135:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.267137:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.267138:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.267140:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.267141:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.267142:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.267190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.267238:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.267241:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.267242:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.267243:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.267245:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.267248:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.267258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.267264:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.267266:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.267267:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.267267:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677447.267270:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.267271:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.267273:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.267274:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.267275:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.267277:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.267278:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.267280:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.267281:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.267282:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.267284:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96730672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.267285:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.267285:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.267286:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.267287:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.267287:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.267291:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.267292:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.267292:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.267293:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.267299:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.267302:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.267303:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.267303:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.267304:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.267304:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.267305:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.267305:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.267306:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.267306:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677447.267307:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.267307:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.267308:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.267308:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.267308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.267387:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.267390:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.267391:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.267393:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.267395:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.267397:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.267398:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.267399:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.267401:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.267401:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.267405:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.267406:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.267409:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.267409:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.267410:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.267412:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.267412:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.267413:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.267413:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.267649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.267699:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.267702:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.267702:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.267703:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.267705:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.267708:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.267827:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.267829:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.267830:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.267831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:12.0:1619677447.267832:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.267834:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.267835:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.267839:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.267842:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.267843:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.267845:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.267848:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.267850:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.267851:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.267852:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.267853:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.267854:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.267857:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.267859:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.267862:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.267863:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.267865:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.267867:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.267868:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.267868:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.267869:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.267923:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.267925:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.267926:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.267926:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.267928:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.267931:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.267974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.267981:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.267984:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.267984:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.267985:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.267987:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.267990:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.268092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.268114:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.268116:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.268117:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.268119:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.268121:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.268121:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677447.268139:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.268142:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.268142:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.268143:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.268145:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.268148:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.268157:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.268160:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.268161:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.268162:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.268163:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.268164:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.268265:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.268267:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.268268:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.268270:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.268272:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.268274:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.268275:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.268276:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.268277:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.268278:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.268282:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.268283:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.268286:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.268287:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.268287:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.268289:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.268290:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.268291:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.268292:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.268293:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.268305:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.268307:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.268307:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.268308:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.268429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.268499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.268514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:4.2:1619677447.268522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.268529:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.268532:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.268533:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.268534:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.268537:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.268539:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.268541:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.268543:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.268544:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.268546:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.268552:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.268555:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.268558:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.268559:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.268559:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.268561:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.268562:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.268562:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.268563:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.268682:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.268684:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.268685:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.268687:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.268689:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.268690:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.268730:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.268733:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.268734:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.268734:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.268735:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.268736:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.268892:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.268894:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.268895:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.268896:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.268898:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.268899:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.268916:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.268919:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.268919:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.268922:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.268924:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.268926:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.268927:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.268928:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.268929:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.268930:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.268933:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.268935:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.268938:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.268939:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.268939:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.268941:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.268941:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.268942:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.268943:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.269059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.269066:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.269068:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.269069:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.269070:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.269072:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.269075:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.269084:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.269086:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.269087:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.269087:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.269089:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.269092:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.269132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.269140:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.269143:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.269143:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.269145:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.269148:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.269150:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.269151:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.269152:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.269154:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.269155:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.269157:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.269159:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.269162:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.269163:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.269163:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.269165:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.269166:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.269167:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.269169:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.269169:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.269175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.269182:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.269184:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.269184:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.269185:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.269262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.269267:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.269270:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.269271:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.269272:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.269275:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.269277:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.269279:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.269280:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.269281:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.269282:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.269285:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.269286:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.269289:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.269290:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.269291:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.269292:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.269302:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.269303:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.269304:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.269305:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.269318:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.269320:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.269321:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.269322:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.269460:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.269463:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.269464:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.269465:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.269467:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.269468:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.269681:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.269683:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.269684:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.269684:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.269686:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.269689:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.269803:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.269806:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.269807:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.269808:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.269810:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.269810:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:4.2:1619677447.269818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.269825:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.269828:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.269829:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.269831:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.269834:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.269835:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.269837:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.269839:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.269841:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.269842:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.269848:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.269850:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.269853:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.269854:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.269855:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.269856:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.269857:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.269858:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.269858:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.269865:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.269868:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.269868:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.269870:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.269873:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.269874:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.269875:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96730637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.269877:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.269878:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.269879:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.269883:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.269885:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.269888:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.269889:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.269890:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.269892:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.269892:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.269893:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.269893:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.270145:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.270148:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.270149:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.270150:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.270152:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.270155:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.270214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:4.2:1619677447.270220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677447.270221:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.270224:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.270225:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.270225:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.270227:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.270231:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.270232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.270238:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.270241:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.270242:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.270243:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.270243:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.270244:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.270267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:9.2:1619677447.270305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.270355:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.270358:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.270359:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.270359:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.270361:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.270364:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.270465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.270627:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.270630:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.270631:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.270633:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.270635:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.270636:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.270644:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.270646:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.270646:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.270646:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:13.0:1619677447.270648:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.270648:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.270648:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.270649:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.270650:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.270651:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.270651:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.270652:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.270658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.270666:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.270667:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.270668:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.270669:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.270669:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.270670:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.270670:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677447.270671:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.270671:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:13.0:1619677447.270672:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.270672:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.270674:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.270997:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.270999:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.271000:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.271002:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.271004:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.271005:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.271012:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.271015:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.271016:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:13.0:1619677447.271016:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:6.0:1619677447.271016:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.271017:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.271018:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.271019:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.271020:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.271022:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.271023:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.271025:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.271287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.271290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.271310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:9.2:1619677447.271394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.271962:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677447.271964:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677447.271967:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.271969:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677447.271969:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.271971:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:12.0:1619677447.271972:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96730615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677447.271974:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677447.271975:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677447.271976:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:6.0:1619677447.271976:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.271979:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:12.0:1619677447.271979:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.271979:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.271982:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.271985:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:6.0:1619677447.271985:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.271986:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:12.0:1619677447.271988:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677447.271988:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96730614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.271989:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:12.0:1619677447.271990:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:6.0:1619677447.271990:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.271991:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.271992:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.271992:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000400:00000200:6.0:1619677447.271992:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:12.0:1619677447.271993:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677447.271994:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000400:00000200:6.0:1619677447.271995:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.271997:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.271999:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.272000:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.272001:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.272002:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.272003:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.272003:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.272004:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.272005:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.272007:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.272008:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.272009:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.272011:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.272012:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.272086:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.272088:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.272088:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.272090:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.272091:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.272092:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.272093:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.272093:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677447.272094:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.272094:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.272095:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.272096:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.272102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.272108:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.272111:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.272112:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.272113:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.272113:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.272114:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.272403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.272411:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.272414:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.272415:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.272416:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.272419:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.272420:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.272422:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.272423:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.272424:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.272425:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.272428:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.272430:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.272432:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.272433:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.272434:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.272435:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.272436:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.272437:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.272437:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.272466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.272507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.272512:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.272515:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.272516:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.272516:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.272518:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.272522:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.272531:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.272534:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.272534:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.272535:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.272536:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.272536:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.272556:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.272558:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.272559:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.272561:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.272563:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.272564:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.272590:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.272592:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:9.2:1619677447.272592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.272593:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.272594:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.272596:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.272598:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.272680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.272687:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.272690:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.272691:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.272691:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.272693:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:24.0:1619677447.272696:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.272701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.272741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.272870:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.272873:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.272873:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.272875:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.272877:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.272879:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.272880:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730602] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.272881:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.272882:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.272883:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.272886:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.272888:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.272890:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.272891:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.272892:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.272893:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.272894:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.272895:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.272896:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.272897:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.272907:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.272909:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.272909:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.272910:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.272958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.272977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:4.2:1619677447.273101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.273139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.273285:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.273288:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.273289:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.273290:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.273297:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.273299:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677447.273300:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.273303:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.273304:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.273305:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677447.273308:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.273310:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.273313:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.273314:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677447.273315:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677447.273316:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.273317:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.273317:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.273318:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.273331:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.273334:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.273335:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.273335:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.273337:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.273340:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.273418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.273438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.273483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.273490:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.273493:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.273494:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.273495:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.273498:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.273500:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.273501:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96730593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.273503:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.273503:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.273504:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.273507:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.273509:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.273511:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.273512:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.273513:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.273514:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.273515:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.273516:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.273516:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.273540:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.273543:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.273543:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.273545:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.273547:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.273549:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.273551:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.273552:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.273553:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.273554:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.273557:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.273559:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.273562:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.273563:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.273564:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.273565:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.273566:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.273567:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.273567:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.273571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677447.273622:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.273624:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.273626:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.273628:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.273631:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.273632:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.273633:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.273635:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.273636:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.273637:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.273640:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.273642:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.273645:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.273645:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.273646:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.273648:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.273648:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.273649:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.273649:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.273661:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.273663:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.273664:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.273665:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.273668:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.273669:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.273684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.273727:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.273730:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.273730:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.273731:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.273734:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.273737:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.273976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.273994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:4.2:1619677447.274207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.274214:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.274217:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.274218:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.274222:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.274224:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.274226:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.274228:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96730582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.274229:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.274230:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.274231:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.274236:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.274238:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.274242:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.274243:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.274244:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.274245:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.274246:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.274247:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.274248:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.274249:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.274255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.274260:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.274263:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:8.0:1619677447.274263:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677447.274264:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.274265:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.274265:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.274266:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.274267:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677447.274268:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.274270:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.274271:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96730580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.274272:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.274273:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.274274:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.274277:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.274279:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.274282:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.274282:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.274283:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.274285:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.274285:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.274286:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.274286:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.274423:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.274425:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.274426:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.274428:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.274430:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.274431:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.274434:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.274435:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.274435:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.274436:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.274437:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.274611:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.274614:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.274614:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.274616:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.274618:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.274619:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.274718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:12.0:1619677447.274725:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677447.274726:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:12.0:1619677447.274728:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:24.0:1619677447.274729:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:12.0:1619677447.274729:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.274730:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.274730:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:12.0:1619677447.274731:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.274733:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.274733:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.274734:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.274736:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.274740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:8.0:1619677447.274746:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.274749:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.274750:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.274750:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.274751:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.274752:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.274799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.274806:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.274809:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.274810:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.274812:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.274814:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.274815:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.274817:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730572] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.274818:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.274819:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.274820:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.274823:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.274825:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.274828:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.274829:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.274829:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.274831:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.274831:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.274832:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.274832:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.274986:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.274989:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.274990:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.274992:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.274995:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.274998:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:8.0:1619677447.274999:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.275001:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.275003:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.275004:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:8.0:1619677447.275009:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.275011:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.0:1619677447.275013:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.275014:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:8.0:1619677447.275015:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:8.0:1619677447.275016:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:8.0:1619677447.275017:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.275017:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.275018:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.275032:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.275034:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677447.275035:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.275036:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.275038:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.275039:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677447.275039:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677447.275040:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.275040:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.275041:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.275043:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.275050:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.275053:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.275054:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.275054:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.275055:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.275057:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.275151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.275193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:12.0:1619677447.275280:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.275283:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.275284:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.275286:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.275287:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.275288:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.275375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.275403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:13.0:1619677447.275540:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.275543:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.275544:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.275546:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.275548:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.275549:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.275645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.275652:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.275655:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.275656:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.275660:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.275662:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.275664:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.275666:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96730558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.275667:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.275669:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.275670:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.275675:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.275677:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.275680:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.275680:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.275681:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677447.275681:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:24.0:1619677447.275683:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.275683:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:12.0:1619677447.275683:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:24.0:1619677447.275684:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:12.0:1619677447.275684:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.275685:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:12.0:1619677447.275686:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.275687:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.275689:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.275700:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.275703:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.275704:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.275704:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.275705:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677447.275705:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:13.0:1619677447.275707:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.275708:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.275709:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.275710:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.275712:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.275715:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.276095:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.276097:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.276098:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.276100:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.276102:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.276104:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.276105:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.276107:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.276108:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.276109:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.276113:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.276114:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.276117:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.276118:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.276119:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.276121:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.276121:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.276122:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.276122:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.276275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.276286:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.276289:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.276290:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.276290:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.276300:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.276310:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677447.276332:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.276335:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.276336:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.276339:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.276340:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.276344:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.276344:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.276347:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.276348:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.276348:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.276351:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.276354:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.276367:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.276370:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.276370:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.276371:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.276372:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.276372:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.276404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.276411:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.276415:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.276416:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.276419:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.276421:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.276426:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.276437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.276444:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.276446:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.276447:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.276447:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.276450:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.276454:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677447.276461:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:4.2:1619677447.276462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.276464:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.276465:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.276467:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.276467:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.276473:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.276492:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.276494:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.276495:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.276498:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.276498:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.276504:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.276522:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.276525:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.276526:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.276527:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.276529:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.276532:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.276611:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.276613:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.276614:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.276616:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.276617:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:4.2:1619677447.276619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:13.0:1619677447.276620:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.276626:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.276629:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.276630:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.276631:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.276635:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.276637:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.276638:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96730529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.276639:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.276640:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.276641:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.276644:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.276647:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.276650:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.276651:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.276652:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.276654:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.276654:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.276655:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.276656:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.276923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.276968:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.276971:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.276972:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.276972:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.276974:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.276977:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.277082:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.277084:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.277085:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.277087:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.277089:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.277090:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.277099:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.277101:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.277102:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.277102:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.277103:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.277104:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.277303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.277623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.277627:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.277629:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.277630:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.277632:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.277635:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.277637:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.277639:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96730520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.277640:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.277641:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.277642:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.277645:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.277647:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.277650:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.277651:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.277651:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.277653:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.277654:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.277655:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.277656:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.277656:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.277659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.277666:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.277668:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.277669:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.277670:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.277710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.277751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:12.0:1619677447.277971:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:4.2:1619677447.277973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.277974:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.277975:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.277976:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.277978:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.277979:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.278018:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.278020:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.278021:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.278022:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.278024:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.278027:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.278135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.278143:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.278147:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.278148:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.278149:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.278153:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.278154:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.278156:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.278157:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.278158:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.278159:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.278162:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.278164:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.278166:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.278167:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.278168:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.278170:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.278170:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.278172:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.278173:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.278173:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.278179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.278186:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.278188:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.278189:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.278190:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.278612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.278733:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.278736:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.278737:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.278739:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.278741:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.278743:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.278744:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.278745:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.278746:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.278747:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.278750:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.278752:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.278755:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.278756:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.278756:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.278758:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.278759:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.278759:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.278760:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.278812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.278844:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.278847:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.278848:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.278848:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.278850:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.278852:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.278916:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.278919:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.278919:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.278920:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.278922:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.278926:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.279029:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.279032:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.279033:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.279033:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.279035:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.279037:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.279048:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.279050:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.279051:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.279051:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.279052:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.279053:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.279199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.279237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.279468:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.279471:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.279472:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.279474:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.279476:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.279477:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.279541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (22)++
00000800:00000200:4.2:1619677447.279709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.279716:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.279719:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.279720:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.279722:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.279725:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.279727:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.279728:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.279730:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.279731:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.279732:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.279736:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.279737:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.279740:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.279741:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.279741:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.279743:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.279744:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.279744:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.279745:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.279977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.280014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.280017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677447.280073:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.280076:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.280077:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.280079:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.280082:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.280083:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.280084:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.280086:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.280087:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.280087:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.280090:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.280092:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.280095:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.280095:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.280096:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.280098:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.280099:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.280100:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.280101:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.280101:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677447.280104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.280114:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.280116:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.280117:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.280118:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.280173:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.280176:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.280177:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.280178:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.280179:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.280182:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.280196:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.280198:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.280199:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.280199:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.280200:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.280201:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.280284:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.280287:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.280288:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.280290:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.280297:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.280299:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.280300:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96730489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.280301:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.280302:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.280303:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.280307:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.280309:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.280312:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.280313:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.280313:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.280315:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.280316:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.280316:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.280317:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.280508:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.280511:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:4.2:1619677447.280511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:12.0:1619677447.280512:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.280514:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.280516:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.280517:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.280518:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.280521:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677447.280524:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.280524:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.280525:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.280526:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:24.0:1619677447.280528:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.280529:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.280530:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.280531:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.280533:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.280534:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.280535:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.280538:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.280539:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.280539:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.280541:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96730486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.280542:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.280543:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.280543:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.280545:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.280548:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.280549:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.280550:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:8.0:1619677447.280550:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.280552:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.280552:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.280553:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.280553:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.280554:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677447.280555:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.280557:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.280558:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.280563:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.280565:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.280566:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.280566:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.280567:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.280571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.280589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:22.0:1619677447.280665:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.280668:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.280669:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.280669:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.280672:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.280674:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.280891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.280937:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.280940:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.280941:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.280941:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.280944:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.280947:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.281086:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.281089:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.281090:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.281092:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.281094:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:22.0:1619677447.281095:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.281097:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:13.0:1619677447.281097:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:22.0:1619677447.281098:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96730478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.281098:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.281100:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677447.281100:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.281101:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.281101:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:22.0:1619677447.281102:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:13.0:1619677447.281103:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:22.0:1619677447.281106:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.281108:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.281111:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.281112:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.281112:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.281114:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.281115:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.281115:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.281116:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.281225:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677447.281228:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677447.281228:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677447.281231:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.281234:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.281235:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677447.281237:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96730476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.281238:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.281239:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.281240:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677447.281243:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.281245:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677447.281248:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.281249:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677447.281249:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677447.281251:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677447.281251:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677447.281252:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677447.281253:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:6.0:1619677447.281267:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.281270:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.281271:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.281272:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.281273:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.281273:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.281274:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.281276:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.281277:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.281278:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.281279:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.281279:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.281351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.281359:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.281361:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.281363:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.281364:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.281368:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.281369:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.281371:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.281372:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.281373:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.281374:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.281377:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.281378:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.281381:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.281382:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.281382:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.281384:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.281384:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.281385:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.281385:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.281496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.281502:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.281505:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.281506:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.281506:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.281508:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.281511:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.281549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.281582:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.281584:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.281585:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.281586:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.281589:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.281589:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677447.281594:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.281597:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.281598:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.281598:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.281600:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.281603:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.281874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.281889:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.281892:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.281893:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.281893:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.281895:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.281900:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.281902:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.281904:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.281904:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.281905:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677447.281905:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.281906:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.281907:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.281907:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.281908:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.281909:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.281910:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.281911:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.281913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.281949:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.281951:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:22.0:1619677447.281954:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.281954:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.281956:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:22.0:1619677447.281957:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.281958:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.281959:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.281959:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.281960:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96730465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.281961:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.281962:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.281963:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.281964:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.281967:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.281968:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.281968:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.281970:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.281970:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.281971:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.281971:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.281972:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.281973:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.281974:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.281975:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.281976:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.281978:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.281978:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.281980:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.281980:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.281981:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:4.2:1619677447.281989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:6.0:1619677447.281991:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.281993:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.281994:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.281995:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.281996:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.281999:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.282000:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.282001:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.282004:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.282005:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.282006:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96730463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.282008:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.282009:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.282011:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.282014:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.282016:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.282019:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.282020:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.282021:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.282022:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.282023:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.282024:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.282025:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.282026:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.282031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.282038:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.282041:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.282041:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.282042:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.282064:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.282067:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.282068:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.282069:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677447.282071:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.282071:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.282081:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.282084:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.282085:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.282085:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.282086:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.282088:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.282254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.282269:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.282271:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.282272:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.282274:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.282275:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.282276:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.282296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.282299:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.282302:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.282302:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.282303:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.282305:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.282308:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.282367:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.282369:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.282370:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.282371:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.282374:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.282374:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.282383:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.282386:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.282386:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.282387:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.282388:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.282388:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.282443:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.282445:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.282446:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.282448:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.282449:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.282451:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.282676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.282723:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.282725:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.282726:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.282727:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.282729:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.282732:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.282763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.282771:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.282774:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.282775:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.282778:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.282779:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677447.282780:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.282782:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:8.0:1619677447.282782:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677447.282783:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.282784:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96730452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.282786:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.282786:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.282786:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.282788:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.282790:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:12.0:1619677447.282792:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677447.282796:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.282799:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.282802:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.282803:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.282803:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.282805:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.282806:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.282806:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.282807:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.283033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.283226:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.283228:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.283229:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.283231:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.283232:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677447.283233:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.283243:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.283246:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.283247:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677447.283248:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677447.283249:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.283250:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.283253:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.283253:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.283255:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.283257:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.283258:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.283356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.283592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.283598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.283663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677447.283670:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677447.283671:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.283673:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:24.0:1619677447.283674:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:6.0:1619677447.283674:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677447.283675:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.283678:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.283678:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.283681:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.283682:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.283683:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.283683:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:24.0:1619677447.283685:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.283685:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.283687:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.283687:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.283688:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.283689:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.283690:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.283690:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.283695:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.283696:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.283698:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:24.0:1619677447.283699:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.283701:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.283702:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.283702:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:24.0:1619677447.283703:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.283703:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:24.0:1619677447.283704:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.283705:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.283705:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677447.283706:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.283706:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.283706:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677447.283707:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677447.283707:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.283708:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.283985:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.283988:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.283989:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.283989:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.283991:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.283994:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.284108:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.284111:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.284111:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.284113:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.284115:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.284116:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.284194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.284300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.284338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:9.2:1619677447.284408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.284425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:22.0:1619677447.284522:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.284525:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.284526:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.284526:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.284528:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.284531:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.284570:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.284572:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.284573:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.284575:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.284577:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.284579:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.284580:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.284581:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.284582:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.284583:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.284587:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.284589:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.284591:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.284592:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.284593:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.284594:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.284595:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.284595:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.284596:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.285020:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.285023:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.285024:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.285026:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.285029:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.285031:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.285032:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.285034:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.285035:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.285036:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.285039:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.285041:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.285044:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.285045:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.285045:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.285048:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.285048:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.285049:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.285049:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.285054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.285181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.285187:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.285190:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.285191:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.285191:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.285193:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:24.0:1619677447.285197:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.285202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:8.0:1619677447.285208:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.285210:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677447.285211:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.285212:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.285212:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.285213:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677447.285215:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:4.2:1619677447.285217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.285218:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.285219:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.285219:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.285222:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.285225:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.285237:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.285240:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.285241:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.285242:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.285243:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.285243:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.285432:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.285435:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.285436:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.285436:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.285438:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.285440:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.285555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.285563:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.285565:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.285566:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.285568:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.285571:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.285573:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.285574:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.285576:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.285576:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.285578:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.285581:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.285583:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.285586:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.285587:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.285588:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.285589:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.285590:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.285590:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.285591:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.285642:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.285644:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.285645:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.285647:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.285649:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.285650:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.285666:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.285668:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.285669:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.285671:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.285672:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.285674:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.285803:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.285806:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.285807:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.285808:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677447.285810:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.285811:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.285844:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.285847:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.285848:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.285850:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.285852:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.285854:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.285855:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.285856:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.285857:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.285858:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.285861:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.285863:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.285865:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.285866:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.285867:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.285868:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.285869:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.285870:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.285870:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.286050:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.286052:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.286053:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.286055:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.286056:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.286058:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.286176:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.286178:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.286179:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.286180:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.286182:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.286183:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.286195:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.286198:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.286198:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.286199:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677447.286199:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:13.0:1619677447.286200:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.286201:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.286202:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677447.286203:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677447.286205:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.286208:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.286209:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677447.286210:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.286211:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.286212:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.286213:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677447.286216:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.286218:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.286220:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.286220:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677447.286221:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.286222:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.286223:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677447.286224:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677447.286225:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677447.286225:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:9.2:1619677447.286228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.286236:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.286237:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.286238:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.286239:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000400:00000200:24.0:1619677447.286240:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.286240:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677447.286241:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677447.286242:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.286242:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677447.286243:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:8.0:1619677447.286289:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.286292:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.286293:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.286301:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.286303:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.286305:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.286306:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96730408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.286308:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.286310:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.286311:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.286315:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.286317:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.286324:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.286325:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.286326:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.286328:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.286329:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.286329:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.286330:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.286335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.286426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.286483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.286491:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.286494:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.286495:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.286497:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.286500:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.286501:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.286503:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.286504:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.286505:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.286506:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.286509:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.286511:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.286514:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.286514:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.286515:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.286517:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.286517:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.286518:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.286520:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.286520:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.286526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.286533:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.286535:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.286536:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.286537:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.286622:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.286624:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.286625:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.286626:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.286628:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.286629:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.286683:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.286686:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.286687:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.286689:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.286692:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.286693:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.286694:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.286695:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.286696:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.286697:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.286702:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.286704:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.286707:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.286708:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.286709:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.286711:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.286711:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.286713:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.286714:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.286714:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.286724:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.286726:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.286728:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.286729:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.286736:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.286739:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.286740:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.286741:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.286742:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.286744:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.286758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.286798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.286851:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.286854:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.286855:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.286856:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.286858:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.286858:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.286871:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.286873:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.286874:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.286875:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.286875:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.286877:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.287030:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.287032:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.287033:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.287035:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.287036:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.287037:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.287085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.287126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:4.2:1619677447.287307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.287356:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.287359:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.287360:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.287360:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.287362:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.287365:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.287404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.287412:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.287415:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.287416:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.287418:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.287421:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.287422:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:24.0:1619677447.287423:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.287424:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96730392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.287425:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.287425:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.287425:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.287426:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.287427:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677447.287427:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.287429:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:24.0:1619677447.287431:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.287431:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.287432:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.287435:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.287436:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.287437:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.287439:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.287439:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.287440:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.287440:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.287555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.287594:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:9.2:1619677447.287596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.287597:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.287597:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.287599:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.287602:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.287603:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.287605:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96730388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.287606:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.287607:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.287608:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.287612:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.287613:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:6.0:1619677447.287614:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:9.2:1619677447.287615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677447.287616:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.287617:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.287617:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.287618:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.287618:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:13.0:1619677447.287619:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.287620:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677447.287620:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:13.0:1619677447.287621:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.287621:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.287622:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.287622:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.287701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.287742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.287794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.287833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.287869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.287876:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.287878:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.287879:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.287879:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.287887:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.287889:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.287974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.287980:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.287983:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.287984:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.287984:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.287987:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.287990:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.287991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677447.287998:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:12.0:1619677447.288000:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.0:1619677447.288001:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.288002:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.288002:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:12.0:1619677447.288003:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.288003:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.288003:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.288003:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:12.0:1619677447.288005:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.288006:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:13.0:1619677447.288006:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.288008:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.288009:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.288010:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.288011:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.288012:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.288016:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.288148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.288155:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.288157:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.288158:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.288160:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.288163:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.288164:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.288166:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.288167:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.288168:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.288169:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.288173:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.288175:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.288178:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.288178:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.288179:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.288181:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.288181:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.288182:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.288182:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.288266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.288317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.288521:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.288524:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.288525:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.288525:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.288527:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.288529:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.288668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.288784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.288792:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.288794:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.288796:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.288797:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.288800:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.288801:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.288802:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96730371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.288804:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.288805:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.288805:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:9.2:1619677447.288808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677447.288809:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.288811:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.288813:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.288814:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.288815:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.288816:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.288817:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.288818:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.288819:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.288827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:22.0:1619677447.288922:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.288924:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.288925:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.288927:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.288929:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.288931:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.288932:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.288933:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.288934:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.288935:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.288939:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.288940:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.288943:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.288944:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.288944:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.288946:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.288946:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.288947:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.288947:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.289032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.289065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.289187:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.289190:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.289191:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.289191:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.289193:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.289199:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.289686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.289763:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:4.2:1619677447.289764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677447.289765:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677447.289767:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677447.289768:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.289768:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.289770:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:22.0:1619677447.289771:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.289772:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:22.0:1619677447.289773:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.289774:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.289775:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:22.0:1619677447.289775:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.289776:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677447.289777:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.289778:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.289780:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.289780:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.289782:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.289784:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.289784:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:24.0:1619677447.289786:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.289786:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.289787:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96730357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.289788:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96730357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.289789:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:24.0:1619677447.289790:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.289790:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.289792:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.289792:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.289794:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.289794:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.289798:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.289799:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.289799:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.289801:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000400:00000200:6.0:1619677447.289802:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.289803:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.289803:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.289804:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677447.289804:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:24.0:1619677447.289806:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.289806:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.289806:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:9.2:1619677447.289808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.289810:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.289811:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.289811:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:24.0:1619677447.289812:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.289812:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:24.0:1619677447.289813:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.289814:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:24.0:1619677447.289815:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.289815:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677447.289816:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677447.289816:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677447.289817:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677447.289817:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.289818:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.289819:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.289824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.289825:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.289828:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.289829:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.289830:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.289831:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.289834:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.289835:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.289839:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.289842:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.289843:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.289844:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.289847:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.289848:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.289848:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.289849:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.289850:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.290003:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.290006:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.290007:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.290009:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.290010:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.290011:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.290023:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.290026:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.290027:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.290027:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.290028:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.290029:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.290264:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677447.290267:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677447.290268:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677447.290270:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677447.290272:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.290273:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677447.290274:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.290275:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.290277:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.290277:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677447.290280:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677447.290282:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.290283:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.290284:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.290285:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.290286:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.290287:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677447.290287:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677447.290288:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677447.290292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.290301:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.290303:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.290303:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.290305:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.290305:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677447.290307:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.290367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.290443:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.290446:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.290447:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.290447:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.290449:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.290451:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.290594:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677447.290597:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677447.290597:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000400:00000200:6.0:1619677447.290600:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.290602:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.290603:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:6.0:1619677447.290604:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.290606:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.290607:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.290608:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:6.0:1619677447.290611:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.290613:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677447.290615:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.290616:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677447.290617:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.290619:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677447.290619:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.290621:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:6.0:1619677447.290622:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:6.0:1619677447.290622:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:9.2:1619677447.290625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.290632:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677447.290634:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677447.290634:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677447.290635:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677447.290645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:9.2:1619677447.290769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.290775:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.290778:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.290778:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.290779:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.290780:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.290783:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677447.290863:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677447.290866:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677447.290867:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:13.0:1619677447.290867:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:8.0:1619677447.290869:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.290870:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.290870:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.290872:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.290872:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.290873:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:12.0:1619677447.290874:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.290875:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:8.0:1619677447.290875:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.290876:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.290877:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.290878:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677447.290881:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.290883:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:12.0:1619677447.290884:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.290885:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.290886:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:12.0:1619677447.290887:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.290887:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.0:1619677447.290887:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:12.0:1619677447.290888:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:8.0:1619677447.290888:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:12.0:1619677447.290889:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.290889:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:12.0:1619677447.290890:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.290890:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677447.290890:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:9.2:1619677447.290942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.291083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.291269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.291284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.291291:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.291298:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.291298:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.291300:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.291303:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.291305:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.291306:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.291308:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.291308:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.291310:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.291313:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.291316:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.291319:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.291320:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.291321:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.291323:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.291323:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.291324:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.291324:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.291346:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.291349:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.291350:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.291352:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.291353:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.291355:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.291488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.291529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.291617:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.291620:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.291621:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.291623:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.291625:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.291626:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.291628:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96730332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.291630:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.291631:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.291632:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.291635:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.291637:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.291639:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.291640:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.291641:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.291642:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.291643:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.291644:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.291645:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.291646:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.291656:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.291658:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.291659:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.291660:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.291758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:13.0:1619677447.291765:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.0:1619677447.291765:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:13.0:1619677447.291768:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:8.0:1619677447.291768:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:13.0:1619677447.291769:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.291770:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.291770:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.291771:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.291772:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:8.0:1619677447.291772:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.291773:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.291775:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.291905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.291906:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.291909:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.291910:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.291912:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.291914:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.291916:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.291918:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.291919:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.291920:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.291921:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.291926:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.291928:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.291931:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.291932:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.291933:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.291934:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.291935:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.291936:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.291937:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.291938:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.291951:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.291953:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.291954:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.291955:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:4.2:1619677447.292148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.292194:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.292197:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.292198:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.292198:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.292201:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.292204:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.292212:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.292215:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.292216:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.292216:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.292217:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.292218:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.292234:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.292237:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.292238:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.292238:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.292240:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.292243:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.292371:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.292373:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.292375:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.292376:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.292380:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.292381:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.292382:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.292384:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.292385:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.292386:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.292389:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.292391:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.292394:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.292394:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.292395:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.292397:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.292397:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.292398:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.292398:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.292404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.292405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:4.2:1619677447.292409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.292412:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.292414:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.292415:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.292416:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.292419:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.292420:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.292422:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.292423:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.292424:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.292425:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.292429:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.292431:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.292434:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.292435:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.292436:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.292438:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.292438:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.292439:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.292440:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.292455:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.292459:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.292459:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.292462:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.292464:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.292466:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.292467:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96730321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.292469:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.292471:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.292472:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.292474:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.292475:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.292476:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.292476:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.292478:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.292478:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.292481:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.292482:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.292482:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.292483:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.292484:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.292486:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.292487:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.292488:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.292488:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.292536:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.292538:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.292539:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.292541:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.292543:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.292549:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.292695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.292704:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.292706:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.292707:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.292707:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.292710:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.292712:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.292736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.292740:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.292743:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.292744:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.292744:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.292747:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.292750:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.292852:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.292855:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.292855:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.292856:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.292858:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.292861:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.292869:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.292872:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.292872:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.292873:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.292874:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.292874:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.292964:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.292967:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.292968:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.292970:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.292972:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.292973:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:9.2:1619677447.292975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677447.292975:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96730313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.292976:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.292977:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.292978:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.292981:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.292982:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.292985:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.292986:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.292986:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.292988:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.292989:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.292990:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.292990:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.292994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677447.293064:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.293067:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.293068:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.293071:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.293071:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.293074:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.293083:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.293086:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.293086:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.293087:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.293087:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.293088:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.293098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.293104:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:4.2:1619677447.293106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.293107:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.293109:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.293110:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.293111:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:8.0:1619677447.293111:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.293113:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.293114:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:24.0:1619677447.293115:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.293115:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.293118:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.293120:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.293121:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.293122:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.293123:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.293124:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.293127:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.293129:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.293131:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.293132:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.293133:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.293134:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.293135:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.293135:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.293136:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.293366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.293421:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.293424:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.293425:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.293426:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.293427:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.293430:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.293438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.293459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:9.2:1619677447.293464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.293471:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.293474:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.293475:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.293475:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.293477:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.293480:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.293548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.293603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.293867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.293873:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.293876:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.293877:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.293877:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.293879:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.293882:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677447.293949:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.293952:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.293953:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.293955:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.293958:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.293959:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.293960:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.293962:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.293963:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.293964:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.293967:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.293969:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.293971:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.293972:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.293973:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.293975:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.293975:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.293976:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.293976:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.294112:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.294114:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.294115:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.294116:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.294119:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.294120:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.294177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.294185:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.294189:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.294193:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:22.0:1619677447.294194:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:8.0:1619677447.294196:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.294197:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.294198:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.294198:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.294198:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:8.0:1619677447.294200:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.294201:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.294201:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96730296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.294203:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.294204:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.294205:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:8.0:1619677447.294205:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.294209:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.294211:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.294213:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.294214:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.294215:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.294217:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.294217:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.294219:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.294220:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.294221:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.294226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.294233:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.294235:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.294236:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.294237:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.294512:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.294514:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.294515:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.294517:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.294517:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.294518:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.294519:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.294520:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.294521:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:12.0:1619677447.294521:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.294522:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.294523:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.294554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.294600:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.294603:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.294604:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.294604:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.294607:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.294613:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.294614:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.294616:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.294617:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.294618:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.294618:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.294619:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.294699:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.294701:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.294702:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.294702:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.294705:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:13.0:1619677447.294707:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.294709:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:22.0:1619677447.294710:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:13.0:1619677447.294710:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.294712:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.294713:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.294719:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.294722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.294733:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.294737:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.294737:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:9.2:1619677447.294741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:6.0:1619677447.294741:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.294744:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.294746:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.294748:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.294750:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.294752:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.294754:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.294759:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.294763:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.294765:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.294766:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.294767:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.294769:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.294769:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.294771:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.294772:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.294772:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.294775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.294781:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:22.0:1619677447.294783:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.294785:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.294785:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677447.294786:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:6.0:1619677447.294786:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.294789:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.294790:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.294792:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:9.2:1619677447.294793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.294796:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.294798:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.294800:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.294800:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.294801:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.294803:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.294804:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677447.294984:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.294986:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.294987:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.294987:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.294990:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.294992:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.295128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.295136:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.295139:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677447.295141:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.295144:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.295145:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.295146:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.295148:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96730283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.295149:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.295150:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.295151:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.295156:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.295158:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.295161:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.295162:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.295163:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.295164:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.295165:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.295166:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.295166:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.295190:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.295193:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.295194:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.295195:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.295197:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.295197:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.295380:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.295383:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.295384:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.295385:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.295387:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.295388:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.295398:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.295400:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.295401:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.295402:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.295402:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.295403:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.295440:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.295442:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.295443:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.295445:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.295446:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.295448:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.295525:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.295528:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.295529:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.295531:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.295532:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.295533:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.295602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.295652:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.295655:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.295657:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.295657:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.295659:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.295662:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.295883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677447.295890:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677447.295893:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677447.295894:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677447.295896:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.295899:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.295901:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.295902:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.295904:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.295905:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.295905:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.295909:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.295910:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.295913:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.295914:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.295914:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.295916:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.295917:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677447.295917:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677447.295918:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.2:1619677447.295964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.295970:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.295973:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.295974:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.295974:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.295976:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.295979:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.296029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.296063:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.296065:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.296066:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.296068:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.296069:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.296071:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.296078:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.296081:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.296081:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.296082:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.296084:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.296087:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.296104:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.296106:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.296107:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.296108:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.296109:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.296110:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.296219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.296226:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.296229:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.296229:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.296230:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.296233:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.296238:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.296240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677447.296246:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.296249:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.296250:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.296250:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.296251:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.296251:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677447.296260:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.296263:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.296264:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.296265:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.296266:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.296268:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.296275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.296323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.296327:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.296330:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.296331:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.296332:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.296334:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.296337:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.296348:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.296351:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.296352:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.296353:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.296356:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.296357:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.296364:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:13.0:1619677447.296367:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677447.296367:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.296368:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.296368:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.296371:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:6.0:1619677447.296371:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.296372:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.296373:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.296374:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.296375:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.296377:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.296407:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.296409:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.296411:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.296413:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.296413:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.296416:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.296426:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.296429:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.296430:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.296431:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.296431:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.296433:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.296529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.296567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.296574:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.296577:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.296577:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.296580:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.296582:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.296584:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.296585:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.296587:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.296588:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.296589:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.296592:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.296593:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.296596:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.296597:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.296598:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.296599:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.296600:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.296601:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.296601:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.296651:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.296654:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.296655:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.296656:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.296657:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.296659:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.296864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.296911:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.296914:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.296914:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.296915:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.296917:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.296920:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.297070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.297073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.297095:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.297098:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.297098:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.297101:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.297104:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.297106:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.297107:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96730249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.297109:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.297110:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.297111:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.297116:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.297118:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.297120:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.297122:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.297122:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677447.297123:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.297123:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.297124:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:22.0:1619677447.297125:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.297126:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.297127:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.297128:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:12.0:1619677447.297128:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:6.0:1619677447.297128:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.297129:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.297129:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.297131:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.297132:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.297133:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.297135:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.297136:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.297341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.297360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.297366:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.297369:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.297370:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.297372:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.297375:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.297376:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.297377:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96730244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.297379:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.297380:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.297380:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:24.0:1619677447.297381:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.297382:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.297382:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.297383:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.297383:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.297384:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.297385:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.297385:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.297386:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.297386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677447.297387:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.297390:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.297390:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.297390:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:24.0:1619677447.297391:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.297391:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.297391:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.297392:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677447.297393:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.297393:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.297394:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677447.297394:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.297394:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677447.297395:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.297395:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.297396:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.297397:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.297643:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.297646:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.297646:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.297647:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.297649:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.297652:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.297864:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.297867:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.297868:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.297870:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.297877:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.297879:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.297880:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96730239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.297881:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.297882:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.297883:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.297887:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.297889:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.297893:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.297894:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.297895:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.297897:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.297897:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.297898:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.297899:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.297899:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.297900:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:6.0:1619677447.297900:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.297901:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.297902:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.297905:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.298247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.298252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:13.0:1619677447.298253:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.298256:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.298256:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.298258:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.298259:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:12.0:1619677447.298260:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.298261:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.298262:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.298262:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.298265:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.298267:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.298269:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.298270:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.298271:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.298272:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.298273:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.298277:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.298279:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.298281:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.298282:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.298283:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.298284:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.298300:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.298302:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.298303:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.298304:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.298310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.298317:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.298320:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.298321:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.298322:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.298350:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.298354:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.298355:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.298356:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.298357:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.298360:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.298369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.298465:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.298468:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.298468:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.298470:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.298473:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.298475:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.298477:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96730228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.298478:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.298479:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.298480:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.298483:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.298485:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.298488:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.298489:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.298489:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.298491:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.298492:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.298492:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.298493:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.298722:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.298725:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.298725:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.298726:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.298728:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.298731:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.298742:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.298745:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.298745:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.298746:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.298747:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.298748:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.298931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.298938:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.298942:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.298942:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.298946:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.298949:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.298951:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.298953:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.298955:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.298957:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.298958:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.298964:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.298966:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.298969:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.298970:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)++
00000800:00000200:8.0:1619677447.298972:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (23)++
00000800:00000200:8.0:1619677447.298974:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (23)--
00000800:00000200:8.0:1619677447.298975:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.298980:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.298981:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677447.299069:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.299071:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.299072:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.299074:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.299077:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.299079:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.299080:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.299081:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.299082:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.299084:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.299088:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.299091:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.299094:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.299095:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.299096:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.299098:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.299098:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.299099:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.299100:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.299104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.299126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.299131:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.299133:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.299134:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.299134:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.299136:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.299138:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.299248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.299254:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.299256:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677447.299257:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677447.299259:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.299261:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.299263:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.299264:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.299266:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.299267:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.299268:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.299271:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.299272:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.299275:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.299277:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.299277:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.299279:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.299280:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.299280:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677447.299281:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677447.299316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.299324:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.299327:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.299327:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.299328:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.299330:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:8.0:1619677447.299334:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.299442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.299458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.299528:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.299531:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.299532:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.299534:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.299536:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.299538:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.299539:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.299541:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.299542:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.299543:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.299546:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.299548:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.299551:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.299551:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.299552:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.299554:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.299554:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.299555:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.299556:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.299557:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677447.299558:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.299561:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.299562:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.299563:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.299566:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.299567:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:24.0:1619677447.299568:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.299569:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96730212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.299569:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.299570:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.299570:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:24.0:1619677447.299571:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.299572:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.299574:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.299577:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.299579:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.299582:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.299583:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.299584:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.299586:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.299586:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.299587:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.299588:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.299592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.299732:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.299734:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.299735:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.299736:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.299739:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.299740:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.299810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:12.0:1619677447.299811:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.299813:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.299814:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.299815:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.299817:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.299818:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.299818:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.299820:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.299821:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.299823:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.299826:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.299827:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.299828:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.299830:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.299831:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.299832:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.299835:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.299837:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.299840:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.299841:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.299841:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.299843:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.299843:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.299844:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.299844:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677447.299983:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.299986:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.299987:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.299988:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.299991:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.299993:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.299994:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.299995:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.299996:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.299998:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.300002:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.300004:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.300007:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.300007:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.300008:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.300010:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.300010:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.300011:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.300012:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.300013:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.300027:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.300029:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.300030:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.300031:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.300038:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.300041:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.300042:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.300043:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.300044:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.300046:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.300135:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.300138:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677447.300138:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000400:00000200:6.0:1619677447.300141:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.300144:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.300146:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:6.0:1619677447.300147:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96730204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.300148:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.300149:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.300150:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:6.0:1619677447.300154:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.300155:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677447.300158:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.300159:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677447.300160:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.300162:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677447.300163:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677447.300164:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677447.300166:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.300166:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:22.0:1619677447.300168:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:9.2:1619677447.300168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:22.0:1619677447.300169:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.300169:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.300171:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.300172:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.300191:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.300194:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.300195:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.300195:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.300196:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.300197:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.300216:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.300219:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.300220:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.300221:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.300223:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.300223:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.300237:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.300239:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.300240:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.300241:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.300241:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.300242:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.300272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.300582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.300591:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.300594:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.300595:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.300599:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.300602:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.300603:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.300605:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.300606:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.300608:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.300610:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.300614:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.300617:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.300622:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.300623:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.300623:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.300625:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.300625:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.300626:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.300627:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.300642:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.300644:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.300645:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.300645:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.300648:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.300652:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.300678:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.300681:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.300683:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.300685:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.300686:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.300687:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.300768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:4.2:1619677447.300861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.300908:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.300911:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.300911:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.300912:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.300914:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.300917:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.301006:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.301008:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.301010:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.301012:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.301012:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.301015:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.301028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.301036:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.301038:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.301039:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.301039:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.301042:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.301045:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.301167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.301209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:9.2:1619677447.301430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.301436:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.301439:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.301440:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.301440:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.301442:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.301445:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.301485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.301532:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.301535:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.301535:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.301536:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.301538:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.301541:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.301552:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.301555:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.301556:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.301556:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.301558:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.301558:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.301759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.301871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.301877:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.301880:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.301881:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.301881:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.301883:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.301885:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.302062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.302118:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.302121:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.302122:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.302122:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.302125:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.302127:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.302306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.302313:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.302316:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.302317:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.302319:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.302321:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.302323:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.302324:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.302327:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.302328:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.302329:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.302332:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.302334:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.302337:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.302338:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.302338:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.302340:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.302340:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.302341:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.302342:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.302368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.302419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677447.302565:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.302569:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.302569:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.302571:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.302573:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.302574:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.302584:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.302587:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.302588:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.302589:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.302589:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.302591:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.302725:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.302728:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.302729:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.302731:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.302734:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.302735:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.302737:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.302738:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.302739:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.302741:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.302745:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.302748:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.302751:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.302752:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.302753:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.302755:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.302755:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.302756:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.302757:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.302916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.302939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:9.2:1619677447.303015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.303022:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.303024:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.303026:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.303026:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.303028:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.303031:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.303082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.303126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.303134:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.303136:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.303137:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.303139:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.303142:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.303144:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.303145:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.303147:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.303148:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.303149:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.303151:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.303153:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.303156:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.303156:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.303157:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.303159:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.303159:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.303160:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.303162:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.303162:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.303168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.303176:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.303178:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.303179:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.303180:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.303243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.303478:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.303481:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.303482:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.303482:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.303484:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.303486:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.303580:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.303583:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.303583:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.303584:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.303586:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.303588:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.303708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:6.0:1619677447.303714:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.303716:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:24.0:1619677447.303717:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.303717:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.303719:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.303720:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.303721:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677447.303722:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.303724:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.303724:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.303725:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96730161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.303727:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.303727:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.303728:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.303729:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.303729:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.303730:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96730160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.303730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677447.303732:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.303733:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.303734:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.303735:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.303736:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.303739:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.303740:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.303740:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.303740:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.303742:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.303742:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.303742:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.303743:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.303744:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.303745:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.303746:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.303747:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.303749:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (24)--
00000800:00000200:24.0:1619677447.303751:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.303752:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.303753:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677447.303755:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.303757:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.303758:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.303760:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.303762:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.303764:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.303765:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96730157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.303767:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.303768:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.303769:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:9.2:1619677447.303771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:24.0:1619677447.303774:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.303776:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.303779:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.303779:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (23)++
00000800:00000200:24.0:1619677447.303780:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (24)++
00000800:00000200:24.0:1619677447.303783:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (25)--
00000800:00000200:24.0:1619677447.303784:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.303784:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.303785:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.303788:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:9.2:1619677447.303789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (22)++
00000800:00000200:22.0:1619677447.303791:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.303793:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.303793:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.303795:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.303799:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.303827:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.303829:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.303830:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.303830:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.303832:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.303835:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.303996:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.303999:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.304000:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.304002:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677447.304003:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.304005:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.304034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.304049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:9.2:1619677447.304053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.304057:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677447.304059:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677447.304060:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677447.304062:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.304066:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.304068:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677447.304069:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.304070:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.304071:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.304072:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677447.304076:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.304078:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677447.304080:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.304081:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:8.0:1619677447.304082:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:8.0:1619677447.304084:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:8.0:1619677447.304084:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677447.304085:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677447.304086:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:12.0:1619677447.304159:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.304162:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.304163:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.304165:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.304166:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.304169:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.304178:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.304181:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.304181:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.304182:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677447.304182:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.304183:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.304184:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.304185:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.304185:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677447.304187:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.304189:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.304204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.304227:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.304230:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.304232:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.304233:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.304236:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.304238:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.304240:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.304241:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.304242:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.304243:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.304249:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677447.304249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677447.304252:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.304255:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.304256:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.304257:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.304259:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.304260:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.304260:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677447.304262:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:22.0:1619677447.304262:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.304263:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.304263:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:24.0:1619677447.304265:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.304268:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.304270:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:24.0:1619677447.304271:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.304272:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96730147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.304272:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:24.0:1619677447.304274:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.304274:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.304275:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:24.0:1619677447.304276:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.304276:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.304276:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.304277:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.304278:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:22.0:1619677447.304278:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.304279:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.304280:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:6.0:1619677447.304280:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.304283:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.304285:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.304286:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.304287:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.304288:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:6.0:1619677447.304288:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:8.0:1619677447.304289:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96730145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.304291:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.304292:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.304308:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.304312:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.304313:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:8.0:1619677447.304313:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.304314:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.304315:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.304316:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.304317:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.304317:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.304318:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.304319:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.304319:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.304320:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.304320:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.304325:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.304327:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.304328:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.304328:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.304329:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.304334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.304592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.304621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:4.2:1619677447.304643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.304692:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.304695:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.304696:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.304697:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.304698:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.304703:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.304706:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.304706:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.304707:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.304707:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.304709:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000400:00000200:22.0:1619677447.304710:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.304710:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.304711:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.304712:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.304713:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.304713:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:22.0:1619677447.304715:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.304717:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96730139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.304719:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677447.304719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:22.0:1619677447.304720:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.304721:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.304726:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.304727:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.304730:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.304732:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:22.0:1619677447.304732:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.304732:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677447.304733:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.304734:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:24.0:1619677447.304735:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.304735:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.304736:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.304737:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.304737:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:24.0:1619677447.304738:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.304740:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.304742:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96730137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.304744:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.304745:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:24.0:1619677447.304747:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.304748:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:12.0:1619677447.304748:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.304749:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.304752:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.304752:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.304753:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.304755:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.304756:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.304759:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.304760:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.304761:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.304763:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.304763:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:24.0:1619677447.304764:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.304764:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.304765:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.304766:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.304767:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.304767:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.304769:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.304772:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.304783:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.304786:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.304787:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.304788:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:13.0:1619677447.304789:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.304790:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677447.304791:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:13.0:1619677447.304791:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.304792:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.304792:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.304793:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.304793:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.305079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.305194:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.305196:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.305197:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.305198:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.305200:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.305201:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.305214:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.305216:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.305217:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.305218:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.305218:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.305219:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.305337:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.305339:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.305343:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.305344:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677447.305345:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.305348:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.305361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.305376:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.305379:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.305380:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.305381:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.305383:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.305387:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.305401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677447.305475:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.305477:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.305478:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.305480:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.305481:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.305482:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.305533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.305541:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.305544:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.305545:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.305547:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.305549:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.305551:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.305552:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.305553:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.305554:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.305555:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.305558:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.305560:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.305563:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.305564:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.305564:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.305566:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.305567:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.305568:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.305569:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.305569:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.305575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.305582:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.305585:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.305585:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.305587:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.305829:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.305832:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.305832:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.305834:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.305836:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.305838:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.305839:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.305841:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.305842:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.305843:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.305846:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.305848:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.305850:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.305851:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.305852:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.305853:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.305854:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.305855:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.305855:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.305919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.305976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.305998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:12.0:1619677447.306141:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.306144:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.306145:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.306147:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.306147:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.306150:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.306161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:13.0:1619677447.306330:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.306333:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.306333:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.306335:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.306337:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.306338:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.306341:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.306341:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.306342:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.306342:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.306343:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.306371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.306379:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.306382:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677447.306383:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677447.306387:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.306390:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.306392:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:22.0:1619677447.306394:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96730120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.306396:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.306397:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.306399:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677447.306402:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.306404:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.306409:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.306410:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677447.306411:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.306412:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.306413:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.306413:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677447.306414:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:6.0:1619677447.306437:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:22.0:1619677447.306440:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (24)++
00000800:00000200:6.0:1619677447.306440:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:22.0:1619677447.306441:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000400:00000200:6.0:1619677447.306441:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.306442:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.306443:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.306443:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.306446:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.306447:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.306448:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.306472:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.306475:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.306475:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.306477:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.306478:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.306479:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.306581:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.306584:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.306585:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.306587:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.306588:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.306589:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.306807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.306854:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:13.0:1619677447.306857:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677447.306857:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.306858:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.306859:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.306860:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.306860:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.306861:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:13.0:1619677447.306862:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.306863:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677447.306863:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.306865:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.306942:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.306945:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.306946:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.306947:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.306949:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.306950:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.307161:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.307163:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.307165:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.307166:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.307167:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.307169:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.307181:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.307184:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.307185:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.307186:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.307186:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.307188:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.307299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.307342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.307348:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.307351:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.307351:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.307353:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.307355:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.307358:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.307367:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.307369:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.307370:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.307370:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.307372:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.307372:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.2:1619677447.307591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (21)++
00000800:00000200:24.0:1619677447.307595:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)++
00000800:00000200:24.0:1619677447.307598:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e49d37c00] (23)--
00000800:00000200:8.0:1619677447.307599:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)--
00000400:00000200:24.0:1619677447.307600:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.307602:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.307606:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)--
00000800:00000200:24.2:1619677447.307616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (20)++
00000800:00000200:24.0:1619677447.307619:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)++
00000800:00000200:24.0:1619677447.307622:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:8.0:1619677447.307622:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)--
00000800:00000200:24.0:1619677447.307623:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e49d37c00] (22)++
00000800:00000200:24.0:1619677447.307624:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e49d37c00] (22)--
00000800:00000200:24.0:1619677447.307625:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)--
00000800:00000200:13.0:1619677447.307645:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.307647:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.307648:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.307650:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.307652:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.307653:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.307669:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.307671:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.307672:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.307675:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.307675:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.307678:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.307691:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.307695:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.307696:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.307697:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.307697:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.307699:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.307801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.307808:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.307811:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.307812:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.307812:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.307814:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677447.307815:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.307817:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.307818:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.307820:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.307820:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.307820:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.307828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:13.0:1619677447.307830:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.307834:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.307837:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.307838:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.307838:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.307840:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.307840:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677447.307850:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.307852:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.307853:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.307856:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.307857:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677447.307859:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96730093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.307861:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.307862:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.307863:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:4.2:1619677447.307865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.307866:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677447.307867:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.42@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.307869:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.307871:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.307872:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.307873:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.307875:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.307875:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39c000] (24)++
00000800:00000200:2.0:1619677447.307876:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39c000] (24)--
00000800:00000200:2.0:1619677447.307877:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39c000] (24)--
00000800:00000200:2.0:1619677447.307878:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39c000] (21)++
00000800:00000200:2.0:1619677447.307881:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.42@o2ib3
00000400:00000200:2.0:1619677447.307884:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.42@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.307886:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.307888:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.307889:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96730091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.307891:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.307892:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.307893:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.307901:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.42@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.307904:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.307909:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.307911:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.307912:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.307914:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:9.2:1619677447.307920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.307921:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39c000] (23)++
00000800:00000200:2.0:1619677447.307922:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39c000] (24)--
00000800:00000200:2.0:1619677447.307923:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39c000] (23)--
00000800:00000200:2.0:1619677447.307924:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39c000] (22)--
00000800:00000200:2.0:1619677447.307926:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.307928:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.307930:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.307932:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.307936:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.307937:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.307938:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39c000] (23)++
00000800:00000200:2.0:1619677447.307941:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.42@o2ib3
00000400:00000200:2.0:1619677447.307943:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.42@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.307946:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.307947:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:2.0:1619677447.307949:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:13.0:1619677447.307950:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:2.0:1619677447.307950:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96730088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.307951:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.307952:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677447.307953:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.307953:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:2.0:1619677447.307954:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.307955:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:13.0:1619677447.307957:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:2.0:1619677447.307959:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.42@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.307961:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677447.307964:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.307965:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:2.0:1619677447.307966:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:2.0:1619677447.307967:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:2.0:1619677447.307968:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39c000] (21)++
00000800:00000200:2.0:1619677447.307969:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39c000] (22)--
00000800:00000200:2.0:1619677447.307970:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39c000] (21)--
00000800:00000200:2.0:1619677447.307984:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39c000] (22)--
00000800:00000200:2.0:1619677447.308003:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39c000] (22)++
00000800:00000200:2.0:1619677447.308004:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.42@o2ib3
00000400:00000200:2.0:1619677447.308007:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.42@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.308008:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:2.0:1619677447.308009:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.308011:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.308011:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.308011:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677447.308013:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96730085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677447.308014:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.308015:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:2.0:1619677447.308015:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.308016:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:2.0:1619677447.308016:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.308017:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677447.308021:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.42@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.308023:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677447.308024:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677447.308025:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.308026:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:22.0:1619677447.308027:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677447.308027:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:22.0:1619677447.308028:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.308028:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677447.308029:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:2.0:1619677447.308029:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39c000] (21)++
00000400:00000200:22.0:1619677447.308030:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.308030:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39c000] (22)--
00000800:00000200:2.0:1619677447.308031:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39c000] (21)--
00000800:00000200:22.0:1619677447.308033:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.308315:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.308318:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.308318:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.308320:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.308322:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.308323:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.308353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.308366:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:4.2:1619677447.308368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.308407:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.308410:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.308412:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.308414:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.308416:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.308418:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.308420:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96730079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.308422:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.308423:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.308425:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.308432:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.308436:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.308439:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.308440:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.308441:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.308443:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.308443:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.308444:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.308445:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.308454:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.308457:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.308458:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.308458:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.308460:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.308462:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.308542:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.308545:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.308546:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.308548:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.308551:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.308552:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.308553:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.308555:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.308556:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.308557:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.308562:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.308565:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.308567:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.308568:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.308568:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.308570:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.308570:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.308571:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.308572:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:2.0:1619677447.308578:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:13.0:1619677447.308582:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)++
00000800:00000200:2.0:1619677447.308582:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.308584:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.308585:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677447.308586:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.308587:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.308588:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.308589:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000400:00000200:2.0:1619677447.308589:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:12.0:1619677447.308590:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:2.0:1619677447.308590:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96730075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.308591:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.308591:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.308591:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.308592:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.308593:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.308593:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.308594:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:12.0:1619677447.308596:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.308597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:2.0:1619677447.308597:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.308599:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.308601:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.308602:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.308602:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.308604:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677447.308604:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.308604:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.308605:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.308606:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:24.0:1619677447.308607:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:13.0:1619677447.308608:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.308608:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677447.308610:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.308611:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.308611:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.308612:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.308613:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677447.308614:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.308614:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677447.308616:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.308618:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96730074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.308619:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.308620:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.308621:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.308626:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.308629:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.308632:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.308632:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.308633:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.308635:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.308635:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.308636:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.308637:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.308801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.308813:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:12.0:1619677447.308821:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.308824:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.308824:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.308826:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.308828:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.308829:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.308857:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.308859:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.308861:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.308861:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.308864:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.308867:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.308896:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.308898:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.308900:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.308900:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.308902:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.308904:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.308915:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.308918:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.308918:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.308920:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.308920:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.308921:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.309023:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.309026:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.309027:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.309027:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.309030:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.309032:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.309062:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.309064:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.309065:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.309067:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.309069:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.309071:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.309072:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.309073:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.309075:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.309075:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.309079:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.309081:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.309083:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.309084:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.309085:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.309087:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.309087:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.309089:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.309090:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.309090:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.309100:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.309102:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.309103:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.309104:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.309164:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.309167:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.309170:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.309173:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.309174:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.309175:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.309177:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.309178:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.309179:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.309182:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677447.309184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:2.0:1619677447.309184:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.309186:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.309187:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.309187:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.309189:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.309189:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:8.0:1619677447.309191:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677447.309191:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.309192:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.309192:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:24.0:1619677447.309196:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.309199:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.309202:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.309205:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.309206:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.309207:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96730064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.309210:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.309216:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.309217:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.309221:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.309224:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.309226:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.309227:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.309228:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.309229:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.309230:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.309231:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.309231:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.309281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.309326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.309330:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.309333:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.309334:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.309334:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.309336:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.309340:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.309397:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.309399:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.309400:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.309401:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677447.309401:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.309402:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.309404:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:22.0:1619677447.309405:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:13.0:1619677447.309405:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.309407:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.309409:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.309409:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677447.309461:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.309463:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.309464:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.309466:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.309469:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.309470:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.309472:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.309473:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.309474:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.309475:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.309479:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.309481:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.309484:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.309484:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.309485:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.309487:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.309487:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.309488:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.309488:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.309554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.309562:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.309565:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.309566:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.309568:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.309572:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.309574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.309584:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:4.2:1619677447.309603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.309610:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.309614:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.309615:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.309616:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.309620:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.309621:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.309622:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.309623:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.309625:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.309625:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.309628:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.309630:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.309633:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.309634:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.309634:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.309636:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.309636:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.309637:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.309637:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677447.309678:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.309681:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:4.2:1619677447.309683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677447.309684:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.309686:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.309687:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.309688:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.309689:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.309690:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.309691:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.309696:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.309699:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.309701:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.309703:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.309703:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.309705:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.309705:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.309706:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.309707:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.0:1619677447.309760:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.309763:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.309764:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.309766:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.309769:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.309770:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.309772:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.309773:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.309774:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.309775:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.309778:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.309780:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.309783:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.309784:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.309784:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.309786:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.309787:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.309788:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.309789:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.309790:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.309803:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.309805:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.309806:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.309807:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.309885:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.309888:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677447.309888:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677447.309891:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.309893:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.309895:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677447.309896:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.309898:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.309898:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.309900:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677447.309903:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.309905:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677447.309907:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.309908:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:22.0:1619677447.309909:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677447.309910:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:22.0:1619677447.309911:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.309912:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:22.0:1619677447.309913:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:22.0:1619677447.309913:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677447.309923:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000800:00000200:6.0:1619677447.309925:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:6.0:1619677447.309925:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677447.309926:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:22.0:1619677447.310012:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.310015:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.310015:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.310017:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.310020:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.310021:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.310023:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.310024:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.310025:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.310026:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.310029:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.310030:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.310033:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.310033:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.310034:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.310035:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.310036:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.310037:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.310037:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.310188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.310195:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.310198:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.310200:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.310201:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.310205:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.310206:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.310207:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.310209:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.310210:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.310211:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.310214:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.310216:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.310219:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.310219:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.310220:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.310222:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.310222:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.310223:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.310224:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.310225:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.310225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.310230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.310237:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.310240:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.310240:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.310241:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.310271:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.310274:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.310274:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.310275:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.310277:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.310280:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.310432:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:12.0:1619677447.310470:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.310475:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677447.310476:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.310477:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.310477:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.310482:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.310483:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.310484:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.310485:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.310486:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.310486:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.310488:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.310511:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.310514:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.310515:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.310515:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.310516:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.310517:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.310533:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.310535:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.310536:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.310538:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.310539:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.310540:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.310552:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.310555:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.310556:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.310557:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.310557:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.310559:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.310575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:4.2:1619677447.310581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.310585:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.310587:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.310588:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.310590:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.310591:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.310591:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.310594:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.310595:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.310596:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:24.0:1619677447.310597:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.310597:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.310599:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.310600:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.310602:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.310603:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.310605:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.310608:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.310610:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.310613:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.310614:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.310614:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.310616:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.310616:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.310617:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.310618:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.310635:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.310638:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.310639:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.310641:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.310642:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.310643:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.310655:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.310658:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.310659:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.310661:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.310663:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.310665:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.310666:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.310667:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.310669:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.310669:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.310673:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.310674:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.310677:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.310678:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.310679:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.310681:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.310681:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.310682:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.310682:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.310687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.310694:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.310697:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.310698:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.310701:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.310704:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.310772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.310923:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.310925:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.310926:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.310928:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.310931:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.310932:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.310933:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96730039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.310935:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.310936:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.310937:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.310940:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.310942:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.310944:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.310945:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.310946:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.310947:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.310948:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.310949:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.310949:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.311568:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.311572:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.311573:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.311575:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677447.311575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:24.0:1619677447.311578:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.311580:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.311582:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96730033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.311583:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:24.0:1619677447.311584:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.311585:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.311585:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.311586:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.311586:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.311588:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.311590:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.311591:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.311592:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.311593:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677447.311596:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.311597:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.311598:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.311600:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.311600:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.311601:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.311601:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.311725:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.311727:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.311728:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.311729:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.311732:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.311733:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.311791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:12.0:1619677447.311795:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.311797:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.311798:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.311798:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:12.0:1619677447.311799:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.311800:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.311801:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677447.311801:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.311803:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677447.311803:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.311806:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.311807:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.311809:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.311810:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.311811:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.311812:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.311815:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.311817:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.311819:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.311820:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.311821:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.311823:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.311823:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.311824:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.311824:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.311969:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.311972:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.311973:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.311975:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.311976:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.311977:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.312141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.312181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.312187:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.312190:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.312191:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.312191:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.312194:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.312197:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.312283:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.312286:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.312287:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.312289:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.312292:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.312299:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.312300:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.312301:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.312302:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.312303:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.312308:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.312310:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:24.0:1619677447.312311:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.312315:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.312316:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.312317:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.312319:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.312319:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.312320:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.312321:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.312326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.312335:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.312337:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.312338:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.312341:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.312345:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677447.312366:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.312368:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.312369:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.312371:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.312373:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.312374:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.312596:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.312599:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.312600:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.312602:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.312603:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.312604:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.312628:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.312631:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.312632:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.312634:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.312635:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.312636:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.312648:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.312651:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.312652:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.312652:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.312653:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.312654:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.312733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.312772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.312785:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.312910:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.312913:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677447.312917:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.312920:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.312922:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.312924:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96730016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.312925:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.312926:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.312927:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.312931:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.312933:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.312935:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.312936:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.312936:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.312938:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.312938:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.312939:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.312940:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:22.0:1619677447.312952:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.312954:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.312955:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.312955:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.312958:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.312960:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.312969:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.312972:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.312972:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.312974:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.312975:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.312975:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.312995:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.312998:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.312999:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.313001:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.313002:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.313004:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.313017:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.313020:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.313021:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.313021:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.313022:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.313023:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.313030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.313041:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.313044:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.313045:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.313045:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.313047:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.313049:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.313069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.313189:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.313192:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.313193:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.313193:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.313196:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.313198:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.313343:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.313346:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.313347:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.313349:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.313351:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.313353:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.313354:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96730009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.313355:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.313356:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.313357:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.313360:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.313361:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.313364:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.313365:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.313366:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.313368:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.313368:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:6.0:1619677447.313368:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677447.313370:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:6.0:1619677447.313370:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:24.0:1619677447.313371:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:22.0:1619677447.313371:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677447.313372:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:6.0:1619677447.313373:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.313375:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.313377:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.313378:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96730008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.313379:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.313380:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.313381:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.0:1619677447.313384:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:9.2:1619677447.313385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677447.313385:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.313386:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.313386:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.313387:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.313387:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.313390:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.313390:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.313391:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.313393:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.313394:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.313395:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.313396:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.313397:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677447.313397:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:9.2:1619677447.313404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:22.0:1619677447.313407:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.313409:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.313410:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.313411:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.313411:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.313414:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.313415:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.313416:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.313417:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.313483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.313500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.313511:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.313525:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.313527:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677447.313530:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.313533:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.313534:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.313535:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96730004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.313537:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.313538:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.313539:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.313544:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.313547:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.313549:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.313550:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.313550:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.313553:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.313553:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.313554:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.313554:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.313569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.313580:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.313771:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.313774:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.313775:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.313777:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.313778:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:24.0:1619677447.313780:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.313781:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:12.0:1619677447.313781:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.313781:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.313783:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677447.313783:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.313784:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.313785:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.313785:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677447.313786:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:12.0:1619677447.313786:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677447.313789:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.313791:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.313794:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.313795:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.313795:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.313797:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.313797:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.313798:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.313798:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:4.2:1619677447.314000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.314039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:4.2:1619677447.314071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.314078:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.314081:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.314082:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.314083:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.314087:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.314088:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.314089:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.314090:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.314091:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.314092:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.314095:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.314096:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.314099:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.314100:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.314100:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.314102:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.314103:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.314103:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.314104:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677447.314168:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.314171:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.314174:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.314176:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.314177:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.314179:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.314180:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.314181:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.314182:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.314185:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.314186:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.314189:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.314190:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.314191:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.314192:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.314193:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.314193:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.314194:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:6.0:1619677447.314362:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.314365:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.314366:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.314369:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.314372:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.314373:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.314375:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.314377:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.314377:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.314379:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.314384:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.314387:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.314390:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.314391:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.314391:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.314393:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.314394:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.314394:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.314395:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.314398:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.314401:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.314401:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.314403:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.314405:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.314406:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.314463:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:12.0:1619677447.314498:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.314501:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.314502:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.314504:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.314505:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.314507:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.314584:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.314587:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.314588:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.314588:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.314590:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.314593:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.314615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.314677:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.314680:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.314681:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.314681:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.314683:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.314686:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.314697:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.314699:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.314700:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.314700:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.314701:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.314702:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.314745:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.314748:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.314751:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.314754:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.314756:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.314757:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.314758:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.314759:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.314760:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.314764:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.314765:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.314767:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.314769:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.314769:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.314771:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.314771:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.314773:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.314774:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.314774:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:13.0:1619677447.314961:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.314964:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.314964:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.314966:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.314968:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.314968:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:4.2:1619677447.315008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.315015:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.315018:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.315019:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.315020:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.315023:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.315025:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.315026:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.315027:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.315028:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.315029:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.315032:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.315034:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.315036:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.315037:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.315038:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.315039:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.315040:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.315041:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.315042:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.315042:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.315048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.315055:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.315057:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.315058:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.315059:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.315061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.315068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.315076:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.315078:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.315080:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.315082:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.315086:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.315123:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.315126:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.315127:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.315127:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.315129:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.315132:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.315147:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:22.0:1619677447.315206:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.315208:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.315209:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.315209:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.315211:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.315213:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.315224:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.315227:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.315227:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.315230:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.315232:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.315234:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.315235:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.315236:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.315237:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.315238:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.315241:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.315243:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.315246:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.315247:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.315247:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.315249:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.315250:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.315250:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.315251:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.315406:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.315409:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.315410:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.315412:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.315415:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.315416:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.315418:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.315419:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.315420:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.315421:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.315426:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677447.315427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.315428:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.315431:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.315432:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.315432:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.315434:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.315435:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.315435:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.315436:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.315478:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.315481:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.315482:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.315483:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.315485:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.315487:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.315589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.315596:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.315599:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.315599:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.315601:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.315604:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.315606:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.315608:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.315609:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.315610:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.315611:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.315615:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.315616:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.315619:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.315620:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.315621:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.315622:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.315623:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.315623:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.315624:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.315681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.315689:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.315692:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.315693:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.315695:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.315699:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677447.315787:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:12.0:1619677447.315823:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.315825:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.315826:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.315828:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.315830:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.315831:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.315840:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.315843:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.315844:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.315844:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.315845:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.315846:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.315859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.315896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.315899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.315903:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.315906:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.315908:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.315910:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.315913:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.315916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.315926:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677447.315979:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.315982:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.315982:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.315984:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.315986:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.315988:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.315989:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.315991:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.315992:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.315993:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.315996:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.315998:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.316000:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.316001:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.316002:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.316003:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.316004:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.316004:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.316005:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.316009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.316017:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.316019:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.316020:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.316022:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.316024:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.316252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:9.2:1619677447.316256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.316262:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.316264:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.316264:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.316266:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.316267:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.316267:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.316269:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.316270:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.316272:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.316273:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.316274:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.316276:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.316277:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.316279:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.316280:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.316281:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000400:00000200:24.0:1619677447.316282:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.316282:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677447.316283:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.316284:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.316285:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.316286:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.316287:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.316287:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677447.316289:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.316290:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.316291:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.316292:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.316292:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.316293:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.316546:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.316548:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.316549:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.316550:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.316552:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.316553:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.316644:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.316647:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.316648:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.316648:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.316650:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.316653:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.316697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.316743:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.316746:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.316746:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.316747:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.316749:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.316752:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.316754:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:22.0:1619677447.316761:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.316764:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.316764:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.316765:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.316766:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.316766:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.316781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.316789:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.316790:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.316791:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.316793:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.316795:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.316884:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.316887:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.316888:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.316890:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.316891:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.316893:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.316950:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677447.316982:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677447.316983:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:6.0:1619677447.316985:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677447.316985:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:22.0:1619677447.316986:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677447.316986:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:6.0:1619677447.316987:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.316987:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:6.0:1619677447.316989:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.316991:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.317044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.317056:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:9.2:1619677447.317063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.317070:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.317072:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.317074:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.317075:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.317075:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:22.0:1619677447.317156:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.317159:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.317160:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.317162:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.317164:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.317165:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.317167:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.317168:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.317169:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.317170:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.317173:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.317174:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.317177:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.317178:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.317179:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.317181:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.317181:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.317182:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.317183:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.317184:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.317194:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.317196:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.317197:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.317198:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.317235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:4.2:1619677447.317241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.317246:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.317248:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.317249:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.317250:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.317251:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.317251:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.317254:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.317255:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.317256:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:24.0:1619677447.317257:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.317257:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.317259:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.317261:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.317262:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.317263:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.317264:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.317267:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.317269:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.317272:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.317273:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.317274:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.317275:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.317276:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.317276:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.317277:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.317557:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.317559:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.317560:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.317562:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.317564:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.317565:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.317592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.317639:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.317642:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.317642:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.317643:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.317645:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.317648:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.317735:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677447.317736:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:13.0:1619677447.317738:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.317738:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.317740:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.317741:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.317742:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.317753:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.317755:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.317756:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.317757:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.317757:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.317759:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.317770:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.317773:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.317774:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.317774:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.317776:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.317778:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.317979:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.317981:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.317982:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.317983:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.317985:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.317985:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:4.2:1619677447.318215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.318298:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.318301:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677447.318305:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.318308:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.318310:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.318312:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96729936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.318314:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.318316:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.318318:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.318324:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.318328:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.318331:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.318332:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.318333:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.318335:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.318335:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.318337:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:12.0:1619677447.318338:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.318338:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677447.318341:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.318342:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.318344:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.318346:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.318346:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.318352:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.318354:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.318355:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.318355:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.318357:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.318360:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.318455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.318462:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.318465:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.318466:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.318468:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.318471:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.318473:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.318474:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.318476:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.318477:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.318478:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.318481:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.318483:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.318486:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.318487:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.318488:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.318490:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.318490:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.318492:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.318493:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.318494:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.318499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.318506:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.318508:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.318509:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.318510:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.318562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:13.0:1619677447.318573:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.318574:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:13.0:1619677447.318575:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.318576:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.318578:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.318579:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.318580:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.318591:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.318594:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.318595:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.318595:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.318596:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.318597:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.318602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.318609:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.318612:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.318613:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.318616:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.318619:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.318759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677447.318767:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.318771:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.318772:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.318774:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.318776:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.318778:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.318779:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.318781:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.318782:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.318783:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.318787:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.318789:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.318791:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.318792:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.318792:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.318794:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.318794:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.318795:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.318796:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:4.2:1619677447.318802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:12.0:1619677447.318809:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.318811:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.318812:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.318813:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.318815:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.318817:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.318930:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677447.318964:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.318967:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.318968:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.318968:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.318970:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.318972:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.318981:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.318984:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.318984:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.318985:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.318985:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.318986:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.318986:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.318987:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.318987:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.318989:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.318992:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.318994:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.318995:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.318996:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.318997:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.318998:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.319002:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.319004:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.319008:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.319008:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.319009:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.319011:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.319012:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.319012:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.319013:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.319039:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.319041:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.319042:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.319042:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.319044:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.319047:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.319060:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.319063:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.319064:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.319064:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.319065:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.319065:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.319091:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.319094:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.319094:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.319096:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.319097:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.319100:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.319297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.319304:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.319306:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.319307:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.319311:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.319313:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.319315:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.319316:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.319318:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.319320:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.319321:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.319326:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.319328:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.319331:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.319331:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.319332:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.319334:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.319335:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.319336:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.319337:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.319338:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.319347:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.319349:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.319350:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.319350:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:12.0:1619677447.319352:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.319352:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.319352:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.319354:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.319354:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.319355:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.319373:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.319376:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.319379:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.319382:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.319383:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.319385:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.319386:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.319387:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.319388:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.319392:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.319394:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.319396:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.319397:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.319397:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.319399:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.319400:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.319400:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.319401:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:4.2:1619677447.319658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.319704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.319711:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.319713:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.319714:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.319716:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.319718:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.319720:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.319721:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729916] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.319723:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.319724:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.319725:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.319728:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.319730:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.319732:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.319733:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.319734:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.319735:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.319736:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.319737:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.319737:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677447.319782:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:12.0:1619677447.319817:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.319820:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.319821:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.319822:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.319823:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.319825:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.319834:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.319837:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.319838:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.319839:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.319840:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.319841:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.319867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.319877:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.319879:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.319880:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.319883:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.319886:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.319887:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677447.319888:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.319919:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.319921:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.319921:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.319922:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:4.2:1619677447.320170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.320229:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.320232:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.320233:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.320234:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.320236:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.320237:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.320270:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:13.0:1619677447.320307:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.320309:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.320310:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.320312:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.320314:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.320315:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.320477:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.320479:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.320480:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.320481:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.320483:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.320484:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.320534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.320560:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.320563:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677447.320566:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.320569:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.320570:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.320571:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.320572:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677447.320573:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.320574:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.320575:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:4.2:1619677447.320575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677447.320575:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:22.0:1619677447.320577:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.320578:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.320579:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:2.0:1619677447.320579:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.320581:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.320582:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.320582:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.320582:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:22.0:1619677447.320584:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.320584:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:2.0:1619677447.320585:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677447.320586:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96729904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.320586:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.320586:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:6.0:1619677447.320587:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.320588:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.320588:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:6.0:1619677447.320589:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.320589:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.320589:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.320591:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.320591:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:22.0:1619677447.320593:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.320593:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.320594:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.320595:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.320598:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.320599:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.320599:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.320601:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.320602:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.320602:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.320603:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.320605:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.320607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.320619:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.320727:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.320730:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.320734:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.320737:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.320739:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.320741:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.320742:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.320743:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.320744:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.320747:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.320749:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.320752:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.320753:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.320753:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.320755:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.320755:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.320756:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.320756:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.320875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.320886:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:9.2:1619677447.320892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.320899:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.320902:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.320903:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.320904:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.320905:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.320928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.320936:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.320938:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.320939:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.320941:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.320944:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.320945:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.320947:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.320948:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.320949:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.320950:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:22.0:1619677447.320951:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000400:00000200:8.0:1619677447.320953:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.320954:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677447.320954:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:8.0:1619677447.320955:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:22.0:1619677447.320957:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.320958:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677447.320959:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.320959:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.320960:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:22.0:1619677447.320961:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:22.0:1619677447.320962:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.320962:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.320962:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.320963:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.320963:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677447.320964:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.320965:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.320966:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:22.0:1619677447.320969:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.320971:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:22.0:1619677447.320974:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.320974:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:22.0:1619677447.320975:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:22.0:1619677447.320977:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:22.0:1619677447.320977:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677447.320978:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677447.320979:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677447.320983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.320989:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.320992:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.320993:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.320993:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.320995:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.320998:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.321069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.321109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.321121:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:9.2:1619677447.321213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.321219:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.321222:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.321223:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.321223:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.321225:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.321227:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.321238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.321244:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.321247:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.321247:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.321248:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.321248:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.321249:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.321405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.321413:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.321416:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.321417:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.321419:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.321421:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:6.0:1619677447.321444:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.321446:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.321447:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.321449:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.321452:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.321454:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.321455:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.321457:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.321458:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.321459:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.321463:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.321464:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.321467:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.321467:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.321468:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.321470:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.321470:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.321471:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.321471:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.321476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.321487:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:9.2:1619677447.321700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.321707:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:4.2:1619677447.321707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677447.321710:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.321711:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.321712:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.321714:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.321717:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677447.321751:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.321754:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.321755:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.321756:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.321758:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.321761:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.321808:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.321811:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:24.0:1619677447.321814:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.321816:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.321817:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.321819:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.321820:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.321822:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.321823:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.321825:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.321829:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.321831:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.321834:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.321835:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.321836:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.321838:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.321839:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.321839:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.321840:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.321845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.321853:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.321856:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.321857:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.321860:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.321863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.321867:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.321868:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.321871:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.321872:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.321873:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.321874:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.321880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.321888:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.321890:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.321891:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.321893:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.321896:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.321898:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.321899:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.321900:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.321902:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.321902:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.321906:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.321908:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.321911:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.321911:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.321912:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.321914:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.321914:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.321915:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.321916:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.322036:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.322038:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.322039:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.322040:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.322042:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.322043:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.322082:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.322085:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.322086:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.322088:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.322090:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.322091:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.322093:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.322094:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.322095:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.322096:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.322099:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.322101:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.322103:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.322104:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.322105:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.322107:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.322107:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.322108:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.322109:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.322110:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.322120:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.322121:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.322122:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.322123:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.322209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.322215:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677447.322266:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.322269:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.322270:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.322270:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.322272:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.322276:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.322306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.322314:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.322316:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.322317:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.322318:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.322320:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.322323:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.322328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:13.0:1619677447.322331:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.322334:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.322334:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.322334:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000400:00000200:13.0:1619677447.322336:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.322337:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.322337:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:12.0:1619677447.322338:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.322338:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:13.0:1619677447.322339:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.322339:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.322340:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677447.322381:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.322384:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.322385:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.322385:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.322387:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.322391:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.322575:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677447.322578:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677447.322579:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.322581:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.322582:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677447.322583:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677447.322584:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.322585:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.322587:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.322587:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.322588:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.322599:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.322600:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.322601:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.322601:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.322602:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.322603:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.322603:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.322604:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:12.0:1619677447.322604:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.322605:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.322606:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.322608:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.322618:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.322621:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.322621:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.322622:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.322623:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.322623:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.322749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.322757:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.322759:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.322761:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.322763:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.322766:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.322854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.322866:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.322868:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.322869:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.322871:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.322872:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.322874:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.322892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.322899:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.322902:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.322903:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.322904:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.322906:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.322909:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.322945:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.322948:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.322949:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.322951:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.322954:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.322956:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.322958:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.322959:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.322961:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.322962:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.322966:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.322968:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.322972:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.322972:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.322973:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.322975:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.322975:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.322976:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.322977:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.322979:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.322982:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.322982:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.322984:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.322986:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.322987:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.323036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.323043:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.323046:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.323047:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.323048:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.323051:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.323052:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.323054:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.323055:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.323056:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.323057:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.323061:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.323062:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.323065:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.323066:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.323067:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.323068:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.323069:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.323070:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.323071:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.323072:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.323077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.323085:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.323087:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:2.0:1619677447.323087:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:8.0:1619677447.323088:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.323089:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.323114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.323121:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.323124:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.323124:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.323126:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.323128:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.323366:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.323369:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.323370:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.323370:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.323372:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.323374:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.323394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.323411:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.323413:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.323414:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.323415:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.323417:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.323421:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.323429:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.323431:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.323432:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.323433:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.323434:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.323434:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.323772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.323783:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.323786:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.323787:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.323790:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.323793:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.323794:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.323796:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.323799:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.323801:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.323802:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:4.2:1619677447.323802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:24.0:1619677447.323806:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.323808:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.323811:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.323811:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.323812:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.323813:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.323813:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.323815:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.323815:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.323816:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.323816:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.323817:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677447.323818:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.323819:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.323820:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.323822:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96729855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.323823:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.323824:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.323825:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677447.323827:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677447.323828:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.323830:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677447.323830:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:13.0:1619677447.323831:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.323833:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.323833:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.323834:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.323834:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:13.0:1619677447.323835:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.323835:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.323836:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.323837:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.323838:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.323838:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.323839:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.323842:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.323842:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.323843:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.323846:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.323849:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.323861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.323873:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:12.0:1619677447.323933:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.323935:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.323936:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.323938:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.323940:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:9.2:1619677447.323940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:12.0:1619677447.323941:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.323945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:2.0:1619677447.323948:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:24.0:1619677447.323951:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:13.0:1619677447.323951:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.323951:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.323952:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.323954:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:13.0:1619677447.323954:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.323955:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.323955:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677447.323955:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:2.0:1619677447.323955:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.323956:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.323957:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:9.2:1619677447.323957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:24.0:1619677447.323958:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.323958:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.323964:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:24.0:1619677447.323966:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:2.0:1619677447.323966:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.323968:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.323969:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.323970:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.323971:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:12.0:1619677447.323983:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.323985:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.323986:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.323988:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.323989:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.323996:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.324020:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.324023:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.324024:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.324024:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.324027:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.324038:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.324067:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.324070:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.324071:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.324074:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.324077:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.324079:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.324081:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96729838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.324083:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.324084:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.324086:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.324091:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.324093:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.324097:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.324097:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.324098:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.324100:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.324101:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.324102:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.324102:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.324147:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.324149:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.324150:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.324152:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.324153:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.324162:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.324195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677447.324345:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.2:1619677447.324346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.324348:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.324350:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.324351:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.324352:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.324353:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.324354:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.324354:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.324356:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.324356:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.324358:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.324359:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.324361:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.324362:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.324364:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.324365:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:8.0:1619677447.324366:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.324368:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.324368:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.0:1619677447.324371:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.324371:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.324372:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.324374:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.324374:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.324375:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.324375:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.324386:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.324389:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.324389:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.324391:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.324393:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.324394:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.324605:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.324607:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.324608:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.324610:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.324610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:12.0:1619677447.324611:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.324612:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.324618:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.324621:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.324621:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.324622:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677447.324625:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.324625:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.324628:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.324628:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.324630:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.324632:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.324634:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.324635:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.324636:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.324638:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.324640:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96729824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.324641:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.324642:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.324643:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.324643:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.324644:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.324648:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677447.324648:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.324649:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.324650:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.324650:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.324652:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.324652:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.324653:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (23)++
00000800:00000200:24.0:1619677447.324654:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (24)--
00000800:00000200:24.0:1619677447.324654:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.324654:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.324654:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.324656:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.324656:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677447.324661:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.324661:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.324662:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.324815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.324842:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.324845:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677447.324848:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.324851:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.324852:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.324854:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96729820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.324855:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.324856:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.324858:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.324861:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.324863:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.324867:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.324868:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.324869:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:4.2:1619677447.324871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.324871:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.324871:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.324872:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.324873:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:6.0:1619677447.324876:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.324879:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.324879:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.324880:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.324882:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.324883:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.324885:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.324886:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.324886:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:12.0:1619677447.324888:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.324889:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.324891:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.324902:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.324905:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.324906:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.324906:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.324907:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.324908:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.324924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.324931:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.324933:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.324935:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.324935:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.324937:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.324940:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:2.0:1619677447.325016:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.325047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.325055:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.325056:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.325057:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.325059:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.325062:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677447.325064:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.325067:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.325067:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.325069:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.325070:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.325071:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.325317:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.0:1619677447.325331:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.325333:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.325334:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.325336:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.325339:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.325341:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.325342:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.325343:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.325344:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.325345:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.325349:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.325351:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.325353:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677447.325353:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.325354:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:22.0:1619677447.325355:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:8.0:1619677447.325355:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677447.325356:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.325356:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677447.325356:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.325357:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:22.0:1619677447.325358:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:8.0:1619677447.325358:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.325359:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.325360:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677447.325361:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.325369:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.325371:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:24.0:1619677447.325372:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:13.0:1619677447.325372:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.325374:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:13.0:1619677447.325374:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.325374:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677447.325375:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.325376:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.325376:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.325377:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677447.325377:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.325378:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.325378:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:12.0:1619677447.325379:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:6.0:1619677447.325379:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000400:00000200:12.0:1619677447.325380:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.325380:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.325382:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.325382:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.325384:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.325486:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.325489:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.325492:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.325495:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.325496:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.325498:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.325499:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.325500:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.325501:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.325504:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.325506:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.325509:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.325510:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.325511:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.325513:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.325513:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.325514:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.325514:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677447.325525:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.325528:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.325529:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.325530:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.325532:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.325533:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.325573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:9.2:1619677447.325580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677447.325581:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.325584:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.325585:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.325587:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.325589:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.325591:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:2.0:1619677447.325591:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:8.0:1619677447.325593:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.325596:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.325597:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.325598:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.325601:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.325603:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.325606:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.325607:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.325607:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.325609:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.325610:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.325610:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.325611:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.325612:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.325617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.325624:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.325626:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.325627:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.325628:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.326054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.326074:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.326077:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.326077:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.326081:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.326084:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.326086:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.326087:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729805] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.326088:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.326089:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.326090:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.326094:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.326096:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.326099:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.326100:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.326101:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.326102:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.326102:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.326103:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.326103:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.326104:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.326104:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.326105:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.326106:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.326106:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.326109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.326109:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.326117:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677447.326119:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677447.326120:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.326121:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.326122:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.326123:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.326123:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:2.0:1619677447.326123:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.326124:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.326125:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.326126:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:6.0:1619677447.326173:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.326176:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.326176:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.326177:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.326179:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.326181:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.326219:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.326222:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.326222:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.326224:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.326226:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.326226:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.326270:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:13.0:1619677447.326316:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.326319:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.326320:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.326321:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.326323:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.326324:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.326334:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.326337:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.326338:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.326339:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.326340:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.326341:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.326523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.326531:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.326533:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.326534:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.326536:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.326539:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.326540:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.326541:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.326543:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.326543:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:8.0:1619677447.326544:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.326545:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.326546:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:8.0:1619677447.326550:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.326550:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.326552:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.326553:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.326554:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:8.0:1619677447.326555:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.326556:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.326556:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677447.326556:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.326557:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.326558:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:2.0:1619677447.326558:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.326559:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.326559:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677447.326559:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.326560:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677447.326562:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.326564:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.326567:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.326568:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.326568:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.326570:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.326570:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.326571:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.326572:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:22.0:1619677447.326583:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.326586:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.326587:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.326588:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.326590:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.326592:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.326632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.326643:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:4.2:1619677447.326800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.326841:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.326844:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.326845:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.326845:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.326847:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.326850:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.327057:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.327060:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.327061:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.327061:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.327063:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.327066:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.327113:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.327115:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.327116:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.327118:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.327119:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.327120:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.327131:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.327134:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.327134:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.327135:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.327136:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.327137:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.327347:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.327350:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.327351:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.327353:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.327354:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.327355:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.327391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.327440:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.327440:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.327441:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.327442:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.327443:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.327444:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.327445:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.327448:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.327448:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.327449:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:6.0:1619677447.327449:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.327450:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:22.0:1619677447.327451:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.327452:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.327454:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.327455:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.327458:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.327459:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.327462:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.327463:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.327463:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.327465:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.327466:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.327467:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.327468:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.327468:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.327471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.327479:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.327481:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.327481:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.327482:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.327482:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.327484:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.327486:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.327488:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.327490:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.327491:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.327493:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.327494:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.327495:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.327496:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.327497:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.327498:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.327521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.327528:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677447.327530:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:12.0:1619677447.327531:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.327531:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.327531:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.327532:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.327533:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.327534:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.327535:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.327536:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.327537:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.327539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.327550:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:4.2:1619677447.327625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.327627:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.327630:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.327631:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.327633:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.327634:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.327635:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.327666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:12.0:1619677447.327710:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.327713:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.327713:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.327715:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.327716:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.327718:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.327879:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.327881:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.327882:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.327884:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.327885:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:4.2:1619677447.327886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.327887:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.327955:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.327957:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.327958:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.327958:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.327961:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.327963:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.327990:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.327992:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677447.327995:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.327997:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.327998:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.328000:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.328001:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.328002:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.328003:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.328006:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.328008:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.328009:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.328010:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.328011:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.328012:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.328013:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.328014:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.328015:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.328015:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:9.2:1619677447.328018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.328028:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.328029:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.328029:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.328030:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.328031:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677447.328116:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.328118:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.328119:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.328120:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.328122:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.328123:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.328134:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.328137:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:6.0:1619677447.328137:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:12.0:1619677447.328138:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.328139:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.328139:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.328140:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.328140:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.328141:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.328143:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.328145:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.328147:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.328148:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.328150:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.328151:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.328152:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.328155:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.328157:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.328159:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.328160:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.328161:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.328162:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.328163:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.328164:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.328164:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.328185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.328222:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.328225:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.328225:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.328226:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.328228:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.328231:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.328233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.328241:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.328243:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:6.0:1619677447.328243:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:8.0:1619677447.328245:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677447.328246:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000400:00000200:24.0:1619677447.328247:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.328247:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.328247:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.328249:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.328249:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:24.0:1619677447.328250:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.328251:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.328253:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.328254:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.328255:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.328256:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.328260:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.328262:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.328264:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.328265:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.328266:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.328267:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.328268:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.328268:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.328269:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.328298:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.328301:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.328302:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.328304:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.328306:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.328307:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.328449:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.328452:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.328453:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.328455:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.328458:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.328460:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.328461:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.328462:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.328463:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.328464:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.328468:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.328469:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.328472:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.328473:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.328474:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.328476:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.328476:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.328477:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.328477:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.328482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.328488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.328491:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.328494:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.328495:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.328497:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.328500:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.328533:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.328535:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.328536:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.328537:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.328539:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.328542:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.328744:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.328746:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.328747:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.328749:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.328751:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.328753:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.328754:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.328756:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.328758:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.328759:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:6.0:1619677447.328759:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.328761:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.328762:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.328763:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.328764:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.328765:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.328767:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.328769:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.328769:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.328770:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.328771:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.328773:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.328773:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.328774:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.328775:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.328833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.328874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677447.328929:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.328932:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.328933:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.328935:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.328936:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.328938:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.328948:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.328950:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.328951:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.328952:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.328953:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.328954:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.329184:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.329209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.329216:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.329218:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.329218:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.329220:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.329221:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.329270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.329277:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.329280:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.329281:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.329283:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.329286:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.329288:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.329289:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.329290:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.329291:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.329292:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.329301:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.329303:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.329305:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.329306:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.329307:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.329309:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.329309:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.329310:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.329310:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.329419:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.329422:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.329423:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.329424:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.329426:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.329428:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.329441:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.329443:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.329444:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.329444:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.329445:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.329446:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.329532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.329579:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.329582:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.329583:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.329583:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.329585:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.329593:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.329632:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:13.0:1619677447.329667:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.329669:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.329670:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.329672:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.329674:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.329675:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.329813:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.329816:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.329817:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.329819:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.329822:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.329823:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.329825:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.329826:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.329827:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.329828:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.329831:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.329833:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.329836:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.329837:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.329838:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.329840:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.329841:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.329842:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.329842:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.329942:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.329945:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:4.2:1619677447.329948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:2.0:1619677447.329948:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.329951:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.329952:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.329954:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.329955:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.329956:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:2.0:1619677447.329956:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.329957:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.329959:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.329960:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677447.329961:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.329963:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.329963:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.329965:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677447.329966:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.329966:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.329966:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677447.329967:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:2.0:1619677447.329968:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.329968:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:8.0:1619677447.329969:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.329969:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:8.0:1619677447.329970:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.329970:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000400:00000200:8.0:1619677447.329971:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.329971:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:8.0:1619677447.329972:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:9.2:1619677447.329974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677447.329976:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.329977:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.329980:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.329982:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.329982:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.329983:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:8.0:1619677447.329984:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.329984:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677447.329984:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.329985:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.0:1619677447.329986:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677447.329986:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.329987:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:2.0:1619677447.329987:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.0:1619677447.329988:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.329993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.2:1619677447.329997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.330000:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.330002:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.330003:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.330004:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677447.330004:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.330007:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.330008:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.330009:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.330010:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.0:1619677447.330187:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.330190:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.330191:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.330192:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.330195:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.330197:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.330199:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.330200:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.330201:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.330203:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.330206:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.330208:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.330211:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.330212:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.330212:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.330214:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.330215:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.330215:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.330216:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.330217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.330229:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:12.0:1619677447.330306:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.330308:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.330309:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.330311:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.330313:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.330314:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.330330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.330338:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.330341:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.330342:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.330344:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.330348:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.330358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.330402:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.330405:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.330406:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.330406:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.330409:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.330411:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.330719:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.330721:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.330722:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.330723:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.330725:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.330726:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.330769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:2.0:1619677447.330770:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:24.0:1619677447.330777:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.330779:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.330781:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.330784:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.330787:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.330789:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.330790:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.330792:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.330792:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:24.0:1619677447.330793:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.330794:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.330794:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.330797:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.330800:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.330801:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.330802:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.330803:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:24.0:1619677447.330805:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677447.330805:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96729738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.330806:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.330807:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677447.330807:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.330808:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:2.0:1619677447.330808:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.330809:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.330809:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:2.0:1619677447.330809:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.330810:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.330811:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.330811:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.330812:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.330812:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677447.330814:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.330815:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.330815:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677447.330816:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.330818:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.330819:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.330820:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677447.330822:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:2.0:1619677447.330822:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.330823:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.330824:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.331197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.331204:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.331206:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.331207:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.331207:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.331209:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.331212:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:2.0:1619677447.331252:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:4.2:1619677447.331253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.331318:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.331321:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.331322:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.331322:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.331325:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.331327:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.331454:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.331458:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.331459:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.331460:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.331464:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.331466:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.331467:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.331468:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.331469:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.331471:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.331474:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.331476:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.331478:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.331479:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.331480:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.331482:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.331482:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.331483:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.331483:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:2.0:1619677447.331648:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.331652:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.331655:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.331658:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.331659:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.331660:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.331662:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.331663:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.331664:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.331667:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.331669:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.331671:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.331672:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.331673:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.331674:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.331675:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.331675:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.331676:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677447.331730:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.331732:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.331733:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.331734:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.331737:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.331738:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.331748:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.331750:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.331750:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.331752:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.331753:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.331754:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.331772:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.331775:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.331776:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.331776:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.331777:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.331779:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.331799:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.331802:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.331802:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.331804:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.331805:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.331807:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.331899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.331907:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.331910:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.331911:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.331913:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.331916:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.331917:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.331919:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.331920:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.331921:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.331922:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.331925:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.331927:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.331929:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.331930:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.331931:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.331932:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.331933:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.331934:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.331934:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.331974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.331986:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:4.2:1619677447.332170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.332205:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.332207:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.332208:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.332209:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.332210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.332211:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.332212:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.332213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.332218:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.332221:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.332222:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.332224:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.332227:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.332230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:12.0:1619677447.332231:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.332233:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.332234:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.332235:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.332236:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.332237:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.332242:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677447.332310:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.332313:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.332313:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.332315:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.332317:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.332318:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.332474:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.332476:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.332477:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.332478:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.332480:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.332481:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.332635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.332643:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.332645:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.332646:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.332648:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.332649:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.332650:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.332682:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.332685:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.332686:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.332686:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.332688:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.332691:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.332700:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.332703:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.332704:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.332706:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.332708:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.332710:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.332711:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.332711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:6.0:1619677447.332712:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.332713:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.332714:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:24.0:1619677447.332716:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:6.0:1619677447.332718:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.332719:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.332719:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677447.332720:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.0:1619677447.332721:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677447.332722:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.332722:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677447.332723:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.332723:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.332724:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:24.0:1619677447.332726:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.332726:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.332726:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.332726:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.332727:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:24.0:1619677447.332728:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:6.0:1619677447.332728:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000400:00000200:24.0:1619677447.332729:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.332729:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:24.0:1619677447.332731:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.332732:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.332733:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.332737:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.332738:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96729714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.332739:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.332739:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:8.0:1619677447.332739:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.332741:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.332741:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.332741:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:8.0:1619677447.332741:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.332742:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:8.0:1619677447.332742:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.332746:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677447.332746:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.332747:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.332748:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.332748:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.332750:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.332751:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.332751:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.332752:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.332753:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (23)++
00000800:00000200:8.0:1619677447.332753:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.332754:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (24)--
00000800:00000200:24.0:1619677447.332755:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.332755:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.332756:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.332756:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.332757:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (23)--
00000800:00000200:9.2:1619677447.332757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.332757:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.332758:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677447.332765:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677447.332767:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:2.0:1619677447.332768:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:24.0:1619677447.332769:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:2.0:1619677447.332769:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.332770:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.332771:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:2.0:1619677447.332771:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.332774:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.332972:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.332974:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.332976:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.332976:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.332978:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.332980:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.332989:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.332992:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.332993:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.332994:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.332995:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.332996:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.333002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.333016:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.333019:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.333020:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.333021:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.333023:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.333024:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.333040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.333053:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677447.333406:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.333409:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.333409:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.333411:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.333414:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.333415:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.333417:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.333418:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.333419:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.333420:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.333423:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.333425:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.333427:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.333428:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.333429:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.333431:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.333431:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.333432:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.333432:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.333437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.333444:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.333447:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.333448:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.333450:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.333453:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:13.0:1619677447.333476:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:9.2:1619677447.333476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:13.0:1619677447.333478:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.333479:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.333481:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.333483:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:12.0:1619677447.333484:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.333485:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.333485:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.333486:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.333487:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.333489:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.333492:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677447.333494:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.333497:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.333498:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.333498:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.333499:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.333500:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.333546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.333619:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:9.2:1619677447.333650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.333657:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.333659:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.333660:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.333663:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.333666:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677447.333867:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:22.0:1619677447.333901:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.333904:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.333905:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.333905:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.333907:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.333909:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.333920:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.333922:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.333923:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.333923:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.333924:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.333925:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.334012:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.334015:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.334017:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.334020:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.334021:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.334023:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.334024:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.334025:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.334026:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:4.2:1619677447.334027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677447.334028:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.334029:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.334031:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.334031:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.334032:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.334033:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.334034:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.334035:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.334035:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.334039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.334051:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:9.2:1619677447.334074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:4.2:1619677447.334076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.334081:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677447.334082:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677447.334084:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.334084:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.334085:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.334085:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:22.0:1619677447.334087:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.334087:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.334089:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.334090:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.334092:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:6.0:1619677447.334096:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.334099:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.334100:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.334102:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.334104:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.334106:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.334107:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.334109:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.334109:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.334111:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.334114:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.334116:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.334118:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.334119:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.334120:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.334122:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.334122:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.334123:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.334123:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.334314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.334321:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.334324:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.334325:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.334325:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.334327:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.334329:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677447.334436:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.334439:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.334440:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.334441:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.334444:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.334445:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.334446:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.334448:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.334449:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.334450:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.334453:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.334454:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.334457:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.334458:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.334458:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.334460:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.334460:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.334460:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.334461:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.334473:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.334475:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.334476:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.334477:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.334480:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:4.2:1619677447.334480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:13.0:1619677447.334481:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.334487:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.334490:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.334491:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.334492:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.334495:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.334497:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.334498:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.334499:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.334500:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.334501:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.334504:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.334506:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.334508:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.334509:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.334510:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.334511:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.334512:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.334513:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.334514:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.334515:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.334520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.334527:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.334530:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.334530:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.334532:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.334620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.334769:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.334771:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.334773:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.334774:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.334777:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.334779:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.334780:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.334782:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.334783:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.334784:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.334788:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.334789:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.334792:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.334793:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.334794:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.334796:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.334796:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.334797:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.334798:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.334802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.334814:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:9.2:1619677447.334821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.334828:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.334831:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.334832:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.334833:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.334834:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.334837:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.334840:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.334841:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.334841:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.334843:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.334846:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.334937:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.334962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.334970:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.334971:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.334972:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.334974:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677447.334974:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.334975:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.334977:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.334978:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.334980:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.334982:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.334983:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.334984:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.334986:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.334987:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.334988:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.334991:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.334993:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.334996:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.334997:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.334997:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.334999:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.334999:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.335000:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.335000:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.335183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.335227:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.335230:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.335231:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.335232:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.335234:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.335236:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.335301:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.335304:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677447.335306:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.335309:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.335310:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.335311:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.335313:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.335313:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.335314:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.335317:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.335319:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.335321:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.335322:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.335323:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.335325:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.335325:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.335326:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.335327:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.335328:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:6.0:1619677447.335340:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.335342:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.335343:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.335343:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.335345:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.335348:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.335381:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:4.2:1619677447.335382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677447.335383:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.335384:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.335385:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.335387:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.335390:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.335392:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.335393:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.335394:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677447.335394:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.335396:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677447.335396:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.335397:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.335398:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.335399:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.335399:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:8.0:1619677447.335399:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.335401:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.335402:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.335403:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.335404:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.335405:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.335408:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.335410:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.335413:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.335414:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.335414:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.335416:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.335416:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.335417:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.335417:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.335825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:12.0:1619677447.335828:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.335830:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.335831:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.335832:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.335833:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:12.0:1619677447.335834:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.335835:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.335836:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.335837:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.335840:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.335842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.335845:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.335847:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.335849:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.335850:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.335851:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.335852:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677447.335879:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:12.0:1619677447.335955:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.335957:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.335958:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.335960:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.335961:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.335963:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.336003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.336037:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.336040:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.336041:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.336043:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.336044:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.336045:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.336052:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.336055:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.336055:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.336056:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.336058:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.336061:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.336155:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.336158:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:4.2:1619677447.336159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:2.0:1619677447.336161:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.336163:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.336165:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:24.0:1619677447.336166:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:2.0:1619677447.336166:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.336168:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.336169:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:2.0:1619677447.336169:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.336170:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677447.336170:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.336172:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.336174:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.336175:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.336176:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:2.0:1619677447.336176:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.336178:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.336179:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.336179:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.336180:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:24.0:1619677447.336181:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.336181:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677447.336182:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:2.0:1619677447.336182:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.336183:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.336183:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.336184:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:24.0:1619677447.336186:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.336188:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.336190:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.336191:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.336192:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.336194:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.336194:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:12.0:1619677447.336194:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677447.336195:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.336196:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.336197:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.336198:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.336200:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.336201:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.336203:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.336216:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.336219:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.336220:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.336220:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.336221:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.336223:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.336370:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:22.0:1619677447.336373:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000400:00000200:22.0:1619677447.336374:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.336375:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:6.0:1619677447.336375:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000400:00000200:22.0:1619677447.336376:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.336376:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.336378:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.336380:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.336380:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.336381:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.336427:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.336430:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.336431:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.336432:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.336434:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.336435:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.336526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.336565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:9.2:1619677447.336727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.336738:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.336741:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.336742:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.336743:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.336746:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.336749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.336752:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.336753:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:9.2:1619677447.336755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.336756:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.336757:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.336759:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.336759:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.336760:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:2.0:1619677447.336767:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.336788:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.336790:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.336791:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.336793:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.336795:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:24.0:1619677447.336796:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.336797:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:6.0:1619677447.336797:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:24.0:1619677447.336799:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.336799:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.336799:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.336800:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.336802:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.336802:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.336803:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:6.0:1619677447.336803:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:24.0:1619677447.336807:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.336809:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.336812:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.336813:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.336813:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.336815:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.336816:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.336816:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.336817:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.336819:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.336821:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.336822:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.336822:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.336823:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.336824:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.336958:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.336961:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677447.336964:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.336966:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.336967:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677447.336968:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.336970:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.336971:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.336972:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677447.336975:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.336977:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677447.336979:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.336980:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677447.336981:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677447.336982:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677447.336983:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.336984:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.336984:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677447.336996:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.336998:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.337000:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.337000:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.337002:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.337004:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.337008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.337015:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.337018:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.337019:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.337022:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.337025:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.337026:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.337027:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.337029:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.337029:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.337030:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.337034:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.337035:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.337038:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.337039:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.337039:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.337041:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.337041:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:12.0:1619677447.337042:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677447.337042:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.337042:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.337045:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.337045:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.337047:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.337049:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.337050:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.337121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.337194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.337205:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.337488:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.337491:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.337492:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.337495:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.337498:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.337500:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.337502:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.337504:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.337506:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.337508:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.337518:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.337520:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.337523:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.337524:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.337524:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (22)++
00000800:00000200:24.0:1619677447.337526:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (23)--
00000800:00000200:24.0:1619677447.337527:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.337529:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.337530:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.337531:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.337533:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.337536:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.337536:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.337538:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.337540:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.337541:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.337543:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.337546:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.337547:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677447.337547:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.337618:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.337620:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.337621:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.337623:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.337624:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.337624:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.337636:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.337639:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.337640:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.337642:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.337644:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.337646:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.337647:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.337649:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.337650:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.337651:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.337654:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.337656:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.337659:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.337661:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.337661:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.337663:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.337664:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.337665:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.337666:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.337667:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.337677:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.337679:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.337680:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.337681:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.337944:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39ea00] (21)++
00000800:00000200:24.0:1619677447.337947:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39ea00] (22)--
00000800:00000200:8.0:1619677447.337947:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.17@o2ib3
00000400:00000200:8.0:1619677447.337949:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.5.17@o2ib3 : GET - for me
00000400:00000200:8.0:1619677447.337952:0:19939:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.5.17@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:8.0:1619677447.337954:0:19939:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.5.17@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:8.0:1619677447.337956:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39ea00] (21)++
00000800:00000200:8.0:1619677447.337957:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39ea00] (22)--
00000400:00000200:8.0:1619677447.337958:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.5.17@o2ib3 local destination
00000400:00000200:8.0:1619677447.337962:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.5.17@o2ib3(10.51.5.17@o2ib3:10.51.5.17@o2ib3) : REPLY try# 0
00000800:00000200:8.0:1619677447.337964:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.5.17@o2ib3
00000800:00000200:8.0:1619677447.337966:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919fce371780] -> 10.51.5.17@o2ib3 (2) version: 12
00000800:00000200:8.0:1619677447.337967:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919fee39ea00] (21)++
00000800:00000200:8.0:1619677447.337967:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39ea00] (22)++
00000800:00000200:8.0:1619677447.337969:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919fee39ea00] (23)--
00000800:00000200:8.0:1619677447.337969:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39ea00] (22)--
00000800:00000200:24.0:1619677447.337982:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39ea00] (22)++
00000800:00000200:24.0:1619677447.337984:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39ea00] (23)--
00000400:00000200:24.0:1619677447.337985:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.337986:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39ea00] (22)--
00000400:00000200:24.0:1619677447.337987:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.5.17@o2ib3: REPLY: OK
00000800:00000200:24.0:1619677447.337989:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39ea00] (21)--
00000800:00000200:2.0:1619677447.337990:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.337993:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.337996:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.337999:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.338000:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.338001:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.338003:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.338004:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.338005:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.338008:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.338010:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.338012:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.338013:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.338014:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.338015:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.338016:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.338017:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.338017:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:4.2:1619677447.338095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.338100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.338107:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.338109:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.338110:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.338112:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.338115:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.338117:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.338118:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.338120:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.338121:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.338122:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.338126:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.338128:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.338131:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.338132:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.338133:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.338134:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.338135:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.338145:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.338146:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.338147:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.338153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.338160:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.338163:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:13.0:1619677447.338163:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677447.338164:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.338165:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.338165:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.338166:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.338168:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.338169:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.338175:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.338176:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.338178:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.338179:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.338179:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.338180:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.338181:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.338287:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:22.0:1619677447.338485:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.338487:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.338488:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.338490:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.338493:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.338495:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.338496:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.338497:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.338498:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.338500:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.338503:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.338505:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.338508:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.338509:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.338509:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.338511:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.338511:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.338512:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.338513:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.338638:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.338640:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.338641:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.338643:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.338645:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.338646:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.338908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.338961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:2.0:1619677447.338963:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.338965:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677447.338968:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:24.0:1619677447.338969:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:2.0:1619677447.338971:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.338972:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:2.0:1619677447.338972:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:8.0:1619677447.338973:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677447.338974:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.338976:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.338976:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.338977:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.338978:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677447.338980:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.338982:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.338982:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000400:00000200:24.0:1619677447.338984:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.338984:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:24.0:1619677447.338985:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.338986:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.338987:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.338987:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:24.0:1619677447.338988:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.338988:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.338989:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.338990:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.338991:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:24.0:1619677447.338992:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.338992:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.338993:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:24.0:1619677447.338994:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.2:1619677447.338995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:24.0:1619677447.338996:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.338997:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.338998:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.338999:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.339000:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.339000:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.339001:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677447.339005:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.339006:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.339006:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.339007:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.339008:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677447.339167:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.339170:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.339171:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.339171:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.339173:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.339176:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.339196:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.339199:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.339201:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.339204:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.339206:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.339207:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.339208:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.339209:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.339210:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.339214:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.339216:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.339218:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.339219:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.339220:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.339221:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.339222:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.339222:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.339223:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:22.0:1619677447.339356:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.339359:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.339359:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.339362:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.339365:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.339367:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.339368:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.339370:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.339371:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.339372:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.339376:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.339378:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.339380:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.339381:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.339382:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.339384:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.339384:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.339385:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.339386:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.339458:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677447.339480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.339488:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.339490:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.339491:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.339492:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.339494:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.339497:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.339498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677447.339504:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.339506:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677447.339507:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.339508:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.339509:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.339509:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.339541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.339585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.339588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.339593:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.339596:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.339597:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.339599:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.339603:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.339605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.339616:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:4.2:1619677447.339966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.340010:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.340011:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.340012:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.340013:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.340013:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.340016:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.340016:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.340019:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.340020:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.340021:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:22.0:1619677447.340022:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:6.0:1619677447.340022:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.340023:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.340026:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.340027:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.340028:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.340033:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.340035:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.340038:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.340039:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.340040:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.340041:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.340042:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.340043:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.340044:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.340048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.340055:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.340058:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.340059:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.340061:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.340063:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.340104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.340111:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.340114:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.340115:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.340116:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.340118:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.340120:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:2.0:1619677447.340133:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:12.0:1619677447.340169:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.340170:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.340171:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:12.0:1619677447.340172:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677447.340172:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:12.0:1619677447.340173:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.340173:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:12.0:1619677447.340174:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.340176:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.340177:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.340217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:13.0:1619677447.340227:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.340229:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677447.340229:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:13.0:1619677447.340230:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.340233:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.340233:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.340236:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.340250:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.340253:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.340253:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.340254:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.340256:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.340259:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.340283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.340290:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.340293:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.340294:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.340302:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.340304:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.340306:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.340307:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.340309:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.340310:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.340311:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.340314:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.340316:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.340318:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.340319:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.340320:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.340321:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.340322:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.340322:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.340323:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.340375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.340416:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.340418:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.340419:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.340420:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.340422:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.340424:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.340582:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.340585:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.340588:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.340591:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.340592:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.340593:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.340595:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.340596:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.340597:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.340600:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.340602:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.340605:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.340606:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.340607:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.340609:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.340609:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.340610:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.340611:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.340612:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:12.0:1619677447.340622:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.340624:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.340626:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.340627:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.340629:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.340630:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.340640:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.340643:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.340644:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.340645:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.340645:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.340646:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.340695:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.340697:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.340698:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.340700:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.340702:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.340704:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.340705:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.340706:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.340707:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.340708:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.340711:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.340713:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.340715:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.340716:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.340717:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.340719:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.340719:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.340720:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.340721:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.340722:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.340732:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.340734:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.340735:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.340736:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.340815:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.340817:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.340818:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.340820:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.340823:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.340824:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.340826:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.340828:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.340829:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.340830:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.340834:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.340836:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:4.2:1619677447.340837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677447.340838:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.340839:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.340840:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.340841:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.340842:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.340843:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.340843:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.340849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.340855:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.340857:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677447.340857:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:12.0:1619677447.340858:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.340859:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.340860:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:13.0:1619677447.340861:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677447.340861:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.340863:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677447.340863:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.340866:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.340886:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.340889:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.340889:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.340890:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.340892:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.340895:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.340909:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.340912:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.340912:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.340913:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.340914:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.340915:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.340960:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.340962:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.340963:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.340965:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.340966:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.340967:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.341274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.341300:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.341303:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.341304:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.341306:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.341309:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.341310:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.341311:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.341313:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.341314:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.341315:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.341319:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.341321:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.341323:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.341324:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.341325:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:4.2:1619677447.341325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677447.341326:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.341327:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.341327:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.341328:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.341331:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.341334:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.341335:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.341335:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.341338:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.341339:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677447.341341:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.341342:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.341342:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.341344:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.341346:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.341347:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:4.2:1619677447.341348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.341355:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.341356:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:8.0:1619677447.341357:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.341358:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677447.341359:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.341359:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.341360:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:12.0:1619677447.341360:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677447.341360:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.341361:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.341362:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.341362:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.341363:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.341363:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.341365:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.341365:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:13.0:1619677447.341367:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:8.0:1619677447.341367:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.341368:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677447.341368:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.341369:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.341370:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.341374:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.341376:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677447.341379:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.341379:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.341379:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.341380:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:13.0:1619677447.341381:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.341382:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677447.341382:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.341382:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677447.341383:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677447.341383:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677447.341384:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.341384:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.341385:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.341514:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:12.0:1619677447.341550:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.341552:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.341553:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.341555:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.341556:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.341557:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.341632:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.341635:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677447.341638:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.341641:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.341642:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677447.341644:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.341645:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.341646:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.341647:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677447.341651:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.341652:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677447.341655:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.341655:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677447.341656:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677447.341658:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677447.341658:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.341659:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.341659:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677447.341671:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.341674:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.341675:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.341675:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.341677:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.341679:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.341713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.341760:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.341762:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.341763:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.341764:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.341766:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.341769:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.341894:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.341896:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.341897:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.341899:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.341900:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.341902:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.342069:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.342072:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.342073:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.342075:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.342078:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.342079:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.342080:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.342082:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.342083:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.342084:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.342087:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.342089:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.342092:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.342093:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.342094:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.342096:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.342096:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.342097:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.342097:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.342102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.342114:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:9.2:1619677447.342204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.342211:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.342213:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.342215:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.342216:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.342219:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677447.342319:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.342321:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.342322:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000400:00000200:12.0:1619677447.342322:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.342323:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.342324:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:4.2:1619677447.342324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:13.0:1619677447.342326:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.342326:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.342327:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:13.0:1619677447.342328:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.342329:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677447.342330:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677447.342331:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96729583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.342333:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677447.342333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:13.0:1619677447.342334:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677447.342335:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:13.0:1619677447.342340:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.342340:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677447.342342:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677447.342343:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.342344:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677447.342345:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.342346:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:8.0:1619677447.342346:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.342347:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:13.0:1619677447.342349:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000400:00000200:8.0:1619677447.342349:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.342350:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677447.342351:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000400:00000200:8.0:1619677447.342351:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.342352:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96729581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.342353:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677447.342353:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:8.0:1619677447.342353:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.342354:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677447.342355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:8.0:1619677447.342355:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.342358:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.342360:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.342363:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.342363:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.342364:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:12.0:1619677447.342366:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:8.0:1619677447.342366:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.342366:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.342367:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677447.342367:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:12.0:1619677447.342368:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:8.0:1619677447.342368:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:12.0:1619677447.342369:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:8.0:1619677447.342369:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:12.0:1619677447.342370:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:4.2:1619677447.342374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.2:1619677447.342375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:24.0:1619677447.342381:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.342383:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:2.0:1619677447.342383:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:24.0:1619677447.342384:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.342385:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677447.342386:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.342387:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.342388:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.342389:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:12.0:1619677447.342443:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.342445:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.342446:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.342447:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.342449:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.342451:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.342462:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.342464:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.342465:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.342466:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.342466:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.342467:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.342651:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.342654:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.342655:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.342655:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.342657:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.342659:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.342842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.342919:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.342922:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.342923:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.342925:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.342927:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.342929:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.342930:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.342931:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.342933:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.342933:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.342937:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.342940:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.342943:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.342944:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.342944:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.342946:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.342947:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.342948:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.342948:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.342970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.342982:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:9.2:1619677447.342990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.342997:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.342999:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.343001:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.343002:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.343002:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677447.343043:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.343045:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.343046:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.343048:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.343049:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.343049:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.343058:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.343061:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.343062:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.343064:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.343067:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.343069:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.343071:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.343072:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.343073:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.343074:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.343078:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.343081:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.343084:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.343085:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.343085:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.343087:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.343088:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.343089:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.343090:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.343091:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677447.343093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.343104:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:2.0:1619677447.343105:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.343107:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.343108:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.343109:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.343120:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.343123:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.343124:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.343126:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.343127:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.343128:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.343141:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.343144:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.343145:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.343145:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.343146:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.343147:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.343334:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.343337:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.343340:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.343343:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.343344:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.343346:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.343347:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.343348:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.343349:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.343352:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.343354:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.343357:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.343357:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.343358:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.343360:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.343360:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.343361:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.343361:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:4.2:1619677447.343379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.343386:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.343389:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.343390:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.343392:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.343395:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.343397:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.343398:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.343399:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.343400:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.343402:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.343405:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.343407:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.343410:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.343410:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.343411:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.343413:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.343414:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.343414:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.343415:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.343775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.343817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.343822:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.343825:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.343826:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.343826:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.343829:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.343832:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.344074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.344086:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.344088:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.344091:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.344094:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.344096:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.344098:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.344099:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.344100:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.344101:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.344104:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.344106:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.344108:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.344109:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.344110:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.344111:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.344112:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.344112:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.344113:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677447.344154:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.344156:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.344157:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.344158:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.344161:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.344162:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.344166:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.344169:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.344170:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.344172:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.344174:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.344176:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.344177:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.344178:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.344179:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.344180:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.344184:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.344186:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.344188:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.344189:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.344189:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.344191:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.344191:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.344192:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.344193:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.344331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.344339:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.344342:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.344343:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.344345:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.344348:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.344348:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:24.0:1619677447.344350:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.344351:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.344353:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.344354:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.344356:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.344362:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.344365:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.344368:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.344368:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.344369:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.344371:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.344371:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.344372:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.344373:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.344389:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.344392:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.344393:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.344394:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.344396:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.344397:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.344454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:13.0:1619677447.344460:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:4.2:1619677447.344460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.344461:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:13.0:1619677447.344463:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.344464:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.344464:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.344465:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.344466:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.344466:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.344468:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677447.344468:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.344471:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:6.0:1619677447.344508:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.344511:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.344512:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.344512:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.344514:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.344517:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.344522:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.344524:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.344525:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.344527:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.344528:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.344529:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.344759:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677447.344762:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677447.344763:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677447.344765:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.344768:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.344769:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677447.344771:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.344772:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.344773:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.344774:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677447.344778:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.344781:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677447.344783:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.344784:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:22.0:1619677447.344784:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677447.344786:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:22.0:1619677447.344787:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677447.344788:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677447.344788:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:6.0:1619677447.344813:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.344816:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.344817:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.344817:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.344819:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.344822:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.344831:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.344834:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.344835:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.344835:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.344836:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.344837:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.344916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.344960:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.344963:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.344963:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.344964:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.344966:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.344969:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.345066:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.345069:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.345070:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.345072:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.345074:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.345075:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.345077:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.345078:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.345079:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.345080:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.345083:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.345085:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.345087:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.345088:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.345088:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.345090:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.345091:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.345091:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.345092:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.345195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.345202:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.345205:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.345206:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.345208:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.345210:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.345212:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.345213:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729547] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.345214:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.345215:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.345216:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.345220:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.345221:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.345224:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.345225:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.345225:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.345227:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.345227:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.345228:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.345228:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.345328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.345362:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677447.345374:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.345377:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.345378:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.345379:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.345381:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.345384:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.345392:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.345395:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.345396:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.345396:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.345397:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.345397:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.345524:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.345527:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.345527:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.345528:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.345530:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.345533:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.345612:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.345615:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.345615:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.345616:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.345618:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.345621:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.345633:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.345635:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.345636:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.345638:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.345640:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.345642:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.345644:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.345645:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.345646:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.345647:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.345651:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.345653:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.345656:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.345656:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.345657:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.345659:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.345660:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.345661:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.345662:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.345663:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.345673:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.345675:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:2.0:1619677447.345675:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:22.0:1619677447.345676:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.345677:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.345677:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.345681:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.345684:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.345686:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.345687:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.345689:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.345691:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.345693:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.345696:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.345699:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.345710:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.345710:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.345711:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.345714:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.345714:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.345716:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (23)++
00000800:00000200:2.0:1619677447.345718:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (24)--
00000800:00000200:2.0:1619677447.345719:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (23)--
00000800:00000200:9.2:1619677447.345720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.345721:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.345723:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.345725:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.345726:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.345726:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.345729:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.345731:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.345736:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.345739:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.345739:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.345741:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.345743:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.345744:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.345755:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.345758:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.345759:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.345759:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.345760:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.345761:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.345780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.345821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.346158:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.346161:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.346162:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.346162:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.346164:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.346167:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.346180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.346187:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.346190:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.346191:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.346195:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.346197:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.346199:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.346201:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.346202:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.346203:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.346205:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.346208:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.346210:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.346213:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.346214:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.346214:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.346216:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.346216:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.346217:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.346219:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.346219:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.346225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.346232:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677447.346233:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677447.346234:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.346235:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.346236:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.346237:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.346237:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.346240:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.346243:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.346244:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.346246:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.346248:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.346249:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.346250:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.346254:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.346257:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.346259:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.346260:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.346261:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.346262:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.346263:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.346264:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.346264:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.346393:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.346396:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.346397:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.346397:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.346399:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.346402:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.346411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.346452:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:22.0:1619677447.346455:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.346457:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.346458:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.346459:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.346461:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.346464:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.346481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.346488:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.346490:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.346491:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.346494:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.346495:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.346610:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.346613:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.346614:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.346615:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.346617:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.346620:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.346666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.346673:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.346676:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.346677:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.346677:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.346679:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.346682:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677447.346810:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.346812:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.346813:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.346815:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.346818:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.346819:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.346821:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.346822:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.346824:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.346824:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.346828:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.346830:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.346833:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.346833:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.346834:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.346836:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.346836:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.346837:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.346837:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:4.2:1619677447.346892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.346937:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.346940:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.346941:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.346941:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.346943:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.346946:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.346955:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.346958:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.346959:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.346959:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:13.0:1619677447.346960:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677447.346960:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.346961:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.346963:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.346964:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.346965:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.346967:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.346968:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.346981:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677447.347016:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.347019:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.347020:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.347020:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.347022:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.347024:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.347097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.347104:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.347107:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.347108:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.347110:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.347114:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.347115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.347126:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677447.347189:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.347192:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.347193:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.347193:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.347195:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.347197:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.347209:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.347212:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.347212:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.347213:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.347214:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.347215:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.347334:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.347336:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.347337:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.347338:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.347339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.347341:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.347341:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.347351:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.347353:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.347354:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.347355:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.347355:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.347356:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.347384:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.347387:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.347387:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.347388:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.347390:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.347393:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.347529:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.347532:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677447.347534:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.347537:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.347538:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.347539:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.347540:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.347541:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.347542:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.347546:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.347548:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.347549:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.347550:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.347550:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.347552:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.347553:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.347553:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.347554:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.347558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.347569:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:12.0:1619677447.347590:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.347593:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.347594:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.347595:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.347598:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.347598:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.347621:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.347623:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.347624:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.347626:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.347628:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.347630:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.347631:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.347633:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.347634:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.347635:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.347639:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.347641:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.347644:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.347645:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.347645:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.347647:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.347648:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.347649:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.347649:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.347691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.347734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.347738:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.347741:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.347742:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.347742:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.347744:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.347747:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.347824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.347831:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.347834:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.347835:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.347837:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.347840:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.347842:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.347843:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.347844:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.347845:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.347846:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.347850:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.347851:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.347854:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.347855:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.347855:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.347857:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.347857:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.347858:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.347858:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677447.347974:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.347977:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677447.347979:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.347981:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.347982:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.347983:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.347985:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.347986:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.347987:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.347990:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.347992:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.347994:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.347995:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.347996:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.347997:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.347998:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.347999:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.348000:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.348000:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:22.0:1619677447.348011:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.348014:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.348015:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.348015:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.348018:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.348021:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.348046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.348053:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.348055:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.348057:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.348057:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.348059:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.348062:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.348202:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.348205:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.348206:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.348206:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.348208:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.348210:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.348305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.348352:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.348354:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.348355:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.348356:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.348358:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.348360:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.348469:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.348472:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677447.348474:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.348477:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.348478:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.348479:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.348481:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.348482:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.348483:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.348486:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.348488:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.348490:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.348491:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.348492:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.348493:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.348494:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677447.348494:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.348494:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.348495:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:6.0:1619677447.348496:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.348497:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.348497:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.348499:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.348504:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.348507:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.348509:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.348509:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.348510:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.348511:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.348512:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000400:00000200:6.0:1619677447.348512:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.348513:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.348513:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.348514:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.348514:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.348515:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.348526:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:22.0:1619677447.348528:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.348530:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.348531:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.348531:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.348532:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.348533:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.348804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.348826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.348833:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.348835:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.348836:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.348837:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.348840:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.348841:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.348842:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.348844:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.348844:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.348845:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.348848:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.348849:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.348849:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:24.0:1619677447.348850:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.348850:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.348851:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000400:00000200:6.0:1619677447.348852:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.348853:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677447.348853:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.348854:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.348855:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:22.0:1619677447.348855:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.348856:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.348857:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.348857:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677447.348857:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:24.0:1619677447.348858:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677447.348858:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.348858:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:22.0:1619677447.348860:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:6.0:1619677447.348860:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677447.348861:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.348863:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.348864:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.348865:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:22.0:1619677447.348869:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.348871:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677447.348874:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.348874:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:22.0:1619677447.348875:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:22.0:1619677447.348877:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:22.0:1619677447.348878:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677447.348879:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677447.348880:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677447.348881:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677447.348891:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:6.0:1619677447.348893:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:6.0:1619677447.348894:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677447.348895:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:13.0:1619677447.348914:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.348916:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.348917:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.348919:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.348921:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.348922:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.349151:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.349154:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.349155:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.349158:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.349161:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.349162:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.349164:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.349165:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.349166:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.349167:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.349172:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.349175:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:4.2:1619677447.349176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.349178:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.349179:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.349180:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.349181:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.349182:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.349183:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.349183:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.349188:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.349191:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.349192:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.349194:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.349197:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.349199:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:6.0:1619677447.349199:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:8.0:1619677447.349201:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96729489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.349201:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.349202:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.349202:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.349203:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.349204:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.349205:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:6.0:1619677447.349205:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.349209:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.349212:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:6.0:1619677447.349214:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.349215:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.349216:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.349217:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.349219:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.349219:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.349221:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.349221:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.349233:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.349236:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.349236:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.349237:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.349239:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.349239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:22.0:1619677447.349242:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.349247:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:24.0:1619677447.349248:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.349250:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:2.0:1619677447.349250:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:8.0:1619677447.349251:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.349252:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.349253:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.349254:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.349256:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.349257:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:24.0:1619677447.349258:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:2.0:1619677447.349258:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.349260:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.349261:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.349262:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.349266:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.349268:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.349270:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.349271:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.349271:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.349273:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.349273:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.349274:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.349275:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.349424:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.349426:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.349427:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.349428:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.349431:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.349432:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.349573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.349614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.349620:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.349623:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.349623:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.349624:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.349626:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.349629:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.350020:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.350022:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.350023:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.350024:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.350026:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.350027:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.350069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.350077:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.350079:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.350081:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.350081:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.350084:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.350086:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.350087:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.350088:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.350089:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.350090:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.350093:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.350095:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.350098:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.350098:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.350099:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.350101:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.350101:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.350102:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.350102:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.350182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.350191:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.350193:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.350194:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.350196:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.350204:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.350205:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.350206:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677447.350226:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:22.0:1619677447.350227:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677447.350229:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:22.0:1619677447.350230:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677447.350230:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.350230:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:22.0:1619677447.350231:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.350231:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.350233:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.350235:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.350247:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.350250:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.350251:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.350251:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.350252:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.350252:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.350343:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.350346:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.350346:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.350349:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.350352:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.350353:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.350355:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.350357:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.350358:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.350359:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.350359:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:22.0:1619677447.350364:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.350367:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.350370:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.350370:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.350371:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.350373:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.350373:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.350375:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.350376:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.350376:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.350379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.350387:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677447.350388:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:6.0:1619677447.350389:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:2.0:1619677447.350389:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:6.0:1619677447.350390:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677447.350391:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.350392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.350392:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.350394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:2.0:1619677447.350395:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.350398:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.350399:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.350400:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.350401:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.350403:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.350404:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.350405:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.350523:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.350526:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.350527:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.350527:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.350530:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.350532:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.350682:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.350685:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.350686:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.350686:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.350688:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.350690:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.350804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.350811:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.350814:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.350815:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.350817:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.350820:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.350821:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.350823:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.350824:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.350825:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.350826:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.350829:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.350831:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.350834:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.350834:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.350835:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.350836:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.350837:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.350839:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.350839:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.350840:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.350846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.350853:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.350855:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.350856:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.350857:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.350894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.350895:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.350898:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.350901:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.350903:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.350904:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.350906:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.350908:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.350909:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.350910:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.350914:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.350916:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.350918:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.350918:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.350919:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.350920:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.350924:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.350925:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:9.2:1619677447.350926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.350926:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.350929:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.350932:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:2.0:1619677447.350935:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000400:00000200:2.0:1619677447.350936:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.350938:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.350940:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.350941:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677447.350943:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:9.2:1619677447.350948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.350954:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.350957:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.350958:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.350959:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.350960:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.351011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.351022:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:9.2:1619677447.351046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.351054:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.351056:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.351057:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.351059:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.351061:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:22.0:1619677447.351225:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.351228:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.351229:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.351231:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.351233:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.351234:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.351236:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.351237:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.351238:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.351239:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.351242:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.351244:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.351246:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.351247:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.351248:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.351249:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.351250:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.351250:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.351251:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.351255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.351267:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:4.2:1619677447.351442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.351488:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.351491:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.351492:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.351493:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.351495:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.351498:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.351530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.351537:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.351539:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.351540:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.351540:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.351542:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.351545:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.351551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677447.351557:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.351559:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.351561:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.351561:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.351562:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.351562:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677447.351677:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.351680:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.351681:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.351682:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.351683:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.351686:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.351701:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.351704:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.351705:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.351705:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.351706:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.351707:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.351767:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.351769:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.351770:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.351773:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.351776:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.351777:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.351778:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.351780:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.351782:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.351782:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.351786:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.351788:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:9.2:1619677447.351789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:22.0:1619677447.351791:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.351792:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.351793:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.351795:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.351795:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677447.351796:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:22.0:1619677447.351796:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.351796:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.351798:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.351799:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.351799:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.351802:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.351804:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:2.0:1619677447.351845:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.351847:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677447.351850:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.351852:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.351853:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.351855:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.351856:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.351857:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.351858:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:4.2:1619677447.351861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677447.351861:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.351863:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.351865:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.351866:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.351866:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.351868:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.351869:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.351869:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.351870:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.351874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.351885:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677447.351910:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.351913:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.351914:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.351914:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.351916:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.351920:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.351970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.351977:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.351980:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.351981:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.351982:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.351986:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.351992:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.351993:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.351994:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.351995:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.351996:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.351999:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.352001:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.352004:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.352005:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.352005:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.352007:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.352008:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.352008:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.352009:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.352046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.352054:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.352056:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.352057:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.352060:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.352062:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.352119:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:12.0:1619677447.352154:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.352157:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.352158:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.352161:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.352161:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.352164:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.352173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.352179:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.352182:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.352183:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.352183:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.352185:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.352188:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.352306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.352354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677447.352358:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.352361:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.352362:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.352363:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.352365:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.352367:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.352377:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.352380:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.352380:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.352381:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.352382:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.352384:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.352562:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.352565:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.352565:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.352566:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.352568:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.352571:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.352803:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.352805:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.352806:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.352808:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.352809:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.352811:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.352841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.352847:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:9.2:1619677447.352849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.352851:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.352851:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.352854:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.352857:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.352857:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:6.0:1619677447.352859:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:2.0:1619677447.352860:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:6.0:1619677447.352861:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.352861:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.352863:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.352864:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.352865:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.352866:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:9.2:1619677447.352867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.352871:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.352872:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.352874:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.352876:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (23)++
00000400:00000200:6.0:1619677447.352878:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.352878:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.352878:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.352880:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:9.2:1619677447.352881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.352883:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.352884:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.352885:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.352887:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.352887:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.352888:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.352889:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.352894:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:9.2:1619677447.352921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.352927:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677447.352929:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:12.0:1619677447.352930:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.352931:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.352932:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:12.0:1619677447.352933:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.352933:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.352935:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:2.0:1619677447.352935:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.352936:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.352939:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.352995:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.352997:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.352998:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.352998:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:13.0:1619677447.352999:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:22.0:1619677447.353000:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.353002:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.353003:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.353004:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:13.0:1619677447.353005:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.353006:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.353008:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.353028:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.353031:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.353031:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677447.353033:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.353036:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.353037:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.353039:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.353040:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.353042:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.353043:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.353047:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.353049:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.353053:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.353054:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.353054:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.353056:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.353065:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.353066:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.353067:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.353075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:9.2:1619677447.353079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.353081:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.353084:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.353085:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.353085:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.353089:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.353090:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.353093:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.353100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.353107:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.353110:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.353111:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.353112:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.353113:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.353116:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.353118:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.353119:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.353119:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.353122:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.353124:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.353138:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.353219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.353225:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.353228:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.353229:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.353229:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.353231:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.353234:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.353379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.353425:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.353428:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.353429:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.353429:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.353431:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.353434:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.353527:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.353530:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677447.353533:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.353536:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.353537:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677447.353539:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.353540:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.353542:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.353543:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677447.353546:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.353549:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.353553:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.353554:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677447.353555:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677447.353556:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.353558:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.353560:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.353561:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.353562:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:12.0:1619677447.353570:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:9.2:1619677447.353572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:12.0:1619677447.353573:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.353574:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.353576:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.353578:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.353579:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.353580:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677447.353584:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:13.0:1619677447.353589:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.353592:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.353593:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.353594:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.353594:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.353596:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.353642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.353649:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.353652:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.353653:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.353655:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.353658:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.353660:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.353661:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729416] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.353662:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.353663:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.353664:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.353667:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.353668:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.353671:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.353672:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.353672:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.353674:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.353675:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.353675:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.353676:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.353716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.353739:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.353742:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.353742:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.353744:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.353746:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.353747:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.353759:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.353762:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.353762:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.353763:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.353765:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.353767:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.353779:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.353782:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.353783:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.353783:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.353785:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.353785:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.353794:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.353797:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.353798:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.353800:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.353801:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.353803:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.353814:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.353817:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.353818:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.353818:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.353819:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.353820:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.353918:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.353921:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.353922:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.353923:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.353925:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.353927:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.353954:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.353957:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.353960:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.353962:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.353964:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.353965:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.353966:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.353967:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.353968:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.353972:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.353973:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.353976:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.353977:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.353978:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.353979:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.353980:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.353981:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.353981:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677447.354001:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:4.2:1619677447.354009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.354050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:12.0:1619677447.354228:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.354231:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.354231:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.354234:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.354235:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.354238:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.354255:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.354258:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:9.2:1619677447.354258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:13.0:1619677447.354259:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.354261:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.354262:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.354263:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.354265:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.354267:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.354268:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.354269:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.354271:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.354274:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.354367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.354400:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.354402:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.354403:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:12.0:1619677447.354403:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677447.354403:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.354405:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:6.0:1619677447.354405:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:12.0:1619677447.354406:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.354408:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.354408:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.354410:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.354410:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.354418:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.354421:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.354422:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.354422:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.354423:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.354423:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.354660:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.354662:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.354663:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.354664:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.354665:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.354668:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.354716:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.354718:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.354719:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.354721:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.354723:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.354725:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.354726:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.354727:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.354728:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.354730:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.354731:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:22.0:1619677447.354733:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.354734:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:22.0:1619677447.354735:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.354737:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.354738:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.354739:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.354739:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:2.0:1619677447.354740:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.354741:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.354741:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.354742:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:2.0:1619677447.354742:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677447.354743:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000400:00000200:2.0:1619677447.354743:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.354744:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677447.354744:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.354745:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.354746:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.354750:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.354751:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.354754:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677447.354754:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.354754:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.354755:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.354756:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:2.0:1619677447.354756:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.354757:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.354757:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.354757:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.354758:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.354758:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:22.0:1619677447.354770:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.354772:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.354773:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.354773:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.354775:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.354778:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.354781:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.354787:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.354788:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.354789:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.354790:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.354791:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.354794:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.354795:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.354797:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.354798:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.354798:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.354800:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.354800:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.354801:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.354801:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677447.354914:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.354917:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.354917:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.354919:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.354921:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.354922:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.354958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.355006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.355006:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.355009:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.355010:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.355010:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.355012:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.355014:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677447.355015:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.355016:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.355017:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.355018:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.355019:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.355022:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677447.355131:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.355134:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.355135:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.355136:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.355138:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.355139:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.355284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.355291:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.355297:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.355298:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677447.355300:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.355303:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.355305:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.355306:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.355307:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.355308:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.355309:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.355312:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.355313:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.355316:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.355317:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.355317:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.355319:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.355320:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.355320:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.355321:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677447.355402:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.355405:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.355406:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.355407:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.355410:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.355411:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.355413:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.355414:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.355415:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.355416:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.355418:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.355420:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.355422:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.355423:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.355423:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.355424:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.355425:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.355426:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.355426:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.355431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.355439:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.355440:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.355441:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.355442:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.355443:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.355444:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.355560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.355567:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.355570:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.355571:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.355571:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.355573:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.355576:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.355581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677447.355587:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.355590:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.355591:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.355591:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.355592:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.355593:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:18.0:1619677447.355704:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.355707:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.355708:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.355710:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.355712:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.355713:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677447.355715:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.355716:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.355717:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.355718:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677447.355721:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.355723:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677447.355726:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.355727:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:13.0:1619677447.355727:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:18.0:1619677447.355728:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:13.0:1619677447.355729:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:18.0:1619677447.355730:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677447.355731:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.355731:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:13.0:1619677447.355731:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.355732:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:13.0:1619677447.355732:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.355732:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677447.355733:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:12.0:1619677447.355734:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.355734:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:13.0:1619677447.355735:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:2.0:1619677447.355736:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.355738:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.355739:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677447.355740:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.355742:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.355743:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.355743:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:12.0:1619677447.355746:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:2.0:1619677447.355746:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.355748:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.355748:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:12.0:1619677447.355749:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.355750:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.355751:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.355752:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.355752:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.355753:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:2.0:1619677447.355753:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:2.0:1619677447.355755:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:2.0:1619677447.355756:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677447.355757:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677447.355758:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677447.355761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.355768:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.355770:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.355771:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.355771:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.355774:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.355776:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.355782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.355788:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.355791:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.355792:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.355792:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.355793:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.355793:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.355805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.355851:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.355854:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.355855:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.355856:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.355858:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.355861:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.356036:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.356038:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.356039:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.356041:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.356042:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.356042:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.356044:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.356044:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.356046:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.356047:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.356049:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.356050:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.356051:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.356052:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.356053:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.356054:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:6.0:1619677447.356054:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.356057:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.356059:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.356060:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.356061:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.356063:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.356063:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.356064:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.356064:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.356069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.356076:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.356079:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.356080:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.356080:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.356082:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.356085:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.356136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.356143:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.356145:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.356147:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.356149:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.356152:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.356153:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.356155:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.356156:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.356157:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.356158:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.356161:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.356163:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.356166:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.356167:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.356174:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.356174:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.356176:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.356176:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.356177:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:24.0:1619677447.356178:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.356179:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.356180:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000400:00000200:24.0:1619677447.356181:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.356184:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.356185:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96729377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.356186:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.356187:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677447.356188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:24.0:1619677447.356189:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.356189:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.356190:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.356190:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.356192:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.356193:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.356194:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.356194:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677447.356197:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.356201:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.356202:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.356202:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.356205:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.356206:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.356207:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.356208:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.356209:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677447.356211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.356218:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.356221:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.356222:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.356222:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.356224:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.356227:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:2.0:1619677447.356227:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.0:1619677447.356229:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.356230:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.356231:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.356539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.356546:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.356549:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.356549:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.356550:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.356552:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.356555:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.356607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.356649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.356655:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.356658:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.356658:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.356659:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.356662:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.356664:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.356682:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.356684:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.356685:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.356686:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.356687:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.356687:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.356709:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.356712:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.356713:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.356714:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.356717:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.356718:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.356719:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.356720:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.356721:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.356722:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.356725:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.356727:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.356729:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.356730:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.356731:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.356732:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.356733:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.356734:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.356734:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.356739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.356747:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.356749:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.356750:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.356750:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.356753:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.356755:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.356757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.356764:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.356767:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.356768:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.356768:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.356769:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.356770:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.356805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.356813:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.356816:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.356817:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.356818:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.356821:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.356822:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.356824:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.356825:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.356826:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.356827:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.356831:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.356832:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.356835:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.356835:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.356836:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.356838:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.356838:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.356839:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.356839:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.356929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.356936:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.356938:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.356939:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.356939:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.356943:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.356946:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677447.356965:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.356968:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.356969:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.356971:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.356972:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.356974:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.357066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.357073:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.357076:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.357077:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.357077:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.357079:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.357082:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:18.0:1619677447.357125:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.357128:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.357129:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.357132:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.357134:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.357136:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.357138:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.357140:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.357141:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.357142:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.357145:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.357148:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.357149:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.357150:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.357151:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.357152:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.357153:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.357154:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.357155:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.357156:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.357158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.357159:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.357161:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.357162:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.357164:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.357165:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.357167:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.357167:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.357168:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.357168:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.357169:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:2.0:1619677447.357169:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.357170:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.357171:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.357171:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.357173:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677447.357173:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.357175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.357183:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.357183:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.357186:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.357187:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:13.0:1619677447.357187:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.357187:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.357188:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.357188:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.357188:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.357189:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.357190:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.357190:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.357228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.357236:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.357239:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.357240:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.357241:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.357242:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.357246:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:12.0:1619677447.357287:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.357289:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.357290:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.357292:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.357292:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677447.357293:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677447.357299:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677447.357300:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.357302:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.357305:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.357307:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.357309:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.357310:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.357310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:6.0:1619677447.357311:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.357312:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.357313:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.357318:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.357321:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.357324:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.357325:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.357325:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.357327:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.357327:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.357328:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.357329:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677447.357373:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.357375:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.357376:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.357378:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.357380:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.357382:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.357384:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.357385:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.357386:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.357387:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.357391:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.357393:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.357395:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.357396:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.357396:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.357398:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.357398:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.357399:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.357400:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.357401:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677447.357404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.357412:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.357413:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.357413:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.357414:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:2.0:1619677447.357414:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.357415:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.357416:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.357417:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:2.0:1619677447.357417:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.357418:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.357467:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.357470:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.357470:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.357472:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.357473:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.357476:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.357673:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.357677:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.357678:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.357678:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.357680:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.357683:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.357700:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.357703:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.357703:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.357704:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.357706:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.357708:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.357766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.357774:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.357777:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.357778:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.357779:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.357782:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.357784:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.357785:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.357786:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.357787:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.357788:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.357791:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.357792:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.357795:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.357796:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.357796:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.357798:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.357799:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.357799:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.357800:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.357922:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.357924:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.357925:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.357925:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.357927:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.357929:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.357938:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:9.2:1619677447.357940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.357941:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.357942:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.357942:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.357943:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.357944:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677447.357948:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.357951:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.357952:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.357952:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.357955:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.357957:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.357961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.358033:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.358035:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.358036:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.358037:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.358039:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.358040:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.358103:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.358106:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.358107:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.358107:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.358110:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.358113:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.358161:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.358164:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.358165:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.358167:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.358168:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.358170:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.358171:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.358172:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.358174:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.358175:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.358178:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.358180:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.358183:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.358184:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.358184:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.358186:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.358186:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.358187:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.358188:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:4.2:1619677447.358232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.358239:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.358242:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.358243:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.358247:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.358250:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.358252:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.358254:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.358256:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.358257:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.358258:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.358264:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.358268:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.358271:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.358272:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.358273:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.358275:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.358276:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.358278:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.358279:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.358280:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.358290:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.358292:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.358293:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.358301:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.358315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.358323:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.358325:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.358326:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.358326:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.358328:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.358331:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677447.358381:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.358384:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.358384:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.358385:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.358387:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.358390:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.358486:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.358488:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.358489:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.358491:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.358493:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.358494:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.358524:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.358526:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.358527:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.358528:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.358530:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.358532:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.358595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.358634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.358638:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.358641:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.358642:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.358642:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.358645:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.358647:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.358692:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.358695:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.358696:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.358696:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.358698:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.358700:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.358732:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.358734:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.358735:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.358735:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.358737:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.358740:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.358748:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.358750:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.358751:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.358751:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.358752:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.358753:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677447.358839:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.358842:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.358843:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.358844:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.358847:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.358849:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677447.358850:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.358851:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.358852:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.358853:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677447.358857:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.358860:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677447.358862:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.358863:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677447.358863:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677447.358865:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677447.358866:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.358866:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.358867:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677447.358869:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.358872:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.0.10.103@o2ib7
00000400:00000200:12.0:1619677447.358874:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.103@o2ib7 : GET - for me
00000800:00000200:13.0:1619677447.358875:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.358877:0:19943:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.103@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:12.0:1619677447.358879:0:19943:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.103@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:6.0:1619677447.358879:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:12.0:1619677447.358881:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.358881:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677447.358881:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.358882:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.358882:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.358883:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:6.0:1619677447.358884:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:12.0:1619677447.358887:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : REPLY try# 0
00000800:00000200:6.0:1619677447.358887:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.358889:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.358892:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.358892:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.358893:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.358895:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.358895:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.358907:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.358909:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.358910:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.358912:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: REPLY: OK
00000800:00000200:12.0:1619677447.358913:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.358914:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.358970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677447.359040:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:2.0:1619677447.359042:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:24.0:1619677447.359044:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.359044:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:18.0:1619677447.359045:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.359045:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:8.0:1619677447.359048:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.359049:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.359051:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.359051:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.359053:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.359053:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.359055:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.359055:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.359058:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.359058:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.359059:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.359060:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.359061:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.359062:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.359067:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.359068:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.359070:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.359073:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.359074:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.359074:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.359076:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.359076:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.359077:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.359078:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.359078:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.0:1619677447.359079:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.359079:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.359080:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.359082:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.359082:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.359083:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.359084:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.359099:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.359101:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.359102:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.359104:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.359106:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.359107:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.359240:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.359243:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.359244:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.359244:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.359246:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.359250:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.359344:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:4.2:1619677447.359346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.359348:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.359348:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.359351:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.359352:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.359354:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.359361:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.359364:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.359365:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.359366:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.359367:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.359368:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.359396:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:12.0:1619677447.359398:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:22.0:1619677447.359399:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.359399:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.359400:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.359401:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.359401:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.359402:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:12.0:1619677447.359404:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.359405:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677447.359406:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.359407:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.359417:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.359420:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.359421:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.359421:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.359422:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.359423:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.359827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.359869:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.359872:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.359872:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.359873:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.359875:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.359878:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.359977:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.359980:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.359981:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.359983:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.359983:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.359990:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.360009:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:12.0:1619677447.360010:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:22.0:1619677447.360012:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:12.0:1619677447.360013:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.360013:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.360013:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.360015:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.360016:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.360016:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.360019:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.360019:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:22.0:1619677447.360021:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.360024:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.360026:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.360027:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.360028:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.360033:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.360036:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.360039:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.360041:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.360041:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (22)++
00000800:00000200:22.0:1619677447.360044:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677447.360045:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.360046:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.360047:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.360055:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.360058:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.360059:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.360062:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677447.360063:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.360066:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.360103:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.360105:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.360106:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.360107:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.360109:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.360110:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.360301:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.360304:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.360305:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.360306:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.360310:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.360311:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.360312:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.360313:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.360314:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.360315:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.360318:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.360320:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.360323:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.360323:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.360324:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.360326:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.360326:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.360327:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.360327:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:18.0:1619677447.360378:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.360381:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.360382:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.360384:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.360387:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.360388:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.360389:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.360391:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.360392:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.360393:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.360397:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.360398:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.360401:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.360401:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.360402:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.360403:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.360404:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.360405:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.360405:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.360458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.360465:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:4.2:1619677447.360468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.360468:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.360469:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.360470:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.360472:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.360475:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.360507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.360513:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.360516:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.360516:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.360517:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.360519:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.360522:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.360554:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feaea5a00] (21)++
00000800:00000200:18.0:1619677447.360556:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.0:1619677447.360557:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.12@o2ib3
00000800:00000200:24.0:1619677447.360558:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feaea5a00] (22)--
00000800:00000200:18.0:1619677447.360559:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.360560:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:8.0:1619677447.360561:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.5.12@o2ib3 : GET - for me
00000400:00000200:18.0:1619677447.360563:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.360565:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.360565:0:19939:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.5.12@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:18.0:1619677447.360567:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677447.360568:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.360568:0:19939:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.5.12@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:8.0:1619677447.360569:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feaea5a00] (21)++
00000400:00000200:18.0:1619677447.360570:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.360571:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.360571:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feaea5a00] (22)--
00000400:00000200:18.0:1619677447.360573:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677447.360573:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.5.12@o2ib3 local destination
00000400:00000200:18.0:1619677447.360579:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000400:00000200:8.0:1619677447.360582:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.5.12@o2ib3(10.51.5.12@o2ib3:10.51.5.12@o2ib3) : REPLY try# 0
00000800:00000200:18.0:1619677447.360583:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:8.0:1619677447.360584:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.5.12@o2ib3
00000800:00000200:18.0:1619677447.360586:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.360587:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:8.0:1619677447.360587:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919fce371900] -> 10.51.5.12@o2ib3 (2) version: 12
00000800:00000200:18.0:1619677447.360588:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:8.0:1619677447.360588:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919feaea5a00] (21)++
00000800:00000200:8.0:1619677447.360589:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feaea5a00] (22)++
00000800:00000200:18.0:1619677447.360590:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677447.360590:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.0:1619677447.360591:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919feaea5a00] (23)--
00000800:00000200:18.0:1619677447.360592:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:8.0:1619677447.360592:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feaea5a00] (22)--
00000800:00000200:18.0:1619677447.360593:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.360593:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:24.0:1619677447.360606:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feaea5a00] (22)++
00000800:00000200:24.0:1619677447.360608:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feaea5a00] (23)--
00000800:00000200:2.0:1619677447.360608:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:24.0:1619677447.360609:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.360610:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feaea5a00] (22)--
00000800:00000200:2.0:1619677447.360610:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:24.0:1619677447.360611:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.5.12@o2ib3: REPLY: OK
00000800:00000200:2.0:1619677447.360611:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.360613:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677447.360614:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feaea5a00] (21)--
00000800:00000200:13.0:1619677447.360669:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.360672:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.360673:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.360674:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.360676:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.360677:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.360679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.360686:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.360688:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.360689:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.360689:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.360690:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:8.0:1619677447.360691:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.360693:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.360693:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.360694:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.360695:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.360695:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677447.360696:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.360726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.360733:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.360735:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.360736:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.360738:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.360741:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.360742:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.360744:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.360745:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.360746:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.360747:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.360750:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.360752:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.360754:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.360755:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.360756:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.360757:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.360758:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.360758:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.360759:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.360861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.360869:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.360872:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.360873:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.360873:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.360875:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.360878:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.360984:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.360987:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.360987:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.360989:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.360990:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.360992:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.361004:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.361006:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.361007:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.361007:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.361008:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.361009:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.361136:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.361138:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.361139:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.361139:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.361141:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.361144:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.361153:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.361156:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.361157:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.361157:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.361158:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.361158:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.361159:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.361162:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.361163:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.361164:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.361168:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.361169:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.361171:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.361172:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.361173:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.361174:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.361177:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.361179:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.361181:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.361182:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.361183:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.361184:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.361185:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.361185:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.361186:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.361196:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.361199:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.361199:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.361201:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.361203:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.361204:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.361310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.361322:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.361325:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.361325:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.361328:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.361330:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.361331:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.361333:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.361335:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.361336:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.361337:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.361342:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.361346:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.361349:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.361350:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.361351:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.361353:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.361353:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.361354:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.361355:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.361360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.361368:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:6.0:1619677447.361369:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677447.361371:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:6.0:1619677447.361372:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.361373:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.361373:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:6.0:1619677447.361373:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.361373:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.361376:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.361377:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.361379:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:6.0:1619677447.361381:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.361581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.361588:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.361591:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.361592:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.361594:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.361596:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.361598:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.361599:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.361601:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.361602:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.361603:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.361606:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.361608:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.361611:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.361612:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.361612:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.361614:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.361614:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.361616:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.361617:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.361617:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.361623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.361630:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.361632:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.361633:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.361634:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.361664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.361671:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.361673:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.361674:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.361675:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.361677:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.361680:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.361682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:8.0:1619677447.361688:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.361691:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.361692:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.361692:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.361693:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.361694:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:18.0:1619677447.361889:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.361892:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.361893:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.361895:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.361897:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.361899:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.361900:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.361901:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.361902:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.361904:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.361907:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.361909:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.361913:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.361913:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.361914:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.361916:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.361916:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.361917:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.361918:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677447.361930:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.361932:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.361933:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.361935:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.361937:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.361938:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.362059:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677447.362062:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677447.362063:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677447.362066:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.362069:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.362070:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:2.0:1619677447.362072:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.362073:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.362074:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.362075:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:9.2:1619677447.362077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677447.362079:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.362081:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:18.0:1619677447.362085:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.362085:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.362086:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:18.0:1619677447.362087:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.362087:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:18.0:1619677447.362088:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.362089:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.362089:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:2.0:1619677447.362090:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677447.362091:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677447.362092:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.362092:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:2.0:1619677447.362093:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.362095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:9.2:1619677447.362098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.362101:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.362104:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.362105:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.362105:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:18.0:1619677447.362106:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:24.0:1619677447.362107:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.362108:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.362110:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:18.0:1619677447.362110:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.362110:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.362111:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.362111:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.362156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.362164:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.362168:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.362168:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.362172:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.362176:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.362178:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.362179:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96729287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.362182:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677447.362182:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.362183:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.362185:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.362186:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.362186:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677447.362191:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.362194:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:22.0:1619677447.362195:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.362196:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.362197:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:22.0:1619677447.362198:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.362198:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:22.0:1619677447.362200:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:2.0:1619677447.362200:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.362200:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:22.0:1619677447.362201:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:508, d:32, s:-96729285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.362202:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.362202:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.362202:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.362203:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000400:00000200:22.0:1619677447.362204:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.362204:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.362204:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:12.0:1619677447.362205:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.362206:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.362206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:22.0:1619677447.362209:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.362209:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.362209:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.362212:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.362213:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.362215:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:22.0:1619677447.362216:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.362216:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.362217:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.362217:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.362217:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:22.0:1619677447.362218:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:18.0:1619677447.362218:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.362218:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677447.362218:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.362219:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:22.0:1619677447.362220:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.362220:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.362220:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.362220:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677447.362221:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677447.362221:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:22.0:1619677447.362222:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.362222:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.362223:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677447.362223:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.362224:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:9.2:1619677447.362226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.362226:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.362234:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.362236:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.362237:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.362238:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.362239:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.362242:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.362245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.362252:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.362255:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.362256:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.362256:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.362257:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.362257:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.362337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.362344:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.362347:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.362348:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.362350:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.362353:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.362354:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.362355:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.362357:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.362358:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.362359:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.362362:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.362364:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.362366:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.362367:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.362368:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.362369:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.362370:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.362371:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.362371:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677447.362449:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.362451:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.362453:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.362454:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.362456:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.362458:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.362459:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.362460:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.362461:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.362462:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677447.362465:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.362467:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.362470:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.362471:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.362472:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.362474:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.362474:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.362475:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.362475:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.362717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.362725:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.362728:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.362729:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.362729:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.362731:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.362734:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.362791:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.362794:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.362795:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.362797:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.362799:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.362801:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.362802:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.362803:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.362805:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.362805:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.362809:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.362811:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.362814:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.362814:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:13.0:1619677447.362814:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:22.0:1619677447.362815:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.362817:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:13.0:1619677447.362817:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:22.0:1619677447.362818:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:13.0:1619677447.362818:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.362819:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.362820:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000400:00000200:13.0:1619677447.362820:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.362821:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677447.362822:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.362823:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.362831:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.362833:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.362834:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.362834:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:4.2:1619677447.362849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.363010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.363017:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.363019:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.363020:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.363022:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.363025:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.363027:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.363028:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.363030:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.363031:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.363032:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.363035:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.363037:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.363040:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.363040:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.363041:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.363043:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.363043:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.363044:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.363044:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.363104:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.363107:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.363108:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.363109:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.363111:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.363111:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.363127:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.363130:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.363130:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.363131:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.363131:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.363132:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.363154:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.363157:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.363158:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.363160:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.363163:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.363164:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.363165:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.363167:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.363167:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.363168:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677447.363171:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.363173:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.363176:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.363177:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.363177:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.363179:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.363180:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.363180:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.363181:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:4.2:1619677447.363266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.363290:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.363293:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.363293:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.363302:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.363306:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.363308:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.363310:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.363312:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.363313:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.363315:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:4.2:1619677447.363316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.363318:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:18.0:1619677447.363319:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.363320:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.363321:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677447.363321:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.363325:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677447.363325:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.363326:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.363327:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.363328:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:6.0:1619677447.363328:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.363329:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.363329:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:6.0:1619677447.363329:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:18.0:1619677447.363330:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:6.0:1619677447.363331:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96729268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.363332:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.363334:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.363335:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.363339:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.363341:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.363344:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.363344:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.363345:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.363347:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.363348:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.363348:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.363349:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.363439:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.363442:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.363443:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.363444:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.363446:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.363447:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.363530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.363548:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.363551:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.363551:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.363553:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.363555:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.363556:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.363579:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.363581:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.363582:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.363583:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.363585:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.363588:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.363598:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.363601:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.363602:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.363604:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.363607:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.363608:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677447.363610:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.363611:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.363612:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.363613:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677447.363616:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.363618:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.363620:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.363620:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.363621:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.363622:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.363623:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.363624:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.363624:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.363628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.363635:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.363637:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.363638:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.363639:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.363640:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.363642:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.363832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.363896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.363904:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.363906:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.363907:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.363908:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.363910:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.363913:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.364006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.364013:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.364015:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.364016:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.364016:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.364018:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.364021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.364025:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677447.364026:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.364029:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.364029:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.364032:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.364035:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.364037:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.364039:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.364041:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.364042:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.364044:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.364048:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.364050:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.364053:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.364054:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.364054:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.364056:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.364056:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.364057:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.364058:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.364209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.364254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.364262:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.364265:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.364266:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.364267:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.364270:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.364274:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.364275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.364282:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.364285:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.364286:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.364286:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.364288:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.364288:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.364346:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.364349:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.364350:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.364351:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.364353:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.364354:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.364363:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.364365:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.364366:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.364367:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.364369:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.364371:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.364537:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.364539:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.364540:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.364541:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.364543:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.364546:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677447.364572:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.364575:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.364576:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.364577:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.364579:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.364581:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.364582:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.364583:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.364584:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.364585:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.364588:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.364590:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.364593:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.364593:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.364594:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.364596:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.364596:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.364597:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.364598:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.364608:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.364611:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.364612:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.364613:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.364615:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.364616:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.364674:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.364677:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.364678:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.364682:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.364685:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.364686:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.364688:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.364690:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.364691:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.364693:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.364697:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.364699:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.364704:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.364705:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.364705:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.364707:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.364708:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.364709:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.364710:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.364711:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677447.364714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (22)++
00000800:00000200:4.2:1619677447.364717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.364723:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:2.0:1619677447.364724:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.364725:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000800:00000200:2.0:1619677447.364725:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:18.0:1619677447.364726:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.364726:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.364727:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000400:00000200:18.0:1619677447.364728:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.364728:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000400:00000200:2.0:1619677447.364729:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.364731:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.364731:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.364732:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677447.364734:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.364735:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.364764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.364768:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.364771:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.364772:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.364772:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.364774:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.364781:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.364783:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.364785:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.364785:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.364786:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.364787:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.364787:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.364952:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.364955:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.364956:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.364956:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.364959:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.364961:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.365108:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.365110:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.365111:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.365111:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.365113:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.365116:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.365125:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.365128:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.365129:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.365129:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.365130:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.365131:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.365154:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.365157:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.365158:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.365160:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.365162:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.365164:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.365165:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.365167:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.365168:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.365169:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.365172:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.365174:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.365177:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.365177:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.365178:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.365180:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.365180:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.365181:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.365181:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.365472:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.365475:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.365476:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.365477:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.365479:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.365480:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.365489:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.365491:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.365492:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.365493:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:22.0:1619677447.365494:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:12.0:1619677447.365494:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.365495:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.365497:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.365498:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.365498:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.365500:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.365502:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.365693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.365774:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.365777:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.365778:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.365779:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.365781:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.365782:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.365783:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.365785:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.365786:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.365787:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.365790:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.365792:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.365793:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.365794:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.365795:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.365796:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.365796:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.365797:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.365797:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.365802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.365809:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.365811:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.365812:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.365813:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.365813:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.365815:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.365887:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.365889:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.365890:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.365891:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.365893:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677447.365893:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.365894:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.365896:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.365897:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.365899:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.365901:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.365902:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.365904:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.365905:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.365906:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.365907:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.365911:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.365912:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.365915:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.365915:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.365916:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.365918:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.365918:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.365919:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.365919:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:24.0:1619677447.366044:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.366047:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.366048:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.366050:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.366052:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.366054:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.366055:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.366056:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.366057:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.366058:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.366062:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.366064:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.366067:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.366068:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.366068:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.366070:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.366071:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.366072:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.366074:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.366074:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.366088:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.366090:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.366091:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.366092:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.366267:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.366270:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.366271:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.366273:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.366274:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.366275:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.366292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.366337:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.366339:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.366340:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.366342:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.366344:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677447.366346:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.366369:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.366371:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.366372:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.366374:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.366375:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.366376:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.366668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.366698:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.366701:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.366702:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.366704:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.366706:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.366707:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.366710:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.366711:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.366713:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.366714:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:6.0:1619677447.366714:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.366715:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.366717:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.366718:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.366718:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.366720:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.366722:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.366723:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.366723:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.366724:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.366725:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.366727:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.366727:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.366728:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.366729:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.366764:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.366766:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.366767:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.366770:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.366773:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.366774:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.366776:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.366777:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.366778:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.366779:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.366784:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.366786:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.366789:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.366790:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.366791:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.366792:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.366793:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.366794:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.366794:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:4.2:1619677447.367062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.367101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.367146:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.367149:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.367150:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.367152:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.367155:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.367156:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:2.0:1619677447.367157:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.367158:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.367159:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:2.0:1619677447.367159:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677447.367160:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.367160:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677447.367162:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.367162:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:12.0:1619677447.367163:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.367165:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:2.0:1619677447.367165:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.367168:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.367170:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.367171:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.367172:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.367173:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.367174:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.367175:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.367176:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.367176:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:12.0:1619677447.367179:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:9.2:1619677447.367179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.367182:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.367183:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.367184:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.367184:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.367186:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677447.367188:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.367189:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.367190:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.367190:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.367190:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.367191:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.367192:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.367193:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.367193:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677447.367195:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.367196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:9.2:1619677447.367200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:2.0:1619677447.367203:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.367206:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.367207:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.367207:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:2.0:1619677447.367207:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.367208:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.367208:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.0:1619677447.367209:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.367210:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.367210:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.367213:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.367216:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677447.367295:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.367301:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.367303:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:24.0:1619677447.367306:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.367308:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.367309:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.367310:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.367312:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.367313:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.367314:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.367317:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.367319:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.367322:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.367322:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.367323:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.367325:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.367325:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.367326:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.367326:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.367433:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.367436:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.367436:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.367437:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.367439:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.367442:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.367514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.367559:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.367562:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.367563:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.367563:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.367565:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.367569:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.367576:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.367579:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.367579:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.367581:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.367583:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.367585:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.367586:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.367587:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.367588:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.367589:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:4.2:1619677447.367591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:22.0:1619677447.367593:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.367595:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.367598:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.367598:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:8.0:1619677447.367598:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.367599:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (24)++
00000800:00000200:8.0:1619677447.367601:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:22.0:1619677447.367602:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:24.0:1619677447.367603:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677447.367603:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:8.0:1619677447.367604:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.367605:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.367606:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.367606:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:8.0:1619677447.367608:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.367609:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.367611:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96729209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.367612:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.367613:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.367614:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:6.0:1619677447.367616:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:8.0:1619677447.367617:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.367618:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:8.0:1619677447.367619:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677447.367619:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.367620:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.367622:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.367623:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.367623:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.367625:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.367625:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.367626:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.367628:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.367628:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.367634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.367641:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.367644:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.367645:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.367645:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677447.367833:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.367835:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.367836:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.367838:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.367840:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.367841:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.367843:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.367844:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.367846:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.367847:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.367850:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.367852:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.367856:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.367857:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.367857:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.367859:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.367859:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.367860:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.367861:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677447.367872:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.367875:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.367876:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.367878:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.367879:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.367881:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.367893:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.367895:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.367896:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.367897:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.367897:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.367899:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.368058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.368103:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.368104:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.368105:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.368106:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:22.0:1619677447.368107:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.368109:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.368109:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:6.0:1619677447.368110:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.368112:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.368112:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:22.0:1619677447.368113:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.368113:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:22.0:1619677447.368114:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:6.0:1619677447.368115:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:22.0:1619677447.368116:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677447.368116:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.368116:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.368117:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.368118:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.368118:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.368118:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.368118:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.368119:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.368119:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.368120:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:2.0:1619677447.368120:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.368121:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.368121:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.368124:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.368125:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.368126:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.368126:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.368129:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.368131:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.368132:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.368133:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.368133:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.368133:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.368135:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.368135:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.368136:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.368136:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677447.368136:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.368137:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.368137:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.368138:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.368139:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.368140:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.368141:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:4.2:1619677447.368340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.368348:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.368351:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.368352:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.368354:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.368356:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:24.0:1619677447.368357:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.368358:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.368359:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.368359:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.368360:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.368361:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.368362:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677447.368362:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.368363:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677447.368364:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677447.368365:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677447.368367:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.368368:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.368371:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.368372:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.368372:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.368374:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.368374:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.368375:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.368376:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.368601:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:4.2:1619677447.368603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.368604:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.368605:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.368606:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.368608:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.368610:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.368664:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.368667:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.368668:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.368668:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.368670:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.368673:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.368687:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.368690:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.368690:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.368690:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.368692:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.368693:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.368788:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.368790:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.368791:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.368793:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.368795:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.368796:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.368797:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.368799:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.368800:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.368802:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.368805:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.368807:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.368810:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.368811:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.368811:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.368813:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.368813:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.368814:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.368815:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.369014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.369021:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.369023:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.369024:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.369025:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.369027:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.369030:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.369033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.369039:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.369041:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.369042:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.369042:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:4.2:1619677447.369042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.369043:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.369044:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.369057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.369064:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.369066:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.369068:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.369069:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.369071:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.369073:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.369074:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.369076:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.369076:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.369077:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.369080:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.369082:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:4.2:1619677447.369082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677447.369084:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.369085:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.369086:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.369087:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.369088:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.369088:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.369089:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.369124:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.369127:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.369128:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.369129:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.369130:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.369132:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.369225:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.369228:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.369229:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.369231:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.369233:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.369234:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677447.369236:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.369237:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.369238:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.369239:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677447.369243:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.369244:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677447.369247:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.369248:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677447.369248:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677447.369250:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677447.369251:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.369251:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.369252:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677447.369290:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.369293:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.369294:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.369304:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.369306:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.369308:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.369309:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.369310:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.369311:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.369312:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.369316:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.369317:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.369321:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.369322:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.369322:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.369324:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.369325:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.369325:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.369326:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.369331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.369339:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.369342:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.369343:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.369343:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.369346:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.369350:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.369378:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.369380:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.369381:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.369384:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.369385:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.369386:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.369496:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.369499:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.369500:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.369502:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.369504:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.369506:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.369507:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.369508:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.369509:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.369511:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.369514:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.369516:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.369519:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.369519:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.369520:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.369522:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.369523:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.369523:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.369524:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.369534:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.369537:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.369538:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.369540:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.369541:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.369543:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.369561:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.369564:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.369565:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.369565:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.369566:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.369567:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.369689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.369728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.369736:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.369738:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.369740:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.369740:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.369742:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.369745:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677447.369833:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.369835:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.369836:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.369837:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.369839:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.369840:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.369904:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.369907:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.369908:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.369908:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.369910:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.369912:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.370168:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.370171:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.370172:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.370172:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.370174:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.370177:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.370311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.370355:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.370358:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.370358:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.370360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (22)++
00000400:00000200:22.0:1619677447.370361:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.370364:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.370366:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.370367:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96729169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.370369:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.370369:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:22.0:1619677447.370370:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.370370:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000400:00000200:22.0:1619677447.370371:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.370371:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.370372:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.370373:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000800:00000200:2.0:1619677447.370373:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (24)++
00000400:00000200:18.0:1619677447.370374:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.370375:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.370375:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000400:00000200:22.0:1619677447.370376:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.370376:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.370377:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.370378:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.370379:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.370379:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.370380:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.370382:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.370382:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.370382:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677447.370383:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.370383:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.370384:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.370385:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.370385:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.370386:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677447.370386:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:22.0:1619677447.370387:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677447.370387:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.370387:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677447.370388:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677447.370389:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.370923:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.370926:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.370927:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.370927:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.370930:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.370932:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.370964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.371005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.371006:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.371008:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.371009:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.371011:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.371012:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:12.0:1619677447.371013:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.371014:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.371015:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.371017:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.371017:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.371019:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.371022:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.371075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.371083:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.371086:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.371087:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.371087:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.371091:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677447.371095:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677447.371149:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.371152:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.371153:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.371155:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.371157:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.371158:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:24.0:1619677447.371159:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.371160:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.371161:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.371161:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:24.0:1619677447.371162:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.371164:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.371164:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.371165:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.371168:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.371170:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.371171:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.371173:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.371173:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:24.0:1619677447.371174:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:18.0:1619677447.371174:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.371175:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677447.371175:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.371177:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.371177:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.371177:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.371178:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.371179:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.371180:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.371181:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:18.0:1619677447.371181:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.371183:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.2:1619677447.371183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.371185:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.371186:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.371187:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.371189:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.371190:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.371191:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.371191:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.371192:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677447.371193:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.371193:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.371194:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.371196:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.371198:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.0:1619677447.371200:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.371202:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.371203:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.371204:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:4.2:1619677447.371442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.371450:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.371453:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.371454:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.371456:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.371458:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.371460:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.371461:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.371463:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.371464:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.371465:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.371468:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.371470:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.371473:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.371474:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.371474:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.371476:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.371476:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.371477:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.371477:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.371547:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.371549:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.371550:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.371552:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677447.371554:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.371555:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.371639:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.371642:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.371643:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.371644:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.371646:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.371647:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.371695:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.371698:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.371699:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.371699:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.371702:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.371708:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.371709:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.371711:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.371712:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.371713:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.371713:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.371714:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.371763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.371805:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:4.2:1619677447.371805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677447.371807:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.371808:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.371809:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.371810:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:12.0:1619677447.371811:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.371812:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.371813:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.371813:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.371814:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.371816:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.371819:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677447.371963:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.371966:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.371967:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.371969:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.371972:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.371973:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.371974:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.371975:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.371977:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.371978:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.371981:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.371983:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.371985:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.371986:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.371987:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.371988:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.371989:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.371990:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.371991:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.371992:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.372002:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.372004:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.372005:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.372006:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677447.372133:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.372136:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.372137:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.372139:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.372142:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.372143:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677447.372145:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.372146:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.372147:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.372148:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677447.372152:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.372155:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.372158:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.372159:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:18.0:1619677447.372159:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677447.372161:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.372162:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.372162:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.372163:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677447.372174:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.372177:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.372178:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.372179:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.372181:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.372182:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.372183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.372236:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.372239:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.372240:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.372242:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:22.0:1619677447.372243:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.372245:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:2.0:1619677447.372245:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.372246:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677447.372247:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:22.0:1619677447.372249:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.372249:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.372251:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.372252:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.372252:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.372253:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677447.372253:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:22.0:1619677447.372255:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.372256:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.372257:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.372258:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.372264:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.372265:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:22.0:1619677447.372267:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677447.372268:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.372271:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.372271:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.372272:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:2.0:1619677447.372272:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:22.0:1619677447.372273:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:2.0:1619677447.372273:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677447.372274:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.372275:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:2.0:1619677447.372275:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677447.372276:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677447.372276:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:22.0:1619677447.372277:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677447.372277:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:22.0:1619677447.372278:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:2.0:1619677447.372278:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677447.372279:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677447.372289:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:6.0:1619677447.372290:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:18.0:1619677447.372291:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:6.0:1619677447.372291:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.372292:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677447.372292:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.372293:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:6.0:1619677447.372301:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.372302:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.372303:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.372303:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.372304:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.372305:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.372306:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.372383:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.372385:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.372386:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.372388:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.372390:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.372391:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.372399:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.372402:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.372403:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.372404:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.372404:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.372406:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.372566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.372572:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.372574:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.372576:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.372577:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.372578:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.372578:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.372580:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.372581:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.372584:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.372585:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.372587:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.372588:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.372589:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.372590:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.372591:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.372593:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:18.0:1619677447.372594:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.372596:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.372599:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.372600:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.372600:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.372602:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.372603:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.372604:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.372604:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:4.2:1619677447.372663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.372708:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.372711:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.372712:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.372712:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.372714:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.372718:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.372894:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.372898:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.372899:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:6.0:1619677447.372901:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677447.372901:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.372903:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.372904:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677447.372904:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.372906:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.372906:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.372907:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.372908:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.372909:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.372909:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.372911:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677447.372911:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.372913:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.372915:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.372915:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.372916:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.372917:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:2.0:1619677447.372918:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.372920:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677447.372922:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.372922:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.372922:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.372924:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677447.372924:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.372924:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.372925:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.372926:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:6.0:1619677447.372927:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.372928:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.372929:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.372931:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.372931:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.372931:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.372932:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677447.372943:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.372946:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.372947:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.372950:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.372952:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.372954:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.372955:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.372956:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.372957:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.372958:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.372962:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.372964:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.372966:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.372967:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.372968:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.372970:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.372970:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.372971:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.372972:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.372973:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.372984:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.372986:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.372987:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.372988:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:4.2:1619677447.372990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.373037:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.373039:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.373040:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.373041:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.373043:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.373046:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.373190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.373197:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.373201:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.373202:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.373205:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.373206:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:8.0:1619677447.373208:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.373209:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:8.0:1619677447.373209:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:13.0:1619677447.373210:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.373211:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96729131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.373212:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.373212:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:8.0:1619677447.373213:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.373214:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.373214:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:13.0:1619677447.373215:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:8.0:1619677447.373218:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.373220:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677447.373223:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.373223:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.373223:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:12.0:1619677447.373226:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.373227:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.373228:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.373228:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.373228:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:12.0:1619677447.373229:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.373230:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.373231:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.373232:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.373233:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.373233:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.373239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.373246:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.373248:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.373249:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.373250:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.373374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677447.373391:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677447.373395:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677447.373395:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000400:00000200:8.0:1619677447.373399:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677447.373401:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.373402:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677447.373404:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96729128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.373406:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.373407:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.373408:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677447.373413:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677447.373415:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.373418:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.373419:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677447.373419:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677447.373421:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.373422:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677447.373422:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:9.2:1619677447.373423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.373423:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:4.2:1619677447.373425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677447.373432:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.373435:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.373436:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.373436:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677447.373437:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.373439:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:9.2:1619677447.373439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.373439:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.373440:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.373441:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.373442:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:6.0:1619677447.373443:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.373444:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.373446:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677447.373447:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.373448:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.373448:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.373448:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.373449:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.373449:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.373451:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.373452:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.373452:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.373453:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.373453:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.373486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.373494:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.373496:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.373498:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.373498:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.373500:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.373503:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.373630:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.373633:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.373633:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.373636:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.373639:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.373640:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.373641:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.373643:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.373644:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.373646:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.373649:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.373651:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.373653:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.373654:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.373655:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.373657:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.373657:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.373658:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.373659:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677447.373703:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.373706:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.373707:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.373709:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.373711:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.373712:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.373714:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.373715:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.373716:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.373717:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.373721:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.373723:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.373726:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.373727:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.373727:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.373729:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.373730:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.373730:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.373731:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:4.2:1619677447.373767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.373860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.373867:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:9.2:1619677447.373869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.373870:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.373871:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.373872:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.373873:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.373874:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.373875:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.373876:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:8.0:1619677447.373876:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.373878:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.373878:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.373882:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:2.0:1619677447.373935:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.373938:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.373939:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.373941:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.373943:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.373944:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.373946:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.373947:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.373948:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.373949:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.373953:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.373956:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.373958:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.373959:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.373960:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.373961:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.373962:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.373962:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.373963:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677447.373975:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.373978:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.373979:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.373979:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.373981:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.373983:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677447.374085:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.374087:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.374088:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.374090:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.374093:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.374094:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.374096:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.374097:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.374098:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.374099:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.374101:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.374102:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.374104:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.374104:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.374105:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.374106:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.374107:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.374107:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.374108:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.374112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.374119:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.374120:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.374121:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.374122:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.374123:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.374125:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.374198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.374205:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.374208:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.374209:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.374209:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.374211:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.374216:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.374234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.374241:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.374244:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.374245:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.374246:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.374249:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.374251:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.374252:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.374254:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.374254:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.374255:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.374258:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.374260:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.374263:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.374264:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.374264:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.374266:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.374266:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.374267:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.374267:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.374346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.374364:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.374367:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.374368:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.374371:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.374373:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.374375:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.374377:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.374378:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:18.0:1619677447.374379:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.374380:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.374381:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:18.0:1619677447.374381:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677447.374381:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.374385:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.374387:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.374388:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.374390:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:18.0:1619677447.374390:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:22.0:1619677447.374391:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96729109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.374393:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.374393:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.374394:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:22.0:1619677447.374395:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.374395:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677447.374396:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.374397:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.374397:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.374398:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.374399:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:22.0:1619677447.374400:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.374402:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.374404:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.374405:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.374406:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.374407:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.374408:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.374409:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.374410:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.374410:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.374413:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.374414:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.374415:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.374415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:13.0:1619677447.374417:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.374418:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.374422:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.374425:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.374426:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.374427:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.374430:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.374433:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.374440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:8.0:1619677447.374446:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.374448:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.374449:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.374449:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.374450:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.374451:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.374791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:9.2:1619677447.374795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.374798:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.374800:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.374801:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.374802:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.374803:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.374803:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677447.374805:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:2.0:1619677447.374806:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.374807:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.374807:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.374809:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.374810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:2.0:1619677447.374812:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.0:1619677447.374819:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.374821:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.374822:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.374822:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.374823:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.374823:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.374824:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.374825:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.374825:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677447.374826:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.374829:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.374831:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.374832:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.374834:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.374835:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.374836:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.374839:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.374841:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.374843:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.374844:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.374845:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.374847:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.374847:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.374848:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.374848:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.374853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.374860:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.374863:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.374864:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.374864:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.374866:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.374869:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.374871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.374878:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.374880:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.374881:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.374882:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.374882:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.374883:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.374964:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.374967:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.374968:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.374968:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.374970:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.374973:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.375101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.375108:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.375110:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.375111:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.375111:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.375114:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677447.375117:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.375256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.375306:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.375309:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.375310:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.375311:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.375313:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.375315:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.375403:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.375406:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.375406:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.375408:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.375410:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.375411:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.375537:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.375540:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.375541:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.375541:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.375543:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.375545:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.375572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.375581:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.375583:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.375585:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.375585:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.375587:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.375590:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.375606:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677447.375608:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677447.375609:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677447.375611:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.375613:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.375615:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677447.375616:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.375617:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.375618:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.375619:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677447.375622:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.375624:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677447.375628:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.375628:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677447.375629:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677447.375631:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677447.375631:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677447.375632:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677447.375632:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:4.2:1619677447.375683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.375691:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.375693:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.375694:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.375696:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.375699:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.375701:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.375702:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.375703:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.375704:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.375705:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.375708:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.375710:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.375713:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.375713:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.375714:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.375716:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.375716:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.375717:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.375717:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.375858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.375899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.375903:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.375906:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.375907:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.375908:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.375910:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.375915:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.375916:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.375918:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.375919:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.375920:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.375920:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.375921:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.376159:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.376161:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.376162:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.376163:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.376165:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.376166:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.376252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.376300:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.376303:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.376304:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.376304:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.376306:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.376309:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677447.376411:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.376414:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.376414:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.376417:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.376420:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.376421:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.376422:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.376424:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.376425:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.376426:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.376429:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.376431:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.376433:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.376434:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.376434:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.376436:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.376436:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.376437:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.376437:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.376443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.376449:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.376451:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.376452:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.376454:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.376454:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.376456:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.376640:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.376643:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.376643:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.376645:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.376647:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.376649:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.376650:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.376652:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.376653:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.376654:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:4.2:1619677447.376655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:22.0:1619677447.376657:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.376659:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.376662:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.376663:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.376663:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.376665:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.376665:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.376666:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.376668:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.376668:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.376679:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.376680:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.376681:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.376682:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.376714:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.376717:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.376718:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.376718:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.376721:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.376723:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.376814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.376822:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.376825:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.376826:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.376826:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.376828:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.376831:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677447.376869:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.376872:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.376873:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.376875:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.376876:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.376878:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677447.376879:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.376880:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.376881:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.376882:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677447.376886:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.376887:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.376890:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.376891:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677447.376892:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677447.376894:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.376894:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.376895:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.376895:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677447.376907:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.376910:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.376911:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.376912:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.376914:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.376915:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677447.376975:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.376978:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.376979:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.376981:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.376984:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.376986:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.376987:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.376988:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.376989:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.376990:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:9.2:1619677447.376991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677447.376994:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.376997:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.377000:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.377000:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.377001:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (23)++
00000800:00000200:18.0:1619677447.377001:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (24)++
00000800:00000200:2.0:1619677447.377001:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.377002:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.377003:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (24)--
00000400:00000200:2.0:1619677447.377003:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.377004:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.377006:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.377006:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.377007:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.377007:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.377008:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.377010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.377017:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.377018:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.377019:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.377020:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.377021:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.377021:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.377021:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.377022:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.377023:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.377023:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.377027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.377034:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.377037:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.377038:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.377038:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.377038:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.377039:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.0:1619677447.377113:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677447.377115:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677447.377116:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677447.377119:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677447.377121:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.377122:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677447.377124:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.377125:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.377126:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.377127:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677447.377130:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677447.377132:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.377135:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.377135:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677447.377136:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677447.377138:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.377138:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677447.377139:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677447.377139:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:6.0:1619677447.377152:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.377154:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.377155:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.377155:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.377157:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.377160:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.377168:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.377170:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.377171:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.377173:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.377174:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.377175:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.377244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.377250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.377257:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.377259:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.377261:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.377262:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.377265:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.377267:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.377269:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.377270:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.377271:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.377272:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.377275:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.377277:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.377279:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.377280:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.377281:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.377282:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.377283:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.377283:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.377284:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.377292:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.377297:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.377300:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.377301:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.377303:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.377309:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.377310:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.377313:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.377313:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.377314:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.377315:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.377316:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.377390:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.377393:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.377394:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:13.0:1619677447.377395:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:2.0:1619677447.377395:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:13.0:1619677447.377397:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.377398:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.377398:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.377400:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:13.0:1619677447.377401:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.377402:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:2.0:1619677447.377402:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.377403:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:2.0:1619677447.377403:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.377405:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.377406:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.377410:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.377412:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.377415:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.377416:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.377416:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.377418:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.377418:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.377419:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.377420:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677447.377431:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.377434:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.377435:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.377435:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.377437:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.377439:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.377448:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.377451:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.377451:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.377452:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.377453:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.377453:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677447.377715:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.377718:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.377718:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.377721:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.377723:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.377724:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.377725:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.377727:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.377728:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.377729:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.377731:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.377733:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.377735:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.377736:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.377737:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.377738:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.377739:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.377740:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.377740:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.0:1619677447.377742:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.377746:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.377751:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:12.0:1619677447.377753:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000400:00000200:8.0:1619677447.377753:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.377755:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:8.0:1619677447.377755:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.377757:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677447.377758:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.377759:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:8.0:1619677447.377759:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.377761:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677447.377762:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.377762:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.377763:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677447.377764:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677447.377765:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96729070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677447.377767:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677447.377768:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.377768:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677447.377769:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.377770:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:12.0:1619677447.377772:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.377773:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.377774:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.377774:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.377775:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.377777:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:12.0:1619677447.377778:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.377778:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.377778:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.377778:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.377779:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.377779:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.377783:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:9.2:1619677447.377783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.377784:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677447.377784:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677447.377785:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:2.0:1619677447.377790:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.377793:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.377794:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.377794:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.377796:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.377799:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.378084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.378124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.378129:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.378132:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.378133:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.378133:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.378136:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.378138:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.378167:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.378169:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.378170:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.378172:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.378174:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.378176:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.378178:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.378179:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.378181:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.378182:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.378193:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.378195:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.378198:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.378199:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.378199:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.378202:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.378202:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.378202:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.378203:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.378232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.378240:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.378242:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.378243:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.378243:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.378245:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.378249:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.378474:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.378477:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.378478:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.378480:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.378482:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.378483:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.378485:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.378486:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.378487:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.378488:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.378491:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.378493:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.378496:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.378496:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.378497:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.378499:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.378499:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.378500:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.378502:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.378502:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.378514:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.378516:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.378517:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.378517:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:4.2:1619677447.378527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.378549:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.378551:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.378552:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.378555:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.378556:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.378558:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.378566:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.378569:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.378570:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.378570:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.378571:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.378572:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.378589:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.378591:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.378592:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.378593:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.378595:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.378596:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.378697:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677447.378700:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677447.378701:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677447.378703:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.378706:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.378707:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.378709:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.378710:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.378711:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.378712:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.378715:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.378717:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.378719:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.378720:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.378721:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.378723:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.378723:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677447.378724:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677447.378724:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:6.0:1619677447.378735:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.378738:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.378739:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.378741:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.378744:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.378746:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.378747:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.378748:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.378750:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.378750:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.378754:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.378756:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.378759:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.378760:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.378760:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.378762:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.378763:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.378764:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.378764:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.378768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.378776:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.378778:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.378780:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.378780:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.378782:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.378784:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.378888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:4.2:1619677447.378892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.378895:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.378898:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.378899:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.378901:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.378902:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:24.0:1619677447.378903:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.378905:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:13.0:1619677447.378905:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:24.0:1619677447.378906:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.378906:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.378907:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.378908:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.378909:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.378909:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677447.378910:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677447.378910:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677447.378914:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.378916:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.378919:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.378920:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.378920:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.378922:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.378922:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.378924:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.378925:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.378926:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.378931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.378939:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677447.378939:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677447.378942:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:8.0:1619677447.378942:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.378942:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677447.378943:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.378944:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.378944:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677447.378946:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.378949:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.379090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.379097:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.379099:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.379100:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.379101:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.379103:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.379107:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.379172:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.379174:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.379175:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.379177:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.379179:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.379180:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.379182:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.379183:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.379184:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.379185:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.379188:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.379190:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.379192:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.379193:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.379194:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.379195:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.379196:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.379197:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.379197:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.379202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.379209:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:4.2:1619677447.379212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.379212:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.379213:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.379213:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.379215:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.379217:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.379223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.379230:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.379233:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.379234:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.379234:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.379235:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.379236:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:12.0:1619677447.379315:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.379318:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.379318:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.379320:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.379322:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.379322:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.379573:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.379576:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.379577:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.379578:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.379580:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.379581:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.379593:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.379596:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.379597:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.379598:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.379598:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.379600:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.379673:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.379675:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.379676:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:4.2:1619677447.379676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:22.0:1619677447.379678:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.379681:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.379682:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:8.0:1619677447.379682:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:22.0:1619677447.379684:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.379685:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.379685:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:22.0:1619677447.379686:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.379687:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.379689:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677447.379689:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.379692:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.379692:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.379694:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:8.0:1619677447.379694:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.379696:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.379697:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.379698:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:8.0:1619677447.379698:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.379698:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.379699:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.379700:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000400:00000200:8.0:1619677447.379700:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.379701:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.379702:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.379703:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.379704:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:8.0:1619677447.379704:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677447.379706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677447.379706:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.379709:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.379710:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.379711:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.379713:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.379713:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.379714:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.379714:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.379714:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677447.379715:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:6.0:1619677447.379717:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.379717:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.379718:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.379718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.379718:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.379719:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.379719:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.379720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677447.379721:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.379725:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677447.379727:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.379728:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.379729:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.379729:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.379732:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.379735:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.379736:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.379738:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.379739:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.379741:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.379743:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.379745:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.379746:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.379747:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.379748:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.379749:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.379753:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.379755:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.379758:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.379758:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.379759:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.379761:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.379761:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.379762:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.379762:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.379773:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.379776:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.379777:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.379779:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.379780:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.379782:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.379789:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.379792:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.379793:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.379794:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.379794:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.379796:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.379843:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:9.2:1619677447.379843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.379846:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.379846:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.379847:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.379849:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.379851:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:22.0:1619677447.379852:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677447.379853:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.379854:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.379855:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.379857:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.379860:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.379861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.379868:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.379871:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.379871:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.379872:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.379872:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.379873:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.379989:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.379991:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.379992:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.379992:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.379995:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.379997:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.380321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.380342:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.380345:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.380346:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.380350:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.380352:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.380354:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.380355:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.380357:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.380359:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.380360:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.380364:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.380367:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.380369:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.380370:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.380371:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.380373:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.380373:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677447.380374:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677447.380374:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.380375:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.380376:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.380377:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.380377:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:9.2:1619677447.380379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:22.0:1619677447.380380:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.380383:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.380386:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.380388:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.380389:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.380390:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.380391:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.380393:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.380395:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.380398:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.380399:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.380399:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.380401:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.380403:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.380498:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.380501:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.380502:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.380503:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.380506:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.380506:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.380527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.380534:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.380537:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.380538:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.380540:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.380542:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.380544:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.380545:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.380546:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.380547:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.380548:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.380551:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.380553:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.380555:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.380556:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.380557:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.380558:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.380559:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.380559:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.380560:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.380627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.380634:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.380637:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.380638:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.380638:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.380640:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.380644:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.0:1619677447.380759:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677447.380762:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677447.380763:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677447.380765:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677447.380768:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.380769:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677447.380770:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.380772:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.380773:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.380774:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677447.380776:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677447.380778:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.380779:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.380780:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677447.380780:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.380781:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.380782:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677447.380783:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677447.380784:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677447.380785:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:9.2:1619677447.380787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.380795:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.380797:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.380797:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.380799:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677447.380799:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677447.380800:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677447.380801:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.380801:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677447.380802:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677447.380802:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:4.2:1619677447.380960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.380988:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.380991:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.380992:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.380993:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.380996:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.380997:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.380998:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96729026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.380999:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.381000:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.381001:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.381004:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.381006:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.381008:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.381009:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.381009:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.381011:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.381011:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.381012:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.381013:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.381014:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677447.381023:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.381025:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.381026:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.381027:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:13.0:1619677447.381028:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.381029:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.381029:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.381030:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.381031:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.381031:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.381332:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677447.381333:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677447.381335:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.381336:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.381337:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.381337:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.381338:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.381339:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:12.0:1619677447.381340:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:12.0:1619677447.381341:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96729024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677447.381342:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677447.381343:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677447.381344:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:12.0:1619677447.381347:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.381349:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.381350:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.381351:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677447.381351:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.381352:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.381352:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.381352:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.381354:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.381355:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677447.381356:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677447.381356:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:13.0:1619677447.381366:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.381368:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.381369:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.381370:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.381372:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.381373:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.381386:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.381388:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.381389:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.381390:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.381390:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.381392:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.381450:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.381453:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.381454:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.381454:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.381456:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.381458:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.381467:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.381470:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.381471:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.381471:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.381472:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.381472:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.381562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.381573:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.381576:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.381576:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677447.381578:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.381581:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.381582:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.381584:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.381586:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.381587:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.381588:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:12.0:1619677447.381594:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:2.0:1619677447.381594:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.381597:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:2.0:1619677447.381597:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:12.0:1619677447.381598:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.381599:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677447.381600:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.381600:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:13.0:1619677447.381601:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.381601:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:12.0:1619677447.381603:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.381603:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.381604:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.381604:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.381605:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:22.0:1619677447.381618:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.381621:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.381622:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.381622:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.381624:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.381627:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677447.381816:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.381819:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.381819:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.381822:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.381824:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.381825:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.381827:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.381829:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.381830:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.381831:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.381834:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.381836:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.381838:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.381839:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.381839:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.381841:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.381842:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.381842:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.381843:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.381847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.381854:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.381856:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.381856:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.381858:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.381858:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.381860:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.382035:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.382038:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.382038:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.382040:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.382042:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.382042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:6.0:1619677447.382044:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.382046:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.382047:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.382048:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.382049:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.382053:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.382056:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.382058:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.382059:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.382060:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.382062:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.382062:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.382063:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.382064:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.382083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.382086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677447.382090:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.382093:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.382094:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.382094:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.382096:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.382100:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.382103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.382110:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.382113:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.382114:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.382114:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.382115:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.382116:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.382286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.382293:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.382300:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.382302:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677447.382305:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.382307:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.382309:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.382310:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.382312:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.382313:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.382315:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.382321:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.382324:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.382328:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.382329:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.382330:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.382332:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.382332:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.382333:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.382334:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.382346:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.382348:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.382349:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.382350:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.382352:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.382353:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:13.0:1619677447.382354:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677447.382355:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.382356:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.382357:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:18.0:1619677447.382358:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.382360:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.382360:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.382361:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.382363:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.382365:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.382366:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.382367:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.382369:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.382371:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:2.0:1619677447.382371:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.382372:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96729009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.382373:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.382374:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.382375:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.382376:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.382379:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.382380:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:2.0:1619677447.382380:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.382381:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.382382:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.382382:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.382384:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.382385:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (23)++
00000800:00000200:2.0:1619677447.382386:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.382387:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (24)--
00000800:00000200:2.0:1619677447.382387:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.382387:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.382388:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.382389:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.382389:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.382390:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.382390:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.382391:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.382391:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.382391:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677447.382393:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.382393:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.382394:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:18.0:1619677447.382395:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.382395:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:22.0:1619677447.382396:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677447.382396:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.382396:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:18.0:1619677447.382397:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.382399:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:18.0:1619677447.382399:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.382400:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.382400:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96729008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.382400:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:18.0:1619677447.382402:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.382403:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:18.0:1619677447.382404:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.382405:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.382406:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:18.0:1619677447.382408:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.382411:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.382414:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.382414:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.382415:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.382416:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.382417:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.382418:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.382419:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.382419:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.382430:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.382432:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.382433:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.382434:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:4.2:1619677447.382544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.382597:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.382600:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.382601:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.382601:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.382603:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.382606:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.382645:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.382647:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.382649:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.382650:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.382650:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:13.0:1619677447.382652:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.382653:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.382653:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677447.382654:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677447.382656:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.382659:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.382660:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:8.0:1619677447.382662:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96729003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.382663:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.382664:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.382665:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677447.382668:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.382669:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:8.0:1619677447.382672:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.382673:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:8.0:1619677447.382673:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:8.0:1619677447.382675:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:8.0:1619677447.382675:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677447.382676:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677447.382677:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:13.0:1619677447.382919:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.382921:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.382922:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.382923:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.382925:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.382926:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.382952:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.382954:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.382955:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.382957:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.382960:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.382961:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.382962:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96729000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.382964:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.382965:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.382966:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.382971:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.382974:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.382977:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.382978:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.382979:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.382980:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.382985:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.382985:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.382986:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.382997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.383005:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.383008:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.383009:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.383010:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.383012:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.383016:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:12.0:1619677447.383079:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.383081:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:4.2:1619677447.383081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:12.0:1619677447.383082:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.383083:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.383085:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.383086:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.383113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:9.2:1619677447.383119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.383120:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.383123:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.383124:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.383125:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.383127:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.383127:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:24.0:1619677447.383129:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:18.0:1619677447.383129:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:24.0:1619677447.383130:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96728995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.383130:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:24.0:1619677447.383131:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.383131:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.383133:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.383133:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.383134:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.383137:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.383137:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677447.383139:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.383141:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.383142:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.383143:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.383144:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.383145:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.383145:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.383146:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.383174:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.383177:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.383178:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.383180:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.383182:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.383184:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.383185:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.383186:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.383187:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.383189:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.383192:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.383194:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.383197:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.383198:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.383199:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.383201:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.383201:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.383202:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.383203:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.383632:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.383635:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.383635:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.383638:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.383639:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.383641:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.383648:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.383651:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.383652:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.383653:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.383653:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.383655:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.383716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677447.383723:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677447.383726:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677447.383727:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677447.383729:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.383732:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.383734:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677447.383735:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.383736:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.383737:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.383738:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677447.383742:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.383744:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677447.383747:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.383747:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:8.0:1619677447.383748:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677447.383750:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:8.0:1619677447.383750:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677447.383751:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677447.383751:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.2:1619677447.383756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:4.2:1619677447.383760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.383764:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.383766:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.383768:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.383768:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.383770:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.383773:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:6.0:1619677447.383960:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.383962:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.383963:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.383966:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.383969:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.383970:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.383971:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.383973:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.383974:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.383975:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.383978:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.383980:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.383983:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.383984:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.383984:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.383986:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.383987:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.383987:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.383988:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.383992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.383999:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.384002:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.384003:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.384003:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.384005:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.384009:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:9.2:1619677447.384010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.384018:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.384020:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.384021:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (23)++
00000800:00000200:2.0:1619677447.384021:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (23)++
00000800:00000200:18.0:1619677447.384022:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (25)--
00000800:00000200:2.0:1619677447.384022:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (25)--
00000400:00000200:18.0:1619677447.384023:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.384023:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.384025:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.384025:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.384028:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:22.0:1619677447.384096:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.384098:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.384099:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.384099:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.384101:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.384103:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.384116:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.384120:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.384120:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.384121:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.384122:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.384122:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.384220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.384227:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.384230:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.384231:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.384233:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.384235:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:24.0:1619677447.384236:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.384237:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:24.0:1619677447.384238:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.384239:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.384239:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:24.0:1619677447.384241:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.384241:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.384243:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.384243:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.384244:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.384245:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.384247:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.384248:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.384248:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.384249:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.384251:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.384251:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.384254:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.384254:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:2.0:1619677447.384254:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.384255:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.384256:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.384257:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.384257:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.384258:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677447.384258:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.384259:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:2.0:1619677447.384259:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.384260:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677447.384260:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.384262:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.384262:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.384263:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.384264:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:4.2:1619677447.384266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.2:1619677447.384268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.384273:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677447.384275:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.0:1619677447.384275:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.384276:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677447.384277:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:8.0:1619677447.384277:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677447.384278:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.384279:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.384280:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.384283:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.384415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.384451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677447.384542:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.384545:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.384546:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.384547:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.384549:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.384550:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.384613:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.384616:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.384616:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.384618:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.384621:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.384622:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:2.0:1619677447.384622:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:22.0:1619677447.384623:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.384625:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.384625:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.384625:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:22.0:1619677447.384626:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.384628:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677447.384629:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.384632:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.384634:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:2.0:1619677447.384634:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.384636:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677447.384636:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:22.0:1619677447.384637:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:2.0:1619677447.384637:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728978] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.384638:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.384639:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.384639:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.384639:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.384640:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.384641:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:2.0:1619677447.384641:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.384642:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.384643:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677447.384644:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677447.384646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.384646:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.384649:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.384650:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.384651:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.384654:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.384654:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.384654:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.384656:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.384656:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.384656:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:18.0:1619677447.384657:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.384657:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.384658:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.384659:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677447.384661:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:6.0:1619677447.384661:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.384662:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.384663:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.384666:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.384669:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.384670:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.384672:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.384673:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.384675:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.384780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.384965:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.384968:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.384969:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.384970:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.384972:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.384973:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.384979:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.384982:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.384983:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.384985:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.384987:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.384989:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.384990:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96728973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.384992:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.384993:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.384994:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:6.0:1619677447.384995:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:24.0:1619677447.384997:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.384998:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:24.0:1619677447.384999:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:22.0:1619677447.384999:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.384999:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.385001:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.385002:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.385003:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.385004:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.385005:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:6.0:1619677447.385005:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.385006:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.385007:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.385007:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.385013:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.385016:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.385016:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.385017:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.385018:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.385019:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.385097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.385104:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.385107:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.385108:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.385108:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.385111:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.385114:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.385156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.385199:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.385202:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.385203:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.385204:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.385206:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.385209:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677447.385274:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.385277:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.385278:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.385280:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.385282:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.385284:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677447.385285:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.385286:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.385287:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.385288:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677447.385292:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:22.0:1619677447.385299:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:18.0:1619677447.385299:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.385302:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:18.0:1619677447.385302:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677447.385303:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.385303:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)++
00000800:00000200:18.0:1619677447.385303:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (23)++
00000800:00000200:6.0:1619677447.385304:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.385305:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.385306:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (23)--
00000800:00000200:18.0:1619677447.385306:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.385307:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.385307:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.385308:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.385320:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.385322:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.385323:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.385323:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.385325:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.385326:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.385480:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.385483:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.385484:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.385486:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.385488:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.385489:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.385491:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.385492:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.385493:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.385494:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.385497:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.385499:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.385502:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.385502:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.385503:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.385504:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.385505:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.385506:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.385507:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.385507:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.385518:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.385520:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.385520:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.385521:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:22.0:1619677447.385556:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.385559:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.385560:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.385563:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.385566:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.385567:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.385569:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.385570:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.385571:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.385572:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.385576:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.385578:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.385580:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677447.385580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.385581:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.385582:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.385583:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.385584:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.385585:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.385585:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.385630:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.385633:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.385634:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.385634:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.385636:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.385639:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.385758:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.385760:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.385761:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.385763:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677447.385765:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.385766:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.385779:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.385782:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.385783:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.385783:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.385784:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.385786:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.385829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.385836:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.385839:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.385840:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.385840:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.385843:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.385846:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.385965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.386004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.386089:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.386091:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.386092:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.386092:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.386095:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.386097:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677447.386124:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.386127:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.386128:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.386130:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.386133:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.386134:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.386136:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.386137:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.386138:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.386139:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.386142:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.386144:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.386147:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.386148:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.386149:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.386150:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.386151:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.386152:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.386152:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:4.2:1619677447.386287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.386315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.386324:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.386327:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.386328:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.386329:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.386331:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.386335:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.386360:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:9.2:1619677447.386362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:6.0:1619677447.386363:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.386364:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.386364:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.386366:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.386369:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:6.0:1619677447.386369:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.386372:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.386373:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.386374:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.386376:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.386379:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.386380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677447.386385:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.386388:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.386389:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.386389:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.386389:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.386390:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677447.386404:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.386406:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.386407:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.386409:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.386411:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.386412:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.386441:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.386444:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.386445:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.386446:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.386447:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.386449:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677447.386631:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677447.386633:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677447.386634:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677447.386637:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.386640:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.386642:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677447.386643:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.386644:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.386645:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.386646:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:9.2:1619677447.386646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:18.0:1619677447.386650:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.386651:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:24.0:1619677447.386652:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.386655:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:18.0:1619677447.386655:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.386656:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.386656:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:8.0:1619677447.386656:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:18.0:1619677447.386657:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:24.0:1619677447.386658:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.386659:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.386660:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677447.386661:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677447.386661:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.386662:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:24.0:1619677447.386663:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:18.0:1619677447.386663:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:18.0:1619677447.386664:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.386664:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.386666:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.386668:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.386670:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96728944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.386671:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.386673:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.386674:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.386677:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.386679:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.386681:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.386682:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.386683:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.386684:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.386685:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.386685:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.386686:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677447.386687:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.386690:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.386691:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.386692:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.386694:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.386695:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.386698:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.386700:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.386701:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.386701:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.386704:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.386707:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.386718:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.386721:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.386721:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.386722:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.386723:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.386724:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.387139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.387147:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.387150:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.387151:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.387151:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.387153:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.387155:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.387159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:4.2:1619677447.387164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.387166:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.387169:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.387170:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.387170:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.387171:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.387171:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.387197:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:20.0:1619677447.387199:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.387201:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:18.0:1619677447.387201:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.387201:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:20.0:1619677447.387203:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.387204:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.387205:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.387206:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.387207:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.387207:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.387208:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.387210:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96728935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.387211:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.387212:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:18.0:1619677447.387212:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.387213:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.387214:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.387214:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.387215:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.387216:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.387217:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.387217:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.387219:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.387220:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.387221:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.387221:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.387222:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.387222:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677447.387222:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.387223:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677447.387224:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.387224:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.387225:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.387226:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:18.0:1619677447.387226:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.387227:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:18.0:1619677447.387227:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.387227:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.387228:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677447.387229:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.387238:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677447.387238:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.387240:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:6.0:1619677447.387240:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.387241:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:20.0:1619677447.387241:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.387241:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.387244:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.387245:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.387247:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:6.0:1619677447.387247:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.387357:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.387360:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.387361:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.387363:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.387365:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.387366:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.387368:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.387370:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.387371:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.387372:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.387375:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.387377:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.387379:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.387380:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.387381:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.387383:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.387383:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.387384:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.387385:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.387404:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:4.2:1619677447.387423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.387430:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.387433:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.387434:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.387436:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.387439:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.387440:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.387442:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.387443:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.387444:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.387445:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.387448:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.387450:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.387453:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.387453:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.387454:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.387456:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.387457:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.387457:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.387458:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677447.387695:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.387698:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.387699:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.387701:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.387704:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.387705:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677447.387706:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.387708:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.387709:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.387710:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:18.0:1619677447.387713:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.387714:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677447.387715:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000400:00000200:18.0:1619677447.387717:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.387717:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:18.0:1619677447.387720:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.387721:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:2.0:1619677447.387721:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.387722:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.387722:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677447.387723:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:18.0:1619677447.387724:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.387725:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.387725:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000400:00000200:18.0:1619677447.387726:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.387726:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.387728:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.387728:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:18.0:1619677447.387729:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.387729:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.387731:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.387733:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.387734:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.387734:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.387736:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.387736:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.387737:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.387737:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:6.0:1619677447.387750:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.387752:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.387753:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.387753:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.387755:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.387757:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.387889:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.387891:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.387892:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.387894:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.387896:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.387897:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.387906:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.387909:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.387909:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.387910:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.387911:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.387912:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.387948:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.387951:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.387952:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.387954:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.387957:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.387958:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.387959:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.387961:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.387962:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.387963:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.387966:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677447.387968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.387968:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.387970:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.387970:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.387971:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.387973:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.387973:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.387975:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.387976:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.387976:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.387979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.387987:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.387988:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.387988:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:18.0:1619677447.387989:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.387989:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.387990:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.387991:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.387992:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.387992:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677447.387997:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.388007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.388015:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.388017:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.388018:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.388019:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.388021:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.388024:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.388170:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.388173:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.388173:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.388175:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.388177:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.388178:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.388192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.388199:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.388202:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.388203:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.388205:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.388207:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.388209:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.388210:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.388211:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.388212:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.388213:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.388216:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.388218:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.388220:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.388221:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.388221:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.388223:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.388223:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.388224:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.388225:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.388343:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.388346:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.388347:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.388347:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.388349:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.388351:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.388369:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.388372:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.388372:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.388373:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.388374:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.388374:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.388431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.388445:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.388447:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.388448:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.388451:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.388453:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.388455:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.388456:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.388458:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.388458:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.388460:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.388465:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.388468:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.388471:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.388472:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.388472:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677447.388473:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.388474:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:22.0:1619677447.388475:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.388475:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.388476:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.388477:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:20.0:1619677447.388477:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.388479:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.388480:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.388482:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96728920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.388483:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.388485:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677447.388485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677447.388486:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.388491:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.388492:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.388494:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677447.388495:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.388496:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677447.388497:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677447.388497:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.388498:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.388498:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677447.388499:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.388500:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.388501:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.388502:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.388502:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.388502:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.388508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.388512:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.388515:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.388516:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677447.388516:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677447.388516:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.388517:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.388517:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677447.388519:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.388520:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.388520:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.388522:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.388525:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.388560:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.388562:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.388563:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.388564:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.388566:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.388567:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.388622:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.388625:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.388626:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.388626:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.388628:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.388631:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.388802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.388843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.388851:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.388853:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.388855:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.388855:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.388857:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.388860:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.388862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.388869:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.388871:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.388872:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.388872:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.388873:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.388873:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.388978:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:22.0:1619677447.389032:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677447.389034:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677447.389037:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:22.0:1619677447.389038:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.389039:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.389040:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.389041:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.389042:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.389043:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:22.0:1619677447.389044:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.389046:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.389047:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.389048:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:6.0:1619677447.389051:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.389052:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.389052:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677447.389054:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677447.389054:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.389058:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.389058:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:22.0:1619677447.389059:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:22.0:1619677447.389061:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:22.0:1619677447.389062:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677447.389062:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677447.389063:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:4.2:1619677447.389192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.389199:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.389202:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.389203:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.389205:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.389208:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.389209:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.389211:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.389212:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.389213:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.389214:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.389217:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.389219:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.389222:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.389222:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.389223:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.389225:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.389225:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.389226:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.389226:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.389273:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.389276:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.389277:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.389279:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.389282:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.389283:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.389285:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.389287:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.389288:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.389289:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.389297:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.389300:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.389303:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.389304:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.389305:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.389306:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.389307:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.389309:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.389309:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.389310:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.389313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.389321:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677447.389321:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677447.389324:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.389324:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677447.389324:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:22.0:1619677447.389325:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:2.0:1619677447.389325:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.389326:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.389327:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.389330:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.389345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.389352:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.389355:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.389355:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.389355:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.389356:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.389357:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.389371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.389555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.389562:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.389565:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.389566:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.389567:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.389569:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.389572:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:2.0:1619677447.389601:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.389604:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.389605:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.389607:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.389609:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.389611:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.389612:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.389613:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.389614:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.389615:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.389619:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.389621:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.389623:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.389624:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.389624:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.389626:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.389627:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.389627:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.389628:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:4.2:1619677447.389704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.389751:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.389754:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.389755:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.389755:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.389757:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.389760:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.389772:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.389774:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.389777:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.389779:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.389781:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.389782:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.389783:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.389784:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.389786:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.389789:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.389791:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.389794:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.389795:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.389795:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.389797:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.389798:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.389798:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.389799:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.389811:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.389814:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.389814:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.389816:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.389818:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.389819:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.389906:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:22.0:1619677447.389915:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.389918:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.389918:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.389919:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.389922:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.389925:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.389950:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.389952:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.389953:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.389955:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677447.389956:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.389958:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.389958:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.389960:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.389961:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.389961:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.389963:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.389965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.389966:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677447.389973:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:13.0:1619677447.389973:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.389976:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:13.0:1619677447.389976:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.389977:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.389977:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677447.389977:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.389978:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.389978:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:18.0:1619677447.389980:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.389980:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.389984:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.390015:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.390018:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.390019:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.390021:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.390024:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:9.2:1619677447.390310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.390317:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.390320:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.390321:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.390321:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.390323:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.390327:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.390330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677447.390336:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.390338:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.390339:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.390339:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.390340:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.390341:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.390435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.390443:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.390446:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.390447:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.390447:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.390449:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.390453:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.390459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.390499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.390504:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.390507:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.390508:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.390509:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.390511:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.390516:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.390517:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.390519:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.390520:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.390521:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.390521:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.390522:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:5.0:1619677447.390527:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.390528:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.390530:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.390531:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.390533:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.390534:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.390535:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.390536:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.390539:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.390541:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.390544:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.390545:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.390546:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.390548:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.390548:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.390549:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.390550:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.390562:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.390564:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.390565:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.390567:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.390569:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.390570:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.390733:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.390736:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.390736:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.390738:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.390739:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.390741:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.390781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.390789:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.390791:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.390793:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.390794:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.390797:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.390798:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.390800:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.390801:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.390802:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.390803:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.390806:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.390808:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.390810:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.390811:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.390812:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.390813:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.390814:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.390815:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.390817:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.390817:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.390822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.390830:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.390832:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.390832:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.390833:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.390911:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.390913:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.390914:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.390915:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.390917:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.390920:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.390929:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.390931:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.390932:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.390934:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.390937:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.390938:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.390939:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.390940:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.390942:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.390943:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.390946:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.390948:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.390951:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.390952:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.390952:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.390954:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.390955:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.390955:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.390956:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.391053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.391070:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.391072:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.391073:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.391076:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.391079:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.391081:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.391084:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.391086:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.391087:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.391088:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:18.0:1619677447.391089:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.391090:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677447.391093:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.391095:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:2.0:1619677447.391097:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.391098:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.391098:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.391100:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96728886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.391101:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.391103:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.391104:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.391107:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.391110:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.391111:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.391112:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677447.391112:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677447.391112:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.391113:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.391113:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.391113:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.391115:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.391115:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677447.391115:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677447.391116:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677447.391116:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:6.0:1619677447.391116:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.391116:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.391117:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.391118:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:6.0:1619677447.391118:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.391118:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (23)++
00000800:00000200:18.0:1619677447.391119:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (24)--
00000800:00000200:2.0:1619677447.391119:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (24)--
00000800:00000200:2.0:1619677447.391119:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677447.391121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.391121:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.391130:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677447.391131:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.391132:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.391132:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:22.0:1619677447.391133:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.391133:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.391133:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:18.0:1619677447.391134:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.391134:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.391134:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.391136:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.391137:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:22.0:1619677447.391138:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:18.0:1619677447.391138:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.391140:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.391141:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.391147:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677447.391148:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.391151:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:18.0:1619677447.391151:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.391151:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:6.0:1619677447.391152:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.391153:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.391154:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.391155:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.391155:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.391156:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.391156:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000400:00000200:20.0:1619677447.391157:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.391157:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000400:00000200:2.0:1619677447.391157:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.391158:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:2.0:1619677447.391159:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.391160:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677447.391161:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.391162:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96728883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.391163:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.391164:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.391165:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.391166:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.391167:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677447.391167:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.391170:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.391171:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:2.0:1619677447.391172:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.391173:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.391175:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000400:00000200:20.0:1619677447.391176:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.391176:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.391178:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.391179:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677447.391179:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:9.2:1619677447.391180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.0:1619677447.391180:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.391181:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677447.391181:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.391182:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.391182:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.391182:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.391183:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677447.391183:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677447.391183:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.391184:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.391184:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.391186:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.391186:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.391187:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677447.391187:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:2.0:1619677447.391187:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677447.391189:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:8.0:1619677447.391189:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:2.0:1619677447.391189:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.391189:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:24.0:1619677447.391190:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677447.391190:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000400:00000200:8.0:1619677447.391190:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.391191:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:9.2:1619677447.391193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:8.0:1619677447.391193:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.391196:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677447.391200:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677447.391201:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:5.0:1619677447.391201:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677447.391202:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677447.391203:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:12.0:1619677447.391203:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.391203:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:18.0:1619677447.391204:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.391204:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677447.391205:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677447.391206:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.391206:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.391206:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677447.391208:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.391209:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.391212:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677447.391222:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.391224:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.391225:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.391227:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.391227:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.391230:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.391491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.391502:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.391505:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.391506:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.391508:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.391511:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.391513:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.391514:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.391516:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.391517:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.391519:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677447.391524:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.391527:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.391530:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.391531:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.391532:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:13.0:1619677447.391533:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.391534:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.391534:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:13.0:1619677447.391535:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677447.391535:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:13.0:1619677447.391536:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.391536:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:13.0:1619677447.391538:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.391540:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.391541:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.391547:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.391550:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.391551:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.391551:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.391554:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.391557:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677447.391697:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.391700:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.391702:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.391703:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.391704:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.391706:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677447.391707:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.391708:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.391709:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.391710:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677447.391714:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.391716:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677447.391718:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.391719:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677447.391719:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.391721:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.391721:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.391722:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.391723:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.391727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.391735:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.391737:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.391738:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.391738:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.391741:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.391743:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:12.0:1619677447.391800:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.391803:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.391803:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.391805:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.391807:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.391808:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.391818:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.391821:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.391822:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.391823:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.391824:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.391825:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.391928:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.391931:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.391931:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.391933:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.391936:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.391938:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.391939:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.391941:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.391941:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.391943:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.391946:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.391948:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.391950:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.391951:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.391952:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.391954:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.391954:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.391955:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.391956:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.391980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.392033:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.392036:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.392037:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.392038:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.392041:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.392043:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.392044:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.392045:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.392046:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.392048:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677447.392050:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.392052:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.392055:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.392056:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.392056:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.392058:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.392058:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.392059:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.392060:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677447.392125:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.392128:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.392129:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.392130:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.392133:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.392134:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.392135:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728869] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.392137:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.392138:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.392139:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.392141:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.392142:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.392144:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.392144:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.392144:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.392146:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.392146:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.392147:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.392147:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.392152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.392159:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.392161:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.392161:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.392162:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.392163:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.392164:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.392257:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.392261:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.392261:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.392263:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.392265:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.392266:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.392336:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.392338:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677447.392340:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677447.392341:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.392343:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.392344:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.392346:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.392347:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.392348:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.392349:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.392353:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.392354:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.392357:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.392357:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.392358:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.392360:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.392361:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.392362:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.392363:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.392364:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:9.2:1619677447.392367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.392371:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.392373:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.392374:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.392375:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677447.392375:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.392378:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.392379:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.392379:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.392381:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.392384:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.392385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.392392:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.392395:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.392395:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.392396:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.392397:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.392398:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677447.392432:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.392436:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.392437:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.392438:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.392441:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.392443:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.392444:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.392445:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.392446:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.392447:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.392450:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.392452:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.392455:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.392456:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.392457:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.392458:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.392459:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.392460:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.392460:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677447.392472:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.392475:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.392476:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.392478:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.392479:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.392480:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.392491:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.392494:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.392496:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.392496:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.392498:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.392499:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.392501:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.392501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:5.0:1619677447.392502:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.392503:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.392504:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.392508:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.392509:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.392512:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.392513:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.392513:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.392515:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.392516:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.392517:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.392517:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.392521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.392529:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.392531:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.392532:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.392532:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.392535:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.392537:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.392543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.392548:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.392551:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.392552:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.392552:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.392554:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.392557:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.392924:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.392927:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.392928:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.392929:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.392932:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.392933:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.392934:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.392936:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.392937:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.392938:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.392941:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.392943:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.392945:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.392946:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.392947:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.392948:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.392949:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.392950:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.392950:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:6.0:1619677447.392962:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.392964:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.392965:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.392965:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.392968:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.392970:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677447.393088:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.393091:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.393092:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.393094:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.393096:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.393097:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.393099:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.393100:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.393101:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.393102:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:4.2:1619677447.393104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:18.0:1619677447.393105:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.393106:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.393107:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.393108:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.393108:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.393110:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.393111:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.393112:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.393113:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.393113:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.393116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.393125:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.393126:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.393126:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.393127:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:2.0:1619677447.393127:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.393128:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.393129:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.393129:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.393130:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677447.393131:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.393153:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.393156:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.393157:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.393157:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.393159:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.393162:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.393275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.393282:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.393284:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.393285:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.393287:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.393290:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.393292:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.393299:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.393300:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.393301:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.393302:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.393305:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.393307:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.393310:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.393310:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.393311:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.393313:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.393313:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.393314:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.393314:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.393341:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.393344:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.393344:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.393346:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.393348:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.393350:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.393353:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.393354:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.393354:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.393355:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.393356:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677447.393410:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.393413:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.393415:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.393417:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.393417:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.393419:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.393421:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.393422:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.393423:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.393424:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.393428:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.393430:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.393433:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.393434:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.393434:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.393436:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.393437:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.393438:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.393438:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.393679:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.393682:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.393683:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.393685:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.393687:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.393688:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.393690:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.393691:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.393692:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.393693:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.393694:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:6.0:1619677447.393696:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.393697:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:6.0:1619677447.393698:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.393698:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.393700:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.393701:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.393701:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.393702:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.393702:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:18.0:1619677447.393703:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.393704:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.393704:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.393704:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.393705:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.393705:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:18.0:1619677447.393706:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.393707:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.393708:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.393712:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.393715:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.393718:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.393719:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.393719:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.393719:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677447.393721:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.393721:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:18.0:1619677447.393722:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.393722:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000400:00000200:5.0:1619677447.393722:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.393723:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:20.0:1619677447.393724:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.393724:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:5.0:1619677447.393724:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.393726:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677447.393735:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677447.393735:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.393737:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:22.0:1619677447.393738:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.393738:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677447.393738:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:22.0:1619677447.393739:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.393740:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677447.393740:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:22.0:1619677447.393741:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.393743:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.393743:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.393744:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.393745:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.393747:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.393748:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677447.393820:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.393823:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.393824:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.393826:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.393828:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.393829:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.393831:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.393832:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.393833:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.393834:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.393837:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.393839:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.393840:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.393841:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.393842:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.393843:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.393843:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.393844:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.393845:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.393846:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:9.2:1619677447.393849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.393857:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:4.2:1619677447.393858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.393858:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.393859:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:2.0:1619677447.393859:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.393860:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.393860:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.393861:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.393861:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.393862:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677447.393862:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.393905:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.393907:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.393908:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.393909:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.393911:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.393913:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.393921:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.393924:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.393925:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.393925:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.393926:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.393927:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.394059:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.394062:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.394063:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.394064:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.394066:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.394069:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.394082:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.394084:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.394085:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.394085:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.394086:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.394087:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.394113:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.394116:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.394117:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.394118:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.394120:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.394120:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677447.394133:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.394135:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.394136:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.394137:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.394139:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.394140:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.394171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.394190:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.394192:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.394193:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.394196:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.394198:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.394200:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.394202:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.394203:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.394205:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.394206:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.394210:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.394212:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:9.2:1619677447.394215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.394215:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.394215:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.394216:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.394218:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.394218:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.394220:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.394221:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.394221:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677447.394223:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.394226:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.394227:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.394227:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.394229:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.394231:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:9.2:1619677447.394232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.394234:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.394235:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.394235:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.394235:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.394237:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.394239:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.394240:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.394241:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.394241:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.394242:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.394348:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.394351:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.394353:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.394354:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.394355:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.394357:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.394358:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.394360:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.394361:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.394362:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.394365:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.394368:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:4.2:1619677447.394370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.0:1619677447.394371:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.394373:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.394374:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.394375:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.394376:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:24.0:1619677447.394377:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677447.394377:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.394378:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.394379:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:24.0:1619677447.394380:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.394381:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.394383:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.394386:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.394388:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.394389:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96728844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.394389:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:24.0:1619677447.394390:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.394391:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.394392:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.394392:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.394393:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.394394:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:24.0:1619677447.394396:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.394398:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.394401:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.394402:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.394402:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.394404:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.394404:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.394405:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.394405:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.0:1619677447.394479:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.394481:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.394482:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.394484:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.394484:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.394487:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.394631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.394647:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.394650:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.394653:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.394653:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.394656:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.394657:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677447.394659:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.394661:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.394662:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.394663:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677447.394666:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.394669:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677447.394672:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.394673:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677447.394674:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677447.394676:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677447.394676:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.394678:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.394678:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:4.2:1619677447.394691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677447.394949:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.394952:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.394953:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.394955:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.394956:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.394958:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.394959:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.394960:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.394962:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.394963:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.394966:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.394968:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.394970:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.394971:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.394971:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.394973:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.394973:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.394974:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.394974:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:22.0:1619677447.395010:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.395013:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.395014:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.395016:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.395019:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.395020:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.395022:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.395023:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.395024:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.395025:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.395029:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.395030:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.395033:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.395034:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.395035:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.395036:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.395037:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.395038:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.395038:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.395043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.395050:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.395053:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.395054:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.395054:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.395056:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.395059:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.395064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.395071:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.395074:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.395075:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.395075:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.395076:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.395076:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.395114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.395121:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.395124:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.395126:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.395127:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.395130:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.395132:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.395134:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96728832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.395135:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.395137:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.395138:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:4.2:1619677447.395143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:8.0:1619677447.395146:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.395148:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.395151:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.395152:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)++
00000800:00000200:8.0:1619677447.395154:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (24)++
00000800:00000200:8.0:1619677447.395156:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677447.395157:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.395157:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677447.395157:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.0:1619677447.395158:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.395160:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.395161:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.395161:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.395163:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.395167:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.395183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.395190:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.395193:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.395194:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.395194:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.395197:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.395200:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.395657:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.395660:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.395660:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.395663:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.395664:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.395666:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.395682:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.395684:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.395687:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.395687:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.395694:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.395696:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.395697:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.395699:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.395700:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.395701:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.395705:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.395707:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.395711:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.395712:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.395713:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.395716:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.395716:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.395717:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.395718:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.395719:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.395722:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.395723:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.395724:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.395726:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.395728:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:12.0:1619677447.395728:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:18.0:1619677447.395729:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.395730:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.395730:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:18.0:1619677447.395731:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677447.395731:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.395733:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:13.0:1619677447.395734:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.395734:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.395737:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:12.0:1619677447.395737:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.395739:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.395741:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.395742:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.395743:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.395744:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.395745:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.395746:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.395746:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.395758:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.395761:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.395762:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.395762:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.395764:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.395767:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.395979:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.395981:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.395982:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.395984:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.395987:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.395988:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677447.396003:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.396006:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.396007:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.396009:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.396012:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.396014:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.396015:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.396017:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.396018:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.396019:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.396022:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.396024:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.396027:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.396028:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.396029:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:2.0:1619677447.396029:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:2.0:1619677447.396031:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (25)--
00000800:00000200:13.0:1619677447.396032:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:2.0:1619677447.396032:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.396032:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:13.0:1619677447.396033:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.396033:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:13.0:1619677447.396035:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.396035:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.396037:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.396037:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677447.396037:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.396038:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:12.0:1619677447.396039:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.396040:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.396041:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677447.396041:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.396042:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677447.396042:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.396043:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.396043:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.396046:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.396046:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.396048:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.396051:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.396052:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.396053:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.396055:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.396056:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.396057:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.396060:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.396062:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.396064:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.396065:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.396066:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.396067:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.396068:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.396068:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.396069:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.396181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.396228:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.396230:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.396231:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.396231:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.396234:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.396237:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.396243:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.396246:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.396247:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.396249:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.396250:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.396251:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.396625:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.396628:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.396630:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.396632:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.396632:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677447.396633:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:20.0:1619677447.396634:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.396635:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.396636:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.396636:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:20.0:1619677447.396637:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677447.396637:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.396638:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:12.0:1619677447.396639:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.396641:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677447.396642:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.396644:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.396645:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.396647:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.396648:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.396649:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.396651:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.396652:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.396653:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.396653:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.396663:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.396665:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.396666:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.396668:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.396669:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.396670:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.396680:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.396683:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.396684:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.396684:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.396685:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.396686:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.396788:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.396790:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.396791:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.396793:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.396793:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.396796:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.396804:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.396807:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.396808:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.396808:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.396809:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.396810:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.396885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.396931:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.396934:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.396935:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.396936:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.396938:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.396941:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.396990:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.396992:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.396993:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.396993:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.396996:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.396999:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.397089:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.397092:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.397093:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.397095:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.397098:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.397099:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.397101:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.397102:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.397103:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.397104:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.397107:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.397109:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.397111:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.397112:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.397112:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.397114:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.397114:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.397115:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.397115:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.397126:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.397129:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.397129:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.397131:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.397133:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.397134:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.397172:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.397174:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.397177:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.397178:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.397179:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.397180:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677447.397182:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.397183:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.397184:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.397185:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677447.397188:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.397190:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677447.397193:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.397193:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677447.397194:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.397195:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677447.397196:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.397196:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.397197:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.397198:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.397198:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.397199:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.397201:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677447.397202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:18.0:1619677447.397205:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.397207:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.397208:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.397210:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.397210:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:18.0:1619677447.397212:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.397212:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.397213:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.397213:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.397215:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.397216:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.397218:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.397219:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.397220:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:18.0:1619677447.397221:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.397222:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.397223:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.397225:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.397226:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.397226:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.397227:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.397239:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.397241:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.397242:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.397244:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.397245:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.397247:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.397422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.397461:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.397463:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.397464:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.397466:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.397468:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.397469:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.397474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677447.397478:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.397481:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.397482:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.397482:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.397483:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.397484:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.397484:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.397486:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.397488:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.397488:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.397490:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.397493:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677447.397719:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.397722:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.397725:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.397725:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.397728:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.397730:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677447.397731:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.397733:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.397734:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.397735:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677447.397738:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.397740:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677447.397743:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.397744:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.397744:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:9.2:1619677447.397744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.397746:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.397747:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.397748:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.397749:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.397750:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.0:1619677447.397750:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:2.0:1619677447.397751:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.397754:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:24.0:1619677447.397755:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:18.0:1619677447.397756:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.397756:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.397758:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:2.0:1619677447.397758:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.397761:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.397761:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:2.0:1619677447.397761:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:8.0:1619677447.397763:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.397763:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:8.0:1619677447.397764:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:5.0:1619677447.397764:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.397765:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:8.0:1619677447.397766:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96728802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.397767:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.397773:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.397774:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.397778:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.397780:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.397783:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.397784:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.397785:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.397787:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.397787:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.397788:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.397790:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.397790:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677447.397802:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:24.0:1619677447.397803:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:5.0:1619677447.397804:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:24.0:1619677447.397805:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:5.0:1619677447.397805:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.397806:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.397807:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:5.0:1619677447.397807:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.397808:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.397810:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.397820:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.397823:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.397824:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.397825:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.397825:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.397826:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.397868:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.397870:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.397872:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.397873:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.397874:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.397876:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.397877:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.397878:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.397879:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.397880:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.397883:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.397885:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.397887:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.397888:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.397889:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.397890:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.397891:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.397891:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.397892:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:4.2:1619677447.397956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.397981:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.397984:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.397984:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677447.397986:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:22.0:1619677447.397987:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.397988:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:22.0:1619677447.397989:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.397990:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.397991:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:13.0:1619677447.397992:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:22.0:1619677447.397993:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96728798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677447.397993:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.397994:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.397995:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:22.0:1619677447.397996:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.397996:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.398000:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.398002:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.398004:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.398005:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.398006:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.398008:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.398008:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.398009:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.398010:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.398022:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.398024:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.398025:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.398027:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.398027:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.398029:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677447.398043:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.398045:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.398046:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.398048:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.398050:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.398052:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.398053:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.398054:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.398055:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.398056:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.398059:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.398061:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.398063:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.398064:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.398065:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.398066:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.398067:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.398068:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.398069:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.398070:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.398072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.398081:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.398082:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.398082:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.398083:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:2.0:1619677447.398083:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.398084:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.398085:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.398085:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.398086:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677447.398086:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.0:1619677447.398175:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.398178:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.398179:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.398181:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.398184:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.398186:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.398187:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.398188:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.398189:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.398190:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.398194:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.398195:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.398198:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.398199:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.398200:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.398201:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.398202:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.398203:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.398203:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.398207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.398214:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.398217:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.398218:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.398219:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.398221:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.398224:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.398344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.398390:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.398393:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.398394:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.398394:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.398398:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.398401:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.398410:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.398413:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.398413:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.398414:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.398415:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.398415:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677447.398422:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.398424:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.398425:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.398427:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.398429:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.398431:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.398510:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.398513:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.398515:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.398515:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.398518:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.398520:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.398521:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.398523:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.398523:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.398525:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.398528:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.398530:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.398533:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.398534:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.398534:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.398536:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.398537:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.398538:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.398538:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:24.0:1619677447.398595:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.398598:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.398599:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.398601:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.398604:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.398605:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.398606:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96728791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.398608:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.398609:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.398610:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:13.0:1619677447.398613:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:24.0:1619677447.398614:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.398616:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:13.0:1619677447.398616:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.398617:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.398619:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677447.398619:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.398620:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.398621:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:12.0:1619677447.398621:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.398622:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.398623:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.398624:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.398624:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.398628:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677447.398637:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.398639:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.398640:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.398642:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.398643:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.398645:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:6.0:1619677447.398650:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.398653:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.398654:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.398655:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.398657:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.398658:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:6.0:1619677447.398659:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.398661:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.398662:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.398664:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.398666:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.398667:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.398668:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.398670:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.398671:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.398672:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.398675:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.398676:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.398678:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.398678:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.398679:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.398681:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.398681:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.398682:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.398682:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.398687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.398694:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.398696:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.398697:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.398697:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.398698:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.398700:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.398704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.398711:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.398713:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.398714:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.398714:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.398715:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.398715:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677447.398791:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.398793:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.398794:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.398795:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.398797:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.398798:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.398821:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.398823:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.398826:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.398826:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.398828:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.398830:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:5.0:1619677447.398832:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.398833:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.398834:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.398835:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:5.0:1619677447.398838:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.398840:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:5.0:1619677447.398843:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.398844:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677447.398844:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.0:1619677447.398846:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677447.398847:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.398847:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.398848:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677447.398852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.398859:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.398861:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.398862:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.398863:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.398865:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677447.398868:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.398868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.398908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677447.398922:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.398924:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.398925:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.398927:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.398928:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.398929:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.398959:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.398962:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.398963:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.398965:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.398967:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.398969:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.398970:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.398971:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.398972:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.398973:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.398977:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.398979:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.398982:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.398982:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.398983:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.398985:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.398986:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.398987:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.398988:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.398988:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.398999:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.399000:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.399001:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.399002:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677447.399147:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.399150:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.399150:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.399153:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.399154:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.399156:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677447.399157:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.399158:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.399159:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.399161:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677447.399164:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.399166:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.399169:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.399170:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:18.0:1619677447.399170:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677447.399172:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.399173:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.399173:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.399174:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677447.399185:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.399188:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.399188:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.399190:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.399191:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.399193:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.399202:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.399206:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.399207:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.399207:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.399208:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.399209:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.399286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.399542:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.399545:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.399546:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.399548:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.399552:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.399553:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:4.2:1619677447.399554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:22.0:1619677447.399555:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96728776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.399557:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.399559:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.399560:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.399564:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.399567:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.399569:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.399570:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.399571:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.399572:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.399573:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.399573:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.399574:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.399580:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.399583:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.399583:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.399585:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.399588:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.399589:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.399591:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.399593:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.399594:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.399596:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.399598:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.399599:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.399600:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.399603:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.399604:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.399606:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.399606:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.399609:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677447.399630:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.399632:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.399633:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.399635:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.399636:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.399637:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677447.399737:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.399740:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.399741:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.399742:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.399743:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.399745:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.399758:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.399761:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.399762:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.399762:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.399763:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.399764:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.399928:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.399931:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.399933:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.399933:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.399936:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.399939:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.399940:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.399942:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.399943:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.399944:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.399948:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.399950:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.399953:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.399954:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.399955:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.399956:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.399957:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.399957:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.399958:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677447.399963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.399968:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.399971:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.399972:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.399974:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.399975:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.399977:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.399987:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.399990:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.399990:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.399991:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.399992:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.399993:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.400010:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.400013:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.400014:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.400016:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.400018:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.400019:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:24.0:1619677447.400025:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.400028:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:5.0:1619677447.400028:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.400029:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.400031:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.400031:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.400032:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.400033:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000400:00000200:24.0:1619677447.400034:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.400034:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.400034:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:24.0:1619677447.400036:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.400037:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.400038:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.400039:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.400040:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.400044:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.400045:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.400048:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.400049:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.400050:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.400052:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.400052:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.400053:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.400053:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:4.2:1619677447.400054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:9.2:1619677447.400058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.400062:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.400064:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.400065:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.400066:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.400066:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:8.0:1619677447.400069:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.400069:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.400070:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:8.0:1619677447.400070:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.400071:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96728768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.400071:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.400073:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.400073:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.400074:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.400075:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.400077:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:8.0:1619677447.400079:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.400081:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.400083:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.400084:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.400085:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.400086:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.400087:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.400088:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.400089:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.400090:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.400095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.400102:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.400105:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.400105:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.400107:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677447.400244:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.400247:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.400247:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.400249:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.400252:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.400253:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.400255:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.400256:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.400257:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.400258:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.400261:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.400263:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.400265:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.400266:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.400267:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.400269:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.400269:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.400270:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.400271:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:6.0:1619677447.400411:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:9.2:1619677447.400412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:6.0:1619677447.400414:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.400415:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.400417:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.400419:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:6.0:1619677447.400419:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.400421:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:8.0:1619677447.400422:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:6.0:1619677447.400422:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.400423:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.400423:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.400425:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.400426:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.400426:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.400427:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:8.0:1619677447.400429:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:6.0:1619677447.400430:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.400432:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.400435:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.400436:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.400436:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.400438:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.400439:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.400440:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.400440:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.400445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.400452:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.400455:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.400456:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.400456:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.400458:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.400461:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:12.0:1619677447.400566:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.400568:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.400569:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.400570:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.400572:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.400573:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.400705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.400743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.400748:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.400751:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.400752:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.400752:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.400755:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.400758:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.400766:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.400768:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.400769:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.400769:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.400770:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.400771:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.401015:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.401018:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:18.0:1619677447.401019:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:5.0:1619677447.401019:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.401020:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677447.401021:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.401022:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:5.0:1619677447.401023:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:2.0:1619677447.401023:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.401024:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.401026:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.401028:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.401029:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.401031:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.401032:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.401033:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.401037:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.401038:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.401041:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.401042:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.401042:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.401044:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.401044:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.401045:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.401046:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677447.401269:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.401272:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.401273:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.401275:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677447.401276:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:2.0:1619677447.401277:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.401278:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:20.0:1619677447.401279:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.401279:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.401279:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.401281:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.401282:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.401282:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:2.0:1619677447.401282:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.401284:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:20.0:1619677447.401285:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:2.0:1619677447.401287:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.401289:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677447.401292:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.401293:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677447.401299:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677447.401301:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677447.401301:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.401302:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.401303:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.401578:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.401581:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.401582:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.401582:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.401584:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.401587:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.401633:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.401636:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.401638:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.401639:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.401641:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.401643:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677447.401644:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.401645:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.401646:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.401647:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677447.401651:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.401653:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677447.401655:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.401656:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677447.401657:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677447.401659:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677447.401659:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.401660:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.401661:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.401665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:4.2:1619677447.401665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.401673:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.401676:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.401677:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.401677:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.401679:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.401682:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.401685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.401692:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.401695:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.401696:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.401696:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.401697:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.401698:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:6.0:1619677447.401710:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.401713:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.401714:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.401715:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.401717:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.401720:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.401750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.401758:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.401761:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.401762:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.401763:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.401766:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.401768:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.401769:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.401770:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.401771:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.401772:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.401775:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.401777:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.401779:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.401780:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.401781:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.401782:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.401783:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.401783:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.401784:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.401813:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.401816:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.401816:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.401817:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.401820:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.401822:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.401840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.401846:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.401849:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.401850:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.401850:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.401853:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.401855:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.401873:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.401875:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.401876:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.401878:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.401880:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.401882:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.401883:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.401884:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.401886:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.401887:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.401890:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.401892:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.401895:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.401896:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.401896:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.401898:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.401899:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.401900:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.401900:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.401989:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.401992:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.401993:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.401993:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.401995:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.401998:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.402013:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.402015:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.402016:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.402017:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.402017:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.402018:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.402105:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.402107:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.402108:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.402109:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.402111:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.402113:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677447.402180:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.402183:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.402184:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.402185:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.402187:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.402189:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.402190:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.402191:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.402193:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.402193:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.402196:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.402198:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.402200:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.402201:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.402202:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.402204:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.402204:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.402205:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.402206:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677447.402218:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.402221:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.402222:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.402223:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.402225:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.402226:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.402306:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.402309:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.402311:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.402311:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.402313:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.402315:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677447.402316:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.402317:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:5.0:1619677447.402317:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.402318:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.402319:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.402320:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677447.402323:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.402324:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.402325:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.402326:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:20.0:1619677447.402327:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677447.402328:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.402329:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.402330:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.402331:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.402334:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.402335:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.402335:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677447.402336:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677447.402337:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.402338:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677447.402338:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.402339:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.402340:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677447.402340:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.402341:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.402341:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.402341:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.402343:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.402343:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.402343:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.402345:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.402349:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.402350:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.402351:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.402352:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.402353:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.402354:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:5.0:1619677447.402361:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.402363:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.402364:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.402365:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.402419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.402428:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.402431:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.402432:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.402433:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.402434:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:2.0:1619677447.402434:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.402436:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.402438:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:18.0:1619677447.402439:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.402439:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.402441:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.402443:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.402445:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.402447:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.402448:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.402449:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.402454:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.402457:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.402460:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.402460:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.402461:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.402463:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.402464:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.402465:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.402465:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:6.0:1619677447.402478:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677447.402479:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677447.402480:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.402481:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.402481:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.402482:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.402483:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.402484:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.402485:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.402485:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.402487:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.402488:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:9.2:1619677447.402601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.402608:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.402611:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.402612:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.402612:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.402614:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.402616:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.402622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677447.402628:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.402631:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.402631:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.402632:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.402632:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.402633:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677447.402698:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.402700:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.402701:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.402703:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.402705:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.402706:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.402716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.402809:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.402811:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.402814:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.402815:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.402816:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.402818:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677447.402820:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.402820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:20.0:1619677447.402821:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.402822:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.402824:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:24.0:1619677447.402827:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:20.0:1619677447.402827:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.402829:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677447.402830:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:20.0:1619677447.402831:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.402831:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677447.402832:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:24.0:1619677447.402833:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.402833:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:24.0:1619677447.402835:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.402835:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.402835:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.402836:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:24.0:1619677447.402837:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.0:1619677447.402837:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:24.0:1619677447.402838:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96728734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.402839:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.402840:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.402841:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:9.2:1619677447.402841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:24.0:1619677447.402845:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.402846:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.402848:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:24.0:1619677447.402849:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.402850:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.402850:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.402851:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:24.0:1619677447.402852:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:2.0:1619677447.402852:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.402853:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.402853:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677447.402853:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:24.0:1619677447.402854:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677447.402854:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.402857:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:22.0:1619677447.403079:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.403082:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.403083:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.403083:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.403085:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.403087:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677447.403200:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.403202:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.403204:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.403205:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.403207:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.403207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:5.0:1619677447.403208:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.403210:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.403211:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.403212:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.403213:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.403216:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.403218:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.403221:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.403222:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.403223:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.403225:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.403225:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.403226:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.403227:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.403237:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.403240:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.403240:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.403242:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.403244:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.403245:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.403252:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.403255:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.403255:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.403256:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.403258:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.403261:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.403277:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.403280:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.403280:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.403281:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.403282:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.403282:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677447.403300:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.403302:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.403303:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.403305:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.403305:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.403307:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.403308:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (24)--
00000800:00000200:5.0:1619677447.403308:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:5.0:1619677447.403309:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:5.0:1619677447.403309:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.403310:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.0:1619677447.403389:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.403392:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.403393:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.403394:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.403397:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.403399:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.403400:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.403401:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.403402:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.403403:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.403409:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.403412:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.403415:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.403416:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.403416:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.403418:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.403419:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.403420:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.403422:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.403422:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677447.403424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:9.2:1619677447.403428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.403433:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.403435:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.403436:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.403437:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.403437:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.0:1619677447.403437:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.403438:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.403439:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.403440:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:18.0:1619677447.403440:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:8.0:1619677447.403440:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:18.0:1619677447.403441:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.403441:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:2.0:1619677447.403441:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.403443:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.403446:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677447.403572:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.403575:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.403577:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.403578:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.403580:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.403582:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.403583:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.403585:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.403586:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.403587:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.403590:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.403592:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.403595:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.403596:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.403596:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.403598:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.403599:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.403600:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.403600:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677447.403621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.403662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.403668:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.403671:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.403672:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.403672:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.403674:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.403677:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.403917:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.403920:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.403921:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.403922:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.403923:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:12.0:1619677447.403924:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.403925:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.403926:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.403927:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.403929:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.403931:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.403933:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.403934:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728722] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.403935:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.403936:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.403937:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.403940:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.403942:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.403945:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.403946:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.403947:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.403948:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.403949:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.403950:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.403950:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.403957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.403965:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.403967:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.403969:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.403969:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.403971:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.403974:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.403982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.404005:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.404007:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.404010:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.404011:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.404013:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.404014:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.404016:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.404018:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.404019:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.404020:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.404025:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.404028:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.404031:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.404032:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.404033:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.404035:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.404037:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.404038:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.404040:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.404040:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677447.404047:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677447.404050:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677447.404051:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.404051:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:12.0:1619677447.404053:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.404054:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.404055:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:20.0:1619677447.404056:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (23)--
00000400:00000200:13.0:1619677447.404056:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.404056:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.404057:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.404057:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.404058:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.404059:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.404062:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.404063:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.404065:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.404066:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.404067:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.404067:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.404067:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677447.404117:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.404120:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.404120:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.404122:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.404125:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.404126:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.404128:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.404129:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.404130:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.404131:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.404134:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.404136:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.404138:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.404139:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.404139:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.404141:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.404142:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.404143:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.404144:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.404145:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677447.404154:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677447.404155:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:22.0:1619677447.404157:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677447.404157:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:22.0:1619677447.404158:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.404158:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677447.404158:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.404159:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:22.0:1619677447.404161:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.404163:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.404216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677447.404228:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.404230:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.404232:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.404233:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.404236:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.404237:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.404239:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.404240:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.404241:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.404242:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.404246:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.404247:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.404250:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.404251:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.404252:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.0:1619677447.404254:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (23)--
00000800:00000200:8.0:1619677447.404254:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.404255:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677447.404256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.0:1619677447.404256:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:18.0:1619677447.404264:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:18.0:1619677447.404267:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000800:00000200:2.0:1619677447.404268:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (24)++
00000400:00000200:18.0:1619677447.404269:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.404269:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000400:00000200:2.0:1619677447.404269:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.404271:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.404271:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.404275:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.404275:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.404276:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.404278:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.404279:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.404279:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.404280:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.404280:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.404292:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.404299:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:5.0:1619677447.404299:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.404301:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677447.404304:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.404306:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.404307:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.404309:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.404310:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.404311:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.404315:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.404317:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.404319:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.404320:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.404321:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.404322:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677447.404323:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.404323:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.404324:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.404325:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:6.0:1619677447.404325:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.404326:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.404327:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.404330:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.404332:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.404333:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.404335:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.404335:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.404337:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.404340:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.404342:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.404345:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.404345:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.404346:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.404348:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.404349:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.404350:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.404351:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.404352:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.404362:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.404364:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.404365:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.404366:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.404402:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.404404:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.404405:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.404405:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.404407:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.404409:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.404470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.404483:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.404486:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.404486:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.404488:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.404490:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.404491:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.404517:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.404519:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.404520:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.404522:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.404523:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.404525:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.404561:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.404563:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.404565:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.404566:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.404568:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.404570:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.404571:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.404573:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.404574:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.404575:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.404578:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.404580:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.404583:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.404584:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.404584:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.404587:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.404587:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.404588:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.404588:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.404599:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.404602:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.404603:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.404604:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.404605:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.404607:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.404620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.404627:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.404630:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.404631:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.404633:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.404636:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.404638:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.404639:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.404641:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.404642:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.404644:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.404647:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.404650:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.404653:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.404654:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.404654:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.404656:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.404656:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.404657:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.404657:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.404664:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.404667:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.404668:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.404669:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.404671:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:9.2:1619677447.404671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.404672:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.404679:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.404682:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.404683:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.404683:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.404685:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.404688:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.0:1619677447.404756:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.404759:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.404760:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.404761:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.404764:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.404766:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.404767:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.404768:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.404769:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.404770:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.404773:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.404775:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.404778:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.404779:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.404779:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.404781:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.404782:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.404782:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.404783:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.404787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.404795:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677447.404795:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:22.0:1619677447.404798:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:18.0:1619677447.404798:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.404799:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.404799:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677447.404799:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:22.0:1619677447.404802:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.404802:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.404804:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.404805:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:22.0:1619677447.404806:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.404808:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.404810:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.404811:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.404812:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.404816:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.404818:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.404821:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.404822:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.404822:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.404824:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.404824:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.404825:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.404826:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.404830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.404837:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.404840:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.404841:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.404842:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.404843:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.404845:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.404881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.404923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677447.404929:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.404932:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.404932:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.404934:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.404935:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.404937:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677447.404975:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.404978:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.404978:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.404980:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.404983:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.404984:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.404985:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.404987:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.404988:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.404989:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.404993:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.404995:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.404997:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.404998:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.404999:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.405000:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.405001:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.405002:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.405003:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.405003:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.405006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.405015:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.405016:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.405016:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.405017:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:2.0:1619677447.405017:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.405018:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.405019:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.405019:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.405020:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677447.405022:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.405024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.405031:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.405034:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.405035:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.405035:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.405037:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.405037:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.405196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.405203:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.405206:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.405207:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.405208:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.405210:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.405213:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.405243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.405249:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:6.0:1619677447.405250:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677447.405252:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:6.0:1619677447.405252:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:24.0:1619677447.405253:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:22.0:1619677447.405253:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:8.0:1619677447.405253:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.405256:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.405256:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.405258:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:6.0:1619677447.405258:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.405260:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.405261:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.405263:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.405264:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.405265:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.405268:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.405270:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.405272:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.405273:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.405274:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.405276:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.405276:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.405277:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.405277:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.405359:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.405362:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.405363:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.405364:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.405366:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.405368:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.405369:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.405371:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.405372:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.405373:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677447.405376:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.405378:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.405381:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.405382:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.405382:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.405384:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.405385:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.405386:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.405386:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:4.2:1619677447.405494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.405541:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.405544:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.405545:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.405545:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.405547:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.405550:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.405695:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677447.405698:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677447.405698:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.405699:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677447.405701:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:6.0:1619677447.405701:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.405701:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.405704:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.405704:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.405705:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.405706:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.405707:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.405707:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.405709:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.405709:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.405710:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.405711:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.405713:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.405714:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.405715:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.405715:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.405717:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.405719:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.405720:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.405721:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.405721:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.405722:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.405724:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.405724:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.405724:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.405724:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677447.405725:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.405725:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677447.405725:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677447.405727:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677447.405727:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.405728:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.405729:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.405835:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.405837:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.405838:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.405840:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.405842:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.405843:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.405852:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.405854:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.405855:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.405856:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.405856:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.405857:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.405881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.405888:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.405891:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.405892:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.405893:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.405896:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.405898:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.405899:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.405900:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.405901:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.405902:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.405906:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.405907:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.405910:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.405911:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.405911:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.405913:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.405914:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.405915:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.405916:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.405917:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.405922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.405929:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.405931:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.405932:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.405933:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.406167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.406214:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.406217:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.406218:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.406218:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.406221:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.406224:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.406233:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.406236:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.406237:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.406237:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.406238:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.406238:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.406308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.406315:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.406318:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:2.0:1619677447.406318:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:24.0:1619677447.406319:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.406319:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:2.0:1619677447.406321:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:24.0:1619677447.406322:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.406322:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.406325:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.406326:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:2.0:1619677447.406327:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.406328:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.406329:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.406331:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.406331:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.406333:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.406336:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.406338:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.406340:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.406341:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.406341:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.406343:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.406343:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.406344:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.406345:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:22.0:1619677447.406466:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.406468:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.406470:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.406471:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.406474:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.406476:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.406477:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.406478:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.406479:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.406480:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.406484:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.406485:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.406489:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.406489:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.406490:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.406492:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.406492:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.406493:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.406494:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.406495:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.406505:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.406507:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.406508:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.406509:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.406544:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.406546:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.406549:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.406550:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.406552:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.406553:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677447.406555:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.406557:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.406558:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.406559:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677447.406562:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.406564:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677447.406567:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.406568:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677447.406568:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677447.406570:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677447.406571:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.406571:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.406572:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:4.2:1619677447.406991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.407021:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.407024:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.407025:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.407027:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.407029:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.407030:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677447.407032:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.407033:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.407034:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.407034:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:18.0:1619677447.407035:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.407037:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.407037:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:18.0:1619677447.407038:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000400:00000200:6.0:1619677447.407038:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.407040:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:6.0:1619677447.407040:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.407043:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.407043:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:6.0:1619677447.407043:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677447.407044:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677447.407046:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677447.407046:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.407047:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.407048:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.407060:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.407062:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.407063:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.407063:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.407065:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.407067:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.407122:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.407125:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.407126:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.407127:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.407130:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.407132:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.407133:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.407134:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.407135:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.407136:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.407139:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.407140:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.407143:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.407143:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.407144:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.407146:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.407146:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.407147:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.407147:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677447.407159:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.407161:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.407161:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:20.0:1619677447.407162:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.407163:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.407164:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.407165:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.407167:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.407168:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.407169:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.407169:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.407171:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677447.407172:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.407173:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.407174:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.407175:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677447.407179:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.407181:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.407183:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.407184:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677447.407185:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.407186:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.407187:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.407188:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.407188:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677447.407278:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.407281:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.407282:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.407283:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.407284:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.407286:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.407327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.407335:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.407338:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.407339:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.407341:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.407344:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.407346:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.407347:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.407348:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.407349:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.407350:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.407354:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.407355:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.407358:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.407358:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.407359:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.407361:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.407361:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.407362:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.407362:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.407407:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.407409:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.407410:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.407411:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.407413:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.407419:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.407421:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.407423:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.407424:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.407424:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.407425:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.407425:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677447.407753:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.407755:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.407756:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.407759:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.407761:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.407763:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677447.407764:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.407765:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.407766:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.407767:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677447.407771:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.407773:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.407776:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.407776:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:22.0:1619677447.407777:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:18.0:1619677447.407777:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677447.407779:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.407779:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:22.0:1619677447.407780:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.407781:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.407781:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:6.0:1619677447.407781:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677447.407782:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000400:00000200:22.0:1619677447.407783:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.407783:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:22.0:1619677447.407785:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.407792:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677447.407793:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:12.0:1619677447.407795:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:2.0:1619677447.407795:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:12.0:1619677447.407796:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.407796:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.407797:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:12.0:1619677447.407798:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.407800:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.407801:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.407811:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.407814:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.407815:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.407816:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.407816:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.407818:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.407919:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.407922:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.407923:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.407925:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.407925:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.407928:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677447.407958:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.407960:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.407961:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.407962:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.407964:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.407965:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.407986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.407999:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.408002:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.408005:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.408005:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.408007:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.408009:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.408011:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.408012:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.408014:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.408015:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.408018:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.408021:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.408024:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.408025:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.408025:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.408027:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677447.408029:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.408030:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.408030:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677447.408030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.408035:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:22.0:1619677447.408038:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (24)++
00000800:00000200:6.0:1619677447.408038:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:22.0:1619677447.408039:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000400:00000200:6.0:1619677447.408039:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.408040:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.408042:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.408042:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.408045:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.408046:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.408046:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:9.2:1619677447.408050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.408058:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.408061:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.408062:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.408062:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.408065:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.408068:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.408072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.408078:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.408081:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.408082:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.408082:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.408083:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.408084:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:12.0:1619677447.408186:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.408189:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.408190:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.408191:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.408192:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.408194:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.408254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.408261:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.408264:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.408265:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.408267:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.408270:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.408271:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.408272:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.408273:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.408274:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.408275:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.408278:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.408279:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.408282:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.408283:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.408283:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.408285:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.408285:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.408286:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.408287:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.0:1619677447.408324:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.408326:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.408329:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.408329:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.408331:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.408332:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677447.408334:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.408335:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.408336:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.408337:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677447.408340:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.408342:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.408344:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.408345:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677447.408346:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.408347:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.408348:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.408349:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.408351:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.408351:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.408359:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.408361:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.408362:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.408362:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.0:1619677447.408488:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.408492:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677447.408496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677447.408496:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677447.408498:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.408500:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:8.0:1619677447.408501:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.408502:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:8.0:1619677447.408502:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677447.408503:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.408504:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96728662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.408505:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.408505:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:8.0:1619677447.408506:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.408506:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.408508:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677447.408508:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.408509:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.408509:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.408511:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.408513:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.408514:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.408515:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:8.0:1619677447.408518:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.408520:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.408522:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.0:1619677447.408523:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.408523:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.408524:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.408525:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:8.0:1619677447.408526:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.408526:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677447.408526:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.408527:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:8.0:1619677447.408528:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:5.0:1619677447.408528:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.408529:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.0:1619677447.408529:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.408529:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677447.408540:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:24.0:1619677447.408543:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:5.0:1619677447.408543:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:24.0:1619677447.408544:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:5.0:1619677447.408544:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.408545:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.408546:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677447.408546:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.408546:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.408549:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.408558:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.408561:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.408562:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.408563:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.408563:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:6.0:1619677447.408563:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:20.0:1619677447.408565:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:6.0:1619677447.408566:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.408567:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.408568:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.408570:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.408573:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677447.408608:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.408610:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.408612:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.408613:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.408615:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.408616:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.408617:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.408619:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.408624:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.408625:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.408628:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.408630:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.408633:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.408634:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.408634:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.408636:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.408637:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.408638:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.408638:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677447.408639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.408650:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.408653:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.408654:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.408655:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.408657:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.408658:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.408685:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.408688:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.408689:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.408690:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.408691:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.408694:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:9.2:1619677447.408780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.408786:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.408788:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.408789:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.408789:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.408792:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.408794:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.0:1619677447.408822:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.408825:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.408827:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.408827:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.408830:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.408831:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.408832:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.408834:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.408835:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.408836:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.408839:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.408840:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.408843:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.408844:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.408845:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.408846:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.408847:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.408847:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.408848:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.408859:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.408861:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.408862:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.408864:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.408865:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.408867:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.408956:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.408960:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.408960:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.408963:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.408965:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.408967:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.408968:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96728651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.408969:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.408969:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:24.0:1619677447.408970:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.408971:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.408972:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.408973:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.408974:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.408974:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.408976:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.408976:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:24.0:1619677447.408977:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.408978:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.408979:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.408979:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.408981:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.408981:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:13.0:1619677447.408981:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.408982:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.408982:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.408984:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.408984:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:13.0:1619677447.408984:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.408985:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.408986:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.408995:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.408998:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.408999:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.409000:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.409002:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.409003:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.409015:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.409017:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.409018:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.409019:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.409019:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.409021:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.409063:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.409066:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.409069:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.409070:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.409073:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.409074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:20.0:1619677447.409076:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.409078:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.409080:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.409082:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.409083:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.409089:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.409092:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.409095:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.409096:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.409096:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.409099:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.409099:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.409100:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.409100:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.409110:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.409113:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.409114:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.409114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:12.0:1619677447.409116:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.409117:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.409118:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.409121:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.409124:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.409125:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.409126:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.409126:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:2.0:1619677447.409127:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.409129:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.409130:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.409131:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:5.0:1619677447.409132:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.409136:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.409137:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677447.409176:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.409179:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.409180:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.409181:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.409183:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.409184:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677447.409225:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.409228:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.409230:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.409231:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.409233:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.409235:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677447.409236:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.409237:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.409238:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.409239:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677447.409243:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.409245:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.409247:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.409248:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677447.409249:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.409251:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.409252:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.409252:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.409253:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.0:1619677447.409287:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.409290:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.409291:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.409299:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.409301:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.409303:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.409304:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.409304:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:8.0:1619677447.409306:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.409306:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:8.0:1619677447.409307:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.409309:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:8.0:1619677447.409309:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.409309:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.409312:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.409314:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.409315:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.409316:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96728643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.409318:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.409318:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.409319:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.409320:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.409321:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.409322:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.409322:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677447.409323:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.409324:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.409325:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.409325:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677447.409325:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.409326:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677447.409328:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.409329:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677447.409330:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:9.2:1619677447.409331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.409332:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.409332:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.409334:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.409335:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.409336:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677447.409340:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.409342:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.409343:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.409343:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.409344:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.409346:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:18.0:1619677447.409346:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.409347:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.409348:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.409349:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.409424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.409432:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.409435:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.409436:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.409436:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.409438:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.409441:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.409447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.409486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.409487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.409494:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677447.409494:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677447.409497:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:18.0:1619677447.409497:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.409498:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.409498:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677447.409498:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:24.0:1619677447.409500:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.409501:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.409503:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.409504:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:24.0:1619677447.409505:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:9.2:1619677447.409505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:24.0:1619677447.409507:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.409509:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.409510:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.409511:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.409513:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:24.0:1619677447.409516:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.409516:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.409516:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.409517:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677447.409518:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.409518:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.409518:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677447.409521:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.409522:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.409522:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.409525:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.409525:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.409526:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.409527:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.409586:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.409588:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.409589:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.409591:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.409593:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.409594:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.409605:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.409608:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.409608:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.409611:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.409613:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.409615:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.409616:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.409618:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.409619:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.409620:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.409623:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.409625:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.409628:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.409628:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.409629:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.409631:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.409631:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.409632:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.409632:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.409787:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.409790:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.409791:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.409793:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.409796:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.409798:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.409799:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.409801:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.409802:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.409803:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.409806:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.409808:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.409811:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.409812:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.409813:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.409814:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.409815:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.409816:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.409816:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677447.409828:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.409834:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.409834:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.409835:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:20.0:1619677447.409836:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.409836:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.409838:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.409839:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.409840:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.409841:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.409844:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.409846:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.409847:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.409848:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.409849:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.409851:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.409855:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.409857:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.409860:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.409861:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.409861:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.409863:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.409864:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.409865:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.409865:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.409869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.409872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.409878:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.409880:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.409881:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.409881:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.409884:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.409887:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.409904:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.409906:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.409907:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.409909:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.409910:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.409911:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:13.0:1619677447.409912:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677447.409914:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.409914:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.409916:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.409917:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.409919:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.409921:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.409922:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.409923:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:12.0:1619677447.409925:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:2.0:1619677447.409925:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96728629] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.409926:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.409927:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.409928:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:2.0:1619677447.409928:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.409929:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.409930:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:2.0:1619677447.409930:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:9.2:1619677447.409931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677447.409932:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:12.0:1619677447.409932:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:18.0:1619677447.409934:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:508, d:32, s:-96728628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.409935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677447.409935:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.409936:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.409937:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.409937:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.409938:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.409942:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.409943:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677447.409945:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677447.409945:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.409946:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.409946:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.409947:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:13.0:1619677447.409947:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:2.0:1619677447.409947:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.409948:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000400:00000200:13.0:1619677447.409948:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.409949:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:13.0:1619677447.409950:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.409950:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.409951:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (24)--
00000800:00000200:18.0:1619677447.409951:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)++
00000800:00000200:12.0:1619677447.409951:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.409951:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:13.0:1619677447.409952:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.409952:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.409953:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (23)++
00000800:00000200:2.0:1619677447.409953:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677447.409954:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (24)--
00000800:00000200:2.0:1619677447.409954:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (24)--
00000800:00000200:18.0:1619677447.409955:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.409955:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:2.0:1619677447.409957:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000400:00000200:2.0:1619677447.409958:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.409959:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:18.0:1619677447.409960:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000400:00000200:2.0:1619677447.409960:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.409961:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.409962:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:18.0:1619677447.409962:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.409964:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:22.0:1619677447.409965:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:18.0:1619677447.409965:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.409965:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:22.0:1619677447.409966:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.409966:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.409966:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:22.0:1619677447.409968:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.409968:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.409968:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.409969:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:22.0:1619677447.409970:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.409980:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.409983:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.409984:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.409984:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.409985:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.409986:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.410165:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.410167:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.410168:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.410170:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.410172:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.410173:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.410175:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.410176:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.410177:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.410178:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.410181:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.410183:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.410186:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.410187:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.410188:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.410190:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.410190:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.410191:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.410191:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.410246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.410292:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.410298:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.410299:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.410301:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.410303:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.410306:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.410316:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.410318:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.410319:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.410319:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.410320:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.410321:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.410414:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.410418:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.410419:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.410420:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.410423:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.410425:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.410426:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96728623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.410427:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.410428:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677447.410428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:24.0:1619677447.410429:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.410433:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.410435:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.410435:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.410436:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.410438:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.410438:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:8.0:1619677447.410438:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.410439:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:8.0:1619677447.410439:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.410441:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.410442:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.410442:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.410443:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.410443:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677447.410445:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.410446:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677447.410446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:12.0:1619677447.410454:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677447.410456:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:12.0:1619677447.410457:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.410457:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.410458:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.410458:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.410459:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.410459:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.410460:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:12.0:1619677447.410460:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.410460:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677447.410461:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.410462:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677447.410462:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.410462:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.410462:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:4.2:1619677447.410466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.410473:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.410477:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.410478:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.410479:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.410482:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.410483:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.410484:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96728621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.410485:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:24.0:1619677447.410486:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.410486:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.410487:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:13.0:1619677447.410488:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.410488:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.410490:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.410491:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.410491:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.410492:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.410493:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.410496:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.410496:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.410497:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.410498:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.410499:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.410500:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.410501:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.410502:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.410507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.410514:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.410516:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.410517:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.410519:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.410544:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.410547:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.410548:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.410550:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.410552:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.410554:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.410555:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.410557:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.410558:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.410559:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.410562:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.410564:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.410567:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.410567:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.410568:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.410570:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.410570:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.410571:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.410571:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.410576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.410584:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.410587:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.410588:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.410588:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.410590:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.410593:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:12.0:1619677447.410606:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.410608:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.410609:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.410610:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.410612:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.410613:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.410624:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.410626:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.410627:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.410628:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.410628:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.410630:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.410741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.410747:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.410750:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.410751:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.410751:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.410754:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.410757:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.410760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:8.0:1619677447.410766:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.410769:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.410769:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677447.410770:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.410771:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.410771:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.410830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.410831:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.410834:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.410835:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.410837:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.410839:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.410841:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.410843:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.410844:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.410846:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.410847:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.410850:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.410853:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.410855:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.410856:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.410857:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.410859:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.410860:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.410860:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.410861:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677447.410873:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.410878:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.410878:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.410879:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:20.0:1619677447.410880:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.410880:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:22.0:1619677447.410881:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677447.410881:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:20.0:1619677447.410883:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:5.0:1619677447.410883:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.410884:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677447.410884:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:6.0:1619677447.410884:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.410885:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.410886:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.410887:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.410888:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677447.410889:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.410890:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:5.0:1619677447.410890:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.410891:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.410892:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677447.410896:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.410898:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.410901:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.410901:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677447.410902:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677447.410904:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677447.410904:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:5.0:1619677447.410906:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:5.0:1619677447.410907:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:5.0:1619677447.410908:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677447.410916:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000800:00000200:20.0:1619677447.410918:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000800:00000200:20.0:1619677447.410919:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677447.410920:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:6.0:1619677447.410979:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.410981:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.410982:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.410984:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.410986:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.410987:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.410989:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.410990:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.410991:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.410992:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.410995:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.410996:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.410999:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.410999:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.411000:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.411002:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.411002:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.411003:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.411004:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.411005:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.411008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.411015:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677447.411015:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677447.411017:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.411018:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677447.411018:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:22.0:1619677447.411019:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:18.0:1619677447.411019:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.411019:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.411022:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.411024:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:18.0:1619677447.411024:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.0:1619677447.411026:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.411029:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.411030:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.411031:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.411033:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677447.411034:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.411036:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.411037:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.411038:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677447.411042:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.411044:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677447.411046:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.411047:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677447.411048:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677447.411050:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677447.411051:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.411052:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.411053:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.411054:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.411064:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.411066:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.411067:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.411068:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:4.2:1619677447.411259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.411306:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.411308:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.411309:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.411310:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.411312:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.411315:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.411435:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.411438:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.411439:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.411440:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.411442:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.411444:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.411446:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.411448:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.411449:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.411450:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.411455:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.411458:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.411460:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.411461:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.411462:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.411464:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.411465:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.411465:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.411466:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:4.2:1619677447.411498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.411506:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.411509:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.411510:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.411512:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.411515:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.411516:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.411517:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.411519:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.411520:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.411521:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.411524:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.411526:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.411529:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.411530:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.411530:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.411532:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.411532:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.411533:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.411533:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.411676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.411718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.411721:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.411724:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.411725:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.411726:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.411728:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.411730:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.411805:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.411808:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.411809:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.411811:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.411812:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.411814:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.411815:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.411816:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.411817:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.411818:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.411822:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.411824:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.411826:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.411827:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.411827:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.411829:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.411829:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.411830:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.411830:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.411835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.411843:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.411846:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.411847:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.411847:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.411849:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.411851:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.411855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.411862:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.411864:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.411865:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.411865:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.411866:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.411867:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.412081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.412207:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.412211:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.412211:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.412213:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.412215:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.412217:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.412218:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.412219:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.412220:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.412221:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.412225:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.412226:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.412229:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.412230:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.412231:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.412232:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.412233:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.412234:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.412234:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:22.0:1619677447.412288:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.412291:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.412292:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.412292:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.412298:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.412300:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677447.412588:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.412590:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.412593:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.412594:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.412594:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.412598:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.412600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.412611:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.412613:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.412614:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.412616:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.412617:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.412619:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.412629:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.412631:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.412632:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.412633:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.412634:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.412635:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:6.0:1619677447.412647:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.412650:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.412651:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.412651:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.412653:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.412659:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.412660:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.412662:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.412663:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.412664:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.412664:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.412665:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.412755:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.412758:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.412760:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.412761:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.412764:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.412765:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.412767:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.412768:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.412769:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.412770:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.412773:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.412775:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.412777:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.412778:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.412778:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.412779:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.412780:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.412781:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.412781:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.412786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.412793:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.412795:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.412795:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.412796:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.412797:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.412799:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.412816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.412823:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.412826:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.412826:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.412827:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.412827:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.412828:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.0:1619677447.412879:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.412882:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.412883:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.412885:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.412888:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.412896:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.412898:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96728596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.412900:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.412901:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.412902:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.412905:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.412907:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:9.2:1619677447.412910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.412910:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.412911:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.412912:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.412915:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.412915:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.412917:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.412917:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.412918:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677447.412919:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.412919:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.412920:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.412920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:24.0:1619677447.412923:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.412926:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:18.0:1619677447.412928:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.412931:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.412932:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.412933:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.412935:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.412938:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677447.413173:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.413175:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.413178:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.413178:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.413180:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.413182:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677447.413183:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.413185:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.413186:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.413187:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677447.413190:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.413192:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677447.413195:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.413195:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677447.413196:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677447.413198:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677447.413198:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.413199:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.413200:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.413204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.413211:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.413214:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.413215:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.413215:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.413217:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.413219:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:22.0:1619677447.413261:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.413263:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:4.2:1619677447.413263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:22.0:1619677447.413264:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.413264:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.413267:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.413269:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.413271:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.413273:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.413274:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.413276:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.413279:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.413281:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.413282:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.413283:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.413285:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.413287:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.413291:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.413300:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.413303:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.413304:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.413305:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.413306:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.413307:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.413308:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.413309:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677447.413330:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.413332:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.413333:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.413335:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.413337:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.413338:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.413447:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.413449:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677447.413451:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677447.413452:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.413454:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.413456:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677447.413457:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.413459:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.413460:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.413461:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.413465:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.413467:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.413469:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.413470:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.413470:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.413472:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.413473:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.413473:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.413474:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677447.413514:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e8d3aa00] (21)++
00000800:00000200:4.2:1619677447.413516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.413518:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.50@o2ib3
00000800:00000200:5.0:1619677447.413521:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3aa00] (22)--
00000400:00000200:20.0:1619677447.413522:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.50@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.413525:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.413527:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:20.0:1619677447.413529:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.413531:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.413532:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.413533:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:20.0:1619677447.413538:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.413541:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.0:1619677447.413544:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.413545:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:20.0:1619677447.413545:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.0:1619677447.413547:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:20.0:1619677447.413548:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e8d3aa00] (21)++
00000800:00000200:20.0:1619677447.413549:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e8d3aa00] (22)--
00000800:00000200:20.0:1619677447.413550:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3aa00] (21)--
00000800:00000200:9.2:1619677447.413553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.413560:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:5.0:1619677447.413560:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e8d3aa00] (21)++
00000800:00000200:24.0:1619677447.413563:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:5.0:1619677447.413563:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.50@o2ib3
00000400:00000200:24.0:1619677447.413564:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.413564:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:6.0:1619677447.413565:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:20.0:1619677447.413566:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3aa00] (22)--
00000800:00000200:6.0:1619677447.413566:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:5.0:1619677447.413566:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.50@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.413567:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.413568:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.413568:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.413569:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:24.0:1619677447.413570:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677447.413570:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.413570:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.413570:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:22.0:1619677447.413571:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.413572:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96728587] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.413573:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.413574:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.413575:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.413576:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:22.0:1619677447.413577:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.413578:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.413578:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.413579:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:5.0:1619677447.413580:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.413582:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:5.0:1619677447.413585:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.413585:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677447.413586:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.0:1619677447.413588:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677447.413589:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e8d3aa00] (21)++
00000800:00000200:5.0:1619677447.413590:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e8d3aa00] (22)--
00000800:00000200:5.0:1619677447.413590:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3aa00] (21)--
00000800:00000200:9.2:1619677447.413594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677447.413598:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e8d3aa00] (21)++
00000800:00000200:20.0:1619677447.413600:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.50@o2ib3
00000800:00000200:8.0:1619677447.413600:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:20.0:1619677447.413602:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.50@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.413603:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.413604:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.413604:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.413604:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3aa00] (22)--
00000400:00000200:20.0:1619677447.413605:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.413607:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677447.413607:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.413608:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.413610:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.413611:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.413612:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.413614:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:20.0:1619677447.413617:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.413620:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677447.413622:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.413623:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:20.0:1619677447.413624:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677447.413626:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677447.413627:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e8d3aa00] (21)++
00000800:00000200:20.0:1619677447.413628:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197e8d3aa00] (22)++
00000800:00000200:20.0:1619677447.413629:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e8d3aa00] (23)--
00000800:00000200:20.0:1619677447.413630:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3aa00] (22)--
00000800:00000200:5.0:1619677447.413636:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e8d3aa00] (22)++
00000800:00000200:12.0:1619677447.413639:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677447.413639:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.50@o2ib3
00000800:00000200:20.0:1619677447.413641:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e8d3aa00] (23)++
00000800:00000200:20.0:1619677447.413642:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197e8d3aa00] (24)--
00000800:00000200:12.0:1619677447.413642:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:5.0:1619677447.413642:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.50@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.413643:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3aa00] (23)--
00000400:00000200:12.0:1619677447.413643:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.413644:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3aa00] (22)--
00000400:00000200:12.0:1619677447.413645:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.413645:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.413646:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:5.0:1619677447.413647:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:5.0:1619677447.413649:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.413650:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.413651:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.413652:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:5.0:1619677447.413656:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.413658:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:12.0:1619677447.413661:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.413661:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.413661:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:12.0:1619677447.413662:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.413662:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:13.0:1619677447.413664:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.413664:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677447.413664:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000400:00000200:12.0:1619677447.413665:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.413665:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e8d3aa00] (21)++
00000800:00000200:5.0:1619677447.413666:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e8d3aa00] (22)--
00000800:00000200:5.0:1619677447.413666:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3aa00] (21)--
00000400:00000200:12.0:1619677447.413667:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.413669:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.413670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.413676:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.413679:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:20.0:1619677447.413679:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e8d3aa00] (21)++
00000400:00000200:24.0:1619677447.413680:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.413680:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:20.0:1619677447.413681:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.50@o2ib3
00000400:00000200:24.0:1619677447.413683:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.413684:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.50@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.413684:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3aa00] (22)--
00000800:00000200:24.0:1619677447.413686:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:20.0:1619677447.413688:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.413690:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:20.0:1619677447.413691:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.413693:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.413694:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.413695:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:20.0:1619677447.413698:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.413700:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:20.0:1619677447.413703:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.413704:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:20.0:1619677447.413704:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:20.0:1619677447.413706:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:20.0:1619677447.413707:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e8d3aa00] (21)++
00000800:00000200:20.0:1619677447.413708:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e8d3aa00] (22)--
00000800:00000200:20.0:1619677447.413708:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3aa00] (21)--
00000800:00000200:5.0:1619677447.413732:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e8d3aa00] (21)++
00000800:00000200:5.0:1619677447.413735:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.50@o2ib3
00000800:00000200:20.0:1619677447.413737:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3aa00] (22)--
00000400:00000200:5.0:1619677447.413738:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.50@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.413741:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.413742:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.0:1619677447.413744:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:5.0:1619677447.413744:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.413745:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.413745:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.413746:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.413747:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:9.2:1619677447.413749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:20.0:1619677447.413751:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.413753:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.413753:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.413755:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:5.0:1619677447.413755:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:20.0:1619677447.413756:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96728580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.413756:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:20.0:1619677447.413758:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.413758:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.413758:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.413759:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:20.0:1619677447.413760:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:8.0:1619677447.413760:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.413762:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677447.413763:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.413763:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:20.0:1619677447.413764:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.413764:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.413765:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.0:1619677447.413767:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:9.2:1619677447.413767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677447.413769:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.413770:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.413770:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.413771:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.413771:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e8d3aa00] (22)++
00000800:00000200:5.0:1619677447.413771:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e8d3aa00] (23)--
00000800:00000200:5.0:1619677447.413772:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3aa00] (22)--
00000800:00000200:24.0:1619677447.413773:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677447.413773:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.413773:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.413774:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.413774:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e8d3aa00] (21)++
00000800:00000200:20.0:1619677447.413775:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.413775:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:24.0:1619677447.413776:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:20.0:1619677447.413776:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3aa00] (22)--
00000800:00000200:5.0:1619677447.413776:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.50@o2ib3
00000800:00000200:24.0:1619677447.413777:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000400:00000200:5.0:1619677447.413778:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.50@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.413779:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.413779:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.413780:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:5.0:1619677447.413781:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.413782:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:5.0:1619677447.413783:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96728579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.413785:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.413787:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.413788:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:5.0:1619677447.413792:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.413795:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.413798:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.413798:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677447.413799:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.413801:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.413801:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e8d3aa00] (21)++
00000800:00000200:5.0:1619677447.413803:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197e8d3aa00] (22)++
00000800:00000200:5.0:1619677447.413804:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e8d3aa00] (23)--
00000800:00000200:5.0:1619677447.413805:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3aa00] (22)--
00000800:00000200:20.0:1619677447.413815:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.413816:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.413817:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.413819:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.413819:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e8d3aa00] (22)++
00000800:00000200:5.0:1619677447.413820:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197e8d3aa00] (23)--
00000800:00000200:5.0:1619677447.413821:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3aa00] (22)--
00000800:00000200:20.0:1619677447.413822:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.413824:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.413825:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3aa00] (21)--
00000800:00000200:13.0:1619677447.413828:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.413830:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.413831:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.413832:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.413834:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.413835:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.413838:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.413841:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.413842:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.413843:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.413844:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.413845:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.413896:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.413899:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.413900:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.413902:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.413902:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.413905:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677447.413994:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.413996:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.413997:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.413999:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.414000:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.414002:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.414014:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.414017:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.414018:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.414019:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.414019:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.414021:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677447.414106:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.414109:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.414111:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.414112:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.414113:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.414115:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677447.414116:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.414117:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.414118:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.414119:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677447.414123:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.414125:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677447.414127:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.414128:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.414129:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.414130:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.414131:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.414132:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.414134:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.414134:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.414145:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.414148:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.414148:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.414150:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:24.0:1619677447.414229:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.414231:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.414232:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.414234:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.414237:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.414239:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.414240:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.414241:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.414242:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.414244:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.414247:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.414249:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.414252:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.414252:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.414253:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.414255:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.414255:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.414256:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.414256:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677447.414269:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.414272:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.414273:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.414274:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.414275:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.414277:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.414299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.414329:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.414332:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.414334:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.414335:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.414337:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.414339:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.414341:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96728570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.414342:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.414343:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.414345:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.414348:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.414348:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.414350:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.414350:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.414351:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.414351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677447.414352:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.414355:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.414355:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.414356:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677447.414356:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.414357:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:20.0:1619677447.414359:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (25)--
00000800:00000200:20.0:1619677447.414360:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.414361:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.414361:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.414437:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.414440:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.414440:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.414442:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.414444:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.414445:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.414455:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.414459:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.414460:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.414462:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.414464:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.414465:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.414467:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.414468:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.414469:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.414470:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.414473:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.414475:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.414477:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.414478:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.414478:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.414480:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.414481:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.414482:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.414483:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.414484:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.414486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.414495:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.414496:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.414496:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.414497:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:2.0:1619677447.414497:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.414498:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.414499:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.414499:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.414500:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677447.414500:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.414582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.414589:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.414592:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.414593:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.414595:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.414598:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.414599:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.414601:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.414603:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.414604:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.414605:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.414609:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.414611:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.414614:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.414614:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.414615:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.414617:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.414617:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.414618:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.414618:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.414635:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.414637:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.414638:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.414640:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.414641:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.414642:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.414668:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.414671:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.414674:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.414674:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.414677:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:5.0:1619677447.414678:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.414680:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:24.0:1619677447.414681:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.414681:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:5.0:1619677447.414682:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.414684:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.414685:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.414685:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.414686:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.414688:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.414690:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.414691:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.414692:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96728564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.414693:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.414694:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.414695:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.414696:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.414697:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.414698:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.414698:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.414700:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677447.414701:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.414701:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.414702:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:24.0:1619677447.414703:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.414703:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:24.0:1619677447.414706:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.414707:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677447.414708:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:24.0:1619677447.414710:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:24.0:1619677447.414710:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.414711:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.414711:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.414712:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (25)++
00000800:00000200:13.0:1619677447.414715:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (25)--
00000400:00000200:13.0:1619677447.414716:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.414718:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.414719:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)++
00000800:00000200:13.0:1619677447.414720:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677447.414720:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (25)--
00000400:00000200:12.0:1619677447.414720:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.414722:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000400:00000200:12.0:1619677447.414722:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.414723:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677447.414724:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.414724:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.414725:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.414725:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.414727:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.414740:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.414742:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.414743:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.414744:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.414744:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.414745:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.414836:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.414838:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.414839:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.414841:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.414843:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.414844:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.414961:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.414963:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.414964:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.414967:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.414969:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.414970:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.414972:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.414973:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.414974:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.414975:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.414978:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.414980:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.414982:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.414983:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.414984:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.414985:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.414986:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.414987:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.414988:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.414988:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.414998:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:22.0:1619677447.414999:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677447.415000:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.415001:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.415002:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.415003:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.415004:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.415004:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.415006:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.415009:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.415019:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.415022:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.415023:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.415025:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.415027:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.415028:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.415030:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.415031:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.415032:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.415033:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.415036:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.415038:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.415041:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.415042:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.415043:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.415044:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.415045:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.415046:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.415047:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.415048:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677447.415050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.415050:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.415053:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.415054:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.415056:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.415058:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.415058:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:6.0:1619677447.415059:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:24.0:1619677447.415060:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:6.0:1619677447.415061:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.415061:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:20.0:1619677447.415062:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.415062:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:6.0:1619677447.415062:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.415062:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.415063:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.415064:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:6.0:1619677447.415064:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.415065:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677447.415065:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:6.0:1619677447.415065:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677447.415065:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.415068:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.415068:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:6.0:1619677447.415069:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.415069:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:20.0:1619677447.415072:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.415072:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:20.0:1619677447.415074:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:6.0:1619677447.415075:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.415076:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.415076:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.415076:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:20.0:1619677447.415077:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677447.415077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.415078:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.415079:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.415079:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677447.415080:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.415080:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:20.0:1619677447.415081:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.415084:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.415086:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.415089:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.415090:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.415091:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.415092:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.415093:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.415094:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.415095:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.415105:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.415107:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.415108:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.415110:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.415111:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.415113:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.415124:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.415126:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.415127:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.415128:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.415128:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.415128:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:13.0:1619677447.415129:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.415131:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.415132:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.415134:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.415134:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.415137:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:12.0:1619677447.415234:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.415237:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.415237:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.415239:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.415241:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.415242:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.415316:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.415318:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.415319:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.415320:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.415322:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.415323:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677447.415333:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.415335:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.415338:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.415339:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.415340:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.415341:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677447.415343:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.415344:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.415345:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.415346:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677447.415349:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.415350:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677447.415353:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.415354:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677447.415354:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677447.415356:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677447.415357:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.415358:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.415359:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677447.415401:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.415403:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.415405:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.415407:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.415410:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.415411:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.415412:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.415413:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.415414:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.415415:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.415419:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.415421:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:13.0:1619677447.415421:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677447.415424:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.415424:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.415424:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.415425:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.415425:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:13.0:1619677447.415426:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.415427:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.415427:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:12.0:1619677447.415427:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.415428:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.415429:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677447.415429:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.415441:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.415443:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.415444:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.415445:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.415447:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.415449:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.415463:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.415465:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677447.415466:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677447.415468:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.415471:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.415472:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:6.0:1619677447.415474:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.415475:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.415476:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.415477:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:6.0:1619677447.415480:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.415482:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677447.415484:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.415485:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677447.415486:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.415488:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677447.415489:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.415489:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677447.415489:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677447.415494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.415502:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.415504:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.415505:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.415505:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.415508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:2.0:1619677447.415508:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.415511:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.415514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.415516:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.415519:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.415520:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677447.415522:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677447.415522:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.415525:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.415525:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.415525:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.415526:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000400:00000200:8.0:1619677447.415526:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:18.0:1619677447.415527:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.415527:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:8.0:1619677447.415527:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96728548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.415528:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677447.415529:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.415530:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.415531:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:8.0:1619677447.415531:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:12.0:1619677447.415532:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.415534:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.415534:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.415535:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677447.415536:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.415537:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.415537:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677447.415538:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.415539:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677447.415540:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.415541:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:22.0:1619677447.415542:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.415542:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:22.0:1619677447.415544:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.415544:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.415544:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.415545:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:22.0:1619677447.415546:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:8.0:1619677447.415546:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000400:00000200:22.0:1619677447.415547:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728547] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.415547:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677447.415549:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.415550:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.415551:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:4.2:1619677447.415552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:22.0:1619677447.415555:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.415557:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:24.0:1619677447.415559:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677447.415560:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.415561:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:22.0:1619677447.415561:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:24.0:1619677447.415562:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677447.415562:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:24.0:1619677447.415563:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.415564:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.415565:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.415565:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.415566:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.415679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.415727:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.415730:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.415731:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.415732:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.415734:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.415736:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677447.415739:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.415739:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.415740:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677447.415740:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.415742:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.415743:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000400:00000200:2.0:1619677447.415743:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.415744:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.415744:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.415745:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:2.0:1619677447.415746:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.415748:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.415749:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.415751:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.415751:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.415753:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.415756:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.415758:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.415760:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.415761:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.415762:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.415763:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.415764:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.415764:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.415765:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677447.415776:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.415779:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.415780:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.415782:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.415782:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.415784:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.415896:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.415898:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.415901:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.415901:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.415904:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.415905:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677447.415906:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.415908:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.415909:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.415910:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677447.415913:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.415915:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.415918:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.415918:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.415919:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.415921:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.415921:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.415923:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.415924:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.415924:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.415936:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.415938:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.415938:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.415939:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.415939:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.415939:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.415940:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.415941:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.415942:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.415943:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.415953:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.415956:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.415957:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.415959:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.415962:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.415963:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.415965:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.415966:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.415967:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:6.0:1619677447.415967:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.415968:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:18.0:1619677447.415971:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.415971:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:6.0:1619677447.415973:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.415974:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.415975:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:18.0:1619677447.415976:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.415978:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:6.0:1619677447.415978:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.415979:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:18.0:1619677447.415980:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.415980:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:18.0:1619677447.415981:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.415982:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.415982:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.415982:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:18.0:1619677447.415983:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677447.415984:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.415985:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.415985:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:18.0:1619677447.415986:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.415988:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.415991:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.415992:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.415993:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677447.415995:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677447.415995:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.415995:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.415996:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.415997:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:22.0:1619677447.415998:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.415999:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.416000:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.416008:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.416010:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.416012:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.416013:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.416014:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.416016:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.416023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.416069:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.416072:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.416073:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.416073:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.416076:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.416079:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.416087:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.416090:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.416091:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.416091:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.416093:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.416093:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.416307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.416330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.416338:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.416341:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.416342:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.416343:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.416346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:8.0:1619677447.416347:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.416349:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.416350:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96728536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.416351:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.416352:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.416352:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.416352:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:8.0:1619677447.416355:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.416355:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.416356:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:8.0:1619677447.416357:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.416357:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.416358:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.416360:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.416361:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.416361:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.416362:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.416363:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.416364:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.416364:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.416365:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.416369:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.416372:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.416372:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.416373:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.416374:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.416375:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.416439:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.416442:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.416442:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.416444:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.416447:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.416449:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.416450:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.416452:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.416453:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.416454:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.416457:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.416458:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.416461:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.416462:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.416463:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.416464:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.416465:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.416465:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.416466:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.416568:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.416571:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.416572:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.416574:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.416577:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.416578:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.416579:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.416581:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.416582:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.416583:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677447.416585:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.416587:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.416590:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.416591:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.416592:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.416593:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.416594:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.416595:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.416595:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677447.416615:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.416618:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.416619:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.416620:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.416623:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.416624:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.416626:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.416627:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.416628:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.416629:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.416632:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.416633:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.416635:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.416636:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.416636:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.416637:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.416638:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.416639:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.416639:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:4.2:1619677447.416645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.416692:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.416695:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.416696:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.416697:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.416699:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.416702:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.416928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.416971:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677447.416972:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677447.416975:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:6.0:1619677447.416976:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.416977:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.416978:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.416979:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.416980:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.416980:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:6.0:1619677447.416981:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.416983:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.416984:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.416985:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677447.416989:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.416991:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.416992:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.416993:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677447.416993:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.416994:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.416994:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.416995:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.416997:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.416997:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677447.416998:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677447.416999:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:22.0:1619677447.417009:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.417012:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.417012:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.417013:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.417014:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.417015:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.417253:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.417255:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.417256:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.417256:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.417258:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.417261:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.417292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.417301:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.417304:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.417305:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.417307:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.417310:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.417311:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.417312:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.417314:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.417315:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.417316:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.417319:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.417321:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.417324:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.417325:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.417325:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.417327:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.417328:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.417328:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.417329:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.417363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.417397:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.417400:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.417400:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.417402:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.417404:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.417406:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.417407:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.417409:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.417409:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.417411:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.417415:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.417417:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.417419:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.417420:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.417421:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.417422:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.417423:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.417424:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.417424:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.417429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.417436:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.417439:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.417440:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.417441:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.417443:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.417445:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677447.417505:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.417508:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.417509:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.417511:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.417511:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.417514:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677447.417741:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.417744:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.417745:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.417747:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.417749:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.417750:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.417752:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.417753:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.417755:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.417756:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.417759:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.417761:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.417764:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.417765:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.417766:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677447.417768:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677447.417768:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.417770:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.417770:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677447.417809:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907b781800] (21)++
00000800:00000200:2.0:1619677447.417812:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.30@o2ib3
00000800:00000200:18.0:1619677447.417813:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907b781800] (22)--
00000400:00000200:2.0:1619677447.417814:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.30@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.417818:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.417820:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:2.0:1619677447.417821:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.417822:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.417823:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.417824:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:2.0:1619677447.417828:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.30@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.417830:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:2.0:1619677447.417833:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f90d380] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.417834:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e49d37c00] (20)++
00000800:00000200:2.0:1619677447.417835:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e49d37c00] (21)++
00000800:00000200:2.0:1619677447.417836:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e49d37c00] (22)--
00000800:00000200:2.0:1619677447.417837:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907b781800] (21)++
00000800:00000200:2.0:1619677447.417838:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907b781800] (22)--
00000800:00000200:2.0:1619677447.417839:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907b781800] (21)--
00000800:00000200:24.2:1619677447.417843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (21)++
00000800:00000200:24.0:1619677447.417846:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)++
00000800:00000200:24.0:1619677447.417849:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e49d37c00] (23)--
00000400:00000200:24.0:1619677447.417851:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.417851:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)--
00000400:00000200:24.0:1619677447.417853:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.417855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.417857:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)--
00000800:00000200:24.0:1619677447.417859:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.417861:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.417862:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.417862:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.417863:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.417866:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.417912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.417952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.417957:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.417961:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.417961:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.417962:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.417964:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.417971:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.417972:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:6.0:1619677447.417972:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.417975:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.417976:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:6.0:1619677447.417976:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.417976:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677447.417977:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.417978:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.417979:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:18.0:1619677447.417980:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.417982:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.417983:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.417985:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.417986:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.417987:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.417988:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.417991:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.417993:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.417994:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.417995:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.417996:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.417997:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.417998:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.417998:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.417999:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.418003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.418010:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.418013:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.418013:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.418014:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.418015:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.418017:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.418101:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.418104:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.418106:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.418107:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.418109:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.418110:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677447.418112:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.418113:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.418114:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.418115:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677447.418119:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.418120:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677447.418124:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.418124:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677447.418125:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677447.418127:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677447.418127:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.418128:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.418129:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677447.418133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.418139:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.418142:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.418143:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.418143:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.418145:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.418148:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677447.418214:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.418217:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.418217:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.418220:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.418223:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.418224:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.418226:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96728514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.418227:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.418228:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.418229:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.418230:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:24.0:1619677447.418232:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.418232:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.418233:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.418233:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:24.0:1619677447.418234:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:22.0:1619677447.418235:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.418237:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.418238:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.418239:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.418239:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.418241:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.418242:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.418243:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.418243:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.418249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.418256:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.418259:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.418260:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.418260:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.418262:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.418266:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.418374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37e000] (20)++
00000800:00000200:24.2:1619677447.418419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (21)++
00000800:00000200:24.0:1619677447.418422:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)++
00000800:00000200:24.0:1619677447.418425:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e49d37c00] (23)--
00000400:00000200:24.0:1619677447.418426:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.418426:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)--
00000400:00000200:24.0:1619677447.418427:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.418430:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)--
00000800:00000200:5.0:1619677447.418442:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.418444:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.418446:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.418447:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.418448:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.418450:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677447.418558:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.418561:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.418561:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.418563:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.418565:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.418566:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.418574:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.418576:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.418577:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.418578:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.418578:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.418579:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.418706:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.418709:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.418711:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.418712:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.418713:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.418715:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.418716:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.418717:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.418718:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.418719:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.418722:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.418724:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.418727:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.418728:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.418728:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.418730:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.418730:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.418731:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.418732:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.418742:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.418745:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.418746:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.418748:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.418749:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.418750:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.418889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.418936:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.418939:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.418940:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.418940:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.418942:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.418945:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.418976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.418984:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.418986:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.418987:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.418988:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.418991:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.418994:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677447.419028:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.419030:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.419031:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.419033:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.419033:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.419036:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.419099:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.419102:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.419105:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.419105:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.419108:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.419110:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.419111:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.419113:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.419114:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.419115:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.419119:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.419122:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.419125:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.419126:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.419126:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.419128:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.419129:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.419130:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.419130:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.419142:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.419144:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.419144:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:13.0:1619677447.419145:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.419146:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.419146:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.419147:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:12.0:1619677447.419148:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.419150:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.419151:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.419152:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.419153:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.419163:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.419165:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.419166:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.419167:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.419167:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.419168:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.419175:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.419178:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.419178:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.419179:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.419181:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.419184:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677447.419391:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677447.419394:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677447.419395:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677447.419397:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.419400:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.419401:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677447.419402:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.419403:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.419404:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.419405:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677447.419408:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.419410:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.419412:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.419413:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.419413:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.419415:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.419415:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677447.419417:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677447.419418:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677447.419418:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:9.2:1619677447.419422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.419431:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.419432:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677447.419432:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.419433:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000400:00000200:2.0:1619677447.419433:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.419434:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677447.419435:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:2.0:1619677447.419435:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.419436:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.419436:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.419437:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:18.0:1619677447.419438:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.419438:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:18.0:1619677447.419439:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.419439:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.419441:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.419442:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.419443:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.419444:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.419445:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.419448:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.419449:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.419451:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.419451:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.419451:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.419453:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.419453:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.419454:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.419455:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.419455:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.419459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.419468:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.419468:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.419469:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.419469:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:2.0:1619677447.419469:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.419470:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.419471:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.419471:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.419472:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677447.419472:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.419478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.419485:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.419487:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.419488:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.419489:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.419489:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.419490:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677447.419546:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fdae00] (21)++
00000800:00000200:5.0:1619677447.419548:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.38@o2ib3
00000400:00000200:5.0:1619677447.419550:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.38@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677447.419551:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (22)--
00000400:00000200:5.0:1619677447.419553:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.419554:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:13.0:1619677447.419555:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:5.0:1619677447.419556:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.419557:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:5.0:1619677447.419557:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677447.419558:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.419558:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.419559:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:13.0:1619677447.419560:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.419562:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:5.0:1619677447.419563:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.38@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:13.0:1619677447.419564:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.419565:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:5.0:1619677447.419568:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.419569:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677447.419569:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.0:1619677447.419571:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677447.419572:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fdae00] (21)++
00000800:00000200:5.0:1619677447.419573:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fdae00] (22)--
00000800:00000200:5.0:1619677447.419573:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (21)--
00000800:00000200:9.2:1619677447.419577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.419584:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.419587:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.419588:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.419588:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.419590:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677447.419593:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677447.419618:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.419621:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:20.0:1619677447.419622:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:12.0:1619677447.419622:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.419623:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.419625:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:13.0:1619677447.419625:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.419625:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:20.0:1619677447.419627:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.419628:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.419629:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.419631:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.419632:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.419634:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.419635:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.419636:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.419639:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.419641:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.419644:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.419645:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.419645:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.419647:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:18.0:1619677447.419647:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.0:1619677447.419648:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.419649:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.419649:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.419650:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.419651:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.419653:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.419654:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.419656:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.419657:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.419658:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.419659:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.419660:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.419663:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.419665:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.419666:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.419667:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.419668:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.419670:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.419670:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.419671:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.419672:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.419673:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677447.419676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.419684:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.419685:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.419685:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.419686:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:2.0:1619677447.419686:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.419687:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.419688:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.419689:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:2.0:1619677447.419689:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.419691:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.419712:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)++
00000800:00000200:4.2:1619677447.419712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.419715:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677447.419715:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.419717:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.419718:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.419719:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677447.419719:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677447.419720:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.419720:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.419722:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.419724:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.419742:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.419744:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.419745:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.419746:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.419746:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.419747:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.419872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.419879:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.419881:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.419882:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.419884:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.419886:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.419888:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.419889:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.419890:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.419891:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.419892:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.419895:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.419897:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.419899:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.419900:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.419901:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.419903:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.419903:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.419904:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.419904:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.0:1619677447.419968:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.419971:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677447.419973:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.419976:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.419977:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:20.0:1619677447.419978:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.419979:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.419980:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.419981:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.419982:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.419987:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.419990:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.419993:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.419994:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.419994:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.419996:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.419997:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.419999:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.420000:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.420000:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:9.2:1619677447.420002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.420008:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.420010:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:18.0:1619677447.420010:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677447.420011:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.420012:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677447.420013:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.420014:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.420014:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.420016:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.420019:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.420028:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.420030:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.420031:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.420033:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.420034:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.420036:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.420036:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.420037:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:20.0:1619677447.420038:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.420038:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.420039:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677447.420042:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.420045:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.420046:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.420048:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.420050:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.420051:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677447.420052:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.420054:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.420055:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.420056:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677447.420059:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.420061:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677447.420063:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.420064:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677447.420065:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677447.420066:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677447.420067:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.420067:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.420068:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677447.420080:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.420083:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.420084:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.420084:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.420087:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.420089:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.420111:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.420114:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.420114:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.420116:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.420118:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.420119:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.420254:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.420257:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.420257:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.420259:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.420262:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.420263:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.420265:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.420266:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.420267:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.420268:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.420272:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.420273:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.420276:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.420277:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.420277:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.420279:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.420280:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.420280:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.420281:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.420292:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.420299:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.420300:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.420300:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:20.0:1619677447.420301:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.420303:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677447.420354:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.420356:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.420357:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.420359:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.420361:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.420363:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677447.420364:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.420366:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.420367:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.420368:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677447.420371:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.420373:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.420376:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.420377:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:18.0:1619677447.420378:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677447.420379:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.420380:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:4.2:1619677447.420380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.420381:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.420381:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677447.420393:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.420396:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.420397:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.420400:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.420400:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.420410:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.420421:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677447.420424:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677447.420425:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.420426:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.420428:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677447.420429:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677447.420429:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677447.420430:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.420430:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677447.420431:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.420433:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.420434:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.420437:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.420438:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.420439:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.420440:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.420442:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:12.0:1619677447.420443:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.420446:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.420447:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.420448:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.420448:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.420449:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677447.420451:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.420453:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.420454:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.420455:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.420456:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.420457:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:6.0:1619677447.420532:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.420535:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677447.420535:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000400:00000200:6.0:1619677447.420538:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.420540:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.420542:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677447.420543:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.420544:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.420545:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.420546:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677447.420550:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.420552:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677447.420554:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:6.0:1619677447.420554:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.420555:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:13.0:1619677447.420556:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:6.0:1619677447.420556:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:13.0:1619677447.420557:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.420558:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:6.0:1619677447.420558:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000400:00000200:13.0:1619677447.420559:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.420559:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:12.0:1619677447.420560:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677447.420560:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:13.0:1619677447.420562:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.420613:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.420616:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.420618:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.420618:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.420620:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.420622:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.420623:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.420624:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.420625:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.420626:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.420630:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.420632:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.420634:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.420635:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.420636:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.420638:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.420639:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.420640:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.420641:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.420642:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.420650:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677447.420650:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677447.420652:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.420652:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.420653:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.420653:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.420654:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.420655:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.420657:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.420659:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.420911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.420918:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.420920:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.420922:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.420923:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.420926:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.420928:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.420929:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.420930:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.420931:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.420932:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.420935:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.420937:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.420939:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.420940:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.420941:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.420942:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.420943:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.420944:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.420946:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.420946:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.420952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.420959:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.420961:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.420962:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.420963:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677447.420984:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.420987:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.420988:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.420991:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.420992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677447.420993:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.420995:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.420996:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.420997:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.420998:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.420999:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.421004:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.421006:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.421010:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.421011:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.421012:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.421014:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.421014:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.421015:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.421016:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677447.421026:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.421029:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.421030:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.421032:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.421033:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.421035:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.421037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677447.421043:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.421045:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.421046:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.421048:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.421049:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.421051:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:12.0:1619677447.421055:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.421058:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.421059:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.421061:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.421062:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.421063:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.421143:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.421146:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.421147:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.421149:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.421152:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.421153:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:8.0:1619677447.421154:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.421155:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96728472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.421156:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:8.0:1619677447.421156:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.421156:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:8.0:1619677447.421158:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.421159:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.421159:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677447.421162:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.421163:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.421163:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.421165:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96728471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.421165:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:22.0:1619677447.421167:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.421168:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.421169:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677447.421170:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.421170:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.421170:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.421172:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:22.0:1619677447.421173:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.421173:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.421173:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.421174:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.421175:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.421178:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.421179:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.421179:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:9.2:1619677447.421179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.421181:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.421181:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.421182:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.421183:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.421184:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677447.421187:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.421190:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.421191:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.421191:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.421193:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.421194:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677447.421197:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.421197:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.421198:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.421198:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.421200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.421207:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.421210:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.421211:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.421211:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.421213:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.421216:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.421223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.421230:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.421233:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.421234:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.421234:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.421235:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.421236:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677447.421301:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.421304:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.421304:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.421306:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.421307:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.421309:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:13.0:1619677447.421377:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.421380:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.421381:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.421383:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.421384:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.421385:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.421394:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.421396:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.421397:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.421398:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.421398:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.421399:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.421414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.421460:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.421463:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.421464:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.421465:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.421466:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.421470:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.421569:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.421572:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.421574:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.421575:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.421577:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.421579:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.421580:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.421581:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.421582:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.421583:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.421587:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.421589:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.421592:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.421593:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.421594:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.421596:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.421596:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.421597:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.421598:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.421608:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.421611:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.421612:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.421613:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.421614:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677447.421615:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.421616:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.421617:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.421617:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.421620:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.421623:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.421624:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.421626:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.421626:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677447.421628:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.421630:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.421630:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677447.421631:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677447.421631:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.421631:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.421632:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.421633:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:22.0:1619677447.421640:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.421642:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.421645:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.421646:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.421646:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.421648:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.421648:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.421649:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.421650:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.421676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.421683:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.421686:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.421687:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.421689:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.421692:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.421694:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.421695:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.421697:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.421698:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.421699:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.421702:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.421704:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.421707:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.421707:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.421708:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.421709:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.421710:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.421711:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.421711:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.421817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.421863:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.421866:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.421867:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.421868:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.421870:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.421873:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.421883:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.421886:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.421887:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.421887:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.421888:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.421889:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.421977:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.421980:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.421981:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.421983:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.421985:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.421987:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.421988:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.421990:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.421991:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.421992:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.421995:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.421997:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.422000:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.422001:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.422001:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.422003:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.422003:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.422004:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.422004:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.422017:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.422020:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.422020:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.422022:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.422023:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.422025:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:6.0:1619677447.422081:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.422084:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.422085:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.422086:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.422087:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.422090:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.422098:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.422101:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.422102:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.422102:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.422103:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.422104:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.422116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.422129:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.422132:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.422133:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.422134:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.422137:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.422138:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.422140:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.422142:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.422143:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.422144:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.422149:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.422152:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.422155:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.422156:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.422157:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.422159:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.422159:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.422160:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.422160:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.422165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.422174:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:18.0:1619677447.422174:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677447.422177:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:18.0:1619677447.422177:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.422178:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.422178:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677447.422178:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:22.0:1619677447.422179:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.422181:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.422182:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.422186:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677447.422186:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677447.422334:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677447.422337:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677447.422338:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677447.422339:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.422342:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.422343:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677447.422345:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.422346:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.422347:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.422348:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677447.422351:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.422353:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.422354:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.422355:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.422356:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.422357:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.422358:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:8.0:1619677447.422359:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:2.0:1619677447.422359:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677447.422360:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.0:1619677447.422362:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.422363:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677447.422363:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:9.2:1619677447.422364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.422365:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:8.0:1619677447.422365:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.422367:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.422368:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:8.0:1619677447.422369:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.422370:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.422370:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677447.422372:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:18.0:1619677447.422372:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:8.0:1619677447.422372:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96728455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.422374:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.422374:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:8.0:1619677447.422374:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.422375:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.422375:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.422375:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:8.0:1619677447.422376:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.422377:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.422377:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.422377:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677447.422379:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.422379:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:20.0:1619677447.422384:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.422385:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.422387:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677447.422388:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.422390:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.422390:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677447.422391:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677447.422391:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.422392:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.422393:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.422393:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677447.422393:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.422394:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.422395:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.0:1619677447.422395:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.422395:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.422396:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.422397:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.422399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:4.2:1619677447.422405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:2.0:1619677447.422406:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:5.0:1619677447.422408:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677447.422408:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.422409:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.422410:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677447.422410:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.422411:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.422411:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.422413:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:5.0:1619677447.422413:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.422414:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.422414:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:24.0:1619677447.422415:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:5.0:1619677447.422416:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.0:1619677447.422417:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.422418:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.422421:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.422423:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.422424:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.422425:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.422426:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.422427:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.422430:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.422432:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.422434:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.422435:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.422435:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.422437:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.422437:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.422438:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.422439:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677447.422543:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.422546:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.422547:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.422549:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.422550:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.422552:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.422559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.422582:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.422585:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.422587:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.422588:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.422591:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.422593:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.422598:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.422600:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.422601:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677447.422601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677447.422603:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677447.422607:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:5.0:1619677447.422608:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.422609:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.422610:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.422610:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.422612:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:6.0:1619677447.422613:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.422615:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.422616:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:6.0:1619677447.422617:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677447.422617:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.422619:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.422620:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.422620:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.422621:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:13.0:1619677447.422633:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.422635:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.422636:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.422638:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.422640:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.422641:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.422732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.422740:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.422742:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.422743:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.422744:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.422746:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.422749:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.422750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.422758:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.422760:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.422761:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.422761:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.422762:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.422763:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.423005:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.423008:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:4.2:1619677447.423008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.423009:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.423011:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.423012:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.423014:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.423015:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.423017:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.423018:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.423019:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.423023:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.423025:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.423028:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.423029:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.423030:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.423032:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.423032:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.423033:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.423034:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:22.0:1619677447.423060:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.423063:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.423064:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.423064:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.423066:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.423069:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.423186:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.423188:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.423189:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.423191:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.423193:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.423195:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.423196:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.423197:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.423198:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.423199:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.423203:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.423204:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.423207:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.423207:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.423208:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.423210:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.423210:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.423211:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.423211:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.423216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.423226:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.423228:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.423229:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:2.0:1619677447.423229:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.423231:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.423231:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.423233:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.423234:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.423235:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000400:00000200:18.0:1619677447.423236:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.423237:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:2.0:1619677447.423237:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.423238:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000400:00000200:18.0:1619677447.423239:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.423239:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.423239:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.423240:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.423240:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.423241:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:18.0:1619677447.423242:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.423243:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.423246:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.423248:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.423248:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.423251:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:18.0:1619677447.423251:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.423252:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.423252:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677447.423254:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.423254:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.423254:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677447.423254:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677447.423255:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.423256:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:20.0:1619677447.423257:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.423259:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677447.423261:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.423262:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.423264:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.423265:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677447.423270:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.423273:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677447.423276:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.423277:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677447.423278:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.423279:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.423280:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.423281:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.423282:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.423285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.423293:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.423299:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.423301:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.423303:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.423304:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.423307:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.423327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.423336:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.423339:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.423340:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.423342:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.423345:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.423346:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.423348:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.423350:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.423351:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.423352:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.423355:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.423357:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.423360:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.423361:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.423361:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.423363:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.423363:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.423364:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.423364:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.423418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.423465:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.423468:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.423468:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.423469:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.423471:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.423474:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.423482:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.423484:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.423485:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.423486:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.423487:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.423487:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.423655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.423662:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.423664:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.423666:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.423666:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.423668:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.423672:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.423696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.423704:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.423707:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.423708:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.423708:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.423710:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.423713:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.423801:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.423804:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.423804:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677447.423806:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.423809:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.423810:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.423811:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.423813:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.423814:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.423815:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677447.423818:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.423820:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.423823:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.423824:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.423824:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.423826:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.423826:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.423827:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.423828:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677447.423846:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.423849:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.423851:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.423851:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.423853:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.423855:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677447.423856:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.423857:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.423858:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.423860:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677447.423863:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.423865:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677447.423867:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.423868:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677447.423869:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677447.423870:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677447.423871:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.423872:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.423874:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.423874:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:9.2:1619677447.423876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.423882:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.423884:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:18.0:1619677447.423884:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:20.0:1619677447.423885:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.423885:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.423886:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.423888:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.423888:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.423890:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.423892:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677447.424003:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.424006:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.424007:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.424010:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.424013:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.424014:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.424016:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.424018:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.424018:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.424020:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.424023:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.424025:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.424027:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.424028:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.424029:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.424030:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.424031:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.424032:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.424032:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.424033:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.424035:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.424036:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.424036:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.424038:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.424042:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.424043:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.424045:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.424045:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.424046:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.424047:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.424049:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.424098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.424144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:6.0:1619677447.424147:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.424150:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.424150:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:8.0:1619677447.424151:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:6.0:1619677447.424151:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.424153:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.424154:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.424155:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.424155:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.424156:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:8.0:1619677447.424157:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.424159:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.0:1619677447.424430:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.424432:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.424435:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.424436:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.424438:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.424440:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.424442:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.424443:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.424445:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.424446:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.424452:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.424454:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.424457:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.424458:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.424459:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.424460:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.424461:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.424462:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.424463:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.424468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.424477:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:18.0:1619677447.424477:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:22.0:1619677447.424480:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:18.0:1619677447.424480:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:22.0:1619677447.424481:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.424481:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:18.0:1619677447.424482:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.424482:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:22.0:1619677447.424485:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.424486:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.424488:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677447.424489:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.424551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.424559:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.424561:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.424563:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.424564:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.424567:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.424568:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.424569:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96728425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.424571:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.424572:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.424574:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.424578:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.424579:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.424582:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.424583:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.424584:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.424586:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.424586:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.424588:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.424589:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.424589:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.424595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.424602:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.424604:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.424605:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.424606:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.424738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.424767:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.424770:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.424772:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.424772:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.424774:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.424776:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.424777:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.424778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677447.424779:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.424779:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.424780:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677447.424783:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:5.0:1619677447.424784:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:6.0:1619677447.424786:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:5.0:1619677447.424786:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.424787:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.424787:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.424788:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.424789:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.424789:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677447.424790:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.424791:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677447.424792:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677447.424792:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.424792:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.424793:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677447.424798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.424802:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.424804:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.424805:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.424805:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677447.424805:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:22.0:1619677447.424806:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.424807:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.424808:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.424809:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.424809:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.424811:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.424814:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.424922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.424929:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.424931:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.424932:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.424933:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.424935:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.424937:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:24.0:1619677447.424938:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.0:1619677447.424940:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:9.2:1619677447.424940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000400:00000200:20.0:1619677447.424942:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.424943:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.424944:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.424946:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.424947:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.424947:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000400:00000200:20.0:1619677447.424948:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.424950:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:20.0:1619677447.424950:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.424950:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677447.424951:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.424951:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.424952:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.424953:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:20.0:1619677447.424954:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.424956:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.424958:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.424959:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.424960:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.424962:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.424962:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.424963:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.424964:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.424968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.424975:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.424977:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.424979:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.424979:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.424981:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.424983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.424987:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.424989:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.424991:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.424992:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.424992:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.424993:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.424994:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677447.425028:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.425031:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.425032:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.425033:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.425036:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.425037:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.425039:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.425040:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.425041:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.425042:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.425045:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.425047:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.425050:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.425050:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.425051:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.425053:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.425053:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.425055:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.425056:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.425057:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677447.425059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.425067:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.0:1619677447.425069:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:2.0:1619677447.425069:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.425070:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.425071:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.425071:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.425072:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.425073:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:2.0:1619677447.425073:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.425076:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.425177:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677447.425180:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677447.425180:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677447.425182:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.425184:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.425186:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677447.425187:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728416] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.425189:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.425190:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.425191:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677447.425195:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.425198:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677447.425201:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.425202:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:18.0:1619677447.425203:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:18.0:1619677447.425206:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:2.0:1619677447.425206:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.425207:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677447.425208:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677447.425208:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:2.0:1619677447.425208:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.425209:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.425211:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.425214:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.425215:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.425216:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96728414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.425218:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.425219:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677447.425220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677447.425221:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.425227:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.425228:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.425230:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.425230:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.425231:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.425232:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.425233:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.425233:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.425234:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000800:00000200:2.0:1619677447.425236:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (24)--
00000800:00000200:18.0:1619677447.425237:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (24)--
00000800:00000200:2.0:1619677447.425237:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.425238:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.425239:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.425240:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.425241:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.425242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.425250:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.425252:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.425252:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:18.0:1619677447.425253:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.425253:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.425253:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.425254:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.425255:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.425255:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677447.425257:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677447.425260:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.425262:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.425263:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.425265:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.425265:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.425268:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.425280:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.425282:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.425283:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.425284:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.425285:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.425286:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:9.2:1619677447.425470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.425477:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.425480:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.425481:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.425481:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.425483:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.425485:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677447.425512:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.425514:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.425515:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.425516:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.425519:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.425520:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.425712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.425727:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.425730:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.425731:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.425734:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.425736:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.425738:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677447.425739:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.425740:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.425741:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.425743:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677447.425747:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.425749:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.425752:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.425753:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677447.425754:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677447.425756:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.425756:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.425757:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.425758:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.425759:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.0:1619677447.425761:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.425763:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.425764:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.425766:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.425768:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677447.425768:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.425769:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677447.425770:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:12.0:1619677447.425771:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:18.0:1619677447.425772:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:12.0:1619677447.425772:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.425773:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:12.0:1619677447.425773:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.425774:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677447.425774:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.425776:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.425780:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.425782:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.425783:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.425784:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.425785:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.425786:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677447.425876:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.425877:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:13.0:1619677447.425879:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.425880:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.425880:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.425881:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:13.0:1619677447.425882:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.425883:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.425884:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677447.425884:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.425887:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.425888:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.425889:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.425891:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.425892:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.425893:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.425895:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.425897:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.425899:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.425900:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.425901:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.425903:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.425903:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.425904:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.425905:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.425906:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677447.425916:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677447.425917:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.425918:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.425919:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677447.425919:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:18.0:1619677447.425920:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:6.0:1619677447.425920:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.425921:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.425922:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.425925:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.425980:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.425983:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.425983:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.425987:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.425990:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.425991:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.425992:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.425994:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.425996:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.425997:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:12.0:1619677447.425998:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.426000:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.426001:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.426002:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.426004:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.426005:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:22.0:1619677447.426009:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.426011:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.426014:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.426015:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.426016:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.426018:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.426018:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.426020:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.426021:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.426021:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.426031:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.426033:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.426034:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.426035:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.426061:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.426064:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.426065:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.426067:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.426069:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.426071:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.426072:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.426073:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.426074:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.426075:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.426079:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.426081:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.426083:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.426084:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.426085:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.426087:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.426087:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.426088:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.426088:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.426101:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.426103:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.426104:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.426106:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.426106:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.426108:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.426291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.426340:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.426343:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.426343:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.426344:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.426346:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.426349:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.426515:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.426518:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.426520:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.426521:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.426523:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.426524:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.426525:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.426527:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.426528:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.426533:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.426536:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:20.0:1619677447.426537:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.426537:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.426538:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.426539:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.426540:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.426542:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.426542:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.426543:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.426544:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.426544:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.426546:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.426546:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.426547:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.426548:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.426559:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.426562:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.426563:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.426565:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.426566:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.426568:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.426580:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.426582:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.426583:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.426585:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.426588:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.426589:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.426591:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.426592:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.426593:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.426594:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.426597:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.426599:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.426601:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.426602:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.426602:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.426604:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.426605:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.426605:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.426606:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.426618:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.426620:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.426621:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.426623:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.426623:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.426625:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677447.426711:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.426714:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.426715:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.426717:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.426719:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.426720:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677447.426722:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.426723:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.426724:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.426725:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677447.426729:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.426731:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.426735:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.426736:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:18.0:1619677447.426736:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677447.426738:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.426739:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.426739:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.426740:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677447.426750:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.426753:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.426754:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.426756:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.426757:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.426758:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.426762:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.426764:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.426765:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.426766:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.426768:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.426770:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677447.426771:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.426773:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.426774:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.426774:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.426775:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.426776:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.426822:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.426825:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.426826:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.426827:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.426829:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.426830:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.426836:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.426838:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.426840:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.426841:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.426842:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.426844:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.426844:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.426845:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:2.0:1619677447.426845:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:13.0:1619677447.426846:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.426846:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.426847:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677447.426847:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.426848:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.426849:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.426850:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.426853:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.426855:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.426858:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.426859:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.426860:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.426861:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.426862:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.426862:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.426863:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677447.426875:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.426878:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.426879:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.426880:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.426882:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.426883:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.426895:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.426897:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.426898:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.426899:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.426900:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.426901:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.426996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.427003:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.427006:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.427007:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.427009:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.427010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:8.0:1619677447.427011:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.427013:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.427014:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.427015:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.427016:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.427018:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.427021:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.427023:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.427026:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.427027:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.427027:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.427029:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.427030:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.427030:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.427031:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.427068:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.427071:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.427071:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.427073:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.427075:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.427077:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.427078:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.427079:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.427080:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.427081:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.427084:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.427086:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.427089:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.427090:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.427091:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.427092:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.427093:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.427093:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.427094:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.427122:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.427125:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.427126:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.427128:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.427129:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.427130:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.427406:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.427409:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.427410:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.427412:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.427412:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:13.0:1619677447.427414:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.427414:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.427415:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.427417:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.427417:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.427420:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.427421:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677447.427423:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.427424:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.427425:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.427426:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677447.427429:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.427431:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677447.427434:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.427435:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677447.427435:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677447.427437:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677447.427438:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.427439:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.427439:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:4.2:1619677447.427480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.427520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.427525:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.427528:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.427529:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.427530:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.427531:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:22.0:1619677447.427532:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.427534:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.427536:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.427537:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.427537:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677447.427539:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:20.0:1619677447.427539:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.427541:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.427541:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:22.0:1619677447.427542:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000400:00000200:20.0:1619677447.427542:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.427542:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.427543:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.427543:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:20.0:1619677447.427544:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.427545:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.427546:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677447.427549:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.427551:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677447.427553:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.427554:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677447.427555:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.427556:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.427557:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.427558:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.427559:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.427560:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:9.2:1619677447.427563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.427571:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:5.0:1619677447.427571:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:18.0:1619677447.427574:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:5.0:1619677447.427574:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:18.0:1619677447.427575:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.427575:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677447.427575:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677447.427576:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677447.427577:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.427580:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.427586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.427594:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.427597:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.427598:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.427600:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.427603:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.427604:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.427606:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.427607:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.427608:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.427609:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.427613:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.427614:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.427617:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.427618:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.427619:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.427620:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.427621:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.427621:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.427621:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677447.427629:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.427632:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.427633:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.427635:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.427637:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.427640:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.427641:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.427643:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.427643:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.427645:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677447.427648:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.427650:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.427653:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.427654:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.427654:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.427656:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.427656:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.427657:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.427658:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677447.427675:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:5.0:1619677447.427678:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:20.0:1619677447.427680:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:5.0:1619677447.427681:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.427684:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.427686:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:5.0:1619677447.427687:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.427689:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.427690:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.427692:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:5.0:1619677447.427698:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.427700:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:5.0:1619677447.427704:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.427705:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:5.0:1619677447.427705:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677447.427707:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677447.427708:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:5.0:1619677447.427708:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:5.0:1619677447.427709:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:13.0:1619677447.427717:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.427718:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.427719:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.427720:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:13.0:1619677447.427721:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.427721:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.427722:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.427723:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.427723:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.427724:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.427725:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.427726:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.427911:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.427914:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.427914:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.427916:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.427919:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.427920:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.427921:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.427923:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.427924:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.427925:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.427928:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.427930:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.427932:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.427933:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.427934:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.427935:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.427936:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.427937:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.427937:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.427963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.428006:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.428009:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.428010:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.428010:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.428012:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.428015:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.428030:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.428033:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677447.428033:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000400:00000200:6.0:1619677447.428035:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.428037:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.428039:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:6.0:1619677447.428040:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.428042:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.428043:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.428044:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:6.0:1619677447.428048:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.428050:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677447.428053:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.428054:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677447.428054:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.428056:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677447.428057:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677447.428057:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677447.428058:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677447.428062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.428070:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.428072:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.428073:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.428073:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.428075:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.428078:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.428079:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:9.2:1619677447.428079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.428081:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.428082:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.428084:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.428085:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.428087:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:2.0:1619677447.428087:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.428090:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.428091:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.428091:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.428092:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.428092:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.428165:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.428168:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.428169:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.428170:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.428173:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.428175:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.428176:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96728371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.428177:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.428178:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.428179:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.428182:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.428184:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.428186:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.428187:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.428188:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.428189:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.428190:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.428191:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.428191:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:22.0:1619677447.428203:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.428206:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.428207:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.428207:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.428209:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.428211:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.428309:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.428312:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.428314:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.428315:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.428317:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.428318:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677447.428319:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.428321:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.428322:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.428323:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677447.428326:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.428328:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.428331:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.428332:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.428332:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.428334:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.428334:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.428335:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.428336:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677447.428347:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.428349:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.428350:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.428352:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.428353:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.428355:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.428355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.428395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.428403:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.428406:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.428407:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.428408:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.428409:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.428413:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.428423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.428431:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.428434:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.428435:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.428436:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.428439:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.428441:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.428442:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.428443:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.428444:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.428445:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.428448:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.428450:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.428452:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.428453:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.428454:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.428455:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.428456:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.428456:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.428457:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677447.428514:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.428516:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.428517:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.428519:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.428521:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.428523:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.428524:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.428525:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.428526:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.428527:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.428531:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.428533:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.428535:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.428536:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.428536:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.428538:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.428538:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.428539:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.428539:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677447.428552:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.428554:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.428555:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.428555:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.428557:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.428559:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.428625:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.428628:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.428629:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.428632:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.428635:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.428636:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.428638:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96728364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.428640:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.428641:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.428642:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.428645:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.428647:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.428650:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.428651:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.428652:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677447.428654:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (24)--
00000800:00000200:2.0:1619677447.428655:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.428655:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.428656:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677447.428678:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.428681:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.428682:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.428683:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.428686:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.428687:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.428689:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.428690:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.428691:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.428692:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.428695:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.428696:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.428698:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.428699:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.428699:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677447.428700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.428701:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.428702:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.428703:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.428704:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.428705:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:22.0:1619677447.428715:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677447.428715:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:22.0:1619677447.428717:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677447.428717:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:22.0:1619677447.428718:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.428718:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.428718:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.428719:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:22.0:1619677447.428721:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.428724:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.428747:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:9.2:1619677447.428749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.428750:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.428751:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.428753:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.428754:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.428756:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.428757:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677447.428758:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:18.0:1619677447.428760:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.428761:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.428761:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677447.428761:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677447.428762:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.428762:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.428764:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.428765:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.428768:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.428768:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.428771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.428777:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.428780:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.428781:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.428781:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.428784:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.428787:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.428795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:12.0:1619677447.428798:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.428801:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.428801:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.428801:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000400:00000200:12.0:1619677447.428803:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.428804:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.428805:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:13.0:1619677447.428805:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.428805:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:12.0:1619677447.428806:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.428806:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.428807:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677447.428818:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.428820:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.428821:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.428822:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.428822:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.428823:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.428844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.428852:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.428854:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.428855:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.428856:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.428858:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.428862:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677447.428960:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.428963:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.428965:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.428965:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.428968:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.428969:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.428971:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.428972:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.428973:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.428974:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.428978:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.428979:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.428983:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.428983:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.428984:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.428986:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.428987:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.428987:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.428988:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.428998:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.429001:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.429002:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.429004:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.429005:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.429006:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.429147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.429154:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.429157:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.429158:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.429160:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.429163:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.429164:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.429166:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.429167:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.429168:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.429169:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.429172:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.429174:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.429177:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.429177:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.429178:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.429180:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.429180:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.429182:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.429183:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.429184:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.429189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.429196:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.429199:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.429199:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.429201:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677447.429215:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.429218:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.429219:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.429223:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.429226:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.429231:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.429233:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96728353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.429235:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.429236:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.429237:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.429240:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.429243:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:9.2:1619677447.429243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.429247:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.429248:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.429249:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.429251:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.429251:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.429252:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.429252:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.429253:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.429253:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.429254:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677447.429255:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:9.2:1619677447.429257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677447.429257:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.429261:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:18.0:1619677447.429265:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.429267:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.429269:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.429269:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.429271:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.429272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:18.0:1619677447.429274:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677447.429278:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:9.2:1619677447.429278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.429281:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:4.2:1619677447.429281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677447.429282:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.429282:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.429284:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.429285:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.429286:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:2.0:1619677447.429287:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.429288:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.429288:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.429289:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.429290:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.429329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677447.429335:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.429337:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.429338:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.429341:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:20.0:1619677447.429344:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.429346:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.429348:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.429348:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.429349:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.429351:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.429354:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.429356:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677447.429358:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.429359:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.429360:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.429361:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677447.429365:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.429367:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.429369:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.429370:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.429371:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.429373:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.429374:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.429375:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.429375:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677447.429386:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.429388:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.429389:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.429390:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.429392:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.429393:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.429411:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.429414:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.429414:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.429415:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.429416:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.429416:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:22.0:1619677447.429539:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.429541:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.429542:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.429542:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.429545:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.429547:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.429556:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.429559:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.429559:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.429560:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.429561:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.429561:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.429647:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.429649:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.429650:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.429651:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.429653:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.429656:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.429669:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.429671:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.429671:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.429673:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.429673:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.429674:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.429674:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.429675:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:22.0:1619677447.429676:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.429676:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.429679:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.429680:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.429682:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.429683:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.429684:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.429685:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.429688:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.429690:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.429693:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.429693:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.429694:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.429696:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.429696:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.429698:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.429699:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.429699:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.429702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.429709:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677447.429710:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:6.0:1619677447.429711:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.429711:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.429712:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677447.429713:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.429714:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.429714:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.429716:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.429719:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.429723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677447.429724:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.429727:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.429730:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:5.0:1619677447.429730:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.429730:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.429731:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.429733:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.429733:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.429734:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.429734:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000400:00000200:20.0:1619677447.429735:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.429735:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.429736:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:5.0:1619677447.429741:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677447.429742:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.429744:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.429745:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.429746:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677447.429747:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.429749:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.429750:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96728342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.429750:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.429752:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.429752:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:20.0:1619677447.429753:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.429754:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.429757:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.429758:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.429758:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677447.429758:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.429760:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.429760:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.429760:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)++
00000800:00000200:20.0:1619677447.429762:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.429762:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (24)++
00000800:00000200:20.0:1619677447.429763:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.429763:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (25)--
00000800:00000200:20.0:1619677447.429764:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.429764:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (24)--
00000800:00000200:5.0:1619677447.429765:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.429766:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.429766:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.429767:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.429768:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.429769:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (24)--
00000400:00000200:5.0:1619677447.429769:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.429770:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.429772:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:9.2:1619677447.429772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677447.429772:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.429773:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.429774:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.429775:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.429777:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.429777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.429778:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.429780:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.429782:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.429783:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.429784:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.429786:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.429788:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.429833:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.429836:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.429837:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.429837:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.429840:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.429842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677447.429843:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.429849:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.429852:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.429853:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.429855:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.429857:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.429859:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.429860:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.429861:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.429862:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.429863:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.429866:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.429868:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.429871:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.429871:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.429872:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.429874:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.429874:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.429875:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.429875:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.430033:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.430036:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.430037:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.430038:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.430040:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.430041:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.430211:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.430213:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.430214:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.430216:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.430217:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.430218:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.430298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.430335:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.430338:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.430339:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677447.430340:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:2.0:1619677447.430340:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.430342:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.430342:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:20.0:1619677447.430343:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.430343:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:2.0:1619677447.430343:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.430345:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.430345:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.430346:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.430347:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.430348:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.430349:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.430349:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.430349:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.430350:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.430351:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.430351:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.430351:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.430353:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.430354:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:5.0:1619677447.430354:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.430355:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.430356:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677447.430356:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.430357:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.430358:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.430360:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677447.430361:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.430361:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.430361:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.430363:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.430364:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.430364:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:20.0:1619677447.430365:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:2.0:1619677447.430365:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:5.0:1619677447.430366:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.430366:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.430366:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:20.0:1619677447.430367:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.430367:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.430368:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:20.0:1619677447.430369:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:5.0:1619677447.430369:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:20.0:1619677447.430370:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96728332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677447.430370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677447.430371:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.430372:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.430373:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:5.0:1619677447.430376:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.430376:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.430377:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.430377:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.430378:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677447.430378:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.430379:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677447.430380:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.430380:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.430380:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677447.430380:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:20.0:1619677447.430381:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.430381:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:2.0:1619677447.430381:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:20.0:1619677447.430382:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000400:00000200:5.0:1619677447.430382:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.430382:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677447.430383:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:18.0:1619677447.430383:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.430383:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:5.0:1619677447.430384:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.430384:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677447.430385:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:18.0:1619677447.430385:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.430386:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.430386:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.430387:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.430387:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.430388:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.430389:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:9.2:1619677447.430391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.430397:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.430400:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:5.0:1619677447.430400:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:8.0:1619677447.430401:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.430402:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.430402:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.430402:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.430403:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:24.0:1619677447.430404:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677447.430407:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.430532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.430539:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.430542:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.430543:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.430544:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.430547:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.430553:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.430554:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96728330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.430556:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.430557:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.430557:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:8.0:1619677447.430558:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.430560:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:8.0:1619677447.430561:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.430561:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.430563:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:8.0:1619677447.430563:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:5.0:1619677447.430563:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.430565:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.430566:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677447.430566:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.0:1619677447.430567:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.430568:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.430569:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.430570:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.430570:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.430686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.430732:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.430734:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.430735:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.430737:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.430738:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.430740:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677447.430766:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.430768:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.430769:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.430771:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.430773:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.430774:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.430849:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.430851:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.430852:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.430854:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.430855:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.430857:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.430868:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.430871:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.430872:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.430873:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.430873:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.430875:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.430916:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.430919:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.430919:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.430921:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.430923:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.430924:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.430956:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.430959:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.430960:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.430961:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.430964:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.430965:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.430967:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.430968:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.430969:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.430970:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.430973:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.430974:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.430977:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.430978:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.430979:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.430980:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.430981:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.430982:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.430982:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.430995:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.430997:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.430998:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.431000:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.431000:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.431002:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:24.0:1619677447.431043:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.431046:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.431047:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.431049:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.431052:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.431053:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.431055:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.431056:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.431057:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.431058:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.431061:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.431063:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.431065:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.431066:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.431067:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.431068:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.431069:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.431069:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.431070:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.431080:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.431082:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.431083:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.431085:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.431086:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.431087:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.431100:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.431103:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.431104:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.431104:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.431105:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.431106:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.431139:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.431142:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.431143:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.431145:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.431147:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.431149:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.431150:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.431151:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.431152:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.431153:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:20.0:1619677447.431154:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.431156:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.431157:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.431157:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.431159:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.431160:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:5.0:1619677447.431160:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.431162:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:6.0:1619677447.431162:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.431163:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.431164:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.431166:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.431166:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.431167:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.431167:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.431171:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.431173:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.431174:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.431175:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.431176:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.431176:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.431335:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.431338:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.431339:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.431339:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.431342:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.431345:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.431414:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.431416:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.431417:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.431418:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.431420:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.431422:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.431532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.431540:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.431543:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.431544:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.431546:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.431546:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.431549:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.431574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677447.431577:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:6.0:1619677447.431578:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677447.431579:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.431580:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:6.0:1619677447.431581:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.431582:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:18.0:1619677447.431582:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.431582:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.431584:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.431585:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.431586:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.431587:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.431588:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:18.0:1619677447.431589:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.431590:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.431591:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.431594:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.431596:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.431598:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.431598:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.431599:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.431601:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.431601:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.431602:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.431602:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.431607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.431614:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.431616:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.431616:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.431617:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.431618:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.431620:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.431651:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.431653:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.431654:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.431656:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.431659:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.431661:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.431662:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.431663:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.431664:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.431665:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.431669:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.431671:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.431673:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.431674:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.431675:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.431676:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.431677:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.431678:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.431678:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.431689:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.431692:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.431692:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.431694:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.431695:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.431696:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677447.431711:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.431713:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.431714:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.431716:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.431718:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.431719:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.431720:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.431721:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.431723:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.431724:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.431727:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.431729:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.431731:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.431732:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.431733:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.431735:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.431735:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.431736:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.431736:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.431748:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.431751:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.431752:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.431753:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.431754:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.431756:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677447.431828:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.431831:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.431832:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.431834:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.431836:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.431837:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.431838:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.431840:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.431841:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.431842:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.431845:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.431846:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907b781800] (21)++
00000800:00000200:2.0:1619677447.431847:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.431848:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.30@o2ib3
00000800:00000200:2.0:1619677447.431848:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.431849:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.431849:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677447.431850:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.30@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.431851:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.431852:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:18.0:1619677447.431853:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.431853:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.431853:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677447.431854:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907b781800] (22)--
00000400:00000200:18.0:1619677447.431855:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:18.0:1619677447.431856:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677447.431857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677447.431858:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.431858:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.431860:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:18.0:1619677447.431863:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.30@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.431865:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:2.0:1619677447.431865:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.431866:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.431867:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.431868:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f90d380] -> 10.0.10.54@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677447.431868:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.431869:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e49d37c00] (20)++
00000800:00000200:18.0:1619677447.431870:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e49d37c00] (21)++
00000800:00000200:2.0:1619677447.431870:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.431872:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e49d37c00] (22)--
00000800:00000200:18.0:1619677447.431872:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907b781800] (21)++
00000800:00000200:2.0:1619677447.431872:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.431873:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907b781800] (22)--
00000800:00000200:18.0:1619677447.431874:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907b781800] (21)--
00000800:00000200:9.2:1619677447.431875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.2:1619677447.431878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (21)++
00000800:00000200:24.0:1619677447.431882:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)++
00000800:00000200:2.0:1619677447.431882:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.431885:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e49d37c00] (23)--
00000800:00000200:2.0:1619677447.431885:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.431886:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.431886:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.431886:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.431886:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:8.0:1619677447.431887:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)--
00000800:00000200:2.0:1619677447.431887:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:24.0:1619677447.431888:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.431892:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)--
00000800:00000200:6.0:1619677447.431915:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc4600] (21)++
00000800:00000200:22.0:1619677447.431918:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (22)--
00000800:00000200:6.0:1619677447.431918:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.64@o2ib3
00000400:00000200:6.0:1619677447.431922:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.64@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.431924:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.431926:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:6.0:1619677447.431928:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.431930:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.431931:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.431932:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:6.0:1619677447.431936:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.431938:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:6.0:1619677447.431941:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.431942:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:6.0:1619677447.431943:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:6.0:1619677447.431945:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:6.0:1619677447.431945:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919874bc4600] (21)++
00000800:00000200:6.0:1619677447.431946:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919874bc4600] (22)--
00000800:00000200:6.0:1619677447.431947:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (21)--
00000800:00000200:9.2:1619677447.431950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:20.0:1619677447.431951:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.431954:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.431955:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.431956:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.431957:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:5.0:1619677447.431957:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.431959:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:24.0:1619677447.431961:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.431965:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.431966:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.431968:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.431969:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.0:1619677447.431971:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.0:1619677447.431972:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.431973:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.431974:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.431975:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.431975:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.432092:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc4600] (21)++
00000800:00000200:22.0:1619677447.432095:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.64@o2ib3
00000800:00000200:6.0:1619677447.432096:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (22)--
00000400:00000200:22.0:1619677447.432098:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.64@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.432101:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.432102:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.432103:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.432105:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.432106:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.432106:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.432110:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.432112:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.432115:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.432115:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.432116:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.432118:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.432118:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919874bc4600] (21)++
00000800:00000200:22.0:1619677447.432119:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919874bc4600] (22)--
00000800:00000200:22.0:1619677447.432120:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (21)--
00000800:00000200:6.0:1619677447.432155:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.432158:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.432159:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.432159:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.432162:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.432164:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.432180:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.432183:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.432183:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.432185:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.432188:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.432189:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.432190:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.432192:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.432193:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.432194:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.432197:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.432199:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.432201:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.432202:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.432202:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.432204:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.432204:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.432205:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.432205:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.432269:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.432272:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677447.432274:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677447.432275:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.432276:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.432278:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.432280:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.432281:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.432282:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.432283:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.432287:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.432289:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.432291:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.432292:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.432293:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.432301:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.432301:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.432302:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.432303:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:13.0:1619677447.432315:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.432318:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.432319:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.432320:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677447.432322:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.432323:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.432341:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.432343:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.432347:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.432347:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.432350:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.432352:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.432354:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.432356:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.432357:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.432359:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.432364:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.432367:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.432370:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.432371:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.432372:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.432374:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.432374:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.432375:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.432376:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.432386:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.432389:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.432390:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.432390:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:12.0:1619677447.432391:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.432392:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:13.0:1619677447.432393:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677447.432393:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.432394:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.432395:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.432395:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.432399:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.432492:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.432494:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.432495:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.432497:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.432498:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.432500:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.432513:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.432517:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.432518:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.432518:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.432519:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.432520:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.432526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.432551:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.432554:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.432555:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.432555:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.432557:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.432559:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.432571:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.432573:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.432574:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.432575:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.432576:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.432576:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677447.432583:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.432585:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.432588:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.432588:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.432592:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.432594:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:5.0:1619677447.432595:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.432597:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.432598:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.432599:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:5.0:1619677447.432604:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.432606:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 440 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:24.0:1619677447.432609:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677447.432609:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.432610:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:5.0:1619677447.432611:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:24.0:1619677447.432612:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.432613:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677447.432613:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:5.0:1619677447.432613:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.432614:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.432614:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:24.0:1619677447.432615:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.432617:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.432619:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.432620:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96728297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.432621:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.432622:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.432623:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.432626:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.432628:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.432631:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.432632:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.432632:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.432634:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.432635:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.432635:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677447.432635:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677447.432636:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.432637:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.432638:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.432639:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.432640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.432641:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.432642:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.432648:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.432651:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:12.0:1619677447.432652:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.432652:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.432653:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.432655:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.432655:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.432657:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.432657:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.432658:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.432659:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677447.432659:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.432945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.432948:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.432950:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.432951:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.432953:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.432954:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.432956:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.432957:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.432959:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.432962:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.432963:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.432965:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.432967:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.432969:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.432971:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.432972:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.432974:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.432978:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.432980:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.432982:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.432983:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.432984:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.432985:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.432986:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.432988:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.432989:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.432990:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677447.432997:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.432999:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.433000:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.433000:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:12.0:1619677447.433002:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.433002:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.433003:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677447.433004:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.433005:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.433005:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677447.433141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.433148:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.433151:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.433152:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.433154:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.433157:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.433159:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.433160:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.433161:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.433162:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.433163:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.433166:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.433168:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.433171:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.433172:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.433172:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.433174:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.433174:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.433175:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.433176:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.433332:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.433335:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.433335:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.433337:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.433339:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.433341:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.433377:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.433380:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.433381:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.433383:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.433385:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.433391:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.433392:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.433393:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.433395:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.433395:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.433399:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.433400:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:9.2:1619677447.433402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.433403:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.433404:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.433405:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.433407:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.433408:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.433409:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.433409:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677447.433411:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.433414:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.433415:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.433415:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.433417:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.433420:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:6.0:1619677447.433421:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.433423:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.433424:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.433425:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.433427:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.433429:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.433450:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.433453:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.433453:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.433455:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.433457:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.433459:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.433460:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.433461:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.433462:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.433463:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.433467:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.433469:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.433471:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.433472:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.433473:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.433475:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.433475:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.433476:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.433477:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.433478:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.433488:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.433490:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.433490:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.433491:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.433495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.433498:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.433501:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.433503:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.433503:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.433506:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.433508:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.433509:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.433511:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.433512:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.433513:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.433517:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.433520:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.433523:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.433524:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.433524:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.433526:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.433527:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.433527:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.433528:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.433539:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.433541:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.433542:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.433544:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.433545:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:13.0:1619677447.433545:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.433547:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.433548:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.433549:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.433549:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.433551:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.433554:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.433718:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.433720:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.433721:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.433723:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.433724:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.433726:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.433734:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.433737:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.433738:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.433738:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.433739:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.433740:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.433744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.433753:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.433756:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.433757:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.433757:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.433759:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.433763:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.433812:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.433815:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.433817:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.433818:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:4.2:1619677447.433819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:20.0:1619677447.433821:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.433823:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.433824:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.433826:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.433826:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.433828:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.433831:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.433834:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.433836:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.433837:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.433838:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.433839:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.433840:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.433841:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.433842:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677447.433843:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.433854:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.433856:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677447.433856:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.433857:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000400:00000200:5.0:1619677447.433857:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.433858:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.433858:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.433859:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:9.2:1619677447.433859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.433860:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.433861:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:4.2:1619677447.433862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.433867:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.433870:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.433871:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.433871:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.433873:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.433876:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.434033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.434040:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.434043:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.434044:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.434045:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.434048:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.434050:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.434051:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.434052:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.434053:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.434054:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.434056:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:24.0:1619677447.434057:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.434059:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.434059:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.434060:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.434062:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.434063:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.434064:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.434064:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:18.0:1619677447.434065:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.434066:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.434066:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:18.0:1619677447.434066:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.434067:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.434068:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.434069:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000400:00000200:18.0:1619677447.434069:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.434070:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:18.0:1619677447.434070:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.434071:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.434074:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.434075:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:4.2:1619677447.434075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677447.434077:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.434078:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.434079:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.434080:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.434080:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.434081:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.434082:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.0:1619677447.434082:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.434084:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.434085:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677447.434086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.434086:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677447.434093:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.434095:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.434096:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.434097:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.434098:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.434100:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.434103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.434110:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.434112:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.434113:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.434114:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.434114:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.434114:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677447.434200:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.434202:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.434203:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.434204:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.434206:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.434209:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.434289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.434308:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.434311:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.434312:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.434313:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.434315:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.434318:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.434334:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.434337:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.434338:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.434338:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.434340:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.434342:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677447.434495:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.434497:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.434500:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.434501:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.434502:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.434503:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677447.434505:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.434506:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.434507:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.434508:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677447.434515:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.434517:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:5.0:1619677447.434518:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.434519:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.434520:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.434521:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677447.434521:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:5.0:1619677447.434521:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.434523:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677447.434524:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:5.0:1619677447.434524:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.434525:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.434525:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677447.434526:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.434526:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.434527:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.434528:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677447.434529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:5.0:1619677447.434529:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.434530:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.434533:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.434535:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677447.434536:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:5.0:1619677447.434537:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.434538:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.434538:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677447.434539:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.434540:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.434540:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677447.434540:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.434540:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.434541:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:24.0:1619677447.434542:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.434542:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:24.0:1619677447.434546:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677447.434553:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.434556:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.434557:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.434557:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.434557:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:22.0:1619677447.434559:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.434560:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:22.0:1619677447.434561:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677447.434561:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.434562:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.434565:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.434566:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.434567:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.434568:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.434569:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.434570:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677447.434573:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:2.0:1619677447.434573:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.434574:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.434576:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.434576:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.434576:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.434577:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.434577:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.434577:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677447.434578:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.434578:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.434579:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.434580:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.434581:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.434581:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.434585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.434592:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.434594:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.434595:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.434596:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.434597:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.434599:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.434619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.434627:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.434629:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.434630:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.434632:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.434635:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.434636:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.434637:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.434639:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.434640:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.434641:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.434644:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.434645:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.434648:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.434649:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.434650:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.434651:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.434652:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.434652:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.434653:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677447.434707:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.434709:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.434710:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.434713:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.434715:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.434717:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.434719:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.434721:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.434722:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.434723:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677447.434727:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.434729:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.434732:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.434732:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.434733:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.434735:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.434735:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.434736:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.434737:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:4.2:1619677447.434743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.434791:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.434794:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.434795:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.434796:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.434798:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.434799:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:9.2:1619677447.434964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.434972:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.434975:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.434976:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.434976:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.434978:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.434982:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677447.435015:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.435018:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.435020:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.435020:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.435022:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.435024:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.435025:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.435026:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.435027:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.435028:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.435032:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.435033:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.435036:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.435037:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.435037:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.435038:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.435039:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.435040:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.435040:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677447.435052:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.435054:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.435055:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.435057:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.435058:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.435059:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.0:1619677447.435077:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.435080:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.435081:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.435082:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677447.435083:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.435084:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:9.2:1619677447.435099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.435105:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.435108:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.435109:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.435109:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.435112:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.435115:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.435117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677447.435123:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.435125:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.435126:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.435126:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.435127:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.435128:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.435187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.435194:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.435196:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.435197:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.435199:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.435201:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.435203:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.435204:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.435205:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.435206:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.435207:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.435211:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.435212:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.435215:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.435216:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.435217:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.435218:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.435219:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.435219:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.435220:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.0:1619677447.435241:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.435243:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.435246:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.435246:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.435248:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.435250:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.435251:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.435253:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.435254:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.435255:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.435260:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.435262:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.435265:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.435266:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.435266:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.435268:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.435269:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.435269:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.435270:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.435275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.435282:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.435284:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.435285:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.435286:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:18.0:1619677447.435286:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.435287:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.435289:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.435290:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.435290:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677447.435290:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:22.0:1619677447.435293:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.435303:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.435304:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.435306:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.435307:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.435307:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.435308:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.435309:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.435316:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.435319:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.435320:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.435322:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.435325:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.435327:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.435328:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.435330:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.435331:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.435332:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.435335:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.435337:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.435340:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.435341:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.435341:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.435343:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.435343:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.435344:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.435346:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.435346:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677447.435357:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:24.0:1619677447.435359:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:20.0:1619677447.435359:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.435360:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.435361:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.435362:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:20.0:1619677447.435362:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.435363:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.435364:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677447.435364:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677447.435399:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.435403:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.435403:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.435405:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.435407:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.435408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:18.0:1619677447.435409:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.435410:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.435412:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.435413:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.435414:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.435417:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.435419:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.435422:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.435422:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.435423:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.435425:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.435425:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.435427:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.435428:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.435428:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.435439:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.435441:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:5.0:1619677447.435442:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677447.435442:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.435443:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.435444:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.435445:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.435447:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.435447:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.435449:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:6.0:1619677447.435454:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.435457:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.435458:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.435458:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.435460:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.435463:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.435515:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.435517:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.435518:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.435520:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.435522:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.435524:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.435525:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.435526:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.435527:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.435528:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.435533:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.435535:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.435538:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.435539:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.435540:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:12.0:1619677447.435541:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:22.0:1619677447.435542:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.435542:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.435543:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.435543:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.435544:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677447.435547:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.435549:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677447.435549:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.435550:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:12.0:1619677447.435552:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96728257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677447.435553:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677447.435554:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677447.435555:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:12.0:1619677447.435559:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677447.435559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:12.0:1619677447.435561:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:12.0:1619677447.435564:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.435565:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:12.0:1619677447.435566:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:18.0:1619677447.435567:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:12.0:1619677447.435567:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:12.0:1619677447.435568:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677447.435569:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677447.435569:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:18.0:1619677447.435570:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:6.0:1619677447.435570:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:18.0:1619677447.435571:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.435571:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.435573:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.435573:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.435574:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.435574:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.435576:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:6.0:1619677447.435576:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.435579:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.435622:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.435625:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.435626:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.435628:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.435629:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.435631:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.435711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.435718:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.435721:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.435722:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.435724:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.435727:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.435729:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.435730:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.435731:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.435732:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.435733:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.435737:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.435738:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.435741:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.435742:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.435743:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.435744:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.435745:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.435745:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.435746:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.435792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.435834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.435838:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.435840:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.435841:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.435842:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.435844:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.435847:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677447.435871:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.435873:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.435875:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.435876:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.435878:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.435879:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677447.435881:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.435882:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.435883:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.435884:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677447.435888:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.435889:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.435892:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.435892:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.435893:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.435895:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.435895:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.435896:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.435898:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.435898:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.435907:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.435908:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.435908:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.435909:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.435909:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.435910:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.435910:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.435911:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.435912:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.435913:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677447.435987:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.435990:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.435991:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.435994:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.435997:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.435998:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.436000:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.436002:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:2.0:1619677447.436002:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.436003:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.436004:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.436005:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.436008:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.436008:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.436009:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.436011:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.436011:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.436012:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.436014:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96728250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.436015:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677447.436016:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.436016:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:5.0:1619677447.436017:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.436017:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677447.436018:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:2.0:1619677447.436019:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.436019:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.436020:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.436020:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:5.0:1619677447.436022:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.436025:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.436027:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.436028:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.436028:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.436030:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:12.0:1619677447.436031:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.436031:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.436031:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.436032:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:12.0:1619677447.436033:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.436034:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.436036:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.436037:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.436039:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.436039:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.436042:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.436043:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.436044:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:13.0:1619677447.436044:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.436046:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:12.0:1619677447.436046:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:20.0:1619677447.436047:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.436047:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:20.0:1619677447.436048:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.436050:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.436051:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:12.0:1619677447.436059:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.436062:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.436062:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.436063:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.436064:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.436065:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.436087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.436093:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.436096:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.436097:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.436098:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.436100:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.436103:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677447.436106:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.436108:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:9.2:1619677447.436108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000400:00000200:13.0:1619677447.436109:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.436110:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.436112:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.436113:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.436114:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.436116:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.436117:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.436117:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.436118:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.436119:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677447.436220:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.436223:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.436224:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.436224:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.436226:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.436229:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677447.436277:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.436280:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.436281:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.436283:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.436286:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.436287:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.436289:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.436290:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.436291:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.436292:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.436300:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.436302:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.436305:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.436306:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.436306:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.436308:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.436308:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.436309:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.436310:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.436311:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.436322:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.436324:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.436325:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.436326:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:22.0:1619677447.436333:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.436336:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.436337:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.436337:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.436339:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.436341:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.436370:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.436372:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.436373:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.436375:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.436377:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.436379:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.436380:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728245] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.436381:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.436382:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.436383:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.436386:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.436388:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.436390:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.436391:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.436392:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.436394:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.436394:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.436395:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.436395:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.436409:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.436411:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.436412:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.436413:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.436415:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.436415:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:4.2:1619677447.436424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.436471:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (23)++
00000800:00000200:22.0:1619677447.436474:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000800:00000200:6.0:1619677447.436474:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (24)++
00000400:00000200:22.0:1619677447.436475:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.436475:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000400:00000200:6.0:1619677447.436476:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.436477:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.436478:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.436481:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.436481:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.436483:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.436486:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.436487:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.436487:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.436488:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.436488:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677447.436491:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.436494:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.436495:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.436497:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.436497:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.436500:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.436509:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.436512:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.436513:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.436514:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.436515:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.436515:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.436586:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.436589:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.436589:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.436591:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.436592:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.436594:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.436723:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.436725:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.436727:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.436728:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.436730:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.436735:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677447.436736:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.436737:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.436738:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:5.0:1619677447.436738:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.436739:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:18.0:1619677447.436741:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.436741:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:5.0:1619677447.436743:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.436744:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.436746:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:18.0:1619677447.436747:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.436748:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.436749:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.436751:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.436752:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.436752:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.436753:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.436753:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677447.436753:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.436755:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.436756:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.436756:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:18.0:1619677447.436757:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.436757:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.436759:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.436763:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.436764:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.436764:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.436766:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.436767:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.436767:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.436768:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.436780:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.436782:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.436783:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.436784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:12.0:1619677447.436785:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.436787:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.436788:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.436793:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.436796:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.436797:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.436798:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.436801:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.436802:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.436804:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.436805:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.436806:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.436807:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.436810:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.436811:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.436814:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.436815:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.436816:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.436817:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.436818:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.436819:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.436820:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.436822:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.436826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.436834:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.436836:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.436836:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.436837:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677447.436885:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.436888:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.436889:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.436890:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.436892:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.436893:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.436933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.436959:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.436961:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.436962:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.436963:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.436965:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.436966:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.436988:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.436991:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.436991:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.436992:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.436994:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.436997:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.437008:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.437011:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677447.437013:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677447.437014:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.437016:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.437017:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.437018:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.437020:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.437021:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.437022:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.437025:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.437027:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.437030:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.437031:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.437031:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.437033:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.437034:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.437035:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.437035:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:12.0:1619677447.437045:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.437048:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.437049:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.437050:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677447.437052:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.437053:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.437062:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677447.437062:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677447.437066:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:13.0:1619677447.437066:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677447.437066:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:13.0:1619677447.437068:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.437068:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:22.0:1619677447.437069:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.437069:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.437069:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:22.0:1619677447.437071:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.437073:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.437075:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.437076:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.437077:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.437078:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.437081:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.437083:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.437086:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.437087:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.437088:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.437090:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.437091:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.437091:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.437092:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.437103:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.437106:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.437108:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.437109:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.437111:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.437112:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.437113:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.437115:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.437116:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.437117:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.437120:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.437122:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.437124:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.437125:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.437125:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.437126:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.437127:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.437128:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.437128:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.437140:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.437146:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677447.437149:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.437150:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.437152:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.437154:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677447.437328:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.437331:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.437332:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.437333:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.437335:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.437336:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.437336:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.437337:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.437338:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.437338:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.437339:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:12.0:1619677447.437444:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.437447:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.437448:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.437449:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.437451:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.437452:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.437470:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.437472:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.437474:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.437475:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.437477:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.437478:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677447.437479:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.437481:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.437481:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.437483:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677447.437486:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.437487:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.437489:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.437490:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.437491:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.437492:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.437493:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.437493:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.437494:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677447.437504:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.0:1619677447.437507:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677447.437507:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.437508:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:5.0:1619677447.437509:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.437510:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.437510:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.437511:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.437512:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.437513:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:2.0:1619677447.437514:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.437515:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677447.437516:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.437518:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.437519:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.437520:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677447.437523:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.437525:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677447.437528:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.437529:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677447.437529:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677447.437531:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677447.437531:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.437532:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.437533:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.437534:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.437545:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:6.0:1619677447.437545:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:18.0:1619677447.437547:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.437548:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677447.437548:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.437549:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677447.437549:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:6.0:1619677447.437549:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.437552:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.437555:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.437709:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:4.2:1619677447.437710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.437712:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.437713:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.437715:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.437717:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.437719:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.437720:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.437722:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.437723:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.437724:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.437728:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.437731:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.437734:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.437735:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.437736:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.437738:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.437738:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.437740:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:24.0:1619677447.437741:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:22.0:1619677447.437741:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.437741:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677447.437743:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.437744:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.437746:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.437749:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.437751:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:6.0:1619677447.437751:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:24.0:1619677447.437752:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96728220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.437753:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.437753:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:24.0:1619677447.437754:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.437754:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:24.0:1619677447.437755:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:22.0:1619677447.437755:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:24.0:1619677447.437758:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.437760:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.437763:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.437764:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.437764:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.437766:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.437766:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.437767:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.437768:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677447.437779:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.437781:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.437785:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:20.0:1619677447.437787:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.437789:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.437792:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.437829:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.437831:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.437832:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.437834:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.437835:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.437837:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.438129:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.438132:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.438134:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.438135:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.438136:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.438138:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.438139:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.438140:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.438141:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.438142:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.438145:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.438147:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.438149:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.438150:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.438151:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.438153:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.438153:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.438154:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.438155:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.438165:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.438168:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.438169:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.438170:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.438172:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.438173:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.438213:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.438216:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677447.438218:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.438219:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.438221:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.438222:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.438223:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.438225:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.438226:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.438227:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.438229:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.438231:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.438233:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.438234:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.438234:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.438236:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.438237:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.438238:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677447.438239:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:5.0:1619677447.438240:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.438249:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.438250:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.438251:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677447.438251:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000400:00000200:20.0:1619677447.438252:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.438252:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.438253:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.438253:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.438254:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677447.438255:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:9.2:1619677447.438459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.438467:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.438470:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.438471:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.438471:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.438473:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.438475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.438477:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.438514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.438520:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.438523:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.438524:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.438525:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.438527:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.438530:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.438539:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.438541:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.438542:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.438542:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.438544:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.438544:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677447.438767:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.438770:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.438771:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.438773:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.438775:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.438776:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.438778:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.438779:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.438779:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:18.0:1619677447.438780:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.438781:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.438782:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.438783:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:18.0:1619677447.438786:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:8.0:1619677447.438786:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.438788:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:8.0:1619677447.438790:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.438791:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.438791:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:8.0:1619677447.438791:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:18.0:1619677447.438792:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:8.0:1619677447.438793:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96728213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.438794:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.438794:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:8.0:1619677447.438794:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.438795:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:8.0:1619677447.438795:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.438796:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:8.0:1619677447.438796:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.438800:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.438801:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.438805:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.438805:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.438806:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:6.0:1619677447.438807:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677447.438808:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.438808:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.438809:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:6.0:1619677447.438809:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.438810:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:8.0:1619677447.438810:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:6.0:1619677447.438810:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.438813:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.438815:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.438822:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.438825:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.438826:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.438828:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.438828:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.438830:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.438875:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.438878:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.438880:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.438881:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.438882:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.438884:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.438885:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.438886:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.438888:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.438888:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.438892:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.438893:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.438897:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.438897:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.438898:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.438900:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.438900:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.438902:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.438903:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.438904:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677447.438913:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.438914:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677447.438916:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677447.438916:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.438917:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:13.0:1619677447.438918:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.438918:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:13.0:1619677447.438919:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.438921:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.438922:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.438930:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.438932:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.438933:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.438935:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.438936:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.438938:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.438938:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.438939:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.438939:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.438940:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.438940:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677447.438941:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.438942:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.439021:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.439024:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.439024:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.439026:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.439029:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.439031:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.439032:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.439034:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.439035:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.439036:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.439039:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.439041:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.439044:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.439045:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.439045:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.439047:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.439048:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.439048:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.439049:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.439168:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.439171:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.439173:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.439173:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.439176:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.439177:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.439179:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.439180:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.439181:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.439182:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.439185:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.439187:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.439189:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.439190:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.439191:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.439192:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.439193:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.439193:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.439194:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677447.439205:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.439207:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.439208:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.439210:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.439211:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.439212:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.0:1619677447.439235:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.439238:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.439239:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.439240:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677447.439240:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.439241:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:6.0:1619677447.439274:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.439276:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.439277:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.439277:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.439279:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.439282:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.439335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.439342:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.439345:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.439346:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.439348:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.439351:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.439353:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.439354:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.439356:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.439356:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.439358:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.439361:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.439363:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.439366:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.439366:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.439367:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.439369:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.439369:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.439370:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.439370:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.439408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.439463:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.439465:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.439466:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.439468:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.439468:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.439470:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.0:1619677447.439471:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677447.439472:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.439473:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.439474:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.439476:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.439479:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677447.439484:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.439487:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.439488:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.439488:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.439489:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.439490:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:6.0:1619677447.439490:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.439493:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.439493:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.439494:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.439495:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.439495:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.439626:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.439629:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.439629:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.439632:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.439634:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.439636:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.439637:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.439639:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.439640:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.439641:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.439644:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.439646:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.439648:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.439649:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.439650:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.439651:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.439651:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.439652:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.439653:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:4.2:1619677447.439851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.439865:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.439868:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.439870:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.439870:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.439873:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.439874:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677447.439877:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.439880:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.439881:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.439882:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677447.439886:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.439888:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677447.439891:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.439892:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677447.439893:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677447.439895:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677447.439895:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.439896:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.439897:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.439901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.439908:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.439911:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.439912:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.439912:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.439914:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.439917:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.439925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.439927:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.439930:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.439931:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.439931:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677447.439932:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000400:00000200:22.0:1619677447.439934:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.439935:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.439936:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.439936:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677447.439937:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.439937:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.439938:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:13.0:1619677447.440056:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.440059:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.440060:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.440061:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.440063:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.440064:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.440094:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.440096:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.440097:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.440100:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.440102:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.440103:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.440105:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.440106:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.440108:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.440110:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.440113:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.440115:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.440118:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.440119:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.440120:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.440122:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.440122:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.440123:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.440123:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677447.440135:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.440137:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.440138:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.440140:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.440142:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.440143:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.440144:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.440146:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.440147:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.440148:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.440149:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.440151:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.440167:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.440169:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.440170:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.440171:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.440172:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.440172:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.440254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.440345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:2.0:1619677447.440348:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.440351:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.440352:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.0:1619677447.440352:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:2.0:1619677447.440353:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.440355:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.440356:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677447.440357:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.440358:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.440358:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.440359:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.440361:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.440362:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.440362:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.440363:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.440364:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.440365:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96728194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.440366:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.440367:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.440368:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.440369:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:2.0:1619677447.440369:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.440370:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.440371:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.440371:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.440372:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:2.0:1619677447.440372:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.440374:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.440374:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:20.0:1619677447.440376:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.440376:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:8.0:1619677447.440377:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.440377:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:2.0:1619677447.440377:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.440377:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677447.440378:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:20.0:1619677447.440379:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.440380:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.440380:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.440381:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:9.2:1619677447.440381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677447.440382:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96728193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.440383:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.440384:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.440384:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:20.0:1619677447.440385:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.440385:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677447.440386:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:8.0:1619677447.440387:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.440387:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.440388:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.440388:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677447.440390:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:5.0:1619677447.440390:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.440391:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.440391:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.440391:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:18.0:1619677447.440392:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.440392:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.440392:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:20.0:1619677447.440393:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677447.440393:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:5.0:1619677447.440394:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:2.0:1619677447.440394:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.440395:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.440395:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:5.0:1619677447.440396:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96728192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.440397:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:5.0:1619677447.440397:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.440398:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.440399:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:9.2:1619677447.440402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:5.0:1619677447.440402:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.440404:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.440406:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.440407:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677447.440407:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.440408:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677447.440408:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.440409:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:2.0:1619677447.440409:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.440411:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677447.440411:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.440411:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.440412:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.440412:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:2.0:1619677447.440412:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.440413:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.440413:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:2.0:1619677447.440413:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677447.440414:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.440414:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:2.0:1619677447.440414:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:20.0:1619677447.440415:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.440415:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:2.0:1619677447.440415:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.440424:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.440426:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.440426:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.440426:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.440427:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:20.0:1619677447.440428:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.440428:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.440430:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.440431:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.0:1619677447.440432:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:6.0:1619677447.440494:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.440497:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.440498:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.440499:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.440502:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.440504:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.440532:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.440535:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.440535:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.440536:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.440538:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.440541:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.440621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.440662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677447.440667:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.440669:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.440670:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.440672:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.440673:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.440675:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.440774:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.440777:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.440779:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.440780:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.440782:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.440783:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.440785:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.440786:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.440787:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.440788:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.440791:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.440793:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.440795:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.440796:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.440796:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.440798:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.440799:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.440799:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.440800:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.440811:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.440814:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.440815:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.440815:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.440817:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.440819:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.440925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.440927:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.440929:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.440931:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.440931:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.440933:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.440936:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.440945:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.440948:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.440948:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.440949:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.440950:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.440951:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.440974:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.440977:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.440978:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.440978:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.440980:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.440980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.440983:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677447.440988:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.440991:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.440992:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.440992:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:4.2:1619677447.440993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:18.0:1619677447.440994:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.440997:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677447.441000:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.441003:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.441004:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.441006:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.441008:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.441010:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.441011:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.441012:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.441013:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.441014:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.441017:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.441019:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.441021:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.441022:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.441023:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.441024:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.441025:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.441025:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.441026:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.441050:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.441053:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.441054:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.441056:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.441058:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.441060:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.441061:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.441062:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.441063:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.441065:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.441069:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.441072:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.441075:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.441076:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.441077:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.441079:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.441080:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.441081:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.441081:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.441103:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.441106:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.441106:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.441108:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.441111:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.441111:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.441199:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.441202:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.441203:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.441205:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.441207:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.441208:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.441210:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.441211:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.441212:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.441213:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.441216:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.441218:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.441221:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.441222:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.441222:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.441224:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.441225:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.441225:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.441226:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677447.441237:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.441239:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.441240:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.441242:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.441243:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.441245:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.441340:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.441342:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.441344:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.441345:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:20.0:1619677447.441347:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.441347:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.441348:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677447.441348:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.441350:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96728176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.441350:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000400:00000200:20.0:1619677447.441351:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.441353:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.441354:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:5.0:1619677447.441356:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:20.0:1619677447.441357:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.441357:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.441357:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.441358:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677447.441359:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677447.441359:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.441360:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.441362:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677447.441362:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.441363:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677447.441363:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:13.0:1619677447.441364:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.441365:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:12.0:1619677447.441365:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.441366:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.441367:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.441368:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.441372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.441380:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.441383:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.441384:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.441384:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.441386:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.441389:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677447.441412:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677447.441415:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677447.441416:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677447.441418:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.441421:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.441422:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.441423:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.441425:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.441426:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.441427:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.441430:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:2.0:1619677447.441430:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.441431:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.441432:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.441434:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.441434:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.441435:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.441435:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.441435:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.441436:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.441438:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:2.0:1619677447.441438:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.441438:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677447.441439:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677447.441439:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.0:1619677447.441451:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.441453:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.441454:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.441456:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.441456:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.441458:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:9.2:1619677447.441717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.441724:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.441727:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.441729:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.441729:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.441731:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.441734:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.441741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.441748:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.441751:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.441752:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.441753:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.441756:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.441758:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.441759:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.441760:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.441761:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.441762:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.441765:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.441767:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.441770:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.441771:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.441771:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.441773:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.441773:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.441775:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.441776:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.441776:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.441783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.441790:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.441792:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.441793:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.441794:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.441833:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677447.441835:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:13.0:1619677447.441836:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.441836:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.441837:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:13.0:1619677447.441838:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.441838:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.441840:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:12.0:1619677447.441840:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:5.0:1619677447.441840:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.441842:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.441843:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.441852:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.441854:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.441855:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.441856:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.441856:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.441857:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.441912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.441921:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.441924:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.441925:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.441928:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.441930:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.441932:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.441933:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.441935:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.441937:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.441938:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.441943:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.441946:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.441949:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.441950:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.441951:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.441953:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.441953:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.441954:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.441955:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.441996:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.441998:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.441999:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.442001:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.442004:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.442005:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.442006:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.442008:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.442009:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.442010:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.442013:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.442015:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.442017:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.442018:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.442018:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.442020:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.442020:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.442021:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.442022:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.442026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.442033:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.442035:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.442036:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.442037:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.442038:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.442039:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.442136:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.442138:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.442139:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.442141:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.442143:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.442144:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677447.442145:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.442147:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.442147:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.442148:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677447.442152:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.442153:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677447.442156:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.442157:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677447.442157:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677447.442159:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677447.442159:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.442160:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.442160:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.0:1619677447.442197:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.442200:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.442204:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.442204:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.442207:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.442209:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.442211:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.442212:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.442214:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.442215:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.442221:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.442224:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.442228:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.442229:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.442229:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.442231:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.442232:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.442233:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.442235:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.442235:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677447.442243:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.442246:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677447.442246:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:12.0:1619677447.442247:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.442248:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.442248:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.442249:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677447.442250:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.442250:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:12.0:1619677447.442251:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.442252:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677447.442252:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.442253:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.442253:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.442255:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.442258:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677447.442262:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.442264:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.442265:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.442266:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.442267:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.442268:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.442279:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.442281:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:18.0:1619677447.442281:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:20.0:1619677447.442283:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.442284:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:5.0:1619677447.442284:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:2.0:1619677447.442285:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:20.0:1619677447.442286:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.442287:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.442288:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.442289:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.442290:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.442291:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.442291:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.442292:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.442297:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.442299:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.442300:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677447.442300:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.442301:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.442304:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.442306:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.442307:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677447.442309:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.442310:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.442311:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.442311:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:18.0:1619677447.442312:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.442313:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.442313:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:18.0:1619677447.442313:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.442314:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.442315:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.442315:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:18.0:1619677447.442316:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.442317:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.442317:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.442318:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.442328:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:12.0:1619677447.442330:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.442330:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.442331:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.442332:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.442333:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:12.0:1619677447.442333:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.442334:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.442335:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.442335:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:13.0:1619677447.442337:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.442338:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.442469:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.442472:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.442473:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.442475:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.442478:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.442479:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.442481:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.442482:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.442483:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.442484:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677447.442488:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.442490:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.442492:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.442493:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.442494:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.442495:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.442496:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.442497:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.442497:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677447.442621:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.442623:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:5.0:1619677447.442626:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.442627:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677447.442629:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.442632:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:20.0:1619677447.442633:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728156] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.442635:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.442637:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.442638:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:20.0:1619677447.442644:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.442647:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:20.0:1619677447.442650:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.442651:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:20.0:1619677447.442652:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:20.0:1619677447.442654:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:20.0:1619677447.442655:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.442655:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.442656:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677447.442660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:6.0:1619677447.442663:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677447.442666:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:6.0:1619677447.442666:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:24.0:1619677447.442667:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:22.0:1619677447.442667:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:13.0:1619677447.442668:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:24.0:1619677447.442669:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:13.0:1619677447.442669:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.442669:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.442670:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.442670:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:13.0:1619677447.442671:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.442672:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:6.0:1619677447.442672:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.442673:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:13.0:1619677447.442673:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:6.0:1619677447.442674:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.442675:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.442677:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:6.0:1619677447.442677:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.442678:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.442679:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.442682:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.442683:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677447.442684:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677447.442686:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.442687:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.442687:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:6.0:1619677447.442687:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.442687:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:12.0:1619677447.442688:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677447.442688:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677447.442689:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.442690:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.442690:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.442692:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.442693:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.442693:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.442704:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.442706:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.442707:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.442708:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.442760:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.442763:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.442765:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.442765:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.442768:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.442770:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.442771:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.442772:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.442773:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.442775:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.442778:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.442780:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.442783:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.442783:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.442784:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.442786:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.442787:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.442787:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.442788:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.442799:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.442802:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.442803:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.442804:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.442805:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.442807:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.442836:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.442839:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.442840:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.442841:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.442844:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.442846:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677447.442847:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.442849:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.442850:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.442851:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677447.442854:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.442856:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.442858:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.442859:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.442860:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.442861:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.442862:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.442862:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.442863:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.442873:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.442875:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.442876:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.442877:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.442879:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.442880:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.442927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.443223:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.443226:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.443227:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.443229:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.443231:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.443233:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.443234:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728149] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.443236:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.443236:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.443237:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.443241:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.443242:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.443245:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.443246:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.443246:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.443248:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.443248:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.443250:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.443251:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.443251:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.443269:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.443271:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.443271:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677447.443272:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.0:1619677447.443272:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677447.443274:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.443275:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.443277:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.443278:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.443280:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:9.2:1619677447.443294:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.443304:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.443308:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.443309:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.443309:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.443311:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.443315:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:6.0:1619677447.443370:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.443373:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.443373:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.443375:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.443378:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.443380:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.443381:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.443381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:6.0:1619677447.443383:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.443383:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.443385:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.443388:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.443390:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.443393:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.443394:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.443395:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.443396:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.443397:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.443397:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.443398:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.443422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677447.443434:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677447.443437:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677447.443437:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.443439:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.443440:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.443440:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.443441:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.443441:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.443442:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:22.0:1619677447.443443:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.443443:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677447.443443:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.443445:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:22.0:1619677447.443446:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:5.0:1619677447.443446:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.443447:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.443449:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.443450:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.443453:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.443454:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.443455:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.443457:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:5.0:1619677447.443457:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.443458:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.443459:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.443460:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.443461:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.443464:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.443466:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.443468:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.443469:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.443469:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.443470:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.443470:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.443470:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.443472:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.443473:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.443473:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.443474:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677447.443474:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.443475:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677447.443476:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (23)++
00000800:00000200:5.0:1619677447.443476:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (24)--
00000800:00000200:20.0:1619677447.443477:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (24)--
00000800:00000200:20.0:1619677447.443478:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.443488:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.443490:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677447.443490:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.443490:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.443491:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677447.443491:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.443493:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.443493:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.443494:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.443496:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677447.443508:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.443511:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.443512:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.443513:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.443513:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.443514:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677447.443548:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.443550:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.443551:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.443552:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.443554:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.443555:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677447.443594:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.443597:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.443598:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.443599:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.443602:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.443604:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.443605:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.443606:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.443607:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.443608:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.443611:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.443613:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.443616:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.443617:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.443618:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.443619:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.443620:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.443621:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.443622:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.443623:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:13.0:1619677447.443633:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677447.443634:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:13.0:1619677447.443635:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677447.443636:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:13.0:1619677447.443636:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.443637:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:13.0:1619677447.443637:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.443638:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677447.443640:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.443640:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677447.443696:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.443699:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.443699:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.443700:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.443702:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.443705:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.443796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.443840:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.443843:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.443843:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.443844:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.443846:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.443848:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.443988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.443996:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.443999:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.444000:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.444000:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.444002:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.444006:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.444179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.444222:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.444225:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.444226:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.444226:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.444228:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.444231:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.444239:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.444242:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.444243:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.444243:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.444244:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.444245:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.444270:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.444273:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.444275:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.444275:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.444277:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.444279:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677447.444280:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.444281:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.444282:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.444283:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.444286:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.444288:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.444290:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.444291:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.444292:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.444302:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:5.0:1619677447.444303:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.444303:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.444304:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677447.444344:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.444346:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.444349:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.444349:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:20.0:1619677447.444351:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.444351:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:5.0:1619677447.444351:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:2.0:1619677447.444352:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:20.0:1619677447.444353:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.444355:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.444355:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.444357:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.444358:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.444358:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.444359:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677447.444360:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.444362:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.444363:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.444364:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.444365:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.444366:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.444367:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.444369:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.444370:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.444370:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.444372:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000400:00000200:18.0:1619677447.444372:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.444373:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.444373:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.444374:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:18.0:1619677447.444374:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.444377:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.444378:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.444379:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.444381:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.444381:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.444382:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.444383:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677447.444385:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.444388:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.444388:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.444390:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.444391:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.444392:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:12.0:1619677447.444394:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.444397:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.444398:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.444400:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.444401:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.444402:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.444413:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.444415:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.444416:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.444417:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.444417:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.444418:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.444617:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.444620:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.444621:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.444623:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.444625:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.444627:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.444628:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.444629:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:2.0:1619677447.444630:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.444630:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.444632:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:2.0:1619677447.444632:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.444635:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.444636:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.444637:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.444638:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.444639:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:20.0:1619677447.444640:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.444642:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.444642:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.444643:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.444643:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.444643:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677447.444644:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.444645:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.444645:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.444645:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.444646:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.444647:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:20.0:1619677447.444649:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.444651:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.444654:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.444654:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.444655:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.444657:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.444657:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.444658:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.444658:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.444669:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.444671:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.444672:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.444672:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:12.0:1619677447.444674:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.444675:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677447.444675:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.444676:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677447.444677:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:6.0:1619677447.444678:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.444681:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.444684:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.444685:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.444687:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.444688:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.444689:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.444693:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.444695:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.444698:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.444698:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.444699:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.444701:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:13.0:1619677447.444702:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677447.444702:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.444702:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.444703:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.444704:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.444705:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.444707:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.444708:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.444709:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.444930:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.444933:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.444934:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.444936:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.444939:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.444940:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.444941:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.444943:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.444944:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.444945:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.444948:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.444950:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.444953:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.444954:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.444955:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.444957:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.444957:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.444958:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.444958:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.444963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.444971:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.444973:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.444974:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.444975:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.444977:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.444980:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.444985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.444993:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.444994:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.444996:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677447.444996:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677447.444997:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.444997:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677447.444997:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677447.444997:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.444998:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.444999:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:22.0:1619677447.445000:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.445002:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.445012:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.445014:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.445015:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.445015:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.445016:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.445017:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.445171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.445193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.445200:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.445203:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.445204:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.445206:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.445209:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.445210:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.445212:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96728122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.445213:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.445214:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.445215:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.445218:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.445219:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.445227:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.445228:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.445228:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.445230:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.445231:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.445231:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.445232:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.0:1619677447.445234:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.445236:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.445237:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.445239:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.445239:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.445242:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:12.0:1619677447.445259:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.445262:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.445263:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.445265:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.445266:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.445267:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.445431:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.445433:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.445435:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.445436:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.445438:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.445439:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.445440:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.445442:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.445443:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.445444:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.445447:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.445449:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.445452:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.445452:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.445453:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.445454:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.445455:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.445456:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.445456:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.445469:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.445472:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.445472:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.445474:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:20.0:1619677447.445475:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.445476:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:9.2:1619677447.445479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.445486:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.445489:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.445490:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.445490:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.445492:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.445496:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:18.0:1619677447.445581:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.445584:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.445585:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.445587:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.445589:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.445591:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.445592:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.445593:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.445595:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.445595:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.445599:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.445601:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.445603:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.445604:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.445604:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.445606:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.445606:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.445608:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.445609:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.445609:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677447.445620:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677447.445620:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.445622:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:20.0:1619677447.445623:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677447.445623:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.445624:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:20.0:1619677447.445626:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.445627:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.445628:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.445630:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677447.445631:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.445633:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.445634:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.445635:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677447.445640:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.445642:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677447.445645:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.445646:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677447.445646:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.445648:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.445649:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.445650:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.445651:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.445652:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:9.2:1619677447.445654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.445662:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:5.0:1619677447.445663:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:18.0:1619677447.445664:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:5.0:1619677447.445665:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:18.0:1619677447.445666:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.445666:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677447.445666:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677447.445667:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677447.445668:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.445671:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.445674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.445681:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.445684:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.445685:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.445686:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.445687:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.445687:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.445880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.445886:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.445889:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.445890:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.445890:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.445892:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.445895:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.445899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.445948:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.445950:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.445951:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.445952:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.445952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677447.445954:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.445956:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677447.446067:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.446070:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.446071:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.446072:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.446075:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.446077:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.446078:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728112] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.446079:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.446080:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.446081:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.446085:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.446086:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.446088:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.446089:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.446089:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.446091:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.446091:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.446092:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.446092:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.446097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.446104:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.446106:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.446106:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.446107:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.446108:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.446110:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.446155:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677447.446158:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677447.446160:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677447.446163:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.446163:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677447.446165:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677447.446166:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96728111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.446168:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677447.446169:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677447.446170:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677447.446174:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.446176:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677447.446179:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.446180:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:13.0:1619677447.446181:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677447.446183:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:13.0:1619677447.446183:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677447.446185:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677447.446186:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677447.446187:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677447.446200:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677447.446202:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677447.446203:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677447.446204:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:6.0:1619677447.446219:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.446221:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.446222:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.446222:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.446225:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.446227:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677447.446254:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.446257:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.446257:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.446260:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.446262:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.446264:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.446265:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.446267:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.446268:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.446269:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.446272:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.446274:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.446277:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.446278:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.446279:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.446281:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.446281:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.446282:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.446283:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.446288:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.446291:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.446292:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.446293:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677447.446294:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.446298:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:13.0:1619677447.446301:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.446301:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.446302:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.446303:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:12.0:1619677447.446304:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.446304:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.446306:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.446307:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.446308:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:12.0:1619677447.446308:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.446308:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.446309:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:12.0:1619677447.446310:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.446312:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.446312:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.446313:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.446314:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.446314:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.446504:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.446507:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.446509:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.446510:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.446511:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.446513:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.446515:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.446515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:20.0:1619677447.446516:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.446518:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.446519:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.446522:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.446524:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.446527:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.446527:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.446528:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.446530:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.446531:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.446531:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.446532:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.446536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.446544:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.446546:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.446547:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.446548:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.446551:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.446553:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.446567:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.446570:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.446571:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.446571:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.446573:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.446577:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.446749:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.446752:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.446753:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.446754:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.446757:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.446759:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.446760:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.446761:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.446763:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.446763:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.446767:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.446769:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.446772:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.446773:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.446774:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.446775:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.446776:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.446776:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.446777:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677447.446790:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.446792:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.446793:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.446795:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.446795:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.446799:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:6.0:1619677447.446803:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.446805:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.446806:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.446809:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.446812:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.446814:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.446816:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.446818:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.446820:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.446821:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.446826:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.446830:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.446833:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.446834:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.446835:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:13.0:1619677447.446836:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:6.0:1619677447.446837:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.446837:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.446838:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.446838:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.446839:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.446840:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.446841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:13.0:1619677447.446842:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.446843:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.446844:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677447.446850:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:8.0:1619677447.446851:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677447.446853:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.446854:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.446854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677447.446854:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.446855:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:20.0:1619677447.446856:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.446856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677447.446857:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.446857:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.446858:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:8.0:1619677447.446860:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.446862:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:18.0:1619677447.446863:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:8.0:1619677447.446863:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96728096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.446864:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.446865:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:8.0:1619677447.446865:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.446865:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:18.0:1619677447.446866:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.446866:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.446866:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.446867:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.446868:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.446870:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:2.0:1619677447.446870:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.446871:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.446872:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.446873:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.446873:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.446873:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677447.446874:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:8.0:1619677447.446874:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.446875:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.446875:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.446876:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.0:1619677447.446877:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.446878:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.446878:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.446878:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.446880:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.446881:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.446881:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.446882:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.446889:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.446891:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.446892:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.446892:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.446893:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.446894:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677447.447006:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.447009:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677447.447011:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.447012:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.447013:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.447015:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.447016:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.447018:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.447019:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.447020:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.447023:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.447025:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.447028:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.447028:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.447029:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:4.2:1619677447.447030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.447031:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.447031:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.447032:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.447033:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677447.447044:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.447046:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.447047:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.447049:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.447050:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.447051:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:9.2:1619677447.447069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.447077:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.447079:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.447081:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.447081:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.447083:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.447086:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.447090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.447097:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.447099:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.447100:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.447100:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.447101:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.447102:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677447.447147:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677447.447147:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677447.447150:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:18.0:1619677447.447151:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.447151:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:20.0:1619677447.447153:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.447155:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.447156:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.447157:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.447159:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.447159:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.447160:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:5.0:1619677447.447160:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.447162:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:5.0:1619677447.447162:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.447162:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.447163:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.447164:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.447165:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.447166:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.447168:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.447169:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.447170:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.447172:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.447173:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.447175:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.447175:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.447176:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:2.0:1619677447.447176:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.447177:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:5.0:1619677447.447177:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:18.0:1619677447.447178:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96728090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.447178:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.447178:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.447179:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.447179:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.447179:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:18.0:1619677447.447180:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.447180:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677447.447181:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.447184:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.447185:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.447185:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.447186:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.447187:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.447188:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.447189:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.447189:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.447190:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677447.447190:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.447191:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (23)++
00000800:00000200:2.0:1619677447.447191:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:22.0:1619677447.447192:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (24)++
00000800:00000200:2.0:1619677447.447192:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.447194:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (25)--
00000800:00000200:18.0:1619677447.447194:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677447.447195:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:18.0:1619677447.447195:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:22.0:1619677447.447196:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.447196:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:6.0:1619677447.447196:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:18.0:1619677447.447197:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:22.0:1619677447.447198:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.447201:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:22.0:1619677447.447202:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:12.0:1619677447.447203:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:22.0:1619677447.447204:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:12.0:1619677447.447204:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.447206:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:12.0:1619677447.447206:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.447207:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.447208:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677447.447208:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.447208:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:22.0:1619677447.447209:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:12.0:1619677447.447209:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.447210:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.447211:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677447.447212:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.447212:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677447.447264:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.447266:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.447267:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.447267:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.447270:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.447272:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677447.447331:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.447333:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.447334:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.447336:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.447337:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.447339:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677447.447480:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.447483:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.447484:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.447486:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.447489:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.447490:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.447492:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.447493:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.447494:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.447495:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.447499:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.447500:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.447503:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.447504:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.447505:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.447507:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.447507:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.447508:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.447509:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:4.2:1619677447.447540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.447546:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.447549:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.447549:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.447551:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.447553:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.447554:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.447556:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.447558:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.447559:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.447562:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.447564:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.447566:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.447567:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.447569:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.447570:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.447571:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.447575:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.447577:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.447580:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.447581:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.447581:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.447583:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.447584:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.447585:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.447586:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.447587:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.447589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.447597:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677447.447597:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.447599:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:6.0:1619677447.447600:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.447601:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677447.447601:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.447601:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677447.447601:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.447603:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.447606:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677447.447744:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.447747:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677447.447749:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.447751:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.447752:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.447753:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:20.0:1619677447.447754:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:5.0:1619677447.447754:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.447755:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.447756:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.447757:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.447758:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677447.447761:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.447763:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000400:00000200:20.0:1619677447.447764:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.447765:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:20.0:1619677447.447766:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.447767:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96728081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.447768:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.447769:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.447770:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.447773:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.447774:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.447774:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677447.447775:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677447.447776:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.447777:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677447.447777:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.447778:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.447779:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677447.447780:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.447780:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.447780:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.447781:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.447782:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.447783:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677447.447783:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.447784:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.447784:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.447785:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.447785:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.447786:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677447.447787:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.447788:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677447.447795:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.447797:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.447798:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.447800:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.447801:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.447803:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.447910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.447930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.447937:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.447939:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.447941:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.447941:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.447943:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.447946:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.447951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.447956:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.447958:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.447959:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.447960:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.447962:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.447965:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.448029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.448036:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.448039:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.448040:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.448041:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.448044:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.448045:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.448047:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.448048:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.448049:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.448050:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.448053:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.448054:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.448057:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.448058:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.448058:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.448060:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.448060:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.448061:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.448061:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.448146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.448149:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.448151:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.448152:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:5.0:1619677447.448152:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.448153:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677447.448154:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.448156:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.448157:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.448158:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:20.0:1619677447.448159:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.448159:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677447.448160:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:5.0:1619677447.448160:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000400:00000200:20.0:1619677447.448161:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.448161:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.448161:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.448162:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.448163:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.448163:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:20.0:1619677447.448164:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.448165:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.448169:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.448171:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.448174:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.448175:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.448176:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.448177:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.448178:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.448180:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.448181:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677447.448182:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.448192:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.448193:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677447.448193:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.448194:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000400:00000200:5.0:1619677447.448194:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.448195:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.448196:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.448196:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.448197:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.448197:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.448199:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:22.0:1619677447.448200:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.448200:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.448201:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.448203:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.448206:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.448210:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.448212:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.448213:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.448214:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.448214:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.448216:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:24.0:1619677447.448277:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.448280:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.448282:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.448283:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.448286:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.448288:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.448289:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.448290:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.448291:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.448292:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.448301:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.448302:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.448305:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.448306:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.448307:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.448309:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.448309:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.448310:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.448310:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677447.448323:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.448326:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.448327:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.448328:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.448329:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.448331:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.448341:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.448344:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.448345:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.448346:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.448346:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.448347:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.448448:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.448450:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.448450:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.448452:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.448453:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.448455:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.448490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.448714:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.448717:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.448717:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.448719:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.448722:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.448723:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.448724:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.448726:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.448727:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.448728:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.448731:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.448733:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.448737:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.448737:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.448738:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.448740:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.448740:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.448741:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.448741:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.448956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.449001:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.449004:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.449005:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.449005:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.449007:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.449010:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.449019:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.449021:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.449022:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.449022:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.449023:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.449024:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.449047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.449055:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.449058:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.449059:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.449059:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.449061:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.449065:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.449093:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677447.449095:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:2.0:1619677447.449095:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.449096:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677447.449098:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:2.0:1619677447.449099:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.449100:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.449101:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.449102:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.449103:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.449104:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.449105:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677447.449105:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.449107:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.449107:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.449109:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.449109:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.449110:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.449111:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.449112:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677447.449116:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.449117:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.449119:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.449120:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677447.449121:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.449122:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.449123:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.449123:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.449124:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677447.449125:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:2.0:1619677447.449125:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.449125:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.449126:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677447.449127:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:2.0:1619677447.449127:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677447.449128:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.449129:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.449130:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.449131:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:22.0:1619677447.449138:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.449138:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:22.0:1619677447.449141:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.449141:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.449142:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.449143:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:22.0:1619677447.449144:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.449144:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.449145:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:22.0:1619677447.449146:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.449146:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.449148:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.449151:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.449151:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.449154:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.449156:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677447.449158:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.449160:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.449162:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.449162:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677447.449166:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.449167:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.449170:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.449171:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.449171:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.449173:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.449173:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.449174:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.449176:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.449176:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.449188:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.449189:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.449189:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.449190:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:5.0:1619677447.449190:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.449191:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.449192:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.449192:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.449193:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677447.449195:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:12.0:1619677447.449203:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.449206:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.449207:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.449209:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.449211:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.449212:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.449264:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.449267:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.449269:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.449270:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.449271:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.449273:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.449274:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.449275:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.449276:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.449277:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.449281:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.449282:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.449285:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.449285:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.449286:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.449288:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.449288:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.449289:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.449290:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.449302:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.449304:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.449305:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.449307:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.449307:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.449309:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:6.0:1619677447.449431:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.449434:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.449435:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.449437:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.449439:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.449441:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.449442:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.449443:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.449444:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.449445:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.449449:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.449450:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.449453:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.449454:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.449454:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.449456:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.449457:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.449457:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.449458:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.449560:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.449562:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.449563:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.449565:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.449566:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.449568:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.449575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.449616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.449622:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.449624:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.449625:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.449626:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.449628:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.449631:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.449675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.449683:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.449686:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.449687:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.449687:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.449689:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.449692:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.449793:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.449795:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.449796:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.449798:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.449800:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.449800:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677447.449801:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.449803:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.449804:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.449806:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.449808:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.449809:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.449811:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.449812:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:2.0:1619677447.449812:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.449813:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.449814:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:13.0:1619677447.449816:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.449817:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.449818:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.449818:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677447.449818:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.449819:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.449820:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.449822:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.449823:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.449823:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.449825:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.449825:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.449826:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.449827:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.449834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.449841:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.449843:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.449844:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.449845:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.449846:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.449848:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.450249:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.450252:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.450253:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.450255:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.450257:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.450258:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.450260:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.450261:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.450262:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.450264:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.450266:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.450268:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:2.0:1619677447.450268:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.450269:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.450271:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.450271:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.450272:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.450272:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.450273:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:5.0:1619677447.450274:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:2.0:1619677447.450274:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677447.450275:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.450276:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.450276:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.450276:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:5.0:1619677447.450277:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.450278:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.450278:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000400:00000200:5.0:1619677447.450279:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.450279:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:5.0:1619677447.450282:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.450285:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.450287:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.450288:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.450289:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.450290:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:13.0:1619677447.450290:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.450291:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.450291:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.450292:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:5.0:1619677447.450292:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.450293:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677447.450295:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (23)--
00000800:00000200:12.0:1619677447.450298:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677447.450300:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677447.450301:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.450302:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.450304:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.450307:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:13.0:1619677447.450307:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.450309:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.450310:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.450312:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.450313:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.450314:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.450408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.450432:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.450435:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677447.450437:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.450438:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.450439:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.450441:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.450442:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.450443:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.450444:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.450445:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.450449:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.450450:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.450453:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.450453:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.450454:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.450456:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.450456:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:6.0:1619677447.450457:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677447.450457:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.450458:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:6.0:1619677447.450460:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.450461:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.450462:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.450463:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.450466:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.450469:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.450471:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.450472:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.450474:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.450475:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.450476:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:22.0:1619677447.450493:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.450495:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.450496:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.450498:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.450501:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.450502:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.450504:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.450505:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.450506:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.450507:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.450511:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.450513:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.450516:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.450517:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.450517:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.450517:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.450519:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.450519:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.450520:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:6.0:1619677447.450520:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.450521:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.450521:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.450522:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.450523:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.450525:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.450573:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.450576:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.450577:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.450579:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.450580:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.450582:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.450595:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.450597:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.450598:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.450599:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.450599:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.450600:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677447.450606:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.450609:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.450610:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.450612:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.450613:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.450615:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.450776:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.450778:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.450779:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.450781:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677447.450782:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.450784:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677447.450868:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.450870:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.450871:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.450871:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.450874:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.450876:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.450885:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.450888:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.450889:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.450889:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.450890:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.450891:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.451033:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.451036:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.451037:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.451039:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.451042:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.451043:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.451045:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.451046:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.451047:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.451048:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.451050:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:2.0:1619677447.451052:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.451053:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.451054:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.451056:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:2.0:1619677447.451056:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.451057:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:8.0:1619677447.451058:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.451058:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.451059:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:8.0:1619677447.451060:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.451060:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.451061:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:8.0:1619677447.451062:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:2.0:1619677447.451062:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.451062:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:8.0:1619677447.451063:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96728042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.451069:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.451070:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.451071:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.451073:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.451075:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:8.0:1619677447.451075:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.451076:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.0:1619677447.451077:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.451080:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.451080:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.0:1619677447.451081:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.451082:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.451084:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.451084:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:8.0:1619677447.451086:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:2.0:1619677447.451086:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:8.0:1619677447.451087:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.451088:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:2.0:1619677447.451088:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.451090:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000400:00000200:2.0:1619677447.451090:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:8.0:1619677447.451091:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.451091:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:2.0:1619677447.451091:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.451094:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.451095:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.451096:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.451099:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.451101:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677447.451101:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.451103:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:2.0:1619677447.451103:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.451103:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.451104:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:20.0:1619677447.451104:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.451104:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.451105:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.451106:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:2.0:1619677447.451106:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.451106:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:24.0:1619677447.451107:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:20.0:1619677447.451107:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.451107:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.451107:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677447.451109:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677447.451110:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:9.2:1619677447.451112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.451119:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.451120:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677447.451122:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.451123:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:5.0:1619677447.451123:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.451123:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000400:00000200:18.0:1619677447.451124:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.451124:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.451124:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.451126:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:18.0:1619677447.451126:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.451129:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.451130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:22.0:1619677447.451132:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.0:1619677447.451134:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677447.451135:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.451136:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.451136:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:5.0:1619677447.451137:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677447.451137:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:22.0:1619677447.451139:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.451140:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677447.451140:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.451141:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.451142:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677447.451142:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.451142:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000400:00000200:5.0:1619677447.451143:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.451143:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.451144:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:5.0:1619677447.451145:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677447.451146:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.451148:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.451149:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.451150:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677447.451154:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.451156:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.451159:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.451160:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.451160:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.451162:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.451163:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.451163:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.451164:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677447.451175:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.451177:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.451178:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.451179:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.451181:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.451182:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:6.0:1619677447.451269:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.451272:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.451273:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.451273:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.451275:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.451277:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677447.451383:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.451385:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677447.451387:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677447.451388:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.451390:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.451391:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677447.451392:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.451394:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.451395:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.451396:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.451400:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.451401:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.451404:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.451404:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.451405:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.451407:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.451407:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.451408:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.451408:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677447.451419:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.451424:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.451425:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.451428:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.451432:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.451433:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.451435:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.451437:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.451438:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.451439:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.451443:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.451446:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.451448:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.451448:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.451449:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.451451:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.451451:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.451452:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.451453:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.451464:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.451467:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.451467:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.451469:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.451470:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.451471:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:18.0:1619677447.451616:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:22.0:1619677447.451618:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:18.0:1619677447.451618:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.451619:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:22.0:1619677447.451621:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:18.0:1619677447.451621:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.451621:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.451622:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.451623:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.451624:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.451625:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.451626:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.451627:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:18.0:1619677447.451628:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.451629:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.451630:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.451633:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.451635:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.451639:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.451640:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.451640:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.451642:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.451643:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.451644:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.451644:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677447.451656:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.451659:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.451659:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.451661:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.451663:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.451664:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.451925:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.451927:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.451930:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.451931:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.451932:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.451934:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.451936:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.451937:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.451938:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.451939:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.451942:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.451944:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.451947:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.451948:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.451948:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677447.451949:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (24)++
00000800:00000200:20.0:1619677447.451951:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (25)--
00000800:00000200:20.0:1619677447.451951:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677447.451951:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:20.0:1619677447.451952:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:12.0:1619677447.451952:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.451953:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:12.0:1619677447.451954:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.451955:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.451956:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677447.451957:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.451957:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.451958:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677447.451959:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.451961:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.451961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.451983:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.451985:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.451986:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.451986:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.451987:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.451988:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.451995:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.451998:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.451999:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.452001:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.452003:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.452005:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:2.0:1619677447.452005:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677447.452006:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.452007:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.452008:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:6.0:1619677447.452008:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:24.0:1619677447.452009:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:2.0:1619677447.452009:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.452010:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.452011:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.452011:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:8.0:1619677447.452012:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.452012:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.452015:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.452015:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.452015:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000400:00000200:8.0:1619677447.452016:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:2.0:1619677447.452017:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:8.0:1619677447.452018:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96728025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.452018:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:8.0:1619677447.452019:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.452021:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.452021:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677447.452022:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.452022:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677447.452023:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677447.452025:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677447.452025:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677447.452026:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:8.0:1619677447.452026:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.452026:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.452026:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677447.452028:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:22.0:1619677447.452029:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.452030:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.452030:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.452031:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:8.0:1619677447.452031:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.452032:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.452032:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.452033:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.452034:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.452035:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.452035:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.452036:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677447.452050:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.452053:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.452054:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.452056:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.452056:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.452059:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:12.0:1619677447.452091:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.452093:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.452094:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.452096:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.452098:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.452099:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.452203:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.452206:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677447.452207:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677447.452209:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.452211:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.452213:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:6.0:1619677447.452214:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.452216:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.452216:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.452217:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:6.0:1619677447.452221:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.452222:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:6.0:1619677447.452225:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.452226:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:6.0:1619677447.452227:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677447.452229:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677447.452229:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.452230:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677447.452230:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:4.2:1619677447.452231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.452238:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.452241:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:12.0:1619677447.452241:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.0:1619677447.452242:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.452244:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.452244:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.452245:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.452246:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.452247:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.452248:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:13.0:1619677447.452248:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:24.0:1619677447.452249:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.452249:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:24.0:1619677447.452251:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.452252:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.452253:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.452256:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.452257:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.452260:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.452261:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.452262:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.452264:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.452264:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.452265:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.452266:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.452267:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.452272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.452280:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.452282:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.452282:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.452283:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.452314:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.452317:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.452318:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.452321:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.452324:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.452326:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.452328:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.452329:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.452331:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.452332:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.452337:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.452340:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.452343:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.452344:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.452344:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.452346:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.452347:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.452348:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.452349:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.452350:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:4.2:1619677447.452352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.452361:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.452363:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.452364:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.452364:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.452365:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.452365:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.452366:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:20.0:1619677447.452366:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.452367:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.452367:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.452367:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.452368:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.452369:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:6.0:1619677447.452369:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.452372:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.452374:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.452376:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.452377:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.452378:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.452378:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.452378:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677447.452379:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.452419:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.452422:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.452422:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.452424:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.452425:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.452427:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.452448:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.452451:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.452453:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.452453:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.452456:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.452457:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677447.452458:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.452459:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.452460:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.452461:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677447.452464:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.452467:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.452470:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.452471:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.452472:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.452473:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.452474:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:24.0:1619677447.452475:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677447.452475:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677447.452477:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:5.0:1619677447.452477:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:24.0:1619677447.452478:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:5.0:1619677447.452478:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:24.0:1619677447.452481:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.452483:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.452484:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.452485:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.452486:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:24.0:1619677447.452487:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:20.0:1619677447.452487:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:24.0:1619677447.452488:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96728016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.452488:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:18.0:1619677447.452488:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:20.0:1619677447.452489:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.452489:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.452491:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.452491:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:5.0:1619677447.452491:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.452492:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.452492:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.452492:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:5.0:1619677447.452493:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.452494:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.452494:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.452496:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.452496:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.452498:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.452499:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.452499:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.452501:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.452502:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.452502:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.452503:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677447.452503:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.452503:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.452504:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:18.0:1619677447.452504:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.452505:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:18.0:1619677447.452505:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677447.452505:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.452506:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.452507:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:18.0:1619677447.452507:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96728014] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.452508:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.452508:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.452509:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.452509:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.452509:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.452510:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.452510:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.452511:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:18.0:1619677447.452511:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.452511:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.452512:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.452512:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677447.452512:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.452513:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:18.0:1619677447.452514:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.452515:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.452515:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677447.452516:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:9.2:1619677447.452517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.452519:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.452521:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677447.452521:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:18.0:1619677447.452523:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:20.0:1619677447.452524:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:18.0:1619677447.452524:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.452525:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:13.0:1619677447.452525:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:20.0:1619677447.452526:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:18.0:1619677447.452526:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677447.452526:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677447.452527:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.452527:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.452528:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677447.452529:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.452529:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.452530:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.452531:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:13.0:1619677447.452531:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.452532:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000400:00000200:2.0:1619677447.452533:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.452534:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677447.452534:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677447.452534:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.452535:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.452536:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.452536:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677447.452539:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.452630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.452669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:4.2:1619677447.452827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.452859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.452866:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.452868:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.452869:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.452869:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.452871:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.452874:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.452977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.452984:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.452987:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.452988:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.452989:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677447.452990:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.452991:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.452992:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.452993:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.452994:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.452996:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96728010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.452997:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.452998:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.452999:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.453001:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.453002:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96728009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.453003:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.453003:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.453004:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.453005:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.453005:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.453009:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.453010:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.453010:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:8.0:1619677447.453010:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.453012:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.453013:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (23)++
00000800:00000200:8.0:1619677447.453013:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.453014:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (24)--
00000800:00000200:24.0:1619677447.453015:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.453016:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.453016:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.453017:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.453022:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.453024:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.453024:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.453025:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.453025:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.453041:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.453044:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.453045:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.453047:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.453048:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.453049:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.453055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.453090:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.453093:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:9.2:1619677447.453093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677447.453096:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.453096:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.453098:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:18.0:1619677447.453099:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.453099:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.453100:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.453101:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.453103:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.453103:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.453104:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.453105:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.453107:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.453110:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.453111:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.453112:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.453113:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.453116:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.453117:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.453117:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.453119:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.453120:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.453121:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.453121:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677447.453133:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.453136:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.453137:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.453138:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.453140:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.453141:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.453150:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.453152:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.453153:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.453154:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.453154:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.453155:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.453260:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.453262:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.453265:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.453265:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.453269:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.453270:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:4.2:1619677447.453270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:5.0:1619677447.453272:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.453274:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.453275:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.453276:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677447.453281:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.453283:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677447.453286:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.453287:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677447.453287:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677447.453289:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677447.453290:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.453290:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.453291:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.453298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.453306:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.453309:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.453310:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.453311:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.453312:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.453315:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:12.0:1619677447.453322:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.453324:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.453325:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.453327:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.453329:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.453329:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.453338:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.453340:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.453341:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.453342:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.453342:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.453343:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.453444:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.453446:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.453448:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.453449:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.453451:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.453452:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.453454:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96728002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.453455:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.453456:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.453457:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.453460:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.453462:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.453464:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.453464:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.453465:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.453467:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.453467:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.453468:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.453469:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:22.0:1619677447.453472:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.453475:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.453476:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.453478:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.453481:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.453481:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:22.0:1619677447.453482:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:5.0:1619677447.453483:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:22.0:1619677447.453484:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96728001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.453484:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.453485:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.453486:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:22.0:1619677447.453487:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.453487:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.453488:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.453489:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:22.0:1619677447.453492:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.453494:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.453497:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.453498:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.453498:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.453500:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.453501:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.453501:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.453502:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.453503:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.453509:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:4.2:1619677447.453509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.453516:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.453517:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.453517:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.453518:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:4.2:1619677447.453558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677447.453562:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.453564:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.453565:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.453567:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.453571:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.453574:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.453771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.453778:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.453781:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.453782:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.453784:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.453787:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.453788:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.453789:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.453791:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.453792:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.453793:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:4.2:1619677447.453794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677447.453796:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.453798:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.453800:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.453801:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.453802:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.453803:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.453804:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.453804:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.453805:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.453839:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.453842:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.453843:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.453843:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.453846:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.453849:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.453857:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.453860:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.453860:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.453861:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.453862:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.453863:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.453961:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.453963:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.453966:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.453967:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.453968:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.453969:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:20.0:1619677447.453971:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.453972:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.453973:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.453974:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:20.0:1619677447.453978:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.453980:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:20.0:1619677447.453983:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.453984:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:20.0:1619677447.453984:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:20.0:1619677447.453986:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:20.0:1619677447.453987:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.453988:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.453988:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677447.453992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.453998:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.454001:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.454002:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.454002:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.454005:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.454007:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.454012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677447.454018:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.454021:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677447.454021:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.454022:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.454023:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.454023:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:18.0:1619677447.454262:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.454265:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.454266:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.454268:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.454270:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.454272:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.454273:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.454275:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.454276:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.454277:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.454280:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.454282:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.454285:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.454286:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.454287:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.454289:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.454289:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.454290:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.454290:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677447.454308:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.454311:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.454313:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.454313:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.454316:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.454317:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:5.0:1619677447.454318:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:13.0:1619677447.454319:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:5.0:1619677447.454319:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.454320:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.454320:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677447.454322:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.454322:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.454323:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:12.0:1619677447.454324:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.454325:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677447.454327:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.454329:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.454332:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.454333:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.454334:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.454336:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.454337:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.454338:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.454338:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.454344:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.454347:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.454348:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.454350:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.454352:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.454355:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.454356:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.454358:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.454359:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.454360:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.454363:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.454364:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.454367:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.454368:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.454368:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.454370:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.454370:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.454371:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.454372:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.454376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.454384:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.454387:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.454388:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.454388:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.454390:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.454393:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.454396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.454403:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.454405:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.454406:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.454407:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.454408:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.454408:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.0:1619677447.454434:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.454437:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.454439:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.454440:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.454442:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.454444:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677447.454446:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.454447:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.454448:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.454449:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677447.454452:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.454454:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.454456:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.454457:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.454458:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.454459:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.454460:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.454461:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.454461:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.454474:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.454476:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.454478:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.454479:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.454479:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.454481:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.454688:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.454690:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.454691:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.454693:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.454693:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.454696:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677447.454774:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.454776:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.454777:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.454779:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.454781:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.454782:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.454869:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.454873:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.454873:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.454875:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.454878:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.454879:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.454880:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.454881:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.454883:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.454883:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.454886:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.454888:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.454890:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.454890:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.454891:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.454892:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.454893:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.454894:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.454894:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.454909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.454916:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.454918:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.454919:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.454920:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.454920:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.454922:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.454925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.454959:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.454961:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.454962:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.454964:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.454967:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.454968:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.454970:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.454971:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.454972:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.454972:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:22.0:1619677447.454973:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.454974:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.454975:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.454977:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.454977:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.454978:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.454980:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.454980:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677447.454983:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.454984:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.454985:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.454987:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.454987:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.454988:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.454989:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.455159:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.455162:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.455163:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.455165:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.455167:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.455169:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.455170:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.455171:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.455172:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.455173:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.455176:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.455178:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.455181:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.455182:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.455182:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.455184:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.455184:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.455185:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.455186:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677447.455199:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.455201:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.455202:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.455204:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.455205:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.455207:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.455588:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.455590:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677447.455592:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.455594:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.455594:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.455596:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.455597:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.455598:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.455599:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.455600:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.455603:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.455605:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.455608:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.455609:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.455609:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.455611:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.455612:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.455612:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.455613:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:13.0:1619677447.455625:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.455627:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.455628:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.455630:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.455632:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.455633:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.455641:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.455644:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.455644:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.455647:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.455649:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.455651:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.455652:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.455654:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.455655:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.455656:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.455659:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.455661:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.455664:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.455665:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.455666:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.455667:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.455668:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.455669:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.455670:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.455670:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.455681:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.455683:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.455684:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.455685:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.455705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.455754:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.455756:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.455757:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.455759:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.455760:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.455762:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.455772:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.455775:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.455776:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.455777:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.455778:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.455779:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.455999:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.456001:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.456004:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.456004:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.456007:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.456009:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677447.456011:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.456013:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.456015:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.456017:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677447.456021:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.456022:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677447.456023:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.456025:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.456026:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677447.456026:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.456026:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677447.456027:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677447.456029:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677447.456029:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:2.0:1619677447.456029:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.456030:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.456031:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:2.0:1619677447.456031:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.456032:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.456034:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677447.456035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:2.0:1619677447.456036:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.456037:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.456039:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.456044:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:2.0:1619677447.456044:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.456045:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.456046:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.456046:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.456049:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.456049:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.456050:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.456051:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.456052:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.456053:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677447.456053:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.456053:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.456054:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.456055:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:6.0:1619677447.456067:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.456069:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.456070:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.456070:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:6.0:1619677447.456070:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.456073:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:6.0:1619677447.456073:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.456075:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.456075:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.456076:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.456078:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.456079:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677447.456081:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.456082:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.456083:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.456084:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677447.456087:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.456089:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.456092:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.456093:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677447.456093:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.456095:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.456096:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.456097:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.456097:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677447.456140:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.456144:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.456144:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.456146:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.456148:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.456151:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.456152:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.456154:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.456155:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.456155:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:18.0:1619677447.456156:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.456156:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677447.456160:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.456162:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.456163:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.456164:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.456165:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.456167:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.456169:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.456171:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.456172:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.456176:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.456178:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:2.0:1619677447.456178:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.456179:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.456181:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.456181:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.456182:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.456183:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.456184:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.456185:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.456185:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.456185:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.456186:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.456186:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.456188:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.456188:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.456189:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.456190:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.456193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:18.0:1619677447.456198:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:5.0:1619677447.456198:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:24.0:1619677447.456200:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:18.0:1619677447.456200:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.456201:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:22.0:1619677447.456202:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677447.456202:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:24.0:1619677447.456203:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:20.0:1619677447.456203:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000400:00000200:24.0:1619677447.456204:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.456204:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:22.0:1619677447.456205:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677447.456205:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (22)--
00000400:00000200:24.0:1619677447.456206:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.456206:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.456206:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:5.0:1619677447.456207:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.456209:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.456210:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.456212:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.0:1619677447.456212:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.456213:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.456214:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.456216:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.456217:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:5.0:1619677447.456217:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.456218:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:5.0:1619677447.456218:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.456218:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.456219:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:20.0:1619677447.456221:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.456221:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677447.456223:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.456226:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677447.456227:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.456229:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.456230:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.456231:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677447.456235:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.456237:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677447.456240:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.456241:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677447.456242:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677447.456244:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677447.456244:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677447.456245:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677447.456246:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:6.0:1619677447.456385:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.456387:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.456388:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.456390:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.456393:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.456394:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.456396:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.456397:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.456398:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.456399:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.456402:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.456404:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 504 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.456407:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.456408:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.456408:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.456410:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.456410:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.456411:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.456412:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.456428:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.456431:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.456431:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.456433:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.456434:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.456436:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.456511:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.456514:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.456516:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.456517:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.456519:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.456520:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.456521:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.456523:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.456524:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.456525:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.456528:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.456530:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.456532:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.456533:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.456534:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.456535:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.456536:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.456537:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.456539:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677447.456539:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.456548:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.456549:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.456549:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.456549:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000400:00000200:20.0:1619677447.456551:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.456551:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:20.0:1619677447.456552:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.456553:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.456554:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677447.456554:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.456554:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.456555:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677447.456556:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:5.0:1619677447.456556:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.456557:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:5.0:1619677447.456558:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:5.0:1619677447.456559:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.456561:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.456562:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.456562:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:5.0:1619677447.456566:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.456567:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:5.0:1619677447.456570:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.456571:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677447.456572:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.0:1619677447.456574:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677447.456574:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:5.0:1619677447.456575:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:5.0:1619677447.456575:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:9.2:1619677447.456579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.456586:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.456588:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.456590:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.456590:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.456592:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.456595:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.456597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.456605:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.456607:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.456607:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.456608:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.456608:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.456609:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.0:1619677447.456611:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.456613:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.456614:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.456616:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.456617:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.456619:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677447.456702:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.456705:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.456706:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:22.0:1619677447.456707:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677447.456708:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.456710:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.456711:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677447.456711:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.456713:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:2.0:1619677447.456713:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.456714:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : REPLY - routed
00000400:00000200:2.0:1619677447.456715:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.456716:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.456717:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677447.456717:0:19936:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 174539
00000800:00000200:8.0:1619677447.456717:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:2.0:1619677447.456717:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.456719:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677447.456719:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.456720:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.456721:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.456721:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:8.0:1619677447.456721:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.456723:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.456724:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.456725:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:2.0:1619677447.456725:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.456727:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96727955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.456727:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677447.456728:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.456728:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:8.0:1619677447.456729:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.456729:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:8.0:1619677447.456730:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.456730:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.456731:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677447.456731:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.456732:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.456732:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:8.0:1619677447.456733:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.456733:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.456734:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677447.456740:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.456743:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.456743:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.456744:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.456746:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.456746:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.456748:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.456749:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.456750:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.456759:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:22.0:1619677447.456762:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:12.0:1619677447.456762:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.456763:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.456765:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.15.18@o2ib3
00000400:00000200:12.0:1619677447.456765:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.456765:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (23)--
00000800:00000200:24.0:1619677447.456766:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:22.0:1619677447.456766:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (22)--
00000800:00000200:13.0:1619677447.456766:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:22.0:1619677447.456767:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.456767:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.456769:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.456769:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:22.0:1619677447.456769:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:24.0:1619677447.456770:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:22.0:1619677447.456770:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.456771:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.456773:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.456774:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.456777:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : REPLY try# 0
00000800:00000200:22.0:1619677447.456780:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 174539 bytes in 256 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.456781:0:19936:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 174539
00000800:00000200:22.0:1619677447.456784:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.456785:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.456786:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.456787:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.456788:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.456789:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.456789:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.456802:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.456804:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.456806:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.456807:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.456809:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.456810:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.456816:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.456818:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.456819:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:5.0:1619677447.456819:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677447.456821:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677447.456821:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:5.0:1619677447.456821:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.456822:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.456822:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.456823:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:20.0:1619677447.456823:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[2] from 10.0.10.115@o2ib7
00000400:00000200:5.0:1619677447.456824:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.456825:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000400:00000200:5.0:1619677447.456825:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:6.0:1619677447.456826:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.456828:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:5.0:1619677447.456829:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.456830:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:20.0:1619677447.456831:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.456831:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.456832:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.456834:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.456835:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.456836:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.456838:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.456838:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.456840:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.456841:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.456842:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.456849:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.456851:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.456852:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.456853:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.456855:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.456858:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.456859:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.456860:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.456862:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:12.0:1619677447.456862:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.456863:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:13.0:1619677447.456863:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.456865:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.456866:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.456866:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:6.0:1619677447.456866:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:20.0:1619677447.456867:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.456869:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.456869:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: REPLY: OK
00000800:00000200:5.0:1619677447.456869:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.456872:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677447.456873:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.456873:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.456876:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.456877:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.456877:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.456878:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.456879:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.456922:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.456925:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.456926:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.456927:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.456929:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.456929:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:4.2:1619677447.457138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.457145:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.457148:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.457149:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.457151:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.457153:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.457154:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.457156:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727949] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.457157:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.457158:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.457159:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.457162:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.457164:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.457167:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.457168:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.457168:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.457170:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.457170:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.457171:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.457173:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.457173:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.457179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:4.2:1619677447.457183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.457186:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.457187:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.457188:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.457190:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.457194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.457201:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.457203:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.457204:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.457205:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.457207:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.457210:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.457226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677447.457234:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:6.0:1619677447.457234:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:24.0:1619677447.457237:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:22.0:1619677447.457238:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:8.0:1619677447.457238:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:6.0:1619677447.457238:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:24.0:1619677447.457239:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.457239:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.457241:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.457242:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.457244:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:6.0:1619677447.457244:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:24.0:1619677447.457245:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.457246:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.457248:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.457248:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677447.457252:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.457253:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.457256:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.457257:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677447.457257:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677447.457259:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.457259:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.457260:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.457261:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.457271:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.457273:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.457274:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.457275:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:12.0:1619677447.457276:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.457277:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677447.457277:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:12.0:1619677447.457279:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677447.457280:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.457280:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.457283:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.457285:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.457286:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.457287:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.457289:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:5.0:1619677447.457289:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.457289:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.457292:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.457299:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.457299:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.457300:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.457300:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:5.0:1619677447.457300:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.457302:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.457305:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.457306:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.457307:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.457308:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.457309:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.457310:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.457310:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.457321:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.457324:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.457325:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.457327:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.457328:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.457329:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.457484:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.457487:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.457488:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.457491:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.457493:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.457494:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.457495:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.457497:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.457498:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.457499:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.457502:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.457504:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.457507:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.457507:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.457508:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.457509:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.457510:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.457511:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.457511:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:4.2:1619677447.457545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.457570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.457577:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.457579:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.457580:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.457581:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.457582:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.457585:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.0:1619677447.457686:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.457687:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:5.0:1619677447.457689:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:20.0:1619677447.457690:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.457690:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.457692:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.457694:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:5.0:1619677447.457694:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.457696:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.457697:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.457698:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.457699:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:5.0:1619677447.457702:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.457703:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.457704:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677447.457705:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:5.0:1619677447.457705:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.457706:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.457707:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.457708:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.457711:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.457712:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:5.0:1619677447.457712:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.457713:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677447.457714:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.457715:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677447.457715:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.457716:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677447.457717:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.457717:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.457718:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.457718:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.457719:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.457719:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.457721:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.457722:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.457723:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.457723:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:24.0:1619677447.457726:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:13.0:1619677447.457727:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:24.0:1619677447.457729:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:13.0:1619677447.457730:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:8.0:1619677447.457730:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:13.0:1619677447.457731:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.457732:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677447.457733:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.457734:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.457734:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.457735:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:24.0:1619677447.457736:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.457737:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96727938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.457738:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.457739:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.457740:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.457744:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.457745:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:24.0:1619677447.457746:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.457748:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:24.0:1619677447.457749:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.457749:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:22.0:1619677447.457749:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.457749:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677447.457750:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:22.0:1619677447.457751:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.457752:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.457753:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.457753:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677447.457753:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.457754:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.457768:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:9.2:1619677447.457769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.457771:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.457771:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.457772:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.457773:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.457774:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.457777:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.457780:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.457781:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.457781:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.457783:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.457787:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.457788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:12.0:1619677447.457795:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:18.0:1619677447.457796:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.457797:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:18.0:1619677447.457798:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677447.457798:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.457799:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.457800:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000400:00000200:12.0:1619677447.457800:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.457801:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677447.457801:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.457802:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677447.457802:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.457841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.457886:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.457889:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.457890:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.457890:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.457892:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.457895:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.457906:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.457909:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.457909:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.457910:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.457911:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.457911:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.457949:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.457952:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.457952:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.457953:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.457955:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.457958:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.457975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.457982:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.457985:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.457986:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.457988:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.457991:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.457992:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.457994:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.457995:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.457996:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.457997:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.458000:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.458002:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.458005:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.458006:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.458006:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.458008:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.458008:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.458009:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.458009:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.458086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.458104:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.458107:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.458108:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.458112:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.458114:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.458116:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.458117:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.458119:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.458120:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.458121:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.458124:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.458126:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.458129:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.458129:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.458130:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.458132:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.458133:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.458133:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.458134:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.458146:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.458149:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.458149:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.458151:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.458152:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.458153:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.458293:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.458299:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.458299:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677447.458302:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.458304:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.458306:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:5.0:1619677447.458307:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.458308:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.458309:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.458310:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:4.2:1619677447.458311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:5.0:1619677447.458314:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.458316:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:5.0:1619677447.458319:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.458320:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:5.0:1619677447.458321:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:5.0:1619677447.458322:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:5.0:1619677447.458323:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.458325:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.458326:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.458326:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:9.2:1619677447.458329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:20.0:1619677447.458334:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.458336:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:24.0:1619677447.458337:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:20.0:1619677447.458337:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.458338:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:24.0:1619677447.458340:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.458341:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.458341:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.458343:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.458346:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.458350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677447.458355:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.458357:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.458360:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.458360:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.458364:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.458365:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.458367:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.458368:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.458369:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.458370:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.458373:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.458375:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.458378:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.458378:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.458379:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.458381:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.458381:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.458382:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.458383:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.458387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.458396:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.458398:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.458399:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.458400:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.458402:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.458405:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.0:1619677447.458433:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.458436:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.458437:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677447.458439:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:8.0:1619677447.458439:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.458441:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.458442:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:8.0:1619677447.458442:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:13.0:1619677447.458443:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.458444:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.458444:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96727927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.458446:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:8.0:1619677447.458446:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.458447:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:8.0:1619677447.458447:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.458448:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677447.458451:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.458453:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.458455:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.458456:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.458456:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.458458:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.458458:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.458459:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.458459:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.458470:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.458473:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.458473:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.458475:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.458476:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.458477:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.458487:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.458490:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.458491:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.458492:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.458492:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.458494:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.458505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.458509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:18.0:1619677447.458513:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.458516:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:24.0:1619677447.458517:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:18.0:1619677447.458517:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.458517:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.458520:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:18.0:1619677447.458520:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.458520:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.458523:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.458524:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:24.0:1619677447.458525:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.458526:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.458528:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.458529:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.458530:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.458531:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.458534:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.458536:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.458538:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.458539:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.458539:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.458541:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.458541:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.458542:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.458543:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.458575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.458596:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.458598:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.458599:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.458601:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.458602:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.458603:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.458622:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.458624:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.458625:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.458626:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.458628:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.458631:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677447.458659:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.458662:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.458664:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.458664:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.458666:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.458668:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.458669:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.458670:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.458672:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.458673:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.458676:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.458678:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 568 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.458680:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.458681:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.458682:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.458683:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.458684:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.458684:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.458685:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677447.458696:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.458698:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.458699:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.458701:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.458702:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.458703:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:4.2:1619677447.458885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.458908:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.458911:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.458912:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.458914:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.458917:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.458918:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677447.458920:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.458921:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.458922:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.458923:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677447.458926:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.458928:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677447.458931:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.458931:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677447.458932:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677447.458934:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677447.458934:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.458936:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.458937:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.458937:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677447.458947:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:18.0:1619677447.458949:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:6.0:1619677447.458950:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.458951:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677447.458951:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:6.0:1619677447.458951:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.458952:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.458953:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:6.0:1619677447.458953:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.458956:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.458976:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.458978:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.458979:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.458980:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.458982:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.458983:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.459099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.459106:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.459108:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.459109:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.459110:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.459112:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.459115:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.459267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.459275:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.459277:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.459279:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.459279:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.459281:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.459285:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.459339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.459347:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.459350:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.459351:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.459352:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.459355:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.459357:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.459358:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.459359:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.459361:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.459361:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.459365:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.459367:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.459370:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.459370:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.459371:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.459373:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.459373:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.459374:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.459374:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.459387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.459446:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.459449:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.459450:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.459450:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.459452:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.459455:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.459517:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.459520:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.459522:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.459523:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.459529:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.459531:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.459532:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.459533:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.459534:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.459535:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.459539:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.459541:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.459543:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.459544:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.459545:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:12.0:1619677447.459546:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.459546:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.459547:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:12.0:1619677447.459548:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677447.459548:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.459548:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:12.0:1619677447.459549:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.459550:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:12.0:1619677447.459551:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.459552:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.459552:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.459553:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:13.0:1619677447.459553:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.459554:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677447.459554:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.459555:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.459556:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.459557:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.459557:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.459558:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:20.0:1619677447.459559:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.459561:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:13.0:1619677447.459562:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.459565:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.459566:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.459566:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.459567:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.459568:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.459641:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.459644:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.459646:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.459649:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.459651:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.459654:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.459656:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.459658:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.459659:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.459659:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.459664:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.459666:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.459669:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.459670:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.459671:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.459673:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.459674:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.459675:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.459676:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.459677:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.459684:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.459687:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.459687:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.459688:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.459797:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.459800:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.459800:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.459802:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.459804:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.459805:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.459843:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.459846:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.459847:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.459849:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.459851:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.459853:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.459854:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.459855:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.459856:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.459857:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.459860:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.459862:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.459864:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.459864:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.459865:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.459867:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.459868:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.459868:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.459869:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.459873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.459880:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.459883:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.459883:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.459884:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.459885:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.459887:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.459894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.459901:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.459903:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.459904:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.459905:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.459905:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.459906:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.460001:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.460004:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.460005:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.460007:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.460009:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.460010:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.460012:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.460013:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.460014:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.460015:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.460019:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.460021:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.460023:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.460024:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.460025:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.460027:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.460027:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.460027:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.460028:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.460055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.460096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.460111:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.460114:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.460115:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.460115:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.460117:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.460120:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.460253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.460260:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677447.460263:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:2.0:1619677447.460263:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.460264:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.460265:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:18.0:1619677447.460265:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.460266:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.460268:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.460269:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:2.0:1619677447.460270:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:20.0:1619677447.460271:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.460273:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677447.460274:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.460275:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.460276:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.460277:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677447.460281:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.460283:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.460285:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.460286:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677447.460287:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.460288:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.460289:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.460290:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.460290:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:6.0:1619677447.460379:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677447.460381:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677447.460382:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:6.0:1619677447.460384:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.460386:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.460388:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.460389:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.460390:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.460391:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.460392:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.460395:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.460397:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.460399:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.460400:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.460401:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.460402:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.460403:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677447.460404:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:6.0:1619677447.460405:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:6.0:1619677447.460405:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:9.2:1619677447.460409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.460415:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000800:00000200:22.0:1619677447.460417:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:18.0:1619677447.460417:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677447.460418:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677447.460419:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:18.0:1619677447.460420:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.460421:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.460421:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.460423:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.460425:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:6.0:1619677447.460592:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.460594:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.460596:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.460596:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.460598:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.460600:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.460609:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.460611:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.460612:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.460612:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.460613:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.460614:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.460641:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.460643:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.460644:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.460645:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.460647:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.460648:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.460649:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.460651:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.460652:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.460653:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.460656:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.460657:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:4.2:1619677447.460659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.460660:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.460660:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.460661:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.460663:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.460663:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:4.2:1619677447.460663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:6.0:1619677447.460664:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.460665:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.460665:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.460675:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:8.0:1619677447.460675:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.460677:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:8.0:1619677447.460677:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:22.0:1619677447.460678:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677447.460679:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677447.460680:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:8.0:1619677447.460680:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.460684:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.460684:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:8.0:1619677447.460686:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.460687:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96727897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.460687:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.460689:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:8.0:1619677447.460689:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.460690:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.460690:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.460691:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.460693:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.460694:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:8.0:1619677447.460695:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.460696:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.460698:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.460698:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.460699:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.460700:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:8.0:1619677447.460702:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.460703:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.460703:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677447.460704:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.460705:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.460706:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.0:1619677447.460706:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 568 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:8.0:1619677447.460707:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.460708:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.460708:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677447.460708:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.460709:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.0:1619677447.460709:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.460710:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.460711:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:6.0:1619677447.460712:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:5.0:1619677447.460712:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:22.0:1619677447.460713:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.460713:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.460713:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677447.460714:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:4.2:1619677447.460714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:6.0:1619677447.460715:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.460715:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:6.0:1619677447.460718:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.460722:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.460725:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.460726:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677447.460726:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:24.0:1619677447.460727:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677447.460727:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.460727:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677447.460728:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677447.460728:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.460729:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.460729:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:20.0:1619677447.460730:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.460731:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.460732:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.0:1619677447.460746:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.460749:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.460750:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.460750:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677447.460751:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.460752:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.0:1619677447.460847:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.460850:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.460851:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.460853:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.460853:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.460856:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.461047:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.461050:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.461052:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.461053:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.461054:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.461055:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677447.461057:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.461058:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.461059:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.461060:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677447.461063:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.461065:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677447.461068:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.461068:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677447.461069:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.461071:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.461071:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.461072:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.461072:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.461077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.461084:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.461087:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.461088:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.461089:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.461090:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.461092:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:18.0:1619677447.461109:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.461112:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.461112:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.461115:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.461117:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.461119:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.461121:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.461122:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.461123:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.461125:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.461128:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.461130:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.461133:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.461134:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.461134:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.461136:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.461137:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.461138:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.461139:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.461139:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:13.0:1619677447.461145:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.461148:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.461149:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.461150:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:13.0:1619677447.461151:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.461152:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677447.461152:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677447.461152:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.461153:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:13.0:1619677447.461154:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.461155:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.461155:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.461155:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:5.0:1619677447.461157:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.461158:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.461160:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.461169:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.461172:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.461173:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.461173:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.461174:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.461175:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677447.461233:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.461236:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.461237:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.461238:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.461240:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.461242:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.461243:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.461245:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.461246:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.461247:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.461250:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.461252:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:22.0:1619677447.461254:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677447.461255:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.461255:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:22.0:1619677447.461256:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:18.0:1619677447.461256:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:6.0:1619677447.461257:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677447.461258:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.461258:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:22.0:1619677447.461259:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.461259:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.461260:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000400:00000200:22.0:1619677447.461261:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.461261:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:22.0:1619677447.461263:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.461264:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.461266:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.461267:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.461267:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.461270:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.461271:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:22.0:1619677447.461272:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.461273:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.461274:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:22.0:1619677447.461275:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.461275:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:22.0:1619677447.461276:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.461276:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.461278:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.461279:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.461279:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.461280:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.461284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.461301:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.461302:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.461302:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:18.0:1619677447.461303:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.461303:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:18.0:1619677447.461306:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.461307:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.461309:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.461310:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.461311:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000400:00000200:2.0:1619677447.461311:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:18.0:1619677447.461312:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677447.461312:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.461313:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.461314:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.461314:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.461314:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.461315:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.461315:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.461316:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:18.0:1619677447.461317:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:2.0:1619677447.461321:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.461323:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677447.461326:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.461327:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677447.461327:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677447.461329:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677447.461330:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.461330:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.461331:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.0:1619677447.461483:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.461486:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.461487:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.461488:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.461490:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.461492:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.461674:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.461676:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677447.461678:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.461679:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.461681:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.461682:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.461683:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.461685:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.461686:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.461687:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.461690:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.461692:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.461694:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.461695:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.461696:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.461698:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.461698:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.461699:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.461700:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677447.461711:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.461713:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.461713:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.461715:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.461716:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.461717:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:12.0:1619677447.461786:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:9.2:1619677447.461788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:12.0:1619677447.461789:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.461790:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.461791:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.461793:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.461794:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677447.461796:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.461799:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.461800:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.461800:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.461802:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.461805:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:6.0:1619677447.461914:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.461917:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.461918:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.461920:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.461922:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.461924:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.461925:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.461927:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.461928:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.461929:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.461932:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.461934:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.461937:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.461938:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.461939:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.461940:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.461941:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.461941:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.461942:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.461984:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.461987:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.461989:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.461989:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.461991:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.461993:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.461994:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.461995:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.461997:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.461997:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.462001:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.462003:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.462006:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.462007:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.462007:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.462009:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.462009:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.462011:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.462012:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677447.462019:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.462021:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.462022:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:13.0:1619677447.462022:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.462024:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:2.0:1619677447.462024:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677447.462025:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.462027:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.462027:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.462028:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677447.462029:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:13.0:1619677447.462030:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677447.462030:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.462032:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.462033:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.462035:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677447.462038:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.462040:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.462044:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.462044:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.462045:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.462047:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.462047:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.462048:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.462049:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:4.2:1619677447.462069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.0:1619677447.462072:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.462075:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.462076:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.462076:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:20.0:1619677447.462078:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.462079:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:5.0:1619677447.462079:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.462080:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677447.462081:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:8.0:1619677447.462082:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.462085:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.462086:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.462088:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.462089:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.462090:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.462091:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.462094:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.462096:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.462099:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.462100:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.462100:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.462102:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.462102:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.462103:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.462103:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.462131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.462139:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.462142:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.462143:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.462143:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.462145:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.462148:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.462205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.462253:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.462256:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.462256:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.462257:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.462259:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.462262:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.462410:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.462413:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.462414:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.462416:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.462418:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.462420:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.462421:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.462423:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.462424:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.462425:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:12.0:1619677447.462426:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.462428:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:2.0:1619677447.462428:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677447.462429:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.462429:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:12.0:1619677447.462431:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.462433:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.462433:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.462433:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.462434:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677447.462435:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.462436:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.462436:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.462437:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.462438:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.462442:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.462445:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.462446:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.462447:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.462447:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.462448:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.462618:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.462621:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.462624:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.462624:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.462625:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d0600] (21)++
00000400:00000200:5.0:1619677447.462626:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.462628:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d0600] (22)--
00000800:00000200:6.0:1619677447.462628:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.28@o2ib3
00000400:00000200:5.0:1619677447.462628:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.462629:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.462631:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.28@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.462631:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.462632:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.462633:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677447.462634:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.462635:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677447.462637:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.462637:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.462639:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.462640:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.462640:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:6.0:1619677447.462641:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.462644:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.462644:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000400:00000200:6.0:1619677447.462645:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.28@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.462645:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:6.0:1619677447.462647:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.462647:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.462647:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.462648:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.462648:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:6.0:1619677447.462650:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.462651:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (21)++
00000800:00000200:6.0:1619677447.462651:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (22)++
00000800:00000200:6.0:1619677447.462653:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (23)--
00000800:00000200:6.0:1619677447.462653:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d0600] (21)++
00000800:00000200:6.0:1619677447.462654:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d0600] (22)--
00000800:00000200:6.0:1619677447.462655:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d0600] (21)--
00000800:00000200:20.0:1619677447.462660:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (23)++
00000800:00000200:20.0:1619677447.462662:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (24)--
00000400:00000200:20.0:1619677447.462663:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.462665:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.462666:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (23)++
00000800:00000200:20.0:1619677447.462667:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (24)--
00000800:00000200:5.0:1619677447.462667:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (24)--
00000400:00000200:5.0:1619677447.462668:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.462669:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.462670:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.462672:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677447.462760:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.462762:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.462765:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.462765:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.462767:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.462768:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677447.462769:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.462771:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.462772:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.462773:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677447.462775:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.462777:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:4.2:1619677447.462779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.462780:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.462781:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.462782:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.462784:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.462785:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.462785:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.462786:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677447.462797:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.462799:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.462800:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.462802:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.462802:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.462804:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.462815:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.462818:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.462819:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.462819:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.462820:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.462821:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677447.462827:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.462830:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.462831:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.462831:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.462834:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.462837:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.462848:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.462851:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.462852:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.462852:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.462853:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.462853:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.462932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.462939:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.462942:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.462943:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.462945:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.462948:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.462950:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.462951:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.462952:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.462953:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.462955:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.462958:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.462960:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.462963:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.462963:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.462964:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.462966:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.462966:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.462967:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.462967:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.463018:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.463021:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.463022:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.463024:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.463026:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.463028:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.463029:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.463031:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.463032:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.463033:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.463037:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.463038:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.463041:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.463042:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.463042:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.463044:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.463044:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.463046:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.463047:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.463047:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677447.463059:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:24.0:1619677447.463060:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.463062:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:5.0:1619677447.463062:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.463062:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.463063:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.463064:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:5.0:1619677447.463064:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.463065:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.463067:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:12.0:1619677447.463114:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.463117:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.463117:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.463119:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.463121:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.463122:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.463168:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.463171:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.463172:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.463174:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.104@o2ib7 : PUT - routed
00000800:00000200:22.0:1619677447.463176:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.463183:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.463186:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.463188:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.463189:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.463190:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.463192:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.463194:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.463195:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.463196:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.463197:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.463200:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.463202:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.463205:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.463206:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.463206:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.463208:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.463209:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.463209:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.463210:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:4.2:1619677447.463213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.463221:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.463224:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.463226:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.463227:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.463230:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.463233:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.463234:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.463236:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.463238:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.463239:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.463243:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.463246:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.463249:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.463250:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.463250:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.463252:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.463253:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.463254:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.463255:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.463256:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.463264:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.463266:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.463267:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:4.2:1619677447.463267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677447.463268:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.463275:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.463277:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.463278:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:6.0:1619677447.463279:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.463280:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.463280:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.463281:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.463282:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.463284:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.463285:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.463285:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.463286:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.463536:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.463539:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.463540:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.463541:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.463543:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.463546:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.463560:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.463563:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:18.0:1619677447.463564:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677447.463564:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.463566:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.463567:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:22.0:1619677447.463568:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.463568:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:22.0:1619677447.463569:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.463570:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.463571:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.463572:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.463573:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.463573:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.463574:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.463575:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.463576:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.463578:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.463579:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.463579:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.463580:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.463582:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.463584:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.463585:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.463586:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.463586:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.463587:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.463589:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.463589:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677447.463589:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.463590:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.463590:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677447.463590:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.463590:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.463592:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.463592:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.463593:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.463594:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677447.463604:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.463606:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.463607:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.463609:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.463609:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.463611:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.463647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.463658:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.463662:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.463664:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.463664:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:5.0:1619677447.463665:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.463666:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677447.463667:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.463670:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.463673:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.463675:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.463676:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:5.0:1619677447.463677:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.463678:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.463679:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:5.0:1619677447.463679:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:12.0:1619677447.463680:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.463682:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.463682:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.463683:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:13.0:1619677447.463683:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677447.463685:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.463685:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:20.0:1619677447.463686:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.463688:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.463688:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.463689:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.463691:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.463693:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.463695:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.463697:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.463697:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.463698:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.463698:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.463699:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.463699:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.463700:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.463701:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.463701:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.463702:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677447.463702:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.463703:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:5.0:1619677447.463703:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677447.463704:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:6.0:1619677447.463704:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.0:1619677447.463705:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:6.0:1619677447.463706:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.463708:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.463709:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.463710:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.463711:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.463712:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.463712:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.463713:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:22.0:1619677447.463714:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.463714:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.463715:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:22.0:1619677447.463716:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.463717:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.463717:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.463719:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.463721:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.463721:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677447.463734:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.463736:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.463737:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.463738:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.463738:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.463740:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:2.0:1619677447.463860:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.463863:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.463864:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.463866:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.463868:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.463870:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677447.463871:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.463873:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.463874:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.463875:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677447.463879:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.463880:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.463883:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.463884:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677447.463885:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677447.463887:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.463887:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.463888:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.463889:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:4.2:1619677447.463892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:13.0:1619677447.463900:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.0:1619677447.463900:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677447.463903:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:8.0:1619677447.463903:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.463904:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:13.0:1619677447.463904:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.463906:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.463906:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:8.0:1619677447.463906:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.463909:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:8.0:1619677447.463910:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.463911:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.463912:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.463914:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.463915:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.463916:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.463920:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.463921:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.463924:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.463925:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.463926:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.463928:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.463928:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.463929:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.463929:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.464027:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.464029:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.464030:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.464032:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.464033:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.464034:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.464040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.464048:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.464051:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.464052:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.464053:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.464053:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.464055:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.464085:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.464087:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.464089:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.464089:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.464092:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.464094:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.464105:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.464108:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.464109:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.464109:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.464111:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.464111:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.464118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.464125:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.464127:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.464128:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.464128:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.464131:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.464134:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.464348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.464373:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.464376:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.464379:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.464379:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.464381:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.464382:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.464384:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.464385:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.464386:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.464387:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.464391:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.464393:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.464395:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.464396:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.464397:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677447.464397:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.464399:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.464399:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677447.464400:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.464400:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677447.464400:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:6.0:1619677447.464401:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.464401:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:6.0:1619677447.464403:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.464406:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.464415:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.464418:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.464419:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.464420:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.464421:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.464421:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677447.464500:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.464503:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.464504:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.464506:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.464508:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.464510:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.464511:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.464513:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.464514:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.464515:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.464518:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.464520:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.464522:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.464523:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.464524:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.464525:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.464525:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.464526:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.464527:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677447.464538:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.464541:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.464542:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.464542:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.464545:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.464547:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.464565:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.464566:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.464567:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.464568:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.464569:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.464569:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.464569:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000400:00000200:13.0:1619677447.464571:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.464571:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:4.2:1619677447.464572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:12.0:1619677447.464573:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.464574:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.464575:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.464580:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.464583:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.464584:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.464586:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.464589:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.464591:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.464592:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96727841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.464593:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.464594:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.464595:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.464598:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.464598:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:8.0:1619677447.464600:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:2.0:1619677447.464601:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.464602:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:12.0:1619677447.464603:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677447.464603:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.464604:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:2.0:1619677447.464604:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.464605:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:12.0:1619677447.464606:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.464606:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.464606:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.464607:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:2.0:1619677447.464607:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.464608:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.464608:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.464609:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000400:00000200:2.0:1619677447.464609:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:13.0:1619677447.464610:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.464610:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677447.464610:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.464611:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677447.464611:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.464612:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.464614:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:4.2:1619677447.464616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:2.0:1619677447.464617:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.464618:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.464620:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.464621:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.464622:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.464623:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677447.464623:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.464624:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.464624:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:24.0:1619677447.464625:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:2.0:1619677447.464625:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:24.0:1619677447.464626:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.464626:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.464630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.464637:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.464639:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.464640:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.464641:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.464642:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.464644:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.464688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.464734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677447.464739:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.464741:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.464742:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.464744:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.464745:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.464748:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.464844:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.464846:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.464849:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.464849:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.464851:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.464853:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.464854:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.464855:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.464856:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.464857:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.464860:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.464862:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.464864:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.464865:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.464866:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.464867:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.464868:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.464869:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677447.464870:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:5.0:1619677447.464871:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.464880:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.464881:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.464882:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.464882:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.464882:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:5.0:1619677447.464883:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.464884:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.464884:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.464885:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677447.464886:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.0:1619677447.465157:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.465159:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.465161:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.465162:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.465163:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.465165:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.465166:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.465166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:5.0:1619677447.465167:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.465168:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.465169:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.465172:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.465173:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.465176:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.465177:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.465177:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.465183:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.465183:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.465184:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.465185:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.465210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.465217:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.465220:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.465221:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.465222:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.465224:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.465237:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.465250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.465257:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.465260:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.465261:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.465261:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.465263:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.465267:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.465271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.465278:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.465280:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.465281:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.465281:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.465283:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.465283:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.465318:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.465320:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.465321:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.465323:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.465324:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.465326:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677447.465363:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.465366:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.465367:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.465369:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.465371:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.465372:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.465373:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727830] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.465374:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.465376:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.465377:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.465380:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.465382:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.465384:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.465384:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.465385:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.465386:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.465387:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.465388:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.465389:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.465389:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:24.0:1619677447.465393:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:9.2:1619677447.465393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.465396:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677447.465397:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677447.465399:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.465401:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:24.0:1619677447.465402:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.465402:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.465403:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.465403:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:2.0:1619677447.465403:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.465404:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:18.0:1619677447.465404:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:24.0:1619677447.465405:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96727829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.465405:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.465406:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:2.0:1619677447.465406:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.465407:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.465408:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.465408:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:24.0:1619677447.465409:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677447.465412:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.465414:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.465416:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.465417:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.465418:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.465419:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.465420:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677447.465420:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677447.465421:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677447.465426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.465432:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.465434:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.465434:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.465435:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.465436:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.465438:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:18.0:1619677447.465493:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.465496:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.465496:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.465498:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:18.0:1619677447.465499:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.465500:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.465501:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.465502:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.465503:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.465504:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.465506:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.465508:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.465510:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.465512:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.465513:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.465515:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.465515:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.465516:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.465517:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.465517:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.465523:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.465526:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.465528:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.465528:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.465529:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.465529:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.465529:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.465530:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.465531:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.465532:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.465532:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)++
00000800:00000200:18.0:1619677447.465533:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (24)++
00000800:00000200:2.0:1619677447.465533:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (24)--
00000800:00000200:18.0:1619677447.465534:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (24)++
00000800:00000200:2.0:1619677447.465534:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (24)--
00000800:00000200:18.0:1619677447.465535:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (24)--
00000800:00000200:2.0:1619677447.465535:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (24)--
00000800:00000200:18.0:1619677447.465536:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.465537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.465541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:20.0:1619677447.465545:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.465546:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:2.0:1619677447.465546:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:20.0:1619677447.465547:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.465548:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:2.0:1619677447.465548:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.465548:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.465549:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.465549:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.465549:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:24.0:1619677447.465550:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:18.0:1619677447.465550:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.465551:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.465551:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.465552:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.465552:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:8.0:1619677447.465552:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:20.0:1619677447.465553:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:18.0:1619677447.465553:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:24.0:1619677447.465554:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.465554:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677447.465555:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.465556:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:5.0:1619677447.465556:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:24.0:1619677447.465557:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:20.0:1619677447.465558:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677447.465559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000400:00000200:20.0:1619677447.465560:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.465562:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677447.465563:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.465564:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.465565:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.465565:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000400:00000200:20.0:1619677447.465567:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.465568:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.465569:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.465569:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000400:00000200:20.0:1619677447.465570:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.465570:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.465570:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.0:1619677447.465572:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677447.465576:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.465577:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677447.465577:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677447.465579:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677447.465579:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.465580:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.465581:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677447.465585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.465591:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.465593:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.465594:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.465594:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.465596:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.465600:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.465600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677447.465607:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.465609:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.465610:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.465610:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.465611:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.465612:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.465640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.465647:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.465651:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.465652:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.465652:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677447.465654:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:18.0:1619677447.465654:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.465657:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:18.0:1619677447.465657:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.465657:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.465659:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.465662:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.465664:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.465665:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.465667:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.465668:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677447.465668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:22.0:1619677447.465669:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.465673:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.465675:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.465678:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.465679:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.465680:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.465682:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.465683:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.465684:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.465685:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.465686:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.465696:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.465697:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.465698:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.465699:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.465716:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.465718:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.465720:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.465721:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.465721:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.465725:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.465829:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.465832:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.465834:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.465835:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.465837:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.465838:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.465840:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.465842:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677447.465842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:20.0:1619677447.465843:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.465844:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.465848:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.465849:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677447.465849:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:24.0:1619677447.465852:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:20.0:1619677447.465852:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.465852:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.465853:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:8.0:1619677447.465853:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.465854:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.465854:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.465855:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.465856:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.465856:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:24.0:1619677447.465857:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.465858:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.465860:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96727818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.465862:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.465863:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.465864:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.465867:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.465869:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.465869:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.465871:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:24.0:1619677447.465872:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677447.465872:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.465873:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.465873:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:5.0:1619677447.465874:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.465875:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.465875:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677447.465875:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:24.0:1619677447.465876:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677447.465876:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:24.0:1619677447.465877:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.466186:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.466189:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:20.0:1619677447.466189:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:22.0:1619677447.466190:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.466190:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:20.0:1619677447.466191:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:22.0:1619677447.466192:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.466192:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.466194:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.466196:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.466197:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677447.466197:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.466198:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677447.466199:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.466200:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.466202:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.466204:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.466206:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.466207:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.466208:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000400:00000200:22.0:1619677447.466209:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.466210:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.466211:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.466211:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.466212:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.466213:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.466213:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:22.0:1619677447.466215:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.466215:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677447.466217:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.466219:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.466220:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.466221:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.466222:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.466223:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.466223:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.466224:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.466515:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.466517:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.466519:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.466520:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.466521:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.466523:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.466524:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.466525:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.466526:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.466527:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.466531:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.466533:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.466536:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.466537:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.466538:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.466539:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.466540:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.466541:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.466542:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.466543:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.466551:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.466553:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.466554:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.466554:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.466554:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.466557:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.466558:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.466560:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.466561:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.466563:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.466764:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.466767:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.466768:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.466770:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.466772:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.466773:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677447.466775:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:2.0:1619677447.466775:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.466776:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:2.0:1619677447.466776:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.466777:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:2.0:1619677447.466777:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.466778:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.466778:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:2.0:1619677447.466778:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.466780:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.466783:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.466783:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.466784:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:4.2:1619677447.466784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.466785:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.466786:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.466787:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.466787:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.466788:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:2.0:1619677447.466789:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677447.466790:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.466790:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.466791:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677447.466792:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.466793:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.466793:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.466793:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:5.0:1619677447.466794:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:2.0:1619677447.466794:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.466794:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:5.0:1619677447.466798:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.466801:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.466804:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.466805:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.466805:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.466806:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:12.0:1619677447.466807:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.466808:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.466808:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.466808:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.466809:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.466810:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:12.0:1619677447.466811:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.466812:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.466814:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.466821:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.466823:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:13.0:1619677447.466823:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677447.466824:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.466825:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.466826:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677447.466826:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677447.466827:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.466827:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.466828:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:13.0:1619677447.466828:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.466828:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677447.466829:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:13.0:1619677447.466830:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.466831:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.466832:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.466835:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.466837:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.466839:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.466840:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.466842:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.466843:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.466846:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.466848:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.466851:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.466852:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.466852:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.466854:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.466855:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.466856:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.466856:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.466861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.466868:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.466871:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.466872:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.466872:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.466874:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.466876:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:12.0:1619677447.467000:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.467002:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.467003:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.467005:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.467006:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.467008:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.467022:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.467024:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.467025:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.467026:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.467026:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.467028:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.467036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.467044:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.467047:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.467048:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.467050:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.467053:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.467054:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:8.0:1619677447.467055:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.467056:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.467057:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.467057:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.467058:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:8.0:1619677447.467059:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.467060:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.467061:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.467064:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.467064:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.467065:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:8.0:1619677447.467066:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.467067:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.467068:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.467069:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.467070:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677447.467070:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:8.0:1619677447.467071:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.467071:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.467073:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.467074:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.467074:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677447.467074:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.467075:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677447.467076:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.467079:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.467080:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.467080:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.467082:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.467082:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.467083:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.467084:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677447.467281:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.467284:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.467286:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.467286:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.467288:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.467290:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677447.467291:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.467292:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.467298:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.467299:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677447.467303:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.467305:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.467308:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.467309:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677447.467309:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677447.467311:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677447.467312:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.467313:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.467314:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.467315:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.467324:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.467326:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.467326:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.467327:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:4.2:1619677447.467419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.467459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677447.467461:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.467464:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.467465:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.467467:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.467470:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.467471:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.467472:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.467474:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:18.0:1619677447.467474:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.467475:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.467476:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.467477:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.467478:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.467478:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:18.0:1619677447.467479:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000400:00000200:22.0:1619677447.467480:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.467481:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.467482:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.467483:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677447.467483:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.467484:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.467485:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.467486:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.467487:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.467488:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.467492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.467499:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.467501:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.467502:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.467502:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.467504:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.467506:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.467531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.467538:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.467540:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.467541:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.467541:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.467542:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.467543:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.467562:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.467565:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.467565:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.467567:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.467569:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.467571:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.467572:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.467573:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.467574:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.467575:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.467578:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.467580:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.467583:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.467583:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.467584:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.467586:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.467586:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.467587:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.467587:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.467592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.467599:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.467601:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.467602:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.467603:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.467605:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.467607:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.467611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.467618:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.467621:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.467622:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.467622:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.467623:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.467624:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677447.467655:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.467657:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.467659:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.467660:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.467663:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.467665:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677447.467667:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:2.0:1619677447.467667:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.467668:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.467669:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:2.0:1619677447.467670:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.467671:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.467672:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.467673:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.467675:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.467676:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.467677:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.467678:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677447.467679:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.467680:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.467681:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.467681:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:20.0:1619677447.467682:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.467682:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677447.467683:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.467684:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677447.467686:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677447.467686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677447.467688:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.467688:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.467688:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.467689:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677447.467691:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.467692:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677447.467692:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:24.0:1619677447.467693:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:5.0:1619677447.467693:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:24.0:1619677447.467696:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:5.0:1619677447.467696:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.467697:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.467697:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677447.467697:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677447.467698:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:12.0:1619677447.467698:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:5.0:1619677447.467698:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.467699:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.467700:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:5.0:1619677447.467700:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:12.0:1619677447.467701:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.467701:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96727795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.467702:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:5.0:1619677447.467703:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677447.467704:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.467704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:5.0:1619677447.467704:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.467705:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677447.467705:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:12.0:1619677447.467707:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677447.467710:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.467712:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.467712:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677447.467712:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:5.0:1619677447.467712:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.467713:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677447.467714:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:18.0:1619677447.467714:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.467715:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.467715:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.467715:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:2.0:1619677447.467715:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677447.467716:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:18.0:1619677447.467718:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.467718:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.467718:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.467719:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.467719:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:18.0:1619677447.467725:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.0:1619677447.467731:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.467733:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.467734:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.467736:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.467738:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.467739:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.0:1619677447.467740:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.467742:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.467743:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.467744:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.467745:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.467747:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.467757:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.467759:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.467760:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.467761:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.467762:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.467763:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.0:1619677447.467764:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.467767:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.467768:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.467768:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.467769:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.467771:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.467832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.467839:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.467842:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.467843:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.467844:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.467847:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.467848:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.467850:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.467851:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.467852:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.467853:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.467857:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.467858:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.467861:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.467862:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.467862:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.467864:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.467864:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.467865:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.467866:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677447.467989:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.467992:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.467992:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.467994:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.467997:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.467998:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.468000:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.468001:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.468002:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.468003:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.468006:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.468008:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.468011:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.468011:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.468012:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.468014:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.468014:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.468015:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.468015:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:4.2:1619677447.468139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.468223:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.468226:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.468229:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.468231:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.468233:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.468234:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.468235:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.468237:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.468238:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.468242:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.468244:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.468246:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.468247:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.468248:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.468250:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.468250:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.468252:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.468253:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.468253:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.468265:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.468266:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.468267:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.468268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:1.0:1619677447.468268:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.468269:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677447.468276:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.468278:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.468279:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.468280:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.468282:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.468285:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:13.0:1619677447.468354:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.468357:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.468357:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.468359:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.468361:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.468362:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.468562:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.468565:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.468566:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.468567:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.468570:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.468572:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.468573:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.468574:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.468575:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.468576:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.468580:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.468581:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.468584:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.468585:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.468586:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.468588:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.468588:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.468589:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.468589:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677447.468602:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:4.2:1619677447.468603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:13.0:1619677447.468604:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.468605:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.468607:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.468608:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.468610:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.468611:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.468613:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.468614:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.468616:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.468619:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.468621:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.468622:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.468623:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.468624:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.468625:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.468629:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.468631:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.468634:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.468634:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.468635:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.468637:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.468637:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.468639:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.468640:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.468640:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.468646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.468651:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677447.468653:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.468654:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677447.468655:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677447.468655:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.468656:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.468657:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.468658:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.468660:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677447.468685:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.468687:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.468689:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.468690:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.468691:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.468693:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.468694:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.468696:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.468697:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.468698:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.468701:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.468703:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.468706:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.468707:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.468708:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.468710:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.468710:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.468711:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.468711:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.468722:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.468725:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.468726:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.468727:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.468729:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.468730:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.468801:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.468803:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.468805:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.468807:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.468808:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.468809:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.468810:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.468812:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.468813:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.468814:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.468817:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000800:00000200:4.2:1619677447.468817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.468819:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.468822:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.468823:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.468823:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.468825:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.468825:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.468827:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.468828:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677447.468829:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.468840:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.468841:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677447.468841:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.468842:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000400:00000200:5.0:1619677447.468842:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.468843:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.468844:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.468844:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:20.0:1619677447.468846:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.468846:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677447.468863:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.468866:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.468867:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.468869:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.468871:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.468873:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.468878:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677447.468913:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.468916:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.468916:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.468918:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.468919:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.468920:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.468944:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.468946:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.468947:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.468948:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.468948:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.468949:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.469055:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.469058:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.469059:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.469061:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.469063:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.469065:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.469066:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.469067:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.469069:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.469070:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.469073:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.469075:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.469078:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.469078:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.469079:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.469081:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.469081:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.469083:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.469084:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.469084:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.469095:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:5.0:1619677447.469096:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677447.469098:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:5.0:1619677447.469098:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677447.469099:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.469099:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:5.0:1619677447.469099:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.469101:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.469101:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.469104:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677447.469165:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.469168:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.469168:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.469170:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.469172:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.469174:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677447.469188:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.469191:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.469193:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.469194:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.469196:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.469197:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.469199:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.469200:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.469201:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.469202:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.469205:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.469206:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.469209:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.469210:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.469210:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.469212:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.469213:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.469214:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.469214:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.469218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.469226:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.469229:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.469230:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.469230:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.469232:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.469235:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.469358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.469366:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.469369:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.469370:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.469370:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.469372:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.469376:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:18.0:1619677447.469391:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.469394:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.469395:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.469397:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:13.0:1619677447.469398:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:18.0:1619677447.469399:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.469400:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.469401:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727775] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.469401:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.469401:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.469403:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.469404:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677447.469404:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000400:00000200:18.0:1619677447.469405:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:12.0:1619677447.469405:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.469407:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:18.0:1619677447.469408:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:4.2:1619677447.469408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.469410:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.469412:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.469413:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.469413:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.469416:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.469416:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.469418:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.469419:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.469419:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:9.2:1619677447.469421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.469429:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.469431:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.469431:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.469432:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.469432:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.469432:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.469433:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.469434:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:2.0:1619677447.469434:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.469436:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.469458:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.469461:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.469461:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.469462:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.469464:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.469467:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.469473:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677447.469474:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.469476:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677447.469476:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:12.0:1619677447.469477:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.469479:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.469480:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.469480:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.469482:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:1.0:1619677447.469482:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.469484:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.469485:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.469486:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.469487:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.469488:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.469492:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.469494:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.469497:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.469498:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.469499:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.469500:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.469501:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.469501:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.469502:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677447.469594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.469601:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.469604:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.469605:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.469607:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.469609:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.469611:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.469612:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.469613:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.469614:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.469615:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.469619:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.469621:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.469623:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.469624:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.469625:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.469626:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.469627:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.469628:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.469628:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.469644:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.469665:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.469668:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.469670:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.469671:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.469673:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.469675:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.469676:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.469677:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.469678:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:5.0:1619677447.469678:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.469679:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.469680:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.469683:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.469684:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.469685:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.469686:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.469687:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677447.469688:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.469690:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.469691:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.469692:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677447.469695:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.469695:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.469696:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.469697:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.469697:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.469699:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.469699:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.469700:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.469701:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677447.469701:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.469701:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.469702:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.469703:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.469703:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.469704:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.469704:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.469705:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.469705:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:13.0:1619677447.469711:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.469713:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677447.469714:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.469715:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.469715:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.469716:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677447.469716:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:13.0:1619677447.469717:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.469717:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.469718:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:4.2:1619677447.469718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.469718:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.469719:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.469720:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677447.469721:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677447.469724:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.469741:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:4.2:1619677447.469760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.469765:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.469767:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.469768:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.469769:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.469771:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.469773:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.469800:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.469803:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.469804:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.469804:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.469806:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.469809:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.469869:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677447.469872:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677447.469873:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677447.469874:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677447.469877:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.469878:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677447.469879:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.469881:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.469882:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.469883:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677447.469886:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677447.469888:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.469889:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.469890:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677447.469890:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.469891:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.469892:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677447.469893:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677447.469893:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677447.469897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.469903:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.469905:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.469905:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.469907:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677447.469907:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.469909:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.0:1619677447.469913:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.469919:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.469919:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677447.469921:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.469922:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:5.0:1619677447.469924:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.469925:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:5.0:1619677447.469926:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:2.0:1619677447.469926:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:5.0:1619677447.469927:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.469928:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.469929:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.469930:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.469930:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.469931:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677447.469932:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.469933:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.469935:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.469935:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.469936:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.469937:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.469938:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:18.0:1619677447.469941:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.469943:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.469943:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.469944:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.469944:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:18.0:1619677447.469945:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.469946:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677447.469946:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:18.0:1619677447.469947:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.469947:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:18.0:1619677447.469948:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.469948:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.469948:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:18.0:1619677447.469949:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.469950:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.469950:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677447.469961:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.469963:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.469963:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.469965:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.469966:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.469967:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.469967:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.469968:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:5.0:1619677447.469968:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.469970:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.469970:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.469973:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.470008:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.470010:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.470011:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.470013:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.470014:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.470016:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677447.470074:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.470076:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.470079:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.470081:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.470082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.470083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677447.470083:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.470084:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.470085:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.470086:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.470087:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:2.0:1619677447.470090:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:1.0:1619677447.470090:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.470092:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677447.470093:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.470094:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.470094:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.470095:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.470096:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:2.0:1619677447.470097:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.470097:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.470099:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.470099:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.470100:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.470101:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677447.470102:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.470124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677447.470132:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:5.0:1619677447.470133:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677447.470134:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677447.470135:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:5.0:1619677447.470136:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.470138:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:8.0:1619677447.470139:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.470139:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.470141:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.470142:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.470143:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677447.470144:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.470145:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.470145:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.470148:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.470148:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.470149:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.470150:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.470151:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677447.470151:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677447.470156:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.470157:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.470159:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.470160:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677447.470161:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.470162:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.470163:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.470163:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677447.470163:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.470164:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677447.470165:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.470165:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677447.470166:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:8.0:1619677447.470166:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:5.0:1619677447.470166:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.0:1619677447.470167:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:5.0:1619677447.470167:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.0:1619677447.470168:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:8.0:1619677447.470174:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:4.2:1619677447.470175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (21)++
00000800:00000200:20.0:1619677447.470180:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:24.0:1619677447.470182:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:20.0:1619677447.470182:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.470183:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.470184:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:24.0:1619677447.470185:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:20.0:1619677447.470185:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.470186:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:5.0:1619677447.470186:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.470187:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.470380:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.470383:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.470384:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.470386:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.470386:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.470389:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.470398:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.470401:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.470402:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.470402:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.470403:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.470404:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677447.470454:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:4.2:1619677447.470477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.470482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.470489:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:1.0:1619677447.470490:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677447.470492:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.470492:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677447.470492:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:8.0:1619677447.470493:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.470493:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677447.470495:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.470498:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.470749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.470756:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.470758:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677447.470760:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.470760:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.470763:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.470765:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.470766:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.470767:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.470768:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.470769:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.470772:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.470773:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.470776:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.470777:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.470778:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.470779:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.470780:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.470780:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.470781:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.470793:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677447.470795:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:1.0:1619677447.470796:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677447.470797:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:1.0:1619677447.470797:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.470800:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.470800:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:1.0:1619677447.470800:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.470802:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.470803:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:1.0:1619677447.470803:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:20.0:1619677447.470805:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.470806:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.470807:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.470808:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.470811:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.470813:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.470816:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.470816:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.470817:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:22.0:1619677447.470818:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677447.470818:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677447.470819:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.470819:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.470820:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:22.0:1619677447.470821:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:20.0:1619677447.470821:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:18.0:1619677447.470822:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:6.0:1619677447.470822:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677447.470822:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:22.0:1619677447.470825:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.470826:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.470827:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.470828:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677447.470828:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.470830:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.470830:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.470831:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.470831:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677447.470832:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.470833:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.470833:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:22.0:1619677447.470834:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.470835:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677447.470835:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.470835:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:18.0:1619677447.470836:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.470836:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.470838:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.470838:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.470841:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:22.0:1619677447.470842:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.470842:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.470845:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.470846:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677447.470848:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.470848:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.470849:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.470850:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.470850:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.470851:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677447.470851:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.470852:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:22.0:1619677447.470853:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677447.470853:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:22.0:1619677447.470854:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677447.470854:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:22.0:1619677447.470855:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677447.470855:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:22.0:1619677447.470856:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.470858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.470863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:2.0:1619677447.470866:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.470867:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.470868:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.470869:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:8.0:1619677447.470869:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:2.0:1619677447.470869:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.470870:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.470871:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.470871:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.470872:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.0:1619677447.470873:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:2.0:1619677447.470873:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677447.470874:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.470874:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.470876:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.470879:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.470880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.470887:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.470890:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.470891:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.470891:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.470892:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.470892:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.470904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.470942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.470950:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.470952:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.470953:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.470953:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.470955:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.470958:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.471038:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.0:1619677447.471041:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.471041:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.471042:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.0:1619677447.471044:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:18.0:1619677447.471044:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.471047:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.471047:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.471047:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:18.0:1619677447.471048:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.471049:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.471049:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.471050:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.471051:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.471052:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.471053:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.471054:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.471054:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.471055:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.471056:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.471058:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.471061:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.471062:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.471063:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.471064:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.471065:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.471065:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.471066:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.471067:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.471067:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.0:1619677447.471068:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.471068:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:20.0:1619677447.471069:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.471069:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.471069:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.0:1619677447.471070:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.471071:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.471072:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.471072:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.471072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.471080:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.471081:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.471082:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:2.0:1619677447.471082:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.471083:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.471084:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677447.471084:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:18.0:1619677447.471085:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.471085:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.471086:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677447.471086:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.471087:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.471089:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.471090:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.471091:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:2.0:1619677447.471091:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.471102:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:9.2:1619677447.471103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.471105:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.471106:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.471107:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.471107:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.471108:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677447.471110:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.471113:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.471114:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.471115:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.471116:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.471119:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.471216:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.471218:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.471221:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.471224:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.471225:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.471227:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.471228:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.471229:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.471230:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677447.471234:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.471235:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.471238:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.471239:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.471240:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.471241:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.471242:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.471243:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.471243:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.471247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.471254:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.471257:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.471257:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.471258:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.471260:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.471263:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.0:1619677447.471553:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.471555:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677447.471557:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677447.471558:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.471560:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.471561:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677447.471562:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.471564:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.471565:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.471566:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677447.471570:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.471571:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.471574:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.471575:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677447.471575:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677447.471578:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677447.471578:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.471579:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.471580:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:6.0:1619677447.471616:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.471619:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.471620:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.471622:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.471625:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.471626:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.471628:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.471629:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.471630:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.471631:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.471635:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.471637:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.471640:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.471640:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.471641:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.471642:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.471643:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.471644:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.471644:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.471645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.471661:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:4.2:1619677447.471687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677447.471694:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.471697:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.471701:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.471704:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.471706:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.471707:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.471709:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.471710:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.471711:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.471717:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.471720:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.471724:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.471725:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.471725:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.471727:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.471727:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.471728:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.471729:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.471740:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.471743:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.471744:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.471745:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.471748:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.471748:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:13.0:1619677447.471749:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677447.471750:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:13.0:1619677447.471752:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677447.471753:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:13.0:1619677447.471753:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:20.0:1619677447.471754:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.471754:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.471755:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:20.0:1619677447.471756:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.471756:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.471757:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.471759:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.471763:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.471766:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.471767:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:1.0:1619677447.471767:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.471768:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:5.0:1619677447.471769:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:1.0:1619677447.471769:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.0:1619677447.471772:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.471773:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.471776:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.471777:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.471779:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.471780:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:5.0:1619677447.471780:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.471781:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.471783:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:5.0:1619677447.471783:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:24.0:1619677447.471784:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.471786:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.471787:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.471789:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.471789:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:8.0:1619677447.471791:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.471792:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96727733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.471792:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.471792:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000400:00000200:8.0:1619677447.471793:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.471793:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:8.0:1619677447.471794:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.471795:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.471795:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.471795:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.471796:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.471796:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:8.0:1619677447.471804:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.471805:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.471805:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677447.471808:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:2.0:1619677447.471808:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.471809:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.0:1619677447.471809:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.471809:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.471810:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:8.0:1619677447.471810:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677447.471811:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.471811:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.471812:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:20.0:1619677447.471813:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.471813:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.471813:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:2.0:1619677447.471813:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.471814:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677447.471814:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:2.0:1619677447.471814:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:20.0:1619677447.471815:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:2.0:1619677447.471816:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.471817:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677447.471818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677447.471818:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.471819:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.471823:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.471825:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.471827:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677447.471827:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:18.0:1619677447.471829:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.471829:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.471830:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.471830:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:2.0:1619677447.471830:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (23)++
00000800:00000200:2.0:1619677447.471830:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000800:00000200:5.0:1619677447.471831:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000400:00000200:18.0:1619677447.471832:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.471832:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677447.471832:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:2.0:1619677447.471832:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (24)--
00000800:00000200:20.0:1619677447.471833:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:2.0:1619677447.471834:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.471834:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.471835:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677447.471836:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.471836:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.471839:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.471840:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:2.0:1619677447.471840:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (23)++
00000800:00000200:18.0:1619677447.471841:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000400:00000200:18.0:1619677447.471841:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.471841:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (25)--
00000800:00000200:2.0:1619677447.471841:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (24)--
00000800:00000200:2.0:1619677447.471842:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.471843:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.471844:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677447.471965:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.471967:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.471969:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.471970:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.471972:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.471973:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677447.471975:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.471976:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.471977:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.471978:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677447.471981:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.471983:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.471985:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.471986:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.471987:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.471989:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.471989:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.471990:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.471990:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677447.472002:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.472004:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.472004:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.472006:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.472007:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.472008:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.472112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:2.0:1619677447.472118:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:24.0:1619677447.472119:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677447.472121:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:24.0:1619677447.472122:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:18.0:1619677447.472122:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:8.0:1619677447.472123:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677447.472125:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.472127:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.472128:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.472129:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:24.0:1619677447.472130:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.472131:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.472132:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.472133:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.472133:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.472134:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.472135:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.472135:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:24.0:1619677447.472136:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.472137:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:2.0:1619677447.472140:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.472142:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.472142:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:24.0:1619677447.472144:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:2.0:1619677447.472145:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.472146:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:24.0:1619677447.472147:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.472147:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:24.0:1619677447.472148:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.472149:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677447.472149:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:24.0:1619677447.472150:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:2.0:1619677447.472150:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:24.0:1619677447.472151:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677447.472151:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.472151:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:24.0:1619677447.472152:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.472153:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.472167:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:4.2:1619677447.472269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.472369:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.472372:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.472372:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.472375:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.472377:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.472379:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.472380:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.472381:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.472382:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.472383:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.472387:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.472388:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.472391:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.472392:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.472393:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.472394:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.472395:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.472396:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.472396:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677447.472410:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.472413:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.472416:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.472419:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.472421:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.472422:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.472423:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.472424:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.472425:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.472428:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.472430:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.472432:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.472433:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.472433:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.472434:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.472435:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.472437:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.472438:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.472438:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677447.472444:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.472447:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.472448:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.472449:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:1.0:1619677447.472451:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:18.0:1619677447.472452:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.472452:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:18.0:1619677447.472453:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:1.0:1619677447.472453:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.472454:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:18.0:1619677447.472455:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.472455:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:18.0:1619677447.472456:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.472457:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.472458:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.472461:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.472462:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.472464:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.472465:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.472465:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.472467:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.472467:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.472468:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.472469:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.472473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.472480:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.472482:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.472482:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.472483:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.472484:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.472486:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677447.472542:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.472544:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.472545:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.472546:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.472549:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.472550:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.472593:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.472596:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.472598:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.472599:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.472602:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.472604:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.472605:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.472606:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.472607:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.472608:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.472611:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.472613:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.472615:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.472616:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.472617:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.472618:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.472619:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.472619:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.472620:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677447.472632:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.472635:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.472636:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.472638:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.472638:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.472641:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.472825:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.472827:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.472829:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.472830:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.472832:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.472833:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.472834:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727722] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.472836:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.472836:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.472838:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.472841:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.472843:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.472845:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.472846:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.472846:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.472848:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.472849:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.472850:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.472851:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677447.472852:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.472863:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.472864:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677447.472864:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.472865:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000400:00000200:5.0:1619677447.472865:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.472866:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.472867:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.472867:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.472868:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.472868:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:4.2:1619677447.472921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.472928:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.472931:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.472933:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.472934:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.472937:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.472939:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.472940:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.472942:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.472942:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.472943:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.472947:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.472948:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.472951:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.472952:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.472952:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.472954:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.472954:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.472956:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.472958:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.472958:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.472964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.472968:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677447.472971:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.472971:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.472972:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.472973:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.472974:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677447.472974:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.472975:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.472977:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.473060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.473199:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.473202:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.473203:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.473205:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.473207:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.473209:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.473211:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.473212:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.473213:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.473215:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.473219:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.473221:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.473224:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.473225:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.473225:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.473227:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.473227:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.473228:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.473229:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:1.0:1619677447.473275:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:2.0:1619677447.473361:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.473364:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.473365:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.473367:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.473370:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.473372:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.473373:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.473375:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.473376:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.473377:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.473380:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.473382:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.473384:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.473385:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.473386:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.473387:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.473388:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.473388:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.473389:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677447.473403:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.473405:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.473406:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.473408:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.473410:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.473412:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.473413:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.473414:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.473415:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.473415:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.473416:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.473418:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677447.473438:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.473441:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.473441:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.473444:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.473445:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:6.0:1619677447.473447:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.473448:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:6.0:1619677447.473448:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:12.0:1619677447.473449:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.473449:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677447.473451:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.473451:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.473452:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:6.0:1619677447.473453:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.473453:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:12.0:1619677447.473454:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:6.0:1619677447.473457:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.473459:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.473462:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.473462:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.473463:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.473465:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.473466:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.473467:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.473468:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.473469:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.473478:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.473480:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:20.0:1619677447.473480:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:22.0:1619677447.473481:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.473482:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.473482:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.473483:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.473485:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.473485:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.473487:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.473635:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:5.0:1619677447.473638:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:20.0:1619677447.473640:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:5.0:1619677447.473640:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.473642:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.473643:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677447.473644:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.473646:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.473647:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.473648:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677447.473651:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.473653:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.473655:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.473656:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.473656:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.473658:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.473658:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:5.0:1619677447.473659:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:5.0:1619677447.473660:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:20.0:1619677447.473671:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.473673:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.473673:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.473675:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.473676:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.473677:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.473690:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.473693:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.473694:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.473694:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.473695:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.473696:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.473699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.473706:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.473709:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.473710:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.473712:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.473715:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.473717:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.473718:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.473719:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.473720:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.473721:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.473724:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.473726:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.473729:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.473730:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.473730:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.473732:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.473732:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.473733:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.473734:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.473745:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.473748:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.473749:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.473751:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.473754:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:13.0:1619677447.473763:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.473766:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677447.473766:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:13.0:1619677447.473767:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.473769:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.473770:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.473771:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.473781:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.473784:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.473786:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.473786:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.473788:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.473789:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.473791:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.473792:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.473793:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.473794:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.473797:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.473799:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.473801:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.473802:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.473802:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.473804:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.473804:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.473806:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.473807:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.473807:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.473815:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.473817:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:6.0:1619677447.473817:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.473818:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.473818:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.473820:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.473821:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.473821:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.473823:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.473825:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.473836:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.473838:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:22.0:1619677447.473839:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.473840:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677447.473840:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:6.0:1619677447.473840:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.473841:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.473841:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:22.0:1619677447.473842:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:18.0:1619677447.473844:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.473846:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.473847:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.473849:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.473850:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.473851:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.473852:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.473855:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.473857:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.473860:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.473861:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.473862:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.473864:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.473864:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.473865:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.473866:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677447.473876:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.473879:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.473880:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.473881:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.473883:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.473884:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.473895:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.473898:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.473900:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.473901:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.473902:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.473904:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.473905:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.473906:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.473907:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.473908:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.473912:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.473914:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.473916:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.473916:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.473917:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.473919:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.473919:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.473920:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.473921:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.473932:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.473935:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.473935:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.473937:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.473938:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.473939:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:4.2:1619677447.473964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.473989:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.473991:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.473992:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.473994:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.473995:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.473997:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.474008:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.474010:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.474011:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.474013:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.474013:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.474016:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677447.474124:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.474126:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.474127:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.474129:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.474131:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.474132:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.474146:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.474148:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.474149:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.474150:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.474150:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.474151:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.474180:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.474182:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.474184:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.474185:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.474187:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.474188:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677447.474190:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.474191:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.474192:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.474193:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677447.474197:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.474198:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.474198:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677447.474201:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.474201:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.0:1619677447.474202:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677447.474202:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.474204:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.474204:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.474205:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.474205:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.474206:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:1.0:1619677447.474206:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.474208:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.474209:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.474210:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.474211:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.474213:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.474217:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.474219:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.474223:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.474224:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.474225:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.474227:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.474228:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.474229:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.474229:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.474239:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.474242:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.474243:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.474245:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.474246:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.474248:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.474256:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.474259:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.474260:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.474260:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.474262:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.474263:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.474271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.474278:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.474281:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.474282:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.474283:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.474287:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.474288:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.474289:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.474291:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.474292:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.474293:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.474301:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.474303:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.474306:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.474307:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.474307:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.474309:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.474310:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.474310:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.474311:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.474326:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:4.2:1619677447.474447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.474486:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.474489:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:4.2:1619677447.474489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677447.474491:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.474493:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677447.474494:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:5.0:1619677447.474494:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.474495:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:6.0:1619677447.474497:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:5.0:1619677447.474497:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.474498:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.474498:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.474498:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.474499:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.474500:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:6.0:1619677447.474501:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.474504:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:5.0:1619677447.474504:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.474506:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.474509:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.474510:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.474510:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.474512:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.474513:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.474513:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.474514:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.474525:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.474528:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.474529:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.474530:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.474532:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.474533:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.474659:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.474662:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.474666:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.474668:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.474670:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.474671:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.474673:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.474674:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.474675:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.474678:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.474680:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.474683:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.474684:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.474685:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.474687:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.474687:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.474688:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.474688:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.474720:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.474723:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.474724:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.474726:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.474728:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.474730:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.474731:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.474732:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.474733:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.474734:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.474738:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.474740:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.474742:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.474743:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.474744:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.474745:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.474746:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.474747:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.474747:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.474760:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.474762:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.474763:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.474765:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.474766:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.474767:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677447.474776:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.474779:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.474780:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.474782:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.474785:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.474786:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.474787:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.474789:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.474790:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.474791:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.474794:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.474795:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.474798:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.474799:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.474799:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.474801:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.474801:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.474802:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.474802:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.474815:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.474817:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.474818:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.474820:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.474820:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.474822:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.474835:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.474838:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.474839:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.474840:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.474841:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.474841:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677447.474865:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.474868:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.474869:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.474871:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.474873:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.474875:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.474876:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.474877:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:18.0:1619677447.474878:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.474878:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.474879:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:18.0:1619677447.474880:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.474882:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:4.2:1619677447.474883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:18.0:1619677447.474884:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.474885:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.474886:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.474888:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.474889:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.474890:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:18.0:1619677447.474890:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.474890:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677447.474891:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.474892:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.474893:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.474893:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.474893:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.474894:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000400:00000200:20.0:1619677447.474895:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.474895:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.474896:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:20.0:1619677447.474899:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.474901:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.474904:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.474904:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.474905:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:12.0:1619677447.474905:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677447.474907:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.474907:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677447.474907:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677447.474907:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:20.0:1619677447.474908:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.474908:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:12.0:1619677447.474908:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.474908:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.474909:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:12.0:1619677447.474910:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.474910:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677447.474912:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.474913:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.474920:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.474923:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.474924:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.474924:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.474926:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.474928:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.474990:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.474992:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.474995:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.474995:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.474997:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.474998:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.475000:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.475001:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.475002:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.475003:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.475006:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.475008:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.475010:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.475011:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.475011:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.475013:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.475013:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.475014:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.475015:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677447.475026:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.475028:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.475029:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.475031:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.475032:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.475033:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:4.2:1619677447.475061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.475069:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.475072:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.475073:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.475075:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.475077:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.475079:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.475080:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.475081:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.475082:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.475083:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.475086:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.475088:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.475090:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.475091:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.475092:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.475094:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.475094:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.475095:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.475095:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.475110:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.475131:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.475134:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.475136:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.475139:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.475140:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.475142:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.475143:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677447.475143:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.475144:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.475145:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:1.0:1619677447.475145:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:13.0:1619677447.475146:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.475148:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.475149:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:1.0:1619677447.475149:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.475152:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.475153:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.475155:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.475156:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.475156:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.475158:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.475159:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.475160:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.475161:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.475162:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677447.475170:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.475173:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.475174:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.475175:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677447.475176:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.475177:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.475178:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.475194:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.475196:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.475197:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.475198:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.475198:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.475199:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.475236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.475245:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.475247:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.475248:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.475249:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.475251:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.475256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.475260:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.475261:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.475263:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.475264:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.475265:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.475265:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.475266:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.475269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.475276:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.475279:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.475280:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.475280:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.475282:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.475285:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.475305:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.475307:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:4.2:1619677447.475307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.475308:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.475310:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.475311:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.475313:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.475314:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.475315:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.475317:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.475318:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.475322:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.475324:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.475326:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.475327:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.475328:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.475330:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.475330:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.475331:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.475331:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.475344:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.475347:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.475348:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.475348:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.475350:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.475350:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.475351:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.475352:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.475353:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:6.0:1619677447.475354:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.475355:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.475357:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.475376:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.475379:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.475379:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.475380:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.475381:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.475381:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677447.475453:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.475456:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.475458:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.475459:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.475460:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.475462:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.475463:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.475465:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.475466:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.475467:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.475470:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.475472:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.475475:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.475476:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.475476:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.475478:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.475479:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.475480:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.475480:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.475492:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.475495:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.475495:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.475497:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.475499:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.475500:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.475545:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.475581:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.475583:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.475584:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.475584:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.475586:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.475589:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.475741:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:4.2:1619677447.475743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.475744:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.475745:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.475746:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.475748:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.475749:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.475764:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.475767:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.475768:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.475770:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.475773:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.475774:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.475776:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.475777:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.475778:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.475779:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.475782:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.475784:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.475786:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.475787:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.475787:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.475789:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.475789:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.475790:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.475791:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.475792:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.475795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.475803:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.475804:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.475804:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:18.0:1619677447.475805:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.475805:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.475806:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.475807:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.475807:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.475808:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677447.475809:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.475815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.475822:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.475825:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.475826:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.475826:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.475826:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.475827:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.475865:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.475867:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.475868:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.475870:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:1.0:1619677447.475874:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.0:1619677447.475942:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.475945:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.475947:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.475948:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.475950:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.475951:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.475953:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.475954:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.475955:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.475956:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.475960:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.475962:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.475965:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.475965:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.475966:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.475968:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.475968:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.475970:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.475971:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.475972:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.475982:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.475984:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.475985:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.475986:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677447.476054:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.476057:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677447.476059:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.476060:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.476062:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.476063:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.476065:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.476066:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.476067:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.476068:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.476071:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.476073:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.476075:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.476076:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.476076:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.476078:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.476078:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.476079:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.476080:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677447.476091:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.476093:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.476094:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.476095:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.476097:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.476099:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:22.0:1619677447.476103:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.476106:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.476107:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.476108:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.476111:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.476111:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000400:00000200:22.0:1619677447.476112:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.476114:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.476114:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677447.476115:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.476115:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000400:00000200:22.0:1619677447.476116:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.476116:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000400:00000200:22.0:1619677447.476117:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.476117:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.0:1619677447.476117:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:22.0:1619677447.476121:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.476123:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.476126:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.476127:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.476128:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.476130:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.476130:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.476131:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.476131:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.476198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.476206:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.476209:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.476210:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.476210:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.476211:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:18.0:1619677447.476212:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.476214:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:18.0:1619677447.476215:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.476215:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:5.0:1619677447.476215:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.476217:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.476217:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.476218:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.476219:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.476219:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.476221:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:6.0:1619677447.476222:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.476225:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.476234:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.476236:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.476237:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.476237:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.476238:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.476239:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.476277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.476325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.476329:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.476332:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.476333:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.476334:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.476336:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.476339:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.476510:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.476538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.476545:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.476548:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.476549:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.476549:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.476551:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.476554:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.476555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677447.476562:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.476564:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.476565:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.476565:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.476566:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.476567:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677447.476620:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.476623:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.476624:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.476624:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.476626:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.476629:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.476689:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.476692:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.476694:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.476696:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.476697:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.476698:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677447.476698:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:20.0:1619677447.476699:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727667] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.476700:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.476701:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.476703:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.476703:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.476704:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.476705:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.476708:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.476708:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.476709:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.476710:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.476710:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.476711:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.476713:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.476714:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:5.0:1619677447.476714:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.476715:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677447.476716:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.476717:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677447.476717:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.476719:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.476720:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.476721:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.476722:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.476723:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.476724:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.476724:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677447.476725:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.476726:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.476728:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.476730:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.476731:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.476731:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.476732:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:13.0:1619677447.476733:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.476734:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.476734:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.476734:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.476735:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000400:00000200:13.0:1619677447.476736:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.476736:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.476737:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:12.0:1619677447.476738:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.476739:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.476745:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.476747:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.476748:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.476749:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:4.2:1619677447.476781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.476815:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.476818:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.476819:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677447.476820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677447.476821:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.476823:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.476826:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.476827:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.476828:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:2.0:1619677447.476829:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.476830:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.476830:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.476831:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.476831:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.476832:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.476835:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.476837:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.476838:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.476839:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.476840:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.476840:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.476841:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.476842:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.476843:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.476844:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.476844:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.476848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.476855:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.476857:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.476857:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.476858:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.476859:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.476860:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.476981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.476988:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677447.476988:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.476990:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:24.0:1619677447.476992:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677447.476992:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.0:1619677447.476992:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:2.0:1619677447.476994:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.476995:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.476995:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.476997:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.476998:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.476998:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.477000:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.477000:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.477001:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.477002:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.477002:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.477003:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.477005:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.477006:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.477007:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.477008:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.477010:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.477011:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.477012:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.477012:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.477013:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.477013:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.477015:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:2.0:1619677447.477015:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.477015:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.477015:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:22.0:1619677447.477016:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677447.477016:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.477016:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.0:1619677447.477017:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.477017:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:6.0:1619677447.477018:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.477019:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.477019:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.477021:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:6.0:1619677447.477021:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.477022:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.477022:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677447.477022:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.477024:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.477025:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.477026:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.477027:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677447.477027:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:4.2:1619677447.477028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:12.0:1619677447.477029:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.477030:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.477031:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677447.477032:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.477033:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677447.477033:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677447.477033:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.477035:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:12.0:1619677447.477035:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.477036:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.477036:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677447.477037:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:6.0:1619677447.477037:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.477037:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:1.0:1619677447.477037:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.477038:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.477039:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.477039:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000400:00000200:1.0:1619677447.477039:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.477040:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.477040:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.477041:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677447.477043:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:13.0:1619677447.477046:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.477048:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.477049:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.477050:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.477051:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.477052:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.477055:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:4.2:1619677447.477349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.477360:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.477363:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.477365:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.477365:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677447.477366:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.477368:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.477368:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.477369:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:20.0:1619677447.477370:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.477372:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.477372:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.477374:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.477375:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.477376:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677447.477376:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.477377:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677447.477379:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.477381:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.477382:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.477383:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.477383:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677447.477385:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 568 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.477388:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.477388:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.477389:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.477390:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:18.0:1619677447.477390:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677447.477391:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.477392:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.477393:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:18.0:1619677447.477393:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.477394:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:20.0:1619677447.477395:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:18.0:1619677447.477395:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677447.477396:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:18.0:1619677447.477397:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.477397:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.477398:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.477399:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677447.477407:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.477409:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677447.477409:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.477410:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000400:00000200:5.0:1619677447.477410:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.477411:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:12.0:1619677447.477411:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.0:1619677447.477412:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.477413:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:12.0:1619677447.477413:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:5.0:1619677447.477413:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:12.0:1619677447.477414:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.477415:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:12.0:1619677447.477416:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.477418:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.477419:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.477494:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.477496:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.477497:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.477498:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.477500:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.477502:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677447.477540:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.477543:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.477545:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.477548:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.477549:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.477551:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.477552:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.477553:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.477554:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.477557:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.477559:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.477560:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.477561:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.477562:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.477564:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.477564:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.477565:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.477565:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.477580:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:2.0:1619677447.477858:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.477860:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.477861:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.477863:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.477866:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.477868:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.477869:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.477870:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.477871:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.477872:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.477876:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.477878:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.477880:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.477881:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.477882:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.477884:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.477885:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.477885:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.477886:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.477887:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.477887:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677447.477888:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.477889:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.477891:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.477893:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677447.477893:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.477895:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.477895:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.477896:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:22.0:1619677447.477898:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.477900:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:18.0:1619677447.477900:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.477901:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.477901:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:6.0:1619677447.477901:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.477903:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.477904:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.477905:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:18.0:1619677447.477905:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677447.477906:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677447.477907:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.477908:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.477908:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.477909:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677447.477919:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.477922:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.477923:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.477925:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.477925:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.477928:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.478043:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.478046:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.478047:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.478049:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.478052:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.478053:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.478054:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.478056:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.478057:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.478058:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.478061:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.478063:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.478065:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.478066:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.478067:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.478069:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.478069:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.478070:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.478071:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.478072:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.478082:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:1.0:1619677447.478082:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677447.478085:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.478085:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677447.478085:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.478086:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.478087:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:1.0:1619677447.478089:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.478091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:1.0:1619677447.478091:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.0:1619677447.478098:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.478101:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.478102:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.478104:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.478107:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.478108:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.478110:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.478111:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.478112:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.478113:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.478117:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.478119:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.478122:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.478123:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.478123:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.478125:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.478125:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.478126:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.478126:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.478142:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677447.478161:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.478164:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.478166:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.478167:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.478168:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.478170:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677447.478171:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.478172:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.478174:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.478175:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677447.478179:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.478180:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677447.478183:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.478183:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677447.478184:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.478186:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.478186:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.478187:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.478188:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.478192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.478199:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.478202:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.478203:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.478203:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.478206:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.478208:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677447.478213:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.478215:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.478218:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.478218:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.478220:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.478222:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.478223:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.478225:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.478226:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.478228:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.478232:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.478234:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.478235:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677447.478237:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.478238:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.478238:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:5.0:1619677447.478239:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:1.0:1619677447.478239:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.478241:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.478241:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:1.0:1619677447.478241:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.478242:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.478243:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:1.0:1619677447.478244:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:13.0:1619677447.478250:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.478253:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.478254:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.478255:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.478256:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.478258:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.478269:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.478269:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677447.478272:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.478273:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.478273:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.478274:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.478276:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677447.478305:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.478305:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.478306:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.478307:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.478308:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.478332:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.478338:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.478341:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.478343:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.478344:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.478347:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.478348:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.478350:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.478352:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.478353:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.478354:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.478358:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.478361:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.478363:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.478364:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.478365:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.478366:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.478367:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.478368:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.478369:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:18.0:1619677447.478375:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677447.478377:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677447.478378:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677447.478380:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.478383:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.478384:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:5.0:1619677447.478384:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:18.0:1619677447.478385:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.478387:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.478387:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.478387:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.478388:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.478389:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:20.0:1619677447.478390:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.478390:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.478393:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.478394:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:18.0:1619677447.478395:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:18.0:1619677447.478397:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.478398:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:18.0:1619677447.478399:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:18.0:1619677447.478400:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:18.0:1619677447.478401:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677447.478402:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:18.0:1619677447.478403:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:18.0:1619677447.478404:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:1.0:1619677447.478414:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:2.0:1619677447.478415:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:1.0:1619677447.478416:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:2.0:1619677447.478417:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677447.478417:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:1.0:1619677447.478417:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.478419:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000400:00000200:1.0:1619677447.478419:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.478422:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677447.478566:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.478569:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.478570:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.478572:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.478574:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.478576:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.478577:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.478578:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.478579:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.478580:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.478583:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.478585:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.478587:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.478588:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.478588:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.478590:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.478590:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.478591:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.478592:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.478592:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.0:1619677447.478603:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677447.478603:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:20.0:1619677447.478605:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:2.0:1619677447.478605:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:20.0:1619677447.478606:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.478606:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.478607:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:20.0:1619677447.478608:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.478610:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.478610:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:4.2:1619677447.478764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.478804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.478811:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677447.478811:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:24.0:1619677447.478814:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:6.0:1619677447.478814:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.478815:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.478815:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677447.478815:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.478816:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.478817:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.478820:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.478821:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.478823:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.478823:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:24.0:1619677447.478824:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.478825:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.478825:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:24.0:1619677447.478826:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.478827:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.478828:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.478829:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.478830:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000400:00000200:24.0:1619677447.478831:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.478831:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.478833:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.478836:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.478837:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.478837:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.478839:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.478840:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.478840:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.478841:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.478856:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (24)++
00000800:00000200:1.0:1619677447.478857:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000400:00000200:1.0:1619677447.478857:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.478859:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.478863:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.478885:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.478888:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.478889:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677447.478889:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677447.478891:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.478892:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:18.0:1619677447.478893:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.478895:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.478895:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.478897:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.478897:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.478898:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.478898:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.478900:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.478900:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.478901:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.478902:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.478903:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.478904:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.478907:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.478909:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.478912:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.478914:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.478914:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.478915:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.478916:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.478916:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.478917:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.478918:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677447.478919:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (23)++
00000800:00000200:1.0:1619677447.478919:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.478920:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.478920:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677447.478921:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677447.478921:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.478922:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.478923:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677447.478934:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (24)++
00000800:00000200:1.0:1619677447.478934:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000400:00000200:1.0:1619677447.478935:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.478937:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.478939:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.478952:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.478954:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.478955:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.478956:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.478957:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677447.479073:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.479076:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.479076:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.479079:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.479081:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.479082:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.479084:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.479085:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.479086:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.479087:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.479091:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.479092:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.479095:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.479096:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.479096:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.479098:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.479099:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.479099:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.479100:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.479112:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.479115:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.479115:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.479117:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.479118:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.479119:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.479187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.479229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.479234:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.479237:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.479238:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.479238:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.479241:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.479244:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.479331:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.479333:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.479336:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.479336:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.479338:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.479339:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677447.479341:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.479341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:20.0:1619677447.479343:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.479344:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.479345:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:5.0:1619677447.479346:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:8.0:1619677447.479348:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.0:1619677447.479348:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.479349:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.479350:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.479351:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677447.479351:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:5.0:1619677447.479352:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.479353:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677447.479354:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677447.479354:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:20.0:1619677447.479355:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677447.479356:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:8.0:1619677447.479356:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.479356:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.479357:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000400:00000200:5.0:1619677447.479357:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.479359:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.479359:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.479359:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:8.0:1619677447.479361:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.479363:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96727633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.479363:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677447.479364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:8.0:1619677447.479364:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.479365:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.479366:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:8.0:1619677447.479366:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.479366:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.479367:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:8.0:1619677447.479367:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:20.0:1619677447.479368:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.479368:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.479369:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.479370:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:8.0:1619677447.479371:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.479372:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.479372:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:2.0:1619677447.479372:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:8.0:1619677447.479373:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.479373:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.479374:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:2.0:1619677447.479375:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.479376:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:8.0:1619677447.479376:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677447.479376:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.479377:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:2.0:1619677447.479378:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.479380:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:8.0:1619677447.479382:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.479384:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.479384:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.479385:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677447.479386:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:8.0:1619677447.479386:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677447.479388:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.479389:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.479391:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.479392:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.479393:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:1.0:1619677447.479401:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.479414:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.479417:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.479417:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.479418:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.479419:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.479461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.479508:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.479511:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.479512:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.479513:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.479515:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.479517:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.479534:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.479560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.479567:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.479570:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.479571:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.479571:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.479573:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.479576:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677447.479610:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.479613:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.479613:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.479615:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.479617:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.479618:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.479641:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.479643:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.479644:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.479646:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.479648:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.479650:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.479651:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.479653:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.479654:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.479655:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.479658:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.479660:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.479663:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.479664:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.479665:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.479666:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.479667:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.479668:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.479668:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677447.479679:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.479681:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.479682:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.479684:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.479685:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.479687:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.479807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.479855:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.479858:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.479859:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.479860:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.479861:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.479863:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.479874:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.479877:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.479877:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.479878:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.479879:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.479880:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:2.0:1619677447.479921:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.479924:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.479925:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.479927:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.479927:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677447.479930:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.479930:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:2.0:1619677447.479932:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.479933:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.479934:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.479935:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.479936:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.479937:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677447.479937:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.479939:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.479940:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.479941:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.479942:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.479943:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.479944:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:1.0:1619677447.479944:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:2.0:1619677447.479946:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.479947:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.479948:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:1.0:1619677447.479948:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.479950:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.479951:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.479951:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677447.479951:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677447.479952:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.479954:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677447.479954:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.479955:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.479955:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.479957:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677447.479957:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.479957:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.479958:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677447.479959:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.479959:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677447.479960:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.479961:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.479962:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:5.0:1619677447.479962:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:20.0:1619677447.479963:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.479964:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.479964:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.479965:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.479966:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.479967:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.479968:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.479970:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.479971:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:5.0:1619677447.479971:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.479973:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.479976:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.479977:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.479978:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677447.479980:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.479981:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.479982:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.479984:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.479984:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.479993:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.479995:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.479996:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.479996:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.480144:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.480147:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.480148:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.480150:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.480152:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.480153:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.480155:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.480156:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.480157:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.480158:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.480161:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.480163:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.480166:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.480167:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.480167:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.480169:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.480169:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.480170:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.480170:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.480179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.480183:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.480186:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.480187:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.480189:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.480189:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.480192:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.480202:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.480205:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.480206:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.480207:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.480207:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.480209:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677447.480226:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.480229:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.480230:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.480230:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.480233:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.480236:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.480337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.480345:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.480347:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.480348:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.480350:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.480353:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.480354:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.480355:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96727620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.480362:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:8.0:1619677447.480363:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.480364:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.480364:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:8.0:1619677447.480365:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.480367:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.480369:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.480370:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:8.0:1619677447.480370:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.480371:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.480371:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.480372:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.480373:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.480374:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.480375:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677447.480375:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.480376:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:5.0:1619677447.480376:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.480377:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677447.480377:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.480378:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:5.0:1619677447.480378:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:8.0:1619677447.480379:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.480380:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.480381:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.480382:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:5.0:1619677447.480382:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.480383:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.480385:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.480386:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.480387:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.480387:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.480387:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:4.2:1619677447.480387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677447.480388:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.480389:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.480389:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.480390:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:5.0:1619677447.480390:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.480391:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.480392:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:20.0:1619677447.480393:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.480393:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:24.0:1619677447.480395:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:20.0:1619677447.480395:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.480396:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.480397:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:20.0:1619677447.480397:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:24.0:1619677447.480398:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:20.0:1619677447.480398:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.480400:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:20.0:1619677447.480400:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.480401:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.480402:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.480404:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.480405:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.480406:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.480406:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.480408:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.480408:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.480410:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.480410:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.480411:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.480411:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677447.480412:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.480413:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.480414:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.480415:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.480415:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677447.480423:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.480426:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.480426:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.480427:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677447.480428:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:6.0:1619677447.480429:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677447.480429:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:6.0:1619677447.480431:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.480432:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.480433:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.480434:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.480436:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.480464:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677447.480467:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677447.480470:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.480472:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.480474:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677447.480475:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.480476:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.480477:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.480478:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677447.480482:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.480485:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677447.480487:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.480488:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:1.0:1619677447.480488:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677447.480490:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677447.480490:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677447.480491:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677447.480492:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:22.0:1619677447.480503:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.480506:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.480507:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.480507:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.480509:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.480511:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.480518:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.480521:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.480521:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.480522:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:12.0:1619677447.480523:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.480525:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677447.480525:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.480526:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:12.0:1619677447.480526:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.480526:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.480528:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.480529:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.480670:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.480672:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.480674:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.480675:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.480677:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.480678:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.480679:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.480681:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.480682:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.480683:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.480686:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.480688:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.480690:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.480691:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.480692:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.480694:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.480694:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.480695:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.480696:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.480700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.480709:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.480711:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.480713:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.480713:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.480715:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.480718:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.480719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.480727:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.480729:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.480730:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.480730:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.480731:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.480732:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.480860:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.480862:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.480863:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.480864:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.480867:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.480868:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.480869:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.480870:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.480871:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.480872:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.480875:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.480877:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.480880:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.480881:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.480881:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.480883:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.480884:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.480884:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.480885:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:4.2:1619677447.480917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.480921:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.480924:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.480925:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.480926:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.480929:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.480930:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677447.480931:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.480932:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.480934:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.480935:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677447.480939:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.480941:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.480943:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.480944:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.480944:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.480946:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.480946:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.480947:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.480948:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.480949:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677447.480952:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:9.2:1619677447.480952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.480955:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.480955:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.480957:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.480960:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.480960:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:22.0:1619677447.480961:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:18.0:1619677447.480961:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.480961:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:18.0:1619677447.480962:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.480962:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:22.0:1619677447.480963:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.480963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.480963:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.480964:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.480964:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:22.0:1619677447.480965:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.480965:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:22.0:1619677447.480966:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.480966:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:22.0:1619677447.480967:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.480969:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.480970:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.480972:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.480972:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.480973:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.480975:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.480977:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.480978:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.480979:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.480980:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.480981:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.480983:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.480984:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.480985:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.480985:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.481003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.481011:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.481013:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.481015:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.481015:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.481017:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.481021:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.481023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.481030:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.481033:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.481034:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.481034:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.481035:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.481035:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677447.481083:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.481086:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.481087:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.481088:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.481089:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.481091:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.481120:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.481122:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.481124:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.481126:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.481128:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677447.481128:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.481129:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.481130:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677447.481131:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.481133:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.481134:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.481138:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.481142:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.481145:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.481146:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.481146:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.481148:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.481148:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.481149:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.481150:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.481160:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.481163:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677447.481164:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.481166:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.481167:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677447.481168:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677447.481169:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.481169:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.481170:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:9.2:1619677447.481170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:12.0:1619677447.481171:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.481171:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677447.481173:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.481173:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.481174:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.481175:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677447.481177:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.481180:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.481181:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.481181:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.481182:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.481185:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677447.481187:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.481190:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.481190:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.481191:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:9.2:1619677447.481191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:13.0:1619677447.481192:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.481193:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.481197:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.481200:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.481201:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.481201:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.481202:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.481203:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:2.0:1619677447.481217:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.481220:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.481221:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.481224:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.481226:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.481228:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.481229:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.481231:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:2.0:1619677447.481231:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.481233:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.481234:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:2.0:1619677447.481234:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.481237:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.481237:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.481239:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.481240:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.481241:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.481242:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:5.0:1619677447.481244:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727602] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.481245:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.481246:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.481248:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:2.0:1619677447.481249:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.481249:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.481250:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677447.481252:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.481254:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:2.0:1619677447.481254:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.481254:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.481256:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:5.0:1619677447.481257:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.481257:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.481257:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677447.481258:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:5.0:1619677447.481258:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:9.2:1619677447.481259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677447.481260:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:5.0:1619677447.481265:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:9.2:1619677447.481266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.0:1619677447.481266:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.481267:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677447.481269:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:18.0:1619677447.481270:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000800:00000200:2.0:1619677447.481270:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.481270:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:18.0:1619677447.481271:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.481271:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:24.0:1619677447.481272:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:2.0:1619677447.481272:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000400:00000200:18.0:1619677447.481273:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.481273:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000400:00000200:2.0:1619677447.481274:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.481275:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:2.0:1619677447.481275:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.481276:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:8.0:1619677447.481276:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.481277:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.481278:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:24.0:1619677447.481279:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.481279:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.481279:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.481281:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.0:1619677447.481441:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.481444:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.481446:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.481448:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.481449:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.481450:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.481451:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.481453:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.481453:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.481453:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:20.0:1619677447.481455:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:6.0:1619677447.481456:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.481457:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.481457:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.481458:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.481459:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.481463:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.481465:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.481467:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.481468:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.481469:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.481470:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.481471:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.481472:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.481474:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677447.481474:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.481486:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.481487:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677447.481487:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.481488:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677447.481488:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.481488:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.481490:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.481490:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.481491:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.481492:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677447.481509:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.481511:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.481512:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.481514:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.481515:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.481517:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.481530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.481545:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.481582:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.481585:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.481586:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.481587:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.481588:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.481589:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.481590:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.481590:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.481592:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.481593:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.481593:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.481596:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.481780:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677447.481783:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677447.481784:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677447.481785:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.481788:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.481789:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.481790:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96727594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.481792:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.481792:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.481793:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.481796:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.481798:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.481800:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.481801:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.481802:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.481803:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.481804:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677447.481805:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed9d3400] (22)++
00000800:00000200:8.0:1619677447.481806:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (23)--
00000800:00000200:8.0:1619677447.481807:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:12.0:1619677447.481816:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.481818:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:24.0:1619677447.481819:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)++
00000400:00000200:12.0:1619677447.481819:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.481820:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.481821:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed9d3400] (23)--
00000800:00000200:24.0:1619677447.481822:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:13.0:1619677447.481823:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.481823:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.481824:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:22.0:1619677447.481834:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.481836:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.481837:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.481839:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.481842:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.481843:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.481845:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.481846:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.481847:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.481848:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.481851:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.481853:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.481856:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.481857:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.481858:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.481859:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.481860:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.481862:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.481863:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.481863:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.481874:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.481875:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.481876:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.481877:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.481934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.481943:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.481946:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.481947:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.481947:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.481950:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.481953:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.481961:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.481964:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.481964:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.481965:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.481966:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.481967:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677447.481973:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.481976:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.481978:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.481978:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.481979:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:5.0:1619677447.481980:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.481981:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:22.0:1619677447.481982:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.481983:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.481983:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:5.0:1619677447.481983:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.481984:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.481985:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.481985:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.481986:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.481989:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:5.0:1619677447.481990:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.481992:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.481995:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.481996:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.481996:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.481998:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.481999:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.482000:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.482001:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.482002:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677447.482003:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:9.2:1619677447.482004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.482006:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.482006:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.482007:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.482008:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.482008:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.482009:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.482012:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.482012:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677447.482012:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677447.482013:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677447.482015:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.482016:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.482016:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.482018:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.482021:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.482081:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.482083:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.482084:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.482086:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.482088:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.482089:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.482090:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.482092:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.482093:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.482094:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.482098:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.482099:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.482102:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.482103:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.482103:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.482105:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.482105:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.482106:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.482106:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.482110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.482118:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.482120:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:1.0:1619677447.482120:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677447.482121:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.482122:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.482123:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.482123:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:2.0:1619677447.482125:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:1.0:1619677447.482126:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677447.482127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:1.0:1619677447.482128:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.482130:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.482131:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727587] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.482132:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.482133:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:1.0:1619677447.482133:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.482134:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677447.482134:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.482135:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.482136:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.482137:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.482137:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.482138:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000400:00000200:20.0:1619677447.482139:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.482139:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.482139:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:1.0:1619677447.482139:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.482140:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.482142:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.482143:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.482145:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.482145:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.482146:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.482148:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.482149:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.482149:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.482150:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.482160:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.482163:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.482164:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.482166:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.482167:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.482169:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.482214:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.482217:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.482218:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.482220:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.482221:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.482223:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.482224:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.482226:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.482227:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.482228:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.482232:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.482234:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.482236:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.482237:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.482237:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.482239:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.482239:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.482240:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.482240:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.482245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.482252:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.482253:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.482254:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.482255:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.482256:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.482257:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677447.482415:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.482417:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.482420:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.482420:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.482422:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.482423:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.482425:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.482426:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.482427:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.482428:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.482431:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.482433:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.482435:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.482435:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.482436:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.482438:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.482438:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.482439:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.482440:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677447.482452:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.482454:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.482455:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.482457:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.482458:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.0:1619677447.482458:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:4.2:1619677447.482569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.482618:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.482621:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.482621:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.482622:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.482624:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.482627:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.482632:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.482635:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677447.482636:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677447.482638:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.482641:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.482642:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:6.0:1619677447.482644:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.482645:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.482646:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.482647:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:6.0:1619677447.482650:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.482652:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677447.482655:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.482655:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:6.0:1619677447.482656:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:6.0:1619677447.482659:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:6.0:1619677447.482659:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.482660:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:6.0:1619677447.482661:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:6.0:1619677447.482662:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677447.482672:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677447.482674:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677447.482674:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677447.482675:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:6.0:1619677447.482845:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.482848:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.482848:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.482849:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.482851:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.482854:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.483011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.483018:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.483021:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.483022:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.483024:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.483026:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.483028:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.483029:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.483030:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.483031:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.483032:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.483036:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.483037:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.483040:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.483041:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.483041:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.483043:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.483043:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.483043:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677447.483044:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.483045:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.483058:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.483059:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.483060:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.483062:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.483065:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.483067:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.483082:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.483099:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.483102:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.483103:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.483105:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.483108:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.483109:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.483110:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96727578] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.483112:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.483112:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.483113:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.483116:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.483118:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677447.483118:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.483120:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:24.0:1619677447.483121:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.483121:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:13.0:1619677447.483121:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.483122:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:13.0:1619677447.483123:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.483124:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.483124:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.483125:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.483125:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.483125:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.483127:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.483128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.483130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.483138:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.483141:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.483142:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.483142:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.483144:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.483147:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.483171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677447.483325:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.483328:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.483329:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.483330:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.483332:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.483333:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.483335:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.483336:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.483337:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.483338:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.483340:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.483342:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.483344:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.483344:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.483345:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.483347:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.483347:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.483348:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.483348:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.483353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.483361:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.483362:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.483363:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.483364:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.483365:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.483366:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677447.483471:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.483474:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.483476:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.483476:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.483478:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.483480:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677447.483482:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.483483:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.483484:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.483485:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677447.483488:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.483490:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677447.483493:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.483494:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677447.483494:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677447.483497:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677447.483498:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.483499:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.483500:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:5.0:1619677447.483501:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.483502:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.483502:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677447.483503:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.483505:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.483506:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.483507:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727573] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.483508:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.483509:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.483510:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.483512:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.483514:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.483516:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.483517:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.483517:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.483519:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.483520:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.483520:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.483521:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677447.483533:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.483536:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.483536:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.483538:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.483538:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:5.0:1619677447.483539:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:4.2:1619677447.483540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.483541:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:20.0:1619677447.483542:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:1.0:1619677447.483544:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.483546:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.483548:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.483549:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727572] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.483552:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.483553:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.483554:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.483559:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.483561:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.483569:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.483571:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.483575:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.483576:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.483577:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:17.0:1619677447.483578:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.483579:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.483580:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677447.483581:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.483581:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677447.483582:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:17.0:1619677447.483583:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.483583:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.483584:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.483585:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.483587:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.483588:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.483589:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.483590:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.483590:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677447.483592:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.483595:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.483596:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.483598:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.483600:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.483601:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.483601:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677447.483602:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677447.483603:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:13.0:1619677447.483604:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677447.483604:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.483605:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:13.0:1619677447.483605:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.483605:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677447.483607:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.483608:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:12.0:1619677447.483608:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677447.483609:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.483610:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.483610:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:22.0:1619677447.483612:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.483615:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677447.483740:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677447.483744:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677447.483744:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677447.483746:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.483748:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.483750:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.483751:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.483753:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.483754:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.483755:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.483758:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.483760:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.483763:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.483764:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.483765:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.483766:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.483767:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677447.483768:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677447.483769:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677447.483769:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677447.483780:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677447.483782:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677447.483783:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677447.483784:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:6.0:1619677447.483809:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.483812:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.483813:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.483815:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.483818:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.483819:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.483821:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.483822:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.483823:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.483824:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.483828:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.483829:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.483832:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.483833:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.483833:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.483835:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.483835:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.483836:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.483837:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.483841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.483849:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:4.2:1619677447.483850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.483851:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.483852:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.483852:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.483854:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.483857:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:22.0:1619677447.483895:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.483898:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.483899:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.483899:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.483902:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.483905:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.483922:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.483925:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.483926:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.483927:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.483929:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.483930:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.483943:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.483945:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.483946:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.483947:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.483947:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.483948:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.483955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.483963:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.483966:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.483967:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.483969:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.483971:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.483973:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.483974:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.483975:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.483976:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.483977:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.483980:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.483982:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.483985:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.483985:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.483986:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.483988:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.483988:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.483989:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.483989:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.484001:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.484004:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.484005:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.484005:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.484007:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.484010:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:24.0:1619677447.484067:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.484070:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.484071:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.484073:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.484076:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.484078:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.484079:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.484081:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.484082:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.484083:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.484087:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.484089:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677447.484090:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677447.484092:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.484092:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:1.0:1619677447.484092:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:24.0:1619677447.484093:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677447.484093:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677447.484095:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.484095:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:1.0:1619677447.484095:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.484096:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.484097:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000400:00000200:1.0:1619677447.484097:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.484098:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:1.0:1619677447.484099:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.484100:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677447.484101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677447.484102:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.484102:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.484104:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.484107:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.484108:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677447.484109:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.484110:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:2.0:1619677447.484111:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.484112:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.484112:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.484112:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.484113:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.484113:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:1.0:1619677447.484113:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.484113:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.484114:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:2.0:1619677447.484114:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.484115:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.484116:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677447.484117:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.484117:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.484117:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.484118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.484126:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.484129:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:12.0:1619677447.484129:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:18.0:1619677447.484130:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.484130:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.484131:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:12.0:1619677447.484131:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:18.0:1619677447.484132:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:12.0:1619677447.484132:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.484134:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.484135:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.484136:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.484138:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.484140:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.484143:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.484143:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.484145:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.484147:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677447.484148:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.484150:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.484151:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.484152:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677447.484155:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.484157:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.484159:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.484160:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677447.484161:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.484162:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.484163:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.484164:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.484164:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677447.484177:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.484179:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.484180:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.484180:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.484182:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.484185:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.484263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.484267:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.484269:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.484270:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.484270:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.484273:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.484276:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:6.0:1619677447.484316:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.484319:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.484319:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.484320:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.484322:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.484325:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.484357:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.484360:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.484361:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.484363:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.484366:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.484368:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.484370:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.484371:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.484372:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.484373:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.484377:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.484379:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.484382:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.484383:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.484383:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.484385:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.484386:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.484386:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.484387:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677447.484394:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.484397:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.484397:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.484399:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.484400:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:12.0:1619677447.484401:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.484402:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677447.484403:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.484404:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.484405:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.484407:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.484408:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677447.484480:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.484483:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.484483:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.484485:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.484487:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.484488:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.484490:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.484491:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.484492:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.484493:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.484496:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.484498:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.484501:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.484502:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.484502:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.484504:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.484505:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.484505:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.484506:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.484510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:17.0:1619677447.484515:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.484517:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:18.0:1619677447.484518:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:1.0:1619677447.484518:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677447.484520:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:17.0:1619677447.484520:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.484521:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.484521:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:17.0:1619677447.484523:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.484524:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.484525:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:18.0:1619677447.484526:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:17.0:1619677447.484526:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.484527:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.484528:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.484529:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.484533:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.484535:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.484538:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.484538:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.484539:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.484541:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.484541:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.484542:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.484543:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677447.484545:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.484548:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.484550:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.484550:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.484553:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.484554:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.484555:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.484555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:5.0:1619677447.484556:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.484557:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.484558:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:24.0:1619677447.484562:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:5.0:1619677447.484562:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.484565:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:5.0:1619677447.484565:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:8.0:1619677447.484566:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677447.484567:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.484568:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000400:00000200:24.0:1619677447.484569:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.484569:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.484571:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.484571:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000400:00000200:24.0:1619677447.484572:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.484572:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.484572:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:24.0:1619677447.484573:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.484574:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96727554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.484576:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.484578:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.484579:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.484583:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.484585:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.484586:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:4.2:1619677447.484586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.484588:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.484589:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.484589:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.484589:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.484590:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.484591:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.484591:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.484592:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:20.0:1619677447.484592:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.484592:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:24.0:1619677447.484593:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677447.484594:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:1.0:1619677447.484604:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.484607:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.484608:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.484608:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.484610:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.484611:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.484613:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:1.0:1619677447.484613:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677447.484614:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.484615:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677447.484615:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.484617:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:17.0:1619677447.484624:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.484627:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.484628:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.484628:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.484629:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.484629:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.484629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.484633:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.484636:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.484637:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.484637:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.484640:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.484642:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.484650:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.484653:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.484654:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.484654:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.484655:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.484656:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.484729:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.484731:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.484733:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.484734:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.484735:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.484737:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.484738:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.484740:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.484741:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.484742:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.484745:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.484747:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.484750:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.484751:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.484751:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.484753:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.484754:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.484754:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.484755:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.484766:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.484769:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.484769:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.484771:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.484772:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.484774:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.484783:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.484785:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:13.0:1619677447.484786:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677447.484787:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.484788:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:13.0:1619677447.484788:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.484789:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.484790:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:5.0:1619677447.484790:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.484791:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.484792:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677447.484792:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.484793:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.484794:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.484796:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.484796:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.484800:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.484802:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.484804:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.484805:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.484806:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.484808:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.484809:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.484810:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.484811:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.484812:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:9.2:1619677447.484815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.484820:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.484822:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:2.0:1619677447.484822:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677447.484823:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.484824:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:2.0:1619677447.484825:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.484826:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.484826:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.484828:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.484831:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.484947:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.484949:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.484950:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.484952:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.484954:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.484956:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.484957:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.484958:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.484959:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.484960:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:4.2:1619677447.484962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:18.0:1619677447.484963:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.484965:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.484968:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.484969:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.484970:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.484970:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677447.484971:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.484972:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.484973:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:8.0:1619677447.484973:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.484974:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677447.484974:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.484975:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:8.0:1619677447.484976:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.484978:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.484980:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.484981:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96727549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.484982:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.484983:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.484984:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:2.0:1619677447.484985:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.484987:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.484987:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:8.0:1619677447.484988:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.484988:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.0:1619677447.484989:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.484992:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.484993:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:12.0:1619677447.484994:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677447.484994:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.484995:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:12.0:1619677447.484996:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677447.484996:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:12.0:1619677447.484997:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.484997:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.484998:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000400:00000200:12.0:1619677447.484999:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.484999:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677447.485000:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.485003:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.485004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.485009:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677447.485011:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.485011:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.485012:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.485013:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677447.485013:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.485014:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.485015:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.485015:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.485018:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.485029:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.485031:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.485032:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.485032:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.485033:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.485034:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.485044:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.485047:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.485048:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.485048:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.485050:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.485053:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:9.2:1619677447.485054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.485062:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.485065:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.485066:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.485066:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.485068:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.485072:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.485094:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.485097:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.485098:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.485100:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.485102:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.485104:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.485105:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.485106:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.485107:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.485108:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.485112:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.485114:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.485117:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.485117:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.485118:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.485120:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.485120:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.485121:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.485121:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677447.485133:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.485136:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.485137:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.485139:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.485142:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.485144:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.485145:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.485146:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.485147:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.485148:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.485152:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.485154:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.485156:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.485157:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.485157:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.485159:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.485160:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.485160:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.485161:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677447.485172:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.485175:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.485176:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.485176:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.485178:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.485180:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677447.485288:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.485291:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.485293:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.485294:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.485300:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.485302:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677447.485303:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.485304:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.485305:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.485307:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677447.485310:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.485312:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677447.485315:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.485315:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677447.485316:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.485318:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.485318:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.485319:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.485319:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.485320:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.485323:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.485323:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:9.2:1619677447.485325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:6.0:1619677447.485325:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.485326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:6.0:1619677447.485327:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.485329:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.485330:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.485331:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.485332:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.485333:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:6.0:1619677447.485333:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:18.0:1619677447.485336:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.485337:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.485337:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.485337:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.485339:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.485340:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677447.485342:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:6.0:1619677447.485344:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.485344:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.485345:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.485347:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.485347:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.485348:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.485349:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.485378:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.485380:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.485382:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.485384:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.485384:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.485387:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.485395:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.485398:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.485399:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.485399:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.485400:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.485401:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.485565:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.485569:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.485571:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.485571:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.485571:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:5.0:1619677447.485573:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.485574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677447.485574:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:2.0:1619677447.485574:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:5.0:1619677447.485576:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.485577:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:5.0:1619677447.485578:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.485579:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:5.0:1619677447.485579:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.485579:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.485580:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:5.0:1619677447.485580:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677447.485582:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.485582:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.485584:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.485585:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.485585:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.485586:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96727538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.485588:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:2.0:1619677447.485588:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.485590:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677447.485590:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.485591:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000400:00000200:2.0:1619677447.485591:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.485592:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.485593:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.485594:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:18.0:1619677447.485595:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.485595:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000400:00000200:2.0:1619677447.485595:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.485596:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:18.0:1619677447.485597:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677447.485597:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:2.0:1619677447.485597:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.485601:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.485602:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.485603:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.485605:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.485605:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.485606:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677447.485607:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:2.0:1619677447.485607:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677447.485608:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.485609:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:5.0:1619677447.485609:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.485609:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.485610:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.485611:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.485612:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.485612:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.485614:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:4.2:1619677447.485617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.485624:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.485627:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.485628:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677447.485629:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677447.485630:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.485632:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.485633:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.485634:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.485634:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.485635:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:12.0:1619677447.485636:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:8.0:1619677447.485636:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96727537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.485637:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677447.485637:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.485639:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.485640:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.485643:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.485645:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.485647:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.485648:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.485649:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.485651:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.485651:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.485652:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.485652:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.485665:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.485667:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.485668:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.485669:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.485671:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.485673:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677447.485782:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.485784:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.485786:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.485787:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.485789:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.485790:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.485792:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.485793:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.485794:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.485795:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.485799:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.485800:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.485803:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.485804:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.485805:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.485806:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.485807:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.485808:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.485809:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.485810:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677447.485815:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.485817:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.485818:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677447.485820:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:1.0:1619677447.485820:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.485822:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:1.0:1619677447.485822:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.485823:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.485824:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677447.485824:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.485825:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.485826:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.485827:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.485828:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.485831:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.485833:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.485835:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.485836:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.485837:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.485839:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.485839:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.485840:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.485841:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.485842:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.485852:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677447.485852:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677447.485855:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677447.485855:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:22.0:1619677447.485856:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.485856:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.485856:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.485857:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677447.485858:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.485862:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677447.485997:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.486000:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.486000:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.486002:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.486002:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.486006:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:6.0:1619677447.486017:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.486020:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.486021:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.486023:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.486026:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.486027:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.486028:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.486029:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.486031:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.486032:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.486035:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.486036:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.486039:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.486040:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.486041:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.486043:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.486043:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.486045:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.486046:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.486046:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.486056:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.486058:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.486059:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.486060:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.486111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.486164:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677447.486164:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677447.486165:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:6.0:1619677447.486166:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.486167:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.486167:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:18.0:1619677447.486168:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:6.0:1619677447.486168:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.486168:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:20.0:1619677447.486170:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.486170:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.486170:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.486173:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.486173:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.486174:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:20.0:1619677447.486175:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677447.486175:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.486177:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.486177:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677447.486179:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.486179:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.486180:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.486181:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677447.486181:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.486182:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.486184:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677447.486186:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.486188:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.486189:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677447.486191:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677447.486192:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.486193:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.486194:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:18.0:1619677447.486194:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.486195:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677447.486196:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.486196:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.486196:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677447.486197:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677447.486197:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677447.486198:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.486198:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.486199:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.486200:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.486389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.486396:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.486399:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.486400:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.486401:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.486403:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.486405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.486408:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.486410:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.486412:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.486413:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.486413:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.486414:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.486414:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.486550:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.486553:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.486554:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.486557:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.486560:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.486561:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.486563:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.486564:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677447.486564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:17.0:1619677447.486565:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.486566:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.486570:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.486571:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.486574:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.486574:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.486575:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.486576:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:17.0:1619677447.486576:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.486577:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677447.486578:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.486578:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:24.0:1619677447.486579:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.486579:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.486579:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677447.486580:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:24.0:1619677447.486582:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.486582:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:24.0:1619677447.486583:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677447.486583:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.486585:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96727525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.486586:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.486586:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.486587:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.486588:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.486589:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.486589:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:12.0:1619677447.486591:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:24.0:1619677447.486593:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.486593:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.486594:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.486595:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:12.0:1619677447.486596:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.486597:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.486598:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:13.0:1619677447.486598:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.486599:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677447.486599:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.486600:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.486601:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.486602:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.486602:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.486614:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.486617:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.486617:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.486618:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.486620:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.486623:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.486631:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.486634:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.486635:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.486635:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.486636:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.486637:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677447.486710:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.486713:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.486714:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.486716:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.486719:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.486720:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.486721:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.486722:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.486723:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.486724:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.486727:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.486728:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:22.0:1619677447.486729:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677447.486731:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.486732:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:18.0:1619677447.486732:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:6.0:1619677447.486732:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.486733:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.486733:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:22.0:1619677447.486735:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.486735:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.486735:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.486736:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.486736:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:22.0:1619677447.486738:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.486743:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.486746:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.486748:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.486749:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.486750:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.486751:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.486753:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.486755:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.486756:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.486757:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.486760:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.486762:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.486764:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.486765:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:6.0:1619677447.486765:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:20.0:1619677447.486766:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.486767:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:6.0:1619677447.486767:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677447.486768:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:20.0:1619677447.486768:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:2.0:1619677447.486768:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.0:1619677447.486769:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.486769:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:6.0:1619677447.486770:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.486771:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.486772:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:6.0:1619677447.486773:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.486774:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:6.0:1619677447.486775:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:6.0:1619677447.486776:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.486777:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.486777:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.486779:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.486779:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:6.0:1619677447.486780:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:5.0:1619677447.486781:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:2.0:1619677447.486781:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.486782:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.486783:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:2.0:1619677447.486783:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.486784:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.486785:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.486785:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.486786:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.486787:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.486789:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:5.0:1619677447.486789:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:2.0:1619677447.486790:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.486793:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.486794:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.486795:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:6.0:1619677447.486796:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:2.0:1619677447.486796:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.486796:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.486797:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.486798:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:6.0:1619677447.486799:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:2.0:1619677447.486799:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.486800:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:2.0:1619677447.486800:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:6.0:1619677447.486801:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:2.0:1619677447.486801:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.486802:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.486812:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.486814:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.486815:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.486815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:6.0:1619677447.486815:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.486818:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.486820:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.486822:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.486824:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.486826:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.486826:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.486828:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.486830:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677447.486831:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677447.486832:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.486833:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.486835:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.486838:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.486840:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.486841:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.486842:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.486843:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.486844:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.486848:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.486850:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.486852:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.486853:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.486854:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.486856:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.486856:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.486857:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.486858:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677447.486998:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.487001:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.487001:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.487004:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.487007:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.487007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677447.487008:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.487009:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.487011:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.487013:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.487014:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.487016:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:18.0:1619677447.487017:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.487017:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.487018:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.487019:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.487020:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.487022:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.487023:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.487023:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.487024:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.487024:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.487026:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.487026:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.487027:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.487028:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.487029:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.487031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.487040:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.487041:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.487041:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.487041:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.487042:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.487042:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.487043:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.487043:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.487044:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677447.487044:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.487055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.487167:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.487170:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.487170:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.487173:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.487175:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.487177:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.487178:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.487179:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.487180:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.487181:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.487185:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.487186:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.487189:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.487190:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.487191:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.487193:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.487193:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.487194:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.487194:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.487206:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.487208:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.487209:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.487211:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.487212:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.487214:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.487233:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.487239:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.487240:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.487240:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.487241:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.487242:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.487351:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.487355:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.487355:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.487356:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.487358:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.487361:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.487369:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.487372:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.487373:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.487373:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.487374:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.487375:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.487391:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.487393:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.487396:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.487397:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.487398:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.487400:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.487401:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.487403:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.487404:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.487405:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.487408:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.487410:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.487413:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.487413:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.487414:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.487416:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.487417:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.487417:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.487418:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.487429:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.487432:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.487432:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.487434:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.487436:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.487437:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.487478:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:5.0:1619677447.487481:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:20.0:1619677447.487483:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:5.0:1619677447.487483:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.487486:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.487487:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:5.0:1619677447.487489:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.487490:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.487491:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.487492:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:5.0:1619677447.487495:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.487497:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:5.0:1619677447.487500:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.487501:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:5.0:1619677447.487501:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:5.0:1619677447.487503:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:5.0:1619677447.487504:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:5.0:1619677447.487505:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:5.0:1619677447.487506:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:5.0:1619677447.487507:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677447.487514:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000800:00000200:20.0:1619677447.487517:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000800:00000200:20.0:1619677447.487517:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677447.487518:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:12.0:1619677447.487691:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.487693:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.487694:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.487695:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.487697:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.487698:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.487723:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.487726:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.487727:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.487729:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.487731:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.487732:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677447.487734:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.487735:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.487736:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.487737:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677447.487740:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.487741:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677447.487745:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.487746:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677447.487746:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677447.487748:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677447.487749:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.487750:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.487750:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:2.0:1619677447.487772:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.487775:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.487776:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:4.2:1619677447.487778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677447.487778:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.487781:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.487782:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.487783:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.487784:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.487786:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.487787:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.487791:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.487794:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.487796:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.487796:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.487797:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.487799:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.487799:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.487800:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.487800:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.487805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.487812:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.487814:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.487815:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.487816:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.487817:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.487819:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.487825:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:4.2:1619677447.487826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.487828:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677447.487829:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:22.0:1619677447.487831:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:2.0:1619677447.487831:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.487833:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.487834:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:2.0:1619677447.487834:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:22.0:1619677447.487835:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.487835:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:2.0:1619677447.487836:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.487837:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.487837:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:17.0:1619677447.487838:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677447.487838:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.487839:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:2.0:1619677447.487839:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.487840:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:22.0:1619677447.487841:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:17.0:1619677447.487841:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:5.0:1619677447.487841:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.487841:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677447.487842:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.487844:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:5.0:1619677447.487844:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.487845:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000400:00000200:17.0:1619677447.487845:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.487846:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (24)--
00000400:00000200:2.0:1619677447.487846:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.487847:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.487847:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:17.0:1619677447.487848:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.487849:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:5.0:1619677447.487849:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.487849:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:17.0:1619677447.487850:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:6.0:1619677447.487850:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:17.0:1619677447.487851:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.487851:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:20.0:1619677447.487852:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.487852:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:2.0:1619677447.487852:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677447.487853:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.487853:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:22.0:1619677447.487854:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.487854:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677447.487854:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:20.0:1619677447.487855:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.487855:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.487855:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.487855:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000400:00000200:20.0:1619677447.487856:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:6.0:1619677447.487856:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.487856:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.487856:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:2.0:1619677447.487856:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677447.487857:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:20.0:1619677447.487858:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.487858:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:17.0:1619677447.487859:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.487860:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.487861:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.487862:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:17.0:1619677447.487862:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.487864:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.487865:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.487865:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677447.487866:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.487867:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.487867:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677447.487868:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677447.487868:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.487868:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677447.487871:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.487871:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.487872:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.487874:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.487874:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.487875:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.487876:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:1.0:1619677447.487880:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677447.487882:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:1.0:1619677447.487882:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.487883:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.487884:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.487885:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.487886:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:1.0:1619677447.487886:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.487887:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:18.0:1619677447.487888:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.487888:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.487890:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:18.0:1619677447.487891:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.487891:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.487893:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:18.0:1619677447.487893:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677447.487894:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.487895:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.487896:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.487896:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:18.0:1619677447.487897:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.487898:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.487901:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.487903:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.487906:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.487906:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.487907:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.487909:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.487909:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.487911:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.487912:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.487913:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.0:1619677447.487923:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677447.487923:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:20.0:1619677447.487925:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:2.0:1619677447.487925:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:20.0:1619677447.487926:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.487926:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.487927:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:20.0:1619677447.487928:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.487929:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.487931:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677447.488079:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.488082:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.488082:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.488083:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.488085:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.488088:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.488120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.488165:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.488168:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.488169:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.488169:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.488171:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.488174:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.488332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.488347:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.488350:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.488351:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.488353:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.488355:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.488357:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.488358:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.488361:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.488362:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.488363:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.488367:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.488370:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.488372:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.488372:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.488373:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.488375:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.488375:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.488376:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.488377:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.488378:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.488390:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.488390:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.488391:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.488391:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.488391:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.488392:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.488393:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677447.488394:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.488394:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.488395:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.488396:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:1.0:1619677447.488396:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:6.0:1619677447.488397:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.488398:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.488400:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.488403:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.488498:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.488501:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.488503:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.488503:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.488505:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.488507:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.488508:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.488509:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.488511:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.488511:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.488515:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.488517:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.488519:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.488520:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.488521:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.488523:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.488523:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.488524:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.488525:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.488529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.488537:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.488540:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.488541:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.488541:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.488543:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.488546:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.488551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.488558:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.488561:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.488562:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.488562:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.488563:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.488563:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.488572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.488618:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.488621:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.488621:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.488622:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.488624:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.488627:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.488638:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.488641:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.488642:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.488642:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.488643:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.488644:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.488649:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.488652:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.488653:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.488654:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.488656:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.488657:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.488667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.488674:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.488676:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.488677:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.488679:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.488681:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.488683:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.488684:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.488685:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.488686:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.488687:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.488690:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.488692:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.488695:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.488695:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.488696:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.488698:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.488698:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.488699:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.488699:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.488712:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.488714:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.488715:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.488716:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.488718:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.488721:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677447.488808:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677447.488809:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.488811:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.488812:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677447.488812:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.488813:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677447.488815:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677447.488816:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.488818:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.488818:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.488820:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677447.488820:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677447.488822:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.488822:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.488824:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.488824:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.488825:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.488825:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.488826:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677447.488827:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.488832:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:1.0:1619677447.488832:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.488835:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.488836:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677447.488838:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.488839:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.488839:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:1.0:1619677447.488839:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:18.0:1619677447.488840:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677447.488840:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.488841:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.488842:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677447.488842:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.488843:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677447.488843:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677447.488844:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677447.488844:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677447.488845:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677447.488854:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.488857:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.488858:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.488859:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:12.0:1619677447.488860:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.488861:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677447.488861:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.488862:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.488863:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:12.0:1619677447.488863:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:6.0:1619677447.488865:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.488867:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.488873:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.488876:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.488877:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.488878:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.488878:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.488880:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.488893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.488909:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.488912:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.488912:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.488914:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.488916:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.488917:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.488935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.488939:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.488942:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.488943:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.488943:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.488945:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.488948:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677447.488996:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.488999:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.489001:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.489002:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.489003:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.489005:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.489006:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.489007:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.489008:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.489009:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.489013:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.489015:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.489018:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.489019:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.489024:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.489026:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.489026:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.489027:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.489028:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:9.2:1619677447.489061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.489068:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.489071:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.489072:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.489072:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.489074:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.489077:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.489141:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.489144:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.489144:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.489146:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.489148:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.489150:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.489151:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.489152:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.489153:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.489154:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.489158:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.489160:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.489162:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.489163:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.489164:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.489166:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.489166:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.489167:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.489167:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.489180:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.489182:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.489183:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.489185:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.489185:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.489187:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.489206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.489253:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.489256:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.489256:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.489257:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.489259:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.489262:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.489333:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.489335:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.489336:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.489338:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.489340:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.489342:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.489343:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.489344:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.489345:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.489346:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.489349:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.489351:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.489353:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.489354:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.489355:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.489357:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.489357:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.489358:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.489358:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677447.489358:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.489361:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.489362:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.489363:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.489367:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.489369:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.489371:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.489372:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.489372:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.489373:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.489374:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.489374:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.489375:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677447.489376:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.489379:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.489379:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.489380:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.489382:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.489385:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.489385:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.489386:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.489388:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.489388:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.489389:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.489389:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.489394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.489401:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.489404:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.489404:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.489405:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.489407:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.489410:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677447.489460:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.489463:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.489463:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.489465:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.489467:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.489469:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.489535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.489542:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.489545:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.489546:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.489547:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.489551:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.489552:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.489553:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.489555:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.489555:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.489556:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.489560:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.489561:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.489564:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.489565:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.489565:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.489567:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.489567:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.489569:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.489570:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.489570:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.489576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.489581:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677447.489583:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.489584:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.489585:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.489585:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677447.489585:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.489586:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.489587:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.489587:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.489590:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.489641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.489688:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.489691:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.489692:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.489692:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.489694:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.489697:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677447.489721:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.489723:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.489725:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.489727:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.489728:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.489730:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.489731:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.489733:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.489734:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.489735:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.489738:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.489740:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.489743:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.489744:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.489745:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.489746:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.489747:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.489749:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.489750:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.489750:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677447.489759:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.489761:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677447.489761:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:12.0:1619677447.489762:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.489763:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:12.0:1619677447.489764:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.489764:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677447.489765:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.489765:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.489767:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.489778:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.489781:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.489782:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.489783:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.489783:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.489785:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.489830:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.489832:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.489833:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.489834:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.489836:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.489837:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677447.489882:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.489885:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.489885:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.489886:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.489888:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.489891:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.0:1619677447.490032:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:1.0:1619677447.490032:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.490035:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:1.0:1619677447.490035:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.490036:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677447.490037:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.490039:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.490039:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.490042:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.490042:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.490044:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:1.0:1619677447.490044:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:5.0:1619677447.490046:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.490046:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.490047:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.490049:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.490049:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.490050:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.490050:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:5.0:1619677447.490052:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.490057:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.490057:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.490059:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:5.0:1619677447.490061:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 568 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.490063:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.490063:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.490064:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:1.0:1619677447.490064:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.490065:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677447.490066:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:1.0:1619677447.490067:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.490067:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.490068:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.490068:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:1.0:1619677447.490068:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.490068:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677447.490070:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:22.0:1619677447.490071:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677447.490071:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:5.0:1619677447.490072:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:22.0:1619677447.490074:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.490075:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.490077:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.490079:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.490079:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:4.2:1619677447.490080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:22.0:1619677447.490081:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:5.0:1619677447.490081:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:22.0:1619677447.490082:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.490082:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:20.0:1619677447.490083:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000400:00000200:13.0:1619677447.490083:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.490083:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.490083:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.490084:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.490084:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677447.490085:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:22.0:1619677447.490086:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677447.490086:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.490086:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:5.0:1619677447.490086:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.490087:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:22.0:1619677447.490087:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.490087:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.490088:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677447.490089:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:13.0:1619677447.490089:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.490090:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.490091:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677447.490092:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.490094:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.490094:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:24.0:1619677447.490097:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.490097:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.490098:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:22.0:1619677447.490098:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.490098:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:24.0:1619677447.490100:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96727473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.490100:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:24.0:1619677447.490101:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.490101:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.490101:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:24.0:1619677447.490102:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.490102:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:24.0:1619677447.490103:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.490107:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.490110:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.490111:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.490113:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:24.0:1619677447.490114:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.490115:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677447.490115:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:5.0:1619677447.490115:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.490116:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.490117:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.490117:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.490117:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.490117:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.490118:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.490119:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.490119:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:5.0:1619677447.490119:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.490120:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.490121:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.490122:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.490123:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.490124:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.490126:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.490127:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.490128:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.490129:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.490129:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.490130:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.490131:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:17.0:1619677447.490131:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677447.490132:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.490132:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.490132:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.490133:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677447.490133:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:5.0:1619677447.490133:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.490134:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.0:1619677447.490134:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.490134:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.490135:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.490136:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.490136:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:17.0:1619677447.490137:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.490137:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.490138:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677447.490140:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:9.2:1619677447.490142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.490150:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677447.490151:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.490153:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:1.0:1619677447.490153:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.490154:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.490154:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.490154:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.490155:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.490155:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000400:00000200:2.0:1619677447.490156:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.490156:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.490159:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.490240:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.490243:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.490243:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.490245:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.490247:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.490248:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.490249:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.490251:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.490251:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.490252:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.490255:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.490257:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.490260:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.490260:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.490261:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.490263:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.490263:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.490264:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.490264:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677447.490276:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.490278:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.490279:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.490281:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.490282:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.490283:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.490370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.490419:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.490422:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.490423:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.490423:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.490426:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.490428:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677447.490459:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.490462:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.490464:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.490464:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.490467:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.490468:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.490469:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.490471:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.490471:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.490472:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.490475:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.490477:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.490479:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.490480:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.490484:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.490486:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.490487:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677447.490488:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.490488:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.490488:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677447.490490:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.490492:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.490493:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.490496:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.490497:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.490498:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.490499:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:17.0:1619677447.490500:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.490500:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.490501:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.490502:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.490503:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.490503:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.490505:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.490506:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.490508:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.490508:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.490512:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.490513:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.490513:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.490515:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.490516:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.490517:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.490518:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.490519:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.490529:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.490531:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.490532:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.490533:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677447.490611:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.490614:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.490615:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.490617:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.490620:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.490621:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.490622:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.490624:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.490625:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.490626:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.490629:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.490631:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.490633:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.490633:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.490634:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.490635:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.490636:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.490636:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.490637:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.490643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.490650:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.490652:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.490653:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.490654:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.490655:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.490657:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.490702:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.490704:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.490706:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.490707:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.490709:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.490710:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.490712:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.490713:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.490714:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.490715:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.490719:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.490720:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.490724:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.490724:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.490725:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.490727:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.490728:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:4.2:1619677447.490732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.0:1619677447.490733:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.490734:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677447.490735:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.490737:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:5.0:1619677447.490737:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.490739:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.490739:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:20.0:1619677447.490741:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.490742:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.490743:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:20.0:1619677447.490743:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677447.490744:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.490746:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.490746:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.490747:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.490748:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677447.490749:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.490750:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677447.490752:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.490752:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96727464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.490753:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.490754:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:8.0:1619677447.490754:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.490755:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.490757:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.490758:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677447.490758:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.490760:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000400:00000200:8.0:1619677447.490760:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.490761:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.490761:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.490762:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.0:1619677447.490762:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.490765:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677447.490766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677447.490766:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.490766:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.490768:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.490769:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.490769:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.490770:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677447.490773:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.490776:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.490777:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.490777:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.490780:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.490782:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:1.0:1619677447.490782:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.490784:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.490785:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.490785:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.490786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000400:00000200:1.0:1619677447.490787:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.490791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.490791:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677447.490796:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.490798:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.490799:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.490800:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.490800:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.490801:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.490801:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.490802:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:2.0:1619677447.490802:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.490803:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.490804:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.490807:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.490811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677447.490817:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677447.490819:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.490820:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.490821:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677447.490822:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.490823:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000400:00000200:6.0:1619677447.490823:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.490823:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.490824:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.490824:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:6.0:1619677447.490826:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.490828:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.490829:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.490830:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.490831:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.490832:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.490836:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.490837:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.490840:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.490840:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.490841:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.490843:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.490843:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.490845:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.490846:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.490846:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.490849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.490856:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677447.490857:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677447.490858:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:22.0:1619677447.490859:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.490860:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677447.490860:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.490860:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.490861:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:2.0:1619677447.490862:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.490864:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:17.0:1619677447.490943:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.490946:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:5.0:1619677447.490947:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:1.0:1619677447.490947:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.490950:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.490950:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.490953:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.490954:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.490957:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.490957:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.490959:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:5.0:1619677447.490959:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:17.0:1619677447.490961:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.490961:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.490963:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.490963:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.490965:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.490965:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.490966:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:5.0:1619677447.490966:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.490972:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.490973:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.490975:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677447.490976:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.490976:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:5.0:1619677447.490977:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.490978:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:1.0:1619677447.490978:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.490979:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.490979:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:1.0:1619677447.490979:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.490980:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677447.490980:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:17.0:1619677447.490981:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.0:1619677447.490981:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000400:00000200:1.0:1619677447.490981:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.490982:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:17.0:1619677447.490983:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677447.490983:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:17.0:1619677447.490984:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.490985:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.490985:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.490986:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.490986:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.490989:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.490989:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:9.2:1619677447.490990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.490990:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.490990:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.490991:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:24.0:1619677447.490996:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:20.0:1619677447.490998:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:24.0:1619677447.490999:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.491000:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.491000:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:8.0:1619677447.491000:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:20.0:1619677447.491001:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.491004:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.491004:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:20.0:1619677447.491005:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.491007:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:24.0:1619677447.491008:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.491145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.491186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.491191:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.491194:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.491194:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.491195:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.491197:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.491200:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.491210:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.491212:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.491213:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.491214:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.491214:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.491215:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.491234:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.491237:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.491239:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.491240:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.491241:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.491242:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.491244:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.491245:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.491246:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.491247:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.491251:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.491252:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.491255:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.491256:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.491256:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.491258:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.491259:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.491260:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.491261:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.491262:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.491270:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:6.0:1619677447.491271:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.491272:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.491273:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.491274:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.491274:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.491275:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.491275:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.491277:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.491279:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.491330:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.491333:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.491333:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.491335:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.491338:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.491339:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.491340:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.491341:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.491343:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.491343:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.491347:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.491349:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.491352:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.491353:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.491353:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.491355:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.491356:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.491356:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.491357:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.491364:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.491367:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.491369:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.491370:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.491373:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.491375:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.491376:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96727454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.491378:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.491379:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.491380:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.491383:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.491385:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.491388:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.491389:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.491390:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.491392:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.491393:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.491393:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.491394:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.491398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.491407:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.491410:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:4.2:1619677447.491410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:18.0:1619677447.491411:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.491411:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.491414:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.491416:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677447.491417:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.491420:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.491421:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.491423:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.491426:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.491427:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.491428:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.491430:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.491431:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.491432:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.491435:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.491436:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.491439:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.491440:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.491441:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.491442:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.491443:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.491443:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.491444:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.491456:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.491459:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.491460:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.491461:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.491462:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.491462:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.491463:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.491465:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.491466:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.491466:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.491468:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.491470:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.491471:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.491472:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.491473:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.491474:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.491478:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.491479:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.491482:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.491482:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.491483:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.491485:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.491485:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.491486:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.491486:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.491498:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.491501:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.491502:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.491502:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.491504:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.491507:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.491515:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.491517:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.491518:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.491519:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.491519:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.491520:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.491583:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.491585:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677447.491586:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677447.491588:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.491590:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.491592:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.491593:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.491594:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.491595:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.491597:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.491600:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.491602:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.491605:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.491606:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.491607:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.491608:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.491608:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.491609:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677447.491610:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677447.491614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.491621:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.491624:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.491625:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.491625:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.491627:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.491629:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.0:1619677447.491648:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.491651:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.491653:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.491654:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.491655:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.491656:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.491658:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.491659:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.491660:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.491661:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.491664:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.491666:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.491668:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.491669:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.491670:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.491672:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.491672:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.491673:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.491674:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:5.0:1619677447.491685:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.491688:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.491690:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.491690:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.491692:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.491694:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677447.491695:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.491696:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:5.0:1619677447.491696:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.491697:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.491698:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:5.0:1619677447.491698:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:12.0:1619677447.491700:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.491702:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000400:00000200:5.0:1619677447.491702:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.491703:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.491705:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.491705:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.491707:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.491708:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:17.0:1619677447.491709:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:5.0:1619677447.491709:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.491710:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677447.491711:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:5.0:1619677447.491711:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.491712:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.491712:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677447.491712:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.491713:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.491715:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.491718:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.0:1619677447.491724:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.491726:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.491727:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.491729:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.491730:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.491731:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.491826:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.491828:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:5.0:1619677447.491828:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.491829:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:5.0:1619677447.491829:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.491831:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.491832:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.491834:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:20.0:1619677447.491835:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.491835:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:20.0:1619677447.491836:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:5.0:1619677447.491836:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677447.491837:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.491839:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.491840:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.491841:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677447.491844:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.491846:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.491846:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.491847:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.491848:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.491848:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.491849:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:5.0:1619677447.491849:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.491849:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.491850:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (23)++
00000800:00000200:5.0:1619677447.491851:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.491852:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.491852:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677447.491853:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.0:1619677447.491854:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.0:1619677447.491865:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677447.491866:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:8.0:1619677447.491868:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.491869:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677447.491869:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.491869:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.491872:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.491872:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.491873:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.491874:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:8.0:1619677447.491876:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.491877:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.491879:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.491880:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.491881:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.491882:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.491885:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.491887:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.491890:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.491891:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.491892:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.491894:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.491895:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.491895:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.491896:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.491901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.491908:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.491911:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.491912:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.491912:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.491915:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.491917:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.491964:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.491966:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.491967:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.491969:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.491971:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.491972:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.491973:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.491975:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.491976:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.491977:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.491980:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.491982:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.491984:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.491984:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.491985:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.491987:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.491987:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.491988:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.491989:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.491990:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.491992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.492000:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.492001:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.492002:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.492002:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.492002:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.492003:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.492004:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.492004:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.492005:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.492005:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677447.492094:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.492096:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.492097:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.492098:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.492099:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.492100:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.492101:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.492103:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.492103:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.492104:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.492106:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.492108:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.492109:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.492109:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.492110:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.492111:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.492112:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.492112:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.492113:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.492114:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:9.2:1619677447.492117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.492118:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.492121:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.492122:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.492124:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.492125:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.492126:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.492126:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.492127:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.492127:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.492128:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.492128:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:22.0:1619677447.492129:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:18.0:1619677447.492129:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.492129:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.492130:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:22.0:1619677447.492131:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.492132:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.492132:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:22.0:1619677447.492133:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.492134:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:9.2:1619677447.492134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:22.0:1619677447.492138:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.492140:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677447.492141:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.492143:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.492144:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.492144:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677447.492144:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.492145:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.492145:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677447.492146:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677447.492146:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677447.492147:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677447.492147:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.492148:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.492148:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.492227:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.492230:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.492232:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.492233:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.492234:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.492235:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.492237:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.492238:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.492239:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.492240:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.492243:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.492245:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.492248:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.492248:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.492249:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.492250:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.492251:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.492252:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.492253:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:2.0:1619677447.492259:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.492261:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.492262:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.492264:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.492265:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:2.0:1619677447.492266:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.492267:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:2.0:1619677447.492267:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:5.0:1619677447.492268:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.492269:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.492270:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.492270:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.492271:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.492272:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:2.0:1619677447.492272:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.492273:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677447.492276:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.492278:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.492282:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.492283:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677447.492283:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677447.492285:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.492285:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.492286:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.492287:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677447.492299:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.492303:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.492304:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.492305:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.492307:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.492308:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.492316:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.492319:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.492320:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.492320:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.492321:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.492322:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.492403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.492459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.492467:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.492469:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.492471:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.492472:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.492475:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.492476:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.492477:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.492479:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.492480:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.492481:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.492483:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.492485:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.492488:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.492488:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.492489:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.492491:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.492491:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.492492:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.492493:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.492494:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.492499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.492506:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.492506:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677447.492508:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.492509:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677447.492509:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.492510:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.492510:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.492510:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.492512:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.492515:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:13.0:1619677447.492616:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.492619:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.492620:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.492621:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677447.492623:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.492624:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.492663:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.492665:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.492666:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.492668:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.492671:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.492672:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.492674:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.492675:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.492676:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.492677:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.492679:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.492681:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.492684:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.492684:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677447.492685:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.492686:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677447.492687:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.492688:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677447.492688:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.492688:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:6.0:1619677447.492688:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.492689:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.492689:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677447.492690:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.492693:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.492700:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.492702:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.492703:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.492705:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.492706:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.492708:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.492807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.492984:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.492987:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.492987:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.492990:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.492993:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.492995:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.492997:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.492999:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.493000:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.493001:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.493005:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.493006:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:18.0:1619677447.493007:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.493009:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.493010:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:18.0:1619677447.493011:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.493012:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:24.0:1619677447.493013:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.493013:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.493015:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.493015:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.493016:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.493016:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:24.0:1619677447.493017:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.493019:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677447.493020:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96727429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.493022:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.493022:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:24.0:1619677447.493023:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.493024:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677447.493024:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.493025:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.493027:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.493027:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:24.0:1619677447.493029:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.493029:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:13.0:1619677447.493030:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:24.0:1619677447.493031:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677447.493031:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:13.0:1619677447.493031:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.493032:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.493033:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.493034:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.493034:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677447.493035:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677447.493037:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.493037:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.493038:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.493039:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.493039:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.493048:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.493050:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.493050:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:9.2:1619677447.493050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.493051:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.493051:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.493051:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.493052:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.493052:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677447.493053:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.493053:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.493054:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.493056:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.493058:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.493061:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.493062:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.493062:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.493064:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.493068:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.493188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.493278:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.493281:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.493282:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.493282:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.493284:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.493287:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.493300:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.493303:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.493304:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.493304:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.493305:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.493306:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677447.493336:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.493339:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677447.493341:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.493342:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.493343:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.493344:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.493346:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.493347:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.493348:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.493349:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.493352:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.493354:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.493357:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.493358:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.493358:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.493360:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.493361:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.493361:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.493362:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677447.493374:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.493377:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.493377:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.493379:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.493380:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.493381:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:1.0:1619677447.493398:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.493401:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.493401:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.493403:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.493404:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.493405:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.493408:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.493408:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.493409:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.493411:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.493412:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.493413:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.493414:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.493417:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.493419:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:1.0:1619677447.493419:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.493420:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727423] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.493422:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.493423:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.493423:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.493426:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.493428:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.493429:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.493430:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.493430:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.493431:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.493431:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.493431:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.493433:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.493433:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)++
00000800:00000200:1.0:1619677447.493433:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.493433:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)++
00000800:00000200:17.0:1619677447.493435:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (25)++
00000800:00000200:17.0:1619677447.493436:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (26)--
00000800:00000200:1.0:1619677447.493436:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (26)++
00000800:00000200:17.0:1619677447.493437:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (25)--
00000800:00000200:1.0:1619677447.493437:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (26)--
00000800:00000200:17.0:1619677447.493438:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (24)--
00000800:00000200:1.0:1619677447.493439:0:19931:0:(o2iblnd_cb.c:929:kiblnd_post_tx_locked()) 10.51.2.27@o2ib3(1): redundant or enough NOOP
00000800:00000200:1.0:1619677447.493440:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.493443:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.493448:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.493448:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.493449:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:1.0:1619677447.493449:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.493450:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.493450:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.493451:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.493452:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.493453:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.493453:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.493454:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.493454:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.493454:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.493455:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.493455:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.493456:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677447.493560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.493599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:12.0:1619677447.493668:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.493670:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.493671:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.493673:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.493674:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.493675:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677447.493693:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.493696:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.493697:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.493699:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.493702:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.493704:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.493705:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.493706:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.493707:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.493708:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.493714:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.493716:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.493719:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.493720:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.493721:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.493723:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.493723:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.493724:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.493725:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.493730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.493738:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:5.0:1619677447.493739:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677447.493740:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:5.0:1619677447.493741:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:2.0:1619677447.493741:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.493742:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:5.0:1619677447.493742:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.493743:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.493745:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.493745:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.493747:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677447.493748:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.493761:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.493762:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.493762:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.493763:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:5.0:1619677447.493764:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.493764:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.493765:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:20.0:1619677447.493766:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.493766:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.493767:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.493768:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.493769:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.493771:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.493772:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.493773:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.493774:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.493778:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.493779:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.493781:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.0:1619677447.493783:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.493783:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:20.0:1619677447.493784:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.493784:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.0:1619677447.493785:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677447.493786:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677447.493787:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.493787:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.493788:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.493788:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677447.493789:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.493791:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677447.493792:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.493794:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.493795:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.493796:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:12.0:1619677447.493800:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:18.0:1619677447.493801:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:12.0:1619677447.493803:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677447.493804:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:12.0:1619677447.493804:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.493806:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.493807:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.493807:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.493808:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677447.493809:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677447.493810:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677447.493811:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.493811:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:12.0:1619677447.493811:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.493812:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.493824:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.493827:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.493828:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.493828:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677447.493828:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.493830:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:13.0:1619677447.493831:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.493832:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.493833:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:13.0:1619677447.493833:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.493835:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.493837:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.493908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.493953:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.493956:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.493957:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.493957:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.493960:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.493963:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.493967:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.493970:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.493971:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.493973:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.493975:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.493977:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.493978:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.493979:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.493981:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.493982:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.493985:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.493987:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.493990:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.493991:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.493992:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.493994:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.493994:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.493995:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.493995:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.494006:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.494009:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.494010:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.494011:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.494013:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.494014:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.494272:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.494275:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.494275:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.494278:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.494280:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.494281:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.494283:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.494286:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.494288:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.494289:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.494299:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.494302:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.494304:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.494305:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.494305:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.494307:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.494307:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.494308:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.494308:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.494321:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.494322:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.494323:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.494324:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:13.0:1619677447.494324:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:1.0:1619677447.494325:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.494327:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:4.2:1619677447.494327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.494327:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:13.0:1619677447.494328:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.494330:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.494331:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.494332:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.494376:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.494379:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.494380:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.494380:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:12.0:1619677447.494381:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:22.0:1619677447.494382:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.494384:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:22.0:1619677447.494385:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:12.0:1619677447.494385:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.494386:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.494388:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.494389:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.494391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:6.0:1619677447.494394:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.494397:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.494398:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.494398:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677447.494398:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.494399:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:13.0:1619677447.494400:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677447.494400:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.494401:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.494402:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677447.494403:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.494404:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677447.494404:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.494405:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.494405:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:8.0:1619677447.494406:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.494407:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677447.494408:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.494409:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.494410:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.494411:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.494412:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.494416:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.494417:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.494420:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.494421:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.494422:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.494423:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.494424:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.0:1619677447.494424:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677447.494425:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.494425:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.0:1619677447.494426:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.494429:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.494429:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.494431:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.494433:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.494434:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.494436:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.494436:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.494437:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:5.0:1619677447.494437:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:17.0:1619677447.494439:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:5.0:1619677447.494440:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.494440:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.494441:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.494443:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.494443:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.494445:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.494446:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.0:1619677447.494446:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.494447:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.494449:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.494450:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.494451:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.494451:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677447.494454:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:9.2:1619677447.494455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677447.494457:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.494457:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.494458:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.494459:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.494459:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677447.494463:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.494466:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.494467:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.494467:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.494469:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.494472:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.494486:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.494488:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.494490:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.494491:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.494493:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.494494:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.494495:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.494497:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.494498:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.494499:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.494501:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.494503:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.494505:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.494506:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.494507:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.494508:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.494509:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.494510:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.494511:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677447.494512:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.494523:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.494524:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677447.494524:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.494525:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000400:00000200:5.0:1619677447.494525:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.494526:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.494526:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.494527:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.494528:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.494528:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677447.494542:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.494545:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.494546:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.494546:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.494547:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.494548:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:24.0:1619677447.494714:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.494717:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.494718:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.494720:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.494723:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.494725:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.494726:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.494727:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.494728:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677447.494728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677447.494729:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.494733:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.494734:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.494738:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.494739:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.494739:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.494741:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.494741:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.494743:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.494744:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.494744:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677447.494756:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.0:1619677447.494758:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:5.0:1619677447.494758:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:8.0:1619677447.494759:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:5.0:1619677447.494759:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.494760:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677447.494761:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.494761:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.494762:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677447.494764:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677447.494772:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677447.494772:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677447.494775:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:17.0:1619677447.494775:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.494775:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.494776:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.494777:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.494777:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.494779:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.494779:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.494781:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:17.0:1619677447.494781:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.494782:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.494783:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.494785:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.494785:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.494789:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.494791:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.494794:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.494794:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.494795:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.494797:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.494797:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.494798:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.494799:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.494809:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.494812:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.494813:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.494815:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.494816:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.494818:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.494827:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.494830:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.494830:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.494831:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.494832:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:9.2:1619677447.494832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:13.0:1619677447.494833:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.494839:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.494842:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.494843:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.494844:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.494845:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.494849:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.494852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.494859:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.494862:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.494863:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.494863:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.494864:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.494865:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:12.0:1619677447.494943:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.494946:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.494946:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.494948:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.494950:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.494950:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677447.495051:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.495054:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.495056:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.495057:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.495058:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.495060:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677447.495061:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.495063:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.495064:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.495065:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677447.495068:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.495070:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:4.2:1619677447.495073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.495074:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.495075:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677447.495076:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.495078:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.495078:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.495079:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.495080:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.495081:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:1.0:1619677447.495091:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677447.495092:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.495094:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:1.0:1619677447.495094:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.495095:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:5.0:1619677447.495095:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.495096:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:1.0:1619677447.495096:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.495098:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.495101:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.495116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.495121:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.495124:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.495124:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.495125:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.495127:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.495130:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.495131:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.495134:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.495136:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.495136:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.495138:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.495140:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.495141:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.495142:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.495144:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.495145:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.495148:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.495150:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.495153:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.495154:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.495154:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.495156:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.495157:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.495158:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.495159:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.495160:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.495167:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.495169:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:12.0:1619677447.495169:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677447.495170:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.495171:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.495171:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.495172:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.495174:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.495175:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.495176:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.495218:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.495221:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.495221:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.495224:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.495227:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.495228:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.495230:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.495231:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.495232:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.495234:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.495237:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.495239:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.495242:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.495243:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.495244:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.495245:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.495246:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.495247:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.495247:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677447.495259:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.495261:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.495262:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.495263:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.495264:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.495267:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677447.495372:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.495374:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.495375:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.495376:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.495378:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.495381:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.495390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.495397:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.495400:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.495401:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.495403:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.495406:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.495407:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.495409:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.495410:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.495411:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.495412:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.495415:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.495417:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.495420:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.495421:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.495421:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.495423:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.495423:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.495424:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.495425:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.495437:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:4.2:1619677447.495439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.495440:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.495440:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.495441:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.495443:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.495445:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:9.2:1619677447.495477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.495486:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.495489:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.495490:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.495490:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.495492:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.495495:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.495549:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.495552:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.495552:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.495554:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.495557:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.495558:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.495559:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.495560:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.495561:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.495562:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.495566:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.495568:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.495570:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.495571:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.495572:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.495573:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.495574:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.495574:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.495574:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677447.495586:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.495589:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677447.495590:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:17.0:1619677447.495590:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.495590:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.495592:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.495593:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:17.0:1619677447.495595:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:20.0:1619677447.495596:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.495596:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.495599:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.495601:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.495602:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.495603:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.495604:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.495605:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:1.0:1619677447.495605:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.495608:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677447.495609:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.495609:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.495610:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677447.495611:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:1.0:1619677447.495611:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.495611:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677447.495614:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.495614:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.495615:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.495616:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.495617:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.495618:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.495619:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:13.0:1619677447.495620:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.495623:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.495624:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.495625:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.495627:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.495628:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.495630:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.495632:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.495633:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.495635:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.495635:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.495637:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:17.0:1619677447.495695:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.495698:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.495698:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.495700:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.495703:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.495705:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.495706:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.495707:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.495708:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.495709:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.495713:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.495715:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.495717:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.495718:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.495718:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.495720:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.495721:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.495722:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.495722:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.495734:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.495736:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.495737:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.495737:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.495739:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.495742:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677447.495761:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.495764:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.495766:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.495767:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.495768:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.495770:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.495771:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.495772:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.495773:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.495774:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.495778:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.495780:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.495783:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.495783:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.495784:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.495786:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.495786:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.495787:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.495788:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677447.495878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.495917:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.495920:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.495920:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.495922:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.495925:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.495926:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.495928:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.495929:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.495930:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.495931:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.495932:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.495934:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.495934:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.495935:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:6.0:1619677447.495935:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.495935:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.495936:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.495937:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:20.0:1619677447.495938:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.495938:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.495940:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.495940:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677447.495941:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (23)++
00000800:00000200:5.0:1619677447.495942:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:6.0:1619677447.495943:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:20.0:1619677447.495944:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:6.0:1619677447.495944:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677447.495945:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:6.0:1619677447.495945:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:5.0:1619677447.495945:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.495946:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.495946:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.495946:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677447.495948:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:24.0:1619677447.495950:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:2.0:1619677447.495951:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.495952:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:8.0:1619677447.495954:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:2.0:1619677447.495955:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.495957:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:8.0:1619677447.495957:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.495957:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:2.0:1619677447.495958:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.495959:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.495960:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:8.0:1619677447.495960:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.495960:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:2.0:1619677447.495960:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:22.0:1619677447.495961:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:5.0:1619677447.495961:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.495961:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.495963:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:8.0:1619677447.495963:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.495963:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.495963:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.495964:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96727386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.495964:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.495965:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.495966:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000400:00000200:8.0:1619677447.495966:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.495966:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000400:00000200:2.0:1619677447.495966:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:20.0:1619677447.495967:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000400:00000200:8.0:1619677447.495967:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.495967:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.495968:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:8.0:1619677447.495968:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.495971:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.495973:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.495974:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.0:1619677447.495976:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.495978:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.495979:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.495979:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.495979:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:8.0:1619677447.495980:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.495981:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.495981:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.495982:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:8.0:1619677447.495983:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.495984:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:2.0:1619677447.495984:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:8.0:1619677447.495985:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:2.0:1619677447.495985:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:8.0:1619677447.495986:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:2.0:1619677447.495986:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677447.495989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.495999:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.496000:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.496000:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.496001:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.496001:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.496002:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.496003:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.496003:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:18.0:1619677447.496004:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.496007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.496012:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.496014:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.496016:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.496017:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.496018:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.496019:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.496020:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677447.496050:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.496052:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.496053:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.496055:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.496055:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.496058:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.496103:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.496106:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.496108:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.496109:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.496111:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.496112:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.496114:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.496115:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.496116:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.496117:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.496120:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.496122:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.496125:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.496126:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.496126:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.496128:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.496128:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.496129:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.496130:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.496141:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.496144:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.496145:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.496145:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.496147:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.496149:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.496278:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.496281:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.496282:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.496283:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.496285:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.496288:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.496294:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.496299:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.496300:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677447.496302:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.496304:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.496306:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.496307:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.496308:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.496309:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.496310:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:4.2:1619677447.496310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:17.0:1619677447.496313:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.496315:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.496317:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.496318:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.496318:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.496319:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.496320:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:24.0:1619677447.496321:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:17.0:1619677447.496321:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.496322:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677447.496322:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677447.496323:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000400:00000200:24.0:1619677447.496324:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.496324:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677447.496327:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.496329:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.496330:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96727379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.496332:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.496333:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.496334:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.496334:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677447.496335:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677447.496336:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:24.0:1619677447.496337:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.496337:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.496338:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677447.496338:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677447.496339:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.496339:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.496339:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.496340:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.496342:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.496343:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.496343:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.496343:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.496345:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.496345:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.496346:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.496347:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.496358:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677447.496358:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.496360:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.496361:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:6.0:1619677447.496361:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.496362:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677447.496362:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.496363:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:17.0:1619677447.496363:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.496363:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.496363:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:17.0:1619677447.496365:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.496365:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:22.0:1619677447.496366:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.496366:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677447.496368:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:22.0:1619677447.496369:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.496370:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.496372:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.496373:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.496374:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.496375:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.496379:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.496380:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.496383:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.496383:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.496384:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.496386:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.496386:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.496387:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.496388:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.496400:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.496402:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.496403:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.496405:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.496406:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.496407:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.496434:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.496437:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.496439:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.496440:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.496441:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.496443:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677447.496444:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.496445:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.496446:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.496448:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677447.496451:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.496453:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677447.496456:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.496456:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677447.496457:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677447.496459:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677447.496459:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.496460:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.496461:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:1.0:1619677447.496516:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677447.496519:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677447.496520:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677447.496522:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.496524:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677447.496524:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.496525:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:5.0:1619677447.496526:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:1.0:1619677447.496527:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.496528:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.496529:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.496529:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.496530:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.496531:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:5.0:1619677447.496532:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.496535:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.496537:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:5.0:1619677447.496538:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.496540:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.496540:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.496541:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:1.0:1619677447.496541:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:5.0:1619677447.496542:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.496543:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.496543:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.496544:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:5.0:1619677447.496544:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.496544:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677447.496544:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:20.0:1619677447.496545:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:1.0:1619677447.496545:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000400:00000200:20.0:1619677447.496549:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.496549:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.496551:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.496551:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.496553:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.496554:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.496555:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.496556:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.496557:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.496560:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.496562:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.496564:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.496565:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.496565:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.496566:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.496566:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.496567:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.496568:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.496568:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.496569:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677447.496569:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.496570:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.496571:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:5.0:1619677447.496571:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677447.496572:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.496573:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677447.496574:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.496576:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.496577:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.496579:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.496581:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.496582:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.496583:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.496585:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.496586:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.496589:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.496589:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.496592:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:2.0:1619677447.496670:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.496673:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.496674:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.496675:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.496679:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.496680:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.496682:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.496683:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.496684:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677447.496684:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.496685:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:17.0:1619677447.496687:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:2.0:1619677447.496688:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.496688:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.496690:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.496690:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:17.0:1619677447.496692:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.496692:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.496693:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:17.0:1619677447.496694:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:2.0:1619677447.496694:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:17.0:1619677447.496695:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.496695:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:17.0:1619677447.496696:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.496696:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.496696:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:17.0:1619677447.496697:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.496697:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:17.0:1619677447.496698:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:9.2:1619677447.496701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:17.0:1619677447.496702:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.496703:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.496706:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.496706:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.496707:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677447.496709:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:17.0:1619677447.496709:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.496709:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.496710:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.496710:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677447.496711:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.496712:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.496713:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.496714:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.496716:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.496723:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.496725:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.496726:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.496726:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.496728:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.496731:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.496872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.496879:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.496882:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.496883:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.496884:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.496887:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.496889:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.496890:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.496891:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.496892:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.496893:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.496896:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.496898:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.496899:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:22.0:1619677447.496900:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677447.496901:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.496902:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.0:1619677447.496902:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.496903:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:8.0:1619677447.496903:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.496903:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677447.496904:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.496905:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:2.0:1619677447.496905:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.496906:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.496906:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.496907:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:2.0:1619677447.496907:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.496908:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000400:00000200:22.0:1619677447.496909:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.496909:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677447.496909:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.496910:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677447.496911:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.496912:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.496913:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.496914:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677447.496914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:2.0:1619677447.496914:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.496915:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.496915:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.496916:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:1.0:1619677447.496919:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:2.0:1619677447.496920:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.496921:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.496921:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:22.0:1619677447.496922:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.496922:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.496923:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:2.0:1619677447.496923:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.496923:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.496924:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.496925:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.496925:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:1.0:1619677447.496925:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.496926:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.496927:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.496928:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.496928:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.496928:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.496929:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.496930:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677447.496930:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.496931:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677447.496931:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:22.0:1619677447.496932:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677447.496932:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:22.0:1619677447.496933:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677447.496933:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:22.0:1619677447.496934:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.496934:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677447.496944:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677447.496945:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:17.0:1619677447.496946:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677447.496947:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:17.0:1619677447.496947:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.496947:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.496948:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.496949:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:17.0:1619677447.496951:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.496953:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.496980:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.496982:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.496983:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.496985:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.496987:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.496989:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.496990:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.496991:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.496992:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.496993:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.496997:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.496999:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.497002:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.497003:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.497004:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.497005:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.497006:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.497007:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.497007:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.497019:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.497021:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.497022:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.497024:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.497026:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.497027:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.497319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.497327:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.497330:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.497331:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.497331:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.497333:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.497335:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:2.0:1619677447.497337:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.497338:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.497339:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.497339:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.497341:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.497344:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.497362:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.497365:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.497366:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.497368:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.497370:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.497372:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.497373:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.497374:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:1.0:1619677447.497374:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.497375:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.497377:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:1.0:1619677447.497377:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677447.497378:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.497382:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.497382:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.497385:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.497385:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:8.0:1619677447.497386:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677447.497388:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96727363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.497388:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.497389:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.497389:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:8.0:1619677447.497390:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.497391:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.497391:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:8.0:1619677447.497392:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.497392:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.497392:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.497393:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677447.497395:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.497397:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.497399:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.497400:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.497401:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.497402:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.497403:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.497404:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.497404:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.497416:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.497418:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.497419:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.497420:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.497423:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.497423:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.497512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.497562:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.497565:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.497567:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.497568:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.497570:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.497571:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.497573:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.497574:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.497575:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.497576:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.497579:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.497581:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.497583:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.497584:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.497585:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.497586:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.497587:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.497588:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.497588:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.497599:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.497602:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.497603:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.497604:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.497605:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.497606:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677447.497671:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.497673:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.497675:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.497676:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.497678:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.497679:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.497680:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.497681:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.497682:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.497683:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.497686:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.497687:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.497689:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.497690:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.497690:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.497692:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.497692:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.497694:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.497695:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.497695:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677447.497705:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677447.497706:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:6.0:1619677447.497708:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677447.497708:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:22.0:1619677447.497709:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.497709:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.497709:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.497710:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:6.0:1619677447.497711:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.497714:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.497722:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.497725:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.497725:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.497726:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.497727:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.497727:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.497737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.497744:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.497746:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.497748:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.497749:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.497752:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.497754:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.497755:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.497756:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.497757:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.497758:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.497761:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.497763:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.497766:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.497767:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.497767:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.497769:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.497770:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.497770:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.497771:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.497783:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.497785:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.497786:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.497787:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.497789:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.497792:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.497800:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.497803:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.497804:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.497804:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.497805:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.497805:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677447.497935:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.497938:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.497939:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.497940:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:18.0:1619677447.497941:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.497942:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.497943:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.497944:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.497946:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:17.0:1619677447.497946:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:18.0:1619677447.497947:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.497949:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.497949:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677447.497949:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:18.0:1619677447.497950:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.497952:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.497952:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.497954:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.497955:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.497955:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.497957:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.497957:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677447.497957:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.497959:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.497959:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.497959:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.497960:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.497961:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.497962:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.497963:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677447.497963:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677447.497970:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.497970:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.497973:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677447.497973:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.497974:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.497975:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677447.497975:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.497976:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.497976:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.497976:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:17.0:1619677447.497977:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:2.0:1619677447.497977:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.497978:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.497978:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:2.0:1619677447.497978:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.497979:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.497979:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.497980:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677447.497980:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.497980:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:2.0:1619677447.497980:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.497981:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677447.497981:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:2.0:1619677447.497981:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.497982:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677447.497982:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677447.497985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.497992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:2.0:1619677447.497992:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677447.497993:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677447.497994:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:6.0:1619677447.497995:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:2.0:1619677447.497995:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.497996:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.497996:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.497997:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.497998:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:2.0:1619677447.497998:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.497999:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.498000:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.0:1619677447.498001:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.498002:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.498002:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.498002:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:8.0:1619677447.498005:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.498007:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.498017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.498057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677447.498076:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.498079:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.498080:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.498082:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.498085:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:1.0:1619677447.498085:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.498086:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:24.0:1619677447.498088:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:1.0:1619677447.498088:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.498089:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:1.0:1619677447.498089:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.498091:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.498092:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.498093:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.498095:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.498096:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.498097:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677447.498098:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96727353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.498098:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.498099:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.498100:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.498101:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.498102:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.498102:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.498102:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.498104:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677447.498105:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.498105:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.498106:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677447.498107:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.498107:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.498108:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677447.498110:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.498111:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677447.498111:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677447.498113:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.498113:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.498114:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.498115:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:17.0:1619677447.498118:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677447.498118:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677447.498120:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.498121:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677447.498121:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.498122:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.498122:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677447.498123:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.498125:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.498126:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.498127:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.498128:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.498129:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.498129:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.498130:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.498132:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.498144:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.498146:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.498147:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.498147:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.498148:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.498149:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.498159:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.498161:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.498162:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.498164:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.498166:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.498167:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.498175:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.498178:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.498179:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.498179:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.498180:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.498181:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.498279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.498286:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.498289:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.498290:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.498291:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.498298:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.498299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.498302:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.498303:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.498305:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.498306:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.498307:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.498307:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.498307:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:4.2:1619677447.498465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.498489:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.498492:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.498493:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.498494:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.498495:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.498497:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.498692:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.498695:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.498697:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.498697:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.498699:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.498700:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.498701:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.498703:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.498704:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.498704:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.498707:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.498709:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.498711:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.498712:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.498712:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.498714:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.498715:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.498716:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.498717:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677447.498718:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.498728:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.498729:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.498730:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:5.0:1619677447.498730:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.498730:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.498731:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.498732:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.498732:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.498733:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.498734:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:17.0:1619677447.498743:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677447.498745:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:17.0:1619677447.498746:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:22.0:1619677447.498747:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:1.0:1619677447.498747:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677447.498748:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.498749:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000400:00000200:17.0:1619677447.498749:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.498750:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677447.498750:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.498750:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:6.0:1619677447.498751:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.498751:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:22.0:1619677447.498752:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.498752:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.498753:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.498754:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.498755:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.498756:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.498757:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.498758:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:17.0:1619677447.498758:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.498761:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.498764:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.498765:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.498766:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.498767:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.498769:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.498769:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.498770:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.498770:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.498782:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.498784:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.498785:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.498785:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.498787:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.498788:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.498802:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.498805:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.498805:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.498806:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.498806:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.498807:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.498842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.498850:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.498852:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.498854:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.498855:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.498858:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.498859:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.498861:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.498863:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.498864:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.498865:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.498869:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.498871:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.498874:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.498874:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.498875:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.498877:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.498877:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.498878:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.498878:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677447.498883:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.498885:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.498887:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.498888:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.498889:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.498890:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.498892:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.498893:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.498893:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.498895:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.498896:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.498897:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.498898:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:1.0:1619677447.498898:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:20.0:1619677447.498899:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.498899:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.498899:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.498900:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.498901:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.498902:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.498905:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.498905:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.498906:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.498907:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.498910:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.498911:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.498911:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.498913:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.498914:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.498915:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.498915:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677447.498916:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.498918:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.498919:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.498920:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.498920:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.498921:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677447.498937:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.498939:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.498940:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.498942:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.498944:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.498945:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.499002:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.499005:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.499006:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.499008:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.499010:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.499012:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.499013:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.499014:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.499015:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.499016:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.499020:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.499022:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.499025:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.499025:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.499026:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.499028:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.499029:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.499030:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.499030:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677447.499051:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.499053:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.499054:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.499056:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.499057:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.499059:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.499131:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.499134:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677447.499136:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677447.499137:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.499139:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.499141:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677447.499142:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.499143:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.499145:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.499145:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.499149:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.499150:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.499153:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.499154:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.499155:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.499156:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.499157:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.499158:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.499158:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677447.499169:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.499172:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.499173:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.499173:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.499175:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677447.499178:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677447.499189:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.499191:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.499192:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.499194:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.499195:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.499197:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677447.499235:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.499238:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.499238:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.499241:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.499244:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.499245:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.499246:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.499247:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.499249:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.499249:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.499253:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.499254:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.499257:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.499257:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.499258:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.499260:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.499261:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.499261:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.499262:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.499274:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.499276:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.499277:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.499279:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.499279:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.499281:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677447.499345:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.499347:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.499348:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.499350:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.499352:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.499353:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.499355:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.499356:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.499357:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.499358:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.499362:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.499365:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.499367:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.499368:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.499368:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.499370:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.499370:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.499371:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.499372:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677447.499384:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.499386:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.499387:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.499388:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.499389:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.499389:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677447.499391:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:12.0:1619677447.499391:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.499392:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.499394:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.499396:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.499397:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.499409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.499455:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.499458:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.499459:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.499460:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.499462:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.499463:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.499473:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.499478:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.499480:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.499481:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.499482:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.499482:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677447.499530:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.499533:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.499533:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.499535:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.499536:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.499537:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.499558:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.499560:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.499562:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.499564:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.499565:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.499566:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677447.499567:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.499569:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.499570:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.499571:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677447.499575:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.499576:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677447.499579:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.499580:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677447.499580:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.499582:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.499583:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.499584:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.499584:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.499588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.499596:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:5.0:1619677447.499598:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:2.0:1619677447.499599:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.499600:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677447.499600:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:2.0:1619677447.499600:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.499602:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.499603:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.499603:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.499605:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:5.0:1619677447.499607:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.499608:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.499609:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.499611:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.499612:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.499613:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.499616:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.499618:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.499622:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.499623:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.499623:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.499625:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:18.0:1619677447.499626:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.0:1619677447.499626:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.499627:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.499627:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:18.0:1619677447.499629:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.499630:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.499631:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.499634:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.499635:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.499637:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.499638:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:18.0:1619677447.499638:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.499639:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.499640:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.499641:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.499641:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.499643:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.499643:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.499644:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.499646:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:18.0:1619677447.499646:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.499648:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.499649:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.499649:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.499651:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.499652:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.499652:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.499653:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677447.499665:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.499668:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.499669:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.499669:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.499671:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.499673:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.499780:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.499783:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.499784:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.499785:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.499787:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.499788:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.499823:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.499825:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.499826:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.499828:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.499828:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:22.0:1619677447.499830:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.499830:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.499831:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.499832:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.499833:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.499833:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.499834:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:22.0:1619677447.499835:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.499835:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:22.0:1619677447.499836:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.499837:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.499840:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.499841:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.499844:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.499844:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.499845:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677447.499845:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677447.499847:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.499847:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.499848:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677447.499848:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.499849:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.499849:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.499849:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.499850:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.499852:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.499860:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.499863:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.499864:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.499864:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.499867:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.499869:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.499895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.499941:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.499944:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.499944:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.499945:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.499947:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.499950:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.500120:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.500123:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.500125:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.500126:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.500127:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.500129:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.500131:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.500132:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.500133:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.500134:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.500139:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.500142:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.500145:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.500146:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.500146:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.500148:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.500149:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.500150:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.500150:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.500161:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.500164:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.500164:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.500166:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.500167:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.500169:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.500276:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.500279:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.500279:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.500282:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.500283:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.500285:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.500286:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.500288:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.500289:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.500290:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.500297:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.500299:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.500302:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.500303:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.500303:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.500305:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.500306:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.500306:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.500307:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.500312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.500319:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.500322:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.500323:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.500323:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.500325:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.500328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.500331:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.500332:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.500334:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.500335:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.500335:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.500336:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.500337:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677447.500409:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.500411:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.500413:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.500414:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.500415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:8.0:1619677447.500417:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.500419:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.500420:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.500422:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.500423:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:8.0:1619677447.500423:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.500424:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.500425:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.500427:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.500429:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.500429:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.500431:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:8.0:1619677447.500431:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.500432:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96727323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.500433:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.500434:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.500435:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.500438:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.500439:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.500439:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.500440:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.500441:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.500442:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.500443:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.500444:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.500444:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.500445:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.500445:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.500446:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.500446:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.500447:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.500448:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.500448:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.500449:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677447.500457:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.0:1619677447.500458:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677447.500459:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:8.0:1619677447.500459:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:20.0:1619677447.500460:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.500461:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.500461:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:1.0:1619677447.500461:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:20.0:1619677447.500462:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.500462:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677447.500463:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677447.500464:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:20.0:1619677447.500465:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.500465:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.500465:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.500467:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.500471:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:22.0:1619677447.500481:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.500483:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.500484:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.500486:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.500488:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.500490:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.500491:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.500492:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.500493:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.500494:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:4.2:1619677447.500496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:22.0:1619677447.500497:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.500498:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.500501:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.500502:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.500502:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.500504:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.500504:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.500505:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.500506:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.500507:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677447.500517:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677447.500517:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.500519:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:5.0:1619677447.500519:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677447.500520:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:6.0:1619677447.500520:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:17.0:1619677447.500521:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.500521:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.500522:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:1.0:1619677447.500522:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.500524:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:17.0:1619677447.500524:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.500525:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:17.0:1619677447.500527:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:5.0:1619677447.500528:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.500530:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.500531:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.500532:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.500533:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.500534:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.500538:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.500540:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:4.2:1619677447.500540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677447.500543:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.500543:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.500544:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677447.500544:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.500546:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.500546:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:22.0:1619677447.500547:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677447.500547:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:6.0:1619677447.500548:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.500548:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:22.0:1619677447.500549:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.500551:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.500554:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.500563:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.500565:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.500566:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.500567:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.500568:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.500568:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.500627:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.500630:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.500631:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.500632:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.500634:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.500635:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.500636:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.500636:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.500637:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.500638:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.500641:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.500642:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.500643:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.500645:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.500646:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.500647:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.500650:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.500652:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.500654:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.500655:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.500656:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.500657:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.500658:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.500658:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.500659:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677447.500670:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.500673:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.500673:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.500675:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.500676:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.500677:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:1.0:1619677447.500833:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.500836:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.500837:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.500837:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.500839:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.500842:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.500851:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.500854:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.500854:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677447.500855:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.500855:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.500856:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.500856:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:17.0:1619677447.500857:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.0:1619677447.500859:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.500859:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.500861:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.500863:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677447.500864:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.500865:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.500866:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.500867:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677447.500871:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.500872:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.500875:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.500875:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677447.500876:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.500878:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.500878:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.500880:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.500881:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.500882:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.500889:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.500891:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:1.0:1619677447.500891:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:20.0:1619677447.500892:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.500893:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677447.500893:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.500894:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.500894:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.500896:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.500898:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:22.0:1619677447.500906:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.500909:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.500910:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.500912:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.500914:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.500916:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.500917:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.500918:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.500919:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.500920:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.500922:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.500924:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.500927:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.500927:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.500928:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.500930:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.500930:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.500931:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.500931:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677447.500943:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.500945:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.500946:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.500946:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.500948:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.500950:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.500960:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.500962:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.500963:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.500964:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.500964:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.500965:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.501039:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.501042:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.501043:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.501043:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.501046:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.501048:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.501077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.501117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.501125:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.501128:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.501129:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.501129:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.501132:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.501135:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.501144:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.501147:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.501147:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.501149:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.501152:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.501154:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.501155:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.501156:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.501157:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.501158:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.501162:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.501164:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.501167:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.501168:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.501168:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.501170:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.501170:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.501172:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.501173:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.501173:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.501176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.501183:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677447.501184:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677447.501185:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:17.0:1619677447.501186:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:24.0:1619677447.501187:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.501187:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677447.501187:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:1.0:1619677447.501188:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677447.501190:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.501193:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:18.0:1619677447.501213:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.0:1619677447.501216:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:18.0:1619677447.501216:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.501216:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.0:1619677447.501218:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:18.0:1619677447.501218:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.501219:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.501221:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.501221:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.501222:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:5.0:1619677447.501222:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.501224:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:18.0:1619677447.501224:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727309] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.501225:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.501226:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.501227:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.501230:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.501232:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.501234:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.501235:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.501236:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.501236:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fdae00] (21)++
00000800:00000200:18.0:1619677447.501237:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.501238:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.501238:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677447.501238:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.38@o2ib3
00000800:00000200:18.0:1619677447.501239:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677447.501241:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (22)--
00000400:00000200:5.0:1619677447.501241:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.38@o2ib3 : ACK - routed
00000800:00000200:9.2:1619677447.501243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677447.501243:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.501245:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.501247:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.501248:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.501249:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.501250:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:5.0:1619677447.501251:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.501252:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.501253:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.501254:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:5.0:1619677447.501254:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.38@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:2.0:1619677447.501255:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.501256:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.501257:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677447.501258:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.501259:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.501260:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.501262:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.501262:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fdae00] (21)++
00000800:00000200:5.0:1619677447.501263:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fdae00] (22)--
00000800:00000200:5.0:1619677447.501264:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (21)--
00000800:00000200:6.0:1619677447.501275:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.501278:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.501279:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.501279:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.501281:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.501283:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.501466:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.501469:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.501469:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:4.2:1619677447.501469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:22.0:1619677447.501471:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.501474:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.501475:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.501476:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.501476:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:22.0:1619677447.501477:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.501479:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.501479:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.501480:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677447.501480:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677447.501483:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.501484:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.501486:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.501487:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:8.0:1619677447.501487:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.501489:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96727305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.501490:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.501490:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000400:00000200:8.0:1619677447.501490:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.501491:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:8.0:1619677447.501491:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.501492:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.501493:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.501493:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.501494:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.501495:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:8.0:1619677447.501496:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.501498:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.501501:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.501501:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.501502:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.501504:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.501504:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.501506:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.501507:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.501508:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.501513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.501518:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.501520:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677447.501521:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677447.501522:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677447.501522:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.501522:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.501523:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.501524:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677447.501524:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.501527:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677447.501565:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.501568:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.501569:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.501571:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.501573:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.501574:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.501576:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.501577:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.501578:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.501579:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.501582:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.501584:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.501586:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.501587:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.501588:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.501589:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.501590:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.501590:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.501591:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677447.501603:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.501605:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.501606:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.501606:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.501608:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.501610:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.501684:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.501687:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.501687:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.501689:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.501692:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.501693:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.501694:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.501695:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.501696:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.501697:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.501700:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.501702:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.501703:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.501704:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.501704:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.501706:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.501706:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.501707:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.501707:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.501718:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.501720:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.501721:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.501722:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.501722:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.501724:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.501736:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.501739:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.501740:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.501740:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.501740:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.501741:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.501754:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.501757:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.501758:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:4.2:1619677447.501759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677447.501760:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.501762:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.501763:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.501764:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.501766:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.501767:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.501768:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.501769:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.501771:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677447.501771:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:2.0:1619677447.501772:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.501773:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.501774:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.501774:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.501775:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.501775:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677447.501776:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.501777:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.501777:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.501778:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.501779:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:2.0:1619677447.501779:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.501779:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.501780:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:20.0:1619677447.501781:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.501781:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:20.0:1619677447.501782:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.501782:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677447.501783:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:2.0:1619677447.501783:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000400:00000200:20.0:1619677447.501784:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.501784:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000400:00000200:20.0:1619677447.501785:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.501785:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:2.0:1619677447.501785:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677447.501787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677447.501788:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.501791:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.501792:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.501793:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:20.0:1619677447.501794:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:18.0:1619677447.501795:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:5.0:1619677447.501795:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96727298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.501797:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:5.0:1619677447.501797:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.501798:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.501798:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.501798:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:5.0:1619677447.501799:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:2.0:1619677447.501799:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.501799:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.501800:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.501801:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.501801:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677447.501803:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:5.0:1619677447.501803:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.501806:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.501807:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.501807:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.501808:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.501808:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.501809:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.501810:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.501810:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.501811:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.501811:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.501812:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.501812:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.501813:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677447.501813:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677447.501814:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.501814:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:6.0:1619677447.501816:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.501819:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.501820:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.501821:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.501823:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.501824:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.501826:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:6.0:1619677447.501826:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:17.0:1619677447.501827:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.501828:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:1.0:1619677447.501828:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.501829:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.501830:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:17.0:1619677447.501831:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:5.0:1619677447.501831:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.501833:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.501834:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.501835:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:1.0:1619677447.502059:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.502062:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.502062:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.502063:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.502065:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.502069:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.502070:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.502071:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.502071:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.502074:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.502075:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.502077:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.502077:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.502079:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.502080:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.502081:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.502084:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.502086:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.502088:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.502088:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.502089:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.502090:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.502091:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.502091:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.502092:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677447.502103:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.502105:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.502106:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.502107:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.502107:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.502109:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.502184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.502230:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.502233:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.502233:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.502234:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.502236:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.502239:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.502261:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.502264:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.502265:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.502266:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.502269:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.502271:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.502272:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.502273:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.502274:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.502275:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.502279:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.502280:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.502283:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.502284:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.502285:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.502287:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.502287:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.502288:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.502288:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.502293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.502302:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.502304:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.502306:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:2.0:1619677447.502306:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.502308:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.502308:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (23)++
00000400:00000200:2.0:1619677447.502308:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.502309:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (24)--
00000800:00000200:18.0:1619677447.502310:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.502311:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.502312:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.502326:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.502328:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.502329:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.502331:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.502333:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.502334:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.502335:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.502337:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.502338:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.502339:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.502342:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.502344:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.502346:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.502347:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.502348:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.502349:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.502350:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.502351:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.502352:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.502352:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.502362:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.502364:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.502365:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677447.502365:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677447.502366:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677447.502367:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.502368:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.502369:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.502370:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.502372:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.502457:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.502459:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.502460:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.502462:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.502463:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.502464:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.502466:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.502467:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:17.0:1619677447.502467:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.502468:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.502469:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:17.0:1619677447.502469:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.502471:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.502473:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677447.502473:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.502475:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.502476:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.502477:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.502478:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:17.0:1619677447.502478:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:18.0:1619677447.502479:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.502479:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:18.0:1619677447.502480:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.502480:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:18.0:1619677447.502481:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.502481:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.502481:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677447.502482:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677447.502482:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:18.0:1619677447.502486:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.502488:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.502490:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.502491:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.502491:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.502493:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.502493:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677447.502493:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677447.502494:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.502495:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:1.0:1619677447.502495:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677447.502496:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:1.0:1619677447.502496:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.502497:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.502498:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.502500:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677447.502507:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677447.502507:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.502509:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:6.0:1619677447.502510:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677447.502510:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677447.502511:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677447.502511:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:6.0:1619677447.502511:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.502511:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:6.0:1619677447.502513:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:17.0:1619677447.502514:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.502515:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.502515:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.502516:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.502516:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.502517:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677447.502526:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.502529:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.502529:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.502530:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.502531:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.502531:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.502538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.502545:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.502548:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.502549:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.502551:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.502554:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.502555:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.502557:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.502558:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.502559:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.502560:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.502563:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.502565:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.502568:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.502569:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.502570:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.502571:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.502572:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.502572:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.502573:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.502585:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.502588:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.502589:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.502589:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.502591:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.502594:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.502602:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.502604:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.502605:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.502606:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.502607:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.502607:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.502628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.502678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677447.502683:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.502686:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.502686:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.502687:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.502689:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.502691:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677447.502794:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.502796:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.502798:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.502800:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.502801:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.502802:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.502803:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.502805:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.502806:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.502807:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.502810:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.502811:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.502814:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.502814:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.502815:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.502817:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.502817:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.502819:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.502820:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677447.502820:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.502831:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.502832:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.502833:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000400:00000200:5.0:1619677447.502833:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.502834:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677447.502834:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.502835:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.502835:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.502836:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.502837:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677447.502846:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.502848:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.502849:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.502850:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.502850:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.502852:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:18.0:1619677447.503034:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.503037:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.503037:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.503040:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.503042:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.503044:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.503045:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.503046:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.503047:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.503048:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.503052:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.503053:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.503056:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.503057:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.503058:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.503060:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.503060:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.503061:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.503061:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.503073:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.503076:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.503077:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.503079:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.503080:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.503082:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.503122:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.503125:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.503126:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.503128:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.503131:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.503133:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.503135:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.503136:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:2.0:1619677447.503137:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.503138:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.503139:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.503139:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.503140:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:13.0:1619677447.503141:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.503142:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.503143:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677447.503147:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.503150:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.503153:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.503154:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.503155:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.503157:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.503158:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.503159:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.503159:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.503165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.503172:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.503175:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.503176:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.503176:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.503179:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.503182:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.503184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.503248:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.503251:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.503253:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.503254:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.503255:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.503257:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.503258:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.503259:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.503260:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.503261:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.503264:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.503266:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.503268:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.503269:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.503270:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.503271:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.503272:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.503272:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.503273:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.503284:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.503287:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.503288:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.503288:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.503290:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.503292:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.503297:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.503300:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.503302:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.503303:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.503305:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.503306:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677447.503307:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.503309:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.503310:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.503310:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677447.503314:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.503315:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677447.503318:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.503319:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677447.503319:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677447.503321:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677447.503322:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.503323:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.503324:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.503325:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.503335:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.503338:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.503339:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.503340:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677447.503349:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.503352:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.503353:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.503355:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.503356:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.503358:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.503360:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.503361:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.503362:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.503363:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.503366:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.503368:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.503371:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.503372:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.503372:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.503374:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.503375:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.503376:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.503377:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.503378:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.503388:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.503390:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.503391:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.503392:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.503596:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.503599:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.503600:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.503601:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.503603:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.503606:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.503709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.503717:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.503720:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.503721:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.503723:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.503726:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.503727:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.503728:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727276] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.503729:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.503730:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.503731:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.503735:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.503736:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.503739:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.503740:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.503741:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.503742:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.503743:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.503743:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.503744:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.503756:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.503758:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.503759:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.503760:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.503762:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.503764:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.503799:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.503802:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.503802:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.503804:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.503806:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.503808:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.503809:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.503810:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.503811:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.503812:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.503815:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.503817:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.503820:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.503821:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.503821:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677447.503822:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:1.0:1619677447.503824:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.503824:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.503825:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:1.0:1619677447.503825:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.503825:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677447.503827:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.503827:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.503830:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.503831:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.503833:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.503834:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.503835:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.503836:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:12.0:1619677447.503837:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:5.0:1619677447.503839:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.503840:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.503840:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.503842:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:12.0:1619677447.503843:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.503843:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.503844:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677447.503844:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:12.0:1619677447.503845:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.503845:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.503846:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.503847:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.503848:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.503849:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677447.503860:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.503862:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.503863:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.503864:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.503866:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.503867:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.0:1619677447.504010:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.504013:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.504015:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677447.504015:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:5.0:1619677447.504015:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.504017:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:5.0:1619677447.504017:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.504018:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.504019:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:13.0:1619677447.504020:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.504021:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677447.504021:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.504023:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677447.504023:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.504024:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.504025:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677447.504028:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.504030:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.504032:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.504033:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.504033:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.504033:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.504035:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:12.0:1619677447.504036:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.504036:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.504036:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677447.504037:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.504037:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.504038:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.504038:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.504039:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.504048:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.504050:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.504051:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.504053:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.504054:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.504055:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.504066:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.504069:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.504070:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.504070:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.504071:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.504072:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.504155:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.504157:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.504158:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.504160:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.504163:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.504164:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.504165:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.504167:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.504168:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.504169:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.504173:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.504174:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677447.504176:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.504177:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.504177:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:17.0:1619677447.504179:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.504180:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.504181:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:6.0:1619677447.504181:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.504183:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.504183:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.504184:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.504184:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677447.504184:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.504186:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.504187:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.504188:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.504189:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.504190:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.504194:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.504196:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.2:1619677447.504198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.504199:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.504199:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.504200:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.504202:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.504202:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.504203:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.504204:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.504206:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:9.2:1619677447.504208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.504208:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.504210:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.504211:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.504212:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.504213:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.504214:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.504214:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.504215:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:18.0:1619677447.504216:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.504216:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.504219:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677447.504245:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.504249:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.504250:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.504251:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.504254:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.504255:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.504256:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.504258:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.504259:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.504260:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.504263:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.504265:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.504268:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.504269:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.504269:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.504271:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.504271:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.504272:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.504272:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677447.504283:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.504286:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.504287:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.504289:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.504290:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.504292:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.504395:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.504398:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.504399:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.504400:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.504401:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.504408:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677447.504431:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.504434:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.504435:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.504436:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.504438:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.504439:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.504558:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.504561:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.504562:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.504564:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.504567:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.504568:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.504570:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.504571:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.504572:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.504573:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.504576:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.504578:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.504580:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.504581:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.504581:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.504583:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.504583:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.504584:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.504584:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677447.504596:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.504598:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.504598:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.504599:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.504600:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.504602:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.0:1619677447.504672:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.504675:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.504677:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.504678:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.504679:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.504680:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.504682:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.504683:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.504684:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.504685:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.504688:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.504690:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.504693:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.504694:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.504694:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.504696:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.504697:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.504698:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.504698:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:4.2:1619677447.504749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.504795:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.504798:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.504799:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.504799:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.504802:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.504805:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.504894:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677447.504896:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677447.504897:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677447.504899:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.504901:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.504903:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677447.504904:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.504906:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.504907:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.504909:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677447.504913:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.504916:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677447.504919:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.504920:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:1.0:1619677447.504921:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:1.0:1619677447.504922:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:1.0:1619677447.504923:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677447.504924:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677447.504925:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677447.504926:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:5.0:1619677447.504935:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.504938:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.504940:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.504940:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.504943:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.504944:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:5.0:1619677447.504945:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.504946:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.504947:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.504947:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.504947:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.504949:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677447.504950:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.504950:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.504952:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:5.0:1619677447.504953:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:17.0:1619677447.504955:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:20.0:1619677447.504957:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:5.0:1619677447.504957:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.504958:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:17.0:1619677447.504958:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677447.504959:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:5.0:1619677447.504959:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:1.0:1619677447.504959:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:20.0:1619677447.504961:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.504961:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.504961:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.504962:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:12.0:1619677447.504963:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:20.0:1619677447.504964:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.504964:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.504965:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:12.0:1619677447.504966:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:5.0:1619677447.504966:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.504967:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:12.0:1619677447.504967:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677447.504967:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:20.0:1619677447.504968:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.504968:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.504968:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677447.504968:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.504969:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.504970:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.504970:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:20.0:1619677447.504971:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.504975:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.504976:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.504977:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.504977:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.504978:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.504980:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677447.504980:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:20.0:1619677447.504981:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.504982:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.504982:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.504983:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.504983:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677447.504984:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.504985:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.504987:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.504987:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677447.504996:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.504998:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677447.504999:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.505000:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.505000:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:13.0:1619677447.505001:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.505001:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.505001:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.505003:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.505004:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.505052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.505060:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.505063:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.505064:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.505065:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.505068:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.505070:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.505071:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.505073:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.505074:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.505075:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.505078:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.505080:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.505082:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.505083:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.505084:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.505085:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.505086:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.505086:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.505087:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.505099:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.505102:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.505103:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.505103:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.505105:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.505109:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.505110:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.505112:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.505112:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.505115:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.505117:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.505118:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.505119:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.505120:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.505121:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.505122:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.505126:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.505127:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.505130:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.505131:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.505132:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.505133:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.505134:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.505135:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.505136:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.505136:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.505140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.505146:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:1.0:1619677447.505147:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677447.505149:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677447.505149:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:24.0:1619677447.505150:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.505150:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.505150:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.505151:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677447.505152:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.505156:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677447.505199:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.505202:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.505203:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.505205:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.505207:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.505208:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.505209:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.505211:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.505212:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.505213:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.505215:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:6.0:1619677447.505216:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.505217:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:6.0:1619677447.505218:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.505218:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.505221:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.505221:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.505222:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.505223:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:18.0:1619677447.505224:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.505224:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:18.0:1619677447.505225:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:6.0:1619677447.505225:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:18.0:1619677447.505226:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.505226:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.505227:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000400:00000200:18.0:1619677447.505228:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.505228:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:18.0:1619677447.505229:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.505230:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.505233:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.505235:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.505238:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677447.505238:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.505238:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677447.505239:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.505239:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677447.505240:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.505241:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677447.505241:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.505241:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677447.505242:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.505242:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:17.0:1619677447.505242:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.505242:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:1.0:1619677447.505242:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.505243:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:1.0:1619677447.505244:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.505247:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:12.0:1619677447.505254:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.505256:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.505257:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.505259:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.505261:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.505262:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.505272:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.505275:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.505275:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.505276:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.505276:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.505278:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.505335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.505370:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.505372:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.505373:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.505374:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.505376:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.505377:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.505385:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.505388:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.505389:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.505391:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.505391:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.505394:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677447.505402:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.505405:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.505405:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.505407:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.505409:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.505410:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.505638:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.505640:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.505641:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.505643:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.505646:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.505647:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.505648:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.505650:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.505651:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.505652:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.505655:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.505657:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.505660:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.505660:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.505661:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.505663:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.505663:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.505664:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.505664:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677447.505677:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.505680:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.505681:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.505681:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.505683:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.505686:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:12.0:1619677447.505712:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.505714:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.505715:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.505716:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.505718:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.505719:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677447.505723:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.505725:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.505728:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.505729:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.505730:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.505732:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.505733:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.505734:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.505736:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.505737:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.505740:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.505742:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.505745:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.505746:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.505746:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.505748:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.505749:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.505750:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.505750:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677447.505869:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.505872:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.505874:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.505874:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.505877:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.505878:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.505879:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.505881:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.505882:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.505883:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.505886:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.505887:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 440 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.505890:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.505891:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.505891:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.505894:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.505894:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.505895:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.505896:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:9.2:1619677447.505900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.505907:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.505910:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.505911:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.505912:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.505913:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.505915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.505916:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.505957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.505962:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.505965:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.505965:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.505966:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.505968:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.505972:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.505980:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.505983:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.505983:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.505984:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.505985:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.505986:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.506006:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.506008:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.506009:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.506010:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.506012:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.506013:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.506027:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.506030:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.506031:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.506033:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.506035:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.506036:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.506038:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.506039:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.506040:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.506041:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.506044:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.506046:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677447.506047:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677447.506048:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.506049:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.506049:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.506050:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.506051:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677447.506051:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.506051:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.506052:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:6.0:1619677447.506052:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.506053:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:6.0:1619677447.506055:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.506056:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:9.2:1619677447.506057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677447.506057:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.506059:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.506059:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.506061:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.506064:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:6.0:1619677447.506065:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.506066:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.506067:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.506068:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677447.506068:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.506070:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.506071:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.506072:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.506072:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.506072:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.506074:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.506074:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.506075:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.506076:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.506080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.506087:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.506089:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677447.506090:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.506091:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677447.506091:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677447.506092:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:17.0:1619677447.506092:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.506093:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.506093:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:1.0:1619677447.506093:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.506095:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.506097:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:12.0:1619677447.506098:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.506100:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.506101:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.506103:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.506104:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.506105:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677447.506107:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.506110:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.506111:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.506111:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.506113:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.506115:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.506117:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677447.506118:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677447.506119:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677447.506121:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.506124:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.506125:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.506126:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.506128:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.506129:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.506130:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.506133:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.506135:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.506138:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.506139:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.506140:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.506142:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.506142:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677447.506143:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677447.506143:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:13.0:1619677447.506154:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.506157:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.506158:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.506159:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.506161:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.506162:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.506290:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.506294:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.506299:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.506302:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.506305:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.506306:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.506308:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.506309:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.506310:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.506311:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.506313:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.506315:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.506316:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.506317:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.506317:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.506319:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.506319:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.506320:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.506320:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677447.506324:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.506326:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.506329:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.506331:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.506332:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.506333:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:20.0:1619677447.506334:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.506335:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.506335:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.506336:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.506337:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.506337:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:20.0:1619677447.506338:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.506338:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.506339:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.506340:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:20.0:1619677447.506343:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.506345:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.506348:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.506349:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.506350:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.506352:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.506352:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.506353:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.506353:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.506365:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677447.506367:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:12.0:1619677447.506368:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.506369:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.506369:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.506370:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:12.0:1619677447.506371:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.506371:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:1.0:1619677447.506371:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.506372:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:1.0:1619677447.506373:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.506374:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.506374:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.506375:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.506377:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.506379:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.506379:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:2.0:1619677447.506382:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677447.506383:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.506384:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:2.0:1619677447.506385:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.506386:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.506387:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.506388:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.506388:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:2.0:1619677447.506388:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:17.0:1619677447.506390:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:13.0:1619677447.506390:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.506391:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:2.0:1619677447.506391:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677447.506393:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:2.0:1619677447.506393:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677447.506394:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.506394:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.506395:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.506396:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:2.0:1619677447.506396:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.506397:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677447.506397:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677447.506399:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677447.506400:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.506401:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.506402:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677447.506404:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.506407:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.506407:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.506409:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.506411:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.506411:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.506521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.506551:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.506554:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.506555:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.506557:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.506559:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.506561:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.506562:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96727233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.506563:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.506563:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:24.0:1619677447.506564:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.506565:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:24.0:1619677447.506566:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.506568:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.506568:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:22.0:1619677447.506569:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:24.0:1619677447.506570:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.506571:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.506572:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:24.0:1619677447.506573:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:22.0:1619677447.506573:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.506573:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:5.0:1619677447.506573:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.506574:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96727232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.506575:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.506576:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.506576:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.506578:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677447.506578:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.506579:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.506579:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.506579:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.506582:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.506582:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677447.506582:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:5.0:1619677447.506582:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677447.506583:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.506583:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677447.506583:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.506584:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.506585:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.506587:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.506587:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.506587:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:9.2:1619677447.506588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.506588:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.506590:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.506590:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.506591:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.506591:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.506592:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.506592:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677447.506593:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.506593:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.506594:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.506594:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.506595:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677447.506595:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.506598:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.506599:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.506599:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.506601:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.506602:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677447.506604:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.506605:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.506606:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.506608:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677447.506609:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.506610:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.506618:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.506621:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.506622:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.506624:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.506626:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.506627:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.506628:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.506630:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677447.506630:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.506631:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.506632:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677447.506633:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.506634:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677447.506636:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.506637:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.506639:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:6.0:1619677447.506640:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.506641:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677447.506642:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:2.0:1619677447.506642:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.506642:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677447.506643:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.506644:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.506644:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.506644:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:6.0:1619677447.506645:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.506645:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.506645:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:6.0:1619677447.506646:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:9.2:1619677447.506650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677447.506650:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.506652:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.506654:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.506655:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.506656:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:18.0:1619677447.506657:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677447.506657:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.506658:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677447.506659:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:6.0:1619677447.506659:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.506659:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:18.0:1619677447.506660:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.506661:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.506662:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.506664:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.506697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c78b90000] (20)++
00000800:00000200:8.0:1619677447.506706:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c78b90000] (21)++
00000800:00000200:8.0:1619677447.506708:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.26@o2ib3
00000800:00000200:24.0:1619677447.506709:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78b90000] (22)--
00000400:00000200:8.0:1619677447.506711:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.26@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.506714:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.506715:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:8.0:1619677447.506717:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.506718:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.506719:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.506720:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:8.0:1619677447.506723:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.26@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.506724:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:8.0:1619677447.506727:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.506728:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:8.0:1619677447.506729:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:8.0:1619677447.506730:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:8.0:1619677447.506731:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c78b90000] (21)++
00000800:00000200:8.0:1619677447.506732:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c78b90000] (22)--
00000800:00000200:8.0:1619677447.506732:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78b90000] (21)--
00000800:00000200:4.2:1619677447.506735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c78b90000] (20)++
00000800:00000200:24.0:1619677447.506742:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c78b90000] (21)++
00000800:00000200:24.0:1619677447.506744:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.26@o2ib3
00000800:00000200:5.0:1619677447.506745:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:24.0:1619677447.506746:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.26@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.506746:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78b90000] (22)--
00000400:00000200:24.0:1619677447.506748:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.506748:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.506749:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.506750:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:20.0:1619677447.506751:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.506751:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.506752:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96727228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.506753:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.506754:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.506755:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677447.506758:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.26@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.506760:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:5.0:1619677447.506760:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:24.0:1619677447.506761:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.506762:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.506762:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.506764:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.506764:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c78b90000] (21)++
00000800:00000200:24.0:1619677447.506765:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c78b90000] (22)--
00000800:00000200:24.0:1619677447.506765:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78b90000] (21)--
00000800:00000200:9.2:1619677447.506769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.506775:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.506778:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.506778:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.506779:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.506780:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.506781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c78b90000] (20)++
00000800:00000200:8.0:1619677447.506784:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677447.506785:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c78b90000] (21)++
00000800:00000200:8.0:1619677447.506787:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.26@o2ib3
00000800:00000200:24.0:1619677447.506789:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78b90000] (22)--
00000400:00000200:8.0:1619677447.506789:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.26@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677447.506790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000400:00000200:8.0:1619677447.506790:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.506792:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677447.506794:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.506795:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.506796:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000400:00000200:8.0:1619677447.506796:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.506797:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:24.0:1619677447.506798:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.506798:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.506799:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.506800:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.506801:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:8.0:1619677447.506801:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.26@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.506803:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.506806:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.506806:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677447.506807:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677447.506808:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.506809:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c78b90000] (21)++
00000800:00000200:8.0:1619677447.506809:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c78b90000] (22)--
00000800:00000200:8.0:1619677447.506810:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78b90000] (21)--
00000800:00000200:22.0:1619677447.506822:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.506825:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.506825:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.506826:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.506828:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.506831:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.506850:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.506853:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.506854:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.506856:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.506857:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.506859:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:4.2:1619677447.506860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c78b90000] (20)++
00000400:00000200:1.0:1619677447.506860:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.506862:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.506863:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.506864:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:24.0:1619677447.506867:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c78b90000] (21)++
00000400:00000200:1.0:1619677447.506867:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.506869:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.26@o2ib3
00000800:00000200:1.0:1619677447.506869:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:8.0:1619677447.506870:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78b90000] (22)--
00000800:00000200:1.0:1619677447.506872:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.506873:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.26@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.506873:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.506874:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.506875:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:24.0:1619677447.506876:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.506876:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:24.0:1619677447.506877:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:1.0:1619677447.506877:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.506877:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677447.506878:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96727225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.506880:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.506881:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.506882:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:24.0:1619677447.506885:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.26@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.506887:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.506888:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.506889:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:6.0:1619677447.506889:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:24.0:1619677447.506890:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.506891:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.506891:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c78b90000] (21)++
00000800:00000200:6.0:1619677447.506891:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:24.0:1619677447.506893:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919c78b90000] (22)++
00000800:00000200:22.0:1619677447.506893:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.506893:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.506894:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c78b90000] (23)--
00000800:00000200:24.0:1619677447.506895:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78b90000] (22)--
00000400:00000200:6.0:1619677447.506895:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.506896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:6.0:1619677447.506897:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.506900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c78b90000] (21)++
00000800:00000200:8.0:1619677447.506905:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.506906:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.506907:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.506909:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c78b90000] (22)++
00000800:00000200:24.0:1619677447.506910:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919c78b90000] (23)--
00000800:00000200:24.0:1619677447.506910:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78b90000] (22)--
00000400:00000200:8.0:1619677447.506910:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.506911:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677447.506911:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.506912:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78b90000] (21)--
00000800:00000200:2.0:1619677447.506987:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.506990:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.506991:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.506992:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.506994:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.506995:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.506997:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.506998:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.506999:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.507000:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.507003:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.507005:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.507006:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.507007:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.507008:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.507009:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.507009:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.507011:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.507012:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.507012:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677447.507015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.507023:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.507024:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.507025:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.507025:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.507025:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.507026:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.507027:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.507027:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.507028:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.507028:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:4.2:1619677447.507059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.507085:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.507087:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.507090:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.507091:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.507093:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.507095:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.507096:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.507098:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.507099:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.507100:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.507104:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.507107:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.507109:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.507110:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.507111:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.507112:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.507113:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.507115:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.507116:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677447.507116:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:22.0:1619677447.507121:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.507124:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.507125:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.507125:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.507128:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.507128:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.507129:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677447.507129:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.507130:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677447.507130:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.507130:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.507131:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.507132:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.507132:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.507133:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.507134:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677447.507147:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.507149:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.507150:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.507151:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.507151:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.507153:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:17.0:1619677447.507279:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.507282:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.507282:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.507284:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.507287:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.507288:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.507289:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.507291:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.507292:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.507296:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.507300:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.507302:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.507304:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.507305:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.507306:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.507307:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.507308:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.507308:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.507309:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677447.507317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.507322:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.507324:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.507325:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.507326:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.507328:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.507330:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.507340:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.507342:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.507343:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.507344:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.507345:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.507345:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.507364:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.507367:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.507367:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.507368:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.507370:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.507373:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.507386:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.507388:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.507389:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.507389:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.507390:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.507391:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.507415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c78b90000] (20)++
00000800:00000200:24.0:1619677447.507423:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c78b90000] (21)++
00000800:00000200:24.0:1619677447.507426:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.26@o2ib3
00000800:00000200:8.0:1619677447.507428:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78b90000] (22)--
00000400:00000200:24.0:1619677447.507429:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.26@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.507431:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.507433:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.507434:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.507436:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.507436:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.507437:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.507441:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.26@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.507442:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.507445:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.507446:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.507446:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.507448:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:4.2:1619677447.507448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c78b90000] (21)++
00000800:00000200:24.0:1619677447.507449:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c78b90000] (22)++
00000800:00000200:24.0:1619677447.507450:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c78b90000] (23)--
00000800:00000200:24.0:1619677447.507451:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78b90000] (22)--
00000800:00000200:24.0:1619677447.507452:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c78b90000] (21)++
00000800:00000200:24.0:1619677447.507454:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.26@o2ib3
00000400:00000200:24.0:1619677447.507456:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.26@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.507459:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.507461:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:8.0:1619677447.507461:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78b90000] (22)--
00000400:00000200:24.0:1619677447.507462:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96727217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.507463:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.507464:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.507464:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:24.0:1619677447.507465:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.507466:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.507467:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.507468:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.26@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.507469:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.507469:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.507470:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.507475:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:24.0:1619677447.507477:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.507478:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.507478:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.507478:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:24.0:1619677447.507480:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.507480:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c78b90000] (21)++
00000800:00000200:24.0:1619677447.507481:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c78b90000] (22)--
00000800:00000200:18.0:1619677447.507481:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:24.0:1619677447.507482:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78b90000] (21)--
00000800:00000200:2.0:1619677447.507482:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.507484:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.507486:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.507487:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.507489:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.507490:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.507491:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.507492:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.507494:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677447.507495:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:18.0:1619677447.507496:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.507497:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.507497:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.507498:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.507498:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:1.0:1619677447.507498:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.507499:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:18.0:1619677447.507500:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.507501:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000400:00000200:1.0:1619677447.507501:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.507502:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (22)++
00000400:00000200:8.0:1619677447.507502:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.507504:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677447.507504:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.507505:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:1.0:1619677447.507505:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.507506:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.507507:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:8.0:1619677447.507507:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.507513:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.507514:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96727215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.507515:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.507516:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.507517:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677447.507517:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:8.0:1619677447.507518:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:17.0:1619677447.507520:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:2.0:1619677447.507520:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.507520:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677447.507521:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.507521:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.507521:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.507522:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:17.0:1619677447.507523:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.507523:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677447.507525:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.0:1619677447.507526:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.507527:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.507528:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.507530:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.507530:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.507531:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.507532:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.507536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.507544:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.507547:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.507548:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.507548:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.507550:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.507554:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.507573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.507580:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.507583:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.507584:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.507584:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.507585:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.507586:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.507638:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.507640:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.507642:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.507643:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.507645:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.507646:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677447.507648:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.507649:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.507650:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.507651:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677447.507655:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.507656:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677447.507659:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.507659:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.507660:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.507662:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.507662:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.507663:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.507663:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677447.507674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.507686:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.507688:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.507689:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.507689:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.507692:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.507695:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.507715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.507721:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.507724:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.507724:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.507725:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.507727:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.507730:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.507790:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.507793:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.507793:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.507794:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.507796:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.507799:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.507807:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.507809:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.507810:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.507811:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.507812:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.507812:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.507940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.507947:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.507949:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.507951:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.507952:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.507955:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.507957:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.507958:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.507959:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.507960:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.507961:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.507964:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.507966:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.507969:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.507969:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.507970:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.507972:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.507972:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.507974:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.507975:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.507975:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.507981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.507992:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.507994:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677447.507995:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:8.0:1619677447.507995:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.507996:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.507998:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.507999:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.507999:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.508001:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.508003:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677447.508182:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.508185:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.508185:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.508187:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.508190:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.508191:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.508192:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.508194:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.508195:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.508196:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.508199:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.508201:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.508204:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.508205:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.508206:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:4.2:1619677447.508206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.508208:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.508208:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.508209:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.508209:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677447.508220:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:12.0:1619677447.508222:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.508223:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.508224:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:12.0:1619677447.508224:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.508225:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.508226:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:12.0:1619677447.508227:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.508229:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677447.508229:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.508231:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677447.508232:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.508233:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.508235:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.508237:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.508238:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.508242:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.508245:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.508246:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.508247:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.508247:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.508249:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.508250:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.508250:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.508251:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.508255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.508260:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:18.0:1619677447.508262:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677447.508263:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:18.0:1619677447.508265:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:5.0:1619677447.508265:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.508266:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:18.0:1619677447.508266:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.508266:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:5.0:1619677447.508267:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.508268:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000400:00000200:5.0:1619677447.508269:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:18.0:1619677447.508270:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:5.0:1619677447.508270:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.508272:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.508273:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.508275:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.508278:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.508281:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.508283:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.508284:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.508284:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.508286:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.508287:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.508288:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.508288:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677447.508301:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.508304:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.508304:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.508306:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.508307:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.508308:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:2.0:1619677447.508327:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.508330:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.508331:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.508332:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.508335:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.508336:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.508337:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.508339:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.508340:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.508341:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.508344:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.508346:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.508348:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.508349:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.508350:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.508351:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.508352:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.508353:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.508353:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677447.508367:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.508369:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.508370:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.508372:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.508372:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.508374:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.508380:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.508382:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.508385:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.508385:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.508387:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.508388:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.508390:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.508391:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.508392:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.508393:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.508396:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.508398:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.508400:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.508401:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.508402:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.508404:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.508404:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.508405:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.508406:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.508407:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677447.508418:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.508418:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.508420:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:13.0:1619677447.508421:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.508421:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.508421:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.508422:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:13.0:1619677447.508423:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.508425:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.508426:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.508437:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.508440:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.508441:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.508441:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.508442:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.508443:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.508549:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.508551:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.508552:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.508553:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.508555:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.508555:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677447.508556:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.508558:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.508559:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.508560:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.508562:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.508564:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.508565:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.508567:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.508568:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.508569:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.508572:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.508574:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.508575:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.508576:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.508577:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.508578:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.508579:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.508580:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.508581:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.508582:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.508592:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.508594:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.508595:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.508598:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.508598:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.508599:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.508600:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.508601:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.508602:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.508603:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.0:1619677447.508709:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.508711:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.508712:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.508714:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.508714:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.508717:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.508732:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.508735:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.508736:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.508736:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.508737:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.508738:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.508936:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.508938:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.508941:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.508942:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.508943:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.508945:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.508946:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.508947:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.508949:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.508950:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.508953:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.508955:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.508957:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.508958:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.508959:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.508960:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.508961:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.508962:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.508962:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677447.508992:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.508995:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:9.2:1619677447.508995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.508995:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.508997:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.508999:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.509001:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:18.0:1619677447.509002:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:17.0:1619677447.509002:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.509003:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.509004:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.509005:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:17.0:1619677447.509005:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677447.509006:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.509006:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.509008:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.509009:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.509012:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.509012:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.509015:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.509016:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.509017:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.509018:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.509019:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.509019:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.509020:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.509031:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.509033:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.509034:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.509036:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.509037:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.509039:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.509050:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.509053:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.509054:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.509055:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.509055:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.509056:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:13.0:1619677447.509057:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677447.509059:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.509060:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.509060:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.509062:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.509065:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.509074:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.509077:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.509078:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.509078:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.509079:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.509080:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:2.0:1619677447.509103:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.509106:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.509107:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.509109:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.509111:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.509112:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.509114:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.509115:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.509116:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.509117:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.509121:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.509122:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.509125:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.509125:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.509126:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.509128:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.509128:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.509129:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.509129:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.509132:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.509134:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:4.2:1619677447.509134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:5.0:1619677447.509137:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.509139:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.509139:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.509140:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:5.0:1619677447.509141:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677447.509142:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.509142:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.509143:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.509143:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.509144:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.509145:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.509146:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.509149:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.509150:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:5.0:1619677447.509150:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.509153:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.509156:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.509157:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.509158:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.509160:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.509164:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.509164:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.509165:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.509172:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.509175:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.509176:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.509177:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.509179:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.509180:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.509184:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.509186:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.509187:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.509189:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.509190:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.509192:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.509233:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.509235:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.509236:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.509238:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.509240:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.509242:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.509243:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.509244:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.509245:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.509246:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.509249:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.509251:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.509253:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.509254:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.509255:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.509256:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.509257:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.509257:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.509258:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.509259:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.509262:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.509263:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.509265:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.509268:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.509269:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:17.0:1619677447.509269:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:22.0:1619677447.509271:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.509271:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:22.0:1619677447.509272:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.509272:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.509273:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.509273:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.509274:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.509275:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:17.0:1619677447.509276:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:22.0:1619677447.509279:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.509280:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.509283:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.509284:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.509284:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.509286:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.509287:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.509288:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.509289:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.509290:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.509292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.509301:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.509303:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:6.0:1619677447.509303:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:18.0:1619677447.509305:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.509305:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:2.0:1619677447.509305:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.509306:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:18.0:1619677447.509307:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.509307:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677447.509309:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677447.509407:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.509409:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677447.509411:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.509412:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.509414:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.509415:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.509417:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.509418:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.509419:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.509420:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.509422:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.509424:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.509427:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.509427:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.509428:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.509429:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.509430:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.509431:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.509431:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:4.2:1619677447.509443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.0:1619677447.509444:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.509448:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.509448:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.509450:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.509451:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.0:1619677447.509451:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.509452:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:24.0:1619677447.509454:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.509455:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.509457:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.509459:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.509461:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.509462:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.509463:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.509464:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.509465:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.509469:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.509470:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.509473:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.509474:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.509474:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.509476:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.509477:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.509478:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.509478:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.509490:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.509492:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.509493:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.509494:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.509496:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.509499:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.509561:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.509563:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.509564:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.509564:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.509567:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.509570:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.509598:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.509601:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.509602:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.509603:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.509605:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.509606:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.509607:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.509609:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.509610:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.509611:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677447.509613:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.509615:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.509618:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.509619:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.509620:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.509622:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.509622:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.509623:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.509623:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.509628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.509634:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.509637:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.509638:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.509638:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.509640:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.509644:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.509955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.509971:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.509974:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.509975:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.509978:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.509981:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.509983:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.509984:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.509986:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.509987:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.509988:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.509992:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.509994:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.509997:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.509997:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.509998:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.510000:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.510000:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.510002:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.510003:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.510003:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677447.510006:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.510012:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:13.0:1619677447.510013:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677447.510013:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.510014:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.510015:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:5.0:1619677447.510015:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.510015:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:13.0:1619677447.510016:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.510017:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:13.0:1619677447.510018:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.510018:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677447.510019:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.510019:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.510020:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.510021:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.510136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:24.0:1619677447.510143:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677447.510145:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677447.510146:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677447.510148:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.510151:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.510152:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.510153:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.510154:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.510155:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.510156:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.510160:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.510161:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.510164:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.510165:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.510166:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.510167:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.510168:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677447.510168:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677447.510169:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:17.0:1619677447.510182:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.510184:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.510185:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.510185:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.510188:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.510191:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:22.0:1619677447.510521:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.510524:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.510525:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.510527:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.510530:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.510531:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.510533:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.510534:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.510535:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.510536:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.510539:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.510541:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.510544:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.510545:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.510545:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.510547:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.510547:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.510548:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.510549:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.510589:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.510592:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.510594:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.510595:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.510595:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.510597:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.510598:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.510600:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.510600:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.510602:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.510605:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.510610:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.510612:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.510613:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.510613:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.510614:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.510615:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.510615:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.510616:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.510617:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.510617:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:5.0:1619677447.510617:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.510618:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.510620:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.510622:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677447.510623:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.510624:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.510625:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.510627:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.510630:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.510630:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.510630:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.510631:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.510633:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:20.0:1619677447.510634:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.510635:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.510636:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.510636:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.510637:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.0:1619677447.510637:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.510638:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.510639:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.510639:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.510640:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677447.510653:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.510655:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.510656:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.510658:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.510659:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.510660:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.510692:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.510695:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.510696:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.510697:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.510698:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.510700:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.510709:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.510711:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.510712:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.510713:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.510713:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.510715:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.510757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.510765:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.510768:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.510769:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677447.510769:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677447.510771:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.510772:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.510773:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677447.510774:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.510775:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.510776:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.510777:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.510777:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.510779:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.510779:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.510780:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.510780:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.510781:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.510782:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.510783:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.510784:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.510788:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.510789:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.510793:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.510795:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.510795:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.510796:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.510797:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.510798:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.510798:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.510799:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.510799:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.510799:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.510800:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677447.510801:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.510801:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677447.510802:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.510803:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.510811:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.510813:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.510815:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.510816:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.510816:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.510817:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.510818:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.510818:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.510820:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.510820:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.510821:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.510822:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.510823:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.510832:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.510834:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.510835:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.510835:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.510836:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.510837:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:12.0:1619677447.511056:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.511058:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.511059:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.511060:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.511062:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.511063:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.511153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.511186:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.511189:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.511190:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.511191:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.511192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677447.511193:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.511194:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.511196:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.511197:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:1.0:1619677447.511197:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.511198:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.511199:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.511200:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.511200:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.511201:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.511203:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.511204:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.511205:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.511207:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.511208:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.511209:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.511209:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.511211:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.511211:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.511212:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.511213:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.511224:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.511227:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.511228:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.511228:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.511230:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.511232:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.511265:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.511267:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.511268:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.511271:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.511274:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.511276:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.511278:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.511278:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:6.0:1619677447.511280:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.511281:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.511281:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.511282:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:6.0:1619677447.511282:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677447.511285:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.511287:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.511290:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:2.0:1619677447.511291:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.511293:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:6.0:1619677447.511294:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.511297:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:2.0:1619677447.511300:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96727169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.511301:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.511302:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.511302:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.511303:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677447.511303:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.511304:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677447.511304:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:6.0:1619677447.511305:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:2.0:1619677447.511307:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.511308:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677447.511309:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:9.2:1619677447.511311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.511312:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677447.511313:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.511314:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:2.0:1619677447.511314:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (23)++
00000800:00000200:20.0:1619677447.511315:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.511315:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.511318:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.511318:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (25)--
00000800:00000200:9.2:1619677447.511319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:5.0:1619677447.511319:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:2.0:1619677447.511319:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.511320:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:5.0:1619677447.511321:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:508, d:32, s:-96727168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.511321:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:5.0:1619677447.511322:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.511323:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:5.0:1619677447.511324:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.511324:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.511325:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:5.0:1619677447.511325:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:2.0:1619677447.511325:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.511326:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.511326:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.511329:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.511329:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000400:00000200:5.0:1619677447.511331:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.511332:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.511334:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677447.511334:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.511335:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677447.511335:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677447.511338:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.511339:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.511340:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:9.2:1619677447.511342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677447.511342:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.511342:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.511343:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.511344:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.511350:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.511354:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.511355:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.511355:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.511356:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.511357:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.0:1619677447.511431:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.511434:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:4.2:1619677447.511435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:20.0:1619677447.511436:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.511437:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:20.0:1619677447.511438:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.511441:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677447.511541:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.511544:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.511545:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.511547:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.511550:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.511551:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.511552:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.511554:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.511555:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.511555:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.511559:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.511560:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.511562:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.511563:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.511564:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.511565:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.511566:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.511566:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.511567:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.511572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.511582:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.511583:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.511584:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.511585:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:18.0:1619677447.511586:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.511586:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.511587:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.511588:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.511589:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.511589:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.511590:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.511591:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.511593:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677447.511594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:2.0:1619677447.511594:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.511594:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.511595:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.511596:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.511598:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:2.0:1619677447.511599:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.511601:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.511601:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:2.0:1619677447.511601:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.511602:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.511603:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.511603:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.511604:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.511604:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.511604:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000800:00000200:2.0:1619677447.511605:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (24)--
00000800:00000200:18.0:1619677447.511606:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (24)++
00000800:00000200:18.0:1619677447.511606:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (24)--
00000400:00000200:5.0:1619677447.511606:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:2.0:1619677447.511606:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.511607:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:5.0:1619677447.511607:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.511607:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.511608:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:5.0:1619677447.511608:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.511608:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:5.0:1619677447.511609:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.511610:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:9.2:1619677447.511611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677447.511613:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.511615:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.511618:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.511618:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.511618:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677447.511619:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.511620:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.511621:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.511621:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:2.0:1619677447.511621:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.511622:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.511623:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000400:00000200:2.0:1619677447.511623:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.511624:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:2.0:1619677447.511624:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677447.511625:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.511633:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.511635:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.511636:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.511637:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677447.511648:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.511651:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.511653:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.511654:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.511656:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.511657:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.511658:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.511660:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.511661:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.511662:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.511665:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.511666:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.511668:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.511669:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.511669:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.511671:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.511671:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.511672:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.511673:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677447.511674:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.511684:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.511686:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.511687:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677447.511687:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.511688:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.511689:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.511690:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.511690:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.511692:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:17.0:1619677447.511693:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.511694:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:17.0:1619677447.511696:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.511697:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.511699:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.511701:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.511703:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.511704:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.511705:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.511706:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.511707:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000400:00000200:17.0:1619677447.511708:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.511710:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677447.511710:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.511710:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.511711:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.511711:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:17.0:1619677447.511711:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.511712:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:22.0:1619677447.511713:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:17.0:1619677447.511714:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.511714:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677447.511716:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677447.511717:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.511717:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.511718:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:22.0:1619677447.511720:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.511720:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.511720:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677447.511721:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:17.0:1619677447.511721:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.511721:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677447.511723:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.511725:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.511726:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.511728:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.511735:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:22.0:1619677447.511736:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.511736:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.511737:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.511739:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.511740:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.511743:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.511744:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.511744:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.511745:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.511746:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.511748:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.511749:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.511750:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.511750:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.511757:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.511760:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.511761:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.511763:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.511765:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.511766:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.511804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.511851:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.511854:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.511854:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.511855:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.511857:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.511860:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.511905:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.511908:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.511910:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.511910:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.511912:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.511914:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.511915:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727156] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.511916:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.511917:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.511918:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.511921:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.511923:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.511926:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.511927:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.511928:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.511929:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.511930:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.511931:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.511932:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.511933:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.511941:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677447.511941:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677447.511943:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.511943:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.511944:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.511944:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.511945:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.511946:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.511948:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.511950:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.511960:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.511963:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.511964:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.511965:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.511966:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.511967:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.511977:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.511980:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.511980:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.511982:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.511985:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.511987:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.511988:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.511989:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.511990:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.511991:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.511994:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.511996:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.511998:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.511999:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.512000:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.512001:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.512002:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.512003:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.512004:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.512005:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677447.512015:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677447.512015:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.512017:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:20.0:1619677447.512018:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.512018:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.512018:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.512019:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:20.0:1619677447.512020:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.512021:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.512023:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:9.2:1619677447.512145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.512154:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.512156:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.512158:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.512158:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.512160:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.512163:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.512192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.512203:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.512206:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.512207:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.512209:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.512210:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.512213:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.512223:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.512225:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.512226:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.512227:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.512227:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.512229:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:24.0:1619677447.512284:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.512288:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.512290:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.512291:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.512291:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.512299:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677447.512301:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.512301:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:24.0:1619677447.512303:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677447.512304:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.512305:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.512306:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.512307:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.512307:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.512308:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:24.0:1619677447.512309:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677447.512310:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.512312:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.512313:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.512314:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:24.0:1619677447.512315:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.512318:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:1.0:1619677447.512319:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.512321:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.512321:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:24.0:1619677447.512322:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.512322:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.512324:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.512325:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.512325:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:1.0:1619677447.512325:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.512326:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:24.0:1619677447.512327:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:1.0:1619677447.512327:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:24.0:1619677447.512328:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:1.0:1619677447.512328:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677447.512329:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:1.0:1619677447.512329:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.512330:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:9.2:1619677447.512331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.512331:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677447.512340:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:22.0:1619677447.512343:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677447.512343:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:2.0:1619677447.512343:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.512344:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.512344:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:2.0:1619677447.512344:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.512345:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.512346:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:2.0:1619677447.512346:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.512347:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677447.512347:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677447.512347:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:22.0:1619677447.512348:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.512348:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:6.0:1619677447.512348:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677447.512349:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:22.0:1619677447.512350:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.512350:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.512354:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.512556:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.512558:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.512559:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.512559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:12.0:1619677447.512560:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.512561:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:12.0:1619677447.512562:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.512563:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.512564:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677447.512566:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.512567:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.512569:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.512571:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677447.512572:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.512574:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.512575:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.512576:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677447.512581:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.512583:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 440 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.512585:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.512586:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.512587:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.512588:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.512589:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.512590:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.512591:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:4.2:1619677447.512601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677447.512604:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.512607:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.512607:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.512609:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.512610:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.512611:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677447.512699:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.512702:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.512703:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.512705:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.512706:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.512708:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.512709:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.512711:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.512712:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.512712:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.512716:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.512718:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.512720:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.512720:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.512721:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.512723:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.512723:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.512725:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.512726:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.512726:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.512730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.512735:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.512737:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.512739:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:2.0:1619677447.512739:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.512740:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.512741:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.512741:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.512743:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.512743:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.512744:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.512745:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.512746:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.512747:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.512748:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.512750:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.512750:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.512751:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.512751:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.512752:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.512753:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.512753:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.512754:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677447.512756:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.512757:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.512758:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.512759:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.512760:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.512760:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.512761:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677447.512772:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.512775:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.512776:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.512777:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.512779:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.512780:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.513042:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.513045:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.513046:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.513048:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.513050:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.513051:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.513053:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.513054:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.513055:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.513056:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.513059:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.513060:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.513062:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.513063:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.513063:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.513065:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.513065:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.513066:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.513067:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.513068:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.513078:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:6.0:1619677447.513079:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677447.513080:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:6.0:1619677447.513081:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677447.513081:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677447.513082:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.513082:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.513082:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677447.513083:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:6.0:1619677447.513084:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.513085:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:24.0:1619677447.513086:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.513087:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677447.513088:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:24.0:1619677447.513090:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.513090:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:6.0:1619677447.513091:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:5.0:1619677447.513091:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.513092:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:24.0:1619677447.513093:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.513093:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.513095:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677447.513096:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:22.0:1619677447.513096:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.513096:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000400:00000200:5.0:1619677447.513096:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:6.0:1619677447.513097:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000400:00000200:5.0:1619677447.513097:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.513098:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.513098:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:5.0:1619677447.513099:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.513100:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.513100:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.513101:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.513102:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.513103:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.513107:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.513108:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.513110:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.513111:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.513113:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.513114:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.513114:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.513115:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.513115:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.513115:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.513116:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.513117:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:5.0:1619677447.513117:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.513117:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:24.0:1619677447.513118:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.513118:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677447.513118:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:24.0:1619677447.513119:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:20.0:1619677447.513119:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.513119:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:24.0:1619677447.513120:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677447.513120:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.513121:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.513123:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677447.513124:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677447.513124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677447.513126:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.513127:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.513128:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:13.0:1619677447.513130:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:20.0:1619677447.513131:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.513133:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:18.0:1619677447.513133:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:13.0:1619677447.513133:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.513134:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.513135:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:20.0:1619677447.513136:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677447.513136:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.513136:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.513137:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.513137:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:18.0:1619677447.513137:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.513137:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.513138:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.513139:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:18.0:1619677447.513139:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.513140:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:9.2:1619677447.513140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.513141:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.513141:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.513144:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.513145:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.513147:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.513148:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.513148:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.513149:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.513150:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.513153:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.513156:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.513157:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.513157:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.513159:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.513162:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.513229:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.513232:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.513233:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.513235:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.513237:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.513239:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677447.513239:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.513241:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.513242:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.513243:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677447.513246:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.513248:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677447.513250:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.513251:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677447.513252:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677447.513253:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.513254:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.513255:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.513255:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.513259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.513266:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.513269:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.513269:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.513270:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.513272:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.513275:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:2.0:1619677447.513328:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.513330:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.513332:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.513333:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.513336:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.513338:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.513339:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.513340:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.513341:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.513342:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.513345:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.513347:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.513349:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.513350:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.513350:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.513352:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.513352:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.513353:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.513354:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677447.513367:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.513370:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.513371:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.513371:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.513373:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.513376:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:24.0:1619677447.513408:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.513411:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.513412:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.513414:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.513416:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.513418:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.513419:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.513421:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.513422:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.513423:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.513426:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.513428:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.513431:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.513432:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.513433:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.513434:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.513435:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.513435:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.513436:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:4.2:1619677447.513442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.513450:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.513453:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.513455:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.513456:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.513456:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.513459:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:9.2:1619677447.513483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.513491:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.513494:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.513495:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.513495:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.513497:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.513498:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.513501:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:18.0:1619677447.513501:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:22.0:1619677447.513502:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.513502:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.513504:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.513507:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.513571:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.513574:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.513576:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.513577:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.513578:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.513580:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.513581:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.513583:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.513584:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.513585:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.513588:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.513590:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.513593:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.513594:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.513595:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.513596:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.513597:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.513598:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.513599:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.513609:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.513612:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.513613:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.513615:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.513617:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.513617:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.513653:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.513656:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.513656:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.513658:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.513661:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.513663:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.513664:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.513665:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.513666:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.513667:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.513671:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.513673:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.513675:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.513676:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.513677:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.513679:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.513679:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.513680:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.513680:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.513701:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.513703:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.513704:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.513706:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.513707:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.513709:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.513722:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.513725:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.513726:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.513727:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.513727:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.513728:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.513756:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.513758:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677447.513761:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.513762:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.513763:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:5.0:1619677447.513763:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.513764:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:20.0:1619677447.513765:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.513766:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.513766:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.513768:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000400:00000200:5.0:1619677447.513768:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.513769:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.513770:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.513773:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.513775:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.513776:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.513777:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.513778:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.513779:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.513779:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.513781:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.513782:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.513782:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.513783:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677447.513795:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.513797:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.513797:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.513799:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.513800:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.513801:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:2.0:1619677447.513834:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.513837:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.513838:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.513839:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.513842:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.513843:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.513844:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.513845:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.513847:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.513847:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.513852:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.513853:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.513854:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.513855:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.513856:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.513857:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.513857:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.513858:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677447.513859:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.513860:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:2.0:1619677447.513860:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.513861:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.513862:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.513863:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.513864:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.513865:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:9.2:1619677447.513866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.513866:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677447.513867:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.513868:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.513869:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.513869:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.513870:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.513872:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.513872:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.513872:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.513873:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.513873:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.513873:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.513875:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:2.0:1619677447.513875:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.513876:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.513876:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.513877:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.513877:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677447.513878:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.513878:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.513880:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.513880:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.513881:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677447.513890:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.513892:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:4.2:1619677447.513892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:5.0:1619677447.513893:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.513895:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.513896:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.513897:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.0:1619677447.513900:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.513902:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.513903:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.513905:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.513908:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.513910:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.513911:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.513912:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.513914:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.513914:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.513918:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.513920:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.513923:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.513924:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.513924:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.513926:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.513927:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.513927:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.513928:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.513939:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.513942:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.513943:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.513943:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.513946:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.513948:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.0:1619677447.513967:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.513970:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.513972:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.513973:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.513974:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.513976:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677447.513977:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.513979:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.513980:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.513981:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677447.513984:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.513986:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677447.513988:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.513989:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677447.513990:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.513992:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.513992:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.513993:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.513994:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.513998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.514005:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.514008:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.514009:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.514010:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.514011:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.514014:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:17.0:1619677447.514024:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.514026:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.514027:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.514027:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.514030:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.514033:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.514123:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.514125:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.514126:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.514128:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.514129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677447.514131:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.514132:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.514133:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.514134:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.514135:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.514136:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.514141:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.514143:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.514146:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.514147:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.514147:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.514149:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.514149:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.514150:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.514151:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.514172:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.514174:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.514175:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.514177:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.514177:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:13.0:1619677447.514179:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.514179:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677447.514180:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.514180:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.514181:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.514183:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.514186:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.514428:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.514431:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.514433:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.514434:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.514435:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.514437:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.514438:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.514439:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.514441:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.514442:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.514445:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.514446:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.514449:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.514450:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.514450:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.514452:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.514452:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.514454:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.514455:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.514456:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.514463:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.514465:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:22.0:1619677447.514466:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.514466:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.514467:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.514469:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.514470:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.514470:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.514472:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.514474:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.514529:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.514532:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.514532:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.514535:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000400:00000200:17.0:1619677447.514536:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.514538:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677447.514539:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:17.0:1619677447.514539:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.514541:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:6.0:1619677447.514541:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.514542:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.514544:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.514544:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.514546:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.514546:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.514547:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:6.0:1619677447.514548:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.514549:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.514551:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.514552:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.514553:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.514556:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:6.0:1619677447.514559:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.514561:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.514563:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.514563:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.514564:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677447.514564:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.514565:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.514566:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677447.514566:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.514567:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.514568:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677447.514568:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.514569:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677447.514569:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:17.0:1619677447.514570:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.514570:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677447.514570:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:12.0:1619677447.514579:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677447.514580:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.514581:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:12.0:1619677447.514582:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:17.0:1619677447.514583:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:12.0:1619677447.514583:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.514583:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.514584:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.514585:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.514586:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:17.0:1619677447.514587:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:12.0:1619677447.514587:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.514587:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.514591:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677447.514591:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.514592:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.514593:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.514593:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.514593:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.514594:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.514595:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.514595:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.514596:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.514597:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.514597:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.514597:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.514598:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.514598:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.514598:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.514599:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.514600:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.514600:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.0:1619677447.514887:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.514890:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.514892:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.514893:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.514895:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.514896:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.514897:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.514899:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.514900:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.514901:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.514904:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.514906:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.514908:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.514909:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.514909:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.514911:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.514911:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.514912:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.514913:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.514924:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.514926:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.514927:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.514929:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.514929:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.514931:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:18.0:1619677447.514988:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.514991:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.514992:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.514994:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.514996:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.514998:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.514999:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.515000:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.515001:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.515002:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.515006:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.515008:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.515011:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.515012:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.515013:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.515015:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.515015:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.515016:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.515017:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677447.515020:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.515023:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.515024:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.515025:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:17.0:1619677447.515026:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.515028:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:17.0:1619677447.515028:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.515029:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:4.2:1619677447.515029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:17.0:1619677447.515030:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:22.0:1619677447.515031:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.515032:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.515033:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.515033:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.515035:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.515036:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677447.515036:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677447.515037:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.515039:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.515041:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.515041:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.515042:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:17.0:1619677447.515044:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:22.0:1619677447.515046:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.515048:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.515049:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677447.515049:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.515049:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.515051:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677447.515052:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.515052:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.515052:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.515053:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.515053:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:17.0:1619677447.515053:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.515055:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.515056:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.515056:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.515057:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.515064:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.515066:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.515067:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.515069:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.515070:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.515071:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.515081:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.515083:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:12.0:1619677447.515084:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.515084:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677447.515085:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:12.0:1619677447.515085:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:20.0:1619677447.515086:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.515087:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677447.515088:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.515088:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.515089:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.515091:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.515223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.515230:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.515233:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.515235:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.515236:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.515239:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.515241:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.515242:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.515243:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.515244:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.515245:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.515248:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.515250:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.515252:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.515253:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.515254:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.515255:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.515256:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.515257:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.515259:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.515259:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.515265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.515269:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677447.515272:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.515272:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.515273:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.515273:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.515274:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.515274:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.515275:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.515275:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.515278:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.515320:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.515323:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.515324:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.515324:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.515326:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.515329:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.0:1619677447.515473:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.515475:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.515478:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.515478:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.515480:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.515482:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677447.515483:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.515484:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.515485:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.515486:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.515489:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.515491:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.515493:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.515494:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.515494:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.515496:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.515497:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.515497:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.515498:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:1.0:1619677447.515510:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.515512:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.515513:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.515513:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.515516:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.515518:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:9.2:1619677447.515648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.515656:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.515659:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.515660:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.515660:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.515662:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.515666:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.515902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.515942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.515948:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.515950:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.515951:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.515952:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.515954:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.515957:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.515966:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677447.515966:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:22.0:1619677447.515969:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677447.515969:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:22.0:1619677447.515970:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000400:00000200:17.0:1619677447.515970:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.515970:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677447.515970:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:22.0:1619677447.515971:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.515972:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:17.0:1619677447.515972:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.515975:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.0:1619677447.515995:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.515998:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.516000:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.516001:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.516002:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.516004:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.516005:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.516007:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.516008:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.516009:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.516012:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.516013:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.516015:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.516016:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.516017:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.516018:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:1.0:1619677447.516018:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677447.516019:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.516020:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.516021:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:17.0:1619677447.516021:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.516021:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.0:1619677447.516022:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:17.0:1619677447.516023:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.516024:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.516025:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.516027:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.516028:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.516029:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.516030:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.516031:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.516033:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:1.0:1619677447.516033:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.516034:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677447.516034:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:1.0:1619677447.516034:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677447.516035:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000400:00000200:17.0:1619677447.516035:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.516035:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.516036:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.516037:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:17.0:1619677447.516037:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.516038:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:5.0:1619677447.516038:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.516039:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.516039:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.516040:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.516040:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:17.0:1619677447.516041:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.516042:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.516045:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.516045:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.516046:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.516047:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.516047:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.516049:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.516050:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.516050:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)++
00000800:00000200:17.0:1619677447.516051:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.516051:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (24)--
00000800:00000200:17.0:1619677447.516052:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.516052:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.516053:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.516054:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.516055:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.516055:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.516056:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.516063:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.516065:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.516066:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.516066:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.516069:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.516071:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.516081:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.516084:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.516084:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.516085:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.516086:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.516086:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.516133:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:4.2:1619677447.516134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:6.0:1619677447.516135:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.516136:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.516138:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.516141:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.516142:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:6.0:1619677447.516142:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.516143:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.516145:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.516146:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:6.0:1619677447.516146:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.516147:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677447.516148:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677447.516152:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.516152:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.516154:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.516155:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:24.0:1619677447.516156:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:6.0:1619677447.516157:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.516158:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96727102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.516158:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.516159:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:24.0:1619677447.516160:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.516161:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.516161:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.516161:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.516162:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.516162:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:24.0:1619677447.516163:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:9.2:1619677447.516167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:24.0:1619677447.516168:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.516170:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.516173:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.516174:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.516174:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:18.0:1619677447.516174:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:24.0:1619677447.516176:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.516176:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.516177:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.516177:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677447.516177:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.516178:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.516178:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.516180:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.516182:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.0:1619677447.516193:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:1.0:1619677447.516193:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.516196:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:20.0:1619677447.516197:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:17.0:1619677447.516197:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000400:00000200:20.0:1619677447.516198:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.516198:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.516200:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.516200:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.516201:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677447.516201:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.516202:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (23)++
00000800:00000200:5.0:1619677447.516203:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (24)--
00000800:00000200:5.0:1619677447.516204:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.516205:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677447.516205:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.0:1619677447.516206:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.516316:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.516319:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:4.2:1619677447.516319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:20.0:1619677447.516321:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.516322:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:20.0:1619677447.516323:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.516326:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677447.516365:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.516368:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.516368:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.516369:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.516371:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.516374:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.516499:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.516502:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.516502:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.516503:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.516506:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.516508:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:12.0:1619677447.516532:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.516534:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.516535:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.516537:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.516539:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.516540:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.516565:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.516568:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.516569:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.516571:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.516573:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.516574:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.516576:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.516577:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.516578:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.516579:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.516581:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:6.0:1619677447.516582:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.516584:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.516585:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677447.516585:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.516587:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677447.516587:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:6.0:1619677447.516588:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.516589:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677447.516589:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.516590:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:2.0:1619677447.516590:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:6.0:1619677447.516591:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.516591:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.516592:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:2.0:1619677447.516592:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.516593:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.516594:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.516596:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.516599:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:2.0:1619677447.516599:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.516601:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.516601:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.516602:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:18.0:1619677447.516603:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.516604:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.516604:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:1.0:1619677447.516604:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:18.0:1619677447.516605:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.516605:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:2.0:1619677447.516605:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.516606:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677447.516607:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:2.0:1619677447.516607:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:1.0:1619677447.516607:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.516608:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:17.0:1619677447.516608:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:5.0:1619677447.516608:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.516609:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.516610:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.516610:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.516612:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.516612:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:18.0:1619677447.516613:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677447.516613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.516613:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:18.0:1619677447.516614:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.516614:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.516615:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.516617:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.516618:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:18.0:1619677447.516620:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.516622:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677447.516622:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.516622:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.516623:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.516623:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.516624:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677447.516625:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.516626:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.516626:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677447.516626:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:2.0:1619677447.516626:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677447.516626:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.516627:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000800:00000200:17.0:1619677447.516627:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.516628:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.516628:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.516628:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:18.0:1619677447.516629:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (25)--
00000800:00000200:5.0:1619677447.516629:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.516629:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:2.0:1619677447.516629:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (25)--
00000800:00000200:18.0:1619677447.516630:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677447.516630:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:2.0:1619677447.516630:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.516631:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.516632:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.516632:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000400:00000200:2.0:1619677447.516632:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.516633:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:5.0:1619677447.516633:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:18.0:1619677447.516634:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.516634:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.516634:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677447.516635:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (24)++
00000800:00000200:2.0:1619677447.516635:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (24)--
00000800:00000200:18.0:1619677447.516636:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (24)--
00000800:00000200:2.0:1619677447.516636:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (24)++
00000800:00000200:18.0:1619677447.516637:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (24)--
00000800:00000200:2.0:1619677447.516637:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000800:00000200:18.0:1619677447.516638:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:9.2:1619677447.516638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:2.0:1619677447.516638:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.516639:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.516641:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.516642:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677447.516644:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:17.0:1619677447.516645:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.516645:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:8.0:1619677447.516647:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.516648:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.516648:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.516648:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.516650:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.516651:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.516652:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.516653:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.516654:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:1.0:1619677447.516655:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.516656:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.516657:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:9.2:1619677447.516659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000400:00000200:1.0:1619677447.516660:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.516662:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:24.0:1619677447.516665:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.516665:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.516665:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.516666:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:24.0:1619677447.516668:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677447.516668:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.516668:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.516668:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677447.516669:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.516670:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.516670:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677447.516671:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:1.0:1619677447.516671:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.516672:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677447.516676:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.516679:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.516680:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:22.0:1619677447.516681:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:18.0:1619677447.516682:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.516682:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677447.516683:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:18.0:1619677447.516684:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.516684:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:22.0:1619677447.516685:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.516685:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:17.0:1619677447.516685:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.516685:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.516686:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:18.0:1619677447.516687:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.516688:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.516688:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.516689:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.516690:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:18.0:1619677447.516690:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.516694:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.516696:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.516699:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.516699:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.516700:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.516702:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.516703:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.516704:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.516704:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:1.0:1619677447.516712:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677447.516714:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677447.516715:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677447.516717:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.516719:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.516721:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677447.516723:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.516724:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.516725:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.516726:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:4.2:1619677447.516727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677447.516729:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.516731:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677447.516734:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.516735:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:1.0:1619677447.516735:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:1.0:1619677447.516737:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:1.0:1619677447.516738:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677447.516738:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677447.516739:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:6.0:1619677447.516775:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.516778:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.516779:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.516779:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.516781:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.516784:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.516891:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.516894:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.516897:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.516898:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.516901:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.516904:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.516906:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.516908:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.516909:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.516911:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.516919:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.516922:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.516926:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.516927:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.516928:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.516930:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.516931:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.516932:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.516932:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.516942:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.516945:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.516945:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.516947:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.516947:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:13.0:1619677447.516949:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677447.516949:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:12.0:1619677447.516950:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:5.0:1619677447.516950:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.516952:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:5.0:1619677447.516952:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.516954:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.516955:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:20.0:1619677447.516955:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:5.0:1619677447.516956:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (24)--
00000800:00000200:20.0:1619677447.516957:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.516958:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677447.517023:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.517026:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.517026:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.517028:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.517031:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.517032:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.517034:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.517035:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.517036:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.517037:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.517041:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.517042:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.517046:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.517046:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.517047:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.517049:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.517049:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:22.0:1619677447.517050:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.517051:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:20.0:1619677447.517051:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:22.0:1619677447.517052:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.517053:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677447.517053:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.517055:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.517056:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.517058:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.517059:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.517060:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.517061:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.517062:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:6.0:1619677447.517063:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:20.0:1619677447.517065:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.517066:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:20.0:1619677447.517067:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677447.517067:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.517068:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.517070:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.517071:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.517072:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.517073:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.517074:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.517075:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.517076:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.517087:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.517089:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.517090:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.517092:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.517092:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.517094:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677447.517110:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.517113:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.517114:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.517114:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:17.0:1619677447.517114:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677447.517115:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.517116:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:17.0:1619677447.517117:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.517118:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.517120:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.517123:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.517124:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.517125:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.517126:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.517127:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.517128:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:4.2:1619677447.517130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:17.0:1619677447.517132:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.517133:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.517136:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.517137:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.517137:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.517139:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.517139:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.517140:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.517140:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.517153:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.517155:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.517156:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.517156:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.517159:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.517161:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.517194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.517232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.517236:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.517239:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.517240:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.517240:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.517242:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.517246:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.517263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.517270:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.517273:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.517274:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.517276:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.517278:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.517280:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.517281:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.517282:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.517284:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.517284:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.517287:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.517289:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.517292:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.517293:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.517298:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.517300:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.517301:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.517302:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.517302:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.517315:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.517317:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.517318:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.517319:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.517321:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.517323:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.517410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.517457:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.517460:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.517460:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.517461:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.517463:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.517469:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.517470:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.517472:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.517472:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.517473:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.517473:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.517474:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.517475:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.517476:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:6.0:1619677447.517476:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.517477:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.517478:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.517480:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.517481:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.517483:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.517484:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.517485:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.517488:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.517489:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.517492:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.517493:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.517493:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.517495:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.517496:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.517496:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.517497:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.517511:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.517513:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.517514:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.517515:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.517516:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.517517:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.517525:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.517527:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.517529:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.517530:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.517532:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.517533:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.517535:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.517536:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.517537:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.517538:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.517541:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.517543:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.517546:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.517547:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.517547:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.517549:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.517550:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.517551:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.517551:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.517563:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.517566:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.517567:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.517568:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.517570:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.517572:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.517667:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.517669:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.517670:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.517672:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.517672:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.517675:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677447.517894:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.517896:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.517897:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.517899:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.517901:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.517902:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.517904:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.517905:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.517906:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.517907:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.517911:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.517912:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.517915:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.517916:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.517916:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.517918:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.517919:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.517919:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.517920:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.517931:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.517934:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.517935:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.517935:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.517938:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.517940:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677447.518017:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.518020:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.518022:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.518023:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.518025:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.518026:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.518028:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.518029:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.518030:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.518031:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.518034:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.518036:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.518039:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.518039:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.518040:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.518042:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.518042:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.518043:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.518044:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:24.0:1619677447.518052:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.518054:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:5.0:1619677447.518055:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:8.0:1619677447.518056:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.518057:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.518058:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.518058:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.518060:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.518061:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.518061:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.518062:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:24.0:1619677447.518063:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.518064:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.518066:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.518067:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.518068:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.518072:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.518074:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.518077:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.518078:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.518079:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.518080:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.518081:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.518081:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.518082:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:4.2:1619677447.518083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.518092:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.518094:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.518095:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.518097:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.518099:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.518100:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.518113:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.518116:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.518119:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.518119:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.518121:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.518123:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.518124:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.518126:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.518127:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.518127:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:20.0:1619677447.518128:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677447.518130:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.518131:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.518132:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.518133:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.518134:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.518136:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677447.518138:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.518141:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.518142:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.518143:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.518145:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.518145:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.518146:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.518147:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.518157:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.518159:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.518160:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.518160:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:13.0:1619677447.518162:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.518162:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:12.0:1619677447.518163:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677447.518163:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.518165:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:13.0:1619677447.518165:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677447.518165:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.518168:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677447.518177:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.518180:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.518181:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.518182:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.518182:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.518183:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.518228:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.518232:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.518233:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.518235:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.518237:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.518238:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:2.0:1619677447.518239:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:12.0:1619677447.518240:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:2.0:1619677447.518240:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677447.518241:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.518242:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.518243:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677447.518244:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.518244:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.518245:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677447.518249:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.518250:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.518251:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.518253:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.518254:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.518255:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.518257:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.518257:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.518259:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.518260:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.518260:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:13.0:1619677447.518269:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677447.518271:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:13.0:1619677447.518271:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.518272:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.518273:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.518274:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:13.0:1619677447.518274:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.518275:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.518275:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.518277:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.518558:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.518560:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.518563:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.518564:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.518565:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.518567:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.518568:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.518570:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.518571:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.518572:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.518575:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.518577:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.518580:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.518581:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.518581:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.518583:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.518584:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.518585:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.518586:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.518587:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677447.518597:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.518597:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.518599:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:12.0:1619677447.518600:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677447.518600:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:12.0:1619677447.518601:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.518601:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:12.0:1619677447.518603:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.518604:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.518605:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.518816:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.518819:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.518821:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.518822:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.518826:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.518828:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.518829:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.518831:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.518831:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.518832:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.518835:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.518837:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.518839:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677447.518840:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.518841:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.518841:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.518843:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.518843:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:1.0:1619677447.518843:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.518844:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677447.518844:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.518845:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:1.0:1619677447.518846:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.518848:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.518850:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.518851:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.518853:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.518854:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.518855:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677447.518856:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:13.0:1619677447.518858:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.518859:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.518859:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677447.518860:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.518861:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.518861:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677447.518863:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.518863:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.518864:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.518865:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.518866:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.518867:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.518868:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.518868:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.518873:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.518875:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.518876:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.518877:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.518877:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.518878:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.518881:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.518883:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.518884:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.518885:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.518886:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.518888:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.518899:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.518901:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.518902:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.518903:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.518903:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.518904:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.0:1619677447.519047:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.519050:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.519052:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.519053:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.519055:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.519057:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.519058:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.519059:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.519060:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677447.519061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:5.0:1619677447.519062:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.519066:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.519068:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.519072:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.519073:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.519074:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.519075:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.519076:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.519077:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.519078:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.519082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.519090:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.519093:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.519094:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.519094:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.519097:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.519101:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:9.2:1619677447.519102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.519108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.519113:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.519115:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.519117:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:2.0:1619677447.519117:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.519118:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.519119:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.519119:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.519120:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.519123:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.519124:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.519124:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.519125:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:2.0:1619677447.519125:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.0:1619677447.519128:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:9.2:1619677447.519129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000400:00000200:20.0:1619677447.519130:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.519131:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.519132:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.519134:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.519135:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.519136:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.519137:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.519137:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000400:00000200:20.0:1619677447.519138:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:18.0:1619677447.519140:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.519141:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.519141:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:20.0:1619677447.519142:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.519142:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.519142:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.0:1619677447.519144:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.519146:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.519147:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.519148:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.519149:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.519150:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.519151:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.519151:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.519162:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.519165:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.519165:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.519167:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:20.0:1619677447.519168:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.519170:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:2.0:1619677447.519358:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.519361:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.519362:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.519363:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.519366:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.519368:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.519369:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.519371:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.519372:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.519373:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.519375:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.519377:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.519380:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.519380:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.519381:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.519383:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.519383:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.519384:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.519384:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677447.519395:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.0:1619677447.519397:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677447.519397:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.519398:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.0:1619677447.519399:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:18.0:1619677447.519399:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.519400:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.519402:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.519402:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.519403:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:18.0:1619677447.519404:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677447.519405:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.519406:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.519407:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.519408:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677447.519410:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:18.0:1619677447.519412:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.519414:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.519416:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.519417:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:18.0:1619677447.519418:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677447.519419:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.519420:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.519420:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.519421:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.519421:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677447.519422:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.519423:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.519424:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.519425:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.519426:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677447.519433:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.519435:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.519436:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.519438:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.519440:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.519441:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.519452:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.519454:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.519455:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.519456:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.519457:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.519458:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677447.519488:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.519491:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.519492:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.519494:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.519495:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.519497:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.519498:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.519500:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.519501:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.519502:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.519505:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.519506:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.519509:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.519510:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.519511:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.519512:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.519513:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.519514:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.519515:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.519516:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677447.519525:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677447.519526:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677447.519527:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.519528:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.519528:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.519529:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677447.519530:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.519530:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.519531:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.519532:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.519546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.519554:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.519557:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.519557:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.519559:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.519562:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.519563:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.519565:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.519566:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.519567:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.519568:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.519571:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.519573:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.519576:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.519576:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.519577:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.519579:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.519580:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.519580:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.519581:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.519593:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.519595:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.519596:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.519596:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.519598:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.519601:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.0:1619677447.519674:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:4.2:1619677447.519675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.519677:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.519679:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.519681:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.519681:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.519683:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.519684:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.519685:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.519686:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.519687:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.519691:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.519694:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.519696:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.519697:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.519697:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.519699:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.519700:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.519700:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.519701:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.519714:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.519717:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.519718:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.519719:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.519720:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.519722:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.519722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677447.519728:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.519731:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.519732:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.519733:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.519734:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.519737:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.519768:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.519771:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.519772:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.519772:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.519774:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.519777:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:2.0:1619677447.519782:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.519785:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.519786:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.519788:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.519790:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.519792:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677447.519793:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.519794:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.519795:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.519796:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677447.519800:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.519801:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677447.519804:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.519804:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677447.519805:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677447.519807:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677447.519807:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.519808:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.519808:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677447.519820:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.519823:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.519824:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.519824:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.519826:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.519828:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.519839:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.519842:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.519843:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.519843:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.519844:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.519844:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677447.519918:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.519920:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.519923:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.519923:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.519925:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.519927:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677447.519928:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.519930:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.519931:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.519932:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677447.519935:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.519937:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677447.519940:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.519941:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677447.519942:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677447.519943:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677447.519944:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.519945:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.519946:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.519947:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.519955:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.519957:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.519958:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.519959:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677447.520031:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.520034:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.520035:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.520037:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.520039:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.520041:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.520042:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.520044:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.520045:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.520046:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.520050:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.520053:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.520056:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.520057:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.520058:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.520060:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.520060:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.520061:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.520061:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677447.520062:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.520064:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.520066:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.520067:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.520069:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.520070:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:20.0:1619677447.520072:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.520072:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:20.0:1619677447.520073:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.520074:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.520075:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:13.0:1619677447.520075:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.520075:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.520078:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.520079:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.520079:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.520080:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677447.520081:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 440 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.520084:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.520084:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677447.520085:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.520087:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.520088:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.520088:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.520089:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677447.520090:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.520102:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.520103:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677447.520103:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.520104:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000400:00000200:5.0:1619677447.520104:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.520105:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.520106:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.520106:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.520107:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.520108:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:24.0:1619677447.520167:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.520170:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.520171:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.520173:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.520176:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.520177:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.520178:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.520180:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.520181:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.520182:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.520185:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.520187:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.520190:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.520191:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.520191:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.520193:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.520193:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.520195:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.520197:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.520197:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677447.520207:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.520209:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.520210:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.520210:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.520211:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:20.0:1619677447.520212:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.520212:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677447.520213:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.520214:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677447.520215:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:6.0:1619677447.520250:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.520253:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.520254:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.520256:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.520257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:6.0:1619677447.520258:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.520259:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.520261:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.520262:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.520263:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.520264:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.520268:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.520270:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.520273:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.520274:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.520274:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.520276:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.520277:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.520277:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.520278:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.520317:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.520320:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.520321:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.520322:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.520325:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.520326:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.520371:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.520373:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.520374:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.520376:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.520376:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.520378:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.520392:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.520394:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.520395:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.520396:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.520397:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.520397:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.520592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.520599:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.520603:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.520603:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.520606:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.520608:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.520610:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.520611:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96727039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.520612:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.520613:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.520614:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.520617:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.520619:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:9.2:1619677447.520621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.520621:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.520622:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.520623:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.520624:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.520625:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.520626:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.520627:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.520628:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677447.520630:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.520633:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:4.2:1619677447.520633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:18.0:1619677447.520635:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.520635:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.520637:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.520638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.520640:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.520640:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:18.0:1619677447.520641:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677447.520641:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.520642:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:24.0:1619677447.520643:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:18.0:1619677447.520643:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677447.520643:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.520643:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.520644:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.520645:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677447.520645:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677447.520645:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.520646:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000400:00000200:17.0:1619677447.520646:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.520647:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.520647:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.520647:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:17.0:1619677447.520649:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.520649:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.520650:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:17.0:1619677447.520651:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:1.0:1619677447.520652:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.520653:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.520655:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.520656:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.520657:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.520662:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.520664:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.520667:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.520668:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.520669:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.520671:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.520671:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.520672:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.520673:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.520688:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.520689:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.520690:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.520691:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.520692:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.520692:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:13.0:1619677447.520693:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.520694:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677447.520695:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.520695:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.520697:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.520697:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.520709:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.520712:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.520713:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.520715:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.520717:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.520719:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.520720:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.520721:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.520723:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.520723:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.520727:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.520729:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.520732:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.520732:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.520733:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.520735:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.520736:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.520736:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.520737:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.520741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.520749:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.520752:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.520753:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.520753:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.520756:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.520758:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.520805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.520977:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.520980:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.520981:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.520983:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.520984:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.520986:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677447.521105:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.521108:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.521109:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.521109:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.521111:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.521115:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677447.521182:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.521185:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677447.521187:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.521188:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.521189:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.521190:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.521192:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.521193:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.521194:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.521195:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.521200:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.521202:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677447.521203:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677447.521205:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.521206:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:5.0:1619677447.521206:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.521206:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:2.0:1619677447.521207:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677447.521208:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000400:00000200:18.0:1619677447.521209:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.521209:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.521209:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.521210:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:18.0:1619677447.521212:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.521213:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.521214:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.521215:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.521216:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.521217:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.521220:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.521221:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:18.0:1619677447.521222:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.521223:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.521224:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.521225:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.521226:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:18.0:1619677447.521226:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:20.0:1619677447.521227:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.521227:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677447.521227:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:20.0:1619677447.521228:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.521228:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:5.0:1619677447.521228:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:18.0:1619677447.521229:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.521230:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677447.521230:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.521231:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677447.521232:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:9.2:1619677447.521232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677447.521232:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.521235:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.521237:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.521239:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.521240:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.521241:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.521243:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.521243:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.521243:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.521244:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.521245:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:5.0:1619677447.521245:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.521246:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.521247:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:5.0:1619677447.521247:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677447.521247:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.521248:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677447.521249:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.521250:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.521251:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677447.521252:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000400:00000200:17.0:1619677447.521253:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.521254:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.521255:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.521256:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:17.0:1619677447.521256:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.521257:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677447.521258:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:5.0:1619677447.521258:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.521258:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:20.0:1619677447.521259:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.521259:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.521260:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:17.0:1619677447.521261:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.521261:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:20.0:1619677447.521262:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.521262:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.521262:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.521263:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.521263:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:5.0:1619677447.521263:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:5.0:1619677447.521264:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.521265:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96727027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.521267:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.521267:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.521268:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.521269:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.521270:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:20.0:1619677447.521272:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.521273:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.521274:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:20.0:1619677447.521275:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.521275:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.521277:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.521277:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677447.521278:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.521278:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.521278:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677447.521279:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677447.521280:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:20.0:1619677447.521283:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677447.521284:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.521284:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.521285:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.521289:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.521292:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.521293:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.521297:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.521302:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.521304:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.521306:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.521310:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.521312:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.521313:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.521316:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.521318:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.521319:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.521320:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.521321:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.521323:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.521326:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.521328:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.521331:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.521332:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.521332:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.521334:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.521335:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.521335:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.521336:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.521379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.521400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:4.2:1619677447.521517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.521524:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.521527:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.521528:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.521530:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.521533:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.521534:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.521535:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.521537:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.521538:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.521538:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.521542:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.521543:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.521546:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.521547:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.521547:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.521549:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.521549:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.521550:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.521550:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.521563:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.521566:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.521567:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.521567:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.521569:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.521572:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677447.521602:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.521605:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.521606:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.521607:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.521610:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.521611:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.521612:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.521612:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:18.0:1619677447.521614:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.521614:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.521615:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:13.0:1619677447.521615:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.521615:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.521617:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.521618:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:18.0:1619677447.521619:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.521620:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.521622:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.521623:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.521624:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.521625:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.521627:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.521627:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.521628:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.521628:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.521633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.521640:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.521642:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.521643:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.521644:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.521645:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.521646:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.0:1619677447.521653:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.521655:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.521656:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.521658:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.521661:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.521663:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.521664:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96727019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.521666:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.521667:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.521667:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.521671:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.521673:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.521676:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.521677:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.521677:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.521679:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.521679:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.521680:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.521680:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677447.521693:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.521696:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.521697:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.521699:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.521699:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.521702:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.521748:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.521751:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.521751:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.521753:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.521757:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.521758:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.521759:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.521761:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.521762:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.521763:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.521767:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.521768:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.521771:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.521772:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.521772:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.521774:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.521775:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.521776:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.521777:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.521778:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.521788:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.521789:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.521790:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.521791:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.521894:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.521897:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.521898:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.521900:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.521903:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.521904:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.521906:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.521907:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.521908:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.521909:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.521913:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.521915:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.521917:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.521918:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.521919:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.521921:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.521921:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.521922:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.521923:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.521924:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.521934:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.521937:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.521938:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.521938:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.522028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.522068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.522074:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.522077:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.522077:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.522078:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.522080:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.522083:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.522090:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.522092:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.522093:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.522095:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.522098:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.522100:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.522101:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.522103:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.522104:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.522105:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.522108:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.522109:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.522112:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.522112:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.522113:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.522115:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.522115:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.522116:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.522116:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.522128:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.522131:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.522132:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.522132:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.522134:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.522136:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677447.522225:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.522228:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.522229:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.522230:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.522233:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.522234:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.522235:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.522235:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:18.0:1619677447.522236:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.522237:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.522238:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:12.0:1619677447.522238:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.522238:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.522240:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.522242:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:13.0:1619677447.522242:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677447.522243:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:12.0:1619677447.522244:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677447.522245:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.522246:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:18.0:1619677447.522246:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.522247:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.522248:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677447.522249:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.522250:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.522250:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.0:1619677447.522250:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:18.0:1619677447.522251:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:20.0:1619677447.522252:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.522252:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:20.0:1619677447.522253:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.522254:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96727012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.522256:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.522256:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:9.2:1619677447.522256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677447.522257:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.522258:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:13.0:1619677447.522259:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.522260:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.522261:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.522261:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:20.0:1619677447.522262:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.522263:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.522263:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677447.522264:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677447.522265:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:20.0:1619677447.522266:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677447.522266:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.522267:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.522267:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:18.0:1619677447.522267:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.522268:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:20.0:1619677447.522269:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.522270:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.522270:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.522271:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:2.0:1619677447.522271:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.522276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.522283:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:18.0:1619677447.522284:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.522285:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:18.0:1619677447.522286:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.522286:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.522287:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677447.522288:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:18.0:1619677447.522288:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000400:00000200:5.0:1619677447.522288:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.522288:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.522289:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677447.522290:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677447.522303:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.522305:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.522306:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.522307:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.522307:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.522308:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:22.0:1619677447.522328:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.522330:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.522331:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.522333:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.522336:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.522337:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.522339:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.522340:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.522341:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.522342:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.522345:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.522347:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.522350:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.522350:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.522351:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.522353:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.522353:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.522354:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.522354:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.522368:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.522371:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.522372:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.522374:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.522374:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.522375:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677447.522377:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:24.0:1619677447.522378:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.522379:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.522381:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.522384:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.522387:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.522389:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.522390:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.522391:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.522392:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.522395:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.522398:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.522401:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.522402:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.522402:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.522404:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.522405:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.522406:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.522406:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677447.522417:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.522420:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.522421:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.522422:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.522422:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677447.522424:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.522425:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.522425:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.522426:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.522426:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.522429:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.522432:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.522442:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.522444:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.522445:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.522445:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.522446:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.522447:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.522523:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.522525:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.522526:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.522528:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.522530:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.522532:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.522533:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.522534:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.522535:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.522536:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.522539:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.522541:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.522544:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.522544:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.522545:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.522547:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.522547:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.522548:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.522548:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.522561:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.522564:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.522565:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.522565:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.522567:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.522569:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.522584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677447.522628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:12.0:1619677447.522631:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.522633:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.522634:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.522634:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:12.0:1619677447.522635:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.522636:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:12.0:1619677447.522637:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.522638:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:5.0:1619677447.522638:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.522639:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.522641:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.522643:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.522644:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.522646:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.522646:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.522648:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.522651:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.522652:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.522656:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.522656:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.522657:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.522659:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.522659:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.522661:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.522662:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.522663:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.522667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.522671:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677447.522671:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677447.522673:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.522673:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.522674:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.522674:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.522675:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.522676:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.522678:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.522679:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.522688:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.522691:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.522692:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.522692:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.522693:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.522695:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.522847:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.522850:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.522850:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.522853:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.522855:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.522856:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.522857:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.522859:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.522860:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.522861:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.522864:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.522865:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.522868:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.522869:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.522869:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.522871:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.522872:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.522873:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.522873:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.522880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.522885:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.522888:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.522889:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.522889:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.522891:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.522894:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677447.522926:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.522929:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.522930:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.522930:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.522933:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.522936:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.522938:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.522940:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.522941:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.522944:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:1.0:1619677447.522944:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.522946:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.522947:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.522947:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.522948:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000400:00000200:1.0:1619677447.522948:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:22.0:1619677447.522949:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.522949:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:1.0:1619677447.522949:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.522951:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.522952:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.522954:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.522958:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.522960:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.522963:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.522964:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.522964:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.522966:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.522966:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.522967:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.522968:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677447.522974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:6.0:1619677447.522979:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.0:1619677447.522981:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677447.522981:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.522982:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.522983:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:24.0:1619677447.522985:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.522985:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:6.0:1619677447.522985:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.522988:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.522988:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:8.0:1619677447.522990:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.522992:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.522993:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96727001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.522995:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.522996:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.522997:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.523000:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.523002:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.523005:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.523006:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.523006:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.523008:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.523008:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.523009:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.523010:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.523021:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.523024:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.523025:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.523025:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.523027:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.523030:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.523055:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.523058:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.523059:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.523060:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.523063:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.523064:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.523065:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96727000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.523067:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.523068:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.523069:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.523072:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.523073:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.523076:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.523077:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.523078:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.523079:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.523080:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.523080:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.523082:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.523082:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.523092:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:20.0:1619677447.523093:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.523095:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:18.0:1619677447.523095:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.523095:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:20.0:1619677447.523096:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.523096:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:20.0:1619677447.523098:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.523099:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.523101:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.523117:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.523119:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.523120:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.523121:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.523122:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.523123:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.523197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677447.523197:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677447.523200:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677447.523201:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677447.523203:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.523205:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.523207:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.523208:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.523210:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.523211:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.523212:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.523216:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.523218:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.523221:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.523221:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.523222:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.523223:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.523224:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677447.523225:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677447.523226:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:1.0:1619677447.523236:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.523239:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.2:1619677447.523240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677447.523240:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.523240:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.523242:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.523243:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.523245:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677447.523245:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:13.0:1619677447.523246:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.523248:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.523249:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.523250:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.523405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.523411:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.523414:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.523416:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.523416:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.523418:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.523420:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677447.523421:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.523422:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.523423:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:5.0:1619677447.523423:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.523424:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:12.0:1619677447.523425:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.523426:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.523428:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677447.523429:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.523430:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.523430:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:12.0:1619677447.523431:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.523433:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.523433:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.523434:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.523436:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.523436:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.523437:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.523438:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677447.523450:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.523452:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.523453:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.523454:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.523455:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.523457:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.523467:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.523469:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.523472:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.523472:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.523474:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.523475:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.523476:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.523477:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.523478:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.523479:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.523482:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.523484:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.523486:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.523487:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.523487:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.523488:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.523489:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.523490:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.523490:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677447.523502:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.523503:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.523504:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.523506:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.523507:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.523508:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:17.0:1619677447.523607:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.523610:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.523610:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.523612:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.523615:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.523616:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677447.523617:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.523619:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.523620:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.523621:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677447.523623:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.523625:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677447.523628:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.523629:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677447.523629:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677447.523631:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.523632:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.523633:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.523634:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.523634:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.523637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.523643:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:1.0:1619677447.523645:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677447.523646:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.523647:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:1.0:1619677447.523647:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:24.0:1619677447.523648:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.523648:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.523649:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677447.523650:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.523653:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.523657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.523703:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.523706:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.523706:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.523707:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.523709:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.523712:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.523792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.523797:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.523800:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.523802:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.523802:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.523804:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.523806:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.523807:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.523808:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.523810:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.523811:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.523818:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.523819:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677447.523819:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.523821:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:5.0:1619677447.523822:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.523822:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677447.523823:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:18.0:1619677447.523824:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.523824:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.523825:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:18.0:1619677447.523826:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.523826:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.523827:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.523827:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677447.523828:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.523829:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.523830:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.523831:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.523832:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.523835:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.523837:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:13.0:1619677447.523838:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677447.523839:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.523840:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.523841:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:13.0:1619677447.523841:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677447.523842:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:13.0:1619677447.523842:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.523843:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.523844:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:13.0:1619677447.523844:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.523845:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.523845:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.523846:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.523856:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.523858:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.523859:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.523861:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.523862:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.523863:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.523918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677447.523965:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.523968:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.523969:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.523971:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.523971:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.523974:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677447.524117:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.524119:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.524120:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.524122:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.524125:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.524126:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.524127:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.524129:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.524130:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.524131:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.524134:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.524136:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.524138:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.524139:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.524139:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.524141:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.524141:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.524142:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.524143:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677447.524154:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.524156:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.524157:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.524158:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.524159:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.524160:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.524172:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.524174:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.524175:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.524176:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.524176:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.524176:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.524195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.524198:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.524200:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.524202:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.524204:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.524205:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.524206:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.524208:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.524209:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.524210:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.524211:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.524214:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.524216:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.524218:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.524219:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.524220:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.524222:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.524222:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.524223:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.524224:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.524235:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.524238:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.524239:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.524239:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.524241:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.524243:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.524261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.524335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:9.2:1619677447.524371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677447.524377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:2.0:1619677447.524378:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.524381:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.524382:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.524383:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.524383:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:2.0:1619677447.524384:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.524385:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:2.0:1619677447.524387:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.524388:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.524388:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.524390:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.524391:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677447.524393:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.524394:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.524395:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.524396:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677447.524399:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.524401:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677447.524404:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.524405:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677447.524405:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677447.524411:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677447.524411:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.524412:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:18.0:1619677447.524413:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677447.524413:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677447.524416:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.524416:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.524418:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.524420:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.524422:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.524423:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.524425:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.524426:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.524427:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.524428:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.524431:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:18.0:1619677447.524431:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.524433:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.524434:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.524434:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.524436:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.524438:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:18.0:1619677447.524438:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.524439:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:20.0:1619677447.524440:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.524440:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677447.524441:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.524442:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.524442:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.524442:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:20.0:1619677447.524443:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:18.0:1619677447.524443:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.524443:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:20.0:1619677447.524447:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.524449:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.524451:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.524452:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.524453:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.524454:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.524455:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:12.0:1619677447.524455:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677447.524456:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.524457:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:12.0:1619677447.524457:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677447.524458:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:12.0:1619677447.524458:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.524460:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.524462:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.524463:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.524469:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.524470:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677447.524470:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.524471:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000400:00000200:5.0:1619677447.524471:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.524472:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.524473:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.524473:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.524474:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.524475:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:17.0:1619677447.524590:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.524593:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.524594:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.524596:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.524598:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.524599:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.524601:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.524602:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.524603:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.524604:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.524607:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.524609:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.524611:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.524612:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.524613:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.524615:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.524615:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.524616:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.524616:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.524627:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.524630:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.524631:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.524633:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.524634:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.524636:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.524644:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.524647:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.524648:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.524648:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.524649:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.524650:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.524650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.524689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.2:1619677447.524885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.524888:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.524891:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.524893:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.524894:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.524895:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.524896:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677447.524898:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.524899:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.524900:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.524901:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677447.524905:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.524906:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.524909:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.524910:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.524910:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.524912:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.524912:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.524913:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.524914:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.524915:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.524921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.524924:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.524925:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.524925:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.524926:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.524926:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.524927:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.524927:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.524928:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.524930:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677447.524930:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.524946:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.524949:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.524950:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.524950:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.524951:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.524952:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.525000:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.525003:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.525003:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.525005:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.525008:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.525009:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.525011:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.525012:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.525013:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.525014:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.525019:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.525021:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.525025:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.525025:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.525026:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.525028:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.525029:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.525030:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.525030:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.525037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:12.0:1619677447.525041:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.525043:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.525044:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.525044:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.525046:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.525047:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:12.0:1619677447.525047:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.525047:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.525048:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.525049:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677447.525049:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.525052:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.525075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.525081:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.525084:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.525085:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.525087:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.525090:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.525091:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.525092:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.525094:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.525095:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.525096:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:4.2:1619677447.525096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677447.525099:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.525101:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.525104:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.525105:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.525105:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.525107:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.525108:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.525108:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.525109:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.525121:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.525123:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.525124:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.525125:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.525126:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:17.0:1619677447.525127:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.525129:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.525129:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:17.0:1619677447.525130:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:8.0:1619677447.525131:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.525134:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.525136:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.525137:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.525138:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.525139:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.525140:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.525143:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.525145:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.525147:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.525148:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.525149:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.525150:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.525151:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.525152:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.525153:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.525154:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677447.525162:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.525164:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.525165:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.525166:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:13.0:1619677447.525167:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.525168:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:12.0:1619677447.525168:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.525169:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677447.525169:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.525170:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.2:1619677447.525228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677447.525274:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.525277:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.525278:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.525280:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.525280:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.525283:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677447.525299:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.525302:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.525303:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.525305:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.525307:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.525309:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.525310:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.525311:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.525312:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.525314:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.525317:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.525319:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.525322:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.525323:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.525323:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.525325:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.525325:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.525326:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.525327:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.525435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.525500:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.525503:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.525505:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.525506:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.525507:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.525508:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.525510:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.525511:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.525512:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.525513:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.525516:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.525518:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.525520:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.525521:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.525521:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.525523:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.525523:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.525524:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.525525:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.525536:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.525538:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.525539:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.525542:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.525542:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.525544:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:6.0:1619677447.525572:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.525575:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.525575:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.525576:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.525579:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.525581:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.525641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.525646:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677447.525648:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677447.525649:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.525650:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677447.525651:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:18.0:1619677447.525652:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:17.0:1619677447.525652:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677447.525653:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.525653:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.525655:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.525656:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.525656:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.525658:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677447.525658:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.525659:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.525660:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.525661:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.525662:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.525663:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.525664:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.525664:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.525665:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677447.525666:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.525671:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.525671:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.525672:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.525674:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.525674:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677447.525674:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.525675:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.525677:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.525677:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.525678:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:1.0:1619677447.525678:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.525679:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.525679:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.525680:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.525681:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677447.525681:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677447.525682:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677447.525682:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:2.0:1619677447.525683:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:1.0:1619677447.525683:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:2.0:1619677447.525684:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:1.0:1619677447.525684:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677447.525685:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:9.2:1619677447.525687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.525695:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:17.0:1619677447.525695:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.525696:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677447.525697:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:17.0:1619677447.525697:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.525697:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:18.0:1619677447.525698:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.525698:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:1.0:1619677447.525698:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:17.0:1619677447.525699:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.525699:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:1.0:1619677447.525699:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677447.525700:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.525701:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.525701:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677447.525701:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.525702:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.525702:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.525703:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677447.525703:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677447.525704:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.525704:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.525704:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.525705:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.525706:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.0:1619677447.525730:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.525733:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.525734:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.525736:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.525738:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.525740:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677447.525741:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.525743:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.525744:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.525745:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677447.525748:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.525750:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.525752:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.525753:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.525754:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.525756:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.525756:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.525757:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.525757:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.525768:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.525771:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.525772:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.525773:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.525775:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.525776:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.525868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:4.2:1619677447.525959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.525975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.525983:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.525986:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.525987:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.525987:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.525989:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.525992:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:22.0:1619677447.526003:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.526006:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.526007:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.526007:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.526009:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.526014:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.526015:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.526018:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.526019:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.526020:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.526020:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.526020:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:4.2:1619677447.526131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.2:1619677447.526133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:6.0:1619677447.526138:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:24.0:1619677447.526139:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677447.526140:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677447.526141:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:24.0:1619677447.526142:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.526142:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677447.526144:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.526145:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.526147:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.526148:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.526148:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677447.526150:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.526150:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.526152:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.526152:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.526153:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.526154:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.526154:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677447.526155:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.526156:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.526158:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.526161:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:24.0:1619677447.526162:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.526163:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.526164:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677447.526165:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.526165:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.526166:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.526167:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677447.526167:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:24.0:1619677447.526168:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.526168:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:24.0:1619677447.526169:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.526169:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.526171:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.526172:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.526173:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.526174:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.526175:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677447.526179:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:4.2:1619677447.526180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.526181:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.526182:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.526183:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.526184:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.526185:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677447.526186:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.526188:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.526188:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.526189:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.526189:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.526190:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.526191:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.526192:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.526192:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.526195:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677447.526196:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.526199:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.526199:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.526201:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.526203:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.526204:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677447.526206:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.526207:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.526208:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.526209:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677447.526213:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.526215:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.526218:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.526219:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:18.0:1619677447.526219:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677447.526221:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.526222:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.526222:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.526223:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677447.526234:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.526236:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.526237:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.526239:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.526241:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.526242:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.526251:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.526254:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.526255:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.526255:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.526256:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.526257:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677447.526284:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.526286:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.526287:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.526289:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.526290:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.526292:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.526299:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.526300:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.526301:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.526302:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.526306:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.526308:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.526311:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.526311:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.526312:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.526314:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.526315:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.526315:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.526316:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.526328:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.526331:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.526332:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.526334:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.526336:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.526336:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677447.526407:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.526410:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.526412:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.526413:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.2:1619677447.526414:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:20.0:1619677447.526418:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.526420:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:20.0:1619677447.526421:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.526423:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.526423:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.526425:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:20.0:1619677447.526429:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.526432:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 440 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677447.526435:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.526436:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:20.0:1619677447.526437:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677447.526439:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677447.526439:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.526440:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.526441:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:13.0:1619677447.526451:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)++
00000800:00000200:13.0:1619677447.526454:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677447.526454:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.526456:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.526458:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677447.526458:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677447.526459:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677447.526459:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.526460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677447.526460:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.526461:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.526463:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.526637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.526642:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.526645:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.526647:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.526647:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.526650:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.526652:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.526653:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.526655:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.526656:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.526657:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.526661:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.526662:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.526666:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.526667:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.526667:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.526669:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.526670:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.526671:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.526671:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677447.526684:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.526687:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.526688:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.526690:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.526691:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:2.0:1619677447.526693:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.526694:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:6.0:1619677447.526694:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677447.526695:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:2.0:1619677447.526695:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:22.0:1619677447.526697:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.526697:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:2.0:1619677447.526697:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.526699:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.526699:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.526700:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.526701:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.526701:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.526702:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.526703:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.526705:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:6.0:1619677447.526705:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677447.526705:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.526706:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.526707:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.526708:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.526708:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:6.0:1619677447.526709:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.526710:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.526710:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.526711:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.526711:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677447.526712:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:2.0:1619677447.526712:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.526712:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.526714:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.526714:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.526715:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:24.0:1619677447.526716:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.526716:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:6.0:1619677447.526717:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.526719:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:9.2:1619677447.526720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.526720:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:24.0:1619677447.526722:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.526723:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:6.0:1619677447.526723:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.526724:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677447.526724:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.526725:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:24.0:1619677447.526726:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.526726:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.526727:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:18.0:1619677447.526727:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677447.526727:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.526727:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677447.526728:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.526729:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677447.526730:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:6.0:1619677447.526730:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.526730:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:18.0:1619677447.526731:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.526731:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.526733:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.526735:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.526738:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:9.2:1619677447.526738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677447.526740:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:22.0:1619677447.526741:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:13.0:1619677447.526741:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.526742:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.526743:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.526744:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677447.526744:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677447.526745:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.526745:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.526746:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.526748:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.526749:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.526750:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.526750:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.526751:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677447.526755:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.526757:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.526758:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.526759:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.526759:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.526760:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.526786:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.526788:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.526789:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.526790:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.526792:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.526793:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.526834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.526873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.526878:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.526881:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.526881:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.526882:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.526884:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.526887:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.527011:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.527013:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.527014:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.527016:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.527017:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.527018:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.527029:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.527032:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.527033:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.527033:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.527034:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.527034:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.527133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:20.2:1619677447.527152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.527155:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.527157:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.527160:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.527161:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.527163:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.527165:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.527166:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726945] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.527167:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.527168:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.527169:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.527173:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.527175:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.527178:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.527179:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.527179:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.527179:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.527180:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.527181:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.527181:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:5.0:1619677447.527181:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.527182:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.527183:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:5.0:1619677447.527183:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.527184:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.527186:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.527199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.527206:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.527209:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.527210:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.527212:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.527214:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.527215:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.527217:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.527218:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.527219:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.527220:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.527223:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.527225:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.527228:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.527228:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.527229:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.527231:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.527231:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.527232:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.527232:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.527247:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.527248:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.527248:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.527249:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.527250:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.527251:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:1.0:1619677447.527252:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.527253:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.527254:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.527254:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:17.0:1619677447.527255:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.527256:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.527256:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.527257:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:18.0:1619677447.527258:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.527258:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:1.0:1619677447.527258:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677447.527259:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677447.527259:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.527261:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.527262:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.527263:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.527263:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.527264:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.527265:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677447.527266:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.527271:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.527271:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.527274:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677447.527274:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677447.527276:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.527277:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:17.0:1619677447.527277:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.527278:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677447.527278:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677447.527279:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.527279:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677447.527279:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677447.527280:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:17.0:1619677447.527280:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677447.527281:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677447.527281:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.527282:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.527282:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.527286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.527293:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.527300:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:22.0:1619677447.527301:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.527303:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677447.527304:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677447.527305:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:2.0:1619677447.527305:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.527306:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.527307:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.527308:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (23)++
00000400:00000200:2.0:1619677447.527308:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.527309:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.527309:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (24)--
00000800:00000200:6.0:1619677447.527310:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677447.527310:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.527311:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.527312:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677447.527322:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919078fb5400] (21)++
00000800:00000200:18.0:1619677447.527325:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.6@o2ib3
00000800:00000200:2.0:1619677447.527325:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (22)--
00000400:00000200:18.0:1619677447.527327:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.6@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.527329:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.527331:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:18.0:1619677447.527332:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.527333:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.527335:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.527336:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:18.0:1619677447.527339:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.527341:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:18.0:1619677447.527344:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f90d380] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.527345:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e49d37c00] (20)++
00000800:00000200:18.0:1619677447.527346:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e49d37c00] (21)++
00000800:00000200:18.0:1619677447.527347:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e49d37c00] (22)--
00000800:00000200:18.0:1619677447.527348:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919078fb5400] (21)++
00000800:00000200:18.0:1619677447.527349:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919078fb5400] (22)--
00000800:00000200:18.0:1619677447.527349:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (21)--
00000800:00000200:24.2:1619677447.527354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (21)++
00000800:00000200:24.0:1619677447.527358:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)++
00000800:00000200:24.0:1619677447.527361:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e49d37c00] (23)--
00000400:00000200:24.0:1619677447.527362:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.527362:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)--
00000400:00000200:24.0:1619677447.527364:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.527368:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)--
00000800:00000200:24.2:1619677447.527387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (20)++
00000800:00000200:24.0:1619677447.527390:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)++
00000800:00000200:24.0:1619677447.527393:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677447.527394:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e49d37c00] (22)++
00000800:00000200:8.0:1619677447.527394:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)--
00000800:00000200:24.0:1619677447.527395:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e49d37c00] (22)--
00000800:00000200:24.0:1619677447.527395:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)--
00000800:00000200:6.0:1619677447.527583:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.527586:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.527587:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.527589:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.527591:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:6.0:1619677447.527591:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.527593:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:6.0:1619677447.527593:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.527594:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.527596:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.527596:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.527597:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:6.0:1619677447.527598:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.527599:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.527599:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677447.527601:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.527602:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.527603:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.527604:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.527605:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.527606:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:6.0:1619677447.527606:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.527608:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.527609:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.527609:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.527609:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.527611:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.527612:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677447.527612:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.527612:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.527613:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.527614:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.527615:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.527616:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.527617:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.527618:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:9.2:1619677447.527618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.527619:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.527619:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:2.0:1619677447.527625:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.527628:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.527629:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.527629:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.527631:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:2.0:1619677447.527631:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.527633:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677447.527634:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.527635:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.527636:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.527638:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:4.2:1619677447.527638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.527639:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677447.527651:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.527654:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.527655:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.527655:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.527656:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.527657:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:22.0:1619677447.527684:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.527687:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.527688:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.527688:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.527691:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.527693:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.527743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677447.527747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:5.0:1619677447.527749:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.527751:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:6.0:1619677447.527752:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:20.0:1619677447.527754:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.527754:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.527755:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677447.527756:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:5.0:1619677447.527756:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.527758:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.527758:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.527759:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.527760:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.527761:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.527761:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.527762:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.527763:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.527763:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:6.0:1619677447.527764:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.527765:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.527767:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.527767:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.527770:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:6.0:1619677447.527771:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.527773:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.527773:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.527774:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:6.0:1619677447.527775:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.527775:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677447.527776:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677447.527776:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.527777:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677447.527778:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.527778:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.527778:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677447.527779:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677447.527779:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:5.0:1619677447.527779:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.527780:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:12.0:1619677447.527790:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.527793:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:22.0:1619677447.527794:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:12.0:1619677447.527794:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.527795:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.527796:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.527796:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.527797:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.527798:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.527798:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677447.527800:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.527800:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.527809:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.527811:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.527812:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.527812:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.527813:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.527813:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.527816:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.527819:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.527820:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.527822:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.527824:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.527826:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.527827:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.527828:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.527830:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.527831:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.527835:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.527838:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.527841:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.527842:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.527843:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.527845:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.527845:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.527846:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.527846:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677447.527871:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.527874:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.527874:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.527875:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.527877:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.527880:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:13.0:1619677447.527914:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.527917:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.527917:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.527919:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.527921:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.527921:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.528017:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.528021:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.528021:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.528023:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.528026:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.528027:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.528028:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.528030:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.528031:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.528032:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.528035:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.528037:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.528038:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.528039:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.528040:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.528041:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.528042:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.528043:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.528044:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.528044:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.528048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.528056:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.528057:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.528057:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.528058:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:2.0:1619677447.528058:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.528059:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.528059:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.528060:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.528061:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677447.528061:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.528069:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.528072:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.528072:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.528074:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.528076:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.528078:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.528079:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.528080:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.528081:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.528083:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.528086:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.528088:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.528090:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.528091:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.528092:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.528093:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.528094:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.528094:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.528095:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.528107:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.528109:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.528110:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.528112:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.528113:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.528114:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.528124:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.528127:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.528128:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.528128:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.528129:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.528130:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.528153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677447.528155:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677447.528157:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677447.528158:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677447.528161:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677447.528163:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.528165:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677447.528167:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.528169:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.528170:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.528171:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677447.528174:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677447.528178:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677447.528181:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.528182:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:1.0:1619677447.528183:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:1.0:1619677447.528185:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:1.0:1619677447.528185:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677447.528186:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677447.528186:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:12.0:1619677447.528204:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.528206:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.528207:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.528208:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.528210:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.528211:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677447.528260:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.528263:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.528264:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.528266:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.528269:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.528270:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.528271:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.528272:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.528274:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.528275:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.528278:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.528280:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.528282:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.528283:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.528284:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.528285:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.528286:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.528287:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.528288:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.528289:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.528300:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.528302:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677447.528302:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677447.528303:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.528303:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.528304:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:6.0:1619677447.528305:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.528305:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.528306:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.528308:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.528414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.528422:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.528425:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.528426:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.528429:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.528431:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.528432:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.528434:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.528436:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.528437:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.528438:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.528442:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.528445:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.528448:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.528449:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.528450:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.528451:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.528452:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.528453:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.528453:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.528465:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:22.0:1619677447.528468:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:6.0:1619677447.528468:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (24)++
00000400:00000200:22.0:1619677447.528469:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.528469:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000400:00000200:6.0:1619677447.528470:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.528471:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:6.0:1619677447.528472:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.528475:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.528475:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.528476:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.528562:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.528565:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.528565:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.528567:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.528569:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.528570:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.528660:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.528663:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.528664:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.528666:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.528668:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.528670:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.528671:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.528672:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.528673:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.528674:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.528677:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.528679:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.528681:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.528682:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.528683:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.528684:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.528685:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.528685:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.528686:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.528695:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.528697:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.528699:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:6.0:1619677447.528700:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.528701:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.528702:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.528702:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.528704:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.528704:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.528705:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.528707:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.528708:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.528709:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.528711:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.528712:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.528712:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.528713:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.528714:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.528717:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.528717:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.528718:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.528720:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.528720:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.528721:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.528721:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677447.528732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.528735:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.528736:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.528737:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.528738:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.528738:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.528740:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.528741:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.528742:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.528743:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:20.0:1619677447.528744:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.528745:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.528746:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.528747:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.528748:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.528749:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.528750:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.528753:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.528754:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.528757:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.528758:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.528759:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.528760:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.528761:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.528762:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.528762:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.528767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.528774:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.528777:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.528778:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.528778:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.528780:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.528783:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.528851:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.528854:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.528855:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.528856:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.528858:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.528860:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.528861:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.528862:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.528863:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.528864:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.528867:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.528869:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.528872:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.528873:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.528873:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.528875:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.528875:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.528876:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.528876:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677447.528888:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.528891:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.528892:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.528893:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.528894:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.528896:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.528992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.528998:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.529001:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.529003:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.529003:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.529005:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.529007:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677447.529008:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.529009:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.529010:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.529011:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.529015:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.529016:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.529019:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.529020:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.529020:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.529022:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.529023:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.529024:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.529025:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.529026:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.2:1619677447.529031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.529034:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:1.0:1619677447.529035:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677447.529036:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.529037:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.529037:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:1.0:1619677447.529038:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.529039:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.529039:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.529041:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.529044:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.529053:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.529053:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:17.0:1619677447.529056:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.529056:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.529057:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677447.529057:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.529058:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:17.0:1619677447.529058:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.529059:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:8.0:1619677447.529059:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.529060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:8.0:1619677447.529062:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.529064:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.529066:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.529067:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.529068:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.529069:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:8.0:1619677447.529070:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.529071:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677447.529073:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.529075:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.529076:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.529076:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.529078:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.529079:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726916] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.529080:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.529081:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.529081:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.529085:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.529086:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.529087:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.529087:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.529088:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.529090:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.529090:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.529091:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.529091:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.529091:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.529092:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.529093:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.529093:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677447.529094:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.529094:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.529095:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.529095:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677447.529104:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.529106:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.529107:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.529108:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:20.0:1619677447.529109:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.529110:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677447.529111:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677447.529112:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.529112:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.529112:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.529114:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.529117:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.529121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:4.2:1619677447.529124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.529171:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.529173:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.529174:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.529175:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.529175:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.529176:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.529176:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.529178:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.529180:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.529181:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.529182:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.529183:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.529353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677447.529360:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.529360:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677447.529363:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:5.0:1619677447.529363:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677447.529363:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677447.529365:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:17.0:1619677447.529367:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.529367:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.529370:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.529370:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.529372:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.529372:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677447.529374:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.529374:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.529376:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.529376:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.529378:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.529378:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.529379:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.529379:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.529384:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.529385:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.529388:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.529388:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.529390:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.529391:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.529391:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:17.0:1619677447.529392:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.529392:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677447.529393:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.529394:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677447.529395:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.529395:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.529395:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677447.529396:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677447.529396:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:17.0:1619677447.529397:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677447.529397:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.529398:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:9.2:1619677447.529402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677447.529403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.529406:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.529408:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.529409:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.529409:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.529410:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:12.0:1619677447.529411:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677447.529413:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:12.0:1619677447.529414:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677447.529414:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.529415:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.529415:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.529418:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.529418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.529419:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:12.0:1619677447.529419:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.529420:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.529421:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.529422:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677447.529422:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:18.0:1619677447.529423:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677447.529424:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.529425:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.529425:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.529426:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.529427:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.529428:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.529429:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.529429:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.529686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:4.2:1619677447.529721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.529731:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.529734:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.2:1619677447.529736:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:5.0:1619677447.529737:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.529739:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.529741:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.529745:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.529774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677447.529777:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.529779:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:13.0:1619677447.529781:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:5.0:1619677447.529781:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.529782:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.529782:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.529783:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.529784:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.529785:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:13.0:1619677447.529786:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677447.529787:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.529789:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.529798:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.529800:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.529801:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.529802:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.529802:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.529804:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.529838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.529843:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.529846:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.529848:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.529848:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.529850:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.529852:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.529853:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.529854:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.529855:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.529856:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.529859:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.529861:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.529863:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.529864:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.529865:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.529866:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.529867:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.529871:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.529872:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677447.529874:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.529877:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.529878:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.529880:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.529882:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.529884:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.529885:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.529887:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.529888:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.529889:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.529891:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:2.0:1619677447.529892:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.529893:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677447.529893:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.529894:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.529894:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.529896:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677447.529897:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.529897:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.529898:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.529899:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.529899:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.529899:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.529900:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.529901:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677447.529912:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.529915:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.529916:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.529916:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.529918:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.529920:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.0:1619677447.530155:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.530158:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.530159:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.530161:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.530161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:8.0:1619677447.530164:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.530165:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.530167:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.530168:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.530169:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.530170:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:6.0:1619677447.530173:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:8.0:1619677447.530175:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.530176:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.530177:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677447.530177:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:6.0:1619677447.530179:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.530180:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.530181:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:6.0:1619677447.530181:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.530182:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.530183:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:6.0:1619677447.530183:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:8.0:1619677447.530184:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:6.0:1619677447.530184:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.530186:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000400:00000200:6.0:1619677447.530186:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.530187:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000400:00000200:6.0:1619677447.530187:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.530188:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:6.0:1619677447.530189:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.530193:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.530196:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:20.0:1619677447.530198:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:6.0:1619677447.530199:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.530200:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:6.0:1619677447.530200:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:20.2:1619677447.530201:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:6.0:1619677447.530201:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:24.0:1619677447.530203:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.530203:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:6.0:1619677447.530203:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.530203:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.530204:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:5.0:1619677447.530204:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:24.0:1619677447.530205:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677447.530205:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.530205:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.530205:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.530206:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677447.530207:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.530210:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677447.530216:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.530218:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.530219:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.530220:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.530223:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.530226:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.530227:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.530229:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.530230:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.530231:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.530242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677447.530247:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.530250:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.530251:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.530253:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.530253:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.530256:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:9.2:1619677447.530259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.530266:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.530266:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677447.530268:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.530269:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677447.530269:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:20.0:1619677447.530270:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:2.0:1619677447.530270:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:20.0:1619677447.530271:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:18.0:1619677447.530271:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.530272:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:18.0:1619677447.530273:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.530276:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.530414:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.530417:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.530418:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.530420:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.530423:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.530424:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.530425:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.530427:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.530428:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.530429:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.530432:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.530433:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.530435:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.530436:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.530437:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.530438:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.530438:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.530439:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.530439:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.530452:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.530454:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.530455:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.530456:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.530457:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.530458:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.530509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.530512:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.530515:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.530517:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.530518:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.530519:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.530521:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677447.530522:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.530523:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.530524:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.530525:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677447.530529:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.530531:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.530533:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.530534:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.530534:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.530536:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.530537:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.530538:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.530538:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677447.530549:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.530552:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.530552:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.530554:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.530555:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.530556:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.530571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.530646:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.530649:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.530650:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.530651:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.530654:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.530655:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.530656:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.530658:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.530659:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.530660:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.530662:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.530664:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.530665:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.530666:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.530667:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.530668:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.530668:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.530669:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.530670:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.530674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.530681:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.530682:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.530683:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.530684:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.530684:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.530686:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.530756:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.530759:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.530760:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.530760:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.530762:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.530765:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.530777:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677447.530777:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677447.530780:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:6.0:1619677447.530780:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.530781:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677447.530781:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.530782:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.530783:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.530783:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:17.0:1619677447.530784:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.530785:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.530787:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.530788:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.530789:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.530790:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.530791:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.530794:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.530796:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.530798:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.530799:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.530799:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.530801:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.530801:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.530803:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.530804:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.530804:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.530815:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.530816:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.530816:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.530816:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.530817:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.530817:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.530818:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.530819:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.530819:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.530820:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677447.530861:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.530863:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.530865:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.530866:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.530868:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.530870:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.530871:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.530872:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.530873:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.530874:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.530877:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.530879:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.530881:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.530882:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.530882:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.530884:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.530885:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.530885:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.530886:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:4.2:1619677447.530892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.530899:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677447.530900:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.530902:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:8.0:1619677447.530902:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.530903:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:20.0:1619677447.530903:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.530905:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.530905:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.530906:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:8.0:1619677447.530907:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.530908:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:8.0:1619677447.530909:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.530910:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.530912:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.530913:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.530914:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.530917:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.530919:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.530922:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.530923:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.530924:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.530926:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.530926:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.530927:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.530928:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.530939:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.530942:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.530942:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.530943:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.530945:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.530948:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.530974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:4.2:1619677447.531003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.531047:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.531050:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.531051:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.531051:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.531053:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.531056:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.531151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.531157:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.531160:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.531162:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.531162:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.531164:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.531165:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.531167:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.531168:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.531169:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.531170:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.531174:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.531175:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.531179:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.531180:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.531180:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.531182:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.531183:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.531183:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.531184:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.531359:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.531362:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.531363:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.531364:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.531366:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.531369:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.531472:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.531474:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.531475:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.531477:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.531479:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.531480:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.531482:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.531483:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.531484:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.531485:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.531488:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.531490:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.531492:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.531493:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.531494:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.531495:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.531496:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.531497:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.531497:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.531506:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.531508:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.531509:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.531511:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.531514:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.531516:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.531518:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.531519:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.531521:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.531521:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:22.0:1619677447.531522:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.531522:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.531523:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.531525:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.531526:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.531527:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.531528:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.531528:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.531531:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.531532:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.531533:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.531535:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.531536:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.531536:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.531537:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677447.531543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:18.0:1619677447.531544:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.531547:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.531547:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.531549:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.531552:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.531554:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.531556:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.531557:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.531559:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.531559:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.531564:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.531566:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.531570:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.531571:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.531571:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.531573:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.531574:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.531575:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.531576:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.531577:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677447.531586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:13.0:1619677447.531587:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.531590:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677447.531590:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:13.0:1619677447.531591:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.531592:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.531592:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.531593:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:12.0:1619677447.531594:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.531595:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.531595:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.531596:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677447.531597:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.531597:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.531598:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.531600:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.531603:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:18.0:1619677447.531645:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.531648:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.531649:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.531651:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.531653:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.531654:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.531655:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.531657:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.531658:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.531659:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.531662:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.531664:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.531665:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.531666:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.531667:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.531668:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.531669:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.531669:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.531670:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.531674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.531681:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.531683:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.531683:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.531685:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.531685:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.531687:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.531793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.531801:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.531804:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.531805:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.531807:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.531810:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.531811:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.531812:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.531814:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.531815:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.531816:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:4.2:1619677447.531818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677447.531819:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.531820:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.531823:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.531824:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.531824:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.531826:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.531826:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.531828:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.531829:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.531829:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.531835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.531840:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677447.531842:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.531842:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.531843:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.531844:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.531844:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677447.531844:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.531846:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.531846:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.531848:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.531857:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.531860:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.531860:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.531861:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:4.2:1619677447.531861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677447.531862:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.531862:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677447.531865:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.531868:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.531868:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.531869:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.531871:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.531873:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.531931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.531934:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.531937:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.531939:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.531940:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.531942:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.531944:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677447.531945:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.531946:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.531947:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.531948:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677447.531951:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.531953:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677447.531956:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.531957:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677447.531958:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677447.531959:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677447.531960:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.531961:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.531961:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.2:1619677447.532083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.532089:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.532092:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.532094:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.532094:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.532097:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.532099:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.532100:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.532101:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.532102:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.532103:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.532107:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.532109:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.532112:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.532112:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.532113:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.532115:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.532115:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.532117:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.532118:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.532119:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.532124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.532126:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.532129:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.532129:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.532130:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.2:1619677447.532335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:22.0:1619677447.532341:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.532344:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677447.532345:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677447.532347:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.532349:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.532351:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677447.532352:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.532353:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.532354:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.532356:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677447.532359:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.532361:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677447.532363:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.532364:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:22.0:1619677447.532365:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677447.532366:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:22.0:1619677447.532367:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.532368:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677447.532368:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:1.0:1619677447.532382:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.532384:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.532385:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.532385:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.532387:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.532390:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.532399:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.532400:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.532401:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.532402:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.532403:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.532404:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.532405:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.532405:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.532406:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.532406:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.532407:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:17.0:1619677447.532408:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.532409:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.532410:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.532412:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.532412:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.532416:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.532417:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.532420:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.532420:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.532421:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.532423:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.532423:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.532424:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.532424:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.532438:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.532440:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.532441:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.532441:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.532444:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.532446:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.532455:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.532458:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.532458:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:24.0:1619677447.532459:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:22.0:1619677447.532459:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.532460:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.532460:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.532461:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.532462:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.532464:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.532467:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.532469:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:4.2:1619677447.532469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677447.532470:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726874] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.532472:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.532472:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.532473:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677447.532476:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.532478:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.532480:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.532481:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677447.532482:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677447.532484:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.532484:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.532485:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.532485:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.532497:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.532499:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.532500:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.532501:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.532503:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.532504:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.532633:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.532636:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.532636:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.532638:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.532640:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.532642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:6.0:1619677447.532642:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.532643:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.532645:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.532646:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.532647:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.532649:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.532651:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.532653:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.532655:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.532655:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.532657:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.532657:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.532658:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.532659:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.532663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.532671:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.532674:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.532675:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.532675:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.532677:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.532680:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:12.0:1619677447.532686:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:9.2:1619677447.532686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:12.0:1619677447.532689:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.532690:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.532691:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.532693:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.532693:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:13.0:1619677447.532694:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.532696:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.532697:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.532697:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.532698:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.532699:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:12.0:1619677447.532704:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.532706:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.532707:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.532708:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.532709:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.532710:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.532720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:12.0:1619677447.532726:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.532728:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.532729:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.532731:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.532732:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.532733:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.532743:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.532746:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.532747:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.532747:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.532748:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.532750:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.532762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677447.532769:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:12.0:1619677447.532770:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677447.532772:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:12.0:1619677447.532772:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:20.0:1619677447.532773:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.532774:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.532775:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:13.0:1619677447.532776:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677447.532777:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000400:00000200:12.0:1619677447.532777:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.532780:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677447.532780:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.532856:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.532859:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.532859:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.532862:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.532863:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.532865:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.532867:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.532868:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.532869:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.532870:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.532873:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.532875:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.532878:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.532879:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.532880:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.532881:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.532882:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.532883:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.532884:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.532884:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:4.2:1619677447.532891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:2.0:1619677447.532894:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:5.0:1619677447.532897:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677447.532897:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.532897:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.0:1619677447.532898:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677447.532898:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.532899:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.532900:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.532901:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.532902:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677447.532902:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.532902:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.532904:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.532904:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:8.0:1619677447.532906:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.532907:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.532909:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.532910:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.532911:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.532912:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.532916:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.532917:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.532920:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.532921:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.532922:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.532923:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.532924:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.532924:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.532925:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677447.532931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.532934:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.532936:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:1.0:1619677447.532937:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:20.0:1619677447.532938:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677447.532940:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.532940:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:1.0:1619677447.532940:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.532941:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:20.0:1619677447.532942:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.532942:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.532943:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.532944:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.532945:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.532946:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.532947:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.532947:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:20.0:1619677447.532951:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.532952:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:20.0:1619677447.532955:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.532956:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:20.0:1619677447.532956:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:20.0:1619677447.532958:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:20.0:1619677447.532959:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.532960:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.532961:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:4.2:1619677447.532962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.532964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.532971:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.532974:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.532975:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.532975:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.532977:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:24.0:1619677447.532980:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:17.0:1619677447.532980:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.532983:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.532984:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.532986:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.532988:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.532990:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.532991:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.532993:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.532994:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.532995:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.532998:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.533000:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.533004:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.533005:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.533005:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.533007:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.533008:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.533008:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.533009:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.533011:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:9.2:1619677447.533013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:22.0:1619677447.533014:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.533015:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.533015:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.533018:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.533019:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:22.0:1619677447.533021:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.533022:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.533023:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.533023:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.533025:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.533029:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677447.533029:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.533032:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.533032:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.533033:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.533034:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.533034:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.533080:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.533083:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.533084:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.533087:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.533089:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.533091:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.533093:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.533094:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.533095:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.533097:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.533101:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.533103:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.533105:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.533106:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.533107:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.533108:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.533109:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.533110:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.533110:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.533114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:13.0:1619677447.533115:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.533118:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.533118:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.533120:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.533121:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.533122:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:13.0:1619677447.533122:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677447.533125:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.533126:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.533126:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.2:1619677447.533127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:18.0:1619677447.533128:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.533131:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677447.533133:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.533135:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.533138:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.533138:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.533140:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.533142:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.533143:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.533145:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.533146:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.533147:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.533150:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.533152:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.533155:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.533156:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.533157:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.533158:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.533159:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.533160:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.533161:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.533171:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.533173:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.533174:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.533176:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.533177:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.533179:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.533337:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.533340:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.533341:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.533343:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.533345:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.533347:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677447.533349:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.533351:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:2.0:1619677447.533351:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.533352:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677447.533352:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.533353:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677447.533356:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.533357:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000400:00000200:18.0:1619677447.533359:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.533359:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:18.0:1619677447.533360:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.533361:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.533362:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.533363:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.533364:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.533366:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.533368:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.533369:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.533369:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.533371:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.533371:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.533371:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677447.533371:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677447.533373:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.533374:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.533374:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677447.533374:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.533375:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.533375:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.533376:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.533376:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677447.533377:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.533381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.533388:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677447.533389:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677447.533390:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.533391:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.533392:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.533393:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677447.533393:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:6.0:1619677447.533393:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.533393:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.533395:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.533396:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.533398:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.533399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.533406:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.533409:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.533410:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.533410:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.533411:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.533411:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.533423:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.533426:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.533426:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.533429:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.533431:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.533433:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.533434:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.533435:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.533436:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.533437:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.533441:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.533442:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.2:1619677447.533445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677447.533445:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.533446:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.533447:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.533448:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.533449:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.533451:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.533452:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.533452:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.533462:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677447.533463:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677447.533464:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677447.533465:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:22.0:1619677447.533466:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.533466:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.533466:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.533468:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.533468:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.533471:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.533492:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.533494:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.533495:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.533496:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.533498:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.533500:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.533696:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.533699:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.533700:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:6.0:1619677447.533700:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677447.533702:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.533703:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.533704:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677447.533705:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.533707:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.533707:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.533708:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.533710:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.533710:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.533711:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.533712:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.533712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677447.533712:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.533713:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:6.0:1619677447.533714:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.533716:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.533717:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.533719:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.533722:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:6.0:1619677447.533723:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.533725:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.533726:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.533727:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.533728:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677447.533729:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.533730:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.533730:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.533730:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677447.533731:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.533731:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.533732:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:6.0:1619677447.533733:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.533733:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.533735:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.533735:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.533736:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.533737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:12.0:1619677447.533744:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677447.533745:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677447.533746:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:12.0:1619677447.533746:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677447.533747:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:12.0:1619677447.533747:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.533748:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:20.0:1619677447.533748:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:22.0:1619677447.533749:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:18.0:1619677447.533749:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.533749:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.533749:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.533750:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:13.0:1619677447.533750:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677447.533751:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:20.0:1619677447.533751:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.533752:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.533752:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:20.0:1619677447.533753:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.533754:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.533755:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.0:1619677447.533756:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677447.533838:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.533841:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.533841:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.533843:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.533846:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.533847:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.533848:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.533850:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.533851:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.533852:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.533855:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.533856:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.533859:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.533859:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.533860:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.533862:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.533862:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.533863:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.533863:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.533875:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.533878:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.533879:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.533879:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.533881:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.533883:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.533944:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.533946:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.533947:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.533949:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.533952:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.533953:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.533955:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.533956:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.533957:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.533958:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677447.533961:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.533963:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.533966:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.533967:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.533967:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.533969:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.533970:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.533971:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.533971:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677447.534206:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.534209:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.534210:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.534212:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.534215:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.534216:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.534217:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.534219:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.534220:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.534221:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.534224:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.534226:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.534228:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.534229:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.534230:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.534232:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.534232:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.534233:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.534234:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.534245:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.534248:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.534248:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.534250:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.534251:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.534253:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.534603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:1.0:1619677447.534609:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.534611:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.534612:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.534614:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.534617:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.534619:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.534620:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.534621:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:1.0:1619677447.534622:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.534623:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.534624:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:1.0:1619677447.534624:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.534625:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.534628:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.534629:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.534631:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.534632:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:24.0:1619677447.534633:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.534634:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96726845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.534635:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.534636:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.534636:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.534636:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:24.0:1619677447.534637:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.534638:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:1.0:1619677447.534638:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.534638:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.534639:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.534640:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677447.534642:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.534644:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.534647:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.534647:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.534648:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.534649:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.534650:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.534651:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.534651:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.534655:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.534656:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.534657:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.534658:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:12.0:1619677447.534658:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.534659:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.534660:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.534661:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677447.534661:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.534662:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.534664:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.534664:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.534664:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.2:1619677447.534665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:13.0:1619677447.534665:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.534665:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.534666:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677447.534666:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.534668:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677447.534668:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.534669:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.534670:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.534670:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.534672:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.534673:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.534675:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677447.534676:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.534676:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.534677:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000400:00000200:20.0:1619677447.534678:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:12.0:1619677447.534678:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.534679:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.534679:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677447.534680:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.534680:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677447.534680:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000400:00000200:20.0:1619677447.534681:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.534681:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.534681:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.534681:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:20.0:1619677447.534682:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.534682:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:20.0:1619677447.534683:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.534683:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.534684:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:20.0:1619677447.534688:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.534690:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677447.534693:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.534694:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.534695:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.534697:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.534697:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.534698:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.534699:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677447.534714:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.534715:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.534717:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.534718:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:17.0:1619677447.534719:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.534719:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.534720:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.534722:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.534722:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.534723:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.534724:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.534725:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.534772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.534776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.534783:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.534785:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.534786:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.534788:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.534791:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.534793:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.534794:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.534795:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.534796:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.534797:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.534800:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.534802:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.534805:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.534806:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.534806:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.534808:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.534808:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.534809:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.534810:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677447.534813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.534818:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.534821:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:17.0:1619677447.534822:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:6.0:1619677447.534822:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.534823:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.534824:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:22.0:1619677447.534825:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.534825:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.534826:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.534828:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:17.0:1619677447.534828:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.534830:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.534834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:6.0:1619677447.534854:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.534857:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.534858:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.534861:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.534864:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.534865:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.534867:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.534868:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.534869:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.534870:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.534874:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.534876:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.534878:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.534879:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.2:1619677447.534880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:6.0:1619677447.534880:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.534881:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.534882:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.534882:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.534883:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.534890:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.534892:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.534893:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.534895:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:5.0:1619677447.534895:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.534895:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677447.534897:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.534898:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:1.0:1619677447.534898:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677447.534899:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.534899:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:17.0:1619677447.534899:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.534899:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000400:00000200:1.0:1619677447.534899:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.534900:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:1.0:1619677447.534901:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.534903:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:9.2:1619677447.534926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.534934:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.534936:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.534937:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.534938:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.534940:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.534943:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.535096:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.535098:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.535099:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.535101:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.535103:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.535105:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.535106:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726837] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.535107:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.535108:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.535109:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.535113:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.535115:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.535119:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.535119:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.535120:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.535122:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.535123:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.535124:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.535125:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.535126:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677447.535135:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677447.535136:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677447.535138:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (25)--
00000800:00000200:1.0:1619677447.535138:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.535138:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677447.535139:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.535139:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677447.535140:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.535142:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.535143:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677447.535143:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677447.535144:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.535144:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:13.0:1619677447.535145:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.535145:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.535146:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.535147:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.535276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:4.2:1619677447.535371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.535398:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.535401:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.535402:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.535404:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.535407:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.535408:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.535409:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.535412:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.535412:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.535414:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.535416:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.2:1619677447.535416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:2.0:1619677447.535417:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.535418:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.535419:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.535420:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.535420:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677447.535421:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.535422:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.535422:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:22.0:1619677447.535423:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.535423:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677447.535424:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.535425:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.535425:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677447.535425:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.535425:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.535426:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:5.0:1619677447.535427:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.535428:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.535429:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.535431:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.535432:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.535433:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.535434:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.535435:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677447.535439:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:5.0:1619677447.535439:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.535441:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677447.535442:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.535443:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.535443:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.535443:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.535444:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.535444:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:17.0:1619677447.535445:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.535446:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.535447:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.535447:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.535448:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677447.535451:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.535452:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.535454:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.535455:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.535456:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.535456:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.535457:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677447.535460:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.535462:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.535463:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.535463:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.535465:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.535467:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.535494:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.535496:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.535497:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.535499:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.535502:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.535503:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.535504:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.535506:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.535507:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.535508:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.535511:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.535513:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.535517:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.535517:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.535518:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.535520:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.535520:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.535521:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.535522:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.535533:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.535536:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.535537:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.535538:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.535540:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.535541:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.535672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677447.535687:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.535690:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.535690:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.535693:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.535695:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.535697:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.535698:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.535700:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.535701:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.535702:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.535706:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.535708:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.535711:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.535711:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.535712:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.535714:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.535714:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.535715:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.535716:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677447.535719:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.535721:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.535722:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.535724:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.535726:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.535727:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.535728:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.535730:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.535731:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.535733:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.535733:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.535735:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:9.2:1619677447.535926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.535933:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.535936:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.535937:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.535937:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.535939:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.535943:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.535966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.535973:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.535975:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677447.535976:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.535977:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677447.535978:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.535978:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:17.0:1619677447.535980:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.535981:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677447.535983:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.535984:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.535985:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677447.535986:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:17.0:1619677447.535987:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.535988:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.535989:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.535990:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.535990:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.535991:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.535992:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677447.535993:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677447.535998:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000400:00000200:17.0:1619677447.535999:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.536001:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:17.0:1619677447.536002:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.536003:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.536004:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.536005:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:17.0:1619677447.536005:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.536006:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.536006:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677447.536006:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.536007:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677447.536008:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.536009:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.536009:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677447.536009:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.536010:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.536011:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.536011:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.536012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:4.2:1619677447.536017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:2.0:1619677447.536020:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.536021:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.536021:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.536022:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:2.0:1619677447.536022:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.536023:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677447.536024:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677447.536024:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.536024:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:24.0:1619677447.536025:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677447.536025:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677447.536027:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:22.0:1619677447.536027:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:13.0:1619677447.536027:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677447.536027:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:22.0:1619677447.536028:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.536028:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.536029:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677447.536030:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:24.0:1619677447.536031:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.536031:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.536034:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.536034:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.536034:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:24.0:1619677447.536035:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:13.0:1619677447.536036:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.536037:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96726822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.536039:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.536039:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:13.0:1619677447.536039:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:24.0:1619677447.536040:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.536041:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:22.0:1619677447.536041:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.536044:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677447.536045:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.536045:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.536045:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.536046:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.2:1619677447.536046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:24.0:1619677447.536047:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677447.536047:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.536049:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:24.0:1619677447.536050:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.536050:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.536051:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.536051:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:24.0:1619677447.536053:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.536053:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.536054:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:20.0:1619677447.536054:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.536055:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.536055:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.0:1619677447.536055:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.536056:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.536058:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:6.0:1619677447.536058:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.536059:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:20.0:1619677447.536059:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96726821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.536059:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.536061:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.536061:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.536062:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.536063:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.536064:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677447.536066:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:20.0:1619677447.536068:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.536069:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.536069:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.536070:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.536070:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.536072:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.536073:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:1.0:1619677447.536073:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.536074:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.536075:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.0:1619677447.536076:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.536076:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.0:1619677447.536077:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677447.536078:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.536079:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:20.0:1619677447.536079:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:6.0:1619677447.536079:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.536080:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:20.0:1619677447.536080:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:22.0:1619677447.536081:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677447.536081:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.536085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.536091:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.536093:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.536094:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.536095:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.2:1619677447.536213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.536219:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.536221:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.536224:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.536224:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.536227:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.536228:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677447.536230:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.536231:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677447.536232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:5.0:1619677447.536232:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.536233:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677447.536236:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.536238:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.536241:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.536242:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677447.536242:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677447.536244:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677447.536245:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.536246:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.536246:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:6.0:1619677447.536306:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.536309:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.536310:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.536312:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.536314:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.536316:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.536318:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.536320:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.536321:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.536322:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.536326:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.536328:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.536331:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.536332:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.536333:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.536335:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.536336:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.536336:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.536337:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677447.536422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.536425:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.536427:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.536430:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.536431:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.536432:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.536434:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.536435:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.536437:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.536438:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.536439:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.536442:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.536444:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.536446:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.536447:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.536448:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.0:1619677447.536449:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677447.536450:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.536451:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677447.536451:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.536452:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677447.536452:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.536452:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:8.0:1619677447.536454:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.536458:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.536459:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.536461:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.536462:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.536463:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.536464:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.536467:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.536469:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.536473:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.536474:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.536474:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.536476:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.536477:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.536477:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.536478:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.536482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.536491:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.536494:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.536494:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.536495:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.536497:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.536500:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.536610:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.536613:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.536614:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.536616:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.536619:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.536620:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.536621:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.536622:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.536624:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.536625:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.536628:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.536630:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.536632:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.536633:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.536634:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.536636:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.536636:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.536637:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.536637:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.536650:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.536652:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.536653:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.536655:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.536657:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.536658:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.536727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.536802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.536808:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.536810:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.536813:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.536813:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.536816:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.536818:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.536820:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.536822:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.536823:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.536825:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.536829:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.536832:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.536835:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.536836:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.536837:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.536839:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.536839:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.536840:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.536841:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677447.536850:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.536852:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.536853:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.536854:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.536856:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.536859:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:13.0:1619677447.536865:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.536867:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.536868:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.536870:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.536871:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.536873:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.536886:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.536889:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.536890:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.536890:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.536891:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.536892:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.537073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.537111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.537113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.537118:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.2:1619677447.537121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:2.0:1619677447.537121:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.537122:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.537122:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.537124:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:2.0:1619677447.537124:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.537126:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:2.0:1619677447.537127:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:20.0:1619677447.537128:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677447.537129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.537129:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.537131:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.537133:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677447.537134:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.537136:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.537136:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677447.537137:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.537138:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:18.0:1619677447.537139:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.537140:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.537140:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.537141:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.537141:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:20.0:1619677447.537142:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.537144:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677447.537147:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.537148:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677447.537148:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677447.537150:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677447.537151:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.537152:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.537153:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.537154:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.2:1619677447.537159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:2.0:1619677447.537159:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.537162:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.537163:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.537164:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.537165:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.537167:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:2.0:1619677447.537167:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.537168:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.537169:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:2.0:1619677447.537169:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.537170:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.537172:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.537173:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.537174:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.537177:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.537178:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.537181:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.537181:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.537182:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.537184:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.537184:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.537185:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.537186:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.537187:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677447.537198:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:18.0:1619677447.537199:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.537201:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:5.0:1619677447.537201:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:18.0:1619677447.537202:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:5.0:1619677447.537202:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.537203:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677447.537204:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.537204:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.537207:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:12.0:1619677447.537272:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677447.537275:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677447.537278:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.537279:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677447.537280:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.537282:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:12.0:1619677447.537283:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96726807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677447.537284:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677447.537285:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677447.537286:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:12.0:1619677447.537290:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.537292:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:12.0:1619677447.537300:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.537301:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:12.0:1619677447.537302:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.2:1619677447.537304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:12.0:1619677447.537304:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:12.0:1619677447.537305:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677447.537306:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677447.537306:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:20.0:1619677447.537307:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.537309:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.537312:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.537313:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.537314:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.537316:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677447.537317:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.537318:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.537319:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.537320:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677447.537324:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.537327:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.537329:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.537330:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.537331:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.537332:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.537333:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.537334:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.537334:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677447.537346:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.537352:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.537353:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.537354:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.537356:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.537356:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677447.537358:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:2.0:1619677447.537359:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.537360:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.537362:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.537364:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.537366:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.537367:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726805] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.537368:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.537369:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.537370:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.537374:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.537375:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.537377:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.537378:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.537379:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.537380:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.537381:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.537382:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.537382:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.537386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.537393:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.537395:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.537396:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.537397:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.537398:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.537399:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.537407:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.537410:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.537411:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.537413:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.537415:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.537417:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.537418:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.537420:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.537421:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.537422:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.537425:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.537427:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.537430:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.537430:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.537431:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.537433:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.537433:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.537435:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.537436:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.537436:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.537447:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677447.537447:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.537449:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.537449:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.537450:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677447.537450:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677447.537451:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.537451:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.537453:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.537456:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.537473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.537481:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919078fb5400] (21)++
00000800:00000200:2.0:1619677447.537484:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.6@o2ib3
00000800:00000200:18.0:1619677447.537486:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (22)--
00000400:00000200:2.0:1619677447.537488:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.6@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.537490:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.537492:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:2.0:1619677447.537493:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.537495:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.537497:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.537498:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:2.0:1619677447.537503:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.537507:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:2.0:1619677447.537510:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f90d380] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.537511:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e49d37c00] (20)++
00000800:00000200:2.0:1619677447.537511:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e49d37c00] (21)++
00000800:00000200:2.0:1619677447.537513:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e49d37c00] (22)--
00000800:00000200:2.0:1619677447.537514:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919078fb5400] (21)++
00000800:00000200:2.0:1619677447.537515:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919078fb5400] (22)--
00000800:00000200:2.0:1619677447.537515:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (21)--
00000800:00000200:24.2:1619677447.537519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (21)++
00000800:00000200:9.2:1619677447.537521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.537523:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)++
00000800:00000200:24.0:1619677447.537526:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e49d37c00] (23)--
00000400:00000200:24.0:1619677447.537527:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.537527:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)--
00000800:00000200:18.0:1619677447.537528:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:24.0:1619677447.537529:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.537531:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:24.0:1619677447.537532:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)--
00000400:00000200:18.0:1619677447.537532:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.537532:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.537535:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.537538:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.537613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:17.0:1619677447.537656:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.537659:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.537660:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.537660:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.537663:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:17.0:1619677447.537668:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.537669:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.537672:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.537673:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.537673:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.537674:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.537674:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:6.0:1619677447.537698:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.537701:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.537701:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.537704:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.537706:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.537708:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.537709:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.537710:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.537712:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.537713:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.537716:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.537718:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.537721:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.537722:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.537723:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.537725:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.537725:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.537726:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.537726:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.537823:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (21)++
00000800:00000200:2.0:1619677447.537826:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:18.0:1619677447.537827:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000400:00000200:2.0:1619677447.537829:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.537832:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.537834:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677447.537835:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.537836:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.537837:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.537838:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677447.537842:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.537843:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 600 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:4.2:1619677447.537846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:2.0:1619677447.537846:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.537846:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:2.0:1619677447.537847:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:2.0:1619677447.537849:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:2.0:1619677447.537849:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (21)++
00000800:00000200:2.0:1619677447.537850:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (22)--
00000800:00000200:2.0:1619677447.537850:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (21)--
00000800:00000200:8.0:1619677447.537858:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.537861:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.537862:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677447.537863:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:8.0:1619677447.537864:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.537865:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.537866:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.537867:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.537868:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.537869:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.537870:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.537870:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.537871:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:8.0:1619677447.537872:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.537873:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.537874:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.537877:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.537879:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.537882:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.537883:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.537884:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.537885:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.537886:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.537887:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:4.2:1619677447.537887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677447.537889:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.537889:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.537895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.537899:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677447.537902:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677447.537902:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.537902:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.537903:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.537904:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.537905:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.537906:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.537906:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.537908:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:9.2:1619677447.537928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.537936:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.537938:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.537939:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.537940:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.537942:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.537945:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.538006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677447.538047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677447.538051:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.538053:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.538054:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.538055:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.538057:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.538058:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.538129:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.538132:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.538133:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.538135:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.538138:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.538139:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677447.538141:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.538142:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.538143:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.538144:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677447.538147:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.538148:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.538151:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.538151:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.538152:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.538154:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.538154:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.538155:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.538156:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.538157:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.538166:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.538169:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.538169:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.538169:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:24.0:1619677447.538171:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:12.0:1619677447.538171:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.538172:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:24.0:1619677447.538173:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:18.0:1619677447.538173:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:13.0:1619677447.538173:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.538174:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.538175:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:2.0:1619677447.538175:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.538176:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:2.0:1619677447.538177:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.538178:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.538180:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.538181:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.538182:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.538183:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:2.0:1619677447.538183:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:13.0:1619677447.538186:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.538187:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:2.0:1619677447.538187:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.538188:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.538188:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.538189:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.538189:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.538191:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.538192:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.538192:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.538194:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.538194:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.538196:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.538197:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.538198:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.538200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.538209:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.538210:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.538210:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:18.0:1619677447.538211:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.538211:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.538212:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.538213:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.538213:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.538214:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.538214:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.538216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.538224:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.538227:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.538227:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.538227:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.538228:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.538229:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.538387:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.538389:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.538390:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.538392:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.538394:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.538395:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.538408:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.538411:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.538411:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.538413:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.538415:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.538415:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.538475:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.538478:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.538478:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.538480:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.538483:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.538484:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.538485:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.538487:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.538488:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.538489:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.538492:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.538494:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.538497:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.538498:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.538498:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.538500:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.538500:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.538501:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.538501:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.538513:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.538516:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.538517:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.538517:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.538520:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.538523:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.538546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.538553:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.538556:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.538557:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.538559:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.538559:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.538562:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.538593:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.538596:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.538597:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.538598:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.538599:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.538601:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.538611:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.538614:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.538615:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.538616:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.538616:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.538618:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677447.538774:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.538777:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.538777:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.538779:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.538781:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.538782:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.538784:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.538785:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.538786:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.538787:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.538791:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.538792:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.538795:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.538796:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.538797:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.538798:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.538799:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.2:1619677447.538800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677447.538800:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.538800:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.538813:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.538816:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.538817:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.538819:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.538820:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.538821:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.538823:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:5.0:1619677447.538823:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:24.0:1619677447.538824:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.538827:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.538830:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.538831:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.538832:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.538834:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677447.538835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:8.0:1619677447.538835:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.538836:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.538838:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:8.0:1619677447.538839:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.538841:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:8.0:1619677447.538841:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.538843:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.538843:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.0:1619677447.538844:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.538845:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:20.0:1619677447.538846:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.538846:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677447.538847:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:12.0:1619677447.538848:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.0:1619677447.538848:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.538848:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:20.0:1619677447.538849:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96726784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.538849:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.538849:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:20.0:1619677447.538850:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.538851:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.538851:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.538851:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.538852:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:12.0:1619677447.538854:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.538854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.538855:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:20.0:1619677447.538856:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.538859:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.538860:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677447.538861:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.538861:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677447.538862:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.538863:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.538864:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:20.0:1619677447.538865:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.538865:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:18.0:1619677447.538865:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.538865:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.538866:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.538867:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:2.0:1619677447.538868:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.538871:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.538879:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.538882:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.538882:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.538884:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677447.538885:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.538887:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.539052:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.539054:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.539055:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.539057:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.539058:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.539059:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677447.539074:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.539077:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.539077:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.539079:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.539082:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.539083:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.539085:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.539086:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.539087:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.539088:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.539091:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.539093:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.539095:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.539096:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.539097:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.539099:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.539099:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.539100:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.539100:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.539113:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.539115:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.539116:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.539116:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.539118:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.539120:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.539216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.539223:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677447.539225:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677447.539226:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:22.0:1619677447.539228:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:20.0:1619677447.539228:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.539228:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.539229:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:5.0:1619677447.539230:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.539231:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.539232:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677447.539233:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.539234:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.539235:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677447.539235:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.539236:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.539237:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.539237:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.539238:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.539239:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.539241:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677447.539242:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.539244:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.539246:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.539247:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.539248:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677447.539248:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677447.539248:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677447.539250:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.539251:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.539251:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677447.539252:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.539252:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (22)++
00000800:00000200:5.0:1619677447.539252:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.539253:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:22.0:1619677447.539254:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (23)--
00000800:00000200:22.0:1619677447.539254:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677447.539254:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677447.539256:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:9.2:1619677447.539256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677447.539257:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.539257:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.2:1619677447.539259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.539262:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.539264:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.539264:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677447.539264:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (23)++
00000800:00000200:20.0:1619677447.539265:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.539266:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (24)--
00000800:00000200:2.0:1619677447.539267:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (24)++
00000800:00000200:2.0:1619677447.539267:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (24)--
00000400:00000200:18.0:1619677447.539268:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.539268:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.539269:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:18.0:1619677447.539270:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.539270:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:2.0:1619677447.539270:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.539271:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.539272:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677447.539273:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.539274:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.539275:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.539278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.539285:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.539288:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.539289:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.539289:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.539290:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.539293:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677447.539301:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.539303:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.539305:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.539306:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.539308:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.539309:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.539311:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.539312:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.539313:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.539314:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.539317:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.539319:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.539322:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.539322:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.539323:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.539325:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.539325:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.539325:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.539326:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.539340:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.539342:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.539343:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.539343:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.539346:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.539348:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.539507:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.539510:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.539512:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.539513:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.539516:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.539518:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.539519:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.539520:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.539521:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.539522:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.539526:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.539528:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.539531:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.539532:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.539533:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.539535:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.539535:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.539536:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.539536:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.539541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.539548:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.539551:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.539552:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.539552:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.539554:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.539557:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.539646:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.539648:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.539649:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.539652:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.539654:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.539655:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.539657:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726777] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.539658:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.539659:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.539660:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.539663:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.539665:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.539668:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.539669:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.539669:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.539672:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.539672:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.539673:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.539673:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.539685:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.539688:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.539689:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.539689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:13.0:1619677447.539691:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.539693:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.539694:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.539703:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:12.0:1619677447.539704:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677447.539705:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.539706:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677447.539707:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.539708:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.539709:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:6.0:1619677447.539709:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.539710:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.539711:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:6.0:1619677447.539711:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.539713:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.539715:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726775] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.539716:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.539717:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.539719:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.539727:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.539729:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.539732:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.539733:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.539734:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.539736:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.539736:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.539737:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.539738:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.539750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677447.539755:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.539758:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.539759:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.539761:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.539762:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.539763:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.539776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677447.539823:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.539825:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.539826:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.539828:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.539828:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.539831:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:9.2:1619677447.539904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.539912:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.539915:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.539916:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.539916:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.539918:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.539921:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.539978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f3a7a00] (20)++
00000800:00000200:4.2:1619677447.540018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f3a7a00] (20)++
00000800:00000200:1.0:1619677447.540042:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.540045:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.540045:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:4.2:1619677447.540048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f3a7a00] (21)++
00000400:00000200:1.0:1619677447.540048:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.540050:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.540052:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.540053:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.540055:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.540056:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.540057:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.540060:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.540061:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677447.540062:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677447.540063:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.540064:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.540065:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.540065:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:12.0:1619677447.540066:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.540066:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.540067:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:13.0:1619677447.540068:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.540068:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.540069:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677447.540070:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677447.540070:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.540071:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.540082:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:4.2:1619677447.540083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f3a7a00] (21)++
00000800:00000200:1.0:1619677447.540083:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.540084:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.540084:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:17.0:1619677447.540085:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.540085:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.540087:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.540087:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.540088:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677447.540092:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.540094:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.540096:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.540097:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.540098:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677447.540098:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:22.0:1619677447.540100:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:17.0:1619677447.540100:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.540101:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.540102:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.540104:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.540107:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:17.0:1619677447.540108:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.540113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f3a7a00] (21)++
00000800:00000200:6.0:1619677447.540117:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.540119:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.540120:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.540121:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.540121:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.540122:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.540140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f3a7a00] (21)++
00000800:00000200:22.0:1619677447.540158:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.540161:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.540161:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.540163:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.540164:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.540164:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.540166:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.540167:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.540169:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.540171:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726763] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.540172:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.540173:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.540174:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.540175:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.540176:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.540178:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:22.0:1619677447.540179:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.540181:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.540184:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.540185:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.540185:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.540187:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.540188:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.540189:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.540189:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.540216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f3a7a00] (20)++
00000800:00000200:4.2:1619677447.540245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.540250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:4.2:1619677447.540254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f3a7a00] (21)++
00000800:00000200:8.0:1619677447.540257:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.540260:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.540261:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.540261:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.540263:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.540265:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.540291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f3a7a00] (20)++
00000800:00000200:18.0:1619677447.540299:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.540303:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.540303:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.540306:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.540309:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.540310:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.540312:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726760] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.540314:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.540315:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.540316:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.540326:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.540329:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.540332:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.540333:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.540333:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.540335:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.540336:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.540336:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.540337:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:6.0:1619677447.540355:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.540358:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.540358:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.540359:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.540361:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.540365:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.540388:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.540390:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.540391:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:9.2:1619677447.540392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:1.0:1619677447.540392:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.540394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:1.0:1619677447.540394:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.540397:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.540399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f3a7a00] (21)++
00000800:00000200:24.0:1619677447.540400:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.540402:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.540403:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.540404:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:24.0:1619677447.540405:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.540405:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.540408:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.540408:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.540409:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677447.540410:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.540411:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.540412:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:9.2:1619677447.540414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000400:00000200:8.0:1619677447.540414:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.540415:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.540417:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.540418:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.540420:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.540422:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.540422:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.540423:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.540423:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.540424:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.540425:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677447.540426:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.540429:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.540430:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.540430:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.540432:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:4.2:1619677447.540432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f3a7a00] (21)++
00000800:00000200:8.0:1619677447.540433:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.540433:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.540434:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.540442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:17.0:1619677447.540446:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.540448:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:4.2:1619677447.540448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f3a7a00] (21)++
00000800:00000200:2.0:1619677447.540449:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:1.0:1619677447.540449:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:17.0:1619677447.540450:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.540451:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.540451:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:2.0:1619677447.540452:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.540453:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.540454:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:17.0:1619677447.540455:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.540455:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.540456:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:2.0:1619677447.540456:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.540457:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.540457:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.540459:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:2.0:1619677447.540460:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:1.0:1619677447.540462:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.540464:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.540465:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.540466:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:1.0:1619677447.540467:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.540468:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.540469:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.540469:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.540470:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.540471:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:1.0:1619677447.540472:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.540474:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.540477:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.540478:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.540478:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.540480:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.540480:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677447.540481:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677447.540481:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.540482:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.540484:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.540485:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.540485:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.540487:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.540490:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:17.0:1619677447.540493:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.540495:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.540496:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.540497:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.540498:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.540500:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.540517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.540520:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.540523:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.540526:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.540526:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.540528:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.540529:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.540531:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.540532:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.540533:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.540534:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.540538:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.540539:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.540542:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.540543:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.540543:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.540545:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.540545:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.540546:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.540547:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.540558:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.540561:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.540562:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.540562:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.540564:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.540566:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.540572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677447.540626:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:13.0:1619677447.540629:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.540629:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.540630:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.540631:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.540632:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:13.0:1619677447.540632:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.540633:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.540634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:13.0:1619677447.540634:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.540635:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:12.0:1619677447.540636:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.540637:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.540679:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.540681:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.540682:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.540684:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.540685:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.540687:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.540699:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.540702:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.540703:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.540703:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.540704:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.540705:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677447.540975:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.540978:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.540978:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.540981:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.540982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f3a7a00] (20)++
00000400:00000200:22.0:1619677447.540984:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.540985:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.540987:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.540989:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.540990:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.540991:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.540995:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.540997:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.541000:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.541001:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.541002:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.541004:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.541004:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.541005:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.541005:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.541005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.541013:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.541016:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.541016:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.541017:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.541019:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.541019:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.541022:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.541023:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677447.541025:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:1.0:1619677447.541025:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677447.541026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:18.0:1619677447.541027:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:1.0:1619677447.541027:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.541028:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:18.0:1619677447.541029:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.541029:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:4.2:1619677447.541030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f3a7a00] (21)++
00000400:00000200:1.0:1619677447.541030:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.541031:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:1.0:1619677447.541031:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.541032:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.541032:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.541033:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:24.0:1619677447.541034:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.541035:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.541035:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.541035:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:18.0:1619677447.541036:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.541038:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96726742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.541038:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.541039:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.541040:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.541040:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.541042:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:18.0:1619677447.541042:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.541042:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:18.0:1619677447.541045:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.541047:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.2:1619677447.541047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.541048:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.541049:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677447.541050:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.541050:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677447.541051:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.541051:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.541052:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677447.541053:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.541053:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.541053:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.541054:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:1.0:1619677447.541054:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.541054:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677447.541055:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.541055:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:24.0:1619677447.541056:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.541056:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.541057:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.541062:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.541063:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.541066:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677447.541067:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.541069:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:6.0:1619677447.541069:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.541070:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:17.0:1619677447.541070:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.541070:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.541071:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.541074:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.541074:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.541077:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:17.0:1619677447.541080:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.541081:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.541084:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:4.2:1619677447.541084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.541085:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.541085:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.541086:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.541086:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.541123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:22.0:1619677447.541131:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.541134:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.541134:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.541135:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.541137:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.541143:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.541145:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.541147:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.541148:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.541149:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.541150:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.541150:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.541168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:20.2:1619677447.541172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.541175:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.541177:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.541177:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.541178:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.541179:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.541180:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.541181:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.541183:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:20.0:1619677447.541184:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.541184:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:20.0:1619677447.541186:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:5.0:1619677447.541186:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677447.541187:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.541187:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.541187:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.541188:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:20.0:1619677447.541189:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.541189:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.541190:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.541190:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:20.0:1619677447.541191:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:4.2:1619677447.541194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f3a7a00] (20)++
00000400:00000200:20.0:1619677447.541195:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.541198:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677447.541200:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.541201:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.541202:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.541203:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.541204:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.541204:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.541205:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677447.541217:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.541220:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.541221:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.541221:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.541223:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.541225:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.541241:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.541243:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.541244:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.541244:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.541247:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.541249:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.541553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.541559:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.541561:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.541563:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:24.0:1619677447.541564:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.541564:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.541566:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.541567:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.541569:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.541570:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.541571:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.541572:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.541573:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.541575:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.541576:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.541578:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.541578:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.541579:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.541580:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.541580:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.541581:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.541585:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677447.541585:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.541586:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.541587:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.541587:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.541589:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.541589:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (22)++
00000800:00000200:20.2:1619677447.541589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:24.0:1619677447.541590:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.541591:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.541591:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.541592:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677447.541592:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.541592:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.541593:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.541599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677447.541600:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.541600:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:8.0:1619677447.541601:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.541601:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.541602:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.0:1619677447.541602:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677447.541604:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.541605:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.541606:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.541606:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.541607:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.541608:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677447.541610:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.541610:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96726734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.541611:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.541612:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.541613:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.541614:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:5.0:1619677447.541617:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.541618:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.541619:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.541619:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.541620:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677447.541621:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:1.0:1619677447.541621:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.541622:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.541623:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.541624:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.541625:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677447.541625:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.541625:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677447.541628:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677447.541628:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.541629:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.541630:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677447.541634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.541640:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.541642:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.541643:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.541644:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.541645:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.541647:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677447.541649:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:17.0:1619677447.541650:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.541651:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.541653:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.541655:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.541657:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.541658:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.541659:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.541661:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.541662:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.541665:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.541667:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.541669:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.541670:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.541671:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.541672:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.541673:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.541673:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.541674:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.541686:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.541690:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.541691:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.541692:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.541693:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.541697:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.541698:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.541700:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.541701:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.541702:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.541702:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.541703:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.541717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.541720:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.541722:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.541724:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.541726:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.541727:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.541729:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.541730:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.541732:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.541733:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.541734:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.2:1619677447.541736:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:20.0:1619677447.541742:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.541743:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.541746:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.541747:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.541748:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.541749:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.541750:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.541751:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.541751:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.541763:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.541766:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.541767:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.541768:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.541770:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.541771:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.541783:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.541786:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.541787:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.541788:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.541789:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.541792:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:13.0:1619677447.541836:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.541839:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.541839:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.541841:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.541843:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.541843:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.541857:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.541861:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.541863:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.541864:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.541866:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.541868:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.541869:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.541871:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.541871:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.541873:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.541876:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.541877:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.541880:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.541881:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.541881:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.541883:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.541884:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.541884:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.541885:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.541897:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.541899:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.541900:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.541900:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.541902:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.541905:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.542092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.542132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.542137:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.542140:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.542141:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.542141:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.542143:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.542146:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.542209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.542217:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.542220:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.542221:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.542221:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.542223:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.542226:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.542230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.542237:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.542240:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.542240:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.542240:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.542241:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.542242:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:18.0:1619677447.542255:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.542258:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.542259:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.542261:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.542263:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.542264:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.542266:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.542267:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.542268:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.542269:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.542272:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.542274:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.542276:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.542277:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.542278:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.542279:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.542280:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.542280:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.542281:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677447.542293:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.542300:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.542301:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.542303:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.542303:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.542303:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677447.542305:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677447.542306:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.542306:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:20.2:1619677447.542308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:2.0:1619677447.542308:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.542311:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.542312:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:5.0:1619677447.542314:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:2.0:1619677447.542314:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.542315:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.542317:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:2.0:1619677447.542317:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.542318:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.542319:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.542320:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.542323:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.542323:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.542325:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:2.0:1619677447.542325:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.542327:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.542328:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.542328:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.542329:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.542329:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677447.542330:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.542331:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.542331:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.542332:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.542332:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.542333:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:5.0:1619677447.542334:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.542336:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.542339:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.542340:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.542340:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.542342:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.542343:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.542344:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.542345:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:1.0:1619677447.542345:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677447.542346:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677447.542348:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.542349:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.542349:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.542351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677447.542351:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.542353:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677447.542356:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677447.542357:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677447.542358:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.542359:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.542359:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.542359:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.542360:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.542362:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.542364:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.542365:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.542374:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:4.2:1619677447.542375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.542376:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.542377:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.542378:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.542378:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.542379:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.542414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.542422:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.542425:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.542426:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.542426:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677447.542426:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.542428:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.542429:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.542430:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.542431:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.542432:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:17.0:1619677447.542434:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.542435:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:9.2:1619677447.542435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:17.0:1619677447.542437:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.542438:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.542439:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.542440:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.542442:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:17.0:1619677447.542443:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.542445:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.542445:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.542446:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.542447:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.542447:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677447.542448:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.542448:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.542449:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.542450:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.542452:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.542452:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.542453:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.542454:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.542455:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677447.542466:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677447.542467:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.542468:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:13.0:1619677447.542469:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.542470:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.542470:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.542471:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677447.542472:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.542473:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.542474:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.542484:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.542486:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.542487:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.542488:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.542488:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.542489:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.542568:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.542570:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.542571:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.542572:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.542574:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.542575:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.542785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:4.2:1619677447.542797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.542831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.2:1619677447.542835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:18.0:1619677447.542838:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:18.0:1619677447.542841:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000800:00000200:2.0:1619677447.542841:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (24)++
00000400:00000200:18.0:1619677447.542842:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.542842:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000400:00000200:2.0:1619677447.542843:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.542844:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000400:00000200:2.0:1619677447.542845:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.542848:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.542849:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.542850:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.542886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.542889:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.542891:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.542893:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.542894:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.542897:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.542899:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.542900:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.542901:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.542902:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.542903:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.542907:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.542908:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.542911:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.542912:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.542912:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.542914:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.542915:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.542916:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.542916:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.542927:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:22.0:1619677447.542929:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677447.542930:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.542931:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.542932:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.542932:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:13.0:1619677447.542933:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.542933:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:22.0:1619677447.542935:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.542936:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:22.0:1619677447.542938:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.542939:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.542940:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.542942:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.542943:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.542944:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.542947:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.542949:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.542952:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.542952:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.542953:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.542955:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.542956:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.542957:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.542958:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.542959:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.542969:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.542971:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.542971:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.542972:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677447.543023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:8.0:1619677447.543030:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677447.543030:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:24.0:1619677447.543033:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.543033:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:5.0:1619677447.543033:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.543035:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.543035:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677447.543037:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.543038:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.543040:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.543040:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677447.543041:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.543042:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.543043:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.543043:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.543044:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.543045:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.543045:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.543047:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.543049:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.543049:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.543052:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.543053:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.543054:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.543055:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.543056:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.543056:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.543057:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.543058:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:8.0:1619677447.543059:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.543059:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.543059:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.0:1619677447.543060:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.543061:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.543068:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.543068:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:9.2:1619677447.543069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.543070:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.543071:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.543072:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:17.0:1619677447.543073:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.543075:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.543076:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.543076:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677447.543076:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.543078:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.543079:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.543080:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.543080:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.543081:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:2.0:1619677447.543082:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.543084:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:2.0:1619677447.543086:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677447.543087:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.543089:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.543090:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:1.0:1619677447.543094:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.543097:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.543097:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.543098:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.543099:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.543099:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677447.543292:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.543299:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.543300:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:18.0:1619677447.543303:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.543305:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.543306:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.543308:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726712] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.543309:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.543310:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.543311:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.543314:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.543316:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.543318:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.543319:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.543320:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.543321:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.543322:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.543323:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.543324:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.543324:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.543328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.543336:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.543337:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.543337:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.543337:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.543338:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.543339:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.543339:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.543340:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.543341:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677447.543341:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.543343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.543350:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.543352:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.543353:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.543354:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.543354:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.543354:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677447.543403:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.543406:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.543406:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.543408:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.543410:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.543411:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.543526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:12.0:1619677447.543569:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.543571:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.543572:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.543573:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.543575:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.543576:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.543628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.543667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.543675:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.543677:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.543678:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.543679:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.543681:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.543684:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.543790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.543796:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.543799:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.543800:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.543802:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.543805:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.543807:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.543808:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.543809:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.543810:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.543811:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.543814:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.543816:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.543819:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.543819:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.543820:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.543822:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.543822:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.543823:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.543823:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.543836:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.543839:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.543840:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.543840:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.543842:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.543845:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:2.0:1619677447.543909:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.543912:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.543913:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.543914:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.543916:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.543918:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.543919:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.543920:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.543921:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.543922:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.543925:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.543927:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.543930:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.543931:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.543932:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.543934:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.543934:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.543935:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.543935:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.543947:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.543949:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.543951:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.543952:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.543954:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.543955:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.544007:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.544009:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.544010:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.544011:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.544013:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.544016:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.544083:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.544086:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.544087:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.544089:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.544091:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.544093:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.544094:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.544095:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.544096:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.544097:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.544101:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.544103:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.544106:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.544107:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.544107:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.544109:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.544109:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.544110:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.544110:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.544115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.544122:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.544124:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.544125:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.544125:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.544128:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.544131:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:18.0:1619677447.544139:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.544142:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.544142:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.544145:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.544148:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.544149:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.544151:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.544152:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.544153:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.544154:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.544158:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.544159:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.544162:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.544163:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.544164:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.544165:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.544166:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.544167:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.544167:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677447.544179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:13.0:1619677447.544221:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.544224:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.544224:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.544226:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.544228:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.544229:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.544246:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.544249:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.544250:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.544252:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.544255:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.544257:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.544258:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726702] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.544259:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.544261:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.544262:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.544265:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.544267:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.544270:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.544270:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.544271:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.544273:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.544273:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.544274:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.544274:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677447.544286:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.544289:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.544290:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.544290:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.544292:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.544297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.544301:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.544340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.2:1619677447.544344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.544349:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.544351:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.544351:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:20.0:1619677447.544352:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.544353:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.544354:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.544355:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.544359:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.544359:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.544360:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:5.0:1619677447.544360:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:20.0:1619677447.544361:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.544362:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96726698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.544363:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.544364:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.544365:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677447.544369:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.544371:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.544373:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.544374:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:13.0:1619677447.544375:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677447.544375:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.544376:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.544377:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677447.544378:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.544379:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.544381:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.544383:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677447.544383:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677447.544384:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.544385:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.544385:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.544386:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.544386:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.544386:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.544386:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677447.544386:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:12.0:1619677447.544387:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:5.0:1619677447.544387:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:13.0:1619677447.544388:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.544388:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.544388:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:5.0:1619677447.544388:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.544389:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.544390:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.544390:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.544391:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.544391:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677447.544401:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.544404:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.544405:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.544406:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.544406:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.544407:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:13.0:1619677447.544408:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.544409:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.544410:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.544411:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.544412:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677447.544412:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.544412:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.544414:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.544415:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.544417:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.544420:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.544421:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.544423:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.544424:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.544425:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.544426:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.544430:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.544431:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.544434:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.544435:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.544436:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.544438:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.544438:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.544438:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.544439:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.544450:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.544452:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.544453:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.544455:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.544456:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.544458:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677447.544705:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.544708:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.544709:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.544710:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.544711:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.544713:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:1.0:1619677447.544713:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.544715:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677447.544715:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.544716:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.544717:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.544718:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.544719:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.544719:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.544720:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.544720:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.544722:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.544724:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.544724:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.544725:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.544726:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.2:1619677447.544727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677447.544727:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.544729:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.544731:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.544732:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.544732:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.544733:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.544734:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.544734:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.544735:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.544735:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.544735:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.544736:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.544736:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.544737:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:6.0:1619677447.544738:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.544739:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.544739:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677447.544748:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000800:00000200:17.0:1619677447.544750:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000400:00000200:17.0:1619677447.544751:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.544751:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (24)++
00000800:00000200:1.0:1619677447.544751:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000400:00000200:1.0:1619677447.544752:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.544753:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.544754:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.544756:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.544757:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.544758:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.544770:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.544772:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.544773:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.2:1619677447.544774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677447.544774:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.544774:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.544775:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677447.544779:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.544782:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.544783:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.544784:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.544785:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.544788:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.544841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.544866:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.544869:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.544871:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.544872:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.544875:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.544876:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677447.544877:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.544879:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.544880:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.544881:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677447.544883:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.544885:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.544888:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.544888:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677447.544889:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677447.544891:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.544891:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.544892:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.544893:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.544903:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.544905:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.544906:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.544908:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.544909:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.544911:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677447.544956:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.544959:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.544959:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.544961:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.544963:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.544964:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.544966:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.544967:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.544968:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.544969:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.544972:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.544974:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.544977:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.544977:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.544978:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.544980:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.544980:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.544982:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.544983:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.544983:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.544993:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677447.544994:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.544995:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677447.544996:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677447.544996:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677447.544997:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.544997:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.544997:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.544999:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.545002:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.545149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.545150:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.545153:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.545154:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.545156:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.545156:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:6.0:1619677447.545158:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.545159:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:2.0:1619677447.545159:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:6.0:1619677447.545160:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726688] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.545161:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.545162:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:6.0:1619677447.545162:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.545163:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:6.0:1619677447.545163:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.545163:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.545164:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.545165:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.545167:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.545168:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.545169:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.545169:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.545172:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.545172:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.545173:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:6.0:1619677447.545174:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.545175:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.545175:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:18.0:1619677447.545176:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.545176:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.545177:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:18.0:1619677447.545178:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.545178:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677447.545178:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.545179:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.545179:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:18.0:1619677447.545182:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.545183:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.545186:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.545186:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.545187:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.545188:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.545189:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.545190:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.545190:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677447.545191:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.545193:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:9.2:1619677447.545194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677447.545195:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.545195:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.545197:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.545199:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.545201:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.545204:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.545205:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.545205:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.545207:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.545208:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.545264:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.545267:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.545267:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.545270:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.545272:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.545273:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.545275:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.545276:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.545277:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.545278:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.545281:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.545283:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.545286:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.545287:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.545288:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.545290:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.545290:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.545291:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.545291:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.545305:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.545307:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.545308:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.545310:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.545312:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.545313:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.545387:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.545390:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.545391:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.545393:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.545395:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.545396:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.545397:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.545399:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.545400:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.545401:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.545404:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.545406:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.545407:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.545408:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.545408:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.545410:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.545410:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.545411:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.545412:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.545413:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677447.545415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.545423:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.545424:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.545424:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.545425:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:2.0:1619677447.545425:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.545426:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.545427:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.545427:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.545428:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677447.545428:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.545434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.545441:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.545443:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.545444:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.545444:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.545445:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.545445:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.545526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677447.545532:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.545535:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.545536:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.545539:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.545542:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.545544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:1.0:1619677447.545544:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.545545:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.545548:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.545549:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.545550:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677447.545550:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:5.0:1619677447.545553:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.545555:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:1.0:1619677447.545557:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.545560:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:5.0:1619677447.545562:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.545563:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677447.545564:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.545564:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.545565:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.545566:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000400:00000200:5.0:1619677447.545567:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:1.0:1619677447.545567:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.545567:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677447.545568:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96726681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.545568:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.545570:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:5.0:1619677447.545570:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.545571:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677447.545572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:5.0:1619677447.545572:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:13.0:1619677447.545573:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.545574:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.545576:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.545577:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677447.545577:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.545579:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.545579:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.545581:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.545582:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677447.545583:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:5.0:1619677447.545583:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.545585:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.545585:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.545585:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.545586:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:8.0:1619677447.545586:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:5.0:1619677447.545586:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:24.0:1619677447.545587:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:20.0:1619677447.545587:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.545587:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:8.0:1619677447.545588:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.545589:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:20.0:1619677447.545590:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.545591:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.545594:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.0:1619677447.545594:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677447.545600:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.545603:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.545604:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.545604:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.545606:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.545608:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.545679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.545685:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.545688:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.545689:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.545691:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.545693:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.545695:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.545696:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.545697:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.545698:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.545699:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.545702:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.545704:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.545707:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.545708:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.545708:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.545710:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.545710:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.545712:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.545713:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.545713:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.545719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.545724:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:8.0:1619677447.545726:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677447.545727:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.545727:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.545728:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.545728:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.545729:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.545729:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677447.545730:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.545733:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.545793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677447.545801:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677447.545804:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677447.545804:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000400:00000200:8.0:1619677447.545806:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.545809:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.545810:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:8.0:1619677447.545811:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.545813:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.545814:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.545814:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677447.545817:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.545818:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:8.0:1619677447.545820:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.545821:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:8.0:1619677447.545822:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:8.0:1619677447.545823:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:8.0:1619677447.545823:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677447.545824:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677447.545825:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:4.2:1619677447.545904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.545907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.545915:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.545918:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.545919:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.545919:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.545921:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.545925:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:1.0:1619677447.545948:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.545951:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.545951:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.545953:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.545956:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.545957:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.545958:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.545959:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.545960:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.545961:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.545964:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.545965:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.545969:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.545970:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.545970:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.545972:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.545972:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.545973:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.545974:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.545978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.2:1619677447.545980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:6.0:1619677447.545980:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.545982:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.545983:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677447.545984:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:6.0:1619677447.545985:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.545986:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.545987:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:6.0:1619677447.545987:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.545988:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.545989:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.545991:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.545992:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.545993:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.545994:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.545995:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:6.0:1619677447.545996:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.546000:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.546002:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.546005:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.546005:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.546006:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.546008:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.546008:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.546009:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.546010:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.546011:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.546021:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.546023:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.546024:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.546025:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.546026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.546034:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.546036:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.546037:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.546038:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.546040:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.546043:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.546154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.546161:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.546164:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.546166:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.546168:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.546171:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.546172:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.546174:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.546175:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.546176:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.546178:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.546182:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.546185:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.546188:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.546188:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.546189:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.546191:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.546191:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.546193:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.546194:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.546195:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.546199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.546202:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.546205:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.546205:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677447.546205:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:20.0:1619677447.546206:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.546208:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (24)++
00000800:00000200:6.0:1619677447.546208:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:22.0:1619677447.546209:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000400:00000200:6.0:1619677447.546209:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.546210:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.546211:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.546212:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.546214:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.546215:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.546216:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.546227:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.546230:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.546230:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.546231:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.546232:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.546232:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.546353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.546356:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.546358:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.546361:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.546362:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.546363:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.546364:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.546366:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.546367:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.546368:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.546369:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.546373:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.546375:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.546378:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.546379:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.546379:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.546381:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.546382:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.546383:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.546384:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.546385:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.2:1619677447.546390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.546395:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.546397:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.546398:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.546399:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677447.546432:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.546435:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.546436:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.546438:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.546441:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.546442:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.546444:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.546445:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.546446:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.546447:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.546450:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.546452:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.546455:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.546456:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.546456:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.546458:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.546459:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.546460:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.546461:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.546462:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677447.546472:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677447.546472:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.546474:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:12.0:1619677447.546475:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677447.546475:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677447.546476:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.546476:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677447.546478:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.546479:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.546481:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.546529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:4.2:1619677447.546534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677447.546538:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.546540:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.546541:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.546543:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.546545:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.546547:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.546548:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.546550:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.546551:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.546552:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.546555:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.546557:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.546560:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.546561:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.546562:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.546563:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.546564:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.546564:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.546567:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.546573:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.546576:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.546576:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:17.0:1619677447.546577:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:8.0:1619677447.546578:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.546579:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677447.546580:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:8.0:1619677447.546580:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.546581:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.546581:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:1.0:1619677447.546581:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.546582:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.546583:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:8.0:1619677447.546583:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96726666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.546583:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.546584:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.546584:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.546586:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.546586:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.2:1619677447.546587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:8.0:1619677447.546587:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:17.0:1619677447.546588:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677447.546590:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:8.0:1619677447.546591:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.546592:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.546593:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677447.546596:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.546597:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.546597:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.546598:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.546599:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:8.0:1619677447.546600:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.546600:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.546601:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:5.0:1619677447.546601:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.546602:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:2.0:1619677447.546602:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:5.0:1619677447.546604:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.546605:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:9.2:1619677447.546606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677447.546606:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:18.0:1619677447.546607:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.546608:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:5.0:1619677447.546608:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.546609:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.546610:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:5.0:1619677447.546610:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.546611:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.546611:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.546611:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.546612:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677447.546613:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.546614:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.546615:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96726664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.546617:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.546617:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.546618:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.546619:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.546620:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677447.546622:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.546623:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.546624:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677447.546624:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:2.0:1619677447.546624:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.546625:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.546627:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.546627:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677447.546627:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.546628:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.546629:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677447.546629:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.546630:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.546631:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.546632:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.546633:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.546633:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.546634:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677447.546647:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.546649:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.546650:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.546650:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.546652:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.546655:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.546952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677447.546998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.547003:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.547005:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.547005:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:20.0:1619677447.547006:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.547006:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.547008:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.547009:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.547012:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.547013:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.547014:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.547014:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.547016:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.547018:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.547019:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.547020:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.547021:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.547025:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.547027:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.547030:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.547031:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.547032:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.547033:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.547034:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.547035:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.547035:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.547062:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.547065:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.547066:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.547066:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.547069:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.547069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.547072:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.547111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677447.547165:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.547168:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.547169:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.547171:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.547173:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.547175:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.547176:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.547178:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.547179:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.547180:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.547183:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.547185:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.547187:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.547188:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.547189:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.547190:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.547191:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.547191:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.547191:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.547203:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.547206:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.547207:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.547207:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.547209:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.547211:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.547466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.547469:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.547472:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.547474:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.547475:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.547477:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.547478:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677447.547480:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.547481:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.547482:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.547483:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677447.547486:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.547488:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677447.547491:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.547491:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677447.547492:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.547494:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.547495:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.547495:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.547496:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.547501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.547510:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.547512:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.547513:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.547513:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.547515:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.547518:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677447.547534:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.547537:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.547538:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.547540:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.547541:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.547543:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.547545:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.547546:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.547548:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.547549:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.547552:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.547555:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.547558:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.547559:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.547559:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.547561:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.547561:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.547562:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.547562:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677447.547575:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.547578:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.547579:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.547579:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.547581:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.547583:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677447.547584:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.547587:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.547588:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.547588:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.547590:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.547593:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.547594:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.547597:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.547597:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.547597:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.547598:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.547599:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.547676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.547740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677447.547787:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.547790:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.547791:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.547793:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.547793:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.547796:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.547904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.547911:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.547914:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.547915:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.547917:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.547920:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.547922:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.547923:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.547925:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.547925:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.547927:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.547930:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.547932:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.547935:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.547936:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.547936:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.547936:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:24.0:1619677447.547938:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.547938:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677447.547938:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:24.0:1619677447.547939:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677447.547939:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:24.0:1619677447.547940:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677447.547941:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.547943:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.547945:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.547946:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.547948:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.547948:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.547950:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677447.547951:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:18.0:1619677447.547953:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677447.547954:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677447.547955:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677447.547955:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.547955:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.547957:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.547958:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:1.0:1619677447.547958:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.547959:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.547961:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.547961:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677447.547961:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677447.547962:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.547963:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.547964:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.547974:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:6.0:1619677447.547975:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677447.547976:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:6.0:1619677447.547977:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677447.547977:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:6.0:1619677447.547978:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.547978:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.547979:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.547980:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.547983:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.547998:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.548000:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.548001:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.548001:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.548002:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.548003:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.548010:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.548013:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.548014:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.548016:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.548018:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.548019:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.548021:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.548022:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.548023:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.548024:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.548026:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.548028:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.548029:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.548030:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.548030:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.548032:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.548032:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.548033:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.548033:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.548038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.548044:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.548046:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.548047:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.548048:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.548049:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.548050:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.548107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:20.0:1619677447.548154:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.548157:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.548157:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.548159:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.548160:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.548162:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.548174:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.548178:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.548179:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.548180:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.548180:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.548181:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:6.0:1619677447.548183:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.548186:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.548187:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.548189:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.548191:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.548193:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.548195:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.548196:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.548197:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.548198:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.548202:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.548204:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.548206:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.548207:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.548208:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.548210:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.548210:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.548211:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.548212:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.548247:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.548249:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.548250:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.548251:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.548254:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.548255:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.548322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.548325:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.548328:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.548330:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.548331:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.548333:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.548334:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677447.548335:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.548337:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.548338:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.548338:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677447.548342:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.548343:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677447.548346:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.548347:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677447.548348:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677447.548350:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677447.548350:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.548351:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.548352:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:2.0:1619677447.548369:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.548372:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.548373:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.548374:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.548376:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.548378:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.548379:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.548380:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.548381:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.548382:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.548386:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.548388:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.548389:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.548390:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.548390:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.548392:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.548392:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.548393:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.548394:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.548395:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.548397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.548406:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.548407:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.548407:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:18.0:1619677447.548408:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.548408:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.548408:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.548409:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.548410:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.548410:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677447.548411:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.548417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.548424:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.548427:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.548427:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.548428:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.548428:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.548428:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.548520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.548559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.548566:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.548569:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.548570:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.548570:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.548572:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.548575:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.548693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.548699:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.548701:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.548703:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.548704:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.548706:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.548708:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.548709:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.548710:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.548711:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.548712:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.548715:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.548717:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.548720:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.548721:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.548721:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.548724:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.548724:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.548726:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.548727:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.548727:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.548732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.548735:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.548737:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:13.0:1619677447.548737:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677447.548738:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.548738:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.548740:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.548741:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.548742:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.548744:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.548745:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.548753:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.548755:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.548756:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.548757:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.548757:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.548759:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.548843:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.548846:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.548846:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.548848:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.548850:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.548852:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.548853:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.548855:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.548856:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.548857:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.548860:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.548861:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.548864:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.548865:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.548866:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.548867:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.548868:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.548869:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.548869:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.548880:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.548883:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.548883:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.548885:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.548886:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.548888:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.548899:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.548901:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.548902:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.548903:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.548903:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.548905:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.548910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.548976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677447.549014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677447.549016:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.549018:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.549019:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.549021:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.549022:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.549023:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.549079:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.549082:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.549082:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.549084:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.549085:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.549087:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.549304:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:8.0:1619677447.549307:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:2.0:1619677447.549307:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.549308:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.549309:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.549310:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.549311:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:2.0:1619677447.549311:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.549312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:8.0:1619677447.549313:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.549313:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.549315:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.549316:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.549317:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.549317:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.549318:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.549319:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677447.549320:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.549322:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.549322:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677447.549323:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.549323:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.549324:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.549325:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677447.549325:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677447.549325:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:2.0:1619677447.549328:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.549329:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.549329:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677447.549329:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.549330:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.549331:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.549331:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.549332:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:1.0:1619677447.549332:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.549333:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.549333:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:1.0:1619677447.549333:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.549335:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96726635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.549336:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.549336:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.549337:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:1.0:1619677447.549337:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677447.549338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.549338:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:1.0:1619677447.549338:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.549339:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:1.0:1619677447.549339:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.549340:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.549341:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:1.0:1619677447.549344:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.549345:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677447.549347:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.549348:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.549348:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.549349:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677447.549349:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.549350:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:18.0:1619677447.549351:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.549351:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677447.549351:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.549352:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.549352:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.549353:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.549353:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677447.549354:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.549354:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.549354:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677447.549357:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.549358:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.549359:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.549362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677447.549365:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677447.549367:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677447.549368:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.549369:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.549369:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.549370:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:17.0:1619677447.549371:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.549371:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.549372:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.549373:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:22.0:1619677447.549374:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.549378:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.549380:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.549382:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.549383:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.549383:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.549385:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.549388:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.549399:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.549402:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.549403:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.549403:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.549404:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.549404:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.549518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.0:1619677447.549566:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.549568:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.549569:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.549571:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.549572:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.549574:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677447.549663:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.549666:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.549667:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.549668:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.549671:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.549673:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.549674:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.549675:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.549676:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.549677:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.549681:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.549683:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.549685:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.549686:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.549687:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.549688:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.549689:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.549689:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.549690:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.549702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.549708:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:1.0:1619677447.549708:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.549710:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.549711:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.549712:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.549712:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.0:1619677447.549713:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.549713:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.549715:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:1.0:1619677447.549715:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.549717:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.549719:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.549720:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.549723:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.549724:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.549726:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.549732:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.549734:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.549737:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.549739:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.549739:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.549741:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.549742:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.549744:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.549745:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.549745:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.549755:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.549757:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.549758:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.549759:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.549759:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.549762:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.549763:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.549763:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.549765:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.549768:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.549891:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.549893:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.549894:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.549895:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.549897:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.549898:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.549997:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.550000:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.550001:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.550002:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:1.0:1619677447.550002:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.550004:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.550005:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:2.0:1619677447.550005:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.550007:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677447.550008:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.550008:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.550010:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.550011:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.550011:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.550012:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677447.550012:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.550014:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.550016:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.550017:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000400:00000200:1.0:1619677447.550017:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.550018:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.550019:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:24.0:1619677447.550020:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.550023:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.550023:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:2.0:1619677447.550023:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677447.550023:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.550024:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677447.550024:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:1.0:1619677447.550025:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.550026:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.550026:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677447.550026:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.550027:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.550028:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677447.550028:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.550029:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.550029:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:24.0:1619677447.550030:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:1.0:1619677447.550030:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.550031:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677447.550032:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96726625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.550032:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.550032:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677447.550033:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.550033:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677447.550034:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.550035:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.550038:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.550040:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.550044:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.550044:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.2:1619677447.550044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:13.0:1619677447.550044:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677447.550045:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:13.0:1619677447.550046:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:24.0:1619677447.550047:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.550047:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:13.0:1619677447.550047:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.550049:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000400:00000200:13.0:1619677447.550049:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.550050:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.550050:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.550050:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.550052:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.550061:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.550063:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:8.0:1619677447.550063:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:20.0:1619677447.550064:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.550065:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:20.0:1619677447.550066:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.550066:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.550067:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677447.550067:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.550069:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.550081:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.550084:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.550085:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:12.0:1619677447.550086:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677447.550086:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.550087:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.550088:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:12.0:1619677447.550089:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.550090:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.550092:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.550094:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.550095:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.550111:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.550114:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.550114:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.550115:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.550116:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.550117:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.550166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.550173:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.550175:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.550176:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.550180:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.550183:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.550185:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.550186:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.550188:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.550189:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.550190:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.550195:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.550198:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.550201:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.550202:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.550202:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.550204:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.550205:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.550206:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.550207:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.550215:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.550217:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677447.550218:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:12.0:1619677447.550218:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.550220:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.550221:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:13.0:1619677447.550221:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:17.0:1619677447.550222:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.550222:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:12.0:1619677447.550223:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:17.0:1619677447.550225:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.550228:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.550239:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.550242:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.550243:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.550243:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.550244:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.550244:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:22.0:1619677447.550317:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.550320:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.550321:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.550323:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.550325:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.550327:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.550328:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.550330:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.550331:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.550332:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.550336:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.550338:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.550341:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.550342:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.550342:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.550344:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.550345:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.550345:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.550346:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.550351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.550359:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.550362:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.550363:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.550364:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.550365:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.550368:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.550375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.550423:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.550426:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.550427:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.550428:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.550429:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.550431:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.550494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.550497:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.550500:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.550502:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.550503:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.550505:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.550506:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677447.550508:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.550509:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.550510:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.550511:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677447.550514:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.550516:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.550518:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.550519:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.550520:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.550521:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.550522:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.550523:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.550523:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677447.550533:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.550534:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677447.550536:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:5.0:1619677447.550536:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.550537:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.550537:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677447.550539:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.550539:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.550540:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.550542:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:17.0:1619677447.550543:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.550544:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.550545:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.550547:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.550547:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.550549:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.550552:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.550554:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.550557:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.550557:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.550558:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.550560:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.550560:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.550561:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.550562:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.550566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.550572:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.550575:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.550576:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.550576:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.550578:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.550581:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:1.0:1619677447.550640:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.550642:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.550643:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.550644:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.550646:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.550649:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.550689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:9.2:1619677447.550723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.550728:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.550730:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.550733:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000400:00000200:2.0:1619677447.550733:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.550734:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000400:00000200:18.0:1619677447.550736:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.550736:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.550737:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.550738:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.550739:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96726614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.550740:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.550741:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677447.550741:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.550743:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:2.0:1619677447.550743:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:22.0:1619677447.550744:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.2:1619677447.550746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677447.550746:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.550747:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:6.0:1619677447.550747:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.550748:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:2.0:1619677447.550748:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.550749:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.550750:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.550750:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.550751:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:2.0:1619677447.550751:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.550752:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000400:00000200:6.0:1619677447.550753:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96726613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.550754:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.550755:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:6.0:1619677447.550755:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.550755:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.550755:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.550755:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.550756:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677447.550757:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.550758:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.550758:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.550759:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.550759:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.550760:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.550760:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.550760:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677447.550761:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.550761:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:6.0:1619677447.550762:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.550765:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.550768:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.550768:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.550769:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.550771:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.550771:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677447.550772:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:6.0:1619677447.550773:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:20.0:1619677447.550774:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:6.0:1619677447.550774:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.550774:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677447.550775:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.550776:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.550780:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.550781:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.550781:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.550783:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.550784:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.550785:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:20.0:1619677447.550785:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.550786:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.550788:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.550788:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677447.550788:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677447.550789:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.551072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.551079:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.551082:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:17.0:1619677447.551083:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677447.551083:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.551085:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.551086:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:24.0:1619677447.551088:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.551088:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677447.551090:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.551090:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.551091:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.551092:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.551093:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.551094:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.551094:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:24.0:1619677447.551096:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.551096:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.551097:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.551098:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.551099:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.551104:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.551105:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.551107:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.551108:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.551110:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.551111:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.551111:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677447.551112:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.551112:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.551113:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.551114:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.551114:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677447.551115:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.551115:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.551115:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.551116:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677447.551117:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.551118:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.551119:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677447.551120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:4.2:1619677447.551124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.551129:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:8.0:1619677447.551130:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:1.0:1619677447.551131:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:24.0:1619677447.551132:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.551132:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:4.2:1619677447.551132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.551133:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.551133:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.551134:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:8.0:1619677447.551134:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.551135:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677447.551135:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.551136:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.551137:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:8.0:1619677447.551137:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.551138:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.0:1619677447.551141:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677447.551179:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.551181:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.551182:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.551183:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.551185:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.551188:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.551252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.551258:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.551260:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.551263:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.551263:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.551265:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.551266:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.551268:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.551269:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.551270:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.551271:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.551274:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.551276:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.551278:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.551279:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.551280:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.551282:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.551283:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.551283:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.551284:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677447.551288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.551297:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.2:1619677447.551298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.551300:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.551300:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.551301:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.551301:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.551303:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:18.0:1619677447.551304:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000400:00000200:20.0:1619677447.551305:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.551306:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677447.551307:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:20.0:1619677447.551308:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.551310:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677447.551311:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.551312:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.551313:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.551314:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677447.551318:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.551320:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677447.551322:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.551323:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.551323:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.551325:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.551325:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.551326:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.551327:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677447.551340:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.551342:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.551343:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.551344:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.551346:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.551348:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.551463:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.551465:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.551466:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.551466:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.551468:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.551471:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677447.551481:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.551484:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.551485:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.551486:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.551489:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.551490:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.551492:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.551493:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.551494:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.551495:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.551498:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.551500:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.551502:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.551503:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.551504:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.551505:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.551506:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.551507:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.551507:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677447.551519:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.551522:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:17.0:1619677447.551522:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.551523:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.551523:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.551525:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.551526:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.551527:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.551528:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.551528:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:24.0:1619677447.551532:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.551534:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.551536:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.551537:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.551537:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:24.0:1619677447.551538:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.551539:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677447.551540:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.551540:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.551541:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (21)++
00000400:00000200:24.0:1619677447.551543:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.551543:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.551543:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:24.0:1619677447.551545:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.551548:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.551549:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.551549:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.551551:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.551552:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.551552:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.551553:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677447.551565:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.551568:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.551569:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.551570:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.551571:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.551573:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.551584:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.551587:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.551588:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.551588:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.551589:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.551590:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.0:1619677447.551602:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677447.551605:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677447.551606:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677447.551607:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677447.551610:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.551611:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677447.551613:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.551614:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.551615:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.551616:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677447.551619:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677447.551621:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.551624:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.551624:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677447.551625:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677447.551626:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.551627:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677447.551628:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677447.551629:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:6.0:1619677447.551641:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.551643:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.551644:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.551644:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.551646:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.551649:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.551661:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.551664:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.551665:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.551665:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.551666:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.551667:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.551768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.551770:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.551773:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.551775:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.551776:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.551777:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.551779:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.551780:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.551782:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.551783:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.551784:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.551787:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.551789:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.551791:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.551792:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.551793:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.551795:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.551795:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.551796:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.551797:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.551801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.551806:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.551808:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.551810:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:2.0:1619677447.551811:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.551812:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.551813:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.551813:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.551815:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.551815:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.551816:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.551818:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.551819:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.551820:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.551823:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.551825:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.551825:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.551826:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.551827:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.551828:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.551829:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.551830:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.551832:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.551833:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.551833:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.551834:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.551847:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.551849:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.551850:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.551851:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.551852:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.551854:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.551908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.551949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.551954:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.551957:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.551958:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.551958:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.551960:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.551963:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.552007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.552015:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.552018:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.552018:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.552020:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.552023:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.552025:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.552026:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.552027:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.552028:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.552029:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.552032:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.552034:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.552037:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.552037:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.552038:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.552040:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.552040:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.552041:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.552041:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.552044:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.552048:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677447.552050:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.552051:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.552053:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.552054:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.552055:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.552055:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.552057:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.552057:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.552058:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.552059:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.552060:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.552062:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.552065:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.552065:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.552066:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.552068:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.552070:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.552071:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.552072:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.552074:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.552074:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.552076:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.552077:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.552077:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.552087:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677447.552088:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677447.552089:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677447.552090:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:22.0:1619677447.552091:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.552091:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.552091:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.552092:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677447.552093:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.552095:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.552171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.552174:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.552176:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.552180:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.552180:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.552182:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.552184:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.552185:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.552187:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.552188:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.552189:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.552194:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.552197:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.552200:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.552200:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.552201:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.552203:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.552203:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.552205:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.552206:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.552206:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.552211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677447.552216:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677447.552218:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:6.0:1619677447.552219:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.552220:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677447.552220:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:6.0:1619677447.552220:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.552222:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:6.0:1619677447.552222:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.552222:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.552222:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677447.552223:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.552224:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.552224:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.552224:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677447.552226:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.552229:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.552403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:5.0:1619677447.552409:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:5.0:1619677447.552412:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:5.0:1619677447.552414:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.552415:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:5.0:1619677447.552416:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.552418:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677447.552419:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.552421:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.552422:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.552423:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677447.552426:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.552428:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.552431:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.552432:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677447.552432:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677447.552434:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677447.552435:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:5.0:1619677447.552436:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:5.0:1619677447.552437:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:18.0:1619677447.552724:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.552727:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.552728:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.552730:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.552732:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.552734:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.552735:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.552736:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.552737:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.552738:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.552742:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.552743:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.552746:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.552747:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.552748:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.552750:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.552750:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.552751:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.552754:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677447.552755:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.552757:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.552758:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.552759:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.552761:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.552763:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.552764:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.552764:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:18.0:1619677447.552765:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.552766:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.552766:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:18.0:1619677447.552767:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677447.552767:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.552770:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.552771:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.552771:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.552773:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.552774:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.552776:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.552777:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.552778:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.552779:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.552780:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.552780:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.552781:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.552785:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.552788:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.552789:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.552790:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.552790:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.552791:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.552795:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.552797:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.552798:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.552798:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.552800:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.552802:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.552815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.552818:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.552821:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.552823:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.552824:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.552826:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.552827:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677447.552828:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.552830:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.552831:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.552832:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677447.552835:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.552837:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677447.552839:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.552840:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.552841:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.552842:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.552843:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.552844:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.552844:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677447.552856:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.552858:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.552859:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.552859:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.552861:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.552863:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.552874:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.552877:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.552877:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.552878:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.552878:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.552879:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.552940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677447.552989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677447.552993:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.552996:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.552997:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.552998:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.552999:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.552999:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.553001:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.553002:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.553002:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:22.0:1619677447.553003:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.553005:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.553008:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.553013:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:20.0:1619677447.553016:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.553019:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.553020:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.553021:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.553021:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.553022:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.553052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:13.0:1619677447.553056:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.553058:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.553059:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.553060:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677447.553062:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.553063:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.553082:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677447.553085:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677447.553086:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677447.553088:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.553090:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.553092:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:2.0:1619677447.553093:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.553095:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.553096:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.553097:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:2.0:1619677447.553100:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.553102:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:2.0:1619677447.553105:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.553106:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:2.0:1619677447.553106:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:2.0:1619677447.553108:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:2.0:1619677447.553109:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677447.553109:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677447.553110:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:9.2:1619677447.553114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.553121:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.553124:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.553125:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.553125:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.553127:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.553131:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.553142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677447.553147:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.553150:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.553151:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.553151:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.553152:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.553152:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:18.0:1619677447.553266:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.553269:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.553269:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.553271:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.553274:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.553275:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677447.553276:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.553277:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.553279:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.553279:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677447.553283:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.553284:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677447.553286:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.553287:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677447.553287:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677447.553289:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677447.553290:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.553291:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.553291:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677447.553305:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.553307:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.553308:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.553308:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.553311:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.553313:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.553321:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.553323:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.553324:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.553326:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.553328:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.553329:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.553331:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.553332:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.553333:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.553334:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.553337:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.553339:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.553342:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.553343:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.553343:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.553345:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.553346:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.553346:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.553347:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.553459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.553710:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.553713:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.553714:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.553716:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.553720:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.553721:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.553723:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.553725:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.553726:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.553728:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.553731:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.553734:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.553736:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.553736:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.553737:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.553739:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.553739:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.553741:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.553742:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.553743:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.553744:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.553745:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.553745:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:9.2:1619677447.553746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677447.553748:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.553750:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.553752:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.2:1619677447.553753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:2.0:1619677447.553754:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96726580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.553755:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:2.0:1619677447.553755:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.553756:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.553757:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.553757:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.553758:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.553759:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.553761:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.553762:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:2.0:1619677447.553762:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.553763:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.553763:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.553764:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.553765:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677447.553765:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.553768:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.553769:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.553770:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.553772:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.553772:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.553773:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.553774:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.553775:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.553785:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:12.0:1619677447.553785:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677447.553787:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.553787:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.553788:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.553788:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.553789:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.553791:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.553793:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.553794:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.553801:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.553804:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.553805:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.553806:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.553808:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.553809:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.553824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.553829:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.553832:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.553834:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.553834:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.553837:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.553838:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.553840:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726578] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.553841:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.553842:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.553843:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.553846:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.553848:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.553851:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.553852:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.553852:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.553854:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.553855:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.553856:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.553856:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:13.0:1619677447.553867:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.553869:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.553870:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.553872:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.553873:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.553875:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.553992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.553994:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.553997:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.553999:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.554000:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.554002:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.554003:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677447.554004:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.554006:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.554007:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.554008:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677447.554011:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.554013:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677447.554016:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.554016:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677447.554017:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.554019:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.554020:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.554021:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:4.2:1619677447.554021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.0:1619677447.554022:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.554025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677447.554028:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.554031:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.554032:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677447.554032:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:8.0:1619677447.554033:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.554035:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.554036:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.554036:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.554037:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.554038:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.554038:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.554040:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.554041:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.554041:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:8.0:1619677447.554042:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.554043:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:9.2:1619677447.554045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000400:00000200:8.0:1619677447.554046:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.554048:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.554050:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677447.554051:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.554051:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.554052:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677447.554052:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.554053:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:8.0:1619677447.554054:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.554054:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.554054:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677447.554055:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:8.0:1619677447.554055:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.554055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.554055:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.554056:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000400:00000200:17.0:1619677447.554056:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.554056:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677447.554057:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.554058:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:17.0:1619677447.554059:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.554061:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.554063:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.554064:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.554067:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.554068:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.554069:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.554070:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.554071:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.554073:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.554074:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.554076:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.554077:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.554078:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.554081:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.554082:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.554083:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.554086:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.554086:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.554087:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.554088:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677447.554089:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.554092:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.554093:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.554093:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.554094:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.554095:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:12.0:1619677447.554098:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.554100:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.554101:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.554103:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.554104:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.554106:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.554114:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.554117:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.554118:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.554118:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.554121:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.554123:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.554140:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.554142:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.554143:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.554143:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.554144:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.554145:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.554320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:24.0:1619677447.554353:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.554356:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.554357:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.554359:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.554362:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.554364:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.554365:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726572] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.554366:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.554366:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:24.0:1619677447.554367:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.554368:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:13.0:1619677447.554369:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.554370:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.554372:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677447.554372:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.554374:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:12.0:1619677447.554374:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.554375:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.554377:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.554378:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.554379:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.554381:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.554381:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.554382:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.554382:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677447.554395:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.554398:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.554399:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.554401:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.554402:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.554404:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.554414:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.554417:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.554417:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.554419:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.554420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677447.554422:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.554423:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.554425:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.554426:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.554427:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.554428:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.554432:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.554434:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.554437:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.554438:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.554439:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.554440:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.554441:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.554442:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.554442:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.554446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.554453:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.554455:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.554456:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.554457:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.554459:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.554462:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.554469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.554476:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.554478:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.554479:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.554479:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.554480:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.554481:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.554711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.554720:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.554723:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.554724:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.554724:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.554726:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.554729:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.554732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.554735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.554742:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.554744:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.554745:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.554746:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.554747:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.554747:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.554763:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.554766:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.554767:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.554769:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.554771:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.554773:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.554774:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.554775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677447.554776:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.554776:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.554777:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.554780:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677447.554781:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677447.554782:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.554784:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.554785:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677447.554785:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.554786:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:20.0:1619677447.554788:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:17.0:1619677447.554788:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.554788:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:20.0:1619677447.554789:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.554789:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.554789:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:20.0:1619677447.554790:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.554792:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.554793:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:9.2:1619677447.554793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.554800:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.554802:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.554803:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.554804:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.554806:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.554809:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.0:1619677447.554829:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.554832:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.554833:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.554834:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.554835:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.554838:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:6.0:1619677447.554935:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.554938:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.554938:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.554940:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.554943:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.554944:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.554946:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.554947:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.554948:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.554949:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.554952:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.554954:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.554956:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.554957:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.554958:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.554960:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.554960:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.554961:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.554961:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.554974:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.554976:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.554977:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.554978:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.554980:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.554981:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677447.555007:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.555010:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.555010:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.555012:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.555015:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.555017:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.555018:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.555019:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.555020:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:1.0:1619677447.555020:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.555021:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:2.0:1619677447.555023:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.555024:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:1.0:1619677447.555026:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.555027:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:1.0:1619677447.555029:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:2.0:1619677447.555030:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.555031:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:1.0:1619677447.555032:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677447.555033:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.555033:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:2.0:1619677447.555034:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.555034:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:2.0:1619677447.555035:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.555035:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:2.0:1619677447.555036:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677447.555036:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.555038:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:2.0:1619677447.555039:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677447.555039:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.555039:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677447.555041:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.555042:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.555043:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.555044:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.555046:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.555046:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.555047:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.555048:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677447.555049:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.555051:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.555052:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.555052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677447.555053:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677447.555058:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.555061:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.555061:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.555062:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.555063:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.555065:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.555077:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677447.555079:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677447.555080:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677447.555083:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.555085:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.555087:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.555088:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.555089:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.555090:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.555091:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.555095:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.555097:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.555100:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.555101:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.555102:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.555104:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.555104:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677447.555105:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677447.555106:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677447.555106:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:12.0:1619677447.555116:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677447.555117:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677447.555119:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:12.0:1619677447.555119:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:17.0:1619677447.555120:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677447.555120:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000400:00000200:12.0:1619677447.555121:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.555122:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.555123:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.555125:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.555172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.555211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677447.555213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677447.555220:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.555222:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.555222:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.555223:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.555224:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.555225:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.555229:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.555229:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677447.555230:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677447.555230:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.555231:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.555232:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.555233:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:18.0:1619677447.555234:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:1.0:1619677447.555234:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677447.555236:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.555237:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.555238:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.555240:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.555241:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677447.555241:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.555242:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.555244:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96726557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.555245:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.555247:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.555247:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.555248:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:18.0:1619677447.555249:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:17.0:1619677447.555252:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.555253:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.555254:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:17.0:1619677447.555254:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:18.0:1619677447.555256:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677447.555256:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.555256:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677447.555257:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.555257:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677447.555257:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.555258:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.555259:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677447.555259:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:18.0:1619677447.555260:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677447.555260:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677447.555260:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.555261:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677447.555261:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.555261:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677447.555262:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.555262:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.555264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677447.555264:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677447.555266:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96726556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.555268:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677447.555269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:2.0:1619677447.555269:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.555270:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.555272:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.555273:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:1.0:1619677447.555273:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:18.0:1619677447.555274:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.555275:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.555276:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.555276:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:2.0:1619677447.555277:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.555277:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.555278:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677447.555278:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:18.0:1619677447.555280:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:1.0:1619677447.555281:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.555282:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.555283:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677447.555284:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.555284:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:2.0:1619677447.555286:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.555286:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.555287:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.555288:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677447.555300:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.555303:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.555304:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.555306:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.555308:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.555309:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.555310:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.555310:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677447.555311:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677447.555311:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.555312:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.555313:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.555314:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.555332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.555340:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.555342:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.555343:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.555343:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.555345:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.555347:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677447.555365:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.555369:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.555370:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.555373:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.555376:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.555377:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.555379:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.555381:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.555382:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.555383:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.555387:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.555390:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.555392:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.555393:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.555393:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.555395:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.555395:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.555397:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.555398:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.555399:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.555409:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:5.0:1619677447.555410:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677447.555411:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.555412:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.555413:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.555413:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.555413:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.555415:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.555416:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.555418:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.555565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.555600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.555603:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:9.2:1619677447.555604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.555605:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.555608:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.555608:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.555610:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.555612:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:18.0:1619677447.555612:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:20.0:1619677447.555613:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.555614:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.555615:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.555615:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:20.0:1619677447.555616:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.555616:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.555617:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.555618:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.555620:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.555620:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:9.2:1619677447.555621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.555622:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677447.555623:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:18.0:1619677447.555624:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677447.555624:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677447.555625:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.555626:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.555626:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:17.0:1619677447.555626:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.555627:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000800:00000200:18.0:1619677447.555628:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677447.555628:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.555629:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (24)--
00000800:00000200:20.0:1619677447.555630:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (24)--
00000800:00000200:20.0:1619677447.555630:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:17.0:1619677447.555630:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:18.0:1619677447.555631:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000400:00000200:17.0:1619677447.555631:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.555632:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:18.0:1619677447.555632:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.555632:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:17.0:1619677447.555632:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.555633:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.555634:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.555634:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:17.0:1619677447.555634:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:9.2:1619677447.555635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:17.0:1619677447.555638:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677447.555639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677447.555640:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.555643:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.555643:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:17.0:1619677447.555644:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.555645:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.555645:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:2.0:1619677447.555645:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.555646:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.555647:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677447.555647:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.555647:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.555647:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:17.0:1619677447.555648:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677447.555648:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:2.0:1619677447.555648:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.555649:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677447.555649:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:2.0:1619677447.555651:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.555723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.555731:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.555733:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.555734:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.555736:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.555739:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.555740:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.555741:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.555743:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.555744:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.555745:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.555748:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.555749:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.555752:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.555753:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.555754:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.555755:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.555756:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.555757:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.555757:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.555770:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.555773:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.555773:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.555774:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.555776:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.555779:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:22.0:1619677447.555810:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.555812:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.555813:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.555815:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.555817:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.555819:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.555820:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.555821:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.555822:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.555823:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.555826:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.555828:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.555830:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.555831:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.555832:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.555833:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.555834:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.555834:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.555834:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.555848:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.555851:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.555851:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.555853:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.555854:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.555855:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.555867:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.555870:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.555871:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.555872:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.555872:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.555873:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.555918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:12.0:1619677447.555962:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.555964:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.555965:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.555967:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.555969:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.555970:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.556120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.556358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:20.2:1619677447.556389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.556393:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.556395:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.556397:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.556400:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.556400:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.556402:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677447.556402:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.556403:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.556404:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.556404:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.556405:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.556405:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000400:00000200:20.0:1619677447.556406:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.556407:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.556411:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.556411:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:20.0:1619677447.556412:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.556415:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.556418:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.556419:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.556420:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.556422:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.556423:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.556424:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.556424:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.556436:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:4.2:1619677447.556436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:13.0:1619677447.556439:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677447.556439:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.556441:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.556443:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:12.0:1619677447.556443:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.556444:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677447.556444:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677447.556445:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.556445:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.556446:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:12.0:1619677447.556447:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.556447:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.556449:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.556450:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.556451:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677447.556451:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677447.556452:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.556452:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:24.0:1619677447.556453:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:12.0:1619677447.556454:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.556455:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.556455:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:17.0:1619677447.556455:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:13.0:1619677447.556455:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.556455:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.556455:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677447.556456:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.556456:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:12.0:1619677447.556457:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677447.556458:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.556459:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.556460:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.556460:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.556461:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.556462:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.556464:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:24.0:1619677447.556465:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.556465:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:2.0:1619677447.556466:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:18.0:1619677447.556467:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.556467:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.556468:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.556469:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:18.0:1619677447.556470:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.556471:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.556471:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.556471:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.556472:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.556472:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:18.0:1619677447.556472:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677447.556472:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.556473:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.556474:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.556474:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:17.0:1619677447.556475:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.556476:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:2.0:1619677447.556476:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.556477:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.556478:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677447.556478:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.556479:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.556480:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.556480:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:18.0:1619677447.556482:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:2.0:1619677447.556482:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96726538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.556483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.556484:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:2.0:1619677447.556484:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.556485:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.556486:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.556487:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.556488:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.556489:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.556489:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677447.556490:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.556490:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.556490:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677447.556491:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:17.0:1619677447.556491:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677447.556491:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:2.0:1619677447.556491:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.556491:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.556492:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:17.0:1619677447.556492:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.556493:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677447.556493:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:2.0:1619677447.556493:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.556494:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:17.0:1619677447.556494:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.556494:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677447.556494:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.556495:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677447.556497:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677447.556497:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.556498:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677447.556498:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677447.556498:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677447.556499:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:8.0:1619677447.556500:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677447.556500:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.556500:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677447.556501:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.556501:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.556502:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:1.0:1619677447.556502:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:2.0:1619677447.556503:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:1.0:1619677447.556504:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.556504:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.556506:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.556506:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.556508:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677447.556510:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677447.556512:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677447.556513:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:13.0:1619677447.556513:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:18.0:1619677447.556514:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:13.0:1619677447.556515:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.556516:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.556518:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.556519:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.556521:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.556586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.556597:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677447.556600:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677447.556600:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677447.556603:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.556605:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.556607:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677447.556609:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.556611:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.556612:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.556613:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677447.556618:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.556620:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677447.556624:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.556624:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677447.556625:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677447.556627:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (23)--
00000800:00000200:17.0:1619677447.556627:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677447.556628:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677447.556628:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:4.2:1619677447.556631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.556636:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:6.0:1619677447.556638:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:22.0:1619677447.556639:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:22.0:1619677447.556640:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000400:00000200:6.0:1619677447.556640:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.556641:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.556642:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.556643:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.556646:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.556646:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.556647:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.556831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.556837:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.556840:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.556842:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.556842:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.556845:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.556847:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.556848:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.556849:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.556851:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.556851:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.556855:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.556857:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.556860:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.556861:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.556861:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.556863:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.556864:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.556865:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.556867:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.556867:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.556872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.556875:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.556877:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.556878:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.556879:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677447.556935:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.556938:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.556939:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.556941:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.556944:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.556945:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.556947:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.556948:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.556949:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.556950:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.556953:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.556955:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.556958:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.556959:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.556959:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.556961:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.556962:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.556962:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.556963:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.556974:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.556976:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.556977:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.556979:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.556980:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.556982:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.557024:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.557027:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.557027:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.557029:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.557031:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.557033:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.557034:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.557035:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.557037:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.557038:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.557041:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.557043:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.557046:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.557047:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.557047:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:24.0:1619677447.557049:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:22.0:1619677447.557049:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.557050:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.557050:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.557051:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.557052:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677447.557053:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677447.557054:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677447.557057:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.557058:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677447.557060:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.557061:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.557062:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.557063:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677447.557067:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677447.557068:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.557070:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.557071:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.557072:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.557073:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.557073:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677447.557074:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677447.557076:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677447.557076:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:9.2:1619677447.557078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.557087:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.557088:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.557089:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.557091:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000400:00000200:8.0:1619677447.557091:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677447.557092:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677447.557092:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677447.557093:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.557094:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677447.557094:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:20.2:1619677447.557132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.557135:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.557138:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.557140:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.557141:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.557142:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.557144:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.557145:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.557147:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.557148:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.557149:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.557152:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.557154:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.557157:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.557158:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.557158:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.557160:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.557161:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.557161:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.557162:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.557173:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.557175:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.557176:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.557178:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.557179:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.557181:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.557245:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.557248:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.557249:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.557251:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.557253:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.557255:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:1.0:1619677447.557255:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.557256:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.557257:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.557258:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:1.0:1619677447.557258:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.557259:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.557259:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.557259:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:6.0:1619677447.557262:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.557264:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.557265:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.557266:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.557269:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.557270:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.557271:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.557272:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.557273:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.557274:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.557275:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.557276:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677447.557284:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677447.557286:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677447.557287:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677447.557288:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.557288:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677447.557288:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.557289:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677447.557290:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.557291:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.557293:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.557305:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.557308:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.557309:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.557310:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.557310:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.557312:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.557384:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.557387:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.557387:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.557390:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.557393:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.557394:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.557396:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.557397:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.557398:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.557399:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.557402:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.557404:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.557407:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.557408:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.557409:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.557411:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.557412:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.557412:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.557413:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677447.557424:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.557427:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.557428:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.557429:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.557431:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.557432:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.557462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.557467:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.557470:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.557472:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.557473:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.557475:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.557477:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.557478:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.557479:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.557480:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.557481:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.557485:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.557486:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.557489:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.557490:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.557491:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.557492:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.557493:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.557494:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.557494:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.557506:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.557508:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.557509:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.557509:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.557511:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.557513:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.557522:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.557524:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.557525:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.557526:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.557526:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.557527:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.557844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.557847:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.557849:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.557851:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.557852:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.557853:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.557855:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.557856:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.557857:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.557858:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.557860:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.557863:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.557865:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.557867:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.557868:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.557869:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.557871:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.557871:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.557872:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.557873:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.557877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.557884:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.557887:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.557888:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.557888:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.557890:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.557892:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.557946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.558042:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.558045:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.558046:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.558048:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.558050:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.558052:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.558053:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.558054:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.558055:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.558057:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.558060:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.558062:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.558064:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.558065:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.558065:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.558067:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.558067:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.558068:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.558069:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677447.558077:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.558080:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.558081:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.558082:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.558083:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:2.0:1619677447.558084:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.558085:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:1.0:1619677447.558086:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.558087:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:2.0:1619677447.558087:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.558087:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.558088:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.558090:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.558090:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.558091:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.558094:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.558096:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677447.558096:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.558097:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.558097:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.558098:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.558100:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.558100:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.558102:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.558103:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.558103:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677447.558106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.558114:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.558115:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.558115:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677447.558116:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.558116:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.558116:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.558117:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.558118:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.558118:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677447.558119:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.558130:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.558133:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.558133:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.558135:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.558138:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.558139:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.558140:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.558141:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.558142:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.558143:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.558146:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.558148:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.558150:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.558150:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.558151:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.558152:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.558153:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.558154:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.558154:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.558165:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.558167:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.558168:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.558169:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.558170:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.558171:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.558310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.558316:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.558319:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677447.558322:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.558322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.0:1619677447.558323:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.558324:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.558326:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:5.0:1619677447.558328:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.558330:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:5.0:1619677447.558330:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.558331:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.558332:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.558333:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.558334:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:5.0:1619677447.558335:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.558337:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:24.0:1619677447.558338:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.558340:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.558341:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.558341:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677447.558341:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:24.0:1619677447.558342:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:5.0:1619677447.558343:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.558344:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96726518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.558344:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.558344:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000400:00000200:24.0:1619677447.558345:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.558345:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:24.0:1619677447.558346:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.558347:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:9.2:1619677447.558349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:24.0:1619677447.558352:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.558354:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.558356:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.558358:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.558358:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.558359:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.558359:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:8.0:1619677447.558359:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.558361:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:8.0:1619677447.558361:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.558362:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.558364:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.558364:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677447.558365:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677447.558365:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.2:1619677447.558372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.558375:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677447.558377:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677447.558378:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.558380:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.558380:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.558381:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.558381:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677447.558381:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:20.0:1619677447.558382:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.558383:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.558384:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.558385:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.558386:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:20.0:1619677447.558387:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.558388:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.558389:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.558392:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.558394:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677447.558395:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.558396:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.558397:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.558398:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677447.558398:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.558398:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.558399:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677447.558399:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677447.558400:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677447.558400:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.558400:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677447.558401:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.558403:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.558403:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:9.2:1619677447.558405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677447.558408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677447.558413:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677447.558414:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.558416:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:2.0:1619677447.558416:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.558417:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.558417:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:2.0:1619677447.558417:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.558418:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:2.0:1619677447.558419:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.558422:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.558423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.558430:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.558433:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.558434:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.558434:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.558435:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.558436:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.558516:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.558519:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.558520:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.558522:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.558523:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.558525:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.558526:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.558527:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.558528:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.558529:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.558532:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.558534:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.558536:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.558537:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.558537:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.558538:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.558539:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.558540:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.558540:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.558552:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.558555:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.558555:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.558556:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.558557:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.558558:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.558622:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.558625:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.558626:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.558627:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.558629:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.558631:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.558632:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.558633:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.558634:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.558635:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.558639:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.558640:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.558643:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.558644:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.558644:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.558646:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.558647:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.558648:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.558648:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677447.558661:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.558663:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.558664:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.558665:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.558666:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.558668:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.558679:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.558682:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.558683:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.558683:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.558684:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.558685:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.559046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:13.0:1619677447.559092:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.559094:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.559095:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.559096:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.559098:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.559100:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677447.559103:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.559106:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.559106:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.559108:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.559110:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.559111:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.559113:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.559114:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.559115:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.559116:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.559119:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.559121:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.559122:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.559123:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.559124:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.559125:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.559125:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.559126:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.559127:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.559138:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.559140:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.559140:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.559141:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.559142:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.559143:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.559144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.559183:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.559186:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.559187:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.559189:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.559190:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:18.0:1619677447.559191:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.559192:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:22.0:1619677447.559193:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:18.0:1619677447.559194:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.559194:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.559195:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.559195:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.559196:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.559197:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.559198:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.559201:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:18.0:1619677447.559201:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.559203:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.559205:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.559205:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.559206:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.559208:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.559208:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.559209:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.559210:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.559211:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:9.2:1619677447.559214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.559222:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.559223:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.559223:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.559224:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.559224:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.559224:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.559226:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.559226:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.559227:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677447.559227:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.559270:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.559273:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.559273:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.559276:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.559278:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.559280:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.559281:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.559283:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.559283:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.559285:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:6.0:1619677447.559285:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.559288:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.559288:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:18.0:1619677447.559289:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.559292:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:6.0:1619677447.559292:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.559301:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.559302:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:6.0:1619677447.559302:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.559303:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:6.0:1619677447.559303:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:12.0:1619677447.559304:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677447.559305:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.559305:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:6.0:1619677447.559305:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96726508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.559306:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:6.0:1619677447.559306:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.559307:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677447.559307:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:6.0:1619677447.559307:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677447.559308:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.559308:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677447.559309:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.559312:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:6.0:1619677447.559312:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.559313:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.559314:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.559317:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.559318:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.559319:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.559320:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.559321:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.559322:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.559322:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677447.559440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.559446:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.559448:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.559451:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.559451:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.559453:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.559455:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677447.559456:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.559457:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.559458:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.559459:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:20.2:1619677447.559461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:5.0:1619677447.559463:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.559465:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677447.559468:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.559469:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677447.559469:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677447.559471:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677447.559472:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.559473:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.559473:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677447.559510:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.559511:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.559512:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.559513:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.559513:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677447.559516:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.559516:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.559518:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.559519:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.559520:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:1.0:1619677447.559520:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:17.0:1619677447.559521:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.559522:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000400:00000200:1.0:1619677447.559522:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.559523:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.559523:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.559524:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.559524:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.559525:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.559525:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.559526:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:1.0:1619677447.559529:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.559531:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.559533:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.559533:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.559534:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.559535:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.559536:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.559537:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.559538:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.559538:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.559550:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.559550:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.559550:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:17.0:1619677447.559551:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.559551:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.559552:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.559553:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.559553:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.559554:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.559554:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677447.559558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.0:1619677447.559603:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.559606:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.559606:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.559608:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.559609:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.559612:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:9.2:1619677447.559660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.559667:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.559670:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.559671:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.559671:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.559674:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.559676:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.559759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.559803:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.559806:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.559807:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.559809:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.559809:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.559812:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.559913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677447.559914:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.559916:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.559917:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.559920:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.559921:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.559923:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.559925:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.559927:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.559928:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.559929:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.559929:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.559931:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.559932:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.559934:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.559935:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.559938:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.559938:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.559939:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.559941:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.559941:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.559942:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.559942:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.559943:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.559943:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.559944:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.559944:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.559946:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.559947:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.559948:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.559948:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.559952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:20.2:1619677447.559959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:8.0:1619677447.559961:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.559963:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.559964:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:20.0:1619677447.559965:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:8.0:1619677447.559965:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.559967:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:8.0:1619677447.559967:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.559968:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.559970:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.559970:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.0:1619677447.559973:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.559974:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677447.559975:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.559978:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.559978:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.559980:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.559983:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.559985:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.559986:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.559987:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.559988:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.559990:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.559993:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.559995:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.559997:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.559997:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.559998:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.560000:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.560000:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.560001:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.560003:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.560003:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.560006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.560014:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.560014:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.560015:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:2.0:1619677447.560015:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.560016:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.560016:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.560017:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.560017:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.560018:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.560019:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:4.2:1619677447.560104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.560146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.560152:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.560155:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.560156:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.560156:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.560158:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.560161:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.560193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:13.0:1619677447.560238:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.560240:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.560241:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.560243:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.560244:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.560245:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.560256:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:9.2:1619677447.560257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.560258:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.560259:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.560260:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.560260:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.560262:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.560265:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.560268:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.560269:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.560269:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.560271:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677447.560271:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.560274:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:2.0:1619677447.560274:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.560275:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.560276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:6.0:1619677447.560277:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.560279:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.560281:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.560282:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.560283:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677447.560283:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.560285:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.560286:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:6.0:1619677447.560286:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.560287:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.560287:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.560288:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.560289:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:6.0:1619677447.560289:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.560291:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.560297:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.560298:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.560299:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.560301:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.560301:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.560303:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.560304:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.560304:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.560315:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:20.0:1619677447.560316:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:22.0:1619677447.560317:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.560317:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.560318:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.560318:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.560319:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.560321:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.560322:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.560323:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.560336:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.560339:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.560339:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.560341:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.560344:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.560345:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677447.560348:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.560350:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.560351:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.560353:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.560356:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.560358:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.560359:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.560360:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.560361:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.560362:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.560366:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.560368:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.560371:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.560372:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.560373:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.560374:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.560375:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.560376:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.560376:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677447.560396:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677447.560399:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:1.0:1619677447.560400:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:17.0:1619677447.560402:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.560405:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.560406:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677447.560408:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.560409:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:17.0:1619677447.560409:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.560410:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.560411:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.560412:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.560413:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:17.0:1619677447.560415:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.560416:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.560417:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.560419:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.560420:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.560421:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:17.0:1619677447.560421:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:17.0:1619677447.560421:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:24.0:1619677447.560422:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.560423:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.560423:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:24.0:1619677447.560424:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.560424:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677447.560424:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:24.0:1619677447.560425:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677447.560425:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000400:00000200:24.0:1619677447.560430:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.560432:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.560436:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.560436:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)++
00000800:00000200:24.0:1619677447.560437:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (23)++
00000800:00000200:20.0:1619677447.560437:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (23)++
00000800:00000200:24.0:1619677447.560439:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (25)--
00000800:00000200:24.0:1619677447.560440:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677447.560440:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (24)--
00000800:00000200:24.0:1619677447.560441:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.560441:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:20.0:1619677447.560441:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.560443:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.560444:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (23)++
00000800:00000200:20.0:1619677447.560445:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (24)--
00000800:00000200:5.0:1619677447.560445:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (24)--
00000400:00000200:5.0:1619677447.560446:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.560447:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.560448:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.560448:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.560449:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.560451:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.560452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.560459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677447.560475:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.560478:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.560479:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.560484:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.560486:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.560487:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677447.560489:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96726488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.560490:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.560491:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.560492:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:9.2:1619677447.560494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677447.560496:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.560498:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677447.560502:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.560502:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.560503:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677447.560504:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.560504:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:18.0:1619677447.560505:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.560506:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:18.0:1619677447.560507:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.560507:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:13.0:1619677447.560509:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:2.0:1619677447.560509:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.560510:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:13.0:1619677447.560510:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.560510:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.560511:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:13.0:1619677447.560511:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.560511:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677447.560513:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.560514:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.560686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.560728:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.560731:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.560731:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.560733:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.560735:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.560736:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.560777:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.560780:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.560781:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.560783:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.560785:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.560787:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:1.0:1619677447.560787:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:6.0:1619677447.560788:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.560789:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.560790:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677447.560790:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.560791:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.560791:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.560793:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.560795:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.560797:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.560798:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.560799:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677447.560799:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:6.0:1619677447.560801:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.560801:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:1.0:1619677447.560801:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.560802:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.560803:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.560803:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.560804:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.560804:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677447.560804:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:6.0:1619677447.560805:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:1.0:1619677447.560809:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.560812:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.560815:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.560816:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.560816:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.560818:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.560818:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.560819:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.560819:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:17.0:1619677447.560820:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.560820:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.560821:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.560822:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.560823:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.560827:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.560829:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.560830:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.560831:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.560832:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.560833:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.560834:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:12.0:1619677447.560843:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.560846:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.560847:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.560848:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.560850:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.560851:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.560867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.560870:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.560873:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.560875:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.560876:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.560878:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.560879:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.560881:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.560882:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.560883:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.560884:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.560887:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.560889:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.560892:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.560893:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.560894:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.560896:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.560896:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.560897:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.560898:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.560908:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.560911:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.560912:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.560914:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.560915:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.560917:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.560931:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.560934:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.560935:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.560935:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.560936:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.560937:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.561105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:13.0:1619677447.561114:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.561116:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.561117:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.561118:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.561120:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.561121:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.561191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.561236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.561244:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.561246:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.561248:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.561249:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.561252:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.561254:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.561255:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.561256:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.561257:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.561258:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.561261:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.561262:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.561265:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.561266:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.561267:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.561268:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.561269:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.561270:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.561270:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.561282:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.561285:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.561285:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.561286:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.561288:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.561290:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.561293:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.561300:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.561301:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.561304:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.561307:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.561308:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.561309:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.561311:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.561312:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.561313:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.561316:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.561318:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.561320:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.561320:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.561321:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.561322:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.561323:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.561324:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.561324:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.561329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.561337:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.561339:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.561340:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.561340:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.561342:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.561343:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.561375:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.561378:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.561379:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.561381:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.561384:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.561385:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.561386:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.561387:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.561389:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.561390:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.561393:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.561395:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.561397:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.561398:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.561399:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.561401:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.561401:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.561402:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.561402:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.561416:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.561418:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.561419:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.561421:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.561422:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.561423:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:24.0:1619677447.561589:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677447.561592:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:17.0:1619677447.561592:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677447.561593:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:17.0:1619677447.561595:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:24.0:1619677447.561596:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000800:00000200:1.0:1619677447.561596:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677447.561598:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.561598:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.561600:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677447.561600:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.561602:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.561602:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.561603:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.561604:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.561605:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.561606:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.561607:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.561608:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677447.561608:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.2:1619677447.561609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.561612:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:24.0:1619677447.561613:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000400:00000200:17.0:1619677447.561614:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.561615:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:24.0:1619677447.561617:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677447.561617:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.561618:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.561618:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:24.0:1619677447.561619:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.561619:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.561620:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677447.561620:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.561621:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.561621:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.561622:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.561622:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:17.0:1619677447.561622:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.561623:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000400:00000200:20.0:1619677447.561623:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:24.0:1619677447.561624:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000400:00000200:20.0:1619677447.561624:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96726473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.561624:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.561624:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:20.0:1619677447.561625:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.561626:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.561626:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:20.0:1619677447.561627:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.561627:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:9.2:1619677447.561627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677447.561628:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677447.561631:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.561633:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.561634:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:20.0:1619677447.561635:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.561636:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677447.561636:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.561636:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.561637:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.561638:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677447.561638:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.561638:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:1.0:1619677447.561638:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:20.0:1619677447.561639:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:12.0:1619677447.561639:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677447.561639:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:20.0:1619677447.561640:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:1.0:1619677447.561640:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:12.0:1619677447.561641:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677447.561641:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:1.0:1619677447.561641:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.561642:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677447.561642:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.561644:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.561646:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:9.2:1619677447.561646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:12.0:1619677447.561647:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.561652:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.561653:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.561655:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.561655:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.561656:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.561656:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.561657:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.561657:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.561657:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.561658:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:1.0:1619677447.561658:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677447.561661:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.561758:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.561761:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.561762:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.561763:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.561765:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:2.0:1619677447.561765:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.561767:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:13.0:1619677447.561768:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:2.0:1619677447.561768:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.561769:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.561770:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.561771:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677447.561772:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.561772:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:2.0:1619677447.561772:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677447.561777:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.561779:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.561779:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.561782:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.561783:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677447.561784:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677447.561786:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.561786:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.561787:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.561788:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677447.561809:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.561811:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.561812:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.561814:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.561815:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.561816:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.561816:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.561817:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.561818:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.561818:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.561819:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.561851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.561857:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.561860:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.561862:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.561863:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.561865:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.561867:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.561868:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.561870:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.561871:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.561872:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.561875:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.561877:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.561880:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.561881:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.561882:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.561884:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.561884:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.561885:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.561886:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.561896:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.561899:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.561900:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.561901:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.561902:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.561904:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.561970:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.561972:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.561973:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.561974:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.561976:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.561977:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.561989:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.561991:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.561992:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.561993:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.561993:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.561994:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.562067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.562106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.562112:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.562115:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.562115:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.562116:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.562118:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.562121:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.562135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:12.0:1619677447.562176:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.562179:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.562180:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.562182:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.562184:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.562185:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.562260:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.562263:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.562263:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.562265:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.562268:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.562269:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.562271:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.562272:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677447.562273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:18.0:1619677447.562273:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.562274:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.562276:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:18.0:1619677447.562278:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.562279:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:18.0:1619677447.562280:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.562281:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.562282:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677447.562283:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.562284:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.562284:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.562284:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677447.562285:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:18.0:1619677447.562286:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677447.562287:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.562287:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.562287:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:20.0:1619677447.562288:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.562288:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:20.0:1619677447.562289:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.562290:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.562295:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677447.562300:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677447.562300:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677447.562301:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677447.562301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677447.562302:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:13.0:1619677447.562303:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.562304:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.562305:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.562305:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:13.0:1619677447.562306:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.562307:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.562308:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.562308:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.562308:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:22.0:1619677447.562309:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:20.0:1619677447.562309:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.562309:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:22.0:1619677447.562310:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.562310:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677447.562311:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.562312:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677447.562312:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.562312:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.562312:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:22.0:1619677447.562313:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.562313:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:22.0:1619677447.562314:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.562314:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:8.0:1619677447.562314:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.562317:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.562319:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.562319:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.562321:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96726462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.562322:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:8.0:1619677447.562323:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.562324:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677447.562324:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.562325:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677447.562325:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.562326:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.562326:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:12.0:1619677447.562327:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:22.0:1619677447.562328:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:12.0:1619677447.562328:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.562329:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.562329:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:8.0:1619677447.562329:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.562330:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.562330:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.562330:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.562331:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677447.562333:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.562335:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.562336:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.562336:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.562338:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.562339:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.562339:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.562340:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.562353:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.562356:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.562356:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.562357:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.562359:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.562362:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.562371:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.562374:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.562374:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.562375:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.562376:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.562376:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.562729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.562735:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.562737:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.562740:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.562740:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.562743:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.562744:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.562746:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.562747:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.562748:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.562749:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.562753:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.562755:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.562758:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.562759:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.562759:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.562761:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.562762:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.562763:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.562764:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.562765:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.562770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.562773:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.562775:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.562775:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.562776:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677447.562845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.562890:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.562893:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.562894:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.562895:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.562897:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.562899:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.562910:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.562912:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.562913:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.562914:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.562915:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.562915:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.562952:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.562955:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.562956:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.562958:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.562960:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.562962:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677447.562963:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.562965:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.562966:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.562967:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677447.562970:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.562971:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.562974:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.562975:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677447.562975:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677447.562977:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.562977:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.562979:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.562980:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.562980:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677447.562989:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.562991:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.562992:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.562993:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:13.0:1619677447.562994:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.562995:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.562995:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.562995:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677447.562997:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.562997:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:17.0:1619677447.563054:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.563056:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:9.2:1619677447.563057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:1.0:1619677447.563057:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.563059:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.563062:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.563063:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.563064:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.563065:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:17.0:1619677447.563066:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.563067:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.563068:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677447.563068:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.563069:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.563069:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.563071:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.563072:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.563073:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.563076:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.563076:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:17.0:1619677447.563077:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.563078:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.563079:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.563080:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.563081:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.563081:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.563093:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.563096:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.563097:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.563097:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.563099:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.563101:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.563602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.563605:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.563607:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.563610:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.563610:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677447.563610:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.563612:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.563613:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.563613:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:20.0:1619677447.563614:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677447.563616:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.563617:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.563618:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.563619:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.563619:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.563620:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.563621:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.563623:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.563624:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.563625:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.563626:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.563627:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.563627:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.563629:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.563630:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.563631:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:18.0:1619677447.563631:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.563632:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.563633:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.563633:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.563634:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.563634:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.563635:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.563636:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.563637:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.563639:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.563639:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.563640:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.563640:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.563654:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677447.563655:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.563655:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677447.563656:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:1.0:1619677447.563656:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.563657:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.563657:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.563658:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.563659:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:17.0:1619677447.563659:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.563660:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.563661:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.563662:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (23)++
00000400:00000200:17.0:1619677447.563662:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.563663:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (24)--
00000800:00000200:20.0:1619677447.563663:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.563663:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.563664:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:17.0:1619677447.563664:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.563665:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.563666:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:17.0:1619677447.563667:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.563668:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.563669:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.563670:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.563672:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.563672:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.563674:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.563677:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.563679:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.563680:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.563681:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.563682:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.563683:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.563683:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.563684:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.563697:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:4.2:1619677447.563698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.563700:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.563701:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.563702:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.563704:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.563707:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.563710:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.563711:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.563712:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.563712:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.563713:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.563714:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000400:00000200:22.0:1619677447.563715:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.563715:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.563716:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.563717:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.563718:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.563718:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677447.563746:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.563748:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.563749:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.563751:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.563752:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.563754:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:2.0:1619677447.564034:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.564037:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.564038:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.564040:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.564043:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.564044:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.564046:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.564047:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.564048:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.564049:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.564052:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.564054:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.564055:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.564056:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.564057:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.564058:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.564059:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.564060:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.564060:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.564064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.564071:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.564073:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.564073:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.564075:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.564075:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.564077:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.564128:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.564130:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.564131:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.564133:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.564135:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.564137:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.564138:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.564139:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.564140:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.564141:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677447.564145:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.564146:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.564149:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.564150:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.564151:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.564152:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.564153:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.564154:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.564154:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.564158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.564165:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.564168:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.564170:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.564170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.0:1619677447.564170:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.564172:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.564175:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.2:1619677447.564213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677447.564219:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.564221:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.564222:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.564224:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.564224:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.564227:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.564248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.564251:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.564254:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:8.0:1619677447.564255:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000400:00000200:20.0:1619677447.564256:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.564256:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.564258:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.564259:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000400:00000200:20.0:1619677447.564260:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:24.0:1619677447.564261:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:20.0:1619677447.564262:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.564263:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.564264:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.564265:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.564266:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:8.0:1619677447.564267:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.564269:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:20.0:1619677447.564270:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.564271:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.564273:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:8.0:1619677447.564273:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.564274:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.564275:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677447.564276:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.564276:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.564277:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:8.0:1619677447.564278:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.564279:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.564279:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.564280:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.0:1619677447.564280:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677447.564281:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.0:1619677447.564283:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.564284:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.564285:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.564286:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.564287:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677447.564288:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677447.564289:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:12.0:1619677447.564301:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.564303:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.564304:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.564305:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.564308:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.564309:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.564331:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.564335:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.564336:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.564338:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.564341:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.564341:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:6.0:1619677447.564343:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.564345:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.564345:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.564346:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:6.0:1619677447.564347:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.564348:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.564349:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677447.564349:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.564352:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.564353:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:6.0:1619677447.564354:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.564355:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96726444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.564356:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:2.0:1619677447.564356:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.564357:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.564359:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677447.564359:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:6.0:1619677447.564360:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.564361:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:2.0:1619677447.564362:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.564363:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.564364:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.564364:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677447.564365:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677447.564366:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.564366:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677447.564368:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.564368:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.564369:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.564370:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:12.0:1619677447.564371:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.564372:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.564372:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.564372:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:12.0:1619677447.564374:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.564374:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:13.0:1619677447.564375:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.564375:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.564375:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:22.0:1619677447.564376:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:12.0:1619677447.564376:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.564379:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.564380:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.564380:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677447.564387:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:13.0:1619677447.564387:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.564389:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.564390:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:13.0:1619677447.564391:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.564391:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.564391:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677447.564392:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.564393:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.564394:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.564446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.564456:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677447.564459:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677447.564459:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677447.564462:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.564465:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.564467:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677447.564469:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.564470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:18.0:1619677447.564471:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.564473:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.564474:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.564477:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:18.0:1619677447.564479:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.564480:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:18.0:1619677447.564481:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.564481:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.564484:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.564484:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.564485:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000400:00000200:24.0:1619677447.564486:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.564486:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:24.0:1619677447.564488:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:18.0:1619677447.564488:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.564489:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96726441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.564489:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677447.564489:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677447.564490:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000400:00000200:24.0:1619677447.564491:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.564492:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.564493:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:9.2:1619677447.564494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:24.0:1619677447.564496:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.564499:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.564501:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:24.0:1619677447.564502:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.564503:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.564503:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.564504:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.564504:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.564505:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.564505:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:6.0:1619677447.564505:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.564506:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:8.0:1619677447.564506:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.564507:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.564508:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:8.0:1619677447.564508:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.564509:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677447.564510:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.564511:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.564512:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.564513:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.564517:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.564518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.564521:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.564524:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677447.564525:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:17.0:1619677447.564525:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.564525:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.564527:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677447.564527:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.564528:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.564529:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.564530:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.564554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.564560:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.564562:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.564565:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.564565:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.564567:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.564569:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677447.564570:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.564572:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.564573:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.564574:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677447.564578:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.564579:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.564582:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.564583:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677447.564583:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677447.564585:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677447.564586:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.564587:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.564588:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.564589:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.2:1619677447.564594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.564597:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:1.0:1619677447.564598:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677447.564599:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.564599:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.564600:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677447.564601:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.564601:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.564603:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.564606:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.564608:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.564609:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.564610:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.564611:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.564612:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.564615:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.564617:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.564619:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.564620:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.564620:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.564622:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.564622:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.564624:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.564625:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.564625:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.564636:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.564637:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.564637:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:17.0:1619677447.564638:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.564638:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.564639:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.564640:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.564641:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.564641:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677447.564643:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:9.2:1619677447.564654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.564661:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.564664:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.564665:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.564665:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.564667:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.564671:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.564672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.564679:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.564682:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.564683:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.564683:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.564685:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.564685:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.564722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:12.0:1619677447.564765:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.564768:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.564769:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.564770:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.564772:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.564773:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.564910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677447.564928:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.564931:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.564932:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.564934:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.564937:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.564939:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.564940:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.564942:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.564943:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.564944:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.564947:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.564953:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:4.2:1619677447.564953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677447.564956:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.564957:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.564958:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.0:1619677447.564958:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.564960:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.564960:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677447.564961:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:8.0:1619677447.564961:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:20.0:1619677447.564962:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.564962:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:20.0:1619677447.564963:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.564965:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.564966:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:9.2:1619677447.564966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.564973:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.564976:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.564977:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.564977:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.564979:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.564982:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.565056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677447.565103:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.565106:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.565107:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.565109:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.565109:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.565112:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:13.0:1619677447.565215:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.565218:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.565218:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.565220:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.565222:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.565223:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.565233:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.565236:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.565237:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.565237:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.565238:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.565239:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677447.565319:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.565323:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.565323:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.565326:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.565327:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.565329:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.565330:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.565331:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.565332:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.565333:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677447.565337:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.565339:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.565342:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.565343:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.565344:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.565345:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.565346:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.565346:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.565347:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.565352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.565359:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.565362:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.565363:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.565363:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:20.2:1619677447.565364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:24.0:1619677447.565366:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.565369:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.565370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:8.0:1619677447.565376:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:13.0:1619677447.565377:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677447.565379:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:13.0:1619677447.565379:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677447.565379:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000400:00000200:13.0:1619677447.565380:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.565380:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.565381:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000400:00000200:13.0:1619677447.565382:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.565382:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677447.565384:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.565386:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.565389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.565391:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.565394:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.565396:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.565397:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.565399:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.565401:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677447.565402:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.565404:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.565404:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.565406:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677447.565410:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.565412:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677447.565415:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.565416:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677447.565416:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.565418:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.565419:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.565420:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.565420:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.565424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.565432:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.565435:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.565436:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.565436:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.565438:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.565441:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677447.565455:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.565458:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.565459:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.565461:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.565463:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.565464:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.565465:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.565467:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.565469:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.565470:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:2.0:1619677447.565470:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.565471:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677447.565473:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.565475:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.565476:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.565477:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.565478:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.565478:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:18.0:1619677447.565479:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:2.0:1619677447.565479:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677447.565481:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.565481:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.565482:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.565483:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.565484:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.565485:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.565486:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:18.0:1619677447.565487:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677447.565487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.565487:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.565488:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.565490:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.565490:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.565492:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.565493:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.565493:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.565493:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.565494:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.565495:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:2.0:1619677447.565495:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.565496:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.565497:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.565498:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.565498:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.565498:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.565499:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677447.565499:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.565500:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.565501:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.565502:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.565511:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.565514:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.565515:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.565516:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.565518:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.565520:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.565546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.565592:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.565595:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.565596:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.565596:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.565598:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.565601:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.565617:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.565620:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.565620:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.565622:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.565625:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.565626:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.565627:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.565629:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.565630:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.565631:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.565634:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.565636:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.565638:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.565639:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.565640:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.565641:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.565642:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.565643:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.565643:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.565647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.565655:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.565658:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.565658:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.565659:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.565660:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677447.565661:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.565663:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:17.0:1619677447.565663:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.565664:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.565666:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.565669:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.565670:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.565671:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.565672:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.565673:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.565675:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.565678:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.565680:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.565681:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.565682:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.565683:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.565684:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.565684:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.565685:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.565685:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.565697:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.565699:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.565700:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.565701:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.565702:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.565703:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.565762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677447.565767:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.565769:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.565770:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.565770:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.565772:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.565776:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:13.0:1619677447.565789:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.565791:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.565792:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.565794:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.565795:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.565796:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.565803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677447.565837:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.565840:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.565841:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.565843:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.565845:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.565846:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.565848:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.565849:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.565850:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.565851:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.2:1619677447.565854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:2.0:1619677447.565854:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.565856:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.565858:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.565859:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.565859:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.565860:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:2.0:1619677447.565861:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.565862:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:2.0:1619677447.565862:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.565863:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.565863:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:5.0:1619677447.565864:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677447.565865:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.565867:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.565869:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677447.565870:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.565871:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.565872:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.565874:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:5.0:1619677447.565874:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.565877:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:5.0:1619677447.565877:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000400:00000200:22.0:1619677447.565878:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.565878:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.565880:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.565880:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.565882:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.565883:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.565883:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.565884:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.565885:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.565886:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.565887:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.565887:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:6.0:1619677447.565891:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.565894:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.565895:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.565895:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.565896:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.565897:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.565899:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.565902:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.565903:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.565903:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.565905:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677447.565907:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.565916:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.565918:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.565919:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.565919:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.565920:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.2:1619677447.565921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:17.0:1619677447.565921:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.565963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677447.565971:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.565973:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.565974:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.565976:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.565977:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.565979:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677447.566068:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.566071:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.566072:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.566074:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.566075:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.566077:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.566078:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.566079:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.566080:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.566081:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.566084:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.566086:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.566088:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.566088:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.566089:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.566091:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.566091:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.566092:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.566092:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677447.566104:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.566106:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.566106:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.566107:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.566109:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.566111:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.566124:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.566127:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.566127:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.566128:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.566129:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.566129:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:12.0:1619677447.566136:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.566139:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.566140:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.566142:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.566143:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.566144:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.566153:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.566156:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.566157:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.566158:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.566161:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.566163:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.566164:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.566165:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.566166:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.566167:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.566170:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.566171:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.566175:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.566175:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.566176:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.566178:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.566178:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.566179:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.566180:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.566184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.566191:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.566194:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.566195:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.566196:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.566198:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.566199:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.566201:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:18.0:1619677447.566201:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.566202:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.566204:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.566206:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.566207:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.566209:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.566210:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.566211:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.566212:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.566216:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.566218:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.566220:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.566221:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.566222:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.566223:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.566224:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.566224:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.566225:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.566234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.566241:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.566243:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.566244:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.566245:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.566247:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.566249:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.566251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.566261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.566269:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.566271:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.566272:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.566272:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.566273:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.566274:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:13.0:1619677447.566275:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.566278:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.566279:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.566280:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.566282:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.566282:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.566299:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.566302:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.566303:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.566304:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.566304:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.566305:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.566438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677447.566450:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.566453:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.566453:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.566455:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.566457:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.566458:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.566460:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726411] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.566461:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.566462:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.566463:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.566468:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.566471:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.566473:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.566474:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.566475:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.566476:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.566476:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.566478:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.566479:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.566479:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677447.566488:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.566491:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677447.566491:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.566492:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:12.0:1619677447.566492:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.566493:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.566493:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.566493:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.566494:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677447.566494:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.566495:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:13.0:1619677447.566495:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.566496:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:1.0:1619677447.566496:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.566497:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.566497:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:2.0:1619677447.566518:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.566520:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.566521:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.566523:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.566526:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.566527:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.566529:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.566530:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.566531:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.566532:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.566536:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.566537:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.566541:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.566542:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.566542:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.566544:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.566544:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.566545:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.566545:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677447.566556:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.566559:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.566560:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.566562:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.566563:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.566565:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.566779:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.566782:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.566783:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.566785:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.566787:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.566789:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.566790:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.566792:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.566793:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.566794:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.566797:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.566799:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:4.2:1619677447.566800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.566802:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.566802:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.566803:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.566805:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.566805:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.566806:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.566806:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677447.566818:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.566821:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.566822:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.566822:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.566824:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.566828:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677447.566848:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.566851:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.566852:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.566852:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.566855:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.566867:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.566876:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.566878:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.566879:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.566881:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.566883:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.566884:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:6.0:1619677447.566885:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:1.0:1619677447.566886:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96726404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677447.566887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:1.0:1619677447.566887:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.566888:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:1.0:1619677447.566888:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.566889:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.566889:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.566889:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677447.566892:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.566893:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677447.566893:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.566894:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677447.566895:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677447.566896:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677447.566896:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677447.566897:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677447.566897:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677447.566898:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.566898:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.566898:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.566899:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677447.566899:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:12.0:1619677447.566900:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.566901:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677447.566901:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.566901:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.566901:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677447.566902:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.566902:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677447.566903:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677447.566903:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:5.0:1619677447.566904:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.566906:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.566907:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.566908:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.566911:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.566913:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:5.0:1619677447.566913:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.566915:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.566916:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.566917:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.566918:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677447.566919:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.566919:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.566919:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.566920:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.0:1619677447.566921:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677447.566921:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.566922:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677447.566923:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:5.0:1619677447.566923:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.566924:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:24.0:1619677447.566925:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677447.566925:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:8.0:1619677447.566926:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.566929:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.566931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:8.0:1619677447.566931:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:4.2:1619677447.566931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:8.0:1619677447.566932:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.566933:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677447.566933:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.566935:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:8.0:1619677447.566935:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.566936:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:8.0:1619677447.566936:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.566937:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:13.0:1619677447.566937:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.566938:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.566939:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:13.0:1619677447.566939:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.566940:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:12.0:1619677447.566940:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.566941:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.566942:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.566943:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.566943:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.566944:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.566947:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.566947:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.566948:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:24.0:1619677447.566949:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.566950:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.566950:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:24.0:1619677447.566951:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.566952:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.566952:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:24.0:1619677447.566953:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.566954:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.566955:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.566956:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:24.0:1619677447.566957:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.566957:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.566958:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:9.2:1619677447.566958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.566962:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.566963:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.566963:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.566964:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.566965:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.566965:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677447.566965:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677447.566966:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677447.566968:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.566969:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.566969:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.566972:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.566974:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.566977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:1.0:1619677447.566978:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.566980:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.566981:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.566982:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.566984:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677447.566984:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.566986:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:2.0:1619677447.566987:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.566988:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.566988:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.566989:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.566990:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.567096:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.567099:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.567100:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.567101:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.567104:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.567105:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.567106:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.567107:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.567108:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.567109:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.567112:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.567114:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.567116:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.567117:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.567118:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.567120:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.567120:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.567121:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.567122:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.567132:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.567135:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.567136:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.567137:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.567139:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.567140:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.567151:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.567153:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.567154:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.567155:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.567155:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.567157:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.567239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.567280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:12.0:1619677447.567356:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677447.567359:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677447.567361:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.567363:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677447.567363:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.567365:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677447.567366:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96726398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677447.567367:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677447.567368:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677447.567369:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677447.567372:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.567374:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677447.567376:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.567377:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:12.0:1619677447.567378:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677447.567379:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677447.567380:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677447.567381:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677447.567381:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:17.0:1619677447.567395:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.567398:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.567398:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.567399:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.567401:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.567404:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677447.567443:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.567446:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.567447:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.567449:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.567451:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.567452:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.567454:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.567455:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.567456:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.567457:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.567460:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.567462:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.567464:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.567465:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.567466:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.567468:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.567468:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.567469:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.567470:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.567471:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.567481:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:1.0:1619677447.567481:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677447.567483:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.567483:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.567484:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677447.567484:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.567485:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.567486:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.567487:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.567489:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.567762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.567790:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.567792:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.567793:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.567794:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.567796:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.567800:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.567802:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.567805:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.567805:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.567805:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.567807:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.567809:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.568053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.568061:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.568064:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.568065:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.568065:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.568067:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.568071:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677447.568091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:17.0:1619677447.568137:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.568140:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.568141:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.568141:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.568143:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.568146:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.568155:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.568157:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.568158:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.568158:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.568159:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.568160:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.568280:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.568282:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.568283:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.568285:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.568286:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.568288:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.568289:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.568290:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.568291:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.568292:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.568300:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.568302:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.568304:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.568305:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.568305:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.568307:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.568307:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.568308:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.568308:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.568322:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.568324:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.568324:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.568326:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.568326:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.568327:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.568343:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.568345:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.568346:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.568347:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.568347:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.568347:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.568422:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.568424:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.568425:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.568427:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.568430:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.568431:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.568432:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.568434:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.568435:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.568436:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.568439:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.568440:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.568443:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.568444:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.568445:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.568446:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.568447:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.568448:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.568448:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677447.568460:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.568462:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.568463:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.568465:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.568466:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.568468:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.568470:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.2:1619677447.568476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.568478:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.568480:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.568480:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.568482:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677447.568482:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677447.568483:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.568484:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.568484:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.568485:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:5.0:1619677447.568485:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.568486:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.568487:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677447.568488:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.568490:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.568491:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.568492:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677447.568495:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.568497:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000400:00000200:20.0:1619677447.568498:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.568499:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:20.0:1619677447.568500:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.568502:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.568503:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.568504:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.568505:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.568507:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.568508:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:20.0:1619677447.568509:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.568509:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677447.568511:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.568511:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677447.568511:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.568512:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.568513:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.568513:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.568514:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.568514:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677447.568515:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.568516:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.568516:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.568517:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.568517:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.568518:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.568518:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.568530:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.568533:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.568534:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.568534:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.568536:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.568539:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.568618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.568664:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.568667:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.568668:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.568669:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.568671:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.568679:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.568681:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.568683:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.568683:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.568684:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.568686:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.568686:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.568861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677447.568874:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.568876:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.2:1619677447.568877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677447.568877:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.568880:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.568882:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677447.568882:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.568884:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:5.0:1619677447.568885:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:1.0:1619677447.568885:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.568887:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.568888:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:1.0:1619677447.568888:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.568890:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.568894:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.568895:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.568897:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.568898:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.568899:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677447.568900:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:1.0:1619677447.568900:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:5.0:1619677447.568901:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96726384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.568901:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677447.568902:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.568902:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:5.0:1619677447.568903:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.568903:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677447.568904:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677447.568904:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.568904:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:5.0:1619677447.568908:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.568909:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.568912:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.568913:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.568913:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.568914:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.568915:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.568915:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:20.0:1619677447.568916:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.568916:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.568917:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.568917:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:20.0:1619677447.568918:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.568918:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677447.568919:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677447.568920:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677447.568921:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:17.0:1619677447.568921:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.568922:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.568923:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.568924:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677447.568929:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.568931:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.568932:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.568932:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.568934:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.568936:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.568967:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.568969:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.568971:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.568972:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.568975:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.568976:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677447.568978:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.568979:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.568980:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.568982:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677447.568985:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.568987:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677447.568990:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.568991:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677447.568992:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677447.568993:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677447.568994:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.568995:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.568996:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.568997:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.569008:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.569010:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.569011:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.569011:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.569040:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.569043:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.569043:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.569045:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.569048:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.569049:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.569051:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.569052:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.569053:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.569054:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.569058:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.569059:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.569062:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.569063:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.569064:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.569066:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.569066:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.569067:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.569067:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.569199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.569213:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.569216:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.569217:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.569217:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.569220:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.569227:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.569246:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.569249:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.569250:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.569250:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.569252:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.569254:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.569297:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.569300:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.2:1619677447.569301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677447.569301:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.569303:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.569304:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.569306:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.569307:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.569308:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677447.569309:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.569310:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:1.0:1619677447.569310:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.569311:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677447.569313:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.569313:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.569315:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.569315:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.569317:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:1.0:1619677447.569317:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:5.0:1619677447.569319:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.569320:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.569320:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677447.569321:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.569321:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:5.0:1619677447.569322:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.569322:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.569323:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:5.0:1619677447.569326:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.569327:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.569328:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.569328:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.569329:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:5.0:1619677447.569330:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.569330:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677447.569331:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677447.569331:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.569333:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.569337:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677447.569338:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.0:1619677447.569338:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.569339:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.569340:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.569341:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.569342:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.569343:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.569344:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.569345:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.569346:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.569346:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:17.0:1619677447.569348:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.569348:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677447.569349:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.569349:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.569350:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.569352:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.569354:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.569622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.569666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.569666:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:22.0:1619677447.569668:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677447.569669:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.569670:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:22.0:1619677447.569671:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.569672:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.569672:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:2.0:1619677447.569672:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.569674:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.569675:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.569676:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:22.0:1619677447.569677:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:2.0:1619677447.569678:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.569680:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.569681:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.569682:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.569686:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.569687:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.569688:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.569689:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.569690:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.569691:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677447.569691:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.569692:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.569692:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.569692:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.569692:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.569694:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.569694:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.569695:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.569696:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677447.569708:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.569710:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.569711:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.569712:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.569713:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.569716:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.569837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677447.569843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.569846:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.569849:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.569852:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.569852:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.569855:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.569856:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.569858:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.569861:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.569862:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.569864:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.569871:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.569871:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.569873:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.0:1619677447.569874:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677447.569874:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.569876:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.569877:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.569878:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:17.0:1619677447.569878:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.569879:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:17.0:1619677447.569880:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:20.0:1619677447.569881:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:17.0:1619677447.569881:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96726372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.569882:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:17.0:1619677447.569883:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.569883:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.569884:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.569885:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000400:00000200:17.0:1619677447.569885:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677447.569886:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:17.0:1619677447.569888:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677447.569888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677447.569890:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.2:1619677447.569891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677447.569892:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.569893:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.569894:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677447.569895:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:17.0:1619677447.569896:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.569896:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.569896:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:17.0:1619677447.569897:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677447.569898:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:17.0:1619677447.569898:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:18.0:1619677447.569899:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.569899:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:2.0:1619677447.569899:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:5.0:1619677447.569900:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.569901:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677447.569902:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.569905:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.569909:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.569912:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.569912:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.569913:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.569915:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.569917:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.569930:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.569932:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.569933:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.569934:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.569935:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.569935:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.570001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.570008:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.570011:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.570012:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.570012:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.570014:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.570018:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.570022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.570030:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.570032:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.570034:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.570036:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.570039:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.570041:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.570043:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.570044:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.570045:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.570046:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.570049:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.570051:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.570054:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.570054:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.570055:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.570057:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.570057:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.570058:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.570058:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.570070:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:4.2:1619677447.570073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.570074:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.570075:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.570075:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.570076:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:1.0:1619677447.570077:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.570080:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:1.0:1619677447.570080:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.570081:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.570083:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.570085:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.570088:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.570089:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.570090:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.570092:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.570093:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.570097:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.570100:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.570103:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.570103:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.570104:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.570106:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.570106:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.570107:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.570108:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677447.570120:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.570122:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.570123:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.570125:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.570126:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.570128:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.570500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:9.2:1619677447.570514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.570522:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.570524:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.570526:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.570526:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.570528:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.570531:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.570543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677447.570551:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.570554:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.570555:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.570557:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.570557:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.570560:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.570565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.570568:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.570570:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.570570:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.570571:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.570571:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.570572:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.570572:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.570573:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.570574:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.570574:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.570576:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.570577:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677447.570579:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.570580:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.570581:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.570582:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677447.570585:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.570587:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677447.570589:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.570590:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677447.570591:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677447.570592:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677447.570593:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.570594:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.570594:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.570599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.570607:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.570609:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.570610:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.570610:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.570613:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.570615:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:17.0:1619677447.570740:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.570743:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.570744:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.570746:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.570748:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.570750:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.570751:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.570752:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.570754:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.570755:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.570757:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.570759:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.570762:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.570763:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.570764:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.570765:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.570766:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.570766:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.570767:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.570779:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.570782:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.570783:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.570785:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.570786:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.570788:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.570812:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.570814:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.570814:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.570815:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677447.570817:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.570818:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.570820:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.570821:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.570822:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.570823:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.570825:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.570826:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.570827:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.570829:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677447.570830:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.570831:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.570832:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.570832:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.570833:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.570833:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.570837:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.570837:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.570838:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.570838:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.570839:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.570840:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.570841:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.570842:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.570843:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677447.570843:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (23)++
00000800:00000200:18.0:1619677447.570844:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677447.570844:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (24)--
00000800:00000200:2.0:1619677447.570845:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.570846:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.570846:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.570846:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.570847:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.570848:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.570849:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.570852:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.570853:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.570854:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.570854:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677447.570854:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677447.570856:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677447.570856:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:2.0:1619677447.570857:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:1.0:1619677447.570857:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.570858:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.570858:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:12.0:1619677447.570859:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677447.570859:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.570860:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:1.0:1619677447.570860:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.570862:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677447.570862:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:12.0:1619677447.570863:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.570865:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.570865:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.570868:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677447.570872:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.570875:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.570876:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.570876:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.570877:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.570878:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:13.0:1619677447.570882:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677447.570884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:13.0:1619677447.570885:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677447.570886:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:13.0:1619677447.570886:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.570887:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.570888:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677447.570889:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:13.0:1619677447.570889:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:20.0:1619677447.570891:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.570892:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.570893:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.570895:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.570896:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.570898:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.570899:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.570900:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.570903:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.570904:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.570905:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677447.570907:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.570908:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:20.0:1619677447.570908:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.570908:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.570910:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.570910:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.570911:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.570911:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.570912:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.570912:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:22.0:1619677447.570914:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.570916:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.570917:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.570918:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.570920:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.570920:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.570924:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.570926:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.570928:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.570929:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.570930:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.570931:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.570932:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.570932:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.570933:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.570945:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.570948:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.570949:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.570950:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.570951:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.570953:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677447.571008:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.571010:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.571011:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.571012:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.571014:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.571015:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.571081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:13.0:1619677447.571122:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.571124:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.571125:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.571126:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.571128:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.571129:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.571169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.571171:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.571174:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.571176:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.571177:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.571178:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.571180:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.571181:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.571183:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.571184:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.571185:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.571188:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.571190:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:4.2:1619677447.571193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.0:1619677447.571195:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.571196:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.571197:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.571199:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.571200:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677447.571200:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.571200:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.571201:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:24.0:1619677447.571203:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.571204:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.571206:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.571207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677447.571209:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.571210:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:12.0:1619677447.571211:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:24.0:1619677447.571212:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.571213:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.571213:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:24.0:1619677447.571214:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677447.571214:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.571215:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677447.571215:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.571217:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677447.571218:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.571219:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.571220:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.571224:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.571225:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.571225:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.571227:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.571228:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.571228:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.571229:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.571241:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.571244:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.571245:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.571246:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.571246:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.571248:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.571248:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.571251:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.571251:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.571252:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.571253:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:17.0:1619677447.571255:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.571256:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:17.0:1619677447.571256:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:6.0:1619677447.571256:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.571257:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.571258:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.571259:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.571260:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.571261:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.571262:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:6.0:1619677447.571262:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:17.0:1619677447.571265:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.571267:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.571269:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.571270:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.571271:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.571273:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.571273:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.571274:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.571274:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.571285:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.571288:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.571289:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.571290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:13.0:1619677447.571290:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.571292:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.571299:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.571339:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.571341:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.571342:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.571343:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.571345:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.571346:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.571448:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.571451:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.571451:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.571452:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:13.0:1619677447.571453:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.571454:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.571454:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:13.0:1619677447.571455:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.571456:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.571457:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.571460:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.571461:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.571463:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.571464:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.571465:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.571466:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.571469:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.571471:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.571473:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.571474:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.571474:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.571476:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.571476:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.571478:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.571479:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.571480:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.571487:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.571490:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.571490:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.571492:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:12.0:1619677447.571492:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.571494:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.571495:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:12.0:1619677447.571495:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.571496:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677447.571496:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677447.571497:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.571499:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.571500:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.571502:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.571504:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.571506:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:18.0:1619677447.571506:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:22.0:1619677447.571507:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.571507:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677447.571508:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.571510:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.571510:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:22.0:1619677447.571511:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.571511:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:13.0:1619677447.571511:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.571512:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.571513:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.571514:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:18.0:1619677447.571515:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.571515:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:22.0:1619677447.571516:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.571517:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.571519:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.571519:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.571520:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.571521:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.571522:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:18.0:1619677447.571522:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.571523:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:18.0:1619677447.571523:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.571524:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:18.0:1619677447.571524:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.571525:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.571525:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.2:1619677447.571525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:22.0:1619677447.571526:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:18.0:1619677447.571528:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.571531:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.571533:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.571534:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.571534:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.571536:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:2.0:1619677447.571536:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.571537:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.571538:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677447.571539:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677447.571539:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.571540:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677447.571541:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677447.571541:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:1.0:1619677447.571541:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677447.571542:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:17.0:1619677447.571542:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.571542:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.571543:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96726348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.571544:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.571545:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.571545:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.571547:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.571548:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677447.571548:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:13.0:1619677447.571552:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:2.0:1619677447.571552:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.571554:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:13.0:1619677447.571555:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.571555:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.571557:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677447.571558:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.571558:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.571558:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.571558:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:13.0:1619677447.571560:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.571560:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.571561:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.571561:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.571562:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.571575:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)++
00000800:00000200:5.0:1619677447.571578:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (24)--
00000400:00000200:5.0:1619677447.571579:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.571581:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)++
00000400:00000200:5.0:1619677447.571581:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.571582:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (24)--
00000400:00000200:20.0:1619677447.571583:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.571584:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.571584:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.571586:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.571587:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677447.571600:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.571602:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.571603:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.571605:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.571607:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.571609:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.571610:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.571612:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.571613:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.571614:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.571617:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.571620:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.571622:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677447.571623:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677447.571624:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.571625:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.571626:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:1.0:1619677447.571626:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.571628:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:1.0:1619677447.571628:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.571629:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:1.0:1619677447.571630:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677447.571631:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.571632:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.571633:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.571634:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677447.571635:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.571636:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.571637:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.571637:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.571638:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677447.571639:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:12.0:1619677447.571640:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677447.571641:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.571642:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:12.0:1619677447.571643:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.571643:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.571643:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:12.0:1619677447.571645:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.571645:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:13.0:1619677447.571646:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.571646:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677447.571646:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677447.571647:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:12.0:1619677447.571648:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677447.571648:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.571649:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.571651:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.571651:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.571658:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.571661:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.571662:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:13.0:1619677447.571662:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:6.0:1619677447.571662:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.571664:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.571665:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.571666:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.571666:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.571667:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677447.571667:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.571668:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677447.571727:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.571730:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.571731:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.571733:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.571735:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.571736:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677447.571738:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.571739:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.571740:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.571741:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677447.571744:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.571746:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.571747:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.571748:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.571749:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.571750:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.571750:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.571751:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.571752:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.571756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.571763:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.571765:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.571765:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.571766:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.571767:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.571770:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.571776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.571783:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.571785:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.571786:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.571786:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.571787:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.571787:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:17.0:1619677447.571851:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.571854:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.571855:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.571855:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.571857:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.571858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.571860:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.571952:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.571955:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.571956:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.571957:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.571960:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.571961:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.571962:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.571964:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.571965:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.571966:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.571969:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.571971:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.571974:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.571975:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.571976:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.571977:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.571978:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.571979:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.571979:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677447.572048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677447.572086:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.572089:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.572091:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.572091:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.572092:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.572094:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677447.572094:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.572095:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:18.0:1619677447.572096:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.572096:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.572097:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96726338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.572098:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.572099:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.572099:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677447.572101:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.572101:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.572102:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:13.0:1619677447.572102:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:2.0:1619677447.572102:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:12.0:1619677447.572104:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:18.0:1619677447.572105:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.572105:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.572106:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.572106:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:18.0:1619677447.572107:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.572107:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.572108:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96726337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.572108:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.572109:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.572110:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.572111:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.572112:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:17.0:1619677447.572112:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:18.0:1619677447.572115:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.572116:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.572118:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677447.572118:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.572118:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.572119:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.572119:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.572120:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.572120:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.572121:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.572122:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677447.572123:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.572123:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:22.0:1619677447.572124:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.572124:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.572124:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.572125:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.572125:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:22.0:1619677447.572126:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.572126:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.572126:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:22.0:1619677447.572127:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:18.0:1619677447.572127:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:22.0:1619677447.572129:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96726336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677447.572129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:22.0:1619677447.572130:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.572131:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.572132:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.572135:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.572136:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:22.0:1619677447.572137:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677447.572137:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677447.572138:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.572139:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.572140:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.572140:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.572140:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677447.572141:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.572141:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:5.0:1619677447.572141:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.572142:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.572143:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.572143:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677447.572143:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:5.0:1619677447.572143:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.572144:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.572145:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:20.0:1619677447.572145:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.572146:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.572146:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677447.572146:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:5.0:1619677447.572146:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.572147:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.572147:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.572148:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:9.2:1619677447.572149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.572157:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:6.0:1619677447.572157:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.572159:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.572160:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677447.572160:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:6.0:1619677447.572160:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:18.0:1619677447.572161:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.572161:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.572163:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.572165:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:1.0:1619677447.572336:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.572339:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.572340:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.572342:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.572344:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.572346:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.572347:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.572348:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.572350:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.572350:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.572354:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.572356:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.572359:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.572360:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.572360:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.572362:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.572363:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.572364:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.572364:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.572377:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.572379:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.572380:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.572382:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.572383:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.572385:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.572448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.572487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677447.572676:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.572679:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.572680:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.572680:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.572682:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.572685:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677447.572743:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.572746:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.572747:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.572749:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.572751:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.572752:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.572754:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.572755:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.572756:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.572757:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.572761:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.572763:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.572766:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.572767:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.572767:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.572770:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.572770:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.572771:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.572771:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.572840:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.572843:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.572844:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.572846:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.572848:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.572849:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.572851:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.572852:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.572853:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.572854:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.572858:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.572859:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.572862:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.572863:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.572864:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.572866:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.572866:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.572867:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.572867:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.572872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.572880:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.572882:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.572883:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.572884:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.572886:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.572889:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.572892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.572899:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.572901:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.572902:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.572902:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.572903:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.572904:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.572918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.572924:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.572927:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.572929:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.572929:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.572932:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.572933:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677447.572935:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.572936:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.572937:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.572938:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677447.572946:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.572947:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677447.572950:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.572951:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677447.572951:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677447.572953:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677447.572953:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.572954:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.572955:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.572960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.572968:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.572970:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.572971:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.572972:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.572973:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.572976:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:12.0:1619677447.572985:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.572987:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.572988:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.572990:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.572991:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.572993:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.573047:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677447.573050:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677447.573050:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677447.573052:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.573055:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.573056:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677447.573057:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.573059:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.573060:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.573061:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677447.573064:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.573066:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677447.573068:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.573069:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:1.0:1619677447.573069:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677447.573071:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677447.573071:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677447.573073:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677447.573074:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677447.573074:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:22.0:1619677447.573085:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.573088:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.573088:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.573089:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.573091:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.573093:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.573093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.573129:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677447.573131:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677447.573132:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677447.573132:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677447.573133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.573134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.573141:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677447.573141:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.2:1619677447.573142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:24.0:1619677447.573144:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:18.0:1619677447.573144:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:20.0:1619677447.573145:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:8.0:1619677447.573145:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677447.573145:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.573146:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.573147:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.573148:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:18.0:1619677447.573148:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.573150:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.573151:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:24.0:1619677447.573152:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677447.573152:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:18.0:1619677447.573152:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:24.0:1619677447.573153:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.573154:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.573155:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.573156:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.573156:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:24.0:1619677447.573157:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677447.573158:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.573160:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.573161:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.573162:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.573162:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:24.0:1619677447.573165:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:20.0:1619677447.573167:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677447.573168:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.573169:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.573169:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.573169:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677447.573169:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677447.573171:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.573171:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.573172:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:20.0:1619677447.573172:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.573172:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.0:1619677447.573173:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:1.0:1619677447.573173:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677447.573174:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:20.0:1619677447.573174:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:24.0:1619677447.573175:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677447.573176:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677447.573176:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:17.0:1619677447.573176:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.573178:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000400:00000200:17.0:1619677447.573178:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.573179:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.0:1619677447.573180:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:4.2:1619677447.573180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677447.573181:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:17.0:1619677447.573181:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96726322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.573183:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.573184:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677447.573186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000400:00000200:17.0:1619677447.573186:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.573186:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:8.0:1619677447.573187:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.573187:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.573188:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.573189:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677447.573190:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.573190:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677447.573190:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.573191:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.0:1619677447.573192:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:17.0:1619677447.573193:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.573193:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677447.573194:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.573195:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:1.0:1619677447.573195:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:22.0:1619677447.573196:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:17.0:1619677447.573196:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.573196:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.573196:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677447.573196:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677447.573197:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:6.0:1619677447.573197:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.573197:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.573198:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677447.573199:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.573200:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.573200:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.573200:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.573201:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.573201:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.573202:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.573202:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:6.0:1619677447.573203:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.573204:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.573213:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677447.573215:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677447.573215:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.573216:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.573217:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.573218:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677447.573219:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.573220:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.573220:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.573221:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677447.573221:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677447.573222:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.573235:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.573237:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.573238:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.573238:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.573240:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.573240:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.573328:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.573331:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.573332:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.573334:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.573336:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.573337:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.573339:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.573340:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.573341:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.573342:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.573345:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.573347:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.573349:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.573350:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.573350:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.573352:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.573352:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.573353:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.573354:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.573359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.573366:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.573368:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.573369:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.573370:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.573371:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.573372:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.573504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.573552:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.573555:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.573555:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.573556:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.573558:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.573561:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.573610:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.573613:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.573614:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.573614:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.573616:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.573620:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.573773:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.573776:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.573777:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.573779:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.573781:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.573782:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.573783:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.573785:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.573786:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.573787:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.573790:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.573792:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.573793:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.573794:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.573794:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.573796:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.573796:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.573798:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.573799:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.573799:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.573811:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.573812:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.573812:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.573812:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.573812:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.573813:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.573814:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.573814:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.573815:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.573815:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.573887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:4.2:1619677447.573905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.573930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677447.573936:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.573938:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.573939:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.573941:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.573941:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.573945:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677447.573952:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:9.2:1619677447.573953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.573954:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677447.573955:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.573957:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.573959:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.573960:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677447.573960:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677447.573961:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.573961:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.573961:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:12.0:1619677447.573963:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.573964:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.573965:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.573965:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.573968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:12.0:1619677447.573968:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677447.573968:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.573969:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.573971:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677447.573971:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.573971:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.573972:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.573972:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677447.573973:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:12.0:1619677447.573973:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.573973:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:20.0:1619677447.573975:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.573976:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.573978:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.573980:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.573981:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.573982:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.573983:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.573984:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.573988:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.573990:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.573993:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.573994:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.573994:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.573996:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.573997:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.573998:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.573999:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.574000:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.574005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.574010:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.574013:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.574013:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.574014:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.574038:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677447.574040:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677447.574041:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677447.574043:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.574045:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.574046:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.574048:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.574049:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.574050:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.574052:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.574055:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:18.0:1619677447.574056:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.574057:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.574058:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.574059:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.574061:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.574062:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.574063:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:2.0:1619677447.574063:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:18.0:1619677447.574064:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:2.0:1619677447.574064:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.574065:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.574066:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.574067:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677447.574067:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:18.0:1619677447.574068:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677447.574069:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677447.574070:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:18.0:1619677447.574071:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.574071:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.574073:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677447.574076:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.574077:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677447.574077:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:13.0:1619677447.574079:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.574079:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677447.574079:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.574080:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.574080:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677447.574081:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.574082:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.574084:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.574085:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.574087:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.574243:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.574246:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.574247:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.574248:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.574250:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.574252:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.574253:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.574254:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.574255:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.574256:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.574259:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.574261:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.574263:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.574264:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.574265:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.574267:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.574267:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.574268:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.574268:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.574281:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.574283:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.574284:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.574284:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.574286:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.574288:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.574305:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.574308:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.574308:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.574310:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.574313:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.574314:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.574315:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726309] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.574317:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.574318:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.574319:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.574322:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.574325:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:4.2:1619677447.574325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.574328:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.574329:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.574329:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.574331:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.574332:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.574332:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.574333:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.574370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.574375:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.574377:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.574378:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.574379:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.574381:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.574384:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.574539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.574546:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.574548:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.574550:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.574551:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.574553:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.574555:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.574556:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.574557:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.574558:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.574559:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.574563:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.574565:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.574568:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.574569:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.574569:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.574572:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.574572:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.574573:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.574574:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.574677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.574685:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.574688:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.574688:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.574689:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.574691:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.574692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.574695:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.574697:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.574699:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.574700:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.574700:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.574701:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.574702:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.574809:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.574811:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.574812:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.574814:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.574816:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.574818:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.574819:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.574821:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.574822:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.574823:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677447.574826:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677447.574827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677447.574828:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.574830:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.574831:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.574832:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.574834:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.574834:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.574835:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.574835:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677447.574850:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.574852:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.574853:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.574855:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.574857:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.574859:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.574860:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.574861:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.574863:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.574863:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.574867:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.574869:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.574871:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.574871:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677447.574872:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.574873:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:12.0:1619677447.574874:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.574874:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.574875:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.574875:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.574876:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.574876:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677447.574876:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.574878:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.574879:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.574888:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.574891:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.574892:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.574892:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.574894:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.574896:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.574913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.574932:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.574935:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.574936:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.574937:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.574939:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.574939:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.2:1619677447.574956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:6.0:1619677447.574956:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.574959:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677447.574959:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.574960:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.574960:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.574961:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:6.0:1619677447.574963:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.574964:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.574965:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677447.574966:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:20.0:1619677447.574967:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.574969:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.574970:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.574971:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.574972:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.574973:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.574975:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:20.0:1619677447.574977:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.574978:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.574978:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.574979:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.574979:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.574980:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.574981:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.574982:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.574983:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.574983:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.574985:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.574986:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.574987:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.574987:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.574997:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.575000:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.575001:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.575003:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.575004:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.575005:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.575015:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.575018:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.575019:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.575020:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.575020:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.575022:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.575217:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.575220:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.575220:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.575224:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.575227:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.575229:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.575230:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.575232:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.575232:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.575234:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.575237:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.575239:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.575242:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.575243:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.575243:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.575245:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.575245:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.575246:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.575247:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677447.575260:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.575262:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.575263:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.575263:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.575266:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.575268:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.575299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:5.0:1619677447.575348:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.575351:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.575351:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.575353:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677447.575354:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.575357:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.575383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.575386:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.575389:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.575391:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.575391:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.575394:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.575396:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.575397:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.575398:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.575399:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.575400:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.575404:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.575405:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.575408:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.575409:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.575409:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.575412:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.575412:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.575413:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.575414:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.575419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.575426:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.575429:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.575430:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.575430:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.575432:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.575435:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.575618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677447.575623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.575629:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.575632:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.575634:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.575634:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.575637:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.575639:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677447.575640:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.575642:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.575644:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.575645:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.575650:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.575652:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.575655:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.575656:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.575656:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.575658:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.575659:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.575660:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.575660:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:12.0:1619677447.575668:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.575670:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.575671:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.575672:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:12.0:1619677447.575673:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.575674:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:13.0:1619677447.575675:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.575675:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:17.0:1619677447.575676:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.575676:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.575678:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677447.575680:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:24.0:1619677447.575887:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.575890:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.575891:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.575893:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.575896:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.575898:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677447.575899:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.575900:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.575901:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.575902:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677447.575906:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.575907:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.2:1619677447.575908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:24.0:1619677447.575910:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.575911:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:20.0:1619677447.575911:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:24.0:1619677447.575912:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677447.575913:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.575913:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677447.575913:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:24.0:1619677447.575914:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.575915:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:20.0:1619677447.575915:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.575917:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.575918:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.575919:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.575920:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.575921:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.575922:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.575923:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:12.0:1619677447.575925:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:20.0:1619677447.575926:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.575927:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:20.0:1619677447.575928:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:12.0:1619677447.575928:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.575930:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.575931:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.575931:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.575932:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.575932:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.575932:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677447.575934:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.575935:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.575937:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.575938:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.575938:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:9.2:1619677447.575941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677447.575943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677447.575945:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.575948:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.575948:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:18.0:1619677447.575949:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:12.0:1619677447.575949:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:5.0:1619677447.575949:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677447.575950:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.575951:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677447.575952:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:5.0:1619677447.575952:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:18.0:1619677447.575953:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.575953:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.575954:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.575955:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677447.575956:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.575958:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.575969:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.575971:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.575972:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.575974:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.575976:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.575978:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.575979:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.575980:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.575981:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.575982:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:4.2:1619677447.575985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:1.0:1619677447.575985:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.575987:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.575990:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.575991:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:8.0:1619677447.575992:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.575992:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.575994:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:8.0:1619677447.575995:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:1.0:1619677447.575995:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.575995:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677447.575996:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677447.575996:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677447.575997:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.576000:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.576002:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.576003:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.576005:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.576006:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.576007:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.576009:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:8.0:1619677447.576010:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.576012:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.576012:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.576013:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.0:1619677447.576015:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677447.576015:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.576016:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.576017:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677447.576017:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.576018:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.576019:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.576019:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677447.576019:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:8.0:1619677447.576020:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677447.576020:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96726289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.576021:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.576022:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.576024:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677447.576024:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.576027:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.576028:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677447.576028:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.576030:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.576030:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.576032:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:6.0:1619677447.576033:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.576033:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.576034:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)++
00000800:00000200:17.0:1619677447.576035:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000400:00000200:6.0:1619677447.576035:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:2.0:1619677447.576035:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (24)++
00000400:00000200:17.0:1619677447.576036:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.576036:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96726288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.576036:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (25)--
00000400:00000200:6.0:1619677447.576037:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.576037:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000400:00000200:6.0:1619677447.576038:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.576038:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:17.0:1619677447.576039:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.576039:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:6.0:1619677447.576040:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.576040:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.576040:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677447.576042:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:6.0:1619677447.576044:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.576045:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.576048:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.576049:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.576049:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:1.0:1619677447.576050:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677447.576051:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:6.0:1619677447.576051:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.576052:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.576052:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.576053:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677447.576053:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.576053:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.576054:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.576054:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677447.576054:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677447.576055:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:1.0:1619677447.576055:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.576056:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.576057:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (23)++
00000800:00000200:17.0:1619677447.576058:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (24)--
00000800:00000200:17.0:1619677447.576058:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.576059:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.576059:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.576064:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.576067:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.576068:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.576069:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.576071:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.576072:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.576218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.576265:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.576268:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.576268:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.576269:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.576271:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.576274:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.576316:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.576318:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.576319:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.576321:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.576322:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.576323:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.576428:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.576430:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.576431:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.576432:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.576434:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.576435:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677447.576456:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.576459:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.576460:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.576462:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.576464:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.576466:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.576467:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726284] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.576469:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.576470:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.576471:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.576475:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.576477:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.576478:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.576479:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.576480:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.576481:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.576481:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.576483:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.576484:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.576484:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.576497:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.576498:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.576498:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.576498:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.576499:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.576499:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.576500:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.576500:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.576501:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677447.576502:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677447.576502:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677447.576504:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.576505:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.576507:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.576509:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.576511:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.576512:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.576513:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.576514:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.576515:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.576518:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.576520:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.576522:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.576523:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.576523:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.576525:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.576526:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.576526:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.576527:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.576531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.576538:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.576540:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.576541:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.576541:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.576543:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.576544:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.576843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677447.576882:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.576885:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.576886:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677447.576888:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:1.0:1619677447.576888:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.576889:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.576891:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:1.0:1619677447.576891:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:12.0:1619677447.576892:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.576892:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677447.576894:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.576895:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:1.0:1619677447.576895:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.576896:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:1.0:1619677447.576896:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.576897:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:18.0:1619677447.576899:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:1.0:1619677447.576900:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.576902:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:1.0:1619677447.576902:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677447.576903:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.576905:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.576905:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.576906:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.576906:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.576907:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:18.0:1619677447.576908:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.576909:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.576909:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.576909:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677447.576910:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.576910:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.576910:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:18.0:1619677447.576911:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.576912:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.576916:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.576918:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.576920:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.576921:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.576922:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.576924:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.576924:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.576925:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.576925:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.576939:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.576941:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.576942:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.576944:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.576945:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.576946:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:6.0:1619677447.577005:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.577007:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.577008:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.577010:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.577011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:6.0:1619677447.577012:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.577014:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.577015:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.577016:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.577017:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.577018:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:6.0:1619677447.577019:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.577021:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.577022:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677447.577022:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.577024:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.577025:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.577027:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.577027:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.577027:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.577028:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:24.0:1619677447.577029:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:6.0:1619677447.577029:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:24.0:1619677447.577030:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.577030:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:24.0:1619677447.577031:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.577031:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:24.0:1619677447.577032:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.577032:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000400:00000200:24.0:1619677447.577033:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:6.0:1619677447.577033:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:24.0:1619677447.577038:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.577040:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.577043:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.577043:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:24.0:1619677447.577044:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.577044:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.577045:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677447.577045:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.577045:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677447.577046:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:22.0:1619677447.577046:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677447.577046:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677447.577047:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.577047:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677447.577047:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.577048:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.577048:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.577049:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.577056:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.577058:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.577059:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.577060:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.577061:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.577062:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.577064:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.577069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.577072:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.577074:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.577077:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.577078:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.577079:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.577081:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677447.577082:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.577083:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.577084:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.577085:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677447.577088:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.577090:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677447.577092:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.577093:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677447.577093:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.577095:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.577096:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.577097:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.577098:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.577101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.577108:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.577111:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.577112:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.577112:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.577114:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.577116:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.577169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.577194:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.577197:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.577198:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.577199:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.577202:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.577204:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.577205:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.577207:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.577208:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.577209:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.2:1619677447.577211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:1.0:1619677447.577214:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.577216:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:5.0:1619677447.577218:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:1.0:1619677447.577219:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677447.577221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:5.0:1619677447.577221:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:1.0:1619677447.577221:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.577221:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.577222:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:1.0:1619677447.577223:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:5.0:1619677447.577224:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:1.0:1619677447.577224:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.577224:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677447.577225:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:1.0:1619677447.577225:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677447.577226:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677447.577227:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.577227:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.577229:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.577229:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.577230:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96726273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.577232:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.577232:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.577233:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.577234:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677447.577235:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96726272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.577235:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:13.0:1619677447.577236:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000400:00000200:8.0:1619677447.577237:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.577238:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677447.577239:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.577239:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.577240:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:5.0:1619677447.577240:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000400:00000200:13.0:1619677447.577242:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.577242:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677447.577243:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677447.577244:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.577244:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.2:1619677447.577245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:12.0:1619677447.577245:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.577245:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.577247:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:5.0:1619677447.577247:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.577247:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.577248:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:12.0:1619677447.577249:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.577250:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.0:1619677447.577250:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.577250:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:8.0:1619677447.577251:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.0:1619677447.577251:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (24)++
00000800:00000200:5.0:1619677447.577251:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:13.0:1619677447.577252:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677447.577252:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:5.0:1619677447.577252:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.577253:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:13.0:1619677447.577254:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:8.0:1619677447.577254:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677447.577255:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.0:1619677447.577255:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:12.0:1619677447.577256:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:8.0:1619677447.577256:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.577257:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.577257:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.2:1619677447.577266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:13.0:1619677447.577266:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.577268:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.577269:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.577271:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.577272:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:12.0:1619677447.577272:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.577273:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677447.577274:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.577275:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.577276:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.577278:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.577282:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.577293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677447.577300:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.577302:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.577303:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.577305:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.577306:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.577309:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.577319:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.577322:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.577322:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.577323:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.577324:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.577324:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.577583:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.577586:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.577586:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.577588:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.577591:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.577592:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.577594:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.577595:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.577596:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.577597:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.577601:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.577603:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.577605:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.577606:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.577606:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.577608:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.577608:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.577609:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.577609:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.577621:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.577624:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.577625:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.577626:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.577627:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.577628:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677447.577708:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.577711:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.577712:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.577712:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.577714:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.577717:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.577727:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.577729:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.577730:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.577732:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.577734:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.577736:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.577737:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.577738:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.577739:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.577740:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.577743:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.577744:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.577747:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.577747:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.577748:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.577750:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.577750:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.577751:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.577751:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677447.577760:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.577763:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:22.0:1619677447.577764:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677447.577764:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.577765:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.577767:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.577768:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.577768:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.577768:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:18.0:1619677447.577769:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:22.0:1619677447.577771:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.577771:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.577773:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:18.0:1619677447.577773:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677447.577773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:18.0:1619677447.577774:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.577775:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.577778:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.577780:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.577782:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.577782:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.577783:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.577785:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.577785:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.577786:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.577787:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.577788:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677447.577791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.577799:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.577800:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.577800:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.577801:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:2.0:1619677447.577801:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.577802:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.577803:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.577803:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.577804:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677447.577805:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.577816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:9.2:1619677447.577817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.577824:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.577827:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.577829:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.577829:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.577830:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.577830:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.577840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677447.577883:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.577886:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:20.0:1619677447.577887:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677447.577887:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.0:1619677447.577889:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:2.0:1619677447.577889:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.577890:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.577891:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.577892:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.577893:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.577893:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:2.0:1619677447.577894:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.577895:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.577896:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:2.0:1619677447.577896:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.577897:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.577900:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.577902:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.577904:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.577905:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.577905:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.577907:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.577907:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.577908:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.577909:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.577913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.577920:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.577923:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.577924:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.577924:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.577926:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.577927:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.578194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.578236:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.578240:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.578240:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.578243:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.578245:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.578247:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.578248:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.578250:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.578250:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.578252:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.578255:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.578256:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.578259:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.578260:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.578261:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.578262:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.578263:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.578264:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.578265:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.578266:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.578269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.578276:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677447.578276:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.578278:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:24.0:1619677447.578279:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:17.0:1619677447.578279:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677447.578280:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.578280:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677447.578280:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.578282:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.578286:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.578303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.578311:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.578313:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.578314:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677447.578315:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:8.0:1619677447.578316:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.578317:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.578318:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:8.0:1619677447.578318:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.578321:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677447.578321:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.578322:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.578323:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.578324:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.578324:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.578326:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677447.578326:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.578328:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.578330:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.578330:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.578331:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.578332:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.0:1619677447.578333:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:6.0:1619677447.578335:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.578336:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.578337:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.578338:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677447.578338:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.0:1619677447.578339:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.578340:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.578340:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.578341:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.578341:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.578342:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.578343:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.578345:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.578345:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.578346:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.578346:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677447.578353:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.578356:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.578357:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.578357:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.578359:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.578362:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.578371:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.578374:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.578375:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.578375:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.578376:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.578377:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:24.0:1619677447.578458:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.578461:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.578462:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.578464:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.578466:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.578468:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.578469:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.578471:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.578472:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.578473:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.578476:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.578478:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.578481:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.578481:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.578482:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.578484:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.578484:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.578485:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.578487:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.578487:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.578496:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.578499:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.578499:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.578499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677447.578500:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:12.0:1619677447.578501:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.578502:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:12.0:1619677447.578503:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.578503:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677447.578504:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.578504:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.578545:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.578548:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.578549:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.578550:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.578552:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.578554:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.578704:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.578706:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.578707:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.578708:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.578710:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.578711:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.578790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.578791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677447.578838:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (23)++
00000800:00000200:5.0:1619677447.578841:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (24)--
00000400:00000200:5.0:1619677447.578842:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.578844:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (23)++
00000400:00000200:5.0:1619677447.578844:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.578845:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (24)--
00000400:00000200:20.0:1619677447.578846:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.578847:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.578847:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.578848:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.578850:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.578850:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.578851:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (23)++
00000800:00000200:5.0:1619677447.578851:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.578852:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.578853:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.578919:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.578922:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.578923:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.578925:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.578928:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.578929:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.578930:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.578932:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.578933:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.578934:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.578938:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.578940:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.578942:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.578943:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.578944:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.578945:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.578946:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.578946:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.578947:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.578959:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.578962:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.578962:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.578963:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.578965:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.578967:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677447.578968:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.578970:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.578971:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.578973:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.578974:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.578975:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.578977:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.578980:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.578980:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.578981:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.578982:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.578982:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.578990:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.578993:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.578994:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.578995:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.578995:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.578997:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.579049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.579052:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.579055:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.579057:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.579058:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.579060:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.579061:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677447.579062:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.579064:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.579065:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.579066:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677447.579069:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.579071:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.579073:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.579074:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.579075:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.579076:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.579077:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.579078:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.579078:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677447.579090:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.579092:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.579093:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.579095:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.579095:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.579097:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.579127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.579169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.579181:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.579185:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.579186:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.579187:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.579190:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.579191:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.579193:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726247] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.579194:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.579195:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.579196:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.579199:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.579200:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.579202:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.579203:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.579204:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.579206:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.579206:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.579207:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.579208:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.579208:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.579219:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:6.0:1619677447.579219:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677447.579221:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.579222:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677447.579222:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.579223:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.579223:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:6.0:1619677447.579223:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.579225:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.579228:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.579361:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.579364:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.579365:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.579368:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.579371:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.579373:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.579375:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.579377:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.579378:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.579380:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.579387:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.579390:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:24.0:1619677447.579391:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:22.0:1619677447.579393:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.579394:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:22.0:1619677447.579394:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:8.0:1619677447.579394:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677447.579395:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.579396:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000400:00000200:24.0:1619677447.579397:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.579397:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.579398:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.579398:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:24.0:1619677447.579399:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.579400:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.579402:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96726244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.579403:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.579404:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.579405:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677447.579405:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.579407:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.579408:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677447.579410:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.579411:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.579413:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:17.0:1619677447.579414:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.579415:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:24.0:1619677447.579417:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677447.579417:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96726243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.579418:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.579418:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:17.0:1619677447.579418:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.579420:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:17.0:1619677447.579420:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.579420:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:24.0:1619677447.579421:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.579421:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.579421:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:17.0:1619677447.579424:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.579426:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.579429:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.579429:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.579430:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.579432:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.579432:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.579433:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.579434:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677447.579435:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.579437:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.579438:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.579440:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.579441:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.579443:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:6.0:1619677447.579445:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.579448:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.579449:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.579449:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.579451:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.579453:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.579555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.579568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.579576:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.579579:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.579581:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.579581:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.579583:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.579585:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.579587:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.579589:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.579590:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.579591:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.579595:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.579598:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.579601:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.579602:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677447.579602:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.579603:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677447.579605:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677447.579605:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:22.0:1619677447.579606:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.579606:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.579606:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.579606:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.579607:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:22.0:1619677447.579608:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.579612:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:12.0:1619677447.579618:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.579621:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.579622:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.579622:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:12.0:1619677447.579624:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.579625:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677447.579625:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.579626:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:6.0:1619677447.579626:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (24)++
00000800:00000200:22.0:1619677447.579627:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:12.0:1619677447.579627:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.579627:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (24)--
00000400:00000200:22.0:1619677447.579628:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.579628:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.579629:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.579630:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.579633:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.579703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677447.579892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.579895:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.579897:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.579899:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.579901:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.579902:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.579904:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.579905:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.579907:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.579908:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.579909:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.579912:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.579914:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.579916:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.579917:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.579917:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.579919:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.579919:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.579921:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.579922:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.579923:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.579927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677447.579932:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677447.579933:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:6.0:1619677447.579935:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677447.579935:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:22.0:1619677447.579936:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.579936:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.579936:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:6.0:1619677447.579938:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.579938:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.579940:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.580165:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.580167:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.580168:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.580170:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.580172:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.580174:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.580175:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.580176:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.580177:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.580178:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.580181:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.580183:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.580185:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.580186:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.580187:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.580189:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.580189:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.580189:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.580190:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677447.580193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.580198:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.580201:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:22.0:1619677447.580202:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:20.0:1619677447.580203:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.580203:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.580205:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:5.0:1619677447.580205:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.580206:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.580206:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:5.0:1619677447.580208:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.580209:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.580209:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.580211:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:5.0:1619677447.580211:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.580211:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.580213:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.580216:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.580218:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.580221:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.580222:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.580222:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.580224:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.580225:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.580225:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.580226:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.0:1619677447.580285:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.580288:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.580289:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.580291:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.580299:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.580301:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.580302:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.580304:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.580305:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.580306:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:4.2:1619677447.580306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:8.0:1619677447.580311:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.580313:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.580314:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.580316:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.580316:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.580317:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:24.0:1619677447.580318:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.580318:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:24.0:1619677447.580321:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.580321:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:24.0:1619677447.580322:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.580323:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.580324:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.580325:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.580326:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:9.2:1619677447.580327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.580327:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.580328:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.580329:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:24.0:1619677447.580330:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.580330:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.580331:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.580334:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.580335:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.580335:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677447.580336:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677447.580337:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.580338:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677447.580338:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:24.0:1619677447.580339:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.580340:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:18.0:1619677447.580340:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.580340:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.580341:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677447.580341:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.580345:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.580346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.580351:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.580354:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:8.0:1619677447.580354:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:17.0:1619677447.580355:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.580355:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.580356:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.580357:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677447.580357:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.580357:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677447.580360:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677447.580392:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.580395:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.580395:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.580397:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.580399:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.580400:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.580402:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.580403:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.580404:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.580405:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.580407:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.580409:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.2:1619677447.580411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:18.0:1619677447.580411:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.580412:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.580412:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.580413:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.580414:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.580415:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.580415:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.580420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.580426:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.580428:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.580429:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.580430:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.580431:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.580433:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.580452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.580459:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.580462:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.580463:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.580463:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.580465:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.580469:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.580470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.580477:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.580479:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.580480:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.580481:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.580481:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.580482:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.580547:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677447.580550:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677447.580551:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677447.580554:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.580556:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.580557:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.580559:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.580560:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.580561:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.580562:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.580566:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.580568:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.580571:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.580571:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677447.580572:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:1.0:1619677447.580574:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:1.0:1619677447.580574:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:13.0:1619677447.580575:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:1.0:1619677447.580575:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677447.580576:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:13.0:1619677447.580577:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677447.580578:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.580580:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.580582:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.580583:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.580583:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677447.580583:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.580584:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.580585:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.580587:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.580598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.580600:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.580603:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.580605:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.580607:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.580608:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.580610:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.580611:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.580613:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.580614:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.580615:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.580619:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.580621:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.580624:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.580625:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.580625:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.580627:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.580628:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.580629:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.580629:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677447.580721:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677447.580724:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:1.0:1619677447.580724:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:17.0:1619677447.580726:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.580729:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.580731:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:17.0:1619677447.580732:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.580733:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.580734:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.580735:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:17.0:1619677447.580739:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.580740:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 976 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677447.580743:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.580744:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:17.0:1619677447.580744:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:17.0:1619677447.580746:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:17.0:1619677447.580747:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677447.580747:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:17.0:1619677447.580748:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:5.0:1619677447.580761:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.580763:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.580764:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.580766:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.580766:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.580769:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:6.0:1619677447.580844:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.580847:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.580847:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.580848:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.580850:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.580854:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.580978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:4.2:1619677447.581009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.581019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677447.581022:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.581025:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.581025:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.581027:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.581028:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.581029:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677447.581039:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.581042:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.581042:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.581044:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.581046:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.581047:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.581049:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.581050:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.581051:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.581052:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:22.0:1619677447.581054:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:18.0:1619677447.581056:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.581057:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:18.0:1619677447.581057:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677447.581058:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.581059:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.581061:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.581061:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.581062:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.581062:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:22.0:1619677447.581064:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677447.581064:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.581065:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.581065:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.581066:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677447.581095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.581097:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.581100:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.581102:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.581103:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.581106:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.581107:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677447.581109:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.581110:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.581111:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.581112:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677447.581116:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.581117:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677447.581121:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.581122:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677447.581122:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677447.581124:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677447.581125:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.581126:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.581126:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:1.0:1619677447.581180:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.581183:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.581183:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.581186:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.581188:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.581190:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.581191:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.581192:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.581194:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.581194:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.581198:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.581199:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.581202:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.581203:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.581204:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.581206:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.581206:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.581208:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.581209:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.581209:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.581219:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.581221:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.581222:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.581222:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.581257:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.581260:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.581261:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.581263:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.581265:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.581267:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.581268:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.581270:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.581271:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.581272:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.581275:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.581277:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.581279:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.581280:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.581281:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.581282:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.581283:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.581284:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.581285:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.581286:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.0:1619677447.581298:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.581301:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.581302:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.581304:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.581305:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.581307:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677447.581338:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.581340:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.581341:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.581342:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.581345:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.581346:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.581354:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.581356:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.581357:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.581358:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.581358:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.581360:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.581404:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.581406:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.581407:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.581408:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.581419:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.581422:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.581423:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.581426:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.581428:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.581430:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.581431:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.581432:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.581433:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.581434:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.581437:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.581439:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.581442:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.581442:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.581443:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.581445:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.581445:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.581446:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.581447:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.581448:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.2:1619677447.581458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677447.581458:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.581460:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.581461:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.581462:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677447.581463:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.581465:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.581465:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.581465:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.581466:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.581467:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.581469:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.581470:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.581471:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.581472:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.581473:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000400:00000200:5.0:1619677447.581473:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.581474:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.581474:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.581474:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677447.581476:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.581476:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:5.0:1619677447.581476:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.581477:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.581478:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.581479:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.581483:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.581484:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.581487:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.581488:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.581489:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.581491:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.581492:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.581493:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.581493:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.581503:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.581506:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.581507:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.581508:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.581510:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.581511:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.581772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.581808:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.581811:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.581811:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.581814:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.581816:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.581818:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.581819:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96726214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.581820:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.581821:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677447.581822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:1.0:1619677447.581822:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.581826:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.581828:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.0:1619677447.581829:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.581831:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:1.0:1619677447.581831:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.581832:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:1.0:1619677447.581832:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:24.0:1619677447.581833:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677447.581833:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.581834:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.581835:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.581835:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.581835:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:1.0:1619677447.581835:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.581835:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.581836:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677447.581837:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.581837:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677447.581838:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.581839:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:22.0:1619677447.581840:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:8.0:1619677447.581841:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.581842:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.581843:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.581844:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.581847:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.581849:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.581852:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.581852:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.581853:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.581855:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.581855:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.581856:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.581856:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.581868:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.581871:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.581872:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.581872:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.581874:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.581876:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677447.581877:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677447.581879:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.581880:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.581880:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.581882:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.581882:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.581887:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.581890:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.581891:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.581891:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.581892:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.581893:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.581942:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.581945:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.581946:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.581948:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.581950:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.581952:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.581953:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.581954:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.581955:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.581956:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.581960:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.581962:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.581964:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.581965:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.581966:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.581967:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.581968:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.581968:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.581969:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677447.581981:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.581983:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.581984:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.581985:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.581986:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.581989:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677447.581989:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677447.581992:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.581993:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.581994:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.581997:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.581998:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.582000:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.582002:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.582002:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.582003:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.582007:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.582008:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.582010:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.582011:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.582012:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.582013:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.582013:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.582014:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.582015:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677447.582027:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.582029:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.582030:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.582032:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.582033:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.582035:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.582048:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.582050:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.582051:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.582052:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.582053:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.582054:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.582341:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.582344:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.582345:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.582347:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.582350:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.582351:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.582353:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.582354:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.582355:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.582356:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.582359:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.582361:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.582364:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.582365:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.582366:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.582368:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.582368:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.582369:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.582369:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.582382:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.582384:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.582385:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.582387:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.582389:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.582390:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.582401:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.582403:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.582405:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.582406:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.582409:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.582410:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.582411:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.582412:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.582413:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.582414:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.582418:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.582419:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.582422:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.582423:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.582423:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.582425:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.582425:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.582426:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.582427:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677447.582440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677447.582440:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.582442:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.582443:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.582445:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.582447:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.582447:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.582487:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.582490:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.582491:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.582493:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.582493:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.582496:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677447.582564:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.582567:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.582567:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.582569:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.582572:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.582573:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.582575:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.582576:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.582577:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.582578:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.582581:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.582583:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.582585:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.582586:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.582586:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.582588:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.582589:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.582589:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.582590:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.582594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.582602:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.582604:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.582605:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.582606:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.582608:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.582610:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.582615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.582622:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.582625:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.582626:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.582626:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.582627:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.582628:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.582637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.582638:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.582640:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.582641:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.582643:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.582644:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.582645:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.582654:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.582657:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.582658:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.582659:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.582659:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.582661:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.582677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677447.582734:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.582737:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.582737:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.582740:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.582742:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.582744:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.582745:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.582746:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.582747:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.582748:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.582751:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.582753:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.582756:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.582757:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.582757:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.582759:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.582760:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.582760:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.582761:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.582772:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.582775:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.582775:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.582777:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.582778:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.582780:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.582963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.582970:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.582973:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.582974:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.582975:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.582978:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.582980:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.582981:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.582982:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.582983:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.582984:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.582987:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.582989:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.582991:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.582992:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.582992:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.582994:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.582994:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.582995:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.582996:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677447.583007:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677447.583008:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677447.583009:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.583010:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677447.583010:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.583011:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:18.0:1619677447.583012:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677447.583012:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.583014:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.583014:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.583016:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:1.0:1619677447.583016:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:18.0:1619677447.583017:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.583018:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.583019:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.583020:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.583024:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.583025:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.583027:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.583027:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.583028:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.583030:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.583030:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.583031:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.583031:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.583036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.583043:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.583045:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.583046:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.583046:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.583048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:2.0:1619677447.583048:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.583050:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677447.583097:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.583099:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.583100:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.583102:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.583102:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.583105:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.583151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.583153:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.583156:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.583158:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.583159:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.583161:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.583163:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:20.0:1619677447.583164:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.583165:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.583166:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.583167:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:20.0:1619677447.583171:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677447.583172:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677447.583173:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.583175:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.583176:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677447.583177:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.583178:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:20.0:1619677447.583178:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:17.0:1619677447.583178:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.583180:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:20.0:1619677447.583180:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:17.0:1619677447.583181:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.583182:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.583183:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000400:00000200:17.0:1619677447.583183:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.0:1619677447.583184:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:17.0:1619677447.583184:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.583185:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.583186:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677447.583186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:17.0:1619677447.583187:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.2:1619677447.583189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000400:00000200:17.0:1619677447.583190:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.583192:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.583192:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:17.0:1619677447.583194:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.583194:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:17.0:1619677447.583195:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.583195:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.583195:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.583196:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.583196:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.583197:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.583197:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.583197:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.583197:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:8.0:1619677447.583198:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.583198:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677447.583199:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.583200:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.583200:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677447.583206:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:17.0:1619677447.583212:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.583212:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.583213:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.583213:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.583213:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.583214:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:1.0:1619677447.583214:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.583215:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.583216:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.583218:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.583261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.583312:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677447.583315:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:1.0:1619677447.583316:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:17.0:1619677447.583318:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.583320:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.583321:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:17.0:1619677447.583322:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.583324:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.583325:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.583326:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:17.0:1619677447.583329:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.583331:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 568 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677447.583333:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.583334:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:17.0:1619677447.583335:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:17.0:1619677447.583337:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:17.0:1619677447.583337:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677447.583338:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdbb1200] (22)++
00000800:00000200:17.0:1619677447.583339:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (23)--
00000800:00000200:17.0:1619677447.583339:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:1.0:1619677447.583350:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)++
00000800:00000200:1.0:1619677447.583352:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdbb1200] (23)--
00000800:00000200:5.0:1619677447.583353:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:1.0:1619677447.583353:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:1.0:1619677447.583354:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:5.0:1619677447.583355:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.583356:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.583358:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.583359:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.583360:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:6.0:1619677447.583440:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.583443:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.583443:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:20.2:1619677447.583445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:6.0:1619677447.583445:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.583447:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.583449:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.583450:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.583451:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.583452:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.583454:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.583457:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.583459:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.583462:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.583463:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.583463:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.583465:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.583465:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.583466:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.583466:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.583472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.583480:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.583482:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.583483:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.583483:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.583486:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.583489:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.0:1619677447.583495:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.583497:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.583498:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.583500:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.583501:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.583503:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677447.583515:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.0:1619677447.583516:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.583518:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.583519:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.583519:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:5.0:1619677447.583519:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.583520:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000400:00000200:22.0:1619677447.583521:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.583521:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.583522:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.583523:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677447.583524:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.583535:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.583538:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.583539:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.583539:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.583540:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.583540:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.583559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.583565:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.583570:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.583571:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:17.0:1619677447.583573:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677447.583573:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.583576:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:5.0:1619677447.583576:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.583576:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677447.583578:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.583580:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.583580:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.583582:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.583582:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.583582:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.583584:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677447.583584:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.583585:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.583587:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.583588:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.583589:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.583592:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.583594:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.583596:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.583598:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.583599:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.583600:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.583600:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.583601:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.583601:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.583602:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.583602:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677447.583602:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.583603:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.583603:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677447.583604:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.583605:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.583606:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.583607:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.583608:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.583613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.583616:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677447.583616:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677447.583618:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:1.0:1619677447.583618:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677447.583619:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.583619:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677447.583619:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:1.0:1619677447.583619:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.583620:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.583621:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.583621:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.583623:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.583623:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.583624:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.583625:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.583626:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.583636:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.583638:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.583639:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.583639:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.583640:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.583640:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.583667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.583722:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.583725:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.583726:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.583727:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.583729:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.583732:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.583822:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.583824:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.583825:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.583827:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.583829:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.583831:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.583832:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.583833:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.583834:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.583835:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.583839:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.583841:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.583843:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.583844:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.583845:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.583846:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.583847:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.583848:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.583848:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677447.583871:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.583873:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.583874:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.583876:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.583877:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.583878:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677447.583886:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.583889:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.583890:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.583891:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.583893:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.583894:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.583967:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.2:1619677447.583968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677447.583970:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.583970:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.583972:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.583974:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.583975:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.583977:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.583978:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.583979:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.583981:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.583985:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.583987:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.583989:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.583990:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.583991:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.583992:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.583993:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.583994:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.583994:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.584005:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.584007:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.584008:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.584008:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.584010:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.584011:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.584013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:4.2:1619677447.584024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.584031:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.584034:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.584035:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.584037:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.584039:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.584041:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.584042:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.584044:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.584045:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.584046:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.584050:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.584051:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.584054:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.584055:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.584056:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.584057:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.584058:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.584058:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.584059:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.584070:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.584073:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.584073:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.584074:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.584076:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.584079:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.584223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.584260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.584263:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.584266:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.584266:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:18.0:1619677447.584268:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:6.0:1619677447.584269:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.584271:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.584272:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.584272:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.584272:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:6.0:1619677447.584273:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677447.584274:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.584275:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96726181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.584278:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:6.0:1619677447.584278:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.584279:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.584280:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.584284:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.584287:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.584290:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.584291:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.584291:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.584299:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:12.0:1619677447.584300:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677447.584300:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.584300:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.584301:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.584302:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.584303:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.584304:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.584307:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.584308:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.584345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:5.0:1619677447.584394:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.584397:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.584398:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.584400:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.584400:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.584402:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677447.584457:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.584460:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.584461:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.584462:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.584464:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.584466:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.584467:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.584469:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.584470:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.584471:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.584474:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.584476:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.584479:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.584480:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.584481:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.584483:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.584483:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.584484:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.584484:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.584495:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.584498:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.584499:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.584501:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.584502:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.584503:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.584549:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.584552:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.584553:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.584555:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.584557:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.584559:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.584560:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.584561:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.584562:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.584563:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.584567:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.584569:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.584571:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.584572:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.584573:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.584574:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.584575:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.584576:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.584576:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.584585:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.584588:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.584588:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.584590:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.584592:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.584592:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.584604:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.584607:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.584608:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.584608:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.2:1619677447.584609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:13.0:1619677447.584609:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.584610:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.584612:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.584614:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.584617:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.584618:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.584619:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.584621:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677447.584622:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.584624:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.584625:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.584626:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677447.584629:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.584631:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677447.584634:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.584634:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677447.584635:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.584636:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.584637:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.584638:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.584638:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.584643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.584650:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.584653:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.584654:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.584654:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.584656:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.584659:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677447.584686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:12.0:1619677447.584698:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.584701:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.584701:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.584703:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.584705:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.584706:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.584723:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.584726:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.584726:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.584728:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.584730:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.584732:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677447.584733:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.584734:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.584735:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.584736:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677447.584740:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:4.2:1619677447.584740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.584742:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.0:1619677447.584742:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:18.0:1619677447.584744:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.584745:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:5.0:1619677447.584745:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:18.0:1619677447.584746:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:5.0:1619677447.584746:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.584747:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.584748:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.584748:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677447.584748:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.584749:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.584750:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.0:1619677447.584750:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677447.584761:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.584764:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.584765:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.584765:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.584767:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.584770:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.584781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.584786:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.584788:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.584789:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.584789:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.584791:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.584794:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.584908:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.584911:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.584911:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.584912:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.584914:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.584918:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.584993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.584995:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.584998:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.585000:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.585001:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.585002:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.585004:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.585005:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.585006:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.585007:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.585008:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.585011:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.585013:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.585016:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.585017:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.585017:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.585019:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.585020:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.585020:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.585021:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677447.585026:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.585029:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.585030:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.585032:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.585035:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.585037:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.585038:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.585039:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.585040:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.585041:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.585044:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.585046:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:4.2:1619677447.585048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:2.0:1619677447.585049:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.585050:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.585051:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.585053:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.585053:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.585054:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:24.0:1619677447.585055:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677447.585055:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.585056:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:24.0:1619677447.585058:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.585059:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.585061:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.585063:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.585064:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.585066:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.585067:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.585067:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:13.0:1619677447.585067:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:24.0:1619677447.585068:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.585069:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:18.0:1619677447.585069:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.585070:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:13.0:1619677447.585070:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.585071:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.585071:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:24.0:1619677447.585073:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677447.585073:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.585073:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.585075:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677447.585077:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.585078:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.585079:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.585080:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.585081:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.585082:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.585083:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.585084:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.585085:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.585090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.585095:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:8.0:1619677447.585098:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.585098:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.585099:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.585099:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677447.585099:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.585100:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.585101:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.585101:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.585104:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.585114:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.585117:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.585118:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.585118:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.585119:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.585120:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.585134:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.585136:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.585137:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.585139:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.585141:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.585143:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.585144:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.585145:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.585147:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.585147:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677447.585151:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.585153:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.585156:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.585157:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.585157:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.585159:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.585160:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.585161:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.585162:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.585163:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.585165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.585172:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:17.0:1619677447.585173:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677447.585175:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:17.0:1619677447.585175:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.0:1619677447.585175:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.585176:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.585176:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.585177:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677447.585178:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.585182:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677447.585268:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.585270:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.585271:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.585273:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.585275:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.585275:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.585314:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.585317:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.585317:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.585320:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.585322:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.585324:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.585325:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726166] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.585326:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.585327:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.585328:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.585331:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.585333:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.585334:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.585335:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.585336:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.585337:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.585338:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.585338:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.585339:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.585344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.585351:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.585353:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.585353:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.585355:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.585355:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.585356:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.585392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.2:1619677447.585395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.585399:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.585400:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.585400:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.585402:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.585404:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.585405:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.585407:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.585409:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.585410:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677447.585411:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.585413:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.585414:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.585415:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677447.585416:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.585418:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.585419:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.585419:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.585420:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.585422:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.585422:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:20.0:1619677447.585425:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.585425:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.585426:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677447.585426:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677447.585427:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.585428:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677447.585429:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.585430:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.585430:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.585430:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.585431:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.585431:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677447.585432:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.585433:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.585434:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.585435:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:4.2:1619677447.585435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.585436:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.585437:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677447.585485:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.585487:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.585488:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.585490:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.585490:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.585493:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.0:1619677447.585582:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.585585:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.585586:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.585589:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.585592:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.585593:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.585595:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.585596:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.585597:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.585598:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.585601:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.585603:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.585605:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.585606:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.585607:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.585609:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.585609:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.585611:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:12.0:1619677447.585612:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677447.585612:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.585612:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.585615:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.585615:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.585615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:12.0:1619677447.585617:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.585619:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.585620:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.585623:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677447.585625:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:18.0:1619677447.585626:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.585626:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.585627:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:18.0:1619677447.585627:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.585628:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.585629:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:18.0:1619677447.585629:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.585631:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.585632:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.585634:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.585635:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.585635:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.585636:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:9.2:1619677447.585636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677447.585637:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.585643:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.585646:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.585647:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.585647:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.585648:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.585649:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.585738:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.585741:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.585741:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.585744:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.585746:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.585748:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.585749:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.585750:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.585752:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.585753:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.585756:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.585758:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.585760:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.585761:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.585762:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.585763:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.585764:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.585765:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.585766:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.585766:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.2:1619677447.585768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.585771:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.585773:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.585775:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.585777:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.585778:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.585778:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677447.585778:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:20.0:1619677447.585779:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:6.0:1619677447.585780:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:20.0:1619677447.585781:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.585781:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:6.0:1619677447.585781:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677447.585782:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.585782:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.585782:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:20.0:1619677447.585783:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.585783:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:20.0:1619677447.585785:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677447.585785:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.585787:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:20.0:1619677447.585788:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.585790:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.585793:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.585793:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.585794:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.585796:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.585796:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.585798:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.585799:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.585799:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.585806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.585809:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.585811:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.585811:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.585811:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.585812:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:20.0:1619677447.585813:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.585813:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.585814:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.585815:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.585816:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677447.585830:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.585833:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.585834:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.585835:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.585835:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.585836:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.585902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:9.2:1619677447.585941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.585949:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.585952:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.585953:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.585954:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.585955:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.585959:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.586007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:12.0:1619677447.586049:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.586052:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.586053:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.586054:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.586056:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.586056:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.586071:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.586074:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.586075:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.586076:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.586078:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.586081:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.586185:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.586188:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.586189:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.586190:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.586193:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.586195:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.586196:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.586196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:8.0:1619677447.586198:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.586199:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.586199:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.586202:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.586204:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.586207:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.586208:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.586208:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.586210:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.586211:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.586211:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.586212:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.586216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.586224:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.586226:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.586228:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.586228:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.586230:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.586233:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.586239:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.586242:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.586242:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.586243:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.586245:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.586248:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.586257:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.586259:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.586260:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.586260:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.586261:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.586262:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677447.586300:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.586303:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.586304:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.586306:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.586307:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.586309:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.586310:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.586311:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.586312:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.586313:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.586316:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.586317:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.586319:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.586320:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.586321:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.586322:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.586323:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.586324:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.586325:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.586325:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.586329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.586338:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.586339:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.586339:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.586339:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.586340:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.586340:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.586341:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.586341:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.586342:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677447.586342:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.586346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.586354:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.586357:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.586358:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.586360:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.586363:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.586364:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.586365:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.586367:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.586368:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.586369:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.586372:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.586374:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.586377:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.586378:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.586378:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.586380:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.586380:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.586381:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.586381:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.586394:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.586397:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.586398:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.586399:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.586401:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.586404:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.586510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.586516:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.586519:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.586521:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.586521:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.586523:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.586525:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.586526:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.586527:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.586528:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.586529:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.586533:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.586534:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.586537:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.586538:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.586539:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.586541:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.586542:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.586542:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.586543:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.586554:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.586557:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.586558:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.586560:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.586561:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.586563:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.586774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.586822:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.586825:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.586825:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.586826:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.586828:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.586831:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.586862:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.586865:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.586865:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.586868:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.586870:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.586871:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.586873:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.586874:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.586875:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.586876:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.586880:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.586882:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.586885:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.586886:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.586886:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.586888:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.586888:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.586889:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.586889:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677447.586893:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.586896:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.586897:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.586899:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.586901:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.586902:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.586903:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.586905:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.586906:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.586906:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.586910:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.586912:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.586915:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.586916:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.586916:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.586918:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.586919:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.586919:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.586920:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.586931:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.586933:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.586934:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.586936:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.586937:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.586939:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.587163:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.587165:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.587166:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.587168:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.587171:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.587172:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.587174:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.587175:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.587176:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:1.0:1619677447.587176:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.587177:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:22.0:1619677447.587179:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.587179:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677447.587182:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.587183:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.587185:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.587185:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:22.0:1619677447.587187:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.587188:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.587188:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677447.587189:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.587189:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.587189:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:22.0:1619677447.587190:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.587191:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677447.587191:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.587191:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.587192:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.587193:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677447.587195:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.587196:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:9.2:1619677447.587197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:22.0:1619677447.587199:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.587200:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.587200:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.587202:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.587202:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.587203:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677447.587203:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:22.0:1619677447.587204:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.587208:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.587212:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.587213:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.587215:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.587217:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:8.0:1619677447.587218:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.0:1619677447.587219:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.587220:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.587221:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.587223:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.587224:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.587263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.587312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.587418:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.587421:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.587422:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.587422:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.587424:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:6.0:1619677447.587425:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.587427:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:6.0:1619677447.587428:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677447.587428:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.587431:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.587433:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.587435:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.587437:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.587438:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.587439:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.587440:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.587444:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.587446:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.587449:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.587450:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.587451:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677447.587452:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:17.0:1619677447.587453:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.587454:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.587454:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.587458:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677447.587461:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677447.587462:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.587464:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.587465:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677447.587468:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677447.587469:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.587469:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.587469:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.587471:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:12.0:1619677447.587471:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.587472:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677447.587473:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677447.587473:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.587474:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.587475:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.587616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677447.587617:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.587619:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.587620:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.587622:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.587622:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.587624:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:18.0:1619677447.587625:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.587627:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:18.0:1619677447.587627:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677447.587627:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.587629:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.587630:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.587630:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.587631:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.587632:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677447.587632:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.587633:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.587635:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.587636:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.587637:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.587638:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.587639:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:5.0:1619677447.587642:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.587643:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.587644:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.587644:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.587646:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.587647:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.587648:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677447.587648:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.587649:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677447.587650:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677447.587651:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.587651:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.587652:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:5.0:1619677447.587652:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.587652:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677447.587654:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.587654:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.587655:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.587655:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:18.0:1619677447.587656:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.587656:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677447.587658:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.587659:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96726138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.587660:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.587661:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.587662:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.587666:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.587666:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677447.587668:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677447.587669:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.587670:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.587671:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.587672:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:12.0:1619677447.587672:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.587673:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677447.587673:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.587674:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.587675:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.587675:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:12.0:1619677447.587675:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.587676:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677447.587686:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.587689:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.587691:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:1.0:1619677447.587692:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.587693:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.587694:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.587695:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.587696:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.587697:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.587698:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.587699:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.587700:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.587701:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.587704:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.587705:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.587706:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:1.0:1619677447.587706:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.587707:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.587708:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.587708:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.587710:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.587710:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.587711:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.587712:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.587713:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.587714:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.587716:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.587716:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.587718:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.587719:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.587719:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.587729:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.587731:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.587732:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.587733:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.587909:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.587912:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.587913:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.587915:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.587918:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.587919:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.587920:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.587922:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.587923:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.587923:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.587927:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.587929:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.587931:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.587932:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.587933:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.587934:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.587935:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.587936:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.587936:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677447.587942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:13.0:1619677447.587947:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.587950:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.587951:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.587953:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.587954:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.587956:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.587987:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.587989:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.587990:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.587991:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.587993:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.587995:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.588138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.588141:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.588143:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.588145:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.588148:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.588150:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:5.0:1619677447.588150:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.588151:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.588151:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:20.0:1619677447.588152:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.588153:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:20.0:1619677447.588154:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.588154:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677447.588154:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.588157:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.588159:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.588160:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.588161:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:17.0:1619677447.588161:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.588162:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.588163:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.588164:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:17.0:1619677447.588164:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.588165:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:17.0:1619677447.588165:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.588166:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677447.588167:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.588168:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.588169:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000400:00000200:17.0:1619677447.588169:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.588170:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677447.588171:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:4.2:1619677447.588173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.588174:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677447.588174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677447.588175:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.588176:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.588177:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.588178:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.588178:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.588179:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677447.588181:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.588184:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.588185:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.588185:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.588188:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.588190:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.588191:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:9.2:1619677447.588193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677447.588194:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.588194:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.588196:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.588197:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.588199:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677447.588200:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.588202:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677447.588203:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.588204:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.588204:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677447.588205:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:18.0:1619677447.588205:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677447.588205:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677447.588206:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677447.588207:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:22.0:1619677447.588208:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.588210:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.588210:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.588211:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:22.0:1619677447.588212:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:12.0:1619677447.588212:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.588212:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:22.0:1619677447.588213:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.588214:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:22.0:1619677447.588215:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.588216:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.588217:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.588222:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.588223:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.588224:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.588224:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.588226:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677447.588226:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.588228:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:6.0:1619677447.588228:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.588229:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.588230:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.588231:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.588231:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.588232:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.588232:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.588233:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.588246:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.588248:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.588249:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.588251:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.588251:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.588253:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677447.588275:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.588278:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.588279:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.588281:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.588283:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.588285:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.588286:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.588287:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.588288:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.588289:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.588292:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.588300:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.588302:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.588303:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.588304:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.588306:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.588306:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.588307:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.588308:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677447.588321:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.588323:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.588324:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.588325:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.588326:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.588330:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.588395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.588402:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.588405:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.588406:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.588408:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.588411:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.588413:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.588414:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.588416:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.588417:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.588418:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.588422:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.588423:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.588426:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.588427:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.588427:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.588429:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.588430:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.588431:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.588431:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677447.588438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677447.588444:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.588447:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.588448:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.588448:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.588451:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.588454:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.588471:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.588474:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.588474:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.588477:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.588479:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.588480:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.588481:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.588482:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677447.588483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677447.588483:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.588484:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.588488:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.588489:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677447.588489:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.0:1619677447.588491:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.588492:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.588492:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.588493:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.588493:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:20.0:1619677447.588494:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.588495:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:5.0:1619677447.588495:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677447.588496:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.588496:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677447.588497:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.588497:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.588509:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.588512:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.588513:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.588513:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.588516:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.588518:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.588833:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.588836:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.588836:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.588838:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.588840:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.588841:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.588843:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.588844:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.588845:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.588846:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.588849:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.588851:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.588854:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.588855:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.588855:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.588857:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.588858:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.588858:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.588859:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677447.588876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:13.0:1619677447.588891:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.588894:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.588895:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.588897:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.588898:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.588899:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.588910:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.588913:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.588913:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.588914:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.588915:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.588915:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.588921:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.588924:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.588925:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.588927:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.588927:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.588930:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.588941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.588969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.588971:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.588974:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.588976:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.588977:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.588978:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.588980:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677447.588981:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.588983:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.588983:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.588985:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:20.0:1619677447.588985:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:22.0:1619677447.588988:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:20.0:1619677447.588988:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.588988:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.588989:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.588990:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:22.0:1619677447.588991:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.588993:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.588993:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:22.0:1619677447.588994:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.588994:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.588996:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.588996:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.588997:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.0:1619677447.588997:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677447.588998:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.0:1619677447.588999:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.589000:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.589007:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.589007:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:8.0:1619677447.589010:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.589010:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677447.589010:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:22.0:1619677447.589011:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:8.0:1619677447.589011:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:6.0:1619677447.589011:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000400:00000200:8.0:1619677447.589012:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.589012:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.589012:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.589012:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.589013:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.589014:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:8.0:1619677447.589014:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.589015:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.589016:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.589017:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.589020:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.589023:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677447.589023:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.0:1619677447.589026:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.589027:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.589027:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.589029:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.589030:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.589030:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.589031:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677447.589043:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.589045:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.589046:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.589048:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.589048:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.589051:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.589228:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.589231:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.589232:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.589234:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.589236:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.589237:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.589239:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.589240:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.589241:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.589242:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.589245:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.589247:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.589250:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.589251:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.589251:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.589253:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.589253:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.589255:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.589256:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.589256:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.589259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.589265:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:1.0:1619677447.589266:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677447.589268:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.589269:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677447.589269:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:24.0:1619677447.589270:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.589270:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.589271:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677447.589272:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.589275:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.2:1619677447.589299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677447.589334:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.589337:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.589338:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.589338:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.589341:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.589344:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.0:1619677447.589345:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.589347:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.589348:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.589350:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.589352:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677447.589352:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.589353:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677447.589355:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.589356:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.589358:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.589360:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.589362:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.589363:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.589363:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000400:00000200:22.0:1619677447.589364:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.589366:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.589366:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:22.0:1619677447.589367:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.589367:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.589368:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.589369:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.589370:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:22.0:1619677447.589371:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.589372:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.589375:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.589376:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.589377:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.589378:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.589379:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.589381:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.589382:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.589382:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.589392:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.589394:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.589395:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.589396:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.589418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.589425:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.589428:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.589429:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.589430:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.589433:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.589434:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.589436:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.589437:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.589438:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.589439:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.589443:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.589444:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.589447:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.589448:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.589449:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.589450:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.589451:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.589451:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.589452:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.589464:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.589467:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.589468:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.589468:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.589470:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.589473:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.589617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.589622:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.589625:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.589627:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.589628:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.589629:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.589631:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.589632:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.589634:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.589635:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.589636:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.589639:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.589641:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.589644:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.589645:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.589645:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.589647:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.589648:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.589649:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.589650:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.589651:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.589655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.589658:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.589660:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:12.0:1619677447.589660:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677447.589661:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.589662:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.589663:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.589664:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.589666:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.589667:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.589669:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.589859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.589872:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.589874:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677447.589874:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:13.0:1619677447.589875:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.589877:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.589877:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.589878:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:12.0:1619677447.589878:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:1.0:1619677447.589878:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.589880:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677447.589881:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:20.2:1619677447.589882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677447.589884:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.589894:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.589897:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.589897:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.589898:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.589898:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.589899:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.0:1619677447.589903:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.589905:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.589906:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.589909:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.589909:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.589912:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.589981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:20.2:1619677447.590022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:5.0:1619677447.590026:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.590029:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.590030:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.590031:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677447.590032:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.590034:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.590149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.590152:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.590154:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.590156:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.590158:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.590159:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.590160:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.590162:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.590163:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.590164:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.590165:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.590168:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.590170:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.590173:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.590173:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.590174:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.590176:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.590176:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.590177:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.590178:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.590190:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.590192:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.590193:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.590193:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.590195:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.590198:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.590331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.590345:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.590349:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.590349:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.590352:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.590355:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.590357:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.590358:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96726104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.590360:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.590361:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.590362:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.590365:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.590367:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.590369:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.590369:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.590370:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.590372:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.590372:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.590373:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.590374:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:4.2:1619677447.590377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677447.590377:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:9.2:1619677447.590379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.590380:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.590381:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.590381:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.590382:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:1.0:1619677447.590383:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.590385:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.590386:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:1.0:1619677447.590386:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:6.0:1619677447.590387:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.590387:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:6.0:1619677447.590389:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.590389:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.590390:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.590391:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677447.590392:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:2.0:1619677447.590392:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.590394:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.590417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.590423:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.590426:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.590428:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.590428:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.590431:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.590432:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677447.590434:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.590435:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.590436:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.590437:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677447.590441:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.590443:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677447.590445:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.590446:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677447.590447:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677447.590448:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677447.590449:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.590450:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.590450:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:22.0:1619677447.590658:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.590661:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.590661:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.590662:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.590664:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.590667:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.590668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.590675:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.590678:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.590680:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.590681:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.590684:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.590685:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.590686:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.590688:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.590689:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.590690:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.590693:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.590694:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.590698:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.590699:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677447.590699:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (23)++
00000800:00000200:24.0:1619677447.590701:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (24)--
00000800:00000200:24.0:1619677447.590702:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.590703:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.590705:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.590705:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677447.590705:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (23)++
00000800:00000200:17.0:1619677447.590707:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (24)--
00000800:00000200:1.0:1619677447.590708:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (24)++
00000400:00000200:17.0:1619677447.590709:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.590709:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (24)--
00000400:00000200:1.0:1619677447.590710:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.590711:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.590711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:1.0:1619677447.590712:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.590715:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.590715:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.590716:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.0:1619677447.590720:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.590722:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.590723:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.590724:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.590724:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.590725:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.590725:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677447.590727:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.590728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:8.0:1619677447.590730:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.590732:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.590733:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.590734:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.590735:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.590736:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.590739:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.590741:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.590744:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.590745:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.590745:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.590747:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.590747:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.590748:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.590748:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.590753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.590760:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.590763:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.590764:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.590764:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.590766:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.590770:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.590834:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.590836:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.590837:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.590839:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.590842:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.590843:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.590844:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.590846:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.590847:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.590848:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.590851:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.590853:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.590855:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.590856:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.590857:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.590858:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.590859:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.590860:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.590860:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.590872:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.590875:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.590876:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.590876:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.590878:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.590880:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.590890:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.590893:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.590894:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.590894:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.590895:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.590895:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.590981:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.590984:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.590985:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.590986:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.590989:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.590990:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.590992:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.590993:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.590995:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677447.590996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677447.590996:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.590998:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.591000:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.591002:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.591003:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.591004:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.591005:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.591005:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.591007:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.591008:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.591009:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677447.591018:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677447.591019:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.591021:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:6.0:1619677447.591021:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.591022:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.591022:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.591023:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:6.0:1619677447.591024:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.591024:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677447.591027:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.591038:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.591041:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.591041:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.591042:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.591043:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.591043:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.591043:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.591045:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.591046:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.591048:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.591049:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.591051:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:12.0:1619677447.591165:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.591167:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.591167:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.591169:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.591171:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.591172:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.591235:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.591237:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.591238:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.591240:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.591242:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.591244:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.591245:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.591246:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.591247:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.591248:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.591252:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.591253:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.591256:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.591257:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.591257:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.591259:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.591260:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.591261:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.591262:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.591262:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.591273:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.591274:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:5.0:1619677447.591275:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677447.591275:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.591276:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.591278:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.591279:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.591280:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.591281:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.591283:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.591293:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.591299:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.591300:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677447.591300:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:20.0:1619677447.591301:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.591302:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.591302:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677447.591302:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.591304:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.591304:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.591306:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.591311:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.591312:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.591314:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.591315:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.591316:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.591316:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.591317:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.591326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.591332:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.591334:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.591336:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.591337:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.591339:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.591341:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677447.591342:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.591343:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.591344:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.591345:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677447.591349:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.591351:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.591352:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.591353:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.591354:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.591355:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.591356:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.591356:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.591357:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677447.591368:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.591370:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.591371:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.591372:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.591374:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.591375:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.591386:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.591388:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.591389:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.591390:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.591390:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.591392:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.591399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677447.591406:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.591409:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.591410:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.591413:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.591415:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.591417:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.591419:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.591420:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.591422:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.591423:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.591428:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.591432:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.591434:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.591435:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.591436:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.591437:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.591438:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.591439:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.591439:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.591445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677447.591449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677447.591452:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:13.0:1619677447.591453:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.591455:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:2.0:1619677447.591455:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.591455:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.591456:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:13.0:1619677447.591456:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.591458:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:13.0:1619677447.591459:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.591460:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.591461:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.591461:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.591487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.591494:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.591498:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.591498:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.591501:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.591504:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.591506:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.591508:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.591509:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.591511:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.591512:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.591517:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.591519:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.591522:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.591523:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.591524:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.591526:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.591526:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.591527:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.591528:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677447.591539:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.591542:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.591543:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.591543:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.591546:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.591549:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:9.2:1619677447.591828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.591835:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.591837:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.591839:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.591839:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.591841:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.591844:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.591848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677447.591854:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.591856:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.591857:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.591857:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.591858:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.591859:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677447.591876:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.591878:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.591879:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.591881:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.591883:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.591885:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.591886:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.591887:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.591888:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.591889:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.591893:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.591894:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.591897:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.591898:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.591898:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.591900:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.591900:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.591901:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.591903:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.591903:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.591912:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.591914:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.591915:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.591916:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.591916:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:13.0:1619677447.591918:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.591918:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.591919:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.591920:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.591920:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.591936:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.591939:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.591940:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.591940:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.591941:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.591942:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.591972:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.591975:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.591976:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.591978:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.591978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:18.0:1619677447.591979:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.591980:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.591982:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.591983:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.591984:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.591985:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:18.0:1619677447.591985:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:24.0:1619677447.591988:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:18.0:1619677447.591989:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.591989:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677447.591991:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:24.0:1619677447.591992:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.591994:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.591994:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.591995:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.591995:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:24.0:1619677447.591996:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.591997:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.591997:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.591997:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:24.0:1619677447.591998:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.591998:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:24.0:1619677447.591999:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.591999:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:24.0:1619677447.592000:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.592005:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.592007:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.592011:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.592012:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.592012:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.592012:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:24.0:1619677447.592014:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.592014:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677447.592014:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:24.0:1619677447.592015:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.592015:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:20.0:1619677447.592015:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.592017:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.592018:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.592019:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677447.592028:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.592031:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.592031:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.592032:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.592034:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.592036:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.592075:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.592077:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.592078:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.592080:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.592082:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.592083:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.592084:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.592085:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.592087:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.592087:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.592091:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.592092:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.592094:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.592095:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.592096:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.592097:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.592097:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.592098:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.592099:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677447.592109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.592110:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.592112:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.592112:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.592114:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.592114:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.592115:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.592149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.592337:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.592340:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.592342:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.592343:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.592346:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.592348:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.592349:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.592350:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.592351:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.592352:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.592355:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.592357:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.592358:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.592359:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.592360:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.592361:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.592362:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.592363:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.592363:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.592368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.592376:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.592378:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.592379:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.592379:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.592380:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.592382:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.592383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.592390:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.592393:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.592393:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.592394:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.592394:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.592395:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.592539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.592545:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.592548:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.592550:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.592550:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.592552:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.592554:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677447.592555:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.592556:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.592557:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.592558:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677447.592561:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.592563:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677447.592565:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.592566:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677447.592566:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677447.592568:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677447.592568:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.592569:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.592570:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.592574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:1.0:1619677447.592579:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.592581:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:1.0:1619677447.592582:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.592583:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677447.592584:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:1.0:1619677447.592584:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.592585:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.592585:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:1.0:1619677447.592586:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.592588:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.592588:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.592589:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.592590:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:1.0:1619677447.592591:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.592592:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.592593:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.592596:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.592598:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.592600:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.592600:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.592601:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.592603:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.592603:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.592604:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.592604:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.592606:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.592608:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.592609:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.592611:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.592614:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.592615:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.592617:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.592617:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:22.0:1619677447.592618:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.592618:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:22.0:1619677447.592619:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.592619:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.592620:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.592620:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.592621:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.592623:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.592625:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677447.592626:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.592628:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.592629:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.592630:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677447.592632:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:22.0:1619677447.592632:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.592633:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.592634:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677447.592635:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.592638:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.592639:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000800:00000200:1.0:1619677447.592639:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (23)++
00000800:00000200:17.0:1619677447.592640:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000800:00000200:1.0:1619677447.592640:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (25)--
00000400:00000200:17.0:1619677447.592641:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.592641:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.592642:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.592643:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.592645:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.592794:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.592797:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.592798:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.592800:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.592801:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.592803:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.592804:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.592806:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.592807:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.592808:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.592811:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.592813:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.592816:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.592816:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.592817:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.592819:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.592819:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.592821:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.592822:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.592822:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.592832:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677447.592832:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677447.592834:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.592835:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677447.592835:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677447.592836:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677447.592836:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.592837:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.592839:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.592840:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.592886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677447.592906:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.592908:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.592910:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.592912:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.592915:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.592916:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.592917:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.592919:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.592920:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.592921:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.592924:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.592926:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.592928:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.592928:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.592929:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.592930:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.592931:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.592931:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.592932:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.592933:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.592935:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.592936:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.592936:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.592939:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.592942:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677447.592942:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.592945:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.592945:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.592947:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.592949:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.592950:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.592952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.592957:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.592959:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.592962:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.592963:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.592964:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.592966:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677447.592967:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.592969:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.592970:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.592971:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677447.592974:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.592976:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.592978:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.592979:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677447.592980:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.592981:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.592982:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.592983:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.592984:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.592985:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.2:1619677447.592990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:1.0:1619677447.592995:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677447.592996:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:1.0:1619677447.592997:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:5.0:1619677447.592998:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:17.0:1619677447.592999:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.592999:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:1.0:1619677447.592999:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.593000:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:1.0:1619677447.593001:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677447.593003:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.593059:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.593062:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.593062:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.593064:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677447.593067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:17.0:1619677447.593067:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.593068:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.593069:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.593070:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.593071:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.593072:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.593075:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.593077:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.593079:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.593080:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.593081:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.593082:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.593083:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.593083:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.593084:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.593095:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.593098:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.593098:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.593099:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.593100:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.593102:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:1.0:1619677447.593102:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:2.0:1619677447.593105:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.593106:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.593108:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.593110:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.593111:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:12.0:1619677447.593112:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:2.0:1619677447.593112:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.593114:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:2.0:1619677447.593114:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677447.593115:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.593115:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.593116:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:12.0:1619677447.593117:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.593118:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:2.0:1619677447.593120:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.593122:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.593123:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.593126:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.593127:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.593128:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.593130:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.593130:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.593131:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.593132:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677447.593142:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.593144:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.593145:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.593147:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.593148:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.593150:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.593286:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.593289:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.593290:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.593292:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.593297:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.593299:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.593300:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.593300:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:17.0:1619677447.593302:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.593302:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.593302:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:17.0:1619677447.593303:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:12.0:1619677447.593303:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.593305:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.593306:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.593308:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:17.0:1619677447.593309:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.593309:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.593309:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.593310:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.593310:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.2:1619677447.593311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:13.0:1619677447.593311:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.593312:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:18.0:1619677447.593313:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677447.593315:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.593315:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:18.0:1619677447.593316:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:17.0:1619677447.593316:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:2.0:1619677447.593316:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:5.0:1619677447.593317:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677447.593318:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000400:00000200:18.0:1619677447.593319:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:17.0:1619677447.593319:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.593319:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.593320:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677447.593320:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:18.0:1619677447.593321:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.593322:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.593322:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.593323:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677447.593324:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.593324:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.593326:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.593326:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677447.593327:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.593327:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.593328:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:5.0:1619677447.593329:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.593331:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.593332:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677447.593333:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:9.2:1619677447.593335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:18.0:1619677447.593336:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:5.0:1619677447.593336:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.593338:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.593340:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.593341:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:5.0:1619677447.593341:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.593342:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:5.0:1619677447.593342:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:24.0:1619677447.593343:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:5.0:1619677447.593343:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.593344:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677447.593344:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.0:1619677447.593344:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:24.0:1619677447.593345:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:18.0:1619677447.593345:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:5.0:1619677447.593345:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.593346:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677447.593346:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:5.0:1619677447.593346:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:24.0:1619677447.593347:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.593347:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.593348:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:24.0:1619677447.593349:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.593350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:24.0:1619677447.593352:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.2:1619677447.593352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.593355:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:9.2:1619677447.593356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:20.0:1619677447.593357:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.593358:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677447.593358:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:20.0:1619677447.593359:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677447.593360:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.593362:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:8.0:1619677447.593362:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000400:00000200:2.0:1619677447.593362:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.593365:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000400:00000200:2.0:1619677447.593365:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.593366:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.593366:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.593367:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:2.0:1619677447.593367:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:8.0:1619677447.593368:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.593373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.593380:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.593383:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.593384:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.593384:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.593385:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.593386:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:22.0:1619677447.593444:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.593447:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.593448:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.593450:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.593452:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.593454:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.593455:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.593456:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.593457:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.593458:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.593462:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677447.593462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677447.593464:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.593466:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.593467:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.593467:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:24.0:1619677447.593469:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.593469:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.593469:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.593470:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677447.593471:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:22.0:1619677447.593471:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.593472:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.593474:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.593477:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.593478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:24.0:1619677447.593479:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.593480:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.593482:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.593483:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.593484:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.593486:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:24.0:1619677447.593487:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.593488:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:24.0:1619677447.593489:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:20.0:1619677447.593489:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.593491:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.593492:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.593492:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:24.0:1619677447.593493:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.593494:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.593494:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:24.0:1619677447.593495:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.593496:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.593496:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.593497:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.593509:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.593511:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.593512:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.593513:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.593514:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.593518:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677447.593525:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.593528:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.593529:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.593530:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.593531:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.593533:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677447.593590:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.593593:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.593593:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.593596:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.593599:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.593600:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.593601:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.593603:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.593604:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.593605:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.593608:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.593610:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.593613:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.593614:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.593614:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.593616:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.593616:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.593617:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.593617:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677447.593645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.593689:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.593692:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.593693:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.593693:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.593696:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.593698:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.593712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.593714:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677447.593715:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677447.593717:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:12.0:1619677447.593717:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.593718:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.593719:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.593720:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677447.593721:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.593721:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.593722:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.593722:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:20.0:1619677447.593723:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677447.593725:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.593726:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.593727:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.593728:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677447.593731:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.593732:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677447.593735:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.593736:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677447.593737:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.593739:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.593739:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.593740:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.593740:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.593745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.593753:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.593755:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.593756:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.593757:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.593758:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.593761:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677447.593863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:13.0:1619677447.593906:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.593909:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.593909:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.593911:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.593913:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.593914:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.594084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.594092:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.594095:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.594096:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.594096:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.594099:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.594102:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.594177:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.594180:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.594180:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.594183:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.594185:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.594186:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.594188:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.594189:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.594190:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.594191:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.594194:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.594196:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.594199:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.594199:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.594200:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.594202:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.594202:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.594204:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.594205:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.594205:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.594208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.594216:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.594217:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.594218:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.594219:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677447.594219:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.594220:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:2.0:1619677447.594220:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.594222:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.594222:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.594222:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.594223:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.594224:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.594225:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.594225:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677447.594226:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.594227:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.594228:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.594229:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.594230:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.594231:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.594234:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.594236:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.594239:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.594240:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.594240:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.594242:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.594243:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.594243:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.594244:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.594245:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677447.594248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677447.594256:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.594258:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.594259:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.594260:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677447.594302:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.594304:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.594305:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.594306:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:12.0:1619677447.594307:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.594308:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.594308:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.594309:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677447.594310:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.594310:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.594311:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.594312:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.594312:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.594312:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.594313:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.594314:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.594315:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.594426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.594432:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.594434:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.594436:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.594437:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.594439:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.594440:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.594442:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.594443:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.594444:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.594445:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.594449:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.594451:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.594454:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.594454:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.594455:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.594457:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.594457:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.594458:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.594459:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.594470:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.594472:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.594473:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.594475:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.594476:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:4.2:1619677447.594476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:13.0:1619677447.594478:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.594483:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.594486:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.594487:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.594489:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.594492:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.594493:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.594495:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.594496:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.594497:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.594498:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.594501:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.594502:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.594505:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.594506:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.594507:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.594508:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.594509:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.594510:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.594510:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.594523:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.594526:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.594527:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.594527:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.594529:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.594535:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.594537:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.594539:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.594540:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.594541:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.594541:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.594541:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.594666:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677447.594668:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677447.594669:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677447.594671:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.594674:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.594675:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:20.2:1619677447.594676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:17.0:1619677447.594676:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.594677:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.594678:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.594680:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677447.594683:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.594685:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677447.594687:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.594688:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:17.0:1619677447.594689:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677447.594691:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677447.594691:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677447.594692:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677447.594692:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:20.0:1619677447.594704:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.594706:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.594707:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.594709:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.594710:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.594712:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.594715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677447.594818:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.594821:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.594822:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.594824:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.594827:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.594828:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.594830:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.594831:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.594832:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.594833:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.594836:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.594837:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.594839:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.594840:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.594841:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.594842:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.594843:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.594843:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.594844:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.594848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.594855:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.594857:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.594858:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.594859:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.594859:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.594861:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.594952:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.594954:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.594955:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.594957:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.594960:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.594961:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.594963:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.594964:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.594965:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.594966:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.594969:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.594971:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.594974:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.594975:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.594975:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.594977:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.594978:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.594978:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.594979:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.595019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677447.595026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.595031:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.595033:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.595035:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.595035:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.595036:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.595036:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.595037:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.595038:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.595041:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.595042:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677447.595042:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.595043:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.595228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677447.595238:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.595242:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.595242:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:4.2:1619677447.595245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677447.595245:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.595247:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.595249:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.595251:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.595253:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.595255:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.595256:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.595262:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.595265:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.595268:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.595269:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.595270:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.595271:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.595272:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.595273:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.595274:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.595275:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677447.595277:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.595279:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.595280:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.595282:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.595283:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.595284:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.595285:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677447.595286:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677447.595287:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677447.595288:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:5.0:1619677447.595288:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:22.0:1619677447.595289:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.595289:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677447.595289:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.595290:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677447.595291:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.595291:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.595292:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.595297:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:13.0:1619677447.595300:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677447.595301:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:5.0:1619677447.595302:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.595303:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.595304:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.595304:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.595305:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.595306:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.595306:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.595308:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.595311:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.595312:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.595313:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.595314:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.595315:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.595390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.2:1619677447.595395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:24.0:1619677447.595398:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.0:1619677447.595400:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:24.0:1619677447.595401:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.595402:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677447.595403:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.595405:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:24.0:1619677447.595406:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.595407:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.595409:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.595410:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.595411:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677447.595412:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:24.0:1619677447.595413:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.595413:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.595415:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.595415:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.595416:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.595416:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.595417:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677447.595418:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:24.0:1619677447.595423:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.595423:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.595425:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:24.0:1619677447.595426:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.595428:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.595429:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:24.0:1619677447.595430:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.595430:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677447.595430:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:24.0:1619677447.595431:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.595432:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:24.0:1619677447.595433:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.595433:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:24.0:1619677447.595434:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.0:1619677447.595434:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.595434:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:24.0:1619677447.595435:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.595436:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.595437:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.595442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677447.595444:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.595447:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:17.0:1619677447.595448:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:18.0:1619677447.595449:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:17.0:1619677447.595450:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:8.0:1619677447.595450:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.595451:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:2.0:1619677447.595451:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.595452:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.595452:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677447.595452:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:2.0:1619677447.595453:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.595454:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.595454:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677447.595455:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.595456:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.595458:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.595459:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:2.0:1619677447.595459:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.595460:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677447.595461:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.595463:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:2.0:1619677447.595464:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.595465:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.595465:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.595466:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.595466:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:2.0:1619677447.595466:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.595468:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.595469:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.595470:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.595472:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.595472:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.595473:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.595473:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677447.595486:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.595488:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.595489:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.595491:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.595492:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.595494:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677447.595541:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.595543:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.595544:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.595544:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.595546:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.595550:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.595559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.595565:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.595567:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.595569:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.595570:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.595573:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.595574:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677447.595576:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.595577:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.595578:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.595579:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677447.595582:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.595584:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.595587:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.595587:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677447.595588:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.595590:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.595590:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.595591:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.595591:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677447.595604:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.595606:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.595607:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.595607:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.595609:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.595611:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.595661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.595694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:4.2:1619677447.595703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:9.2:1619677447.595731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.595739:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.595742:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.595743:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.595743:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.595745:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.595748:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.595750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.595756:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.595759:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.595760:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.595760:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.595760:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.595761:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.595814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:20.2:1619677447.595854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:18.0:1619677447.595870:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.595872:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.595873:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.595876:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.595878:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.595880:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.595881:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.595883:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.595884:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.595885:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.595889:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.595890:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.595893:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.595894:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.595894:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.595896:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.595896:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.595898:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.595899:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.595900:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677447.595900:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.595903:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:4.2:1619677447.595903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:17.0:1619677447.595904:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.595904:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.595906:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.595909:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:2.0:1619677447.595910:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:20.0:1619677447.595911:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677447.595912:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:20.0:1619677447.595913:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:2.0:1619677447.595913:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:20.0:1619677447.595914:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.595914:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:20.0:1619677447.595916:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.595917:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.595919:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.595951:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.595954:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.595955:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.595956:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.595957:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.595959:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.596036:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.596039:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.596039:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.596041:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.596044:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.596046:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.596047:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.596048:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.596049:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.596050:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677447.596054:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.596056:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.596059:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.596060:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.596060:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.596062:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.596063:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.596063:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.596064:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.596068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.596074:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.596077:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.596078:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.596078:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.596081:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.596084:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.596206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.596244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.596252:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.596254:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.596255:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.596255:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.596258:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.596261:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.596501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.596540:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (23)++
00000800:00000200:17.0:1619677447.596542:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (24)--
00000800:00000200:1.0:1619677447.596543:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (24)++
00000400:00000200:17.0:1619677447.596544:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.596544:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (24)--
00000400:00000200:1.0:1619677447.596544:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.596546:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.596546:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:20.2:1619677447.596548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677447.596550:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.596551:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.596551:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.596553:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.596555:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.596555:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:20.0:1619677447.596556:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.596557:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.596558:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.596559:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.596562:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.596562:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.596562:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.596563:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000400:00000200:5.0:1619677447.596563:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:20.0:1619677447.596565:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677447.596565:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:5.0:1619677447.596565:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.596565:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:20.0:1619677447.596566:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:1.0:1619677447.596566:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.596567:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.596567:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.596567:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.596567:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.596568:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677447.596568:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.0:1619677447.596569:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.596569:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:20.0:1619677447.596570:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:5.0:1619677447.596572:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.596574:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.596576:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.596577:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677447.596578:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.596579:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.596580:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.596582:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.596586:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.596587:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.596588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.596591:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:17.0:1619677447.596592:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677447.596593:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.596594:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677447.596594:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.0:1619677447.596595:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677447.596595:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.596596:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.596597:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.596597:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.596599:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.596600:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.596601:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.596602:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.596604:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.596605:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.596606:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.596609:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.596611:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.596611:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.596612:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.596613:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.596614:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.596615:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.596615:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.596617:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.596618:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.596618:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.596619:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.596630:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.596632:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.596633:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.596634:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:4.2:1619677447.596635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:1.0:1619677447.596635:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.596636:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.0:1619677447.596643:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.596645:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.596646:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.596648:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.596651:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.596653:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.596654:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.596655:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.596656:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.596657:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.596661:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.596662:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.596665:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.596666:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.596667:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.596668:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.596669:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.596669:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.596670:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.596682:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.596684:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.596685:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.596686:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.596688:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.596690:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:6.0:1619677447.596733:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.596736:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.596736:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.596738:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.596741:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.596743:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.596744:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.596745:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.596746:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.596747:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.596751:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.596753:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.596755:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.596756:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.596756:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.596758:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.596758:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:6.0:1619677447.596759:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.596759:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.596760:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.596760:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.596761:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.596763:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.596765:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.596767:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.596768:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.596769:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.596770:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.596772:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677447.596772:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.596774:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.596775:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.596775:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.596776:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:2.0:1619677447.596776:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.596777:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.596778:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.596778:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.596780:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.596780:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.596780:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.596781:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:18.0:1619677447.596782:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.596783:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677447.596784:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.596785:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.596785:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.596787:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.596788:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.596789:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.596790:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.596791:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.596791:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:18.0:1619677447.596792:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677447.596792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.596793:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.596794:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.596794:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.596795:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.596796:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.596797:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.596798:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:2.0:1619677447.596798:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.596799:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.596799:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.596800:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.596801:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.596801:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.596803:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.596804:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677447.596805:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.596807:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.596807:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.596809:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.596810:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.596812:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677447.596813:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726014] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.596814:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:18.0:1619677447.596814:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.596815:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.596815:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677447.596817:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.596818:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.596818:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.596819:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.596820:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:5.0:1619677447.596820:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.596822:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677447.596824:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.596825:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:18.0:1619677447.596825:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677447.596827:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.596828:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.596828:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.596829:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677447.596842:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.596846:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.596847:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.596848:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.596850:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.596852:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.596940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.596981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677447.596986:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.596989:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.596990:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.596991:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.596992:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.596994:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.597071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:12.0:1619677447.597115:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.597117:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:17.0:1619677447.597118:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:12.0:1619677447.597118:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.597119:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.597121:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:13.0:1619677447.597121:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.597122:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677447.597122:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.597123:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.597126:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.597128:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.597129:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.597131:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.597132:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.597133:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.597136:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.597138:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.597141:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.597142:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.597142:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.597144:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.597145:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.597146:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.597146:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.597157:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.597160:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.597160:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.597162:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.597163:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.597165:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.597471:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.597474:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.597475:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.597476:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.597479:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.597481:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.597482:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.597483:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.597484:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.597485:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.597488:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.597490:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.597493:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.597494:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.597495:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.597497:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.597497:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.597498:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.597498:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.597503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.597510:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.597513:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.597514:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.597514:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.597517:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.597520:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.597520:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.597523:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.597524:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.597526:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.597529:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.597530:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.597532:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.597533:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.597534:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.597535:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.597539:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.597541:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.597544:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.597545:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.597546:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677447.597547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:1.0:1619677447.597548:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.597548:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.597549:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:20.0:1619677447.597550:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:1.0:1619677447.597550:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.597551:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677447.597553:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.597556:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677447.597558:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.597560:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:5.0:1619677447.597560:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.597562:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.597562:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:20.0:1619677447.597564:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.597564:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:20.0:1619677447.597565:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.597565:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677447.597566:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:17.0:1619677447.597566:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:20.0:1619677447.597570:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.597573:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.597575:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.597576:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.597577:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.597579:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.597580:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.597580:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.597581:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677447.597585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.597593:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.597596:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.597597:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.597597:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.597600:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:9.2:1619677447.597601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.597604:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.597606:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.597608:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.597609:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.597609:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.597610:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.597610:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.597666:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.597669:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.597670:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.597672:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.597674:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.597676:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.597677:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.597678:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.597679:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.597681:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.597684:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.597686:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.597688:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.597689:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.597690:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.597691:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.597692:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.597692:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.597693:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.597706:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.597708:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.597709:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.597711:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.597712:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.597713:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.597740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.597766:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.597769:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.597769:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.597771:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.597774:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.597776:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.597777:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96726003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.597778:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.597779:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.597780:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.597783:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.597785:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.597786:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.597788:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:1.0:1619677447.597788:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.597789:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677447.597789:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.597789:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:6.0:1619677447.597790:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.597791:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.597791:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:6.0:1619677447.597792:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.597792:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.597793:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.597795:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.597804:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.597807:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.597808:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.597808:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.597810:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.597812:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:17.0:1619677447.598039:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.598042:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.598043:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.598044:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.598047:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.598048:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.598049:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96726002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.598050:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.598051:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.598052:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.598054:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677447.598055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:17.0:1619677447.598056:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.598057:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.598058:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.598058:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.598060:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.598060:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.598061:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.598061:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677447.598063:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.598067:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.598071:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.598073:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.598074:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:8.0:1619677447.598075:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.598076:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:8.0:1619677447.598077:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.598077:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.598078:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:8.0:1619677447.598079:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96726001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.598080:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.598081:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.598081:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:8.0:1619677447.598082:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.598083:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.598086:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.598088:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.598090:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.598091:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.598092:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.598094:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.598094:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.598095:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.598095:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.598107:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.598110:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.598110:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.598110:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.598112:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.598115:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:24.0:1619677447.598124:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.598126:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.598127:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.598128:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.598129:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:24.0:1619677447.598131:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.598131:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.598132:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.598132:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000400:00000200:24.0:1619677447.598133:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:1.0:1619677447.598133:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000400:00000200:24.0:1619677447.598134:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96726000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.598134:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:24.0:1619677447.598135:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.598136:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.598137:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.598140:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.598141:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.598144:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.598145:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.598146:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.598147:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.598148:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.598148:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.598149:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677447.598161:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.598163:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.598164:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.598166:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.598167:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.598169:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.598181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:12.0:1619677447.598228:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677447.598230:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677447.598231:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.598232:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.598234:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.598235:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.598237:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.598238:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.598239:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.598240:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.598241:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.598250:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.598253:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.598254:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.598254:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.598255:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.598256:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.598341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.598347:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.598350:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.598352:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.598353:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.598355:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.598356:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677447.598357:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.598359:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.598360:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.598361:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677447.598364:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.598366:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.598368:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.598369:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.598369:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.598371:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.598371:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.598372:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.598373:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677447.598384:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.598386:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.598387:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.598389:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.598390:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.598391:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.598505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.598525:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.598528:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.598528:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.598532:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.598534:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.598535:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.598537:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.598538:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.598539:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.598540:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.598544:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.598545:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.598548:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.598549:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.598550:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.598552:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.598552:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.598553:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.598553:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.598555:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.598558:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.598558:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.598559:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.598561:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.598563:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.598565:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.598568:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.598569:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.598570:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.598572:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:12.0:1619677447.598572:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.598573:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.598576:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.598577:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.598577:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.598578:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.598578:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.598703:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.598705:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.598706:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.598708:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.598710:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.598712:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.598713:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.598714:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.598715:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.598716:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.598720:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.598722:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.598725:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.598726:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.598726:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.598728:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.598728:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.598730:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.598731:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.598731:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.598741:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.598743:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.598744:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.598745:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677447.598862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.598868:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.598871:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.598873:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.598873:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.598875:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.598876:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677447.598878:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.598879:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.598880:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.598881:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677447.598884:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.598886:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.598888:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.598889:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.598890:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.598891:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.598892:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.598892:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.598893:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677447.598904:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.598906:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.598907:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.598909:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.598910:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.598911:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.598927:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.598929:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.598930:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.598931:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.598932:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.598932:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.2:1619677447.598949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677447.598992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677447.599072:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.599075:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.599076:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.599078:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.599080:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.599082:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.599083:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.599085:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.599086:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.599087:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.599090:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.599092:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.599095:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.599095:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.599096:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.599098:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.599098:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.599099:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.599099:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677447.599112:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.599114:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.599115:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.599117:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.599117:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.599119:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.599180:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.599183:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.599184:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.599186:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.599189:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.599191:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.599192:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.599193:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.599194:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.599195:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.599199:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.599201:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.599203:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.599205:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.599205:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.599207:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.599208:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.599209:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.599210:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.599210:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.599213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.599220:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677447.599221:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677447.599222:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.599223:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:17.0:1619677447.599223:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.599224:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677447.599224:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.599225:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677447.599226:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.599229:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677447.599312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:9.2:1619677447.599360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:17.0:1619677447.599362:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.599365:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.599365:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.599366:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.599368:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:17.0:1619677447.599368:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.599371:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:17.0:1619677447.599371:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:2.0:1619677447.599371:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.599372:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.599374:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.599377:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.599386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.599392:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.599395:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.599396:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.599397:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.599397:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.599398:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:18.0:1619677447.599408:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.599410:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.599411:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.599413:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.599415:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.599417:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.599418:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.599419:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.599420:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.599421:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.599425:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.599426:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.599429:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.599430:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.599431:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.599432:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.599433:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.599433:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.599434:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677447.599447:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.599449:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.599450:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.599450:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.599452:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.599454:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.599513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:4.2:1619677447.599553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.599561:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677447.599563:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:24.0:1619677447.599564:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:20.0:1619677447.599565:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:8.0:1619677447.599565:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.599566:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.599566:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.599568:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.599569:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.599569:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:24.0:1619677447.599571:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.0:1619677447.599571:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:24.0:1619677447.599573:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.599575:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.599576:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.599577:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.599581:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.599581:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:24.0:1619677447.599582:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.599583:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677447.599584:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.599585:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:24.0:1619677447.599586:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.599586:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.599586:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:5.0:1619677447.599586:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.599586:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:24.0:1619677447.599587:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.599587:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.599588:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:6.0:1619677447.599588:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.599589:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.599590:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.599590:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.599591:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677447.599591:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.599593:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.599594:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.599596:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.599597:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.599598:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.599601:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.599602:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677447.599603:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.599605:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.599606:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.599606:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.599606:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.599607:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:17.0:1619677447.599608:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.599608:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.599609:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.599610:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.599610:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677447.599611:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677447.599611:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677447.599620:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.599622:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.599623:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.599623:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.599625:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.599625:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:9.2:1619677447.599651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.599659:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.599661:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.599663:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.599663:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.599665:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.599668:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.599727:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.599729:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.599730:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.599732:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.599734:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.599735:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.599736:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.599737:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.599738:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.599739:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.599743:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.599744:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.599747:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.599748:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.599748:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.599750:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.599750:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.599751:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.599751:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.599764:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.599766:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.599767:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.599767:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.599769:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.599772:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.599817:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.599819:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.599820:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.599822:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.599825:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.599826:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.599828:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.599829:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.599830:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.599831:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.599834:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.599835:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.599838:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.599838:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.599839:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.599840:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.599841:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.599842:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.599843:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.599843:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.599855:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:5.0:1619677447.599856:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:18.0:1619677447.599857:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.599857:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.599858:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677447.599858:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.599859:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.599861:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.599861:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.599863:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.600100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:9.2:1619677447.600112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.600119:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.600122:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:20.2:1619677447.600123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677447.600123:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.600123:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.600126:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:2.0:1619677447.600126:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.600128:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677447.600128:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:20.0:1619677447.600131:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.600131:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.600133:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.600135:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677447.600136:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.600137:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.600138:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.600139:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677447.600143:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.600145:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677447.600146:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.600147:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.600148:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677447.600148:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.600149:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:5.0:1619677447.600149:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.600151:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.600151:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:5.0:1619677447.600151:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.600152:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.600153:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677447.600154:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.600154:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:9.2:1619677447.600156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.600164:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.600167:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.600168:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.600168:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.600170:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.600172:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.600261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.600268:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.600271:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.600272:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.600274:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.600277:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.600278:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.600279:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.600280:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.600281:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.600282:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.600285:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.600287:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.600289:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.600290:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.600291:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.600292:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.600303:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.600304:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.600305:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677447.600306:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677447.600306:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677447.600309:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.600310:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.600310:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.600312:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.600312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.600315:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:24.0:1619677447.600319:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.600322:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.600323:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.600323:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.600342:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.600345:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.600346:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.600348:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.600351:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.600352:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.600354:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.600355:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.600356:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.600357:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.600361:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.600363:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.600365:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.600366:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.600366:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.600368:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.600369:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.600369:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.600370:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.600402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.600410:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.600412:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.600413:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.600414:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.600415:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.600418:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:1.0:1619677447.600442:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.600445:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.600445:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.600447:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.600450:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.600451:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.600452:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.600454:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.600454:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.600456:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:20.2:1619677447.600459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:1.0:1619677447.600459:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.600460:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.600463:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.600464:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:5.0:1619677447.600465:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:1.0:1619677447.600465:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.600466:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:5.0:1619677447.600467:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:1.0:1619677447.600467:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.600467:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.600468:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677447.600470:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.600470:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:9.2:1619677447.600472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:5.0:1619677447.600472:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.600474:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.600475:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.600476:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.600477:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.600478:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677447.600479:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.600481:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.600482:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:5.0:1619677447.600482:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000400:00000200:24.0:1619677447.600483:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.600484:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:24.0:1619677447.600485:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.600487:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.600488:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.0:1619677447.600488:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.600488:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.600490:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.600491:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.600492:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.600493:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:4.2:1619677447.600524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.600572:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.600575:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.600576:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.600576:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.600578:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.600581:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.600684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.0:1619677447.600731:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.600733:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.600734:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.600736:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.600737:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.600739:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.600893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.600898:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.600900:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.600902:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.600903:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.600905:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.600906:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677447.600908:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.600909:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.600910:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.600911:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677447.600914:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.600916:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.600918:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.600919:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677447.600920:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.600921:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.600922:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.600923:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.600924:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.600925:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.600930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.600932:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.600934:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.600935:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677447.600935:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:20.0:1619677447.600936:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677447.600938:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.600939:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.600939:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.600941:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.600943:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677447.600955:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.600958:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.600959:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.600961:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.600963:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.600964:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.600966:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.600967:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.600968:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.600969:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.600973:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.600974:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.600977:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.600978:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.600979:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.600980:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.600981:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.600981:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.600982:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677447.600993:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.600996:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.600996:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.600998:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.600999:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.601000:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.601118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.601125:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.601128:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.601129:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.601130:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.601133:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.601134:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.601136:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.601137:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.601138:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.601139:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.601143:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.601144:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.601147:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.601148:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.601149:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.601150:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.601151:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.601151:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.601152:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.601165:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.601167:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.601168:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.601168:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.601170:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.601173:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.601180:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.601183:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.601184:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.601186:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.601188:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.601190:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.601191:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.601192:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.601193:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.601194:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.601198:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677447.601199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.601200:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.601201:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.601202:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.601203:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.601204:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.601205:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.601206:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.601206:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.601210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.601217:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.601219:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.601220:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.601221:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.601222:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.601224:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.601227:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.601229:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.601230:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.601232:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.601235:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.601236:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.601237:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.601239:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.601239:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.601241:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:4.2:1619677447.601241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677447.601244:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.601246:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.601249:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.601249:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.601250:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.601252:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.601252:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.601253:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.601253:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.601258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.601264:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.601267:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.601268:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.601268:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.601270:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.601273:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:1.0:1619677447.601392:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.601395:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.601396:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.601396:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.601398:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.601401:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.601444:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.601447:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.601448:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.601449:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.601451:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.601453:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677447.601454:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.601455:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.601456:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.601457:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677447.601460:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.601462:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.601463:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.601464:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.601465:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.601466:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.601467:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.601468:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.601469:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.601469:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.601480:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.601482:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.601483:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.601484:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.601492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.601499:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.601501:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.601502:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.601503:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.601504:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.601505:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.601516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.601524:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.601526:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.601527:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.601527:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.601527:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.601528:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.601595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.601611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.601616:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.601619:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.601621:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.601622:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.601624:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.601626:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.601627:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96725959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.601629:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.601630:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.601631:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.601636:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.601638:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.601641:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.601642:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.601643:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.601644:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.601645:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.601646:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.601646:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677447.601746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.601753:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.601756:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.601757:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.601759:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.601761:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.601763:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.601764:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.601766:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.601767:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.601768:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.601771:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.601773:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.601776:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.601777:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.601777:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.601779:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.601779:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.601780:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.601780:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.601793:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.601795:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.601796:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.601797:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.601799:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.601801:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:2.0:1619677447.602024:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.602027:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.602028:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.602030:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.602031:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.602033:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.602034:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.602036:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.602037:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.602037:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.602040:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.602042:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.602045:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.602045:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.602046:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.602048:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.602048:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.602049:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.602050:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.602050:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.602061:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:20.0:1619677447.602062:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677447.602063:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:20.0:1619677447.602064:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:18.0:1619677447.602064:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:20.0:1619677447.602065:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.602065:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:20.0:1619677447.602067:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.602068:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.602070:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:6.0:1619677447.602084:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.602087:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.602088:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677447.602088:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:6.0:1619677447.602089:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.602091:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.602092:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:6.0:1619677447.602092:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.602093:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.602094:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.602095:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.602096:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.602097:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.602097:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.602098:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.602099:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677447.602100:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.602102:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.602102:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.602103:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.602104:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:6.0:1619677447.602105:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.2:1619677447.602106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:6.0:1619677447.602108:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.602108:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.602109:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677447.602109:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.602111:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.602111:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677447.602111:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:6.0:1619677447.602112:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.602112:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.602114:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.602115:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677447.602116:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677447.602118:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.602118:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.602119:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.602119:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:1.0:1619677447.602124:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.602127:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.602128:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.602128:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.602130:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.602131:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677447.602132:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:12.0:1619677447.602134:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.602136:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.602137:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.602139:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.602140:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:17.0:1619677447.602144:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.602146:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.602147:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.602148:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.602149:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.602149:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.602150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677447.602152:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.602155:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.602156:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.602158:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.602159:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.602160:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.602170:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.602172:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.602173:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.602174:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.602174:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.602175:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.602177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.602222:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.602225:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.602226:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.602228:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.602228:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.602231:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.602241:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.602244:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.602245:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.602246:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.602246:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.602248:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.602277:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.602280:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.602280:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.602282:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.602285:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.602286:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.602288:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.602289:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.602290:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.602291:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.602299:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.602301:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.602303:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.602304:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.602304:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.602306:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.602306:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.602307:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.602308:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677447.602321:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.602323:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.602323:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.602324:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.602325:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.602327:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.602437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.602485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677447.602486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:22.0:1619677447.602488:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:4.2:1619677447.602488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.602491:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.602492:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.602492:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.602493:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677447.602493:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:22.0:1619677447.602494:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.602495:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:18.0:1619677447.602496:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:22.0:1619677447.602497:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:18.0:1619677447.602497:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.602497:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.602498:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.602498:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.602499:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.602500:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.602502:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:18.0:1619677447.602504:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:5.0:1619677447.602504:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.602505:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.602506:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.602507:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.602510:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.602512:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.602515:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.602515:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.602516:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.602518:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.602518:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.602520:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.602521:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.602522:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.2:1619677447.602526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.602529:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.602532:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.602532:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:1.0:1619677447.602532:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677447.602533:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:1.0:1619677447.602534:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.602535:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.602536:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.602537:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677447.602540:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:24.0:1619677447.602684:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.602687:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.602688:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.602690:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.602693:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.602694:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.602696:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.602697:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.602698:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.602699:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.602702:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.602704:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.602706:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.602707:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.602708:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.602709:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.602710:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.602711:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.602712:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.602713:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.602725:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.602727:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.602727:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.602728:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.602733:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.602735:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.602736:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.602737:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.602739:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.602740:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.602800:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.602803:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.602803:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.602805:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.602807:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.602808:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.602892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:20.2:1619677447.602923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.602926:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.602928:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.602930:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677447.602933:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:5.0:1619677447.602933:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.602934:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.602936:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.602937:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.602938:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.602939:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.602940:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.602942:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.602942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:20.0:1619677447.602943:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.602945:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.602946:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.602949:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:20.0:1619677447.602950:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.602950:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.602952:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.602952:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.602953:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.602953:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.602954:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.602955:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.602955:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.602956:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.602957:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.602959:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.602959:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.602960:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.602961:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.602982:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.602985:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.602986:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.602987:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.602989:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.602990:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.602990:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.602993:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.602994:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.602994:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.602996:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.603000:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.603007:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.603010:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.603011:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.603011:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.603013:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.603013:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.603243:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.603243:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:13.0:1619677447.603246:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677447.603246:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.603247:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:13.0:1619677447.603247:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.603250:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.603250:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677447.603250:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.603252:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677447.603253:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.603255:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.603256:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.603257:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.603258:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.603259:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.603262:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.603264:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.603265:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.603268:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.603268:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677447.603269:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677447.603269:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.603269:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:17.0:1619677447.603272:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:13.0:1619677447.603272:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:2.0:1619677447.603272:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677447.603273:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:2.0:1619677447.603273:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:1.0:1619677447.603273:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.603274:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.603274:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677447.603274:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:12.0:1619677447.603275:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.603275:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:17.0:1619677447.603277:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.603278:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.603279:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.603281:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.603282:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.603283:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.603285:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.603287:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.603289:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.603290:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.603290:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.603292:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.603292:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677447.603296:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677447.603300:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.603301:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.603303:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677447.603304:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.603304:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.603304:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677447.603305:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.603306:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:13.0:1619677447.603306:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.603307:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.603309:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.603309:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.603310:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.603311:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.603311:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.603313:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.603313:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.603314:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677447.603314:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.603316:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.603317:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.603319:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.603321:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.603323:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.603324:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.603326:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.603327:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.603328:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:6.0:1619677447.603328:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677447.603331:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677447.603332:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.603332:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.603333:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.603333:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.603334:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:6.0:1619677447.603334:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.603337:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.603338:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.603339:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.603340:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.603341:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.603342:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.603343:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.603344:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.603354:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:5.0:1619677447.603355:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:22.0:1619677447.603356:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.603357:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.0:1619677447.603357:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677447.603358:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:5.0:1619677447.603358:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.603360:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.603361:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.603363:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.603370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:4.2:1619677447.603376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.603422:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (23)++
00000800:00000200:20.0:1619677447.603424:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (24)--
00000400:00000200:20.0:1619677447.603425:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.603427:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.603427:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (23)++
00000800:00000200:20.2:1619677447.603429:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.603429:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (24)--
00000400:00000200:5.0:1619677447.603429:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.603431:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.603433:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.603434:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.603434:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.603435:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.603437:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.603439:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.603440:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.603442:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.603443:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.603444:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.603446:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.603447:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.603448:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.603451:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.603453:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.603455:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.603456:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.603457:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.603458:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.603459:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.603459:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.603460:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.603473:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.603475:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.603476:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.603477:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.603479:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.603481:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.603695:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.603696:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.603697:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.603698:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.603699:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.603699:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.603699:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:13.0:1619677447.603701:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.603701:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.603702:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.603703:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.603704:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.603788:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.603791:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.603791:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.603794:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.603797:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.603798:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.603800:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.603801:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.603802:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.603803:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.603806:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.603808:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.603812:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.603812:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.603813:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.603815:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.603815:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.603816:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.603817:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.603827:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.603830:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.603831:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.603833:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.603834:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.603836:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.603898:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.603901:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.603902:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.603904:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.603906:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.603908:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677447.603909:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.603910:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.603911:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.603912:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677447.603915:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.603916:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.603919:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.603920:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677447.603920:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677447.603922:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.603922:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.603923:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.603924:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.603945:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.603947:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.603948:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.603949:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.603951:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.603952:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.604008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677447.604039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.604042:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677447.604044:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677447.604045:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.604047:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.604047:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.604047:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:5.0:1619677447.604049:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:1.0:1619677447.604049:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.604050:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.604051:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.604052:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677447.604053:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.604053:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.604055:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.604056:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.604057:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.604057:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.604058:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.604059:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677447.604059:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.604060:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.604062:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.604063:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.604065:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.604065:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677447.604066:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.604066:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677447.604068:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677447.604068:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677447.604071:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.604072:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:1.0:1619677447.604072:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.604073:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677447.604073:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.604074:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.0:1619677447.604075:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.604075:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677447.604076:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677447.604076:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:20.0:1619677447.604077:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677447.604077:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677447.604078:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:1.0:1619677447.604078:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.604078:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677447.604079:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.604084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.604087:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.604089:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.604089:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:20.0:1619677447.604090:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.604090:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:20.0:1619677447.604091:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.604091:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.604093:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.604093:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.604094:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.604095:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.604097:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.604097:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.604098:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.604099:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.604099:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:13.0:1619677447.604100:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.604101:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.604101:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.604102:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.604104:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.604105:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.604110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.604113:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.604115:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.604115:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.604115:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.604116:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.604117:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.604117:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.604117:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.604118:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.604118:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.604119:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.604120:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.604198:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.604201:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.604202:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.604204:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.604206:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.604207:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.604209:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.604210:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.604211:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.604212:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.604216:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.604217:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.604220:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.604221:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.604222:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.604223:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.604224:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.604225:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.604225:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.604458:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.604460:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.604462:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.604463:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.604466:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.604468:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.604469:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.604470:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.604471:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.604472:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.604476:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.604478:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.604480:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.604481:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.604481:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.604483:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.604483:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.604485:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.604486:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.604486:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.0:1619677447.604497:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:18.0:1619677447.604497:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.604499:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:20.0:1619677447.604500:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.604500:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.604500:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.604501:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:20.0:1619677447.604502:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.604503:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.604504:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.604513:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.604515:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.604516:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.604517:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.604518:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.604519:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:2.0:1619677447.604540:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.604542:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.604544:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.604545:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.604547:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.604549:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677447.604550:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725923] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.604552:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.604553:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.604553:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677447.604557:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.604559:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677447.604561:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.604561:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677447.604562:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677447.604564:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677447.604564:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.604565:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.604565:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677447.604579:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.604581:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.604582:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.604582:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.604585:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.604587:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.604600:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.604602:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.604603:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.604604:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.604604:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.604605:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.604774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:13.0:1619677447.604815:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677447.604818:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677447.604820:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.604821:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:13.0:1619677447.604822:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.604822:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:5.0:1619677447.604824:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.604824:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.604825:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.0:1619677447.604826:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:13.0:1619677447.604826:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96725921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.604827:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:13.0:1619677447.604828:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677447.604829:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677447.604830:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.604830:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:13.0:1619677447.604834:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.604836:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677447.604839:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.604839:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:13.0:1619677447.604840:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677447.604842:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:13.0:1619677447.604842:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677447.604843:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677447.604844:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:17.0:1619677447.604856:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.604859:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.604860:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.604860:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.604862:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.604865:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:9.2:1619677447.605112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.605120:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.605123:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.605123:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.605124:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.605126:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.605129:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.605130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.605137:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.605140:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.605141:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.605141:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.605142:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.605142:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.605244:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.605246:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.605247:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.605249:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.605252:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.605253:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.605254:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.605256:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.605257:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.605258:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.605261:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.605262:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.605265:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.605266:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.605266:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.605268:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.605269:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.605269:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.605270:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:4.2:1619677447.605281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.605282:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.605286:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.605287:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.605289:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.605290:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.605291:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.605313:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677447.605314:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677447.605317:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.605317:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:1.0:1619677447.605317:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:22.0:1619677447.605318:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.605321:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.605322:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.605324:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.605324:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.605326:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.605327:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.605327:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.605329:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.605330:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.605331:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.605332:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677447.605332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677447.605332:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.605334:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677447.605334:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:6.0:1619677447.605340:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.605341:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.605344:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677447.605344:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:6.0:1619677447.605347:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.605348:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:1.0:1619677447.605348:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.605349:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:1.0:1619677447.605349:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.605350:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:6.0:1619677447.605351:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.605352:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677447.605352:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:6.0:1619677447.605353:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677447.605353:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677447.605354:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677447.605354:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.605355:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.605358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:9.2:1619677447.605364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677447.605366:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.605369:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.605370:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.605370:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.605371:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:8.0:1619677447.605372:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.605374:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:8.0:1619677447.605375:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:2.0:1619677447.605375:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.605376:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:9.2:1619677447.605377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000400:00000200:2.0:1619677447.605378:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.605382:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:24.0:1619677447.605384:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.605387:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.605387:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.605387:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:20.2:1619677447.605388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:24.0:1619677447.605389:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.605389:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.0:1619677447.605394:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.605396:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.605399:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.605399:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.605401:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.605402:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677447.605404:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.605405:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.605406:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.605407:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677447.605411:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.605416:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.605418:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.605419:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677447.605419:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.605420:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.605422:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:5.0:1619677447.605422:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.605422:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677447.605422:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677447.605423:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:18.0:1619677447.605424:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.605424:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677447.605427:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.605428:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.605429:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.605431:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.605432:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.605433:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.605435:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:18.0:1619677447.605435:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.605437:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:18.0:1619677447.605437:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.605438:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.605440:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.605440:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.605441:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.605441:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.605442:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.605443:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677447.605444:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.605444:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.605445:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.605446:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.605450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.605457:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.605459:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.605460:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.605461:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.605462:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.605464:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.605532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677447.605576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677447.605580:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.605583:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.605584:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.605585:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.605586:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.605588:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.605679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.605686:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.605688:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.605690:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.605691:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.605694:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.605695:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.605697:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.605698:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.605699:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.605700:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.605702:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.605704:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.605707:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.605708:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.605708:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.605710:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.605710:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.605711:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.605712:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.605724:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.605727:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.605727:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.605728:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.605730:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.605733:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.605742:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.605745:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.605746:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.605746:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.605747:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.605747:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677447.605754:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.605757:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:17.0:1619677447.605758:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677447.605758:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.605760:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.605761:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.605762:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677447.605764:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.605764:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.605766:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.605766:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.605767:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.605768:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.605769:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.605770:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.605770:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.605771:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.605772:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.605772:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.605774:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.605776:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.605778:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.605781:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.605782:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.605783:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.605783:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.605783:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.605785:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.605786:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.605786:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677447.605786:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.605787:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677447.605787:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.605788:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677447.605790:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:17.0:1619677447.605790:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.605791:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.605791:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677447.605901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:9.2:1619677447.605937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.605945:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.605948:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.605948:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.605949:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.605951:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.605954:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677447.605954:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677447.605957:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677447.605959:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.605961:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677447.605962:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.605964:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677447.605966:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96725906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677447.605967:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677447.605968:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677447.605969:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677447.605972:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.605973:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677447.605976:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.605977:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:12.0:1619677447.605977:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677447.605979:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677447.605979:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677447.605981:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677447.605982:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677447.605983:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:1.0:1619677447.605992:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.605995:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.605996:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:13.0:1619677447.605996:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000400:00000200:1.0:1619677447.605996:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.605998:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677447.605999:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:1.0:1619677447.605999:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.606000:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:1.0:1619677447.606002:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.606184:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.606187:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.606187:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.606190:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.606191:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.606192:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.606194:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.606195:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.606196:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.606197:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.606200:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.606201:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.606203:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.606204:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.606205:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.606206:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.606207:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.606208:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.606208:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.606212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.606212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.606220:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.606222:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.606222:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.606224:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.606225:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.606227:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.606296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.606299:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.606301:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.606303:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.606305:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.606305:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.606307:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.606308:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.606310:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.606311:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.606312:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.606315:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.606317:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.606320:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.606321:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.606321:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.606323:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.606324:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.606325:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.606326:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677447.606460:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.606463:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.606463:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.2:1619677447.606464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:17.0:1619677447.606465:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.606468:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.606469:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:5.0:1619677447.606470:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:17.0:1619677447.606471:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.606472:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.606473:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.606473:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.606475:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:17.0:1619677447.606475:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.606476:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.606478:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.606479:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.606480:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.606481:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.606482:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.606483:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.606484:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.606484:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677447.606484:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.606485:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.606486:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:5.0:1619677447.606486:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.606487:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.606488:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.606490:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.606490:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677447.606490:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.606491:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.606494:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.606495:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.606500:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.606502:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.606502:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:1.0:1619677447.606502:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677447.606503:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:1.0:1619677447.606503:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.606504:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:5.0:1619677447.606504:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:1.0:1619677447.606504:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.606505:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.606506:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.606507:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.606508:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.606509:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.606510:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.606549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:6.0:1619677447.606550:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.606553:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.606554:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677447.606556:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:6.0:1619677447.606556:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.606558:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.606559:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.606560:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677447.606560:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.606561:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.606563:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.606563:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.606564:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.606565:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677447.606566:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.606567:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.606569:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96725899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.606569:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.606570:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.606572:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.606572:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:24.0:1619677447.606573:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.606574:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.606575:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.606576:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:24.0:1619677447.606577:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.606578:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.606578:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677447.606579:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.606579:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.606580:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.606581:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677447.606582:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.606583:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.606584:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:9.2:1619677447.606584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:24.0:1619677447.606585:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.606586:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.606587:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.606588:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.606589:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677447.606591:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677447.606591:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677447.606593:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.606593:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.606594:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.606594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677447.606594:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.606595:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.606596:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.606597:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.606600:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677447.606600:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:17.0:1619677447.606602:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677447.606602:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.606603:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.606604:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.606604:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.606605:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677447.606606:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.606606:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.606608:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677447.606710:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.606713:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.606714:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.606714:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.606716:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.606720:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.606838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.0:1619677447.606885:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.606887:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.606888:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.606890:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.606891:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.606893:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.606903:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.606905:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.606906:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.606907:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.606908:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.606909:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.607002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.607036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:9.2:1619677447.607037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.607043:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.607045:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:18.0:1619677447.607045:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677447.607046:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677447.607048:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.607048:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.607049:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.607049:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:22.0:1619677447.607050:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.607052:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.607052:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.607053:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.607055:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.607055:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:22.0:1619677447.607056:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.607057:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:9.2:1619677447.607058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:22.0:1619677447.607060:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.607062:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.607064:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.607065:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.607065:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.607066:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677447.607066:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677447.607068:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.607069:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (24)--
00000800:00000200:22.0:1619677447.607069:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:1.0:1619677447.607069:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:22.0:1619677447.607070:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:18.0:1619677447.607070:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:17.0:1619677447.607070:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677447.607070:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (23)++
00000800:00000200:22.0:1619677447.607071:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:2.0:1619677447.607071:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:22.0:1619677447.607072:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:2.0:1619677447.607072:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:1.0:1619677447.607072:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677447.607074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677447.607074:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.607076:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.2:1619677447.607077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (21)++
00000400:00000200:1.0:1619677447.607077:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.607078:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.607079:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.607080:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.607082:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677447.607082:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000400:00000200:1.0:1619677447.607083:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.607084:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:18.0:1619677447.607085:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:6.0:1619677447.607085:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:1.0:1619677447.607085:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.607086:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.607086:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:2.0:1619677447.607086:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677447.607087:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.607087:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:18.0:1619677447.607088:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.607088:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.607090:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.607090:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.607091:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.607091:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.607092:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677447.607103:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.607105:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.607106:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.607106:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.607108:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.607109:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.607122:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.607125:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.607125:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.607125:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.607126:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.607126:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.607196:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.607199:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.607200:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.607200:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.607203:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.607205:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:24.0:1619677447.607223:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.607225:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.607227:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.607228:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.607231:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.607233:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.607234:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.607235:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.607237:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.607237:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.607241:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.607242:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.607245:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.607246:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.607247:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.607249:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.607249:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.607250:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.607250:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.607254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.607261:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.607264:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.607265:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.607265:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.607268:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.607271:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.607363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.607369:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.607372:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.607374:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.607375:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.607377:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.607378:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.607380:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.607381:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.607382:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.607383:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.607386:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.607388:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.607392:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.607392:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.607393:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:4.2:1619677447.607394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.607395:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.607395:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.607396:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.607397:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.607409:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.607412:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.607413:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.607415:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.607416:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.607418:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.607423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.607426:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.607428:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.607430:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.607431:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.607433:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.607435:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677447.607436:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.607438:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.607439:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.607440:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.607441:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:20.0:1619677447.607443:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.607444:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.607444:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.607445:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.607445:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:22.0:1619677447.607447:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.607450:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.607450:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.607452:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.607453:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677447.607454:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.607456:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.607456:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.607457:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.607457:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:1.0:1619677447.607471:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.607473:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.607474:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.607474:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.607477:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.607479:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:12.0:1619677447.607488:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.607491:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.607492:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.607493:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.607495:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.607496:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677447.607514:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:4.2:1619677447.607515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:18.0:1619677447.607517:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.607518:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.607520:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.607521:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.607522:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:8.0:1619677447.607522:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:18.0:1619677447.607524:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.607525:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.607526:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:18.0:1619677447.607526:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.607527:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.607529:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677447.607529:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.607532:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.607532:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.607534:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:8.0:1619677447.607534:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.607536:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96725885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.607537:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.607538:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.607538:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:8.0:1619677447.607538:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.607539:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.607540:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000400:00000200:8.0:1619677447.607540:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.607541:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.607541:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.607542:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:8.0:1619677447.607543:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.607545:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.2:1619677447.607546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677447.607548:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.607549:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.607549:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.607551:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.607552:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.607552:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.607553:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677447.607553:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.607555:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.607556:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.607557:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.607558:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.607560:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:17.0:1619677447.607565:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.607567:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.607568:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.607569:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.607571:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.607573:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.607659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.607697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.607733:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.607736:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.607737:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.607737:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.607740:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.607742:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.607751:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.607753:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.607754:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.607755:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.607756:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.607756:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.607868:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.607871:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.607871:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.607873:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.607875:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.607877:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.607878:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.607879:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.607880:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.607882:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.607885:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.607887:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.607890:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.607891:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.607892:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.607894:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.607894:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.607895:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.607895:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.607907:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.607910:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.607910:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.607912:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.607913:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.607915:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.608005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.608024:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.608027:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.608027:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.608033:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.608034:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.608036:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.608037:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.608038:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.608040:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.608041:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.608045:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677447.608045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.608047:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.608052:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.608053:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.608054:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.608054:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.608055:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.608057:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.608057:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:2.0:1619677447.608057:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.608059:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.608060:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000400:00000200:2.0:1619677447.608060:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.608061:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.608062:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.608064:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677447.608069:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (25)++
00000800:00000200:18.0:1619677447.608071:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:12.0:1619677447.608071:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (25)--
00000400:00000200:12.0:1619677447.608072:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.608074:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:12.0:1619677447.608074:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.608075:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:13.0:1619677447.608076:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)++
00000800:00000200:2.0:1619677447.608076:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677447.608077:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (25)--
00000400:00000200:13.0:1619677447.608077:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.608077:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (25)--
00000800:00000200:12.0:1619677447.608078:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000400:00000200:13.0:1619677447.608079:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.608079:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677447.608080:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.608081:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.608082:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.608083:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.608084:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.608091:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.608094:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.608095:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.608095:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.608096:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.608097:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.608143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:5.0:1619677447.608187:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.608190:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.608191:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.608193:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.2:1619677447.608194:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677447.608196:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.608197:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.608239:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.608241:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.608242:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.608244:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.608245:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.608247:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.608258:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.608261:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.608262:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.608262:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.608263:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.608264:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.608347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.608349:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:4.2:1619677447.608352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.608354:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.608356:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.608356:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.608359:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.608361:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.608362:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.608364:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.608365:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.608366:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.608370:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.608373:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.608376:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.608377:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.608378:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.608380:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.608380:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.608382:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.608383:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.608383:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.608388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677447.608393:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.608393:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677447.608396:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677447.608396:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:12.0:1619677447.608397:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.608397:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.608397:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:12.0:1619677447.608399:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.608401:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.608402:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.608402:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.608405:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.608406:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.608406:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.608408:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.608412:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.608424:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.608426:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.608427:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.608429:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.608430:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.608431:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.608593:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.608595:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.608596:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.608598:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.608601:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.608603:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.608604:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.608605:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.608606:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.608607:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.608611:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.608612:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.608614:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.608615:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.608616:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.608617:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.608618:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.608618:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.608619:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.608623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.608630:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.608632:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.608633:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.608634:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.608635:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.608636:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.608641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.608649:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.608651:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.608652:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.608652:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.608652:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.608653:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.608776:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.608778:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.608779:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.608781:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.608783:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.608785:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.608786:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.608787:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.608788:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.608789:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677447.608792:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677447.608793:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.608795:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677447.608795:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.608796:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.608798:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.608799:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:17.0:1619677447.608799:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.608800:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.608801:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.608802:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:17.0:1619677447.608802:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.608803:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.608803:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:17.0:1619677447.608803:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677447.608805:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.608806:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.608807:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.608808:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677447.608811:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.608813:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677447.608816:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.608817:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677447.608817:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:17.0:1619677447.608818:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677447.608819:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677447.608819:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:17.0:1619677447.608820:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.608820:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677447.608820:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.608821:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:5.0:1619677447.608821:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.608822:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.608824:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:9.2:1619677447.608825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.608832:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.608835:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.608836:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.608837:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.608839:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.608842:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677447.608882:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.608884:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.608885:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.608886:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.608888:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.608889:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.608903:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.608906:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.608907:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.608907:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.608908:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.608909:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.608944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.608947:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.608950:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.608952:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.608953:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.608955:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.608956:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.608957:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.608959:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.608960:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.608961:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.608964:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.608966:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.608968:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.608969:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.608970:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.608972:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.608972:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.608974:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.608974:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.608985:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.608987:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.608988:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.608989:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.608991:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.608992:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.609079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.609125:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.609128:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.609129:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.609130:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.609132:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.609134:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.609308:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.609311:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.609312:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.609314:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.609316:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.609317:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.609328:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.609331:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.609332:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.609334:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.609336:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.609338:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.609339:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.609340:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.609341:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.609342:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.609346:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.609348:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.609351:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.609352:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.609353:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.609354:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.609355:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.609356:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.609356:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.609367:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.609369:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.609370:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.609372:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.609374:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.609374:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.609407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.609415:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.609418:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.609419:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.609420:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.609423:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.609424:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.609426:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.609427:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.609428:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.609429:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.609432:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.609433:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.609436:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.609437:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.609438:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.609439:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.609440:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.609440:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.609441:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.609453:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677447.609454:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677447.609456:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:1.0:1619677447.609456:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.609457:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.609457:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677447.609458:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.609459:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.609460:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.609462:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.609464:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.609465:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.609466:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.609466:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:22.0:1619677447.609467:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.609467:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:22.0:1619677447.609469:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.609470:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.609471:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.609471:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.609472:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.609472:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:22.0:1619677447.609473:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.609475:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.609478:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.609479:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.609480:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.609481:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.609482:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.609482:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.609483:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677447.609493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677447.609495:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.609497:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.2:1619677447.609498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:17.0:1619677447.609498:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.609498:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.609499:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:17.0:1619677447.609500:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.609501:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:17.0:1619677447.609503:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677447.609504:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.609504:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.609507:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.609509:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.609511:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.609513:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.609514:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.609515:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.609519:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.609521:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.609525:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.609526:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.609526:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.609528:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.609529:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.609530:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.609530:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.609540:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.2:1619677447.609542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677447.609543:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.609544:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.609546:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.609547:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.609548:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.609594:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.609596:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.609598:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.609599:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.609602:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.609603:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.609604:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.609606:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.609607:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.609608:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.609611:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.609613:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.609616:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.609617:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.609617:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.609619:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677447.609619:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677447.609620:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.609621:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.609621:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:1.0:1619677447.609621:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.609622:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.609624:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677447.609626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677447.609626:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.609628:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.609629:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.609630:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.609631:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.609632:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:18.0:1619677447.609633:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:1.0:1619677447.609635:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.609636:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.609637:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.609637:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677447.609637:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:18.0:1619677447.609639:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.609640:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.609641:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.609641:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:18.0:1619677447.609643:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.609643:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.609644:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.609645:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.609646:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.609646:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.609649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.609655:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:17.0:1619677447.609656:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.609658:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.0:1619677447.609658:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.609659:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.609659:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677447.609660:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.609661:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677447.609662:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.609665:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:1.0:1619677447.610055:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.610057:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.610058:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.610060:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.610063:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.610065:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.610066:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.610069:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.610070:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.610071:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.610075:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.610078:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.610082:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.610082:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.610083:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.610085:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.610085:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.610086:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.610086:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.610097:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:9.2:1619677447.610097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.610099:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.610100:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.610100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:12.0:1619677447.610102:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.610103:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.610105:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.610105:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.610108:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.610109:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.610109:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.610112:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.610115:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.610141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.2:1619677447.610219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.0:1619677447.610264:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.610267:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.610268:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.610270:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.610270:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677447.610271:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.610273:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677447.610273:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.610273:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.610275:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.610278:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.610280:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.610281:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.610282:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.610283:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.610284:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.610288:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.610290:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.610293:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.610293:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.610299:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.610301:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.610301:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.610302:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.610303:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677447.610354:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.610357:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.610357:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.610359:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.610362:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.610364:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.610365:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.610366:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.610367:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.610368:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.610370:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.610371:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.610373:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.610373:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.610374:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.610376:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.610376:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.610377:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.610377:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.610382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.610389:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.610391:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.610392:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.610393:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.610394:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.610395:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.610514:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.610516:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.610517:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.610518:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.610520:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.610522:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.610523:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.610524:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.610524:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.610525:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.610528:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.610529:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.610531:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.610532:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.610532:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.610534:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.610534:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.610535:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.610535:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677447.610548:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.610551:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.610551:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.610552:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.610554:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.610556:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.610599:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.610601:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.610602:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.610604:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.610607:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.610608:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.610610:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.610611:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.610612:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.610613:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.610616:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.610618:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.610620:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.610621:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.610621:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.610623:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.610623:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.610624:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.610624:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.610637:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.610639:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.610640:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.610640:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.610642:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.610643:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:9.2:1619677447.610710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.610718:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.610720:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.610721:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.610722:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.610724:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.610727:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.610728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.2:1619677447.610735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677447.610735:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.610738:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.610739:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.610739:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.610740:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677447.610741:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677447.610741:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.610744:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.610745:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.610747:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.610750:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.610751:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.610753:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.610755:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.610757:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.610758:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.610762:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.610765:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.610768:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.610768:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.610769:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.610771:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.610771:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.610772:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.610773:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.610777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.610784:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.610787:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.610788:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.610788:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677447.610789:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:18.0:1619677447.610790:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.610792:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677447.610792:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.610793:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.610794:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.610795:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:9.2:1619677447.610797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677447.610797:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677447.610804:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.610807:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.610808:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.610808:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.610809:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.610810:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:24.0:1619677447.610942:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.610944:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.610945:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.610947:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.610950:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.610951:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.610953:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.610954:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.610955:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.610956:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.610960:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.610961:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.610964:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.610964:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.610965:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.610966:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.610967:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.610968:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.610970:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.610970:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677447.610979:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.610982:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.610983:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.610983:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:20.2:1619677447.610984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.0:1619677447.610984:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:13.0:1619677447.610985:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.610985:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.610986:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677447.610987:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677447.610987:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.610988:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.610990:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.610992:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.610993:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.610994:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.610996:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677447.610997:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.610998:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677447.610999:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.610999:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.611001:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677447.611001:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677447.611001:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.611002:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.611003:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.611003:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.611004:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.611004:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677447.611005:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.611005:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.611007:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:17.0:1619677447.611008:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.611009:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.611010:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.611011:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.611011:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:17.0:1619677447.611011:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.611012:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.611013:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:17.0:1619677447.611013:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.611014:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.611014:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:17.0:1619677447.611014:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.611015:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677447.611015:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.611019:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.611021:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.611024:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.611025:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.611025:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.611027:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:5.0:1619677447.611027:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677447.611028:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.611028:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.611029:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677447.611029:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.611030:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.611032:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.611033:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:9.2:1619677447.611033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.0:1619677447.611034:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.611039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.611040:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.611042:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:18.0:1619677447.611042:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:24.0:1619677447.611043:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.611043:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:18.0:1619677447.611045:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.611045:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:24.0:1619677447.611046:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.611047:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.611048:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:18.0:1619677447.611049:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.611051:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.611053:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.611054:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.611055:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.611056:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.611061:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.611063:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.611066:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.611068:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.611068:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.611070:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.611071:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.611071:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.611072:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677447.611084:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.611086:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.611087:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.611089:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.611090:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.611092:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.611100:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.611103:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.611104:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.611105:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.611106:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.611107:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.611342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.611347:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.611350:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677447.611352:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.611353:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.611354:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.611356:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.611357:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.611359:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.611359:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.611361:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.611364:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.611365:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.611368:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.611369:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.611370:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.611372:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.611373:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.611373:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.611374:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:6.0:1619677447.611401:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.611404:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.611405:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.611407:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.611409:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.611410:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.611412:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.611413:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.611414:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.611415:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.611418:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.611420:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.611422:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.611423:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.611423:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.611425:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.611426:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.611427:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.611428:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.611428:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.611432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.2:1619677447.611433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:2.0:1619677447.611439:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677447.611440:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677447.611441:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:1.0:1619677447.611441:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677447.611442:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.611443:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677447.611443:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.611443:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.611444:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677447.611444:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.611445:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:2.0:1619677447.611445:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.611448:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:1.0:1619677447.611449:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.611452:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.611454:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.611456:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.611458:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.611459:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.611461:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.611466:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.611468:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.611470:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.611471:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.611471:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.611473:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.611473:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.611475:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.611476:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.611477:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.611483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677447.611488:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.611489:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.611490:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.611490:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:18.0:1619677447.611491:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:17.0:1619677447.611491:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.611491:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.611492:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.611493:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.611493:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677447.611494:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.611495:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.611495:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677447.611496:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:18.0:1619677447.611497:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.611501:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.611557:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:2.0:1619677447.611559:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:1.0:1619677447.611560:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.611561:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.611561:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.611562:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.611563:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:1.0:1619677447.611564:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.611565:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.611567:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:2.0:1619677447.611568:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.611569:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.611571:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.611572:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.611573:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.611574:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:20.2:1619677447.611576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:2.0:1619677447.611578:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.611579:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.611582:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.611583:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.611583:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.611585:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.611585:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.611587:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.611588:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.611588:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.0:1619677447.611598:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:18.0:1619677447.611599:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:20.0:1619677447.611600:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.611601:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.611601:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.611602:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:20.0:1619677447.611603:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.611603:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677447.611604:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.611606:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.611623:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.611626:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.611627:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.611629:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.611629:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.611631:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.611930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.611933:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.611936:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.611938:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.611939:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.611940:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.611942:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677447.611943:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.611945:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.611946:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.611947:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677447.611950:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.611952:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677447.611954:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.611955:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.611956:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.611958:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.611958:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.611960:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.611961:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.611962:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.611966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677447.611969:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.611971:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:5.0:1619677447.611972:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:17.0:1619677447.611973:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.611973:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:5.0:1619677447.611974:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.611975:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677447.611975:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.611976:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.611976:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677447.611976:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.611978:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.611978:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.611979:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.611980:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.611981:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.611982:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.611985:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.611986:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.611986:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.611987:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.611988:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.611992:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.611993:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.611993:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677447.611995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:17.0:1619677447.611995:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.611995:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.611996:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.611997:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677447.612000:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:5.0:1619677447.612003:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:20.0:1619677447.612005:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:5.0:1619677447.612005:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.612007:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.612008:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:13.0:1619677447.612009:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:5.0:1619677447.612009:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.612011:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.612012:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:5.0:1619677447.612012:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677447.612013:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.612013:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:13.0:1619677447.612015:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.612016:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.612017:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677447.612017:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.612019:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.612021:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.612022:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677447.612022:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.612024:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.612025:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:5.0:1619677447.612026:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708600] (22)++
00000800:00000200:5.0:1619677447.612027:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (23)--
00000800:00000200:5.0:1619677447.612028:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.2:1619677447.612033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677447.612040:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (22)++
00000800:00000200:5.0:1619677447.612040:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.612041:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708600] (23)--
00000800:00000200:5.0:1619677447.612041:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.612042:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:5.0:1619677447.612042:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.612043:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.612044:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.612044:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.612045:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:4.2:1619677447.612256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:4.2:1619677447.612260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677447.612263:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.612266:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.612267:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.612269:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.612271:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.612273:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.612274:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.612275:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.612276:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.612277:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.612281:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.612283:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.612286:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.612287:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.612287:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.612289:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.612290:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.612290:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.612291:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.0:1619677447.612308:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.612310:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.612311:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.612311:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:5.0:1619677447.612313:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.612314:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677447.612314:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.612315:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.612315:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.612316:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:1.0:1619677447.612317:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.612320:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.612384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677447.612431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677447.612437:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.612439:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.612440:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.612442:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.612443:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.612445:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.612445:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.612446:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (23)++
00000800:00000200:20.0:1619677447.612447:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.612447:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.612448:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.612496:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.612499:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.612499:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.612500:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.612502:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.612505:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.612514:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.612516:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.612517:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.612517:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.612518:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.612519:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.612561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.612564:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.612566:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.612568:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.612569:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.612571:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.612572:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.612573:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.612575:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.612576:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.612577:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.612580:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.612582:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.612585:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.612586:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.612587:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.612588:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.612589:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.612590:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.612590:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.612603:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.612606:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.612607:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.612608:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.612610:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.612611:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.612646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.612676:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.612679:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.612680:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.612681:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.612684:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.612686:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.612687:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.612688:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.612689:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.612690:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.612691:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.612694:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.612694:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.612694:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.612696:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.612696:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.612697:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.612699:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:2.0:1619677447.612699:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.612699:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.612700:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.612702:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.612702:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.612703:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.612704:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677447.612716:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.612718:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.612719:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.612720:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.612721:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.612722:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.612826:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.612829:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.612830:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.612830:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.612833:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.612835:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677447.612877:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.612880:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.612881:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.612883:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.612886:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.612887:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.612888:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.612890:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.612891:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.612892:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.612896:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.612897:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.612901:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.612901:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.612902:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.612904:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.612904:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.612906:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.612907:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.612907:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:13.0:1619677447.612917:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.612918:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:13.0:1619677447.612919:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677447.612919:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.612920:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:13.0:1619677447.612921:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.612921:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:13.0:1619677447.612922:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.612923:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.612925:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.613028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.613034:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.613036:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677447.613038:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.613039:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.613041:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.613043:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.613044:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.613045:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.613046:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677447.613047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:5.0:1619677447.613048:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.613051:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.613053:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.613056:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.613056:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.613057:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.613059:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.613059:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.613064:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.613066:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.613066:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:1.0:1619677447.613067:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.613070:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.2:1619677447.613071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677447.613071:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677447.613074:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000400:00000200:1.0:1619677447.613074:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.613076:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.613077:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:1.0:1619677447.613077:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.613078:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:1.0:1619677447.613079:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.613080:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.613082:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.613082:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.613084:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.613087:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.613089:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:4.2:1619677447.613092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677447.613092:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.613093:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.613093:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.613095:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.613096:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.613096:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677447.613097:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:1.0:1619677447.613097:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677447.613100:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.613100:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.613102:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.613103:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.613105:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:12.0:1619677447.613108:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.613110:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.613111:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.613113:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.613114:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.613116:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.613122:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:13.0:1619677447.613124:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.613125:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.613125:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:13.0:1619677447.613127:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.613128:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:18.0:1619677447.613129:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.613129:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.613129:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.613131:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:18.0:1619677447.613132:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.613134:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.613135:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.613137:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.613137:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.613139:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.613142:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.613144:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.613147:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.613148:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.613148:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.613150:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.613150:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.613151:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.613152:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.613153:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:12.0:1619677447.613154:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.613156:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.613157:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.613159:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.613160:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.613161:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.613163:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:6.0:1619677447.613164:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677447.613165:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:6.0:1619677447.613166:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677447.613166:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.613167:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.613168:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.613168:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.613170:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.613172:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.613180:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.613183:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.613184:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.613184:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.613185:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.613186:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.613188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677447.613234:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.613237:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.613238:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.613239:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.613240:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.613242:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.613250:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.613253:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.613253:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.613254:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.613255:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.613256:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677447.613393:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.613396:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.613396:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.613398:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.613400:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.613401:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.613419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.613427:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.613430:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.613431:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.613433:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.613436:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.613437:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.613438:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.613440:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.613440:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.613441:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.613444:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.613446:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:4.2:1619677447.613448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.613449:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.613450:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.613450:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.613452:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.613452:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.613454:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.613455:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.613455:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.613461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.613466:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677447.613468:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677447.613469:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677447.613470:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:1.0:1619677447.613470:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.613471:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.613471:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.613472:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677447.613473:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.613476:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:17.0:1619677447.613476:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677447.613479:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677447.613480:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677447.613482:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.613485:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.613486:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:18.0:1619677447.613487:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677447.613488:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.613489:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.613489:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.613490:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.613490:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000400:00000200:18.0:1619677447.613491:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.613491:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.613491:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:18.0:1619677447.613492:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:5.0:1619677447.613494:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:18.0:1619677447.613495:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.613497:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:5.0:1619677447.613497:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.613498:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.613500:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.613500:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677447.613500:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.613501:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677447.613502:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:5.0:1619677447.613502:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677447.613504:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677447.613504:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677447.613505:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677447.613505:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:6.0:1619677447.613645:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.613648:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.613648:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.613651:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.613653:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.613654:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.613656:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.613657:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.613658:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.613660:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.613663:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.613664:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.613667:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.613668:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.613668:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.613670:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.613670:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.613671:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.613672:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.613684:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.613686:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.613687:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.613688:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.613690:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.613691:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:9.2:1619677447.613778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.613785:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.613788:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.613789:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.613789:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.613791:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.613795:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:24.0:1619677447.613806:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.613809:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.613810:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.613812:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.613814:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.613816:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.613818:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.613819:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.613820:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.613821:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.613823:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.613825:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.613828:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.613829:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.613829:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.613831:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.613832:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.613832:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.613833:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677447.613845:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.613848:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.613849:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.613850:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.613851:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.613853:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677447.613940:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.613943:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.613944:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.613945:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.613947:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.613948:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.613949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.613982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.613990:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.613992:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.613993:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.613994:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.613996:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.613999:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.614001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.614004:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.614007:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.614009:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.614010:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.614012:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.614014:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.614015:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.614016:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.614017:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.614018:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.614022:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.614024:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.614026:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.614027:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.614028:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.614029:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.614030:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.614031:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.614031:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.614043:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.614046:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.614047:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.614047:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.614049:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.614051:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.614065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:20.2:1619677447.614106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:17.0:1619677447.614112:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.614115:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.614116:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.614116:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.614118:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.614122:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:9.2:1619677447.614139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.614146:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.614149:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.614150:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.614150:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.614152:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.614155:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.614420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.614428:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.614430:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.614431:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.614433:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.614436:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.614437:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.614439:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.614440:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.614441:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.614442:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.614445:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.614446:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.614449:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.614450:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.614451:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.614452:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.614453:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.614453:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.614454:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.614467:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.614469:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.614470:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.614470:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.614472:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.614475:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677447.614521:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.614524:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.614525:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.614526:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.614529:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.614530:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.614531:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.614533:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.614534:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.614535:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.614538:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.614540:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.614543:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.614544:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.614544:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.614546:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.614546:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.614547:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.614547:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677447.614559:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.614561:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.614562:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.614562:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.614564:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.614567:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677447.614708:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.614711:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.614712:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.614714:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677447.614717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:18.0:1619677447.614717:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.614718:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.614719:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.614721:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.614722:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.614723:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.614723:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.614725:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:18.0:1619677447.614726:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.614728:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677447.614729:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:5.0:1619677447.614730:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.614731:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.614732:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:5.0:1619677447.614732:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.614733:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.614734:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.614734:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:18.0:1619677447.614735:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:5.0:1619677447.614735:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.614736:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.614736:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:5.0:1619677447.614736:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.614737:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.614739:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.614742:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.614744:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.614746:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.614747:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.2:1619677447.614748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677447.614748:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.614749:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.614750:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.614750:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.614751:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677447.614753:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.614755:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.614756:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.614757:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.614758:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.614759:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677447.614762:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.614765:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.614766:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.614766:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.614769:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.614771:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.614864:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.614867:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.614867:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.614870:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.614872:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.614874:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.614875:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.614877:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.614878:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.614879:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.614882:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.614885:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.614887:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.614888:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.614889:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.614890:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.614891:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.614891:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.614892:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677447.614904:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.614906:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.614907:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.614907:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.614910:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.614912:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.615031:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.615034:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.615034:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.615038:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.615041:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.615042:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.615043:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.615045:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.615046:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.615047:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.615050:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.615052:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:9.2:1619677447.615054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677447.615055:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.615056:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.615057:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.615059:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.615059:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.615060:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677447.615061:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:17.0:1619677447.615061:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677447.615064:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.615065:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.615066:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.615068:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.615071:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677447.615072:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.615075:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.615076:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.615078:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.615080:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.615081:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.615090:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.615092:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.615093:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.615094:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.615094:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.615096:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.615177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.615188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.615195:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.615198:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.615199:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.615202:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.615204:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.615206:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.615207:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96725794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.615209:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.615210:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.615211:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.615214:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.615215:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.615218:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.615219:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.615219:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.615221:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.615221:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.615222:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.615222:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.615224:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.615227:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.615228:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.615229:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.615231:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.615234:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.615234:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.615237:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.615238:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.615238:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.615240:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.615243:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.615253:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.615255:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.615256:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.615256:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.615257:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.615258:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.615386:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.615388:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.615389:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.615389:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.615392:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.615394:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.615500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677447.615507:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.615509:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.615510:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.615514:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.615517:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.615519:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:2.0:1619677447.615520:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:17.0:1619677447.615521:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.615523:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.615523:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.615524:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677447.615525:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.615527:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.615528:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.615531:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.615532:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677447.615533:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.615534:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96725789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.615535:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.615536:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.615537:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677447.615537:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.615538:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:2.0:1619677447.615538:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677447.615539:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.615540:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.615541:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.615542:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:2.0:1619677447.615542:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.615544:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.615544:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677447.615544:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.615546:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.615547:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.615548:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.615549:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.615550:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.615550:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.615551:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:22.0:1619677447.615553:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677447.615555:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677447.615556:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:1.0:1619677447.615556:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.615557:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677447.615557:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677447.615557:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.615558:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.615559:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:1.0:1619677447.615559:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.615560:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.615561:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.615561:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:22.0:1619677447.615562:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.615562:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677447.615563:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:22.0:1619677447.615564:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.615565:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96725788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.615566:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.615566:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.615567:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.615568:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.615569:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.615569:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.615569:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.615572:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:22.0:1619677447.615573:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.615575:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.615578:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.615579:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.615580:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.615581:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.615582:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.615582:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.615583:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.615583:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.615584:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.615585:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.615585:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.615586:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.615587:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.615606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.615612:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.615614:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.615617:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.615617:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.615619:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.615621:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.615622:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.615623:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.615624:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.615625:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.615629:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.615631:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.615634:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.615634:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.615635:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.615637:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.615637:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.615639:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.615640:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.615640:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.615645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.615648:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.615650:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.615651:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.615652:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.615877:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.615880:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.615880:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.615883:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.615885:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.615886:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.615888:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.615889:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.615890:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.615891:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.615894:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.615896:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.615898:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.615899:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.615899:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.615901:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.615901:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.615902:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.615902:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.615907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.615913:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.615916:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.615916:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.615917:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.615918:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.615919:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.615926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.615934:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.615936:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.615936:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.615937:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.615937:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.615937:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.2:1619677447.615976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.615979:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.615982:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.615984:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.615985:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.615987:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.615989:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677447.615990:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.615992:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.615993:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.615994:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677447.615997:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.615999:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677447.616002:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.616003:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677447.616004:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677447.616006:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677447.616006:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.616007:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.616008:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:2.0:1619677447.616062:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.616064:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.616066:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.616067:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.616069:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.616071:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.616072:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.616073:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.616074:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.616075:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.616078:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.616080:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.616082:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.616083:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.616083:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.616085:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.616085:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.616086:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.616086:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.616090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.616097:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.616099:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.616100:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.616101:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.616102:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.616103:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.616122:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.616125:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.616125:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.616127:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.616129:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.616130:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.2:1619677447.616131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:1.0:1619677447.616131:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725783] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.616133:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.616134:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.616135:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.616138:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.616140:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.616141:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.616142:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.616143:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.616144:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.616145:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.616145:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.616146:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677447.616157:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.616159:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.616159:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.616160:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.616161:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.616163:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:12.0:1619677447.616175:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.616177:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.616178:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.616180:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677447.616182:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.616183:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.616199:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.616202:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.616203:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.616203:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.616204:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.616205:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.616219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.616258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:4.2:1619677447.616434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.616441:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677447.616441:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.616444:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677447.616444:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:6.0:1619677447.616444:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:24.0:1619677447.616445:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677447.616448:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.616449:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.616451:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.616452:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.616452:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:8.0:1619677447.616454:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677447.616454:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.616456:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.616456:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.616457:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.616458:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.616459:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.616459:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677447.616461:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677447.616464:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.616466:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:8.0:1619677447.616468:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.616469:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.616470:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:6.0:1619677447.616470:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.616470:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:8.0:1619677447.616471:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677447.616471:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:24.0:1619677447.616473:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.616473:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:8.0:1619677447.616474:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.616474:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.616474:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:8.0:1619677447.616475:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:6.0:1619677447.616475:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000400:00000200:24.0:1619677447.616476:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.616476:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677447.616476:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:24.0:1619677447.616477:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:8.0:1619677447.616478:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:24.0:1619677447.616479:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96725778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.616480:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.616481:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.616482:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.616486:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.616486:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:8.0:1619677447.616487:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.616488:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.616488:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.616488:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677447.616488:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.616489:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.616490:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.616491:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.616492:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.616492:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677447.616493:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.616494:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.616495:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.616495:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:24.0:1619677447.616496:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.616496:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:17.0:1619677447.616496:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.616496:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.616497:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:1.0:1619677447.616498:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.616501:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:12.0:1619677447.616512:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.616514:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.616515:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.616516:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.616519:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.616520:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.616605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677447.616649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677447.616658:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.616661:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.616662:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.616664:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.616665:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.616669:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677447.616910:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.616912:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.616913:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.616914:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.616915:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.616918:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.616933:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.616935:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.616936:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.616936:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.616937:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.616938:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.616968:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.616971:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.616971:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.616973:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.616975:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.616976:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.616977:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.616979:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.616980:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.616981:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.616984:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.616986:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.616987:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.616988:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.616989:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.616990:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.616990:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.616991:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.616991:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.617002:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.617004:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.617005:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.617006:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.617006:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.617007:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.617008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.617054:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677447.617056:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:22.0:1619677447.617057:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.617057:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.617058:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.617060:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:6.0:1619677447.617060:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.617060:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677447.617063:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.617064:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:2.0:1619677447.617064:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.617066:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.617067:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.617069:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.617070:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.617071:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:2.0:1619677447.617071:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:22.0:1619677447.617074:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.617075:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.617075:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:2.0:1619677447.617075:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.617077:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.617077:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.617077:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.617080:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.617080:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.617081:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.617083:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.617083:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.617084:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.617085:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677447.617113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.617116:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.617119:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.617121:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.617122:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.617123:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.617125:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.617126:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.617127:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.617128:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.617129:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.617132:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.617134:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.617138:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.617138:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.617139:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.617141:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.617142:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.617142:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.617143:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.617154:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.617156:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.617157:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.617159:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.617160:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.617162:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.617354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677447.617401:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.617404:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.617405:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.617407:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.617408:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.617409:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677447.617425:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.617428:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.617429:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.617429:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.617431:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.617434:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677447.617706:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.617709:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.617709:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.617712:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.617715:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.617716:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.617718:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.617719:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.617720:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.617721:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.617725:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.617726:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.617730:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.617730:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.617731:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.617733:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.617733:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.617735:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.617736:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.617736:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:4.2:1619677447.617742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.617747:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.617748:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:12.0:1619677447.617750:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677447.617750:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.617750:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:12.0:1619677447.617751:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.617751:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:12.0:1619677447.617753:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.617755:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.617756:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.617790:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.617793:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.617794:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.617794:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.617796:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.617798:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.617983:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.617985:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.617986:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.617988:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.617989:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.617991:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.618002:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.618004:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.618005:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.618006:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.618006:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.618007:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.618074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:12.0:1619677447.618116:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.2:1619677447.618118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:12.0:1619677447.618118:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.618119:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.618120:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:12.0:1619677447.618120:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.618122:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677447.618123:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:13.0:1619677447.618123:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:20.0:1619677447.618125:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.618126:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.618127:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.618129:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677447.618130:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.618133:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.618134:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.618135:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677447.618138:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.618140:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677447.618143:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.618144:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677447.618145:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677447.618147:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677447.618148:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.618148:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.618149:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677447.618153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.618160:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.618162:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.618164:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.618164:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.618166:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677447.618169:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:18.0:1619677447.618183:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.618186:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.618186:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.618189:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.618191:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.618193:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.618194:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725763] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.618196:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.618197:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.618198:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.618202:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.618203:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.618207:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.618207:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.618208:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.618210:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.618210:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.618211:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.618212:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.618213:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:12.0:1619677447.618222:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.618223:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:12.0:1619677447.618225:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677447.618225:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:12.0:1619677447.618226:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.618226:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:12.0:1619677447.618227:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.618227:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677447.618228:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.618230:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.618275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.618280:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.618283:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.618286:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.618289:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.618292:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.618292:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.618298:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677447.618299:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:1.0:1619677447.618300:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:5.0:1619677447.618301:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.618303:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.618303:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.618304:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.618305:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:1.0:1619677447.618305:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.618307:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.618308:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.618309:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.618310:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.618311:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.618312:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:1.0:1619677447.618312:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.618314:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.618315:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677447.618316:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:1.0:1619677447.618317:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.618318:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677447.618319:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677447.618319:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.618320:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.618320:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677447.618322:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.618323:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.618324:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:9.2:1619677447.618325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:1.0:1619677447.618325:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.618326:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.618326:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.618327:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677447.618334:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.618337:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.618338:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.618338:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.618338:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.618340:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.618340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.618341:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677447.618342:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:13.0:1619677447.618342:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.618344:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.618346:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.618347:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.618347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.618353:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.618356:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.618357:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.618358:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.618358:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.618359:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:22.0:1619677447.618388:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.618391:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.618392:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.618393:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.618395:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.618398:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.618408:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.618410:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.618411:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.618413:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.618414:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.618415:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.618533:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.618536:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.618537:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.618539:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.618541:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.618543:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.618544:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.618545:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.618546:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.618547:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.618550:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.618552:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.618555:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.618556:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.618557:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.618559:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.618559:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.618560:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.618560:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.618565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.618573:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.618575:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.618576:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.618577:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.618579:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.618582:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.618818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:6.0:1619677447.618833:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.618836:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.618837:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.618837:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.618839:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.618842:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.618856:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.2:1619677447.618857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:13.0:1619677447.618858:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.618859:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.618861:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.618862:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.618863:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.618894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.618901:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.618904:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.618905:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.618905:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.618907:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.618911:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.618944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.618958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.618965:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.618968:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.618969:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.618971:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.618974:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.618975:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.618976:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96725753] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.618978:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.618978:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.618979:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.618982:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.618984:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:4.2:1619677447.618984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677447.618986:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.618987:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.618988:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.618989:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.618990:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.618991:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.618992:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.618993:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.618998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.619003:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.619005:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677447.619006:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.619007:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.619007:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.619008:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.619008:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.619009:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677447.619009:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.619012:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.619022:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.619024:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.619025:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.619025:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.619026:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.619027:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677447.619039:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.619042:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.619042:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.619044:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.619047:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.619049:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.619050:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.619051:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.619052:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.619053:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.619057:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.619059:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.619061:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.619062:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.619063:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.619065:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.619065:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.619066:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.619067:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.619079:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.619081:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.619082:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.619083:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.619085:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.619086:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.619202:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.619205:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677447.619207:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.619208:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.619210:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.619211:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.619212:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.619214:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.619215:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.619216:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.619219:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.619221:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.619223:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.619224:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.619225:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.619226:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.619227:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.619229:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677447.619230:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:5.0:1619677447.619230:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.619240:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.619241:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.619241:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677447.619242:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677447.619242:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.619243:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.619243:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.619244:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:20.0:1619677447.619245:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.619247:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.0:1619677447.619263:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.619266:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.619267:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.619268:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677447.619268:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.619270:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:12.0:1619677447.619346:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.619348:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.619349:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.619350:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.619352:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.619354:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.619365:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.619367:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.619368:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.619369:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.619369:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.619370:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.619404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.619435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:6.0:1619677447.619449:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.619452:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.619453:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.619453:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.619456:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.619462:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.619464:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.619466:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.619466:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.619468:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.619468:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.619469:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.619476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677447.619477:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.619480:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.619481:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.619483:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.619484:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:17.0:1619677447.619486:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.619487:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.619488:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.619488:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:2.0:1619677447.619488:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:17.0:1619677447.619489:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.619490:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.619491:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.619491:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.619493:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:18.0:1619677447.619494:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:17.0:1619677447.619496:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677447.619497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:17.0:1619677447.619498:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677447.619502:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.619502:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677447.619503:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:2.0:1619677447.619504:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677447.619505:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.619505:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.619507:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:2.0:1619677447.619507:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.619508:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677447.619508:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.619508:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677447.619508:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.619509:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.619510:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.619511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.619518:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:1.0:1619677447.619518:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677447.619521:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:1.0:1619677447.619521:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:24.0:1619677447.619522:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.619522:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.619523:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677447.619523:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.619525:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.619528:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.619530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.619536:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.619539:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.619540:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.619540:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.619541:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.619541:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677447.619566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.619571:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.619574:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.619576:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.619576:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.619578:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.619580:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.619582:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.619583:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.619584:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.619585:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.619589:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.619591:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.619593:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.619594:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.619595:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.619596:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.619597:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.619598:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.619598:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.619654:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.619657:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.619657:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.619659:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.619661:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.619663:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.619664:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.619666:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.619667:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.619668:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.619671:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.619673:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.619675:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.619676:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.619677:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.619679:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.619679:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.619680:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.619680:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.619692:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.619695:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.619696:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.619696:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.619698:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.619700:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.619743:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.619746:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.619747:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.619749:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.619752:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.619753:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.619755:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.619756:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.619757:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.619758:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.619760:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.619762:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.619764:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.619764:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.619764:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.619766:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.619766:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.619767:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.619768:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.619772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.619779:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.619781:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.619781:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.619783:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.619784:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.619786:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.619817:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.619820:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.619820:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.619822:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.619823:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.619824:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677447.619887:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.619890:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.619890:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.619893:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.619895:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.619897:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.619898:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.619899:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.619900:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.619901:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.619905:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677447.619906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.619907:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.619909:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.619910:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.619911:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.619913:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.619913:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.619914:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.619914:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.619918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.619925:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.619927:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.619928:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.619929:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.619931:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.619934:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.619946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.619953:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.619956:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.619957:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.619957:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.619959:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.619963:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.619985:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.619988:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.619988:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.619991:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.619992:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.619994:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.619995:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.619997:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.619997:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.619999:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.620002:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.620004:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.620007:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.620008:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.620008:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.620010:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.620010:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.620011:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.620012:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677447.620024:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.620027:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.620027:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.620029:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.620030:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.620032:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.620047:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.620049:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.620050:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.620051:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.620052:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.620053:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:6.0:1619677447.620093:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.620096:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.620097:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.620099:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.620101:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.620102:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.620103:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.620105:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.620106:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.620107:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.620111:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.620112:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.620115:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.620116:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.620117:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.620118:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.620119:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.620119:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.620120:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677447.620133:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.620135:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.620136:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.620137:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.620139:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.620141:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.620194:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.620196:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.620197:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.620199:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.620202:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.620203:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.620204:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725737] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.620206:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.620207:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.620209:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.620212:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.620214:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.620218:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.620219:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.620220:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.620222:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.620222:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.620223:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.620223:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.620235:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677447.620238:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677447.620238:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.620240:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.620242:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677447.620243:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677447.620243:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.620243:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677447.620245:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.620245:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.620247:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.620254:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.620257:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.620257:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.620258:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.620258:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.620259:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.620299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.620305:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.620308:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.620310:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.620311:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.620313:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.620314:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.620315:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.620317:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.620318:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.620319:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.2:1619677447.620320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:5.0:1619677447.620323:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.620325:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.620327:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.620328:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.620329:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.620331:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.620331:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.620332:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.620333:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.620334:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.620339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.620342:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.620344:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.620345:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.620346:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.2:1619677447.620363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:4.2:1619677447.620377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.620397:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.620399:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.620403:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.620403:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.620407:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.620408:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.620409:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.620411:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.620411:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.620413:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.620415:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.620417:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:9.2:1619677447.620417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677447.620418:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677447.620419:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.620420:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.620420:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:17.0:1619677447.620420:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.620421:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677447.620422:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.620422:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.620423:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000400:00000200:17.0:1619677447.620423:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.620424:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:17.0:1619677447.620425:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.620425:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:17.0:1619677447.620427:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:2.0:1619677447.620427:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.620429:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:17.0:1619677447.620429:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96725731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.620429:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.620430:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.620431:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.620432:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677447.620432:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.620434:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:17.0:1619677447.620436:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.620437:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:17.0:1619677447.620438:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.620439:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.620440:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.620441:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.620442:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:17.0:1619677447.620442:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:5.0:1619677447.620442:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.620443:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.620444:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.620444:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:17.0:1619677447.620445:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.620445:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.620446:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677447.620597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.620603:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.620606:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.620607:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.620609:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.620612:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.620614:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.620615:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96725729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.620616:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.620617:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.620618:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.620621:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.620623:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.620626:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.620626:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.620627:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.620629:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.620629:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.620630:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.620630:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.620632:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.620634:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.620636:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.620637:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.620638:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.620639:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.620641:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.620641:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.620643:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.620643:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.620644:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.620646:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.620647:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.620648:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.620651:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.620652:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:1.0:1619677447.620652:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.620654:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.620657:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.620658:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.620659:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.620660:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.620661:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.620662:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.620663:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.620664:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.2:1619677447.620668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.620671:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.620673:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:17.0:1619677447.620674:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:20.0:1619677447.620676:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.620676:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.620677:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677447.620678:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.620678:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677447.620678:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.620680:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.620681:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.620683:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.620684:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.620685:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:20.0:1619677447.620685:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.620688:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.620688:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677447.620689:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.620691:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.620692:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.620694:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.620695:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:20.0:1619677447.620695:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.620696:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:22.0:1619677447.620697:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.620697:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:22.0:1619677447.620698:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.620699:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.620699:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.620699:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:22.0:1619677447.620700:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:20.0:1619677447.620700:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.620701:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:22.0:1619677447.620704:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.620705:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.620708:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.620709:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.620710:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.620712:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.620712:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.620713:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.620713:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677447.620725:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.620728:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.620728:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.620731:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.620733:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.620735:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.620737:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.620738:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.620739:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.620740:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.620743:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.620745:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.620748:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.620749:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.620749:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.620751:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.620751:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.620752:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.620753:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677447.620765:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.620767:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.620768:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.620768:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.620770:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.620773:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677447.620827:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.620829:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677447.620830:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677447.620833:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.620835:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.620837:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:6.0:1619677447.620838:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.620839:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.620841:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.620842:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:6.0:1619677447.620845:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.620847:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677447.620850:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.620850:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:6.0:1619677447.620851:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:6.0:1619677447.620853:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:6.0:1619677447.620854:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.620855:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:6.0:1619677447.620856:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:6.0:1619677447.620856:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677447.620867:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677447.620869:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677447.620869:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677447.620870:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:5.0:1619677447.620962:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.620964:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.620965:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.620967:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.620967:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.620970:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677447.620986:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.620989:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.620989:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.620992:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.620995:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.620996:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.620997:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.620999:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.621001:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.621001:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.621005:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.621008:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.621009:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.621010:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.621011:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.621012:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.621013:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.621014:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.621014:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.621026:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.621028:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.621028:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.621030:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.621030:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.621032:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.621101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677447.621105:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.0:1619677447.621108:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.621111:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.621113:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.621114:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.621115:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.621117:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.621119:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.621120:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.621121:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.621122:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.621127:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.621129:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.621133:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.621134:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.621134:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.621136:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.621136:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.621137:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.621138:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.621149:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.621152:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.621153:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.621154:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.621157:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.621158:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.621192:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.621195:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.621196:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.621198:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.621200:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.621201:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677447.621203:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.621204:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.621205:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.621206:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677447.621210:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.621212:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677447.621215:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.621215:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677447.621216:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677447.621218:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677447.621218:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.621219:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.621221:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677447.621222:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.621223:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.621225:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.621226:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.621228:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.621230:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.621231:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.621232:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.621232:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.621233:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.621237:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.621239:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.621241:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.621242:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.621242:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.621244:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.621245:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.621245:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.621246:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:22.0:1619677447.621250:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:9.2:1619677447.621250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.621253:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.621254:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.621256:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.621258:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:22.0:1619677447.621259:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.621259:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:22.0:1619677447.621260:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677447.621260:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.621261:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:22.0:1619677447.621262:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96725715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.621263:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.621264:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.621265:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.621266:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:22.0:1619677447.621267:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.621270:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.621272:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:9.2:1619677447.621273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.621273:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:22.0:1619677447.621275:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.621276:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677447.621276:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:8.0:1619677447.621276:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:22.0:1619677447.621277:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:8.0:1619677447.621278:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.621279:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.621280:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677447.621280:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.621281:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:8.0:1619677447.621281:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.621282:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.621282:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677447.621283:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.621284:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.621284:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.621285:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.621286:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:8.0:1619677447.621287:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.621289:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96725714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.621290:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.621291:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:8.0:1619677447.621291:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.621292:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:6.0:1619677447.621292:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:4.2:1619677447.621299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677447.621300:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (23)--
00000800:00000200:1.0:1619677447.621300:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000800:00000200:17.0:1619677447.621301:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000400:00000200:17.0:1619677447.621302:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.621302:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677447.621302:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.621303:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677447.621303:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (23)++
00000800:00000200:1.0:1619677447.621303:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (24)--
00000800:00000200:1.0:1619677447.621304:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.621305:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.621306:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.621306:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.621307:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.621308:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:4.2:1619677447.621308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.621309:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.0:1619677447.621309:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.621311:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.621313:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.621314:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.621314:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.621315:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:24.0:1619677447.621316:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.621317:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96725713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.621317:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:24.0:1619677447.621319:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.621320:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.621322:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.621326:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.621326:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.621328:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.621328:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.621329:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.621330:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.621331:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677447.621331:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:24.0:1619677447.621332:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.621333:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.621334:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.621334:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.621335:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.621336:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.621336:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.621337:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677447.621337:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.621338:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.621339:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677447.621340:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.621342:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:5.0:1619677447.621342:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:24.0:1619677447.621343:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.621344:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:4.2:1619677447.621351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677447.621352:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.621355:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.621356:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:1.0:1619677447.621356:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:20.0:1619677447.621357:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.621357:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.621359:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.621359:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (23)++
00000800:00000200:1.0:1619677447.621359:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.621360:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.621361:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.621362:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (23)++
00000800:00000200:17.0:1619677447.621363:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (24)--
00000800:00000200:17.0:1619677447.621364:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.621364:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.621365:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.621368:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.621419:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.621422:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.621423:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.621425:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.621426:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.621428:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.621429:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.621430:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.621432:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.621432:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.621436:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.621438:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.621441:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.621442:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.621442:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.621445:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.621445:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.621446:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.621446:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.621459:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.621462:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.621463:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.621465:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.621466:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.621468:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.621523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:5.0:1619677447.621556:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.621558:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677447.621561:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.621562:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.621564:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:5.0:1619677447.621564:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.621565:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.621565:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.621566:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.621566:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.621568:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000400:00000200:5.0:1619677447.621568:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.621569:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.621570:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.621573:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.621575:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.621577:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.621577:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.621578:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.621579:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.621579:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.621581:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.621582:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.621582:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.621583:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677447.621594:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.621596:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.621597:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.621599:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.621600:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.621601:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.2:1619677447.621924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.621930:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.621932:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.621934:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.621935:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.621937:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.621938:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.621939:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.621941:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.621942:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.621942:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.621946:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.621947:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.621950:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.621950:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.621951:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.621953:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.621954:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.621954:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.621955:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.621965:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.621968:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.621969:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.621971:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.621972:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.621973:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.622055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:4.2:1619677447.622075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.622102:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.622104:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.622105:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.622107:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.622108:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.622110:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677447.622266:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.622269:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.622269:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.622271:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.622273:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.622275:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.622276:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.622277:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.622279:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.622280:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.622283:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.622284:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.622287:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.622288:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.622288:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.622290:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.622290:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.622291:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.622292:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677447.622306:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.622308:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.622309:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.622311:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.622312:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.622313:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.622437:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.622440:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.622440:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.622443:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.622445:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.622447:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.622448:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.622449:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.622450:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.622451:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.622455:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.622456:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.622459:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.622460:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.622461:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.622463:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.622463:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.622464:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.622464:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.622477:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.622480:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.622481:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.622483:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.622484:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.622485:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.622493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.0:1619677447.622540:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.622542:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.622544:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.622546:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.622547:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.622548:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.622549:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.622550:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.622550:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677447.622552:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.622555:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.622556:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.622558:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.622559:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.622560:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:5.0:1619677447.622560:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.622561:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:12.0:1619677447.622562:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.622563:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.622564:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677447.622565:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.622566:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.622566:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:12.0:1619677447.622567:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.622568:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.622569:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.622569:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.622571:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.622571:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.622572:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.622573:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:13.0:1619677447.622578:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.622581:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.622582:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.622582:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.622583:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.622584:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:13.0:1619677447.622585:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.622586:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.622587:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.622588:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.622589:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.622590:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.2:1619677447.622895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.622901:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.622903:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677447.622905:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.622906:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.622908:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.622909:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677447.622911:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.622912:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.622913:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.622914:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.622918:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.622920:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.622923:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.622924:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.622924:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.622926:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.622927:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677447.622927:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.622928:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.622929:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:22.0:1619677447.622930:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:5.0:1619677447.622930:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:6.0:1619677447.622931:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.622933:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677447.622935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000400:00000200:22.0:1619677447.622936:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.622938:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:20.0:1619677447.622938:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000400:00000200:22.0:1619677447.622939:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.622939:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:22.0:1619677447.622940:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.622940:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:22.0:1619677447.622941:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.622941:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.622941:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:22.0:1619677447.622942:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:17.0:1619677447.622942:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.622943:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.622943:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.622945:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.622946:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.622948:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677447.622949:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.622951:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.622952:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.622953:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.622954:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.622955:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.622955:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.622956:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677447.622959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:9.2:1619677447.622961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.622968:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.622971:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.622972:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.622972:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.622973:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:2.0:1619677447.622974:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.622975:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.622977:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677447.622978:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.622979:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.622981:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.622983:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.622984:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.622986:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.622988:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.622989:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.622994:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.622996:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.622999:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.623000:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.623001:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.623003:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.623003:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.623005:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.623006:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.623006:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.2:1619677447.623007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677447.623009:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.623012:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.623013:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.623015:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.623016:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.623017:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.623017:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:13.0:1619677447.623018:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.623019:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.623020:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:13.0:1619677447.623021:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677447.623021:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:13.0:1619677447.623022:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.623023:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.623025:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.623026:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.623053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.623110:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.623113:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.623114:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.623115:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.623117:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.623120:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.623157:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.623160:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.623160:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.623162:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.623164:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.623165:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.623424:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.623428:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.623429:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.623429:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.623431:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.623434:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677447.623443:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.623446:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.623448:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.623449:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.623452:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.623454:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.623455:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.623457:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.623458:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.623459:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.623462:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.623464:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.623467:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.623468:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.623469:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.623470:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.623471:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.623472:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.623472:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677447.623485:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.623488:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.623489:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.623490:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.623493:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.623494:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.623565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.0:1619677447.623611:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.623613:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.623614:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.623616:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.623617:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.623619:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.623629:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.623632:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.623633:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.623633:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.623634:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.623635:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.623712:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.623714:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:4.2:1619677447.623715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:5.0:1619677447.623716:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677447.623718:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.623718:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.623720:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.623720:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.623722:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.623723:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.623724:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.623727:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.623730:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.623733:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.623734:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.623734:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.623736:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.623736:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.623738:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677447.623739:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:5.0:1619677447.623739:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.623750:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.623750:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.623751:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677447.623751:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:5.0:1619677447.623751:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.623752:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.623753:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:20.0:1619677447.623754:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.623754:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.623759:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677447.623760:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.623762:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.623762:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.623764:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.623766:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.623767:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.623768:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.623772:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.623772:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.623773:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.623774:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.623775:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:13.0:1619677447.623806:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.623808:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.623809:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.623810:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.623812:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.623813:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.623823:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.623826:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.623826:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.623827:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.623828:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.623828:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.623867:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.623869:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.623870:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.623872:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.623875:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.623877:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.623878:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725688] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.623880:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.623881:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.623882:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.623885:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.623887:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.623889:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.623890:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.623891:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.623892:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.623893:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.623894:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.623895:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.623895:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677447.623907:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:18.0:1619677447.623908:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:5.0:1619677447.623909:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:18.0:1619677447.623910:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:5.0:1619677447.623910:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.623911:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.0:1619677447.623912:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.623912:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:5.0:1619677447.623912:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.623914:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.623953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.623956:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.623959:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.623961:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.623962:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.623963:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.623965:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.623966:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.623967:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.623968:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.623970:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.623973:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.623975:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.623977:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.623978:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.623978:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.623980:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.623981:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.623982:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.623982:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.624005:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.624008:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.624009:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.624010:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.624011:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.624013:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.624125:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:9.2:1619677447.624125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:12.0:1619677447.624127:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.624128:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.624130:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.624132:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.624133:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.624133:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.624135:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.624136:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.624136:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.624139:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.624142:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.624215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.624256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677447.624303:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.624306:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.624307:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.624309:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.624312:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.624313:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.624314:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.624315:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.624316:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.624317:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.624320:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.624322:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.624324:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.624325:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.624326:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.624327:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.624328:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.624328:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.624329:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677447.624342:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.624344:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.624345:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.624347:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.624348:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.624349:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.624454:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.624457:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.624457:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.624460:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.624462:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.624464:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.624465:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725682] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.624467:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.624468:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.624469:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.624478:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.624481:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.624484:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.624485:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.624486:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.624488:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.624489:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.624490:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.624491:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.624492:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.624495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:1.0:1619677447.624502:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:2.0:1619677447.624503:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:1.0:1619677447.624504:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.624505:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677447.624506:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:1.0:1619677447.624506:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677447.624507:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.624507:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.624510:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.624514:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677447.624570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:20.2:1619677447.624612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677447.624619:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.624622:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.624623:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.624624:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.624626:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.624627:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.624660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.624706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.624714:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.624716:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.624717:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.624718:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.624720:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.624723:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.624778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.624784:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.624786:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.624789:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.624789:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.624791:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.624792:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677447.624794:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.624795:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.624796:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.624797:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677447.624801:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.624806:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677447.624809:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.624809:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677447.624810:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677447.624811:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:2.0:1619677447.624812:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.624813:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677447.624813:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677447.624813:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.624814:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.624815:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:2.0:1619677447.624815:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.624817:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.624819:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.624820:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.624821:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.624822:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.624823:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.624826:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.624828:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.624831:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.624832:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.624833:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.624835:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.624835:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.624836:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.624836:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677447.624848:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.624850:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.624851:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.624853:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.624854:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.624855:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677447.624855:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.624858:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.624858:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.624860:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.624863:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.624865:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.624866:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.624866:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:18.0:1619677447.624867:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.624868:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.624869:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.624869:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.624870:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.624871:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.624871:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.624872:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:18.0:1619677447.624873:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.624875:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.624877:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.624878:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.624879:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.624881:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.624881:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.624882:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.624883:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677447.624995:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677447.624995:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677447.624999:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:2.0:1619677447.624999:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:1.0:1619677447.624999:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677447.625000:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:17.0:1619677447.625003:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.625003:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.625006:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.625007:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.625007:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677447.625008:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677447.625009:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.625010:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.625011:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.625012:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.625012:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.625013:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.625013:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677447.625014:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677447.625018:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000400:00000200:17.0:1619677447.625019:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.625021:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677447.625022:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677447.625023:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.625023:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.625024:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677447.625025:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.625025:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.625025:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677447.625026:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677447.625027:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:2.0:1619677447.625027:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.625028:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.625028:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677447.625029:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.625029:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.625030:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.625030:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.625030:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:9.2:1619677447.625031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.625034:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.625034:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:9.2:1619677447.625035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:6.0:1619677447.625038:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.625039:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.625040:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:24.0:1619677447.625041:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:18.0:1619677447.625041:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.625041:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.625041:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.625041:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.625042:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:6.0:1619677447.625044:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:2.0:1619677447.625044:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.625044:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677447.625045:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:18.0:1619677447.625045:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000400:00000200:24.0:1619677447.625046:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.625046:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:6.0:1619677447.625046:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96725672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.625048:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.625048:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:6.0:1619677447.625048:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.625049:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.625050:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:24.0:1619677447.625053:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:6.0:1619677447.625057:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.625059:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:4.2:1619677447.625061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:5.0:1619677447.625062:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:6.0:1619677447.625063:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.625064:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.625065:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:5.0:1619677447.625065:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:6.0:1619677447.625066:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.625067:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:5.0:1619677447.625067:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.625068:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:8.0:1619677447.625068:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677447.625068:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.625069:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:5.0:1619677447.625069:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.625071:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:24.0:1619677447.625072:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.625072:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:5.0:1619677447.625073:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96725670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.625075:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.625076:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.625076:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.625077:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:8.0:1619677447.625079:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.625081:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.625082:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:13.0:1619677447.625083:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677447.625083:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96725669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.625083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.625085:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:13.0:1619677447.625086:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:8.0:1619677447.625086:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677447.625087:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.625087:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677447.625088:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.625088:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.625089:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.625090:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.625090:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:12.0:1619677447.625091:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.625092:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677447.625092:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.625092:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.625092:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.625093:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:8.0:1619677447.625094:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.625094:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.0:1619677447.625103:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.625104:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.625104:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.625106:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:8.0:1619677447.625106:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.625107:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.625107:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677447.625108:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:8.0:1619677447.625108:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:20.0:1619677447.625109:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.625111:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.625111:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.625113:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.2:1619677447.625120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677447.625120:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.625122:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.625123:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:9.2:1619677447.625124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677447.625124:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.625126:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677447.625126:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.625128:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677447.625129:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:5.0:1619677447.625130:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.625131:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677447.625132:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:5.0:1619677447.625133:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.625133:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:5.0:1619677447.625134:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:12.0:1619677447.625135:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.625135:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.625135:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96725666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.625136:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.625137:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:5.0:1619677447.625137:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.625137:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.625138:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.625138:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.625138:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.625139:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.625140:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.625141:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.625142:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:5.0:1619677447.625143:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.625145:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.625148:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.625148:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.625149:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.625151:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.625151:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.625153:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.625154:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.625154:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.625159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.625162:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:22.0:1619677447.625164:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.625164:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.625165:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.625166:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.625167:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.625168:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.625168:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.625170:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.625172:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.625186:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.625189:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.625189:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.625190:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.625191:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.625191:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.625400:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.625403:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.625403:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.625405:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.625408:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.625409:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.625410:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.625412:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.625413:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.625414:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.2:1619677447.625416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:17.0:1619677447.625417:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.625419:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.625421:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.625422:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.625423:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.625425:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.625425:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.625426:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.625426:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.625552:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677447.625555:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677447.625557:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.625559:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677447.625560:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.625561:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:13.0:1619677447.625562:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96725662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.625564:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677447.625564:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677447.625566:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:13.0:1619677447.625569:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.625570:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677447.625573:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.625574:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677447.625575:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.625577:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677447.625578:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677447.625579:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677447.625579:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677447.625584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.625592:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.625594:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.625595:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.625596:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.625598:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.625601:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.625604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.625611:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.625614:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.625615:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.625615:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.625616:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.625617:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677447.625655:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.625658:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.625659:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.625660:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.625662:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.625662:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.625671:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.625673:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.625674:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.625675:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.625675:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.625676:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.625697:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677447.625698:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677447.625700:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:4.2:1619677447.625700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.625701:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677447.625701:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.625702:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677447.625705:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.625706:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.625707:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.625709:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.625709:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.625710:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677447.625711:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.625712:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.625714:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.625714:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.625715:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.625716:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.625717:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.625717:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.625723:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.625724:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.625727:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.625727:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.625729:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.625730:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.625730:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677447.625731:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.625731:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.625732:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.625732:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.625733:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677447.625734:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.625734:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.625735:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677447.625735:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.625736:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.625736:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677447.625747:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.625749:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.625750:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.625751:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:1.0:1619677447.625751:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:18.0:1619677447.625754:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:17.0:1619677447.625755:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.625755:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.625758:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.625758:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:18.0:1619677447.625761:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.625762:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.625764:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.625765:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.625766:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.625767:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.625771:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.625772:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.625775:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.625776:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.625777:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.625779:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.625779:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.625780:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.625781:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677447.625793:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.625795:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.625796:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.625798:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.625799:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.625800:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677447.625947:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.625949:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.625950:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.2:1619677447.625952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:1.0:1619677447.625953:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.625955:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.625957:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:5.0:1619677447.625958:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677447.625958:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.625960:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:1.0:1619677447.625960:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.625961:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.625962:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677447.625963:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.625963:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.625965:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.625966:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.625967:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.625968:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96725653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.625968:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:5.0:1619677447.625970:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.625971:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.625972:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677447.625972:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.625973:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.625974:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:5.0:1619677447.625976:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.625976:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.625976:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.625977:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:5.0:1619677447.625978:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677447.625978:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.625979:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677447.625981:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.625982:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.625986:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.625988:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677447.625988:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677447.625989:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:5.0:1619677447.625989:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.625989:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.625990:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677447.625991:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:13.0:1619677447.625991:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:17.0:1619677447.625992:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677447.625992:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.625993:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677447.625994:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.625994:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:9.2:1619677447.625994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.625996:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.626002:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.626004:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.626006:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:12.0:1619677447.626006:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:2.0:1619677447.626006:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.626008:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.626009:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.626010:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677447.626010:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677447.626011:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.626012:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.626014:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677447.626022:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.626025:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.626027:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.626028:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.626029:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.626031:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.626032:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.626034:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677447.626034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:20.0:1619677447.626035:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.626037:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.626041:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.626042:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677447.626043:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:24.0:1619677447.626044:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:20.0:1619677447.626045:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.626045:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:8.0:1619677447.626045:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677447.626046:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:24.0:1619677447.626047:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.626048:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.626048:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.626049:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000400:00000200:24.0:1619677447.626050:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.626050:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:24.0:1619677447.626052:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.626054:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96725651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.626056:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.626057:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.626058:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.626061:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.626061:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:24.0:1619677447.626063:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.626064:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.626065:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.626066:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.626067:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.626067:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.626067:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.626068:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.626069:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:24.0:1619677447.626070:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.626070:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.626072:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.626073:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.626074:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.626079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.626084:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.626086:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.626087:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.626087:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:1.0:1619677447.626088:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.626089:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.626089:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.626090:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.626090:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.626093:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.626103:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.626106:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.626106:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.626107:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.626108:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.626108:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.626123:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677447.626125:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677447.626126:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677447.626128:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.626130:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.626131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:1.0:1619677447.626132:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677447.626133:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.626134:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.626135:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.626136:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677447.626140:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.626142:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677447.626145:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.626146:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:1.0:1619677447.626146:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:1.0:1619677447.626148:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:1.0:1619677447.626149:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677447.626149:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677447.626150:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:18.0:1619677447.626156:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.626158:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.626159:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.626162:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.626164:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.626166:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.626168:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96725648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.626169:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.626170:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.626171:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.626175:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.626177:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.626179:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.626180:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.626181:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.626183:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.626183:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.626184:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.626185:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677447.626197:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.626199:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.626200:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.626202:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.626203:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.626204:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.626216:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.626219:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.626220:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.626220:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.626221:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.626222:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.626246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.626262:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.626264:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.626265:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.626268:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.626271:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.626273:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.626274:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.626276:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.626277:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.626278:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.626281:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.626283:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.626286:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.626287:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.626288:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.626290:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.626290:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.626291:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.626291:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677447.626291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.626301:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.626304:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.626305:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.626306:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.626307:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:6.0:1619677447.626308:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.626310:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.626311:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.626311:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:13.0:1619677447.626313:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.626314:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.626315:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.626366:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.626369:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.626370:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.626372:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.626375:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.626376:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.626377:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.626378:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.626379:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:22.0:1619677447.626380:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.626381:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.626381:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.626383:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:22.0:1619677447.626386:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.626386:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.626389:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.626389:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.626390:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:22.0:1619677447.626391:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677447.626391:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.626392:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.626393:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677447.626393:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.626394:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.626395:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.626395:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677447.626395:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.626396:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.626397:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.626398:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677447.626398:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.626400:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.626403:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.626403:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.626404:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677447.626406:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:2.0:1619677447.626406:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.626407:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:6.0:1619677447.626408:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677447.626408:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:6.0:1619677447.626410:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.626411:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.626412:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677447.626421:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000800:00000200:17.0:1619677447.626424:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (24)++
00000800:00000200:1.0:1619677447.626424:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000800:00000200:17.0:1619677447.626425:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000400:00000200:1.0:1619677447.626425:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.626426:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.626427:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.626428:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.626431:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.626431:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.626432:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.626712:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.2:1619677447.626715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677447.626715:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.626716:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.626718:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.626719:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:17.0:1619677447.626720:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.626721:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:5.0:1619677447.626721:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.626722:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:17.0:1619677447.626723:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.626723:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:17.0:1619677447.626724:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.626724:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.626726:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:18.0:1619677447.626726:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677447.626726:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.626727:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677447.626728:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.626728:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.626730:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.626730:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.626731:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.626732:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.626732:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.626734:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.626734:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:5.0:1619677447.626734:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.626735:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.626736:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.626737:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677447.626738:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677447.626738:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.626739:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.626739:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:5.0:1619677447.626739:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.626741:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677447.626741:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.626742:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.626742:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.626743:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:5.0:1619677447.626743:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.626744:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.626746:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.626746:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.626747:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.626748:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.626749:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.626750:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.626750:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:5.0:1619677447.626750:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.626751:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.0:1619677447.626751:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.626751:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.626753:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.626754:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677447.626754:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.626754:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.626755:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.626757:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.626758:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.626759:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.626761:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.626762:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.626763:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:13.0:1619677447.626763:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:22.0:1619677447.626766:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:13.0:1619677447.626766:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:6.0:1619677447.626766:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.626767:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.626767:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.626770:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:12.0:1619677447.626770:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677447.626771:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.626773:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.626774:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.626796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.626803:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.626806:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.626807:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.626809:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.626812:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.626814:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.626815:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.626817:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.626818:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.626819:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.626822:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.626824:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.626827:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.626828:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.626829:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.626830:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.626831:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.626832:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.626832:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.626854:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.626857:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.626857:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.626858:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.626860:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.626863:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.626874:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.626877:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.626878:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.626878:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.626878:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.626879:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.626895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:20.0:1619677447.626942:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.626944:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.626945:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.626947:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.626948:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.626949:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.626994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.627065:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.627068:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.627069:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.627069:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.627071:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.627074:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.627088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.627094:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.627097:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.627099:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.627100:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.627103:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.627104:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:5.0:1619677447.627105:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:6.0:1619677447.627106:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.627107:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:5.0:1619677447.627107:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.627109:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.627109:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.627110:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:5.0:1619677447.627110:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.627111:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:8.0:1619677447.627112:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:6.0:1619677447.627112:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.627113:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:24.0:1619677447.627114:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:6.0:1619677447.627115:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.627116:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.627117:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.627117:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.627118:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.627119:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.627119:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677447.627121:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.627122:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96725634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.627123:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.627124:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677447.627124:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.627124:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:8.0:1619677447.627125:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.627126:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:6.0:1619677447.627126:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:17.0:1619677447.627127:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:5.0:1619677447.627127:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.627127:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:6.0:1619677447.627128:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.627128:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:1.0:1619677447.627128:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.627129:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.627130:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677447.627130:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:17.0:1619677447.627131:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.627132:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.627132:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000400:00000200:17.0:1619677447.627133:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.627133:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.627133:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677447.627134:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:6.0:1619677447.627134:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.627134:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:17.0:1619677447.627135:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677447.627135:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.627135:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:17.0:1619677447.627136:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96725633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.627138:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.627138:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677447.627139:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677447.627139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677447.627139:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.627139:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:17.0:1619677447.627140:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.627141:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.627141:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.627142:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.627143:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:17.0:1619677447.627144:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.627145:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.627146:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:17.0:1619677447.627149:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.627149:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.627150:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:17.0:1619677447.627150:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.627150:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:2.0:1619677447.627150:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.627152:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677447.627152:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.627153:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677447.627154:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677447.627154:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.627154:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677447.627155:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.0:1619677447.627157:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.627158:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.627159:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.627160:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.627162:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:9.2:1619677447.627163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:12.0:1619677447.627166:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.627168:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.627169:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.627170:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000400:00000200:12.0:1619677447.627171:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.627172:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.627173:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:12.0:1619677447.627174:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.627174:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.627175:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.627176:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.627176:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677447.627322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.627328:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.627330:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.627332:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.627333:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.627335:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.627337:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.627338:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.627341:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.627341:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.627343:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.627346:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.627347:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.627350:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.627351:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.627352:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.627354:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.627354:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.627355:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.627356:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.627368:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.627370:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.627371:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.627373:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.627374:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.627376:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.627431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677447.627475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677447.627483:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.627486:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.627487:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.627489:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.627490:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.627492:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.627877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.627899:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.627901:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.627902:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.627903:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.627905:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.627908:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.0:1619677447.627920:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.627922:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.627924:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.627925:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.627925:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.627928:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.628033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.0:1619677447.628079:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.628081:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.628082:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.628083:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.628085:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.628086:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.628095:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.628098:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.628099:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.628100:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.628100:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.628102:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677447.628172:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.628174:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.628175:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.628177:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.628180:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.628181:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.628183:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.628184:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.628185:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.628186:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.628189:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.628191:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.628194:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.628195:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.628195:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.628197:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.628197:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.628199:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.628200:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.628200:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.628211:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677447.628211:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.628213:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.628213:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.628214:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677447.628214:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677447.628215:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.628215:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.628217:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.628220:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.628265:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.628267:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.628268:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.628269:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.628271:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.628272:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.628284:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.628286:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.628287:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.628288:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.628288:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.628289:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.628353:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.628356:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.628357:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.628358:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.628360:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.628362:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.628363:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.628364:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.628365:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.628366:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.628370:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.628372:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.628375:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.628376:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.628376:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.628378:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.628378:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.628380:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.628381:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.628382:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:12.0:1619677447.628392:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677447.628393:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:12.0:1619677447.628394:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677447.628395:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:12.0:1619677447.628395:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.628396:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.628397:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:12.0:1619677447.628397:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.628399:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.628400:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.628455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.628491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:6.0:1619677447.628503:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.628506:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.628507:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.628507:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.628510:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.628514:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.628540:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.628543:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.628544:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.628546:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.628546:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.628548:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677447.628593:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.628595:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.628596:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.628598:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.628600:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.628602:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.628603:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.628605:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.628606:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.628607:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.628610:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.628612:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.628614:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.628615:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.628616:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.628618:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.628618:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.628619:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.628619:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.628624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.628631:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.628634:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.628635:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.628635:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.628637:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.628640:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:18.0:1619677447.628917:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.628919:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.628920:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.628922:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.628925:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.628927:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677447.628928:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.628929:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.628930:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.628931:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677447.628935:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.628937:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.628940:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.628941:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:18.0:1619677447.628942:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677447.628943:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.628944:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.628945:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.628946:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.628946:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:13.0:1619677447.628956:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677447.628957:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.628959:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.628959:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:13.0:1619677447.628960:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.628960:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.628961:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:13.0:1619677447.628962:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.628963:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.628965:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.629193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.629212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.629214:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.629217:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.629219:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.629220:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.629222:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.629224:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677447.629225:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.629226:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.629227:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.629228:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677447.629232:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.629235:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:4.2:1619677447.629235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677447.629238:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.629239:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:6.0:1619677447.629239:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.629240:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677447.629242:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:6.0:1619677447.629242:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.629243:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677447.629243:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.629243:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000400:00000200:6.0:1619677447.629243:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.629244:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:6.0:1619677447.629245:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.629248:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.629259:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.629263:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.629263:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.629264:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.629265:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.629266:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.629300:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.629302:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.629304:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.629304:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.629306:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.2:1619677447.629307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:6.0:1619677447.629309:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.629355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.629359:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677447.629359:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.629360:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.629361:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:5.0:1619677447.629362:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.629362:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.629363:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.629363:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677447.629364:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.629365:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.629367:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.629368:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.629368:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:17.0:1619677447.629369:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.629369:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.629370:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:5.0:1619677447.629370:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:17.0:1619677447.629371:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677447.629372:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.629372:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.629374:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.629375:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.629375:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.629376:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.629377:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:17.0:1619677447.629377:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677447.629383:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.629383:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.629387:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:17.0:1619677447.629387:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.629389:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.629390:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.629390:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:17.0:1619677447.629390:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677447.629391:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:17.0:1619677447.629391:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.629392:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.629393:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:17.0:1619677447.629393:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677447.629394:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677447.629394:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.629394:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677447.629395:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.629396:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.629397:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:9.2:1619677447.629400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.2:1619677447.629402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:4.2:1619677447.629403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:1.0:1619677447.629405:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.629407:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677447.629408:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:5.0:1619677447.629408:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:1.0:1619677447.629408:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.629409:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.629410:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:5.0:1619677447.629410:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:1.0:1619677447.629410:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.629411:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:5.0:1619677447.629411:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:18.0:1619677447.629412:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.629412:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677447.629412:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:1.0:1619677447.629412:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:24.0:1619677447.629415:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000400:00000200:18.0:1619677447.629415:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.629416:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677447.629418:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.629418:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:24.0:1619677447.629420:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.629422:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677447.629423:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.629425:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.629426:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.629427:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677447.629431:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.629433:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677447.629435:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.629436:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:24.0:1619677447.629437:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:24.0:1619677447.629438:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:24.0:1619677447.629439:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677447.629440:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677447.629440:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:2.0:1619677447.629670:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.629673:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.629674:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.629675:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.629678:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.629679:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.629680:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.629682:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.629683:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.629684:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.629687:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.629689:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.629692:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.629693:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.629694:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.629695:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.629696:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.629696:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.629697:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.0:1619677447.629705:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.629707:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.629708:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.629709:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677447.629710:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.629711:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.629712:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.629712:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.629714:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.629714:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:13.0:1619677447.629716:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:8.0:1619677447.629716:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96725610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.629717:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.629718:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677447.629718:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.629719:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677447.629722:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.629724:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.629727:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.629728:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.629728:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.629730:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.629730:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.629731:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.629732:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.629742:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.629744:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.629745:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.629746:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.629748:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.629749:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.629760:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.629762:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.629763:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.629764:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.629765:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.629766:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.629783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.629830:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.629833:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.629833:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.629834:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.629836:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.629839:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.629956:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.629958:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.629959:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.629961:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.629963:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.629964:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.629966:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.629967:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.629968:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.629969:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.629972:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.629974:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.629977:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.629977:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.629978:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.629980:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.629980:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.629981:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.629981:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.629994:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.629996:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.629997:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.629997:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.629999:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.630002:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.630100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.630106:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.630109:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.630111:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.630112:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.630114:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.630115:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.630117:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.630118:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.630119:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.630120:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.630124:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.630125:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.2:1619677447.630129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:5.0:1619677447.630129:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.630130:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.630131:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.630134:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.630135:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.630136:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.630137:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677447.630168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.630177:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677447.630178:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677447.630180:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:13.0:1619677447.630181:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.630181:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.630181:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677447.630181:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:24.0:1619677447.630182:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677447.630183:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:1.0:1619677447.630184:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:24.0:1619677447.630185:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.630185:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677447.630185:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:1.0:1619677447.630185:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.630186:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:13.0:1619677447.630186:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677447.630187:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96725603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.630188:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.630189:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.630190:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.630191:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:24.0:1619677447.630192:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.630195:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.630197:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.630200:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.630200:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.630201:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.630203:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.630203:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.630204:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.630204:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.630216:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.630219:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.630219:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.630220:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.630222:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.630224:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.630244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.630300:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.630303:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.630304:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.630307:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.630310:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.630312:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.630313:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.630316:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.630317:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.630319:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677447.630320:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.630323:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677447.630323:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.630324:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677447.630325:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.630326:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.630327:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:1.0:1619677447.630327:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.630329:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.630330:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677447.630330:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.630332:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677447.630332:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:22.0:1619677447.630333:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:18.0:1619677447.630333:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.630334:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:18.0:1619677447.630334:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677447.630334:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96725599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.630335:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.630336:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:18.0:1619677447.630336:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96725599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.630337:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:1.0:1619677447.630337:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.630338:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.630338:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:18.0:1619677447.630338:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.630338:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677447.630340:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.630341:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:18.0:1619677447.630342:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:9.2:1619677447.630343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.630343:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:1.0:1619677447.630344:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.630347:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677447.630348:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677447.630349:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.630351:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.630351:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.630351:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.630352:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:1.0:1619677447.630352:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:18.0:1619677447.630353:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.630354:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:18.0:1619677447.630354:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000400:00000200:2.0:1619677447.630354:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.630354:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:20.0:1619677447.630355:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.630355:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000800:00000200:1.0:1619677447.630355:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.630355:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677447.630356:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.630356:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:1.0:1619677447.630356:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677447.630357:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (24)--
00000400:00000200:2.0:1619677447.630357:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:508, d:32, s:-96725597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.630358:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:2.0:1619677447.630359:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.630360:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:2.0:1619677447.630360:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.630361:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.630362:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.630363:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.630364:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677447.630365:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.630366:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.630367:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:2.0:1619677447.630367:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.630368:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000400:00000200:18.0:1619677447.630369:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.630371:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.630371:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.630372:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.630372:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.630375:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.630375:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.630375:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.630375:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.630376:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.630377:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.630377:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.630377:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:18.0:1619677447.630379:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.630379:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.630381:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677447.630389:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.630391:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.630392:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.630394:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.630395:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.630397:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.630397:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.630398:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.630399:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.630399:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)++
00000800:00000200:5.0:1619677447.630400:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.630401:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.630402:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677447.630456:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.630458:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.630459:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.630461:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.630464:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.630465:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.630467:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.630468:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.630469:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.630470:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677447.630474:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.630475:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.630478:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.630478:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.630479:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.630481:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.630482:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.630483:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.630483:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:4.2:1619677447.630603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677447.630610:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677447.630614:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677447.630615:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677447.630617:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.630619:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.630621:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:8.0:1619677447.630622:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.630623:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.630624:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.630625:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:8.0:1619677447.630629:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.630630:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677447.630632:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.630633:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:8.0:1619677447.630634:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.630636:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.630636:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677447.630637:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:8.0:1619677447.630639:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:8.0:1619677447.630639:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:9.2:1619677447.630641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:4.2:1619677447.630645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677447.630650:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.630652:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.630652:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.630654:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.630654:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:8.0:1619677447.630654:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:8.0:1619677447.630655:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677447.630656:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.630656:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677447.630657:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:17.0:1619677447.630674:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.630677:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.630678:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.630680:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.630683:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.630684:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677447.630686:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.630687:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.630688:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.630689:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677447.630692:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.630694:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677447.630697:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.630698:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677447.630698:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677447.630700:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.630701:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.630702:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.630703:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.630703:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.630706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.2:1619677447.630710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.0:1619677447.630712:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:20.0:1619677447.630713:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677447.630714:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677447.630715:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.630716:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677447.630716:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677447.630716:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:8.0:1619677447.630717:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.630717:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677447.630718:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.630718:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677447.630719:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.630721:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.630721:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.630722:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:8.0:1619677447.630722:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:20.0:1619677447.630724:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.630726:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.630727:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.630728:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.630732:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.630735:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.630738:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.630739:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.630739:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.630741:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.630741:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.630742:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.630743:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.630754:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:9.2:1619677447.630756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.630758:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.630759:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.630759:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.630761:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.630763:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677447.630764:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.630768:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.630769:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.630769:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.630771:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.630774:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.630775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.630782:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.630785:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.630786:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.630786:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.630787:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.630788:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.630789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677447.630831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:4.2:1619677447.630832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.630834:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.630837:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.630837:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.630839:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.630841:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.630842:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.630998:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:1.0:1619677447.630998:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677447.631001:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:1.0:1619677447.631001:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.631002:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.631002:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.631003:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.631004:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.631005:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.631007:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.631008:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677447.631008:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:24.0:1619677447.631009:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.631011:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.631012:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.631013:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.631016:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.631017:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.631020:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.631020:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.631021:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.631023:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.631023:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.631024:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.631025:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.631035:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.631037:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.631038:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.631039:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.631041:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.631043:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.631053:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.631055:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.631056:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677447.631056:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.631057:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.631057:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.631058:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:13.0:1619677447.631058:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.631059:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.631061:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.631064:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.631065:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.631067:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725587] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.631068:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.631069:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.631070:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.631073:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.631075:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.631078:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.631078:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.631079:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.631081:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.631081:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.631082:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.631082:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.631093:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.631096:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.631097:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.631098:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.631100:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.631101:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.631136:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.631139:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.631140:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.631142:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.631144:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.631146:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.631147:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.631149:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.631149:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.631151:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.631154:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.631156:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.631158:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.631159:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.631160:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677447.631160:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:22.0:1619677447.631162:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.631162:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677447.631162:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:22.0:1619677447.631163:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:13.0:1619677447.631163:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.631164:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.631165:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:13.0:1619677447.631165:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.631166:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.631168:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.631175:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:1.0:1619677447.631176:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677447.631177:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.631178:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677447.631178:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:6.0:1619677447.631179:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677447.631180:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.631180:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.631182:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.631184:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.631193:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.631196:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.631196:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.631197:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.631198:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.631198:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.631198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.631205:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.631208:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.631209:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.631211:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.631213:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.631215:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.631216:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.631217:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.631219:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.631220:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.631223:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.631225:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.631228:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.631229:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.631229:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.631231:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.631231:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.631232:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.631233:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.631244:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.631247:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.631248:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.631248:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.631250:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.631253:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.631257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:24.0:1619677447.631264:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677447.631267:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677447.631268:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677447.631269:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.631272:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.631273:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:24.0:1619677447.631275:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.631276:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.631277:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.631278:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:24.0:1619677447.631280:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.631281:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.631283:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.631284:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677447.631284:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.631285:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.631286:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677447.631287:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677447.631287:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.2:1619677447.631291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.631299:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.631301:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.631301:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.631302:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.631303:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.631305:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.631310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677447.631317:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.631319:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.631320:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.631320:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.631321:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.631321:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.631333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.631373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.631379:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.631382:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.631383:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.631384:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.631386:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.631390:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.631392:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.631394:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.631395:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.631395:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.631396:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.631396:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.631410:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.631413:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.631414:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.631416:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.631418:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.631420:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.631421:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.631422:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.631424:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.631425:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.631428:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.631429:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.631431:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.631432:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.631433:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.631434:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.631435:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.631436:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.631436:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.631448:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.631450:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.631451:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.631451:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.631452:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.631454:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.631465:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.631467:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.631468:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.631468:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.631469:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.631469:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.631533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677447.631560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.631566:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.631568:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677447.631570:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677447.631571:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.631573:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.631574:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677447.631575:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.631577:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.631578:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.631579:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677447.631582:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.631585:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.631588:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.631588:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677447.631589:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677447.631591:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677447.631592:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.631593:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.631594:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.631595:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.2:1619677447.631600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.631603:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.631605:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.631606:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.631607:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:6.0:1619677447.631681:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.631684:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677447.631685:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677447.631687:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.631689:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.631691:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:6.0:1619677447.631693:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725578] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.631695:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.631696:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.631697:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.631698:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.631701:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.631702:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677447.631702:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.631704:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.631705:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.631706:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.631707:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:6.0:1619677447.631709:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677447.631709:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96725576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.631710:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:6.0:1619677447.631710:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677447.631710:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.631711:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.631712:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:1.0:1619677447.631712:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:6.0:1619677447.631713:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.631714:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:9.2:1619677447.631715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.631715:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000400:00000200:1.0:1619677447.631716:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.631718:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.631720:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.631721:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.631722:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:18.0:1619677447.631724:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:1.0:1619677447.631724:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.631724:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.631725:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.631725:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677447.631727:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:12.0:1619677447.631727:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.631728:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.631729:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.631729:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.631730:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.631730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:18.0:1619677447.631731:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:12.0:1619677447.631732:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.631734:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:13.0:1619677447.631734:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.631735:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.631736:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.631739:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.631740:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.631740:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.631742:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.631745:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.631748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.631754:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.631757:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.631758:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.631758:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.631759:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.631760:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677447.631802:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.631805:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.631805:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.631808:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.631811:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.631812:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.631813:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.631815:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.631816:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.631817:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.631820:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.631822:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.631825:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.631825:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.631826:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.631827:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.631828:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.631829:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.631829:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677447.631842:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.631844:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.631845:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.631846:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.631848:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.631850:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.631897:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.631900:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.631900:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677447.631903:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.631905:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.631907:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.631908:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.631909:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.631911:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.631912:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.631915:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.631917:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.631920:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.631921:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.631921:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.631923:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.631923:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.631924:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.631924:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677447.631932:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.631934:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.631935:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677447.631937:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:18.0:1619677447.631937:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.631938:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.631939:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:18.0:1619677447.631939:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677447.631940:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.631940:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725573] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.631942:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.631942:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.631943:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.631943:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:18.0:1619677447.631944:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.631945:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:18.0:1619677447.631947:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.631949:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.631951:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.631952:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.631952:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.631954:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.631954:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.631955:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.631956:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677447.631968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677447.631968:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.631970:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.631971:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.631971:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.631973:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677447.631974:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.631976:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677447.631976:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.631978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:5.0:1619677447.631978:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.631981:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.631982:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677447.631983:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.631984:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725572] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.631985:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.631986:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.631987:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.631991:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.631994:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.631998:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.631999:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.632000:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.632001:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.632002:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.632003:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.632003:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.632014:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.632017:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.632018:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.632019:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.632021:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.632022:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.632028:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.632031:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.632032:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.632032:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.632034:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:17.0:1619677447.632037:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.632038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.632051:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677447.632051:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:12.0:1619677447.632054:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.632054:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.632055:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:4.2:1619677447.632055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:2.0:1619677447.632055:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:12.0:1619677447.632057:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.632058:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:2.0:1619677447.632058:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:12.0:1619677447.632061:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:2.0:1619677447.632061:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.632063:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.632065:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.632066:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677447.632067:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677447.632069:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.632070:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.632071:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.632072:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.632072:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.632073:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.632073:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677447.632075:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96725567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.632077:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.632077:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:9.2:1619677447.632078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677447.632078:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.632079:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.632079:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.632082:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677447.632083:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.632083:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.632084:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677447.632085:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.632086:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.632086:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.632087:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.632088:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.0:1619677447.632088:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.632088:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677447.632089:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.632090:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:8.0:1619677447.632090:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.632090:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.632091:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.632091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677447.632092:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.632092:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:18.0:1619677447.632093:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.632093:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.632095:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.632095:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677447.632097:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.632099:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.632100:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.632101:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.632101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677447.632101:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.632103:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.632104:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:1.0:1619677447.632105:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.632107:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.632108:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.632109:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:9.2:1619677447.632109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000400:00000200:1.0:1619677447.632109:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.632110:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.632110:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.632111:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.632111:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.632113:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.632117:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.632120:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.632121:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.632121:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.632121:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.632122:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:17.0:1619677447.632123:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.632126:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.632127:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.632127:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.632128:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.632129:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.632238:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.632239:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.632241:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:1.0:1619677447.632242:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.632243:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.632244:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.632245:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.632246:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.632247:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.632248:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.632249:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.632251:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.632252:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.632255:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.632256:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.632258:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.632259:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.632259:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.632260:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.632260:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.632261:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.632262:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.632263:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.632265:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.632266:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.632266:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.632276:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.632277:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.632277:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.632277:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.632278:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:17.0:1619677447.632279:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.632279:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.632280:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.632280:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.632281:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677447.632394:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.632397:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.632398:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.632400:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.632402:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.632403:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.632404:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.632406:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.632407:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.632408:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.632411:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.632413:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.632416:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.632416:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.632417:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.632419:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.632419:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.632420:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.632420:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677447.632434:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.632436:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.632437:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.632439:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.632440:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.632442:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.632442:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.632442:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.632443:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.632443:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.632444:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.632581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:13.0:1619677447.632597:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.632600:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.632600:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.632602:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.632604:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.632605:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.632617:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.632619:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.632620:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.632621:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.632621:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.632622:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.632623:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.632626:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.632627:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.632629:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.632629:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.632632:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.632751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.632757:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.632761:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.632761:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.632765:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.632767:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.632769:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.632770:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.632772:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.632774:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.632775:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.632780:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.632782:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.632785:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.632786:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.632787:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.632788:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.632789:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.632789:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.632790:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.632795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.632802:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.632804:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.632805:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.632806:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677447.632806:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:2.0:1619677447.632807:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.632808:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.632809:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.632810:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:5.0:1619677447.632811:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.632812:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.632814:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.632841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.632849:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.632850:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:8.0:1619677447.632852:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.632853:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677447.632853:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677447.632853:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:8.0:1619677447.632856:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.632857:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.632859:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.632859:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.632861:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677447.632861:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.632863:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.632863:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.632864:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.632865:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.632866:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.632867:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677447.632867:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.632869:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.632872:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.632874:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.632875:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.632877:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.632878:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.632879:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:17.0:1619677447.632880:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677447.632880:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.632880:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.632881:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.632881:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.632882:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677447.632883:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:8.0:1619677447.632883:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.632883:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.632884:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.0:1619677447.632884:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.632885:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.632895:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:1.0:1619677447.632896:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677447.632898:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.632899:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.632899:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677447.632899:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.632900:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.632900:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.632902:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.632903:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.632906:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.632907:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:24.0:1619677447.633019:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.633022:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.633023:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.633025:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.633027:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.633029:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.633030:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.633032:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.633033:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.633034:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.633037:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.633039:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.633042:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.633043:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.633044:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.633046:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.633046:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.633047:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.633049:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.633049:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677447.633052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.633060:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.633062:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:8.0:1619677447.633062:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:2.0:1619677447.633063:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.633064:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.633064:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.633064:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.633065:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:18.0:1619677447.633066:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.633069:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.633102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.633109:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.633112:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.633113:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.633113:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.633115:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.633118:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:17.0:1619677447.633193:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.633196:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.633197:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.633199:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.633201:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.633203:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.633204:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.633205:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.633206:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.633207:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.633211:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.633212:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.633216:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.633216:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.633217:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.633219:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.633219:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.633220:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.633220:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.633231:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.633234:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.633235:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.633237:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.633238:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.633239:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.633310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.0:1619677447.633361:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.633363:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.633364:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.633366:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.633368:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677447.633369:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.633370:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677447.633371:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.633371:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.633374:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.633376:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.633377:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.633379:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.633380:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.633381:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.633382:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.633386:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.633387:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.633390:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.633390:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.633391:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.633393:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.633393:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.633394:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.633396:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.633396:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677447.633406:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677447.633407:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.633409:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:1.0:1619677447.633409:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.633410:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.633410:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:1.0:1619677447.633410:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.633411:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:1.0:1619677447.633413:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.633415:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.633465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:4.2:1619677447.633465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.633471:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:5.0:1619677447.633474:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:20.0:1619677447.633476:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:5.0:1619677447.633477:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.633480:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.633483:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:5.0:1619677447.633484:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.633486:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.633488:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.633489:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:5.0:1619677447.633494:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677447.633495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.633496:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677447.633498:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.633499:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:5.0:1619677447.633500:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.633501:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:20.0:1619677447.633502:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.633502:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:20.0:1619677447.633504:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.633504:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677447.633504:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:5.0:1619677447.633505:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677447.633506:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:5.0:1619677447.633506:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000400:00000200:20.0:1619677447.633507:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96725550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.633507:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.633509:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.633510:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.633511:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677447.633514:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.633516:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.633518:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.633518:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.0:1619677447.633519:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677447.633520:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677447.633521:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:20.0:1619677447.633522:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.633522:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:13.0:1619677447.633522:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.633522:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.633523:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000400:00000200:13.0:1619677447.633523:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.633524:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:12.0:1619677447.633525:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677447.633525:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.633526:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:13.0:1619677447.633527:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:6.0:1619677447.633527:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.633529:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.633532:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.633535:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:12.0:1619677447.633536:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.633538:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.633539:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.633539:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.633539:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677447.633540:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.633540:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:17.0:1619677447.633541:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:12.0:1619677447.633541:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.633542:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677447.633544:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:13.0:1619677447.633582:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.633585:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.633585:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.633587:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.633588:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.633590:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.633824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677447.633864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677447.633866:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.633868:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.633869:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.633870:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.633872:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.633873:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677447.633959:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.633962:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.633962:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.633964:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.633967:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.633968:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.633970:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725547] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677447.633971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:1.0:1619677447.633971:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.633972:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.633973:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.633976:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.633977:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677447.633978:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.633979:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.633982:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.633982:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.633982:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.633983:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.633984:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677447.633985:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.633985:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677447.633986:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677447.633986:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.633987:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.633987:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:5.0:1619677447.633988:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.633989:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.633990:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.633991:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.633995:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.633996:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.633998:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.633999:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.634000:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677447.634000:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)++
00000400:00000200:13.0:1619677447.634001:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.634001:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (24)++
00000400:00000200:13.0:1619677447.634003:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.634003:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677447.634004:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677447.634004:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677447.634006:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.634006:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.634007:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.634007:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.634012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.634015:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677447.634015:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677447.634018:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:12.0:1619677447.634018:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677447.634019:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.634019:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:12.0:1619677447.634019:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.634021:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.634022:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.634023:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.634027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.634035:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.634038:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.634039:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.634040:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.634041:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.634042:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.634067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.634247:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.634249:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.634250:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.634252:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.634254:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.634256:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.634257:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.634258:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.634259:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.634260:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.634263:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.634265:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.634267:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.634268:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.634269:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.634270:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.634271:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.634271:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.634272:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.634294:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.634301:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.634302:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.634302:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:20.0:1619677447.634304:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.634306:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:24.0:1619677447.634389:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.634392:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.634393:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.634395:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.634398:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.634399:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.634401:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.634402:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.634403:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.634404:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.634408:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.634410:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.634413:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.634414:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.634415:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.634416:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.634417:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.634417:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.634418:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.634423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.634432:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.634435:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.634435:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.634436:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.634438:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.634441:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.634442:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.634444:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.634445:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.634447:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.634450:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.634452:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.634453:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.634454:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.634456:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.634456:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.634460:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.634462:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.634465:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.634465:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.634466:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.634468:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.634468:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.634469:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.634469:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.634482:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.634484:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.634485:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.634485:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.634488:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.634490:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.634552:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677447.634554:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677447.634556:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677447.634559:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.634559:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677447.634561:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:13.0:1619677447.634562:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96725540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.634564:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677447.634565:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677447.634566:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:13.0:1619677447.634570:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.634572:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:13.0:1619677447.634574:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.634575:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:13.0:1619677447.634576:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:13.0:1619677447.634577:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:13.0:1619677447.634578:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677447.634579:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677447.634579:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:20.2:1619677447.634605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:4.2:1619677447.634640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.634646:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.634649:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.634650:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.634652:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.634653:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.634654:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.634702:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.634705:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.634706:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.634707:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.634709:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.634711:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.634712:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.634714:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.634715:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.634716:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.634718:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.634720:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.634722:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.634723:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.634723:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.634725:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.634725:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.634726:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.634726:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.634730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.634737:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.634739:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.634740:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.634740:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.634741:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.634742:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.634750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.634757:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.634759:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.634760:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.634761:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.634761:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.634761:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.634854:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.634857:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.634858:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.634860:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.634862:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.634863:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.634865:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.634866:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.634867:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.634868:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.634871:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.634873:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.634875:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.634876:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.634877:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.634878:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.634878:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.634880:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.634881:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.634881:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.634892:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.634893:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.634893:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:17.0:1619677447.634894:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.634894:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.634895:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.634896:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.634896:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.634897:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.634897:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.634913:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.634916:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.634917:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.634917:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.634917:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.634918:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:9.2:1619677447.635105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.635111:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.2:1619677447.635112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677447.635113:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.635115:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:18.0:1619677447.635116:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.635116:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.635117:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.635118:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.635118:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:2.0:1619677447.635119:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.635120:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:5.0:1619677447.635121:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:18.0:1619677447.635122:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.635124:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:18.0:1619677447.635124:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.635124:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.635125:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.635126:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677447.635127:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.635130:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.635130:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.635132:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.635133:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.635133:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.635134:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.635134:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:5.0:1619677447.635135:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.635136:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:2.0:1619677447.635136:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.635137:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.635137:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.635138:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.635139:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.635139:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:5.0:1619677447.635140:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.635141:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.635142:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:2.0:1619677447.635142:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:18.0:1619677447.635143:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:5.0:1619677447.635143:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677447.635144:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.635145:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.635145:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.635146:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:5.0:1619677447.635146:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677447.635146:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:5.0:1619677447.635147:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.635147:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:2.0:1619677447.635148:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96725532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.635149:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.635150:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:2.0:1619677447.635150:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.635151:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.635152:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.635152:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:2.0:1619677447.635152:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.635156:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.635158:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.635158:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.635159:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.635160:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.635161:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677447.635161:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.635162:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.635163:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.635165:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.635166:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.635167:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.635168:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.635168:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677447.635178:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677447.635179:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:22.0:1619677447.635181:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:18.0:1619677447.635181:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:22.0:1619677447.635182:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.635182:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677447.635182:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.635183:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:22.0:1619677447.635184:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.635187:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.635230:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.635233:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.635233:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.635234:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.635236:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.635239:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.635251:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.635254:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.635254:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.635255:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.635256:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.635256:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.635339:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.635342:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.635343:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.635345:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.635347:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.635348:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.635350:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.635351:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.635352:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.635353:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.635356:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.635358:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:4.2:1619677447.635360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.635361:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.635361:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.635362:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.635364:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.635364:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.635365:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.635365:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.635370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.635378:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.635380:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.635381:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.635382:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.635384:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.635387:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:18.0:1619677447.635472:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.635474:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.635475:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.635478:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.635480:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.635482:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.635483:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.635484:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.635486:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.635486:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.635490:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.635492:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.635494:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.635495:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.635496:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.635497:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.635498:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.635499:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.635499:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677447.635510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.635516:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.635517:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.635518:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.635519:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.635519:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.635521:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.635522:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.635525:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.635525:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.635526:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.635527:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.635527:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.635529:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.635530:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.635531:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.635532:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.635535:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.635537:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.635541:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.635541:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.635542:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.635545:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.635546:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.635546:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.635547:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.635550:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.635552:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.635553:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.635555:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.635557:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.635559:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.635560:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.635561:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.635563:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.635564:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.635568:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.635570:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.635573:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.635574:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.635574:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.635576:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.635577:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.635578:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.635579:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.635580:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.635590:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.635592:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.635593:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.635594:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677447.635595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:9.2:1619677447.635630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677447.635633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:2.0:1619677447.635637:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.635640:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.635641:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.635641:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.635643:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.635647:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.635703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.635707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:18.0:1619677447.635710:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.635713:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.635714:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.635714:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.635715:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:18.0:1619677447.635716:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.635717:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.635718:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.635720:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.635720:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:24.0:1619677447.635723:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.635724:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.635725:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.635728:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.635729:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.635729:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.635733:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.635734:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.635737:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.635738:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.635738:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.635740:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.635741:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.635741:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.635742:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.635754:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.635757:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.635758:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.635758:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.635760:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.635763:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.635774:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677447.635775:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677447.635777:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.635778:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:2.0:1619677447.635778:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:1.0:1619677447.635778:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:18.0:1619677447.635779:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:17.0:1619677447.635779:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.635780:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:2.0:1619677447.635781:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.635783:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.635785:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677447.635786:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.635787:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.635789:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.635790:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677447.635793:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.635795:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.635798:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.635799:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677447.635800:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677447.635802:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.635802:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.635803:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.635804:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677447.635816:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.635819:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.635820:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.635822:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.635823:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.635825:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.635890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:4.2:1619677447.635947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.635976:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.635978:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.635979:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.635981:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.635983:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.635985:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.635986:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.635987:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.635988:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.635989:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.635994:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.635997:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.636000:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.636000:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.636001:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.636003:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.636004:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.636004:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.636005:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.636032:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.636035:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.636036:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.636036:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.636039:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.636042:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.636110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.636113:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.636116:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.636118:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.636119:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.636121:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.636123:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677447.636124:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.636126:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.636127:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.636128:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677447.636131:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.636133:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677447.636136:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.636137:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677447.636138:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.636139:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677447.636139:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fdae00] (21)++
00000800:00000200:20.0:1619677447.636140:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.636141:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.636141:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677447.636141:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.38@o2ib3
00000800:00000200:20.0:1619677447.636142:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (22)--
00000400:00000200:5.0:1619677447.636143:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.38@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.636145:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.636146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:5.0:1619677447.636147:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.636148:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.636149:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.636150:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.636151:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.636153:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:5.0:1619677447.636154:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.38@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.636155:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.636156:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.636157:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.636157:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677447.636159:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.636160:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.636160:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.636160:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.636162:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677447.636162:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.636163:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fdae00] (21)++
00000800:00000200:5.0:1619677447.636164:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fdae00] (22)++
00000800:00000200:5.0:1619677447.636165:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fdae00] (23)--
00000800:00000200:5.0:1619677447.636166:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (22)--
00000800:00000200:22.0:1619677447.636176:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.636176:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fdae00] (22)++
00000800:00000200:20.0:1619677447.636178:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fdae00] (23)--
00000800:00000200:22.0:1619677447.636179:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677447.636179:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (22)--
00000800:00000200:6.0:1619677447.636179:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.636180:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.636180:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (21)--
00000400:00000200:22.0:1619677447.636182:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.636184:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.636416:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.636419:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.636420:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.636422:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.636425:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.636426:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.636427:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.636427:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:17.0:1619677447.636428:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.636430:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.636430:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.636431:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:17.0:1619677447.636431:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677447.636434:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.636436:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.636436:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.636438:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:6.0:1619677447.636438:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.636439:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.636440:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.636441:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.636441:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:6.0:1619677447.636441:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.636442:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.636443:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.636443:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:6.0:1619677447.636443:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.636444:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.636445:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677447.636446:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.636448:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.636451:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.636452:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.636452:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.636454:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.636454:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.636455:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.636455:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677447.636457:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.636460:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.636461:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.636463:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.636463:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.636465:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.636465:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.636466:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.636466:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.636466:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.636468:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.636470:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.636496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:22.0:1619677447.636502:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.636505:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677447.636505:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677447.636507:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.636510:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.636511:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677447.636513:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.636513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:22.0:1619677447.636514:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.636515:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.636516:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677447.636519:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.636520:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677447.636522:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.636523:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:22.0:1619677447.636524:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677447.636525:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:22.0:1619677447.636526:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.636526:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677447.636527:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:1.0:1619677447.636539:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.636541:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.636543:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.636543:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.636545:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.636547:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.636551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.2:1619677447.636684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677447.636817:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.636819:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.636820:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.636822:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.636824:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.636825:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677447.636826:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.636828:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.636829:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.636830:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677447.636833:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.636835:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677447.636838:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.636838:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677447.636839:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677447.636841:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.636841:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.636843:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.636844:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.636844:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.636847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.636853:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:1.0:1619677447.636855:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677447.636856:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:1.0:1619677447.636856:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:24.0:1619677447.636857:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.636857:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.636857:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.636858:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677447.636859:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.636863:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.637021:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.637024:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.637024:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.637027:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.637029:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.637030:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.637032:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.637033:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.637034:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.637035:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.637039:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.637041:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.637044:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.637045:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.637046:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.637048:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.637048:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.637049:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.637049:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.637080:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.637082:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.637083:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.637084:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.637086:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.637087:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.637098:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.637100:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.637101:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.637102:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.637102:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.637103:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677447.637144:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.637147:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.637147:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.637150:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.637152:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.637154:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.637155:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.637156:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.637157:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.637158:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.2:1619677447.637161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:17.0:1619677447.637161:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.637163:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677447.637164:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.637166:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:17.0:1619677447.637167:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.637167:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.637168:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:20.0:1619677447.637169:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.637169:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:17.0:1619677447.637170:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.637170:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:20.0:1619677447.637171:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.637171:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.637171:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:20.0:1619677447.637173:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:4.2:1619677447.637174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:20.0:1619677447.637175:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.637177:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.637178:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.637179:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677447.637182:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.637182:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.0:1619677447.637184:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677447.637185:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.637186:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.637187:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.637188:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677447.637188:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:12.0:1619677447.637188:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.637189:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677447.637190:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.637190:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.637191:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:12.0:1619677447.637191:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677447.637192:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:2.0:1619677447.637194:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.637197:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.637198:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.637200:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.637204:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.637205:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.637206:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96725504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.637208:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.637209:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.637210:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.637214:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.637214:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677447.637216:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677447.637217:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.637217:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:2.0:1619677447.637218:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677447.637218:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.637219:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.637220:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:1.0:1619677447.637220:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.637221:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.637222:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.637222:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:1.0:1619677447.637222:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.637223:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:22.0:1619677447.637232:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.637235:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.637236:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.637236:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.637236:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:22.0:1619677447.637238:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.637238:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.637239:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.637241:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.637241:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.637242:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.637244:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.637277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.637280:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.637283:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.637285:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.637286:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.637287:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.637288:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677447.637290:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.637291:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.637292:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.637297:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677447.637300:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.637302:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677447.637304:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.637305:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677447.637305:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.637307:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.637307:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.637309:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.637310:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.637311:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:9.2:1619677447.637314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.2:1619677447.637316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.637322:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:5.0:1619677447.637322:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:18.0:1619677447.637324:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:5.0:1619677447.637325:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:2.0:1619677447.637325:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.637326:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.637326:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.637327:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677447.637328:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.637330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.637334:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.637335:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.637337:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.637338:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.637338:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.637339:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.637340:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677447.637348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:5.0:1619677447.637355:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:5.0:1619677447.637358:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:5.0:1619677447.637360:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.637361:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:5.0:1619677447.637363:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.637365:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:5.0:1619677447.637367:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.637369:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.637370:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.637371:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:5.0:1619677447.637375:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.637378:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:5.0:1619677447.637380:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.637381:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:5.0:1619677447.637382:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:5.0:1619677447.637384:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:5.0:1619677447.637385:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:5.0:1619677447.637385:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:5.0:1619677447.637386:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:20.0:1619677447.637563:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.637566:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.637567:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.637568:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.637569:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.637572:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.637584:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.637587:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.637588:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.637589:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.637590:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.637593:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677447.637596:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.637599:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.637600:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.637601:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.637604:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.637606:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.637607:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.637608:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.637609:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.637610:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.637612:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677447.637614:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.637615:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:2.0:1619677447.637615:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.637616:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.637618:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.637618:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.637619:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.637620:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:17.0:1619677447.637621:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.637622:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:2.0:1619677447.637622:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.637622:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:17.0:1619677447.637623:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.637623:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.637623:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:17.0:1619677447.637625:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.637626:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.637627:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677447.637631:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.637633:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.637636:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.637637:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.637637:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.637637:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.637638:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:1.0:1619677447.637638:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.637640:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000400:00000200:1.0:1619677447.637640:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.637642:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.637643:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.637643:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.637644:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.637644:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:9.2:1619677447.637646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.637653:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.637656:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.637657:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.637657:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.637659:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.637662:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677447.637752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:20.2:1619677447.637768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:9.2:1619677447.637790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.637798:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.637800:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.637801:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.637802:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.637804:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.637807:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.637810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:9.2:1619677447.637815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.637818:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.637821:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:4.2:1619677447.637822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:2.0:1619677447.637822:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677447.637823:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.637825:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.637825:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677447.637825:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.637826:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.637826:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.637826:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677447.637827:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (23)++
00000800:00000200:2.0:1619677447.637827:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.637828:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (24)--
00000800:00000200:2.0:1619677447.637828:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.637829:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.637831:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.637832:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.0:1619677447.637832:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.637835:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.637836:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.637837:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.637840:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.637842:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.637843:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.637844:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.637845:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.637847:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.637850:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.637851:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.637854:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.637855:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.637855:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.637857:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.637858:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.637858:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.637859:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.637872:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.637875:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.637876:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.637876:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.637878:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.637882:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.637890:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.637893:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.637894:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.637894:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.637895:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.637896:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.638028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.638073:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.638076:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.638077:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.638077:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.638079:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.638082:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.638091:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.638094:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.638095:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.638095:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.638096:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.638097:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.638355:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.638358:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.638358:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.638360:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.638363:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.638365:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.638366:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.638368:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.638368:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.638370:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.638372:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:6.0:1619677447.638373:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.638375:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:6.0:1619677447.638375:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.638376:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677447.638378:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.638379:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:6.0:1619677447.638379:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.638380:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:18.0:1619677447.638381:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.638381:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.638382:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.638382:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:18.0:1619677447.638383:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:6.0:1619677447.638383:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:18.0:1619677447.638384:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.638385:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.638386:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.638387:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:9.2:1619677447.638388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:18.0:1619677447.638391:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.638394:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.638397:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.638397:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.638398:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (23)++
00000800:00000200:18.0:1619677447.638398:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (24)++
00000800:00000200:2.0:1619677447.638398:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.638399:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.638400:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (24)--
00000800:00000200:18.0:1619677447.638401:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:2.0:1619677447.638401:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.638402:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.638403:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677447.638403:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.638404:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.638406:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.638406:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.638408:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.638408:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.638409:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.638437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.638445:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.638447:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.638448:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.638450:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.638453:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.638454:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.638456:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.638457:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.638458:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.638459:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.638462:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.638463:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.638466:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.638467:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.638468:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.638469:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.638470:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.638471:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.638472:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.638473:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.638478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.638484:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677447.638486:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:4.2:1619677447.638486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.638487:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.638488:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.638488:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.638488:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677447.638488:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.638489:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677447.638490:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.638493:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677447.638544:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.638546:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.638547:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.638549:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.638549:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.638552:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.638641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.638643:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.638646:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.638648:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.638649:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.638650:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.638652:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.638653:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.638655:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.638656:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.638657:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.638660:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.638661:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.638664:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.638664:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.638665:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.638667:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.638667:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.638668:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.638669:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:1.0:1619677447.638811:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.638813:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.638814:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.638815:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.638817:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.638819:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.638861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677447.638908:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.638910:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.638911:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.638913:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.638913:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.638916:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677447.638966:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:4.2:1619677447.638967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.638969:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.638969:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.638971:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.638973:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.638975:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.638976:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.638978:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.638979:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.638980:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.638984:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.638985:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.638988:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.638989:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.638990:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.638992:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.638992:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.638993:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.638993:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.639006:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.639008:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.639009:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.639009:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:4.2:1619677447.639010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677447.639011:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.639014:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.639023:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.639026:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.639027:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.639027:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.639028:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.639029:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.639043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.639046:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.639048:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.639051:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.639052:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.639053:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.639055:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677447.639056:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.639058:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.639059:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.639060:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677447.639064:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.639065:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677447.639068:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.639069:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.639069:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.639071:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.639072:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.639072:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.639073:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677447.639085:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.639087:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.639088:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.639088:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.639090:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.639093:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:24.0:1619677447.639301:0:13351:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:4.2:1619677447.639348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.639383:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.639386:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.639387:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.639389:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.639391:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.639393:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.639394:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.639396:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.639397:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.639398:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.639401:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.639402:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.639405:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.639405:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.639406:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:4.2:1619677447.639406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:17.0:1619677447.639408:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.639408:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.639410:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.639411:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.639412:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677447.639414:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.639416:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.639417:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.639419:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.639422:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:8.0:1619677447.639422:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.639422:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:8.0:1619677447.639424:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:1.0:1619677447.639424:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677447.639425:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:8.0:1619677447.639425:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96725478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.639425:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677447.639426:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.639426:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.639426:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677447.639427:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.639428:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.639429:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.639429:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.639433:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.639435:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.639435:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.639437:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.639438:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.639439:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.639441:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.639441:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.639442:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.639443:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.639454:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.639457:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.639458:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.639458:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.639460:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.639463:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.639624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:6.0:1619677447.639631:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.639633:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.639634:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.639635:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.639637:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.639641:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.639672:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.639675:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.639675:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.639677:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.639679:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.639680:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.639728:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.639732:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.639733:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.639734:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.639737:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.639739:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.639740:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.639741:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.639741:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677447.639742:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.639744:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677447.639744:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.639745:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:2.0:1619677447.639748:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.639748:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.639751:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:1.0:1619677447.639751:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.639752:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:2.0:1619677447.639754:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677447.639754:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.639755:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:1.0:1619677447.639755:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.639756:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.639757:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:1.0:1619677447.639757:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.639758:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:1.0:1619677447.639758:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.639759:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.639760:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.639761:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:1.0:1619677447.639761:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.639764:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.639767:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.639768:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.639769:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.639771:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.639771:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.639772:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:1.0:1619677447.639772:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.639772:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677447.639774:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.639775:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677447.639775:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677447.639776:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:24.0:1619677447.639777:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677447.639777:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.639778:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.639780:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:5.0:1619677447.639780:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.639781:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:8.0:1619677447.639781:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.639783:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.639783:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.639783:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:24.0:1619677447.639786:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.639786:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:24.0:1619677447.639787:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677447.639787:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.639789:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96725472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677447.639789:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.639790:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677447.639791:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.639792:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.639792:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677447.639793:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.639796:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.639798:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.639801:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.639802:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.639802:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.639804:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.639805:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.639805:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.639806:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.639811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.639819:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.639821:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.639823:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.639823:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.639825:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.639828:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.639925:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.639927:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.639928:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.639930:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.639932:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.639934:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.639935:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.639937:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.639938:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.639939:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.639942:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.639944:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.639946:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.639947:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.639948:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.639950:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.639950:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.639951:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.639952:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.639953:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.639963:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:20.0:1619677447.639964:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677447.639965:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.639965:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677447.639966:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.639967:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.639967:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:20.0:1619677447.639969:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.639970:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.639972:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:13.0:1619677447.639997:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.639999:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.640000:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.640001:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.640003:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.640004:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.640105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.640121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.640127:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.640129:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.0:1619677447.640129:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:18.0:1619677447.640131:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:20.0:1619677447.640132:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677447.640132:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:5.0:1619677447.640134:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.640135:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.640137:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.640137:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.640139:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.640140:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:5.0:1619677447.640141:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.640142:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.640144:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.640144:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.640145:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.640145:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.640147:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:5.0:1619677447.640147:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.640149:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.640152:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:18.0:1619677447.640152:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.640152:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.640153:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.640153:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.640156:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.640156:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.0:1619677447.640156:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.640159:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.640159:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.640160:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.640160:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.640161:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677447.640162:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677447.640163:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677447.640164:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.640165:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:5.0:1619677447.640165:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.640166:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.0:1619677447.640167:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.640168:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.640168:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.640169:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.640169:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677447.640171:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.640172:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.640173:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.640176:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.640177:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:17.0:1619677447.640178:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.640179:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96725466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.640179:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:12.0:1619677447.640180:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677447.640180:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.640181:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:17.0:1619677447.640181:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.640182:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.640182:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:17.0:1619677447.640183:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:12.0:1619677447.640183:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.640183:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:12.0:1619677447.640185:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.640186:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:17.0:1619677447.640188:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.640188:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.640189:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:17.0:1619677447.640190:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.640192:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.640193:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.640194:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.640195:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.640196:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.640196:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.640197:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.640209:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.640211:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:8.0:1619677447.640211:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:22.0:1619677447.640212:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.640212:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:24.0:1619677447.640215:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:22.0:1619677447.640215:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.640215:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:22.0:1619677447.640217:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:8.0:1619677447.640217:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.640220:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.640222:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677447.640223:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.640224:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.640226:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.640226:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:8.0:1619677447.640227:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:22.0:1619677447.640229:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.640229:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.640230:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000400:00000200:8.0:1619677447.640231:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.640231:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.640232:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.640233:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.640236:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.640237:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.640238:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.640239:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.640239:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.640240:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.640241:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.640251:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.640254:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.640254:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.640256:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.640258:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.640258:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.640364:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.640367:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.640367:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.640369:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.640371:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.640372:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.640381:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.640383:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.640384:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.640385:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.640385:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.640386:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.640426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:12.0:1619677447.640468:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.640470:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.640470:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.640472:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.640474:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.640475:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.640477:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.640479:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.640480:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.640482:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.640484:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.640486:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677447.640487:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.640489:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.640490:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.640491:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.640491:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.640493:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.640494:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:24.0:1619677447.640495:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.640495:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.640495:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.640496:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.640497:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.640500:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.640500:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677447.640501:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677447.640502:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.640503:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.640504:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.640505:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.640506:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.640516:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.640517:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.640518:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.640518:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:13.0:1619677447.640519:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:12.0:1619677447.640519:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.640520:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.640521:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.640521:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.640521:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.640521:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:13.0:1619677447.640522:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.640522:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.640523:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:2.0:1619677447.640523:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677447.640524:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.640525:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.640526:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.640527:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.640529:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.640532:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.640533:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.640534:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.640536:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.640537:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.640538:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.640541:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.640542:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.640545:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.640546:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.640546:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.640548:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.640548:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.640549:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.640550:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677447.640562:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.640565:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.640565:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.640567:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.640568:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.640570:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.640585:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.640588:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.640589:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.640590:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.640591:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.640592:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.640663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.640668:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.640671:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.640673:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.640674:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.640675:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.640677:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.640678:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.640679:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.640680:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.640681:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.640684:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.640685:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.640688:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.640688:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.640689:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.640690:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.640691:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.640692:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.640692:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.640704:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.640707:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.640708:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.640708:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.640709:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.640712:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.640840:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.640843:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.640844:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.640845:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.640848:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.640849:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.640851:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.640852:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.640853:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.640854:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.2:1619677447.640855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:24.0:1619677447.640857:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.640859:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.640861:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.640862:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.640863:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.640864:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.640865:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.640866:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.640866:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.640876:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.640878:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.640879:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.640880:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.640882:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.640883:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.640900:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.640902:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.640903:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.640905:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.640906:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.640908:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677447.640922:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.640924:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.640926:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.640927:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.640929:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.640930:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.640946:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.640949:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.640950:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.640952:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.640954:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.640955:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.640957:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.640958:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.640959:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.640960:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.640963:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.640965:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.640967:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.640968:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.640969:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.640970:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.640971:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.640972:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.640972:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677447.640980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677447.640984:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.640987:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.640987:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.640989:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.640990:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.640991:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677447.641000:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.641003:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.641003:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.641007:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.641010:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.641011:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.641013:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.641014:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.641015:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.641016:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.641019:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.641021:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.641023:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677447.641024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:1.0:1619677447.641024:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.641024:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:13.0:1619677447.641026:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677447.641026:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.641026:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.641027:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.641027:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.641029:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.641030:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.641032:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.641033:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.641034:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677447.641039:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.641041:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.641042:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.641043:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.641044:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.641046:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.641058:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.641060:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.641061:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.641061:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.641062:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.641062:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.641098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.641145:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.641148:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.641148:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.641149:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.641151:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.641154:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.641190:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.641193:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677447.641194:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677447.641196:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.641198:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.641200:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:6.0:1619677447.641201:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.641202:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.641203:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.641204:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:6.0:1619677447.641207:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.641209:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:6.0:1619677447.641212:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.641213:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:6.0:1619677447.641214:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677447.641216:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677447.641217:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.641217:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677447.641218:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:8.0:1619677447.641222:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.641224:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.641225:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.641227:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.641229:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:8.0:1619677447.641230:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.641231:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:8.0:1619677447.641231:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677447.641232:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.641232:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96725450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677447.641234:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.641234:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.641235:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:8.0:1619677447.641235:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.641236:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:12.0:1619677447.641237:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:8.0:1619677447.641239:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.641241:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.641244:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.641245:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.641246:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.641248:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.641248:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.641249:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.641250:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.641254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.641262:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.641265:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.641266:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.641266:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.641268:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.641271:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.641305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.641308:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.641311:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.641313:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.641314:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.641315:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.641317:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.641318:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.641320:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.641321:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.641322:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.641326:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.641327:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.641330:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.641331:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.641332:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.641334:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.641334:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.641336:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.641337:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.641338:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.641342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.641348:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.641350:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.641352:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.641352:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677447.641517:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.641520:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.641521:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.641523:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.641524:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.641526:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677447.641527:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.641528:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.641529:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.641530:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677447.641534:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.641536:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677447.641539:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.641540:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677447.641540:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677447.641542:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.641542:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.641544:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.641545:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.641545:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.641548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.641555:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:1.0:1619677447.641556:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677447.641558:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:1.0:1619677447.641558:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.0:1619677447.641559:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.641559:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677447.641560:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.641560:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677447.641562:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.641565:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:2.0:1619677447.641664:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.641666:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.641668:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.641669:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.641672:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.641674:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.641675:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.641676:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:2.0:1619677447.641676:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.641677:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.641678:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:22.0:1619677447.641679:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.641680:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.641681:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.641682:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.641683:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.2:1619677447.641683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677447.641684:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.641688:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.641688:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:22.0:1619677447.641689:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.641689:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:22.0:1619677447.641691:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.641691:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.641691:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.641692:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.641693:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:22.0:1619677447.641694:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.641694:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.641695:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.641696:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.641697:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.641734:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.641736:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.641737:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.641739:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.641739:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.641742:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.641888:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.641890:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.641891:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.641893:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.641896:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.641898:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.641900:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.641901:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.641902:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.641903:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.641906:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.641908:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.641911:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.641911:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677447.641912:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.641912:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.641914:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.641915:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677447.641915:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.641915:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.641915:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:17.0:1619677447.641916:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677447.641917:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.641920:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.641922:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.641923:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.641925:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.641925:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.641927:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.641930:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.641932:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.641935:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.641936:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.641937:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.641939:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.641939:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.641940:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.641940:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677447.641972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.641974:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.641977:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.641979:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.641980:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.641982:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.641983:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677447.641985:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.641986:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.641987:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.641988:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677447.641992:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.641994:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677447.641996:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.641997:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677447.641998:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677447.642000:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677447.642000:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.642002:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.642003:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.642004:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:4.2:1619677447.642005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.642008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.642014:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.642016:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.642017:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.642018:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:4.2:1619677447.642045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677447.642255:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.642258:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.642258:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.642261:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.642263:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.642264:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.642266:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.642267:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.642268:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.642269:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.642271:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.642273:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.642275:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.642275:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.642276:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.642278:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.642278:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.642279:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.642279:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677447.642291:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.642297:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.642297:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.642298:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.642299:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.642300:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.642319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677447.642369:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.642372:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.642373:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.642375:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.642375:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.642378:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677447.642549:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677447.642552:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677447.642552:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677447.642555:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.642558:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.642560:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677447.642561:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.642562:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.642563:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.642564:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677447.642568:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.642570:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677447.642573:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.642574:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:18.0:1619677447.642575:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:18.0:1619677447.642577:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:18.0:1619677447.642577:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677447.642578:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677447.642579:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677447.642580:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677447.642590:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677447.642592:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677447.642592:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677447.642593:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:4.2:1619677447.642670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.642691:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.642694:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.642695:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.642698:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.642699:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.642701:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.642702:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.642704:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.642705:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.642706:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.642709:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.642711:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.642714:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.642715:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.642716:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:22.0:1619677447.642718:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:1.0:1619677447.642718:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.642718:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.642719:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.642720:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.642721:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.642722:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.642722:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:9.2:1619677447.642724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:22.0:1619677447.642725:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.642730:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.642730:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.642733:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.642734:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.642734:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.642736:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.642739:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.642742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.642749:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.642751:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.642752:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.642752:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.642753:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.642754:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.642800:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.642803:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.642804:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.642805:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.642808:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.642810:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.642811:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.642812:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.642813:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.642814:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.642818:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.642820:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.642823:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.642824:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.642824:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.642826:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.642827:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.642827:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.642827:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677447.642966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.0:1619677447.642969:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677447.642972:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:20.0:1619677447.642974:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.642975:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677447.642977:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.642978:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677447.642979:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.642981:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.642982:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.642983:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677447.642986:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.642987:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.642990:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.642990:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.642991:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.642993:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.642993:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677447.642994:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.0:1619677447.642995:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:5.0:1619677447.643012:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.643014:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:8.0:1619677447.643015:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000400:00000200:5.0:1619677447.643015:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.643016:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.643017:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.643018:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:5.0:1619677447.643018:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:24.0:1619677447.643019:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677447.643025:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677447.643028:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000400:00000200:8.0:1619677447.643028:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.643029:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677447.643031:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.643031:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.643032:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.643033:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000400:00000200:8.0:1619677447.643033:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.643034:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:8.0:1619677447.643034:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.643035:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.643035:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:8.0:1619677447.643039:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677447.643040:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.643043:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.643044:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677447.643045:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677447.643046:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.643047:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677447.643047:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677447.643048:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:22.0:1619677447.643060:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.643063:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.643063:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.643064:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.643066:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.643068:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.643091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.643095:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:4.2:1619677447.643095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.643098:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.643100:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.643101:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.643103:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.643105:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677447.643107:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.643109:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.643110:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.643111:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677447.643115:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.643117:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677447.643120:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.643120:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.643121:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.643123:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.643123:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.643124:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.643125:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677447.643137:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.643139:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.643140:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.643140:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.643142:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.643144:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677447.643145:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:6.0:1619677447.643147:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.643148:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.643148:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.643151:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.643157:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.643180:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.643182:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.643183:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.643185:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.643187:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.643188:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.643197:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.643200:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.643201:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.643201:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.643202:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.643203:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677447.643411:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.643414:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.643415:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.643417:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.643420:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.643421:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.643423:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.643424:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.643425:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.643426:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.643430:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.643431:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.643434:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.643435:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.643435:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.643437:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.643437:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.643438:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.643438:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:4.2:1619677447.643441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.643452:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.643455:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.643456:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.643456:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.643459:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.643461:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.643470:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.643473:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.643473:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.643474:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.643475:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.643476:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677447.643491:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.643494:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.643495:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.643495:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.643497:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.643502:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.643503:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.643505:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.643506:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.643507:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.643507:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.643508:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.643669:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677447.643672:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677447.643673:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677447.643675:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.643676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677447.643677:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.643679:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677447.643680:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.643682:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.643683:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.643684:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677447.643688:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.643690:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677447.643693:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.643693:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677447.643694:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677447.643696:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677447.643696:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677447.643697:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677447.643698:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:4.2:1619677447.643724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.643726:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.643729:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.643730:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.643731:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.643732:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.643735:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.643793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:12.0:1619677447.643837:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.643840:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.643840:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.643842:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.643844:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.643845:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.643857:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.643857:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677447.643860:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.643861:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677447.643861:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.643861:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677447.643861:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:12.0:1619677447.643862:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.643863:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677447.643863:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.643866:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.643867:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.643869:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.643870:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.643871:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.643872:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.643877:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.643879:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.643882:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.643883:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.643883:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.643885:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.643886:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.643887:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.643888:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.643889:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.643899:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677447.643899:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677447.643901:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677447.643901:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:22.0:1619677447.643902:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.643902:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.643903:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.643904:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677447.643905:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.643907:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.643966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.643972:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.643975:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677447.643977:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.643978:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.643980:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.643982:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.643983:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.643984:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.643985:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.643986:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.643990:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.643992:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.643995:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.643995:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.643996:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.643998:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.643999:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.644000:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.644000:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:4.2:1619677447.644011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.644040:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.644043:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.644044:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.644045:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.644048:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.644049:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.644050:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.644052:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.644053:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.644054:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.644057:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.644059:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.644062:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.644063:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.644063:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.644065:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.644066:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.644066:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.644067:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677447.644078:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.644081:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.644082:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.644082:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.644084:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.644086:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.644178:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fdae00] (21)++
00000800:00000200:20.0:1619677447.644180:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.38@o2ib3
00000400:00000200:20.0:1619677447.644182:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.38@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.644184:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (22)--
00000400:00000200:20.0:1619677447.644185:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.644188:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677447.644189:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.644190:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.644191:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.644192:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677447.644196:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.38@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.644198:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677447.644201:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.644201:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677447.644202:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677447.644204:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677447.644204:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fdae00] (21)++
00000800:00000200:20.0:1619677447.644205:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fdae00] (22)--
00000800:00000200:20.0:1619677447.644206:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (21)--
00000800:00000200:9.2:1619677447.644210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.644217:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.644219:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.644220:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.644220:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.644223:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677447.644226:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.644394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.644401:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.644404:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.644406:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.644406:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.644408:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.644411:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677447.644412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.644421:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677447.644422:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.644424:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677447.644425:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.644426:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677447.644427:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.644427:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.644429:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.644430:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.644431:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.644432:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677447.644433:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.644434:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:5.0:1619677447.644435:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.644435:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.644436:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.644437:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.644437:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.644438:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.644439:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:5.0:1619677447.644445:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.644445:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.644448:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:5.0:1619677447.644449:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.644451:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.644452:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.644453:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.644453:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:5.0:1619677447.644454:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.644454:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677447.644456:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.644456:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677447.644457:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677447.644457:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.644458:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.644458:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677447.644458:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.644459:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.644470:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677447.644471:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:12.0:1619677447.644472:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:17.0:1619677447.644473:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:12.0:1619677447.644473:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.644474:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.644474:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:12.0:1619677447.644475:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.644476:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.644477:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.644479:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677447.644481:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677447.644578:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.644581:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.644581:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.644583:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.644586:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.644587:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.644588:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.644590:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.644590:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.644592:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.644595:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.644597:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.644599:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.644600:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.644601:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.644602:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.644603:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.644603:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.644604:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677447.644617:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.644619:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.644620:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.644620:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.644622:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.644624:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.644636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.644644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:22.0:1619677447.644681:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.644683:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.644684:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.644687:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.644690:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.644692:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.644694:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.644695:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:22.0:1619677447.644696:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.644697:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.644697:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:22.0:1619677447.644698:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677447.644698:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.644700:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.644701:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.644703:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:22.0:1619677447.644704:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.644706:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.644709:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.644711:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.644712:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.644713:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.644714:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.644716:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.644717:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.644718:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.644720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.644728:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677447.644728:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:6.0:1619677447.644730:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:2.0:1619677447.644731:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.644732:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.644732:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677447.644732:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.644733:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:2.0:1619677447.644734:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.644737:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.644739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.644746:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.644749:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.644750:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.644750:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.644751:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.644752:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.644860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.644867:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.644870:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.644871:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.644873:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.644875:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.644877:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.644878:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.644879:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.644881:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.644882:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.644885:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677447.644885:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.644887:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:8.0:1619677447.644887:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.644888:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.644890:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.644890:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.644891:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.644892:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:17.0:1619677447.644893:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.644893:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.644894:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:8.0:1619677447.644894:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.644895:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.644895:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677447.644896:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.644897:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.644898:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.644899:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.644903:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.644904:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.644907:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.644909:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.644909:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.644910:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.644910:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.644911:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.644912:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.644912:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.644913:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.644913:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.644914:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677447.644915:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.644916:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677447.644925:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.644928:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.644929:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.644929:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.644931:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.644933:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.645050:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.645053:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.645054:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.645056:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.645058:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.645060:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677447.645061:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.645062:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.645063:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.645064:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677447.645067:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.645069:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677447.645071:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.645072:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677447.645073:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677447.645075:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677447.645075:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.645077:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.645078:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.645078:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:24.0:1619677447.645090:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.645092:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:18.0:1619677447.645092:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:8.0:1619677447.645093:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:18.0:1619677447.645094:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:24.0:1619677447.645095:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.645095:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.645096:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:24.0:1619677447.645098:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.645099:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.645100:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725405] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.645102:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.645103:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.645104:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.645106:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.645108:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.645111:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.645112:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.645113:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.645115:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.645115:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.645116:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.645116:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677447.645129:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.645132:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.645135:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.645135:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.645137:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.645140:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.645160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.645163:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.645165:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.645168:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.645168:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.645170:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.645171:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.645173:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.645174:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.645175:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.645176:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.645179:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.645181:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.645185:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.645185:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:6.0:1619677447.645185:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.645186:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677447.645188:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677447.645188:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677447.645188:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677447.645189:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.645189:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:6.0:1619677447.645189:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.645190:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:6.0:1619677447.645191:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.645194:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.645201:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.645204:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.645205:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.645207:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.645207:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.645209:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.645274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677447.645323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677447.645327:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.645330:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.645331:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.645333:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.645333:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.645338:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.645340:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.645343:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.645347:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.645348:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.645348:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.645349:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677447.645387:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.645390:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.645391:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.645392:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.645395:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.645396:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.645482:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.645486:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.645486:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.645488:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.645491:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.645493:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.645494:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.645495:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.645496:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.645497:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.645501:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.645503:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.645505:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.645506:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.645506:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.645508:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.645509:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.645509:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.645510:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.645511:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677447.645516:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.645519:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.645520:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677447.645522:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677447.645522:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:17.0:1619677447.645522:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.645523:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.645524:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677447.645524:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.645525:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.645525:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.645525:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:17.0:1619677447.645526:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:20.0:1619677447.645528:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.645528:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.645529:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.645529:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.645530:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:17.0:1619677447.645530:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.645531:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.645535:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.645537:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.645539:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.645540:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.645541:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.645542:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.645543:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.645544:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.645544:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.645556:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.645559:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.645560:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.645560:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.645563:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.645565:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.645587:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.645589:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.645590:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.645592:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.645593:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.645594:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.645641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.645647:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.645649:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.645652:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.645652:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.645654:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.645656:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.645657:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.645659:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.645660:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.645661:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.645664:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.645666:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.645669:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.645670:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.645671:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.645673:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.645674:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.645675:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.645676:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.645677:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.645682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.645685:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677447.645685:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677447.645687:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.645687:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.645688:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.645688:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.645689:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.645691:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.645692:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.645694:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.645709:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.645711:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.645712:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.645713:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.645713:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.645714:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.645752:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.645755:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.645756:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.645758:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.645760:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.645761:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.645763:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.645764:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.645765:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.645766:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.645769:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.645771:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.645774:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.645775:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.645775:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.645777:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.645777:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.645778:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.645778:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.645783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.645790:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.645793:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.645794:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.645794:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.645796:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.645799:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677447.645839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:4.2:1619677447.645888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.645898:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.645900:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.645901:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.645902:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677447.645903:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.645904:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.645906:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.645907:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.645908:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.645909:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.645911:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.645912:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.645913:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.645914:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.645915:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96725393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.645917:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.645917:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.645918:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.645919:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.645919:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.645922:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.645923:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.645923:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.645924:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.645925:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.645926:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.645927:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.645927:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.645928:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.645928:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.645929:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.645929:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677447.645930:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.645932:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.645932:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.645935:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (23)++
00000800:00000200:24.0:1619677447.645936:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (24)--
00000800:00000200:8.0:1619677447.645936:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (24)--
00000800:00000200:24.0:1619677447.645937:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:1.0:1619677447.645940:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.645942:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.645943:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.645944:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.645946:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:1.0:1619677447.645946:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.645948:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.645949:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.645949:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:1.0:1619677447.645949:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:20.0:1619677447.645951:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.645952:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:5.0:1619677447.645952:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.645953:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.645954:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677447.645955:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:12.0:1619677447.645967:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.645969:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.645970:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.645972:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.645974:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.645975:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.646025:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.646028:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.646028:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.646031:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.646033:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.646035:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.646036:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.646037:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.646038:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.646039:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.646043:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.646046:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.646049:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.646050:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.646051:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.646053:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.646054:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.646055:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.646055:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.646066:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.646069:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.646070:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.646071:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.646071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:13.0:1619677447.646073:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.646074:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.646078:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.646081:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.646082:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.646082:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.646085:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.646088:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.646151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.646158:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.646161:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.646161:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.646162:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.646163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:18.0:1619677447.646164:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.646167:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677447.646169:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.646172:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.646174:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.646174:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.646177:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.646178:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.646179:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.646181:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.646182:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.646183:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.646187:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.646188:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.646190:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.646191:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.646192:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.646193:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.646194:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.646194:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.646195:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.646207:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.646209:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.646210:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.646211:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.646213:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.646215:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.646215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.646223:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.646226:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.646227:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.646227:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.646228:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.646229:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.646260:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.646263:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.646264:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.646264:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.646266:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.646269:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.646274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.646277:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.646279:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.2:1619677447.646282:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677447.646283:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.646286:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677447.646289:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.646290:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677447.646292:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.646297:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.646299:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.646300:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677447.646304:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.646306:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.646308:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.646309:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677447.646310:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.646312:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.646312:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.646313:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.646314:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677447.646328:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.646329:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.646330:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.646331:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.646331:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677447.646333:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000400:00000200:1.0:1619677447.646334:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.646336:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:1.0:1619677447.646336:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.646337:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.646338:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:1.0:1619677447.646338:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:17.0:1619677447.646339:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677447.646339:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:1.0:1619677447.646339:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.646340:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.646341:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677447.646342:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.646343:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.646344:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:1.0:1619677447.646344:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:12.0:1619677447.646345:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:1.0:1619677447.646348:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.646350:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.646351:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.646353:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.646353:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.646354:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.646354:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.646355:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (24)--
00000800:00000200:1.0:1619677447.646355:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (24)++
00000800:00000200:17.0:1619677447.646356:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (24)--
00000800:00000200:1.0:1619677447.646356:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (24)--
00000800:00000200:17.0:1619677447.646357:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.646357:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.646358:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.646360:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.646360:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.646370:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.646371:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.646372:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.646372:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.646373:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.646373:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.646374:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.646375:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.646375:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.646376:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677447.646416:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.646419:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.646420:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.646421:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.646424:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.646426:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.646427:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.646428:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.646429:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.646431:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.646434:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.646436:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.646438:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.646439:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.646440:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.646441:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.646442:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.646442:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.646443:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.646454:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.646456:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.646457:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.646459:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.646460:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.646461:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.646482:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.646485:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.646486:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.646488:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.646490:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.646492:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.646493:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.646494:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.646495:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.646496:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.646500:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.646502:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.646505:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.646506:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.646506:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.646508:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.646508:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.646509:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.646509:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677447.646693:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.646696:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.646696:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.646698:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.646700:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.646701:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:4.2:1619677447.646702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:17.0:1619677447.646703:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.646704:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.646705:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.646706:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.646709:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:17.0:1619677447.646710:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.646712:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.646712:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.646713:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677447.646715:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.646716:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:8.0:1619677447.646716:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.646717:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:8.0:1619677447.646718:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.646719:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.646719:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.646720:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.646720:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677447.646720:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.646721:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96725380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.646723:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.646724:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.646725:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.646728:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.646730:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677447.646731:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.0:1619677447.646732:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.646733:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:12.0:1619677447.646734:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.646734:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.646734:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:12.0:1619677447.646736:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.646736:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.646736:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.646737:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:13.0:1619677447.646738:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.646738:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:12.0:1619677447.646739:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.646739:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.646745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.646749:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.646752:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.646752:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.646753:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.646753:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.646754:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.646755:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.646756:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.646756:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.646759:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:9.2:1619677447.646777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.646785:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.646788:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.646789:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.646790:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.646791:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.646794:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677447.646796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:20.2:1619677447.646837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:5.0:1619677447.646844:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.646846:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.646847:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.646849:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.646849:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.646852:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.646956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677447.646966:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.646969:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.646970:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.646973:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.646975:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.646978:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.2:1619677447.646979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:18.0:1619677447.646979:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.646981:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.646982:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:18.0:1619677447.646982:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.646984:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:18.0:1619677447.646984:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.646987:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.646988:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.646989:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.646991:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.646992:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:18.0:1619677447.646993:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.646994:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96725375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.646996:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.646997:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.646997:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.646998:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.646998:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.646998:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.647000:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.647001:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:20.0:1619677447.647002:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.647002:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.647003:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:20.0:1619677447.647004:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.647004:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.0:1619677447.647006:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.647007:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.647008:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.647009:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.647010:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.647011:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.647011:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.647013:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.647014:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:13.0:1619677447.647015:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.647016:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.647016:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.647017:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:13.0:1619677447.647018:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.647018:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677447.647019:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.647020:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.647023:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.647025:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.647026:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.647028:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.647028:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.647029:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:12.0:1619677447.647032:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.647034:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.647036:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.647037:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.647038:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.647039:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.647107:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.647109:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.647110:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.647112:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.647115:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.647116:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.647118:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.647119:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.647120:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.647121:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.647125:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.647126:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.647130:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.647131:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.647131:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.647133:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.647134:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.647134:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.647135:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.647264:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.647267:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.647268:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.647270:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.647273:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.647274:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.647275:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.647277:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.647278:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.647279:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.647282:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.647284:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.647287:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.647288:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.647288:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.647290:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.647291:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.647291:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.647292:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.647549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.647554:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.647556:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.647557:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.647560:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.647563:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.647564:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.647566:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.647569:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.647570:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.647571:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.647576:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.647578:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.647580:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.647581:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.647582:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.647583:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.647584:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.647584:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.647585:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677447.647596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677447.647597:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.647600:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.647600:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.647602:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:1.0:1619677447.647602:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.647603:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.647605:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:6.0:1619677447.647605:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.647606:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.647606:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.647609:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.647611:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.647616:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.647619:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.647620:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.647620:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.647621:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.647622:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677447.647676:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.647679:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.647679:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.647681:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.647684:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.647686:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677447.647687:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.647689:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.647690:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.647691:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677447.647695:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.647697:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.647700:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.647701:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:18.0:1619677447.647701:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677447.647703:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.647704:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.647705:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.647705:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677447.647717:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.647719:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.647720:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.647722:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.647724:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.647725:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.647736:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.647739:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.647740:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.647740:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.647742:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.647744:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.648015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.648059:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.648061:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.648062:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.648064:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.648065:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.648067:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.648086:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.648089:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:22.0:1619677447.648090:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677447.648090:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.648091:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:22.0:1619677447.648092:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:20.0:1619677447.648092:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.648092:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:6.0:1619677447.648093:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.648095:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.648098:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.648100:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.648101:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.648102:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.648103:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.648104:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.648107:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.648109:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.648111:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.648112:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.648113:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.648114:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.648115:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.648116:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.648117:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.648117:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677447.648128:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677447.648128:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.648130:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:17.0:1619677447.648131:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:6.0:1619677447.648131:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:17.0:1619677447.648132:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.648132:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677447.648132:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.648134:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.648137:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.648151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.648159:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.648162:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.648163:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.648165:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.648168:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.648170:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.648171:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.648172:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.648173:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.648174:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.648178:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.648179:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.648182:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.648183:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.648184:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.648185:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.648186:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.648187:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.648187:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.648199:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.648202:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.648202:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.648203:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.648205:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.648208:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.648210:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.648212:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.648213:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.648214:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.648215:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000400:00000200:1.0:1619677447.648215:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.648216:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.648216:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.648217:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.648217:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.648218:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:1.0:1619677447.648218:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.648220:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.648221:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.648223:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.648224:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.648227:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.648229:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.648231:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.648232:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.648233:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.648234:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.648235:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.648236:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.648237:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.648237:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.648248:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.648250:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.648250:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.648251:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677447.648306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.648457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.648464:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.648467:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.648469:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.648469:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.648472:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.648475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.648478:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.648480:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.648482:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.648482:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.648483:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.648484:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.648485:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.648503:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.648506:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.648506:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.648507:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.648509:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.648512:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:24.0:1619677447.648574:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677447.648577:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677447.648578:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677447.648579:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677447.648583:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.648584:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677447.648585:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.648587:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.648588:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.648588:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677447.648591:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677447.648592:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.648594:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.648594:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.648595:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.648596:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.648597:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677447.648598:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677447.648598:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677447.648602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.648608:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.648610:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.648611:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.648612:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.648612:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677447.648614:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.648619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.648644:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.648646:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.648647:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.648648:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.648650:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.648653:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.648847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.648853:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:4.2:1619677447.648853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.648856:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.648858:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.648859:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.648860:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.648862:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677447.648863:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.648865:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.648866:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.648867:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677447.648871:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.648873:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.648876:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.648877:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.648878:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.648879:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.648880:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.648881:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.648881:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677447.648893:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.648895:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.648896:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.648898:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.648898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677447.648899:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.648901:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677447.648903:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.648906:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.648907:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.648907:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.648909:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.648912:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.648937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677447.648948:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.648949:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.648950:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.648951:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:18.0:1619677447.648953:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.648953:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.648956:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.648956:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.648958:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.648960:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.648961:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.648963:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.648965:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677447.648967:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.648968:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96725354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.648970:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.648970:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.648971:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.648972:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.648973:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:2.0:1619677447.648976:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.648978:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.648980:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.648980:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.648981:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.648981:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.648982:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.648982:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677447.648984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677447.648984:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.648984:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.648985:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.648985:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.648986:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.648986:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.648987:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.648987:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.648988:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677447.648988:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677447.648988:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677447.648990:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.648991:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.648991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:12.0:1619677447.648993:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.648995:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.648996:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677447.648998:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.649001:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.649002:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.649002:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.649004:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.649007:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.649042:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.649045:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.649045:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.649047:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.649050:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.649052:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.649053:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.649054:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.649055:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.649056:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.649059:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.649060:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.649062:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.649063:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.649063:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.649065:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.649066:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.649067:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.649067:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677447.649080:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.649082:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.649083:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.649083:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.649085:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.649088:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.649250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.649285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.649292:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.649298:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.649300:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.649301:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.649303:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.649306:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.649420:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.649423:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.649424:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.649426:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.649429:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.649430:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.649432:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.649433:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.649434:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.649435:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.649438:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.649440:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.649443:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.649443:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.649444:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.649446:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.649447:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.649447:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.649448:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.649460:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.649463:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.649464:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.649466:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.649468:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.649468:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.649559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:1.0:1619677447.649606:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.649609:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.649610:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.649611:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.649613:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000400:00000200:1.0:1619677447.649613:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.649616:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:1.0:1619677447.649616:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:13.0:1619677447.649618:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.649619:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677447.649620:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.649621:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677447.649623:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96725348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.649624:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677447.649625:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677447.649626:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:13.0:1619677447.649629:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.649631:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677447.649634:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.649635:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:13.0:1619677447.649636:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:13.0:1619677447.649637:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:13.0:1619677447.649638:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677447.649639:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677447.649639:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677447.649643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.649651:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.649653:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.649655:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.649655:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.649657:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.649660:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.649665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.649672:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.649675:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.649675:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.649676:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.649677:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.649677:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:6.0:1619677447.649677:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.649680:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.649681:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.649681:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.649683:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.649686:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.649696:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.649698:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.649699:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.649699:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.649701:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.649701:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:17.0:1619677447.649815:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.649818:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.649818:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.649819:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.649821:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.649824:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.649869:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.649871:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.649872:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.649874:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.649876:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.649878:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.649879:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.649880:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.649881:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677447.649882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677447.649882:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.649884:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.649886:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.649888:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.649889:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:1.0:1619677447.649889:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.649890:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.649891:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:24.0:1619677447.649892:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:1.0:1619677447.649892:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677447.649893:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:1.0:1619677447.649893:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.649893:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677447.649895:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677447.649897:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.649898:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677447.649900:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96725344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.649902:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.649903:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.649904:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.649905:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:24.0:1619677447.649907:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:6.0:1619677447.649908:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:24.0:1619677447.649909:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.649909:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.649909:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.649912:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.649913:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.649914:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677447.649914:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677447.649916:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677447.649916:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677447.649917:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:6.0:1619677447.649917:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.649918:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677447.649919:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677447.649931:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677447.649933:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677447.649934:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677447.649935:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:22.0:1619677447.649940:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:22.0:1619677447.649943:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:6.0:1619677447.649943:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (24)++
00000400:00000200:22.0:1619677447.649944:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.649944:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000400:00000200:6.0:1619677447.649945:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.649946:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:6.0:1619677447.649947:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.649950:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.649950:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.649951:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.649953:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.649953:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.649954:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.649955:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.649955:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.650016:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.650019:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.650020:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.650022:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.650024:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.650026:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.650027:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677447.650028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:2.0:1619677447.650028:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.650029:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.650030:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.650033:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.650035:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677447.650035:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.650038:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677447.650038:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.650039:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.650039:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.650040:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.650040:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.650041:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.650041:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:24.0:1619677447.650042:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:5.0:1619677447.650042:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.650042:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.650043:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:5.0:1619677447.650044:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:24.0:1619677447.650045:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:5.0:1619677447.650046:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.650048:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.650049:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.650050:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.650050:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:8.0:1619677447.650051:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.650053:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.650054:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.650055:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:20.0:1619677447.650055:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:24.0:1619677447.650056:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96725340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.650056:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.650056:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:24.0:1619677447.650057:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.650057:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.650059:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.650059:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.650059:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.650060:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.650060:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677447.650061:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677447.650063:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677447.650063:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:24.0:1619677447.650064:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.650065:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:24.0:1619677447.650066:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.650066:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.650066:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.650067:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:9.2:1619677447.650068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677447.650068:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:24.0:1619677447.650069:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.650070:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.650071:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677447.650071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:24.0:1619677447.650073:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.650073:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.650074:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677447.650074:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:24.0:1619677447.650075:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677447.650076:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:18.0:1619677447.650076:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:20.0:1619677447.650077:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.650078:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.650078:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:9.2:1619677447.650079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677447.650080:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.650080:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.650082:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.650082:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.650083:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.650084:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.650085:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:18.0:1619677447.650086:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.650086:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.650089:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.650096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.650099:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.650102:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.650104:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.650105:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.650106:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.650108:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.650110:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.650111:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.650112:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.650113:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.650117:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.650119:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.650122:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.650123:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.650123:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.650125:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.650126:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.650127:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.650127:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677447.650191:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.650194:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.650195:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.650197:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.650199:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.650201:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.650202:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.650204:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.650205:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.650206:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.650209:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.650211:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.650214:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.650215:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.650216:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.650217:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.650218:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.650218:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.650219:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.650229:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.650232:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.650233:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.650235:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.650236:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.650238:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.650259:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.650262:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.650263:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.650263:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.650266:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.650268:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.650304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677447.650319:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.650321:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.650325:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.650325:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.650328:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.650329:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.650331:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.650332:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.650333:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.650334:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.650338:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.650340:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.650343:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.650343:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.650344:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.650345:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.650346:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.650347:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.650348:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:13.0:1619677447.650353:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.650355:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.650356:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.650358:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.650360:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:12.0:1619677447.650360:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.650361:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677447.650362:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.650363:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.650364:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.650365:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.650367:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:22.0:1619677447.650375:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.650377:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.650378:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.650380:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.650382:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.650384:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.650385:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.650386:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.650388:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.650389:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.650392:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.650394:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.650397:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.650397:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.650398:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.650400:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.650400:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.650401:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.650401:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.650406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.650414:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.650417:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.650418:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.650418:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.650420:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.650422:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677447.650494:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.650496:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.650497:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.650499:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.650501:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.650503:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.650504:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.650505:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.650506:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.650507:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.650510:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.650512:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.650514:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.650515:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.650515:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.650517:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.650517:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.650519:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.650520:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.650520:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.650531:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:1.0:1619677447.650531:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677447.650533:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.650533:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.650534:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677447.650534:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.650535:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.650535:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.650537:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.650539:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.650571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.650580:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677447.650583:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.650583:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.650585:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.650586:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677447.650586:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.650587:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677447.650588:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.650589:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.650590:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.650591:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.650592:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.650593:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.650593:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.650594:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.650594:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.650595:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.650596:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.650597:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.650598:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.650603:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.650605:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.650605:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.650607:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.650609:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.650610:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.650611:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.650611:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.650611:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.650612:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.650612:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.650613:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.650614:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:5.0:1619677447.650614:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.650615:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.650615:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677447.650615:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.650616:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.650616:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.650625:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677447.650626:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677447.650627:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.650628:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.650628:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.650629:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.650629:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677447.650630:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.650631:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.650633:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.650639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.650648:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.650650:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.650652:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.650652:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.650654:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.650657:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.650661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.650668:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.650671:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.650671:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.650672:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.650674:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.650674:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:30.0:1619677447.650678:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.650679:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.650680:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677447.650684:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.650685:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677447.650688:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.650689:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677447.650689:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677447.650691:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677447.650692:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.650692:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.650693:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677447.650704:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.650707:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.650708:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.650708:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.650710:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.650714:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.650724:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.650727:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.650727:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.650728:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.650728:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.650729:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.650868:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.650870:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.650871:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.650873:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.650875:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.650876:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.650877:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.650879:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.650880:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.650880:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.650883:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.650885:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.650887:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.650887:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.650888:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.650890:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.650890:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.650891:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.650891:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677447.650902:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.650904:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.650905:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.650906:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.650907:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.650908:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.650914:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.650917:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.650918:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.650919:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.650921:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.650923:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.650924:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.650925:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.650927:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.650928:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.650931:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.650933:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.650936:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.650937:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.650937:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.650939:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.650940:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.650940:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.650941:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677447.650952:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.650955:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.650956:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.650957:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.650958:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.650960:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.650971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.650973:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.650976:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.650978:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.650979:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.650981:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.650983:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677447.650984:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725326] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.650986:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.650987:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.650988:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677447.650991:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.650993:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677447.650996:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.650997:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.650997:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.650999:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.651000:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.651000:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.651001:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.2:1619677447.651010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677447.651012:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.651015:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.651016:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.651016:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.651018:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.651020:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677447.651025:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.651026:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.651027:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677447.651031:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.651034:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.651034:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.651035:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000400:00000200:30.0:1619677447.651036:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.651036:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.651036:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:30.0:1619677447.651039:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.651040:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677447.651042:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96725324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.651043:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.651044:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.651045:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677447.651048:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.651049:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677447.651052:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.651053:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677447.651053:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677447.651055:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.651056:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.651057:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.651058:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.651059:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.0:1619677447.651067:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677447.651069:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.0:1619677447.651070:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:6.0:1619677447.651071:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.651072:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:5.0:1619677447.651072:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.651073:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.651074:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:5.0:1619677447.651074:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.651075:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.651076:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.651077:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:20.0:1619677447.651078:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:5.0:1619677447.651078:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (24)--
00000800:00000200:20.0:1619677447.651079:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.651079:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677447.651080:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677447.651221:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677447.651224:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677447.651225:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677447.651227:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.651228:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.651230:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677447.651231:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.651233:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.651234:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.651235:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677447.651239:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.651240:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677447.651243:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.651244:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:1.0:1619677447.651245:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:1.0:1619677447.651246:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:1.0:1619677447.651247:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677447.651247:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677447.651248:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:22.0:1619677447.651310:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.651312:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.651313:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.651315:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.651318:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.651319:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.651320:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.651322:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.651323:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.651324:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.651327:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.651329:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.651331:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.651332:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.651333:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.651334:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.651335:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.651336:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.651336:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.651341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.651350:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.651353:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.651354:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.651354:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.651356:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.651358:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677447.651387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.651390:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.651393:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.651395:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.651396:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.651398:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.651399:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.651400:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.651402:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.651403:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.651404:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.651407:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.651409:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.651412:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.651413:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.651413:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.651415:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.651415:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.651416:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.651417:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.651428:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.651431:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.651432:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.651432:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.651434:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.651436:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.651437:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07de000] (21)++
00000800:00000200:17.0:1619677447.651440:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.1@o2ib3
00000800:00000200:2.0:1619677447.651441:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677447.651441:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07de000] (22)--
00000800:00000200:5.0:1619677447.651442:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:17.0:1619677447.651443:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.1@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.651443:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.651444:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:5.0:1619677447.651445:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:17.0:1619677447.651446:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.651448:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:17.0:1619677447.651448:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.651448:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677447.651449:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.651449:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.651450:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.651451:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.651452:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.651452:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.651452:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:17.0:1619677447.651453:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.651454:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677447.651454:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.651455:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.651456:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.651457:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.651457:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.651458:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.651459:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.651459:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677447.651460:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:17.0:1619677447.651462:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.651466:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677447.651466:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.651466:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677447.651467:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.651468:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.651469:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:17.0:1619677447.651470:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.651470:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07de000] (21)++
00000800:00000200:5.0:1619677447.651470:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677447.651470:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.651471:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07de000] (22)--
00000800:00000200:17.0:1619677447.651471:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07de000] (21)--
00000800:00000200:2.0:1619677447.651471:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677447.651472:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.651472:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:2.0:1619677447.651472:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677447.651473:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:2.0:1619677447.651473:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.651474:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.0:1619677447.651475:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.651475:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:2.0:1619677447.651475:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.651475:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.0:1619677447.651477:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677447.651478:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:5.0:1619677447.651478:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:9.2:1619677447.651480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:12.0:1619677447.651483:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.651486:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.651486:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.651487:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:20.0:1619677447.651489:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:18.0:1619677447.651489:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:12.0:1619677447.651489:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.651489:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677447.651490:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.651490:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.651490:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.651491:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:2.0:1619677447.651491:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:20.0:1619677447.651492:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.651492:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:12.0:1619677447.651492:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.651492:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000400:00000200:20.0:1619677447.651493:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.651493:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:1.0:1619677447.651493:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.651494:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677447.651494:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:18.0:1619677447.651495:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677447.651495:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677447.651496:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:9.2:1619677447.651496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000400:00000200:1.0:1619677447.651496:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.651499:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.651500:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.651501:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.651502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677447.651502:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.651503:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.651504:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.651504:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000400:00000200:1.0:1619677447.651504:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.651506:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:2.0:1619677447.651506:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:20.2:1619677447.651508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677447.651508:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:13.0:1619677447.651508:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.651508:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000400:00000200:1.0:1619677447.651508:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.651509:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.651509:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:2.0:1619677447.651509:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.651509:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677447.651510:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:13.0:1619677447.651511:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.651511:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.651511:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677447.651511:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:1.0:1619677447.651511:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.651512:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.651513:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:1.0:1619677447.651513:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.651514:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.651515:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.651516:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.651517:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.651518:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.651518:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.651523:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677447.651529:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.651531:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.651532:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.651532:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677447.651533:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.651535:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:17.0:1619677447.651535:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.651536:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:30.0:1619677447.651537:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.651537:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:30.0:1619677447.651539:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.651540:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:30.0:1619677447.651541:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:24.0:1619677447.651541:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:30.0:1619677447.651542:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96725314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.651543:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.651544:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.651545:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.651546:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:8.0:1619677447.651546:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.651549:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.651551:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:22.0:1619677447.651552:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:30.0:1619677447.651553:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677447.651553:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:30.0:1619677447.651554:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677447.651554:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:8.0:1619677447.651554:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96725313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.651554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677447.651555:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:30.0:1619677447.651556:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.651556:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:8.0:1619677447.651556:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.651556:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.651557:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677447.651557:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.651557:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.651558:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:8.0:1619677447.651558:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.651558:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:22.0:1619677447.651559:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.651561:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:24.0:1619677447.651562:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.651562:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:5.0:1619677447.651562:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.651563:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.651564:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:20.0:1619677447.651564:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.651564:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.651566:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.651566:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.651567:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:24.0:1619677447.651569:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.651570:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.651571:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:8.0:1619677447.651571:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.651571:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:24.0:1619677447.651573:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96725312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.651573:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:24.0:1619677447.651575:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.651576:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.651576:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.651577:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:24.0:1619677447.651580:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.651582:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.651583:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.651584:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677447.651585:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.651585:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.651586:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.651586:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.651587:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:24.0:1619677447.651589:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.651589:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677447.651589:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:24.0:1619677447.651590:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.651590:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:20.0:1619677447.651590:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.651592:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.651593:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.651594:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.651603:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.651605:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.651606:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.651607:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.651609:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.651611:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.651698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.651704:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.651707:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.651709:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.651709:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.651711:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.651713:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677447.651714:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.651716:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.651717:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.651718:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677447.651721:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.651723:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.651725:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.651727:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677447.651727:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677447.651729:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677447.651730:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.651731:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.651732:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.651733:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.2:1619677447.651738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.651741:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.651743:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.651744:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.651745:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.2:1619677447.651798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.651801:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.651803:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.651806:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.651807:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.651808:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.651810:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.651811:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.651812:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.651813:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.651814:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.651817:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.651819:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.651821:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.651822:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.651823:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.651825:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.651825:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.651826:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.651827:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.651828:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.651833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.651838:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:6.0:1619677447.651839:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.651841:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677447.651841:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:22.0:1619677447.651842:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677447.651842:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:6.0:1619677447.651843:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.651843:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:6.0:1619677447.651845:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.651847:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.651857:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.651859:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.651860:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.651860:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.651861:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.651862:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.651872:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.651875:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.651876:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.651878:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.651880:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.651882:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.651883:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725309] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.651884:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.651885:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.651886:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.651889:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.651890:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.651893:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.651893:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.651894:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.651896:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.651896:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.651896:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.651897:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.651910:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.651912:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.651913:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.651915:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.651916:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.651917:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.651931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.651946:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677447.651978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677447.651984:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:30.0:1619677447.651986:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677447.651986:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.651987:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.651988:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.651989:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677447.651989:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.651990:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677447.651990:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.651992:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:30.0:1619677447.652062:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.652064:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.652066:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.652069:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.652071:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677447.652072:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96725307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.652073:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.652074:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.652075:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677447.652079:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.652080:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.652083:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.652084:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677447.652085:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677447.652086:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.652087:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.652088:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.652088:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677447.652100:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.652103:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.652103:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.652104:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.652106:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.652109:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.652219:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.652222:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.652222:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.652224:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.652226:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.652227:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.652228:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.652229:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.652230:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.652231:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.652234:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.652236:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.652238:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.652238:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.652239:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.652241:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.652241:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.652242:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.652242:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677447.652247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.652254:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.652254:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.652256:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.652257:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.652257:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:1.0:1619677447.652257:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.652258:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677447.652259:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.652260:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:8.0:1619677447.652261:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.652264:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.652265:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.652266:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.652268:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.652269:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.652270:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.652273:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.652275:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.652278:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.652279:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.652279:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.652281:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.652281:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.652282:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.652283:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.652296:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.652301:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.652302:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.652303:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.652306:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.652310:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.652312:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.652314:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.652315:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.652316:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.652316:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.652316:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:12.0:1619677447.652322:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.652325:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.652326:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.652328:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.652329:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.652331:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.652375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.652392:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677447.652406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.652412:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.652414:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.652417:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.652417:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.652419:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:5.0:1619677447.652419:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.652421:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:5.0:1619677447.652421:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677447.652422:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.652422:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.652424:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.652424:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.652425:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.652426:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:5.0:1619677447.652426:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.652430:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.652431:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.652434:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.652435:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.652436:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.652438:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.652439:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.652440:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.652440:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.652444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.652452:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.652455:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.652456:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.652456:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.652458:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.652460:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.652572:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.652575:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.652575:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.652577:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.652580:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.652581:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.652583:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.652584:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.652585:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.652586:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.652590:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.652591:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.652594:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.652595:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.652596:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.652598:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.652598:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.652599:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.652600:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.652601:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.652611:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.652613:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.652613:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.652614:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677447.652618:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.652691:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.652694:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.652695:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.652696:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.652698:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.652699:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.652726:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.652729:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.652731:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.652731:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.652733:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.652734:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.652736:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.652737:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.652738:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.652739:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.652742:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.652744:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.652746:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.652747:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.652747:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.652749:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.652749:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.652750:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.652751:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677447.652762:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.652764:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.652765:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.652767:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.652768:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.652769:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:1.0:1619677447.652897:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.652900:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.652900:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.652903:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.652904:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.652905:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.652907:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.652908:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.652909:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.652910:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677447.652910:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:30.0:1619677447.652913:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:1.0:1619677447.652914:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.652916:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:30.0:1619677447.652917:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.652918:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677447.652919:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.652919:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.652919:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:30.0:1619677447.652921:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:1.0:1619677447.652921:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.652921:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677447.652922:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96725297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.652923:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:30.0:1619677447.652924:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.652924:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.652924:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677447.652925:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.652926:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677447.652929:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.652931:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677447.652934:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.652934:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677447.652935:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677447.652936:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:17.0:1619677447.652936:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.652936:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.652936:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:30.0:1619677447.652937:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677447.652937:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.652937:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677447.652938:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677447.652938:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.652938:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.652939:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:17.0:1619677447.652940:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.652940:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.652942:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.652974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.653013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677447.653033:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.653212:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:24.0:1619677447.653213:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:1.0:1619677447.653214:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677447.653215:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:24.0:1619677447.653216:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.653217:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:1.0:1619677447.653218:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.653220:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.653220:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.653222:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.653222:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.653223:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.653224:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.653225:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.653226:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.653227:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.653228:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.653228:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677447.653229:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.653230:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:4.2:1619677447.653232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677447.653235:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.653236:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.653238:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.653239:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.653240:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.653241:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.653242:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.653242:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.653243:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.653244:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.653244:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.653245:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.653246:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:1.0:1619677447.653246:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.653246:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:24.0:1619677447.653247:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:1.0:1619677447.653247:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:24.0:1619677447.653248:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:1.0:1619677447.653248:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677447.653249:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:9.2:1619677447.653252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:13.0:1619677447.653256:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.653259:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.653259:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.653261:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000400:00000200:13.0:1619677447.653261:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.653261:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677447.653263:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:12.0:1619677447.653263:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.653263:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:17.0:1619677447.653264:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:24.0:1619677447.653265:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:13.0:1619677447.653265:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677447.653265:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.653265:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:24.0:1619677447.653266:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.653267:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.653267:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.653268:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.653269:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677447.653271:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.653280:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.653283:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.653284:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.653285:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.653287:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.653290:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.653323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:17.0:1619677447.653344:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.653347:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.653348:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.653350:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.653353:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.653355:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677447.653356:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.653358:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.653359:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.653360:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677447.653366:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.653369:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677447.653372:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.653373:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677447.653374:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677447.653375:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.653376:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.653377:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.653377:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677447.653381:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:9.2:1619677447.653382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677447.653387:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:24.0:1619677447.653389:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:12.0:1619677447.653389:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677447.653389:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.653389:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.653391:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:24.0:1619677447.653392:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:12.0:1619677447.653392:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.653393:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.653394:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.653394:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:12.0:1619677447.653395:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.653396:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.653396:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:13.0:1619677447.653397:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.653397:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.653397:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677447.653398:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.653399:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:24.0:1619677447.653400:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.0:1619677447.653400:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.653400:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.653400:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:9.2:1619677447.653403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:13.0:1619677447.653407:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.653409:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:13.0:1619677447.653410:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.653411:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.0:1619677447.653411:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.653412:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:13.0:1619677447.653412:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:8.0:1619677447.653412:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:13.0:1619677447.653413:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.653414:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.653414:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.653415:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.653574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.653581:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.653584:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.653586:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.653587:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.653589:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.653590:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.653591:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.653593:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.653594:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.653595:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.653598:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.653600:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.653603:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.653603:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.653604:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.653606:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.653606:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.653608:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.653609:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.653609:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.653615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.653620:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677447.653622:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.653622:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.653623:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.653623:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.653624:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.653625:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677447.653625:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.653626:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.653628:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.653653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677447.653697:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.653700:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.653700:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.653702:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.653704:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.653710:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.653713:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.653714:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677447.653715:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:22.0:1619677447.653716:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.653718:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.653718:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.653719:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.653719:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:22.0:1619677447.653719:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.653720:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:22.0:1619677447.653721:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.653722:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.653723:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.653724:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.653725:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.653729:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.653731:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.653734:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.653735:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.653736:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.653738:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.653738:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.653739:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.653739:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677447.653751:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.653754:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.653754:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.653756:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.653759:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677447.653800:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.653803:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.653804:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.653806:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.653809:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.653810:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.653812:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.653813:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.653814:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.653815:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.653818:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.653820:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.653823:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.653824:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.653825:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.653827:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.653827:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.653828:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.653828:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677447.653837:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.653839:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.653841:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.653843:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.653845:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.653846:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:2.0:1619677447.653847:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677447.653849:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.653850:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.653851:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.653852:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677447.653855:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.653856:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.653860:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.653861:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677447.653862:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677447.653863:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.653864:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.653865:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.653865:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:4.2:1619677447.653875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.653877:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.653879:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.653880:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.653882:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.653883:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.653885:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677447.653901:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.653904:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.653906:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.653907:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.653909:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.653910:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.653911:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.653912:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.653914:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.653914:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.653918:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.653920:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677447.653921:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.653923:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.653924:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.653924:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:6.0:1619677447.653924:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.653925:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677447.653926:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000400:00000200:6.0:1619677447.653926:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.653927:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.653927:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.653928:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:6.0:1619677447.653928:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.653936:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.653940:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.653943:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.653943:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.653945:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.653946:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.653948:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677447.653965:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.653967:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.653968:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.653970:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.653970:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.653973:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677447.653974:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.653977:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.653978:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.653980:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.653982:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.0:1619677447.653999:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.654002:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.654003:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.654004:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.654005:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.654006:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677447.654025:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.654028:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.654029:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.654031:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.654034:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.654036:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677447.654037:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.654039:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.654040:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.654041:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677447.654044:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.654045:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677447.654049:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.654050:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677447.654050:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677447.654052:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677447.654053:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.654054:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.654054:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677447.654076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.654081:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.654084:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.654086:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.654087:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.654089:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.654090:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.654091:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.654092:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.654094:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.654095:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.654098:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.654100:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:4.2:1619677447.654101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.654102:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.654103:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.654103:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.654105:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.654106:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.654106:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.654107:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.654112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.654119:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.654121:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.654122:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.654123:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.654125:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.654128:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.654142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.654147:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.654149:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.654150:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.654151:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.654153:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.654156:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.654167:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.654170:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.654170:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.654171:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.654172:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.654172:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.654239:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.654242:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.654242:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.654244:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.654248:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.654249:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.654250:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.654252:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.654253:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.654254:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:18.0:1619677447.654257:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:1.0:1619677447.654257:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.654259:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.654260:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:1.0:1619677447.654260:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:18.0:1619677447.654262:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.654263:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.654264:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.654264:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:18.0:1619677447.654265:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.654266:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:1.0:1619677447.654266:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.654267:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.654267:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677447.654268:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725276] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.654268:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:18.0:1619677447.654269:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.654270:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.654271:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.654274:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.654276:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.654278:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.654279:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.654279:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.654281:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.654281:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.654283:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.654284:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.654284:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:30.0:1619677447.654285:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:9.2:1619677447.654287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.654298:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.654299:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.654300:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.654300:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.654301:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677447.654301:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.654302:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.654303:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.654304:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.654306:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.654307:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.654309:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.654309:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.654311:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.654313:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.654314:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.654315:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.654317:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.654318:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.654318:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.654321:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.654323:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.654325:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.654325:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.654326:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.654328:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.654328:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.654329:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.654329:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677447.654332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.654339:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.654341:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.654343:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.654343:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.654344:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.654345:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.654345:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.654347:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:5.0:1619677447.654348:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.654349:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.654350:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.654351:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.654352:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:5.0:1619677447.654355:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:20.0:1619677447.654356:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.654356:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.654357:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.654358:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:20.0:1619677447.654361:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.654362:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:20.0:1619677447.654363:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:20.0:1619677447.654365:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:20.0:1619677447.654365:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.654366:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.654367:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677447.654371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.654378:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.654381:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.654382:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.654382:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.654384:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.654387:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:17.0:1619677447.654419:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677447.654422:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677447.654422:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677447.654425:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.654428:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.654429:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677447.654431:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.654432:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.654433:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.654434:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677447.654438:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.654439:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677447.654442:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.654442:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:17.0:1619677447.654443:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677447.654445:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677447.654445:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677447.654446:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677447.654446:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677447.654451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.654458:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.654461:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.654462:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.654462:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.654464:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.654464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.654467:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.654474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.654482:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.654485:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.654485:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.654486:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.654487:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.654487:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.654553:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.654556:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.654556:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.654558:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.654561:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.654562:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.654563:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.654565:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.654566:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.654567:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.654570:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.654571:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.654574:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.654575:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.654576:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.654577:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.654578:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.654579:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.654579:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677447.654590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.654598:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.654601:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.654603:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.654605:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.654607:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.654608:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96725269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.654609:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.654610:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.654611:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.654614:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.654616:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.654619:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.654619:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.654620:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.654621:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.654622:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.654623:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.654623:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.654635:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.654637:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.654638:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.654640:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.654642:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.654642:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.654882:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.654885:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.654885:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.654887:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.654889:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.654891:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.654892:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.654894:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.654894:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.654896:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.654899:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.654901:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.654904:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.654905:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.654906:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.654908:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.654908:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.654909:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.654910:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.654911:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677447.654920:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677447.654921:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677447.654922:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677447.654923:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:12.0:1619677447.654923:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.654924:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.654925:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677447.654925:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.654926:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.654928:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.655060:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677447.655064:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.655067:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.655067:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677447.655071:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.655073:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.655075:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.655076:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96725266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.655078:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.655079:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.655080:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.655083:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.655085:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.655087:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.655088:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.655089:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.655091:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677447.655091:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.655092:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.655092:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.655093:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.655094:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.655095:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.655096:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.655097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677447.655098:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677447.655098:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.655098:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.655100:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.655101:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.655102:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677447.655105:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.655108:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.655109:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.655109:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.655111:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.655114:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677447.655115:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.655118:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.655119:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.655120:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.655120:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.655122:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.655193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.655237:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.655240:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.655241:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.655242:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.655244:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.655246:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677447.655262:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.655265:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.655265:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.655267:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.655270:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.655271:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.655273:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.655274:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.655275:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.655276:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677447.655278:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677447.655279:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.655281:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677447.655281:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.655282:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677447.655284:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.655284:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:1.0:1619677447.655284:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.655285:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.655287:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.655287:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:1.0:1619677447.655287:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.655288:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.655288:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:1.0:1619677447.655288:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.655290:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.655291:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.655292:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677447.655293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677447.655299:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.655300:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.655302:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:1.0:1619677447.655302:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.655303:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.655304:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.655305:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.655305:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.655307:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.655308:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.655308:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.655308:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.655310:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.655310:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.655311:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.655312:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677447.655324:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.655326:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.655327:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.655328:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.655329:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.655330:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.655525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677447.655532:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677447.655532:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.655535:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:5.0:1619677447.655536:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677447.655536:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.0:1619677447.655538:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:18.0:1619677447.655539:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.655539:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.655541:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.655542:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677447.655543:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.655544:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.655545:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.655546:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.655546:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.655547:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.655548:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677447.655548:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.655549:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.655550:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.655556:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.655556:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.655559:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.655560:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.655562:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.655563:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.655563:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.655564:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.655564:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.655565:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.655567:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.655567:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.655567:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.655568:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677447.655568:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677447.655569:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677447.655569:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.655570:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677447.655581:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677447.655582:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:13.0:1619677447.655583:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677447.655584:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:13.0:1619677447.655584:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.655585:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.655587:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.655588:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.655588:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.655588:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.655591:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677447.655591:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.655635:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.655638:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.655639:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.655641:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.655643:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.655644:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.655646:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.655647:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.655648:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.655649:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.655652:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.655654:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.655657:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.655658:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.655659:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.655660:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.655661:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.655661:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.655662:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.655674:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.655677:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.655678:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.655678:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.655680:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.655683:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.655683:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.655686:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.655687:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.655689:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.655692:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.655693:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.655694:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.655695:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.655697:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.655697:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.655698:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.655701:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.655701:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:2.0:1619677447.655701:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.655702:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.655703:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.655704:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.655704:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.655705:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.655706:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.655706:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.655708:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.655708:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.655710:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.655711:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.655712:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.655714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.655723:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.655724:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.655724:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.655724:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.655725:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.655725:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.655726:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.655726:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.655728:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.655728:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:6.0:1619677447.655800:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.655803:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.655804:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.655805:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.655808:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.655810:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.655811:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.655812:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.655813:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.655815:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.655817:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.655819:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.655821:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.655822:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.655823:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.655824:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.655825:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.655826:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.655826:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.655830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.655837:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.655840:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.655841:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.655842:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.655843:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.655845:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677447.655859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.655867:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.655869:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.655871:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.655874:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.655876:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.655878:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96725256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.655878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:30.0:1619677447.655879:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.655880:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.655881:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.655885:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.655887:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.655890:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.655891:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.655892:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.655893:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.655894:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.655895:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.655896:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:9.2:1619677447.655897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.655904:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.655907:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.655907:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.655908:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.655908:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:18.0:1619677447.655910:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.655911:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.655912:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.655913:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.655914:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.655914:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.655917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.655917:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677447.655924:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.655925:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677447.655927:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.655928:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:18.0:1619677447.655928:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.655928:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000400:00000200:22.0:1619677447.655929:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.655929:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677447.655929:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.655930:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:22.0:1619677447.655931:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.655934:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677447.655935:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677447.655978:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.655981:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.655981:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.655982:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.655984:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.655987:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677447.655987:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:17.0:1619677447.655990:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.655991:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.655993:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.655996:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.655997:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.655998:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.655999:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.656000:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.656001:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.656004:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.656006:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.656008:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.656009:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.656010:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.656011:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.656012:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.656012:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.656013:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.656035:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.656038:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.656039:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.656039:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.656041:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.656043:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.656223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.656228:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.656230:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:5.0:1619677447.656230:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.656231:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.656233:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.656234:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.656236:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.656237:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.656238:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.656240:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.656241:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.656242:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.656243:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.656246:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.656248:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.656249:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.656250:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.656252:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.656252:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.656254:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.656256:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.656257:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000400:00000200:5.0:1619677447.656257:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.656258:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.656259:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.656260:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.656260:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:5.0:1619677447.656260:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.656261:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.656262:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677447.656263:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.656263:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.656263:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.656264:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.656266:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.656267:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.656267:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.656268:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:5.0:1619677447.656273:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.656275:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:18.0:1619677447.656276:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:5.0:1619677447.656276:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.656277:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.656278:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:30.0:1619677447.656279:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:18.0:1619677447.656279:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.656279:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677447.656280:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:18.0:1619677447.656281:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.656282:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.656283:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.656283:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.656285:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677447.656286:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.656287:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.656288:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.656289:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:18.0:1619677447.656289:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.656290:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.656297:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.656299:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.656301:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.656302:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.656303:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.656305:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.656305:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.656306:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.656307:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.656312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.656312:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)++
00000800:00000200:6.0:1619677447.656315:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.5.62@o2ib3
00000800:00000200:22.0:1619677447.656316:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)--
00000400:00000200:6.0:1619677447.656318:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.62@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.656319:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:6.0:1619677447.656321:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.656321:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.656322:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.656324:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:6.0:1619677447.656324:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:6.0:1619677447.656325:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.656325:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.656327:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.656327:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:6.0:1619677447.656328:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.656329:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:6.0:1619677447.656332:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.656334:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677447.656337:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.656338:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677447.656339:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.656341:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)++
00000800:00000200:6.0:1619677447.656341:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677447.656342:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.62@o2ib3
00000800:00000200:6.0:1619677447.656342:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d6a00] (23)++
00000800:00000200:6.0:1619677447.656344:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d6a00] (24)--
00000400:00000200:22.0:1619677447.656345:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.62@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.656345:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (23)--
00000800:00000200:6.0:1619677447.656346:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)--
00000400:00000200:22.0:1619677447.656347:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.656347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:22.0:1619677447.656349:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:22.0:1619677447.656350:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.656351:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.656352:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.656352:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677447.656355:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.656355:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:22.0:1619677447.656356:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.656357:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.656358:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677447.656358:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.656358:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.656359:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.656359:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677447.656360:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.656360:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d6a00] (21)++
00000800:00000200:22.0:1619677447.656360:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d6a00] (22)--
00000800:00000200:22.0:1619677447.656361:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)--
00000400:00000200:18.0:1619677447.656361:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.656363:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.656374:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.656375:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.656376:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)++
00000400:00000200:6.0:1619677447.656376:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.656378:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.62@o2ib3
00000400:00000200:6.0:1619677447.656378:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.656380:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.656381:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.62@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.656382:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.656382:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.656382:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)--
00000400:00000200:22.0:1619677447.656384:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677447.656385:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725247] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.656387:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.656388:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.656389:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677447.656392:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.656393:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677447.656395:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.656396:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:22.0:1619677447.656396:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677447.656398:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:22.0:1619677447.656398:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d6a00] (21)++
00000800:00000200:22.0:1619677447.656399:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d6a00] (22)--
00000800:00000200:22.0:1619677447.656399:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)--
00000800:00000200:1.0:1619677447.656413:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.656416:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.656417:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.656417:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.656419:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.656421:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:6.0:1619677447.656434:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)++
00000800:00000200:6.0:1619677447.656436:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.62@o2ib3
00000800:00000200:22.0:1619677447.656437:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)--
00000800:00000200:20.2:1619677447.656437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:6.0:1619677447.656439:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.62@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.656441:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:6.0:1619677447.656441:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.656443:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:6.0:1619677447.656443:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.656444:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.656446:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.656446:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.656447:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.656447:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:6.0:1619677447.656448:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:20.0:1619677447.656449:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.656451:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677447.656452:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725245] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.656452:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.656454:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.656455:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.656455:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:20.0:1619677447.656456:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.656458:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.656459:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000400:00000200:20.0:1619677447.656460:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.656460:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.656461:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)++
00000800:00000200:20.0:1619677447.656462:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.656463:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.62@o2ib3
00000800:00000200:6.0:1619677447.656463:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.656463:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d6a00] (23)++
00000800:00000200:20.0:1619677447.656465:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.656465:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f3d6a00] (24)++
00000400:00000200:22.0:1619677447.656466:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.62@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.656466:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677447.656466:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.656466:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d6a00] (25)--
00000800:00000200:6.0:1619677447.656467:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (24)--
00000800:00000200:20.0:1619677447.656468:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.656468:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677447.656468:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (23)--
00000400:00000200:22.0:1619677447.656469:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.656470:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:22.0:1619677447.656471:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:20.0:1619677447.656471:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000400:00000200:22.0:1619677447.656472:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96725244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.656472:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:22.0:1619677447.656474:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677447.656474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:22.0:1619677447.656475:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.656476:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:20.2:1619677447.656477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677447.656477:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d6a00] (23)++
00000800:00000200:6.0:1619677447.656478:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f3d6a00] (24)--
00000800:00000200:6.0:1619677447.656479:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (23)--
00000800:00000200:6.0:1619677447.656480:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)--
00000400:00000200:22.0:1619677447.656481:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.656481:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:5.0:1619677447.656483:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:22.0:1619677447.656484:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.656484:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.656485:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.656485:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.656486:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.656487:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:2.0:1619677447.656487:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.656488:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.656488:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.656489:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.656489:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:22.0:1619677447.656490:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.2:1619677447.656490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:6.0:1619677447.656491:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.656492:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.656493:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.656493:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d6a00] (21)++
00000800:00000200:22.0:1619677447.656494:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d6a00] (22)--
00000800:00000200:22.0:1619677447.656495:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)--
00000400:00000200:6.0:1619677447.656495:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.656496:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.656498:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.656499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:6.0:1619677447.656499:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.656501:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.656504:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)++
00000400:00000200:6.0:1619677447.656504:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.656505:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:6.0:1619677447.656505:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.656506:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.62@o2ib3
00000400:00000200:6.0:1619677447.656506:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.656507:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:22.0:1619677447.656509:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.62@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.656510:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:6.0:1619677447.656510:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.656511:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.656511:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.656511:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.656513:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.656514:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.656514:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.656516:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96725241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.656518:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.656519:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.656521:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.656523:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:6.0:1619677447.656524:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.656525:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.656525:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:22.0:1619677447.656527:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.656527:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:17.0:1619677447.656528:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677447.656528:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.656529:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677447.656529:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.656530:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677447.656531:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:6.0:1619677447.656531:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)--
00000800:00000200:1.0:1619677447.656532:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.656533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.656534:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677447.656534:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.656535:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:22.0:1619677447.656536:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:17.0:1619677447.656536:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.656537:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677447.656538:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d6a00] (21)++
00000400:00000200:17.0:1619677447.656538:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:22.0:1619677447.656539:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d6a00] (22)--
00000400:00000200:17.0:1619677447.656539:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96725240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.656540:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)--
00000400:00000200:17.0:1619677447.656540:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.656540:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:17.0:1619677447.656541:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.656542:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.656543:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:20.2:1619677447.656544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677447.656544:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.656544:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.656546:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.656547:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.656548:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:17.0:1619677447.656549:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.656550:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:17.0:1619677447.656551:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.656551:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677447.656552:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:17.0:1619677447.656552:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.656552:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:9.2:1619677447.656552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677447.656553:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.656554:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.656554:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.656554:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677447.656554:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:30.0:1619677447.656555:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:17.0:1619677447.656555:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:30.0:1619677447.656556:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677447.656556:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.656556:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000400:00000200:12.0:1619677447.656556:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.656556:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.656557:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:17.0:1619677447.656557:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677447.656558:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.656559:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:13.0:1619677447.656559:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.656560:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:12.0:1619677447.656561:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.656563:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.656564:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.656564:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.656565:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.656565:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:17.0:1619677447.656569:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.656569:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.656570:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.656570:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.656571:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.656571:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.656572:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677447.656573:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.656573:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.656575:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677447.656575:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677447.656578:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.656580:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.656583:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.656584:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677447.656586:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96725239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.656587:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.656588:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.656589:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677447.656593:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.656594:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.656598:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.656598:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677447.656599:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677447.656600:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.656601:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.656602:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.656603:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.656614:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.656617:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.656618:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.656618:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.656620:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.656623:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.656634:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.656637:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.656637:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.656638:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.656639:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.656639:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.656663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:4.2:1619677447.656687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.656702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.656709:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.656712:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.656713:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.656713:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.656716:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.656719:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.656724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.656728:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.656731:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.656732:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.656733:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.656735:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.656738:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.656799:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)++
00000800:00000200:22.0:1619677447.656802:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.62@o2ib3
00000800:00000200:6.0:1619677447.656803:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)--
00000400:00000200:22.0:1619677447.656805:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.62@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.656807:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.656808:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:22.0:1619677447.656810:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.656811:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.656812:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.656813:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:22.0:1619677447.656816:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.656818:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:22.0:1619677447.656821:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.656822:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:22.0:1619677447.656823:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:22.0:1619677447.656825:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:22.0:1619677447.656825:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d6a00] (21)++
00000800:00000200:22.0:1619677447.656826:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d6a00] (22)--
00000800:00000200:22.0:1619677447.656827:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)--
00000800:00000200:9.2:1619677447.656831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.656837:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.656840:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.656841:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.656842:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.656842:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)++
00000400:00000200:24.0:1619677447.656844:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.656845:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)--
00000800:00000200:6.0:1619677447.656845:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.62@o2ib3
00000400:00000200:6.0:1619677447.656847:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.62@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.656848:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:6.0:1619677447.656850:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.656851:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:6.0:1619677447.656853:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.656854:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.656855:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.656856:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:6.0:1619677447.656859:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.656861:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:6.0:1619677447.656863:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.656864:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:6.0:1619677447.656864:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:6.0:1619677447.656866:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:6.0:1619677447.656866:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d6a00] (21)++
00000800:00000200:6.0:1619677447.656867:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f3d6a00] (22)++
00000800:00000200:6.0:1619677447.656868:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d6a00] (23)--
00000800:00000200:6.0:1619677447.656869:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)--
00000800:00000200:22.0:1619677447.656879:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)++
00000800:00000200:5.0:1619677447.656880:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:22.0:1619677447.656881:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f3d6a00] (23)--
00000800:00000200:22.0:1619677447.656881:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)--
00000800:00000200:22.0:1619677447.656882:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)--
00000800:00000200:5.0:1619677447.656883:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.656884:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.656885:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.656886:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.656888:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.656895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:6.0:1619677447.656896:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)++
00000800:00000200:22.0:1619677447.656899:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)--
00000800:00000200:6.0:1619677447.656899:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.62@o2ib3
00000400:00000200:6.0:1619677447.656901:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.62@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.656903:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.656904:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:6.0:1619677447.656905:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.656908:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.656909:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.656910:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:6.0:1619677447.656910:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:6.0:1619677447.656914:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.656915:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:6.0:1619677447.656917:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.656918:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:6.0:1619677447.656918:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:6.0:1619677447.656920:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:6.0:1619677447.656920:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d6a00] (21)++
00000800:00000200:6.0:1619677447.656921:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d6a00] (22)--
00000800:00000200:6.0:1619677447.656922:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)--
00000800:00000200:20.0:1619677447.656933:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.656935:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.656936:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.656938:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.656939:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.656940:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:30.0:1619677447.656944:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.656946:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.656947:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.656949:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.656952:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677447.656960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:22.0:1619677447.656965:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)++
00000800:00000200:5.0:1619677447.656966:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677447.656968:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.62@o2ib3
00000800:00000200:6.0:1619677447.656969:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)--
00000800:00000200:5.0:1619677447.656969:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:22.0:1619677447.656972:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.62@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.656972:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.656973:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.656975:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.656975:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.656976:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.656978:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.656978:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677447.656980:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.656980:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.656982:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.656982:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.656983:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:5.0:1619677447.656983:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.656984:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677447.656990:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.656990:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.656993:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.656993:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.656995:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.656996:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.656996:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:22.0:1619677447.656997:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677447.656997:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677447.656998:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.656999:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.656999:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:22.0:1619677447.657000:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d6a00] (21)++
00000800:00000200:5.0:1619677447.657000:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677447.657001:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d6a00] (22)--
00000800:00000200:5.0:1619677447.657001:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.657001:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.657002:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)--
00000800:00000200:17.0:1619677447.657013:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:20.0:1619677447.657016:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:17.0:1619677447.657016:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.657016:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.657017:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.657019:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.657019:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.657019:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.657021:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:20.0:1619677447.657022:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.657024:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.657025:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677447.657033:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677447.657043:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.657046:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.657047:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.657047:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.657048:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.657049:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677447.657052:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.657055:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.657056:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.657056:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.657057:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.657058:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:13.0:1619677447.657064:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.657067:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.657068:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.657070:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.657071:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.657072:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.657087:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.657090:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.657091:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.657092:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.657092:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.657094:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677447.657123:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.657126:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.657127:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.657129:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.657130:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.657132:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.657133:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.657135:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.657136:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.657137:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.657140:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.657140:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)++
00000800:00000200:18.0:1619677447.657142:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:22.0:1619677447.657143:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)--
00000800:00000200:6.0:1619677447.657143:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.62@o2ib3
00000800:00000200:18.0:1619677447.657145:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.657146:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.657146:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:6.0:1619677447.657146:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.62@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.657148:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.657148:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:6.0:1619677447.657148:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.657149:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:6.0:1619677447.657149:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:18.0:1619677447.657150:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:6.0:1619677447.657151:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.657152:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.657153:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.657154:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.657158:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.657160:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:5.0:1619677447.657161:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:6.0:1619677447.657162:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.657163:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.657164:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:5.0:1619677447.657164:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.657165:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.657166:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.657166:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d6a00] (21)++
00000800:00000200:6.0:1619677447.657166:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d6a00] (22)--
00000400:00000200:5.0:1619677447.657166:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.657167:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:6.0:1619677447.657167:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)--
00000800:00000200:5.0:1619677447.657169:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.657180:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.657183:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.657184:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.657185:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.657186:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.657187:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.657230:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.657233:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.657233:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.657235:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.657237:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.657238:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.657240:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.657241:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.657242:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.657243:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.657246:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.657248:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.657250:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.657251:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.657251:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.657253:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.657253:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.657254:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.657255:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.657266:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.657268:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.657268:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.657269:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.657270:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.657271:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:22.0:1619677447.657336:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.657338:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.657339:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.657342:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.657344:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.657345:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.657347:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.657348:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.657349:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.657350:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.657354:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.657355:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.657358:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.657358:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.657359:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.657361:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.657361:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.657362:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.657363:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.657364:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677447.657375:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677447.657375:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.657376:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:20.0:1619677447.657377:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:6.0:1619677447.657377:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677447.657378:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.657379:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:20.0:1619677447.657381:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.657381:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.657383:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.657392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.657397:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.657400:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.657402:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.657403:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.657405:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.657410:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:20.2:1619677447.657412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:12.0:1619677447.657412:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:5.0:1619677447.657412:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.657413:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.657414:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.657415:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:5.0:1619677447.657415:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:12.0:1619677447.657416:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.657418:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.657419:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.657420:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.657421:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.657422:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.657425:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.657426:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.657426:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.657428:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677447.657428:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.657429:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.657429:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.657430:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.657433:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.657436:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.657437:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.657437:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.657438:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.657439:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.657440:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.657442:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.657443:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.657444:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.657447:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.657448:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.0:1619677447.657461:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.657463:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.657464:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.657466:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.657467:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.657469:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677447.657483:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.657486:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.657488:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.657490:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.657492:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677447.657493:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96725222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.657495:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.657496:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.657497:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677447.657500:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.657501:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677447.657504:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.657504:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677447.657505:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677447.657506:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.657507:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.657508:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.657510:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.657510:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677447.657553:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.657555:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.657557:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.657558:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.657561:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.657563:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677447.657564:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.657565:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.657566:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.657567:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677447.657570:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.657571:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677447.657574:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.657575:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677447.657575:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677447.657577:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677447.657578:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.657579:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.657580:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.657580:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:4.2:1619677447.657581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.657588:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.657591:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.0:1619677447.657591:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.657592:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677447.657592:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:22.0:1619677447.657593:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:8.0:1619677447.657593:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.657594:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:22.0:1619677447.657595:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.657595:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677447.657595:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677447.657596:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:22.0:1619677447.657597:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.657597:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.657599:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:22.0:1619677447.657600:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:8.0:1619677447.657600:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.657601:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.657602:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.657603:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.657607:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.657608:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.657611:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.657612:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.657613:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.657614:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.657615:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.657616:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.657616:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.657629:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.657631:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.657632:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.657633:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.657635:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.657637:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.657747:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.657750:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.657751:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.657751:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.657752:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.657753:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.657817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.657823:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.657825:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.657828:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.657828:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.657831:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.657832:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.657833:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.657835:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.657836:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.657836:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.657839:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.657841:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.657844:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.657844:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.657845:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677447.657845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.657846:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.657847:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.657848:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.657848:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.657860:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.657862:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.657863:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.657863:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.657866:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.657868:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677447.657897:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.657905:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.657908:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.657909:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.657911:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.657913:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.657914:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.657916:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.657917:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.657918:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.657919:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.657922:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.657924:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.657927:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.657927:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.657928:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.657930:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.657930:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.657931:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.657931:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.657944:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.657946:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.657947:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.657949:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.657950:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.657951:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677447.658044:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.658047:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.658047:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.658050:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.658051:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.658053:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.658054:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.658056:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.658057:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.658058:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.658061:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.658063:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.658066:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.658067:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.658067:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.658069:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.658070:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.658070:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.658071:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677447.658072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.658079:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677447.658081:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677447.658082:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:12.0:1619677447.658084:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:30.0:1619677447.658085:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677447.658085:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.658087:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.658087:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.658088:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:30.0:1619677447.658089:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677447.658090:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96725215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.658090:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:30.0:1619677447.658091:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.658093:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.658093:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677447.658097:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.658098:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677447.658101:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.658102:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.658103:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677447.658105:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.658105:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.658106:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.658107:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677447.658119:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.658121:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.658122:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.658124:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.658124:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.658127:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677447.658178:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.658180:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.658181:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.658183:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.658184:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.658185:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.658252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.658286:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.658289:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.658291:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.658293:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.658299:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.658300:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:5.0:1619677447.658300:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.658301:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.658302:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.658302:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.658303:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.658303:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.658304:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.658305:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.658308:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.658310:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.658312:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.658312:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.658313:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.658313:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.658314:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.658316:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.658317:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.658318:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.658320:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677447.658320:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.658331:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.658332:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.658333:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:5.0:1619677447.658333:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.658333:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.658334:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.658335:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.658335:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.658336:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.658337:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677447.658354:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.658357:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.658358:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.658358:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.658359:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.658360:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:4.2:1619677447.658503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.658518:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.658521:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.658521:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.658525:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.658527:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.658529:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.658530:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.658532:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.658533:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.658534:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.658538:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.658540:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.658543:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.658544:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.658545:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.658546:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.658547:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.658547:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.658548:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.658552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:6.0:1619677447.658555:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.658558:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:24.0:1619677447.658559:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:22.0:1619677447.658559:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.658559:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.658562:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:6.0:1619677447.658562:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.658563:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.658563:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.658565:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.658565:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.658568:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677447.658753:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677447.658756:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677447.658757:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677447.658758:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.658761:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.658763:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:8.0:1619677447.658764:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.658765:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.658766:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.658767:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677447.658771:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.658772:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:8.0:1619677447.658775:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.658775:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:8.0:1619677447.658776:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:8.0:1619677447.658778:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:8.0:1619677447.658778:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677447.658778:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677447.658779:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:4.2:1619677447.658866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.658867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.658874:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.658877:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.658880:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.658880:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.658882:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.658884:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677447.658885:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.658888:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.658889:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.658890:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677447.658895:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.658897:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677447.658900:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.658901:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677447.658902:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677447.658904:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677447.658905:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.658906:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.658907:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.658908:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:9.2:1619677447.658911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.2:1619677447.658913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:4.2:1619677447.658914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677447.658918:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.658920:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.658920:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677447.658920:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:20.0:1619677447.658921:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.658922:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.658923:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.658923:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.658926:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.658927:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.658929:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677447.658929:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:30.0:1619677447.658932:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.658935:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.658937:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.658939:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.658940:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96725206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.658941:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.658942:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.658943:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.658947:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.658948:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.658951:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.658952:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.658952:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.658954:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.658954:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.658955:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.658956:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.658967:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:22.0:1619677447.658970:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:12.0:1619677447.658970:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.658970:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.658973:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:12.0:1619677447.658973:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.658973:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:13.0:1619677447.658974:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.658975:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:22.0:1619677447.658976:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.658978:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.658980:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.658981:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.658982:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.658983:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.658984:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.658988:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.658990:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.658993:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.658993:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.658994:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.658996:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.658996:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.658997:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.658998:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677447.659010:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.659013:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.659014:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.659014:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677447.659014:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.659017:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.659017:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.659018:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677447.659020:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:2.0:1619677447.659020:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.659022:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.659023:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.659024:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.659025:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.659027:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.659028:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.659031:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.659033:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.659034:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.659035:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.659036:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.659038:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.659038:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.659039:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.659040:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.659044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.659051:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.659053:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.659054:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.659054:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.659055:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.659057:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.659066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.659073:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.659075:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.659076:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.659076:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.659076:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.659077:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.659083:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.659086:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.659087:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.659088:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.659090:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:1.0:1619677447.659091:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.659092:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:1.0:1619677447.659093:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:8.0:1619677447.659094:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:1.0:1619677447.659095:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.659097:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.659098:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.659098:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.659099:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:24.0:1619677447.659100:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.659103:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.659105:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677447.659105:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:24.0:1619677447.659107:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96725202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.659108:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.659108:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.659109:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.659109:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.659109:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:24.0:1619677447.659110:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.659111:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.659111:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.659113:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:24.0:1619677447.659114:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.659114:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.659114:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677447.659116:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:9.2:1619677447.659117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.659118:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.659119:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.659120:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.659122:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.659123:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.659124:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.659124:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.659125:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:17.0:1619677447.659125:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677447.659126:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.659126:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:17.0:1619677447.659127:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:8.0:1619677447.659127:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.659128:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.659128:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677447.659129:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.659131:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677447.659134:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.659137:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677447.659138:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:13.0:1619677447.659138:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.659139:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.659141:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:13.0:1619677447.659141:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.659141:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.659141:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.659144:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.659146:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.659147:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.659148:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.659150:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.659151:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.659152:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.659155:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.659157:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.659160:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.659161:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.659161:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.659163:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.659164:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.659165:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.659165:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677447.659177:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.659179:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.659180:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.659182:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.659183:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.659185:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.659196:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.659199:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.659200:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.659201:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.659201:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.659202:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677447.659248:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.659250:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.659251:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.659252:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.659254:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.659255:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.659268:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.659271:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.659272:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.659272:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.659273:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.659273:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677447.659487:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.659490:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.659491:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.659493:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.659496:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.659497:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.659498:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.659500:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.659501:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.659502:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677447.659505:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.659507:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.659509:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.659510:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.659511:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.659513:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.659513:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.659513:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.659514:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.659519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.659526:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.659529:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.659530:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.659530:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.659532:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.659535:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.659537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:8.0:1619677447.659543:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.659545:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.659546:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.659546:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.659548:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.659548:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.659636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.659657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.659663:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.659666:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.659668:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.659668:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.659671:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.659672:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677447.659674:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.659675:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.659676:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.659677:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677447.659680:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.659682:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.659685:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.659686:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677447.659686:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (22)++
00000800:00000200:5.0:1619677447.659689:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677447.659689:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.659690:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.659691:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677447.659841:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.659843:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.659846:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.659846:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.659848:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.659850:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.659852:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.659853:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677447.659853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:20.0:1619677447.659855:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.659856:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.659859:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.659860:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677447.659861:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.659862:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.659863:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:20.0:1619677447.659863:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.659864:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:8.0:1619677447.659864:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.659866:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.659866:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.659866:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.659867:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000400:00000200:24.0:1619677447.659868:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.659868:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:24.0:1619677447.659870:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.659872:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96725195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.659874:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.659875:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.659876:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.659880:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:24.0:1619677447.659881:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.659881:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:5.0:1619677447.659881:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:24.0:1619677447.659883:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.659883:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.659883:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.659884:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.659886:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.659886:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.659887:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.659887:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:8.0:1619677447.659887:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:24.0:1619677447.659888:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.659888:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:24.0:1619677447.659889:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677447.659889:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:8.0:1619677447.659889:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96725194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.659891:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:8.0:1619677447.659891:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.659891:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.659892:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:8.0:1619677447.659892:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.659893:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.659894:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677447.659894:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:24.0:1619677447.659895:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677447.659896:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:20.0:1619677447.659897:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.659897:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.659899:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.659901:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.659903:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96725193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.659903:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.659904:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.659905:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.659906:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:17.0:1619677447.659906:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677447.659906:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.659908:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677447.659909:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.659909:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:20.0:1619677447.659910:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.659910:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.659910:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.659910:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.659911:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.659911:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677447.659912:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:17.0:1619677447.659912:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.659912:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.659913:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.659913:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:1.0:1619677447.659913:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.0:1619677447.659915:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.659916:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:17.0:1619677447.659916:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:1.0:1619677447.659916:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.659917:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677447.659917:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.2:1619677447.659918:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:1.0:1619677447.659919:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.659921:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.659922:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96725192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.659923:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:1.0:1619677447.659923:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.659924:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677447.659924:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677447.659924:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.659925:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.659925:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677447.659925:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:13.0:1619677447.659927:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.659927:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.659929:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677447.659930:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.659930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.659931:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.659932:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:13.0:1619677447.659933:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.659934:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:1.0:1619677447.659935:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.659936:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.659937:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:2.0:1619677447.659938:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677447.659938:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.659939:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.659939:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677447.659940:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:1.0:1619677447.659940:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:2.0:1619677447.659941:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.659942:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:12.0:1619677447.659944:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:9.2:1619677447.659944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:2.0:1619677447.659944:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.659947:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.659947:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677447.659948:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.659948:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.659949:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.659951:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:12.0:1619677447.659951:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.659953:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.659954:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.659954:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.659956:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.659959:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.2:1619677447.659964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677447.659966:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.659968:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.659969:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.659971:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.659972:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:12.0:1619677447.659972:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.659974:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.659975:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.659975:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.659976:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.660014:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.660016:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.660017:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.660018:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:12.0:1619677447.660018:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.660020:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:13.0:1619677447.660020:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.660021:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:30.0:1619677447.660022:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.660025:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.660026:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.660027:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96725189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.660028:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.660029:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.660030:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.660033:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.660035:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.660037:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.660037:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.660038:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.660039:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.660040:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.660041:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.660041:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.660051:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.660053:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.660054:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.660056:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.660057:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.660058:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.660151:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677447.660154:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677447.660155:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677447.660157:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.660160:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.660161:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.660163:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.660164:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.660165:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.660166:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.660170:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.660172:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.660175:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.660176:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.660176:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.660178:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.660178:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677447.660179:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677447.660180:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:2.0:1619677447.660312:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.660315:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.660316:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.660318:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.660321:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.660322:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.660324:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.660325:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.660326:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.660326:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.660329:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.660330:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.660333:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.660333:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.660334:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.660335:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.660336:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.660337:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.660337:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677447.660350:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.660353:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.660354:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.660354:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.660356:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.660359:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.0:1619677447.660369:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.660371:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.660372:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.660374:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.660377:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.660378:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.660380:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.660381:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.660382:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.660383:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.660387:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.660389:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.660392:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.660392:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.660393:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.660395:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.660396:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.660397:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.660398:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.660399:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677447.660402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.660410:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677447.660411:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.660413:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:18.0:1619677447.660413:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:24.0:1619677447.660414:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:18.0:1619677447.660414:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.660414:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.660416:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:18.0:1619677447.660417:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.660420:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.660422:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.660425:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.660426:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.660428:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.660430:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.660432:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.660433:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.660434:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.660435:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.660436:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.660439:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.660441:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.660444:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.660445:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.660446:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.660448:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.660448:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.660449:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.660449:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.660460:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.660463:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.660464:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.660466:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.660468:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.660468:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.660611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.660750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:20.2:1619677447.660791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:5.0:1619677447.660798:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.660800:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.660801:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.660803:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.660803:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.660806:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.0:1619677447.660806:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:13.0:1619677447.660835:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.660837:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.660838:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.660840:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.660841:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.660842:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.660880:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.660883:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.660884:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.660886:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.660888:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.660889:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.660891:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.660892:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.660893:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.660894:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.660897:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.660899:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.660901:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.660902:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.660903:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.660904:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.660904:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.660906:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.660907:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.660907:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.660910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.660918:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.660919:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.660919:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:18.0:1619677447.660920:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.660920:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.660920:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.660921:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.660922:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.660923:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.660923:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677447.660926:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.660929:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.660929:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.660931:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.660934:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.660935:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.660937:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.660938:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.660939:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.660940:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.660943:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.660944:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.660948:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.660949:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.660949:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.660951:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.660951:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.660953:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.660954:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.660955:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677447.660963:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677447.660965:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677447.660966:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.660967:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.660967:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.660968:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.660968:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677447.660969:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.660970:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.660971:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677447.661062:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.661064:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.661066:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.661068:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.661069:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.661070:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.661071:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.661073:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.661074:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.661075:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:2.0:1619677447.661075:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:20.0:1619677447.661078:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.661078:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.661079:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.0:1619677447.661080:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:2.0:1619677447.661081:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.661083:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677447.661083:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.661084:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000400:00000200:2.0:1619677447.661084:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:20.0:1619677447.661085:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:2.0:1619677447.661085:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.661086:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.661087:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000400:00000200:2.0:1619677447.661087:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.661088:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.661088:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:2.0:1619677447.661088:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.661089:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.661092:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.661093:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.661095:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.661096:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.661096:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.661098:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.661098:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.661099:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:5.0:1619677447.661100:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:2.0:1619677447.661100:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.661101:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677447.661103:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.661103:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.661104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.661106:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.661106:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.661108:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:18.0:1619677447.661112:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.661114:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.661114:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677447.661115:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.661115:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.661116:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.661117:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.661117:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.661118:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.661118:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.661126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.661132:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.661135:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.661136:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.661136:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.661136:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.661137:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.661139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.661178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.661186:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.661189:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.661190:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.661190:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.661192:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.661195:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.661415:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.661418:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.661419:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.661421:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.661424:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.661425:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.661427:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.661428:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.661429:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.661430:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.661433:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.661435:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.661438:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.661438:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.661439:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.661441:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.661441:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.661442:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.661442:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.661456:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.661459:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.661460:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.661460:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.661462:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.661465:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.661469:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.661472:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.661473:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:22.0:1619677447.661474:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:2.0:1619677447.661474:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.661477:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677447.661477:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.661478:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.661478:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:2.0:1619677447.661478:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:22.0:1619677447.661479:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000400:00000200:2.0:1619677447.661479:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.661480:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:2.0:1619677447.661481:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.661482:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.661483:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.661486:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.661488:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.661490:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.661490:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.661491:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.661492:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.661493:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.661493:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.661494:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.661498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677447.661504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677447.661506:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.661508:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.661508:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.661509:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.661510:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.661511:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677447.661513:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.661516:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.661518:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.661520:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.661522:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.661523:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96725174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.661525:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.661526:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.661527:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.661528:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:30.0:1619677447.661530:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.661531:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:30.0:1619677447.661532:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.661532:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.661534:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.661535:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.661536:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:6.0:1619677447.661536:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.661537:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:6.0:1619677447.661538:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:30.0:1619677447.661539:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.661539:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:6.0:1619677447.661539:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.661540:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:6.0:1619677447.661540:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.661541:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:6.0:1619677447.661541:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.661542:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.661546:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.661548:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.661551:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.661552:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.661553:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:20.0:1619677447.661554:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677447.661555:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.661555:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677447.661556:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:6.0:1619677447.661556:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677447.661557:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.661557:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:20.0:1619677447.661559:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.661560:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.661562:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.661578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.661584:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.661586:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.661589:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.661589:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.661591:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.661593:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:5.0:1619677447.661594:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.661595:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.661596:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.661597:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:5.0:1619677447.661601:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.661602:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:5.0:1619677447.661606:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.661606:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677447.661607:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.0:1619677447.661609:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677447.661609:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.661611:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.661612:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.661613:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:9.2:1619677447.661615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.2:1619677447.661618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.661621:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:8.0:1619677447.661621:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:20.0:1619677447.661623:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.661624:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.0:1619677447.661624:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.661625:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677447.661625:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:8.0:1619677447.661625:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.661627:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.661631:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:30.0:1619677447.661654:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677447.661685:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.661687:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.2:1619677447.661688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:13.0:1619677447.661688:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.661689:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677447.661690:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:13.0:1619677447.661690:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.661692:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.661692:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677447.661692:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.661692:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.661693:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677447.661693:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.661695:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.661696:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.661698:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.661699:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677447.661701:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.661702:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.661703:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.661704:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:13.0:1619677447.661704:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.661707:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.661708:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.661708:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.661709:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.661709:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.661711:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677447.661711:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.661713:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.661714:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.661714:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.661716:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.661717:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.661718:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.661718:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677447.661730:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.661733:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.661734:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.661734:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.661736:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.661738:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677447.661802:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.661804:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.661805:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.661806:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.661810:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677447.661831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.661838:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:22.0:1619677447.661841:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)++
00000800:00000200:2.0:1619677447.661841:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.661842:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:22.0:1619677447.661844:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.13.22@o2ib3
00000800:00000200:6.0:1619677447.661845:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000400:00000200:2.0:1619677447.661846:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.661848:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.51@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.22@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.661848:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.661849:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:22.0:1619677447.661851:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.661851:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.661853:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.661854:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.51@o2ib7 local destination
00000400:00000200:22.0:1619677447.661855:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.661855:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.661856:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677447.661857:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.661858:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.661859:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.51@o2ib7
00000400:00000200:2.0:1619677447.661861:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.661865:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.51@o2ib7(10.0.10.51@o2ib7:10.0.10.51@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.661865:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.661866:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.661867:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:22.0:1619677447.661868:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.51@o2ib7
00000800:00000200:2.0:1619677447.661868:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.661869:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.661870:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.661870:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.661871:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:22.0:1619677447.661872:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494880] -> 10.0.10.51@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.661873:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea3be00] (20)++
00000800:00000200:22.0:1619677447.661874:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea3be00] (21)++
00000800:00000200:9.2:1619677447.661875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.661876:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea3be00] (22)--
00000800:00000200:22.0:1619677447.661877:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d5000] (21)++
00000800:00000200:22.0:1619677447.661878:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d5000] (22)--
00000800:00000200:22.0:1619677447.661879:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)--
00000800:00000200:18.0:1619677447.661883:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.661885:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:4.2:1619677447.661885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:18.0:1619677447.661886:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.661886:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.661888:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.661890:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.661894:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.661897:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.661898:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.661898:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.661900:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.661904:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.661906:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.661908:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.661908:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.661909:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.661910:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.661911:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.661922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.661937:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.661967:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.661969:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.661970:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.661972:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.661974:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:22.0:1619677447.661974:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)++
00000800:00000200:22.0:1619677447.661977:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.22@o2ib3
00000800:00000200:6.0:1619677447.661978:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000400:00000200:22.0:1619677447.661980:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.22@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.661982:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.661984:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:22.0:1619677447.661985:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.661986:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.661987:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.661988:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677447.661991:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.661993:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.661995:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.661996:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:22.0:1619677447.661997:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.661998:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677447.661999:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d5000] (21)++
00000800:00000200:22.0:1619677447.662000:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d5000] (22)--
00000800:00000200:22.0:1619677447.662000:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)--
00000800:00000200:9.2:1619677447.662004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.662012:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.662015:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.662016:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:2.0:1619677447.662016:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.662018:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.662018:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.662019:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (23)++
00000800:00000200:18.0:1619677447.662020:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (24)--
00000800:00000200:18.0:1619677447.662021:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.662022:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.662023:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.662032:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)++
00000800:00000200:6.0:1619677447.662034:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.22@o2ib3
00000800:00000200:22.0:1619677447.662035:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000400:00000200:6.0:1619677447.662037:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.22@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.662039:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.662040:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:6.0:1619677447.662042:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.662043:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.662044:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.662045:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:6.0:1619677447.662048:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.662049:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677447.662051:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.662052:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677447.662052:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.662054:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677447.662055:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d5000] (21)++
00000800:00000200:6.0:1619677447.662056:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d5000] (22)--
00000800:00000200:6.0:1619677447.662056:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)--
00000800:00000200:9.2:1619677447.662060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.662062:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)++
00000800:00000200:22.0:1619677447.662065:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.22@o2ib3
00000800:00000200:6.0:1619677447.662066:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000400:00000200:22.0:1619677447.662067:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.22@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.662067:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:22.0:1619677447.662068:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.662070:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:2.0:1619677447.662070:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:22.0:1619677447.662071:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.662071:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.662072:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:22.0:1619677447.662073:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.662073:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.662074:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.662075:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.662077:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:22.0:1619677447.662079:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.662081:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:22.0:1619677447.662084:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.662085:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:22.0:1619677447.662085:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677447.662087:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677447.662088:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d5000] (21)++
00000800:00000200:17.0:1619677447.662088:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677447.662089:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f3d5000] (22)++
00000800:00000200:22.0:1619677447.662090:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d5000] (23)--
00000800:00000200:22.0:1619677447.662090:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000800:00000200:17.0:1619677447.662091:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.662092:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.662093:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.662096:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.662097:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.662098:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.662100:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.662100:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:6.0:1619677447.662100:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)++
00000400:00000200:17.0:1619677447.662101:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.662102:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:12.0:1619677447.662103:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:6.0:1619677447.662103:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f3d5000] (23)--
00000800:00000200:6.0:1619677447.662103:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000400:00000200:12.0:1619677447.662104:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.662104:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)--
00000400:00000200:17.0:1619677447.662105:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677447.662106:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.662107:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677447.662107:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.662109:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.662110:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:12.0:1619677447.662110:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677447.662111:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.662112:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.662113:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.662114:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.662114:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.662119:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.662122:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.662123:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.662123:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:22.0:1619677447.662124:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)++
00000800:00000200:13.0:1619677447.662125:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677447.662126:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.22@o2ib3
00000800:00000200:12.0:1619677447.662126:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677447.662126:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:6.0:1619677447.662128:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000800:00000200:1.0:1619677447.662129:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.662129:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.662130:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.662130:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.662131:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.662133:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.662133:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:22.0:1619677447.662134:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677447.662136:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.662137:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.662138:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.662139:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677447.662142:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.662144:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677447.662147:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.662147:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:22.0:1619677447.662148:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677447.662149:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:22.0:1619677447.662150:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d5000] (21)++
00000800:00000200:22.0:1619677447.662150:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d5000] (22)--
00000800:00000200:22.0:1619677447.662151:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)--
00000800:00000200:17.0:1619677447.662163:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.662166:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.662167:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.662167:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.662169:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.662172:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.662186:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.662188:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.662189:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.662189:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.662190:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.662191:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677447.662239:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677447.662271:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.662274:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.662275:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.662275:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.662278:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.662280:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.662308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:22.0:1619677447.662309:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)++
00000800:00000200:22.0:1619677447.662311:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.22@o2ib3
00000800:00000200:6.0:1619677447.662312:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000400:00000200:22.0:1619677447.662314:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.22@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.662316:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.662317:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:22.0:1619677447.662318:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677447.662319:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:22.0:1619677447.662319:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.662320:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.662321:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.662322:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.662323:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.662326:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.662328:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:30.0:1619677447.662329:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.662331:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677447.662332:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:22.0:1619677447.662332:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:30.0:1619677447.662333:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96725157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.662333:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:30.0:1619677447.662334:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.662335:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.662335:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.662336:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d5000] (21)++
00000800:00000200:22.0:1619677447.662336:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d5000] (22)--
00000400:00000200:30.0:1619677447.662337:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.662337:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)--
00000400:00000200:30.0:1619677447.662340:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.662342:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.662345:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.662346:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.662346:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.662348:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.662349:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.662350:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.662350:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677447.662351:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.662355:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.662358:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.662359:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.662361:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.662363:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:6.0:1619677447.662364:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.662365:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.662366:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725156] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.662366:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677447.662367:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:5.0:1619677447.662367:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.662368:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.662369:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:6.0:1619677447.662369:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.662370:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.662370:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677447.662370:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.662371:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.662371:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.662372:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.662373:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677447.662374:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:6.0:1619677447.662374:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.662376:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.662378:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.662379:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.662380:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.662381:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.662382:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.662383:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:20.0:1619677447.662384:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:6.0:1619677447.662384:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.662385:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677447.662387:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:9.2:1619677447.662388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:20.0:1619677447.662389:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.662392:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.662392:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.662393:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.662394:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.662395:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.662396:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:20.0:1619677447.662396:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.662396:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677447.662397:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.662398:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677447.662398:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677447.662399:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.662400:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.662400:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:20.0:1619677447.662401:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.662403:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677447.662403:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.662405:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:20.0:1619677447.662406:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.662406:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:6.0:1619677447.662407:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)++
00000800:00000200:5.0:1619677447.662407:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.662408:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677447.662409:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.22@o2ib3
00000400:00000200:5.0:1619677447.662409:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.662410:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000800:00000200:20.0:1619677447.662411:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.662412:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.662412:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:5.0:1619677447.662412:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.662413:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.22@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.662413:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677447.662414:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000400:00000200:6.0:1619677447.662415:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.662415:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.662416:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.662417:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.662417:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.662418:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.662419:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.662419:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.662420:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000400:00000200:6.0:1619677447.662420:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.662421:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.662421:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:6.0:1619677447.662421:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.662423:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.662424:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.662425:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.662426:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.662427:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.662428:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.662428:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.662429:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:4.2:1619677447.662430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.662431:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:6.0:1619677447.662431:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.662432:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.0:1619677447.662433:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.662434:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.662434:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677447.662435:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.662435:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.662436:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.662437:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.662438:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.662438:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d5000] (21)++
00000800:00000200:5.0:1619677447.662438:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.662439:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677447.662440:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d5000] (23)--
00000800:00000200:6.0:1619677447.662441:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000800:00000200:5.0:1619677447.662441:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.662442:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)++
00000800:00000200:22.0:1619677447.662444:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)++
00000800:00000200:6.0:1619677447.662445:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.22@o2ib3
00000800:00000200:22.0:1619677447.662446:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.22@o2ib3
00000400:00000200:6.0:1619677447.662448:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.22@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.662449:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.22@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.662450:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.662451:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.662452:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.662454:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96725152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.662456:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.662457:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:6.0:1619677447.662459:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.662460:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.662464:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:22.0:1619677447.662466:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96725150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.662466:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.662468:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.662469:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.662470:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:6.0:1619677447.662470:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.662473:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.662474:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.662474:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:22.0:1619677447.662475:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.662476:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.662477:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d5000] (23)++
00000800:00000200:22.0:1619677447.662478:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:6.0:1619677447.662479:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f3d5000] (24)++
00000800:00000200:6.0:1619677447.662480:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d5000] (25)--
00000800:00000200:6.0:1619677447.662481:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (24)--
00000800:00000200:22.0:1619677447.662482:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.662482:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (23)--
00000800:00000200:22.0:1619677447.662483:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:22.0:1619677447.662483:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:9.2:1619677447.662483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.662484:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.662485:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.662486:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000400:00000200:6.0:1619677447.662486:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.662487:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d5000] (23)++
00000800:00000200:22.0:1619677447.662487:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d5000] (24)--
00000800:00000200:22.0:1619677447.662488:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (23)--
00000400:00000200:6.0:1619677447.662488:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.662489:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.662490:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)++
00000800:00000200:30.0:1619677447.662491:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677447.662491:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f3d5000] (23)--
00000800:00000200:22.0:1619677447.662491:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000800:00000200:9.2:1619677447.662491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:2.0:1619677447.662491:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677447.662492:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)--
00000800:00000200:6.0:1619677447.662492:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677447.662494:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:2.0:1619677447.662494:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:30.0:1619677447.662495:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.662495:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.662495:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.662498:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.662498:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:2.0:1619677447.662499:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.662501:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:24.0:1619677447.662501:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.662502:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.662502:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:2.0:1619677447.662502:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:24.0:1619677447.662504:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.662507:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677447.662518:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.662520:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.662521:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.662523:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.662526:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.662526:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)++
00000400:00000200:8.0:1619677447.662527:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.662529:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725149] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.662529:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.22@o2ib3
00000800:00000200:22.0:1619677447.662530:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000400:00000200:8.0:1619677447.662530:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.662532:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.662533:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:6.0:1619677447.662534:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.22@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.662537:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.662538:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.662539:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:6.0:1619677447.662540:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.662541:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:6.0:1619677447.662542:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.662543:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.662544:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.662545:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.662545:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:6.0:1619677447.662545:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.662547:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.662548:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.662549:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.662549:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:6.0:1619677447.662552:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.662555:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677447.662558:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.662559:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677447.662559:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677447.662561:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:6.0:1619677447.662561:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677447.662563:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:6.0:1619677447.662563:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d5000] (21)++
00000800:00000200:6.0:1619677447.662563:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d5000] (22)--
00000400:00000200:20.0:1619677447.662564:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.662564:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)--
00000400:00000200:20.0:1619677447.662566:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.662567:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.662569:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677447.662569:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.662572:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677447.662573:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.662575:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.662577:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.662578:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677447.662578:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677447.662579:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.662579:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.662580:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.662583:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.662604:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)++
00000800:00000200:22.0:1619677447.662607:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.22@o2ib3
00000800:00000200:6.0:1619677447.662607:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000400:00000200:22.0:1619677447.662610:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.22@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.662612:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.662615:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.662616:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.662617:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)++
00000400:00000200:22.0:1619677447.662618:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.662619:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.662619:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.22@o2ib3
00000400:00000200:22.0:1619677447.662620:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.662623:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.22@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.662624:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.662624:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.662626:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.662626:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.662628:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.662629:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.662630:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.662631:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.662633:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.662634:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.662634:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:6.0:1619677447.662634:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.662636:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.662637:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.662638:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d5000] (23)++
00000800:00000200:22.0:1619677447.662638:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d5000] (24)--
00000800:00000200:17.0:1619677447.662638:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677447.662639:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (23)--
00000800:00000200:6.0:1619677447.662639:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.662640:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)++
00000800:00000200:6.0:1619677447.662640:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.662640:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677447.662641:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.662641:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.662642:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.22@o2ib3
00000800:00000200:6.0:1619677447.662642:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.662643:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d5000] (23)++
00000800:00000200:6.0:1619677447.662644:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f3d5000] (24)++
00000800:00000200:6.0:1619677447.662645:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d5000] (25)--
00000400:00000200:22.0:1619677447.662646:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.22@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.662646:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.662646:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (24)--
00000800:00000200:6.0:1619677447.662647:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (23)--
00000400:00000200:22.0:1619677447.662649:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.662649:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.662651:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.662651:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:1.0:1619677447.662652:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:22.0:1619677447.662653:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96725144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.662653:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96725144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.662653:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.662654:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.662656:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.662656:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.662656:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.662657:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.662657:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (23)++
00000400:00000200:17.0:1619677447.662658:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.662658:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f3d5000] (24)--
00000800:00000200:6.0:1619677447.662658:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (23)--
00000400:00000200:22.0:1619677447.662659:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.662659:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:5.0:1619677447.662659:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677447.662660:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000800:00000200:5.0:1619677447.662661:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.662662:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.662665:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:17.0:1619677447.662666:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.662666:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.662667:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.662669:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.662670:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.662670:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.662670:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.662672:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:1.0:1619677447.662672:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.662673:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.662673:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677447.662674:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.662674:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:5.0:1619677447.662674:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:1.0:1619677447.662674:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.662674:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.662675:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:22.0:1619677447.662676:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.662676:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.0:1619677447.662676:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677447.662676:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677447.662677:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:22.0:1619677447.662678:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.662678:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:17.0:1619677447.662678:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:5.0:1619677447.662678:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677447.662679:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d5000] (21)++
00000800:00000200:17.0:1619677447.662679:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.662679:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677447.662680:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d5000] (22)--
00000800:00000200:22.0:1619677447.662680:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)--
00000800:00000200:17.0:1619677447.662680:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.662681:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.662690:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.662693:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.662693:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.662694:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:13.0:1619677447.662695:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.662696:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677447.662697:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.662698:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:13.0:1619677447.662698:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:6.0:1619677447.662698:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.662700:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.662703:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.662742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677447.662746:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.662749:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.662750:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.662752:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.662754:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.662755:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:30.0:1619677447.662757:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:18.0:1619677447.662757:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.662758:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.662759:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.662760:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.662764:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.662766:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.662768:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.662769:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.662769:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.662771:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.662772:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.662773:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.662773:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.662777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.662784:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.662786:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.662787:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.662788:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.662789:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:20.2:1619677447.662792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677447.662792:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.662798:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677447.662800:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677447.662800:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.662801:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.662802:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.662803:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677447.662803:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.662804:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677447.662804:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.662806:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.662876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677447.662919:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.662921:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.662922:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.662924:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.662926:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.2:1619677447.662929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.662935:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.662937:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.662939:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.662940:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.662943:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.662945:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.662947:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.662948:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:5.0:1619677447.662948:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.662949:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.662950:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.662951:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.662951:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.662955:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.662955:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.662957:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.662959:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.662960:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:5.0:1619677447.662961:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677447.662962:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.662962:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.662962:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:22.0:1619677447.662963:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.662964:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.662964:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:22.0:1619677447.662965:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.662965:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.662966:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.662967:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000400:00000200:22.0:1619677447.662968:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.662968:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677447.662970:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.662973:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677447.662973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677447.662974:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677447.662975:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:20.0:1619677447.662976:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:30.0:1619677447.662977:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (24)++
00000800:00000200:22.0:1619677447.662978:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:22.0:1619677447.662978:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677447.662978:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.662978:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677447.662979:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677447.662979:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677447.662980:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:22.0:1619677447.662980:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:30.0:1619677447.662981:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.662983:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.662986:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:30.0:1619677447.662987:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.662988:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.662989:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:30.0:1619677447.662990:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:30.0:1619677447.662990:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.662993:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.662994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:30.0:1619677447.663035:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.663036:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.663037:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.0.10.116@o2ib7
00000800:00000200:1.0:1619677447.663039:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.663042:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.663043:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:30.0:1619677447.663044:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.116@o2ib7 : GET - for me
00000400:00000200:1.0:1619677447.663044:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.663046:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.663047:0:19944:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.116@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:30.0:1619677447.663049:0:19944:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.116@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:1.0:1619677447.663049:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677447.663051:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.663051:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.663053:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677447.663056:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : REPLY try# 0
00000800:00000200:30.0:1619677447.663058:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.663060:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.663061:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.663061:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.663062:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677447.663063:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:6.0:1619677447.663068:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.663071:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.663071:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.663072:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.663075:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.663077:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677447.663081:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:18.0:1619677447.663151:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.663154:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.663154:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.663157:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.663158:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.663160:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677447.663161:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.663163:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.663164:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.663165:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677447.663168:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.663170:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.663173:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.663174:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:18.0:1619677447.663175:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.663176:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.663177:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.663178:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.663178:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.663179:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677447.663179:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.663179:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.663181:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.663184:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.663185:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.663186:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.663188:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.663189:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.663189:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.663189:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.663192:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:17.0:1619677447.663193:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677447.663193:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.663195:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:12.0:1619677447.663195:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.663196:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.663199:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.663199:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.663199:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677447.663200:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.663202:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.663203:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.663204:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.663205:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.663206:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677447.663215:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677447.663216:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677447.663217:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.663218:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.663218:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.663219:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677447.663220:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.663220:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.663221:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.663222:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677447.663234:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677447.663238:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:1.0:1619677447.663238:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:17.0:1619677447.663240:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.663243:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.663244:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:17.0:1619677447.663246:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.663247:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.663248:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.663249:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:17.0:1619677447.663252:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.663254:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 600 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677447.663257:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.663257:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:17.0:1619677447.663258:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:17.0:1619677447.663259:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:4.2:1619677447.663259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.663260:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677447.663260:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:17.0:1619677447.663261:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:20.0:1619677447.663273:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.663275:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.663276:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.663278:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.663279:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.663281:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.0:1619677447.663295:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.663301:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.663302:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.663303:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.663304:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.663305:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:30.0:1619677447.663307:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.663310:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.663311:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.663313:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.663317:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:22.0:1619677447.663338:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)++
00000800:00000200:12.0:1619677447.663340:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:22.0:1619677447.663342:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.22@o2ib3
00000800:00000200:12.0:1619677447.663342:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.663343:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.663343:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000400:00000200:22.0:1619677447.663344:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.22@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677447.663345:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.663347:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.663347:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.663348:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:22.0:1619677447.663349:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.663350:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.663351:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.663352:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.663353:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.663357:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.663357:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677447.663358:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:12.0:1619677447.663360:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.663361:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.663361:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:22.0:1619677447.663362:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.663362:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677447.663362:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.663362:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677447.663364:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.663364:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d5000] (21)++
00000800:00000200:12.0:1619677447.663364:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.663365:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d5000] (22)--
00000800:00000200:22.0:1619677447.663366:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)--
00000800:00000200:20.0:1619677447.663378:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.663380:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.663381:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.663383:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.663384:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.663385:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.663498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.663503:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.663506:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.663508:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.663508:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.663511:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.663512:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:4.2:1619677447.663513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:5.0:1619677447.663514:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.663515:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.663516:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.663517:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.663521:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:5.0:1619677447.663521:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.663522:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.663525:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.663526:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677447.663526:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.663527:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.663527:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677447.663528:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.663529:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677447.663530:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.663530:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.663530:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.663531:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:24.0:1619677447.663532:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.663533:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96725127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.663534:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.663535:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.663536:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.663540:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.663541:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677447.663542:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677447.663544:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:24.0:1619677447.663545:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677447.663545:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.663546:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.663546:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:13.0:1619677447.663547:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.663548:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677447.663548:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.663549:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.663549:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677447.663549:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.663550:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.663563:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.663566:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.663567:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.663567:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.663569:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.663572:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677447.663573:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677447.663575:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.663576:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.663579:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.663581:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.663583:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.663584:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.663585:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.663587:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.663588:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.663591:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.663593:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.663597:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.663597:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.663598:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.663600:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.663600:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.663601:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.663601:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677447.663621:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:4.2:1619677447.663645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.663686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.663701:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.663703:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.663704:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.663705:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.663707:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.663710:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:30.0:1619677447.663874:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677447.663877:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:30.0:1619677447.663880:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.663882:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.663884:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:30.0:1619677447.663885:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96725124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.663887:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.663888:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.663889:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677447.663892:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.663894:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677447.663897:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.663898:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:30.0:1619677447.663898:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677447.663900:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677447.663900:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677447.663901:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677447.663902:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:17.0:1619677447.663911:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.663913:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.663915:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:17.0:1619677447.663916:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.663917:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.663918:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.663918:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.663920:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.663921:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.663922:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.663923:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.663924:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.663925:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.663928:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.663929:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.663929:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677447.663929:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.663930:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.663931:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:2.0:1619677447.663932:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.663933:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.663934:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:17.0:1619677447.663935:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.663936:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.663936:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:2.0:1619677447.663937:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.663938:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000400:00000200:2.0:1619677447.663938:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:17.0:1619677447.663939:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.663939:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:2.0:1619677447.663939:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.663940:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:2.0:1619677447.663941:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.663942:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.663943:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:9.2:1619677447.663944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:2.0:1619677447.663946:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.663947:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.663949:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.663949:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.663950:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677447.663951:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:2.0:1619677447.663951:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.663952:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.663952:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.0:1619677447.663953:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:2.0:1619677447.663953:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677447.663954:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.663955:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.663957:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.663958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677447.663960:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.0:1619677447.663963:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.663965:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:18.0:1619677447.663965:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:20.0:1619677447.663968:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.663968:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.663969:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.663969:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:2.0:1619677447.663969:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:20.0:1619677447.663970:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.663971:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000400:00000200:20.0:1619677447.663972:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.663973:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.663973:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:20.0:1619677447.663974:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.663975:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.663976:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:9.2:1619677447.663978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000400:00000200:20.0:1619677447.663980:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.663981:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.663983:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.663984:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.663985:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:2.0:1619677447.663985:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.663986:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.663987:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.663988:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:2.0:1619677447.663988:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:20.0:1619677447.663989:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:18.0:1619677447.663989:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.663989:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:20.0:1619677447.663990:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:2.0:1619677447.663990:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.663991:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677447.664001:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.664003:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.664003:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:5.0:1619677447.664003:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.664004:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.664004:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.664006:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.664006:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:5.0:1619677447.664006:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.664008:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.2:1619677447.664041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.664043:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.664046:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.664048:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.664049:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.664050:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.664052:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677447.664053:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.664055:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.664056:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.664057:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677447.664060:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.664062:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677447.664064:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.664065:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677447.664065:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677447.664067:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677447.664068:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.664069:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.664070:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:6.0:1619677447.664250:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.664253:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.664254:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.664254:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.664256:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.664259:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.664289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.664295:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.664301:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.664302:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.664303:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.664306:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.664307:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.664308:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.664310:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.664311:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.664312:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.664315:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.664317:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.664320:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.664321:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.664321:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.664323:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.664324:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.664324:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.664325:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.664338:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.664341:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.664341:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.664343:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.664345:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.664346:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.664359:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677447.664378:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.664381:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.664382:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.664383:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.664386:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.664388:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677447.664388:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.664389:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.664391:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:1.0:1619677447.664391:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677447.664392:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.664392:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.664393:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:13.0:1619677447.664394:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.664395:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.664397:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677447.664397:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.664399:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.664402:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.664402:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.664403:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.664405:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.664405:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.664406:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.664407:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.664411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.664418:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.664421:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.664422:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.664422:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.664424:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.664427:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.664431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677447.664437:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.664440:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.664440:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.664440:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.664442:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.664442:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:18.0:1619677447.664460:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.664463:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.664464:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.664466:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.664468:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.664469:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.664471:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.664473:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.664474:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.664475:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.664478:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.664480:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.664483:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.664484:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.664484:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.664486:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.664487:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.664487:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.664488:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677447.664501:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.664503:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.664504:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.664505:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.664507:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.664508:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.664536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.664584:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.664586:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.664587:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.664589:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.664590:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.664592:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.664602:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.664604:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.664605:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.664606:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.664607:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.664608:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.664660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.664668:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.664670:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.664672:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.664675:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.664677:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.664678:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96725113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.664679:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.664680:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.664681:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.664685:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.664686:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.664689:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.664689:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.664690:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.664691:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.664692:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.664693:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.664693:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.664703:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.664706:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.664706:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.664708:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.664710:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.664711:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677447.664815:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.664817:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.664818:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.664820:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.664823:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.664825:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677447.664826:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725112] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.664827:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.664828:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.664829:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677447.664832:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.664833:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677447.664836:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.664837:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677447.664838:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677447.664839:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.664840:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.664840:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.664841:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.664845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.664852:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.664854:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.664855:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.664856:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.664858:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.664860:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677447.664920:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.664922:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.664923:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.664924:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.664926:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.664927:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.664942:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.664945:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.664945:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.664946:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:5.0:1619677447.664946:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:12.0:1619677447.664947:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.664948:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.664948:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.664951:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.664951:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.664953:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.664954:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.664956:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.664957:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.664958:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.664959:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.664962:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.664964:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.664966:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.664967:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.664968:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.664969:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.664970:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.664971:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.664971:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677447.664983:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.664986:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.664987:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.664988:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.664989:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.664990:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:30.0:1619677447.665165:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677447.665195:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.665198:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.665200:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:1.0:1619677447.665200:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.665201:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:30.0:1619677447.665202:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:17.0:1619677447.665203:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.665204:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.665205:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.665205:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677447.665205:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.665205:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:30.0:1619677447.665206:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:1.0:1619677447.665207:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.665208:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.665209:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.665210:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:17.0:1619677447.665214:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.665215:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.665215:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.665216:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.665217:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.665220:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.665220:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.665221:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.665223:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.665223:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.665225:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.665226:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.665226:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.665229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.665235:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677447.665237:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677447.665238:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.665239:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.665239:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.665239:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677447.665239:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.665241:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677447.665241:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.665244:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:2.0:1619677447.665270:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.665272:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.665274:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.665275:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.665278:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.665280:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.665281:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.665282:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.665284:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.665284:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677447.665288:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.665290:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.665293:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.665294:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.665299:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.665300:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.665301:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.665302:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.665303:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:22.0:1619677447.665455:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.665458:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.665459:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.665461:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.665464:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.665466:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.665467:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.665468:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.665469:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.665470:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.665474:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.665476:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.665478:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.665479:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:4.2:1619677447.665479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677447.665480:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.665481:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.665482:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.665482:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.665483:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.665488:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.2:1619677447.665489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:24.0:1619677447.665490:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677447.665493:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.665494:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677447.665495:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:24.0:1619677447.665496:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.665496:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:24.0:1619677447.665498:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:5.0:1619677447.665498:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677447.665499:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:24.0:1619677447.665500:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96725105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.665500:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.665500:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.665501:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.665501:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.665502:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.665503:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.665503:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.665503:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.665505:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.665505:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.665507:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.665510:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.665511:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.665511:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.665512:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.665513:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:24.0:1619677447.665514:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.665516:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.665517:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.665518:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.665518:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.665519:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.665521:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.665521:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.0:1619677447.665521:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.665522:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.665522:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.665523:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.665524:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:5.0:1619677447.665524:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.665525:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677447.665525:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.665526:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.665527:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677447.665531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.665534:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.665537:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.665538:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.665538:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677447.665538:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.665538:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.665540:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.665540:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:13.0:1619677447.665541:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677447.665541:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677447.665542:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:13.0:1619677447.665542:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.665542:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:13.0:1619677447.665544:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.665545:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.665547:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.665554:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.665556:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677447.665557:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.665558:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.665559:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.665560:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.665560:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.665561:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677447.665561:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.665561:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.665562:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.665563:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.665576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.665622:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.665625:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.665626:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.665626:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.665629:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.665631:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677447.665631:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.665634:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.665635:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.665636:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.665639:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.665640:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.665642:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.665643:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:17.0:1619677447.665643:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.665644:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.665645:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.665646:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.665646:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.665647:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.665648:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.665648:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:17.0:1619677447.665648:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.665650:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.665652:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.665653:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.665654:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.665656:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.665656:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.665657:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.665658:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.665668:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.665670:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.665671:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.665672:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.665673:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.665675:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.665728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:20.0:1619677447.665776:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.665779:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.665779:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.665781:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677447.665784:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.665785:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677447.665785:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677447.665788:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.665788:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.665791:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.665794:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.665795:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:20.0:1619677447.665797:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:18.0:1619677447.665797:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.665798:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.665799:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.665800:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.665800:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.665801:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.665802:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.665802:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:18.0:1619677447.665803:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.665803:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677447.665805:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.665807:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.665807:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.665808:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.665809:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.665810:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.665811:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.665811:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.665816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.665823:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.665825:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.665825:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.665826:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.665827:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.665829:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677447.665950:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.665953:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.665955:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.665958:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.665959:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.665961:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96725099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.665962:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.665963:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.665964:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.665967:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.665969:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.665971:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.665972:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.665973:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.665974:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.665975:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.665976:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.665976:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.665986:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.665989:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.665989:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.665991:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.665992:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.665994:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.665998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.666012:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677447.666043:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.666045:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.666046:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.666048:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.666050:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.666052:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677447.666053:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.666056:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.666057:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.666059:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.666061:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.666062:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.666063:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.666065:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.666066:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.666067:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.666070:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.666072:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.666075:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.666076:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.666076:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.666078:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.666079:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.666080:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.666081:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.666081:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:12.0:1619677447.666090:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.666092:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:12.0:1619677447.666093:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.666094:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.666094:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:12.0:1619677447.666095:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.666095:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.666096:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677447.666097:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.666099:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.666122:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.666125:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.666127:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.666127:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.666130:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.666131:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.666132:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.666134:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.666135:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.666136:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.666138:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.666140:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.666142:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.666143:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.666144:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.666145:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.666146:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.666147:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.666147:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677447.666158:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.666161:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.666161:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.666163:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.666163:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.666165:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:4.2:1619677447.666189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.666203:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.666206:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.666207:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.666207:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.666209:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.666212:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.666226:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.666229:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.666230:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.666231:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.666231:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.666232:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.666233:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.666233:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.666234:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.666234:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.666235:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.666238:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.666427:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.666431:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.666431:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.666433:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.666435:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.666436:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.666437:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.666439:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.666440:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.666441:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.666443:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.666445:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.666447:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.666447:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.666448:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.666449:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.666450:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.666451:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.666452:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.666464:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.666465:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.666466:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.666467:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.666468:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.666469:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677447.666511:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.666514:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.666516:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.666519:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.666520:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:30.0:1619677447.666521:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677447.666523:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96725092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.666523:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.666523:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:30.0:1619677447.666525:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.666526:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.666526:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.666527:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677447.666530:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.666532:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.666534:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:18.0:1619677447.666535:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.666536:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.666537:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.666538:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:18.0:1619677447.666538:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.666539:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:18.0:1619677447.666539:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.666540:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.666540:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.666541:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.666542:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.666542:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:18.0:1619677447.666543:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.666545:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.666548:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677447.666548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.666549:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.666550:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.666552:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.666553:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.666554:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.666556:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.666556:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.666557:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.666557:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.666558:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.666559:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.666560:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.666564:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677447.666565:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:13.0:1619677447.666566:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.666566:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.666566:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.666567:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677447.666569:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.666569:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.666571:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.666573:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.666574:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.666747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.666789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.666793:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.666796:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.666797:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.666798:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.666799:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.666802:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.666804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.666819:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677447.666853:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.666856:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.666856:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.666859:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.666859:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.666865:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.666868:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.666871:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.666872:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.666874:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.666876:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.666877:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.666878:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.666879:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.666881:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.666882:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.666885:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.666886:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677447.666888:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:17.0:1619677447.666889:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.666890:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:17.0:1619677447.666890:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.666890:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:30.0:1619677447.666891:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.666892:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:30.0:1619677447.666893:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.666893:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.666893:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.666894:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.666896:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:6.0:1619677447.666905:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.666908:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.666909:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.666909:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.666911:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.666913:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.666948:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.666951:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.666951:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.666954:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.666956:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.666957:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.666958:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.666960:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.666961:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.666962:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.666965:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.666967:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.666970:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.666971:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.666972:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.666974:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.666974:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.666975:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.666976:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.666977:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.666987:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.666989:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.666990:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.666991:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677447.666999:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.667001:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.667002:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.667005:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.667008:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.667009:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.667010:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.667011:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.667013:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.667014:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.667017:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.667019:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.667022:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.667023:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.667024:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.667026:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.667026:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.667027:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.667027:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677447.667038:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.667041:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.667042:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.667044:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.667045:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.667046:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.667059:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.667062:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.667063:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.667063:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.667064:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.667065:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.667071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.667074:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.667077:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.667079:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.667080:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.667081:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.667083:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.667084:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.667085:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.667086:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.667087:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.667090:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.667092:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.667094:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.667095:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.667096:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.667097:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.667098:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.667099:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.667100:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.667101:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677447.667105:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.2:1619677447.667106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677447.667110:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677447.667112:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:22.0:1619677447.667113:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.667114:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.667114:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.667114:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.667115:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.667115:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:22.0:1619677447.667117:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.667120:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.667136:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.667138:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.667139:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.667140:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.667142:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.667143:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.667161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.667205:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.667207:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.667208:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.667209:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.667211:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.667213:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.667221:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.667223:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.667224:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.667225:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.667226:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.667226:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.667282:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.667285:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.667286:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.667288:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.667291:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.667292:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.667298:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.667300:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.667301:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.667302:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:2.0:1619677447.667302:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.667305:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:2.0:1619677447.667306:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.667308:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.667308:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.667308:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.667311:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.667311:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677447.667312:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:2.0:1619677447.667312:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.667313:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677447.667314:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.667315:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.667315:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677447.667316:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.667316:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.667316:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:5.0:1619677447.667317:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:2.0:1619677447.667317:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:5.0:1619677447.667320:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.667322:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.667324:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.667324:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.667325:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.667327:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.667327:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.667328:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:13.0:1619677447.667329:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.667329:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:13.0:1619677447.667332:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.667333:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.667334:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.667336:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.667337:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.667341:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.667344:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.667344:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.667347:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677447.667347:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:20.0:1619677447.667351:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.667353:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:30.0:1619677447.667356:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.667358:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:1.0:1619677447.667358:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677447.667361:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.667361:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.667362:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677447.667362:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000400:00000200:30.0:1619677447.667364:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.667365:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:5.0:1619677447.667365:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000400:00000200:1.0:1619677447.667365:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.667366:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.667366:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000400:00000200:30.0:1619677447.667367:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96725079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.667367:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:1.0:1619677447.667367:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.667368:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.667368:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:1.0:1619677447.667369:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.667370:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.667370:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.667371:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:1.0:1619677447.667372:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.667373:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.667374:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.667376:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.667379:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.667380:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:1.0:1619677447.667381:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.667382:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.667383:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.667383:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:1.0:1619677447.667384:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.667385:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.667385:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:1.0:1619677447.667385:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.667386:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.667387:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.667388:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:1.0:1619677447.667388:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.667388:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677447.667389:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:1.0:1619677447.667390:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.667391:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.667391:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.2:1619677447.667395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677447.667401:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677447.667401:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677447.667403:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:17.0:1619677447.667403:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:12.0:1619677447.667403:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:30.0:1619677447.667404:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:17.0:1619677447.667404:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677447.667404:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.667405:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:1.0:1619677447.667405:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677447.667406:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.667407:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:13.0:1619677447.667407:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.667409:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.667453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.667461:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.667464:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.667465:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.667467:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.667470:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.667472:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.667473:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.667474:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.667475:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.667476:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.667480:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.667482:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.667484:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.667485:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.667486:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.667487:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.667488:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.667488:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.667489:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.667501:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.667503:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.667504:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.667505:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.667507:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.667510:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.667573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.667620:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.667623:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.667624:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.667625:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.667627:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.667629:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677447.667666:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.667669:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.667671:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.667673:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.667675:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677447.667676:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96725075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.667678:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.667679:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.667680:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677447.667683:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.667684:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.667687:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.667688:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677447.667689:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677447.667690:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.667691:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.667692:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.667694:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.667694:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677447.667703:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.667706:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.667707:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.667707:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.667710:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.667713:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.667803:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.667806:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.667807:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.667808:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.667811:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.667812:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:1.0:1619677447.667812:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.667814:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.667815:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.667815:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:1.0:1619677447.667815:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.667817:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.667818:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:6.0:1619677447.667819:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.667822:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.667823:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677447.667824:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:6.0:1619677447.667825:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.667826:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.667826:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:6.0:1619677447.667827:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:1.0:1619677447.667827:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.667828:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:18.0:1619677447.667829:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.667829:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.667829:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.667830:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.667831:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.667831:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.667831:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677447.667832:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.667832:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.667832:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677447.667833:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677447.667834:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677447.667835:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.667836:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.667838:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.667839:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677447.667844:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.667845:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:6.0:1619677447.667845:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.667846:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:18.0:1619677447.667846:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.667848:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.667848:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.667849:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.667850:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000400:00000200:22.0:1619677447.667851:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.667851:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.667851:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.667852:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.667853:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677447.667853:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:6.0:1619677447.667853:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.667854:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677447.667854:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:22.0:1619677447.667855:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677447.667855:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:6.0:1619677447.667855:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.667856:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677447.667856:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.667858:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.667858:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.667864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.667870:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.667871:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.667872:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.667873:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677447.667874:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.667876:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.667877:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.667877:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.667879:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.667881:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677447.667887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.667893:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.667896:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.667898:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.667898:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.667900:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.667902:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677447.667903:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.667905:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.667906:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.667907:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677447.667910:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.667912:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.667915:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.667916:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677447.667916:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677447.667918:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677447.667919:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.667920:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.667921:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:4.2:1619677447.667947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.667992:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.667995:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.667995:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.667996:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.667998:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.668000:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.668072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.668075:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.668078:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.668080:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.668081:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.668083:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.668084:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.668086:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.668087:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.668088:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.668089:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.668092:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.668094:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.668096:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.668097:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.668098:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.668099:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.668100:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.668101:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.668101:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.668113:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.668115:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.668116:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.668116:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:6.0:1619677447.668116:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.668118:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.668119:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.668120:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:6.0:1619677447.668120:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:8.0:1619677447.668122:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.668125:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.668127:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677447.668128:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.668130:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.668131:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.668132:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677447.668135:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.668137:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.668140:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.668141:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.668141:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.668143:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.668143:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.668144:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.668145:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.668155:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.668158:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.668158:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.668160:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.668162:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.668163:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.668225:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.668228:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.668228:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.668230:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.668232:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.668233:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.668347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.668386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.668392:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.668395:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.668396:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.668396:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.668398:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.668401:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.668410:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.668413:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.668414:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.668414:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.668415:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.668415:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:30.0:1619677447.668471:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.2:1619677447.668475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:30.0:1619677447.668483:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677447.668485:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:30.0:1619677447.668487:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.668489:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:30.0:1619677447.668489:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.668490:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.668491:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:30.0:1619677447.668491:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:31.0:1619677447.668492:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677447.668492:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96725064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.668493:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.668494:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.668495:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.668497:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.668499:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.668501:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.668502:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.668502:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677447.668502:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:30.0:1619677447.668504:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.668504:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677447.668505:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:22.0:1619677447.668505:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.668505:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:30.0:1619677447.668506:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000400:00000200:22.0:1619677447.668506:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.668508:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.668511:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.668517:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.668519:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.668520:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.668520:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:13.0:1619677447.668522:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.668523:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.668523:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:12.0:1619677447.668524:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677447.668524:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:13.0:1619677447.668525:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.668525:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.668526:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.668577:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.668579:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.668582:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.668582:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.668584:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.668586:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.668587:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.668593:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.668594:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.668595:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677447.668595:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:17.0:1619677447.668599:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:5.0:1619677447.668599:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.668599:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677447.668601:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:17.0:1619677447.668602:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.668604:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677447.668605:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.668605:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000400:00000200:17.0:1619677447.668606:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:5.0:1619677447.668606:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:17.0:1619677447.668607:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.668607:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.668608:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000400:00000200:17.0:1619677447.668609:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.668609:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000400:00000200:17.0:1619677447.668610:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.668610:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000400:00000200:17.0:1619677447.668611:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:5.0:1619677447.668611:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.2:1619677447.668614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000400:00000200:17.0:1619677447.668614:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.668616:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.668619:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.668620:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:17.0:1619677447.668620:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.668620:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:20.0:1619677447.668622:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:17.0:1619677447.668622:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.668622:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677447.668623:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:20.0:1619677447.668623:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:17.0:1619677447.668623:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.668623:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:5.0:1619677447.668623:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000400:00000200:20.0:1619677447.668624:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.668624:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677447.668624:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000400:00000200:22.0:1619677447.668625:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.668625:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.668626:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.668626:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.668627:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:22.0:1619677447.668628:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.668628:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:9.2:1619677447.668628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:22.0:1619677447.668629:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:22.0:1619677447.668630:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.668632:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.668633:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.668634:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:22.0:1619677447.668634:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:24.0:1619677447.668637:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:22.0:1619677447.668637:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.668638:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.668638:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.668638:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.668641:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.668641:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.668642:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677447.668642:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677447.668644:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677447.668644:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.668645:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.668645:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677447.668646:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677447.668648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:8.0:1619677447.668655:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.668658:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.668658:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677447.668658:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:6.0:1619677447.668658:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677447.668660:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.668660:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677447.668660:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.668661:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:6.0:1619677447.668661:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.668663:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.668663:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.668665:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.668665:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.668666:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:22.0:1619677447.668667:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.668667:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.668669:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.668670:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.668672:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.668673:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.668673:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.668677:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.668678:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.668681:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.668681:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.668682:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.668684:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.668684:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.668685:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.668685:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677447.668697:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.668700:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.668700:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.668702:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.668703:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.668705:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.668714:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.668716:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.668718:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.668718:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.668719:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.668720:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677447.668817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.668832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.668838:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.668841:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.668843:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.668843:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.668845:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.668847:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.668848:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.668850:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.668851:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.668852:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.668863:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.668866:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.668869:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.668870:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.668871:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (22)++
00000800:00000200:9.2:1619677447.668873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (23)++
00000800:00000200:5.0:1619677447.668874:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (24)--
00000800:00000200:5.0:1619677447.668874:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.668875:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.668876:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.668881:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:18.0:1619677447.668884:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000800:00000200:2.0:1619677447.668884:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (24)++
00000400:00000200:18.0:1619677447.668885:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.668885:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000400:00000200:2.0:1619677447.668886:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.668887:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.668887:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.668891:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.668891:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.668892:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.668894:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.668895:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.668896:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.668896:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.668897:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.668923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.668926:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.668929:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.668931:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:24.0:1619677447.668933:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:20.0:1619677447.668933:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.668933:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.668934:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:24.0:1619677447.668936:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:20.0:1619677447.668936:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.668937:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.668937:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:20.0:1619677447.668939:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.668940:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.668941:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.668943:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.668944:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000400:00000200:24.0:1619677447.668945:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.668946:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96725055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.668946:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.668948:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.668949:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.668949:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.668949:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:24.0:1619677447.668950:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.668950:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.668952:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:24.0:1619677447.668953:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.668953:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.668953:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.668954:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:24.0:1619677447.668955:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.668958:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677447.668958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.668959:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677447.668960:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000800:00000200:24.0:1619677447.668962:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (24)--
00000800:00000200:24.0:1619677447.668962:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.668963:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.668964:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:18.0:1619677447.668965:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:18.0:1619677447.668967:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000800:00000200:2.0:1619677447.668968:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (24)++
00000400:00000200:18.0:1619677447.668969:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.668969:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000400:00000200:2.0:1619677447.668970:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.668971:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000400:00000200:2.0:1619677447.668972:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.668974:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.668975:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.668975:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.2:1619677447.669004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.669012:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.669015:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.669017:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.669018:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.669020:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.669021:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.669023:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96725054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.669024:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.669025:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.669026:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.669029:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.669031:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.669034:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.669035:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.669036:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.669037:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.669038:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.669039:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.669039:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677447.669052:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.669054:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.669055:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.669057:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.669058:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.669060:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.669282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.669285:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.669287:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.669289:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.669290:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.669292:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.669298:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.669300:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.669301:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.669302:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.669303:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.669307:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.669308:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.669311:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.669312:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.669313:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.669315:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.669315:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.669316:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.669316:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.669329:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.669331:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.669332:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.669334:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.669336:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.669337:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.669348:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.669350:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.669351:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.669352:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.669353:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.669354:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.669475:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.669478:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.669479:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.669481:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.669484:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.669485:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.669486:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.669488:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.669489:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.669490:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.669493:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.669495:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.669498:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.669498:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.669499:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:4.2:1619677447.669500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.669501:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.669501:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.669502:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.669502:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.669507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.669514:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:30.0:1619677447.669516:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677447.669517:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.669517:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.669518:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.669519:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:8.0:1619677447.669521:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.669522:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.669523:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677447.669524:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:30.0:1619677447.669525:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.669526:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.669528:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96725049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.669529:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.669530:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.669531:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.669533:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.669535:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.669537:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.669538:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.669539:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.669540:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:9.2:1619677447.669540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677447.669541:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.669541:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.669542:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677447.669548:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.669551:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:12.0:1619677447.669552:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677447.669553:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.669553:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.669554:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.669555:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.669555:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:12.0:1619677447.669557:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.669557:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.669559:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.669559:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.669559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.669567:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.669570:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.669571:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.669571:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.669572:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.669573:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.669716:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.669718:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.669719:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.669721:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.669724:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.669725:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.669726:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.669727:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.669728:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.669729:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.669733:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.669735:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.669737:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.669737:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.669738:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.669740:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.669740:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.669741:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.669741:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.669746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677447.669750:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:18.0:1619677447.669753:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:17.0:1619677447.669753:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677447.669754:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:18.0:1619677447.669755:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.669755:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.669756:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.669757:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.669757:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:17.0:1619677447.669758:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.669759:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:17.0:1619677447.669760:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.669761:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.669763:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.669764:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.669765:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.669768:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.669770:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.669773:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.669774:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.669774:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.669775:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.669776:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677447.669777:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677447.669779:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677447.669779:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677447.669791:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677447.669791:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.669792:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.669792:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.669793:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677447.669793:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677447.669794:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.669795:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000400:00000200:1.0:1619677447.669795:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.669796:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.0:1619677447.669802:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.669805:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.669806:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.669808:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.669808:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.669811:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677447.669814:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.669817:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.669818:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.669818:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.669819:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.669820:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.669830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.669837:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.669840:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.669842:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.669844:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.669845:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.669846:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.669848:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96725045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.669849:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.669850:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.669851:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.669855:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.669856:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.669859:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.669859:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.669860:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.669861:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.669862:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.669863:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.669864:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677447.669874:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.669876:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.669877:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.669878:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.669880:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.669881:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.669887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.669889:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.669892:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.669894:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.669895:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.669896:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.669897:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:20.0:1619677447.669898:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.669900:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.669900:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.669900:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:20.0:1619677447.669901:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.669901:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:20.0:1619677447.669902:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.669902:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:20.0:1619677447.669903:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.669904:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:20.0:1619677447.669907:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.669909:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.669912:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.669913:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.669913:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.669915:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.669916:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.669917:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.669918:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.669919:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.669924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677447.669927:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.669930:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677447.669930:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:13.0:1619677447.669931:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.669932:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.669932:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:12.0:1619677447.669933:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.669933:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.669934:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.669935:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.669966:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.669969:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.669971:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.669974:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.669977:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.669977:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677447.669978:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:17.0:1619677447.669980:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677447.669980:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.669980:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:5.0:1619677447.669982:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.669983:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.669984:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:1.0:1619677447.669984:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.669986:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.669988:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.669988:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.669989:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.669990:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:1.0:1619677447.669991:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.669992:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677447.669992:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.669993:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000400:00000200:1.0:1619677447.669993:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.669994:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.669996:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.669996:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.669997:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000400:00000200:1.0:1619677447.669997:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.669998:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:1.0:1619677447.669999:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.670002:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.670003:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.670003:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.670005:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.670005:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.670006:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.670007:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.670008:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677447.670009:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.670011:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.670012:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.670014:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.670016:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.670017:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:30.0:1619677447.670018:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677447.670019:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:30.0:1619677447.670020:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.670022:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.670022:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.670022:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677447.670023:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.670023:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.670024:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.670027:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.670063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:4.2:1619677447.670072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.670110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.670118:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.670120:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:20.0:1619677447.670122:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:18.0:1619677447.670122:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.670122:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.670124:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:2.0:1619677447.670124:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.670125:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.670127:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.670127:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677447.670129:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.670131:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.670416:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.670419:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.670419:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.670422:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.670425:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:1.0:1619677447.670425:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.670426:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:6.0:1619677447.670428:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:1.0:1619677447.670428:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.670429:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677447.670430:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.670431:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.670432:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:6.0:1619677447.670433:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.670435:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.670436:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.670437:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.670438:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.670438:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:6.0:1619677447.670440:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.670441:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.670441:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677447.670442:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.670442:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.670443:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:6.0:1619677447.670445:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.670445:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.670446:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677447.670447:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.670447:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.670448:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.670450:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.670451:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.670451:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.670453:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.670454:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.670454:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.670455:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.670459:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.670462:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.670463:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.670465:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.670466:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.670467:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:12.0:1619677447.670467:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677447.670470:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.670471:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.670471:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.670473:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.670476:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:13.0:1619677447.670669:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.670671:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.670672:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.670673:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.670675:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.670675:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.670691:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.670694:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.670694:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.670696:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.670698:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.670699:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.670701:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.670702:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.670703:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.670704:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.670708:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.670709:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.670711:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.670712:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.670713:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.670715:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.670715:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.670716:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.670716:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.670728:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.670731:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.670732:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.670732:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.670734:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.670737:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.670829:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.670831:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.670832:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.670834:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.670837:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.670838:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.670840:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.670841:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.670842:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.670843:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.670846:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.670847:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.670850:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.670851:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.670852:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.670854:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.670854:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.670855:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.670855:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.670866:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.670869:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.670870:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.670871:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.670873:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.670874:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.670875:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.670878:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.670879:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.670881:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.670884:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.670885:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.670886:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96725034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.670888:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.670889:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.670890:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.670893:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.670895:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.670898:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.670899:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.670900:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.670901:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.670902:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.670903:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.670904:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.670905:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.670918:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.670919:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:5.0:1619677447.670919:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.0:1619677447.670920:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.670921:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677447.670921:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.670922:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.670924:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.670925:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.670926:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.670927:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (23)++
00000800:00000200:5.0:1619677447.670927:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.670928:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.670929:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.670930:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:2.0:1619677447.671012:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.671015:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.671016:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.671018:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.671020:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.671022:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.671023:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.671025:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.671025:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.671027:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.671030:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.671032:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.671035:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.671036:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.671036:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.671038:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.671039:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.671039:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.671040:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677447.671052:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.671055:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.671056:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.671058:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.671058:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.671060:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.671122:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.671125:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.671128:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.671130:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.671130:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.671131:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.671133:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96725032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.671134:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.671135:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.671136:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.671139:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.671141:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.671144:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.671145:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.671145:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.671147:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.671147:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.671148:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.671149:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.671159:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.671161:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.671162:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.671163:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.671165:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.671166:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.671179:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.671181:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.671182:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.671183:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.671183:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.671184:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.671218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.671259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677447.671263:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677447.671264:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677447.671266:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:6.0:1619677447.671267:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:1.0:1619677447.671267:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.671268:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.671268:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.671269:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.671270:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.671272:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.671273:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:6.0:1619677447.671273:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:17.0:1619677447.671275:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.671276:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.671277:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.671278:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.671281:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.671283:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.671285:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.671286:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.671287:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.671288:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.671289:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.671290:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.671290:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677447.671305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.671314:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:22.0:1619677447.671314:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:30.0:1619677447.671317:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:22.0:1619677447.671317:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.671318:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.671318:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:6.0:1619677447.671318:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:30.0:1619677447.671320:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.671320:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:31.0:1619677447.671321:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:22.0:1619677447.671321:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.671322:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.671323:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.671323:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:30.0:1619677447.671324:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:22.0:1619677447.671324:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:20.0:1619677447.671325:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.671326:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96725029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.671326:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:30.0:1619677447.671328:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.671328:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.671330:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.671331:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.671332:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.671333:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:20.0:1619677447.671334:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:30.0:1619677447.671339:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.671339:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.671342:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677447.671343:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:30.0:1619677447.671345:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.671345:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.671346:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.671346:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677447.671346:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.671347:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:30.0:1619677447.671348:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677447.671348:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:30.0:1619677447.671349:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677447.671349:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:30.0:1619677447.671350:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677447.671350:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:30.0:1619677447.671351:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677447.671351:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:13.0:1619677447.671361:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.671363:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:5.0:1619677447.671363:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:13.0:1619677447.671364:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.671366:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.671366:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.671366:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.671368:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.671368:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:5.0:1619677447.671368:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.671369:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.671370:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:12.0:1619677447.671379:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.671382:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.671383:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.671384:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.671385:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.671386:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.671390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.671393:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.671395:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.671398:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.671399:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.671400:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.671402:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677447.671403:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.671404:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.671405:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.671406:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677447.671410:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.671412:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677447.671414:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.671415:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677447.671415:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.671417:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.671418:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.671419:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.671419:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.671424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.671432:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.671435:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.671436:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.671436:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.671439:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.671442:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:31.0:1619677447.671470:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.671472:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.671473:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.671475:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.671476:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.671477:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.671527:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.671529:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.671531:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.671533:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.671533:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.671534:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.671535:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96725025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.671537:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.671537:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.671538:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.671541:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.671543:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.671545:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.671545:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.671546:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.671547:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.671548:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.671550:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.671551:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.671552:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677447.671559:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.671562:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.671562:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.671564:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.671565:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.671567:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.671568:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.671570:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.671571:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.671572:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677447.671601:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.671603:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.671604:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.671606:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.671607:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.671608:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.671622:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.671625:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.671626:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.671627:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677447.671627:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.671629:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677447.671629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:20.2:1619677447.671669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:1.0:1619677447.671685:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.671688:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.671688:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.671689:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.671691:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.671694:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.671752:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.671755:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.671756:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.671758:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.671760:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.671761:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.671762:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.671763:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.671764:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.671765:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.671769:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.671770:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.671773:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.671774:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.671774:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.671776:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.671776:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.671777:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.671778:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.671779:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.671789:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.671791:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.671791:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.671792:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.671796:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.671799:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.671801:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.671802:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.671802:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.671804:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.671805:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.671806:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.671806:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:4.2:1619677447.671806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:6.0:1619677447.671808:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.671809:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.671810:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.671811:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.671814:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.671814:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.671815:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:6.0:1619677447.671816:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.671819:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.671822:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.671823:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.671823:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.671825:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.671826:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.671827:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.671828:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.671828:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.671838:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.671841:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.671841:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.671842:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.671845:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.671848:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.671849:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.671850:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.671853:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.671854:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:24.0:1619677447.671855:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:24.0:1619677447.671856:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96725019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.671856:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:24.0:1619677447.671857:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.671857:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:24.0:1619677447.671858:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.671858:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.671860:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:6.0:1619677447.671860:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.671863:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.671865:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:24.0:1619677447.671867:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.671867:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677447.671868:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:24.0:1619677447.671868:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:6.0:1619677447.671868:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677447.671870:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:24.0:1619677447.671870:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.671871:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:6.0:1619677447.671871:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:24.0:1619677447.671872:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.671872:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.671872:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.671873:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.671874:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.671956:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.671959:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.671961:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.671962:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.671965:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.671966:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.671968:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.671969:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.671970:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.671971:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.671974:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.671975:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.671978:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.671979:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.671979:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.671981:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.671981:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.2:1619677447.671982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677447.671983:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.671983:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:31.0:1619677447.671990:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.671992:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.671995:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.671995:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:30.0:1619677447.671996:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.671997:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.671997:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:31.0:1619677447.671998:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:17.0:1619677447.671998:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.671999:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:31.0:1619677447.672000:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96725017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.672001:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.672002:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.672003:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.672003:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:31.0:1619677447.672004:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.672008:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.672010:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.672012:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.672013:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.672013:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.672015:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.672015:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.672017:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:1.0:1619677447.672017:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.672018:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.672019:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677447.672020:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.672020:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.672021:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.672022:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.672022:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.672025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.672030:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.672031:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.672032:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.672033:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.672034:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.672034:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.672034:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.672036:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.672037:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677447.672037:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.672135:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.672137:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.672139:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.672140:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.672141:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.672141:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.672143:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96725016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.672144:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.672145:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.672146:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.672148:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.672150:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.672152:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.672153:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.672153:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.672155:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.672155:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.672156:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.672157:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.672167:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.672169:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.672170:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.672171:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.672173:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.672174:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.672215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677447.672271:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.672273:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.672274:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.672275:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.672277:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.672278:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.672382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.672388:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.672390:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.672393:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.672396:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:17.0:1619677447.672398:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677447.672399:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.672400:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:17.0:1619677447.672401:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:5.0:1619677447.672402:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725014] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.672402:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677447.672403:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.672404:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.672405:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.672406:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677447.672407:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.672409:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.672410:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.672410:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000400:00000200:17.0:1619677447.672413:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.672413:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:17.0:1619677447.672414:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.672415:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:5.0:1619677447.672416:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.672417:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677447.672417:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:17.0:1619677447.672419:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.672419:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677447.672420:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677447.672421:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.0:1619677447.672421:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.672422:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:17.0:1619677447.672423:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.672423:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:17.0:1619677447.672424:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.672425:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.672426:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.672427:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.672427:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.2:1619677447.672428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677447.672428:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677447.672431:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.672433:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.672434:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.672435:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:22.0:1619677447.672441:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.672443:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.672444:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.672444:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.672446:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.672449:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.672454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:18.0:1619677447.672458:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.672461:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:8.0:1619677447.672462:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677447.672462:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:24.0:1619677447.672465:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:18.0:1619677447.672465:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:8.0:1619677447.672465:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:18.0:1619677447.672468:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.672468:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.672470:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:8.0:1619677447.672470:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.672471:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.672472:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.672473:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.672473:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96725011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.672474:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.672475:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677447.672475:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.672476:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.672477:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.672480:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.672482:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:8.0:1619677447.672482:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.672484:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:8.0:1619677447.672484:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.672485:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.672486:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:30.0:1619677447.672487:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:18.0:1619677447.672487:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:8.0:1619677447.672487:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677447.672488:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.672488:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:30.0:1619677447.672489:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.672489:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677447.672489:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.0:1619677447.672489:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.672490:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:31.0:1619677447.672491:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:18.0:1619677447.672491:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677447.672491:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.672491:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.672492:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677447.672493:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:8.0:1619677447.672493:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.672505:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.672507:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.672508:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.672509:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.672511:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.672514:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.672551:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.672554:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.672554:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.672557:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.672559:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.672561:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.672562:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677447.672563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:2.0:1619677447.672563:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.672565:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.672566:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.672569:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.672571:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677447.672571:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.672574:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:2.0:1619677447.672574:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.672575:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.672575:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677447.672577:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.672577:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:2.0:1619677447.672577:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.672577:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:31.0:1619677447.672579:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.672579:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:31.0:1619677447.672580:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:2.0:1619677447.672580:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.672580:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:31.0:1619677447.672582:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96725008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.672583:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.672584:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.672585:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.672588:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.672589:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:17.0:1619677447.672590:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677447.672591:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:31.0:1619677447.672592:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.672592:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.672593:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:18.0:1619677447.672593:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:17.0:1619677447.672593:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:31.0:1619677447.672594:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:18.0:1619677447.672594:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677447.672594:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.672594:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677447.672595:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677447.672595:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:31.0:1619677447.672596:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:17.0:1619677447.672596:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.672597:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:17.0:1619677447.672599:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677447.672606:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.672609:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.672610:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.672612:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.672613:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.672613:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677447.672655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.672844:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.672847:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.672848:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.672849:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.672850:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:1.0:1619677447.672852:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.672853:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:1.0:1619677447.672853:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.672855:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.672856:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.672856:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:1.0:1619677447.672857:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.672858:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.672859:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.672859:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:20.0:1619677447.672860:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.672862:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96725005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.672863:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.672864:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.672865:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.672866:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:1.0:1619677447.672866:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.672869:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.672870:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.672870:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.672871:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677447.672872:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:1.0:1619677447.672872:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.672873:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677447.672874:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.672874:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.672874:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677447.672875:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.672876:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.672877:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.672878:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.672878:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.672879:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:12.0:1619677447.672886:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.672888:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.672889:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.672891:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.672891:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:13.0:1619677447.672893:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677447.672893:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:12.0:1619677447.672894:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:5.0:1619677447.672894:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.672896:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.672897:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.672898:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677447.672911:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.672913:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.2:1619677447.672915:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.672917:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.672917:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677447.672917:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.672918:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.0:1619677447.672919:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.672921:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.672923:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.672924:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.672925:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.672927:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677447.672928:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.672929:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.672930:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.672931:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677447.672935:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.672937:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677447.672939:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.672940:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677447.672940:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677447.672942:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677447.672942:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.672943:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.672944:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.672948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.672956:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.672959:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.672960:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.672960:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.672962:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.672964:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.672967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.672974:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677447.672975:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677447.672977:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.672977:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:31.0:1619677447.672978:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:2.0:1619677447.672978:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.672979:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000400:00000200:31.0:1619677447.672980:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.672980:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:30.0:1619677447.672982:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.672983:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.672984:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.672985:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96725003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.672987:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.672988:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.672989:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.672992:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.672993:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.672996:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.672997:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.672997:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.672999:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.672999:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.673000:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.673001:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.673011:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.673014:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.673015:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.673016:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.673018:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.673019:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.673097:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.673100:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.673101:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.673103:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.673105:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.673106:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.673108:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.673109:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.673110:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.673111:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.673114:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.673116:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.673118:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.673119:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.673120:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.673122:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.673122:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.673123:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.673123:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.673128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.673136:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.673138:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.673139:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.673139:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.673142:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.673146:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:18.0:1619677447.673147:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677447.673149:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677447.673150:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677447.673152:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.673154:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.673156:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677447.673157:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96725001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.673158:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.673159:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.673160:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677447.673163:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.673165:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.673167:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.673168:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.673168:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.673170:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.673170:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677447.673171:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677447.673172:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:9.2:1619677447.673176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.673182:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.673184:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.673185:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.673186:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.673186:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.673188:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.673259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.673300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (23)++
00000800:00000200:30.0:1619677447.673315:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.673317:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.673318:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.673318:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:30.0:1619677447.673320:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.673320:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.673321:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:30.0:1619677447.673322:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.673323:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:18.0:1619677447.673323:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.673325:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.673326:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677447.673328:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.673329:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.673330:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.673331:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.673333:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:18.0:1619677447.673334:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.673336:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.673337:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:18.0:1619677447.673337:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.673338:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.673338:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.673339:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677447.673340:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.673341:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:18.0:1619677447.673341:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677447.673344:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.673344:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.673345:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.673345:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677447.673357:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.673360:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.673360:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.673362:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.673364:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.673365:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.673449:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.673451:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.673453:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.673454:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.673457:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.673459:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.673460:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724998] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.673461:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.673462:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.673463:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.673466:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.673468:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.673470:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.673471:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.673471:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.673472:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.673473:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.673474:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.673474:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677447.673482:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.673485:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.673485:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677447.673487:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:17.0:1619677447.673487:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.673489:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.673490:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.673490:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.673492:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:20.0:1619677447.673493:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.673493:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:17.0:1619677447.673494:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.673495:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:17.0:1619677447.673495:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.673496:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.673497:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.673501:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.673503:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.673506:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.673507:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.673507:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.673509:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.673509:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.673510:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.673511:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.673522:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.673525:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.673525:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.673527:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.673528:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.673530:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.673551:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.673554:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.673555:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.673557:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.673559:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.673560:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.673562:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.673563:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.673564:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.673565:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.673569:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.673570:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677447.673571:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677447.673573:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677447.673573:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.673574:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:30.0:1619677447.673575:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.673575:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.673576:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677447.673577:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.673577:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.673577:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.673577:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.673578:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:30.0:1619677447.673579:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.673580:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.673582:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.673582:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677447.673582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:30.0:1619677447.673584:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.673587:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.673589:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.673589:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:30.0:1619677447.673591:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.673592:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.673592:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:30.0:1619677447.673593:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677447.673593:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.673593:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677447.673594:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.673594:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.2:1619677447.673594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:30.0:1619677447.673596:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.673596:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:18.0:1619677447.673596:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.673599:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677447.673600:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.673602:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.673605:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.673605:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.673606:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.673608:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:5.0:1619677447.673608:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.673609:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.673609:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:12.0:1619677447.673611:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.673611:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96724993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.673612:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677447.673613:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.673614:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.673615:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677447.673615:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.673620:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.673622:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.673624:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.673625:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.673626:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.673628:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)++
00000800:00000200:5.0:1619677447.673628:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:5.0:1619677447.673629:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.673630:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.673630:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.673632:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.673633:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)++
00000800:00000200:13.0:1619677447.673634:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (25)--
00000800:00000200:13.0:1619677447.673635:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677447.673636:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677447.673638:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677447.673639:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.673641:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.673642:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.673643:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677447.673643:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677447.673644:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.673644:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677447.673645:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.673648:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.673649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.673656:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.673659:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.673660:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.673662:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.673665:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.673666:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.673667:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.673669:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.673670:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.673671:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.673675:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.673677:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.673680:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.673680:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.673681:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.673683:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.673683:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.673684:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.673684:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.673697:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.673699:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.673700:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.673701:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.673703:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.673705:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.673857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.673902:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.673904:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.673905:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.673906:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.673908:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.673911:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.673922:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.673925:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.673926:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.673926:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.673927:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.673927:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677447.673970:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.673972:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.673974:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.673975:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.673977:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.673979:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.673980:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.673981:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.673982:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.673983:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.673986:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.673988:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.673989:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.673990:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.673991:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.673992:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.673993:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.673994:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.673995:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677447.673996:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.674007:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.674008:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.674009:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.674009:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000400:00000200:5.0:1619677447.674009:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.674010:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.674011:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.674011:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.674012:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.674013:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:12.0:1619677447.674073:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.674076:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.674077:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.674078:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.674080:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.674081:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.674097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.674100:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.674102:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.674104:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.674105:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.674107:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.674108:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.674109:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.674111:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.674112:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.674113:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.674116:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.674118:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.674121:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.674121:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.674122:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.674124:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.674124:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.674125:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.674127:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.674127:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:9.2:1619677447.674130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677447.674132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677447.674138:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677447.674139:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677447.674139:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:6.0:1619677447.674140:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677447.674141:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:2.0:1619677447.674141:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:31.0:1619677447.674142:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.0:1619677447.674142:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677447.674143:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677447.674143:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:5.0:1619677447.674143:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:2.0:1619677447.674143:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.674144:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677447.674145:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.674146:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.674147:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.674147:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.674148:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:31.0:1619677447.674149:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.674150:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.674152:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:6.0:1619677447.674152:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.674154:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.674154:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.674157:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.674157:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.674158:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.674159:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.674161:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677447.674161:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.674166:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.674167:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.674169:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.674170:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.674172:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.674172:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.674173:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.674174:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.674175:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677447.674175:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.674175:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.674176:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.674176:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:31.0:1619677447.674177:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.674177:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677447.674178:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.674179:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.674181:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.674182:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.0:1619677447.674190:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:22.0:1619677447.674191:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.674193:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677447.674193:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:6.0:1619677447.674194:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:5.0:1619677447.674194:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.674195:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.674196:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677447.674196:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.674196:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.674197:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.674198:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677447.674199:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.674199:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677447.674200:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677447.674201:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677447.674208:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.674211:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.674212:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.674212:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.674213:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.674214:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677447.674299:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.674302:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.674302:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.674304:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.674307:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.674308:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677447.674310:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.674311:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.674312:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.674313:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677447.674317:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.674319:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.674322:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.674323:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:18.0:1619677447.674324:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677447.674326:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.674326:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.674327:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:4.2:1619677447.674327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.674328:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.674329:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:13.0:1619677447.674340:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677447.674340:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.674342:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:13.0:1619677447.674343:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:2.0:1619677447.674343:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:13.0:1619677447.674344:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.674344:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:13.0:1619677447.674346:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.674347:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.674349:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.674356:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.674359:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.674360:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.674360:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.674361:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.674362:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.674375:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.674377:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.674378:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.674379:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.674381:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.674383:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.674396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.674404:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.674407:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.674409:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.674412:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.674412:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.674414:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.674415:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.674416:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.674417:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.674418:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.674421:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.674423:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.674425:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.674426:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.674427:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.674428:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.674429:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.674430:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.674430:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677447.674442:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.674445:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.674446:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.674447:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.674448:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.674450:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.674464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.674472:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.674475:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.674475:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.674477:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.674480:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.674481:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.674482:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.674483:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.674484:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.674485:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.674488:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.674489:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.674492:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.674493:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.674494:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.674495:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.674496:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.674497:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.674498:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.674499:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.674504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.674510:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.674511:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.674513:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677447.674513:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.674513:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677447.674514:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677447.674514:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.674515:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677447.674516:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.674519:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.674528:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.674531:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.674532:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.674532:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.674533:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.674533:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.674607:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.674609:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.674610:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.674612:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.674614:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.674616:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.674617:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.674618:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.674619:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.674620:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.674623:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.674625:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.674627:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.674627:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.674628:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.674629:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.674630:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.674631:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.674632:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.674632:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.674644:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.674645:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.674645:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.674646:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.674646:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.674646:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.674647:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.674647:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.674648:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.674649:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.674691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.674730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.674738:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.674741:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.674742:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.674742:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.674745:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.674748:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.674749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:9.2:1619677447.674750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.674752:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.674754:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677447.674756:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677447.674757:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.674758:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.674759:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.674759:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.674761:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:18.0:1619677447.674761:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.674761:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000400:00000200:20.0:1619677447.674762:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.674762:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.674762:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:20.0:1619677447.674764:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.674765:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.674766:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.674771:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.674774:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.674777:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.674778:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.674779:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.674781:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.674781:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.674782:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.674783:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.674793:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.674796:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.674797:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.674798:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.674800:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.674801:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.674812:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.674815:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.674816:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.674817:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.674817:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.674819:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.675002:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.675005:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677447.675007:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.675008:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:4.2:1619677447.675009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:5.0:1619677447.675010:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.675011:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.675012:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.675014:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.675015:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.675016:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.675019:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.675022:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.2:1619677447.675023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677447.675024:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.675025:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.675025:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.675027:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.675028:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.675029:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.675029:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:31.0:1619677447.675032:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.675035:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.675037:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.675039:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.675040:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.675041:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:31.0:1619677447.675042:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.675043:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96724975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.675043:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:31.0:1619677447.675044:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.675044:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.675045:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.675047:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677447.675047:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.675047:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.675050:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:4.2:1619677447.675051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677447.675052:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.675054:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.675055:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:31.0:1619677447.675058:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.675058:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.675058:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:31.0:1619677447.675059:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:22.0:1619677447.675059:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.675059:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677447.675061:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:22.0:1619677447.675061:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.675062:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.675062:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.675063:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:22.0:1619677447.675065:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677447.675075:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.675078:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.675079:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.675080:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.675080:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.675083:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.675174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:20.2:1619677447.675199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:30.0:1619677447.675207:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677447.675210:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:30.0:1619677447.675212:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677447.675213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677447.675214:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677447.675215:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.675216:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.675217:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.675218:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.675219:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.675220:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.675221:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:30.0:1619677447.675223:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.675224:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.675224:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.675225:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.675225:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677447.675227:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.675228:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:18.0:1619677447.675228:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.675229:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.675231:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.675231:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:18.0:1619677447.675231:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677447.675232:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677447.675233:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:20.0:1619677447.675245:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.675248:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.675249:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.675250:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.675251:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.675253:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677447.675306:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.675309:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.675310:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.675312:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.675314:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.675316:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.675317:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.675318:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.675320:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.675320:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.675324:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.675325:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.675328:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.675329:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.675329:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.675331:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.675332:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.675332:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.675333:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.675346:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.675349:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.675350:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.675351:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.675353:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.675355:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.675499:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.675501:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.675504:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.675505:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.675506:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.675508:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677447.675509:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.675510:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.675511:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.675512:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677447.675515:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.675517:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677447.675520:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.675521:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677447.675521:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677447.675523:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677447.675524:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.675525:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.2:1619677447.675525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.675528:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677447.675529:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:9.2:1619677447.675529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677447.675530:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.675532:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.675533:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.675535:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.675536:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.675537:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.675537:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:30.0:1619677447.675539:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.675539:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.675539:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:30.0:1619677447.675540:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:4.2:1619677447.675540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.675541:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.675541:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.675543:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.675544:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.675546:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:30.0:1619677447.675547:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.675550:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.675550:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.675551:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:9.2:1619677447.675551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677447.675553:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.675553:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.675554:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.675556:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.675557:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:18.0:1619677447.675558:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.2:1619677447.675560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:18.0:1619677447.675561:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:4.2:1619677447.675561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:2.0:1619677447.675561:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.675562:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.675563:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.675564:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677447.675565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677447.675565:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677447.675566:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:13.0:1619677447.675568:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.675568:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.675569:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.0:1619677447.675569:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:13.0:1619677447.675570:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.675571:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:12.0:1619677447.675572:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:5.0:1619677447.675572:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677447.675573:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677447.675573:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:13.0:1619677447.675573:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.675573:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.675575:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.675575:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.675576:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.675577:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.675577:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:5.0:1619677447.675577:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.675578:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.675578:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677447.675580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677447.675580:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.675581:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677447.675581:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.675582:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96724967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.675583:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.675584:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.675586:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.675587:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677447.675588:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.675591:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.675591:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:24.0:1619677447.675593:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.675593:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.675594:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677447.675594:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.675594:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.675595:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:18.0:1619677447.675595:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:5.0:1619677447.675595:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677447.675595:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.675597:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.675597:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:2.0:1619677447.675597:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.675598:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.675598:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.675598:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:24.0:1619677447.675599:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.675599:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:18.0:1619677447.675599:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.675600:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.675602:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.675602:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:18.0:1619677447.675602:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.675602:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677447.675603:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677447.675603:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:24.0:1619677447.675604:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677447.675604:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.675605:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96724966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.675607:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.675608:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.675609:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.675613:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.675615:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677447.675615:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677447.675617:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.675617:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677447.675618:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:1.0:1619677447.675618:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.675619:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.675620:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677447.675620:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:1.0:1619677447.675620:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.675621:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.675621:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:17.0:1619677447.675621:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:18.0:1619677447.675622:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:17.0:1619677447.675622:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.675624:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.675624:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.675625:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.675626:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.675627:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.0:1619677447.675634:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.675636:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.675637:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.675639:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.675639:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677447.675639:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677447.675641:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.675641:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.675643:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.675644:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.675647:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.675649:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.675650:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.675651:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.675652:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.675654:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.675657:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.675659:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.675662:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.675663:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.675663:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.675665:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.675666:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.675667:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.675667:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.675679:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.675681:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.675682:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.675684:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.675685:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.675687:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677447.675722:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677447.675724:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677447.675727:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.675728:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677447.675729:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.675730:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677447.675732:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.675733:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.675734:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.675735:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677447.675739:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.675740:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677447.675744:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.675745:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677447.675745:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677447.675747:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677447.675748:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677447.675749:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677447.675749:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677447.675762:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.675765:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.675766:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.675766:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.675768:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.675771:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.675871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.675877:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.675879:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.675881:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.675882:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.675884:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.675886:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.675887:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.675888:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.675889:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.675890:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.675893:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.675895:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.675898:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.675899:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.675900:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.675902:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.675902:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.675903:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.675904:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.675914:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.675917:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.675918:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.675919:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.675920:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.675922:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.676061:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.676063:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.676064:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.676065:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.676067:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.676068:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677447.676068:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677447.676070:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.676071:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.676073:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.676076:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.676077:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:17.0:1619677447.676077:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.676078:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.676079:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:17.0:1619677447.676080:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.676080:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:17.0:1619677447.676082:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.676083:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.676083:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.676085:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:22.0:1619677447.676086:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.676087:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.676088:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:2.0:1619677447.676088:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:22.0:1619677447.676089:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.676089:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677447.676089:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:22.0:1619677447.676091:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.676091:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:22.0:1619677447.676092:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.676093:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.676093:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:22.0:1619677447.676094:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.676094:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:2.0:1619677447.676094:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.676096:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:2.0:1619677447.676096:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:22.0:1619677447.676098:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.676098:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96724959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.676100:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677447.676100:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.676101:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.676101:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.676101:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:2.0:1619677447.676101:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.676102:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.676104:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.676105:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.676105:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677447.676106:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:22.0:1619677447.676107:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.676108:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677447.676108:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:13.0:1619677447.676109:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:22.0:1619677447.676110:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.676111:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.676111:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:13.0:1619677447.676111:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:2.0:1619677447.676111:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.676111:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:13.0:1619677447.676112:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.676112:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.676113:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:13.0:1619677447.676114:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.676114:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:12.0:1619677447.676115:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.676115:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.676115:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677447.676117:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.676121:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:1.0:1619677447.676121:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.676123:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:12.0:1619677447.676124:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677447.676124:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:17.0:1619677447.676125:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.676125:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677447.676125:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.676126:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.676127:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:12.0:1619677447.676127:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.676127:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.676128:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:22.0:1619677447.676129:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:12.0:1619677447.676129:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.676129:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677447.676129:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.676130:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.676130:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:12.0:1619677447.676131:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:22.0:1619677447.676132:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.676135:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.676186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.676191:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.676193:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677447.676194:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.676196:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.676196:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.676198:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.676199:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.676200:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.676201:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.676203:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.676204:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.676205:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.676206:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.676208:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:8.0:1619677447.676208:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:30.0:1619677447.676209:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.676210:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.676211:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.676211:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.676211:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.676211:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:31.0:1619677447.676213:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.676214:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677447.676214:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.676217:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.676217:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.676217:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:8.0:1619677447.676217:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.676218:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:8.0:1619677447.676218:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:31.0:1619677447.676219:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.676220:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.676220:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677447.676220:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96724956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.676221:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:8.0:1619677447.676221:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.676222:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.676222:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:8.0:1619677447.676222:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.676223:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.676223:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.676223:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:8.0:1619677447.676223:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677447.676224:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.676225:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.676225:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.676226:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:8.0:1619677447.676226:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.676227:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.0:1619677447.676228:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.676231:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.676232:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.676233:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.676234:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.676235:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677447.676235:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.676235:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.676235:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:13.0:1619677447.676236:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.676236:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:13.0:1619677447.676237:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.676237:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.676238:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.676239:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.676239:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.676240:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.676241:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.676241:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.676241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.676243:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677447.676248:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.676251:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.676252:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.676252:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.676255:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.676258:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.676332:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.676334:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.676337:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677447.676340:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.676341:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:5.0:1619677447.676341:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.676342:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.676342:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:20.0:1619677447.676343:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.676345:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.676345:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:20.0:1619677447.676346:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:18.0:1619677447.676346:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.676349:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.676350:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000400:00000200:2.0:1619677447.676351:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.676353:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:2.0:1619677447.676353:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.676354:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.676355:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.676355:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.676356:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:2.0:1619677447.676356:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.676357:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.676358:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.676358:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000400:00000200:2.0:1619677447.676358:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.676359:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.676360:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:2.0:1619677447.676362:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.676363:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.676367:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.676367:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.676368:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.676370:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.676370:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.676371:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.676372:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.676373:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.676375:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.676376:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.676378:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.676378:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.676380:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:13.0:1619677447.676384:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.676387:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.676388:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.676389:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.676391:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.676392:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.676477:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.676480:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.676481:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.676483:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.676485:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.676487:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.676488:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.676490:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.676491:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.676492:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.676495:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.676497:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.676500:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.676501:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.676502:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.676504:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.676504:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.676506:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.676507:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.676507:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677447.676516:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677447.676517:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677447.676519:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677447.676519:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:12.0:1619677447.676520:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.676520:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.676521:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677447.676522:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.676523:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.676525:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.676691:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.676693:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.676696:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.676698:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.676698:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.676699:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.676700:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.676702:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.676703:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.676704:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.676707:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.676708:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.676711:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.676711:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.676712:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.676714:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.676714:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.676715:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.676716:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.676726:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.676728:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.676729:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.676730:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.676732:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.676733:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.676734:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.676737:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.676738:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.676740:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.676742:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.676744:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.676745:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.676746:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.676747:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.676747:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:17.0:1619677447.676748:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:13.0:1619677447.676750:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.676751:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:17.0:1619677447.676752:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.676752:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.676752:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.676754:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:13.0:1619677447.676754:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.676756:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.676757:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.676758:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.676759:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.676760:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.676760:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.676761:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677447.676765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677447.676772:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.676775:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.676776:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.676778:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.676779:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.676781:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.676796:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.676798:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.676799:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.676800:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.676802:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.676803:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677447.676813:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.676816:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.676817:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.676817:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677447.676818:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.676819:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677447.676847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.676855:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.676857:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.676859:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.676861:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.676861:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.676863:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.676864:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.676865:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.676866:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.676867:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.676870:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.676871:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.676873:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.676874:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.676874:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.676876:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.676876:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.676877:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.676878:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677447.676888:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.676890:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.676891:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.676892:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.676894:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.676895:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:24.0:1619677447.676913:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.676916:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.676917:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.676919:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.676921:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.676923:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.676924:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.676925:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.676926:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.676927:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.676930:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.676932:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.676934:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.676935:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.676936:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.676938:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.676938:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.676939:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.676939:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.676944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.676953:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.676955:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.676955:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000400:00000200:18.0:1619677447.676957:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.676957:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000400:00000200:18.0:1619677447.676959:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.676960:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.676963:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.676963:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.676964:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.676965:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.676965:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:18.0:1619677447.676966:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.676967:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.676967:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.676968:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.676968:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.676969:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677447.676969:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.676970:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:2.0:1619677447.676971:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:2.0:1619677447.676975:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.676976:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:2.0:1619677447.676979:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.676980:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:2.0:1619677447.676980:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677447.676982:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:2.0:1619677447.676982:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677447.676983:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677447.676983:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:1.0:1619677447.677006:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.677009:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.677010:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.677010:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.677012:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.677014:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.677091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.677094:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.677096:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.677099:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.677100:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.677101:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.677103:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677447.677104:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724945] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.677105:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.677106:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.677107:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677447.677111:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.677113:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677447.677116:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.677117:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677447.677117:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:17.0:1619677447.677117:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677447.677119:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677447.677120:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677447.677120:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.0:1619677447.677121:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.677121:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:1.0:1619677447.677121:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.677123:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.677125:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.677126:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.677127:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.677129:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.677130:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.677131:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:17.0:1619677447.677131:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:30.0:1619677447.677134:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:17.0:1619677447.677134:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.677135:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.677136:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:30.0:1619677447.677137:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.677138:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:17.0:1619677447.677138:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.677139:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.677139:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677447.677140:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:17.0:1619677447.677141:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.677141:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.677142:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.677143:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.677164:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.677166:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.677167:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.677167:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.677169:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.677170:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.677185:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.677187:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.677188:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.677188:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.677189:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.677189:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677447.677303:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.677305:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.677306:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.677308:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.677310:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.677312:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.677313:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.677315:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.677315:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.677317:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.677320:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.677322:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.677324:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.677325:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.677325:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.677327:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.677327:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.677328:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.677328:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677447.677341:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.677344:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.677345:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.677345:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.677347:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.677349:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.677357:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.677360:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.677361:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.677361:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.677362:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.677362:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.677363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.677411:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.677415:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.677415:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.677416:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.677418:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.677421:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.677448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.677457:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.677460:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.677463:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.677465:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.677465:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.677467:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.677468:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.677469:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.677470:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.677471:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:22.0:1619677447.677473:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:31.0:1619677447.677475:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.677476:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.677476:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:31.0:1619677447.677477:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.677479:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677447.677479:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.677480:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.677480:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:22.0:1619677447.677481:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.677482:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.677482:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:22.0:1619677447.677482:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:31.0:1619677447.677483:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.677484:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:22.0:1619677447.677484:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.677485:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.677486:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.677487:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.677490:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.677492:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:30.0:1619677447.677493:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:22.0:1619677447.677495:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.677496:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:22.0:1619677447.677496:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000400:00000200:30.0:1619677447.677497:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.677497:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.677498:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000400:00000200:30.0:1619677447.677499:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.677499:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677447.677500:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:22.0:1619677447.677500:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.677500:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677447.677501:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:1.0:1619677447.677534:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.677536:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.677537:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.677539:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.677541:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.677542:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.677544:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.677545:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.677546:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.677547:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.677551:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.677552:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.677555:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.677555:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.677556:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.677558:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.677558:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.677559:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.677559:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.677572:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677447.677572:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.677574:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:31.0:1619677447.677575:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:6.0:1619677447.677575:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.677576:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:31.0:1619677447.677578:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.677578:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.677579:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677447.677580:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:31.0:1619677447.677581:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.677582:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677447.677584:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.677585:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.677586:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.677587:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677447.677590:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.677592:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:22.0:1619677447.677593:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677447.677595:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.677595:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:22.0:1619677447.677595:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677447.677596:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:6.0:1619677447.677596:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.677597:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677447.677598:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.677598:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677447.677598:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.677598:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677447.677600:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.677601:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.677602:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.677615:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.677617:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.677618:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.677619:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677447.677663:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.677666:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.677668:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.677668:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.677671:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.677673:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677447.677674:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.677675:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.677676:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.677677:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677447.677681:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.677683:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 440 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.677685:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.677686:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677447.677686:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.677688:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.677689:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.677689:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.677690:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:17.0:1619677447.677712:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.677715:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.677716:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.677716:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.677718:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.677720:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:12.0:1619677447.677721:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677447.677724:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677447.677727:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.677729:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677447.677730:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.677731:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:12.0:1619677447.677733:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677447.677734:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677447.677735:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677447.677736:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:12.0:1619677447.677740:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.677742:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:12.0:1619677447.677744:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.677745:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:12.0:1619677447.677746:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:12.0:1619677447.677747:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:12.0:1619677447.677748:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677447.677749:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677447.677751:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677447.677752:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:9.2:1619677447.677753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.677760:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.677762:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.677763:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.677763:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.677765:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000400:00000200:8.0:1619677447.677765:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.677767:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:8.0:1619677447.677767:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677447.677768:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677447.677769:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:2.0:1619677447.677789:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.677791:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.677793:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.677794:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.677797:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.677798:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.677799:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.677801:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.677802:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.677803:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.677806:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.677807:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.677809:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.677809:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.677810:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.677812:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.677812:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.677813:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.677814:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.677815:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:9.2:1619677447.677818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.677826:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.677827:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.677827:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:18.0:1619677447.677828:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.677828:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.677829:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.677830:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.677830:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.677831:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677447.677832:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.677854:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.677857:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.677858:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.677859:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.677861:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.677862:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.677863:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.677864:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.677865:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.677866:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.677869:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.677870:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.677872:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.677872:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.677873:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.677874:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.677875:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.677876:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.677877:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.677877:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.677888:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.677889:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:1.0:1619677447.677889:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677447.677890:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.677891:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677447.677891:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.677892:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.677892:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.677894:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.677896:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.677950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.677993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.677997:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.678000:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.678001:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.678001:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.678004:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.678006:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.678016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.678023:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.678024:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:22.0:1619677447.678026:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.678027:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.678027:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.678027:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:22.0:1619677447.678028:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.678028:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677447.678029:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:2.0:1619677447.678029:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.678031:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.678035:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.678326:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.678329:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.678329:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.678331:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.678333:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.678335:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.678336:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.678337:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.678338:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.678339:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.678343:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.678344:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.678347:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.678348:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.678348:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.678350:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.678350:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.678351:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.678352:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.678353:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677447.678358:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.678361:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.678362:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.678363:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.678363:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677447.678365:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677447.678365:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:18.0:1619677447.678366:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.678366:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677447.678367:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:6.0:1619677447.678367:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677447.678367:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677447.678368:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.678369:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:18.0:1619677447.678369:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.678370:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.678371:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.678371:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.678372:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:6.0:1619677447.678373:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:18.0:1619677447.678376:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.678378:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.678381:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.678382:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.678383:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.678384:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.678385:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.678386:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.678386:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677447.678464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.678467:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.678470:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.678472:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.678473:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.678474:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.678476:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.678478:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.678479:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.678480:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.678482:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:4.2:1619677447.678482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:20.0:1619677447.678486:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.678488:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.678491:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.678492:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.678493:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.678495:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.678495:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.678496:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.678497:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.678508:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.678511:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.678512:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.678513:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.678515:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.678516:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.678518:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.678521:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.678522:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.678524:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.678527:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:22.0:1619677447.678527:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.678528:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677447.678529:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.678530:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.678530:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.678531:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.678532:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.678532:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.678533:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.678534:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.678535:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:22.0:1619677447.678538:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.678540:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.678543:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.678544:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.678545:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.678547:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.678547:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.678548:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.678549:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677447.678560:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.678563:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.678564:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.678566:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.678566:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.678569:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.678582:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.678584:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.678585:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.678586:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.678587:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.678588:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:2.0:1619677447.678628:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.678630:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.678631:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.678633:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.678637:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.678638:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.678639:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.678641:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.678642:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.678643:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.678646:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.678648:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.678650:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.678651:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.678651:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.678653:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.678653:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.678654:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.678654:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677447.678667:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.678670:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.678671:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.678671:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.678673:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.678675:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.678698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.678704:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.678707:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.678709:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.678710:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.678713:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.678715:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.678716:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.678717:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.678718:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.678719:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.678723:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.678725:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677447.678725:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.678728:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:24.0:1619677447.678728:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.678729:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.678729:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677447.678730:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.678731:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:31.0:1619677447.678732:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.678732:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677447.678732:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.678733:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.678733:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:31.0:1619677447.678734:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677447.678735:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.678736:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.678737:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.678738:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677447.678742:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.678743:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.678745:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677447.678746:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.678747:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)++
00000800:00000200:1.0:1619677447.678747:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:31.0:1619677447.678748:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (24)++
00000800:00000200:17.0:1619677447.678748:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.678749:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.678750:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:31.0:1619677447.678751:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677447.678751:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.678752:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.678752:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677447.678753:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.678764:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.678766:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.678767:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.678767:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.678769:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.678771:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.678780:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.678783:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.678784:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.678784:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.678785:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.678786:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.678933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.678948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.678954:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677447.678958:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677447.678960:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677447.678961:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.678962:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.0:1619677447.678962:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:6.0:1619677447.678965:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.678965:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.678968:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.678969:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677447.678969:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.678971:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.678971:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677447.678972:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.678973:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.678974:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.678975:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.678976:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.678977:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677447.678978:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.678981:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.678982:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.678982:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.678983:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.678984:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.678986:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.678987:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.678988:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.678989:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.678990:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.678990:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677447.678990:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.678991:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.678992:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.678993:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677447.678993:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.678994:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.678994:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.678994:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677447.678994:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (22)++
00000800:00000200:5.0:1619677447.678996:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (23)--
00000800:00000200:5.0:1619677447.678997:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:9.2:1619677447.678998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (22)++
00000800:00000200:5.0:1619677447.678998:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.679000:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.679000:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.679005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.679005:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (23)++
00000800:00000200:20.0:1619677447.679008:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:18.0:1619677447.679008:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (24)--
00000800:00000200:9.2:1619677447.679008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677447.679009:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.679009:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (23)++
00000800:00000200:20.0:1619677447.679010:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.679011:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677447.679011:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (24)--
00000800:00000200:20.0:1619677447.679012:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677447.679012:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.679012:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.679013:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.679015:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.679016:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.679017:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.679018:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.679018:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677447.679018:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:18.0:1619677447.679019:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.679020:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:18.0:1619677447.679021:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.679023:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.679025:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.679025:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.679026:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:2.0:1619677447.679026:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:18.0:1619677447.679027:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.679028:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.679029:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.679031:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.679031:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677447.679035:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.679037:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.679040:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.679041:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677447.679042:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677447.679044:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.679044:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.679045:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.679045:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677447.679057:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.679060:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.679061:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.679063:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.679064:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.679066:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677447.679213:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.679216:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.679218:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.679219:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.679220:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.679222:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.679223:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.679225:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.679226:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.679227:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.679230:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.679232:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.679234:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.679235:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.679236:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.679237:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.679238:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.679239:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.679240:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677447.679240:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.679253:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.679253:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677447.679254:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.679255:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.679256:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.679258:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:17.0:1619677447.679259:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677447.679259:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.679260:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.679261:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:17.0:1619677447.679261:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.679262:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677447.679263:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:17.0:1619677447.679264:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.679266:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.679268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:17.0:1619677447.679268:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.679269:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.679270:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.679271:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.679272:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.679275:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000400:00000200:17.0:1619677447.679275:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.679277:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.679278:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.679280:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.679280:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:17.0:1619677447.679280:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.679281:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:17.0:1619677447.679281:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.679282:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.679282:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:30.0:1619677447.679283:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677447.679283:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.679283:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.679284:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.679285:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.679286:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.679288:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.679290:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.679290:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.679292:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:1.0:1619677447.679299:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:30.0:1619677447.679301:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96724915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.679302:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.679302:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:30.0:1619677447.679303:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.679303:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.679303:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.679304:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.679305:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.679306:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:30.0:1619677447.679308:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.679310:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677447.679312:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.679313:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677447.679314:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677447.679316:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.679316:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.679317:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.679317:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677447.679318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.679322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:13.0:1619677447.679323:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.679325:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677447.679326:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:13.0:1619677447.679326:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.679327:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:20.0:1619677447.679329:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:12.0:1619677447.679330:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677447.679331:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:13.0:1619677447.679331:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.679331:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.679332:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.679333:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:20.0:1619677447.679334:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.679334:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.679335:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.679337:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.679338:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.679338:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.679340:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.679341:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:20.0:1619677447.679341:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.679342:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.679347:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.679350:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.679353:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.679354:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.679355:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.679356:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.679357:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.679358:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.679359:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677447.679370:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.679373:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.679374:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.679374:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:31.0:1619677447.679376:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.679376:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:30.0:1619677447.679377:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.679377:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.679378:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.679379:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:6.0:1619677447.679380:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.679382:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.679459:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.679461:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.679462:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.679463:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.679465:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.679467:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.679501:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.679504:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.679504:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.679506:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.679509:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.679510:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.679512:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.679513:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.679514:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.679515:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.679519:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.679521:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.679523:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.679524:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.679525:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.679527:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.679527:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.679529:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.679530:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.679530:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677447.679540:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:6.0:1619677447.679540:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.679542:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:30.0:1619677447.679543:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:6.0:1619677447.679543:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:30.0:1619677447.679544:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.679544:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677447.679546:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677447.679546:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.679549:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677447.679712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.679751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.679753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677447.679759:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:4.2:1619677447.679761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:18.0:1619677447.679762:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.679763:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.679763:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.679765:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.679768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.679768:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677447.679771:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.679771:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.679772:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677447.679773:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677447.679774:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.679775:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.679776:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.679776:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.679776:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.679777:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:8.0:1619677447.679777:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.679779:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.679780:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.679781:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.679782:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.679783:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.679786:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.679788:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.679791:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.679792:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.679792:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.679794:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.679794:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.679795:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.679796:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.679808:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.679811:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.679812:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.679812:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.679814:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.679817:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.679850:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.679853:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.679853:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.679855:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.679857:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.679858:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.679859:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724908] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.679861:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.679862:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.679863:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.679866:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.679868:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.2:1619677447.679870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677447.679871:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.679871:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.679872:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.679874:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.679875:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.679876:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.679876:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.679878:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.679881:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.679883:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.679885:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.679886:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.679887:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:13.0:1619677447.679887:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:31.0:1619677447.679889:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.679889:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:31.0:1619677447.679890:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.679891:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677447.679891:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.679892:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.2:1619677447.679892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:13.0:1619677447.679893:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.679893:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:31.0:1619677447.679896:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.679896:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.679897:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677447.679898:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.679900:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:31.0:1619677447.679901:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.679902:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.679902:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:5.0:1619677447.679902:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.679903:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677447.679904:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.679905:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:5.0:1619677447.679905:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.679906:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:5.0:1619677447.679907:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:31.0:1619677447.679908:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.679908:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:5.0:1619677447.679908:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.679909:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.679910:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:5.0:1619677447.679910:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.679911:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.2:1619677447.679914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677447.679914:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.679915:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.679915:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:5.0:1619677447.679915:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.679916:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.679917:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.679918:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.679920:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.679921:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:30.0:1619677447.679922:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677447.679922:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.679922:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.679924:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.679924:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.679924:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677447.679925:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:20.0:1619677447.679925:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.679925:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677447.679926:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677447.679926:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:20.0:1619677447.679927:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.679927:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.679928:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:9.2:1619677447.679929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.679930:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677447.679938:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677447.679939:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.679940:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.679941:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.679942:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.679942:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.679943:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.679943:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000400:00000200:2.0:1619677447.679943:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.679944:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.679945:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677447.679946:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677447.679981:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.679984:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.679985:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.679987:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.679989:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.679991:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677447.679992:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.679994:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.679995:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.679996:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677447.679999:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.680001:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.680003:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.680004:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677447.680005:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677447.680006:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.680007:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.680009:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.680010:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.680010:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.680018:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.680021:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.680021:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.680023:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.680023:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:13.0:1619677447.680025:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.680025:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.680025:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.680026:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.680027:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.680046:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.680049:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.680051:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.680053:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.680054:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.680055:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677447.680057:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.680058:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.680059:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.680060:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677447.680063:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.680064:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677447.680067:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677447.680067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677447.680068:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677447.680068:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677447.680070:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677447.680070:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.680071:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.680072:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677447.680084:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.680087:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.680087:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.680088:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.680090:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.680094:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677447.680111:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.680113:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.680115:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.680116:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.680117:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.680119:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:13.0:1619677447.680189:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.680191:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.680192:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.680193:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.680195:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.680196:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.680209:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.680212:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.680213:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.680213:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.680214:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.680215:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.680334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.680342:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.680345:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.680345:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.680347:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.680350:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.680351:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.680352:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.680353:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.680354:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.680355:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.680358:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.680359:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.680363:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.680363:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.680364:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.680366:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.680366:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.680367:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.680368:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.680369:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.680374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.680380:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.680381:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.680382:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677447.680383:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677447.680383:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.680384:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677447.680384:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.680385:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.680386:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.680388:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.680397:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.680400:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.680401:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.680401:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.680402:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.680403:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.680478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677447.680481:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.680483:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.680484:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.680486:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.680487:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:1.0:1619677447.680488:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.680490:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:1.0:1619677447.680490:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.680491:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.680493:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.680493:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.680494:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:1.0:1619677447.680494:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.680495:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.680496:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.680498:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677447.680499:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.680500:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.680502:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.680502:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.680503:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.680504:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677447.680505:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.680506:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.680507:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:30.0:1619677447.680508:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.680509:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.680509:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677447.680510:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677447.680510:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.680510:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.680513:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677447.680513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677447.680514:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.680514:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.680516:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.680517:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.680518:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.680518:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.680523:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.680526:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.680527:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.680529:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.680531:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.680531:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.680531:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.680533:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.680534:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.680536:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.680536:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.680539:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677447.680560:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.680563:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.680564:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.680565:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.680567:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.680570:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.680634:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.680636:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.680639:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.680640:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.680641:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.680642:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.680643:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.680645:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.680646:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.680647:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.680650:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.680652:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.680654:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.680655:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.680656:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.680657:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.680658:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.680658:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.680659:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.680671:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.680673:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.680674:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.680676:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.680677:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.680678:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.2:1619677447.680748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:17.0:1619677447.680794:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.680797:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.680798:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.680798:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.680801:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.680804:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.680905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.680908:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.680910:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.680913:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.680914:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.680915:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.680917:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.680918:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724895] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.680919:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.680920:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.680921:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.680924:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.680926:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.680928:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.680929:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.680930:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.680932:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677447.680933:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677447.680933:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.680933:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.680934:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677447.680936:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.680938:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.680940:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677447.680940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677447.680941:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.680943:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.680944:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.680945:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.680946:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.680947:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.680948:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:31.0:1619677447.680950:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.680950:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.680951:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.680952:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.680952:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.680953:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.680954:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.680955:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.680956:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.680956:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677447.680957:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.680958:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.680960:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.680961:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.680962:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677447.680970:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.680972:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.680973:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.680974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677447.680975:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:13.0:1619677447.680975:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.680976:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.680977:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:13.0:1619677447.680977:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.680978:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.680979:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677447.680981:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.680984:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.680985:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.680987:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.680989:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.680990:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.680992:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.680993:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.680994:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677447.680994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:8.0:1619677447.680995:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.680998:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.681000:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.681003:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.681003:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.681004:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.681006:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.681006:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.681007:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.681007:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.681019:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.681022:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.681023:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.681023:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.681025:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.681028:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:6.0:1619677447.681039:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.681042:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.681043:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.681043:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.681046:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.681048:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.681058:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.681061:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.681061:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.681062:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.681063:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.681063:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.681240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.681247:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.681250:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.681252:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.681254:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.681254:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.681256:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.681257:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.681258:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.681259:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.681260:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.681263:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.681264:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.681265:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.681266:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.681267:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.681267:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.681267:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.681268:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:31.0:1619677447.681269:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.681269:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.681270:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.681271:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.681271:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.681271:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677447.681272:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:12.0:1619677447.681281:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.681284:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:30.0:1619677447.681285:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000400:00000200:12.0:1619677447.681285:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.681286:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.681288:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.681288:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:12.0:1619677447.681288:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.681289:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:13.0:1619677447.681289:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.681290:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.681291:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677447.681298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.681305:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.681307:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.681309:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.681310:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.681312:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.681313:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677447.681315:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.681316:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.681317:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.681318:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677447.681321:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.681323:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677447.681325:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.681326:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677447.681327:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677447.681328:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677447.681329:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.681330:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.681330:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.681335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.681344:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.681346:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.681348:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.681348:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.681349:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.681352:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:17.0:1619677447.681435:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.681439:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.681440:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.681441:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.681442:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677447.681443:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000400:00000200:17.0:1619677447.681445:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.681446:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.681447:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.681448:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.681449:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:17.0:1619677447.681450:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.681451:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.681452:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677447.681452:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.681453:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.681455:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.681455:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.681457:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:17.0:1619677447.681458:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.681458:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.681458:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.681459:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:17.0:1619677447.681460:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.681460:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.681461:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.681462:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.681462:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.681463:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.681464:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.681465:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.681467:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.681468:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.681469:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:6.0:1619677447.681470:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.681470:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.681471:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.681472:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.681473:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:17.0:1619677447.681474:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677447.681474:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.681475:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.681476:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.681476:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.681477:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:6.0:1619677447.681477:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.681477:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:1.0:1619677447.681477:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.681478:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.681478:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.681480:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.681480:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:1.0:1619677447.681480:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:6.0:1619677447.681481:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.681482:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677447.681482:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdf63800] (22)++
00000800:00000200:6.0:1619677447.681483:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677447.681483:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (23)--
00000800:00000200:1.0:1619677447.681484:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:12.0:1619677447.681485:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:9.2:1619677447.681485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:1.0:1619677447.681485:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.681486:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:12.0:1619677447.681488:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677447.681489:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677447.681489:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.681489:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677447.681490:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.681493:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.681493:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677447.681494:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:12.0:1619677447.681494:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.681495:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677447.681496:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:17.0:1619677447.681496:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)++
00000400:00000200:1.0:1619677447.681496:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.681497:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.681497:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdf63800] (23)--
00000800:00000200:2.0:1619677447.681497:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:17.0:1619677447.681499:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677447.681499:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.681500:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.681500:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.681500:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:1.0:1619677447.681502:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:9.2:1619677447.681503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.681507:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.681508:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.681510:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.681511:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.681511:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.681512:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.681513:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677447.681581:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.681584:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.681585:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.681586:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.681589:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.681591:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677447.681592:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.681593:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.681594:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.681595:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677447.681599:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.681601:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677447.681604:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.681605:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677447.681605:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677447.681607:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677447.681608:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.681608:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.681609:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677447.681641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:4.2:1619677447.681646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677447.681649:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.681652:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:24.0:1619677447.681652:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:31.0:1619677447.681655:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.681655:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:31.0:1619677447.681657:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.681657:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.0:1619677447.681657:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:31.0:1619677447.681658:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677447.681658:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.681659:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.681661:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.681661:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.681662:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.681662:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:31.0:1619677447.681663:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677447.681663:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.681665:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.681665:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:24.0:1619677447.681666:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.681667:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.681668:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:18.0:1619677447.681668:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.681669:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677447.681670:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:18.0:1619677447.681672:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.681673:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.681673:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.681674:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.681674:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:18.0:1619677447.681675:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.681676:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.681676:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677447.681677:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:18.0:1619677447.681677:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677447.681678:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.681678:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:18.0:1619677447.681678:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96724882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.681679:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.681680:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.681680:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:18.0:1619677447.681680:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.681681:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.681682:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.681682:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:18.0:1619677447.681682:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.681683:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.681684:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677447.681685:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.681687:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.681688:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:18.0:1619677447.681690:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.681691:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677447.681691:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:18.0:1619677447.681692:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:13.0:1619677447.681692:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.681693:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.681694:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.681694:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.681695:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:12.0:1619677447.681695:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.681696:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677447.681696:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:13.0:1619677447.681696:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677447.681699:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.681700:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.681700:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.681703:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.681705:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:12.0:1619677447.681712:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.681714:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.681714:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:12.0:1619677447.681715:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.681716:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.681717:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.681717:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.681718:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.681718:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.681719:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:12.0:1619677447.681720:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677447.681720:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:13.0:1619677447.681721:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.681724:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.681725:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.681727:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.681728:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.681730:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.681735:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.681738:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.681740:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.681742:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.681742:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.681744:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677447.681745:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.681746:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.681747:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.681749:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677447.681752:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.681754:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677447.681756:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.681757:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677447.681758:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677447.681760:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677447.681761:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.681762:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.681762:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677447.681766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.681774:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.681776:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.681778:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.681778:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.681780:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.681783:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.0:1619677447.681893:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.681895:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.681897:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.681898:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.681900:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.681901:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.681902:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.681904:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.681905:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.681906:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.681909:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.681910:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.681913:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.681914:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.681914:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.681916:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.681916:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.681917:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.681917:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.681930:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.681932:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.681933:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.681935:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.681936:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.681941:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:17.0:1619677447.681942:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.681945:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.681946:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.681947:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.681950:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.681952:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.681953:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.681954:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.681955:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.681956:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.681959:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.681961:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.681963:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.681964:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.681965:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.681967:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.681967:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.681968:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.681968:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.681980:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.681983:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.681984:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.681985:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.681986:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.681989:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.681997:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.681999:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.682000:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.682001:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.682001:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.682003:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.0:1619677447.682008:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.682011:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.682012:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.682015:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.682017:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.682019:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.682021:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.682022:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.682024:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.682025:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.682029:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.682031:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.682035:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.682035:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.682036:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.682039:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.682039:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.682040:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.682041:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.682045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.682052:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:18.0:1619677447.682053:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:12.0:1619677447.682054:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.682055:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.682056:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.682057:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.682057:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.682057:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677447.682058:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:18.0:1619677447.682060:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.682060:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677447.682063:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.682262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.682270:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.682273:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.682275:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.682277:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.682278:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.682279:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.682281:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.682282:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.682283:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.682284:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.682287:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.682288:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.682290:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.682291:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.682292:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.682299:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.682299:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.682301:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.682302:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.682303:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.682304:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.682305:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.682306:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.682307:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.682308:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.682309:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677447.682310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:1.0:1619677447.682317:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677447.682318:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.682320:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:1.0:1619677447.682320:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:30.0:1619677447.682321:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677447.682321:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677447.682322:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:1.0:1619677447.682323:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.682326:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.682328:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.682329:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.682330:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677447.682331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677447.682331:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.682332:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.682336:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.682337:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.682341:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.682342:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.682343:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.682345:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.682345:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.682346:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.682346:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.682358:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.682361:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.682362:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.682363:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.682365:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.682366:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.682372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677447.682376:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.682378:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.682379:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.682381:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.682382:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.682384:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:22.0:1619677447.682460:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.682463:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.682464:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.682466:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.682468:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.682469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:22.0:1619677447.682470:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.682471:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.682472:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.682472:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:22.0:1619677447.682474:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.682475:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:20.0:1619677447.682475:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.682477:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.682478:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:22.0:1619677447.682479:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.682480:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.682481:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:20.0:1619677447.682482:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.682483:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.682484:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.682485:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.682486:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.682486:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:20.0:1619677447.682486:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.682486:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.682488:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.682489:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.682490:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.682490:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:20.0:1619677447.682490:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.682492:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.682495:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.682496:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.682497:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.682499:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.682499:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.682500:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.682502:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.682502:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.2:1619677447.682507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.682513:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.682515:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.682516:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.682517:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:12.0:1619677447.682526:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.682528:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.682529:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.682531:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.682532:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.682534:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.682547:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.682550:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.682551:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.682552:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.682552:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.682554:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.682570:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.682572:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.682575:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.682576:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.682576:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.682578:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677447.682579:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.682580:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.682581:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.682582:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677447.682586:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.682587:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677447.682588:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.682589:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:31.0:1619677447.682591:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.682591:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677447.682592:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677447.682592:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:2.0:1619677447.682592:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.682594:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677447.682595:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:2.0:1619677447.682595:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.682596:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.682596:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:2.0:1619677447.682597:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.682598:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.682600:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.682601:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677447.682602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:2.0:1619677447.682602:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.682606:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.682608:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.682610:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.682612:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.682612:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.682613:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.682613:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.682614:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677447.682615:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.682616:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.682617:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.682617:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.682618:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677447.682619:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.682619:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:17.0:1619677447.682623:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.682626:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.682627:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677447.682628:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:17.0:1619677447.682629:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.682631:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:17.0:1619677447.682632:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.682632:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.682634:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:13.0:1619677447.682634:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.682635:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.682636:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724869] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.682637:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.682638:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.682638:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:17.0:1619677447.682639:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.682642:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.682644:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.682647:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.682648:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.682648:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.682650:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.682651:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.682651:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.682652:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.682664:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.682666:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.682667:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.682667:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.682670:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.682672:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.682694:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.682696:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.682698:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.682700:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.682702:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.682704:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.682705:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.682706:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.682707:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.682708:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.682711:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.682713:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.682716:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.682717:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.682717:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.682719:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.682720:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.682721:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.682721:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.682725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.682733:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.682736:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.682737:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.682737:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.682739:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.682742:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.682747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.682754:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.682757:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.682758:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.682758:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.682758:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.682759:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.682973:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.682975:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.682976:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.682978:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.682980:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.682982:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.682983:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.682985:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.682986:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.682987:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.682990:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.682992:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.682995:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.682996:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.682996:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.682998:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.682999:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.683000:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.683001:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.683002:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.683011:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:30.0:1619677447.683013:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677447.683013:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.683014:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.683015:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.683016:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.683016:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.683018:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.683019:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.683021:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677447.683075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.683081:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.683084:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.683086:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.683087:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.683090:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.683091:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.683093:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.683095:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.683096:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.683098:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.683101:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.683104:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.683106:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.683107:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.683107:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.683109:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.683110:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.683110:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.683111:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677447.683123:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.683125:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.683126:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.683127:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:20.0:1619677447.683128:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.683129:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:22.0:1619677447.683130:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.683130:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.683131:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.683131:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:22.0:1619677447.683133:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.683135:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677447.683226:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.683229:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.683231:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.683233:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.683234:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.683235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:31.0:1619677447.683236:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.683237:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.683238:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.683239:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.683240:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.683241:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.683243:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:31.0:1619677447.683244:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677447.683246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:5.0:1619677447.683246:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.683247:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.683248:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.683249:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:31.0:1619677447.683250:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.683250:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677447.683251:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.683251:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677447.683251:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.683252:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.683253:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.683253:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.683253:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:5.0:1619677447.683253:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.683254:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.683254:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:5.0:1619677447.683254:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.683255:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677447.683256:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.683259:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.683259:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.683261:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.683262:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:30.0:1619677447.683263:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:5.0:1619677447.683264:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677447.683265:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96724862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.683265:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.683265:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:30.0:1619677447.683266:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.683266:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:30.0:1619677447.683267:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.683267:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:30.0:1619677447.683268:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:13.0:1619677447.683268:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.683268:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.683268:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.683269:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:13.0:1619677447.683270:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.683271:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.683271:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.683271:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677447.683272:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.683274:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:13.0:1619677447.683275:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.683276:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677447.683276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677447.683277:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.683278:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.683279:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677447.683279:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:30.0:1619677447.683280:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.683281:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.683281:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677447.683281:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:6.0:1619677447.683281:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.683282:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.683283:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.683283:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.683284:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.683285:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.683286:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.683288:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677447.683291:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.683300:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:22.0:1619677447.683300:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:31.0:1619677447.683301:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.683301:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:22.0:1619677447.683303:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677447.683304:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.683304:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.683305:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.683306:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.683306:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677447.683307:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:1.0:1619677447.683315:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.683317:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.683318:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.683321:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.683324:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.683325:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.683326:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.683328:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.683329:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.683330:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.683333:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.683335:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.683338:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.683339:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.683340:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.683342:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.683342:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.683343:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.683343:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677447.683346:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.683348:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.683349:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.683351:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.683354:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.683356:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:12.0:1619677447.683356:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:18.0:1619677447.683357:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.683358:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.683359:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.683359:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:18.0:1619677447.683360:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:12.0:1619677447.683360:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.683362:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.683363:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:18.0:1619677447.683364:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.683366:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.683368:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677447.683370:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.683370:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.683371:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.683373:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.683374:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.683375:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.683376:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.683377:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:4.2:1619677447.683377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.683387:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.683387:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.683389:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:13.0:1619677447.683390:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677447.683390:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:13.0:1619677447.683391:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.683391:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:13.0:1619677447.683392:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.683393:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.683395:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.683498:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.683501:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.683502:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.683504:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.683507:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.683508:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.683510:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.683511:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.683512:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.683513:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.683517:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.683518:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.683521:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.683522:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.683522:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.683524:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.683524:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.683525:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.683526:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.683527:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:6.0:1619677447.683530:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.683533:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.683533:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.683535:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.683537:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:6.0:1619677447.683538:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.683539:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:6.0:1619677447.683539:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:2.0:1619677447.683539:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.683540:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677447.683541:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:6.0:1619677447.683541:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.683541:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:17.0:1619677447.683542:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.683542:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.683542:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:6.0:1619677447.683543:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.683545:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.683545:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.683548:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:6.0:1619677447.683550:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.683552:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.683555:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.683556:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.683557:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.683559:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.683564:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.683564:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.683565:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677447.683573:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.683575:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.683576:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.683578:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.683579:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.683581:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:22.0:1619677447.683583:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.683586:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.683587:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.683587:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.683590:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.683593:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677447.683627:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.683630:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.683631:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.683633:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.683635:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.683637:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677447.683638:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.683639:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.683641:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.683641:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677447.683645:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.683646:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677447.683649:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.683649:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677447.683650:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677447.683652:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677447.683652:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.683653:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.683653:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677447.683665:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.683667:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.683668:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.683668:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.683670:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.683672:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.683792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677447.683833:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.683835:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677447.683836:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:31.0:1619677447.683837:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.683838:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.683838:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.683839:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.683840:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.683841:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677447.683842:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.683843:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.683844:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.683845:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.683847:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:31.0:1619677447.683848:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.683848:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.683849:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.683849:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.683850:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.683851:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.683852:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.683852:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.683853:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.683853:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677447.683853:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677447.683854:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677447.683856:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.683856:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.683858:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.683859:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.683860:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677447.683869:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.683872:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.683872:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.683873:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:22.0:1619677447.683873:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.683875:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:22.0:1619677447.683875:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.683876:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.683877:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677447.683878:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.683912:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.683915:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.683916:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.683918:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.683918:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677447.683921:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.683921:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.683922:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677447.683923:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.683925:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.683925:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.683927:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.683927:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.683928:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.683929:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677447.683929:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.683931:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.683933:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.683934:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.683935:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.683936:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.683939:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:1.0:1619677447.683939:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.683941:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.683942:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.683942:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.683943:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.683943:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.683944:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.683945:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677447.683945:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.683946:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:1.0:1619677447.683946:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.683947:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.683947:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:1.0:1619677447.683947:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.683948:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.683949:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.683951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.683959:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:17.0:1619677447.683960:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677447.683961:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.683962:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677447.683962:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.683963:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.683963:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.683963:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.683967:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.683967:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.683969:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.683970:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.684034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.684037:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.684039:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.684042:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.684043:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.684044:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.684046:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.684047:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.684049:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.684050:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.684051:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.684054:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.684056:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.684058:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.684059:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.684060:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.684061:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.684062:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.684063:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.684063:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.684075:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.684077:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.684078:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.684078:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.684080:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.684082:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.684148:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:4.2:1619677447.684149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.684151:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.684152:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.684154:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.684156:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.684158:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.684159:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.684161:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.684163:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.684164:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.684168:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.684171:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.684174:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.684175:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.684175:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.684177:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.684178:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.684179:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.684180:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.684180:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.0:1619677447.684189:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:30.0:1619677447.684191:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.684193:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:5.0:1619677447.684193:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:30.0:1619677447.684194:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.684194:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:4.2:1619677447.684194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677447.684195:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : ACK - routed
00000400:00000200:30.0:1619677447.684196:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.684196:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:31.0:1619677447.684197:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.684197:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:5.0:1619677447.684197:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.684198:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:5.0:1619677447.684198:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:30.0:1619677447.684199:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.0:1619677447.684199:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.684200:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.684201:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:5.0:1619677447.684201:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.684202:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.684202:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.684203:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.684203:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.684205:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.684207:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.684209:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.684211:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.684211:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.684211:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.684212:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.684213:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.684214:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.684215:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.684216:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677447.684217:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:5.0:1619677447.684218:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.684229:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.684229:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677447.684229:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:5.0:1619677447.684230:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.684231:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.684231:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.684231:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:20.0:1619677447.684233:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.684233:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.684235:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.0:1619677447.684259:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.684262:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.684263:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.684263:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677447.684264:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.684265:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.2:1619677447.684289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:20.2:1619677447.684336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:5.0:1619677447.684340:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.684343:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.684343:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.684345:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677447.684346:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.684348:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.684471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.684478:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.684481:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.684482:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.684483:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.684486:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.684488:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.684489:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.684490:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.684491:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.684492:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.684495:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.684496:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.684497:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677447.684499:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.684500:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:24.0:1619677447.684500:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.684500:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677447.684502:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.684502:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.684502:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.684503:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.684503:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:31.0:1619677447.684505:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.684505:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.684506:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.684507:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.684509:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.684509:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.684510:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.684513:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.684515:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.684516:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677447.684517:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.684518:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.684518:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.684519:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:31.0:1619677447.684520:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.684520:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677447.684520:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.684520:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.684521:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.684522:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:1.0:1619677447.684522:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.684525:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:12.0:1619677447.684533:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677447.684536:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677447.684536:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.684537:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.684539:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677447.684539:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.684539:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.684540:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:13.0:1619677447.684540:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.684541:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.684541:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:12.0:1619677447.684541:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.684547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.684550:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677447.684550:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.684553:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:13.0:1619677447.684553:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.684554:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.684555:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:20.0:1619677447.684556:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.684556:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.684556:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:4.2:1619677447.684556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.684557:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:20.0:1619677447.684559:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.684562:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.684563:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724842] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.684564:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.684566:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.684567:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.684571:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.684573:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.684576:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.684577:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.684578:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.684579:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.684583:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.684584:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.684585:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.684585:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.684588:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.684589:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.684590:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.684592:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:18.0:1619677447.684594:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.684595:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:18.0:1619677447.684595:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677447.684596:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.684596:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96724840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.684597:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.684598:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.684599:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:18.0:1619677447.684599:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.684600:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:18.0:1619677447.684600:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.684604:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.684604:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:18.0:1619677447.684606:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.684607:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.684608:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.684609:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.684609:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677447.684610:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.684610:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:5.0:1619677447.684610:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.684612:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.684612:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.684613:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677447.684613:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677447.684614:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:6.0:1619677447.684629:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.684632:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.684632:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.684635:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.684637:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.684639:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.684640:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.684642:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.684643:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.684644:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.684648:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.684650:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.684652:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.684653:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.684653:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.684655:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.684656:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.684657:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.684657:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.684659:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:9.2:1619677447.684662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.684662:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.684664:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.684665:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.684667:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.684668:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.684668:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.684669:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.684670:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.684671:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.684671:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.684673:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.684674:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.684675:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.684678:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.684680:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.684682:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.684682:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.684683:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677447.684683:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.684684:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.684686:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.684686:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.684687:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.684688:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.684692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.684699:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.684700:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.684701:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.684702:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.684703:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.684704:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.684832:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.684834:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.684835:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.684837:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.684839:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.684840:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.684841:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724837] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.684843:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.684844:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.684845:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677447.684848:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.684850:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.684852:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.684853:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.684854:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.684856:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.684856:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.684857:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.684858:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.684862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.684869:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.684872:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.684872:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.684873:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.684875:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.684878:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.685073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.685121:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677447.685122:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:9.2:1619677447.685122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.685123:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.685124:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.685125:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:12.0:1619677447.685126:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.685128:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.685128:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.685128:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:12.0:1619677447.685129:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:20.0:1619677447.685130:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.685130:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:20.0:1619677447.685132:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677447.685133:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.685133:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:20.0:1619677447.685134:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.685134:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.685134:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.685136:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.685136:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.685137:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677447.685140:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.685140:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.685142:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 440 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677447.685146:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.685146:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677447.685147:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677447.685149:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677447.685150:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.685151:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.685151:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:9.2:1619677447.685173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.685180:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.685182:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.685183:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.685184:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.685186:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.685189:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.685198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677447.685203:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.685206:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.685207:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.685207:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.685208:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.685209:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677447.685302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.685308:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.685310:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.685313:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.685313:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.685316:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.685318:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.685319:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.685320:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.685321:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.685322:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.685326:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.685328:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.685381:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.685382:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.685383:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.685385:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.685385:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.685386:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.685387:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677447.685400:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.685403:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.685404:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.685406:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.685408:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.685409:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.685427:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.685430:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.685432:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.685433:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.685434:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.685435:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.685436:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.685438:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.685439:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.685440:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.685443:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.685444:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.685447:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.685448:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.685449:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.685450:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.685450:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.685451:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.685452:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.685463:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.685465:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.685466:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.685467:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.685469:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.685470:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.685541:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.685543:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.685544:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.685545:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.685547:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.685548:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.685592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.685635:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.685638:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.685639:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.685639:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.685641:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.685645:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.685655:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.685657:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.685658:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.685658:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.685659:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.685660:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.685680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.685683:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.685686:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.685688:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.685689:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.685691:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.685693:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677447.685694:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.685695:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.685696:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.685697:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677447.685701:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.685703:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677447.685706:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.685707:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677447.685707:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677447.685709:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677447.685710:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.685711:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.685711:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677447.685722:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.685725:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.685726:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.685728:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.685728:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.685731:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677447.685736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.685743:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.685746:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.685748:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.685749:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.685750:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.685751:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.685752:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.685753:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.685754:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.685755:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.685758:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.685760:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.685762:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.685763:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.685764:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.685766:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.685766:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.685767:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.685767:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:17.0:1619677447.685773:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.685777:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:13.0:1619677447.685778:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677447.685778:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.685779:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.685780:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.685781:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.685782:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.685782:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.685784:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:17.0:1619677447.685785:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:13.0:1619677447.685785:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:17.0:1619677447.685786:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.685787:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.685788:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.685790:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.685793:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.685795:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.685798:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.685798:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.685799:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677447.685800:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677447.685801:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.685803:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:17.0:1619677447.685803:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677447.685804:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.685804:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:17.0:1619677447.685805:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677447.685805:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677447.685805:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.685806:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.685806:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677447.685815:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.685817:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677447.685817:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:13.0:1619677447.685818:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.685819:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.685819:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677447.685820:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.685820:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.685821:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.685822:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.685918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:18.0:1619677447.685925:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:24.0:1619677447.685926:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677447.685928:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.685929:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:24.0:1619677447.685930:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.685931:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:18.0:1619677447.685932:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.685934:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.685935:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.685936:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.685936:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.685938:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:18.0:1619677447.685938:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.685940:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.685940:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.685941:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.685942:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.685942:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.685943:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.685944:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:18.0:1619677447.685946:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.685948:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.685949:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.685951:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:18.0:1619677447.685952:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.685953:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.685953:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.685954:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.685955:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:18.0:1619677447.685955:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.685955:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:24.0:1619677447.685956:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.685957:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:18.0:1619677447.685957:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.685957:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:24.0:1619677447.685958:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.685960:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.685961:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.685962:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.685967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.0:1619677447.685971:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.685973:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:17.0:1619677447.685974:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:5.0:1619677447.685974:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.685975:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:20.0:1619677447.685976:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.685976:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.685977:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:8.0:1619677447.685977:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677447.685978:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.685978:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677447.685978:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:5.0:1619677447.685979:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:17.0:1619677447.685980:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.685980:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.685983:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.0:1619677447.685990:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.685992:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.685993:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.685994:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.685995:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.685996:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677447.686169:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.686171:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.686172:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.686174:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.686175:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.686177:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.686189:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.686191:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.686192:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.686193:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677447.686193:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.686195:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677447.686233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f771800] (20)++
00000800:00000200:1.0:1619677447.686265:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.686268:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.686269:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.686271:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.686272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f771800] (21)++
00000400:00000200:1.0:1619677447.686273:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.686275:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.686276:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.686278:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.686279:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.686280:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.686283:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.686284:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.686287:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.686288:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.686289:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.686291:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.686292:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.686292:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.686293:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.686305:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.686308:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.686309:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.686310:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.686312:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.686313:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.686326:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.686328:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.686329:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.686330:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.686330:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.686332:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.686372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.686385:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.686387:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.686389:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.686389:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.686390:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.686391:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.686393:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.686393:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.686394:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.686396:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.686397:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.686398:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.686400:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.686402:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.686402:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.686403:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.686404:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.686405:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.686405:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.686406:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.686409:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.686409:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:30.0:1619677447.686409:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.686410:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.686411:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.686412:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.686412:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.686413:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.686413:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.686414:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.686414:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.686414:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.686415:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677447.686416:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.686416:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.686416:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.686417:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.686418:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.686428:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.686430:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.686430:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.686431:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.686432:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.686432:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:13.0:1619677447.686433:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.686434:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.686434:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.686435:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.686439:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.686439:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.686468:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.686471:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.686473:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.686473:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.686476:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.686477:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.686479:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.686480:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.686481:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.686482:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.686485:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.686487:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.686489:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.686490:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.686491:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.686492:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.686493:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.686494:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.686494:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677447.686507:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.686509:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.686510:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.686512:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.686513:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.686514:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.2:1619677447.686574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.686579:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.686582:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.686584:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.686585:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.686586:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.686588:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677447.686589:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.686590:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.686591:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.686592:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677447.686595:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.686596:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677447.686599:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.686599:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677447.686600:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677447.686601:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:4.2:1619677447.686601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:5.0:1619677447.686602:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.686603:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.686604:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.686605:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:24.0:1619677447.686608:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.2:1619677447.686608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.2:1619677447.686610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:24.0:1619677447.686611:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.686612:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.2:1619677447.686613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:24.0:1619677447.686614:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.686616:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:24.0:1619677447.686617:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.686618:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.686619:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96724818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.686619:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:2.0:1619677447.686619:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:24.0:1619677447.686620:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.686620:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.686620:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.686621:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.686621:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:24.0:1619677447.686622:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.686622:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:2.0:1619677447.686622:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.686623:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:2.0:1619677447.686625:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.686627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000400:00000200:24.0:1619677447.686628:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.686630:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.686634:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.686634:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:18.0:1619677447.686634:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:24.0:1619677447.686635:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.686637:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.686637:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677447.686637:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:24.0:1619677447.686638:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677447.686638:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.686638:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:24.0:1619677447.686639:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677447.686639:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.686640:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:17.0:1619677447.686650:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.686653:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.686654:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.686654:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:9.2:1619677447.686655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:17.0:1619677447.686656:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.686659:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.686665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.686665:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.686667:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.686668:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:2.0:1619677447.686668:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.686669:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677447.686670:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000400:00000200:18.0:1619677447.686672:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.686674:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.686674:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677447.686675:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.686676:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.686677:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:2.0:1619677447.686677:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.686678:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.686679:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.686679:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.686680:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.686681:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.686681:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:18.0:1619677447.686682:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.686684:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.686689:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.686692:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:22.0:1619677447.686695:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677447.686695:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.686696:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.686696:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677447.686698:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:18.0:1619677447.686698:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.686699:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.686699:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:6.0:1619677447.686699:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677447.686700:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:22.0:1619677447.686702:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.686704:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.686706:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.686707:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96724814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.686708:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.686709:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.686710:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.686712:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:4.2:1619677447.686713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677447.686714:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.686715:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.686716:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.686717:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:20.0:1619677447.686718:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.686719:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.0:1619677447.686719:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:22.0:1619677447.686720:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.686721:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677447.686721:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:6.0:1619677447.686721:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.686722:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:6.0:1619677447.686722:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.686724:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:6.0:1619677447.686724:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.686726:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.686727:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.686727:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.686728:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.686728:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677447.686738:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.686741:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.686741:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.686742:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.686744:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.686746:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.686756:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.686759:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.686760:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.686762:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.686765:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.686766:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.686768:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.686769:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.686770:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.686771:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.686775:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.686776:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.686779:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.686780:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.686781:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.686783:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.686784:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.686784:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.686785:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677447.686796:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.686798:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.686800:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.686802:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.686802:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.686804:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.686850:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.686853:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.686854:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.686856:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.686858:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.686860:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.686861:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.686863:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.686864:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.686865:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.686868:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.686870:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.686873:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.686874:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.686875:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.686877:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.686878:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.686878:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.686879:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.686890:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.686892:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.686893:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.686895:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.686896:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.686897:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.686929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.686937:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.686940:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.686942:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.686944:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.686945:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.686946:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.686947:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.686949:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.686950:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.686951:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.686954:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.686956:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.686958:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.686959:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.686960:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.686962:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.686962:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.686964:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.686965:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.686966:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677447.686971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677447.686973:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.686976:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.686976:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.686978:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.686979:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:13.0:1619677447.686980:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.686981:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.686982:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.686983:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.686983:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677447.686994:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.686996:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.686997:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.2:1619677447.686998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:13.0:1619677447.686998:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.686998:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.687000:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.687004:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.687007:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.687009:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.687010:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.687014:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.687016:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.687018:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.687020:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.687022:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.687023:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.687025:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:5.0:1619677447.687026:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.687028:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:5.0:1619677447.687028:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.687029:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.687031:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.687032:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.687033:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.687033:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677447.687034:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.687035:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:18.0:1619677447.687036:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:5.0:1619677447.687036:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:1.0:1619677447.687036:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:18.0:1619677447.687037:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96724808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.687037:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.687037:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:18.0:1619677447.687038:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.687039:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.687039:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:18.0:1619677447.687040:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:17.0:1619677447.687040:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.687040:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.687042:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.687044:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.687045:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677447.687046:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677447.687047:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677447.687048:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.687049:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.687050:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.687050:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677447.687052:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.687052:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:12.0:1619677447.687052:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.687053:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.687053:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:12.0:1619677447.687053:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.687055:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.687056:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.687057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.687064:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.687067:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.687068:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.687068:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.687070:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.687072:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.687184:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.687187:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.687187:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.687189:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.687192:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.687193:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.687195:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.687196:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.687197:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.687198:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.687201:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.687203:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.687206:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.687206:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.687207:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.687209:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.687209:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.687210:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.687210:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.687222:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.687225:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.687226:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.687226:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.687228:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.687230:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:30.0:1619677447.687274:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677447.687277:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677447.687279:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.687280:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677447.687281:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.687283:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677447.687285:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.687286:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.687287:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.687288:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677447.687292:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.687300:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677447.687303:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.687304:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:30.0:1619677447.687304:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:30.0:1619677447.687306:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:30.0:1619677447.687307:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677447.687308:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:30.0:1619677447.687309:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:30.0:1619677447.687310:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:9.2:1619677447.687312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.687320:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.687323:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.687324:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.687324:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.687325:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)++
00000400:00000200:8.0:1619677447.687326:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.687328:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:8.0:1619677447.687328:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:31.0:1619677447.687329:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677447.687330:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:31.0:1619677447.687437:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.687439:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.687441:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.687444:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.687444:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.687446:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.687447:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724805] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.687448:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.687449:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.687450:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.687452:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677447.687453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.687454:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.687456:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.687457:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.687457:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.687459:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.687459:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.687460:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.687461:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.687462:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.687462:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.687462:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677447.687464:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.687464:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.687465:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.687467:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.687468:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.687469:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.687470:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.687471:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:13.0:1619677447.687472:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:30.0:1619677447.687474:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.687474:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677447.687475:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.687475:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:13.0:1619677447.687475:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.687476:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:13.0:1619677447.687476:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.687477:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.687478:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677447.687478:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.687478:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.687479:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:12.0:1619677447.687479:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.687480:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.687482:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.687482:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.687483:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.687484:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677447.687489:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.687492:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.687493:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.687493:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.687494:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.687495:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:13.0:1619677447.687495:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.687498:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.687499:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.687501:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.687502:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.687504:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677447.687517:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.687521:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.687522:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.687524:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.687526:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.687527:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.687529:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.687530:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.687531:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.687532:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.687535:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.687537:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.687539:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.687540:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.687541:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.687542:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.687543:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.687544:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.687545:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.687546:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.687556:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677447.687556:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677447.687558:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.687559:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.687559:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677447.687560:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.687561:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.687561:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.687563:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.687565:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.687608:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.687611:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.687612:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.687614:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.687616:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.687617:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.687618:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.687620:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.687621:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.687622:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.687624:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.687626:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.687629:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.687629:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.687630:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.687631:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.687632:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.687633:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.687633:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677447.687645:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.687648:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.687649:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.687649:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.687651:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.687653:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677447.687672:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:13.0:1619677447.687673:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677447.687674:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:13.0:1619677447.687676:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:5.0:1619677447.687676:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.687677:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:13.0:1619677447.687677:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.687678:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.687679:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.687680:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.687680:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:5.0:1619677447.687681:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.687682:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.687683:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.687684:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.687685:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.687689:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.687691:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.687694:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.687695:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.687695:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:18.0:1619677447.687697:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677447.687697:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.687698:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.687699:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.687699:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:18.0:1619677447.687700:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.687702:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.687704:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.687706:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.687707:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.687708:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.687709:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.687710:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.687711:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:18.0:1619677447.687711:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.687713:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.687714:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.687715:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.687716:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.687716:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.687718:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:18.0:1619677447.687718:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.687720:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.687721:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.687721:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.687723:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.687723:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.687725:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.687726:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.687726:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677447.687729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.687738:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.687739:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.687739:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.687739:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.687740:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.687740:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.687741:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.687741:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.687742:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677447.687743:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.687817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.687826:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.687828:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.687829:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.687832:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.687835:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.687836:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.687838:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.687840:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.687841:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.687843:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.687851:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.687853:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.687856:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.687857:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.687858:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.687859:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.687860:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.687860:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.687861:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.687870:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.687872:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.687873:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.687873:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.687874:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.687874:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.687876:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.687876:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.687879:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.687880:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.687880:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.687881:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.687890:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.687892:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.687893:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.687894:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.687895:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.687895:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.687949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.687957:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.687960:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.687962:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.687964:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.687965:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.687967:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.687968:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.687970:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.687971:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.687972:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.687975:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.687977:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.687979:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.687980:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.687980:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.687982:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.687983:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.687984:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.687984:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677447.687994:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:20.2:1619677447.687995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:30.0:1619677447.687997:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.687998:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.687999:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.688000:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677447.688001:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.688001:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.0:1619677447.688003:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.688005:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.688006:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.688007:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.688009:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.688010:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.688011:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.688012:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.688013:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.688017:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.688019:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.688022:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.688022:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.688023:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.688025:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.688025:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.688026:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.688027:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.688037:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.688040:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.688041:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.688042:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.688043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:13.0:1619677447.688044:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.688045:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.688046:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.688049:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.688051:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.688051:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:31.0:1619677447.688053:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.688053:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677447.688054:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:31.0:1619677447.688055:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:8.0:1619677447.688055:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:31.0:1619677447.688056:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.688056:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.688057:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.688058:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.688059:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:24.0:1619677447.688059:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.688061:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.688062:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96724793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.688063:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.688063:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.688064:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.688065:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:24.0:1619677447.688065:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.688068:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.688069:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:24.0:1619677447.688069:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.688070:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:24.0:1619677447.688071:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.688072:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.688072:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.688073:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.688074:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677447.688074:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.688075:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.688075:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.688077:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.688077:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.688078:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.688079:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.688090:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.688093:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.688094:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.688094:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.688096:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.688099:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.688151:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.688154:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.688155:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.688156:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.688158:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.688160:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.688161:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.688162:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.688163:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.688164:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.688167:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.688169:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.688171:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.688172:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.688172:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.688174:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.688174:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.688175:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.688175:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.688186:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.688188:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.688189:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.688190:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.688191:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.688192:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.688243:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.688245:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.688246:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.688246:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.688247:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.688248:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677447.688288:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.688291:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.688292:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.688298:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.688300:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.688301:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.688302:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.688304:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.688305:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.688306:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.688309:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.688310:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.688312:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.688313:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.688314:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.688315:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.688316:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.688317:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.688317:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677447.688330:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.688332:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.688334:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.688334:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.688336:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.688338:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.688437:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.688440:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.688441:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.688442:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.688445:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.688446:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.688447:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.688448:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.688449:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.688450:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.688453:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.688455:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.688456:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.688456:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.688457:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.688458:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.688458:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.688459:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.688459:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.688465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.688472:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.688474:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.688475:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.688475:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.688477:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.688478:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.688509:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.688512:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.688513:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.688515:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.688517:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.688518:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.688520:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.688521:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.688522:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.688523:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.688526:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.688528:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.688531:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.688532:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.688533:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.688534:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.688535:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.688536:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.688537:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.688538:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.688548:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.688550:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.688550:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.688551:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677447.688592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.688595:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.688598:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.688600:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.688601:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.688603:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.688604:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.688606:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.688607:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.688608:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.688609:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.688612:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.688614:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.688617:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.688618:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.688619:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.688621:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.688621:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.688622:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.688623:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.688634:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.688637:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.688638:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.688640:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.688642:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.688643:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.688709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.688756:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.688759:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.688759:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.688760:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.688762:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.688765:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.688775:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.688777:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.688778:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.688779:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.688780:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.688780:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.688788:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.688791:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.688791:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.688793:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.688795:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.688796:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.688798:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.688799:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.688800:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.688801:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.688804:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.688806:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.688808:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.688809:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.688809:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.688811:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.688811:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.688812:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.688812:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.688823:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.688825:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.688826:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.688827:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.688828:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.688829:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:9.2:1619677447.688864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.688873:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.688875:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.688876:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.0:1619677447.688876:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:2.0:1619677447.688877:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.688878:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:5.0:1619677447.688879:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:2.0:1619677447.688879:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.688882:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:18.0:1619677447.688882:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.688883:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.688883:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.688884:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.688884:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677447.688886:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.688886:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.688886:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.688887:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000400:00000200:18.0:1619677447.688888:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.688888:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:2.0:1619677447.688888:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.688888:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.688889:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.688890:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.688890:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.688890:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:18.0:1619677447.688892:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.688892:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.688893:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.688894:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.688895:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677447.688899:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.688900:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.688902:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.688903:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677447.688904:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.688905:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.688906:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.688906:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.688907:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:18.0:1619677447.688908:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.688908:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.0:1619677447.688908:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:18.0:1619677447.688909:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:5.0:1619677447.688909:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:18.0:1619677447.688910:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:5.0:1619677447.688910:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:18.0:1619677447.688911:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:5.0:1619677447.688911:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677447.688913:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:5.0:1619677447.688913:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:6.0:1619677447.688922:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677447.688922:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:20.0:1619677447.688923:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:6.0:1619677447.688924:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677447.688924:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:22.0:1619677447.688925:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.688925:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:6.0:1619677447.688925:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.688925:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000400:00000200:20.0:1619677447.688926:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.688926:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:2.0:1619677447.688926:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:5.0:1619677447.688927:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:2.0:1619677447.688927:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.0:1619677447.688928:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:20.0:1619677447.688929:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.688929:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.688929:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677447.688931:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:6.0:1619677447.688931:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.689035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.689040:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.689043:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.689045:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.689045:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.689047:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.689049:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.689050:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.689051:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.689053:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.689054:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.689057:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.689059:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.689061:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.689061:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.689062:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.689063:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.689064:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.689064:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.689065:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.689077:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.689079:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.689080:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.689080:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.689082:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.689084:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.689086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.689089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:18.0:1619677447.689115:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.689117:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.689118:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.689121:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.689123:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.689125:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.689126:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96724779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.689128:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.689129:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.689130:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.689134:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.689136:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.689139:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:18.0:1619677447.689139:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.689139:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.689140:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.689142:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.689142:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.689142:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.689143:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:30.0:1619677447.689144:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.689144:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.689145:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.689145:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677447.689146:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.689148:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.0:1619677447.689153:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:31.0:1619677447.689155:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.689155:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.689155:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.689156:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:20.0:1619677447.689157:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.689157:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:31.0:1619677447.689158:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.689158:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677447.689159:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:2.0:1619677447.689159:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:31.0:1619677447.689160:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677447.689160:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.689160:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:5.0:1619677447.689160:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:2.0:1619677447.689160:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677447.689161:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:24.0:1619677447.689161:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677447.689161:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:5.0:1619677447.689161:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.689161:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:31.0:1619677447.689163:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.0:1619677447.689163:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.689163:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.689164:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.689166:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:8.0:1619677447.689167:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.689173:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.689174:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.689176:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.689177:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:8.0:1619677447.689177:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.689178:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.689180:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.689181:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.689182:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.689182:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:8.0:1619677447.689182:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.689185:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.689187:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.689187:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.0:1619677447.689188:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.689189:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.689190:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.689191:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.689192:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.689194:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.689194:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.689203:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.689205:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.689206:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.689207:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:12.0:1619677447.689208:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.689209:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.689209:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677447.689209:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.689210:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.689211:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:17.0:1619677447.689231:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.689234:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.689235:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.689235:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.689238:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.689241:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677447.689242:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677447.689244:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677447.689247:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.689248:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:1.0:1619677447.689249:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:30.0:1619677447.689250:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.689251:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:1.0:1619677447.689252:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677447.689253:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.689253:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:30.0:1619677447.689254:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.689254:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (21)++
00000400:00000200:30.0:1619677447.689255:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.689255:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.689255:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:30.0:1619677447.689256:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.689260:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.689261:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.689264:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.689264:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.689265:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677447.689265:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.689266:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:31.0:1619677447.689267:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.689268:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.689268:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:31.0:1619677447.689269:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.689269:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677447.689270:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:31.0:1619677447.689271:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.689272:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724775] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.689272:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.689273:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.689274:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.689275:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.689278:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.689279:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.689281:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.689281:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677447.689282:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.689283:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.689284:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.689284:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677447.689284:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677447.689285:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677447.689285:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.689286:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:20.0:1619677447.689286:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.689288:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.689289:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:13.0:1619677447.689298:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.689301:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.689302:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.689303:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.689305:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.689306:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.689314:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.689316:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.689317:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.689317:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.689318:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.689319:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.689330:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677447.689333:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:1.0:1619677447.689333:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:17.0:1619677447.689335:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.689337:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.689339:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.689340:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.689341:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.689342:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.689343:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:4.2:1619677447.689345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:17.0:1619677447.689346:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.689347:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 440 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.689349:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.689350:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.689351:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.689352:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.689353:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677447.689354:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:17.0:1619677447.689354:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:1.0:1619677447.689366:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.689368:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.689369:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.689369:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.689371:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.689375:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.689376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:1.0:1619677447.689376:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.689378:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.689378:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:5.0:1619677447.689381:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:1.0:1619677447.689381:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.689382:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.689383:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:5.0:1619677447.689384:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:1.0:1619677447.689385:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.689386:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677447.689387:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:1.0:1619677447.689387:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677447.689388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677447.689388:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.689389:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.689389:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.689390:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677447.689392:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96724771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.689393:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.689394:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:5.0:1619677447.689394:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.689395:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.689396:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:5.0:1619677447.689396:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.689396:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.689397:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677447.689397:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677447.689398:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.689398:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.689399:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:6.0:1619677447.689400:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.689400:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:5.0:1619677447.689401:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677447.689402:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677447.689403:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.689403:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.689403:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.689404:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.689405:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677447.689406:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.689406:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.689408:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (23)++
00000800:00000200:5.0:1619677447.689414:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (24)--
00000800:00000200:17.0:1619677447.689415:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000800:00000200:5.0:1619677447.689415:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.689415:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:17.0:1619677447.689416:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000800:00000200:5.0:1619677447.689416:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:17.0:1619677447.689417:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.689417:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.689417:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.689418:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.689419:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000400:00000200:17.0:1619677447.689420:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.689420:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000800:00000200:17.0:1619677447.689421:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.689421:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.689422:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:1.0:1619677447.689422:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677447.689424:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.689425:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.689440:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.689443:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.689444:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.689444:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.689445:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.689446:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.689518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.689526:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.689528:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.689529:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.689531:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.689534:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.689535:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.689536:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.689538:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.689539:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.689540:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.689543:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.689545:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.689549:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.689549:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.2:1619677447.689550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.0:1619677447.689550:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.689552:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.689552:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677447.689553:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677447.689553:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.689553:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677447.689555:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:22.0:1619677447.689556:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:20.0:1619677447.689557:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.689558:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677447.689559:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:20.0:1619677447.689559:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.689560:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677447.689561:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.689562:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.689563:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.689564:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.689565:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.689565:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.689565:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:20.0:1619677447.689566:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.689567:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:1.0:1619677447.689568:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:22.0:1619677447.689569:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96724768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.689569:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.689569:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.689571:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.689571:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.689572:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.689572:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.689573:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677447.689574:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.0:1619677447.689575:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.689577:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.689579:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.689581:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.689581:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.689582:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677447.689582:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.689583:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.689583:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.689585:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.689585:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.689586:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677447.689586:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677447.689587:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.689587:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.689588:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.689589:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.689589:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.689594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.689598:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677447.689599:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677447.689599:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.689600:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.689600:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.689601:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:30.0:1619677447.689602:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:20.0:1619677447.689602:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.689602:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677447.689603:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.689603:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:30.0:1619677447.689604:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.689604:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677447.689605:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677447.689606:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.689608:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677447.689623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.689669:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.689672:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.689673:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.689673:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.689676:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.689678:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.689687:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.689690:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.689691:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.689691:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.689692:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.689692:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.689734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.689741:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.689744:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.689747:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.689748:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.689749:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.689751:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.689752:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.689753:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.689755:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.689755:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.689759:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.689761:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.689764:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.689765:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.689765:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.689767:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.689768:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.689769:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.689769:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677447.689781:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.689784:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.689785:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.689786:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.689787:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.689790:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677447.689867:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.689870:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.689872:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.689874:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.689875:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.689877:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677447.689878:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.689879:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.689881:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.689881:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:4.2:1619677447.689882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:30.0:1619677447.689885:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.689886:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.689889:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.689890:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677447.689890:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677447.689892:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.689893:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.689894:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.689894:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.689906:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.689908:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.689909:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.689910:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.689912:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.689915:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.689927:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.689929:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.689930:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.689930:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.689932:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.689934:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677447.690136:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.690139:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.690139:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.690140:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:31.0:1619677447.690141:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.690142:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677447.690143:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:31.0:1619677447.690144:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:20.0:1619677447.690145:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.690146:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.690148:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.690149:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.690150:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.690151:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.690153:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.690153:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.690156:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.690158:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.690161:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.690161:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.690162:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.690163:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.690164:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.690165:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.690166:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.690177:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.690179:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.690180:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.690182:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.690182:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.690184:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677447.690199:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.690202:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.690203:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.690203:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.690204:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.690205:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:18.0:1619677447.690274:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.690278:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.690278:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.690280:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.690283:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.690284:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.690286:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.690287:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.690288:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.690289:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.690292:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677447.690297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.690298:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.690300:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.690301:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.690302:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.690303:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.690304:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.690304:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.690305:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.690310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.690318:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.690320:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.690321:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.690322:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.690323:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.690327:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.690338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677447.690341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677447.690346:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677447.690347:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.690348:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.690350:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.690350:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677447.690350:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.690352:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.690353:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.690353:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.690355:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:5.0:1619677447.690355:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.690356:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.690357:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.690359:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.690360:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.690361:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.690365:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.690367:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.690370:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.690370:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.690371:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.690373:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.690374:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.690375:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.690375:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677447.690387:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.690390:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.690391:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.690393:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.690394:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.690396:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:6.0:1619677447.690451:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.690454:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.690455:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.690457:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.690459:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.690460:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.690462:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.690463:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.690464:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.690465:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.690468:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.690470:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.690473:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.690474:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.690474:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.690476:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.690477:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.690477:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.690478:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.690490:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.690492:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.690493:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.690495:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.690496:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.690497:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677447.690507:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.690510:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.690512:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.690514:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.690515:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.690517:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677447.690518:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.690519:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.690520:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.690521:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.690522:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:31.0:1619677447.690525:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.690525:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.690526:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.690527:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.690527:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677447.690528:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.690529:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677447.690531:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.690532:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677447.690533:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677447.690534:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.690535:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.690536:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.690538:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.690539:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677447.690543:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.690546:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.690547:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677447.690548:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:24.0:1619677447.690549:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.690551:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.690551:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.690551:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.690552:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:22.0:1619677447.690552:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.690553:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:30.0:1619677447.690554:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:24.0:1619677447.690554:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96724756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.690554:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.690555:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.690556:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:24.0:1619677447.690556:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.690557:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.690557:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:24.0:1619677447.690558:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677447.690561:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.690562:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.690565:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.690565:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677447.690566:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677447.690568:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.690568:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.690569:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.690569:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.690581:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.690583:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.690584:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.690586:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.690588:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.690589:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677447.690620:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.690623:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.690624:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.690626:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.690628:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.690630:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.690631:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.690632:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.690634:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.690635:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.690638:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.690639:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.690642:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.690643:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.690644:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.690646:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.690646:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.690647:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.690647:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.690658:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.690661:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.690662:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.690664:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.690665:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.690666:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.690692:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.690694:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.690695:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.690696:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.690698:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.690700:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.690727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.690765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.690767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.690773:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.690775:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.690777:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.690777:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.690779:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.690782:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677447.691126:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.691128:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.691129:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.691130:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.691133:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.691134:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677447.691145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.691152:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.691155:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.691156:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.691157:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677447.691159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:8.0:1619677447.691160:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.691161:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.691162:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.691164:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.691165:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.691166:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:8.0:1619677447.691166:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:31.0:1619677447.691169:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:8.0:1619677447.691169:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.691171:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.691171:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677447.691173:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.0:1619677447.691173:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.691174:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.691174:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:31.0:1619677447.691175:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:8.0:1619677447.691175:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.691176:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:31.0:1619677447.691177:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.691177:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:31.0:1619677447.691178:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.691178:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:31.0:1619677447.691179:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.691179:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.691179:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.691179:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:31.0:1619677447.691182:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.691184:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:4.2:1619677447.691185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677447.691186:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.691187:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.691187:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.691189:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.691189:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:1.0:1619677447.691190:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:31.0:1619677447.691191:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.691192:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:24.0:1619677447.691192:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:31.0:1619677447.691193:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677447.691193:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.691193:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:24.0:1619677447.691194:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677447.691194:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.691195:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.691196:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.691196:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.691199:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.691200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677447.691202:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.691204:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:20.0:1619677447.691205:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:13.0:1619677447.691205:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.691207:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.691208:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677447.691208:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:12.0:1619677447.691208:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.691210:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:20.0:1619677447.691210:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.691210:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.691211:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677447.691211:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:30.0:1619677447.691212:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677447.691213:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.691214:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.691215:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.691216:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.691217:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.691218:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:12.0:1619677447.691219:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:20.0:1619677447.691222:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.691222:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.691223:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.691223:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:20.0:1619677447.691224:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:12.0:1619677447.691224:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.691225:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677447.691226:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.691227:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.691228:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.691229:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.691230:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.691231:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.691232:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.691244:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.691246:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.691247:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.691248:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.691249:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.691251:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:17.0:1619677447.691293:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.691298:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.691300:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677447.691302:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.691305:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.691306:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.691308:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.691309:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.691310:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.691311:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.691315:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.691316:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677447.691317:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.691318:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.691319:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:17.0:1619677447.691319:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.691320:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677447.691320:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:17.0:1619677447.691321:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:18.0:1619677447.691322:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.691322:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.691322:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677447.691323:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.691323:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:18.0:1619677447.691325:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.691326:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724747] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.691328:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.691329:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.691330:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.691333:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.691335:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677447.691335:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:18.0:1619677447.691337:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.691337:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.691337:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.691338:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:17.0:1619677447.691338:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:18.0:1619677447.691339:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:1.0:1619677447.691339:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.691341:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.691341:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:1.0:1619677447.691341:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677447.691342:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.691342:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677447.691354:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.691356:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:4.2:1619677447.691356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.691358:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.691358:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.691358:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.691359:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.0:1619677447.691360:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.691363:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.691364:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.691366:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.691367:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.691368:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677447.691405:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.691408:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.691408:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.691409:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.691411:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.691414:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.691595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.691601:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.691603:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.691606:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.691606:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.691607:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.691609:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677447.691610:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.691612:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.691613:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.691614:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677447.691617:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.691619:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.691621:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.691622:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677447.691622:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.691624:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.691625:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.691626:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.691626:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677447.691638:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.691640:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.691641:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.691641:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.691643:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.691646:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:12.0:1619677447.691730:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.691733:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.691733:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.691735:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.691737:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.691738:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.691791:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.691794:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.691796:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.691798:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.691799:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.691799:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677447.691800:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.691802:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.691803:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.691804:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677447.691806:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.691808:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677447.691810:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.691811:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677447.691811:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677447.691813:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.691813:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.691814:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.691815:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.691874:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.691876:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.691877:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.691879:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677447.691880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:6.0:1619677447.691881:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.691883:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.691884:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.691885:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.691886:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.691887:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.691888:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.691891:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:6.0:1619677447.691891:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.691893:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:31.0:1619677447.691894:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.691895:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:6.0:1619677447.691896:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.691896:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:31.0:1619677447.691897:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.691897:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677447.691898:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:6.0:1619677447.691899:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.691899:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:31.0:1619677447.691900:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.691900:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:31.0:1619677447.691901:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.691901:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:31.0:1619677447.691902:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.691903:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.691907:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.691908:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.691911:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.691912:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.691912:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.691914:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.691914:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.691915:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.691916:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:17.0:1619677447.691923:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:30.0:1619677447.691926:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:17.0:1619677447.691926:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.691927:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.691927:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677447.691928:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.691929:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.691929:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.691931:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.691931:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677447.691932:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.691934:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677447.691942:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.691945:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.691946:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.691947:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677447.691947:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.691948:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:1.0:1619677447.691978:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.691980:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.691981:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.691983:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.691985:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.691987:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.691988:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.691989:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.691990:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.691991:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.691995:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.691996:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.691999:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.692000:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.692001:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.692003:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.692003:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.692004:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.692004:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.692015:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.692018:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.692019:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.692020:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.692022:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.692024:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.692048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.692093:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.692095:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.692096:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.692097:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.692097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:22.0:1619677447.692099:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.692100:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:22.0:1619677447.692101:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677447.692103:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.692105:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.692106:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.692107:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.692109:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.692110:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.692111:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.692112:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.692113:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.692117:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.692118:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677447.692119:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677447.692121:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:20.0:1619677447.692122:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.692122:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677447.692123:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.692124:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:2.0:1619677447.692124:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.692126:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.692126:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:2.0:1619677447.692127:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.692128:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.692129:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000400:00000200:2.0:1619677447.692129:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677447.692130:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:2.0:1619677447.692130:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724737] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.692132:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.692133:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:2.0:1619677447.692133:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.692134:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677447.692134:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.2:1619677447.692135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000400:00000200:18.0:1619677447.692137:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677447.692138:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:18.0:1619677447.692139:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.692139:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.692140:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:31.0:1619677447.692141:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:5.0:1619677447.692141:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:2.0:1619677447.692141:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.692142:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.692142:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96724736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.692143:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.692143:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:31.0:1619677447.692144:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.692144:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:18.0:1619677447.692145:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.692145:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.692145:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677447.692146:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.692147:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:18.0:1619677447.692148:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.692150:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.692152:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.692152:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.692153:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.692153:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.692154:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.692154:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.692156:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.692157:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.692157:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.692157:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.692158:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.692158:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.692159:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.692159:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677447.692160:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.692162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.692170:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:12.0:1619677447.692170:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677447.692173:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:12.0:1619677447.692173:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:18.0:1619677447.692174:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.692174:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.692174:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.692177:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:13.0:1619677447.692177:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.692178:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.692180:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.692181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:12.0:1619677447.692182:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.692189:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.692191:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.692192:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.692193:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.692193:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.692194:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.692226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.692233:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.692236:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.692237:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.692239:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.692242:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.692243:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.692244:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.692246:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.692247:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.692248:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.692251:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.692253:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.692256:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.692257:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.692257:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.692259:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.692260:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.692260:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.692261:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.692273:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.692276:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.692277:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.692277:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.692280:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.692283:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.692297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:24.0:1619677447.692299:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.692302:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.692303:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:30.0:1619677447.692305:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:24.0:1619677447.692305:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.692308:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:24.0:1619677447.692308:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.692310:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.692311:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.692311:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.692312:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.692312:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.692313:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.692314:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677447.692314:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.692315:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.692315:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.692317:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.692318:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.692319:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677447.692319:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.692321:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.692323:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.692324:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.692325:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.692325:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677447.692325:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677447.692327:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.692327:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:30.0:1619677447.692328:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.692328:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.692328:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:30.0:1619677447.692329:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.692329:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677447.692331:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:30.0:1619677447.692331:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.692332:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.692333:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677447.692341:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677447.692343:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677447.692344:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.692345:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.692347:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.692348:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.692350:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.692351:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.692353:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.692353:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.692354:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.692389:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.692392:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.692394:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.692394:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.692397:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.692398:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.692400:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.692401:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.692402:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.692403:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.692406:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.692408:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.692411:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.692411:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.692412:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.692413:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.692414:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.692415:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.692415:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677447.692427:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.692429:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.692430:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.692432:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.692433:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.692434:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:1.0:1619677447.692636:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.692638:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.692639:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.692641:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.692644:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.692645:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.692647:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.692648:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.692649:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.692650:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.692653:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.692655:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.692658:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.692659:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.692659:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.692661:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.692662:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.692663:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.692664:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.692665:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677447.692674:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677447.692675:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.692676:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:31.0:1619677447.692677:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:17.0:1619677447.692677:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677447.692678:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.692678:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677447.692680:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.692680:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.692683:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.692687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.692692:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.692694:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.692695:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.692696:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.692696:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.692696:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.692697:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:30.0:1619677447.692697:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:31.0:1619677447.692699:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.692699:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.692701:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.692703:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.692704:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.692705:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.692706:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.692707:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.692710:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.692712:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.692715:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.692716:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.692717:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.692718:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.692719:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.692720:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.692720:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677447.692733:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.692735:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.692736:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.692738:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.692738:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.692741:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:6.0:1619677447.692843:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.692845:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.692846:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.692848:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.692850:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.692851:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.692852:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.692854:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.692855:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.692856:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.692859:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.692861:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.692864:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.692864:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.692865:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.692867:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.692867:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:4.2:1619677447.692868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:6.0:1619677447.692869:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.692870:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.692870:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677447.692879:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677447.692882:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677447.692882:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:1.0:1619677447.692882:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.692883:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677447.692884:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:1.0:1619677447.692884:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:30.0:1619677447.692885:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:8.0:1619677447.692885:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.692885:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.692886:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:17.0:1619677447.692886:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.692887:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:30.0:1619677447.692888:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.692888:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:8.0:1619677447.692888:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.692888:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.692889:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:8.0:1619677447.692890:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677447.692891:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.692891:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.692891:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:30.0:1619677447.692893:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.692893:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.692894:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.692894:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.692896:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.692896:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677447.692898:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.692899:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.692900:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.692902:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.692902:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677447.692904:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.692905:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.692906:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.692906:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677447.692907:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.692907:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677447.692908:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.692909:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.692909:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677447.692910:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677447.692910:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.692910:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677447.692911:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.692911:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677447.692911:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677447.692918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.692921:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677447.692921:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677447.692922:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677447.692923:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:13.0:1619677447.692924:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.692924:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.692925:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:20.0:1619677447.692926:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.692926:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.692926:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.692926:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677447.692926:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.692928:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.692928:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677447.692929:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.692929:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:20.0:1619677447.692930:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677447.692931:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.692932:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:20.0:1619677447.692933:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.692934:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.692935:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677447.692938:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.692940:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677447.692943:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.692944:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677447.692944:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.692946:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.692947:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.692948:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.692948:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.692952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.692961:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.692964:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.692965:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.692965:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.692967:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.692970:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:12.0:1619677447.692997:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.693000:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.693001:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.693002:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.693004:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.693005:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.693020:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.693022:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.693023:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.693024:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.693024:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.693026:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677447.693051:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.693054:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.693054:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.693057:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.693059:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.693060:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.693062:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.693063:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.693064:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.693065:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:4.2:1619677447.693069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677447.693069:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.693070:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.693073:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.693074:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.693075:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.693077:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.693077:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.693077:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.693078:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.693089:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.693092:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.693093:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.693094:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.693096:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.693097:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.693111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:6.0:1619677447.693114:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.693117:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.693117:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.693118:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.693120:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.693123:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.693133:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.693136:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.693136:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.693137:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.693138:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.693139:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.693154:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.693157:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.693157:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.693159:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.693160:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.693161:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.693174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.693182:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.693185:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.693186:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.693188:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.693189:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.693190:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677447.693191:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.693193:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.693194:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.693195:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677447.693198:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.693199:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.693201:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.693202:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.693203:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.693204:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.693204:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.693206:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.693207:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.693208:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:13.0:1619677447.693209:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.693212:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.693212:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.2:1619677447.693213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677447.693213:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.693214:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.693214:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.693218:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.693219:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.693220:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.693221:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.693221:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.693222:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.693223:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.693223:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.693225:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677447.693225:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.0:1619677447.693285:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.693288:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.693290:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.693290:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.693293:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.693300:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:5.0:1619677447.693301:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.693302:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.693304:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.693305:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:5.0:1619677447.693308:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.693310:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 440 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:5.0:1619677447.693313:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.693314:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:5.0:1619677447.693314:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:5.0:1619677447.693316:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:5.0:1619677447.693317:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.693318:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677447.693319:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:5.0:1619677447.693320:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.693331:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.693333:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677447.693334:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.693335:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:2.0:1619677447.693374:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.693378:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.693378:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.693380:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.693383:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.693384:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.693386:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.693387:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.693388:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.693389:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.693392:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.693394:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.693397:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.693398:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.693398:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.693400:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.693401:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.693401:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.693402:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677447.693415:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.693417:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.693417:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.693420:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.693421:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.693423:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.693453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:17.0:1619677447.693460:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.693460:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.693462:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:17.0:1619677447.693464:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.693464:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677447.693465:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.693465:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.693467:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.693468:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.693470:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.693470:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.693472:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:5.0:1619677447.693472:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.693474:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.693474:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.693475:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.693476:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.693476:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.693477:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.693478:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.693483:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.693484:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.693486:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.0:1619677447.693488:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.693489:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.693490:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:5.0:1619677447.693490:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.693491:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.0:1619677447.693491:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.693492:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.693493:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.693493:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.693494:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677447.693494:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.693494:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677447.693495:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677447.693495:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.693496:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:6.0:1619677447.693506:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677447.693507:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677447.693509:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.693510:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.693510:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.693510:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.693511:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.693511:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.693513:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.693514:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.693516:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677447.693518:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.693528:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.693531:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.693532:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.693532:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.693533:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.693533:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:24.0:1619677447.693601:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.693604:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.693605:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.693607:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.693610:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.693612:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.693613:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.693614:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.693615:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.693616:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.693620:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.693622:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.693625:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.693625:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.693626:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.693628:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.693629:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.693629:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.693630:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.693634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.693642:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.693645:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.693646:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.693646:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.693649:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.693652:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.693652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.693660:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.693663:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.693664:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.693665:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.693668:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.693670:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.693671:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.693672:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.693673:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.693674:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.693676:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677447.693678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.0:1619677447.693678:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.693680:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.693681:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677447.693682:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.693682:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.693683:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:8.0:1619677447.693684:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.693684:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:20.0:1619677447.693685:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.693685:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.693685:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.0:1619677447.693687:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.693688:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.693689:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677447.693690:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.693692:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.693693:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.693694:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677447.693697:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.693698:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677447.693699:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:1.0:1619677447.693700:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677447.693701:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.693701:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.693701:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.693702:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677447.693703:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:1.0:1619677447.693703:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.693704:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.693705:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.693706:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677447.693706:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677447.693707:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.693708:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:9.2:1619677447.693710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.2:1619677447.693713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.693718:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:2.0:1619677447.693718:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.693720:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:5.0:1619677447.693721:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.693721:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677447.693722:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.693722:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.693723:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:2.0:1619677447.693724:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.693726:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:17.0:1619677447.693749:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.693752:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.693753:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.693755:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.693757:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.693758:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.693760:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.693761:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.693762:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.693763:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.693766:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.693767:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.693769:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.693770:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.693771:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.693772:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.693772:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.693773:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.693774:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.693785:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.693787:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.693788:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.693788:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.693789:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.693791:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:9.2:1619677447.693905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.693913:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.693915:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.693917:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.693917:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.693919:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.693922:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677447.693976:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.693979:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677447.693979:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:30.0:1619677447.693981:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.693982:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:30.0:1619677447.693983:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.693983:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:31.0:1619677447.693984:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.693985:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:22.0:1619677447.693985:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.693986:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.693988:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.693988:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.693989:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.693990:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:30.0:1619677447.693991:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677447.693992:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.693993:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.693995:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.693996:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:30.0:1619677447.693997:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.693999:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:22.0:1619677447.694000:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.694001:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.694002:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.694003:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677447.694003:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:30.0:1619677447.694004:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.694005:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.694005:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.694006:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677447.694006:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.694006:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.694007:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.694009:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.694010:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.694010:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.694011:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.694017:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.694020:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.694021:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.694022:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.694025:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.694026:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.694065:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.694068:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.694069:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.694070:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.694072:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.694074:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.694075:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.694076:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.694077:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677447.694078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:17.0:1619677447.694078:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.694082:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.694084:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.694086:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.694087:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.694087:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.694089:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.694089:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.694091:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.694092:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.694092:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.694102:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677447.694103:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677447.694105:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677447.694105:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.694105:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.694106:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.694106:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.694107:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677447.694109:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.694112:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.694118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677447.694121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:2.0:1619677447.694126:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.694128:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.694130:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.694130:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.694132:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.694135:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677447.694299:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677447.694303:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677447.694305:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.694307:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677447.694307:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.694309:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:12.0:1619677447.694310:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677447.694312:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677447.694313:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677447.694314:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:12.0:1619677447.694317:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.694318:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677447.694319:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.694321:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:12.0:1619677447.694321:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.694322:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:12.0:1619677447.694322:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.694322:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.694324:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.694324:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:12.0:1619677447.694325:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677447.694325:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000400:00000200:18.0:1619677447.694326:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.694326:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000400:00000200:18.0:1619677447.694328:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.694329:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.694330:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.694331:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.694332:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.694335:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.694337:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.694339:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.694340:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.694341:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.694342:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.694343:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.694344:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.694345:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.694346:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.694357:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:5.0:1619677447.694359:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677447.694359:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.694360:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.694361:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.694362:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.694363:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.694364:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.694366:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.694367:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677447.694397:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677447.694400:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677447.694402:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.694404:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677447.694405:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.694406:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677447.694408:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.694409:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.694410:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.694411:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677447.694415:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.694417:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677447.694419:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.694420:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677447.694421:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677447.694422:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677447.694423:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677447.694424:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677447.694424:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677447.694428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.694435:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.694437:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.694439:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.694439:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.694441:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.694443:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677447.694448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.694454:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.694455:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.694456:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.694457:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.694459:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.694461:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.694462:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.694464:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.694465:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.694466:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.694467:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.694468:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.694470:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.694474:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.694476:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677447.694476:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.694478:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.694479:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677447.694479:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.694480:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.694481:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.694482:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.0:1619677447.694483:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.694485:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.694485:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.694485:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:31.0:1619677447.694486:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.694486:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.694487:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.694487:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677447.694488:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.694488:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000400:00000200:30.0:1619677447.694489:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.694489:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:30.0:1619677447.694490:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:5.0:1619677447.694490:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:30.0:1619677447.694491:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96724702] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.694491:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.694491:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.694492:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:30.0:1619677447.694493:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.694493:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677447.694494:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.694495:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.694495:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.694495:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.694496:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.694497:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:30.0:1619677447.694498:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.694498:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:30.0:1619677447.694500:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.694500:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.694501:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.694501:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:20.0:1619677447.694502:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.694503:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.694504:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.694504:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:30.0:1619677447.694505:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677447.694506:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.694507:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.694508:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.694509:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.694510:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677447.694518:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.694520:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.694521:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.694523:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:13.0:1619677447.694523:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.694524:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677447.694524:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.694526:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:22.0:1619677447.694526:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:13.0:1619677447.694526:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.694526:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.694527:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.694528:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.694529:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:22.0:1619677447.694529:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.694535:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.694536:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.694538:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.694539:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.694540:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.694540:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.694541:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.694565:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.694568:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.694569:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.694571:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.694574:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.694575:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.694576:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.694577:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.694578:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.694579:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.694582:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.694584:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.694587:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.694587:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.694588:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.694590:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.694591:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.694591:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.694592:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.694596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.694602:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.694605:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.694606:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.694606:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.694608:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.694611:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677447.694636:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.694639:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.694639:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.694641:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.694644:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.694645:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.694646:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.694648:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.694649:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.694650:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.694653:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.694655:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.694658:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.694659:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.694659:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.694661:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.694662:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.694663:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.694664:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.694665:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677447.694667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.694675:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.0:1619677447.694677:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:18.0:1619677447.694678:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.694679:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.694679:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:2.0:1619677447.694679:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.694680:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:18.0:1619677447.694681:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.694681:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:18.0:1619677447.694684:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.694685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.694693:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.694695:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.694696:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.694697:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.694697:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.694698:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.694732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.694769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.2:1619677447.694799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.694805:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.694807:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.694810:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.694810:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.694812:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.694814:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:5.0:1619677447.694815:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.694816:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.694818:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.694819:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:5.0:1619677447.694822:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.694824:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:5.0:1619677447.694827:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.694828:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677447.694828:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.0:1619677447.694830:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677447.694831:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.694831:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.694832:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677447.694851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.694858:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.694861:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.694861:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.694862:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.694864:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677447.694867:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.694896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.694903:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.694905:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.694906:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.694908:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.694910:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.694912:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.694913:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.694914:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.694915:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.694916:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.694920:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.694921:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.694924:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.694925:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.694926:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.694927:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.694928:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.694929:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.694931:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.694931:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677447.694936:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:4.2:1619677447.694937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677447.694939:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.694940:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.694941:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.694942:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677447.694944:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:6.0:1619677447.694944:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.694944:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.694945:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:6.0:1619677447.694945:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677447.694945:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.694946:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.694946:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677447.694946:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.694947:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677447.694948:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.694948:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.694949:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.694950:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677447.694951:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:6.0:1619677447.694953:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.694955:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.694957:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.694958:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.694958:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.694960:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.694961:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.694961:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.694962:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.694975:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.694977:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.694978:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.694980:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.694981:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.694982:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.694997:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.695000:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.695001:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.695002:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.695002:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.695004:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677447.695090:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.2:1619677447.695090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677447.695093:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.695095:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.695095:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677447.695097:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.0:1619677447.695097:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:31.0:1619677447.695098:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.695100:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677447.695100:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.695100:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.695101:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.695103:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.695103:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.695104:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.695105:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.695105:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.695106:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.695108:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.695109:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.695109:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.695110:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.695112:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.695114:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.695116:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.695118:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.695119:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677447.695120:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.695120:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.695121:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677447.695122:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.695122:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677447.695122:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677447.695123:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.695124:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677447.695125:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.695125:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.695126:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.695127:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.695135:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677447.695135:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677447.695138:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677447.695138:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:18.0:1619677447.695138:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:22.0:1619677447.695139:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.695139:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.695139:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:30.0:1619677447.695141:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.695142:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.695142:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.695143:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:30.0:1619677447.695144:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.695145:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:18.0:1619677447.695145:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.695147:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:18.0:1619677447.695147:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:4.2:1619677447.695148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:18.0:1619677447.695149:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96724692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.695151:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:18.0:1619677447.695151:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.695152:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.695153:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677447.695156:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.695158:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.695160:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.695161:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.695162:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.695163:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.695164:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.695165:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.695165:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.695169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.695173:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.695176:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.695176:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.695176:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:20.0:1619677447.695178:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.695179:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:2.0:1619677447.695179:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.695180:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.695181:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.695182:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:2.0:1619677447.695182:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.695184:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677447.695193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677447.695195:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.695198:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.695199:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.695199:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677447.695200:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:31.0:1619677447.695201:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677447.695201:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:6.0:1619677447.695201:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:31.0:1619677447.695204:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:6.0:1619677447.695204:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.695205:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.695205:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.695207:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.695207:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.695208:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.695210:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.695210:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:31.0:1619677447.695212:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.695213:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.695214:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.695216:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.695216:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.695220:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.695222:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.695225:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.695226:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.695226:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.695228:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.695229:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.695230:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.695230:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677447.695242:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.695245:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.695246:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.695248:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.695248:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.695250:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.0:1619677447.695301:0:13335:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:30.0:1619677447.695349:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.695352:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.695352:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.695354:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.695356:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.695357:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:18.0:1619677447.695392:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.695395:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.695396:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.695398:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.695400:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.695401:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.695403:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724688] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.695404:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.695405:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.695406:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.695409:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.695411:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.695414:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.695415:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.695415:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.695417:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.695417:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.695418:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.695418:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677447.695432:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.695434:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.695435:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.695435:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.695437:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.695439:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.695440:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.695443:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.695444:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.695446:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.695448:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.695450:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.695451:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.695452:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.695453:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.695454:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677447.695458:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.695459:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.695462:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.695463:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.695463:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.695465:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.695465:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.695466:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.695467:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677447.695507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.695510:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.695512:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.695514:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.695515:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.695517:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.695518:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.695520:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.695521:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.695523:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677447.695523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:20.0:1619677447.695524:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.695528:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.695530:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.695533:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.695534:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.695535:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.695536:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.695537:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.695538:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.695538:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.695550:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.695553:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:1.0:1619677447.695553:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677447.695554:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.695554:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677447.695556:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:22.0:1619677447.695557:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.695557:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.695559:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.695560:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:1.0:1619677447.695562:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.695563:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.695565:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.695566:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.695567:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677447.695567:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.695568:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:30.0:1619677447.695570:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:6.0:1619677447.695570:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.695573:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:1.0:1619677447.695573:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.695574:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.695574:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.695574:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.695574:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.695576:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:30.0:1619677447.695577:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.695577:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.695578:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677447.695579:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677447.695580:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.695580:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000400:00000200:30.0:1619677447.695581:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.695581:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:30.0:1619677447.695582:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.695582:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:30.0:1619677447.695583:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.695583:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.695583:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677447.695584:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677447.695584:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.695585:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.695586:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677447.695587:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.695587:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.695587:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:9.2:1619677447.695588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:6.0:1619677447.695588:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.695588:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677447.695589:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.695592:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.695592:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.695593:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.695595:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.695595:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677447.695595:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:30.0:1619677447.695596:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.695597:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677447.695598:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.695598:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.695599:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.695601:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.695604:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.695607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:8.0:1619677447.695614:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.695616:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.695617:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.695617:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.695618:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.695618:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677447.695619:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.695622:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.695622:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677447.695623:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:22.0:1619677447.695624:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.695625:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:22.0:1619677447.695626:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.695626:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.695628:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:12.0:1619677447.695628:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.695630:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724682] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.695630:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:22.0:1619677447.695631:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.695631:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:22.0:1619677447.695633:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.695634:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.695636:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.695638:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.695640:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.695640:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:22.0:1619677447.695641:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.695642:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.695642:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:22.0:1619677447.695643:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.695644:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.695645:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.695645:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.695645:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.695645:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.695647:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.695649:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.695650:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.695651:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.695652:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.695653:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.695656:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.695658:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.695661:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.695662:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.695662:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.695664:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.695664:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.695665:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.695666:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677447.695667:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.695669:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.695670:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.695671:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.695671:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.695673:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.695722:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.695724:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.695725:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.695727:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.695728:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.695729:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:4.2:1619677447.695768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.695807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.695815:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.695818:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.695819:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.695819:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.695822:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.695825:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.696036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.696073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677447.696077:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.696079:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.696080:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.696081:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.696083:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.696084:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.696227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.696234:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.696237:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.696239:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.696241:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.696241:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.696243:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677447.696244:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.696246:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.696247:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.696247:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677447.696250:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.696252:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677447.696255:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.696256:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.696256:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677447.696258:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.696259:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.696259:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.696260:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677447.696272:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.696275:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.696276:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.696278:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.696278:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.696281:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.696290:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.696292:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.696298:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.696299:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.696300:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.696300:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.696415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677447.696459:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.696462:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.696462:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.696464:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.696467:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.696467:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.696476:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.696478:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.696479:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.696480:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.696480:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.696481:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677447.696511:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.696513:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.696514:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.696517:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.696519:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.696520:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.696521:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.696521:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:17.0:1619677447.696523:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.696523:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:17.0:1619677447.696524:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677447.696524:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.696525:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:13.0:1619677447.696526:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.696528:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.696528:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.696528:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.696530:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.696533:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.696534:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.696534:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.696536:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.696536:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.696537:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.696538:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.696542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:12.0:1619677447.696548:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.696550:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:12.0:1619677447.696551:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.696552:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677447.696553:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:12.0:1619677447.696553:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.696553:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677447.696554:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.696554:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:12.0:1619677447.696555:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677447.696556:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.696558:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.696759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.696799:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.696803:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.696805:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.696805:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.696806:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:5.0:1619677447.696808:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.696809:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:5.0:1619677447.696809:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:24.0:1619677447.696810:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:5.0:1619677447.696811:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.696813:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.696813:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.696814:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.696815:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:8.0:1619677447.696817:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.696818:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:5.0:1619677447.696819:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.696820:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96724671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.696822:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.696822:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:8.0:1619677447.696823:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.696824:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.696824:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000400:00000200:8.0:1619677447.696825:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:5.0:1619677447.696825:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.696827:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.696827:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000400:00000200:8.0:1619677447.696828:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.696828:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.696829:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.0:1619677447.696830:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.696838:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.696839:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.696840:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677447.696841:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:8.0:1619677447.696841:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.696842:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.696842:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677447.696843:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:8.0:1619677447.696843:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:20.0:1619677447.696844:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.696846:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.696848:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.696849:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:2.0:1619677447.696851:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.696854:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.696855:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.696857:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.696859:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.696860:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677447.696862:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.696863:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:2.0:1619677447.696863:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.696864:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.696865:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:13.0:1619677447.696866:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.696867:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.696870:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.696870:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:2.0:1619677447.696870:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.696872:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:13.0:1619677447.696873:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.696875:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.696876:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677447.696876:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677447.696878:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677447.696878:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.696879:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.696880:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677447.696892:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.696894:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.696895:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.696895:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.696898:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.696900:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:30.0:1619677447.697080:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:4.2:1619677447.697080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677447.697083:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.697085:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.697087:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677447.697087:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:30.0:1619677447.697088:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.697089:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:24.0:1619677447.697090:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:30.0:1619677447.697091:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.697091:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:30.0:1619677447.697092:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.697093:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.697093:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.697094:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:24.0:1619677447.697095:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.697097:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:30.0:1619677447.697098:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.697098:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96724668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.697100:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:24.0:1619677447.697100:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.697101:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.697102:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:20.2:1619677447.697102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.697103:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.697103:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677447.697104:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:24.0:1619677447.697105:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.697106:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.697106:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.697107:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677447.697107:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677447.697108:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677447.697109:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:24.0:1619677447.697110:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.697111:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.697111:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677447.697112:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:24.0:1619677447.697113:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.697113:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:30.0:1619677447.697114:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.697114:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.697114:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:31.0:1619677447.697116:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.697117:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.697118:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.697119:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96724667] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.697120:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.697121:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.697123:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.697126:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.697127:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:30.0:1619677447.697128:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.697130:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.697130:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:30.0:1619677447.697131:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.697131:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.697131:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.697132:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.697133:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:1.0:1619677447.697133:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.697134:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.697135:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.697135:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:1.0:1619677447.697136:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.697145:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:12.0:1619677447.697146:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677447.697148:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.697148:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.697149:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (21)++
00000800:00000200:12.0:1619677447.697149:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:17.0:1619677447.697150:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.697150:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:12.0:1619677447.697150:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.697151:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.697153:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.697154:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677447.697157:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.697160:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.697160:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.697162:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.697165:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.697167:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.697168:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.697170:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.697171:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.697172:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.697175:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.697177:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.697180:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.697181:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.697182:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.697183:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.697184:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.697184:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.697185:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:4.2:1619677447.697204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.697211:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.697214:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.697215:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.697217:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.697217:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.697220:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.697251:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.697253:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.697254:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.697255:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.697257:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.697260:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.697357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.697363:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.697366:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.697368:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.697369:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.697370:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.697372:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677447.697373:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.697375:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.697375:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.697377:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677447.697380:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.697381:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.697384:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.697384:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677447.697385:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.697387:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.697387:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.697389:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.697390:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.697390:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.697395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.697398:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.697400:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.697401:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677447.697401:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:20.0:1619677447.697402:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677447.697404:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.697405:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.697405:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.697407:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.697409:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.697583:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677447.697584:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677447.697586:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.697586:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.697587:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.697587:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:17.0:1619677447.697590:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.697591:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.697592:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.697593:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.697595:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:6.0:1619677447.697595:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677447.697597:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.697597:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.697599:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.697599:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.697600:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.697601:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.697602:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:6.0:1619677447.697603:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677447.697607:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.697608:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.697610:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:6.0:1619677447.697612:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677447.697614:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.697615:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.697616:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:6.0:1619677447.697616:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:17.0:1619677447.697617:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677447.697617:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.697618:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:17.0:1619677447.697619:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677447.697619:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677447.697620:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677447.697621:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:17.0:1619677447.697622:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677447.697622:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.697622:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677447.697623:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.697624:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:4.2:1619677447.697631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.697633:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.697635:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:1.0:1619677447.697635:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677447.697636:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.697637:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.697637:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677447.697638:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.697639:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677447.697640:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677447.697640:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677447.697641:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.697643:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.697644:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.697646:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.697662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.697671:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.697673:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.697676:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.697676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677447.697678:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.697678:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.697679:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.697681:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.697681:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:31.0:1619677447.697682:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.697683:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.697684:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:6.0:1619677447.697684:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.697685:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.697686:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.697688:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.697688:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.697690:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:6.0:1619677447.697691:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677447.697693:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.697694:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.697694:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.697696:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.697696:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.697698:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.697699:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.697700:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677447.697706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677447.697708:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.697710:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.697711:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.697713:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.697714:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:12.0:1619677447.697714:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.697715:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.697716:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.697717:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.697717:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.697728:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.697731:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.697732:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.697732:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.697733:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.697735:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.697848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.697857:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.697860:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.697861:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.697861:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.697863:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.697867:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.697930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:20.0:1619677447.697975:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.697977:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.697978:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.697980:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.697981:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.697983:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.698062:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.698065:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677447.698067:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.698067:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.698069:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.698071:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677447.698072:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.698073:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.698074:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.698075:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.698079:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.698080:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.698082:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.698083:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.698084:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.698085:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.698086:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677447.698087:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677447.698088:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:5.0:1619677447.698088:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677447.698099:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.698100:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677447.698100:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.698101:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000400:00000200:5.0:1619677447.698101:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.698102:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677447.698102:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.698104:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.698105:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.0:1619677447.698106:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.2:1619677447.698253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.698261:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.698264:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.698266:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.698268:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.698268:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.698270:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.698271:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.698272:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.698273:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.698274:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.698277:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.698279:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.698282:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.698283:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.698283:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.698285:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.698286:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.698287:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.698287:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677447.698310:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.698312:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.698313:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.698314:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.698316:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.698317:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677447.698350:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.698352:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.698353:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.698355:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.698358:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.698359:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.698361:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.698362:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.698363:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.698364:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.698366:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.698368:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.698370:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.698371:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.698371:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.698373:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.698373:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.698374:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.698374:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.698387:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.698389:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.698389:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.698390:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.698391:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.698393:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.698405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:17.0:1619677447.698405:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.698408:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.698409:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.698409:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.698409:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.698410:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677447.698411:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.698413:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.698416:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.698416:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.698417:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.698419:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.698421:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.698422:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.698423:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.698424:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.698428:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.698429:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.698432:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.698433:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.698433:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.698435:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.698436:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.698437:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.698437:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677447.698449:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.698452:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.698454:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.698455:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:30.0:1619677447.698457:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.698459:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:22.0:1619677447.698629:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.698632:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.698633:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.698635:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.698638:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.698639:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.698640:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.698642:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.698643:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.698643:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.698647:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.698650:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.698652:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.698653:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.698654:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.698655:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:22.0:1619677447.698656:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.698656:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.698657:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.698657:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.698658:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.698659:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.698666:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.698669:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.698670:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:1.0:1619677447.698670:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:8.0:1619677447.698671:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96724651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.698672:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.698672:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.698673:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:8.0:1619677447.698673:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.698673:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.698674:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677447.698675:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.698677:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.698677:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.0:1619677447.698679:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.698682:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.698683:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677447.698684:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.0:1619677447.698684:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.698686:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.698686:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:13.0:1619677447.698687:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.698687:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.698687:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.698688:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:13.0:1619677447.698689:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.698691:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.698692:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.698692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.698701:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.698703:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.698704:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.698705:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.698707:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.698710:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.698725:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.698728:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.698728:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.698730:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.698732:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.698733:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.698735:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.698736:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.698737:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.698738:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.698741:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.698743:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.698746:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.698746:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:4.2:1619677447.698746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.2:1619677447.698747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:17.0:1619677447.698747:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.698748:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.698749:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.698750:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.698750:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677447.698751:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:24.0:1619677447.698753:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677447.698753:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677447.698756:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.698757:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.698757:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677447.698757:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677447.698759:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.698760:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.698760:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:24.0:1619677447.698762:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.698762:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.698762:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:20.0:1619677447.698763:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.698764:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.698765:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677447.698765:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:6.0:1619677447.698765:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:24.0:1619677447.698766:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96724648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.698766:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.698766:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.698768:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.698768:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.698769:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.698770:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677447.698770:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.698770:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677447.698773:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:24.0:1619677447.698775:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.698775:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.698776:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:24.0:1619677447.698777:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.698777:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.698778:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:22.0:1619677447.698779:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.0:1619677447.698779:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:24.0:1619677447.698780:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.698780:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:24.0:1619677447.698781:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.698781:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677447.698781:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:24.0:1619677447.698782:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.698782:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:6.0:1619677447.698782:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.698783:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:24.0:1619677447.698784:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.698784:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:24.0:1619677447.698785:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.698785:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.698786:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.698788:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677447.698788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677447.698791:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.698793:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677447.698793:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000400:00000200:20.0:1619677447.698794:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.698794:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000400:00000200:20.0:1619677447.698795:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.698795:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:5.0:1619677447.698796:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.698797:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.0:1619677447.698798:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:1.0:1619677447.698798:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.698800:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.698801:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.698801:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.698803:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.698806:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677447.698810:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.698812:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.698813:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.698814:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.698815:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.698816:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.2:1619677447.698856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.698861:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.698864:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.698866:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.698866:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.698868:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.698869:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677447.698871:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.698872:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.698873:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.698874:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677447.698877:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.698879:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.698881:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.698882:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677447.698883:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.698884:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.698885:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.698886:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.698887:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677447.698898:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.698901:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.698902:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.698902:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.698904:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.698906:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.0:1619677447.698973:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.698975:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677447.698977:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.698978:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677447.698980:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.698981:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.698982:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.698983:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.698985:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.698985:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677447.698988:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.698990:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.698992:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.698993:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.698993:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.698994:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.698995:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677447.698996:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677447.698996:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:4.2:1619677447.699000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.699001:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.699003:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.699004:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.699006:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.699008:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:1.0:1619677447.699008:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.699010:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:5.0:1619677447.699011:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.699012:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.699012:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.699013:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.699014:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:1.0:1619677447.699014:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.699015:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:1.0:1619677447.699015:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.699016:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.699020:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.699024:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.699027:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.699028:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.699028:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.699030:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.699031:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.699032:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.699032:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.699036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.699043:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:9.2:1619677447.699044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.699045:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.699046:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.699047:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.699049:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.699052:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:2.0:1619677447.699052:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.699054:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.699055:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.699056:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.699057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000400:00000200:2.0:1619677447.699058:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.699061:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.699062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.699063:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.699066:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.699066:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.699067:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.699068:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.699068:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:18.0:1619677447.699069:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.699072:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.699073:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.699073:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.699074:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.699075:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.699149:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07de000] (21)++
00000800:00000200:17.0:1619677447.699152:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.1@o2ib3
00000800:00000200:1.0:1619677447.699152:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07de000] (22)--
00000400:00000200:17.0:1619677447.699155:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.1@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.699158:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.699159:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:20.2:1619677447.699161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:17.0:1619677447.699161:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.699162:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.699163:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.699164:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:17.0:1619677447.699167:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.699168:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677447.699169:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 584 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:30.0:1619677447.699171:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:17.0:1619677447.699172:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677447.699173:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.699173:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:17.0:1619677447.699173:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:31.0:1619677447.699175:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677447.699175:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000400:00000200:30.0:1619677447.699176:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.699176:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07de000] (21)++
00000400:00000200:30.0:1619677447.699177:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:17.0:1619677447.699177:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07de000] (22)++
00000800:00000200:17.0:1619677447.699178:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07de000] (23)--
00000400:00000200:30.0:1619677447.699179:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.699179:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07de000] (22)--
00000400:00000200:30.0:1619677447.699180:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.699181:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.699182:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.699185:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.699187:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.699189:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:1.0:1619677447.699189:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07de000] (22)++
00000800:00000200:30.0:1619677447.699190:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.699191:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.699192:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.699192:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:1.0:1619677447.699192:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07de000] (23)--
00000800:00000200:1.0:1619677447.699192:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07de000] (22)--
00000400:00000200:20.0:1619677447.699193:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.699193:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07de000] (21)--
00000800:00000200:30.0:1619677447.699194:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.699194:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.699195:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677447.699195:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.699196:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677447.699197:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:30.0:1619677447.699198:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677447.699198:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:30.0:1619677447.699201:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.699203:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.699205:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.699205:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.699206:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.699207:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.699208:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:30.0:1619677447.699209:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.699210:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.699210:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.699211:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.699211:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.699213:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.699214:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:30.0:1619677447.699215:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.699217:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677447.699218:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.699220:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.699220:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.699221:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.699223:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.699223:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.699224:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.699226:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:20.0:1619677447.699226:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.699227:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677447.699229:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.699230:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.699230:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.699231:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.699232:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:13.0:1619677447.699235:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.699237:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.699238:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.699240:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.699241:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677447.699241:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.699242:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.699244:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.699245:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.699246:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677447.699264:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.699268:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.699268:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:6.0:1619677447.699270:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677447.699270:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.699272:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.699273:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677447.699273:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.699275:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.699276:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.699276:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.699278:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.699278:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.699279:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.699279:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.699280:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.699281:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.699283:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.699284:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.699285:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.699285:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.699288:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.699289:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.699290:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.699291:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.699292:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.699292:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.699299:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.699300:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.699300:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677447.699301:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.699301:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.699301:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:6.0:1619677447.699302:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.699303:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.699304:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.699304:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.699305:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.699310:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.699312:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.699313:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.699315:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.699315:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.699317:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677447.699317:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.699320:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.699321:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.699321:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.699323:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.699326:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.699327:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.699328:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.699328:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.699331:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.699333:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.699335:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.699336:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.699337:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.699338:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.699339:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.699342:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.699344:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.699345:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.699346:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.699347:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.699348:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.699349:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.699349:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.699350:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.699351:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.699362:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.699363:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.699363:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:17.0:1619677447.699364:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.699364:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:17.0:1619677447.699365:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.699365:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.699366:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.699366:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.699367:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677447.699525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.699532:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.699535:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.699536:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.699538:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.699541:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.699542:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.699544:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.699545:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.699546:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.699547:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.699550:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.699552:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.699554:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.699555:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.699556:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.699557:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.699558:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.699559:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.699559:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.699571:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.699574:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.699575:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.699575:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.699577:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.699579:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.699588:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.699591:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.699591:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.699591:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.699592:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.699593:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.699722:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.699725:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.699726:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.699729:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.699731:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.699732:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.699734:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.699736:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.699737:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.699738:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.699741:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.699744:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.699746:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.699747:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.699747:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.699748:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.699749:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.699749:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.699750:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.699761:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.699763:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.699764:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.699764:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.699766:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.699767:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:1.0:1619677447.699767:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:31.0:1619677447.699769:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.699770:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.699772:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.699774:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.699774:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:17.0:1619677447.699781:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.699784:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.699784:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.699785:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.699785:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.699786:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.699789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.699833:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:22.0:1619677447.699834:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:24.0:1619677447.699835:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.699836:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677447.699837:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:24.0:1619677447.699838:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.699838:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.699838:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.699840:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.699841:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.699842:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.699843:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.699843:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:24.0:1619677447.699845:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.699846:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.699846:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.699849:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.699851:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.699854:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.699855:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.699855:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.699857:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.699858:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.699859:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.699859:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.699864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.699871:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.699874:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.699875:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.699876:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.699877:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.699880:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.699970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.699973:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.699976:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.699978:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.699980:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.699980:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.699982:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.699983:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.699985:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.699985:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.699987:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.699990:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.699992:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.699994:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.699995:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.699995:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.699997:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.699997:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.699998:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.699999:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.700010:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.700013:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.700014:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.700014:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.700016:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.700018:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.700060:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.700063:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.700064:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.700064:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.700065:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.700065:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.700217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.700224:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.700226:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.700228:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.700229:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.700232:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.700234:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.700235:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.700236:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.700237:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.700238:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.700242:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.700243:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.700246:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.700247:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.700247:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.700249:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.700250:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.700251:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.700252:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.700253:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.700259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677447.700263:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677447.700264:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:30.0:1619677447.700265:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:24.0:1619677447.700266:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.700266:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:30.0:1619677447.700267:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.700267:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.700268:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677447.700268:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.700269:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.700269:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.700269:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.700270:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.700270:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.700272:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677447.700273:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.700273:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:30.0:1619677447.700274:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.700275:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.700276:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677447.700280:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.700280:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677447.700281:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.700283:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:30.0:1619677447.700284:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.700284:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677447.700285:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677447.700285:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:6.0:1619677447.700286:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.700287:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.700288:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677447.700288:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.700289:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.700289:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:6.0:1619677447.700290:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.700291:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724629] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.700292:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.700299:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.700300:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.700302:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.700304:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.700304:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.700305:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.700307:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.700307:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.700310:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.700310:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.700311:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.700312:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.700312:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.700314:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.700315:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.700316:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.700317:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677447.700328:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.700331:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.700332:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.700334:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.700335:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.700337:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:2.0:1619677447.700357:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.700361:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.700361:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.700363:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.700366:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.700367:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.700369:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.700370:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.700371:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.700372:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.700375:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677447.700377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.700377:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.700380:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.700380:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.700381:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.700383:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.700383:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.700385:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.700386:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.700386:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.700397:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:17.0:1619677447.700398:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677447.700399:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.700400:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677447.700400:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677447.700401:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:17.0:1619677447.700401:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.700401:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.700404:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.700406:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.700407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.700413:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.700415:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.700417:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.700418:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.700420:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.700421:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.700423:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.700424:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.700425:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:5.0:1619677447.700425:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.700426:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.700428:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.700429:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.700430:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.700430:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.700432:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.700432:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.700434:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.700434:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.700435:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.700436:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.700437:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.700438:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.700438:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.700439:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.700451:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.700454:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.700454:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.700455:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.700456:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.700458:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.700484:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.700487:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.700487:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.700489:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.700491:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.700493:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.700494:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.700495:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.700496:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:1.0:1619677447.700497:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.700497:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:2.0:1619677447.700499:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.700500:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.700503:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677447.700503:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.700505:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:2.0:1619677447.700506:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.700509:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:1.0:1619677447.700509:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677447.700510:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.700510:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:2.0:1619677447.700511:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.700511:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:2.0:1619677447.700512:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.700513:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:1.0:1619677447.700513:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.700513:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.700514:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.700514:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:2.0:1619677447.700517:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.700519:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677447.700522:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.700523:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677447.700523:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677447.700525:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677447.700525:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677447.700526:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:2.0:1619677447.700527:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:30.0:1619677447.700528:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:2.0:1619677447.700528:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.700528:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:30.0:1619677447.700529:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.700531:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.700532:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.700534:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677447.700539:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.700541:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.700542:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.700543:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677447.700695:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677447.700698:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677447.700700:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.700702:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677447.700703:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.700704:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.700705:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.700706:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.700708:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.700709:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.700712:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.700713:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.700716:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.700716:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.700717:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.700719:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.700719:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677447.700720:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677447.700721:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:20.2:1619677447.700721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.700724:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.700726:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.700729:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.700730:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:30.0:1619677447.700731:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:20.0:1619677447.700731:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.700733:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:20.0:1619677447.700733:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677447.700734:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.700734:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724622] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.700736:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.700736:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.700737:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:20.0:1619677447.700737:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.700738:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:20.0:1619677447.700738:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677447.700742:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.700743:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.700746:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.700747:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677447.700747:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.700748:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677447.700749:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.700749:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:31.0:1619677447.700751:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:20.0:1619677447.700751:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:31.0:1619677447.700752:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:20.0:1619677447.700752:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:31.0:1619677447.700753:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677447.700753:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677447.700753:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677447.700755:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677447.700757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677447.700762:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677447.700763:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:17.0:1619677447.700765:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:5.0:1619677447.700765:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:17.0:1619677447.700766:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.700766:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:1.0:1619677447.700766:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.700767:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:17.0:1619677447.700768:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.700771:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.700850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.700895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.700895:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.700897:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:22.0:1619677447.700899:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677447.700899:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.700901:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.700902:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.700903:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.700903:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:2.0:1619677447.700904:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.700905:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.700906:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.700907:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96724619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.700908:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:2.0:1619677447.700909:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.700910:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.700910:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.700913:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.700915:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.2:1619677447.700916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677447.700917:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.700918:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:6.0:1619677447.700919:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.700919:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.700921:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.700921:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:6.0:1619677447.700922:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.700922:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:22.0:1619677447.700923:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.700923:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:30.0:1619677447.700924:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:6.0:1619677447.700924:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.700924:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.700924:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:6.0:1619677447.700925:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.700926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677447.700927:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.700929:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.700931:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.700931:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.700932:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.700933:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.700935:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.700935:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:30.0:1619677447.700936:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.700936:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:5.0:1619677447.700936:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:30.0:1619677447.700937:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:8.0:1619677447.700937:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:5.0:1619677447.700938:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:24.0:1619677447.700939:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677447.700939:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:5.0:1619677447.700939:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.700940:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.700940:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:30.0:1619677447.700941:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.700941:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:8.0:1619677447.700942:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.700942:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.700943:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:8.0:1619677447.700945:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.700945:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:30.0:1619677447.700947:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677447.700947:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:30.0:1619677447.700948:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.700949:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:8.0:1619677447.700949:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96724617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.700950:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:8.0:1619677447.700950:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.700951:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:8.0:1619677447.700951:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.700952:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:8.0:1619677447.700952:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:30.0:1619677447.700953:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.700956:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:8.0:1619677447.700960:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.700962:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677447.700965:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:8.0:1619677447.700965:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.700966:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.700966:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.700967:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.700968:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.700968:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.700969:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.700969:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:20.0:1619677447.700970:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.700970:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.0:1619677447.700970:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.2:1619677447.700973:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677447.700975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.700976:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.700977:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.700979:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:5.0:1619677447.700979:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.700981:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.700982:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:20.0:1619677447.700983:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.700984:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.700985:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724616] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.700985:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.700986:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.700986:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.700987:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.700988:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.700989:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.700990:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.700993:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.700993:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.0:1619677447.700995:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.700998:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.700999:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.700999:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.701001:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.701002:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.701003:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.701004:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.701005:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.701010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677447.701014:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677447.701015:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:22.0:1619677447.701017:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677447.701017:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:22.0:1619677447.701018:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.701018:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.701018:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.701019:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:22.0:1619677447.701020:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.701022:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.701032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:31.0:1619677447.701051:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.701053:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.701057:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.701057:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.701060:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.701061:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677447.701062:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.701064:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.701064:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.701065:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677447.701068:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.701070:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:9.2:1619677447.701072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677447.701073:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.701074:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677447.701074:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677447.701076:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.701077:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.701078:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.701078:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677447.701080:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.701082:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.701083:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.701084:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.701086:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.701089:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.701090:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.701093:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.701093:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.701094:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.701096:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.701099:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.701189:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.701192:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.701193:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.701194:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.701197:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.701198:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.701200:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.701201:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.701202:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.701203:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.701206:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.701208:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.701210:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.701211:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.701212:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.701213:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.701214:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.701215:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.701215:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.701219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.701226:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.701228:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.701229:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.701230:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.701231:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.701232:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.701375:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.701377:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.701379:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.701380:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.701382:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.701383:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677447.701384:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.701385:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.701386:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.701387:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677447.701390:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.701392:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.701393:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.701393:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.701394:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.701395:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.701396:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.701396:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.701397:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.701402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.701409:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.701411:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.701411:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.701413:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.701413:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.701414:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.701420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.701427:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.701429:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.701430:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.701430:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.701430:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.701431:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.701432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.701480:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.701482:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.701483:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.701484:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.701486:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.701488:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.701653:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.701656:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.701656:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.701658:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.701661:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.701662:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.701663:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.701665:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.701666:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.701667:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.701670:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.701672:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.701675:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.701676:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.701676:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.701678:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.701679:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.701680:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.701681:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.701682:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677447.701685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.701692:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.701694:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.701694:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.2:1619677447.701694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677447.701694:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:22.0:1619677447.701695:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.701696:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.701697:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.701699:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.701701:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.701702:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677447.701702:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677447.701705:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.701707:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.701709:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.701710:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.701712:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.701713:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.701714:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.701715:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.701716:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.701720:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.701722:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.701725:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.701725:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.701726:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.701728:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.701729:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.701730:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.701732:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.701732:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677447.701734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677447.701740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677447.701746:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677447.701748:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677447.701748:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.701750:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.701751:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.701751:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677447.701751:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.701751:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677447.701752:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677447.701753:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.701754:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.701754:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:5.0:1619677447.701755:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:5.0:1619677447.701755:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.701756:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677447.701784:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.701787:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.701789:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.701791:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.701792:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.701793:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677447.701795:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.701796:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.701797:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.701798:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677447.701801:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.701802:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.701805:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.701806:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677447.701807:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.701807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677447.701808:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677447.701809:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.701810:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.701811:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677447.701814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.701823:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.701826:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.701827:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.701827:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.701830:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.701833:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.701841:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.701844:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.701845:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677447.701846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:1.0:1619677447.701847:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.701849:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.701851:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.701852:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.701853:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.701854:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:1.0:1619677447.701854:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.701855:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:5.0:1619677447.701857:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.701858:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.701859:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.701859:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.701860:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677447.701861:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:5.0:1619677447.701862:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.701864:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.701864:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.701865:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.701867:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.701867:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.701868:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.701869:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.701873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.701879:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.701882:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.701883:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.701883:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.701886:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.701888:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677447.702059:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.702061:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.702062:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.702064:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.702067:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.702068:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.702069:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.702070:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.702072:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.702072:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.702076:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.702078:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.702080:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.702081:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.702082:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.702084:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.702084:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.702086:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.702087:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.702087:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677447.702090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.702098:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677447.702100:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:18.0:1619677447.702101:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:24.0:1619677447.702102:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.702102:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:18.0:1619677447.702102:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.702102:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.702103:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:18.0:1619677447.702104:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.702108:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.702112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:4.2:1619677447.702118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:2.0:1619677447.702119:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.702121:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.702122:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.702122:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.702123:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.702124:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.0:1619677447.702125:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.702128:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.702129:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.702130:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.702133:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.702134:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.702135:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.702137:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.702138:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.702138:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.702141:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.702143:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.702146:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.702146:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.702147:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.702148:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.702149:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.702150:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.702150:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.702154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.702163:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:4.2:1619677447.702164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.702166:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.702166:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.702167:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.702169:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.702172:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.702201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.702208:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:6.0:1619677447.702209:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:30.0:1619677447.702211:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:6.0:1619677447.702212:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:30.0:1619677447.702213:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.702213:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.702213:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.702215:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:6.0:1619677447.702215:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.702216:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.702218:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:6.0:1619677447.702218:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:30.0:1619677447.702219:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724602] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.702220:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.702221:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.702222:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.702225:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.702226:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.702228:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.702229:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.702230:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.702231:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.702232:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.702233:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.702233:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.702245:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.702247:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.702248:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.702250:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.702252:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.702253:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677447.702261:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.702264:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.702264:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.702267:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.702269:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.702271:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.702272:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.702273:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.702274:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.702275:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677447.702279:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.702280:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.702283:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.702284:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.702284:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.702286:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.702287:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.702288:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.702289:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.702289:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.702292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.702300:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.702302:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:17.0:1619677447.702302:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677447.702303:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.702304:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.702304:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.702305:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677447.702306:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.702306:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677447.702309:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:31.0:1619677447.702544:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.702546:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.702548:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.702551:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.702551:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.702553:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677447.702554:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.702555:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.702556:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.702557:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677447.702560:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.702562:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677447.702565:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.702566:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677447.702567:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677447.702568:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677447.702569:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.702571:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.702572:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.702573:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677447.702575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:4.2:1619677447.702581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.702583:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:30.0:1619677447.702586:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:18.0:1619677447.702586:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.702587:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.702587:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677447.702589:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:18.0:1619677447.702589:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.702590:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.702591:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677447.702592:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:22.0:1619677447.702613:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.702615:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.702616:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.702618:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.702620:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.702622:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.702623:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.702623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677447.702625:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.702626:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677447.702626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000400:00000200:22.0:1619677447.702628:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.702629:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.702630:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:6.0:1619677447.702630:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.702631:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.702632:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.702633:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:22.0:1619677447.702634:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.702634:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.702635:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.702636:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677447.702637:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.702637:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.702638:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.702638:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.702639:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.702639:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:20.0:1619677447.702639:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.702640:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.702640:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.702641:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.702641:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.702642:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.702642:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:20.0:1619677447.702642:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.702642:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.702643:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.702643:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:20.0:1619677447.702643:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.702643:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.702644:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:20.0:1619677447.702647:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.702648:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:9.2:1619677447.702648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.702651:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.702652:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.702652:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.702654:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.702655:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677447.702655:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677447.702656:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:2.0:1619677447.702656:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.702659:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.702660:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.702660:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.702662:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.702664:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:1.0:1619677447.702666:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677447.702667:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:1.0:1619677447.702669:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:22.0:1619677447.702670:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:17.0:1619677447.702670:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677447.702671:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.702671:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:1.0:1619677447.702672:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.702673:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.702675:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.702676:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:1.0:1619677447.702677:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.702678:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.702679:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.702680:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.702681:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677447.702685:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.702687:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.702690:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.702690:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.702691:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.702693:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.702693:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.702694:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.702695:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.702699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.702705:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.702708:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.702709:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.702710:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.702711:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.702714:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.702716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677447.702722:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.702724:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677447.702725:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.702726:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.702726:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.702727:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.2:1619677447.702746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:8.2:1619677447.702807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677447.702810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.702814:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.702815:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.702816:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.702818:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.702819:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.702821:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.702821:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.702822:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.702824:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.702824:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.702826:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.702828:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.702829:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.702831:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.702832:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.702834:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.702838:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.702841:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.702843:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.702844:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.702845:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.702847:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.702848:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.702849:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.702850:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.702854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.702861:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:18.0:1619677447.702864:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000800:00000200:2.0:1619677447.702864:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (24)++
00000400:00000200:18.0:1619677447.702865:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.702865:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000400:00000200:2.0:1619677447.702866:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.702868:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.702868:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.702872:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.702872:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.702873:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.703037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.703083:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.703086:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.703086:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.703087:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.703089:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.703091:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.703107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.703115:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.703118:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.703120:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.703122:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.703123:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.703124:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677447.703125:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.703126:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.703127:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.703128:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677447.703131:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.703133:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.703134:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.703135:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.703135:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.703137:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.703138:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.703139:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.703139:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677447.703149:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.703152:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.703153:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.703154:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.703156:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.703157:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:8.0:1619677447.703196:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.703198:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.703199:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.703201:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.703203:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.703205:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.703206:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.703207:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.703208:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.703209:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.703212:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.703214:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.703217:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.703218:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.703218:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.703221:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.703221:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.703222:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.703222:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.703227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.703234:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.703237:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.703238:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.703238:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.703240:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.703243:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.703264:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.703267:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.703267:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.703271:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.703274:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.703276:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.703278:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.703280:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.703282:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.703283:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.703288:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.703291:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.703293:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.703294:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.703301:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.703303:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.703303:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.703304:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.703305:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.703309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.703318:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.703320:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.703321:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.703322:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.703323:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.703326:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.703329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:9.2:1619677447.703334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677447.703337:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.703340:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.703340:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.703341:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.703342:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.703342:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677447.703343:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.703346:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.703347:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.703347:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.703350:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.703354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.703359:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.703361:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.703362:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.703363:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.703363:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.703364:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.703365:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677447.703384:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.703387:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.703388:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.703390:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.703392:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.703392:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.2:1619677447.703398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.703406:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.703409:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.703410:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.703411:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.703413:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.703416:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.703419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.703426:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.703429:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.703430:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:13.0:1619677447.703430:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677447.703430:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.703431:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:9.2:1619677447.703431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:17.0:1619677447.703432:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:13.0:1619677447.703433:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.703434:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.703435:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.703437:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.703438:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:13.0:1619677447.703441:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.703441:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.703442:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.703442:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.703444:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.703448:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:12.0:1619677447.703453:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.703455:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.703456:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.703457:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.703457:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.703458:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.703459:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.703459:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.703460:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677447.703461:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.703462:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.703472:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677447.703496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.703504:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.703507:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.703508:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.703508:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.703510:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.703513:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677447.703681:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.703684:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.703686:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.703687:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.703690:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.703691:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.703693:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.703694:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.703695:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.703696:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.703699:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.703701:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.703704:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.703705:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.703705:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.703707:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.703708:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.703708:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.703709:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.703713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.703721:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.703723:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.703724:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.703725:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.703727:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.703729:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:9.2:1619677447.703750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:20.2:1619677447.703752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.703756:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:24.0:1619677447.703757:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:20.0:1619677447.703758:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:24.0:1619677447.703760:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:20.0:1619677447.703760:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677447.703761:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.703761:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:12.0:1619677447.703762:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.0:1619677447.703762:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.703763:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.703763:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.703764:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:20.0:1619677447.703765:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677447.703765:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.703766:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:20.0:1619677447.703766:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677447.703767:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.703768:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.703768:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:20.0:1619677447.703769:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.703770:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:12.0:1619677447.703770:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.703772:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:20.0:1619677447.703778:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:31.0:1619677447.703780:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677447.703780:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.703782:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.703783:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:20.0:1619677447.703783:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.703784:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677447.703785:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.703786:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:31.0:1619677447.703787:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.703787:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677447.703787:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:31.0:1619677447.703788:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:20.0:1619677447.703788:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.703788:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:31.0:1619677447.703789:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.703791:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.703792:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677447.703792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677447.703793:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.703796:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.703798:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.703800:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.703800:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677447.703801:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.703802:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.703803:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.703803:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:31.0:1619677447.703804:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:18.0:1619677447.703804:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.703804:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677447.703805:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.703806:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:18.0:1619677447.703806:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:1.0:1619677447.703806:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.703808:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.703809:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.703810:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.703811:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.703813:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.703815:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:13.0:1619677447.703816:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:1.0:1619677447.703816:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.703819:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.703819:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.703819:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.703820:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.703821:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:13.0:1619677447.703822:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.703822:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:1.0:1619677447.703825:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.703827:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.703829:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677447.703830:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.703831:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.703832:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.703834:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.703834:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.703835:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.703835:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.703846:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.703849:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.703850:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.703851:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.703853:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.703854:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.703953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.703958:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.703961:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.703964:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.703964:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.703967:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.703969:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677447.703970:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724572] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.703971:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.703972:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.703978:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677447.703981:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.703983:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677447.703983:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.703985:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.703986:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677447.703986:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:22.0:1619677447.703987:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.703987:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.703987:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.703988:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.703989:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:6.0:1619677447.703990:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.703990:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.703990:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.703998:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.2:1619677447.704005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.704013:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:9.2:1619677447.704014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:17.0:1619677447.704015:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.704016:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.704016:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.704018:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.704021:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:2.0:1619677447.704022:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.704025:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.704027:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.704027:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.704029:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.704032:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.704038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.704046:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.704048:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:4.2:1619677447.704048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.704049:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.704050:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.704051:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.704052:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:22.0:1619677447.704061:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.704063:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.704064:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.704064:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.704067:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.704074:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.704092:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.704095:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.704095:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.704096:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.704098:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.704100:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.704248:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.704251:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.704252:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.704254:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.704256:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.704257:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.704259:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.704260:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.704261:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.704262:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.704265:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.704267:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.704270:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.704270:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.704271:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.704273:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.704273:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.704275:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.704276:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.704276:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.704279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.704285:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677447.704287:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677447.704288:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.704289:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.704289:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.704289:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677447.704289:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.704290:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677447.704291:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.704299:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:30.0:1619677447.704335:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.704338:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.704338:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.704339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:30.0:1619677447.704340:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.704342:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.704343:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:24.0:1619677447.704346:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.704349:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.704350:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.704352:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.704354:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.704356:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.704357:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.704358:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.704359:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.704360:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.704363:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.704364:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.704367:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.704368:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.704368:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.704370:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.704371:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.704371:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.704372:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.704376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.704385:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.704388:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.704388:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.704389:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.704391:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.704394:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.704507:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.704510:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.704510:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.704512:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.704515:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.704516:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.704517:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.704519:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.704520:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.704521:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.704524:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.704526:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.704528:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.704529:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.704530:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.704532:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.704532:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.704533:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.704533:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.704545:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.704548:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.704549:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.704549:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.704551:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.704553:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.704563:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.704565:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.704566:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.704567:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.704568:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.704568:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.704616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.704645:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.704648:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.704649:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.704651:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.704653:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.704655:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.704657:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.704659:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.704660:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.704661:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.704662:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.704663:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.704664:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.704665:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.704666:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.704667:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.704670:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.704671:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.704672:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.704673:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.704674:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.704676:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.704676:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.704677:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.704678:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677447.704688:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.704691:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.704692:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.704694:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.704695:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.704696:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677447.704705:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677447.704709:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.704710:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.704710:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.704712:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.704713:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677447.704901:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.704904:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.704904:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.704906:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.704909:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.704910:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677447.704911:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.704913:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.704914:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.704915:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677447.704918:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.704920:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677447.704923:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.704924:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677447.704924:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677447.704926:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.704927:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.704927:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.704928:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.704932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.704938:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.704942:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.704943:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.704944:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.704945:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:8.0:1619677447.704945:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.704947:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.704948:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:24.0:1619677447.704949:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.704949:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.704952:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.704954:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.704955:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.704956:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.704957:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.704958:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.704962:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.704963:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.704967:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.704967:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.704968:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.704970:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.704970:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.704971:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.704971:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677447.704976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.704984:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.704987:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.704988:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.704990:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.704991:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.704993:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677447.704995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.705003:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.705006:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.705007:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.705008:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.705008:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.705010:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.705283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.705299:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.705302:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.705303:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.705305:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.705307:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.705309:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.705310:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.705312:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.705314:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.705315:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.705319:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.705322:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.705324:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.705325:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.705326:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.705328:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.705328:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.705329:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.705329:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:4.2:1619677447.705334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677447.705339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.705339:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.705342:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.705343:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.705343:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.705345:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.705345:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677447.705347:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677447.705348:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.705350:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.705350:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.705351:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.705352:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:17.0:1619677447.705353:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.705353:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:1.0:1619677447.705353:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.705355:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677447.705356:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.705356:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677447.705356:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.705357:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.705358:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.705358:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:1.0:1619677447.705358:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.705359:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.705360:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677447.705363:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.705364:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677447.705364:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.705365:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.705366:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.705366:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677447.705367:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.705369:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.705369:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.705370:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.705370:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.705370:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:1.0:1619677447.705371:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677447.705373:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677447.705374:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677447.705374:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677447.705375:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:18.0:1619677447.705378:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.705380:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.705381:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.705383:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.705385:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.705387:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:18.0:1619677447.705387:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677447.705388:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.705389:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:18.0:1619677447.705389:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677447.705390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:18.0:1619677447.705390:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.705390:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.705391:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.705391:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:22.0:1619677447.705393:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.705395:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.705395:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.705397:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.705398:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.705398:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:18.0:1619677447.705400:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.705401:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677447.705401:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677447.705401:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:5.0:1619677447.705401:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.705403:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:5.0:1619677447.705403:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.705404:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.705405:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.705405:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:5.0:1619677447.705405:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.705406:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.705407:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.705408:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.705409:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.705413:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.705415:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.705417:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.0:1619677447.705417:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.705418:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.705418:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.705419:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.705420:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.705420:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.705420:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.705421:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.705421:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:6.0:1619677447.705422:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.705422:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.705424:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.705433:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.705435:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.705436:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.705436:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.705438:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.705440:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.705598:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.705601:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.705602:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.705603:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.705607:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.705608:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677447.705609:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.705611:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.705611:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.705612:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677447.705615:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.705617:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:4.2:1619677447.705618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.705619:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.705620:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.705620:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.705623:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.705623:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.705624:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.705625:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677447.705626:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.705628:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.705629:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.705630:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.705632:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.705634:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.705635:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96724553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.705636:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:8.0:1619677447.705636:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.705637:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.705638:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:8.0:1619677447.705638:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677447.705639:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.705641:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.705642:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.705643:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.705644:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.705644:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.705647:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.705648:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.705648:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.705650:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.705650:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.705652:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.705653:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:12.0:1619677447.705654:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.705654:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.705656:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677447.705657:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.705658:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.705658:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.705659:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:4.2:1619677447.705659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:12.0:1619677447.705660:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677447.705664:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.705666:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677447.705667:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.705667:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.705668:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677447.705668:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.705669:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.705670:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677447.705670:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.705673:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.705856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.705873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.705881:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.705884:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.705886:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.705888:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.705889:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.705890:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677447.705891:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.705892:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.705893:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.705894:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:9.2:1619677447.705894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677447.705897:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.705899:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677447.705902:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.705902:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677447.705903:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677447.705903:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.705904:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:31.0:1619677447.705905:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677447.705906:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677447.705906:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.705906:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.705907:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:2.0:1619677447.705908:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.705909:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.705909:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:2.0:1619677447.705911:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.2:1619677447.705912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677447.705916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:6.0:1619677447.705917:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.705920:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.705920:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.705922:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.705924:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677447.705924:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:6.0:1619677447.705925:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.705926:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:6.0:1619677447.705926:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:30.0:1619677447.705927:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:6.0:1619677447.705927:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.705928:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677447.705928:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.705929:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:6.0:1619677447.705929:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.705930:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.705930:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.705930:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:6.0:1619677447.705931:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.705935:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.705936:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:6.0:1619677447.705937:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.705939:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.705940:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.705941:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.705943:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.705943:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.705945:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.705946:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.705946:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677447.705949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.705958:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:5.0:1619677447.705958:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:22.0:1619677447.705960:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:5.0:1619677447.705960:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677447.705961:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.705961:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:5.0:1619677447.705961:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.705963:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.705963:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.705966:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.706000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.706003:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.706005:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.706007:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.706008:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.706009:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.706011:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.706012:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.706013:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.706014:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.706015:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.706019:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.706020:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.706023:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.706023:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.706024:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.706026:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.706026:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.706027:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.706029:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.706029:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.706034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677447.706038:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677447.706040:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:22.0:1619677447.706041:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.706042:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.706042:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.706042:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.706043:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:22.0:1619677447.706044:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.706044:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.706046:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677447.706073:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.706076:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.706077:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.706079:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.706081:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.706082:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.706083:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.706085:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.706086:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.706087:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.706090:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.706092:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.706094:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.706095:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.706095:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.706097:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.706097:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.706098:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.706098:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.706103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.706110:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.706111:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.706112:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.706113:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.706114:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.706116:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677447.706159:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.706162:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.706164:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.706166:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.706167:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.706169:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677447.706170:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724547] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.706171:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.706172:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.706173:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677447.706176:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.706178:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.706181:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.706182:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677447.706182:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677447.706184:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.706185:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.706185:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.706186:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.706198:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.706201:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.706201:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.706202:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.706204:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.706210:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.706211:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.706213:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.706214:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.706215:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.706215:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.706216:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.706382:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.706385:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.706386:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.706387:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.706390:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.706392:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.706393:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.706394:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.706395:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.706396:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.706399:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.706401:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.706404:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.706404:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.706405:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.706407:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.706407:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.706409:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.706410:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.706410:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.2:1619677447.706413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.706425:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.706426:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:5.0:1619677447.706426:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:1.0:1619677447.706426:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677447.706427:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.706428:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677447.706429:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:1.0:1619677447.706429:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.706430:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677447.706430:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.706431:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.706432:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:1.0:1619677447.706432:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.706434:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.706435:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:1.0:1619677447.706435:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.706436:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.706438:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.706439:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.706440:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677447.706442:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.706444:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.706447:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.706448:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.706448:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.706450:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.706450:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.706451:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.706452:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.706456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.706463:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.706466:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.706466:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.706467:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.706469:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.706472:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.706524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.706531:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.706533:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.706534:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.706536:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.706539:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.706540:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.706541:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677447.706541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:24.0:1619677447.706542:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.706543:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.706544:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:4.2:1619677447.706544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677447.706547:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.706550:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.706554:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.706555:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.706556:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.706558:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.706558:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.706559:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.706560:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.706564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677447.706569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.706572:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677447.706572:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677447.706574:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:17.0:1619677447.706575:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.706576:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.706576:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.706578:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.706578:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.706580:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.706581:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:20.0:1619677447.706583:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.706584:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.706586:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96724541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677447.706587:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:20.0:1619677447.706591:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.706591:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:20.0:1619677447.706592:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.706593:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:12.0:1619677447.706593:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.706594:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.706594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:12.0:1619677447.706597:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.706598:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.706598:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.706600:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:12.0:1619677447.706600:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.706602:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.706603:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.706603:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.706604:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.706604:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:5.0:1619677447.706604:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.706606:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.706606:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:5.0:1619677447.706606:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.706607:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.706608:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677447.706609:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.706610:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677447.706611:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.706614:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.706615:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.706616:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.706616:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.706618:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.706620:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.706623:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.706624:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.706624:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.706626:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.706628:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677447.706781:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.706784:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.706786:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.706788:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.706789:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.706791:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677447.706792:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.706793:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.706794:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.706796:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677447.706799:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.706801:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677447.706804:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.706805:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677447.706805:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677447.706807:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677447.706808:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.706809:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.706809:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677447.706813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.706822:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.706824:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.706826:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.706826:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.706828:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.706831:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:1.0:1619677447.706953:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.2:1619677447.706956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677447.706956:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.706956:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.706958:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.706961:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.706962:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.706963:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.706964:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:1.0:1619677447.706964:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.706965:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.706966:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677447.706967:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677447.706970:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.706970:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.706971:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.706973:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.706973:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:30.0:1619677447.706975:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:1.0:1619677447.706975:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.706976:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.706976:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677447.706977:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.706978:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.706978:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.706978:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677447.706979:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.706980:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677447.706980:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.706981:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.706982:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677447.706983:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677447.706984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677447.706985:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.706988:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.706989:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.706990:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.706991:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.706992:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677447.706992:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:30.0:1619677447.706993:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.706993:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:17.0:1619677447.706993:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.706993:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:17.0:1619677447.706994:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.706994:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.706995:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.706996:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.706996:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.2:1619677447.706997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:1.0:1619677447.706997:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677447.706998:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677447.707006:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.707008:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.707009:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.707011:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.707012:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:4.2:1619677447.707012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:5.0:1619677447.707014:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.0:1619677447.707020:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.707023:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.707024:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.707025:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.707029:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.707031:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.707032:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.707033:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.707034:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.707035:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.707038:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.707040:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.707043:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.707043:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.707044:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.707046:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.707046:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.707047:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.707047:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.707052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.707060:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.707063:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.707063:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.707064:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.707066:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.707068:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.707072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:4.2:1619677447.707072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.707079:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.707083:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.707083:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.707084:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.707085:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.707085:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677447.707119:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.707122:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.707123:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.707123:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.707125:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.707128:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.707305:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.707308:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.707308:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.707311:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.707313:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.707314:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.707316:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.707317:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.707318:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.707319:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.707322:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.707323:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.707326:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.707327:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.707327:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.707329:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.707330:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.707330:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.707331:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.707336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.707344:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.707347:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.707348:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.707349:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.707350:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.707353:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:1.0:1619677447.707363:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.707366:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.707366:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.707369:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677447.707370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:1.0:1619677447.707370:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.707372:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.707373:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.707374:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.707376:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.707376:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.707380:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.707382:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.707385:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.707386:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.707387:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.707389:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.707389:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.707390:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.707391:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.707402:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.707405:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.707406:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.707408:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.707410:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.707411:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.2:1619677447.707411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.707419:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.707421:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.707422:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.707423:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.707425:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:17.0:1619677447.707428:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.707431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.707439:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.707442:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.707444:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.707446:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.707447:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.707449:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.707450:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.707451:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.707453:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.707453:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.707457:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.707458:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.707461:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.707462:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.707463:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.707465:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.707465:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.707466:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.707467:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.707471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.707480:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:4.2:1619677447.707481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.707482:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677447.707483:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.707484:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.707485:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:20.0:1619677447.707486:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.707486:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.707488:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.707488:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.707489:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000400:00000200:18.0:1619677447.707490:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.707490:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.707491:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:18.0:1619677447.707491:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:5.0:1619677447.707491:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677447.707492:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677447.707492:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.707492:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677447.707494:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:18.0:1619677447.707494:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.707495:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.707497:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.707497:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.707498:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677447.707498:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.707500:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.707502:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677447.707502:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.707503:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.707505:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.707505:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.707506:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.707507:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.707508:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.707509:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.707510:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.707511:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:30.0:1619677447.707512:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.707514:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:30.0:1619677447.707515:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.707515:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.707516:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.707517:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677447.707517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.707518:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.707519:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677447.707520:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677447.707522:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677447.707523:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.707524:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.707525:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677447.707525:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.707527:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.707528:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.707528:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:9.2:1619677447.707529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.707529:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:20.0:1619677447.707530:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.707531:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:4.2:1619677447.707532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.707532:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677447.707533:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677447.707533:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677447.707534:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:31.0:1619677447.707536:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:18.0:1619677447.707536:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.707537:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.707537:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677447.707538:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.707539:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.707540:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.707542:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.707542:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:18.0:1619677447.707543:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:31.0:1619677447.707546:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:2.0:1619677447.707622:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.707625:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.707627:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.707628:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.707630:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.707632:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677447.707633:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.707635:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.707636:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.707637:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677447.707640:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.707642:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.707643:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.707644:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.707645:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.707646:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.707647:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.707648:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.707648:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.707652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.707659:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.707661:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.707662:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.707662:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.707664:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.707665:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.707670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.707677:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.707680:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.707680:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.707680:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.707681:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.707681:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:1.0:1619677447.707782:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.707785:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.707786:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.707787:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.707790:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.707791:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.707793:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.707794:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.707795:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.707796:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.707799:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.707801:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.707804:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.707805:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.707806:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.707808:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.707808:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.707809:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.707809:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.707821:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.707823:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.707824:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.707826:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.707827:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.707829:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.707858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.707900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.707908:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.707911:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:6.0:1619677447.707911:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:18.0:1619677447.707912:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.707912:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677447.707914:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.707915:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:18.0:1619677447.707915:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.707916:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.707918:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:6.0:1619677447.707918:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.2:1619677447.707919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:9.2:1619677447.707919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677447.707921:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677447.707926:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677447.707927:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.707930:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677447.707931:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:18.0:1619677447.707932:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.707932:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000400:00000200:30.0:1619677447.707933:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.707933:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677447.707934:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:2.0:1619677447.707934:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:30.0:1619677447.707936:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.707937:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677447.707938:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.707940:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.707941:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.707942:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677447.707945:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.707947:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.707949:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.707949:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.707950:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.707951:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.707952:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.707953:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.707954:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.2:1619677447.707955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:22.0:1619677447.707957:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.707960:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.707960:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.0:1619677447.707961:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:22.0:1619677447.707962:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.707963:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:5.0:1619677447.707963:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:22.0:1619677447.707964:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.707965:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.707966:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:22.0:1619677447.707966:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:20.0:1619677447.707966:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:31.0:1619677447.707967:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.707967:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.707968:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.707968:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.707969:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.707969:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677447.707970:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.707970:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:22.0:1619677447.707970:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.707971:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677447.707971:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.707972:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:5.0:1619677447.707972:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.707974:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.707975:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.707976:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.707978:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.707979:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:20.0:1619677447.707980:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.707981:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.707982:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.707982:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:22.0:1619677447.707983:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:20.0:1619677447.707983:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96724521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.707983:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.707984:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:20.0:1619677447.707985:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.707986:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.707986:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:20.0:1619677447.707986:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.707987:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677447.707987:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.707988:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.707990:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.707990:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (21)++
00000400:00000200:20.0:1619677447.707991:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677447.707991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (22)++
00000800:00000200:5.0:1619677447.707991:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (22)++
00000800:00000200:20.0:1619677447.707993:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.707994:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (24)--
00000800:00000200:5.0:1619677447.707994:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)++
00000800:00000200:20.0:1619677447.707995:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.707995:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (24)--
00000800:00000200:20.0:1619677447.707996:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.707996:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.707996:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.707997:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.707998:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.707999:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.708000:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.708000:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.708000:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (23)++
00000800:00000200:18.0:1619677447.708002:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (24)--
00000400:00000200:18.0:1619677447.708003:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.708003:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (23)++
00000800:00000200:9.2:1619677447.708004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.708004:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (24)--
00000400:00000200:18.0:1619677447.708005:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.708005:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.708006:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.708009:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.708010:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.708011:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677447.708011:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.708013:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.708014:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.708015:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.708016:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.708017:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.708082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.708087:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.708090:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677447.708092:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.708093:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.708095:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.708096:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.708098:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.708099:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.708100:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.708101:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.708104:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.708106:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.708109:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.708109:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.708110:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.708112:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.708112:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.708113:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.708114:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677447.708118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.708125:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.708128:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.708129:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.708129:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.708131:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.708133:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677447.708159:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.708162:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.708163:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.708166:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.708169:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.708170:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:24.0:1619677447.708171:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.708172:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.708173:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.708174:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:24.0:1619677447.708177:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.708179:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:24.0:1619677447.708181:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.708182:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:24.0:1619677447.708183:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:24.0:1619677447.708184:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:24.0:1619677447.708185:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.708185:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.708185:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:4.2:1619677447.708330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677447.708373:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.708376:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.708376:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.708378:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.708380:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.708381:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.708478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.708487:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.708489:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677447.708490:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:30.0:1619677447.708490:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.708491:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.708492:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.708493:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.708494:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.708495:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.708496:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.708497:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.708498:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.708500:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.708501:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.708502:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:30.0:1619677447.708502:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.708503:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.708504:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.708505:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.708506:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677447.708508:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.708509:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.708509:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:4.2:1619677447.708509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677447.708510:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.708511:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677447.708511:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.708512:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.708513:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (23)++
00000800:00000200:31.0:1619677447.708514:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.708515:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677447.708515:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (24)--
00000800:00000200:30.0:1619677447.708515:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.708516:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677447.708516:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.0:1619677447.708516:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677447.708518:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677447.708518:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.708518:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677447.708518:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:31.0:1619677447.708520:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:24.0:1619677447.708520:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677447.708521:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:20.2:1619677447.708521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:8.0:1619677447.708521:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.708524:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677447.708524:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.2:1619677447.708524:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.708525:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.708527:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:12.0:1619677447.708527:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.708528:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.708528:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.708530:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:12.0:1619677447.708530:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.708530:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.708531:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96724515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.708532:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:17.0:1619677447.708532:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:13.0:1619677447.708532:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:8.0:1619677447.708532:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.708533:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677447.708533:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:8.0:1619677447.708533:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.708534:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.708535:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:30.0:1619677447.708536:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:17.0:1619677447.708536:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.708536:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677447.708537:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:8.0:1619677447.708537:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.708538:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:17.0:1619677447.708539:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.708540:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.708542:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.0:1619677447.708542:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.708543:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.708544:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.708545:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.708546:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.708546:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.708547:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.708551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.708559:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.708562:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.708562:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.708563:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.708565:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.708567:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.708662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677447.708665:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677447.708668:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677447.708670:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.708671:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677447.708672:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.708674:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677447.708675:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.708677:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.708678:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.708679:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677447.708682:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.708684:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677447.708687:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.708688:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677447.708688:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677447.708690:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677447.708691:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677447.708691:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677447.708692:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:17.0:1619677447.708715:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.708717:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.708718:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.708720:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.708722:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.708724:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.708725:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.708727:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.708728:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.708729:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.708732:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.708733:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.708736:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.708737:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.708738:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.708740:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.708740:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.708741:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.708742:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.708754:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.708757:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.708758:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.708759:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.708760:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.708762:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.708763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.708769:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677447.708772:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677447.708774:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677447.708774:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.708775:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.708776:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.708776:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677447.708776:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:12.0:1619677447.708778:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:5.0:1619677447.708778:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.708781:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.708782:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.708784:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.708786:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.708787:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.708788:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.708792:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.708795:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.708798:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.708798:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.708799:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.708801:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.708801:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.708803:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.708804:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.708805:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:9.2:1619677447.708807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677447.708809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.708812:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.708814:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.708815:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677447.708815:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677447.708816:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677447.708818:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.708819:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.708820:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677447.708820:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:2.0:1619677447.708821:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.708823:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:2.0:1619677447.708823:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:13.0:1619677447.708824:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.708826:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.708826:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.708828:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.708892:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.708894:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.708897:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.708898:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.708899:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.708900:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.708902:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.708903:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.708904:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.708905:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.708908:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.708910:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.708912:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.708913:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.708914:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.708915:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.708916:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.708917:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.708918:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677447.708928:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.708931:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.708932:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.708934:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.708936:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.708938:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.708939:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96724509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.708939:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:24.0:1619677447.708941:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.708941:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.708942:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.708942:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.708942:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.708944:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.708946:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.708946:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.708948:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.708949:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.708950:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.708951:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.708951:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.708953:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.708953:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.708954:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.708954:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.708968:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.708970:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.708971:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.708972:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.708974:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.708975:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.708996:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f7600] (21)++
00000800:00000200:31.0:1619677447.708999:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.2.25@o2ib3
00000400:00000200:31.0:1619677447.709001:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.25@o2ib3 : REPLY - routed
00000800:00000200:30.0:1619677447.709003:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f7600] (22)--
00000800:00000200:31.0:1619677447.709004:0:19945:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000800:00000200:31.0:1619677447.709009:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a8677f7600] (21)++
00000800:00000200:31.0:1619677447.709011:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a8677f7600] (22)++
00000800:00000200:31.0:1619677447.709011:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a8677f7600] (23)--
00000800:00000200:31.0:1619677447.709012:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f7600] (22)--
00000800:00000200:30.0:1619677447.709025:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f7600] (22)++
00000800:00000200:30.0:1619677447.709027:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f7600] (23)--
00000800:00000200:30.0:1619677447.709028:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f7600] (22)--
00000800:00000200:6.0:1619677447.709078:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:4.2:1619677447.709080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.709081:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.709082:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.709084:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.709086:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.709088:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.709089:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.709091:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.709092:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.709094:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.709097:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.709100:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.709103:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.709104:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.709104:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.709106:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.709107:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.709107:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.709108:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.709112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.709120:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.709123:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.709124:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.709124:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.709126:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.709127:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:1.0:1619677447.709128:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677447.709129:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.709130:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.709132:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.709133:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.709134:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677447.709179:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.709181:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.709182:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.709184:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.709186:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.709188:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.709189:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.709190:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.709192:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.709192:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:17.0:1619677447.709193:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:12.0:1619677447.709194:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.709195:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.709196:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677447.709197:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.709198:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:13.0:1619677447.709198:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.709199:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677447.709201:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.709202:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.709202:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.709204:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.709204:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.709206:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.709207:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.709208:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.709217:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677447.709218:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677447.709220:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677447.709220:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:22.0:1619677447.709221:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.709221:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.709221:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.709222:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677447.709223:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.709226:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.709271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.709278:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.709281:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.709284:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.709285:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.709286:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.709287:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.709289:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.709290:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.709291:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.709292:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.709301:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.709302:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.709305:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.709306:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.709307:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677447.709307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:31.0:1619677447.709308:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.709309:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.709310:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677447.709310:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:31.0:1619677447.709311:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677447.709312:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.709315:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677447.709317:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.709317:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.709318:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677447.709320:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.709321:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.709322:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.709322:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:20.0:1619677447.709323:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677447.709325:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:13.0:1619677447.709325:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.709325:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.709327:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.709328:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:12.0:1619677447.709329:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.709330:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677447.709331:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.709331:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677447.709332:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677447.709334:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677447.709335:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.709336:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.709336:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.2:1619677447.709373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.709378:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.709381:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.709383:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.709384:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.709386:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.709387:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.709389:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.709390:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.709391:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.709392:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.709395:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.709397:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.709400:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.709401:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.709402:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.709404:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.709404:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.709405:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.709406:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.709416:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.709419:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.709419:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.709421:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.709422:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.709424:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.709464:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.709467:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.709467:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.709469:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.709472:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.709474:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.709475:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.709476:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.709477:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.709478:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.709481:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.709482:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.709484:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.709485:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.709486:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.709487:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.709488:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.709488:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.709489:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.709491:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f7600] (22)++
00000800:00000200:30.0:1619677447.709494:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.2.25@o2ib3
00000800:00000200:30.0:1619677447.709496:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a8677f7600] (23)--
00000800:00000200:31.0:1619677447.709497:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f7600] (22)++
00000400:00000200:30.0:1619677447.709497:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.709499:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.2.25@o2ib3
00000400:00000200:30.0:1619677447.709499:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677447.709501:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.709502:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.709502:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:30.0:1619677447.709503:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.709504:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:6.0:1619677447.709504:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.709505:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.709505:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.709508:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.709509:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.25@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : REPLY try# 0
00000800:00000200:6.0:1619677447.709510:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:30.0:1619677447.709511:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 1048576 bytes in 256 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.709512:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.25@o2ib3 : REPLY - routed
00000800:00000200:30.0:1619677447.709512:0:19944:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000800:00000200:31.0:1619677447.709516:0:19945:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000800:00000200:30.0:1619677447.709517:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.709518:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.709518:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.709520:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.709521:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a8677f7600] (23)++
00000800:00000200:31.0:1619677447.709522:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a8677f7600] (24)++
00000800:00000200:30.0:1619677447.709522:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a8677f7600] (24)--
00000800:00000200:30.0:1619677447.709522:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f7600] (23)--
00000800:00000200:22.0:1619677447.709522:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677447.709524:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a8677f7600] (23)++
00000800:00000200:30.0:1619677447.709524:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f7600] (23)--
00000800:00000200:22.0:1619677447.709524:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677447.709525:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a8677f7600] (23)--
00000800:00000200:31.0:1619677447.709525:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f7600] (22)--
00000800:00000200:6.0:1619677447.709525:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.709526:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.709527:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.709528:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:30.0:1619677447.709540:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f7600] (22)++
00000800:00000200:31.0:1619677447.709542:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.709542:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f7600] (23)--
00000800:00000200:30.0:1619677447.709543:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f7600] (22)--
00000800:00000200:31.0:1619677447.709544:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677447.709545:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:4.2:1619677447.709554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677447.709571:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.709574:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[3] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.709578:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677447.709579:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)++
00000800:00000200:4.2:1619677447.709593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677447.709594:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677447.709594:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.709596:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.709599:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.709599:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.709601:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.709603:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.709604:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677447.709613:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677447.709616:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.709617:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.709617:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.709618:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.709619:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677447.709663:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.709666:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.709667:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.709669:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.709671:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.709673:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.709674:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.709675:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.709676:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.709677:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.709681:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.709683:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.709685:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.709686:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.709687:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.709689:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.709689:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.709690:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.709690:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:31.0:1619677447.709909:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f7600] (22)++
00000800:00000200:31.0:1619677447.709911:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.2.25@o2ib3
00000800:00000200:31.0:1619677447.709913:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a8677f7600] (23)--
00000400:00000200:31.0:1619677447.709914:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.709915:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.709916:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724497] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.709916:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f7600] (22)--
00000400:00000200:31.0:1619677447.709917:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.709918:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.709919:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.709922:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.25@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : REPLY try# 0
00000800:00000200:31.0:1619677447.709924:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 1048576 bytes in 256 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.709925:0:19945:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000800:00000200:31.0:1619677447.709932:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.709933:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.709934:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.709935:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677447.709935:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a8677f7600] (21)++
00000800:00000200:31.0:1619677447.709936:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a8677f7600] (22)--
00000800:00000200:31.0:1619677447.709937:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f7600] (21)--
00000800:00000200:2.0:1619677447.709940:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.709943:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.709944:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.709946:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.709948:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.709950:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.709951:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96724496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.709952:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.709953:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.709954:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.709958:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.709959:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.709961:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.709962:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.709963:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.709964:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.709965:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.709966:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.709966:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677447.710039:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.710042:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.710043:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.710045:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.710047:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.710049:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.710050:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96724495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.710051:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.710052:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.710053:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:4.2:1619677447.710057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677447.710057:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.710058:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.710060:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.710061:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.710062:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.710063:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.710064:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.710064:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.710065:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.710095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.710102:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.710104:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.710105:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.710106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677447.710106:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.710107:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.710109:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.710114:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:6.0:1619677447.710114:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:9.2:1619677447.710115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:17.0:1619677447.710116:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.710117:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.710117:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.710118:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.710118:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.710118:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.710120:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.710121:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.710122:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:2.0:1619677447.710123:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.710124:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.710125:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.710127:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.710127:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.710127:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.710128:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.710158:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.710160:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.710161:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.710163:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.710165:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.710166:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.710167:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96724493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.710169:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.710170:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.710171:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.710174:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.710175:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.710178:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.710178:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.710179:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.710181:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.710181:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.710183:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.710184:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.710184:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:30.0:1619677447.710185:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (23)++
00000800:00000200:30.0:1619677447.710188:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (24)--
00000400:00000200:30.0:1619677447.710189:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.710191:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: REPLY: OK
00000800:00000200:31.0:1619677447.710192:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (23)++
00000800:00000200:31.0:1619677447.710193:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (24)--
00000800:00000200:30.0:1619677447.710194:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (23)--
00000800:00000200:22.0:1619677447.710194:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:31.0:1619677447.710195:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:2.0:1619677447.710195:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:22.0:1619677447.710196:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.710197:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677447.710197:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:22.0:1619677447.710198:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.710198:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.710199:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:22.0:1619677447.710200:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.710202:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.710211:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.710214:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.710214:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.710215:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.710216:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.710216:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677447.710241:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.710243:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[1] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.710247:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677447.710248:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:13.0:1619677447.710265:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677447.710268:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677447.710268:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:13.0:1619677447.710268:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.710268:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.710270:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.710272:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.710273:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.710277:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.710280:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.710280:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.710282:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.710284:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:22.0:1619677447.710285:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.710286:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:22.0:1619677447.710287:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:12.0:1619677447.710287:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:22.0:1619677447.710288:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.710288:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.710288:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:22.0:1619677447.710289:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.710290:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:22.0:1619677447.710291:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.710291:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.710301:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.710302:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.710306:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.710307:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.710308:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.710310:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.710310:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.710311:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.710312:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.710313:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.710323:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.710325:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.710326:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.710327:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677447.710409:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.710412:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.710413:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.710414:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.710416:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.710418:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.710495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.710498:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.710501:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.710503:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.710504:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.710506:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.710507:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.710509:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.710510:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.710511:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.710512:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.710515:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.710517:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.710520:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.710520:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.710521:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.710523:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.710524:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.710525:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.710525:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677447.710537:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.710540:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.710541:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:4.2:1619677447.710541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677447.710543:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.710545:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.710546:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.710547:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96724489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.710548:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.710549:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.710550:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.710554:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.710557:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.710560:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.710561:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.710562:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.710564:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.710564:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.710565:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.710566:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.710585:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.710588:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.710588:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.710590:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.710591:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.710593:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:9.2:1619677447.710632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.710639:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.710642:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.710643:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.710643:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.710645:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.710648:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677447.710650:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.710653:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.710654:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.710654:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:18.0:1619677447.710655:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:22.0:1619677447.710656:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.710658:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.710659:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:22.0:1619677447.710660:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:18.0:1619677447.710662:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.710664:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.710666:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.710668:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.710670:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.710671:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.710672:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.710675:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.710677:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.710679:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.710680:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.710681:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.710682:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.710683:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.710684:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.710684:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677447.710740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.710749:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.710752:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.710753:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.710754:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.710755:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.710757:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677447.710760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.710767:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.710770:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.710771:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.710772:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.710772:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.710773:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677447.710791:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.710794:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.710795:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.710796:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: REPLY: OK
00000800:00000200:30.0:1619677447.710797:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.710799:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.710840:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.710843:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.710845:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.710847:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.710847:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.710849:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677447.710850:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.710852:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.710853:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.710854:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677447.710857:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.710858:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.710862:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.710863:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677447.710863:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677447.710865:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.710866:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.710867:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.710867:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.710879:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.710882:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.710882:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.710883:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.710885:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.710888:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.710975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.2:1619677447.710980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:24.0:1619677447.710982:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.710984:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.710985:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.710987:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.710988:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:24.0:1619677447.710989:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.710990:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:24.0:1619677447.710991:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.710992:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.710993:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.710994:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:24.0:1619677447.710994:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.710995:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.710996:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.710996:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677447.710998:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677447.710999:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.710999:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.711000:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.711001:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.711001:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:31.0:1619677447.711002:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.711004:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.711005:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.711005:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.711006:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677447.711007:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.711007:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.711008:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.711009:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.711010:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:31.0:1619677447.711011:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.711011:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677447.711012:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677447.711012:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.2:1619677447.711013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677447.711014:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677447.711015:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.711015:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.711016:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:4.2:1619677447.711016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.2:1619677447.711020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677447.711021:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.711024:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.711025:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.711025:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.711027:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.711027:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:20.0:1619677447.711028:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677447.711030:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.0:1619677447.711030:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:20.0:1619677447.711031:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:8.0:1619677447.711031:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:20.0:1619677447.711032:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.711032:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:20.0:1619677447.711034:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.2:1619677447.711035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677447.711035:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.711037:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.711042:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.711045:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.711046:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.711046:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.711047:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.711048:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.711078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.711124:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.711127:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.711127:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.711128:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.711130:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.711133:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.711148:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.711151:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.711152:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.711154:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.711157:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.711158:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.711160:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.711161:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.711162:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.711163:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.711167:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.711169:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.711172:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.711172:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.711173:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.711175:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.711175:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.711177:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.711178:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.711178:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.711181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.711187:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:1.0:1619677447.711189:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677447.711190:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677447.711190:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:24.0:1619677447.711191:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.711191:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.711191:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677447.711193:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.711193:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677447.711196:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677447.711197:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.711199:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.711200:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.711202:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.711205:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.711206:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.711208:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.711209:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.711210:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.711211:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.711214:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.711216:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.711219:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.711220:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.711220:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.711222:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.711223:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.711223:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.711224:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677447.711240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677447.711242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.711247:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.711249:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.711250:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.711252:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.711252:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.711252:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.711254:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.711255:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.711257:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.711258:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.711260:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.711261:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.711262:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.711263:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.711264:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.711265:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:5.0:1619677447.711266:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.711267:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.711271:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.711272:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.711272:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.711274:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.711275:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.711276:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.711276:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677447.711287:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.711290:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.711291:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.711294:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677447.711300:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.711303:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.711515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:2.0:1619677447.711538:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.711541:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.711542:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.711544:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.711547:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.711548:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.711550:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.711551:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.711552:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.711553:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.711556:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.711558:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.2:1619677447.711559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.711561:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.711561:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.711562:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.711564:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.711564:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.711565:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.711566:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677447.711567:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.711569:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.711570:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.711570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:20.0:1619677447.711572:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.711574:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.711575:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.711576:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.711578:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.711579:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.711581:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.711581:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.711583:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677447.711630:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.711633:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.711636:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.711637:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.711638:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.711640:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677447.711641:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.711643:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.711644:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.711645:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677447.711648:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.711650:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.711651:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:31.0:1619677447.711654:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.711654:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:31.0:1619677447.711655:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677447.711655:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677447.711655:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.711656:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.711657:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.711658:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.711659:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.711659:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:8.0:1619677447.711659:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.711661:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.711662:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96724475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.711663:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.711664:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.711665:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.711668:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.711669:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:22.0:1619677447.711671:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677447.711672:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.711673:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:22.0:1619677447.711674:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:8.0:1619677447.711674:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:22.0:1619677447.711675:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.711675:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:8.0:1619677447.711676:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.711676:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.711677:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:22.0:1619677447.711678:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.711678:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.711680:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.2:1619677447.711681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.711689:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:6.0:1619677447.711689:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.711692:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:6.0:1619677447.711692:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.711693:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:20.0:1619677447.711693:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.711693:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000400:00000200:20.0:1619677447.711694:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.711694:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.711694:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.711695:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.711697:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.711740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.711745:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.711748:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.711750:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.711751:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.711753:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.711754:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.711756:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.711757:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.711758:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.711759:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.711762:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.711763:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.711766:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.711767:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.711767:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.711769:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.711769:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.711771:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.711772:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.711773:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.711778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.711781:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:22.0:1619677447.711782:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.711783:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.711783:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.711784:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.711785:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.711785:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.711786:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.711787:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.711790:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.711820:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.711822:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.711823:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.711825:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.711828:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.711830:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677447.711831:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.711832:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.711833:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.711834:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677447.711837:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.711839:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:4.2:1619677447.711841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.711842:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.711842:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677447.711843:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677447.711845:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.711845:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:4.2:1619677447.711845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:17.0:1619677447.711846:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.711846:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.711851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.711855:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.711857:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.711858:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:24.0:1619677447.711860:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.711860:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.711861:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.711863:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.711863:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.711866:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.711868:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96724471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.711869:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.711870:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.711871:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.711872:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.711873:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.711874:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000400:00000200:24.0:1619677447.711875:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.711876:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.711876:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.711877:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.711877:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.711878:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.711880:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.711881:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.2:1619677447.711881:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.711882:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.711883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677447.711884:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.711885:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.711885:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677447.711886:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.711886:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677447.711890:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.2:1619677447.711890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.711892:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.711893:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.711895:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.711896:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.711898:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.711898:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.711901:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.711902:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.2:1619677447.711902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000400:00000200:1.0:1619677447.711902:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.711904:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.711907:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.711908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.711910:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.711912:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.711913:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.711914:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.711914:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677447.711915:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.711916:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.711918:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.711919:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.711919:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.711920:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.711921:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:13.0:1619677447.712104:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677447.712107:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677447.712109:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677447.712111:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.712111:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677447.712113:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:13.0:1619677447.712114:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.712116:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677447.712117:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677447.712118:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:13.0:1619677447.712121:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.712123:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677447.712126:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.712126:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:13.0:1619677447.712127:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:13.0:1619677447.712129:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:13.0:1619677447.712130:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677447.712131:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677447.712131:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677447.712135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.712142:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.712145:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.712146:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.712146:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.712148:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.712151:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677447.712275:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.712278:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.712279:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.712282:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.712284:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.712285:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.712287:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.712288:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.712289:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.712290:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.712298:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.712300:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.712302:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.712303:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.712303:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.712305:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.712306:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.712306:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.712306:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.712312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.712320:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.712322:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.712323:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.712323:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.712326:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.712327:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.712369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.712374:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.712377:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.712379:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.712380:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.712382:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.712383:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.712384:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.712386:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.712387:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.712388:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.712391:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.712393:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.712395:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.712396:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.712397:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.712399:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.712400:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.712401:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.712402:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.712403:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:9.2:1619677447.712406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677447.712408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:1.0:1619677447.712410:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.712412:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.712413:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677447.712413:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:2.0:1619677447.712414:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:1.0:1619677447.712415:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.712416:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.712417:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.712417:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:2.0:1619677447.712417:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.712418:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.712418:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.712418:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.712419:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.712420:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.712420:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.712422:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.712423:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:1.0:1619677447.712423:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.712424:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.712427:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.712429:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.712430:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.712431:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.712432:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.712434:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.712434:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.712435:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.712435:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.712439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.712447:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.712449:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.712449:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.712450:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.712451:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.712453:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.712475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.712481:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677447.712483:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677447.712483:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:22.0:1619677447.712486:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677447.712486:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677447.712486:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:5.0:1619677447.712486:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.712488:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.712489:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.712491:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.712491:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677447.712492:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.712493:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.712494:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.712495:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.712496:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.712497:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.712498:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.712499:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:5.0:1619677447.712499:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:30.0:1619677447.712500:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.712503:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:5.0:1619677447.712504:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.712505:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.712507:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.712507:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677447.712508:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677447.712509:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:30.0:1619677447.712510:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.712510:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.712510:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:4.2:1619677447.712510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.712511:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677447.712512:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677447.712513:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:6.0:1619677447.712513:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:5.0:1619677447.712513:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:30.0:1619677447.712514:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96724464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.712514:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:5.0:1619677447.712514:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677447.712515:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.712515:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677447.712516:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.712516:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677447.712516:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:30.0:1619677447.712517:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.712517:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:30.0:1619677447.712518:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.712518:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.712519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:30.0:1619677447.712522:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.712524:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677447.712527:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.712527:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.712527:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:30.0:1619677447.712528:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.712529:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:30.0:1619677447.712530:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.712530:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677447.712530:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.712531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:17.0:1619677447.712531:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677447.712532:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.712533:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:1.0:1619677447.712533:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.712534:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677447.712536:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:31.0:1619677447.712547:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.712549:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.712551:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.712551:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677447.712553:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.712556:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:9.2:1619677447.712556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:22.0:1619677447.712557:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.712557:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.712560:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.712563:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677447.712565:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.712568:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.712569:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.712569:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.712571:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.712574:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677447.712581:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.712583:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.712584:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.712586:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677447.712588:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.712589:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.712601:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.712603:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.712604:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.712604:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.712605:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.712606:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.712728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.712735:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.712738:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.712740:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.712741:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.712744:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.712746:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.712747:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.712748:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.712749:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.712750:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.712753:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677447.712754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.0:1619677447.712754:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.712757:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.712758:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.712758:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.712760:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.712761:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.712761:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.712761:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:31.0:1619677447.712762:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.712764:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.712766:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677447.712766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677447.712769:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.712769:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.712771:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.712772:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.712773:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.712774:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.712774:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:31.0:1619677447.712775:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677447.712777:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.712777:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:31.0:1619677447.712778:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.712778:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.712780:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:17.0:1619677447.712780:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.712783:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:31.0:1619677447.712784:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.712784:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.712785:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.712787:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.712788:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.712789:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.712789:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.712793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.712801:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.712804:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.712805:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.712807:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.712807:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.712809:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677447.712815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677447.712822:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.712825:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.712826:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.712827:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.712828:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.712829:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677447.712894:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.712897:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.712898:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.712900:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.712902:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.712904:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.712905:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.712906:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.712907:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.712908:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.712912:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.712914:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.712917:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.712918:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.712919:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.712921:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.712921:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.712922:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.712922:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.712933:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.712936:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.712937:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.712939:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.712939:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.712941:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.713081:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.713084:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.713086:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.713088:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.713089:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.713090:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.713091:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.713093:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.713094:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.713095:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.713098:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.713100:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.713103:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.713103:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.713104:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.713105:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.713106:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.713107:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.713107:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.713117:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.713119:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.713120:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.713122:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.713123:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.713124:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.713157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:4.2:1619677447.713163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.713164:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.713167:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.713168:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.713170:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.713172:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.713174:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.713176:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.713178:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.713179:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.713180:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.713184:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.713186:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.713189:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.713189:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.713190:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.713192:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.713192:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.713193:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.713193:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.713198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:9.2:1619677447.713202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677447.713205:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.713208:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.713209:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.713209:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677447.713209:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.713212:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.713212:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.713213:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.713214:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677447.713215:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:2.0:1619677447.713216:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.713219:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.713318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.713326:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.713329:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.713331:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.713333:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.713334:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.713335:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.713336:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.713338:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.713339:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.713340:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.713342:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:31.0:1619677447.713343:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.713345:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.713345:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.713345:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.713347:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.713349:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.713349:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:18.0:1619677447.713349:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.713350:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:18.0:1619677447.713351:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677447.713352:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.713352:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:18.0:1619677447.713352:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.713354:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:18.0:1619677447.713354:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.713355:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000400:00000200:18.0:1619677447.713355:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.713356:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:18.0:1619677447.713356:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.713359:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677447.713359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.713361:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.2:1619677447.713362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677447.713364:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.713365:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.713365:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.713367:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.713367:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677447.713368:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.713369:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:30.0:1619677447.713370:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:18.0:1619677447.713370:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:5.0:1619677447.713370:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:18.0:1619677447.713371:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:5.0:1619677447.713371:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.713373:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.713373:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677447.713373:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.713374:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:5.0:1619677447.713374:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.713379:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677447.713381:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677447.713381:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:5.0:1619677447.713381:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677447.713382:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677447.713382:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:13.0:1619677447.713384:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:6.0:1619677447.713384:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.713384:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.713385:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:13.0:1619677447.713386:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.713386:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:6.0:1619677447.713387:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.713388:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.713388:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:6.0:1619677447.713388:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.713390:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.713391:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:6.0:1619677447.713391:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.713392:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.713393:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.713397:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.713399:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.713402:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.713403:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.713403:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.713405:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.713406:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.713406:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.713407:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677447.713521:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.713524:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.713524:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.713527:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.713529:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.713530:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.713532:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.713533:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.713534:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.713535:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.713539:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677447.713541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677447.713541:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677447.713543:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677447.713544:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.713545:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.713545:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677447.713546:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677447.713547:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:20.0:1619677447.713548:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.713548:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.713549:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677447.713549:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.713550:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000400:00000200:20.0:1619677447.713551:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.713551:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677447.713552:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.713553:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.713555:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.713556:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.713557:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:12.0:1619677447.713559:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:20.0:1619677447.713560:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.713561:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:20.0:1619677447.713562:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:12.0:1619677447.713562:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:17.0:1619677447.713563:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:12.0:1619677447.713563:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.713564:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.713564:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.713564:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677447.713565:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:13.0:1619677447.713565:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.713565:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.713566:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.713567:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.713568:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677447.713568:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677447.713569:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.713569:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.713580:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.713583:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.713584:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.713584:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.713586:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.713588:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.713599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.713647:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.713650:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.713651:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.713651:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.713654:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.713656:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.713669:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.713672:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.713673:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.713673:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.713675:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.713677:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.713703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.713714:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.713716:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.713718:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.713719:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.713720:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.713721:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.713723:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.713723:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.713724:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.713726:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.713727:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.713727:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.713730:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.713730:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.713732:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677447.713732:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.713733:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.713734:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.713735:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.713735:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:31.0:1619677447.713736:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.713738:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:30.0:1619677447.713739:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.713739:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:31.0:1619677447.713740:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.713740:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.713741:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:8.0:1619677447.713741:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.713742:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.713742:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.713743:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.713744:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.713744:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:8.0:1619677447.713744:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.713745:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.713745:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.713745:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:8.0:1619677447.713745:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:31.0:1619677447.713746:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.713747:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.713747:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.713748:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.2:1619677447.713748:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.713749:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.713749:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:8.0:1619677447.713751:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96724446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.713753:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.713754:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.713754:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.713757:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.713757:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:13.0:1619677447.713759:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677447.713759:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.713759:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.713760:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.713761:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677447.713762:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:13.0:1619677447.713762:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.713762:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.713763:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:5.0:1619677447.713763:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:13.0:1619677447.713764:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.713764:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.713765:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.713766:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.713766:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677447.713766:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:13.0:1619677447.713767:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.713768:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.713769:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.713770:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677447.713772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.713780:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677447.713782:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:18.0:1619677447.713783:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:24.0:1619677447.713784:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:18.0:1619677447.713784:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.713784:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.713785:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.713786:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:18.0:1619677447.713786:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.713790:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.713791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.713797:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.713800:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.713801:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.713801:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.713802:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.713803:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.713879:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.713881:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.713882:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.713884:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.713886:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.713887:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677447.713888:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.713890:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.713890:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.713892:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677447.713895:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.713897:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.713898:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.713899:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.713900:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.713901:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.713901:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.713902:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.713903:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.713907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.713914:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.713916:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.713916:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.713918:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.713918:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.713920:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:17.0:1619677447.713998:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.714000:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.714001:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.714003:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.714006:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.714008:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.714009:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.714010:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.714011:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.714012:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.714016:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.714018:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.714021:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.714021:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.714022:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.714024:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.714024:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.714025:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.714025:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.714037:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.714039:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.714040:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.714042:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.714043:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.714045:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677447.714054:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677447.714057:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.714058:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.714059:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.714059:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.714061:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.714182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.714190:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.714192:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.714195:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.714196:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.714197:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.714198:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.714199:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.714201:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.714202:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.714203:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.714206:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.714208:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.714210:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.714211:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.714212:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.714213:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.714214:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:4.2:1619677447.714214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677447.714215:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.714215:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.0:1619677447.714220:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.714223:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.714224:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:12.0:1619677447.714225:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:8.0:1619677447.714226:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.714228:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.714228:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.714229:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.714230:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.714230:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:13.0:1619677447.714232:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:8.0:1619677447.714232:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96724442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.714233:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:8.0:1619677447.714233:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.714234:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.714235:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.714238:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.714240:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.714243:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.714243:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.714244:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.714245:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.714246:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.714247:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.714248:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.714249:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.714254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.2:1619677447.714260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.714263:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.714264:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.714265:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.714266:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.714268:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.714271:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.714272:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.714272:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.714275:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.714277:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.714283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.714290:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.714293:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.714293:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.714299:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.714300:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.714301:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.714301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.714343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.714349:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.714352:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.714353:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.714353:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.714355:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.714358:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.714367:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.714370:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.714370:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.714371:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.714372:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.714373:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.714388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.714396:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.714399:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.714399:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.714400:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.714402:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.714405:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.714560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.714563:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.714565:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.714567:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.714569:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.714570:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.714571:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.714573:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.714574:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.714575:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.714576:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.714579:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.714580:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.0:1619677447.714581:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.714583:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:20.0:1619677447.714584:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.714584:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.0:1619677447.714585:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.714586:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:2.0:1619677447.714586:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.714588:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.714588:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:2.0:1619677447.714588:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.714589:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.714590:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:2.0:1619677447.714590:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677447.714591:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.714593:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.714594:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.714595:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677447.714598:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.714600:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.714601:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.714603:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.714604:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677447.714604:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:13.0:1619677447.714605:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.714605:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.714606:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:13.0:1619677447.714607:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.714607:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.714607:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:12.0:1619677447.714608:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.714608:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:17.0:1619677447.714609:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677447.714609:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:1.0:1619677447.714609:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:13.0:1619677447.714610:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677447.714613:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.714616:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.714618:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:12.0:1619677447.714619:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677447.714619:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.714621:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:1.0:1619677447.714621:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.714622:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.714622:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.714622:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677447.714623:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.714623:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.714623:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.714624:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677447.714625:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.714625:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677447.714625:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.714626:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:1.0:1619677447.714630:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.714632:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.714636:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.714636:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677447.714637:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.714637:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.714638:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.714639:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.714639:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.714640:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677447.714641:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.714641:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.714642:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677447.714643:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (25)--
00000800:00000200:13.0:1619677447.714644:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677447.714649:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.714651:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.714652:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.714653:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.714653:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.714654:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.714655:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:12.0:1619677447.714655:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.714657:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.714657:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.714658:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.714659:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.714660:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.714703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.714707:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.714710:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.714711:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.714714:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.714716:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.714718:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.714720:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.714722:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.714723:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.714724:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.714728:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.714731:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.714734:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.714735:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.714736:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.714738:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.714738:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.714740:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.714742:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.714742:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:31.0:1619677447.714751:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:22.0:1619677447.714753:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:31.0:1619677447.714754:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (24)--
00000400:00000200:31.0:1619677447.714754:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.714755:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.714755:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:31.0:1619677447.714756:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.714756:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677447.714759:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.714760:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.714761:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.714762:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677447.714762:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.714762:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.714765:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.714765:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.714767:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:31.0:1619677447.714768:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.714768:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:31.0:1619677447.714770:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:30.0:1619677447.714770:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.714771:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.714772:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.714773:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.714774:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677447.714778:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.714780:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677447.714783:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.714784:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677447.714784:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677447.714786:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677447.714786:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.714787:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.714788:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.714799:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.714802:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.714803:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.714803:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.714806:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.714809:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.714820:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.714822:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.714823:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.714824:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.714824:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.714825:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677447.714826:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.714828:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.714829:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.714831:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.714834:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.714835:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.714836:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.714838:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.714839:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.714840:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.714843:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.714845:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.714847:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.714847:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.714848:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.714850:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.714850:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.2:1619677447.714851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677447.714851:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.714851:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.714856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677447.714859:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.714862:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:2.0:1619677447.714863:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:30.0:1619677447.714864:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.714865:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:31.0:1619677447.714866:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.714866:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.714866:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.714867:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:30.0:1619677447.714868:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677447.714868:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000400:00000200:30.0:1619677447.714870:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.714870:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:30.0:1619677447.714871:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.714872:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.714873:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.714876:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.714878:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.714881:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.714882:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.714882:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.714884:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.714885:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.714885:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.714886:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.714896:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.714899:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.714900:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.714901:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.714903:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.714904:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677447.715000:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.715002:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.715003:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.715005:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.715008:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.715009:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.715011:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.715012:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.715013:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.715014:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.715017:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.715019:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.715022:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.715023:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.715023:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.715025:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.715026:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.715026:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.715027:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.715038:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.715040:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.715041:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.715043:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.715044:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.715045:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.715073:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.715075:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.715076:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.715078:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.715079:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.715080:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.715270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677447.715317:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.715320:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.715321:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.715322:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.715324:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.715325:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677447.715333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.715340:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.715343:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.715344:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.715346:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.715349:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.715350:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.715352:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.715353:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.715354:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.715355:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.715357:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.715359:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.715362:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.715362:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.715363:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.715365:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.715365:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.715366:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.715366:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677447.715370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.715376:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.715379:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:8.2:1619677447.715381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677447.715381:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.715382:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.715384:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.715386:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677447.715387:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724426] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.715388:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.715389:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.715389:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:5.0:1619677447.715390:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:1.0:1619677447.715392:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.715393:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.715393:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.715394:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.715396:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:1.0:1619677447.715396:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.715398:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.715399:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:1.0:1619677447.715399:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677447.715400:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677447.715402:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677447.715402:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.715404:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.715405:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.715406:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:9.2:1619677447.715408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.2:1619677447.715411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.715413:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.715415:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.715416:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677447.715416:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:20.0:1619677447.715417:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.715419:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.715420:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.715420:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.715422:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.715425:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.715434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.715441:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.715444:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.715444:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.715445:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.715446:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.715446:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677447.715449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.715452:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.715454:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.715457:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.715458:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.715459:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.715461:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:20.0:1619677447.715462:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.715463:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.715464:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.715465:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:20.0:1619677447.715469:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.715471:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:18.0:1619677447.715473:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.0:1619677447.715474:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.715475:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:20.0:1619677447.715476:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:18.0:1619677447.715476:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.715476:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.0:1619677447.715477:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:20.0:1619677447.715478:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:18.0:1619677447.715478:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.715479:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.715479:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:18.0:1619677447.715481:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.715482:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.715483:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677447.715483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:18.0:1619677447.715485:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.715486:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.715487:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.715489:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.715490:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:18.0:1619677447.715491:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.715492:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:18.0:1619677447.715493:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.715493:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.715493:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.715494:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.715494:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:24.0:1619677447.715496:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.715496:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.715496:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.715497:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.715498:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:24.0:1619677447.715499:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:18.0:1619677447.715499:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677447.715503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.715507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:2.0:1619677447.715511:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.715512:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.715512:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.715513:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:8.0:1619677447.715513:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000400:00000200:2.0:1619677447.715513:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.715514:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.715515:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.715515:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.715516:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.715516:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:18.0:1619677447.715517:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.0:1619677447.715517:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.715518:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:2.0:1619677447.715518:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.0:1619677447.715519:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:17.0:1619677447.715550:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.715553:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.715553:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.715555:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.715558:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.715560:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.715561:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724423] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.715563:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.715563:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.715565:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.715568:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.715570:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.715573:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.715574:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.715574:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.715576:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.715577:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.715578:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.715579:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.715579:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677447.715589:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677447.715590:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677447.715592:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677447.715592:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.715593:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677447.715593:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.715593:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677447.715595:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.715598:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.715598:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.715601:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.715602:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.715603:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.715604:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.715605:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.715624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.715665:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.715667:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.715668:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.715669:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.715671:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.715672:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.715672:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.715674:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.715675:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.715676:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.715678:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.715681:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677447.715681:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677447.715684:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.715686:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.715688:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.715688:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.715690:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677447.715691:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.715693:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.715694:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.715694:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677447.715698:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.715699:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.715702:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.715703:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677447.715703:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.715705:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677447.715706:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.715707:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.715709:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.715709:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677447.715712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.715720:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:30.0:1619677447.715723:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:1.0:1619677447.715723:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.715724:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.715724:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.715725:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.715726:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:1.0:1619677447.715726:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.715727:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677447.715730:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677447.715744:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.715747:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.715748:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.715750:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.715752:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.715753:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.715755:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.715756:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.715757:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.715758:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.715761:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.715763:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.715764:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.715765:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.715766:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.715767:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.715768:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.715768:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.715769:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.715774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.715780:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.715783:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.715783:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.715784:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.715785:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.715786:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.715797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.715804:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.715806:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.715807:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.715808:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.715808:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.715809:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.715881:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.715884:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.715884:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.715886:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.715889:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.715891:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.715892:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.715893:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.715894:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.715895:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.715898:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.715900:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.715902:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.715903:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.715903:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.715905:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.715906:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.715907:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.715907:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.715911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.715919:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.715921:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.715922:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.715923:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.715925:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.715927:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.715939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.715947:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.715950:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.715952:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.715954:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.715955:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.715956:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677447.715958:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.715959:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.715960:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.715961:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677447.715965:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.715967:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677447.715970:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.715970:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.715971:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677447.715972:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677447.715973:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.715974:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.715975:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:1.0:1619677447.715975:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:30.0:1619677447.715976:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:17.0:1619677447.715976:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677447.715977:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:1.0:1619677447.715977:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677447.715979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:1.0:1619677447.715980:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.715981:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.2:1619677447.715982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:1.0:1619677447.715982:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724416] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.715984:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.715984:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.715985:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.715986:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677447.715988:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:24.0:1619677447.715989:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.715989:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:1.0:1619677447.715989:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.715991:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:5.0:1619677447.715991:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:1.0:1619677447.715991:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.715992:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.715992:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.715993:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.715994:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677447.715994:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.715994:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.715994:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.715995:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:24.0:1619677447.715995:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.715995:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.715995:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:24.0:1619677447.715997:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:1.0:1619677447.715997:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.715997:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.715998:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.715998:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677447.715999:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96724415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.715999:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677447.716000:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.716001:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.716003:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:8.2:1619677447.716003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:24.0:1619677447.716007:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.716009:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677447.716010:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:4.2:1619677447.716011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.716012:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.716012:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677447.716013:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.716013:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:17.0:1619677447.716013:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.716013:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.716015:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.716015:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:17.0:1619677447.716015:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.716016:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:17.0:1619677447.716016:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:24.0:1619677447.716017:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677447.716021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.2:1619677447.716025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677447.716028:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.716030:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.716031:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.716032:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677447.716034:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.716034:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677447.716035:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.716036:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.716037:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.716037:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.716037:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.716038:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.716054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677447.716057:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.716059:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.716060:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.716062:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.716063:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.716065:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.716082:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.716085:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.716086:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.716086:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.716087:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.716088:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.716183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.716188:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.716191:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.716193:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.716194:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.716196:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.716197:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677447.716198:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.716200:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.716201:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.716202:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677447.716209:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.716211:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.716211:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677447.716213:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.716214:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:5.0:1619677447.716214:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:1.0:1619677447.716214:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677447.716215:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:17.0:1619677447.716216:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.716216:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677447.716217:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:17.0:1619677447.716218:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.716218:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.716218:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677447.716219:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.716220:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.716222:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677447.716222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:17.0:1619677447.716223:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.716224:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.716226:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.716228:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.716229:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.716230:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.716230:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:17.0:1619677447.716231:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.716232:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.716232:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677447.716232:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:17.0:1619677447.716233:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:2.0:1619677447.716233:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.716234:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:17.0:1619677447.716234:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:2.0:1619677447.716235:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.2:1619677447.716238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.716238:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:1.0:1619677447.716245:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.716247:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.716248:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.716249:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.716250:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.716251:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677447.716325:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.716328:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.716330:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.716333:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.716333:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.716335:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677447.716336:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724411] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.716337:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.716338:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.716339:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677447.716342:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.716344:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.716346:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.716347:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677447.716348:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.716350:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677447.716350:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.716351:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.716352:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677447.716356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.716364:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.716367:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.716368:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.716368:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.716370:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.716373:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.716472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:18.0:1619677447.716476:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.716479:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.716480:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.716482:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.716483:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.716485:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.716487:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.716489:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.716490:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.716491:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.716495:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.716498:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.716501:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.716501:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.716502:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.716504:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.716504:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.716505:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.716506:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677447.716510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.2:1619677447.716515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.716517:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.2:1619677447.716520:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:5.0:1619677447.716523:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:1.0:1619677447.716523:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:20.0:1619677447.716524:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (22)--
00000400:00000200:20.0:1619677447.716525:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.716526:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:20.0:1619677447.716527:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.716527:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.716527:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.716530:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:1.0:1619677447.716530:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:1.0:1619677447.716533:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.716535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.716542:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.716545:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.716545:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.716546:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.716547:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.716548:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.716581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.716613:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.716616:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.716617:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677447.716618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:1.0:1619677447.716619:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.716621:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.716622:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.716623:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.716625:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.716626:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:1.0:1619677447.716626:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.716627:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.716628:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.716629:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.716630:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.716631:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.716631:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.716632:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.716634:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.716634:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.716635:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.2:1619677447.716636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:1.0:1619677447.716636:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.716637:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.716638:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.716639:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.716639:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677447.716643:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.2:1619677447.716643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.716645:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.716646:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.716647:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.716647:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.716649:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.716650:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.716652:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.716653:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.716654:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.716655:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.716657:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.716745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.716750:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.716753:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.716755:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.716755:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.716757:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.716759:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:5.0:1619677447.716760:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724405] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.716762:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.716763:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.716764:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:5.0:1619677447.716767:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.716769:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:5.0:1619677447.716772:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.716773:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677447.716773:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.0:1619677447.716775:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677447.716776:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.716776:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.716777:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677447.716781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.716788:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.716790:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.716791:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.716792:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.716794:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.716797:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.716814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.716821:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.716824:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.716824:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.716826:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.716828:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.716830:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.716831:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.716832:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.716833:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.716834:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.716838:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.716839:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.716842:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.716843:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.716844:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.716845:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.716846:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.716847:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.716847:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.716851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.716859:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:20.2:1619677447.716860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677447.716862:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:20.0:1619677447.716863:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677447.716863:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.716863:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.716865:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:1.0:1619677447.716865:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.716868:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.716868:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.0:1619677447.716869:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.716870:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.716872:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.716873:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.716875:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.716876:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.716877:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.716880:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.716882:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.716885:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.716886:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.716886:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.716888:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.716889:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.716890:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.716890:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677447.716902:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.716905:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.716905:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.716907:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.716908:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.716910:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:2.0:1619677447.716930:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.716934:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.716935:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.716936:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.716939:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.716940:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.716941:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.716943:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.716944:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.716945:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.716948:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.716950:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.716951:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.716952:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.716953:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.716954:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.716955:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.716956:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.716956:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.716961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.716968:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.716970:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.716971:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.716971:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.716973:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.716974:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.0:1619677447.716976:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.716979:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.716980:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.716981:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.716981:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:8.0:1619677447.716984:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.716984:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.716985:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:8.0:1619677447.716986:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:6.0:1619677447.716987:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.716988:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.716989:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.716990:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.716990:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.716991:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.716992:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677447.716993:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677447.716993:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.716995:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:6.0:1619677447.716995:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.716996:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.716997:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677447.716998:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.716998:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.716999:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.717001:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.717001:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:31.0:1619677447.717002:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:6.0:1619677447.717002:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.717005:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96724399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.717005:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.717005:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:31.0:1619677447.717006:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.717006:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.717006:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677447.717008:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.717008:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:6.0:1619677447.717008:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.717009:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:8.0:1619677447.717009:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:6.0:1619677447.717009:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.717009:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.717010:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.717010:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.717011:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.717011:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.717012:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:31.0:1619677447.717013:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.717013:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.717014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677447.717015:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677447.717017:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.717018:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:8.2:1619677447.717018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.717019:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677447.717020:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.717021:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.717022:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.717022:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677447.717023:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.717025:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:17.0:1619677447.717026:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:5.0:1619677447.717026:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.717028:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677447.717028:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:5.0:1619677447.717028:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.717029:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.717029:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.717031:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.717031:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.717033:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.717036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.717043:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.717046:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.717047:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.717047:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.717049:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.717049:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.717403:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.717406:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.717407:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.717409:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.717410:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.717412:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.717413:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.717414:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.717415:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.717416:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.717419:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.717421:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.717423:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.717424:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.717425:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.717426:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.717427:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.717428:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.717429:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.717429:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677447.717433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.717441:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.717442:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.717442:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.717443:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.717444:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.717444:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.717444:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.717445:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.717445:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.717446:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677447.717582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.717631:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.717634:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.717634:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.717635:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.717637:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.717641:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677447.717667:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.717670:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.717672:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.717674:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.717675:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.717676:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.717677:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.717678:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.717680:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.717680:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.717684:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.717685:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.717687:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.717688:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.717689:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.717690:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.717691:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.717692:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.717692:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.717703:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.717706:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.717706:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.717708:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.717710:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.717711:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.717745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.717748:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.717750:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.717752:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.717754:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.717755:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.717757:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.717758:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.717759:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.717760:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.717761:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.717764:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.717766:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.717769:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.717769:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.717770:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.717772:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.717773:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.717773:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.717774:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.717784:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.717787:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.717788:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.717790:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.717791:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.717793:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.717804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:12.0:1619677447.717805:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.717808:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.717809:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.717809:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.717810:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.717811:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677447.717811:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.717814:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.717816:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.717818:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.717818:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.717819:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.717821:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.717822:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.717823:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.717824:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.717827:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.717829:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.717831:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.717832:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.717832:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.717834:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.717834:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.717835:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.717836:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677447.717846:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.717848:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.717849:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.717851:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.717852:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.717853:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677447.717869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.717876:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.717879:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.717880:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.717882:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.717885:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.717886:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.717888:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.717889:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.717890:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.717891:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.717894:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.717895:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.717898:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.717899:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.717899:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.717901:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.717901:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.717902:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.717902:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.717907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.717915:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.717918:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.717919:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.717920:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.717922:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.717924:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677447.718301:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.718303:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.718304:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.718306:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.718308:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.718310:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.718311:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.718312:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.718313:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.718314:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.718317:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.718319:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.718322:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.718323:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.718323:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.718325:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.718325:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.718326:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.718327:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.718382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.718399:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.718402:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.718403:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.718406:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.718408:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.718410:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.718411:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.718413:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.718414:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.718414:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677447.718415:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.718417:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.718417:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:2.0:1619677447.718419:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677447.718420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:1.0:1619677447.718420:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.718421:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.718421:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.718423:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:2.0:1619677447.718424:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.718425:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:1.0:1619677447.718425:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96724389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.718426:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677447.718426:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677447.718426:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.718427:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.718428:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.718428:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:1.0:1619677447.718428:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.718429:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677447.718429:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677447.718429:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.718429:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:31.0:1619677447.718431:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677447.718431:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677447.718432:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.718432:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.718433:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.718434:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.718436:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:5.0:1619677447.718436:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.718436:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.718437:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:5.0:1619677447.718438:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.718439:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96724388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.718439:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677447.718440:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.718440:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.718440:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:12.0:1619677447.718441:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:5.0:1619677447.718442:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.718442:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677447.718443:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.718443:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677447.718444:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677447.718444:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.718444:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677447.718445:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.718446:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677447.718447:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.718448:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.718449:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677447.718451:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.718451:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.718452:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.718453:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.718455:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.718455:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677447.718456:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677447.718457:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.718458:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:30.0:1619677447.718459:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.718459:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.718459:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677447.718461:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.718461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677447.718462:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.718464:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.718464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.718467:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.718469:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:18.0:1619677447.718469:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677447.718470:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.718471:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.718472:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.718473:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.718473:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.718475:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.718478:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677447.718508:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.718511:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.718513:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.718515:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.718516:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.718517:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677447.718519:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.718520:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.718521:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.718522:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677447.718526:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.718527:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677447.718531:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.718532:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677447.718532:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677447.718534:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.718535:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.718536:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.718538:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.718538:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677447.718548:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:30.0:1619677447.718551:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677447.718551:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.718552:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.718553:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.718554:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.718554:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677447.718555:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.718556:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677447.718559:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.718565:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677447.718568:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:1.0:1619677447.718568:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:17.0:1619677447.718571:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.718573:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.718575:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.718576:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.718578:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.718579:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.718580:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.718583:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.718585:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.718588:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.718589:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.718589:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.718591:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.718592:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677447.718592:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:17.0:1619677447.718593:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:9.2:1619677447.718597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.718604:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.718607:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.718608:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.718608:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.718610:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.718613:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677447.718642:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.718645:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.718645:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.718647:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.718650:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.718651:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677447.718652:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.718653:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.718654:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.718655:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677447.718659:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.718660:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.718663:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.718664:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677447.718665:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677447.718666:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.718667:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.718667:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.718668:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.718678:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.718681:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.718682:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.718683:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.718685:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.718687:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.718766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.718773:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.718776:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.718777:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.718778:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.718781:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.718781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:8.0:1619677447.718782:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.718783:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.718784:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.718786:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.718786:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.718789:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.718791:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.718794:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.718795:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.718795:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.718797:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.718797:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.718798:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.718800:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.718800:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.718802:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.718806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.718810:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.718813:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.718813:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.718814:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.718814:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.718815:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.718816:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677447.718816:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.718817:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677447.718818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.718819:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.718821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.718826:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.718829:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.718831:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.718831:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.718832:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:2.0:1619677447.718833:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.718834:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:17.0:1619677447.718836:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677447.718837:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:2.0:1619677447.718837:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.718838:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.718839:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.718839:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.718840:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677447.718841:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677447.718842:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.718843:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.718844:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.718845:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.718846:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677447.718846:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.718847:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:20.2:1619677447.718848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:17.0:1619677447.718849:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.718850:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000400:00000200:17.0:1619677447.718850:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.718851:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:17.0:1619677447.718851:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.718852:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:20.0:1619677447.718853:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:18.0:1619677447.718853:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677447.718856:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.718857:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.718857:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000400:00000200:17.0:1619677447.718857:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.718857:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677447.718858:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:17.0:1619677447.718859:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:20.0:1619677447.718860:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.718860:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677447.718860:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.718861:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.718862:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677447.718862:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.718863:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:17.0:1619677447.718863:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.718863:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:20.0:1619677447.718864:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96724380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.718865:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.718865:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.718865:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.718865:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677447.718866:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.718866:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:20.0:1619677447.718867:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.718871:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677447.718871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.718872:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.718876:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.718876:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.718877:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.718878:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:20.0:1619677447.718879:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.718880:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677447.718880:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:20.0:1619677447.718881:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.718881:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677447.718881:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.718882:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.718883:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.718885:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:12.0:1619677447.718891:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.718894:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.718895:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.718897:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.718898:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.718900:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.718969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.718977:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.718979:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.718982:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.718983:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.718984:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.718986:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.718987:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.718988:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.718989:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.718990:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.718994:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.718995:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.718998:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.718999:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.719000:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.719002:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.719002:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.719003:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.719004:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.719008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.719017:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.719019:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.719020:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.719022:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.719022:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.719026:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677447.719068:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.719071:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.719072:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.719074:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.719077:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.719079:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.719080:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.719081:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.719082:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.719083:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.719086:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.719088:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.719090:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.719091:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.719092:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.719094:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.719094:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.719096:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.719097:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.719097:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677447.719100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.719108:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677447.719108:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.719110:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.719110:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677447.719111:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:18.0:1619677447.719111:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:20.0:1619677447.719112:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.719114:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.719114:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.719117:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677447.719121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677447.719129:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.719132:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.719133:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.719133:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.719134:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.719135:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:13.0:1619677447.719139:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.719142:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.719142:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.719144:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.719145:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.719146:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.719270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677447.719279:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677447.719281:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677447.719282:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677447.719284:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.719285:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677447.719286:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.719286:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677447.719288:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.719290:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.719290:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.719291:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.719292:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:31.0:1619677447.719298:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.719299:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.719300:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.719301:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.719302:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.719302:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.719303:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.719304:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.719309:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.719310:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.719312:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.719313:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.719315:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.719316:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.719316:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.719316:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.719317:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677447.719318:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.719318:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677447.719319:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.719319:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677447.719320:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677447.719320:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.719320:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.719321:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.719322:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677447.719325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.719332:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:12.0:1619677447.719333:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677447.719335:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.719335:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.719336:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.719336:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.719337:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677447.719337:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.719337:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:12.0:1619677447.719338:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.719339:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:1.0:1619677447.719339:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:30.0:1619677447.719340:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.719340:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.719341:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:30.0:1619677447.719342:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.719344:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.719345:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.719345:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.719346:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.719347:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.719347:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:30.0:1619677447.719347:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:31.0:1619677447.719348:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:13.0:1619677447.719348:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.719349:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:13.0:1619677447.719351:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.719352:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.719353:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.719353:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.719355:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.719444:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677447.719446:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677447.719447:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677447.719449:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.719451:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.719452:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.719454:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.719455:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.719456:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.719457:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.719460:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.719462:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.719464:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.719465:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.719465:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.719466:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.719467:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677447.719468:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677447.719468:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.2:1619677447.719472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.719479:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.719481:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.719482:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.719483:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.719484:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.719485:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.719490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.719498:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.719500:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.719501:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.719501:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.719502:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.719502:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.719659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677447.719706:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.719709:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.719710:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.719711:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.719713:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.719714:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:12.0:1619677447.719718:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.719721:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.719721:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.719723:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.719725:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.719726:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.719827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.719835:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.719837:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.719839:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.719841:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.719841:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.719843:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.719844:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.719845:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.719846:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.719847:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.719850:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.719852:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.719855:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.719855:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.719856:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.719858:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.719858:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.719859:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.719859:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.719864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.719872:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.719874:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.719875:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.719877:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.719877:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.719880:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.0:1619677447.719919:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.719922:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.719923:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.719925:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.719928:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.719929:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677447.719930:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.719931:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.719933:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.719933:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677447.719936:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.719938:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.719940:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.719941:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.719942:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.719943:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.719943:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:1.0:1619677447.719944:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677447.719945:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.719946:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.719946:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:17.0:1619677447.719947:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.719947:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.719949:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.719951:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.719952:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.719954:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.719955:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:1.0:1619677447.719955:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.719956:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.719957:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:1.0:1619677447.719957:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:13.0:1619677447.719958:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.719959:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.719961:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.719961:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:13.0:1619677447.719961:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.719961:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:1.0:1619677447.719961:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.719963:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:1.0:1619677447.719964:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.719965:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677447.719967:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.719968:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.719969:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.719971:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.719971:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.719973:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.719974:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.719974:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677447.719983:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677447.719984:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677447.719985:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.719986:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.719987:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.719987:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.719988:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677447.719988:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.719989:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.719991:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.719992:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.719995:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.719996:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.719998:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.720001:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.720003:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.720004:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.720005:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.720006:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.720007:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.720011:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.720013:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.720016:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.720016:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.720017:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.720018:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.720019:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.720020:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.720021:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.720022:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.720025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:4.2:1619677447.720027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677447.720032:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677447.720033:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677447.720034:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.720034:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677447.720034:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.720035:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.720035:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:8.0:1619677447.720036:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:2.0:1619677447.720036:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.720037:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677447.720037:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.720039:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.720041:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.720042:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:8.0:1619677447.720043:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.720045:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.720047:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.720048:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.720049:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.720050:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.720054:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.720055:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.720058:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.720059:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.720060:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.720061:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.720062:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.720063:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.720063:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.720068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677447.720075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:1.0:1619677447.720076:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677447.720078:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:1.0:1619677447.720078:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.720079:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.720079:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.720080:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:1.0:1619677447.720081:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.720083:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.720084:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:1.0:1619677447.720084:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:20.0:1619677447.720086:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.720087:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677447.720089:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.720090:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.720091:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.720092:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677447.720096:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.720097:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677447.720100:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.720101:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677447.720102:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677447.720104:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677447.720104:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.720105:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.720106:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677447.720116:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:18.0:1619677447.720117:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677447.720119:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.720120:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.720120:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.720121:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:31.0:1619677447.720122:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.720123:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:18.0:1619677447.720124:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677447.720126:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:18.0:1619677447.720126:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.720128:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677447.720129:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.720130:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.720131:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.720132:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677447.720135:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.720137:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677447.720137:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677447.720139:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.720139:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677447.720140:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.720140:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:13.0:1619677447.720140:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.720142:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.720142:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:13.0:1619677447.720142:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.720143:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.720144:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:13.0:1619677447.720144:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.720145:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.720145:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:9.2:1619677447.720148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.720156:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.720158:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.720158:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.720159:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.720159:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.720159:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.720160:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.720161:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:2.0:1619677447.720161:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.720163:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.720226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.720273:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.720275:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.720276:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.720277:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.720279:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.720282:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.720426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.720434:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.720437:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.720439:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.720441:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.720442:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.720443:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.720445:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.720446:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.720447:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.720448:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.2:1619677447.720450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:30.0:1619677447.720451:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.720452:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.720455:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.720455:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.720456:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677447.720456:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677447.720457:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.720458:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.720459:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:5.0:1619677447.720459:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:30.0:1619677447.720461:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000400:00000200:5.0:1619677447.720461:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.720462:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677447.720462:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.720464:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.720466:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.2:1619677447.720467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:5.0:1619677447.720467:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.720468:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.720469:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.720470:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.720471:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.720473:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:5.0:1619677447.720473:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677447.720474:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.720476:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:12.0:1619677447.720476:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.720476:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.720478:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:13.0:1619677447.720478:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.720479:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:12.0:1619677447.720479:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677447.720480:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677447.720480:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.720480:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.720481:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.720483:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.720483:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.720484:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.720485:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.720496:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.720498:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.720498:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:13.0:1619677447.720499:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.720499:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:13.0:1619677447.720500:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.720500:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.720501:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.720501:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.720502:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.720503:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.720504:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.720513:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.720516:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.720516:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.720517:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.720517:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.720518:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.720584:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.720587:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.720589:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.720591:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.720591:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.720593:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677447.720594:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.720595:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.720596:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.720597:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677447.720600:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.720602:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.720605:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.720606:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677447.720607:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.720608:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677447.720609:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.720610:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.720610:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677447.720615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.720624:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.720627:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.720628:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.720628:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.720631:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.720634:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.720648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:8.2:1619677447.720687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.720695:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.720697:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.720698:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.720700:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.720701:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.720704:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677447.720731:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.720735:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.720735:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.720738:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.720739:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.720741:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.720742:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.720743:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.720744:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.720745:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:4.2:1619677447.720746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:18.0:1619677447.720748:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.720750:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.720751:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.720752:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.720753:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.720754:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.720755:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.720755:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.720756:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.720760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.720768:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.720770:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.720771:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.720772:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.720773:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.720775:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.720781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.720788:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.720788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677447.720791:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.720791:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.720792:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:22.0:1619677447.720793:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677447.720793:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.720793:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.720796:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.720797:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.720798:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.720800:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.720802:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.720813:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.720816:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.720816:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.720817:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.720818:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.720818:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.720883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.720890:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.720894:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:1.0:1619677447.720895:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677447.720896:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.720897:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:1.0:1619677447.720898:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:30.0:1619677447.720899:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.720899:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677447.720900:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.720901:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.720902:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.720903:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.720904:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.720904:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.720905:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.720905:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.720906:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.720908:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.720909:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.720910:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.720911:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677447.720912:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.720915:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677447.720915:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.720916:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.720917:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.720917:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677447.720918:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.720919:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:1.0:1619677447.720919:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.720920:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.720920:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:1.0:1619677447.720920:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.720921:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.720922:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.720923:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.720923:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.720924:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677447.720926:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.720929:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.720930:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.720932:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.720934:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:24.0:1619677447.720935:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.720936:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:22.0:1619677447.720936:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.720937:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:24.0:1619677447.720939:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96724353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677447.720939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:22.0:1619677447.720940:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.720941:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:24.0:1619677447.720942:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.720942:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.720943:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.720943:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:24.0:1619677447.720944:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677447.720944:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:13.0:1619677447.720944:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.720945:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96724352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.720946:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.720947:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.720947:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:12.0:1619677447.720947:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.720948:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:22.0:1619677447.720949:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.720949:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:24.0:1619677447.720950:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.720950:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677447.720950:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.720950:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.720952:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:17.0:1619677447.720952:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.720954:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.720954:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:24.0:1619677447.720955:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.720956:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.720957:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.720957:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:24.0:1619677447.720959:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.720960:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:22.0:1619677447.720960:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.720960:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:24.0:1619677447.720961:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.720961:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.720961:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.720963:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.720963:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.720964:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.720965:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.720965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.2:1619677447.720969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.720972:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.720975:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.720977:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.720977:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:2.0:1619677447.720977:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.720979:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.720980:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.720980:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.720982:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.720982:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.720983:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.720985:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677447.721255:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.721257:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.721259:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.721261:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.721262:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.721263:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.721264:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.721265:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.721266:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.721267:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.721270:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.721272:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.721274:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.721275:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.721275:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.721277:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.721277:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.721278:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.721279:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.721289:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.721291:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.721292:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.721298:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.721300:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.721302:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.721371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.721380:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.721383:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.721384:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.2:1619677447.721385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:1.0:1619677447.721386:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.721388:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677447.721388:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.721390:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:1.0:1619677447.721390:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.721392:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.721394:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.721394:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:1.0:1619677447.721394:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.721395:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.721397:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677447.721398:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.721400:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.721401:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96724347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.721403:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.721403:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.721405:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.721406:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.721407:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:20.0:1619677447.721410:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.721411:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.721412:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.721412:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.721413:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677447.721415:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.721415:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.721415:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677447.721416:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.721416:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677447.721417:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.721417:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677447.721419:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.721419:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.721420:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.721421:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.721422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.2:1619677447.721425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.0:1619677447.721429:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:6.0:1619677447.721431:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677447.721432:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:30.0:1619677447.721433:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:24.0:1619677447.721433:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.721433:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.721433:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.721434:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:13.0:1619677447.721434:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677447.721435:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.721436:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:13.0:1619677447.721436:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:8.0:1619677447.721436:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:13.0:1619677447.721437:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.721437:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.721439:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:30.0:1619677447.721440:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677447.721440:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.721440:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677447.721441:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:12.0:1619677447.721441:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:30.0:1619677447.721442:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.721443:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:30.0:1619677447.721444:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.721445:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.721446:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.721447:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.721448:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.721452:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.721454:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.721457:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.721458:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.721458:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.721460:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.721461:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.721462:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.721462:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.721466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.721475:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.721478:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.721478:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.721480:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.721480:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.721483:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677447.721484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677447.721493:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.721495:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.721496:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.721497:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.721498:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.721498:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677447.721667:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.721670:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.721671:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.721674:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.721676:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.721677:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.721678:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.721679:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.721681:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.721681:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.721685:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.721687:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.721689:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.721689:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.721690:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.721692:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.721692:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.721693:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.721693:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.721698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.721705:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.721707:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.721708:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.721708:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.721709:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.721711:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.721779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:22.0:1619677447.721785:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.721787:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677447.721788:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677447.721790:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.721792:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.721794:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:22.0:1619677447.721795:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.721796:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.721797:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.721798:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677447.721801:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.721803:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.721805:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.721806:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:22.0:1619677447.721807:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.721808:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677447.721809:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.721809:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677447.721810:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677447.721814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.721822:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.721824:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.721825:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.721825:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.721828:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.721830:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.721837:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.721840:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.721840:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.721843:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.721845:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.721846:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.721847:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.721849:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.721850:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.721850:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.721854:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.721855:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677447.721857:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677447.721858:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.721859:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.721859:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:17.0:1619677447.721860:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:6.0:1619677447.721861:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.721861:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677447.721861:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.721862:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:17.0:1619677447.721863:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.721863:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:17.0:1619677447.721866:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.721868:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.721869:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.721870:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.721871:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.721872:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.721875:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.721877:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.721881:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.721881:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.721882:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.721884:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.721885:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.721885:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.721886:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.721906:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.721909:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.721910:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.721912:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.721913:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.721914:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677447.721980:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.721983:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.721985:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.721987:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.721988:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.721990:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677447.721991:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.721992:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.721994:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.721995:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677447.721998:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.722000:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.722003:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.722003:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677447.722004:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.722006:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677447.722006:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.722008:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.722009:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:20.2:1619677447.722009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:31.0:1619677447.722010:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677447.722012:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:8.2:1619677447.722012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.722014:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.722017:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.722019:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.722019:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:1.0:1619677447.722020:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:20.0:1619677447.722021:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:20.0:1619677447.722022:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.722023:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.722023:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.722023:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:30.0:1619677447.722024:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:1.0:1619677447.722024:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.722025:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.722026:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:20.0:1619677447.722026:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677447.722026:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.722027:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.722028:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:20.0:1619677447.722029:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.722030:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677447.722031:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:20.0:1619677447.722035:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.722035:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:20.0:1619677447.722036:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:20.0:1619677447.722038:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:20.0:1619677447.722038:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.722040:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.722041:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.722042:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:9.2:1619677447.722044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:20.2:1619677447.722046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:24.0:1619677447.722050:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:5.0:1619677447.722052:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:24.0:1619677447.722053:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.722053:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.722054:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.722054:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.722055:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:24.0:1619677447.722056:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.722056:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.2:1619677447.722057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:24.0:1619677447.722059:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.722095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.722103:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.722105:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.722107:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.722107:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.722109:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:9.2:1619677447.722111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677447.722113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677447.722114:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.722115:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.722117:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.722118:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.722119:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:5.0:1619677447.722119:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677447.722119:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.722120:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677447.722121:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.722124:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.722124:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.722126:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.722128:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.722129:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.722131:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.722132:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.722133:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.722137:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.722138:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.722141:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.722142:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.722143:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.722145:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.722146:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.722146:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.722148:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.722148:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.722153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.722156:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:30.0:1619677447.722158:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677447.722158:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.722159:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677447.722160:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677447.722160:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:30.0:1619677447.722161:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.722163:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.722164:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.722166:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.0:1619677447.722378:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.722381:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.722382:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.2:1619677447.722383:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000400:00000200:8.0:1619677447.722388:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.722391:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.722392:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:8.0:1619677447.722392:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.722394:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.722395:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.722396:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.722396:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.722397:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (21)++
00000400:00000200:8.0:1619677447.722397:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.722398:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.722398:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:8.0:1619677447.722398:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.722401:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.722403:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.722407:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.722408:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.722408:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.722410:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.722411:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.722411:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.722412:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.722417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.722426:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.722429:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.722430:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.722430:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.722432:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.722435:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.722671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.722680:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.722682:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.722684:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.722687:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.722687:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.722689:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.722690:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.722691:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.722692:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.722693:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.722696:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.722697:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.722699:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.722700:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.722701:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.722702:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.722703:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.722704:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.722704:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677447.722714:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.722717:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.722718:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.722719:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.722721:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.722722:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:1.0:1619677447.722723:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677447.722726:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677447.722726:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:1.0:1619677447.722728:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.722731:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000400:00000200:1.0:1619677447.722731:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.722732:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.722734:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.722735:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.722735:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.722736:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000400:00000200:1.0:1619677447.722736:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.722737:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:1.0:1619677447.722737:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.722738:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:1.0:1619677447.722738:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.722741:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.722743:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.722746:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.722747:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.722747:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.722749:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.722749:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677447.722750:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677447.722751:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:13.0:1619677447.722763:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.722765:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.722766:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.722768:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.722769:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.722771:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.722842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677447.722887:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.722889:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.722890:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.722892:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.722893:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.722894:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:2.0:1619677447.723117:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677447.723120:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677447.723121:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677447.723123:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.723125:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.723127:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.723128:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.723129:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.723130:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.723131:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.723135:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.723136:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.723139:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.723140:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.723140:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.723142:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.723143:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677447.723143:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677447.723144:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677447.723148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.723156:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.723158:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.723159:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.723159:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.723161:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.723164:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677447.723230:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.723233:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.723235:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.723237:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.723237:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.723239:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677447.723240:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.723242:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.723242:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.723244:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677447.723247:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.723249:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677447.723252:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.723253:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677447.723254:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677447.723256:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.723256:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.723257:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677447.723257:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677447.723258:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677447.723259:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.723260:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.723262:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.723265:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.723266:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:20.2:1619677447.723267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:1.0:1619677447.723268:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.723269:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.723270:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677447.723270:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.723271:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.723272:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:1.0:1619677447.723275:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.723276:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.723276:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677447.723278:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.723279:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.723280:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:1.0:1619677447.723281:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.723282:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:1.0:1619677447.723282:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.723282:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.723283:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:20.0:1619677447.723283:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96724330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.723284:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.723284:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677447.723284:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:20.0:1619677447.723285:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.723285:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677447.723286:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.723286:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.723287:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677447.723287:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.723288:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.723288:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677447.723291:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.723300:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677447.723300:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.723300:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.723302:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.723303:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677447.723303:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:17.0:1619677447.723303:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:30.0:1619677447.723304:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.723304:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:30.0:1619677447.723305:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.723305:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677447.723305:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:20.0:1619677447.723306:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677447.723306:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677447.723307:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.723307:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677447.723307:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677447.723308:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677447.723309:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677447.723318:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677447.723319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:31.0:1619677447.723321:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.723322:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.723322:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.723323:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.723323:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.0:1619677447.723325:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.723329:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.723331:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:6.0:1619677447.723331:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:5.0:1619677447.723332:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:22.0:1619677447.723333:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.723333:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.723334:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.723334:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.723335:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:5.0:1619677447.723336:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:6.0:1619677447.723337:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.723338:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.723339:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.723339:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.723341:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.723341:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.723341:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:22.0:1619677447.723344:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.723345:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.723346:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000400:00000200:22.0:1619677447.723347:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.723349:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.723349:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:22.0:1619677447.723350:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.723351:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.723352:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.723352:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.723353:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.0:1619677447.723353:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677447.723354:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:22.0:1619677447.723355:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.723356:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677447.723356:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:22.0:1619677447.723357:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.723357:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.723358:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:22.0:1619677447.723359:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.723360:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.723361:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.723362:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:9.2:1619677447.723362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:22.0:1619677447.723363:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.723364:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.723364:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.723368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.723370:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.723372:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.723374:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.723375:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.723376:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:17.0:1619677447.723377:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.723379:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:17.0:1619677447.723379:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.723380:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.723381:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.723383:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.723385:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.723389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.723396:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.723399:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.723400:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.723401:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.723404:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.723406:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.723407:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.723409:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.723410:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.723411:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.723415:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.723417:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.723420:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.723421:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.723422:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.723423:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.723424:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.723425:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.723425:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.723429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.723437:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.723440:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.723440:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.723441:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.723443:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:1.0:1619677447.723443:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.723445:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677447.723445:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:12.0:1619677447.723446:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.723448:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.2:1619677447.723448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:13.0:1619677447.723450:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.723451:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677447.723456:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.723459:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.723459:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.723460:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.723461:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.723461:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.723491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.723500:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.723503:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.723504:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.723507:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.723509:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.723511:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.723513:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.723515:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.723516:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.723517:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.723522:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.723525:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.723528:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.723529:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.723530:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.723532:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.723533:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.723534:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.723534:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677447.723538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.723542:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.723545:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.723545:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:20.0:1619677447.723546:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:6.0:1619677447.723547:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:2.0:1619677447.723547:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.723548:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.723549:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.723549:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.723550:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.723551:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.723551:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.723551:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.723552:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.723552:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:2.0:1619677447.723553:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96724323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.723554:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.723554:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.723555:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.723556:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677447.723557:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.723558:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:2.0:1619677447.723560:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.723562:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.723565:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.723566:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.723567:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677447.723569:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.723569:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.723569:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.723570:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:22.0:1619677447.723571:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.723571:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:6.0:1619677447.723572:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.723573:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.723574:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.723574:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.723581:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.723584:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.723585:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.723587:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.723588:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.723590:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.723604:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.723606:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.723608:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.723608:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.723609:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.723610:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.723843:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.723845:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.723846:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.723848:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.723851:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.723853:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.723854:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.723855:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.723856:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.723857:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.723860:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.723861:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.723863:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.723863:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.723864:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.723865:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.723866:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.723867:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.723867:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.723872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.723874:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.723877:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.723877:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:2.0:1619677447.723879:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:1.0:1619677447.723879:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.723882:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:1.0:1619677447.723882:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.723883:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.723883:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:18.0:1619677447.723884:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.723885:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.723885:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.723886:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.723887:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:1.0:1619677447.723887:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.723888:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.723892:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.723894:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.723897:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.723898:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.723898:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.723900:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.723901:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.723901:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.723902:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.723913:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.723916:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.723917:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.723919:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.723920:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:4.2:1619677447.723921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677447.723922:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677447.723963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.723968:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.723970:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.723971:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.723972:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.723974:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.723977:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:30.0:1619677447.724044:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.724047:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.724049:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.724051:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.724054:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.724055:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.724057:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.724058:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.724059:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.724059:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.724062:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.724064:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.724066:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.724067:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.724067:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.724069:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.724069:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.724069:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.724070:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677447.724070:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.724071:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.724071:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:31.0:1619677447.724072:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.724072:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.724074:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:13.0:1619677447.724081:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.724083:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.724084:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.724086:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.724087:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.724088:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.724134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.724141:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.724144:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.724144:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.724146:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.724148:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:24.0:1619677447.724149:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.724151:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:17.0:1619677447.724151:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.724151:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677447.724153:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.724154:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.724155:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.724156:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.724157:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.724157:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.724160:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.724161:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.724162:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:24.0:1619677447.724164:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677447.724164:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.724165:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.724165:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:17.0:1619677447.724165:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.724166:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.724167:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.724167:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.724168:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.724168:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.724169:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677447.724171:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.724172:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.2:1619677447.724174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.724175:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.724177:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.724177:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.724180:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.724180:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.724181:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.724182:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.724183:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.724185:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.724186:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.724186:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.724187:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.724190:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:12.0:1619677447.724192:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.724194:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.724195:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.724197:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.724198:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.724200:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.724207:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.724209:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.724210:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.724211:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.724211:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.724213:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.724285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.724299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.724305:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.724308:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.724311:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.724311:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.724313:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.724315:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.724316:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.724318:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.724319:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.724320:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.724323:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.724326:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.724328:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.724329:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.724330:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.724332:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.724332:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:9.2:1619677447.724333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.724333:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.724334:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.724341:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.724344:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:22.0:1619677447.724345:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:18.0:1619677447.724345:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.724345:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.724347:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.724348:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.724349:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.724349:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.724351:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.724351:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.724353:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.724496:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.724499:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.724500:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.724502:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.724505:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.724507:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.724508:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.724509:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.724510:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.724511:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.724514:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.724516:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.724519:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.724519:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.724520:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.724522:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.724523:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.724524:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.724524:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.724536:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.724539:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.724539:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.724541:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.724543:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.724544:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677447.724616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.724669:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.724672:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.724672:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.724674:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.724676:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.724678:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.724680:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.724681:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.724682:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.724683:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.724686:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.724688:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.724691:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.724692:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.724692:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.724694:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.724694:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.724696:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.724697:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.724697:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677447.724701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.724707:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:1.0:1619677447.724708:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677447.724709:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.724709:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.724710:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677447.724711:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.724712:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.724712:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.724714:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.724716:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.724722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.2:1619677447.724723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677447.724729:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677447.724730:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.724731:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.0:1619677447.724731:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.724734:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.724734:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:1.0:1619677447.724734:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:31.0:1619677447.724735:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.724735:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677447.724735:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677447.724735:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.724736:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.724736:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:8.0:1619677447.724737:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.724738:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.724739:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.724740:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.724740:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.724741:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.724742:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:31.0:1619677447.724743:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.724743:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724309] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.724744:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.724745:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.724746:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.724747:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677447.724748:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.724750:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.724751:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.724753:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.724753:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677447.724754:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.724754:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.724755:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.724756:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.724756:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677447.724756:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:31.0:1619677447.724757:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677447.724757:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677447.724758:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677447.724758:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.724759:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.724760:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.724761:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.724762:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.724764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677447.724767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:12.0:1619677447.724769:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.724771:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.724772:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.724773:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.724774:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:12.0:1619677447.724775:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.724775:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:13.0:1619677447.724776:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.724777:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677447.724777:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.724777:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:24.0:1619677447.724778:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.724778:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.724778:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.724779:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.724780:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.724782:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.724784:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.724785:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.724785:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.724786:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.724787:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.724788:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.724790:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.724791:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.724792:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.724795:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.724797:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.724800:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.724801:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.724802:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.724803:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.724804:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.724805:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.724806:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.724807:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677447.724816:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677447.724817:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677447.724818:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677447.724819:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:12.0:1619677447.724820:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.724820:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.724820:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677447.724821:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.724823:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.724824:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.724918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.724927:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.724929:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.724931:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.724933:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.724933:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.724935:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677447.724936:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.724938:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.724939:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.724940:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677447.724943:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.724945:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677447.724948:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.724948:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.724949:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677447.724951:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.724952:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.724953:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.724954:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.724955:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677447.724961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.724969:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.724971:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.724972:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.724973:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.724984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.724992:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.724994:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.724995:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.724997:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.725000:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.725001:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.725031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.725033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.725041:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.725043:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.725045:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.725045:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.725048:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.725053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.725057:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.725059:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.725061:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.725061:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.725062:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.725063:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.725064:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.725315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.725351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.725354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677447.725359:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.725362:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.725363:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.725363:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.725365:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.725370:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.725370:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.725372:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.725373:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.725375:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.725376:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.725378:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.725379:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.725380:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.725381:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.725382:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.725385:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.725386:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.725389:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.725389:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.725390:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.725392:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.725392:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.725392:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.725393:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.725397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.725404:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.725406:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.725406:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.725408:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.725408:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.725409:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.725454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.725463:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.725465:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.725468:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.725470:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.725471:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.725472:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.725474:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.725475:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.725476:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.725477:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.725480:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.725482:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.725483:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:31.0:1619677447.725485:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.725486:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:18.0:1619677447.725486:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.725486:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:31.0:1619677447.725487:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.725488:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:18.0:1619677447.725488:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.725489:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.725490:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.725491:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:18.0:1619677447.725491:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.725492:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.725493:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.725495:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.725496:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.725497:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.725500:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.725502:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:13.0:1619677447.725502:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.725504:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:18.0:1619677447.725505:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677447.725505:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.725506:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.725506:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:13.0:1619677447.725507:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.725508:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.725509:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:12.0:1619677447.725509:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.725510:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:13.0:1619677447.725510:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.725511:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677447.725511:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.725512:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:30.0:1619677447.725513:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.725516:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.725517:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.725518:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.725519:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677447.725521:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.725522:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.725522:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:30.0:1619677447.725523:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.725524:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677447.725524:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.725525:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.725526:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:30.0:1619677447.725528:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.725530:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677447.725533:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.725534:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677447.725535:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677447.725536:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.725537:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.725538:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.725540:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.725540:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677447.725552:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:31.0:1619677447.725553:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:6.0:1619677447.725554:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:31.0:1619677447.725556:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.725556:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677447.725556:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:6.0:1619677447.725556:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.725557:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677447.725557:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.725558:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.725560:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.725561:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:4.2:1619677447.725561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.725562:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:22.0:1619677447.725563:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.725563:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.725565:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.725566:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.725568:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.725569:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.725570:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.725575:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.725577:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.725581:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.725582:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.725582:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.725585:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.725586:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.725586:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.725587:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677447.725598:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.725601:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.725602:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.725603:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.725605:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.725607:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:6.0:1619677447.725611:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.725614:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.725615:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.725615:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.725617:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.725621:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.725749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:9.2:1619677447.725787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.725795:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.725798:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.725799:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.725799:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.725801:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.725804:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.725815:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.725819:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.725819:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.725821:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.725824:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.725826:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.725827:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.725828:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.725829:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.725830:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.725833:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.725835:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.725838:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.725839:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.725839:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.725841:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.725842:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.725842:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.725843:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.725847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.725854:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.725857:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.725858:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.725859:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.725861:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.725864:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:2.0:1619677447.725964:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.725967:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.725968:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.725970:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.725973:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.725974:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.725975:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.725977:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.725978:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.725979:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.725981:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.725983:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.725985:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.725986:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.725986:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.725988:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.725988:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.725990:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.725991:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.725991:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.725994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.726001:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.726003:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.726003:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.726003:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.726004:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:18.0:1619677447.726005:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.726005:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.726006:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.726006:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.726007:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.726013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:4.2:1619677447.726018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.726020:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.726022:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.726023:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.726024:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.726024:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.726025:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.726055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.726065:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.726067:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677447.726069:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:30.0:1619677447.726069:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.726070:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.726071:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.726071:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e2f13e00] (21)++
00000400:00000200:31.0:1619677447.726073:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.726073:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.26@o2ib3
00000400:00000200:30.0:1619677447.726074:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.726075:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.726076:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.26@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.726077:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.726077:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e2f13e00] (22)--
00000400:00000200:30.0:1619677447.726078:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.726079:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:20.0:1619677447.726079:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.726080:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677447.726082:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.726082:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.726083:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.726084:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.726084:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.726085:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.726086:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:20.0:1619677447.726086:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.726087:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.726087:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677447.726089:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.726090:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.726091:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.26@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.726093:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.726094:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.726094:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677447.726094:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 280 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:31.0:1619677447.726095:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.726096:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677447.726096:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.726097:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.726097:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.726097:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:20.0:1619677447.726097:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677447.726098:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677447.726098:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677447.726099:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677447.726099:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677447.726100:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.726100:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e2f13e00] (21)++
00000800:00000200:31.0:1619677447.726101:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.726101:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677447.726101:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e2f13e00] (22)--
00000800:00000200:20.0:1619677447.726101:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e2f13e00] (21)--
00000800:00000200:31.0:1619677447.726102:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.726103:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.726104:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.726105:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.726105:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.726108:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.726111:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.726112:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.726113:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:12.0:1619677447.726114:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.726115:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:13.0:1619677447.726115:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:22.0:1619677447.726116:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.726117:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.726117:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.726117:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.726119:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.726120:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:22.0:1619677447.726121:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:13.0:1619677447.726121:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.726123:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.726124:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.726125:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.726136:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.726139:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.726140:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.726140:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.726141:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.726142:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.726152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.726158:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.726160:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677447.726162:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677447.726163:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.726165:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.726167:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677447.726168:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.726169:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.726170:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.726172:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.726175:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.726177:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.726180:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.726180:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.726181:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.726183:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.726183:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.726184:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.726184:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677447.726189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.726196:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.726199:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.726200:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.726200:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.726202:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677447.726204:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677447.726248:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.726250:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.726251:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.726253:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.726255:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.726257:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.726258:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.726259:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.726260:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.726262:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.726265:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.726267:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.726269:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.726270:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.726271:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.726273:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.726273:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.726274:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.726274:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677447.726286:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.726289:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.726290:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.726291:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.726292:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.726300:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.726301:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677447.726303:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.726303:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.726304:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.726305:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.726306:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.726320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.726323:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.726326:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677447.726326:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:4.2:1619677447.726328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:5.0:1619677447.726329:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677447.726329:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000400:00000200:20.0:1619677447.726330:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.726330:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677447.726332:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.726333:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.726334:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.726334:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.726335:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:2.0:1619677447.726335:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677447.726336:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.726337:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.726338:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.726338:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.726339:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677447.726339:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.726340:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.726341:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.726342:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.726342:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.726343:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.726344:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677447.726345:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.726347:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.726347:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677447.726348:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.726348:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.726349:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.726350:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96724286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.726351:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.726351:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.726352:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.726353:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.726354:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677447.726354:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.726354:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.726355:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.726355:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.726356:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.726356:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.726357:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.726358:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:8.0:1619677447.726358:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.726359:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677447.726359:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.726360:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.726360:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.0:1619677447.726360:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.726360:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677447.726360:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000400:00000200:17.0:1619677447.726361:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.726361:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000400:00000200:17.0:1619677447.726362:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:2.0:1619677447.726362:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:8.0:1619677447.726363:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677447.726364:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96724285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.726364:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000400:00000200:17.0:1619677447.726365:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.726366:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677447.726366:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (22)++
00000400:00000200:17.0:1619677447.726367:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.726369:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (23)++
00000400:00000200:17.0:1619677447.726371:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.726371:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677447.726371:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (24)--
00000800:00000200:18.0:1619677447.726373:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:8.0:1619677447.726373:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.726374:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:12.0:1619677447.726374:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677447.726374:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.726374:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677447.726375:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000400:00000200:12.0:1619677447.726375:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.726375:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000800:00000200:18.0:1619677447.726376:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:12.0:1619677447.726376:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.726376:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:1.0:1619677447.726376:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000800:00000200:17.0:1619677447.726377:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.726378:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:13.0:1619677447.726378:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:1.0:1619677447.726378:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.726379:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:12.0:1619677447.726379:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677447.726380:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.726381:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.726382:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.726383:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.726383:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.726383:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.726384:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.726386:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.726387:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:9.2:1619677447.726387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:1.0:1619677447.726387:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.726388:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677447.726388:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.726391:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:13.0:1619677447.726392:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.726393:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677447.726394:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:13.0:1619677447.726394:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.726395:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.726396:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.726397:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.726398:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.726398:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.726400:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.726403:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.726408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677447.726413:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.726416:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.726417:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.726417:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.726418:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.726418:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:18.0:1619677447.726436:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.726439:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.726439:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.726442:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.726444:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.726445:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.726446:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724284] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.726448:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.726449:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.726450:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.726453:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.726454:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.726457:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.726457:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.726458:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.726459:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.726460:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.726460:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.726461:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.726466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.726473:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.726476:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.726477:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.726477:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.726479:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.726481:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.726485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.726491:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.726494:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.726495:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.726495:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.726496:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.726497:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.726707:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.726710:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.726710:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.726714:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.726717:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.726719:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.726721:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.726723:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.726724:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.726726:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.726730:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.726733:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.726735:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.726736:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.726736:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.726738:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.726738:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.726739:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.726739:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.726743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.2:1619677447.726748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:1.0:1619677447.726750:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.726752:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.726753:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.726754:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.726755:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.726756:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:1.0:1619677447.726757:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677447.726759:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.726760:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.726762:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.726763:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.726767:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:24.0:1619677447.726768:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.726771:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.726771:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.726773:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.726776:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.726777:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677447.726779:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.726780:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.726781:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.726782:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677447.726786:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.726788:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.726790:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.726791:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677447.726791:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677447.726793:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.726793:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.726794:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.726794:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.726806:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.726808:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.726809:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.726810:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.726812:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.726813:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.726821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:12.0:1619677447.726823:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.726826:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.726827:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.726828:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.726828:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.726829:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677447.726829:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677447.726832:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.726834:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.726836:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.726836:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.726837:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.726839:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.726840:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.726841:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.726842:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.726845:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.726847:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.726850:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.726851:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.726851:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.726853:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.726854:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.726854:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.726855:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.726859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.726867:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.726870:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.726871:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.726873:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.726873:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.726876:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:6.0:1619677447.727004:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.727007:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.727007:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.727010:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.727012:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.727013:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.727015:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.727016:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.727017:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.727018:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.727021:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.727023:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.727026:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.727027:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.727028:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.727030:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.727030:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.727031:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.727031:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677447.727115:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.727117:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.727118:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.727120:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.727122:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.727123:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677447.727124:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.727126:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.727127:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.727128:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677447.727131:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.727133:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677447.727136:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.727137:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677447.727137:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677447.727139:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.727139:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.727140:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.727141:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.727142:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.727145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.727152:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:1.0:1619677447.727152:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.727154:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.0:1619677447.727155:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:1.0:1619677447.727155:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677447.727156:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.727156:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.727156:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677447.727158:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.727161:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677447.727259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.727265:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.727267:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.727269:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.727270:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.727272:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.727273:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.727275:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724276] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.727276:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.727277:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.727278:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.727281:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.727283:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.727285:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.727286:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.727287:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.727289:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.727289:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.727291:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.727292:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.727292:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.727298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.727301:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.727303:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.727304:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.727304:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677447.727308:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.727311:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.727312:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.727313:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.727316:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.727317:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677447.727318:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:2.0:1619677447.727318:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.727320:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.727321:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.727321:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.727321:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.727322:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677447.727324:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.727325:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:2.0:1619677447.727326:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:30.0:1619677447.727327:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:2.0:1619677447.727328:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.727330:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.727331:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.727332:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.727333:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.727334:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.727334:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.727335:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:4.2:1619677447.727339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.727348:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.727350:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.727352:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.727352:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.727354:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.727358:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.727387:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.727390:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.727390:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.727391:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.727393:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.727396:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.727405:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.727407:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.727408:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.727408:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.727409:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.727410:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677447.727639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.727647:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.727649:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.727650:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.727651:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.727653:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.727656:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.727678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.727685:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.727688:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.727690:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.727692:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.727693:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.727695:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.727696:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.727698:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.727699:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.727699:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.727703:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.727704:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.727706:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.727707:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.727708:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.727709:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.727710:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.727711:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.727713:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.727713:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677447.727719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.727723:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.727725:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.727726:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.727727:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:30.0:1619677447.727727:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.727728:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.727728:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.727729:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.727731:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.727731:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:24.0:1619677447.728035:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677447.728039:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677447.728039:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677447.728041:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.728043:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.728044:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:24.0:1619677447.728046:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.728047:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.728048:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.728049:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:24.0:1619677447.728052:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.728053:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.728055:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.728056:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677447.728057:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.728058:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.728058:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677447.728059:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677447.728060:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:9.2:1619677447.728064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.728070:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.728072:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.728072:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.728073:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.728074:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.728075:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.728084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677447.728090:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.728093:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.728094:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.728094:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.728095:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.728095:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.2:1619677447.728103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.728106:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.728108:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.728110:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.728112:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.728113:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.728114:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677447.728115:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:20.0:1619677447.728116:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.728117:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.728117:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.728117:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.728118:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677447.728118:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.728119:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.728120:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.728120:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677447.728123:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677447.728123:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.728124:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:20.0:1619677447.728124:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.728125:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.728126:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.728127:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677447.728127:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.728129:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.728130:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677447.728131:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.728131:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:30.0:1619677447.728132:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.728132:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677447.728133:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.728134:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.728135:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677447.728136:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.728136:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.728136:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677447.728138:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.728141:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.728142:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677447.728143:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:30.0:1619677447.728144:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:30.0:1619677447.728145:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.728146:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.728146:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.728147:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677447.728149:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677447.728150:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.728152:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.728154:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.728155:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677447.728155:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677447.728156:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.728156:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677447.728158:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.728159:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.728235:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.728238:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.728240:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:17.0:1619677447.728240:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677447.728241:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.728244:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:2.0:1619677447.728244:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.728245:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:2.0:1619677447.728246:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677447.728247:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.728247:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.728248:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.728250:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.728250:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.728250:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677447.728252:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.728253:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.728254:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.728255:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.728256:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.728257:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.728257:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.728259:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.728260:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:17.0:1619677447.728261:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.728261:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.728262:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677447.728263:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.728263:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.728264:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.728264:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677447.728267:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.728267:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.728268:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:9.2:1619677447.728269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677447.728270:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.728270:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.728271:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.728271:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677447.728276:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.728278:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.728279:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.728280:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.728281:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.728283:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677447.728283:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.728285:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.728286:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.728288:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.728289:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.728291:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.728366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.728390:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.728392:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.728394:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.728395:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.728397:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.728399:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677447.728400:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.728402:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.728403:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.728403:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677447.728407:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677447.728409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.728409:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677447.728412:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.728413:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677447.728413:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.728415:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677447.728415:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677447.728415:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.728416:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.728417:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677447.728418:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.728419:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.728419:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.728422:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.728424:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.728440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.728447:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.728450:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.728451:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.728453:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.728455:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.728456:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.728457:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.728458:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.728459:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.728460:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.728463:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.728465:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.728468:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.728468:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.728469:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.728471:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.728472:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.728472:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.728473:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.728477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.728486:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.728488:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.728489:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.728489:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.728491:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.728494:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:31.0:1619677447.728553:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.728555:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.728557:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.728559:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.728560:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.728561:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.728562:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.728563:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.728564:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.728565:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.728568:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.728569:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.728572:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.728572:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.728573:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.728575:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.728575:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.728576:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.728576:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.728588:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.728590:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.728591:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.728592:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.728594:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.728595:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.728607:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.728609:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.728610:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.728611:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.728611:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.728612:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.728619:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677447.728622:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677447.728623:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677447.728625:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.728628:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.728629:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.2:1619677447.728631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:17.0:1619677447.728631:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.728632:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.728633:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.728634:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.728637:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:17.0:1619677447.728639:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.728640:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:17.0:1619677447.728641:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.728642:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:17.0:1619677447.728644:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677447.728644:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.728645:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.728646:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677447.728647:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.728648:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.728648:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677447.728649:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:5.0:1619677447.728649:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:17.0:1619677447.728650:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000400:00000200:5.0:1619677447.728650:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96724259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.728652:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.728653:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.728654:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677447.728659:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.728660:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.728661:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:13.0:1619677447.728663:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.728664:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.728664:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.728665:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677447.728665:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:13.0:1619677447.728666:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.728666:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.728667:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677447.728668:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.728669:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.728669:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:13.0:1619677447.728671:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.728673:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.728674:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.728675:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.728675:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677447.728676:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.728678:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.728680:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.728682:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.728683:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.728685:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.728686:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.728687:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677447.728691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.728694:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.728696:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.728698:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.728699:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677447.728699:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.728701:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.728702:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.728703:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000400:00000200:20.0:1619677447.728703:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677447.728704:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677447.728705:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:20.0:1619677447.728705:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.728706:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:20.0:1619677447.728706:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.728707:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.728708:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.728711:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.728713:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.728716:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.728717:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.728718:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.728720:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.728721:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.728721:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.728722:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.728732:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.728735:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.728735:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.728737:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.728739:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.728740:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.728854:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.728857:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.728857:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.728860:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.728862:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.728864:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.728865:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.728867:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.728868:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.728869:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.728872:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.728873:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.728876:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.728877:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.728877:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.728879:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.728880:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.728881:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.728881:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.728893:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.728895:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.728896:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.728897:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.728899:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.728901:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:30.0:1619677447.728963:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.728966:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.728968:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.728970:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.728971:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.728972:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.728973:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.728975:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.728976:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.728977:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.728980:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.728982:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.728984:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.728985:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.728986:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.728987:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.728988:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.728990:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.728991:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.728992:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677447.728999:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.729002:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.729002:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.729004:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.729005:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677447.729006:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.729007:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:12.0:1619677447.729007:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.729008:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.729009:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.729056:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.729059:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.729060:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.729062:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.729064:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.729066:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.729067:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.729068:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.729069:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.729071:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.729074:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.729076:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.729079:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.729079:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.729080:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.729082:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.729082:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.729084:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.729085:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.729085:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677447.729088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.729095:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:17.0:1619677447.729096:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677447.729097:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.729097:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.729098:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677447.729099:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.729100:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.729100:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.729102:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.729105:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.729124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677447.729178:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.729180:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.729181:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.729182:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.729184:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.729186:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:9.2:1619677447.729228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.729231:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.729234:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.729234:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:18.0:1619677447.729236:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:1.0:1619677447.729236:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.729238:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:1.0:1619677447.729238:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.729240:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.729240:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:1.0:1619677447.729240:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.729241:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.729242:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.729243:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.729244:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.729245:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:18.0:1619677447.729246:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:1.0:1619677447.729248:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.729249:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.729252:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.729252:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.729253:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.729255:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.729255:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.729256:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.729257:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677447.729265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:6.0:1619677447.729268:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.729271:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:24.0:1619677447.729272:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.729272:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.729272:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.729274:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:6.0:1619677447.729274:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.729275:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677447.729276:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:24.0:1619677447.729277:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.729280:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.729282:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.729283:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.729284:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.729285:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.729286:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.729290:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.729292:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.729301:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.729302:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.729302:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.729304:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.729304:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.729305:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.729306:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.729310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.729318:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.729321:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.729322:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.729322:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.729324:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.729327:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.729542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:8.2:1619677447.729582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677447.729585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:1.0:1619677447.729590:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.729593:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.729594:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677447.729594:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:1.0:1619677447.729596:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.729597:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.729598:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.729598:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.729600:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.729600:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000400:00000200:1.0:1619677447.729600:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.729601:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.729603:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.729604:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:1.0:1619677447.729604:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.729605:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.729608:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.729610:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.729613:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.729614:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.729614:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.729616:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.729617:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.729618:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.729619:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.729620:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.729630:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677447.729631:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677447.729632:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.729633:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.729634:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.729634:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.729635:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.729636:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.729638:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.729639:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.729694:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.729697:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.729699:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.729701:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.729701:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.729703:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677447.729704:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.729705:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.729706:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.729707:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677447.729711:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.729713:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.2:1619677447.729714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:30.0:1619677447.729716:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.729716:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677447.729717:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677447.729719:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.729719:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677447.729719:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677447.729720:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.729721:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677447.729721:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.729724:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.729725:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.729726:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.729728:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.729729:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724247] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.729731:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.729732:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.729733:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:20.0:1619677447.729733:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.729736:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:20.0:1619677447.729736:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.729736:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.729737:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.729737:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:22.0:1619677447.729739:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.729740:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.729741:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.729741:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.729742:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677447.729744:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.729744:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.729745:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.729745:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.0:1619677447.729757:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:6.0:1619677447.729757:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677447.729760:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:6.0:1619677447.729760:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.729761:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.729761:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.729762:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:6.0:1619677447.729763:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.729764:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.729765:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:8.0:1619677447.729767:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.729768:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.729769:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.729771:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.729772:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.729773:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.729776:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.729778:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.729781:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.729782:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.729783:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.729785:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.729785:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.729786:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.729788:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.729788:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677447.729791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.729799:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677447.729801:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:2.0:1619677447.729802:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:24.0:1619677447.729803:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:18.0:1619677447.729803:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.729803:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.729804:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.729805:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:2.0:1619677447.729805:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.729809:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.729817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.729819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.729826:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.729829:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.729830:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.729830:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.729832:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.729832:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.729854:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.729857:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.729857:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.729860:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.729862:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.729863:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:22.0:1619677447.729864:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:2.0:1619677447.729865:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.729866:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:2.0:1619677447.729866:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.729867:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:2.0:1619677447.729867:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.729868:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.729868:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.729870:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.729872:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.729874:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.729875:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.729877:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.729878:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.729879:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.729880:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.729881:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.729881:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.729882:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.729886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.729898:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:1.0:1619677447.729900:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.729901:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.729902:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.729903:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.729903:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.0:1619677447.729904:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:17.0:1619677447.729904:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677447.729906:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:1.0:1619677447.729906:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.729909:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.729910:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.729912:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.729913:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.729914:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.729915:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.729917:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.729919:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.729921:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.729922:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.729922:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.729924:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.729924:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.729925:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.729926:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.729930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.729937:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.729939:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.729940:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.729940:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.729941:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.729943:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.730034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.730042:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.730044:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.730046:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.730047:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.730050:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.730052:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.730053:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.730054:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.730056:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.730056:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.730060:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.730062:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.730065:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.730066:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.730067:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.730068:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.730069:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.730070:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.730071:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.730071:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.730074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.730077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.2:1619677447.730082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:1.0:1619677447.730082:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.730085:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677447.730085:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.730085:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.730086:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.730087:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.730088:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677447.730088:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.730089:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677447.730090:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:1.0:1619677447.730091:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677447.730092:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.730094:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.730095:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.730096:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.730099:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:8.2:1619677447.730101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.730109:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.730112:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.730113:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:8.2:1619677447.730114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.730114:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677447.730114:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.730115:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:17.0:1619677447.730122:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.730125:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.730125:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.730126:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.730127:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.730127:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.730252:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.730255:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.730256:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.730258:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.730259:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.730261:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.730262:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.730263:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.730264:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.730265:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.730269:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.730270:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.730273:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.730274:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.730275:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.730276:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.730277:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.730277:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.730278:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.730292:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.730294:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.730301:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.730301:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.730302:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.730304:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.730304:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.730307:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.730307:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.730308:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:22.0:1619677447.730309:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.730310:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.730310:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.730311:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.730311:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.730312:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.730312:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.730313:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.730313:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.730314:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:22.0:1619677447.730315:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:4.2:1619677447.730316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:22.0:1619677447.730318:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.730320:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.2:1619677447.730321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:22.0:1619677447.730323:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.730324:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.730324:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.730326:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.730327:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.730328:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.730328:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677447.730330:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.730333:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.730335:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.730338:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.730339:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.730339:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:31.0:1619677447.730340:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:30.0:1619677447.730341:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.730342:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96724237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.730342:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.730344:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.730344:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.730345:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.730345:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.730348:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.730349:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:31.0:1619677447.730350:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.730352:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.730355:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.730356:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.730356:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.730358:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.730359:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.730359:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.730360:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.730365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.730373:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.730376:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.730377:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.730379:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.730380:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.730381:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677447.730385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677447.730392:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.730395:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.730396:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.730396:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.730397:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.730398:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.730487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.730492:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.730495:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.730497:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.730497:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.730499:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.730500:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677447.730502:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.730503:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.730504:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.730505:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.730508:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.730510:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.730513:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.730514:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.730514:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.730516:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.730517:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.730518:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.730519:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.730520:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677447.730522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677447.730525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.730528:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.730529:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.730530:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:17.0:1619677447.730530:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677447.730531:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677447.730533:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.730534:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.730534:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.730536:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677447.730539:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.0:1619677447.730545:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.730549:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.730550:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.730552:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.730555:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.730556:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.730558:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.730559:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.730560:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.730561:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.730564:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.730566:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.730569:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.730570:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.730571:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.730572:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.730573:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.730573:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.730574:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677447.730578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.730586:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.730588:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.730589:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.730591:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.730592:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.730594:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.730602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.2:1619677447.730610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.730618:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.730621:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.730621:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.730622:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.730623:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.730624:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.730643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.730647:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.730650:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.730651:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.730651:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.730653:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.730656:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677447.730689:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.730693:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.730694:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.730696:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.730698:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.730699:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.730701:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.730702:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.730703:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.730704:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.730707:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.730709:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.2:1619677447.730711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677447.730711:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.730711:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.730712:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.730714:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.730714:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.730715:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.730716:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.730717:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.730719:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:9.2:1619677447.730720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677447.730721:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.730722:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.730724:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.730726:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.730727:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.730727:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:5.0:1619677447.730729:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.730729:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:5.0:1619677447.730730:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.730730:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.730732:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:5.0:1619677447.730732:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.730733:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.730735:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:5.0:1619677447.730739:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.730741:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.730744:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.730745:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.730746:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.730747:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.730748:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.730749:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.730750:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.730751:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.730756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.730758:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:22.0:1619677447.730759:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.730761:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:20.0:1619677447.730761:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:6.0:1619677447.730761:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:22.0:1619677447.730762:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.730762:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.730762:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.730762:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.730763:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.730764:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.730765:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.730768:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.730768:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.730769:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.730769:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:30.0:1619677447.730771:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.730774:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.730776:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.730777:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677447.730778:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.730779:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.730780:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:30.0:1619677447.730781:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:22.0:1619677447.730781:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:30.0:1619677447.730782:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.730783:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677447.730784:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.730784:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000400:00000200:6.0:1619677447.730784:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.730785:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.730785:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.730785:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:30.0:1619677447.730786:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:22.0:1619677447.730786:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.730787:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:6.0:1619677447.730787:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.730790:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677447.730791:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.730791:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.730793:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:6.0:1619677447.730793:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.730794:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.730795:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:30.0:1619677447.730796:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.730797:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677447.730797:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677447.730799:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.730799:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677447.730799:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.730800:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.730801:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.730801:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.730803:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.730804:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.730805:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.730806:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.730807:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.730807:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.730808:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.730812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.730820:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:20.2:1619677447.730821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677447.730823:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.730824:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.730824:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.730826:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.730828:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:31.0:1619677447.730829:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.730832:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:9.2:1619677447.730833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000400:00000200:31.0:1619677447.730834:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.730836:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.730837:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.730839:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677447.730840:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.730840:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000400:00000200:31.0:1619677447.730841:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.730842:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.730843:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677447.730844:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.730844:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.730844:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.730845:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.730846:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:31.0:1619677447.730848:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.730849:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677447.730853:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.730854:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677447.730854:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677447.730856:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677447.730857:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.730858:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.730858:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.730862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.730870:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.730872:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.730873:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.730875:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.730876:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.730878:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.730884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.730891:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.730895:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.730895:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.730897:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.730900:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.730901:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.730902:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.730904:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.730905:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.730906:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:4.2:1619677447.730908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677447.730909:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.730910:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.730913:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.730914:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.730915:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.730917:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.730917:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.730918:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.730918:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.730923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.730933:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.730935:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.730935:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.730936:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.730937:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.730939:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.730940:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.730942:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.730942:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.730944:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:1.0:1619677447.730944:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.730945:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.730946:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.730946:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.730947:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.730947:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.730948:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.730948:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.730949:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.730949:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:17.0:1619677447.730950:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.730954:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.730956:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.730959:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.730960:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.730961:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.730962:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.730963:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.730963:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.730964:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.730968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.730974:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.730977:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.730978:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.730978:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.730980:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.730984:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677447.731043:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.731045:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.731046:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.731048:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.731051:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.731052:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.731053:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.731055:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.731056:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.731057:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.731060:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.731062:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.731065:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.731066:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.731067:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.731068:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.731069:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.731069:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.731069:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677447.731074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.731083:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.731085:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.731086:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.731088:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.731088:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.731091:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677447.731249:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.731251:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.731252:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.731254:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.731256:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.731258:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.731259:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.731260:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677447.731260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:18.0:1619677447.731261:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.731262:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.731266:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.731267:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.731270:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.731271:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.731272:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.731273:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.731273:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.731275:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.731276:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.731277:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.731287:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.731289:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.731290:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677447.731290:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.2:1619677447.731291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.2:1619677447.731292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:9.2:1619677447.731299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677447.731300:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:1.0:1619677447.731302:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:30.0:1619677447.731303:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.731305:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.731305:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.731306:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.731306:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.731308:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.731308:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.731308:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:1.0:1619677447.731308:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.731310:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:2.0:1619677447.731310:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:30.0:1619677447.731311:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.731311:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:30.0:1619677447.731312:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.731312:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.731312:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.731315:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.731315:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.731316:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.731318:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:30.0:1619677447.731320:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.731322:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.731324:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.731325:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.731325:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.731327:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.731327:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.731328:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.731329:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.731341:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.731343:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.731344:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.731346:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.731348:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.731349:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677447.731420:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.731422:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.731423:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.731425:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.731428:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.731429:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.731430:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.731432:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.731433:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.731434:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.731437:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.731439:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.731442:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.731443:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677447.731443:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.731443:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.731445:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.731445:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677447.731446:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:17.0:1619677447.731447:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677447.731447:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677447.731448:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.731448:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677447.731449:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677447.731451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:22.0:1619677447.731452:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.731453:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.731454:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.731456:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.731457:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.731457:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:22.0:1619677447.731458:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:1.0:1619677447.731458:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677447.731460:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.731461:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:22.0:1619677447.731461:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.731461:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.731461:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.731461:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.731462:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677447.731463:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.731464:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.731466:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.731466:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677447.731467:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.731467:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.731469:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.731470:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.731470:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.731471:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.731475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.731483:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.731485:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.731486:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.731487:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.731489:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.731491:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.731508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.731557:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.731560:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.731561:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.731561:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.731563:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.731566:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.731569:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.731571:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.731572:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.731574:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.731577:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.731578:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.731579:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.731581:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.731582:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.731583:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.731586:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.731588:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.731591:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.731592:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.731592:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.731594:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.731594:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.731595:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.731595:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677447.731600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.731609:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.731611:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.731612:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.731613:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.731615:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.731616:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677447.731719:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.731722:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.731724:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.731725:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.731726:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.731728:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.731729:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.731730:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.731731:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.731732:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.731735:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.731737:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.731739:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.731740:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.731741:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.731742:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.731743:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.731744:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.731744:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.731754:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.731757:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.731757:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.731759:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.731761:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.731762:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.731772:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.731775:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.731775:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.731776:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.731777:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.731777:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.731784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.731792:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.731794:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.731796:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.731798:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.731798:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.731799:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.731800:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.731802:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.731803:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.731804:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.731807:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.731808:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.731810:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.731811:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.731812:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.731813:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.731814:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.731815:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.731817:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.731817:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677447.731823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677447.731825:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.731827:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.731828:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.731830:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.731831:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:13.0:1619677447.731832:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.731832:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677447.731833:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.731834:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.731835:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:22.0:1619677447.731837:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.731840:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.731841:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.731843:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.731844:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677447.731845:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.731845:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:22.0:1619677447.731846:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.731848:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.731848:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.731848:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.731849:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:13.0:1619677447.731849:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:22.0:1619677447.731850:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.731850:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.731850:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:22.0:1619677447.731851:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.731851:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:22.0:1619677447.731852:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677447.731853:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.731856:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.731856:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.731857:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:22.0:1619677447.731858:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:17.0:1619677447.731859:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.731860:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.731862:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677447.731862:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.731863:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.731863:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:17.0:1619677447.731863:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:22.0:1619677447.731865:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.731866:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.731867:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:17.0:1619677447.731867:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.731868:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:17.0:1619677447.731868:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:22.0:1619677447.731869:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677447.731871:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.731872:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.731873:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.731874:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.731875:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.731875:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.731876:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.731879:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:9.2:1619677447.731880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:6.0:1619677447.731881:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.731882:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.731882:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.731886:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.731889:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.731890:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.731890:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.731892:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.731895:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:2.0:1619677447.732070:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.732073:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.732074:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.732076:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.732079:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.732080:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.732081:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.732083:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.732084:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677447.732085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677447.732085:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.732088:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.732090:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.732092:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.732092:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.732093:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.732094:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.732095:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.732096:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.732096:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.732097:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:9.2:1619677447.732100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.732108:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.732109:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.732109:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.732109:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.732110:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.732110:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.732112:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.732112:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.732113:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677447.732114:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.732127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.732132:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.732134:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.732135:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.732136:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.732138:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.732141:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.732190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.732199:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.732201:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.732203:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.732205:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.732206:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.732207:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.732208:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.732210:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.732211:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.732212:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.732215:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.732217:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.732219:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.732219:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.732220:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.732222:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.732222:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.732223:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.732224:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677447.732234:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.732236:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.732237:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.732239:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.732240:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:4.2:1619677447.732240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677447.732241:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:24.0:1619677447.732247:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.732250:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.732251:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.732252:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.732255:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.732257:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.732258:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.732259:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.732260:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.732261:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.732264:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.732265:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.732268:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.732269:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.732270:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.732271:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.732272:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.732273:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.732273:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.732283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.732291:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.732297:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.732299:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.732300:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.732302:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.732305:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.732330:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.732333:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.732334:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.732335:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.732337:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.732339:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.732340:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.732342:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.732343:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.732344:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.732347:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.732349:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.732352:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.732353:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.732353:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.732355:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.732356:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.732356:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.732357:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.732369:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.732372:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.732373:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.732374:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.732376:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.732377:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.732460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.732466:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.732469:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.732471:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.732472:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.732473:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.732475:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677447.732476:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.732478:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.732479:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.732480:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677447.732483:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.732485:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.732487:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.732488:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.732489:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.732490:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.732491:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.732492:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.732493:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677447.732497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.732505:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.732507:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.732508:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.732510:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.732511:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.732512:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677447.732516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.732524:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.732527:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.732528:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.732528:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.732529:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.732530:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.732549:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.732552:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.732553:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.732555:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.732557:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.732558:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677447.732560:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.732561:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.732562:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.732563:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677447.732567:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.732569:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677447.732571:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.732572:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677447.732573:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677447.732575:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.732575:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.732576:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.732576:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.732581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.732587:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.732590:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.732591:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.732591:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.732594:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.732596:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.732760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.2:1619677447.732800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.732809:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:4.2:1619677447.732811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:20.0:1619677447.732813:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.732814:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.732816:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.732816:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.732819:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:20.0:1619677447.732819:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:24.0:1619677447.732822:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677447.732822:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677447.732824:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.732827:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.732828:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.732829:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.732830:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.732831:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.732832:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.732835:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.732837:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.732839:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.732840:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.732841:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.732842:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.732843:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677447.732843:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:18.0:1619677447.732843:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:24.0:1619677447.732844:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:18.0:1619677447.732846:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.732847:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.732848:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:8.2:1619677447.732848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:18.0:1619677447.732851:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.732852:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.732853:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.732855:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.732856:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.732856:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:18.0:1619677447.732857:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677447.732859:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:18.0:1619677447.732860:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677447.732860:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.732860:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.732861:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:1.0:1619677447.732862:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.732864:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.732865:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.732865:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677447.732865:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677447.732867:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.732867:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.732868:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.732869:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677447.732881:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.732884:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.732885:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.732885:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.732887:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.732889:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.732944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.732952:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.732955:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.732957:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.732960:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.732960:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.732961:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.732962:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.732964:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.732965:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.732966:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.732969:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.732971:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.732974:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.732975:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.732976:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.732977:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.732978:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.732979:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.732979:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.732983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.732992:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.732994:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.732996:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.732997:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.732998:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.733002:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677447.733018:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.733021:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.733022:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.733023:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.733025:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.733026:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:2.0:1619677447.733209:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.733212:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.733213:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.733216:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.733218:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.733219:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.733221:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.733222:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.733223:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.733224:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.733227:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.733229:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.733231:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.733231:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.733232:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.733233:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.733233:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.733234:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.733235:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.733239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.733246:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.733248:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.733248:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.733250:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.733250:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.733251:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.733260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.733267:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.733269:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.733270:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.733270:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.733271:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.733271:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.733417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.733464:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.733467:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.733468:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.733468:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.733470:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.733473:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.733548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.733556:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.733559:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.733560:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.733561:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.733564:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.733565:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.733566:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.733568:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.733569:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.733570:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.733573:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.733574:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.733577:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.733578:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.733578:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.733580:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.733580:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.733581:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.733582:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.733586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677447.733588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:17.0:1619677447.733594:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.733597:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.733598:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.733599:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.733601:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.733603:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.733605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.733613:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.733616:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.733616:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.733617:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.733618:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.733618:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.733619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.733622:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.733625:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.733627:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677447.733627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:20.0:1619677447.733629:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.733630:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:5.0:1619677447.733630:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.733631:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:20.0:1619677447.733632:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.733633:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.733633:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.733634:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.733634:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.733635:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677447.733636:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.733639:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.733641:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.733641:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.733642:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.733644:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.733645:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677447.733645:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677447.733648:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677447.733648:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.733649:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.733650:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:13.0:1619677447.733727:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.733729:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.733730:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.733731:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.733734:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.733735:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677447.733776:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.733778:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.733780:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.733783:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.733783:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.733784:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677447.733786:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.733787:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.733788:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.733788:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677447.733792:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.733793:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677447.733795:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677447.733796:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.733797:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677447.733798:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677447.733798:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677447.733799:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677447.733799:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677447.733800:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.733801:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:17.0:1619677447.733801:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.733803:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.733804:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677447.733804:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.733805:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.733806:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.733808:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.733809:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.733810:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.733813:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.733813:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677447.733815:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.733816:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:30.0:1619677447.733817:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677447.733817:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.733817:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.733818:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.733818:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677447.733819:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:17.0:1619677447.733819:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:6.0:1619677447.733819:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.733820:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.733821:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677447.733821:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.733821:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.733822:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.733822:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.733823:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.733832:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.733834:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.733835:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.733835:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.733836:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.733836:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.733836:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.733837:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.733838:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:6.0:1619677447.733838:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.733839:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.733841:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.733857:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.733859:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.733860:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.733860:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.733861:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.733862:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.733959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.733965:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.733967:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.733970:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.733970:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.733972:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.733974:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677447.733975:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.733976:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.733977:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.733978:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677447.733982:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.733983:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.733985:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.733986:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.733986:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.733988:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.733989:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.733989:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.733990:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677447.733994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.734002:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.734004:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.734005:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.734007:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.734008:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.734009:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.734309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.734324:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.734327:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.734328:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.734331:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.734333:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.734335:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.734337:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.734339:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.734339:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.734341:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.734344:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.734346:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.734349:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.734350:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.734351:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.734352:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.734353:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.734354:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.734354:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677447.734360:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.734363:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:8.2:1619677447.734363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677447.734364:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.734365:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.734367:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.734368:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:1.0:1619677447.734371:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.734374:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.734375:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.734375:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.734377:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.734379:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677447.734380:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677447.734383:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.734384:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.734385:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.734385:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.734387:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677447.734404:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.734406:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.734407:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.734409:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.734411:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.734413:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.734414:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.734415:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.734416:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.734418:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.734421:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.734423:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.734425:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.734426:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.734426:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.734428:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.734429:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.734430:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.734431:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.734431:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.734442:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677447.734442:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.734444:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677447.734445:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677447.734445:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677447.734446:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.734446:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.734446:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677447.734448:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.734450:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.734509:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.734512:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.734512:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.734514:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.734516:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.734517:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.734652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.734660:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677447.734661:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:24.0:1619677447.734663:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:18.0:1619677447.734664:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:8.0:1619677447.734664:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677447.734665:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:24.0:1619677447.734667:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.734668:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.734669:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.734669:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.734671:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:18.0:1619677447.734671:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.734673:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.734673:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.734676:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.734676:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.734678:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.734678:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.734680:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:18.0:1619677447.734680:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.734685:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.734686:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.734688:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.734689:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:18.0:1619677447.734691:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.734692:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.734692:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.734693:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:18.0:1619677447.734693:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.734694:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:18.0:1619677447.734695:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.734696:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.734696:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677447.734696:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.734697:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.734697:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:24.0:1619677447.734698:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.734699:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.734700:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.734702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677447.734705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677447.734708:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677447.734709:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:13.0:1619677447.734711:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677447.734712:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:13.0:1619677447.734712:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.734712:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:17.0:1619677447.734713:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.734713:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:13.0:1619677447.734714:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.734714:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.734714:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677447.734715:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.734715:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677447.734716:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.734716:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.734719:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677447.734746:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.734749:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.734751:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677447.734751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677447.734752:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.734754:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.734755:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.734756:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.734757:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677447.734758:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.734758:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677447.734760:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.734760:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:30.0:1619677447.734761:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.734761:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677447.734761:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.734763:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.734764:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.734764:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.734766:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.734766:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.734767:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.734768:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.734768:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:5.0:1619677447.734769:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.734770:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.734771:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677447.734771:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.734772:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:1.0:1619677447.734772:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.734773:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677447.734773:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.734773:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.734774:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677447.734774:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.734774:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:30.0:1619677447.734775:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.734776:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.734776:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:5.0:1619677447.734776:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.734779:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.734780:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.734781:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:5.0:1619677447.734783:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.734784:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.734785:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.734785:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:12.0:1619677447.734786:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.734786:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.734787:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:1.0:1619677447.734787:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:5.0:1619677447.734788:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (24)++
00000800:00000200:1.0:1619677447.734788:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.734789:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.734790:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677447.734791:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (25)--
00000800:00000200:5.0:1619677447.734791:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677447.734792:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:5.0:1619677447.734792:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677447.734793:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.734793:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.734793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.0:1619677447.734793:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:12.0:1619677447.734795:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.734799:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:12.0:1619677447.734799:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.734800:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677447.734802:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:12.0:1619677447.734802:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:24.0:1619677447.734803:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.734803:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.734803:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:13.0:1619677447.734804:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.734805:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.734806:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.734807:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.734809:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.734817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:13.0:1619677447.734824:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.734824:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.734827:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:13.0:1619677447.734827:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.734827:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:13.0:1619677447.734828:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677447.734828:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (21)++
00000800:00000200:13.0:1619677447.734829:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.734830:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.734830:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.734830:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677447.734831:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.735012:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.735015:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.735016:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.735018:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.735021:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.735023:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.735024:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.735025:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.735027:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.735028:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.735031:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.735033:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.735036:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.735037:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.735038:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.735040:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.735040:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.735041:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.735041:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677447.735052:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.735055:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.735055:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.735057:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.735058:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.735060:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.735129:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.735132:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.735134:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.735136:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.735136:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.735138:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.735139:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.735140:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.735141:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.735142:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.735145:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.735147:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.735150:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.735150:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.735151:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.735152:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.735153:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.735154:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.735155:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677447.735161:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.735164:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:12.0:1619677447.735164:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677447.735165:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.735167:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.735167:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.735167:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.735169:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.735170:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.735171:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:17.0:1619677447.735172:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:12.0:1619677447.735172:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:17.0:1619677447.735173:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.735174:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.735175:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.735176:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.735179:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.735180:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.735182:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.735183:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.735184:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.735185:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.735186:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.735186:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.735187:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.735191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.735199:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.735201:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.735201:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.735202:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.735203:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.735204:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.735210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.735213:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.735215:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.735218:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.735219:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.735220:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.735222:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.735223:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.735224:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.735225:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.735226:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.735229:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.735232:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:4.2:1619677447.735232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.735235:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.735236:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.735236:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.735238:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.735239:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.735240:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.735241:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.735242:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.735247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.735252:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.735254:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.735255:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.735256:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.735264:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:22.0:1619677447.735267:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.735267:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000400:00000200:6.0:1619677447.735268:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.735270:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.735272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.735273:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.735286:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.735289:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.735289:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.735290:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.735291:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.735297:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.735385:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.735388:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.735388:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.735390:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.735392:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.735393:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.735418:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.735420:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.735421:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.735423:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.735425:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.735426:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.735428:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.735429:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.735430:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.735431:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.735434:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.735436:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.735439:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.735440:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.735440:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.735442:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.735442:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.735443:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.735444:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.735456:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.735459:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.735459:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.735461:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.735463:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.735464:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:24.0:1619677447.735557:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677447.735560:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677447.735561:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677447.735563:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677447.735565:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.735567:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677447.735568:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.735569:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:24.0:1619677447.735569:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.735570:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.735571:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677447.735572:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.735575:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.735576:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:24.0:1619677447.735576:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:31.0:1619677447.735577:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.735578:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:31.0:1619677447.735579:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.735580:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.735581:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.735581:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.735582:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.735582:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:31.0:1619677447.735583:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.735583:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677447.735585:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677447.735585:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000400:00000200:31.0:1619677447.735586:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.735586:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677447.735586:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:31.0:1619677447.735588:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.735590:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.735591:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.735591:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.735592:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.735593:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.735594:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.735594:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677447.735598:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677447.735598:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677447.735601:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:6.0:1619677447.735601:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.735602:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.735602:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:6.0:1619677447.735603:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.735604:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:18.0:1619677447.735605:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.735605:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:13.0:1619677447.735607:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.735607:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.735608:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:18.0:1619677447.735609:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.735610:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:13.0:1619677447.735610:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.735610:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:18.0:1619677447.735612:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.735612:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:18.0:1619677447.735613:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.735614:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.735615:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.735619:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.735619:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:18.0:1619677447.735621:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.0:1619677447.735622:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.735623:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:18.0:1619677447.735623:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.735624:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.735625:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677447.735625:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:8.0:1619677447.735625:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.735626:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.735627:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.735628:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.735628:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.735628:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.735628:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.735629:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677447.735629:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:12.0:1619677447.735630:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.735630:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.735631:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677447.735631:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96724172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677447.735632:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:8.0:1619677447.735637:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.735638:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.735639:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.735642:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.735642:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:8.0:1619677447.735644:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.735644:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.735645:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.735647:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.735648:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:8.0:1619677447.735648:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.735649:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.735649:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.735650:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.0:1619677447.735652:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.735652:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:20.2:1619677447.735653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.0:1619677447.735654:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.735655:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:20.0:1619677447.735656:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677447.735656:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677447.735658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677447.735659:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677447.735662:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.735663:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.735665:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.735666:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677447.735666:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:20.0:1619677447.735667:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.735668:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:24.0:1619677447.735669:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:20.0:1619677447.735669:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:18.0:1619677447.735669:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677447.735670:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:2.0:1619677447.735670:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.735671:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:20.0:1619677447.735671:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.735672:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:2.0:1619677447.735672:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.735673:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:24.0:1619677447.735673:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:20.0:1619677447.735673:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.735674:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.735675:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.735676:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.735676:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:30.0:1619677447.735677:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.735679:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677447.735680:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677447.735681:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:30.0:1619677447.735682:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.735683:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.735684:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677447.735685:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.735688:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.735689:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.735691:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.735693:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.735694:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.735694:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.735695:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.735695:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.735696:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677447.735696:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677447.735696:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.735697:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.735697:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677447.735698:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677447.735698:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677447.735699:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:22.0:1619677447.735699:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:20.0:1619677447.735699:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.735699:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677447.735700:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:22.0:1619677447.735703:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677447.735703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:22.0:1619677447.735705:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.735706:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.735708:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96724168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.735709:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.735709:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:22.0:1619677447.735711:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.735711:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:22.0:1619677447.735712:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677447.735712:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.735712:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:13.0:1619677447.735715:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.735715:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.735715:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:22.0:1619677447.735716:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.735716:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.735718:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:13.0:1619677447.735718:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677447.735718:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.735719:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.735721:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.735721:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677447.735722:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.735723:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:9.2:1619677447.735724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.735725:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.735725:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.735726:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.735727:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677447.735731:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.2:1619677447.735731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.735734:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.735735:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.735735:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.735738:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.735739:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677447.735741:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.735741:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.735742:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.735742:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.735744:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.735747:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.735901:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.735904:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.735905:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.735907:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.735909:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.735910:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.735911:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.735913:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.735914:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.735915:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.735918:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.735920:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.735922:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.735923:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.735923:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.735925:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.735925:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.735926:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.735927:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.735928:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677447.735931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.735939:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.735940:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.735940:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.735940:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.735941:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.735941:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.735942:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.735942:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.735943:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.735944:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.735970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.736028:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.736031:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.736032:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.736032:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.736035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000400:00000200:6.0:1619677447.736035:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.736038:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.736043:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.736045:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.736046:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.736048:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:20.0:1619677447.736048:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.736050:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:22.0:1619677447.736051:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.736051:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:22.0:1619677447.736052:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.736052:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.736053:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.736054:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.736082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.736089:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.736092:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.736093:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.736095:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.736098:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.736100:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.736101:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.736102:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.736103:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.736104:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.736108:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.736109:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.736112:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.736113:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.736113:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.736115:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.736116:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.736116:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.736117:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.736121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.736129:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.736132:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.736132:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.736133:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.736135:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.736138:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.736140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.736147:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.736150:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.736150:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.736151:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.736152:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.736152:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:31.0:1619677447.736175:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.736178:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.736180:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.736182:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.736184:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.736185:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677447.736187:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.736188:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.736189:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.736190:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677447.736193:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.736194:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677447.736196:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.736197:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677447.736198:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677447.736199:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.736200:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.736201:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.736203:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.736203:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.736216:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.736219:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.736219:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.736220:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677447.736220:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.736223:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.736223:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.736225:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.736228:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.736229:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.736230:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.736231:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.736232:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.736233:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.736236:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.736238:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.736241:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.736242:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.736243:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.736245:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.736245:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.736246:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.736246:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.736251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.736258:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.736261:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.736262:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.736262:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.736265:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.736268:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.736271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.736279:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.736282:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.736284:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.736285:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.736286:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.736288:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677447.736289:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.736290:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.736291:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.736292:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677447.736300:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.736302:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677447.736305:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.736306:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.736306:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677447.736308:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.736309:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.736310:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.736312:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.736313:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677447.736315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677447.736317:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.2:1619677447.736319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677447.736320:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.736320:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.736322:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.736324:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.736324:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:17.0:1619677447.736325:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:6.0:1619677447.736325:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:17.0:1619677447.736327:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.736327:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:31.0:1619677447.736328:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:6.0:1619677447.736328:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:5.0:1619677447.736328:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.736329:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:17.0:1619677447.736329:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.736330:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:20.0:1619677447.736330:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:17.0:1619677447.736330:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.736330:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.736331:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:17.0:1619677447.736331:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.736332:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677447.736332:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:6.0:1619677447.736333:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.736333:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.736334:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.736335:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (22)--
00000400:00000200:17.0:1619677447.736335:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.736335:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.736335:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.736336:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:6.0:1619677447.736337:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:17.0:1619677447.736338:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.736338:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.736340:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677447.736340:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.736341:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:6.0:1619677447.736341:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.736342:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:6.0:1619677447.736342:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:17.0:1619677447.736343:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.736344:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.736344:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.736345:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677447.736345:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.736347:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.2:1619677447.736349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.736350:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.736350:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.736351:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.736353:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.736353:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.736354:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.736355:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677447.736356:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.736358:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.736359:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.736360:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:9.2:1619677447.736360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:1.0:1619677447.736361:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.736363:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677447.736368:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:8.2:1619677447.736368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.736371:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.736372:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.736373:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.736374:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.736376:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:17.0:1619677447.736376:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.736379:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.736380:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.736380:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.736381:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.736381:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.736404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.736451:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.736454:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.736455:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.736455:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.736457:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.736460:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.736532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f3d4600] (20)++
00000800:00000200:6.0:1619677447.736537:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d4600] (21)++
00000800:00000200:22.0:1619677447.736540:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d4600] (22)--
00000800:00000200:6.0:1619677447.736540:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.11@o2ib3
00000400:00000200:6.0:1619677447.736543:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.12.11@o2ib3 : GET - for me
00000400:00000200:6.0:1619677447.736546:0:19937:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.12.11@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:6.0:1619677447.736548:0:19937:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.12.11@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:6.0:1619677447.736549:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d4600] (21)++
00000800:00000200:6.0:1619677447.736550:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d4600] (22)--
00000400:00000200:6.0:1619677447.736552:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.12.11@o2ib3 local destination
00000400:00000200:6.0:1619677447.736556:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.12.11@o2ib3(10.51.12.11@o2ib3:10.51.12.11@o2ib3) : REPLY try# 0
00000800:00000200:6.0:1619677447.736558:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.12.11@o2ib3
00000800:00000200:6.0:1619677447.736561:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91986aff6480] -> 10.51.12.11@o2ib3 (2) version: 12
00000800:00000200:6.0:1619677447.736561:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91986f3d4600] (21)++
00000800:00000200:6.0:1619677447.736562:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f3d4600] (22)++
00000800:00000200:6.0:1619677447.736563:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91986f3d4600] (23)--
00000800:00000200:6.0:1619677447.736564:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d4600] (22)--
00000800:00000200:20.2:1619677447.736569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f3d4600] (21)++
00000800:00000200:22.0:1619677447.736574:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d4600] (22)++
00000800:00000200:22.0:1619677447.736576:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f3d4600] (23)--
00000400:00000200:22.0:1619677447.736576:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.736577:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d4600] (22)--
00000400:00000200:22.0:1619677447.736578:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.12.11@o2ib3: REPLY: OK
00000800:00000200:22.0:1619677447.736579:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d4600] (21)--
00000800:00000200:20.2:1619677447.736717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.736725:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.736728:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.736730:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.736731:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.736733:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.736735:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.736736:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.736737:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.736738:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.736739:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.736743:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.736744:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.736747:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.736748:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.736749:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.736750:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.736751:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.736752:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.736752:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.736763:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.736766:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.736766:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.736768:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.736770:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.736771:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.736821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677447.736838:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.736840:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.736841:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.736844:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.736847:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.736848:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.736850:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96724155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.736851:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.736852:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.736853:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.736857:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.736858:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.736861:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.736862:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.736863:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.736865:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.736865:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.736866:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.736866:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677447.736870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:9.2:1619677447.736875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.736879:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.736881:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.736882:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.736883:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:5.0:1619677447.736884:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.736885:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.736885:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.736887:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.736887:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:2.0:1619677447.736887:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.736889:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.736892:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.736894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.736901:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.736904:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.736905:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.736905:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.736906:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.736907:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.736919:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.2:1619677447.736920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:22.0:1619677447.736922:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.736922:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:20.0:1619677447.736923:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:6.0:1619677447.736924:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.736925:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:6.0:1619677447.736926:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.736928:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.736928:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:5.0:1619677447.736929:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:6.0:1619677447.736930:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.736931:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.736931:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.736932:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.736933:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.736933:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.736935:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.736936:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.736938:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.736938:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.736939:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.736941:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.736943:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.736944:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.736945:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.736945:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.736946:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.736948:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.736948:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.736948:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677447.736949:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.736949:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:6.0:1619677447.736949:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.736950:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.736950:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677447.736951:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677447.736952:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.736952:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.736953:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677447.736961:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:22.0:1619677447.736961:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.736963:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:30.0:1619677447.736964:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:22.0:1619677447.736964:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.736964:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:30.0:1619677447.736965:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.736967:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.736968:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:31.0:1619677447.736969:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:30.0:1619677447.736969:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.736970:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.736971:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.736971:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.736974:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:2.0:1619677447.737042:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.737044:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.737046:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.737047:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.737050:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.737052:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677447.737053:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.737054:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.737056:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.737056:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677447.737060:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.737062:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677447.737065:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.737065:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677447.737066:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677447.737068:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677447.737068:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.737069:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.737069:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677447.737104:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.737107:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.737108:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.737110:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.737113:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.737114:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.737115:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.737116:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.737117:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.737118:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.737121:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.737122:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.737124:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.737125:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.737125:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.737127:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.737127:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.737129:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.737130:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.737130:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677447.737133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.737140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:2.0:1619677447.737141:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:1.0:1619677447.737141:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677447.737143:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.737143:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.737144:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677447.737144:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.737145:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.737145:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.737147:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:1.0:1619677447.737147:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.737150:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.737151:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677447.737151:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.737152:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:24.0:1619677447.737153:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.737154:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.737155:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677447.737156:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.737157:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.737158:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.737158:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.737159:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724149] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.737160:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677447.737161:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.737162:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724149] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.737163:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.737163:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.737165:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.737166:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.737166:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.737171:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.737172:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.737174:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.737176:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.737177:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.737178:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677447.737179:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.737179:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.737179:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677447.737180:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.737181:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.737182:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.737182:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.737182:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677447.737183:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.737183:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677447.737183:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677447.737184:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.737188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677447.737194:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677447.737195:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:12.0:1619677447.737196:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.737197:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.737198:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.737199:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.737199:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.737199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.737199:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:13.0:1619677447.737200:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:17.0:1619677447.737201:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.737201:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677447.737204:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:13.0:1619677447.737212:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.737214:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.737215:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.737216:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.737216:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.737217:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.737240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.737245:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.737248:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.737249:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.737250:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.737252:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.737255:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.737294:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.737303:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.737306:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.737308:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.737310:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.737311:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.737312:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677447.737314:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.737315:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.737316:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.737317:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677447.737320:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.737321:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.737323:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.737324:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.737324:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.737326:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.737326:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.737327:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.737327:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.737328:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677447.737328:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677447.737329:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.737330:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.737332:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.737334:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.737335:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.737336:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677447.737336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:31.0:1619677447.737337:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.737338:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.737338:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.737339:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.737340:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.737340:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.737341:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.737342:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.737342:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677447.737343:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.737344:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677447.737345:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.737345:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:31.0:1619677447.737346:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.737346:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.737347:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.737347:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677447.737348:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.737348:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.737349:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:5.0:1619677447.737349:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.737350:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:5.0:1619677447.737351:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677447.737352:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.737354:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.737354:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.737356:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677447.737359:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:13.0:1619677447.737361:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.737361:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:13.0:1619677447.737363:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.737364:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.737364:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.737365:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677447.737365:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:13.0:1619677447.737366:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.737367:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:13.0:1619677447.737368:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.737368:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:12.0:1619677447.737369:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.737369:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.737369:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677447.737380:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.737382:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.737383:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.737385:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677447.737386:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.737388:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677447.737398:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.737401:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.737402:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.737403:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677447.737403:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.737405:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677447.737453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.737513:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.737516:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.737517:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.737517:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.737519:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.737523:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.737693:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.737696:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.737697:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.737699:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.737702:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.737704:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677447.737705:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.737706:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.737707:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.737708:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677447.737711:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.737712:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.737715:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.737715:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.737716:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.737717:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.737718:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.737718:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.737719:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.737730:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.737732:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.737733:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.737734:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.737736:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.737737:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.737869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.737913:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.737916:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.737917:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.737918:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.737920:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.737923:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.738083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.738086:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.738088:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.738091:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.738092:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.738094:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.738095:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.738096:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.738098:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.738099:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.738100:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.738103:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.738105:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.738109:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.738109:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.738110:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.738112:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.738112:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.738113:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.738114:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677447.738125:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.738128:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.738128:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.738130:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.738131:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.738133:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677447.738146:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677447.738149:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.738150:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.738150:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:6.0:1619677447.738150:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677447.738151:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.738152:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:22.0:1619677447.738153:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.738153:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.738155:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.738158:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.738159:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:31.0:1619677447.738161:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:6.0:1619677447.738161:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.738162:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.738163:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:6.0:1619677447.738163:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.738164:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.738164:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:31.0:1619677447.738166:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.738167:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:6.0:1619677447.738168:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.738170:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:31.0:1619677447.738171:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:6.0:1619677447.738173:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.738174:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.738175:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.738176:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.738177:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.738178:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.738178:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.738191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.738198:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.738201:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.738201:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.738204:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.738206:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.738208:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.738209:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.738210:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.738211:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.738212:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.738215:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.738217:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.738220:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.738221:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.738221:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.738223:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.738223:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.738224:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.738225:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.738229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.738237:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.738240:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.738241:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.738241:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.738243:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.738246:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.738256:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.738258:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.738259:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.738261:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.738263:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.738265:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.738266:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.738267:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.738268:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.738269:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.738273:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.738275:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.738277:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.738278:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.738279:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.738280:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.738281:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.738281:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.738282:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.738299:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.738301:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.738302:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.738302:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.738304:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.738307:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.738429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677447.738475:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.738477:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.738478:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.738479:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.738482:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.738482:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.2:1619677447.738817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.738825:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.738827:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.738829:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.738831:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.738832:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.738832:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.738833:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.738834:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.738835:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.738836:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.738838:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.738840:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.738842:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.738843:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.738843:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.738845:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.738845:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.738846:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.738847:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.738858:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.738860:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.738861:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.738862:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.738864:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.738866:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.738899:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677447.738902:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677447.738903:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677447.738905:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677447.738908:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.738909:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677447.738910:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.738911:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.738912:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.738913:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677447.738916:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677447.738918:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.738919:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.738919:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677447.738920:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.738921:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.738922:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677447.738922:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677447.738923:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677447.738937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.738943:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.738945:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.738946:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.738947:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677447.738947:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.738949:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:1.0:1619677447.738971:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.738974:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.738974:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.738976:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.738979:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.738980:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.738981:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.738983:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.738984:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:1.0:1619677447.738984:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.738985:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:13.0:1619677447.738986:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.738987:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.738989:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677447.738990:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.738990:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.738991:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:13.0:1619677447.738992:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677447.738995:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.738995:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.738996:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.738998:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.738998:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.738999:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.739000:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.739001:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.739003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.739010:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:17.0:1619677447.739011:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677447.739012:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.739013:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.739013:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.739014:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677447.739014:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.739015:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677447.739016:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.739019:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.739053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.2:1619677447.739092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677447.739094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677447.739101:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.739104:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.739105:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.739107:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.739107:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.739110:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677447.739182:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.739184:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.739185:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.739187:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.739190:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.739192:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.739193:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.739194:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.739195:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.739196:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.739199:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.739201:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.739204:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.739205:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.739205:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.739207:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.739208:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.739209:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.739210:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.739210:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.739221:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.739223:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.739224:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.739225:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:31.0:1619677447.739233:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.739236:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.739238:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.739239:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.739240:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.739242:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677447.739243:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.739244:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.739245:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.739246:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677447.739249:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.739251:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677447.739254:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.739255:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677447.739256:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677447.739257:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.739258:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.739259:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.739259:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.739271:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.739274:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.739275:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.739275:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.739277:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.739280:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.739289:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.739292:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.739292:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.739297:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.739298:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.739298:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.739307:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fdae00] (21)++
00000800:00000200:20.0:1619677447.739310:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.38@o2ib3
00000400:00000200:20.0:1619677447.739312:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.38@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.739313:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (22)--
00000400:00000200:20.0:1619677447.739315:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.739316:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677447.739318:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.739319:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.739320:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.739321:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677447.739324:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.38@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.739326:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677447.739329:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.739329:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677447.739330:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677447.739332:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677447.739332:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fdae00] (21)++
00000800:00000200:20.0:1619677447.739333:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fdae00] (22)--
00000800:00000200:20.0:1619677447.739334:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (21)--
00000800:00000200:9.2:1619677447.739348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.739356:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:1.0:1619677447.739357:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677447.739359:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.739359:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.739359:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:24.0:1619677447.739360:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.739360:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677447.739363:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.739364:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.739366:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:1.0:1619677447.739367:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.739369:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.739370:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677447.739371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000400:00000200:1.0:1619677447.739371:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.739372:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.739373:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.739376:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.739377:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.739378:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:24.0:1619677447.739380:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.739380:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.739381:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:1.0:1619677447.739381:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.739382:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.739382:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:1.0:1619677447.739382:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.739382:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.739384:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.739384:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.739385:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.739386:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677447.739395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:6.0:1619677447.739398:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.739400:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.739401:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.739401:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.739402:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:6.0:1619677447.739403:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.739405:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.739406:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677447.739406:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:24.0:1619677447.739407:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.739410:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.739411:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.739412:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.739414:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.739415:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.739416:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.739419:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.739420:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.739420:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:24.0:1619677447.739423:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.739423:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:24.0:1619677447.739424:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.739424:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.739424:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:24.0:1619677447.739425:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.739425:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:24.0:1619677447.739426:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.739426:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.739427:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.739428:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.739430:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.739430:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.739434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.739436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.739442:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677447.739443:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677447.739444:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677447.739445:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:1.0:1619677447.739445:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.739446:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.739446:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.739447:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677447.739448:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.739452:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.739452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.739460:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.739462:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.739463:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.739463:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.739464:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.739465:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677447.739479:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.739483:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.739483:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.739486:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.739488:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.739490:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.739491:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.739493:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.739494:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.739495:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.739498:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.739499:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.739502:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.739503:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.739504:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.739506:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.739506:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.739507:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.739507:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.739520:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.739523:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.739524:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.739525:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.739527:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.739528:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.739683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677447.739687:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.739690:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.739693:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.739695:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.739695:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.739696:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.739698:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.739699:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.739700:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.739702:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.739706:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.739708:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.739711:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.739712:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.739712:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.739714:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.739715:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.739716:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.739716:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677447.739723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:13.0:1619677447.739726:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.739729:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.739729:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.739731:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.739732:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677447.739732:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.739733:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.739734:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:5.0:1619677447.739734:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:1.0:1619677447.739735:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677447.739736:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:17.0:1619677447.739737:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.739737:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.739739:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:17.0:1619677447.739739:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.739739:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.2:1619677447.739741:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:17.0:1619677447.739741:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.739742:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.739742:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:17.0:1619677447.739743:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.739743:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:17.0:1619677447.739744:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.739744:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.739745:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:17.0:1619677447.739746:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677447.739748:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677447.739748:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.739749:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.739749:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:5.0:1619677447.739749:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.739750:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.739750:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.739751:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677447.739752:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.739753:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:5.0:1619677447.739754:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.739755:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.739756:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.739756:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:5.0:1619677447.739756:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.739757:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:5.0:1619677447.739758:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.739759:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.739759:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677447.739759:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.739760:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.739761:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:5.0:1619677447.739762:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.739764:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.739767:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.739768:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.739769:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.739771:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.739772:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.739773:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.739774:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.739775:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677447.739777:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.739779:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.739780:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:20.2:1619677447.739780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:6.0:1619677447.739780:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.739783:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677447.739783:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677447.739783:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.739785:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:6.0:1619677447.739785:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677447.739786:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677447.739786:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677447.739787:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:12.0:1619677447.739787:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.739789:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.739790:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.739792:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.739821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:8.2:1619677447.739858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.739866:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.739868:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.739869:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.739871:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.739872:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.739875:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677447.739879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.739887:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.739889:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.739890:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.739891:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.739892:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.739892:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:13.0:1619677447.739908:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.739911:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.739912:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.739913:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.739915:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.739916:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.740055:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.740058:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.740059:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.740061:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.740063:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.740065:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.740066:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.740067:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.740068:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.740069:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.740073:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.740074:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.740077:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.740077:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.740078:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.740080:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.740080:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.740081:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.740081:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.740085:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.740088:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.740091:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.740092:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.740093:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.740093:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:31.0:1619677447.740095:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677447.740096:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.740096:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:31.0:1619677447.740097:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.740097:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.740097:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:31.0:1619677447.740098:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.740099:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.740100:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:22.0:1619677447.740102:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:31.0:1619677447.740103:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.740105:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677447.740108:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.740108:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677447.740109:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677447.740111:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.740111:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.740113:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.740114:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.740115:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.740128:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.740130:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.740131:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.740132:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677447.740220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.740226:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.740229:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.740231:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.740231:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.740234:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.740235:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.740237:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.740238:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.740239:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.740240:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.740243:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.740245:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.740247:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.740248:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.740249:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.740251:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.740251:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.740252:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.740253:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677447.740257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.740264:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.740267:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.740268:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.740269:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.740270:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.740273:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.740418:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.740421:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.740422:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.740424:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.740426:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.740427:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.740428:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.740430:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.740431:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.740432:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.2:1619677447.740433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:18.0:1619677447.740435:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.740436:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.740436:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.740438:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:18.0:1619677447.740438:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.740439:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:20.0:1619677447.740440:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.740440:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.740441:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.740441:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677447.740441:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677447.740442:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:20.0:1619677447.740443:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.740443:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:20.0:1619677447.740444:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.740446:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724112] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.740447:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677447.740447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677447.740448:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.740449:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.740453:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.740454:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.740455:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.740457:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:20.0:1619677447.740458:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.740458:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:2.0:1619677447.740458:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.740459:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.740459:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.740460:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.740461:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.740462:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677447.740462:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.740463:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.740463:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.740474:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.740477:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.740478:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.740479:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.740481:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.740482:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.740511:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.740514:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.740516:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.740518:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.740519:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.740520:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.740521:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.740522:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.740524:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.740525:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.740528:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.740530:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.740532:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.740533:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.740534:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.740535:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.740536:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.740537:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.740537:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.740547:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.740549:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.740550:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.740551:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.740553:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.740554:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.740566:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.740568:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.740569:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.740570:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.740570:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.740571:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.740572:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.740575:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.740576:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.740578:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.740579:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.740581:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677447.740582:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.740583:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.740584:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.740585:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677447.740589:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.740590:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677447.740593:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.740594:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677447.740594:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677447.740596:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677447.740596:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.740597:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.740597:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677447.740610:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.740612:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.740613:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.740614:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.740616:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.740618:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:17.0:1619677447.740648:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.740651:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.740651:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.740653:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.740656:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.740657:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.740658:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.740659:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.740660:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.740661:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.740664:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.740666:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.740668:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.740669:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.740669:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.740671:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.740671:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.740673:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.740674:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.740674:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.740684:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677447.740685:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.740686:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677447.740687:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677447.740687:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677447.740688:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.740688:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.740689:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677447.740691:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.740693:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.740701:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.740704:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.740705:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.740705:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.740706:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.740706:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.740832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.740877:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.740880:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.740881:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.740881:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.740882:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:22.0:1619677447.740883:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.740883:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.740886:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.740887:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.740887:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.740889:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.740891:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.740892:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.740894:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.740895:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.740896:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.740900:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.740902:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.740904:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.740905:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.740906:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.740908:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.740908:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.740909:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.740909:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677447.740920:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.740923:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.740924:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.740926:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.740927:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.740929:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.740930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.740938:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.740941:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.740943:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.740944:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.740945:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.740946:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677447.740948:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.740949:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.740950:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.740951:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677447.740954:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.740956:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677447.740959:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.740960:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.740961:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677447.740962:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.740963:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.740964:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.740966:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.740966:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677447.740969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677447.740972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677447.740977:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:31.0:1619677447.740980:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:5.0:1619677447.740980:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.740981:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.740982:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.740983:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:20.0:1619677447.740983:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677447.740984:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.740985:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677447.740985:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677447.741052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.741060:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.741063:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.741064:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.741065:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.741066:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.741068:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.741091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.741096:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.741099:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.741101:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.741102:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.741104:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.741105:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677447.741106:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.741108:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.741109:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.741110:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677447.741113:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.741115:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.741117:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.741118:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.741118:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.741120:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.741121:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.741122:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.741122:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677447.741126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.741133:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.741135:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.741136:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.741138:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.741139:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.741140:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.741214:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.741217:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.741218:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.741220:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.741222:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.741224:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.741225:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.741226:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.741227:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.741228:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.741231:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.741233:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.741236:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.741237:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.741237:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.741239:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.741240:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.741240:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.741241:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.741252:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.741254:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.741255:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.741257:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.741259:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.741260:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.741269:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.741272:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.741273:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.741273:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.741274:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.741276:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.741365:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.741368:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.741369:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.741371:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.741373:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.741375:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.741376:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.741377:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.741378:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.741379:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.741382:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.741384:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.741386:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.741387:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.741387:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.741389:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.741390:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.741390:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.741391:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.741396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.741403:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.741406:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.741406:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.741407:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.741409:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.741410:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677447.741522:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:24.0:1619677447.741523:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:31.0:1619677447.741524:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.741525:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.741526:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:31.0:1619677447.741527:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.741527:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:31.0:1619677447.741529:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:24.0:1619677447.741529:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.741530:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:24.0:1619677447.741531:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.741533:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.741534:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.741535:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.741536:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.741537:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.741540:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.741541:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.741544:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.741545:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.741545:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.741548:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.741548:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.741550:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.741551:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.741552:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677447.741554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.741557:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.741559:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.741560:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:4.2:1619677447.741560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:22.0:1619677447.741562:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.741562:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.741564:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:22.0:1619677447.741565:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.741565:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:18.0:1619677447.741566:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.741566:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.741567:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:8.0:1619677447.741567:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.741568:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:2.0:1619677447.741568:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.741569:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.741570:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:22.0:1619677447.741570:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.741570:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.741571:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677447.741571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:22.0:1619677447.741572:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.741572:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.741572:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:2.0:1619677447.741575:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.741576:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:22.0:1619677447.741577:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.741577:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.741578:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.741579:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:22.0:1619677447.741580:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.741580:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:8.0:1619677447.741580:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.741580:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.741580:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:8.0:1619677447.741582:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:22.0:1619677447.741583:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677447.741583:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96724098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.741584:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.741585:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:8.0:1619677447.741585:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.741586:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.741587:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.741587:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:8.0:1619677447.741587:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.741588:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.741589:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:8.0:1619677447.741590:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677447.741593:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.741597:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.741600:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.741600:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677447.741601:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (23)++
00000800:00000200:1.0:1619677447.741601:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000800:00000200:8.0:1619677447.741603:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (25)--
00000800:00000200:8.0:1619677447.741604:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.741604:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677447.741604:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000800:00000200:17.0:1619677447.741605:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677447.741605:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.741605:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.741608:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677447.741609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.741612:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.741614:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.741616:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.741617:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.741618:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.741618:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.741621:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.741640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.741664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.741672:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.741675:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.741677:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.741679:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.741679:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.741681:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.741682:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.741683:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.741684:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.741685:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:6.0:1619677447.741686:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:31.0:1619677447.741688:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.741688:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677447.741689:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:22.0:1619677447.741689:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.741690:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.741692:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677447.741692:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.741693:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.741694:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.741695:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677447.741695:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677447.741696:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.741697:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.741698:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.741708:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.741711:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.741712:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.741713:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.741715:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.741716:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.741730:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.741732:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.741734:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.741736:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.741737:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.741738:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.741739:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.741740:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.741741:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.741742:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.741745:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.741747:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.741749:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.741750:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.741750:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.741752:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.741752:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.741753:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.741753:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.741763:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.741766:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.741766:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.741768:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.741770:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.741770:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.2:1619677447.741779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677447.741788:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.741793:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.741794:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.741795:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677447.741796:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.741796:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.741797:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677447.741799:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.741800:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.741801:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.741804:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.741805:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.741807:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.741808:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.741809:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.741810:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.741813:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.741815:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.741817:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.741817:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.741818:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.741820:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.741820:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.741821:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.741821:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.741826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.741833:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.741835:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.741836:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.741837:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.741838:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.741839:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677447.741893:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.741896:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.741897:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.741898:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.741900:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.741901:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.741903:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.741904:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.741905:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.741906:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.741909:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.741911:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.741912:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.741913:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.741914:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.741916:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.741916:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.741918:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.741919:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.741919:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:9.2:1619677447.741922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.741930:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.741931:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.741931:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.741931:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.741932:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.741932:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.741933:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.741934:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.741934:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677447.741935:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.742126:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.742129:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.742129:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.742131:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.742134:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.742136:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.742137:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.742139:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.742140:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.742142:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.742146:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.742148:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.742151:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.742152:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.742152:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.742154:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.742155:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.742156:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.742157:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.742158:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.742160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677447.742166:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:6.0:1619677447.742168:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:31.0:1619677447.742169:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:18.0:1619677447.742169:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:31.0:1619677447.742170:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.742170:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.742170:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:31.0:1619677447.742171:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.742171:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:6.0:1619677447.742171:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:18.0:1619677447.742172:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.742173:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:2.0:1619677447.742173:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677447.742174:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:18.0:1619677447.742175:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.742177:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677447.742227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.742235:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.742238:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.742240:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.742242:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.742243:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.742244:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.742245:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.742246:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.742247:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.742248:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.742251:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.742253:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.742255:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.742256:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.742257:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.742258:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.742259:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.742260:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.742260:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.742270:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.742273:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.742273:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.742275:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.742277:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.742278:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.742312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.742333:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.742337:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.742338:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.742340:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.742343:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.742344:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.742345:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96724088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.742346:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.742347:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.742348:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.742351:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.742352:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:4.2:1619677447.742352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677447.742355:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.742355:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.742356:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.742357:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.742358:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.742358:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677447.742358:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:24.0:1619677447.742359:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.742360:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.742361:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.742362:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.742364:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.742367:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.742370:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.742373:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.742374:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.742375:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.742377:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.742378:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677447.742378:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.742381:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.742382:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.742384:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.742386:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.742387:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.742389:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.742390:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.742391:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.742392:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.742395:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.742397:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.742400:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.742401:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.742402:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.742403:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.742404:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.742404:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.742405:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677447.742413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.742415:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:12.0:1619677447.742416:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.0:1619677447.742418:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:12.0:1619677447.742418:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.742419:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.742421:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.742421:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:13.0:1619677447.742423:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:20.0:1619677447.742424:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:12.0:1619677447.742424:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:20.0:1619677447.742427:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.742428:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677447.742430:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.742431:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.742432:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.742433:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677447.742436:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:8.2:1619677447.742436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.742438:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677447.742441:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.742442:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677447.742443:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677447.742444:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.742445:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677447.742445:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.742445:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.742446:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000400:00000200:5.0:1619677447.742447:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.742448:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.742448:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.742448:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.742450:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.742452:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.2:1619677447.742453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.742456:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.742458:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.742459:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.742460:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.2:1619677447.742586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:8.2:1619677447.742624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677447.742627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:1.0:1619677447.742631:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.742634:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.742635:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.742635:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.742637:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.742641:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.742740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.742778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.742786:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.742789:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.742790:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.742790:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.742792:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.742795:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.742810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.742812:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.742815:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.742817:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.742818:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.742820:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.742822:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677447.742823:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.742824:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.742825:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.742826:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677447.742830:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.742832:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677447.742834:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.742835:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677447.742835:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.742837:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.742838:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.742839:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.742839:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.742843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.742851:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.742853:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.742855:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.742855:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.742856:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.742859:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.742862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.742869:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.742872:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.742873:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.742874:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.742875:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.742875:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:6.0:1619677447.742896:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.742899:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.742900:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.742900:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.742902:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.742905:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.742943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677447.742946:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.742949:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.742950:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677447.742952:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:18.0:1619677447.742952:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.742953:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.742954:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:18.0:1619677447.742955:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.742957:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.742958:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.742958:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.742959:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.742960:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.742960:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.742960:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.742962:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.742963:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.742964:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.742965:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.742966:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.742967:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:18.0:1619677447.742967:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.742970:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.742971:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.742971:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.742971:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.742973:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677447.742973:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.742973:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.742975:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:31.0:1619677447.742976:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.742976:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.742976:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677447.742977:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.742977:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.742979:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.742979:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:8.2:1619677447.742979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.742980:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.742981:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:2.0:1619677447.742988:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:5.0:1619677447.742989:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677447.742990:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.742990:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:30.0:1619677447.742991:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:5.0:1619677447.742991:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:2.0:1619677447.742991:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:5.0:1619677447.742992:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.742993:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.742994:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.742994:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.742994:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.742996:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.742997:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677447.742997:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677447.742999:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677447.743091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.743118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.743125:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.743128:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.743129:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.743130:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.743133:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.743134:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.743135:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96724077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.743136:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:8.0:1619677447.743136:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.743137:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.743138:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:22.0:1619677447.743139:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.743141:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.743141:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.743141:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.743142:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.743142:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677447.743143:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677447.743144:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.743146:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.743146:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.743147:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677447.743147:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.743147:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:6.0:1619677447.743147:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.743149:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:6.0:1619677447.743149:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.743150:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.743150:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.743151:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677447.743151:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.743152:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.743154:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677447.743155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:6.0:1619677447.743155:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.743156:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.743159:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.743161:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677447.743162:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:6.0:1619677447.743164:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.743165:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:6.0:1619677447.743165:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.743165:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:1.0:1619677447.743166:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.743167:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.743167:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.743168:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.743168:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:17.0:1619677447.743169:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.743169:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677447.743172:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.743225:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.743227:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.743228:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.743230:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.743233:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.743234:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.743235:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.743236:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.743237:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.743239:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677447.743242:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.743244:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.743246:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.743247:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.743248:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.743250:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.743250:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.743251:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.743252:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.743253:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.743255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.743262:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:17.0:1619677447.743263:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677447.743265:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:17.0:1619677447.743265:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.0:1619677447.743265:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.743266:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.743266:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.743267:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677447.743268:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.743272:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677447.743380:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.743382:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.743383:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.743385:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.743387:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.743389:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677447.743390:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.743391:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.743392:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.743393:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677447.743396:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.743398:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.743401:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.743401:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.743402:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.743403:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.743404:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.743405:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.743405:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.743418:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.743421:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.743421:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.743423:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.743425:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.743426:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.743427:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.743427:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.743428:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.743428:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.743429:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.743595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.743632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.743641:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.743643:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.743644:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.743645:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.743647:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.743650:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.743761:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677447.743764:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677447.743764:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:1.0:1619677447.743767:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.743770:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.743771:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.743772:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.743774:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.743775:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.743776:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.743779:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.743781:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.743784:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.743785:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.743785:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.743787:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.743788:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677447.743788:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677447.743789:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:13.0:1619677447.743800:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.743802:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.743803:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.743805:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.743806:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.743808:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.743823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.743830:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.743833:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.743834:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.743835:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.743838:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.743839:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.743840:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.743842:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.743843:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.743844:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.743847:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.743848:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.743851:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.743852:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.743852:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.743854:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.743854:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.743855:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.743856:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.743860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.743867:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.743870:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.743871:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.743871:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.743873:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.743876:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.743880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.743887:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.743889:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.743890:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.743890:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.743891:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.743892:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.0:1619677447.743901:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.743903:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.743905:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.743906:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.743909:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.743910:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.743911:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.743913:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.743914:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677447.743914:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.743914:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677447.743917:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:8.0:1619677447.743918:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.743918:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.743920:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.743920:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:17.0:1619677447.743922:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.743923:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677447.743924:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:8.0:1619677447.743924:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:17.0:1619677447.743925:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.743925:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:17.0:1619677447.743926:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.743926:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:17.0:1619677447.743927:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.743927:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.743927:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:17.0:1619677447.743928:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.743928:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:17.0:1619677447.743932:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677447.743932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677447.743933:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677447.743936:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.743937:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677447.743938:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677447.743939:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677447.743940:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677447.743940:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.743941:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.743941:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677447.743942:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.743943:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.743945:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.743946:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.743948:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:9.2:1619677447.743956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.743962:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.743965:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.743966:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.743966:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.743968:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.743971:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677447.743985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.743996:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.743998:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.744001:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.744001:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.744003:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.744005:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677447.744006:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.744008:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.744010:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.744011:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677447.744015:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.744017:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677447.744020:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.744021:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677447.744022:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677447.744023:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677447.744024:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.744025:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.744026:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.744027:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:9.2:1619677447.744030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.2:1619677447.744032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:9.2:1619677447.744033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.744036:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.744038:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.744038:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677447.744039:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677447.744039:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677447.744039:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.744041:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:31.0:1619677447.744042:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:18.0:1619677447.744043:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:2.0:1619677447.744043:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.744044:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.744044:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.744045:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.744045:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.744046:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.744047:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677447.744047:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.744048:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:2.0:1619677447.744048:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:31.0:1619677447.744049:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.744049:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:2.0:1619677447.744049:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:31.0:1619677447.744051:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.744051:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000800:00000200:18.0:1619677447.744051:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.744051:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.744052:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:31.0:1619677447.744053:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.744053:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677447.744054:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.744054:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.744055:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.744055:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:30.0:1619677447.744056:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.744057:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.744058:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.744060:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.744060:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:30.0:1619677447.744061:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.744063:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.744063:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.744064:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:31.0:1619677447.744067:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.744068:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.744068:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677447.744068:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.744070:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.744070:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677447.744071:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.744071:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.744072:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677447.744073:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677447.744073:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.744073:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:30.0:1619677447.744074:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677447.744075:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677447.744076:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677447.744078:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871380600] (22)++
00000800:00000200:30.0:1619677447.744079:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (23)--
00000800:00000200:30.0:1619677447.744080:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:12.0:1619677447.744082:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.2:1619677447.744083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:12.0:1619677447.744084:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.744085:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.744087:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.744088:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.744088:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:12.0:1619677447.744089:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.744090:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:8.0:1619677447.744091:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677447.744092:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:31.0:1619677447.744093:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (22)++
00000800:00000200:1.0:1619677447.744093:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.744094:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:8.0:1619677447.744094:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677447.744094:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.744096:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871380600] (23)--
00000800:00000200:31.0:1619677447.744096:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:8.0:1619677447.744096:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.744097:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000400:00000200:1.0:1619677447.744097:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.744098:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677447.744099:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96724064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.744100:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.744100:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:8.0:1619677447.744101:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.744102:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677447.744105:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677447.744107:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.744109:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.744109:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677447.744110:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.744111:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.744112:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677447.744112:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677447.744113:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677447.744127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.744133:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.744135:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.744136:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.744136:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.744138:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677447.744139:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:18.0:1619677447.744294:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.744300:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.744301:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:18.0:1619677447.744304:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.744306:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.744308:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.744309:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.744310:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.744311:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.744312:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.744316:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.744318:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.744321:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.744322:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.744323:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.744324:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.744325:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.744326:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.744326:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677447.744339:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:4.2:1619677447.744341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.744342:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.744342:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.744344:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.744346:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.744347:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.744355:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.744356:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:12.0:1619677447.744357:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.744358:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.744359:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.744359:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.744359:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.744360:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.744360:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677447.744363:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.744365:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.744367:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677447.744368:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96724061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.744369:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.744370:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.744371:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677447.744374:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.744376:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.744378:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.744379:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.744379:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.744381:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.2:1619677447.744383:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.744386:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:4.2:1619677447.744386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677447.744388:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.744389:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.744390:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677447.744392:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.744394:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:12.0:1619677447.744394:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:20.0:1619677447.744395:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.744396:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:20.0:1619677447.744397:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:12.0:1619677447.744397:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.744399:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.744400:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.744401:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677447.744401:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.744402:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.744404:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.2:1619677447.744404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.744405:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.744407:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.744408:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677447.744412:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.744415:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.744416:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.744417:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.744417:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.744419:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677447.744471:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.744474:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.744475:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.744476:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677447.744476:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.744477:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677447.744498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.744504:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.744506:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.744508:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.744509:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.744511:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.744513:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.744514:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.744516:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.744517:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.744518:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677447.744522:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:5.0:1619677447.744522:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.744524:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677447.744525:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.744527:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.744527:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.744527:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.744528:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677447.744529:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.0:1619677447.744529:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.744530:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677447.744531:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.744531:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677447.744532:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:5.0:1619677447.744532:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:31.0:1619677447.744534:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.744535:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.744536:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.744537:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677447.744540:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.744542:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677447.744544:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.744544:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:31.0:1619677447.744545:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677447.744546:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:22.0:1619677447.744546:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.744547:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.744548:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.744548:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:22.0:1619677447.744548:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.744550:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:22.0:1619677447.744550:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.744551:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.744552:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677447.744552:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.744564:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677447.744566:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677447.744567:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.744567:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.744568:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:30.0:1619677447.744569:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:6.0:1619677447.744569:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.744570:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677447.744570:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677447.744571:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677447.744583:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fdae00] (21)++
00000800:00000200:20.0:1619677447.744586:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.38@o2ib3
00000400:00000200:20.0:1619677447.744588:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.38@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.744589:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (22)--
00000400:00000200:20.0:1619677447.744590:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.744592:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677447.744593:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.744594:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.744595:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.744596:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677447.744600:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.38@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.744602:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677447.744605:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.744605:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677447.744606:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677447.744608:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677447.744608:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fdae00] (21)++
00000800:00000200:20.0:1619677447.744609:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fdae00] (22)--
00000800:00000200:20.0:1619677447.744610:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (21)--
00000800:00000200:17.0:1619677447.744620:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.744623:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:9.2:1619677447.744623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.744624:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.744626:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.744628:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.744630:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:17.0:1619677447.744630:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.744631:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.744632:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:17.0:1619677447.744632:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.744633:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.744633:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.744634:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.744634:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:24.0:1619677447.744636:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000400:00000200:17.0:1619677447.744638:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.744639:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:17.0:1619677447.744640:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.744643:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.744644:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.744645:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.744647:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.744647:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.744648:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.744648:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.744659:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.744662:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.744663:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.744665:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.744666:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.744668:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:2.0:1619677447.744866:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.744869:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.744870:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.744872:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.744876:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.744877:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677447.744879:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.744880:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.744881:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.744882:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.744884:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:2.0:1619677447.744885:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.744886:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:2.0:1619677447.744887:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.744889:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.744890:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677447.744890:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.744891:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.744891:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677447.744892:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:31.0:1619677447.744893:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.744894:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.744894:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.744894:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:31.0:1619677447.744895:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.744895:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:31.0:1619677447.744896:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.744896:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:31.0:1619677447.744897:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.744901:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.744903:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.744906:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.744907:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.744907:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.744907:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:31.0:1619677447.744909:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.744909:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677447.744909:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.744910:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.744911:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.744911:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:13.0:1619677447.744911:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.744913:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.744914:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.744921:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.744924:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.744925:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.744926:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.744928:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.744928:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.744963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.744971:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.744973:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.744975:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.744977:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.744977:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.744979:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.744980:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.744981:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.744982:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.744983:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.744986:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.744987:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.744989:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.744990:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.744991:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.744992:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.744993:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.744994:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.744995:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.744996:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677447.745001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677447.745004:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.745006:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.745007:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.745008:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.745009:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:12.0:1619677447.745010:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.745011:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.745012:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:13.0:1619677447.745012:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677447.745013:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.745019:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.745022:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.745023:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.745023:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.745024:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.745025:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.745033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.745039:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.745041:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.745043:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.745044:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.745046:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.745048:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.745049:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.745051:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.745051:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.745053:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.745056:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.745058:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.745061:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.745061:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.745062:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.745064:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:4.2:1619677447.745064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:5.0:1619677447.745065:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.745065:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.745066:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.0:1619677447.745071:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.745074:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.745075:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677447.745076:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:8.0:1619677447.745077:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.745079:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:8.0:1619677447.745079:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.745080:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.745081:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677447.745082:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.745083:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96724052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.745084:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:8.0:1619677447.745084:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.745085:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:8.0:1619677447.745085:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.745086:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.745086:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)++
00000400:00000200:8.0:1619677447.745086:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.745087:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.745087:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.745088:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:8.0:1619677447.745090:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.745091:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.745094:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.745095:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.745095:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.745097:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.745097:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.745099:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.745100:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.745101:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.745103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.745106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.745110:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.745113:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.745113:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.745114:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.745114:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.745115:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:4.2:1619677447.745115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.745116:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.745117:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.745117:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.745120:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677447.745130:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.745133:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.745134:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.745137:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.745139:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.745141:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.745142:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.745144:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.745145:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.745146:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.745150:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.745152:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.745154:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.745155:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.745155:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.745157:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.745157:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.745158:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.745159:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.745170:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.2:1619677447.745170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677447.745173:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.745174:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.745174:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.745176:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:20.0:1619677447.745178:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:22.0:1619677447.745179:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.745180:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.745181:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.745183:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.745183:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.745186:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.745202:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.745205:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677447.745206:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677447.745206:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677447.745209:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:17.0:1619677447.745209:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.745211:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.745212:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.745212:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677447.745213:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.745214:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.745215:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.745216:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677447.745216:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.745217:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.745218:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.745219:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.745219:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.745220:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.745221:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677447.745224:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.745226:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.745227:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.745228:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677447.745230:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.745231:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.745232:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.745232:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.745233:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677447.745234:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677447.745234:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.745234:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.745235:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677447.745236:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.745236:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677447.745236:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.745237:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.745238:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.745246:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.745248:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.745249:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.745251:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.745251:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:13.0:1619677447.745253:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.745254:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.745254:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.745255:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.745255:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.745257:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.745261:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.745324:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.745327:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.745328:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.745331:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.745333:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.745334:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.745336:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.745337:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.745338:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.745339:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.745342:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.745344:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.745346:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.745347:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.745348:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.745349:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.745349:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.745350:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.745350:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.745356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.745363:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.745365:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.745366:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.745367:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.745368:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.745370:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.745404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.745409:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.745412:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677447.745414:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677447.745415:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.745417:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.745418:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.745420:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.745421:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.745422:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.745423:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.745427:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.745428:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.745430:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.745431:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.745432:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.745434:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.745435:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.745436:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.745436:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677447.745440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.745447:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.745449:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.745450:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.745451:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.745452:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.745455:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.745465:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.745468:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.745469:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.745470:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.745472:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.745473:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.745474:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.745475:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.745476:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.745477:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.745480:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.745482:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.745484:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.745485:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.745485:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.745487:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.745487:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.745488:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.745488:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.745492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.745496:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677447.745499:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:2.0:1619677447.745499:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677447.745499:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:2.0:1619677447.745501:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.745502:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.745502:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.745503:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.745504:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.745504:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.745506:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:1.0:1619677447.745506:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677447.745507:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.745509:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.745510:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.745511:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677447.745514:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.745516:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677447.745519:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.745520:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:1.0:1619677447.745521:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:1.0:1619677447.745523:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:1.0:1619677447.745523:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677447.745524:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677447.745524:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:30.0:1619677447.745557:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.745559:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.745562:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.745564:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.745564:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.745566:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677447.745567:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.745568:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.745570:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.745571:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677447.745574:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.745576:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677447.745579:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.745580:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677447.745580:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677447.745582:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677447.745583:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.745584:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.745584:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677447.745589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.745597:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.745599:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.745600:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.745600:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.745603:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.745606:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.745633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.745636:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.745639:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.745641:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.745642:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.745644:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.745645:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.745647:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.745648:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.745649:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.745650:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.745654:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.745655:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.745658:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.745658:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.745659:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.745661:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.745661:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.745662:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.745662:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.745675:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.745678:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.745679:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.745679:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.745681:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.745683:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.745740:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.745742:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.745743:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.745745:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.745747:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.745748:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.745877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:9.2:1619677447.745913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.745921:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.745924:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.745925:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.745925:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.745927:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.745930:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.745933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.745940:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.745943:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.745944:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677447.745944:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677447.745944:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.745945:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.745945:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.745947:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.745948:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.745949:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.745952:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.745953:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:6.0:1619677447.745954:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.745955:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.745956:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.745957:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677447.745957:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.745958:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.745960:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.745960:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.745964:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.745965:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.745967:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.745968:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677447.745968:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:6.0:1619677447.745969:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.745969:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:1.0:1619677447.745969:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.745971:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.745971:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:1.0:1619677447.745971:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.745972:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677447.745972:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.745973:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:1.0:1619677447.745973:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.745976:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677447.745977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.745978:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.745981:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.745982:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.745983:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.745985:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.745986:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.745986:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.745988:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.745988:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:17.0:1619677447.745989:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.745989:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000400:00000200:17.0:1619677447.745990:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.745990:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.745991:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.745993:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.745999:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.746001:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.746002:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.746003:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.746009:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.746012:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.746013:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.746013:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.746015:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.746017:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.746146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.746153:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.746156:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.746158:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.746160:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.746161:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.746163:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.746164:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.746166:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.746167:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.746168:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.746171:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.746173:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.746177:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.746177:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.746178:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.746180:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.746181:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.746182:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.746183:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.746186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.746195:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.746198:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.746199:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.746200:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.746200:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.746204:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.746211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.746257:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.746260:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.746261:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.746261:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.746263:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.746266:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.746276:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.746279:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.746280:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.746280:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.746281:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.746282:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677447.746520:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.746523:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.746525:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.746528:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.746528:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.746529:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.746530:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.746532:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.746533:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.746534:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.746536:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.746538:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.746540:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.746541:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.746541:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.746543:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.746543:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.746545:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.746546:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.746547:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677447.746556:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.746559:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677447.746560:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:13.0:1619677447.746560:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.746561:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.746563:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.746563:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677447.746563:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.746564:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.746565:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.746574:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.746576:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.746577:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.746578:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.746578:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.746579:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.746582:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.746585:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.746586:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.746586:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.746588:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.746591:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.746600:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.746603:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.746604:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.746604:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.746605:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.746606:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.746617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.746625:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.746628:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.746630:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.746632:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.746632:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.746634:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.746635:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.746636:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.746637:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.746637:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:31.0:1619677447.746638:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.746640:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.746640:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:31.0:1619677447.746643:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.746644:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.746645:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.746646:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.746648:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:31.0:1619677447.746649:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677447.746649:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.746650:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.746650:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:6.0:1619677447.746651:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.746652:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.746652:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:6.0:1619677447.746652:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.746653:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:6.0:1619677447.746653:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.746654:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:6.0:1619677447.746656:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677447.746658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.746658:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.746661:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.746662:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677447.746663:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (23)++
00000800:00000200:18.0:1619677447.746664:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677447.746665:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:20.0:1619677447.746666:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)++
00000800:00000200:6.0:1619677447.746666:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.746666:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677447.746667:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:6.0:1619677447.746667:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.746668:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (24)--
00000800:00000200:2.0:1619677447.746668:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:20.0:1619677447.746669:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.746670:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.746671:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.746671:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.746672:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)++
00000400:00000200:18.0:1619677447.746673:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:5.0:1619677447.746673:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (24)--
00000800:00000200:20.0:1619677447.746674:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000400:00000200:18.0:1619677447.746674:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.746674:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.746675:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.746676:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:18.0:1619677447.746676:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.746677:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.746678:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.746678:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:18.0:1619677447.746682:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.746684:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.746686:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.746687:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.746688:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.746689:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.746690:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.746690:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.746691:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677447.746695:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.2:1619677447.746695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677447.746698:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.746699:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.746701:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.746703:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:17.0:1619677447.746704:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.746705:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:17.0:1619677447.746705:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:20.0:1619677447.746706:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.746707:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.746708:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.746708:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:17.0:1619677447.746709:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.746710:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.746711:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000400:00000200:17.0:1619677447.746711:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.746711:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000400:00000200:17.0:1619677447.746712:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:5.0:1619677447.746712:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.746712:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.746713:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:17.0:1619677447.746716:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.746719:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.746722:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.746723:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.746723:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.746725:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.746726:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.746726:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.746727:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677447.746731:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.746733:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.746734:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.746736:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:22.0:1619677447.746739:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:2.0:1619677447.746739:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.746740:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:22.0:1619677447.746741:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:12.0:1619677447.746741:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:22.0:1619677447.746742:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.746742:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:2.0:1619677447.746742:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96724028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.746743:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.746744:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:2.0:1619677447.746744:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.746745:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:12.0:1619677447.746745:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.746745:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677447.746747:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.746748:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.746748:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677447.746749:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:12.0:1619677447.746750:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.746752:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.746753:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.746754:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.746754:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.746756:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.746756:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.746757:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.746758:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.746762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.746769:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.746771:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.746772:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.746773:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.746774:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.746776:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.746956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.746990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.746998:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.747001:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.747002:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.747003:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.747003:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:24.0:1619677447.747006:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.747006:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.747007:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.747007:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.747008:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.747009:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96724026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.747010:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.747011:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.747012:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.747013:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.747013:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:24.0:1619677447.747017:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.747018:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.747021:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.747022:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.747022:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.747024:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.747024:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.747025:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.747026:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.747030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.747037:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.747040:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.747041:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.747041:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.747043:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.747046:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.747047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.747054:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.747056:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.747057:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.747057:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.747058:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.747059:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677447.747185:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.747188:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.747188:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.747190:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.747192:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.747194:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.747195:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.747195:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677447.747196:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.747198:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.747198:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.747198:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:22.0:1619677447.747199:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677447.747202:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.747203:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.747204:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.747205:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:22.0:1619677447.747206:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:1.0:1619677447.747207:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96724023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.747208:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.747209:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677447.747209:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.747210:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000400:00000200:1.0:1619677447.747210:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.747211:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.747212:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.747213:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.747214:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:1.0:1619677447.747214:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.747215:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.747216:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677447.747216:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.747218:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.747219:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.747219:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.747220:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.747221:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.747222:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.747222:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.747226:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.747228:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:8.0:1619677447.747230:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:6.0:1619677447.747230:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.747231:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.747231:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.747233:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.747234:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:17.0:1619677447.747234:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.747236:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.747236:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.747236:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.747237:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:8.0:1619677447.747238:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.747239:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.747240:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:8.0:1619677447.747240:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.747242:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.747243:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.747244:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.747245:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.747249:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.747250:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.747254:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.747254:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.747255:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.747257:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.747257:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.747258:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.747259:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677447.747263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.747271:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.747273:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.747274:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.747276:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.747276:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.747279:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.747304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.747306:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.747309:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.747311:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.747312:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.747313:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.747315:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.747316:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.747318:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.747319:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.747320:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.747323:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.747324:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.747327:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.747328:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.747329:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.747331:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.747331:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.747332:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.747333:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.747334:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.747339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.747344:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677447.747346:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.747346:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.747347:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.747348:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.747349:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.747350:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.747351:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.747353:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.747354:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.747379:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.747382:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.747384:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.747387:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.747387:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.747388:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677447.747389:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677447.747390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:30.0:1619677447.747391:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.747391:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.747392:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677447.747395:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.747397:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677447.747398:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.747399:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:30.0:1619677447.747400:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.747401:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.747401:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677447.747401:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:31.0:1619677447.747403:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.747403:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.747404:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:31.0:1619677447.747405:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:30.0:1619677447.747405:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.747405:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:31.0:1619677447.747406:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96724019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.747407:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.747407:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677447.747407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:31.0:1619677447.747408:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.747409:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.747412:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.747414:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.747416:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.747417:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.747417:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677447.747418:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:31.0:1619677447.747419:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.747419:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.747420:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:22.0:1619677447.747420:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:31.0:1619677447.747421:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:6.0:1619677447.747421:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.747422:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.747424:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.747427:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.747431:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.747433:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.747434:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.747436:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.747437:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.747438:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.2:1619677447.747448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.747456:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.747459:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.747460:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.747461:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.747461:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.747464:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.747497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.747536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.747541:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.747543:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.747544:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.747545:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.747547:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.747549:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.747597:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.747600:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.747600:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.747602:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.747605:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.747606:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.747607:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.747609:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.747610:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.747611:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.747614:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.747616:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.747619:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.747619:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.747620:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.747622:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.747622:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.747623:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.747623:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677447.747705:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.747708:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.747708:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.747711:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.747712:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.747713:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.747715:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.747716:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.747717:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.747718:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.747721:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.747723:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.747725:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.747726:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.747726:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.747728:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.747728:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.747729:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.747729:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.747734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.747741:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.747743:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.747744:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.747745:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.747746:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.747748:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.747754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.747761:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.747764:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.747765:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.747765:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.747766:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.747767:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:6.0:1619677447.747775:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.747777:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.747778:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.747779:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.747781:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.747784:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:30.0:1619677447.748145:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677447.748148:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:30.0:1619677447.748150:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.748152:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:30.0:1619677447.748152:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.748154:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.748155:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.748156:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.748157:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.748158:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.748161:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.748163:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.748166:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.748167:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.748167:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.748169:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.748170:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677447.748171:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677447.748171:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:8.2:1619677447.748175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:4.2:1619677447.748181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.748186:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.748188:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.748189:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.748191:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.748192:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.748195:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677447.748227:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.748230:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.748231:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.748231:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.748233:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.748236:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.748273:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.748276:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.748277:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.748278:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.748281:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.748283:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.748284:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96724011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.748285:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.748286:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.748287:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.748290:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.748292:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.748301:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.748301:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.748302:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.748304:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.748305:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.748305:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.748306:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.748311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.748319:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.748322:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.748323:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.748324:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.748325:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.748329:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.748595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.748603:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.748606:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.748608:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.748610:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.748611:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.748613:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.748614:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.748615:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.748616:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.748617:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.748620:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.748622:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.748625:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.748626:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.748626:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.748628:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:4.2:1619677447.748628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677447.748629:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.748630:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.748631:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.748632:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677447.748635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677447.748637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677447.748643:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677447.748646:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:5.0:1619677447.748646:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.748647:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.748648:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.748649:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677447.748649:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.748650:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.748652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:5.0:1619677447.748654:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.748657:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.748658:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.748661:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.748662:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.748663:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.748664:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.748665:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:9.2:1619677447.748688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.748696:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.748698:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.748700:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.748700:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677447.748702:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:18.0:1619677447.748702:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.748704:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:18.0:1619677447.748705:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:30.0:1619677447.748706:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.748707:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.748708:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.748711:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:2.0:1619677447.748820:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.748823:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.748824:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.748826:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.748828:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.748829:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.748830:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.748832:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.748832:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.748833:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.748836:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.748838:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.748840:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.748841:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.748842:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.748843:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.748844:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.748845:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:20.2:1619677447.748846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:2.0:1619677447.748846:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.748847:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677447.748850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.748855:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677447.748857:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677447.748858:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:5.0:1619677447.748858:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:18.0:1619677447.748859:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:20.0:1619677447.748860:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:18.0:1619677447.748860:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677447.748860:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:5.0:1619677447.748860:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.748861:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.748861:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.748862:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:17.0:1619677447.748863:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.748863:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.748864:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:17.0:1619677447.748866:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:5.0:1619677447.748866:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.748867:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.748868:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.748869:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.748873:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.748874:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.748877:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.748878:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.748878:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.748880:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.748881:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.748882:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.748883:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677447.748886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.748893:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.748896:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.748897:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.748897:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.748899:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.748901:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.748918:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.748921:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.748922:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.748924:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.748925:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.748927:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.748928:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.748929:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.748930:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.748931:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.748935:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.748937:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.748940:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.748940:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.748941:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.748943:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.748943:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.748944:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.748945:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.748946:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.748949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.748955:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:1.0:1619677447.748956:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677447.748958:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677447.748958:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.748958:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677447.748959:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.748959:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.748960:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677447.748962:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.748965:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677447.749003:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.749006:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.749007:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.749009:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.749012:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.749013:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.749015:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.749016:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.749017:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.749018:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.749021:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.749023:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.749026:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.749027:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:4.2:1619677447.749027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.749028:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.749030:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.749030:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.749031:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.749031:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677447.749043:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.749046:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.749047:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.749048:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.749050:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.749051:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.2:1619677447.749065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.749073:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.749075:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.749076:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.749078:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.749079:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.749081:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.749163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.749168:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.749171:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.749173:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.749173:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.749175:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.749177:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.749178:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.749180:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.749181:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.749182:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.749185:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.749187:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.749190:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.749191:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.749192:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.749194:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.749194:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.749195:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.749195:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.749207:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.749210:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.749211:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.749213:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.749214:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.749216:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.749386:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.749389:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.749389:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.749391:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.749394:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.749396:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.749397:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96724001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.749398:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.749399:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.749400:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.749403:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.749405:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.749408:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.749409:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.749410:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.749412:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.749412:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.749413:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.749413:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.749426:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.749429:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.749431:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.749432:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.749434:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.749435:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677447.749436:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.749438:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.749440:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.749441:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.749443:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.749444:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.749446:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96724000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.749447:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.749448:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.749449:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.749452:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.749454:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.749457:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.749458:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.749458:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.749460:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.749460:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.749462:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.749462:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.749472:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.749475:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.749475:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.749477:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.749479:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.749480:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.749490:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.749493:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.749493:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.749494:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.749495:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.749495:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.749501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.749509:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.749512:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677447.749514:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.749514:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.749515:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:24.0:1619677447.749517:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.749517:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:24.0:1619677447.749519:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.749519:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.749520:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.749522:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.749523:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.749524:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.749526:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.749527:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.749528:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.749529:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723998] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.749530:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.749530:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.749531:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.749532:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.749535:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.749536:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:8.0:1619677447.749536:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.749537:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.749537:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.749539:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.749540:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.749540:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.749541:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.749541:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.749541:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.749541:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.749543:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.2:1619677447.749545:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.749548:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.749549:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:4.2:1619677447.749549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677447.749550:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.749550:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:1.0:1619677447.749553:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:12.0:1619677447.749556:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677447.749556:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.749557:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.749557:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.749558:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.749559:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.749559:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:12.0:1619677447.749562:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.749562:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:13.0:1619677447.749563:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.2:1619677447.749564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:12.0:1619677447.749565:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.749571:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.749574:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.749574:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.749575:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.749576:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.749577:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.749593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:9.2:1619677447.749600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.749609:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.749611:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.749613:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.749613:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.749615:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.749618:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.749620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.749627:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.749630:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.749631:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.749631:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.749632:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.749632:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.749673:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.749676:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.749676:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.749679:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.749682:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.749684:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.749685:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.749686:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.749687:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.749688:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.749694:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.749696:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.749700:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.749701:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.749702:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.749704:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.749705:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.749706:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.749707:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.749711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:13.0:1619677447.749717:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677447.749718:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:13.0:1619677447.749719:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.749720:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.749721:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.749722:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.749722:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.749722:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:12.0:1619677447.749723:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:17.0:1619677447.749724:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.749725:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.749728:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677447.749839:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.749842:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.749844:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.749846:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.749846:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.749848:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.749849:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.749850:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.749851:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.749852:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.749855:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.749857:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.749859:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.749860:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.749861:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.749862:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.749863:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.749864:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.749864:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.749874:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.749876:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.749877:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.749878:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.749880:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.749881:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.749935:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.749937:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.749938:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677447.749940:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:1.0:1619677447.749940:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.749943:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.749944:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.749944:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:1.0:1619677447.749944:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.749946:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.749947:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.749947:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.749948:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.749949:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.749949:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677447.749950:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.749952:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.749953:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.749954:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.749955:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.749956:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.749957:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:2.0:1619677447.749959:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.749961:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.749963:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.749964:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.749964:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:2.0:1619677447.749965:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.749965:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:2.0:1619677447.749966:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.749967:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:2.0:1619677447.749968:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.749968:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:1.0:1619677447.749968:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.749968:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677447.749969:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677447.749969:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677447.749970:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.749973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.2:1619677447.749973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677447.749978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:24.0:1619677447.749980:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:20.0:1619677447.749981:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677447.749981:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.749983:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:17.0:1619677447.749983:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677447.749984:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:8.0:1619677447.749984:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.749985:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.749985:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.749985:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:20.0:1619677447.749987:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.749987:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:17.0:1619677447.749988:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.749989:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.749990:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.749990:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:24.0:1619677447.749992:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:20.0:1619677447.749992:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.749993:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677447.749993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000400:00000200:20.0:1619677447.749994:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.749996:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.749997:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.749999:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000400:00000200:20.0:1619677447.750000:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.750002:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.750002:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.750003:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.750003:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.750004:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.750004:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.0:1619677447.750005:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.750006:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.750007:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.750008:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.750009:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.750010:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.750011:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.750022:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.750025:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.2:1619677447.750025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000400:00000200:22.0:1619677447.750026:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.750028:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.750028:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.750028:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677447.750030:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677447.750031:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.750032:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:6.0:1619677447.750033:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.750036:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.750038:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677447.750039:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.750040:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.750042:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:6.0:1619677447.750042:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.750043:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:22.0:1619677447.750044:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.750045:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.750046:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.750046:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.750046:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.750047:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.750048:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677447.750051:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.750052:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:6.0:1619677447.750053:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677447.750054:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:6.0:1619677447.750055:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677447.750056:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:6.0:1619677447.750057:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:6.0:1619677447.750057:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:8.2:1619677447.750060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677447.750063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.750068:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000800:00000200:1.0:1619677447.750068:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:22.0:1619677447.750070:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:22.0:1619677447.750071:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:1.0:1619677447.750071:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:22.0:1619677447.750072:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:17.0:1619677447.750072:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.750072:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.750074:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.750077:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.750146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677447.750152:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677447.750153:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677447.750155:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.750155:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677447.750156:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:20.2:1619677447.750157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:17.0:1619677447.750158:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.750160:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.750160:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:17.0:1619677447.750161:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.750162:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:5.0:1619677447.750162:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:31.0:1619677447.750163:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.750164:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.750164:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.750165:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:31.0:1619677447.750166:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.750166:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.750167:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.750168:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:17.0:1619677447.750168:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:5.0:1619677447.750168:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.750169:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.750170:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.750172:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.750172:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.750172:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.750173:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677447.750175:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96723987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.750176:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.750176:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:5.0:1619677447.750176:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.750177:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.750179:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.750179:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000400:00000200:5.0:1619677447.750179:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.750180:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.750180:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.750182:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:31.0:1619677447.750183:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677447.750183:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.750184:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.750185:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.750185:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.750185:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.750186:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.750186:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677447.750187:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677447.750187:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:9.2:1619677447.750187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:31.0:1619677447.750188:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.750188:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:17.0:1619677447.750188:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677447.750188:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.750188:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.750189:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.750191:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.750194:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:5.0:1619677447.750195:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.750196:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.750196:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:24.0:1619677447.750197:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.2:1619677447.750197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:24.0:1619677447.750198:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.750199:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:24.0:1619677447.750201:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.750201:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.750202:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.750202:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677447.750203:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.750204:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:12.0:1619677447.750204:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.750204:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.750205:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677447.750206:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.750206:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.750207:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.750207:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.750208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.750208:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.750209:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.750209:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.750211:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.750213:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.750213:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.0:1619677447.750216:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.750217:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.750217:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.750218:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.750219:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:30.0:1619677447.750220:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.750222:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.750224:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.750226:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.750227:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.750228:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.750229:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.750230:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.750231:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.750232:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.750235:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.750237:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.750239:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.750240:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.750241:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.750242:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.750243:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.750244:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.750245:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.750246:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677447.750254:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.750256:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.750257:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.750259:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:12.0:1619677447.750259:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.750260:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.750261:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.750262:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.750262:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.750263:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.750264:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.750264:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.750265:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.750266:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.750267:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.750405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.2:1619677447.750444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.750453:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.750455:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.750456:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.750458:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.750459:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.750461:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677447.750466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.750473:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.750476:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.750477:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.750478:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.750479:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.750479:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677447.750520:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.750523:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.750525:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.750526:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.750529:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.750530:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677447.750531:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.750533:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.750534:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.750535:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677447.750538:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.750540:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677447.750542:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677447.750542:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.750543:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677447.750544:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:18.0:1619677447.750545:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:17.0:1619677447.750546:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.750546:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677447.750546:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677447.750547:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677447.750548:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.750548:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:18.0:1619677447.750550:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.750552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:18.0:1619677447.750552:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:9.2:1619677447.750552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:18.0:1619677447.750553:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.750554:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.750555:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.750556:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.750558:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:24.0:1619677447.750559:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:18.0:1619677447.750560:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.750560:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:24.0:1619677447.750562:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:18.0:1619677447.750562:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.750563:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.750563:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.0:1619677447.750563:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:5.0:1619677447.750563:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.750565:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.750566:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.750566:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:5.0:1619677447.750566:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.750567:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677447.750567:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:24.0:1619677447.750569:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:18.0:1619677447.750569:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.750569:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:5.0:1619677447.750569:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.750570:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:5.0:1619677447.750570:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.750571:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:5.0:1619677447.750571:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.750572:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.750576:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.750578:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.750580:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.750581:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.750581:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:13.0:1619677447.750582:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.750583:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.750583:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.750584:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677447.750585:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677447.750585:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:13.0:1619677447.750586:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.750588:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.750589:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.750591:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.750597:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.750600:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.750601:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.750601:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.750603:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.750605:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.750626:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.750628:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.750629:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.750631:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.750634:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.750635:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677447.750636:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.750637:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.750638:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.750639:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:2.0:1619677447.750639:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:31.0:1619677447.750642:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:2.0:1619677447.750642:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.750644:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:31.0:1619677447.750645:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.750647:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677447.750647:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.750647:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:31.0:1619677447.750648:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.750648:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:31.0:1619677447.750650:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:2.0:1619677447.750650:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.750650:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:31.0:1619677447.750651:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.750651:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.750651:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:31.0:1619677447.750652:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.750653:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.750654:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.750657:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.750659:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:9.2:1619677447.750659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677447.750662:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.750662:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.750663:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.750664:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.750665:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.750666:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677447.750666:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:31.0:1619677447.750667:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677447.750668:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.750669:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.750670:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.750671:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.750673:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:12.0:1619677447.750676:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.750679:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.750680:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.750681:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.750683:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.750684:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.750720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.750727:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.750729:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.750730:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.750732:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.750734:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.750736:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.750737:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.750738:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.750740:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.750740:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.750744:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.750746:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.750748:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.750749:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.750750:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.750752:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.750752:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.750753:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.750753:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.750758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.750765:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.750768:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.750769:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.750769:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.750771:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.750775:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.750818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:8.2:1619677447.750856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.750864:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.750867:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.750868:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.750870:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.750870:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.750873:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.750970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.750977:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.750980:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.750983:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.750984:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.750985:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.750986:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677447.750987:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.750989:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.750990:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.750990:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677447.750994:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.750995:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677447.750998:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.750999:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.751000:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677447.751002:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.751002:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.751003:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.751003:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.751007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.751016:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.751018:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.751019:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.751021:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.751021:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.751024:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677447.751171:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.751174:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.751176:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.751178:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.751179:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.751181:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.751182:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.751183:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.751184:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.751185:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.751188:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.751190:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.751192:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.751192:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.751193:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.751194:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.751195:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.751196:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.751196:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677447.751201:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.751204:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.751205:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:13.0:1619677447.751207:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:2.0:1619677447.751207:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.751208:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.751209:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.751210:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.751210:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:13.0:1619677447.751212:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.751212:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.751213:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677447.751213:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.751214:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677447.751214:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.751215:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677447.751219:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.751221:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.751223:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.751224:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.751225:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.751227:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.751227:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.751228:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.751229:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:4.2:1619677447.751251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.751297:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.751300:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.751301:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.751301:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.751303:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.751306:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.751433:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.751436:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.751437:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.751438:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.751440:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.751441:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.751451:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.751453:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.751454:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.751455:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.751455:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.751456:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.751512:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.751515:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.751516:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.751518:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.751521:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.751523:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.751524:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.751526:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.751526:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.751528:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.751531:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.751532:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.751535:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.751536:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.751536:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.751538:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.751538:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.751539:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.751540:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.751545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.751553:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.751556:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.751556:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.751558:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.751559:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.751561:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677447.751614:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.751617:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.751617:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.751619:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.751622:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.751623:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.751624:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.751626:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.751627:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.751628:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.751631:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.751633:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.751635:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.751635:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.751636:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.751637:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.751638:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.751638:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.751639:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.751643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.751651:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.751653:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.751653:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.751654:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.751655:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.751656:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.751662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.751669:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.751672:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.751673:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.751673:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.751674:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.751674:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.751730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:5.0:1619677447.751736:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:5.0:1619677447.751739:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:20.0:1619677447.751741:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:5.0:1619677447.751741:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.751743:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.751745:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:5.0:1619677447.751746:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.751747:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.751748:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.751749:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:5.0:1619677447.751753:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677447.751753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:5.0:1619677447.751754:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.751757:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.751758:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677447.751758:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.751760:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.751760:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:24.0:1619677447.751761:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.0:1619677447.751761:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:5.0:1619677447.751762:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:24.0:1619677447.751763:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.751764:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.751766:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677447.751766:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:24.0:1619677447.751769:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.751771:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.751773:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.751774:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.751774:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:24.0:1619677447.751775:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.751776:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.751777:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.751777:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.751780:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.751780:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.751780:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:24.0:1619677447.751783:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.751783:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:24.0:1619677447.751786:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.751787:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.751787:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.751789:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.751789:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.751791:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.751793:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.751794:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.751795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.751799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.751803:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.751805:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.751806:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.751806:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:1.0:1619677447.751807:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.751808:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.751809:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677447.751809:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.751810:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.751812:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677447.751897:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.751900:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.751901:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.751903:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.751905:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.751907:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.751908:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.751909:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.751911:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.751912:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.751915:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.751917:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.751920:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.751921:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.751922:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.751923:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.751924:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.751924:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.751925:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.752050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.752071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.752079:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.752081:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.752083:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.752085:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.752086:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.752086:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677447.752087:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.752089:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.752089:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.752090:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677447.752093:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.752095:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677447.752095:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:30.0:1619677447.752096:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.752097:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.752098:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:22.0:1619677447.752098:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.752098:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.752099:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:22.0:1619677447.752099:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.752100:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.752100:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.752101:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:22.0:1619677447.752101:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.752104:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677447.752113:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.752115:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.752116:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.752118:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.752119:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.752120:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:12.0:1619677447.752158:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.752160:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.752161:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.752162:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.752164:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.752166:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.752177:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.752180:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.752180:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.752182:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.752184:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.752185:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:17.0:1619677447.752186:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.752187:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.752188:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:17.0:1619677447.752189:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.752189:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:17.0:1619677447.752190:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.752192:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677447.752193:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.752196:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.752196:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.752198:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:17.0:1619677447.752198:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:24.0:1619677447.752199:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.752201:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.752201:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.752202:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.752202:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:24.0:1619677447.752203:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677447.752203:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.752204:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.752205:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:24.0:1619677447.752206:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.752206:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.752206:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677447.752208:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.752211:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.752212:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.752213:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.752214:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.752215:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.752216:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.752217:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:12.0:1619677447.752217:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:24.0:1619677447.752218:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.752220:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.752220:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.752221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:12.0:1619677447.752222:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.752224:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.752225:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.752231:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:5.0:1619677447.752231:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.0:1619677447.752233:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.752233:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.752234:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677447.752234:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.752234:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.752236:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.752237:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.752239:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:6.0:1619677447.752524:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.752527:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.752528:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.752530:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.752533:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.752534:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.752536:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.752538:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.752539:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.752540:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.752545:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.752547:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.752550:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.752551:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.752552:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677447.752553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:6.0:1619677447.752553:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.752553:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.752554:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:20.0:1619677447.752556:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:6.0:1619677447.752556:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.752556:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677447.752559:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:8.2:1619677447.752560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:20.0:1619677447.752561:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677447.752563:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.752565:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:5.0:1619677447.752565:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.752566:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96723960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.752566:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:22.0:1619677447.752567:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:20.0:1619677447.752567:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.752568:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.752568:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.752568:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.752569:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.752569:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677447.752569:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.752570:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:5.0:1619677447.752570:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.752571:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.752573:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:20.0:1619677447.752573:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.752573:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:30.0:1619677447.752574:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.752574:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:30.0:1619677447.752576:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.752576:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.2:1619677447.752577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.752578:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677447.752578:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.752579:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.0:1619677447.752579:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.752580:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.752582:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.752583:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.752584:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.752584:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.752586:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.752586:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.752586:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.752586:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.752587:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.752587:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.752588:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.752591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:31.0:1619677447.752595:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677447.752596:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:31.0:1619677447.752598:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.752599:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.752599:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.752599:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:31.0:1619677447.752600:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.752600:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677447.752601:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.752603:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.752613:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.752615:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.752616:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.752617:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.752618:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.752619:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:1.0:1619677447.752628:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.752631:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.752632:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.752634:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.752637:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.752638:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.752639:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.752641:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.752642:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.752643:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.2:1619677447.752646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:1.0:1619677447.752646:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.752648:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.752651:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.752652:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.752653:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.752654:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:1.0:1619677447.752654:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.752655:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.752656:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677447.752657:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:1.0:1619677447.752657:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.752658:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677447.752659:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.752660:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.752661:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.752662:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.752663:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.752665:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.752666:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.752667:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677447.752668:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677447.752669:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:31.0:1619677447.752670:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.752671:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:31.0:1619677447.752672:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677447.752672:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677447.752672:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.752672:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.752672:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.752675:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.752676:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:12.0:1619677447.752676:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.752677:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:13.0:1619677447.752678:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.752679:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.752680:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.752681:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:13.0:1619677447.752681:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677447.752682:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.752683:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677447.752685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677447.752689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677447.752692:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677447.752693:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:12.0:1619677447.752695:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.752695:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677447.752695:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:12.0:1619677447.752696:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:30.0:1619677447.752697:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:12.0:1619677447.752697:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:5.0:1619677447.752697:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.752698:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:12.0:1619677447.752698:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:5.0:1619677447.752698:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.752699:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.752700:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.752701:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677447.752702:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.752745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.752786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.752791:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.752794:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.752794:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.752795:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.752797:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.752800:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.752808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:22.0:1619677447.752810:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.752811:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677447.752812:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.752813:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.752813:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.752814:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.752814:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:22.0:1619677447.752815:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:20.0:1619677447.752816:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.752817:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.752818:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.752820:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.752821:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.752823:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.752823:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.752825:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.752828:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.752830:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.752833:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.752834:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.752835:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.752837:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.752837:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.752839:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.752840:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.752840:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677447.752840:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (21)++
00000800:00000200:2.0:1619677447.752843:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:18.0:1619677447.752844:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000800:00000200:20.2:1619677447.752845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:2.0:1619677447.752846:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.752849:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:2.0:1619677447.752849:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.752851:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:2.0:1619677447.752851:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:12.0:1619677447.752852:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:2.0:1619677447.752852:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677447.752853:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.752853:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:2.0:1619677447.752853:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.752854:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:2.0:1619677447.752854:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677447.752855:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.752855:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.752856:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677447.752856:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677447.752859:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.752861:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:12.0:1619677447.752862:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.752865:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.752866:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.752866:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:2.0:1619677447.752866:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:18.0:1619677447.752868:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.752869:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:2.0:1619677447.752870:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (21)++
00000400:00000200:18.0:1619677447.752871:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.752871:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919065782400] (22)++
00000800:00000200:2.0:1619677447.752872:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (23)--
00000400:00000200:18.0:1619677447.752873:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.752873:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000800:00000200:2.0:1619677447.752874:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.752875:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:8.2:1619677447.752875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000400:00000200:18.0:1619677447.752876:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.752878:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.752878:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.752879:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.752882:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.752884:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.752885:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.752885:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:2.0:1619677447.752885:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (22)++
00000800:00000200:2.0:1619677447.752885:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919065782400] (23)--
00000800:00000200:18.0:1619677447.752886:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.752886:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000800:00000200:18.0:1619677447.752887:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.752887:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (21)--
00000800:00000200:5.0:1619677447.752888:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:18.0:1619677447.752889:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.752889:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:5.0:1619677447.752889:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.752890:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000400:00000200:5.0:1619677447.752890:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.752891:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:18.0:1619677447.752891:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.752892:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677447.752893:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:9.2:1619677447.752895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.752903:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.752904:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.752905:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.752905:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.752906:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.752906:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.752907:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.752908:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:2.0:1619677447.752908:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.752910:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.752991:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.752994:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.752994:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.752996:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.752998:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.753000:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.753001:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.753002:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.753003:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.753004:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.753007:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.753009:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.753011:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.753012:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.753013:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.753014:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.753015:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.753015:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.753016:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.753021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.753029:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.753031:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.753032:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.753033:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.753035:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.753036:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.753055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:8.2:1619677447.753105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677447.753109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:5.0:1619677447.753113:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.753115:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.753116:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.753118:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.753119:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.753122:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.753213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.753220:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.753223:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.753225:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.753227:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.753228:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.753229:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677447.753231:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.753232:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.753233:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.753234:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677447.753237:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.753239:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.753242:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.753242:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.753243:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.753244:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.753245:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.753246:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.753246:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677447.753256:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.753259:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.753260:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.753261:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.753263:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.753263:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.753284:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.753287:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.753289:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.753291:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.753291:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.753293:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677447.753299:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723949] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.753300:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.753301:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.753302:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677447.753305:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.753306:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677447.753309:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.753310:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677447.753310:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677447.753312:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.753312:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.753313:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.753314:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677447.753327:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.753330:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.753330:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.753331:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.753333:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.753336:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.753347:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.753350:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.753350:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.753351:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.753352:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.753352:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:17.0:1619677447.753422:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.753425:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.753426:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.753428:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.753431:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.753432:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.753433:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.753435:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.753436:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.753437:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.753441:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.753442:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.753445:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.753446:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.753447:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.753449:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.753449:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.753449:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.753450:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.753462:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.753464:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.753465:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.753467:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.753469:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.753470:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.753527:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.753530:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.753530:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.753532:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.753535:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.753536:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.753537:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.753537:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:22.0:1619677447.753538:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.753539:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.753540:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.753540:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.753541:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.753545:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:22.0:1619677447.753546:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.753546:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.753549:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.753549:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.753550:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.753551:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.753552:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:18.0:1619677447.753552:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.753553:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:18.0:1619677447.753553:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.753554:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.753555:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:18.0:1619677447.753557:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:22.0:1619677447.753558:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.753559:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.753559:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677447.753559:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.753561:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677447.753561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.753562:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.753563:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.753564:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.753565:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.753566:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.753567:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.753567:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677447.753569:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.753572:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.753573:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.753573:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.753575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677447.753575:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.753577:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677447.753578:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677447.753578:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.753580:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:6.0:1619677447.753581:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677447.753581:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677447.753582:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.753582:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.753583:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:6.0:1619677447.753584:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.753587:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677447.753622:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.753625:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.753625:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.753627:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.753629:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.753630:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.753650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.753653:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.753656:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.753658:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.753659:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.753661:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.753662:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.753663:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.753665:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.753666:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.753667:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.753670:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.753672:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.753675:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.753676:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.753676:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.753678:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.753679:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.753679:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.753680:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.753690:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.753693:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.753694:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.753695:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.753697:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.753698:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.753713:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.753716:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.753717:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.753719:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.753721:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.753723:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.753724:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.753726:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.753726:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.753727:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.753731:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.753732:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.753736:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.753736:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.753737:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.753739:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.753739:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.753740:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.753740:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.753755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.753763:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.753766:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.753767:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.753767:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.753769:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.753772:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.754040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.754048:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.754051:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.754053:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.754055:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.754055:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.754057:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.754058:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.754060:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.754061:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.754062:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.754065:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.754067:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.754070:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.754071:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.754071:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.754073:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.754074:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.754074:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.754075:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.754079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.754087:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.754090:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.754091:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.754092:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.754093:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.754096:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.754137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.754144:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.754147:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.754148:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.754149:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.754152:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.754154:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.754155:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.754156:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.754157:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677447.754158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677447.754158:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.754161:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.754163:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.754165:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.754165:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.0:1619677447.754166:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.754166:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.754168:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.754168:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.754169:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.754169:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:2.0:1619677447.754169:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.754170:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.754170:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677447.754172:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.754175:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.2:1619677447.754176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.754185:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.754188:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.754188:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677447.754189:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677447.754189:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.754191:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:17.0:1619677447.754191:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.754194:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.754194:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677447.754195:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.754197:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.754198:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677447.754199:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.754201:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.754202:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.754203:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677447.754206:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.754207:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677447.754210:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.754211:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677447.754212:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677447.754213:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677447.754214:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.754216:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.754217:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.754218:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677447.754228:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.754230:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:30.0:1619677447.754231:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:6.0:1619677447.754231:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.754232:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.754233:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.754234:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677447.754234:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.754235:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677447.754236:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677447.754370:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.754372:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.754373:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677447.754374:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:1.0:1619677447.754375:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.754377:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.754377:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:1.0:1619677447.754377:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.754379:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677447.754380:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.754380:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.754382:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.754382:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.754383:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.754384:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.754384:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677447.754385:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.754387:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.754388:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.754388:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.754389:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.754391:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:18.0:1619677447.754393:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.754395:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.754396:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.754397:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.754397:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677447.754398:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.754398:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677447.754399:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.754399:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677447.754400:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.754400:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.754401:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:1.0:1619677447.754401:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677447.754402:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:1.0:1619677447.754402:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677447.754403:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.754404:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677447.754407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.754414:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:17.0:1619677447.754415:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677447.754415:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677447.754416:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:17.0:1619677447.754417:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:2.0:1619677447.754417:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677447.754418:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.754418:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677447.754418:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:6.0:1619677447.754419:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.754419:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.754420:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:17.0:1619677447.754422:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.754422:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.754425:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.754426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.754426:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677447.754433:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.754436:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.754437:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.754437:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.754438:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.754439:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.754440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677447.754444:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.754446:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.754448:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.754450:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.754450:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.754451:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.754453:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.754455:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.754455:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677447.754456:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.754458:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.754459:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.754460:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.754462:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.754463:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.754464:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:20.0:1619677447.754465:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.754465:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.754467:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.754468:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.754469:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.754472:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.754473:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:5.0:1619677447.754473:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.754474:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.754475:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.754475:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.754476:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.754477:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.754477:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.754477:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.754478:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677447.754478:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.754479:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.754479:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.754481:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.754481:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.754482:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.754483:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:13.0:1619677447.754489:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.754492:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.754493:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.754495:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.754496:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.754497:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.754497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.754506:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.754507:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.754509:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:12.0:1619677447.754509:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.754510:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.754510:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.754510:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:12.0:1619677447.754511:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.754512:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:18.0:1619677447.754513:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:12.0:1619677447.754513:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.754514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.754516:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.754518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.754525:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.754528:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.754529:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.754529:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.754530:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.754530:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.754561:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.754564:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.754565:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.754565:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.754568:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.754571:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677447.754710:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.754712:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.754715:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.754717:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.754717:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.754719:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.754720:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.754722:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.754723:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.754724:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.754727:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.754729:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.754731:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.754731:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.754732:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.754734:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.754734:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.754735:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.754735:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.754746:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.754748:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.754748:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.754750:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.754752:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.754752:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.754817:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.754820:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.754821:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.754823:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.754826:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.754827:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.754828:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.754830:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.754831:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.754832:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.754835:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.754837:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.754840:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.754841:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.754842:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.754843:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.754844:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.754844:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.754845:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.754856:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.754859:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.754860:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.754862:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.754863:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.754865:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677447.754955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.754961:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.754963:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.754964:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.754967:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.754970:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.754971:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.754973:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.754974:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.754976:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.754977:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.754981:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.754984:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.754987:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.754987:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.754988:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.754990:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.754991:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.754992:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.754992:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.754997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.755005:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.755008:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.755009:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.755009:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.755012:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.755014:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.755059:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.755061:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.755062:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.755064:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.755066:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.755068:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.755069:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.755070:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.755072:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.755072:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.755076:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.755078:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.755081:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.755082:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.755083:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.755085:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.755085:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.755086:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.755086:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677447.755098:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.755101:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.755102:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.755103:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.755105:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.755106:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.755107:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677447.755108:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677447.755108:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.755110:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.755112:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.755114:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677447.755115:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.755117:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.755118:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.755119:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677447.755123:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.755124:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677447.755128:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.755128:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677447.755129:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677447.755131:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.755131:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.755132:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.755133:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677447.755144:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677447.755144:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:22.0:1619677447.755147:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677447.755147:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.755148:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:6.0:1619677447.755148:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.755149:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.755150:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.755151:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.755153:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.755154:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:22.0:1619677447.755155:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:2.0:1619677447.755156:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.755157:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.755158:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.755159:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.755163:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.755164:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.755167:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.755167:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.755168:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.755170:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.755170:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.755171:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.755172:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.755176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.755184:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.755187:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.755188:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.755188:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.755190:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.755194:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.755195:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.755197:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.755198:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.755200:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.755202:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.755203:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.755204:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.755205:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.755206:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.755207:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.755211:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.755213:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.755216:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.755216:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.755217:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.755219:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.755219:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.755220:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.755221:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.755222:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.755225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.755231:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677447.755232:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.755234:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.0:1619677447.755234:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.755235:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:17.0:1619677447.755235:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677447.755235:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.755236:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677447.755238:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.755241:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.755379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.755387:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907e3ce200] (21)++
00000800:00000200:18.0:1619677447.755391:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.7@o2ib3
00000800:00000200:2.0:1619677447.755391:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3ce200] (22)--
00000400:00000200:18.0:1619677447.755394:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.7@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.755396:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.755397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:18.0:1619677447.755398:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000800:00000200:20.0:1619677447.755400:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:18.0:1619677447.755400:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.755402:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:18.0:1619677447.755402:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.755403:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.755404:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000800:00000200:5.0:1619677447.755405:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.755406:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.755409:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.755410:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.755411:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.755412:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96723923] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.755414:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.755414:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000400:00000200:20.0:1619677447.755415:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.755417:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:18.0:1619677447.755418:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f90d380] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.755418:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e49d37c00] (20)++
00000800:00000200:18.0:1619677447.755419:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e49d37c00] (21)++
00000400:00000200:20.0:1619677447.755421:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.755421:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e49d37c00] (22)--
00000800:00000200:18.0:1619677447.755422:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907e3ce200] (21)++
00000800:00000200:20.0:1619677447.755423:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677447.755423:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907e3ce200] (22)++
00000800:00000200:18.0:1619677447.755425:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907e3ce200] (23)--
00000800:00000200:18.0:1619677447.755425:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3ce200] (22)--
00000800:00000200:20.0:1619677447.755426:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.755427:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:24.2:1619677447.755428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (21)++
00000800:00000200:20.0:1619677447.755428:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677447.755430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677447.755431:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)++
00000800:00000200:20.0:1619677447.755431:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.755432:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.755433:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:24.0:1619677447.755434:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e49d37c00] (23)--
00000800:00000200:20.0:1619677447.755434:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:24.0:1619677447.755435:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.755435:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (23)--
00000800:00000200:6.0:1619677447.755436:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677447.755436:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907e3ce200] (22)++
00000400:00000200:24.0:1619677447.755438:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.755438:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907e3ce200] (23)--
00000800:00000200:6.0:1619677447.755439:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677447.755439:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3ce200] (22)--
00000800:00000200:22.0:1619677447.755440:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.755440:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.755440:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3ce200] (21)--
00000800:00000200:24.0:1619677447.755441:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)--
00000400:00000200:6.0:1619677447.755443:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.755444:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.755447:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.755448:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.755448:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:30.0:1619677447.755450:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.755450:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677447.755452:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.755452:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.755453:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:22.0:1619677447.755453:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.755453:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.755454:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.755455:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677447.755462:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677447.755466:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.755466:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.755467:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.755469:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.755469:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677447.755500:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.755502:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.755504:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.755506:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.755507:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.755509:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.755510:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.755511:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.755512:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.755513:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.2:1619677447.755514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:31.0:1619677447.755516:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.755518:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.755520:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.755521:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.755522:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.755522:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.755523:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.755523:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677447.755524:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.755525:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.755525:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:30.0:1619677447.755525:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.755527:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.755527:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.755529:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677447.755530:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.755531:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.755532:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.755533:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677447.755536:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.755536:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677447.755538:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:12.0:1619677447.755538:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.755539:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.755541:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677447.755541:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.755542:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.755542:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:12.0:1619677447.755543:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.755544:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.755544:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677447.755544:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.755545:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.755546:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.755550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677447.755557:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.755559:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677447.755560:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:5.0:1619677447.755561:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:1.0:1619677447.755561:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677447.755562:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.755563:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.755564:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.755564:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.755566:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.755567:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677447.755567:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:17.0:1619677447.755569:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.755570:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.755571:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.755572:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.755576:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677447.755576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:17.0:1619677447.755577:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.755580:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.755581:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.755582:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.755584:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677447.755584:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.755584:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.755585:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.755585:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677447.755586:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.755587:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.755588:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.755589:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.755590:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:12.0:1619677447.755596:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.755599:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.755599:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.755601:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.755602:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.755604:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.755607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.755614:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.755617:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.755618:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.755620:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.755622:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.755624:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.755625:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.755627:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.755628:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.755629:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.755632:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.755634:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.755636:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.755637:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.755638:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.755640:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.755640:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.755641:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.755641:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.755646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.755654:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.755657:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.755657:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.755658:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.755660:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.755662:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:24.0:1619677447.755732:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.755735:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.755736:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.755737:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.755740:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.755741:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.755743:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.755744:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.755745:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.755746:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.755749:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.755751:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.755753:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.755754:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.755755:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.755757:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.755757:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.755758:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.755758:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.755763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.755770:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.755773:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.755774:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.755774:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.755776:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.755779:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.755787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.755827:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:9.2:1619677447.755830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.755830:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.755833:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.755834:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:18.0:1619677447.755835:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.755836:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.755837:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.755837:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.755839:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.755840:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.755840:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96723915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.755842:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.755843:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.755844:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.2:1619677447.755845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:18.0:1619677447.755847:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.755848:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:2.0:1619677447.755848:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.755850:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.755852:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.755853:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.755854:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677447.755854:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.755855:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:2.0:1619677447.755856:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.755856:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677447.755856:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.755857:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:2.0:1619677447.755857:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.755857:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:17.0:1619677447.755858:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.755860:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.755860:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.755863:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.755864:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.755865:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.755865:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:17.0:1619677447.755866:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:1.0:1619677447.755866:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.755868:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.755869:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:1.0:1619677447.755869:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.755870:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:22.0:1619677447.755872:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.755873:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.755873:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:1.0:1619677447.755873:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.755875:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:22.0:1619677447.755876:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.755878:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.755878:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.755879:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.755880:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.755882:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.755882:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.755882:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.755883:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.755887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.755894:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.755896:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.755897:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.755898:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.755900:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.755903:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:18.0:1619677447.756103:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.756106:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.756106:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.756109:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.756111:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.756113:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.756114:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.756115:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.756116:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.756117:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.756119:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.756121:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.756123:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.756124:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.756124:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.756126:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.756126:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.756127:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.756127:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.756132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.756139:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.756141:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.756141:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.756142:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.756143:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.756144:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.756150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.756156:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.756159:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.756159:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.756160:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.756160:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.756161:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.756176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:4.2:1619677447.756182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.756210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.756215:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.756218:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.756221:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.756221:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677447.756223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:5.0:1619677447.756224:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.756225:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.756227:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96723910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.756228:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.756229:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.756230:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677447.756231:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.756233:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:5.0:1619677447.756233:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.756234:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.756235:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.756235:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:17.0:1619677447.756237:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.756237:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677447.756238:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.756239:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (23)++
00000800:00000200:17.0:1619677447.756240:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:6.0:1619677447.756240:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:5.0:1619677447.756240:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (24)++
00000800:00000200:22.0:1619677447.756241:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.756241:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.756242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677447.756242:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.756243:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.756243:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.756245:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:6.0:1619677447.756246:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.756246:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.756246:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677447.756250:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677447.756251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677447.756253:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.756253:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:20.0:1619677447.756254:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677447.756254:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.756255:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:22.0:1619677447.756256:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.756256:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:1.0:1619677447.756256:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.0:1619677447.756257:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677447.756258:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677447.756258:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:22.0:1619677447.756259:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.756260:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.756261:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.756263:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.756295:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.756298:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.756301:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.756303:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.756304:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.756305:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.756307:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677447.756308:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.756310:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.756311:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.756312:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677447.756315:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.756317:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.756319:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.756320:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677447.756321:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.756322:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.756323:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.756324:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.756324:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677447.756329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.756337:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.756340:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.756340:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.756341:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.756343:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677447.756345:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.756397:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677447.756400:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677447.756401:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677447.756403:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677447.756405:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.756406:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677447.756407:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723908] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.756409:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.756410:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.756411:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677447.756415:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677447.756416:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677447.756419:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.756420:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:1.0:1619677447.756421:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:1.0:1619677447.756423:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:1.0:1619677447.756423:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677447.756424:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677447.756424:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:31.0:1619677447.756434:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.756437:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.756440:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.756442:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.756442:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.756444:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.756445:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.756446:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.756447:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.756448:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.756452:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.756452:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:31.0:1619677447.756454:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.756455:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.756456:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:31.0:1619677447.756457:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.756458:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:2.0:1619677447.756458:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.756459:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.756460:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677447.756460:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.756461:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.756462:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:2.0:1619677447.756462:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.756463:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.756464:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.756464:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:2.0:1619677447.756464:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.756465:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.756466:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677447.756470:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.756472:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:13.0:1619677447.756473:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.756475:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.756476:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677447.756476:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.756476:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677447.756477:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:13.0:1619677447.756477:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.756478:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.756478:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.756478:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.756479:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:30.0:1619677447.756480:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:13.0:1619677447.756480:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.756480:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:30.0:1619677447.756481:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677447.756481:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.756482:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.756493:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.756496:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.756497:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.756498:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.756498:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.756500:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.756521:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.756523:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.756524:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.756526:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.756529:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.756530:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.756532:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.756533:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.756534:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.756535:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.756538:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.756540:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.756542:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.756543:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.756544:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.756545:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.756545:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.756546:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.756547:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.756551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.756559:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.756561:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.756561:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.756562:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.756563:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.756565:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.756573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.756580:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.756583:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.756583:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.756584:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.756584:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.756585:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677447.756639:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677447.756642:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677447.756642:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677447.756644:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.756648:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.756650:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677447.756651:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.756652:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.756653:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.756654:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677447.756658:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.756660:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677447.756663:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.756663:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:18.0:1619677447.756664:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:18.0:1619677447.756666:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:18.0:1619677447.756667:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677447.756668:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:18.0:1619677447.756669:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:18.0:1619677447.756670:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:9.2:1619677447.756672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.756679:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:2.0:1619677447.756681:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:24.0:1619677447.756682:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.756683:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.756683:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:2.0:1619677447.756683:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677447.756684:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:24.0:1619677447.756685:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.756685:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:24.0:1619677447.756688:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677447.756720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.756727:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.756730:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.756732:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.756734:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.756734:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.756736:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.756737:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.756738:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.756739:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.756740:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.756743:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.756745:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.756748:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.756748:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.756749:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.756751:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.756751:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.756753:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.756754:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.756755:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677447.756760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677447.756762:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.756765:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.756766:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.756767:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.756768:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:13.0:1619677447.756769:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.756770:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677447.756771:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.756772:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.756773:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:4.2:1619677447.756774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.2:1619677447.756812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.756821:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.756823:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.756824:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.756826:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.756826:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.756830:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677447.756847:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.756849:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.756850:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.756852:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.756853:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.756855:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.756864:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.756867:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.756868:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.756868:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677447.756869:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.756871:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:18.0:1619677447.757166:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.757169:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.757170:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.757172:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.757175:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.757177:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.757178:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.757179:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.757181:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.757181:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.757185:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.757187:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.757189:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.757190:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.757191:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.757192:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.757193:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.757193:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.757194:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.757199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.757204:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.0:1619677447.757206:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677447.757206:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.757207:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:1.0:1619677447.757208:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677447.757209:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.757209:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.757209:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.757211:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.757211:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:20.0:1619677447.757212:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.757212:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677447.757212:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:2.0:1619677447.757213:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677447.757215:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.757215:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677447.757216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:2.0:1619677447.757216:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.757217:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.757217:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.757218:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.757219:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:20.0:1619677447.757220:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:1.0:1619677447.757220:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.757221:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:1.0:1619677447.757222:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.757223:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.757223:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.757224:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.757224:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:1.0:1619677447.757224:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.757225:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.757225:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.757225:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677447.757226:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:1.0:1619677447.757228:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.757229:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.757229:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677447.757230:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.757230:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677447.757232:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.757232:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.757233:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.757233:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:1.0:1619677447.757233:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677447.757234:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:1.0:1619677447.757234:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.757235:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.757236:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.757237:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.757238:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.757239:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677447.757245:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.757247:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:17.0:1619677447.757249:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:13.0:1619677447.757249:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.757249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.757250:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:13.0:1619677447.757251:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.757251:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.757252:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.757253:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.757253:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.757253:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.757254:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:13.0:1619677447.757254:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:6.0:1619677447.757254:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.757256:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.757258:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.757271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.757279:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.757282:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.757284:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.757285:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.757286:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.757288:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.757289:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.757290:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677447.757290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677447.757291:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.757292:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.757293:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.757301:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.757302:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.757302:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.757303:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.757304:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.757306:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.757307:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.757307:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.757307:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677447.757308:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.757310:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.757310:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.757311:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.757312:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.757324:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.757326:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.757327:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.757328:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.757330:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.757331:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.757341:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.757343:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.757344:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.757345:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.757346:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.757347:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.757403:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.757405:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.757406:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.757408:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.757410:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.757411:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677447.757413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.757420:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.757423:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.757423:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.757425:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.757428:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.757430:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.757431:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.757432:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.757433:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.757434:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.757437:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.757438:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.757441:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.757442:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.757443:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.757444:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.757445:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.757445:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.757446:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677447.757450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677447.757450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.757453:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.757456:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:5.0:1619677447.757456:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677447.757457:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677447.757458:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:22.0:1619677447.757459:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.757459:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.757461:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.757461:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.757461:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:22.0:1619677447.757462:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.757462:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.757463:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.757464:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.757465:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677447.757466:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.757466:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.757467:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.757468:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.757469:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.757469:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:5.0:1619677447.757470:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.757471:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.757471:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.757472:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677447.757472:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677447.757478:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.757478:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.757481:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.757482:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.757485:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.757485:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.757485:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:22.0:1619677447.757486:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677447.757486:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677447.757487:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.757489:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677447.757489:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677447.757490:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677447.757490:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.757491:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677447.757492:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:5.0:1619677447.757492:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.757493:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.757494:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.757495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:12.0:1619677447.757503:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677447.757504:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677447.757505:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:12.0:1619677447.757505:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.757506:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.757507:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:6.0:1619677447.757507:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:18.0:1619677447.757508:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.757508:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.757508:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.757508:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.757509:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.757509:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.757511:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.757511:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.757514:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677447.757745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.757753:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.757756:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.757758:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.757760:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.757760:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.757761:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.757763:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.757764:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.757765:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.757766:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.757769:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.757771:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.757774:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.757775:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.757775:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.757777:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.757778:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.757778:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.757779:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.757783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.757791:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.757793:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.757794:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.757796:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.757797:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.757800:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677447.757873:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.757876:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.757877:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.757879:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.757881:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.757883:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.757884:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.757885:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.757886:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.757887:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.757891:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.757893:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.757896:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.757897:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.757897:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.757900:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.757900:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.757901:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.757901:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.757912:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.757915:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.757916:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.757918:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.757919:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.757920:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.757922:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.757925:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.757926:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.757928:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.757931:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.757932:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677447.757933:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.757935:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.757936:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.757936:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677447.757939:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.757941:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.757944:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.757944:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677447.757945:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677447.757947:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.757947:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.757949:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.757950:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.757950:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.757958:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.757961:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.757962:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.757963:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.757963:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:13.0:1619677447.757965:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.757965:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:12.0:1619677447.757966:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.757966:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.757967:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:2.0:1619677447.758029:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.758031:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.758033:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.758034:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.758037:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.758038:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677447.758039:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.758041:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.758041:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.758042:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677447.758045:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.758047:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677447.758049:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.758050:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677447.758051:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677447.758053:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677447.758053:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.758054:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.758054:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.758066:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.758069:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.758070:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.758070:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.758072:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.758074:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.758085:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.758088:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.758088:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.758089:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.758090:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.758091:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.758180:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.758182:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.758183:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.758184:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.758186:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.758187:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.758194:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.758196:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.758197:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.758199:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.758201:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.758203:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.758204:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.758205:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.758206:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.758207:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.758211:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.758213:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.758215:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.758216:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.758216:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.758218:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.758218:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.758219:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.758219:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.758224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.758232:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.758235:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.758236:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.758236:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.758238:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.758240:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.758243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.758244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.758251:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.758255:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.758255:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.758256:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.758257:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.758257:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:31.0:1619677447.758289:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.758292:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.758300:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.758301:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:31.0:1619677447.758302:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.758304:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.758359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.758367:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.758370:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.758372:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.758374:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.758374:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.758376:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.758377:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.758378:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.758379:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.758380:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.758383:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.758384:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.758387:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.758388:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.758389:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.758390:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.758391:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.758392:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.758392:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.758397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.758405:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.758408:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.758409:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.758410:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.758412:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.758413:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677447.758496:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.758499:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.758500:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.758502:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.758505:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.758506:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.758508:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.758510:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.758511:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.758512:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.758515:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.758517:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.758520:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.758521:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.758522:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.758523:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.758524:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.758525:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.758525:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.758538:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.758540:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.758541:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.758543:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.758545:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.758546:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677447.758592:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.758595:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.758596:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.758597:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.758600:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.758602:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.758603:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.758604:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.758605:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.758606:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.758609:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.758611:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.758614:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.758615:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.758615:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.758617:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.758618:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.758618:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.758619:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677447.758623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.758631:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.758633:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.758634:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.758636:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.758637:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.758638:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:6.0:1619677447.758731:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.758733:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.758734:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.758737:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.758740:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.758741:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.758743:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.758745:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.758746:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.758747:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.758752:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.758754:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.758757:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.758758:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:4.2:1619677447.758758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:6.0:1619677447.758759:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.758761:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.758761:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.758762:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.758763:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.758765:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.758768:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.758768:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.758770:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.758773:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.758774:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.758775:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96723878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677447.758778:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:8.0:1619677447.758781:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.758782:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.758783:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.758785:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:8.0:1619677447.758787:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.758788:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.758789:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.758789:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.758789:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.758791:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.758791:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.758792:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.758793:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.758794:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677447.758795:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.758796:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.758797:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.758798:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.758798:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.758804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.758811:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.2:1619677447.758811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.758812:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.758813:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.758814:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.758819:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.758822:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.758823:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.758823:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.758825:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.758827:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.758853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.758859:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.758862:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.758864:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.758864:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.758866:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.758868:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.758869:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.758871:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.758872:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.758873:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.758876:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.758879:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.758882:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.758883:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.758884:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.758886:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.758887:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.758888:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.758888:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677447.758889:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.758892:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.758894:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.758895:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.758897:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.758898:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:13.0:1619677447.758899:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:31.0:1619677447.758900:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.758901:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:31.0:1619677447.758902:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677447.758902:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.758903:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.758904:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:13.0:1619677447.758904:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.758905:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:31.0:1619677447.758907:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.758909:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.758911:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.758911:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.758912:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.758913:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.758914:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.758915:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.758916:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.758917:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.758926:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.758928:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.758929:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.758931:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.758932:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.758933:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.758949:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.758951:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.758952:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.758953:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.758953:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.758954:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.758967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.758970:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.758973:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.758975:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.758976:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.758978:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.758980:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.758981:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.758982:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.758984:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.758985:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.758988:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.758990:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.758993:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.758994:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.758995:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.758997:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.758997:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.758998:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.758999:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677447.759009:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.759012:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.759013:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.759015:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.759016:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.759018:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:2.0:1619677447.759119:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677447.759122:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677447.759123:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677447.759125:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.759128:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.759129:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677447.759131:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723874] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.759132:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.759133:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.759134:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677447.759138:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.759139:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677447.759143:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.759143:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677447.759144:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677447.759146:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677447.759146:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677447.759147:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677447.759147:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:31.0:1619677447.759299:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.759302:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677447.759303:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677447.759304:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.759306:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.759306:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677447.759306:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.759307:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677447.759309:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677447.759310:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.759310:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.759311:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:7.0:1619677447.759311:0:520:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000400:00000200:31.0:1619677447.759312:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.759312:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.759313:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.759314:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.759315:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.759317:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.759318:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.759318:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.759320:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.759321:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.759323:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.759324:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.759325:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.759325:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.759326:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677447.759328:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677447.759328:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677447.759328:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.759329:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677447.759329:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.759330:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677447.759330:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.759331:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.759332:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.759333:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.759333:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.759343:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.759346:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:13.0:1619677447.759346:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:22.0:1619677447.759347:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.759347:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.759349:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.759349:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.759350:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.759352:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:13.0:1619677447.759352:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.759354:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.759355:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.759355:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.759356:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.759356:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.759357:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.759358:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.759424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:9.2:1619677447.759463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.759471:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.759474:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.759475:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.759475:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.759477:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.759481:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.2:1619677447.759554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677447.759562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.759562:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.759565:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.759565:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.759566:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.759568:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.759571:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677447.759606:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.759609:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.759610:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.759611:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.759613:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.759615:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677447.759626:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677447.759628:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.759629:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.759630:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.759630:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.759631:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:2.0:1619677447.759681:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.759684:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.759685:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.759687:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.759690:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.759691:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.759693:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.759694:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.759695:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.759696:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.759699:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.759701:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.759702:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.759703:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.759704:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.759705:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.759706:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.759707:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.759708:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.759709:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677447.759711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.759719:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.759720:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.759721:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.759721:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.759721:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.759722:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.759723:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.759723:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.759724:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677447.759725:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.759794:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.759797:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.759797:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.759799:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.759800:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:2.0:1619677447.759800:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.759801:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.759802:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.759803:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:2.0:1619677447.759805:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.759806:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.759807:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.759808:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.759808:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.759811:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.759811:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.759813:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:2.0:1619677447.759813:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.759814:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.759816:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.759817:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.759817:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.759817:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:31.0:1619677447.759818:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.759818:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.759820:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.759820:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:31.0:1619677447.759821:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.759821:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.759822:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:31.0:1619677447.759823:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.759823:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677447.759827:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.759827:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677447.759828:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:31.0:1619677447.759830:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:31.0:1619677447.759831:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.759832:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.759832:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.759832:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:18.0:1619677447.759833:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.759835:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:13.0:1619677447.759835:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:18.0:1619677447.759836:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:13.0:1619677447.759836:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.759837:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:13.0:1619677447.759838:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.759839:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677447.759840:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677447.759841:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.759841:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.759842:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.759842:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:20.2:1619677447.759844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:12.0:1619677447.759844:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.759849:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.759852:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.759855:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.759855:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.759858:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.759860:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.759861:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.759862:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.759864:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.759864:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.759868:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.759870:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.759873:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.759874:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.759875:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.759876:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.759877:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.759878:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.759879:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677447.759889:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.759892:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.759893:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.759894:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.759895:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677447.759896:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:4.2:1619677447.759896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:6.0:1619677447.759897:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:30.0:1619677447.759898:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:22.0:1619677447.759898:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.759901:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.759902:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.759903:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:6.0:1619677447.759904:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.759905:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.759906:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.759907:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677447.759907:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.759909:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.759910:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:24.0:1619677447.759911:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.759913:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.759914:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.759915:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.759916:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.759916:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:24.0:1619677447.759918:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.759919:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.759919:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.759920:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:6.0:1619677447.759920:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.759921:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:24.0:1619677447.759923:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.759923:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.759923:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.759924:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.759924:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.759925:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.759929:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.759929:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.759930:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.759932:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.759932:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.759933:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.759934:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:31.0:1619677447.759936:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:8.2:1619677447.759938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677447.759939:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.759940:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.759942:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.759942:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.759945:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677447.759945:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.759948:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.759949:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.759949:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.759952:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.759954:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.760063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.760071:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.760074:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.760076:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.760078:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.760078:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.760080:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677447.760081:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.760083:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.760084:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.760085:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677447.760088:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.760090:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677447.760092:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677447.760093:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.760094:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.760094:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677447.760095:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.760095:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:30.0:1619677447.760096:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.760096:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:1.0:1619677447.760097:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.760098:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.760099:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.760100:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:1.0:1619677447.760100:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.760101:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:8.2:1619677447.760102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:1.0:1619677447.760102:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.760104:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.760105:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677447.760106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:1.0:1619677447.760106:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.760109:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.760110:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.760113:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:1.0:1619677447.760113:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.760114:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:1.0:1619677447.760114:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.760115:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.760116:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:1.0:1619677447.760116:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.760117:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:20.0:1619677447.760117:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.760117:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677447.760118:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.760118:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:1.0:1619677447.760118:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:20.0:1619677447.760119:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.760119:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:5.0:1619677447.760120:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677447.760120:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677447.760122:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677447.760129:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677447.760130:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677447.760131:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677447.760132:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:13.0:1619677447.760132:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.760133:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677447.760134:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.760134:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.760135:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.760137:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.760159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.760179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.760187:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.760189:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.760190:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.760191:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.760193:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.760196:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.760204:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.760207:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.760208:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.760208:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.760210:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.760214:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.760223:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.760225:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.760226:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.760226:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.760227:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.760228:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677447.760233:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.760236:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.760238:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.760240:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.760241:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.760243:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677447.760244:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.760246:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.760247:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.760248:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677447.760250:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.760252:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677447.760254:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.760255:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677447.760255:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677447.760257:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.760257:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.760259:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.760260:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.760261:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.760274:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.760277:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.760277:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.760278:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677447.760365:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.760367:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.760368:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.760370:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.760373:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.760374:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.760376:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.760377:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.760378:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.760379:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.760383:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.760384:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.760387:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.760388:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.760389:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.760390:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.760391:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.760391:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.760392:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.760397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.760403:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.760406:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.760407:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.760408:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.760409:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.760413:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.760563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.760604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.760610:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.760613:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.760614:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.760614:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.760616:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.760619:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.760668:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.760670:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.760671:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.760673:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.760676:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.760678:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.760679:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.760680:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.760681:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.760682:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.760686:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.760687:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.760690:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.760691:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.760691:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.760693:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.760693:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.760694:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.760695:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.760706:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.760709:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.760710:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.760710:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.760712:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.760714:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.760727:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.760730:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.760731:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.760732:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.760735:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.760736:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677447.760737:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.760739:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.760740:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.760741:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677447.760745:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.760747:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677447.760749:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.760750:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677447.760751:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677447.760753:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677447.760753:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.760754:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.760754:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677447.760809:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.760811:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.760812:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.760814:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.760817:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.760818:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.760820:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.760821:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.760822:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.760823:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.760826:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.760827:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.760829:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.760830:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.760830:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.760832:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.760833:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.760833:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.760833:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:22.0:1619677447.760840:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677447.760842:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:20.2:1619677447.760843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:6.0:1619677447.760843:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677447.760845:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.760848:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.760849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:22.0:1619677447.760850:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.760851:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.760852:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:22.0:1619677447.760852:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.760853:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.760855:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:22.0:1619677447.760855:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.2:1619677447.760857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677447.760857:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677447.760859:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.760859:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.760860:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.760860:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.760861:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.760861:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:30.0:1619677447.760862:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.760862:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.760863:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677447.760864:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677447.760864:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.760865:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677447.760866:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96723851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.760866:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.760866:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677447.760866:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:30.0:1619677447.760867:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.760867:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.760868:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677447.760869:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.760869:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:5.0:1619677447.760869:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.760870:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677447.760870:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000400:00000200:30.0:1619677447.760871:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:22.0:1619677447.760871:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000400:00000200:5.0:1619677447.760872:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.760875:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677447.760875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677447.760877:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677447.760880:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.760881:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.760881:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:5.0:1619677447.760881:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677447.760883:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.760883:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96723850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.760884:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677447.760884:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677447.760884:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.760884:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:30.0:1619677447.760885:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.760885:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:17.0:1619677447.760885:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:5.0:1619677447.760885:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.760886:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.760887:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.760887:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (23)++
00000800:00000200:1.0:1619677447.760888:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (24)--
00000400:00000200:17.0:1619677447.760889:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677447.760889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677447.760889:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.760890:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677447.760891:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.760891:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:17.0:1619677447.760892:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.760892:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:5.0:1619677447.760893:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.760893:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.760894:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.760895:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96723849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.760895:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.760896:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.760897:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:17.0:1619677447.760897:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.760897:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.760898:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:17.0:1619677447.760898:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.760898:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:20.0:1619677447.760899:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.760899:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.760900:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:20.0:1619677447.760901:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.760901:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.760902:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677447.760902:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.760903:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000400:00000200:17.0:1619677447.760904:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.760904:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677447.760904:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677447.760906:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677447.760906:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.760907:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.760909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677447.760909:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.760910:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.2:1619677447.760910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:17.0:1619677447.760911:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.760913:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677447.760913:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.760913:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677447.760913:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677447.760914:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:17.0:1619677447.760914:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.760915:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.760916:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677447.760916:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.760916:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677447.760916:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:31.0:1619677447.760917:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.760917:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677447.760918:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.760919:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:31.0:1619677447.760920:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.760920:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.760920:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:6.0:1619677447.760920:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.760920:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.760921:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.760922:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.760922:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677447.760923:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:6.0:1619677447.760923:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.760924:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:12.0:1619677447.760926:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677447.760926:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.760927:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.760928:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.760929:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:6.0:1619677447.760929:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677447.760930:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.760932:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.760933:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:6.0:1619677447.760933:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.760936:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.760936:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.760939:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.760940:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.760941:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.760943:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.760943:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.760944:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.760945:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:13.0:1619677447.760946:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677447.760946:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:13.0:1619677447.760949:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.760950:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.760950:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.760951:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.760952:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.760956:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.760958:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.760959:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.760960:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.760997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677447.761012:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.761015:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.761018:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.761019:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.761020:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.761022:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677447.761024:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.761025:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.761026:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.761027:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677447.761030:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.761031:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677447.761034:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.761035:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677447.761036:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677447.761037:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.761038:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.761039:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.761039:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.761043:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.761046:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.761048:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.761048:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.761050:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:6.0:1619677447.761050:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.761051:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.761052:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.761053:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:22.0:1619677447.761054:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.761055:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.761057:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.761102:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677447.761105:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677447.761106:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677447.761108:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.761110:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.761111:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677447.761113:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.761114:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.761115:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.761116:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677447.761120:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.761121:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677447.761124:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.761125:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677447.761126:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677447.761128:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677447.761128:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677447.761129:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677447.761129:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:1.0:1619677447.761165:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.761168:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.761168:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.761171:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.761173:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.761175:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.761177:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.761178:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.761179:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.761180:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.761184:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.761186:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.761189:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.761189:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.761190:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.761192:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.761193:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.761194:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.761195:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.761195:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677447.761205:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677447.761205:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677447.761207:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:17.0:1619677447.761207:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.761208:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677447.761209:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.761209:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677447.761210:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.761211:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.761214:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.761223:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.761226:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.761227:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.761228:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.761229:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.761230:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677447.761303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.761327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:8.2:1619677447.761369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.761377:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.761380:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.761381:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.761383:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.761383:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.761386:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.761508:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.761511:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.761511:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.761514:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.761516:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.761518:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.761519:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.761521:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.761522:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.761523:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.761526:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.761528:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.761531:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.761532:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.761532:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:18.0:1619677447.761533:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:1.0:1619677447.761534:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.761535:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.761535:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677447.761536:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:1.0:1619677447.761536:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677447.761537:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.761539:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677447.761540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:18.0:1619677447.761542:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.761543:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.761544:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.761545:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.761546:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.761547:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:18.0:1619677447.761547:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.761550:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:18.0:1619677447.761550:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.761551:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.761551:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:18.0:1619677447.761552:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.761553:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.761554:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.761554:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.761555:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.761556:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:18.0:1619677447.761557:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.761558:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.761558:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.761559:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.761562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:9.2:1619677447.761566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.761568:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.761571:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.761572:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.761572:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.761573:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.761573:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:2.0:1619677447.761573:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.761575:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.761576:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.761577:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.761578:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.761579:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.761580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.2:1619677447.761588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:18.0:1619677447.761588:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.761590:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.761591:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:18.0:1619677447.761592:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.761592:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.761593:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.761593:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.761594:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.761597:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.761598:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.761600:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.761601:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677447.761602:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.761603:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:20.0:1619677447.761604:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.761605:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.761606:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.761606:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.761607:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:20.0:1619677447.761610:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.761610:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.761611:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.761613:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.761613:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.761614:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.761615:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.761616:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:2.0:1619677447.761616:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.761617:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677447.761617:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.761618:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:20.0:1619677447.761619:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.761619:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.761620:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.761621:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.761622:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:2.0:1619677447.761622:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.761623:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.2:1619677447.761625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.761626:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677447.761627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:2.0:1619677447.761627:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.761628:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.761629:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.761630:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.761631:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.761631:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677447.761632:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677447.761633:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:17.0:1619677447.761635:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:5.0:1619677447.761635:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:17.0:1619677447.761636:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.761636:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:1.0:1619677447.761636:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.761637:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:17.0:1619677447.761639:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.761642:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.761672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.761710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.761718:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.761720:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.761721:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.761722:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.761724:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.761727:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.761744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.761752:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.761755:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.761757:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.761759:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.761760:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.761761:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.761763:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.761764:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.761765:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.761766:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.761769:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.761771:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.761773:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.761774:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.761774:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.761776:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.761777:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.761778:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.761778:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.761789:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.761792:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.761793:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.761794:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.761796:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.761797:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.761836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.761842:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.761845:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.761847:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.761847:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.761849:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.761851:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677447.761852:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.761853:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.761854:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.761855:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677447.761860:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.761862:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.761865:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.761866:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677447.761867:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677447.761868:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:5.0:1619677447.761868:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.761869:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677447.761870:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:5.0:1619677447.761870:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.761870:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:30.0:1619677447.761873:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.761874:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:8.2:1619677447.761874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:30.0:1619677447.761875:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.761877:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677447.761878:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.761879:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.761880:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.761881:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.761882:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:30.0:1619677447.761884:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.761884:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:30.0:1619677447.761885:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.761885:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.761886:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.761887:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.761888:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.761889:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:30.0:1619677447.761890:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.761890:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677447.761891:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:30.0:1619677447.761892:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677447.761893:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677447.761895:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677447.761895:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677447.761897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.761903:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.761906:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.761906:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.761907:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.761909:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.761910:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:24.0:1619677447.761911:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:31.0:1619677447.761913:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677447.761914:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677447.761914:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677447.762077:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.762081:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.762082:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.762082:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.762084:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.762086:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677447.762087:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.762089:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.762090:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.762092:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.762095:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.762096:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:22.0:1619677447.762098:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:2.0:1619677447.762098:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.762099:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.762100:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.762101:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677447.762101:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.762102:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.762102:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.762103:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.762104:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:2.0:1619677447.762104:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.762106:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.762108:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.762109:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.762110:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.762111:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.762112:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.762112:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.762113:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.762117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.762125:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.762127:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.762127:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.762128:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.762129:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.762131:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677447.762150:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.762153:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.762155:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.762157:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.762158:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.762160:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.762161:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.762161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:31.0:1619677447.762162:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.762163:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.762165:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.762168:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.762170:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.762172:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.762173:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.762174:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.762176:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.762176:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.762177:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.762178:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.762198:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.762200:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.762201:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.762202:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.762202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:12.0:1619677447.762204:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.762205:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.762206:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.762209:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.762210:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.762211:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.762213:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.762216:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.762249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.762256:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.762259:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.762260:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.762261:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.762265:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.762267:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.762268:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.762270:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.762270:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.762271:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.762275:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.762276:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.762279:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.762280:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.762280:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.762282:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.762282:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.762283:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.762284:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.762288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.762301:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.762304:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.762304:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.762305:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.762307:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.762309:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.762329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.762337:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.762340:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.762342:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.762344:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.762345:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.762346:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.762347:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.762348:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.762349:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.762350:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.762353:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.762355:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.762357:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.762357:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.762358:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.762360:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.762360:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.762361:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.762361:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.762373:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.762376:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.762376:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.762378:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.762380:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.762381:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677447.762548:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677447.762550:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:1.0:1619677447.762551:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.762552:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677447.762554:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.762554:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:1.0:1619677447.762554:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.762557:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.762557:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.762559:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.762559:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.762560:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.762561:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.762562:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.762562:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.762563:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.762564:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.762564:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.762566:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.762568:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.762568:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.762572:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.762573:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.762573:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.762574:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.762575:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.762575:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.762576:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.762577:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.762577:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677447.762578:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.762578:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.762578:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677447.762579:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.762581:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.762582:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.762583:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.762583:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677447.762583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.762591:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:13.0:1619677447.762592:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677447.762593:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.762594:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.762594:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:13.0:1619677447.762595:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.762595:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.762596:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.762598:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:13.0:1619677447.762598:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.762599:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.762600:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677447.762601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.762608:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.762611:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.762612:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.762612:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.762613:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.762613:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.762637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.762640:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.762643:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.762645:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.762646:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.762647:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.762649:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.762650:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.762651:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.762652:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.762654:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.762657:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.762659:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.762662:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.762663:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.762664:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.762666:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.762666:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.762667:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.762667:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.762678:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.762681:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.762681:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.762683:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.762684:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.762686:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.762687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677447.762732:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:9.2:1619677447.762732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677447.762734:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.762735:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.762737:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.762738:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.762739:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:2.0:1619677447.762740:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.762743:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.762745:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.762745:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.762747:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.762750:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.763035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.763043:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.763046:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.763048:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.763050:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.763050:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.763052:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677447.763053:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.763054:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.763055:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.763056:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677447.763060:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.763061:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.763063:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.763064:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.763065:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.763066:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.763066:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.763068:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.763070:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.763070:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677447.763076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.763080:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.763082:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.763082:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.763084:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.763084:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.763084:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.763085:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.763086:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.763087:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677447.763088:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:22.0:1619677447.763142:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.763144:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.763145:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.763147:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.763150:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.763151:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.763153:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.763154:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.763155:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.763156:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.763159:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.763161:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.763164:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.763165:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.763165:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.763167:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.763168:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.763168:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.763169:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.763173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.763181:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.763183:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.763184:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.763185:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.763187:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.763189:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:6.0:1619677447.763289:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.763291:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.763292:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.763301:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.763303:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.763306:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe9286a00] (21)++
00000800:00000200:6.0:1619677447.763306:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.763309:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.60@o2ib3
00000800:00000200:24.0:1619677447.763310:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe9286a00] (22)--
00000400:00000200:8.0:1619677447.763312:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.60@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.763314:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.763315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:8.0:1619677447.763316:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:8.0:1619677447.763317:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.763319:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.763320:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.763321:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.763321:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.763324:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:8.0:1619677447.763325:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.60@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.763326:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.0:1619677447.763328:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 984 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:5.0:1619677447.763329:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.763331:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.763332:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:8.0:1619677447.763332:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:5.0:1619677447.763332:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.763334:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.763334:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:8.0:1619677447.763335:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe9286a00] (21)++
00000800:00000200:8.0:1619677447.763335:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe9286a00] (22)--
00000400:00000200:5.0:1619677447.763335:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.763336:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe9286a00] (21)--
00000400:00000200:5.0:1619677447.763336:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.763337:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.763338:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:8.2:1619677447.763340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000400:00000200:5.0:1619677447.763341:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.763343:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:5.0:1619677447.763347:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.763348:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.763348:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677447.763348:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677447.763349:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.763350:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.763350:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677447.763351:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:20.0:1619677447.763352:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.763352:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.763352:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:5.0:1619677447.763354:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.763355:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:9.2:1619677447.763357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.2:1619677447.763361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000800:00000200:24.0:1619677447.763365:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.763368:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.763368:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.763369:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.763369:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000400:00000200:24.0:1619677447.763371:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.763372:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.763373:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:24.0:1619677447.763374:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.0:1619677447.763374:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677447.763375:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.763376:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.2:1619677447.763400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.763405:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.763407:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.763409:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.763410:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.763412:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.763414:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.763415:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.763416:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.763417:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.763418:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.763421:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.763422:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.763425:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.763426:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.763426:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.763428:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.763429:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.763429:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.763430:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.763442:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.763445:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.763446:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.763447:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.763449:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.763450:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.763461:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.763464:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.763465:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.763465:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.763466:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.763467:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.763478:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.763480:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.763482:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.763483:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.763486:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.763487:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677447.763489:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.763490:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.763491:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.763492:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677447.763495:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.763497:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.763500:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.763501:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677447.763502:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677447.763504:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.763504:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.763505:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.763505:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677447.763516:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.763519:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.763520:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.763522:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.763522:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.763525:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677447.763626:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.763629:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.763630:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.763632:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.763634:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.763635:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.763637:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.763638:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.763639:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.763640:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.763643:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.763645:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.763648:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.763649:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.763649:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.763651:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.763652:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.763652:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.763653:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.763664:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.763666:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.763667:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.763669:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.763670:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.763672:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.763684:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.763687:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.763688:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.763688:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:22.0:1619677447.763689:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:13.0:1619677447.763689:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.763690:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.763692:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.763693:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.763693:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.763695:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.763698:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:30.0:1619677447.763971:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.763973:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.763975:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.763977:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.763978:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.763979:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677447.763980:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.763982:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.763983:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.763984:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677447.763987:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.763989:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677447.763991:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.763991:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677447.763992:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677447.763993:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.763994:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.763995:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.763995:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677447.764263:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.764266:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.764267:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.764268:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.764271:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.764273:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.764274:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.764275:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.764276:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.764277:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.764279:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.764281:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.764284:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.764285:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.764285:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.764286:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.0:1619677447.764287:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.764287:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.764288:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:18.0:1619677447.764289:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:8.0:1619677447.764289:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:2.0:1619677447.764290:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.764292:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:8.2:1619677447.764294:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:18.0:1619677447.764300:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.764302:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:20.0:1619677447.764303:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:18.0:1619677447.764303:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.764304:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.764305:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:18.0:1619677447.764305:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.764306:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.764306:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.764308:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.764309:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:18.0:1619677447.764310:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.764312:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677447.764312:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.764314:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.764315:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.764316:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.764318:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.764318:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.764320:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.764321:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.764321:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:9.2:1619677447.764324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.764333:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.764335:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.764335:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.764335:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.764336:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.764336:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.764337:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.764338:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:2.0:1619677447.764338:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.764340:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.764346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.764354:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.764356:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.764357:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.764357:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.764358:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.764358:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.764369:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.764372:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.764372:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.764374:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.764376:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.764378:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.764379:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.764381:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.764382:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.764383:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.764386:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.764388:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.764391:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.764392:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.764392:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.764394:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.764394:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.764395:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.764396:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.764400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.764409:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.764411:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.764412:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.764414:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.764415:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.764416:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677447.764438:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.764441:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.764442:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.764444:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.764447:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.764448:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.764449:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.764451:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.764452:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.764453:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:4.2:1619677447.764455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:1.0:1619677447.764456:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.764458:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.764461:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.764462:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.764462:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.764462:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.764464:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.764465:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:1.0:1619677447.764465:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677447.764466:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677447.764466:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.764467:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000400:00000200:24.0:1619677447.764468:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.764468:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677447.764470:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.764471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:24.0:1619677447.764472:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.764473:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.764474:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.764475:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.764476:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:12.0:1619677447.764478:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677447.764479:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:24.0:1619677447.764480:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.764480:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:17.0:1619677447.764481:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:12.0:1619677447.764481:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.764482:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.764482:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.764482:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677447.764483:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.764485:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.764485:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.764486:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.764486:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677447.764486:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.764488:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.764488:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.764489:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.764490:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.764494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.764502:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.764505:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.764506:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.764506:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.764508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677447.764508:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.2:1619677447.764511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:1.0:1619677447.764511:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.764515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.764517:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.764519:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.764521:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.764521:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.764523:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677447.764523:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:17.0:1619677447.764526:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.764526:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.764526:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.764527:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.764528:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.764529:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.764543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.764553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.764561:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.764564:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.764566:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.764568:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.764569:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.764570:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.764572:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.764573:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.764574:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.764575:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.764578:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.764580:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.764583:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.764584:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.764585:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.764586:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.764587:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.764588:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:22.0:1619677447.764588:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:31.0:1619677447.764589:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:22.0:1619677447.764592:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:8.2:1619677447.764592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.764592:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.764593:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.764595:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.764598:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.764600:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.764602:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.764603:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.764605:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.764606:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.764606:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.764608:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:6.0:1619677447.764609:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.764610:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.764610:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.2:1619677447.764611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.764611:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.764612:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.764618:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.764621:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.764622:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.764623:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.764624:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.764625:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677447.764864:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.764867:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.764868:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.764870:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.764872:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.764874:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.764875:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.764876:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.764877:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.764878:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.764882:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.764883:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.764887:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.764887:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.764888:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.764890:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.764890:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.764891:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.764891:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.764896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.764906:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.764908:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.764909:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.764909:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.764911:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:18.0:1619677447.764912:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.764914:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.764914:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.764916:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:18.0:1619677447.764916:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.764916:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.764918:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.764920:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.764921:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.764922:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.764923:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.764926:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.764928:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.764931:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.764932:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.764933:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.764935:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.764935:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.764936:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.764937:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.764938:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.764948:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.764950:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.764951:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.764952:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677447.764958:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.764961:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.764962:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.764963:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.764965:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.764966:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.765042:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.765045:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.765046:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.765048:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.765051:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.765052:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.765053:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.765055:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.765056:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.765057:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.765060:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.765062:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.765064:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.765065:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.765066:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.765067:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.765068:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.765068:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.765069:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.765070:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.765073:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.765075:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.765077:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.765078:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:30.0:1619677447.765078:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.765080:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.765080:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677447.765081:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.765081:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.765082:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.765083:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.765084:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.765085:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677447.765088:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.765088:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:30.0:1619677447.765088:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.765090:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677447.765094:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.765094:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677447.765095:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677447.765097:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.765097:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.765098:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.765100:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.765101:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677447.765110:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:31.0:1619677447.765113:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:6.0:1619677447.765113:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.765114:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.765114:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.765116:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.765117:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.765117:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:6.0:1619677447.765117:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.765120:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.765235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.765243:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.765245:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.765247:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.765247:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.765249:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.765253:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.765285:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.765288:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.765289:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.765291:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.765300:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.765301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:22.0:1619677447.765302:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.765303:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.765304:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.765305:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.765306:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:8.0:1619677447.765308:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:22.0:1619677447.765310:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.765311:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.765312:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677447.765312:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.765314:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.765315:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:8.0:1619677447.765315:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.765316:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.765318:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.765318:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:8.0:1619677447.765318:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.765319:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:8.0:1619677447.765319:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:22.0:1619677447.765320:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.765321:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:8.0:1619677447.765321:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.765322:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.765323:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.765324:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:9.2:1619677447.765326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:8.0:1619677447.765327:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.765329:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.765331:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.765332:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.765332:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:8.0:1619677447.765333:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.765334:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:8.0:1619677447.765334:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.765334:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.765334:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677447.765335:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677447.765335:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677447.765336:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:8.0:1619677447.765336:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:18.0:1619677447.765337:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.765338:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.765338:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677447.765338:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.765339:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.2:1619677447.765340:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.765342:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.765344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.765348:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.765350:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677447.765351:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.765352:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.765352:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.765353:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.765353:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677447.765354:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.765354:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.765358:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.765385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.765391:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.765394:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.765396:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.765396:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.765399:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.765400:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.765402:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.765403:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.765404:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.765405:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.765410:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.765412:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.765414:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677447.765415:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.765416:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.765416:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:5.0:1619677447.765417:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.765418:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677447.765419:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.765419:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:2.0:1619677447.765419:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.765421:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.765422:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000400:00000200:2.0:1619677447.765422:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.765423:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:2.0:1619677447.765424:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.765425:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.765426:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677447.765427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:2.0:1619677447.765427:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.765428:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.765430:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:2.0:1619677447.765431:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.765432:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:12.0:1619677447.765432:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677447.765433:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677447.765433:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.765434:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.765435:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677447.765435:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677447.765436:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.765436:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.765437:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:12.0:1619677447.765438:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.765438:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.765439:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.765439:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:13.0:1619677447.765440:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.765440:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677447.765441:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.765444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.2:1619677447.765445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.765453:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.765455:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.765456:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.765458:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.765461:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.765463:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.765464:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.765466:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.765467:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.765470:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.765473:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.765474:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.765476:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.765477:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.765478:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.765479:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.765483:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.765485:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.765488:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.765489:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.765489:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.2:1619677447.765490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677447.765491:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.765491:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.765492:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.765493:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677447.765498:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.765501:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.765502:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.765503:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.765503:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.765504:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677447.765506:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677447.765507:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.765508:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.765508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000400:00000200:31.0:1619677447.765510:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.765512:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.765513:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.0:1619677447.765515:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.765518:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.765519:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.765521:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.2:1619677447.765522:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.0:1619677447.765525:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.765525:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:30.0:1619677447.765529:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.765532:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.765534:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.765536:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.765537:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.765539:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.765540:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.765541:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.765542:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.765543:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.765547:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.765549:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.765552:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.765553:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.765553:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.765555:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.765556:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.765556:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.765557:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.765563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.765571:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.765573:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.765574:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.765576:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.765576:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.765579:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.765633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.765637:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.765640:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.765642:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.765643:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.765645:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.765646:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:20.0:1619677447.765648:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.765649:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.765650:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.765651:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:20.0:1619677447.765654:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.765656:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:20.0:1619677447.765659:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.765660:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:20.0:1619677447.765660:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:20.0:1619677447.765662:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:20.0:1619677447.765663:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.765663:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.765664:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677447.765668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.765674:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.765677:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.765677:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.765678:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.765680:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:24.0:1619677447.765683:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.765687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:8.0:1619677447.765693:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.765696:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.765696:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677447.765697:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.765698:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.765698:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:17.0:1619677447.765814:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.765816:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.765817:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.765819:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.765822:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.765824:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677447.765825:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.765826:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.765827:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.765828:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677447.765832:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.765833:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677447.765833:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677447.765835:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677447.765837:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.765838:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.765838:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677447.765838:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:30.0:1619677447.765839:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677447.765840:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.765840:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677447.765841:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.765841:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.765842:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677447.765843:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.765844:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.765845:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.765846:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677447.765846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:31.0:1619677447.765847:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.765851:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.765852:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.765852:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:4.2:1619677447.765852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.765854:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:31.0:1619677447.765855:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.765855:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.765855:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677447.765856:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.765856:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677447.765857:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.765858:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.765859:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.765860:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.765860:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677447.765860:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677447.765881:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.765883:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.765884:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.765885:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.765887:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.765888:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677447.765889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677447.765893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.765894:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.765897:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.765898:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.0:1619677447.765898:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:22.0:1619677447.765899:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.765900:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.765901:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.765902:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.765903:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.765903:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.765904:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.765905:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.765906:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.765906:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:22.0:1619677447.765908:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.765909:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.765912:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.765914:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.765917:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.765918:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.765919:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.765920:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.765921:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.765921:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.765922:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677447.765933:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.765936:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.765937:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.765939:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.765940:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.765942:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.765962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:6.0:1619677447.765967:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.765970:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677447.765970:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000400:00000200:6.0:1619677447.765972:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.765975:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.765976:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.765978:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.765979:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.765980:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.765980:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.765983:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.765985:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.765987:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.765987:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.765988:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.765989:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.765990:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677447.765991:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677447.765991:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677447.765995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.766003:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.766005:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.766007:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.766007:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.766009:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.766011:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.766018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.766025:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.766027:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.766028:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.766028:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.766029:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.766030:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:1.0:1619677447.766037:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.766040:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.766041:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.766043:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.766046:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.766047:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.766048:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.766050:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.766051:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.766052:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.766055:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.766057:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.766060:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.766061:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.766062:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.766063:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.766064:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.766065:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.766066:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.766067:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677447.766069:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:9.2:1619677447.766069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:18.0:1619677447.766072:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.766073:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.766074:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.766076:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.766076:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677447.766077:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:18.0:1619677447.766078:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:8.0:1619677447.766078:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.766079:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:18.0:1619677447.766079:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.766079:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.766080:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677447.766080:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.766081:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.766081:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:18.0:1619677447.766082:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.766083:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:8.0:1619677447.766083:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.766085:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.766086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:18.0:1619677447.766087:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.766089:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677447.766091:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.766092:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:24.0:1619677447.766093:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677447.766093:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677447.766094:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:24.0:1619677447.766095:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:18.0:1619677447.766095:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.766096:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.766096:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677447.766096:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.766098:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.766100:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.766102:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.766103:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723783] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.766105:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.766106:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.766107:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.766108:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:24.0:1619677447.766110:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.766110:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.766111:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.766111:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:24.0:1619677447.766112:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.766114:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:24.0:1619677447.766115:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.766116:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.766116:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.766117:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.766118:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.766119:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.766119:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.766120:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.766125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.766128:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.766131:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.766131:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.766132:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677447.766133:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677447.766133:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.766133:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:17.0:1619677447.766136:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.766137:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.766137:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.766140:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.766143:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.766152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.766159:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.766162:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.766164:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.766166:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.766167:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.766168:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.766170:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.766171:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.766172:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.766173:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.766176:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.766178:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.766181:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.766182:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.766182:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.766184:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.766185:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.766186:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.766186:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.766190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.766198:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.766200:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.766202:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.766203:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.766204:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.766206:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.766241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:4.2:1619677447.766246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.766249:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.766251:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.766252:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.766255:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.766257:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.766260:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.766262:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.766263:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.766265:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.766266:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.766270:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.766273:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.766276:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.766277:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.766277:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.766279:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.766279:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.766280:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.766281:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.766292:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.2:1619677447.766292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.2:1619677447.766299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:6.0:1619677447.766299:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.766301:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.766302:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.766304:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.766306:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677447.766308:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677447.766309:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677447.766310:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.766311:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.766311:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.766312:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.766313:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.766313:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:20.0:1619677447.766314:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.766314:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:5.0:1619677447.766314:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.766316:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.766319:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:2.0:1619677447.766319:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.766320:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:2.0:1619677447.766320:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:20.0:1619677447.766321:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.766322:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:2.0:1619677447.766322:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.766323:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.766324:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.766325:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.766328:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.766330:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.766332:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.766333:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.766334:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.766335:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.766335:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.766336:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.766337:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.766342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.766350:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.766352:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.766352:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.766353:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.766354:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.766356:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677447.766456:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.766459:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.766462:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.766464:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.766464:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.766465:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.766467:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723777] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.766468:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.766469:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.766470:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.766473:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.766475:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.766477:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.766478:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.766479:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.766480:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.766481:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.766482:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.766483:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677447.766485:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.766488:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.766488:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.766490:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.766493:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.766493:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:17.0:1619677447.766494:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.766496:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.766496:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:17.0:1619677447.766497:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677447.766497:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.766498:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.766500:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:12.0:1619677447.766500:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.766501:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:17.0:1619677447.766503:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.766505:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.766507:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.766508:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.766509:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.766510:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677447.766512:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677447.766512:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.766513:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.766513:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.766514:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.766517:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.766518:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.766519:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677447.766519:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.766521:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.766524:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.766525:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.766526:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.766528:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.766529:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.766570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.766600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.766604:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.766606:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.766608:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.766609:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.766611:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.766612:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.766614:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.766615:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.766616:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.766617:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677447.766618:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:20.0:1619677447.766620:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.766621:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.766621:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677447.766622:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:6.0:1619677447.766622:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.766624:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.766625:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.766626:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.766627:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677447.766627:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.766629:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.766630:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.766630:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.766631:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.766641:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.766644:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.766645:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.766646:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.766648:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.766649:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.766709:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.766711:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.766712:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.766714:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.766715:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.766716:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.766729:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.766731:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.766732:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.766733:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677447.766733:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.766735:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677447.766756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.766763:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.766766:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.766768:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.766770:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.766770:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.766771:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.766772:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.766773:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.766774:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.766775:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.766776:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:31.0:1619677447.766778:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.766779:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:31.0:1619677447.766780:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.766780:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.766782:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.766784:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.766784:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:2.0:1619677447.766784:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.766785:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:2.0:1619677447.766785:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:31.0:1619677447.766787:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.766787:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:2.0:1619677447.766787:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.766788:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:2.0:1619677447.766788:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.766789:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.766790:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000400:00000200:2.0:1619677447.766790:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.766791:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677447.766793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:2.0:1619677447.766793:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.766795:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.2:1619677447.766796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677447.766798:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.766799:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677447.766800:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (23)++
00000800:00000200:5.0:1619677447.766801:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (24)++
00000800:00000200:2.0:1619677447.766802:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:2.0:1619677447.766802:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677447.766803:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (24)--
00000800:00000200:2.0:1619677447.766803:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:30.0:1619677447.766804:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:5.0:1619677447.766804:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.766804:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:30.0:1619677447.766806:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:20.0:1619677447.766806:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)++
00000400:00000200:5.0:1619677447.766806:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.766807:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677447.766807:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (24)--
00000800:00000200:30.0:1619677447.766808:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677447.766808:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.766809:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.766810:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.766811:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.766813:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677447.766815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677447.766822:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.766824:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.766825:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.766826:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.766827:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.766827:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:8.0:1619677447.766898:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.766901:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.766902:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.766904:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.766907:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.766908:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.766910:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.766911:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.766912:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.766913:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.766916:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.766918:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.766921:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.766921:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.766922:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.766924:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.766925:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.766926:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.766926:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.766931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.766938:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.766941:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.766942:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.766942:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.766944:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.766947:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.766949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.766956:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.766959:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.766960:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.766960:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.766961:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.766961:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.767166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677447.767215:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.767217:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.767218:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.767220:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.767222:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.767222:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677447.767237:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677447.767240:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.767241:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.767242:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.767242:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.767243:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:7.0:1619677447.767305:0:520:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:18.0:1619677447.767306:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:7.0:1619677447.767307:0:520:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:18.0:1619677447.767309:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.767310:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.767311:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.767314:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.767316:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.767317:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.767318:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.767319:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.767320:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.767323:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.767325:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.767327:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.767328:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.767329:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.767330:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.767331:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.767332:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.767332:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.767337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.767345:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.767347:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.767348:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.767348:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.767350:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.767353:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.767375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.767383:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.767386:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.767388:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.767390:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.767391:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.767393:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.767394:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.767395:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.767396:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.767397:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.767401:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.767402:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.767405:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.767406:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.767406:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.767408:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.767409:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.767410:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.767410:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677447.767411:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.767413:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677447.767414:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.767415:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.767417:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.767418:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677447.767419:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.767421:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.767421:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.767422:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:13.0:1619677447.767422:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.767424:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.767425:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.767426:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.767427:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:13.0:1619677447.767427:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.767429:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.767430:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677447.767431:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.767432:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677447.767433:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677447.767433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677447.767434:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.767435:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.767435:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.767436:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677447.767440:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.767443:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.767444:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.767445:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.767448:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.767449:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:24.0:1619677447.767450:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.767451:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.767451:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:24.0:1619677447.767453:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.767453:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.767453:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:24.0:1619677447.767454:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.767455:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.767455:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.767458:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:24.0:1619677447.767459:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.767461:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.767463:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.767464:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.767465:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.767466:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.767467:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.767468:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.767468:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.767472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.767480:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.767483:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.767484:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.767484:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.767486:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.767489:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.767495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.767502:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.767505:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.767505:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.767506:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.767507:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.767508:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.767617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.767623:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.767625:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.767627:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.767628:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.767631:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.767632:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.767634:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.767635:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.767636:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.767637:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.767640:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.767642:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.767645:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.767646:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.767646:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.767648:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.767649:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.767649:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.767650:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.767662:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.767665:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.767665:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.767666:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.767668:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.767670:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.767774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.767815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677447.767818:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.767820:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.767821:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.767822:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.767825:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.767826:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.0:1619677447.767833:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.767835:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.767836:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.767838:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.767841:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.767842:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.767843:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.767844:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.767845:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.767846:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.767849:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.767850:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.767853:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.767854:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.767855:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.767857:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.767857:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.767858:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.767860:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.767860:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677447.767863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.767870:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677447.767873:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:2.0:1619677447.767873:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:24.0:1619677447.767874:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:18.0:1619677447.767874:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.767874:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.767875:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.767876:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:2.0:1619677447.767877:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.767879:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.768078:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.768081:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.768082:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.768084:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.768086:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.768088:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.768089:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.768090:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.768091:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.768092:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.768096:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.768097:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.768100:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.768101:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.768101:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.768103:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.768103:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.768104:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.768105:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.768117:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.768119:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.768120:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.768120:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.768123:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.768125:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.768421:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.768423:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.768424:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.768425:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.768427:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.768430:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.768443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.768482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677447.768489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677447.768490:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677447.768492:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.768493:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:20.0:1619677447.768494:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:18.0:1619677447.768494:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.768494:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.768497:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.768498:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.768498:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.768500:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.768501:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:20.0:1619677447.768502:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677447.768503:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.768504:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.768506:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.768506:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677447.768510:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.768512:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677447.768514:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.768515:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.768515:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.768517:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.768518:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.768518:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.768519:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677447.768524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.768533:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.768535:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.768536:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.768536:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.768539:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.768541:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.768833:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.768836:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.768836:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.768838:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.768840:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.768841:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.768842:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.768843:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.768844:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.768845:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.768848:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.768850:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.768853:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.768853:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.768854:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.768856:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.768856:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.768858:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.768859:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.768859:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.768869:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.768871:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.768872:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.768873:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.768880:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.768883:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.768884:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.768886:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.768887:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.768889:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.768953:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.768955:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.768956:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.768958:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.768961:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.768963:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.768965:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.768966:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.768967:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.768968:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.768973:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.768975:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.768978:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.768979:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.768980:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.768982:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.768982:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.768983:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.768983:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.768988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:9.2:1619677447.768994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677447.768995:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.768998:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.768999:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.768999:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.769001:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.769002:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:17.0:1619677447.769004:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.769004:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.769006:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.769006:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.769008:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.769011:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.769028:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.769030:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.769031:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.769033:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.769035:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.769037:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:20.2:1619677447.769038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:18.0:1619677447.769038:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.769040:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.769041:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.769042:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.769045:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.769047:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.769050:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.769051:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.769052:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.769053:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.769054:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.769055:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.769056:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.769057:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.769068:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.769070:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.769071:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677447.769071:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677447.769079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.769087:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.769090:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.769091:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.769091:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.769093:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.769096:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.769101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.769108:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.769110:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.769111:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.769111:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.769112:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.769113:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.769281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.769286:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.769289:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.769291:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.769291:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.769300:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.769301:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677447.769303:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.769304:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.769305:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.769306:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677447.769309:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.769311:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.769314:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.769315:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677447.769315:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677447.769317:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677447.769318:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.769318:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.769319:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677447.769348:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.769351:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.769353:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.769355:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.769355:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.769357:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677447.769358:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.769359:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.769360:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.769361:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677447.769365:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677447.769365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.769366:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677447.769369:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.769370:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677447.769370:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677447.769372:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.769372:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.769373:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.769374:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.769374:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677447.769375:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.769376:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.769376:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.769377:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.769378:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.769380:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.769381:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.769382:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.769382:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677447.769383:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.769384:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.769385:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.769385:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677447.769389:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677447.769389:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.769389:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.769390:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.769390:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.769391:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677447.769392:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.769392:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.769393:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:30.0:1619677447.769395:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677447.769395:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.769396:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.769397:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:1.0:1619677447.769397:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.769397:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.769398:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:1.0:1619677447.769398:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:30.0:1619677447.769399:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:4.2:1619677447.769399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.769399:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677447.769400:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.769400:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:2.0:1619677447.769402:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:1.0:1619677447.769402:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.769403:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677447.769404:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:2.0:1619677447.769405:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:1.0:1619677447.769406:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.769406:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:2.0:1619677447.769407:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.769407:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:2.0:1619677447.769409:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:1.0:1619677447.769409:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.769409:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677447.769410:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96723748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.769410:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.769411:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:2.0:1619677447.769412:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.769413:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677447.769415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:2.0:1619677447.769415:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.769418:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.769420:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:13.0:1619677447.769422:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677447.769423:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:2.0:1619677447.769423:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.769423:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:13.0:1619677447.769424:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:2.0:1619677447.769424:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677447.769425:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.769426:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.769426:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.769426:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.769426:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677447.769426:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:2.0:1619677447.769427:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.769428:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:13.0:1619677447.769429:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.769429:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:17.0:1619677447.769430:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.769432:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:13.0:1619677447.769432:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.769440:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:22.0:1619677447.769443:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (24)++
00000800:00000200:22.0:1619677447.769444:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:6.0:1619677447.769444:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000400:00000200:22.0:1619677447.769445:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.769445:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:6.0:1619677447.769445:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.769447:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:22.0:1619677447.769448:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.769448:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.769449:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.769450:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.769451:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677447.769451:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.769452:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:13.0:1619677447.769453:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.769453:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.769455:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.769455:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.769456:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.769457:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.769457:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.769563:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.769566:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.769567:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.769569:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.769571:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.769572:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.769574:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.769575:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.769576:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.769577:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.769580:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.769581:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.769583:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.769584:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.769585:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677447.769585:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:22.0:1619677447.769587:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.769587:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.769588:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.769588:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.769588:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.769589:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.769591:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.769593:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.769593:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.2:1619677447.769593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.769601:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.769603:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.769604:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.769606:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.769607:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.769608:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.769965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.769973:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:6.0:1619677447.769975:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677447.769976:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:6.0:1619677447.769978:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:31.0:1619677447.769979:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.769979:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677447.769980:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.769982:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.769982:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.769983:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:6.0:1619677447.769984:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.769985:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.769985:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677447.769987:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.769987:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.769988:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677447.769989:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.769989:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.769991:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677447.769991:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677447.769991:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.769991:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:6.0:1619677447.769992:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:1.0:1619677447.769996:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.769997:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.769998:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.769998:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.769999:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:6.0:1619677447.770000:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:1.0:1619677447.770001:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96723742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.770002:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:1.0:1619677447.770002:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.770003:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.770004:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:1.0:1619677447.770004:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.770005:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:1.0:1619677447.770005:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.770006:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.770006:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677447.770007:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677447.770007:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:6.0:1619677447.770007:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.770008:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:31.0:1619677447.770009:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:6.0:1619677447.770009:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000400:00000200:1.0:1619677447.770009:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.770010:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:6.0:1619677447.770010:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:31.0:1619677447.770011:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:1.0:1619677447.770011:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.770012:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:9.2:1619677447.770013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:1.0:1619677447.770014:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677447.770015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677447.770015:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.770015:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.770017:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.770017:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.770018:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.770019:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.770020:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677447.770020:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677447.770023:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:18.0:1619677447.770023:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:5.0:1619677447.770023:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:22.0:1619677447.770024:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:18.0:1619677447.770024:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.770024:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (23)--
00000800:00000200:22.0:1619677447.770025:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.770026:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.770026:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.770027:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.770029:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.770029:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:5.0:1619677447.770029:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.770030:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677447.770032:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:30.0:1619677447.770033:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.770033:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.770035:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.770036:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.770038:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.0:1619677447.770143:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.770146:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.770147:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.770149:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.770152:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.770152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:8.0:1619677447.770154:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677447.770155:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.770156:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.770158:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.770159:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677447.770162:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.770165:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.770167:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.770168:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.770169:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.770170:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.770171:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.770171:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.770172:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.770183:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.770186:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.770186:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.770188:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.770189:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.770190:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.770198:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.770200:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.770201:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.770202:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.770204:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.770207:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.770223:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.770226:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.770226:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.770229:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.770231:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.770232:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.770233:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.770234:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.770235:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.770236:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.770239:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.770241:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.770243:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.770244:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.770245:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.770246:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.770247:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.770247:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.770248:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.770252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.770260:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.770262:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.770263:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.770264:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.770266:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.770268:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677447.770342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.770345:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.770348:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.770350:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.770351:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.770353:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.770355:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.770356:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.770358:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.770359:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.770360:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.770364:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.770367:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.770370:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.770371:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.770372:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.770374:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.770374:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.770376:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.770377:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.770377:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.770382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:9.2:1619677447.770387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.770388:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677447.770389:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677447.770390:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:31.0:1619677447.770391:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:5.0:1619677447.770391:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677447.770392:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.770392:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:31.0:1619677447.770394:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.770395:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:30.0:1619677447.770396:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.770397:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677447.770398:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.770399:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.770399:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.770402:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.770405:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.770432:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.770435:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.770436:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.770438:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.770441:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.770442:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.770444:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.770445:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.770446:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.770447:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.770451:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.770452:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.770456:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.770457:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.770457:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.770459:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.770460:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.770460:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.770461:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.770473:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.770475:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.770476:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.770478:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.770479:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.770480:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677447.770489:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677447.770492:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.770493:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.770493:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.770494:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.770495:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.770501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.770508:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.770511:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.770513:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.770515:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.770515:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.770517:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677447.770518:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.770520:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.770520:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.770522:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677447.770525:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.770526:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677447.770529:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.770530:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.770531:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677447.770533:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.770533:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.770535:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.770536:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.770537:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677447.770539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677447.770542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677447.770547:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.770549:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:31.0:1619677447.770550:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:5.0:1619677447.770550:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.770552:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:20.0:1619677447.770552:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.770552:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.770553:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.770554:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677447.770555:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677447.770560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.770567:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.770570:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.770570:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.770571:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.770572:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.770573:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.770626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.770675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:9.2:1619677447.770676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.770679:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.770682:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.770682:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.770683:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.770684:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:6.0:1619677447.770685:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.770687:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.770688:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.770688:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:2.0:1619677447.770688:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.770691:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.770694:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:22.0:1619677447.770767:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.770770:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.770771:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.770773:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.770776:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.770777:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.770779:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.770781:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.770782:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.770783:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.770787:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.770788:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.770792:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.770792:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.770793:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.770795:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.770795:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.770796:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.770797:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677447.770811:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.770814:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.770815:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.770817:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.770817:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.770820:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677447.770856:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.770859:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.770860:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.770862:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.770864:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.770865:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.770867:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.770868:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.770869:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.770870:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.770873:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.770875:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.770877:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.770878:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.770879:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.770881:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.770881:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.770882:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.770882:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677447.770894:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.770896:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.770897:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.770897:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.770900:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.770902:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.770919:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.770922:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.770923:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.770923:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.770924:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.770925:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677447.771012:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.771015:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.771017:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.771019:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.771021:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.771022:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677447.771023:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.771025:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.771026:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.771027:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677447.771030:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.771032:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677447.771035:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.771036:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677447.771036:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677447.771038:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.771039:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.771040:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.771040:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.771051:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.771054:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.771055:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.771055:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.771057:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.771060:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.771076:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.771079:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.771079:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.771080:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.771081:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.771081:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.771081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.771089:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.771092:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.771092:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.771094:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677447.771094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:24.0:1619677447.771097:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.771098:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.771099:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.771100:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.771100:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:24.0:1619677447.771102:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.771102:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:24.0:1619677447.771103:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.771105:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.771106:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677447.771107:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.771108:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.771110:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.771110:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677447.771111:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.771113:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677447.771113:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.771114:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:5.0:1619677447.771114:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.771115:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677447.771115:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.771116:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.771117:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.771118:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.771118:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:5.0:1619677447.771118:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.771120:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.2:1619677447.771122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.771123:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.771124:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677447.771124:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (23)++
00000800:00000200:5.0:1619677447.771126:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (24)--
00000800:00000200:5.0:1619677447.771127:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.771128:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.771129:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.771130:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:1.0:1619677447.771130:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000800:00000200:17.0:1619677447.771133:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000800:00000200:1.0:1619677447.771133:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000800:00000200:20.2:1619677447.771134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677447.771134:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000400:00000200:1.0:1619677447.771134:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.771135:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.771136:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.771137:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000400:00000200:17.0:1619677447.771137:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:20.0:1619677447.771139:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:1.0:1619677447.771139:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677447.771140:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:17.0:1619677447.771140:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.771141:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677447.771142:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:2.0:1619677447.771148:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.771151:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.771152:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.771154:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.771157:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.771159:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677447.771160:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.771161:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.771162:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.771163:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677447.771167:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.771169:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.771172:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.771173:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677447.771173:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677447.771175:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.771176:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.771176:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.771177:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677447.771189:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.771192:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.771193:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.771195:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.771196:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.771198:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.771242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.771288:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.771292:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.771292:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:17.0:1619677447.771299:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:6.0:1619677447.771300:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.771302:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:6.0:1619677447.771303:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.771303:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.771305:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.771306:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:17.0:1619677447.771308:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.771309:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.771310:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.771312:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.771313:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.771314:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.771318:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.771319:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.771322:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.771323:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.771324:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.771325:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.771326:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.771327:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.771328:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.771329:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677447.771340:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:1.0:1619677447.771340:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.771342:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.771342:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677447.771343:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:1.0:1619677447.771343:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677447.771344:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.771346:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.771348:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.771349:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677447.771359:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.771362:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.771363:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:22.0:1619677447.771365:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000400:00000200:18.0:1619677447.771365:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.771368:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000400:00000200:18.0:1619677447.771368:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.771368:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:18.0:1619677447.771370:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.771371:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.771371:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.771373:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.771374:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.771374:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.771375:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677447.771375:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.771377:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.771379:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.771379:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.771380:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.771381:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:18.0:1619677447.771381:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.771384:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677447.771385:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.771385:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.771386:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.771387:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677447.771387:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.771388:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.771389:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.771390:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:22.0:1619677447.771391:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.771391:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:22.0:1619677447.771392:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:22.0:1619677447.771392:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:8.2:1619677447.771394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.771395:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:22.0:1619677447.771395:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677447.771396:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677447.771398:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677447.771398:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:20.0:1619677447.771401:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677447.771402:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:20.0:1619677447.771404:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:2.0:1619677447.771404:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:20.0:1619677447.771405:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.771405:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.771406:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:30.0:1619677447.771407:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:20.0:1619677447.771407:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.771408:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.771409:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677447.771410:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:6.0:1619677447.771410:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000400:00000200:30.0:1619677447.771411:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.771412:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000400:00000200:30.0:1619677447.771413:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.771413:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:31.0:1619677447.771414:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:6.0:1619677447.771414:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:30.0:1619677447.771416:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677447.771462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.771470:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.771473:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.771475:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.771477:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.771478:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.771479:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.771481:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723722] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.771482:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.771483:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.771484:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.771487:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.771489:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.771492:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.771493:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.771493:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.771495:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.771496:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.771497:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.771497:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677447.771508:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.771510:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.771511:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.771513:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.771515:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.771516:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.771537:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.771540:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.771540:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.771541:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.771541:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.771542:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677447.771758:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.771761:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.771762:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.771764:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.771767:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.771769:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677447.771770:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.771771:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.771772:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.771773:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677447.771777:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.771779:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677447.771782:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.771783:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677447.771783:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677447.771785:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.771785:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.771786:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.771786:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.771791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.771797:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.771801:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.771801:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.771802:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.771804:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.771807:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.771810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.771816:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.771819:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.771819:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.771819:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.771820:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.771821:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.771842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.771889:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.771891:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.771892:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.771893:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.771895:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.771898:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.771923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.771930:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.771933:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.771934:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.771936:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.771939:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.771940:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.771942:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.771943:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.771944:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.771945:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.771949:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.771950:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.771953:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.771954:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.771954:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.771956:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.771957:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.771958:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.771959:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.771960:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.771962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.771965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.771970:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.771972:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.771972:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.771973:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.771974:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677447.771974:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.771975:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.771976:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.771976:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.771979:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.771982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.771989:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.771991:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.771992:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.771993:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.771993:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.771994:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677447.772112:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.772115:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.772115:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.772117:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.772120:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.772121:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.772122:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.772124:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.772125:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.772126:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.772129:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.772130:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.772133:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.772133:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.772134:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.772136:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.772136:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.772137:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.772137:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.772142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.772150:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.772153:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.772153:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.772155:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.772156:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.772157:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677447.772161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677447.772169:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677447.772173:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677447.772174:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.772176:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.772177:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.772178:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.772178:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677447.772181:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.772183:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.772186:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.772187:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.772188:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.772190:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.772191:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.772192:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.772193:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.772194:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.772198:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.772200:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.772203:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.772204:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.772204:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.772206:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.772206:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.772207:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.772208:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.772212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.772220:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.772223:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.772224:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.772225:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.772226:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.772228:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.772262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.772265:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.772267:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.772270:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.772273:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.772274:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677447.772274:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.772275:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.772276:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.772278:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.772279:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.772283:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.772285:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.772288:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.772290:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.772290:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.772292:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.772293:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.772300:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.772301:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.772306:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677447.772309:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677447.772310:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.772310:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:12.0:1619677447.772311:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.772313:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:1.0:1619677447.772313:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.772314:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677447.772315:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677447.772315:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.772315:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:1.0:1619677447.772316:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677447.772317:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.772317:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.772319:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677447.772319:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.772320:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.772322:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.772323:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.772324:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.772325:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.772328:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.772329:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.772332:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.772333:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.772334:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.772336:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.772336:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.772337:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.772337:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.772345:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.772347:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.772348:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.772349:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.772349:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.772350:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.772351:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.772351:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.772351:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.772352:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677447.772353:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.772356:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677447.772420:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.772423:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.772425:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.772428:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.772428:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.772430:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677447.772431:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.772432:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.772434:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.772435:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677447.772438:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.772440:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677447.772443:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.772444:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677447.772445:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677447.772446:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677447.772447:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.772448:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.772448:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.772461:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.772464:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.772465:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.772466:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.772468:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.772470:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.772502:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.772505:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.772507:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.772508:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.772511:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.772513:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.772514:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723712] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.772516:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.772517:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.772517:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.772520:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.772522:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.772525:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.772526:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.772526:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.772529:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.772530:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.772530:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.772531:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.772535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.772543:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.772546:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.772547:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.772547:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.772549:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.772552:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.772636:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677447.772639:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677447.772640:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677447.772642:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.772644:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.772646:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.772647:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.772648:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.772649:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.772650:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.772653:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.772655:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.772658:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.772659:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.772659:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.772661:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.772662:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677447.772662:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677447.772663:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:12.0:1619677447.772674:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.772676:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.772677:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.772679:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.772680:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.772682:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.772736:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.772739:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.772740:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.772742:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.772745:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.772746:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677447.772747:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.772749:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.772749:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.772750:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677447.772753:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.772755:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677447.772758:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.772758:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677447.772759:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677447.772761:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.772761:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.772762:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.772763:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.772767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.772773:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.772776:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.772777:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.772777:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.772779:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.772783:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677447.772833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:30.0:1619677447.772872:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.772875:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.772877:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.772878:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.772879:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.772879:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:30.0:1619677447.772881:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:13.0:1619677447.772881:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:30.0:1619677447.772882:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.772882:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.772882:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:13.0:1619677447.772883:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000400:00000200:30.0:1619677447.772884:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.772884:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.772885:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.772885:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:30.0:1619677447.772886:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.772886:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:13.0:1619677447.772886:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677447.772890:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.772892:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.772893:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.772894:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.2:1619677447.772894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:2.0:1619677447.772894:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.772896:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.772897:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.772897:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.772898:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:2.0:1619677447.772898:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.772898:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.772899:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.772900:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677447.772900:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:2.0:1619677447.772900:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.772901:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.772902:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677447.772903:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:2.0:1619677447.772903:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.772905:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677447.772905:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.772906:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.772908:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677447.772909:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.772909:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:5.0:1619677447.772910:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:2.0:1619677447.772910:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000400:00000200:5.0:1619677447.772911:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96723706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.772913:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)++
00000400:00000200:5.0:1619677447.772913:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.772913:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:5.0:1619677447.772914:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.772914:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.772914:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:5.0:1619677447.772915:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.772915:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.772916:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677447.772917:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.772919:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.772919:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.772920:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.772921:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677447.772921:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.772922:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:13.0:1619677447.772923:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.772923:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.772924:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.772925:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.772926:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.772926:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.772927:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.772929:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.772930:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.772931:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.772931:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677447.772933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.772941:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.772944:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.772945:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677447.772946:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677447.772946:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.772948:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.772950:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.772951:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.772953:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.772954:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.772955:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.773079:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.773081:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.773082:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.773084:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.773087:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.773088:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.773090:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.773091:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.773092:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.773093:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.773096:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.773097:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.773100:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.773101:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.773102:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.773103:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.773104:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.773104:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.773105:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677447.773109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.773117:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.773120:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.773121:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.773123:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.773123:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.773126:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.773138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.773146:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.773148:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.773149:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.773151:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.773153:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.773155:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:24.0:1619677447.773155:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.773156:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.773157:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.773158:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:24.0:1619677447.773158:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.773160:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677447.773161:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.773162:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.773163:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.773163:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.773164:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:31.0:1619677447.773165:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.773166:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723702] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.773167:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.773167:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.773168:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.773168:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:31.0:1619677447.773169:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.773169:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.773170:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.773171:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.773171:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:31.0:1619677447.773172:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.773172:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:31.0:1619677447.773173:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.773176:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.773176:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.2:1619677447.773176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.773177:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.773178:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.773179:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.773180:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.773181:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.773182:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677447.773184:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.773187:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.773188:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.773188:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.773190:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677447.773190:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.773193:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.773193:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.773194:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677447.773195:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:13.0:1619677447.773195:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.773197:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:13.0:1619677447.773197:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.773197:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.773198:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.773199:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677447.773408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.773414:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.773416:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.773419:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.773419:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.773421:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.773423:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677447.773424:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.773426:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.773427:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.773428:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677447.773431:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.773433:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.773435:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.773436:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677447.773437:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677447.773439:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677447.773439:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.773440:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.773440:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:4.2:1619677447.773642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.773663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.773666:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.773668:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.773670:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.773671:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.773673:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.773675:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.773676:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.773678:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.773679:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677447.773679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677447.773681:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677447.773683:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:20.0:1619677447.773684:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.773686:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677447.773686:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:30.0:1619677447.773687:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.773688:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.773689:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.773690:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.773690:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.773690:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677447.773692:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.0:1619677447.773692:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.773693:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.773693:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.773694:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.773699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.773707:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.773710:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.773711:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.773711:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.773713:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.773716:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.773818:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.773821:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.773822:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.773824:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.773826:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.773828:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.773829:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.773830:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.773832:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.773833:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677447.773836:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.773838:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.773841:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.773842:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.773842:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.773844:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.773844:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.773845:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.773846:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677447.773867:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.773870:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.773871:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.773873:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.773876:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.773878:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.773879:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.773880:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.773881:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.773882:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.773885:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.773887:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.773889:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.773890:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.773891:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.773893:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.773893:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.773895:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.773896:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.773896:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.773906:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677447.773907:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677447.773908:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677447.773909:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677447.773909:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.773910:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677447.773911:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.773911:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.773913:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.773915:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.773938:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.773941:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.773941:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.773943:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.773946:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.773947:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.773949:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.773950:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.773951:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.773952:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.773956:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.773958:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.773960:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.773961:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.773962:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.773963:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.773964:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.773965:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.773965:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.773970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.773979:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.773981:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.773982:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.773983:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.773984:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.773986:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.774128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677447.774152:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.774155:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.774157:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.774159:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.774159:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.774161:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677447.774162:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.774163:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.774164:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.774165:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677447.774169:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.774170:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677447.774173:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.774174:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677447.774174:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (22)++
00000800:00000200:31.0:1619677447.774176:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:31.0:1619677447.774176:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.774177:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.774178:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677447.774239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.774247:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.774250:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.774252:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.774253:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.774254:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.774255:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677447.774256:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.774258:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.774258:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.774259:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677447.774263:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.774264:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.774266:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.774267:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.774267:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.774268:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.774269:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.774270:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.774270:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677447.774280:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.774283:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.774283:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.774285:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.774287:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.774288:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.774305:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.774308:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.774309:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.774310:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.774313:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.774313:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.2:1619677447.774412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.774415:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.774417:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.774419:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.774420:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.774422:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.774423:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.774425:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.774426:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.774427:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.774428:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.774431:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.774433:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.774435:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.774436:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.774436:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.774438:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.774438:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.774439:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.774441:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.774441:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.774446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677447.774452:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677447.774452:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.774454:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:22.0:1619677447.774455:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677447.774455:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:22.0:1619677447.774456:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.774456:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.774456:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677447.774456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:22.0:1619677447.774458:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.774460:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677447.774499:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.774502:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.774503:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.774504:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.774504:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677447.774506:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.774507:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677447.774507:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.774508:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.774510:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.774513:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.774514:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.774515:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723688] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.774517:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.774518:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.774519:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:18.0:1619677447.774519:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677447.774522:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677447.774522:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:30.0:1619677447.774523:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.774524:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:18.0:1619677447.774524:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677447.774525:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.774526:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677447.774526:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.774526:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.774527:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.774529:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.774529:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.774531:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.774532:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.774532:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:9.2:1619677447.774535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.774543:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.774544:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.774544:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.774545:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:2.0:1619677447.774545:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.774546:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.774547:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.774547:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.774548:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677447.774550:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.774552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.774559:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.774562:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.774562:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.774562:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.774563:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.774564:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.774615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:8.2:1619677447.774654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677447.774656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:5.0:1619677447.774663:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.774665:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.774666:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.774667:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.774668:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.774671:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.774686:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.774689:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.774690:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.774692:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.774695:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.774696:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.774697:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.774699:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.774699:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.774701:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.774704:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.774706:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.774707:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.774708:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.774709:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.774710:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.774711:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.774711:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.774711:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.774716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.774723:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.774725:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.774726:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.774727:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.774728:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.774729:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.774810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.774818:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.774821:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.774823:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.774824:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.774825:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.774827:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677447.774828:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.774830:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.774831:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.774832:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677447.774836:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.774838:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677447.774842:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.774843:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677447.774843:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677447.774845:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677447.774846:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.774847:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.774849:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.774849:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677447.774851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677447.774860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.774865:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.774867:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.774867:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.774867:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:31.0:1619677447.774868:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.774869:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:30.0:1619677447.774869:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.774869:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:31.0:1619677447.774870:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677447.774870:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.774871:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.774872:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:20.0:1619677447.774872:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.774873:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677447.774874:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.774875:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:6.0:1619677447.774876:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.774878:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.774879:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:31.0:1619677447.774881:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000400:00000200:6.0:1619677447.774881:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.774883:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.774884:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000400:00000200:6.0:1619677447.774884:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:31.0:1619677447.774885:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.774886:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677447.774886:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:6.0:1619677447.774886:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.774887:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.774888:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:6.0:1619677447.774888:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.774889:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.774893:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.774895:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.774897:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.774898:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.774899:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.774900:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.774901:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.774902:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.774903:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.774904:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.774907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.774913:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677447.774914:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677447.774915:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.774916:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.774917:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.774917:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.774918:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.774919:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.774920:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.774923:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.774928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.774935:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.774938:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.774938:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.774938:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.774939:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.774940:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.774964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.775008:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677447.775011:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677447.775012:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677447.775014:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.775016:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.775018:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677447.775019:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.775020:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.775021:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.775022:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677447.775026:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.775027:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677447.775030:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.775031:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:17.0:1619677447.775031:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677447.775033:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677447.775033:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677447.775034:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:2.0:1619677447.775034:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677447.775036:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677447.775036:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:2.0:1619677447.775037:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.775038:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:8.2:1619677447.775039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:2.0:1619677447.775039:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.775042:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.775044:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677447.775045:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96723680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.775046:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.775047:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.775047:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:5.0:1619677447.775048:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:2.0:1619677447.775048:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.775050:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677447.775050:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:5.0:1619677447.775051:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:1.0:1619677447.775051:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000400:00000200:5.0:1619677447.775052:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.775052:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.775053:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.775054:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.775054:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:5.0:1619677447.775056:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:2.0:1619677447.775057:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.775058:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:22.0:1619677447.775059:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.2:1619677447.775059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.775059:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677447.775061:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.775061:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:22.0:1619677447.775062:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.775062:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677447.775062:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.775062:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:22.0:1619677447.775063:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.775065:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.775066:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677447.775068:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.775069:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.775070:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.775070:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.775071:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.775072:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:12.0:1619677447.775074:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.775077:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.775078:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.775080:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.775081:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677447.775081:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:12.0:1619677447.775083:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.775084:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.775085:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.775085:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.775086:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.775087:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.775148:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.775150:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.775151:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.775153:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.775156:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.775157:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.775158:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.775160:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.775161:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.775162:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.775166:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.775167:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.775170:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.775171:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.775172:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.775174:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.775174:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.775175:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.775175:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677447.775197:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.775200:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.775200:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.775202:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.775203:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.775205:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677447.775222:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.775224:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.775225:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.775228:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.775230:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.775232:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.775233:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.775234:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.775235:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.775236:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.775240:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.775242:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.775245:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.775246:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.775247:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.775249:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.775249:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.775250:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.775250:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.775261:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.775264:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.775265:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.775266:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.775268:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.775269:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:7.0:1619677447.775301:0:520:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:7.0:1619677447.775302:0:520:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:7.0:1619677447.775304:0:520:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:20.2:1619677447.775385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.775393:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.775396:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.775398:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.775400:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.775401:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.775402:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.775404:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.775405:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.775406:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.775407:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.775411:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.775412:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.775416:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.775416:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.775417:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.775419:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.775420:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.775421:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.775421:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.2:1619677447.775460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:4.2:1619677447.775461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.775467:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.775470:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:8.2:1619677447.775471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:5.0:1619677447.775473:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.775474:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.775475:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:5.0:1619677447.775475:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.775477:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.775477:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:4.2:1619677447.775477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:5.0:1619677447.775478:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.775479:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.775480:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.775481:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.775482:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.775483:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677447.775484:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677447.775486:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.775487:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:5.0:1619677447.775487:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.775488:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.775489:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677447.775490:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:8.0:1619677447.775492:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.775493:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.775494:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000400:00000200:8.0:1619677447.775495:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.775495:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:8.0:1619677447.775496:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:5.0:1619677447.775497:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:5.0:1619677447.775497:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:8.0:1619677447.775498:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96723674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.775498:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000400:00000200:8.0:1619677447.775499:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.775499:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:8.0:1619677447.775500:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.775501:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:9.2:1619677447.775503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:20.2:1619677447.775504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:8.0:1619677447.775505:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.775507:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677447.775508:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.775510:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:4.2:1619677447.775510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677447.775511:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.775512:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.775512:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:24.0:1619677447.775513:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:20.0:1619677447.775513:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.775513:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.775513:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:24.0:1619677447.775514:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.775515:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:24.0:1619677447.775516:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.775516:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.775516:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.775516:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.775517:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:20.0:1619677447.775518:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:8.0:1619677447.775518:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.775518:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.775519:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:20.0:1619677447.775519:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96723673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.775519:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:6.0:1619677447.775519:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.775520:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.775522:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677447.775522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:6.0:1619677447.775522:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.775523:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:6.0:1619677447.775525:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.775526:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:20.0:1619677447.775527:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.775529:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.775530:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677447.775532:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.775532:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.775532:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677447.775533:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.775533:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.775534:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.775535:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.775536:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.775536:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:1.0:1619677447.775536:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.775537:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677447.775538:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.775541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.775549:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.775551:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.775552:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.775553:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.775555:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.775557:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.775562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.775569:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.775572:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.775572:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.775573:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:12.0:1619677447.775574:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677447.775574:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.775574:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:12.0:1619677447.775577:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.775578:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.775579:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:12.0:1619677447.775579:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.775581:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:13.0:1619677447.775581:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677447.775582:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677447.775583:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:22.0:1619677447.775584:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.775587:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.775589:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.775590:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.775591:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.775591:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:22.0:1619677447.775592:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.775593:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:13.0:1619677447.775594:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.775596:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.775596:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:22.0:1619677447.775597:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.775597:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.775598:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.775600:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.775602:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.775603:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.775604:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.775606:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.775606:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.775607:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.775607:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677447.775633:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.775636:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.775637:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.775638:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.775641:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.775643:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.775644:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.775645:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.775646:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.775647:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.775651:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.775653:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.775656:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.775657:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.775657:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.775659:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.775659:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.775660:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.775660:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.775665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.775671:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.775674:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.775675:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.775675:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.775677:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.775680:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:18.0:1619677447.775838:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.775841:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.775842:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.775844:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.775847:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.775848:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.775849:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.775851:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.775852:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.775853:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.775856:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.775858:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.775859:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.775860:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.775861:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.775862:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.775863:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.775863:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.775864:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.775869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.775876:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.775878:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.775878:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.775879:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.775880:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.775881:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.775912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.2:1619677447.775951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.775959:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.775962:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.775963:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.775964:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.775965:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.775967:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.775971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.775979:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.775982:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.775984:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.775985:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.775987:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.775988:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.775989:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723667] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.775992:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.775992:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.775993:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.775996:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.775998:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.776000:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.776001:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.776001:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.776003:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.776004:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.776005:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.776005:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677447.776015:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:6.0:1619677447.776016:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677447.776018:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.776019:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.776019:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.776020:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:13.0:1619677447.776021:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.776022:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:6.0:1619677447.776022:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.776023:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677447.776023:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:6.0:1619677447.776025:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.776027:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677447.776027:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:1.0:1619677447.776027:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:6.0:1619677447.776028:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.776030:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.776030:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.776031:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.776032:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677447.776033:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.776035:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.776036:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.776037:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.776038:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.776039:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:1.0:1619677447.776039:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.776040:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.776042:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.776044:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.776044:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:1.0:1619677447.776044:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.776046:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.776046:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.776047:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677447.776047:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.776048:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677447.776050:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.776051:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.776051:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.776053:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.776054:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.776056:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.776057:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.776057:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.776067:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.776070:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.776070:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.776071:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.776077:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.776079:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.776080:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.776082:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.776083:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.776085:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677447.776430:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.776433:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.776434:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.776436:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.776438:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.776439:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677447.776441:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.776442:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.776443:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.776444:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677447.776448:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.776450:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.776453:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.776454:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:18.0:1619677447.776454:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677447.776456:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.776456:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.776457:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.776458:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.776459:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.776470:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:12.0:1619677447.776471:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677447.776471:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.776472:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:12.0:1619677447.776473:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:2.0:1619677447.776473:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:12.0:1619677447.776474:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.776476:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.776478:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.776479:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677447.776540:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.776543:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.776543:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.776545:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.776547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:17.0:1619677447.776549:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.776550:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.776551:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.776553:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.776554:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.776555:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.776559:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.776561:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.776564:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.776564:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.776565:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.776567:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.776567:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.776568:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.776568:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.776581:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.776583:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.776584:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.776584:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.776586:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.776588:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:30.0:1619677447.776594:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.776597:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.776598:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.776599:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.776601:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.776602:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677447.776635:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.776638:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.776638:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.776641:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.776643:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.776645:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.776646:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.776648:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.776649:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.776650:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.776653:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.776655:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.776656:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.776657:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.776657:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.776659:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.776659:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.776661:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.776662:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.776662:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.776665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.776673:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.776674:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.776674:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.776675:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:2.0:1619677447.776675:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.776676:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.776677:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.776677:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.776678:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677447.776678:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.776839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.776842:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.776844:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.776846:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.776847:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.776849:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.776850:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.776852:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.776853:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.776854:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.776855:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.776859:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.776860:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.776864:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.776864:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.776865:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.776867:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.776868:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.776869:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.776869:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.776880:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.776883:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677447.776884:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:13.0:1619677447.776884:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.776885:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.776887:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:12.0:1619677447.776887:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.776888:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677447.776889:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.776891:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.776891:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.776893:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.776894:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.776895:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.776896:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.776897:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.776900:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.776901:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.776904:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.776904:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.776905:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.776907:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.776907:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.776909:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.776909:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.776918:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.776921:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.776921:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.776923:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.776924:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.776926:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.776940:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.776942:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.776943:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.776944:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.776944:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.776945:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.777028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.777035:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.777038:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.777040:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.777042:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.777042:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.777043:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.777044:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.777046:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.777046:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.777047:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.777050:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.777052:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.777055:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.777056:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.777057:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.777058:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.777059:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.777060:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.777060:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.777068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.777076:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.777079:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.777080:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.777082:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.777082:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.777085:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677447.777086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:1.0:1619677447.777093:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677447.777094:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677447.777096:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.777096:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.0:1619677447.777097:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.777098:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.777099:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.777099:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:1.0:1619677447.777099:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.777100:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:1.0:1619677447.777101:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.777103:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.777104:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.777105:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.777106:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.777107:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.777110:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.777112:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.777115:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.777116:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.777116:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.777118:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.777119:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.777119:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.777120:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.777124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.777131:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.777134:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.777134:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.777135:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.777137:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.777140:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:18.0:1619677447.777255:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.777258:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.777259:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.777261:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.777262:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.777263:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.777264:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.777266:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.777266:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.777267:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.2:1619677447.777268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:18.0:1619677447.777270:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.777271:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:18.0:1619677447.777272:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.777273:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.777274:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:18.0:1619677447.777274:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.777275:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677447.777276:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.777276:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.777276:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.777277:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.777277:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.0:1619677447.777277:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.777279:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.777281:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677447.777282:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677447.777282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677447.777283:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.777284:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.777285:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.777288:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:20.0:1619677447.777289:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.777291:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.777291:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.777291:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.777292:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.777300:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.777301:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677447.777301:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:2.0:1619677447.777301:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.777302:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.777303:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677447.777304:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.777304:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.777305:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677447.777347:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907e3ce200] (21)++
00000800:00000200:18.0:1619677447.777351:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.7@o2ib3
00000800:00000200:2.0:1619677447.777351:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3ce200] (22)--
00000400:00000200:18.0:1619677447.777353:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.7@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.777355:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.777357:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:18.0:1619677447.777358:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.777359:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.777360:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.777361:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:18.0:1619677447.777365:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.777367:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:18.0:1619677447.777369:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f90d380] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.777370:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e49d37c00] (20)++
00000800:00000200:18.0:1619677447.777371:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e49d37c00] (21)++
00000800:00000200:18.0:1619677447.777372:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e49d37c00] (22)--
00000800:00000200:18.0:1619677447.777373:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907e3ce200] (21)++
00000800:00000200:18.0:1619677447.777374:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907e3ce200] (22)--
00000800:00000200:18.0:1619677447.777374:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3ce200] (21)--
00000800:00000200:24.2:1619677447.777379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (21)++
00000800:00000200:24.0:1619677447.777384:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)++
00000800:00000200:24.0:1619677447.777387:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e49d37c00] (23)--
00000800:00000200:8.0:1619677447.777388:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (23)--
00000400:00000200:24.0:1619677447.777389:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.777391:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.777394:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)--
00000800:00000200:2.0:1619677447.777415:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.777418:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.777418:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.777420:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.777423:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.777425:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.777426:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.777427:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.777428:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.777429:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.777432:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.777433:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.777436:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.777437:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.777437:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.777439:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.777439:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.777440:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.777440:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.777452:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.777455:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.777456:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.777457:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.777459:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.777460:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.777470:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.777472:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.777475:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.777477:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.777477:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.777479:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.777480:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.777481:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.777482:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.777483:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.777485:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:31.0:1619677447.777487:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.777488:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677447.777488:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:31.0:1619677447.777489:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.777490:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.777492:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.777492:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.777493:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:17.0:1619677447.777494:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.777495:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.777495:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677447.777495:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:31.0:1619677447.777496:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.777497:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:17.0:1619677447.777497:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.777498:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.777499:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.777500:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677447.777504:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.777505:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:13.0:1619677447.777506:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677447.777509:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.777509:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:13.0:1619677447.777509:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.777509:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.777510:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:13.0:1619677447.777511:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.777512:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:17.0:1619677447.777512:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:12.0:1619677447.777512:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.777513:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:13.0:1619677447.777513:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.777515:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677447.777515:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677447.777542:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677447.777545:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677447.777546:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:1.0:1619677447.777546:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677447.777587:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.777589:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.777590:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.777592:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.777594:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.777596:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.777597:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.777599:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.777600:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.777600:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.777603:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.777605:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.777608:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.777608:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.777609:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.777611:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.777611:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.777612:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.777613:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.777623:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.777626:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.777627:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.777628:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.777630:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.777631:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.777727:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.777729:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.777730:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.777732:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.777733:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.777734:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.777751:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.777753:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.777754:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.777755:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.777755:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.777756:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.777828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.777836:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.777838:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.777840:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.777842:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.777842:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.777843:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.777844:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.777846:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.777846:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.777847:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.777850:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.777852:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.777855:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.777855:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.777856:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.777858:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.777858:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.777859:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.777861:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.777861:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677447.777864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677447.777867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677447.777874:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.777877:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677447.777879:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:5.0:1619677447.777879:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.777880:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:20.0:1619677447.777880:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.777880:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.777881:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.777883:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677447.777883:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677447.777884:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677447.777885:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677447.777886:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.777887:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.777890:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.777891:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.777892:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.777894:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.777895:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.777896:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.777898:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.777900:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.777902:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.777903:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.777903:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.777905:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.777905:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.777906:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.777908:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.777908:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677447.777918:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.777920:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.777921:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.777922:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:12.0:1619677447.777922:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.777924:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.777924:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677447.777924:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.777925:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.777926:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677447.777988:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.777991:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.777992:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.777994:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.777994:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:17.0:1619677447.777996:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.777997:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.777998:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:17.0:1619677447.777998:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.778000:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.778001:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.778002:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.778002:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.778004:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.778005:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.778007:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677447.778008:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.778009:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.778010:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.778011:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.778012:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.778013:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677447.778013:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677447.778014:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.778015:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:8.0:1619677447.778016:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.778017:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.778018:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677447.778018:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677447.778019:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.778019:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:4.2:1619677447.778019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:17.0:1619677447.778020:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677447.778021:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.778023:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.778024:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.778026:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.778027:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.778027:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.778028:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:12.0:1619677447.778029:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677447.778029:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000400:00000200:24.0:1619677447.778031:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.778031:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677447.778032:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.2:1619677447.778032:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:24.0:1619677447.778033:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.778033:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.778033:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.778033:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677447.778034:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.778034:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677447.778035:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.778036:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.778036:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.778036:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:12.0:1619677447.778037:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677447.778038:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.778038:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (23)--
00000400:00000200:24.0:1619677447.778039:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.778040:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.778040:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.0:1619677447.778040:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.778041:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.778042:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:20.0:1619677447.778043:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:8.0:1619677447.778043:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:20.0:1619677447.778044:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.778044:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:24.0:1619677447.778045:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.778046:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.778046:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.778047:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.778049:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:24.0:1619677447.778051:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.778052:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.778052:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.778054:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.778054:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.778055:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.778056:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.778060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.778068:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.778071:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.778071:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.778072:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.778074:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.778077:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.778166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.778171:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.778174:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.778176:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.778176:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.778179:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.778181:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.778182:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.778184:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.778184:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.778186:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.778188:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.778190:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.778193:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.778194:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.778195:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.778197:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.778197:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.778198:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.778199:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.778209:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:4.2:1619677447.778209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.778212:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.778213:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.778215:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.778216:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.778218:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.778231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:30.0:1619677447.778254:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.778256:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.778257:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.778259:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.778260:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.778261:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.2:1619677447.778267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.778273:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.778275:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677447.778276:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.778277:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.778277:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.778278:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.778278:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677447.778279:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:20.0:1619677447.778279:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.778281:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.778281:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.778284:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677447.778318:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.778321:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.778323:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.778325:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.778325:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.778327:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.778328:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.778329:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.778330:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.778331:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.778334:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.778336:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.778338:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.778339:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.778339:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.778341:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.778341:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.778342:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.778343:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.778354:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.778357:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.778358:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.778359:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:12.0:1619677447.778359:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.778361:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.778362:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:13.0:1619677447.778362:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.778363:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.778365:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.778366:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.778367:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.778367:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:18.0:1619677447.778368:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.778369:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.778369:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:18.0:1619677447.778370:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.778371:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.778373:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.778374:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.778375:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.778379:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.778380:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.778383:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.778384:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.778385:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.778387:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.778387:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.778388:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.778388:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677447.778400:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.778402:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.778403:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.778403:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:13.0:1619677447.778405:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.778405:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:12.0:1619677447.778406:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.778407:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:13.0:1619677447.778408:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677447.778408:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.778411:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.778413:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.778414:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.778416:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.778417:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.778418:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.778421:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.778422:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.778425:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.778425:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.778426:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.778427:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.778428:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677447.778429:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677447.778430:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677447.778431:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:8.2:1619677447.778434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.778442:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:5.0:1619677447.778443:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677447.778444:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677447.778445:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677447.778446:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.0:1619677447.778449:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.778450:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.778451:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.778453:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.778455:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677447.778490:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.778493:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.778493:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.778495:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.778497:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.778498:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.778577:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.778579:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.778581:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.778582:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.778585:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.778587:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.778588:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.778589:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.778590:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.778591:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.778594:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.778595:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.778598:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.778598:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677447.778599:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.778600:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.778601:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.778602:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677447.778602:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.778602:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.778603:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.778603:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:1.0:1619677447.778603:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.778606:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.778608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677447.778608:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.778609:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.778610:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.778611:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.778612:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.778615:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:1.0:1619677447.778616:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.778618:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:1.0:1619677447.778618:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.778620:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.778620:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.778621:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677447.778622:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.778622:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.778622:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.778624:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.778625:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.778625:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:9.2:1619677447.778626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:1.0:1619677447.778626:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.778626:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677447.778634:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.778637:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.778638:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:13.0:1619677447.778638:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.778638:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.778639:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.778640:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.778641:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.778642:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.778643:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.778645:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.778646:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.778856:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.778859:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.778861:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.778863:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.778863:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.778865:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677447.778866:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.778867:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.778868:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.778869:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677447.778873:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.778874:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677447.778877:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.778878:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677447.778878:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677447.778880:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.778880:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.778881:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.778882:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677447.778927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677447.778930:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:20.0:1619677447.778930:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677447.778933:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:20.0:1619677447.778933:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:31.0:1619677447.778934:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.778935:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.778936:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.778936:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677447.778937:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.778938:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:20.0:1619677447.778938:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.778940:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.778941:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.778943:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.778944:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.778945:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.778948:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.778950:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.778952:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.0:1619677447.778953:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.778954:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.778954:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.778955:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:20.0:1619677447.778956:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.778956:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.0:1619677447.778957:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.778958:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:2.0:1619677447.778958:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.778960:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.778960:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:2.0:1619677447.778960:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.778962:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.778963:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.778964:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677447.778965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:2.0:1619677447.778965:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.778966:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:12.0:1619677447.778968:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:2.0:1619677447.778970:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.778971:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677447.778971:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:12.0:1619677447.778972:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.778972:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:12.0:1619677447.778973:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.778973:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.778974:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677447.778975:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.778975:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677447.778975:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.778976:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.778976:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.778977:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.778979:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.778980:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.778980:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.778981:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677447.778985:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.778988:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.778989:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.778990:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.778990:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.778992:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.778995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.779033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677447.779037:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.779040:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.779040:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.779042:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.779043:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.779045:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677447.779056:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.779059:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.779060:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.779062:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.779065:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.779066:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.779067:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723629] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.779069:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.779070:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.779071:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.779074:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677447.779074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:18.0:1619677447.779076:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.779079:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.779079:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.779080:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677447.779081:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677447.779082:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.779082:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.779083:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.779083:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677447.779084:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.779085:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.779087:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.779090:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.779091:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.779092:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.779094:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.779095:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.779095:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:24.0:1619677447.779096:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.779098:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:24.0:1619677447.779099:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.779099:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.779100:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677447.779101:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.779102:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.779104:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.779104:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.779105:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.779106:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.779107:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.779108:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.779109:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:20.2:1619677447.779109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:24.0:1619677447.779111:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.779111:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.779113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.779117:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:4.2:1619677447.779117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677447.779120:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.779122:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.779122:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:31.0:1619677447.779124:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.779124:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677447.779124:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677447.779124:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:17.0:1619677447.779125:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.779125:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:31.0:1619677447.779126:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:8.0:1619677447.779126:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.779126:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:31.0:1619677447.779127:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.779128:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.779128:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:31.0:1619677447.779129:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.779130:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.779131:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:17.0:1619677447.779131:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:31.0:1619677447.779134:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.779136:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.2:1619677447.779136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677447.779138:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.779139:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.779139:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.779141:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.779141:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.779142:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.779143:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:1.0:1619677447.779143:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.779146:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.779147:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.779147:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.779148:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.779149:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677447.779153:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.779155:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.779156:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.779158:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.779159:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.779160:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677447.779401:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.779403:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.779405:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.779408:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.779408:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.779409:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677447.779410:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.779412:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.779412:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677447.779413:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.779413:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:17.0:1619677447.779415:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.779415:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.779418:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.779419:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.779421:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:17.0:1619677447.779421:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.779422:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:31.0:1619677447.779424:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.779424:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:17.0:1619677447.779424:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.779425:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:17.0:1619677447.779425:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.779426:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:17.0:1619677447.779426:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.779427:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677447.779427:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.779428:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.779428:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:17.0:1619677447.779430:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.779432:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:4.2:1619677447.779433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.779434:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.779435:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.779435:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.779437:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.779438:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.779438:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.779439:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.779444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.779451:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.779453:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.779454:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.779455:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677447.779455:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.779457:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.779458:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.779458:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677447.779459:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:22.0:1619677447.779461:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.779464:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.779466:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.779468:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.779470:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.779471:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.779472:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.779477:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.779479:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.779482:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.779482:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.779483:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.779485:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.779485:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.779486:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.779488:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.779488:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677447.779491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.779499:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.779501:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.779502:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677447.779503:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:6.0:1619677447.779503:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.779503:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:17.0:1619677447.779506:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:5.0:1619677447.779506:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.779506:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.779507:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:20.0:1619677447.779508:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:5.0:1619677447.779508:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.779509:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.779510:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:20.0:1619677447.779510:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.779511:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000400:00000200:17.0:1619677447.779511:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.779511:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677447.779512:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:20.0:1619677447.779512:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000400:00000200:17.0:1619677447.779512:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:5.0:1619677447.779512:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000400:00000200:30.0:1619677447.779513:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.779513:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:17.0:1619677447.779514:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.779515:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.779515:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.779516:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:17.0:1619677447.779517:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.779517:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.779518:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677447.779519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:17.0:1619677447.779521:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.779523:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677447.779525:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.779525:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677447.779526:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:17.0:1619677447.779527:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677447.779528:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.779528:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:17.0:1619677447.779529:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677447.779529:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677447.779530:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.0:1619677447.779531:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.779531:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.779533:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:5.0:1619677447.779534:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.779536:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:2.0:1619677447.779536:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.779537:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:5.0:1619677447.779537:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.779539:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.779539:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.779540:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.779541:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.779542:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:2.0:1619677447.779542:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.779544:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:2.0:1619677447.779544:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677447.779545:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.779545:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96723619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.779546:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.779546:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:2.0:1619677447.779547:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.779548:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.779548:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.779549:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.779550:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:22.0:1619677447.779551:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.779551:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.779552:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.779552:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:2.0:1619677447.779553:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.779554:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.779555:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677447.779555:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.779556:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.779557:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677447.779558:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.779559:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.779560:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.779562:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.779562:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.779563:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.779564:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677447.779569:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.779572:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.779573:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.779575:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.779576:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.779577:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.779723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.779726:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.779729:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.779731:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.779732:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.779734:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.779736:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677447.779737:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.779738:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.779740:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.779741:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677447.779744:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.779746:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677447.779750:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.779750:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677447.779751:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677447.779753:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677447.779753:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.779755:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.779756:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.779757:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.2:1619677447.779762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:31.0:1619677447.779766:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.779768:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:5.0:1619677447.779768:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000400:00000200:31.0:1619677447.779770:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.779770:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.779771:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:31.0:1619677447.779772:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.779772:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677447.779772:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677447.779775:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.779784:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.779786:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.779787:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.779788:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677447.779788:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.779790:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677447.779807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.779852:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.779855:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.779856:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.779857:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.779859:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.779862:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.779870:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.779873:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.779874:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.779874:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.779875:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.779876:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.779993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.780001:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.780004:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.780006:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.780008:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.780009:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.780010:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677447.780012:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723616] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.780013:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.780014:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.780015:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677447.780018:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.780020:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677447.780023:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.780024:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677447.780025:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677447.780026:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677447.780027:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.780028:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.780029:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.780032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.780040:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.780042:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.780043:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.780045:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.780045:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.780048:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.780055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.780102:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.780105:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.780106:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.780106:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.780109:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.780109:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677447.780111:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677447.780112:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.780113:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.780116:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.780118:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.780119:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.780120:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.780122:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:9.2:1619677447.780122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677447.780122:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.780123:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.780124:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:22.0:1619677447.780125:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.780125:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.780127:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.780127:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000400:00000200:1.0:1619677447.780128:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.780129:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.780130:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677447.780130:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677447.780130:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.780131:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.780132:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.780132:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677447.780132:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:18.0:1619677447.780133:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:1.0:1619677447.780133:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:18.0:1619677447.780134:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.780134:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677447.780134:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:22.0:1619677447.780135:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.780135:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677447.780136:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.780136:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.780136:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677447.780137:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.780139:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.780140:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.780140:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.2:1619677447.780140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:22.0:1619677447.780141:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.780142:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.780143:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.780146:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.780148:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677447.780148:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.780150:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:22.0:1619677447.780151:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.780151:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:17.0:1619677447.780151:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.780151:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:22.0:1619677447.780152:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:17.0:1619677447.780153:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.780154:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.780154:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.780155:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677447.780155:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:22.0:1619677447.780156:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.780160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.780168:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.780170:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.780171:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.780172:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.780173:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.780175:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.780290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.780333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.780339:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.780342:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.780342:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.780343:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.780345:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.780347:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.780402:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.780405:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.780406:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.780407:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.780411:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.780412:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.780413:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.780415:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.780416:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.780417:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677447.780421:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.780422:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.780425:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.780426:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.780427:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.780428:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.780429:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.780430:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.780430:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:4.2:1619677447.780505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.2:1619677447.780544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.780552:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.780554:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.780555:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.780557:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.780557:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.780559:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677447.780561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.780567:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.780570:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.780571:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.780572:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.780573:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.780574:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.780632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.780634:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.780636:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.780639:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.780640:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.780642:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.780644:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.780645:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.780646:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.780647:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.780648:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.780652:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.780654:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.780657:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.780657:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.780658:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.780660:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.780661:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.780661:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.780662:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677447.780674:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.780676:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.780677:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.780679:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.780681:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.780682:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:22.0:1619677447.780709:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.780712:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.780713:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.780715:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.780717:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.780718:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.780720:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.780721:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.780722:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.780723:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.780726:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.780728:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.780731:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.780731:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.780732:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.780734:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.780734:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:4.2:1619677447.780734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677447.780735:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.780735:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.780741:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.2:1619677447.780743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.780744:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677447.780747:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.780749:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.780750:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.780752:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:1.0:1619677447.780752:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:24.0:1619677447.780753:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.780754:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:24.0:1619677447.780755:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.780755:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.780755:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.780756:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.780757:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.780758:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.780760:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.780762:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.780763:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:24.0:1619677447.780766:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.780767:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.780767:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.780769:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.780769:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.780770:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.780771:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.780775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.780782:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.780785:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.780786:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.780786:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.780788:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.780790:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677447.780826:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.780829:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.780830:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.780832:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.780835:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.780836:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.780838:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677447.780839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:18.0:1619677447.780839:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.780840:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.780841:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.780844:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.780846:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677447.780847:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677447.780848:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.780848:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:4.2:1619677447.780848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.780849:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677447.780850:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:18.0:1619677447.780851:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.780851:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:31.0:1619677447.780852:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.780852:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.780852:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:30.0:1619677447.780853:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.780855:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.780856:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:9.2:1619677447.780857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677447.780858:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.780859:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.780861:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.780862:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.780864:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:31.0:1619677447.780865:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.780866:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:31.0:1619677447.780867:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.780867:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.780867:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.780869:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677447.780869:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.780870:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.780871:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677447.780871:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677447.780873:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.780874:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.780874:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.780875:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677447.780886:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:9.2:1619677447.780888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.780889:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.780889:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.780891:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.780893:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.780894:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677447.780896:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.780899:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.780900:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.780900:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.780903:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.780906:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.780992:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.780994:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.780995:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.780997:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.780999:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.781000:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.781002:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723602] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.781003:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.781004:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.781005:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677447.781008:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.781010:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.781013:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.781014:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.781015:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.781016:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.781017:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.781018:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.781019:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.781020:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.781022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.781029:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:17.0:1619677447.781030:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677447.781031:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.781032:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.781032:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.781033:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677447.781033:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.781034:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677447.781035:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.781038:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677447.781151:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.781154:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.781154:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.781157:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.781159:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.781160:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.781162:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.781163:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.781164:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.781165:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.781168:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.781170:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.781173:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.781174:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.781174:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.781176:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.781177:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.781177:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.781178:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.781182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.781190:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.781192:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.781194:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.781194:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.781196:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.781198:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.781201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.781208:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.781210:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.781211:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.781211:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.781212:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.781213:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.781423:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.781426:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.781427:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.781429:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.781431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677447.781431:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.781433:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.781434:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.781435:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.781436:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.781437:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677447.781440:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.781442:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.781445:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.781446:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.781446:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.781448:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.781448:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.781449:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.781450:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.781454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.781461:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.781464:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.781465:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.781465:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.781467:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.781469:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.781473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:30.0:1619677447.781477:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.781479:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:8.0:1619677447.781479:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000400:00000200:30.0:1619677447.781480:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.781481:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.781481:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.781482:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.781482:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.781483:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:31.0:1619677447.781484:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:8.0:1619677447.781484:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:30.0:1619677447.781485:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.781552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677447.781555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.781559:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.781560:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.781560:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.781562:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677447.781563:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.781565:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.781566:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.781568:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.781569:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.781571:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.781572:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.781573:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.781574:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.781576:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.781577:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.781578:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.781579:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.781580:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.781581:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.781581:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.781584:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.781585:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.781585:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.781586:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.781587:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.781587:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.781588:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.781589:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.781589:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.781589:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.781590:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.781590:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677447.781591:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.781591:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.781593:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.781594:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.781595:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.781595:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677447.781599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.781601:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.781603:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.781604:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.781606:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.781607:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.781607:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:13.0:1619677447.781609:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.781610:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.781611:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.781612:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.781613:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.781616:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.781620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.781627:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.781629:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.781630:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.781630:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.781631:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.781632:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.781656:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677447.781659:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677447.781659:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677447.781662:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.781665:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.781666:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.781668:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.781669:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.781670:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.781671:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.781674:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.781676:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.781679:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.781680:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.781681:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.781682:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.781683:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677447.781684:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677447.781684:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:12.0:1619677447.781695:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.781698:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.781699:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.781700:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.781702:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.781703:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.781717:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.781720:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.781720:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.781721:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.781721:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.781723:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.781751:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.781754:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.781755:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.781757:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.781759:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.781760:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.781762:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.781763:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.781764:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:2.0:1619677447.781764:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.781765:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.781768:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:2.0:1619677447.781770:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.781771:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.781771:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677447.781773:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.781774:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.781776:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.781777:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:2.0:1619677447.781777:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:31.0:1619677447.781778:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.781778:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:31.0:1619677447.781779:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.781779:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:31.0:1619677447.781780:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.781780:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:31.0:1619677447.781781:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.781781:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.781782:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.781783:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:31.0:1619677447.781785:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677447.781786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.781787:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.781789:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.781790:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.781791:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.781792:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.781793:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.781794:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:18.0:1619677447.781794:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:31.0:1619677447.781796:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:18.0:1619677447.781796:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.781796:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677447.781796:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677447.781797:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677447.781797:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.781798:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.781799:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.781801:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.781802:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.781804:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.781805:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677447.781806:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.781807:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.781809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:12.0:1619677447.781809:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.781810:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.781811:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.781814:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.781816:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.781817:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.781818:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.781819:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677447.781820:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677447.781821:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.781821:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.781823:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.781825:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.781826:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.781827:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.781828:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677447.781829:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.781829:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.781831:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.781831:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:30.0:1619677447.781835:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.781835:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.781837:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.781838:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.781840:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677447.781840:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:30.0:1619677447.781841:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.781842:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:1.0:1619677447.781842:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.781843:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:1.0:1619677447.781843:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.781844:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:1.0:1619677447.781844:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.781845:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:1.0:1619677447.781845:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:30.0:1619677447.781846:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.781847:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:1.0:1619677447.781848:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.781850:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.781852:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677447.781853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:1.0:1619677447.781853:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.781853:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.781854:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:13.0:1619677447.781855:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677447.781855:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.781856:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.781856:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.781858:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.781858:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.781860:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.781861:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:12.0:1619677447.781861:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.2:1619677447.781861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:13.0:1619677447.781862:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677447.781864:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.781864:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.781865:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:17.0:1619677447.781868:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.781871:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.781872:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.781872:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.781874:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.781875:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.781926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.781933:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.781935:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.781936:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.781938:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.781941:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.781943:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.781944:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.781946:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.781947:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.781948:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.781952:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.781954:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.781957:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.781958:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.781958:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677447.781959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:24.0:1619677447.781960:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.781961:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.781961:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.781962:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677447.781963:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677447.781965:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:8.2:1619677447.781966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677447.781967:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.781968:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677447.781970:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.781971:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677447.781973:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.781974:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.781974:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:20.0:1619677447.781975:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.781976:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:1.0:1619677447.781977:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.781978:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.781978:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.781979:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.781980:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.781981:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 584 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:1.0:1619677447.781983:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677447.781984:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.781984:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.781985:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.781987:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.781987:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677447.781988:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677447.781989:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:8.2:1619677447.781993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.782001:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.782003:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.782004:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.782006:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.782006:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.782008:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:17.0:1619677447.782216:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.782219:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.782220:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.782222:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.782223:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.782225:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.782226:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.782228:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.782229:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.782230:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.782233:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.782235:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.782237:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.782238:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.782239:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.782240:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677447.782241:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677447.782241:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.782241:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.782242:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.782243:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.782246:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.782248:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.782248:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.782249:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.782251:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.782252:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.782253:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.782254:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.782255:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:31.0:1619677447.782257:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.782258:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:31.0:1619677447.782259:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:22.0:1619677447.782259:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.782259:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.782261:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.782262:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.782263:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.782264:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677447.782264:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677447.782265:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.782266:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.782267:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.782267:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.782277:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.782280:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.782280:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.782282:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.782284:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.782285:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.782299:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.782301:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.782303:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.782303:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.782304:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.782304:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.782509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.782517:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.782520:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.782522:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.782524:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.782525:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.782526:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.782527:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723587] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.782528:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.782529:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.782530:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.782533:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.782535:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.782538:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.782538:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.782539:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.782541:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.782541:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.782542:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.782543:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.782547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.782555:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.782557:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.782558:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.782561:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.782561:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.782564:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:13.0:1619677447.782583:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.782585:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.782586:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.782588:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.782590:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.782590:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.782639:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677447.782639:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.782643:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:17.0:1619677447.782643:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.782643:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:2.0:1619677447.782644:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:1.0:1619677447.782646:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.782647:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677447.782649:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.782650:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.782650:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677447.782652:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677447.782652:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.782653:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.782654:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.782655:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.782655:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.782656:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.782656:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677447.782657:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677447.782661:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.782663:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677447.782664:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677447.782666:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677447.782667:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.782667:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.782668:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677447.782669:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.782669:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677447.782670:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:1.0:1619677447.782670:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.782671:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.782672:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677447.782673:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677447.782673:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.782673:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:18.0:1619677447.782674:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677447.782674:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677447.782674:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677447.782675:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677447.782676:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.782679:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.782680:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.782681:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:6.0:1619677447.782682:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.782683:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.782684:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.782684:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:17.0:1619677447.782685:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:22.0:1619677447.782686:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.782686:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96723583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.782687:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:6.0:1619677447.782687:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.782688:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677447.782688:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.782688:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677447.782689:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.782692:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.782693:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:6.0:1619677447.782694:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.782697:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.782699:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.782700:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.782701:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.782702:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.782703:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.782704:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.782705:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.782705:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677447.782709:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:9.2:1619677447.782710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.782712:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.782714:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.782715:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.782716:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677447.782716:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677447.782718:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:18.0:1619677447.782718:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.782718:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.782718:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.782719:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.782720:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:18.0:1619677447.782720:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.782720:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.782722:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.782723:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.782724:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.782725:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.782728:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.782729:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:2.0:1619677447.782730:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.782732:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.782735:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.782736:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.782737:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.782739:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.782739:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.782740:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.782740:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677447.782750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:12.0:1619677447.782751:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.782754:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.782755:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.782757:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.782757:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.782760:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677447.782787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.782795:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.782798:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.782799:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.782801:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.782801:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.782804:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677447.782809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677447.782816:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.782819:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.782820:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.782820:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.782821:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.782822:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677447.782824:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.782827:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.782829:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.782831:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.782832:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.782834:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.782835:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.782837:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.782837:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.782839:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.782842:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.782843:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.782846:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.782846:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.782847:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.782849:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.782849:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.782850:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.782851:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.782860:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.782863:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.782864:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.782865:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.782867:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.782867:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.2:1619677447.782885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.782891:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.782893:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.782896:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.782896:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.782897:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.782899:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677447.782900:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.782902:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.782903:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.782904:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677447.782907:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.782909:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.782912:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.782913:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677447.782913:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.782915:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.782915:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.782917:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.782918:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.782919:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677447.782921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677447.782924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.782926:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.782928:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:17.0:1619677447.782928:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:20.0:1619677447.782929:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.782930:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677447.782930:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.782931:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.782931:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.782934:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.782936:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.0:1619677447.783132:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.783135:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.783136:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.783138:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.783141:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.783142:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677447.783144:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723578] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.783145:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677447.783146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:8.0:1619677447.783146:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.783147:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677447.783150:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.783152:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.783154:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:8.0:1619677447.783155:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.783156:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:8.0:1619677447.783156:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.783156:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:30.0:1619677447.783158:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.783158:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.783158:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.783159:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.783159:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:31.0:1619677447.783160:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.783161:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.783162:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.783163:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.783164:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.783165:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.783166:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.783169:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.783169:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677447.783170:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.783172:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.783172:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.783173:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.783174:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677447.783174:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (23)++
00000400:00000200:12.0:1619677447.783174:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.783176:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:30.0:1619677447.783176:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677447.783176:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:30.0:1619677447.783177:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.783177:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.783177:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.783187:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677447.783188:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677447.783190:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.783190:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.783191:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.783191:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:13.0:1619677447.783193:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.783194:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:1.0:1619677447.783194:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.783195:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:1.0:1619677447.783196:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.783198:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.783199:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.783200:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.783201:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.783202:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:12.0:1619677447.783204:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:1.0:1619677447.783206:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.783207:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.783207:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.783208:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677447.783208:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:12.0:1619677447.783209:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.783210:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677447.783211:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.783212:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.783212:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.783214:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.783215:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.783215:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:4.2:1619677447.783216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:1.0:1619677447.783216:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.783220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.783225:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.783227:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.783228:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.783229:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.783230:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.783230:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.783233:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.783234:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.783236:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.783238:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.783239:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.783240:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.783241:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.783242:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.783243:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.783244:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:24.0:1619677447.783245:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.783247:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.783251:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.783251:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.783252:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.783254:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.783254:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.783255:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.783255:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:31.0:1619677447.783257:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.783259:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.783260:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.783260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677447.783262:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.783263:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.783264:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:17.0:1619677447.783267:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.783270:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.783271:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.783271:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.783274:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.783276:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.783287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.783294:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.783300:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.783302:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.783303:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.783304:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.783304:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677447.783650:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.783653:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.783655:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.783657:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.783658:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.783659:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677447.783660:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723572] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.783662:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.783663:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.783664:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677447.783667:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.783669:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677447.783672:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.783673:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677447.783673:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677447.783675:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.783675:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.783676:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.783677:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677447.783681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.783689:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.783692:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.783693:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.783693:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.783696:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.783701:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.783782:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.783785:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.783786:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.783788:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.783791:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.783792:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.783793:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.783795:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.783796:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.783797:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.783800:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.783802:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.783805:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.783806:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.783806:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.783808:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.783809:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.783810:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.783810:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.783814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.783821:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.783824:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.783825:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.783825:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.783827:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.783830:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.783833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.783839:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.783842:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.783842:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.783843:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.783844:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.783844:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677447.783923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:4.2:1619677447.783928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677447.783931:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.783933:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.783936:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.783937:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.783938:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.783940:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.783941:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.783942:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.783943:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.783944:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.783948:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.783950:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.783953:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.783953:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.783954:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.783955:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.783956:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.783957:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.783957:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677447.783967:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:4.2:1619677447.783968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677447.783970:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.783970:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.783972:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.783973:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:31.0:1619677447.783974:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.783974:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:22.0:1619677447.783976:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.783976:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.783977:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.783979:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.783982:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.784126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.784129:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.784131:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.784133:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.784136:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.784138:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677447.784139:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.784139:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.784141:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.784142:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.784143:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677447.784148:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.784150:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677447.784154:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.784155:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677447.784156:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677447.784158:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677447.784158:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.784159:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.784160:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677447.784170:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (23)++
00000800:00000200:30.0:1619677447.784173:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (24)--
00000400:00000200:30.0:1619677447.784174:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.784175:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.784177:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (23)++
00000800:00000200:30.0:1619677447.784178:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (24)--
00000800:00000200:31.0:1619677447.784179:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.784179:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.784180:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:31.0:1619677447.784181:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.784183:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.784196:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.784199:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.784200:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.784201:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677447.784201:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.784203:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677447.784297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.784303:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.784305:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.784308:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.784308:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.784311:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.784312:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.784314:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.784315:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.784316:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.784317:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.784320:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.784322:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.784324:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.784325:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.784326:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.784327:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.784328:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.784329:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.784329:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.784342:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677447.784343:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.784345:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:18.0:1619677447.784345:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:22.0:1619677447.784346:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.784346:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677447.784347:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.784349:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.784349:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:22.0:1619677447.784351:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:18.0:1619677447.784352:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.784354:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.784355:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.784356:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.784358:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.784359:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.784362:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.784364:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.784366:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.784367:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.784368:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.784369:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.784369:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.784371:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.784372:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.784373:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:4.2:1619677447.784380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:6.0:1619677447.784382:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677447.784383:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:6.0:1619677447.784384:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.784385:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.784385:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.784385:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.784386:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.0:1619677447.784387:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677447.784387:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:6.0:1619677447.784388:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:8.0:1619677447.784389:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:6.0:1619677447.784390:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.784391:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.784392:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.784395:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.784397:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.784398:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.784400:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.784401:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.784402:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.784405:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.784407:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.784410:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.784411:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.784411:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.784413:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.784414:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677447.784414:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.784415:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.784416:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677447.784417:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:8.0:1619677447.784417:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677447.784418:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677447.784419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:17.0:1619677447.784420:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.784422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:17.0:1619677447.784423:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.784425:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.784427:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.784429:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:17.0:1619677447.784429:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.784429:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:17.0:1619677447.784430:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.784430:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:24.0:1619677447.784431:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677447.784431:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.784432:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677447.784432:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.784433:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.784434:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.784436:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.784436:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.784437:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.784438:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.784441:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.784442:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.784443:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.784445:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.784445:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.784446:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.784446:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.784459:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:24.0:1619677447.784461:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:13.0:1619677447.784461:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.784462:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.784464:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:13.0:1619677447.784464:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.784465:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.784466:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:24.0:1619677447.784467:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.784467:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:24.0:1619677447.784470:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.784472:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.784473:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.784474:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.784475:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.784476:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.784479:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.784481:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.784485:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.784485:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.784486:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.784488:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.784488:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.784489:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.784490:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677447.784494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.784502:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.784505:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.784506:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.784507:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.784510:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.784511:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:9.2:1619677447.784547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.784553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.784555:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.784558:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.784558:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.784559:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.784561:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.784565:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677447.784598:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.784601:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.784602:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.784603:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.784605:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.784606:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677447.784615:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677447.784618:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.784618:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.784619:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.784619:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.784620:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:22.0:1619677447.784636:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.784639:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.784640:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.784641:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.784642:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.784642:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.784645:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.784648:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.784649:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.784651:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.784653:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.784655:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.784656:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.784657:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.784658:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.784659:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.784662:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.784664:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.784666:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.784667:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.784667:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.784669:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.784669:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.784670:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.784670:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.784675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.784683:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.784686:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.784686:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.784687:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.784689:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.784691:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677447.784703:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.784706:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.784708:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.784709:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.784710:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.784711:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677447.784713:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.784714:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.784715:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.784716:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677447.784719:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.784720:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677447.784723:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.784724:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677447.784724:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677447.784726:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.784726:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.784728:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.784729:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.784730:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.784743:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.784745:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.784746:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.784747:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677447.784829:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.784832:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.784833:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.784835:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.784837:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.784839:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.784840:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.784841:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.784842:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.784843:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.784847:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.784848:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.784851:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.784852:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.784853:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.784855:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.784856:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.784857:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.784858:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.784859:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677447.784868:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.2:1619677447.784869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677447.784869:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677447.784870:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.784871:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.784871:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.784872:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677447.784873:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.784873:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.784875:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.784876:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.784877:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.784880:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.784882:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.784883:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.784884:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.784886:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677447.784887:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.784888:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.784889:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.784890:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677447.784894:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.784895:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677447.784898:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.784899:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.784900:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677447.784901:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.784902:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.784903:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.784904:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.784905:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677447.784907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677447.784911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677447.784918:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:31.0:1619677447.784919:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:4.2:1619677447.784921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677447.784922:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.784923:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677447.784923:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677447.784924:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677447.784926:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.784926:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.784927:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.784927:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.784928:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.784929:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.784931:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.784932:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.784933:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.784934:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.784936:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.784937:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.784939:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.784940:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.784942:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.784946:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.784949:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.784952:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.784953:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.784954:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.784956:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.784957:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.784958:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.784958:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677447.784963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.784970:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.784972:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.784973:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.784975:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.784976:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.784977:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677447.784979:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.784982:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.784983:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.784985:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.784986:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.784987:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.784990:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.784993:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.784994:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.784996:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.784997:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.784999:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.785000:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.785002:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.785003:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.785004:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.785007:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.785009:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.785012:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.785013:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.785013:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.785015:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.785016:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.785016:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.785017:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.785031:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.785034:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.785035:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.785037:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.785039:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.785041:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.785042:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.785043:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.785045:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.785046:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.785049:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.785051:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.785053:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.785054:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.785055:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.785056:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.785057:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.785058:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.785059:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.785060:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.785072:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.785074:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.785075:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.785076:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.785079:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.785081:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.785082:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.785083:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.785085:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.785087:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.785100:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.785102:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.785103:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.785104:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.785104:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.785106:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.785157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.785165:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.785168:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.785168:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.785169:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.785171:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.785174:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.785197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.785242:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.785245:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.785246:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.785246:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.785248:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.785251:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677447.785333:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.785336:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.785338:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.785341:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.785341:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.785343:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677447.785344:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.785345:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.785346:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.785347:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677447.785350:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.785352:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677447.785355:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.785356:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677447.785356:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677447.785357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677447.785358:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.785359:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.785360:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.785360:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677447.785365:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.785367:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.785369:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.785370:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.785372:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.785374:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:6.0:1619677447.785374:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:8.0:1619677447.785375:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723547] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.785376:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.785376:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.785377:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:8.0:1619677447.785377:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.785378:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677447.785378:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.785380:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.785381:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.785383:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.785384:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.785386:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.785387:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.785388:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.785390:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.785390:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.785391:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.785392:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.785395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.785403:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.785406:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.785407:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.785407:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.785409:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.785412:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.785484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.785507:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.785510:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.785511:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.785513:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.785516:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.785518:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.785519:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.785520:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.785521:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.785522:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:4.2:1619677447.785525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677447.785526:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.785528:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.785530:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.785531:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.785531:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.785533:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.785533:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.785534:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.785535:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.785546:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.785549:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.785550:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.785550:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.785552:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.785554:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.785609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.785615:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.785617:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.785619:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.785620:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.785623:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.785624:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.785625:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.785627:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.785628:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.785629:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.785632:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.785634:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.785636:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.785637:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.785637:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.785639:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.785639:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.785640:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.785640:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.785653:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.785655:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.785656:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.785656:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.785658:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.785660:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.785718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.2:1619677447.785756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.785764:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.785766:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.785767:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.785769:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.785770:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.785772:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677447.785774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.785783:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.785786:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:22.0:1619677447.785788:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677447.785788:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.785788:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.785789:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.785790:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677447.785791:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.785791:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.785793:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.785796:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.785798:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.785799:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.785800:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.785802:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.785803:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.785806:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.785808:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.785811:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.785812:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.785812:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.785814:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.785815:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677447.785815:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:22.0:1619677447.785816:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.785816:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.785817:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.785818:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.785820:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.785823:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.785824:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.785826:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.785827:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.785828:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.785829:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677447.785833:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.785835:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.785838:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.785839:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.785839:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.785842:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.785842:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.785843:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.785844:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.785845:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.785857:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.785858:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.785858:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.785859:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.785860:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677447.785860:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.785861:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.785864:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.785867:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.785868:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.785869:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.785871:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.785872:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.785872:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.785875:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.785877:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.785879:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.785880:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.785880:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.785882:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.785882:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.785883:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.785884:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.785888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.785896:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.785898:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.785899:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.785900:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.785901:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.785903:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.785975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.786024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:30.0:1619677447.786028:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677447.786029:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677447.786031:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.786032:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.786032:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.786034:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677447.786035:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677447.786035:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.786035:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.786037:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:5.0:1619677447.786037:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.786039:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.786040:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.786041:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.786042:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.786043:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.786047:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.786048:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.786051:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.786052:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.786052:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.786054:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.786054:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.786056:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.786057:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.786058:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.786063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677447.786065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:6.0:1619677447.786067:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.786070:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:6.0:1619677447.786070:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.786071:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.786071:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:6.0:1619677447.786071:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.786071:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.786072:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.786073:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:6.0:1619677447.786073:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.786073:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.786074:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.786075:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.786076:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:5.0:1619677447.786078:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.786079:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677447.786081:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.786082:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.786083:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.786084:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677447.786087:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.786088:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.786089:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:22.0:1619677447.786090:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.786091:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.786091:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.786091:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.786092:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.786092:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:22.0:1619677447.786093:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.786093:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677447.786094:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677447.786095:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.786096:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.786097:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.2:1619677447.786128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.786135:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.786138:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:6.0:1619677447.786139:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:31.0:1619677447.786140:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.786142:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:6.0:1619677447.786142:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:31.0:1619677447.786143:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.786143:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:31.0:1619677447.786145:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:6.0:1619677447.786145:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.786147:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.786148:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.786148:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.786149:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677447.786150:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.786151:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.786152:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.786152:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677447.786153:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.786154:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:6.0:1619677447.786155:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.786156:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677447.786157:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.786158:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.786160:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:6.0:1619677447.786160:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.786161:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.786163:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.786163:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:30.0:1619677447.786163:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:6.0:1619677447.786163:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677447.786164:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:30.0:1619677447.786164:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96723534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.786165:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:30.0:1619677447.786166:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.786166:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677447.786167:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.786167:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:30.0:1619677447.786168:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.786168:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677447.786169:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677447.786170:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677447.786170:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677447.786171:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.786171:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:30.0:1619677447.786172:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.786172:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677447.786173:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677447.786173:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.786174:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677447.786175:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.786175:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.786175:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:17.0:1619677447.786176:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677447.786176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677447.786178:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.786178:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:17.0:1619677447.786178:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.786178:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677447.786179:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:17.0:1619677447.786179:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:30.0:1619677447.786181:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.786181:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677447.786181:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96723533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.786181:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:30.0:1619677447.786182:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677447.786182:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677447.786182:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.786183:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677447.786183:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:17.0:1619677447.786184:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.786185:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:12.0:1619677447.786185:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.786186:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:13.0:1619677447.786186:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:18.0:1619677447.786187:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.786187:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:17.0:1619677447.786188:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.786188:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:18.0:1619677447.786189:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.786190:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:18.0:1619677447.786192:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:17.0:1619677447.786193:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.786193:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:22.0:1619677447.786194:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677447.786194:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.786196:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.786196:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677447.786197:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677447.786197:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.786197:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.786198:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.786199:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.786201:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.2:1619677447.786202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677447.786204:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677447.786209:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.786211:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.786212:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.786213:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:6.0:1619677447.786213:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:1.0:1619677447.786214:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.786215:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:6.0:1619677447.786216:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.786217:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.786217:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.786218:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.786219:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.2:1619677447.786220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.786227:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.786230:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.786231:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.786231:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.786232:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.786233:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.786270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.786584:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.786587:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.786587:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.786589:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.786591:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.786593:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.786594:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.786595:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.786596:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.786597:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.786601:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.786602:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.786605:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.786605:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.786606:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.786608:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.786608:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.786609:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.786611:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.786611:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677447.786614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.786621:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.786622:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:1.0:1619677447.786622:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677447.786623:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.786624:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677447.786625:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.786626:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.786626:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.786628:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.786630:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.786637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.786678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.786683:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.786685:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.786686:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.786687:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.786689:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.786692:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:17.0:1619677447.786771:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.786773:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.786774:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.2:1619677447.786776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:17.0:1619677447.786776:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.786778:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.786779:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:17.0:1619677447.786779:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.786780:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.786782:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:17.0:1619677447.786782:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.786783:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.786784:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.786785:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:5.0:1619677447.786785:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.786787:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.786788:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.786789:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.786790:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.786790:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:20.0:1619677447.786792:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.786793:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.786793:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.786794:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.786795:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.786795:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.786797:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.786797:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:20.0:1619677447.786798:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.786798:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.786799:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677447.786800:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.786803:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.786804:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.786805:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.786807:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.786807:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.786808:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.786809:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.786810:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.786813:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.786814:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.786814:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.786816:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.786818:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.786820:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.786823:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.786824:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.786825:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.786827:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.786828:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.786913:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.786916:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.786918:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.786920:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.786921:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.786922:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.786923:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.786925:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.786926:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.786927:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.786930:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.786932:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.786935:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.786935:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.786936:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.786938:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.786938:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.786939:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.786940:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.786949:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.786951:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.786952:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.786954:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.786955:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.786956:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.787072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.787079:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.787081:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.787082:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.787085:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.787088:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.787090:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.787091:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.787092:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.787094:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.787094:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.787097:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.787099:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.787102:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.787103:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.787103:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.787105:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.787106:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.787106:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.787106:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.787111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.787119:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.787122:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.787123:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.787123:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.787125:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.787128:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.787129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.2:1619677447.787134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:17.0:1619677447.787136:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.787139:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.787139:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.787140:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.787141:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.787141:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677447.787142:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677447.787145:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:30.0:1619677447.787147:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.787148:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677447.787149:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.787151:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677447.787152:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.787153:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.787154:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.787155:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677447.787158:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.787160:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677447.787162:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.787163:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677447.787164:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677447.787166:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677447.787166:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677447.787168:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87b3e5800] (22)++
00000800:00000200:30.0:1619677447.787169:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:30.0:1619677447.787170:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:9.2:1619677447.787172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.2:1619677447.787175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:1.0:1619677447.787178:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677447.787181:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:1.0:1619677447.787181:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.787182:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677447.787183:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)++
00000800:00000200:2.0:1619677447.787184:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:1.0:1619677447.787184:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.787185:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.787186:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:18.0:1619677447.787186:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:1.0:1619677447.787186:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.787187:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:1.0:1619677447.787187:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:31.0:1619677447.787188:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000400:00000200:2.0:1619677447.787188:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.787189:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.787190:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.787191:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.787192:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:1.0:1619677447.787192:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.787196:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.787198:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.787200:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.787201:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.787201:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.787203:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.787204:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.787205:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.787206:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.787207:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.787217:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677447.787217:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.787220:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.787220:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.787221:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677447.787221:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677447.787222:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.787222:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:20.2:1619677447.787223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:22.0:1619677447.787224:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.787226:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.2:1619677447.787265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.787272:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.787275:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.787276:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.787278:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.787278:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.787281:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677447.787305:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.787307:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.787308:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.787310:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.787312:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.787314:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.787315:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.787316:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.787317:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.787318:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.2:1619677447.787319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:18.0:1619677447.787322:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.787323:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.787326:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.787327:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677447.787327:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.787327:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.787329:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.787329:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677447.787330:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:18.0:1619677447.787330:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.787331:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:31.0:1619677447.787332:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.787334:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.787335:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.787336:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.787337:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.787339:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.787340:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.787341:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.787344:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.787346:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.787349:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.787350:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.787350:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.787352:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.787352:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.787353:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.787354:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:6.0:1619677447.787355:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.787357:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.787358:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.787359:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.787361:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.787363:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.787365:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.787367:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.787368:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.787370:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.787372:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.787373:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.787383:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.787385:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.787386:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.787387:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.787388:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.787389:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.787669:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.787672:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.787674:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.787676:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.787676:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.787677:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.787678:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.787680:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.787681:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.787681:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.787684:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.787686:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.787688:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.787689:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.787689:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.787691:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.787691:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.787693:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.787694:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.787695:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677447.787703:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.787705:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.787706:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.787707:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.787708:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677447.787709:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.787710:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.787711:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.787711:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.787712:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677447.787744:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.787747:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.787748:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.787750:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.787752:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.787753:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.787755:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.787756:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.787757:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.787758:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.787761:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.787762:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.787766:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.787766:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.787767:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.787769:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.787770:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.787771:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.787772:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.787772:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:12.0:1619677447.787782:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677447.787783:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.787785:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:12.0:1619677447.787785:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677447.787786:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:12.0:1619677447.787786:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.787787:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:12.0:1619677447.787788:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.787789:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.787790:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.787800:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.787803:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.787804:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.787805:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.787805:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.787807:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.787807:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.787810:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.787811:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.787812:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.787815:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.787817:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.787818:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.787820:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.787820:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.787822:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.787825:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.787827:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.787829:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.787830:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.787831:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.787832:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.787833:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.787833:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.787834:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.787846:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.787848:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.787849:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.787850:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.787852:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.787854:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.787862:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.787865:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.787866:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.787866:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.787867:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.787868:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.788087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.788095:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.788097:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.788100:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.788101:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.788102:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.788104:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.788105:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.788106:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.788107:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.788108:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.788112:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.788113:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.788116:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.788117:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.788118:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.788119:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.788120:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.788121:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.788122:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.788131:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.788134:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.788134:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.788136:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.788138:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.788139:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.788252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.788259:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.788262:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.788263:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.788265:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.788268:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.788269:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.788270:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.788272:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.788273:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.788273:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.788276:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.788278:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.788281:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.788281:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.788282:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.788284:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.788284:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.788285:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.788285:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.788290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.788300:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.788302:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.788303:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.788303:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.788307:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.788310:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677447.788341:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.788344:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.788345:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.788346:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.788349:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.788349:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:1.0:1619677447.788433:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.788436:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.788437:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.788439:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.788441:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.788443:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.788444:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.788445:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.788446:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.788447:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.788451:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.788452:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.788455:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.788456:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.788456:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.788458:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.788459:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.788460:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.788460:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.788491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.788498:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.788500:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.788501:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.788502:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.788504:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.788507:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677447.788532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.788535:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.788537:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.788540:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.788540:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.788542:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.788544:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677447.788546:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677447.788548:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000400:00000200:20.0:1619677447.788550:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.788551:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.788552:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:5.0:1619677447.788553:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:5.0:1619677447.788554:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:20.0:1619677447.788556:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.788557:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.788558:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:5.0:1619677447.788560:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.788561:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:20.0:1619677447.788562:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.788563:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677447.788563:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:5.0:1619677447.788563:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.788564:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.788565:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:5.0:1619677447.788565:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.788566:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677447.788569:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.788571:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.788571:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.788572:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.788573:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.788573:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.788574:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.788574:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.788575:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:5.0:1619677447.788575:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.788576:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.788577:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:5.0:1619677447.788577:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:5.0:1619677447.788578:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:5.0:1619677447.788578:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:20.2:1619677447.788579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677447.788580:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.788581:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.788582:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000400:00000200:30.0:1619677447.788583:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.788584:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:8.2:1619677447.788584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677447.788586:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677447.788586:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677447.788588:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.0:1619677447.788588:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677447.788589:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:5.0:1619677447.788589:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:30.0:1619677447.788592:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:5.0:1619677447.788592:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.788592:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.788594:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.788595:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.788595:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677447.788596:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:5.0:1619677447.788596:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:30.0:1619677447.788597:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.788599:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:30.0:1619677447.788600:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.788602:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.788603:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.788604:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677447.788607:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.788609:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677447.788612:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.788613:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:30.0:1619677447.788614:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677447.788615:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677447.788616:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677447.788617:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677447.788617:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:8.2:1619677447.788621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.788629:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.788632:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.788632:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.788633:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.788635:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.788638:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.788675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.788683:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.788686:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.788688:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.788689:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.788691:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.788692:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.788694:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.788695:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.788696:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.788697:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.788699:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.788701:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.788703:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.788704:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.788704:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.788706:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.788706:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.788708:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.788709:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.788710:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677447.788715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.788720:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.788721:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.788722:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.788723:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.788724:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.788725:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.788725:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.788726:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.788727:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.788728:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:9.2:1619677447.788904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.788911:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.788914:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.788915:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.788916:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.788918:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.788921:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.788951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:4.2:1619677447.788982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.788988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677447.788992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:18.0:1619677447.788995:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.788998:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.788999:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.788999:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677447.788999:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677447.789002:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:17.0:1619677447.789002:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.789002:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.789005:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.789007:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.789009:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:9.2:1619677447.789010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677447.789010:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.789012:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.789013:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.789014:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:1.0:1619677447.789014:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.789016:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000400:00000200:1.0:1619677447.789017:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.789019:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.789019:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:1.0:1619677447.789019:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.789020:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (23)++
00000800:00000200:18.0:1619677447.789021:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.789022:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677447.789022:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677447.789023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.789023:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.789024:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.789026:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.789026:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.789027:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.789027:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677447.789030:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.789033:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.789034:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.789034:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.789036:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.789039:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.789039:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.789042:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.789043:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.789044:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.789046:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.789047:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.789071:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.789074:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.0:1619677447.789074:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:31.0:1619677447.789076:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.789077:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:31.0:1619677447.789078:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.789078:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677447.789078:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:31.0:1619677447.789080:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.789080:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.789081:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.789082:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.789082:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.789083:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.789084:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.789084:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.789085:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.789087:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.789088:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.789088:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.789089:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.789090:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.789092:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.789093:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.789094:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.789094:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.789095:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.789096:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.789096:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.789097:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677447.789097:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.789098:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677447.789098:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677447.789099:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677447.789099:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677447.789099:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.789100:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.789101:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.789108:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677447.789111:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677447.789112:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.789112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:12.0:1619677447.789113:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.789115:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677447.789115:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677447.789116:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677447.789116:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.789117:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677447.789118:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.789119:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677447.789120:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.789121:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.789122:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.789124:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.789126:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.789127:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.789128:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.789130:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.789131:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.789132:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.789135:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.789136:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.789139:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.789140:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.789140:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.789142:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.789142:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.789143:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.789144:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.789145:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.789148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.789150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677447.789155:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677447.789155:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677447.789157:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677447.789158:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677447.789159:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:17.0:1619677447.789159:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.789159:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:2.0:1619677447.789159:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.0:1619677447.789160:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677447.789160:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.789161:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677447.789162:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.789163:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.789165:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.789166:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:18.0:1619677447.789167:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.789168:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.789169:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.789170:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.789171:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.789174:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.789176:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.789178:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.789179:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.789179:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.789181:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.789182:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.789182:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.789183:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.789187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.789194:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.789196:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.789196:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.789197:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.789198:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.789200:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.789254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.789262:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.789265:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.789267:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.789269:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.789269:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.789271:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677447.789272:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.789273:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.789274:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.789275:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677447.789278:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.789280:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.789282:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.789283:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.789283:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.789285:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.789285:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.789286:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.789287:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677447.789300:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.789303:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.789304:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.789305:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.789307:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.789308:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677447.789318:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.789321:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.789321:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.789322:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677447.789323:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.789323:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677447.789399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.789402:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.789404:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.789407:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.789407:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.789410:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.789411:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.789413:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.789414:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.789415:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.789416:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.789419:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.789421:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.789423:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.789424:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.789425:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.789427:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.789427:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.789428:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.789429:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.789433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.789441:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.789443:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.789444:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.789445:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.789447:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.789448:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.789449:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.789450:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.789451:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.789453:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.789456:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.789457:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.789459:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.789460:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.789461:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.789462:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.789465:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.789467:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.789469:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.789469:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.789470:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.789471:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.789472:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.789473:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.789474:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.789475:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677447.789477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.789486:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.789486:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.789486:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:17.0:1619677447.789487:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.789487:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.789488:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.789489:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.789489:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.789490:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.789490:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.789528:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.789530:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.789531:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.789533:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.789534:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.789535:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677447.789578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.789621:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.789624:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.789625:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.789625:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.789627:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.789630:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.789637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.789646:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.789648:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.789648:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.789649:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677447.789651:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.789652:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.789654:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.789655:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.789656:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.789658:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.789659:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.789660:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.789661:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.789663:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.789664:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.789665:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.789666:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.789667:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.789667:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.789668:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.789672:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.789673:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.789673:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.789674:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.789675:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.789676:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.789676:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.789677:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.789678:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.789678:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.789679:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.789679:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677447.789680:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.2:1619677447.789681:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.789682:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.789683:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.789684:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.789684:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.789686:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.789688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677447.789689:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.789692:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.789693:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.789693:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.789695:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.789695:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:17.0:1619677447.789698:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:2.0:1619677447.789698:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.789700:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.789700:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.789702:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.789705:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.789859:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.789861:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.789862:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.789864:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.789866:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.789867:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.789868:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.789869:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.789871:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.789871:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.789875:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.789876:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.789878:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.789879:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.789879:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.789881:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.789881:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.789882:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.789882:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.789887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.789894:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.789896:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.789896:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.789897:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.789898:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.789899:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677447.789986:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.789988:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.789989:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.789991:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.789993:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.789995:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.789996:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.789997:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.789998:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.789999:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.790003:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.790004:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.790007:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.790008:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.790009:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.790011:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.790011:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.790012:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.790012:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677447.790066:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.790069:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.790071:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.790073:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.790074:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.790075:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.790076:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.790078:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.790079:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.790080:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.790083:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.790085:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.790087:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.790088:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.790088:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.790090:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.790090:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.790092:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.790092:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.790103:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.790105:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.790106:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.790107:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.790109:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.790111:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.790121:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.790123:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.790124:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.790125:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.790125:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.790126:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.790139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677447.790179:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.790181:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.790182:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.790184:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.790185:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.790187:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677447.790221:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.790224:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.790224:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.790227:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.790228:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.790230:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.790231:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.790232:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.790233:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.790234:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.790238:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.790240:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.790242:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677447.790243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677447.790243:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.790243:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.790245:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.790245:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.790246:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.790247:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:30.0:1619677447.790250:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.790253:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:8.2:1619677447.790253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677447.790255:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.790256:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.790257:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.790259:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677447.790260:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.790261:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:30.0:1619677447.790262:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.790263:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.790263:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:30.0:1619677447.790264:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677447.790264:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.790264:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.790266:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.790268:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.790269:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677447.790271:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677447.790273:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.790274:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.790275:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677447.790276:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.2:1619677447.790276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677447.790277:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.790278:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.790279:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:17.0:1619677447.790283:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.2:1619677447.790283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677447.790286:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.790287:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.790287:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.790288:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.790288:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677447.790290:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.790293:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.790299:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.790299:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.790301:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.790304:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.790342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.790344:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.790347:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.790349:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.790350:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.790352:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.790353:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677447.790354:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.790356:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.790357:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.790358:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677447.790361:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.790363:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.790365:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.790366:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677447.790366:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.790368:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.790368:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.790369:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.790370:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677447.790374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.790377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:1.0:1619677447.790381:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.790383:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.790384:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.790385:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.790385:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.790387:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.790387:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677447.790387:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677447.790389:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:24.0:1619677447.790390:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.790392:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.790394:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.790395:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.790396:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.790397:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.790398:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.790401:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.790403:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.790406:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.790407:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.790407:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.790409:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.790410:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.790411:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.790411:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.790415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.790422:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.790424:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.790425:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.790426:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.790428:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.790430:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.790523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.790529:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.790532:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.790534:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.790534:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.790536:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.790537:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677447.790539:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.790540:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.790541:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.790542:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677447.790546:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.790552:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.790554:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.790555:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677447.790556:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.790558:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.790558:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.790559:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.790560:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677447.790564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.790571:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.790574:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.790575:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.790575:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.790577:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.790579:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.790585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.790593:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.790595:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.790596:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.790596:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:22.0:1619677447.790597:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677447.790597:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.790598:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:22.0:1619677447.790599:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.790601:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.790601:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.790603:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.790606:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.790660:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.790663:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.790663:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.790665:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.790668:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.790669:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.790670:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.790672:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.790672:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.790674:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.790677:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.790678:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.790680:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.790681:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:6.0:1619677447.790681:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.790682:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.790683:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.790684:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:6.0:1619677447.790684:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.790684:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677447.790685:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:6.0:1619677447.790685:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:8.0:1619677447.790687:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677447.790689:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:8.0:1619677447.790695:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.790696:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677447.790697:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:8.0:1619677447.790698:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.790699:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.790699:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.790700:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:8.0:1619677447.790700:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.790700:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.790701:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.790703:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.790705:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.790705:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.0:1619677447.790707:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.790710:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.790711:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.790711:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.790713:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.790713:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.790714:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.790714:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.790725:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.790728:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.790728:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.790730:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.790732:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.790733:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.790734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.790741:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.790744:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.790744:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.790747:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.790749:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.790750:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.790751:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.790754:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.790755:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.790756:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.790760:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.790763:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.790766:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.790766:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.790767:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.790769:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.790769:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.790770:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.790771:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.790779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677447.790782:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:4.2:1619677447.790782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.790784:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.790785:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.790785:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.790788:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.790788:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:22.0:1619677447.790790:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677447.790790:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.790791:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.790793:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.790794:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.790796:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.791112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.791123:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.791126:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.791126:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.791129:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.791131:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.791132:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.791134:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.791136:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.791138:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.791140:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.791144:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.791146:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.791150:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.791150:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.791151:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.791153:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.791153:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.791154:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.791155:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677447.791159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677447.791163:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.791165:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:5.0:1619677447.791165:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677447.791166:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.791168:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:31.0:1619677447.791169:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.791170:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.791170:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.791171:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.791172:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:5.0:1619677447.791173:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.791175:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.791176:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.791178:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.791179:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:5.0:1619677447.791179:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.791180:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.791182:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.791183:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.791184:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.791184:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:5.0:1619677447.791184:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.791186:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.0:1619677447.791186:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.791189:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.791190:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.791191:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.791192:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.791193:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.791194:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.791196:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.791196:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.791201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.791204:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677447.791205:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677447.791206:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:31.0:1619677447.791207:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677447.791207:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.791207:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:31.0:1619677447.791208:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.791210:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.791211:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.791213:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:2.0:1619677447.791217:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.791220:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.791222:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.791223:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.791226:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.791227:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677447.791229:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.791230:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.791231:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.791232:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677447.791235:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.791237:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677447.791240:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.791241:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677447.791241:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677447.791243:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677447.791243:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.791244:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.791245:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:30.0:1619677447.791253:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.791256:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.791258:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.791260:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.791261:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.791261:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:30.0:1619677447.791262:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:30.0:1619677447.791264:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.791264:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677447.791264:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:30.0:1619677447.791266:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.791267:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.791267:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.791268:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.791270:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.791273:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.791274:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.791275:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.791276:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.791277:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.791278:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:1.0:1619677447.791278:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.791279:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:1.0:1619677447.791279:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.791280:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.791280:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:30.0:1619677447.791281:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.791282:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.791283:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:1.0:1619677447.791284:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.791285:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.2:1619677447.791286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.791288:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.791290:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.791290:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:9.2:1619677447.791299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (22)++
00000800:00000200:1.0:1619677447.791299:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.791300:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677447.791301:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.791301:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:7.0:1619677447.791302:0:520:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:1.0:1619677447.791302:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:17.0:1619677447.791303:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:7.0:1619677447.791303:0:520:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:1.0:1619677447.791303:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.791304:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.791305:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:7.0:1619677447.791305:0:520:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:24.0:1619677447.791306:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:7.0:1619677447.791307:0:520:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:17.0:1619677447.791308:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:7.0:1619677447.791308:0:520:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:24.0:1619677447.791309:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.791309:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.791310:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.791312:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.791315:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:1.0:1619677447.791343:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.791346:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.791347:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.791348:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.791351:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.791353:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.791354:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.791355:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.791356:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.791357:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.791360:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.791362:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.791364:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.791365:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.791365:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.791367:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.791367:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.791368:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.791368:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.791381:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.791384:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.791385:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.791385:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.791387:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.791390:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.791592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.791626:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.791629:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.791630:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.791632:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.791635:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.791637:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.791638:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.791639:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:6.0:1619677447.791640:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.791641:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.791641:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.791642:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.791642:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.791644:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.791647:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.791647:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.791648:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.791649:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.791652:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.791653:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.791653:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.791655:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.791656:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.791657:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.791658:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.791658:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.791662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.791668:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.791670:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.791670:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677447.791670:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677447.791671:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677447.791673:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.791674:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.791675:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.791676:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.791679:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.791682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.791688:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.791691:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.791692:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.791692:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.791693:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.791694:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677447.791782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.791792:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.791795:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.791797:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.791799:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.791800:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.791802:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.791803:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.791804:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.791805:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.791806:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.791806:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:31.0:1619677447.791809:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.791810:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:31.0:1619677447.791811:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.791811:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.791812:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.791814:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.791814:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.791815:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:8.0:1619677447.791815:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.791816:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:8.0:1619677447.791816:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:31.0:1619677447.791817:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:8.0:1619677447.791817:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.791818:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:8.0:1619677447.791818:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.791819:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:8.0:1619677447.791819:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.791820:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.791823:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.791825:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.791828:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.791829:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:12.0:1619677447.791830:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.0:1619677447.791830:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.791831:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.791832:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:12.0:1619677447.791833:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:8.0:1619677447.791833:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000400:00000200:12.0:1619677447.791834:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.791834:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000400:00000200:12.0:1619677447.791835:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.791835:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.791837:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.2:1619677447.791837:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:12.0:1619677447.791838:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677447.791845:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:24.0:1619677447.791847:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:20.0:1619677447.791847:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.791848:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.791849:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.791850:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:20.0:1619677447.791850:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.791851:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677447.791851:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.791853:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677447.791859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.791867:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.791870:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.791871:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.791873:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.791875:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.791876:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.791878:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.791881:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.791881:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.791883:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.791886:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.791888:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.791889:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.791890:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.791891:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.791892:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.791896:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.791897:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.791900:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.791901:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.791902:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.791904:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.791904:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.791906:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.791907:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.791907:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677447.791912:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.791915:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:13.0:1619677447.791916:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:30.0:1619677447.791917:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.791918:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677447.791918:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:30.0:1619677447.791919:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.791919:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:30.0:1619677447.791920:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:13.0:1619677447.791920:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.791920:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:30.0:1619677447.791921:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.791921:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677447.791922:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.791922:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677447.791923:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.791923:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:30.0:1619677447.791924:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.791925:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:13.0:1619677447.791925:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:30.0:1619677447.791928:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.791930:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677447.791933:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.791934:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:30.0:1619677447.791935:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677447.791936:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677447.791937:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.791938:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.791940:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.791940:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677447.791942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.791950:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.791952:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.791953:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677447.791954:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:17.0:1619677447.791954:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.791956:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.791957:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.791958:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.791959:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677447.791959:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.792025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.792029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.792036:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.792038:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.792039:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.792041:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.792044:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.792046:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.792047:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.792049:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.792049:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.792050:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.792054:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.792055:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.792058:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.792059:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.792059:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.792061:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.792062:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.792062:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.792063:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.792067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.792074:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.792077:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.792078:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.792078:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.792081:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.792083:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.792086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.2:1619677447.792088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.792092:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677447.792093:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.792094:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:17.0:1619677447.792096:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.792096:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:20.0:1619677447.792097:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.792097:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677447.792097:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677447.792098:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.792099:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:20.0:1619677447.792100:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.792102:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.792103:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.792104:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.792105:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.792106:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.792110:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.792112:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.792114:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.792115:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.792115:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.792117:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.792117:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.792118:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.792119:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.792130:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.792133:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.792134:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.792134:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.792136:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.792138:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.792148:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.792151:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.792152:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.792152:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.792153:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.792153:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.792296:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.792300:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.792302:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.792304:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.792307:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.792308:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.792309:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.792311:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.792312:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.792313:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.792316:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.792318:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.792322:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.792322:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.792323:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.792325:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.792325:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.792326:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.792326:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.792340:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:31.0:1619677447.792341:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.792343:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:12.0:1619677447.792343:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.792344:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.792346:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.792347:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.792348:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677447.792348:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.792349:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:31.0:1619677447.792350:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.792351:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.792352:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.792353:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.792354:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.792355:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.792357:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:31.0:1619677447.792358:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.792360:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.792360:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.792361:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.792361:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.792362:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.792363:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.792363:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677447.792363:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677447.792364:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.792366:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.792366:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.792367:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.792368:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.792378:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.792380:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.792381:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.792382:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.792384:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.792385:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.792461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.792497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.2:1619677447.792701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:8.2:1619677447.792737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.792745:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.792748:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.792749:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.792750:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677447.792751:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.792753:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677447.792921:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.792925:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.792925:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.792927:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.792930:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.792932:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.792933:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.792934:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.792935:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.792936:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.792940:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.792942:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.792945:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.792946:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.792946:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.792949:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.792949:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.792950:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.792950:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677447.793037:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:9.2:1619677447.793038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.793039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.793040:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.793041:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.793043:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.793045:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.793045:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:17.0:1619677447.793047:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:2.0:1619677447.793047:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:17.0:1619677447.793048:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.793049:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.793049:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.793050:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.793051:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.793052:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677447.793052:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.793056:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.793056:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.793059:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.793062:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.793062:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.793063:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.793066:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.793066:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.793067:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.793067:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.793078:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.793081:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.793083:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.793084:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.793086:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.793087:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677447.793175:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.793178:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.793179:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.793181:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.793183:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.793184:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677447.793186:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.793187:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.793188:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677447.793189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:18.0:1619677447.793189:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677447.793193:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.793194:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.793196:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.793197:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677447.793197:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.793198:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.793199:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.793199:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677447.793200:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:18.0:1619677447.793200:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.793201:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:31.0:1619677447.793202:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.793204:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.793205:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.793205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:31.0:1619677447.793207:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.793208:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.793209:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.793210:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.793211:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.793212:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:31.0:1619677447.793214:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.793214:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.793215:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.793216:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677447.793217:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:2.0:1619677447.793218:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.793220:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.793220:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:31.0:1619677447.793221:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.793222:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:9.2:1619677447.793222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:31.0:1619677447.793223:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.793224:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.793225:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.793227:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.793227:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:18.0:1619677447.793229:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.793232:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:20.2:1619677447.793233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677447.793233:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.793233:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.793234:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.793234:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.793236:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.793238:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.793239:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.793241:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:13.0:1619677447.793241:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.793242:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.793243:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:13.0:1619677447.793243:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.793244:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.793245:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677447.793357:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.793360:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.793362:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.793364:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.793365:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.793366:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677447.793367:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.793368:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.793370:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.793370:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677447.793374:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.793375:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677447.793378:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.793378:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677447.793379:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677447.793380:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.793381:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.793382:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.793382:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677447.793397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677447.793405:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677447.793407:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677447.793409:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.793411:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:20.2:1619677447.793411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:31.0:1619677447.793412:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.793413:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677447.793414:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.793414:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:31.0:1619677447.793415:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.793416:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:31.0:1619677447.793417:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.793418:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.793419:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.793420:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.793421:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.793422:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.793423:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677447.793424:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.793424:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.793426:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.793427:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.793427:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.793428:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:20.0:1619677447.793428:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.793429:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677447.793431:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677447.793431:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000400:00000200:20.0:1619677447.793431:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.793432:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677447.793433:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:20.0:1619677447.793433:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.793436:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.793437:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:9.2:1619677447.793437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.793438:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.793439:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.793440:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.793441:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.793441:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:2.0:1619677447.793445:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.793448:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.793449:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.793449:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.793451:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.793453:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.793454:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677447.793456:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.793457:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.793458:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.793460:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.793461:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.793563:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.793566:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.793567:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.793569:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.793571:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.793573:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.793574:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.793576:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.793577:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.793578:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.793581:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.793583:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.793586:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.793587:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.793587:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.793589:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.793590:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.793590:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.793591:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.793595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.793604:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.793606:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.793607:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.793608:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.793609:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.793611:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677447.793616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677447.793624:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.793626:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.793627:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.793628:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.793628:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.793630:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.793748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.793756:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.793759:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.793762:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.793763:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.793764:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.793766:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.793767:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.793769:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.793770:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.793771:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.793774:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.793776:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.793778:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.793779:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.793780:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.793781:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.793782:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.793783:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.793783:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:18.0:1619677447.793786:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.793788:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.793789:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.793791:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.793793:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.793793:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:18.0:1619677447.793795:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.793796:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.793796:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:18.0:1619677447.793797:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677447.793797:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.793798:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677447.793799:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.793799:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:18.0:1619677447.793800:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:13.0:1619677447.793802:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:18.0:1619677447.793803:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.793806:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.793809:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.793810:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.793810:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.793812:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.793813:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.793814:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.793814:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.793835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.793842:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.793845:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.793846:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.793846:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.793848:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.793851:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.793994:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.793996:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.793997:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.794000:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.794002:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.794004:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.794005:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.794007:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.794008:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.794009:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.794012:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.794014:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.794016:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.794017:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.794018:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.794019:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.794020:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.794020:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.794021:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.794033:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.794036:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.794037:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.794037:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.794039:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.794041:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.794051:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.794054:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.794054:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.794055:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.794056:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.794056:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.794061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.794089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.794096:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.794099:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.794100:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.794102:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.794105:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.794106:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.794107:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.794107:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:24.0:1619677447.794108:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.794109:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.794110:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.794110:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.794111:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.794111:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:24.0:1619677447.794114:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.794114:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.794116:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.794117:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.794120:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.794121:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.794121:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.794123:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.794123:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.794125:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.794126:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.794126:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.794129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.794132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677447.794134:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.794136:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677447.794137:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:31.0:1619677447.794138:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.794139:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677447.794139:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:30.0:1619677447.794140:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677447.794140:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:31.0:1619677447.794141:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.794141:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.794141:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.794142:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:31.0:1619677447.794143:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:24.0:1619677447.794143:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:31.0:1619677447.794144:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96723441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.794144:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.794146:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.794146:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:31.0:1619677447.794147:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.794148:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.794150:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:31.0:1619677447.794151:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.794151:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.794152:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.794153:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.794154:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.794156:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.794156:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.794156:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.794157:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.794157:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677447.794159:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.794159:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.794160:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.794161:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.794171:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.794173:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.794174:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.794176:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.794178:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.794179:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.794190:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.794193:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.794194:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.794194:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.794195:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.794196:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.794269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.794277:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.794280:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.794282:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.794284:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.794284:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.794285:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677447.794287:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.794288:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.794289:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.794290:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677447.794298:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.794300:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677447.794303:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.794304:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.794304:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677447.794306:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.794307:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.794307:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.794308:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.794313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.794321:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.794324:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.794325:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.794326:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.794327:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.794329:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677447.794354:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.794358:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.794358:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.794360:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.794363:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.794365:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.794366:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.794367:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.794368:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.794369:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.794373:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.794374:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.794377:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.794377:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.794378:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.794380:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.794380:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.794381:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.794381:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.794386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:1.0:1619677447.794388:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.794391:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.794392:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677447.794393:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:1.0:1619677447.794394:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.794396:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:1.0:1619677447.794396:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.794397:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.794397:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.794399:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.794399:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:1.0:1619677447.794399:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.794400:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.794401:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:1.0:1619677447.794401:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.794402:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.794405:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.794406:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.794410:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.794410:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.794411:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.794413:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.794413:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.794415:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.794416:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.794417:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.794427:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677447.794427:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677447.794429:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.794430:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677447.794430:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677447.794431:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677447.794431:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.794432:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.794434:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.794435:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.794678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.794720:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.794723:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.794724:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.794724:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.794726:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.794729:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.794739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.794744:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.794747:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.794749:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.794749:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.794751:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.794753:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677447.794754:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.794756:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.794757:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.794758:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677447.794761:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.794763:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.794765:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.794766:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677447.794770:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677447.794772:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677447.794773:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:1.0:1619677447.794773:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.794774:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.794775:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:1.0:1619677447.794776:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.794777:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.794779:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.794782:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.794783:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.794784:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.794786:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.794787:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.794788:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.794792:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.794794:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.794797:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.794798:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.794799:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.794801:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.794801:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.794802:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.794802:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.794813:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.794816:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.794817:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.794819:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.794820:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.794821:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.794846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.794854:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.794857:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.794859:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.794861:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.794862:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.794864:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.794865:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.794866:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.794867:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.794868:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.794871:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.794872:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.794876:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.794877:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.794877:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.794879:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.794880:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.794881:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.794881:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.794885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.794893:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.794895:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.794896:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.794898:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.794899:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.794902:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677447.795072:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.795075:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.795076:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.795078:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.795081:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.795082:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.795084:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.795085:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.795086:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.795087:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:4.2:1619677447.795087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:17.0:1619677447.795090:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.795092:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677447.795094:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.795095:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.795096:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.795096:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.795097:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.795098:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677447.795098:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.795098:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.795099:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.795100:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677447.795100:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.795102:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.795104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:8.0:1619677447.795104:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.795105:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.795107:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.795108:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.795109:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.795111:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.795113:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.795113:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.795114:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.795115:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.795116:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.795118:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.795119:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.795120:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.795120:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.795121:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677447.795122:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.795123:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.795124:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.795124:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.795128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.795136:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.795138:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:22.0:1619677447.795139:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677447.795139:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.795140:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.795142:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:1.0:1619677447.795142:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.795143:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.795145:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.795146:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:22.0:1619677447.795147:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.795148:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.795150:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.795151:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.795152:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.795153:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.795156:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.795158:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:4.2:1619677447.795158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.795162:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.795163:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.795163:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.795165:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.795166:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.795167:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.795167:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.795198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.795201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.795205:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.795208:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.795209:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.795209:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.795213:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.795216:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.795217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.795224:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.795227:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.795228:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.795228:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.795229:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.795230:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.795440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.795486:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677447.795488:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677447.795489:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.795489:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:17.0:1619677447.795490:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:6.0:1619677447.795490:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.795491:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677447.795492:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.795493:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.795496:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.795496:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:17.0:1619677447.795498:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.795499:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.795500:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.795501:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.795502:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.795505:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.795507:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.795510:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.795511:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.795512:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.795513:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.795514:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.795514:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.795515:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.795519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.795527:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.795530:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.795530:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.795531:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.795533:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.795536:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677447.795704:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.795707:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.795708:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.795708:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.795710:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.795713:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.795783:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.795786:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.795787:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.795788:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.795790:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.795791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:2.0:1619677447.795792:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.795793:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.795794:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.795795:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.795796:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.795799:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:2.0:1619677447.795800:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.795801:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.795802:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677447.795802:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.795805:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.795805:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.795806:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.795806:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:8.0:1619677447.795807:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.795808:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.795808:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:8.0:1619677447.795809:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.795810:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.795810:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:8.0:1619677447.795811:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.795811:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.795812:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677447.795814:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:8.0:1619677447.795817:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.795818:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.795821:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677447.795822:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:8.0:1619677447.795822:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.795823:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.795824:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677447.795825:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.795825:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677447.795825:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.795826:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677447.795826:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.795827:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.0:1619677447.795827:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.795828:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (22)++
00000400:00000200:1.0:1619677447.795828:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.795830:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677447.795830:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.795831:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677447.795831:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.795832:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.795835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.795844:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.795846:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.795847:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.795847:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.795849:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.795851:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.795986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677447.796029:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.796031:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.796032:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.796034:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.796036:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.796037:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:1.0:1619677447.796062:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.796065:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.796065:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.796067:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.796070:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.796072:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:1.0:1619677447.796072:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.796073:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.796074:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.796075:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:1.0:1619677447.796075:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.796076:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677447.796077:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:6.0:1619677447.796078:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.796081:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.796081:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.796082:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:1.0:1619677447.796083:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:6.0:1619677447.796084:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.796085:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.796087:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.796087:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677447.796088:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:1.0:1619677447.796088:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.796088:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.796090:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.796091:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:6.0:1619677447.796092:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.796092:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677447.796093:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:1.0:1619677447.796093:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.796094:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.796096:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.796097:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.796097:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.796099:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.796099:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.796100:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.796100:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677447.796103:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677447.796104:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:9.2:1619677447.796105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:17.0:1619677447.796106:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.796107:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677447.796107:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677447.796107:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677447.796108:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.796109:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.796111:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.796112:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.796112:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.796114:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.796116:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.796116:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.796117:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.796120:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:31.0:1619677447.796179:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.796182:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.796184:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.796185:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.796187:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.796189:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:31.0:1619677447.796190:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.796191:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.796192:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.796193:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677447.796196:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.796198:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677447.796201:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.796202:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:31.0:1619677447.796202:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677447.796204:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677447.796205:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.796206:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.796208:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.796208:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677447.796210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.796217:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.796220:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.796221:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.796221:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677447.796222:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:17.0:1619677447.796223:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.796224:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.796225:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.796226:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677447.796227:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.796231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.796238:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.796241:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.796242:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.796242:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.796243:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.796244:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677447.796287:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.796290:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.796291:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.796301:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.796302:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.796304:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.796305:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.796306:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.796308:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.796309:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.796312:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.796313:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.796316:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.796317:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.796317:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.796319:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.796319:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.796320:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.796321:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.796322:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.796333:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:22.0:1619677447.796334:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.796336:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677447.796336:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.796336:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:22.0:1619677447.796337:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.796337:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677447.796339:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.796340:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.796342:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677447.796525:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677447.796527:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:6.0:1619677447.796527:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677447.796528:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:22.0:1619677447.796530:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:18.0:1619677447.796530:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.796530:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:18.0:1619677447.796533:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.796533:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.796535:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:6.0:1619677447.796535:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.796536:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.796536:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677447.796538:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.796538:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.796540:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.796540:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.796541:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677447.796541:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.796542:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677447.796548:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.796548:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.796552:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.796552:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.796554:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.796555:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.796555:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.796556:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677447.796556:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.796557:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.796558:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.796558:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.796559:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:6.0:1619677447.796559:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677447.796560:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677447.796561:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:6.0:1619677447.796561:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.796561:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.796562:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.796565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.796572:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677447.796573:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.796574:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.796575:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.796575:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:2.0:1619677447.796575:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:22.0:1619677447.796576:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.796576:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.796576:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677447.796576:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.796578:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.796579:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.796579:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:2.0:1619677447.796579:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.796582:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.796583:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:6.0:1619677447.796590:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.796593:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.796594:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.796594:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.796595:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.2:1619677447.796596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:6.0:1619677447.796596:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.796602:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.796605:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.796607:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.796608:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.796611:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.796613:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.796614:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.796615:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.796616:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.796617:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.796621:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.796623:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.796626:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.796627:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.796628:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.796630:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.796630:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.796631:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.796632:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.796643:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:4.2:1619677447.796643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.796646:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.796647:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.796648:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:13.0:1619677447.796649:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.796650:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:12.0:1619677447.796650:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.796651:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:13.0:1619677447.796652:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:18.0:1619677447.796654:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.796657:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.796658:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677447.796660:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.796662:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.796663:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.796664:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:4.2:1619677447.796668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.2:1619677447.796669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:18.0:1619677447.796669:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.796672:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.796676:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.796676:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677447.796677:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677447.796677:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.796678:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677447.796679:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.796680:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:18.0:1619677447.796680:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:30.0:1619677447.796681:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:18.0:1619677447.796681:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:8.0:1619677447.796681:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677447.796682:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:30.0:1619677447.796684:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.796684:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.796685:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.796687:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.796687:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.796687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:30.0:1619677447.796688:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.796690:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96723412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.796690:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:12.0:1619677447.796691:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:30.0:1619677447.796692:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.796692:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96723412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.796693:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.796693:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.796694:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677447.796694:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.796694:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:20.0:1619677447.796695:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:12.0:1619677447.796695:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.796696:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677447.796696:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.796698:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:13.0:1619677447.796698:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:30.0:1619677447.796699:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.796699:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.796701:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.796701:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.796702:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:24.0:1619677447.796702:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.796703:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.796704:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.796705:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.796705:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:24.0:1619677447.796705:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.796705:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:30.0:1619677447.796707:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:30.0:1619677447.796708:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:24.0:1619677447.796708:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.796708:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677447.796709:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:24.0:1619677447.796709:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677447.796710:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677447.796711:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:24.0:1619677447.796711:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.796711:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677447.796712:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:24.0:1619677447.796712:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.796712:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.796713:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.796713:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677447.796714:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677447.796715:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.796716:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.2:1619677447.796717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677447.796717:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.796717:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.2:1619677447.796717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:13.0:1619677447.796718:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.796719:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.796721:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677447.796725:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:17.0:1619677447.796725:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677447.796728:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:17.0:1619677447.796728:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:31.0:1619677447.796729:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:1.0:1619677447.796729:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.796730:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.796731:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:17.0:1619677447.796732:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.796734:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.796742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.796749:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.796752:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.796752:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.796753:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.796754:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.796754:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.796771:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.796774:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.796775:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.796776:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.796779:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.796780:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.796782:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.796783:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.796784:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.796785:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.796788:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.796790:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.796793:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.796794:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.796794:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.796796:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.796796:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.796797:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.796797:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.796809:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.796812:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.796813:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.796813:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.796815:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.796818:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.796888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.796893:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.796898:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.796901:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.796903:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.796904:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000400:00000200:5.0:1619677447.796906:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.796907:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:5.0:1619677447.796908:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.796909:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677447.796910:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.796911:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:5.0:1619677447.796911:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.796912:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677447.796913:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.796913:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:12.0:1619677447.796916:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:12.0:1619677447.796917:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.796918:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677447.796919:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677447.796920:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.796920:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:12.0:1619677447.796921:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.796923:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.796924:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:12.0:1619677447.796925:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.796925:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.796926:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:12.0:1619677447.796927:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.796927:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.796928:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.796928:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.796931:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.796932:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:12.0:1619677447.796935:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677447.796937:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:12.0:1619677447.796938:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677447.796939:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677447.796940:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:6.0:1619677447.796940:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.796942:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.796943:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.796943:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.796946:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.796948:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.2:1619677447.796954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.796962:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.796964:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.796965:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.796965:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:20.0:1619677447.796967:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.796967:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.796968:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.796969:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677447.796970:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:2.0:1619677447.796971:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.796974:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.796975:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.796977:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.796978:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.796979:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.796980:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.796983:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.796985:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.796987:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.796988:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.796988:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.796990:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.796990:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.796991:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.796992:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.796996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.797002:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677447.797004:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:22.0:1619677447.797005:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.797005:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.797007:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.797007:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.797007:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.797008:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.797009:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.797010:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.797011:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:18.0:1619677447.797011:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:22.0:1619677447.797013:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.797014:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.797015:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.797016:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.797020:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.797022:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.797025:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.797026:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.797027:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.797029:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.797029:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.797030:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.797031:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677447.797041:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.797044:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.797045:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.797047:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.797048:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.797050:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.797060:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.797063:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.797064:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.797065:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.797065:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.797066:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.797204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:31.0:1619677447.797220:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.797223:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.797226:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.797227:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.797228:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.797230:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677447.797231:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.797233:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.797234:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.797235:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677447.797239:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.797241:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677447.797244:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.797245:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677447.797245:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677447.797247:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677447.797247:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.797248:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.797249:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677447.797251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.797258:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:6.0:1619677447.797260:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677447.797261:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.797262:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.797262:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.797263:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677447.797263:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.797264:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.797264:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.797265:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:13.0:1619677447.797266:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.797267:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.797267:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.797268:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:13.0:1619677447.797269:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.797269:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677447.797271:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.797272:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.797281:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.797284:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:12.0:1619677447.797284:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677447.797284:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.797285:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.797286:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.797287:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.797287:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.797288:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.797288:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.797289:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.797290:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.797293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.797300:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.797303:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.797305:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.797305:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.797307:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.797309:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.797310:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.797311:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.797312:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.797313:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.797317:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.797319:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.797323:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.797323:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.797324:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677447.797325:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677447.797325:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.797326:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.797328:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:24.0:1619677447.797329:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.797329:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:5.0:1619677447.797329:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.797329:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:8.0:1619677447.797332:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677447.797334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:8.0:1619677447.797334:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.797336:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.797337:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.797338:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677447.797339:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000400:00000200:8.0:1619677447.797339:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.797340:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677447.797340:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.797342:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:8.0:1619677447.797343:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.797343:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.797344:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.797344:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.797344:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.0:1619677447.797344:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:6.0:1619677447.797344:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.797345:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:6.0:1619677447.797346:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.797347:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:8.0:1619677447.797347:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.797348:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.797348:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677447.797348:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677447.797350:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000800:00000200:8.0:1619677447.797350:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.797350:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.797351:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:30.0:1619677447.797352:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.797352:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:31.0:1619677447.797354:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677447.797355:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.797356:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677447.797357:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.797359:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.797359:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.797360:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677447.797363:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.797363:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677447.797365:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:13.0:1619677447.797365:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.797366:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.797367:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677447.797367:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.797368:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677447.797369:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677447.797370:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:13.0:1619677447.797370:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.797370:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.797371:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677447.797371:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677447.797372:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:4.2:1619677447.797388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.797424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677447.797427:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.797430:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.797431:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.797432:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.797434:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.797435:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677447.797604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.797612:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.797615:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.797615:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.797617:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.797619:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.797621:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.797622:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.797623:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.797624:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.797625:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.797628:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.797630:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.797632:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.797633:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.797634:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.797636:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.797636:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.797638:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.797639:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.797639:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.797645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.2:1619677447.797653:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.797657:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.797660:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.797660:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.797661:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.797662:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.797664:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.797665:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.797666:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.797668:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.797671:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.797718:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.797721:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.797722:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.797724:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.797726:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.797727:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.797728:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.797729:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677447.797730:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.797731:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.797732:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677447.797732:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.797733:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677447.797735:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677447.797737:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.797739:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:18.0:1619677447.797739:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:2.0:1619677447.797739:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.797739:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.797741:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:31.0:1619677447.797742:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677447.797742:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.797742:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:1.0:1619677447.797743:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.797744:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.797744:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.797745:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:2.0:1619677447.797745:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:1.0:1619677447.797745:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.797746:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677447.797746:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.797747:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.797747:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.797747:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.797748:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:30.0:1619677447.797750:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.797750:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677447.797751:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.797752:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96723395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.797752:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96723395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.797754:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677447.797754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.797754:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.797755:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.797755:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.797756:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.797757:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677447.797758:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677447.797758:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.797759:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.797760:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:18.0:1619677447.797762:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677447.797762:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.797762:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677447.797763:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.797763:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:1.0:1619677447.797763:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.797763:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677447.797764:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.797764:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.797765:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:30.0:1619677447.797766:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.797766:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.797767:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.797768:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.797768:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:30.0:1619677447.797769:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.797769:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (24)++
00000800:00000200:2.0:1619677447.797769:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000800:00000200:30.0:1619677447.797770:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:2.0:1619677447.797770:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.797771:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.797771:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (24)--
00000800:00000200:18.0:1619677447.797771:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677447.797772:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.797772:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.797772:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.797773:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677447.797773:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677447.797773:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:30.0:1619677447.797774:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.797774:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677447.797774:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:12.0:1619677447.797774:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.797776:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:9.2:1619677447.797776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:12.0:1619677447.797777:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.797779:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.797780:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.797782:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.797784:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.797786:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.797787:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.797788:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677447.797788:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677447.797790:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:2.0:1619677447.797790:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.797791:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.797792:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.797793:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:22.0:1619677447.797794:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.797797:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.797825:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.797828:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.797828:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.797830:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.797832:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.797832:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.797902:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.797905:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.797905:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.797907:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.797909:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.797911:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677447.797912:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.797913:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.797914:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.797915:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677447.797919:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.797920:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.797924:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.797924:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:18.0:1619677447.797925:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677447.797927:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.797927:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.797928:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.797929:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.797930:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:12.0:1619677447.797940:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677447.797940:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.797942:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:12.0:1619677447.797943:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:2.0:1619677447.797943:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:12.0:1619677447.797944:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.797944:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:12.0:1619677447.797945:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.797947:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.797948:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.797957:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.797959:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:4.2:1619677447.797959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.797960:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.797961:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.797961:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.797962:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.798005:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.798007:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.798008:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.798009:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.798011:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.798014:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.798053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:9.2:1619677447.798090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677447.798092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.798095:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.798097:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:18.0:1619677447.798098:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:20.0:1619677447.798100:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.798100:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677447.798101:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:20.0:1619677447.798102:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.798102:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.798102:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:20.0:1619677447.798104:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.798105:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000400:00000200:20.0:1619677447.798106:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.798107:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.798108:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.798108:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:20.0:1619677447.798109:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:9.2:1619677447.798110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:20.0:1619677447.798113:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.798114:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.798117:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.798117:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.798118:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.798119:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.798120:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.798121:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.798121:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.798121:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677447.798122:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.798122:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.798123:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677447.798123:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.798124:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.798135:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.798137:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.798138:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.798138:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.798140:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.798142:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.798153:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.798156:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.798157:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.798157:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.798158:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.798158:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677447.798208:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.798210:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.798213:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.798214:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.798216:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.798217:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677447.798219:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.798220:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.798221:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.798222:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677447.798226:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.798227:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677447.798230:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.798231:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677447.798231:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.2:1619677447.798232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:31.0:1619677447.798233:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.798233:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.798234:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.798235:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677447.798237:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:5.0:1619677447.798239:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:20.0:1619677447.798242:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:5.0:1619677447.798242:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.798244:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.798246:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:5.0:1619677447.798247:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.798249:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.798250:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.798251:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:5.0:1619677447.798255:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.798257:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:5.0:1619677447.798260:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.798261:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:5.0:1619677447.798261:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:5.0:1619677447.798263:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:5.0:1619677447.798264:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:5.0:1619677447.798265:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:5.0:1619677447.798266:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:5.0:1619677447.798267:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.2:1619677447.798269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677447.798273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:30.0:1619677447.798276:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677447.798277:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000800:00000200:30.0:1619677447.798279:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:20.0:1619677447.798279:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000800:00000200:20.0:1619677447.798280:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677447.798281:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000400:00000200:30.0:1619677447.798282:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.798283:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.798285:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.798286:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.798288:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.798289:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.798290:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.798291:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.798300:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:30.0:1619677447.798301:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.798303:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.798303:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677447.798304:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:30.0:1619677447.798307:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.798307:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:17.0:1619677447.798307:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.798308:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.798309:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.798310:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677447.798310:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.798311:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.798311:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:17.0:1619677447.798311:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677447.798313:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.798314:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.798315:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.798316:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677447.798319:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.798321:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677447.798323:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.798324:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:17.0:1619677447.798324:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:13.0:1619677447.798324:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677447.798326:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677447.798327:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:13.0:1619677447.798327:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.798327:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.798328:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677447.798329:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000400:00000200:13.0:1619677447.798329:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.798330:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:13.0:1619677447.798331:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.798332:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.798340:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.798343:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:1.0:1619677447.798343:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000400:00000200:22.0:1619677447.798344:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.798344:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677447.798345:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000400:00000200:22.0:1619677447.798346:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.798346:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677447.798347:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:22.0:1619677447.798349:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.798393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.798428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.798436:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.798439:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.798440:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.798440:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677447.798440:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.798442:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.798443:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.798444:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.798444:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.798446:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:6.0:1619677447.798447:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.798450:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.798525:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.798527:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.798528:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.798530:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.798532:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.798534:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.798535:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.798537:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.798538:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.798539:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.798542:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.798544:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.798547:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.798548:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.798549:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.798551:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.798551:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.798551:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.798552:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.798563:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.798566:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.798566:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.798568:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.798570:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.798571:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.798684:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.798686:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.798687:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.798689:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.798692:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.798693:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.798695:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.798696:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.798697:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677447.798698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:2.0:1619677447.798698:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.798701:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.798703:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.798705:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.798706:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677447.798707:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677447.798707:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677447.798709:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:2.0:1619677447.798709:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.798709:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.798710:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.798710:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:31.0:1619677447.798711:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.798713:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.798714:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.798715:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.798716:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677447.798716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677447.798718:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.798719:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.798720:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.798723:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.798723:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:31.0:1619677447.798724:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.798726:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.798726:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.798728:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.798728:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.798729:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677447.798729:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677447.798730:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.798732:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.798732:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677447.798732:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677447.798733:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.798734:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:4.2:1619677447.798735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.2:1619677447.798748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.798756:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.798759:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.798760:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.798761:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.798763:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.798766:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677447.798768:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.798770:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.2:1619677447.798771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000400:00000200:30.0:1619677447.798773:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.798775:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.798775:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.798777:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677447.798778:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.798778:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677447.798779:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.798780:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.798781:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.798781:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.798782:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:20.0:1619677447.798782:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.798783:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.798784:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:30.0:1619677447.798785:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.798785:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:5.0:1619677447.798785:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:22.0:1619677447.798786:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.798786:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.798787:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:22.0:1619677447.798788:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.798790:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.798790:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677447.798791:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.798791:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:30.0:1619677447.798793:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.798793:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.798795:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.798796:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.798797:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677447.798806:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.798808:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.798809:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.798809:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.798810:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:6.0:1619677447.798811:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.798813:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.798814:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677447.798814:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677447.798815:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.798965:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.798968:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.798969:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.798970:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.798972:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.798973:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.798995:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.798997:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.798998:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.799000:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.799003:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.799005:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.799006:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.799007:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.799008:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.799009:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.799012:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.799014:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.799016:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.799017:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.799018:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.799020:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.799021:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.799022:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.799022:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.799036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.799043:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.799045:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.799046:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.799048:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.799049:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:1.0:1619677447.799049:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677447.799050:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677447.799052:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.799053:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.799055:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.799058:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.799060:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.799061:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.799062:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.799063:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.799064:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.799068:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.799070:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.799073:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.799073:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.799074:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.799076:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.799076:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.799077:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.799078:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.799079:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677447.799088:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677447.799090:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677447.799091:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677447.799092:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.799092:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677447.799092:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.799093:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677447.799094:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.799095:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.799096:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.799125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.799133:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.799136:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.799138:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.799140:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.799141:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.799143:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677447.799144:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.799145:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.799146:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.799147:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677447.799151:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.799153:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677447.799156:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.799157:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677447.799158:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677447.799159:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677447.799160:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.799161:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.799161:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.799165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.799173:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.799176:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.799177:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.799178:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.799179:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.799181:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677447.799185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.799193:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.799195:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.799196:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.799197:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.799198:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.799198:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.799205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.799212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.799215:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.799217:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.799220:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.799220:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.799223:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.799225:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677447.799226:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.799228:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.799229:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.799230:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:6.0:1619677447.799231:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.799233:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677447.799234:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677447.799237:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.799238:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.799240:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:6.0:1619677447.799240:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.799241:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.0:1619677447.799243:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677447.799243:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96723371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.799244:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)++
00000400:00000200:6.0:1619677447.799244:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.799245:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (23)++
00000400:00000200:6.0:1619677447.799245:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.799246:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.799247:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:20.0:1619677447.799248:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.799249:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000400:00000200:6.0:1619677447.799250:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.799251:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.799251:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:6.0:1619677447.799252:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:4.2:1619677447.799253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.799254:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.799255:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.799256:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677447.799257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:6.0:1619677447.799257:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.799258:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.799258:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677447.799259:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:8.2:1619677447.799263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.799263:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.799265:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.799266:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.799267:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:1.0:1619677447.799271:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.799273:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.799274:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.799274:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.799276:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.799279:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:7.0:1619677447.799305:0:520:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:7.0:1619677447.799307:0:520:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:7.0:1619677447.799309:0:520:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:30.0:1619677447.799418:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677447.799422:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:30.0:1619677447.799424:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.799426:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:30.0:1619677447.799427:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.799428:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677447.799430:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.799431:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.799432:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.799433:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:20.2:1619677447.799436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000400:00000200:30.0:1619677447.799437:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.799439:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:4.2:1619677447.799439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677447.799441:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.799442:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677447.799443:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.799443:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:30.0:1619677447.799445:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677447.799445:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677447.799446:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:22.0:1619677447.799446:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:30.0:1619677447.799447:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:24.0:1619677447.799447:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677447.799447:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677447.799449:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.799450:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677447.799451:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.2:1619677447.799451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:22.0:1619677447.799452:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.799452:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677447.799453:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:17.0:1619677447.799454:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:22.0:1619677447.799455:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.799455:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.799456:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.799457:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.799457:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.799458:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.799459:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.799459:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.799460:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:17.0:1619677447.799460:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.799461:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.799462:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677447.799463:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.799463:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96723368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.799464:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96723368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.799464:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.799465:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.799465:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.799465:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:24.0:1619677447.799466:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.799466:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:17.0:1619677447.799467:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.799468:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.799468:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.799469:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:22.0:1619677447.799469:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677447.799469:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.799470:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677447.799470:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.799472:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.799472:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:22.0:1619677447.799473:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.799473:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677447.799474:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000400:00000200:17.0:1619677447.799474:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.799475:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677447.799475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:17.0:1619677447.799476:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.2:1619677447.799477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:24.0:1619677447.799478:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.799479:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.799480:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.799481:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.799481:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.799482:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.799483:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.799483:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.799483:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677447.799484:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:24.0:1619677447.799484:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.799484:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677447.799484:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:24.0:1619677447.799485:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.799485:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677447.799486:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677447.799486:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:31.0:1619677447.799487:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:24.0:1619677447.799487:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677447.799487:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.799488:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677447.799488:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677447.799488:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000400:00000200:6.0:1619677447.799489:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.799489:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:22.0:1619677447.799490:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.799490:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.799490:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677447.799491:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677447.799491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677447.799492:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.799492:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.799492:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677447.799493:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.799494:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.799495:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.799495:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.799496:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:31.0:1619677447.799497:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.799497:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.799498:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:31.0:1619677447.799499:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96723366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.799499:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:12.0:1619677447.799499:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.799500:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.799501:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.799501:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96723366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.799501:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.799501:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:12.0:1619677447.799502:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.799502:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.799503:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.799504:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.799504:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.799504:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:31.0:1619677447.799505:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.799505:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:12.0:1619677447.799505:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677447.799505:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.799506:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677447.799508:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:22.0:1619677447.799509:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.799509:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.799510:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.799511:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000400:00000200:31.0:1619677447.799512:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.799512:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677447.799515:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.799518:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.799520:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.799521:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.799521:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.799522:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.799524:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.799524:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.799525:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677447.799525:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.799525:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677447.799526:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:18.0:1619677447.799527:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677447.799528:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.799528:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:18.0:1619677447.799528:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.799529:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.799530:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.799530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.2:1619677447.799534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677447.799538:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)++
00000800:00000200:30.0:1619677447.799539:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.799541:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:5.0:1619677447.799541:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (24)--
00000800:00000200:31.0:1619677447.799542:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:30.0:1619677447.799542:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.799542:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.799543:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.799544:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677447.799544:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)++
00000800:00000200:31.0:1619677447.799545:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677447.799545:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (24)--
00000800:00000200:31.0:1619677447.799546:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:30.0:1619677447.799546:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.799546:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.799546:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.799548:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.799550:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.0:1619677447.799550:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.799551:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.799552:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677447.799693:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.799696:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.799698:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.799699:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.799701:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.799702:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677447.799704:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.799705:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.799706:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.799707:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677447.799711:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.799713:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677447.799716:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.799717:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677447.799717:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677447.799719:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677447.799720:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.799721:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.799721:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.799732:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.799735:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.799736:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.799736:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:6.0:1619677447.799736:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.799738:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.799739:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.799740:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.799741:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.799741:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:17.0:1619677447.799744:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.799745:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.799747:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.799748:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.799749:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.799750:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.799753:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.799755:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.799757:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.799758:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.799759:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.799760:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.799760:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.799761:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.799762:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.799773:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.799776:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.799777:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.799777:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.799779:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.799781:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.799792:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.799794:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.799795:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.799795:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.799796:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.799797:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.799892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677447.799936:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.799939:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.799940:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.799941:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.799943:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.799944:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:1.0:1619677447.800073:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.800076:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.800076:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.800079:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.800081:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.800082:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.800084:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.800085:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.800086:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.800087:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.800090:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.800092:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.800093:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.800094:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.800095:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.800097:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.800097:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.800098:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.800098:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.800103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.800110:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.800112:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.800112:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.800113:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.800114:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.800115:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.800140:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.2:1619677447.800142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677447.800143:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.800144:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.800146:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.800148:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.800150:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:2.0:1619677447.800150:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.800151:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.800152:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.800153:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:2.0:1619677447.800153:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.800155:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677447.800157:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.800157:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:2.0:1619677447.800159:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.800161:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.800161:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:31.0:1619677447.800162:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.800164:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.800164:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.800165:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:31.0:1619677447.800166:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.800166:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677447.800167:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.800168:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.800168:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.800168:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.800169:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.800169:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:31.0:1619677447.800171:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.800173:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.2:1619677447.800174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677447.800176:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.800177:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.800178:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.800180:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.800181:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.800181:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677447.800181:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:31.0:1619677447.800182:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677447.800183:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.800184:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.800186:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.2:1619677447.800186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.800188:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.800190:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.800191:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.800193:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.800193:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.800195:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.800196:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.800197:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677447.800203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677447.800210:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.800213:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.800214:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.800215:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.800216:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.800217:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.800222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677447.800229:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677447.800232:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677447.800233:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677447.800235:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.800237:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.800239:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677447.800240:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.800242:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.800243:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.800244:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677447.800247:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.800249:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677447.800251:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.800252:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:8.0:1619677447.800252:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:8.0:1619677447.800254:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:8.0:1619677447.800254:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677447.800255:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677447.800255:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:30.0:1619677447.800266:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.800268:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.800269:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.800270:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.800272:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.800273:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:1.0:1619677447.800448:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.800451:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.800452:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.800454:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.800455:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.800456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677447.800456:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.800458:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.800459:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.800460:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.800461:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.800464:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.800466:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.800468:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.800469:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.800470:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.800471:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.800471:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.800473:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.800474:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.800474:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677447.800478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.800487:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.800488:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.800488:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:17.0:1619677447.800489:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.800489:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.800489:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.800490:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.800491:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.800491:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677447.800493:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.800496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:9.2:1619677447.800498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.800504:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.800506:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677447.800507:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.800508:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.800508:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.800508:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.800509:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:1.0:1619677447.800509:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:18.0:1619677447.800510:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.800510:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.800512:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.800515:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.800666:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.800669:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.800669:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.800671:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.800673:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.800675:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.800676:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.800678:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.800678:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.800679:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.800682:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.800684:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.800687:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.800688:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.800688:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.800690:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.800691:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.800691:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.800692:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.800750:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.800752:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.800753:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.800755:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.800757:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.800759:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.800760:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.800761:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.800762:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.800763:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.800767:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.800768:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.800771:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.800771:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.800772:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.800774:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.800774:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.800775:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.800775:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677447.800787:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.800790:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.800791:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.800791:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.800793:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.800795:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.800804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.800851:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.800853:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.800854:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.800855:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.800857:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.800860:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.800866:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.800869:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.800869:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.800871:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.800873:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.800875:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.800876:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.800877:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.800878:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.800879:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.800883:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.800885:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.800888:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.800888:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.800889:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.800891:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.800892:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.800892:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.800893:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.800905:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.800907:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.800908:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.800910:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.800910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:12.0:1619677447.800912:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.800913:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.800917:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.800919:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.800920:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.800922:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.800924:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.800925:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:24.0:1619677447.800926:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.800927:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.800928:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677447.800929:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.800929:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:24.0:1619677447.800930:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.800930:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.800930:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:24.0:1619677447.800931:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:12.0:1619677447.800932:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:24.0:1619677447.800934:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.800935:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.800938:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.800939:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.800940:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.800941:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.800942:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.800942:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.800943:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.800947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.800955:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.800958:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.800959:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.800959:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.800961:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.800964:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.0:1619677447.801230:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.801233:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.801234:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.801236:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.801239:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.801240:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.801242:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.801243:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.801244:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.801245:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.801248:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.801250:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.801252:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.801253:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.801254:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.801256:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.801256:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.801257:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.801257:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.801262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.801270:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.801272:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:4.2:1619677447.801272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.801273:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.801274:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.801276:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.801279:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.2:1619677447.801316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677447.801319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677447.801325:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.801327:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.801328:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.801330:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.801331:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.801333:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:2.0:1619677447.801408:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.801411:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.801412:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.801414:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.801416:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.801418:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.801419:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.801420:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.801421:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.801422:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.801426:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.801427:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.2:1619677447.801430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677447.801431:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.801431:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.801432:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.801434:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.801434:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.801436:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.801437:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.801437:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677447.801438:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.801441:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.801443:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.801446:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.801446:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.801447:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.801448:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.801448:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:13.0:1619677447.801448:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.2:1619677447.801449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:31.0:1619677447.801450:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.801450:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:31.0:1619677447.801451:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.801451:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:13.0:1619677447.801451:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:31.0:1619677447.801452:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.801452:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:13.0:1619677447.801452:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.801454:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:13.0:1619677447.801454:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.801455:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.801455:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677447.801456:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:12.0:1619677447.801456:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.801457:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677447.801457:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:1.0:1619677447.801457:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677447.801460:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677447.801461:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.801461:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.801461:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.801461:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.801462:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:13.0:1619677447.801462:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.801463:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.801464:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:20.0:1619677447.801464:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.801464:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.801464:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.801465:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677447.801465:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.801465:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.801466:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.801466:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677447.801466:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.801466:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:13.0:1619677447.801466:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.801467:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:20.0:1619677447.801468:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.801468:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.801470:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.801470:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.801472:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.801472:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.801473:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:20.0:1619677447.801474:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677447.801477:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677447.801477:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:20.0:1619677447.801479:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.801479:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.801479:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:30.0:1619677447.801480:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:12.0:1619677447.801480:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.801482:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.801482:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677447.801483:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677447.801483:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.801483:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.801484:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.801485:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.801486:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.801486:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.801486:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:30.0:1619677447.801487:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677447.801487:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.801487:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:30.0:1619677447.801488:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96723342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.801488:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.801488:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677447.801489:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:30.0:1619677447.801490:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.801490:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677447.801490:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:30.0:1619677447.801491:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.801491:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:17.0:1619677447.801491:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677447.801492:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.801492:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677447.801492:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.801492:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677447.801495:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677447.801496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677447.801497:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.801500:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.801500:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.801501:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.801503:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.801503:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677447.801503:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677447.801504:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677447.801504:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:30.0:1619677447.801505:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.801506:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.801506:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.801506:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.801507:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.801507:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.801510:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.801510:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:1.0:1619677447.801510:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.801511:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.801513:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.801513:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.801513:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.801513:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.801514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:13.0:1619677447.801515:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.801515:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.801517:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.801519:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.801522:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.801524:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.801525:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.801527:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.801528:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.801529:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.801531:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.801532:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.801533:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.801536:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.801538:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.801541:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.801542:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.801543:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.801545:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.801545:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.801546:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.801547:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.801558:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.801560:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.801561:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.801563:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.801564:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.801565:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.801572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.801579:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.801581:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.801582:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.801584:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.801587:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.801589:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.801590:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.801592:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.801593:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.801594:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.801597:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.801598:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.801601:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.801602:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.801602:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.801604:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.801604:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.801605:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.801606:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.801610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.801617:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.801620:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.801621:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.801621:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.801624:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.801626:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677447.801724:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.801727:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.801728:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.801730:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.801732:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.801734:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.801735:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.801736:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.801737:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.801739:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.801742:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.801744:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.801746:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.801747:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.801748:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.801749:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.801750:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.801751:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.801752:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.801753:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677447.801756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.801763:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677447.801764:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.801766:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.801766:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677447.801766:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.801767:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.801767:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.801768:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:1.0:1619677447.801769:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.801772:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.801773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.801780:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.801783:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.801784:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.801784:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.801785:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.801786:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:12.0:1619677447.801863:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.801865:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.801866:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.801867:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.801869:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.801870:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.802046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:2.0:1619677447.802054:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.802057:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.802058:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.802061:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.802064:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.802066:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.802067:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.802068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677447.802069:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.802070:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.802072:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.802076:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.802079:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.802081:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.802082:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.802083:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.802085:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.802085:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.802086:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.802086:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.802091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.2:1619677447.802095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677447.802098:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.802100:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.2:1619677447.802101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:17.0:1619677447.802101:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.802101:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.802103:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.802106:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677447.802108:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (23)++
00000800:00000200:20.0:1619677447.802110:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (24)--
00000400:00000200:20.0:1619677447.802111:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.802113:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.802113:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (23)++
00000800:00000200:5.0:1619677447.802115:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (24)--
00000400:00000200:5.0:1619677447.802115:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.802116:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.802117:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.802118:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.802120:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677447.802121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.2:1619677447.802124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.802130:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.802131:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.802132:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.802133:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:5.0:1619677447.802133:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.802134:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.802135:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.802135:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.802136:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.802137:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.802138:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.802139:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677447.802140:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.802142:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.802143:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.802144:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677447.802147:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.802149:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.802151:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.802152:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.802153:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.802154:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.802155:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.802155:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.802156:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677447.802160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.802167:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.802169:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.802170:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.802172:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.802173:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.802174:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:6.0:1619677447.802264:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.802267:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.802268:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.802270:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.802272:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.802274:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.802275:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.802277:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.802278:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.802279:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.802282:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.802284:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.802286:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.802287:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.802288:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.802289:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.802290:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.802291:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.802291:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.802297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.802305:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.802308:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.802309:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.802309:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.802311:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.802311:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.802313:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:17.0:1619677447.802314:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.802315:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.802316:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.802319:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.802321:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.802322:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.802324:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.802325:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.802326:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.802329:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.802331:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.802333:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.802334:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.802335:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.802337:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.802337:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.802338:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.802338:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.802352:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.802354:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.802355:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.802355:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.802358:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.802360:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.802427:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.802430:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.802431:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.802432:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.802435:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.802436:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677447.802438:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.802439:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.802440:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.802441:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677447.802445:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.802446:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.802450:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.802450:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677447.802451:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:4.2:1619677447.802453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:2.0:1619677447.802453:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.802454:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.802454:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.802455:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.0:1619677447.802460:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.802463:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.802464:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.802466:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.802467:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:8.0:1619677447.802469:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.802470:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:8.0:1619677447.802470:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677447.802471:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.802472:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.802472:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.802473:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.802474:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:8.0:1619677447.802474:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.802475:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:8.0:1619677447.802475:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.802479:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.802481:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.802483:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.802484:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.802485:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.802486:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.802487:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.802488:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.802489:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.802490:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.802493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.802495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.802500:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.802502:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.802503:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677447.802504:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677447.802504:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.802504:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.802505:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.802506:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.802506:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.802509:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677447.802705:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.802708:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.802709:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.802711:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.802714:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.802715:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.802716:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.802718:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.802719:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.802720:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.802723:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.802725:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.802726:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.802727:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.802728:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.802729:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.802730:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.802730:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.802731:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.802735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.802742:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.802745:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.802745:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.802746:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.802747:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.802748:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.802752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:31.0:1619677447.802756:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.802759:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:18.0:1619677447.802760:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677447.802761:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.802763:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.802763:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:31.0:1619677447.802764:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.802764:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677447.802764:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.802765:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.802765:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:31.0:1619677447.802766:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.802767:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.802768:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.802769:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.802770:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.802774:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.802776:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.802779:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.802780:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.802781:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.802783:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.802783:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.802784:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.802785:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.802796:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.802799:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.802799:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.802801:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.802803:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677447.802803:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677447.802803:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:12.0:1619677447.802804:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.802804:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.802805:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.802806:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.802808:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.802808:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.802811:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.802811:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.802812:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:22.0:1619677447.802813:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:6.0:1619677447.802814:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.802815:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723326] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.802816:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677447.802817:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.802819:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.802820:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677447.802821:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677447.802821:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.802821:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.802822:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.802823:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:22.0:1619677447.802825:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.802827:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.802829:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.802830:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.802831:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.802832:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.802833:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.802834:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:4.2:1619677447.802834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.802835:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.802838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.802846:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.802849:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.802849:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.802850:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.802852:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.802854:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677447.802880:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.802883:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.802884:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.802884:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.802886:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.802889:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.803067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.803070:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.803072:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.803074:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.803075:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.803076:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.803078:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677447.803079:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.803081:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.803082:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.803083:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677447.803086:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.803088:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.803090:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.803091:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.803092:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.803093:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.803093:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.803095:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.803096:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.803097:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677447.803099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677447.803101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.803105:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.803105:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.803106:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.803107:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.803107:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.803108:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.803108:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.803109:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.803111:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.803112:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.803205:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.803207:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.803208:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.803209:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.803211:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.803213:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.803239:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.803242:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.803243:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.803245:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.803247:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.803248:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.803249:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.803251:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.803251:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.803252:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.803255:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.803257:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.803259:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.803260:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.803261:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.803262:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.803262:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.803263:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.803264:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.803274:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.803276:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.803277:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.803278:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.803280:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.803281:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.803304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.803312:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.803314:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.803316:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.803317:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.803319:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.803321:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.803322:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.803324:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.803324:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.803325:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.803328:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.803329:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.803332:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.803333:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.803333:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.803335:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.803335:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.803336:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.803336:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.803341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.803349:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.803352:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.803353:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.803353:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.803355:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.803358:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.803360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.803367:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.803369:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.803370:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.803370:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.803371:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.803372:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.803404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.803422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.803425:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.803427:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677447.803430:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:30.0:1619677447.803430:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.803432:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.803432:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.803434:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.803434:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677447.803435:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.803437:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.803437:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.803438:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677447.803441:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.803441:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.803442:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.803444:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96723318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.803444:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:31.0:1619677447.803445:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.803446:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.803447:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.803450:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.803452:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677447.803452:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:31.0:1619677447.803454:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.803454:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.803455:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.803455:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:22.0:1619677447.803455:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677447.803456:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.803456:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:22.0:1619677447.803456:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.803456:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.803457:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.803457:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.803458:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.803458:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.803458:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.803459:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:22.0:1619677447.803459:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.803460:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (23)++
00000800:00000200:31.0:1619677447.803461:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (24)--
00000800:00000200:31.0:1619677447.803462:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677447.803462:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.803462:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677447.803462:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677447.803474:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.803475:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.803476:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.803477:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:6.0:1619677447.803477:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:31.0:1619677447.803478:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.803478:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.803479:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677447.803479:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677447.803480:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:22.0:1619677447.803480:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:30.0:1619677447.803481:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:22.0:1619677447.803481:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.803481:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:31.0:1619677447.803482:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:22.0:1619677447.803482:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.803482:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677447.803483:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.803483:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:6.0:1619677447.803483:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.803484:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (21)++
00000800:00000200:6.0:1619677447.803484:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677447.803485:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:22.0:1619677447.803485:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.803485:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.803486:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:22.0:1619677447.803489:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.803500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.803506:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.803511:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.803512:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677447.803515:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.803515:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677447.803517:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.803518:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.803519:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677447.803519:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677447.803521:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.803521:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.803523:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.803524:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.803524:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.803525:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677447.803525:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.803527:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.803529:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.803529:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.803530:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.803531:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.803533:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:1.0:1619677447.803535:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.803537:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.803537:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:1.0:1619677447.803537:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.803538:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677447.803540:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677447.803541:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:1.0:1619677447.803541:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.803541:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.803542:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:1.0:1619677447.803542:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.803543:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:1.0:1619677447.803544:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.803545:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.803546:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.803547:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.803548:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677447.803556:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677447.803558:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677447.803559:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.803560:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.803560:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:31.0:1619677447.803561:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.803561:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.803562:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.803563:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.803565:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.803595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:9.2:1619677447.803634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.803642:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.803645:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.803646:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.803646:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.803648:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.803651:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.803782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.803785:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.803787:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.803790:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.803791:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.803792:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.803794:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.803795:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.803796:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.803797:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.803798:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.803802:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.803803:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.803805:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.803806:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.803807:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.803808:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.803809:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.803809:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.803810:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.803822:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.803824:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.803825:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.803825:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.803828:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.803830:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.803874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.803881:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.803884:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.803887:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.803888:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.803890:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.803891:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.803893:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.803895:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.803896:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.803897:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.803900:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.803902:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.803905:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.803906:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.803907:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.803908:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.803909:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.803910:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.803912:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.803912:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677447.803918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:9.2:1619677447.803920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.803921:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.803923:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.803924:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.803926:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:13.0:1619677447.803926:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.803928:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:18.0:1619677447.803928:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:13.0:1619677447.803928:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.803929:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:12.0:1619677447.803929:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677447.803930:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:18.0:1619677447.803931:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.803932:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.803932:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.803935:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.803938:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.803942:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.803945:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.803946:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.803946:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.803947:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.803949:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.803960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.803980:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.803983:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.803983:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.803987:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.803990:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.803991:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.803993:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.803994:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.803995:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.803996:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.804000:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.804001:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.804004:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.804005:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:4.2:1619677447.804005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677447.804006:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.804007:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.804008:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.804008:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.804009:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677447.804009:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.804012:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.804013:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.804013:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.804015:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.804020:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.804022:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.804023:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.804023:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.804023:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.804026:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.804028:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.804039:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.804041:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.804042:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.804043:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.804043:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.804044:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677447.804112:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.804115:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.804117:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.804119:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.804119:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.804121:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:24.0:1619677447.804121:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe9286a00] (21)++
00000400:00000200:31.0:1619677447.804122:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723309] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.804124:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.804124:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.60@o2ib3
00000800:00000200:8.0:1619677447.804124:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe9286a00] (22)--
00000400:00000200:31.0:1619677447.804125:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.804127:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677447.804127:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.60@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.804130:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.804130:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.804131:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:31.0:1619677447.804132:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:24.0:1619677447.804133:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.804134:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.804135:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.804136:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.804136:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:31.0:1619677447.804137:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677447.804138:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:24.0:1619677447.804139:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.60@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.804140:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.804140:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.804141:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677447.804141:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 584 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:31.0:1619677447.804142:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677447.804142:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:24.0:1619677447.804144:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.804145:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:2.0:1619677447.804145:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:24.0:1619677447.804146:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:18.0:1619677447.804146:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:24.0:1619677447.804148:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:24.0:1619677447.804148:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe9286a00] (21)++
00000400:00000200:2.0:1619677447.804148:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.804150:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fe9286a00] (22)++
00000400:00000200:2.0:1619677447.804150:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.804151:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe9286a00] (23)--
00000400:00000200:2.0:1619677447.804151:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:24.0:1619677447.804152:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe9286a00] (22)--
00000800:00000200:22.0:1619677447.804153:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:2.0:1619677447.804153:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96723307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677447.804154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000400:00000200:2.0:1619677447.804154:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.804155:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.804156:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:2.0:1619677447.804156:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:22.0:1619677447.804157:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.804157:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.804160:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.804161:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.804163:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.804163:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.804164:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:8.0:1619677447.804165:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe9286a00] (22)++
00000800:00000200:5.0:1619677447.804166:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:2.0:1619677447.804166:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.804167:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fe9286a00] (23)--
00000400:00000200:5.0:1619677447.804167:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.804167:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:8.0:1619677447.804168:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe9286a00] (22)--
00000800:00000200:2.0:1619677447.804168:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677447.804169:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:8.0:1619677447.804169:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe9286a00] (21)--
00000400:00000200:5.0:1619677447.804169:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.804169:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.804170:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.804171:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.804172:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.804173:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.804184:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.804186:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.804187:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677447.804187:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:18.0:1619677447.804188:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677447.804297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:4.2:1619677447.804303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677447.804305:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.804308:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.804310:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.804313:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.804313:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.804314:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.804316:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.804317:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.804318:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.804319:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.804323:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.804325:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.804328:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.804329:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.804330:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.804331:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.804332:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.804332:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.804333:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.804345:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.804348:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.804349:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.804350:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.804352:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.804353:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.804367:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.804370:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.804371:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.804373:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.804376:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.804378:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.804379:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.804381:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.804382:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.804383:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.804386:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.804389:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.804392:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.804392:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.804393:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.804395:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.804395:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.804396:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.804396:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.804402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.804410:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.804412:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.804413:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.804415:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.804418:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.804419:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677447.804419:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.804421:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.804422:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.804424:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.804427:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.804428:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.804430:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.804431:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.804432:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.804433:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.804437:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.804438:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.804441:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.804442:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.804443:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.804445:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.804445:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.804446:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.804447:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.804457:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.804460:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.804461:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.804463:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.804464:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.804465:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677447.804593:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.804595:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.804596:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.804598:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.804600:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.804602:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677447.804604:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.804605:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.804606:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.804608:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677447.804611:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.804614:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677447.804617:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.804617:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677447.804618:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677447.804620:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677447.804620:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.804621:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.804622:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677447.804634:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.804637:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.804638:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.804638:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.804639:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:6.0:1619677447.804640:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:13.0:1619677447.804642:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:6.0:1619677447.804642:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:13.0:1619677447.804643:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.804644:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.804646:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.804647:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.804670:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.804672:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.804673:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.804675:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.804678:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.804679:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.804681:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.804682:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.804683:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.804684:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677447.804686:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:4.2:1619677447.804686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:24.0:1619677447.804687:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.804689:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:24.0:1619677447.804690:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677447.804690:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.804692:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.804694:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677447.804694:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.804694:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.804695:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.804695:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.804696:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:17.0:1619677447.804696:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.804697:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.804698:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:8.0:1619677447.804698:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.804699:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:17.0:1619677447.804699:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.804700:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000400:00000200:17.0:1619677447.804700:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.804701:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.804702:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.804702:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.804702:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.804703:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.804704:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:17.0:1619677447.804705:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.804706:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.804709:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677447.804709:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:17.0:1619677447.804710:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.804711:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:8.0:1619677447.804711:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96723297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.804713:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677447.804713:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.804713:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.804713:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677447.804713:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.804714:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677447.804714:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.804715:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:24.0:1619677447.804716:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:20.0:1619677447.804716:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.804716:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.804717:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.804718:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:20.0:1619677447.804718:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.804719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.0:1619677447.804719:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.804720:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677447.804721:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:8.0:1619677447.804721:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.804722:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.804723:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.804723:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.804724:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.804726:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.804727:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.804727:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.804728:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.804729:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.804730:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.0:1619677447.804730:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.804731:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.804732:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.804732:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.804736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.804744:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.804747:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.804748:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.804748:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.804750:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.804753:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.804840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.804848:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.804851:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.804853:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.804855:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.804856:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.804857:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.804859:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.804860:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.804861:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.804862:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.804865:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.804866:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.804870:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.804871:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.804871:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.804873:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.804874:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.804875:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.804875:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.804879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.804888:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:4.2:1619677447.804888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.804891:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.804892:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.804893:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.804895:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.804896:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677447.804933:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.804936:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.804937:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.804937:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.804939:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.804942:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:17.0:1619677447.805021:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.805024:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.805025:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.805026:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.805029:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.805031:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.805032:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.805033:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.805034:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.805035:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.805038:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.805040:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.805042:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.805043:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.805044:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.805045:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.805045:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.805046:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.805047:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.805048:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677447.805051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.805059:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.805060:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.805060:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.805060:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.805061:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.805061:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.805062:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.805063:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.805064:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.805064:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677447.805181:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.805184:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.805185:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.805187:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.805189:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.805190:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.805192:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.805193:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.805194:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.805195:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.805199:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.805201:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.805204:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.805205:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.805206:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.805208:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.805209:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.805210:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.805210:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.805234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.805242:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.805244:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.805245:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.805245:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.805247:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.805250:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.805266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06fbb0600] (20)++
00000800:00000200:4.2:1619677447.805298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06fbb0600] (22)++
00000800:00000200:4.2:1619677447.805316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06fbb0600] (21)++
00000800:00000200:30.0:1619677447.805346:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.805349:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.805351:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.805353:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.805353:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.805355:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.805356:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.805357:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.805358:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.805359:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.805362:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.805364:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.805366:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.805366:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.805367:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.805368:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.805369:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.805370:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.805370:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.805382:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.805384:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.805385:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.805386:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.805388:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.805389:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.805474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.805488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.805490:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.805493:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.805496:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.805496:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.805498:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.805500:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.805502:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.805503:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.805504:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.805505:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.805509:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.805510:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.805513:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.805514:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.805515:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.805518:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.805518:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677447.805519:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677447.805519:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.805520:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677447.805521:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:31.0:1619677447.805522:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.805523:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.805524:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:31.0:1619677447.805525:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.805525:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:30.0:1619677447.805526:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.805527:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:2.0:1619677447.805527:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.805530:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:2.0:1619677447.805530:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.805531:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:12.0:1619677447.805532:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.805533:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.805533:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.805534:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677447.805535:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.805536:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:13.0:1619677447.805536:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677447.805536:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.805536:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:12.0:1619677447.805538:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.805539:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.805540:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.805541:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.805541:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:2.0:1619677447.805541:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.805543:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:2.0:1619677447.805543:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.805546:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.805547:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.805547:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.805548:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.805549:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:13.0:1619677447.805550:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.805550:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:13.0:1619677447.805551:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.805551:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:13.0:1619677447.805552:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.805552:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.805552:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677447.805554:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.805669:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.805672:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.805673:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.805675:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.805677:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.805678:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677447.805679:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.805681:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.805682:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.805683:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677447.805686:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.805688:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677447.805691:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.805692:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677447.805693:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677447.805694:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.805695:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.805695:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.805696:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.805711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:4.2:1619677447.805718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.805721:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.805724:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.805725:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.805726:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:24.0:1619677447.805727:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.805728:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.805731:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.805731:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.805732:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677447.805733:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.805734:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.805736:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.805737:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.805738:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.805739:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.805740:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.805744:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.805746:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.805749:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.805749:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.805750:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.805752:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.805752:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.805753:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.805753:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.805758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.805765:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.805768:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.805769:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.805769:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.805771:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.805774:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677447.805775:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.805776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:18.0:1619677447.805777:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.805778:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.805780:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.805783:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.805783:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:18.0:1619677447.805784:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.805785:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.805786:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:18.0:1619677447.805787:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.805787:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.805787:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:18.0:1619677447.805788:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.805788:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000400:00000200:18.0:1619677447.805789:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677447.805789:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:18.0:1619677447.805792:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.805793:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.805796:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.805797:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.805797:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.805799:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.805799:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.805801:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.805802:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.805802:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677447.805813:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677447.805813:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.805814:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:22.0:1619677447.805815:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677447.805815:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.805816:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:22.0:1619677447.805817:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.805817:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.805818:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.805822:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.805830:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.805832:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.805833:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.805833:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.805834:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.805835:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.805950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.805957:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.805960:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.805962:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.805964:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.805965:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.805966:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.805968:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723284] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.805969:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.805970:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.805971:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.805974:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.805976:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.805979:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.805980:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.805981:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.805982:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.805983:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.805984:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.805984:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.805988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.805996:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.805999:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.806000:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.806002:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.806002:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.806005:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.806034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.806065:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.806067:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.806068:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.806070:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677447.806072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:2.0:1619677447.806073:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.806074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677447.806074:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.806075:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.806077:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.806078:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.806079:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.806080:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.806082:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:2.0:1619677447.806082:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.806083:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.806084:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:20.0:1619677447.806085:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.806086:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.806086:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.806087:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.806088:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:2.0:1619677447.806088:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677447.806090:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.2:1619677447.806091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.806091:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.806091:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.806092:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677447.806098:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.806101:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.806102:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (23)++
00000800:00000200:5.0:1619677447.806102:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (24)--
00000800:00000200:5.0:1619677447.806103:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.806104:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.806107:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.806107:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.806109:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.806110:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.806111:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.806186:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.806188:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.806189:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.806191:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.806193:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.806195:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.806196:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.806198:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.806199:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.806200:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.806203:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.806204:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.806207:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.806208:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.806208:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.806210:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.806210:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.806211:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.806212:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.806224:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.806226:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.806227:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.806227:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.806229:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.806232:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.806361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.806366:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.806369:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.806371:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.806372:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.806374:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.806375:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.806377:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.806378:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.806379:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.806380:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.806383:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.806385:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.806387:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.806388:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.806389:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.806391:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.806391:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.806392:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.806393:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677447.806406:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.806408:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.806409:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:30.0:1619677447.806409:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.806410:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.806411:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677447.806413:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:31.0:1619677447.806414:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.806414:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:31.0:1619677447.806416:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.806416:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.806418:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.806419:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.806420:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.806421:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.806422:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.806426:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.806427:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.806430:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.806431:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.806431:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.806433:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.806434:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.806434:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.806435:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.806457:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.806459:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.806460:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.806462:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.806463:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.806465:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.806488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677447.806531:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.806533:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.806534:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.806534:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:30.0:1619677447.806535:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.806537:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.806537:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.806538:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677447.806539:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:6.0:1619677447.806539:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.806541:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.806543:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.806544:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.806546:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.806547:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.806548:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.806551:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.806553:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.806555:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.806556:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.806556:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.806558:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.806559:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.806560:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.806561:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.806562:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677447.806565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.806572:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:17.0:1619677447.806573:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677447.806574:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.806574:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.806575:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677447.806576:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.806577:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.806577:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.806579:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.806582:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.806654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.806657:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.806659:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.806661:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.806662:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.806664:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.806666:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677447.806667:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723276] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.806668:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.806669:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.806670:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677447.806674:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.806675:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.806678:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.806678:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677447.806679:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.806681:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.806681:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.806682:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.806683:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677447.806687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.806694:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.806696:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.806697:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.806698:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.806700:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.806702:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.806704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.806711:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.806714:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.806714:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.806715:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.806716:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.806716:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.806800:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.806802:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.806803:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.806805:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.806807:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.806809:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.806810:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.806811:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.806812:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.806813:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.806816:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.806818:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.806820:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.806821:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.806821:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.806823:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.806823:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.806824:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.806824:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.806837:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.806839:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.806840:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.806840:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.806843:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.806845:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.806857:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.806859:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.806860:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.806860:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.806861:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.806862:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.806887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.806895:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.806898:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.806900:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.806902:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.806902:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.806904:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.806905:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.806906:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.806907:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.806907:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:31.0:1619677447.806908:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.806910:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.806910:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:31.0:1619677447.806912:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.806913:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.806914:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.806916:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.806916:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.806917:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:31.0:1619677447.806918:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.806918:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:18.0:1619677447.806918:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.806920:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.806920:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:18.0:1619677447.806920:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.806920:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.806922:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:18.0:1619677447.806922:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677447.806923:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.806924:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:18.0:1619677447.806925:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.806927:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.806929:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677447.806930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677447.806930:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.806931:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.806932:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:13.0:1619677447.806932:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:18.0:1619677447.806933:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.806933:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.806934:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677447.806934:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.806935:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.806937:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.806938:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:12.0:1619677447.806938:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.2:1619677447.806938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:13.0:1619677447.806939:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.806940:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.806941:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.806942:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677447.806946:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.806948:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.806949:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.806951:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.806951:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.806952:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677447.806954:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:12.0:1619677447.806956:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.806957:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.806957:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.806958:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.806959:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.806988:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.806991:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.806992:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.806994:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.806996:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.806998:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.806999:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.807000:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.807002:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.807002:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.807006:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.807007:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.807010:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.807011:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.807011:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.807013:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.807014:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.807015:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.807016:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.807016:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:4.2:1619677447.807018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:13.0:1619677447.807026:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677447.807027:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677447.807028:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.807029:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:13.0:1619677447.807029:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:24.0:1619677447.807030:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:18.0:1619677447.807030:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:13.0:1619677447.807030:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.807030:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677447.807032:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:13.0:1619677447.807032:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.807032:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677447.807033:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.807034:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677447.807036:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.807037:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.807038:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.807040:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.807041:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.807042:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.807045:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.807047:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.807050:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.807051:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.807051:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.807053:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.807054:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.807055:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.807056:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.807057:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.807059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.807062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.807067:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.807070:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677447.807070:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:17.0:1619677447.807071:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.807071:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.807072:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677447.807073:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.807073:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.807074:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.807076:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.807104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.807155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.807162:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.807165:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.807166:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.807166:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.807168:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.807172:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.807185:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.807188:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.807188:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.807191:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.807193:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.807194:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.807196:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.807197:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.807198:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.807199:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.807202:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.807205:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.807209:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.807209:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.807209:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.807210:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.807211:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.807212:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.807212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:6.0:1619677447.807212:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.807213:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:22.0:1619677447.807214:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.807214:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677447.807215:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677447.807215:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.807216:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.807217:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677447.807217:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.807220:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.807221:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.807222:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.807223:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677447.807225:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.807226:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.807227:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.807228:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677447.807232:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.807235:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677447.807238:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.807238:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.807239:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.807241:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.807241:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.807243:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.807244:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.807245:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677447.807247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677447.807250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.807255:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677447.807255:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:5.0:1619677447.807257:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.807258:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677447.807258:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.807259:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.807259:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.807260:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677447.807261:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.807264:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.807270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:7.0:1619677447.807303:0:520:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:9.2:1619677447.807327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.807335:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.807338:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.807339:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.807339:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.807341:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.807344:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.807350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.807357:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.807359:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.807360:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.807360:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.807361:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.807362:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.807411:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.807413:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.807414:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.807416:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.807419:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.807420:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.807421:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.807423:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.807424:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.807425:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.807428:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.807430:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.807432:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.807433:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.807434:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.807435:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.807436:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.807437:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.807438:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.807439:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.807449:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677447.807449:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677447.807451:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677447.807451:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:6.0:1619677447.807452:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.807452:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.807453:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.807453:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677447.807454:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.807457:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677447.807583:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.807586:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.807586:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.807588:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.807591:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.807593:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.807594:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.807595:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.807596:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.807597:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.807601:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.807603:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.807605:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.807606:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.807606:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.807608:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.807608:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.807609:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.807609:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.807615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.807620:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.807623:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:17.0:1619677447.807623:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677447.807624:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677447.807625:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:22.0:1619677447.807626:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.807627:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.807627:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:22.0:1619677447.807628:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.807629:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.807630:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.807631:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.807631:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:22.0:1619677447.807633:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.807634:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.807635:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.807639:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.807640:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677447.807640:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677447.807643:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677447.807643:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.807644:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:30.0:1619677447.807645:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.807645:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.807646:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:30.0:1619677447.807647:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.807647:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677447.807648:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677447.807648:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.807648:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:30.0:1619677447.807649:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677447.807650:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.807652:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677447.807652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677447.807653:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.807654:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677447.807657:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.807659:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677447.807659:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:30.0:1619677447.807662:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.807662:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:1.0:1619677447.807662:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:30.0:1619677447.807663:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:17.0:1619677447.807663:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.807663:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.807664:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.807665:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677447.807665:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.807666:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:1.0:1619677447.807667:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677447.807668:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.807668:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.807682:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.807685:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.807686:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.807687:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677447.807736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.807774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.807782:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.807784:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.807786:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.807786:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.807788:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.807791:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677447.807942:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.807945:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.807946:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.807948:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.807951:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.807952:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.807953:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.807955:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.807956:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.807957:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.807960:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.807961:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.807964:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.807964:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.807965:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.807966:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.807967:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.807967:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.807968:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.807979:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.807981:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.807982:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.807984:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.807986:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.807987:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.808070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.808077:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.808079:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.808080:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.808081:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.808085:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.808086:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.808087:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.808089:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.808089:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.808090:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.808093:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.808095:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:18.0:1619677447.808096:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:8.0:1619677447.808097:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.808098:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.808098:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:18.0:1619677447.808099:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.808099:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.0:1619677447.808100:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.808100:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:18.0:1619677447.808101:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.808101:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.808102:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677447.808103:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.808105:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.808106:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677447.808106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:18.0:1619677447.808107:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.808108:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.808109:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.808112:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.808113:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677447.808114:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:18.0:1619677447.808115:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.808116:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.808117:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677447.808117:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:18.0:1619677447.808118:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.808118:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:17.0:1619677447.808118:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.808118:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:18.0:1619677447.808119:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.808120:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:17.0:1619677447.808120:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.808123:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:9.2:1619677447.808125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.808132:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.808134:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.808135:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.808136:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.808138:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.808139:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.808165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.808172:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.808174:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.808175:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.808176:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.808178:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.808181:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.808239:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.808242:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.808242:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.808244:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.808247:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.808248:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.808250:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.808252:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.808253:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.808254:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:4.2:1619677447.808254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:6.0:1619677447.808257:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.808259:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.808262:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.808263:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.808263:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.808265:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.808266:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.808266:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.808267:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677447.808278:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.808280:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.808281:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.808283:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.808285:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.808287:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677447.808297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.808302:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.808305:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.808305:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.808306:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.808308:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.808312:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.808431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.808440:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.808443:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.808445:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.808447:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.808447:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.808449:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677447.808450:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723254] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.808451:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.808452:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.808453:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677447.808457:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.808459:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677447.808461:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.808462:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.808463:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677447.808464:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.808465:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.808466:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.808466:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:1.0:1619677447.808466:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.808468:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.808469:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677447.808471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:1.0:1619677447.808471:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.808473:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.808475:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.808476:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.808478:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.808479:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:1.0:1619677447.808479:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.808480:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.808482:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.808483:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.808483:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.808485:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.808485:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.808485:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.808488:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.808489:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.808489:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.808490:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.808492:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.808493:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.808493:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.808494:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.808505:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.808507:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.808508:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.808510:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.808511:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.808513:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.808521:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.808523:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.808524:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.808525:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.808525:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.808527:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.808583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.808586:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677447.808589:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.808591:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:5.0:1619677447.808592:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.808593:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.808593:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.808594:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.808595:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.808597:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.808597:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.808599:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.808600:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.808601:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.808602:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.808605:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.808607:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.808607:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.808608:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000400:00000200:5.0:1619677447.808609:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:17.0:1619677447.808611:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:5.0:1619677447.808611:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.808612:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677447.808614:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.808614:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.808615:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.808616:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.808617:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.808618:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677447.808619:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677447.808619:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:20.0:1619677447.808620:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677447.808621:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677447.808621:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:17.0:1619677447.808621:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96723250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.808621:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.808622:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:17.0:1619677447.808622:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.808623:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.808624:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.808624:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677447.808624:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.808625:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677447.808625:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.808625:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.808626:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:31.0:1619677447.808627:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.808628:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.808629:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.808629:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.808629:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.808630:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:31.0:1619677447.808631:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:17.0:1619677447.808632:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.808632:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.808632:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:31.0:1619677447.808633:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96723249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.808634:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.808634:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000400:00000200:31.0:1619677447.808635:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.808635:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.808635:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.808635:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.808635:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:31.0:1619677447.808636:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:17.0:1619677447.808636:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.808636:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677447.808638:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677447.808639:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677447.808639:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:6.0:1619677447.808639:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:31.0:1619677447.808640:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.808640:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.2:1619677447.808640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677447.808640:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000400:00000200:6.0:1619677447.808640:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.808642:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.808643:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677447.808643:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:30.0:1619677447.808644:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:9.2:1619677447.808644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677447.808645:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.808645:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:6.0:1619677447.808645:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677447.808646:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677447.808646:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677447.808646:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677447.808647:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:30.0:1619677447.808647:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.808647:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677447.808648:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.808649:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:30.0:1619677447.808649:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.808650:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.808650:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677447.808652:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.808652:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:2.0:1619677447.808652:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.808654:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.808655:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.808656:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.808657:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.808660:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.808664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.808672:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.808675:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.808675:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.808676:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.808677:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.808677:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.808851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.808887:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.808890:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.808891:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.808893:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.808895:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.808896:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:1.0:1619677447.808897:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.808898:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723247] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.808899:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:1.0:1619677447.808899:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.808900:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.808900:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:1.0:1619677447.808900:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.808901:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:22.0:1619677447.808902:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.808905:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:1.0:1619677447.808905:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.808907:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.808910:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.808911:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.808911:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.808913:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.808913:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.808914:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.808915:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677447.808917:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.808920:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.808920:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.808923:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.808925:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.808926:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:13.0:1619677447.808926:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:24.0:1619677447.808928:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.808929:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.808929:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:24.0:1619677447.808930:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677447.808930:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.808932:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:13.0:1619677447.808932:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.808933:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:24.0:1619677447.808935:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.808937:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.808938:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.808939:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.808940:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.808941:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.808942:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.808943:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.808944:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.808944:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.808955:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.808957:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.808958:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.808959:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.808961:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.808962:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.808973:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.808975:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.808976:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.808977:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.808977:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.808978:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.809024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.809032:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:4.2:1619677447.809032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677447.809034:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.809036:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.809038:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.809039:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.809039:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:31.0:1619677447.809040:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:8.0:1619677447.809041:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:31.0:1619677447.809042:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723245] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.809042:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:31.0:1619677447.809044:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.809044:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.809045:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.809046:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.809047:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.809049:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677447.809050:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.809050:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.809051:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.809052:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.809052:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.809053:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.809055:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.809056:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677447.809057:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:8.0:1619677447.809057:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.809059:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677447.809059:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:8.0:1619677447.809059:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.809060:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.809061:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.0:1619677447.809061:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.809062:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.809063:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.2:1619677447.809065:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.809069:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.809069:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.809070:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.809071:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677447.809072:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.809075:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:8.2:1619677447.809075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677447.809076:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.809077:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.809080:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.809081:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.809082:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.2:1619677447.809082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.809083:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677447.809085:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.809086:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.809086:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:1.0:1619677447.809086:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.809087:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:17.0:1619677447.809089:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.809089:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000400:00000200:2.0:1619677447.809090:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.809092:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.809092:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.809093:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677447.809093:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.2:1619677447.809094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.809094:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.809094:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:2.0:1619677447.809094:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677447.809095:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:2.0:1619677447.809095:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.809096:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.809098:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.809098:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677447.809101:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677447.809102:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.809104:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677447.809104:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.809105:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.809105:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677447.809106:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.809106:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.809107:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:1.0:1619677447.809107:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.809108:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.809109:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.809110:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.809111:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.809111:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677447.809123:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.809125:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.809126:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.809126:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.809129:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.809131:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.2:1619677447.809135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677447.809146:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.809148:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677447.809149:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.809151:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.809152:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:5.0:1619677447.809152:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.115@o2ib7 : GET - for me
00000800:00000200:2.0:1619677447.809152:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.809155:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.809155:0:19935:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.115@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:5.0:1619677447.809157:0:19935:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.115@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000400:00000200:18.0:1619677447.809158:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.809158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677447.809159:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:5.0:1619677447.809159:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.809159:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:18.0:1619677447.809160:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.809161:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.809163:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.809165:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.809167:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:5.0:1619677447.809167:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : REPLY try# 0
00000800:00000200:2.0:1619677447.809167:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.809169:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:5.0:1619677447.809170:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.809170:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.809171:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.809173:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.809175:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.809175:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.809176:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.809176:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677447.809177:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.809178:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677447.809179:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.809179:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:18.0:1619677447.809180:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:18.0:1619677447.809180:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (24)++
00000800:00000200:5.0:1619677447.809180:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:18.0:1619677447.809183:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:18.0:1619677447.809183:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.809184:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.809184:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.809185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.809193:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)++
00000800:00000200:20.0:1619677447.809196:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (24)--
00000400:00000200:20.0:1619677447.809196:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.809198:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: REPLY: OK
00000800:00000200:5.0:1619677447.809199:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)++
00000800:00000200:5.0:1619677447.809200:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (24)--
00000800:00000200:20.0:1619677447.809201:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.809201:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.809202:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.809202:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.809205:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677447.809225:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.809227:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.809228:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.809230:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.809233:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.809234:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.809235:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.809237:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.809238:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.809239:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.809242:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.809244:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.809247:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.809248:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.809249:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.809251:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.809251:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.809252:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.809252:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.809260:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.809263:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:12.0:1619677447.809263:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:30.0:1619677447.809265:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.809266:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:31.0:1619677447.809267:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:12.0:1619677447.809267:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.809268:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.809268:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.809269:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677447.809270:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.809270:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.809271:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:30.0:1619677447.809272:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.809273:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677447.809273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:30.0:1619677447.809274:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.809276:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677447.809277:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.809278:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:30.0:1619677447.809280:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.809281:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.809281:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677447.809283:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.809283:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.809284:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677447.809285:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:20.0:1619677447.809285:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677447.809286:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.809287:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677447.809287:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.809288:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677447.809288:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.809289:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:20.0:1619677447.809289:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.809290:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.809299:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.809300:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.809301:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.809303:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.809303:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.809304:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.809304:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.809305:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.809305:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677447.809306:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.809307:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.809308:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677447.809309:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.809309:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.809309:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.809320:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.809324:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.809325:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.809325:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:13.0:1619677447.809326:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.809327:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.809328:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:12.0:1619677447.809328:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.809329:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.809329:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.809330:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.809331:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.809454:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.809457:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.809458:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.809460:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.809463:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.809464:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.809465:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.809467:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.809467:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.809468:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.809472:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.809473:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.809476:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677447.809477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677447.809477:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.809478:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.809479:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.809480:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.809481:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.809482:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.809482:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677447.809485:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.809488:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.809490:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.809492:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:31.0:1619677447.809493:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.809493:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677447.809493:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677447.809494:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:31.0:1619677447.809495:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:17.0:1619677447.809495:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:31.0:1619677447.809496:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.809496:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677447.809496:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.809496:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677447.809497:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677447.809497:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.809498:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.809498:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:31.0:1619677447.809499:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:12.0:1619677447.809500:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:31.0:1619677447.809503:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.809505:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.809507:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.809508:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.809509:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.809510:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.809511:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.809512:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.809512:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677447.809532:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.809535:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.809536:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.809537:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.809539:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.809540:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:17.0:1619677447.809861:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.809864:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.809865:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.809867:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.809870:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.809871:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.809872:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.809874:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.809875:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.809876:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.809879:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.809880:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.809883:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.809884:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.809884:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.809886:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.809887:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.809887:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.809888:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.809899:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.809901:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.809902:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.809904:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.809905:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.809907:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.809918:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.809921:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.809921:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.809922:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.809922:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.809923:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.809924:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.809926:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.809927:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.809929:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.809931:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.809933:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677447.809934:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.809935:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.809936:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.809937:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677447.809940:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.809942:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.809944:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.809945:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677447.809945:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677447.809947:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.809947:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.809948:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.809948:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.809958:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.809961:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.809962:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.809963:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.809965:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.809966:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.809974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.809979:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.809982:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.809984:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.809984:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.809987:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.809989:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.809990:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.809991:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.809993:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.809993:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.809997:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.809998:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.810001:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.810002:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.810002:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.810004:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.810005:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.810006:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.810006:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.810011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.810018:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.810021:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.810022:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.810022:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.810023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677447.810024:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.810027:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.810069:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.810071:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.810072:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.810073:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.810075:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.810078:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677447.810081:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.810084:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.810086:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.810088:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.810088:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677447.810089:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:31.0:1619677447.810090:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.810091:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.810092:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.810092:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677447.810093:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.810093:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000400:00000200:31.0:1619677447.810094:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.810094:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.810094:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:31.0:1619677447.810095:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.810099:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.810101:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.810103:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.810104:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.810104:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.810106:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.810106:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.810107:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.810108:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.810117:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.810120:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.810121:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.810122:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.810124:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.810125:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.810387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.810425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.810432:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.810435:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.810436:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.810437:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.810439:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.810441:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.810442:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.810443:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.810444:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.810445:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.810448:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.810449:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.810451:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677447.810452:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.810453:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.810453:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.810454:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.810455:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677447.810455:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.810455:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.810456:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.810457:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.810457:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.810459:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.810461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:1.0:1619677447.810461:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.810462:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.810464:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.810465:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.810467:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.810470:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.810471:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.810471:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.810472:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.810473:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.810474:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.810477:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.810477:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.810478:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.810478:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.810479:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.810481:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.810482:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.810483:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.810483:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.810494:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.810497:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.810498:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.810499:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.810501:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.810502:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.810588:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.810591:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.810591:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.810592:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.810594:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.810597:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.810758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.810780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:4.2:1619677447.810795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.2:1619677447.810809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.810812:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.810814:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.810817:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677447.810817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:20.0:1619677447.810819:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.810820:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.810821:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677447.810821:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:20.0:1619677447.810822:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.810823:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.810824:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.810824:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.810825:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.810826:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.810826:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.810829:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:20.0:1619677447.810830:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.810830:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.810832:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.810835:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.810836:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.810837:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.810839:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.810839:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.810841:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.810842:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.810843:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.810848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677447.810851:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.810854:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.810855:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.810855:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:12.0:1619677447.810856:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.810857:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.810857:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:6.0:1619677447.810858:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677447.810858:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677447.810859:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.810859:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.810860:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.810861:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.810863:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.810865:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.810867:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.810868:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.810870:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.810871:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.810872:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.810875:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.810877:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.810880:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.810881:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.810881:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.810883:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.810884:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.810885:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.810886:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.810886:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677447.810890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.810898:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:5.0:1619677447.810898:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:22.0:1619677447.810900:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.810901:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.0:1619677447.810901:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.810901:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.810902:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677447.810903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:5.0:1619677447.810903:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.810906:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.810906:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677447.810912:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.810915:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.810917:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.810919:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.810919:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.810921:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.810922:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.810924:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.810924:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.810926:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.810929:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.810931:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.810934:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.810935:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.810936:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.810937:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.810938:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.810939:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.810941:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.810941:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677447.810943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677447.810947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677447.810951:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.810953:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.810954:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.810955:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:5.0:1619677447.810955:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.810956:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.810958:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.810958:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677447.810958:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677447.810960:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.810960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677447.810967:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.810970:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.810971:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.810972:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.810973:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.810973:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.811123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.811249:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.811251:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.811252:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.811255:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.811258:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.811259:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.811261:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.811262:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.811263:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.811264:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.811268:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.811270:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.811272:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.811273:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677447.811273:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.811273:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.811275:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.811275:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677447.811276:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677447.811276:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.811277:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677447.811278:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.811278:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677447.811280:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677447.811280:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:31.0:1619677447.811281:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.811281:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.811283:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:13.0:1619677447.811283:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.811284:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96723217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.811284:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:31.0:1619677447.811286:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.811286:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.811286:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:31.0:1619677447.811288:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677447.811288:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:31.0:1619677447.811291:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.811291:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.811292:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.811292:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.811299:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:6.0:1619677447.811301:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.811302:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.811303:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.811303:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677447.811303:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677447.811305:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.811305:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.811307:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.811308:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.811309:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677447.811318:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.811321:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.811322:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.811323:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:12.0:1619677447.811323:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.811325:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.811326:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:13.0:1619677447.811326:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.811327:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.811327:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.811336:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.811339:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.811340:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.811340:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.811341:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.811342:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.811509:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.811512:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.811512:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.811514:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.811517:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.811518:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677447.811519:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.811521:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.811522:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.811523:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677447.811526:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.811527:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.811529:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.811530:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.811530:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.811532:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.811533:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.811533:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.811534:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.811538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.811545:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.811547:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.811548:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.811549:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.811550:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:4.2:1619677447.811551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.811552:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:8.2:1619677447.811590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.811598:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.811601:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.811602:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.811603:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.811604:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.811606:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.811676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.811684:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.811687:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:20.2:1619677447.811687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:8.0:1619677447.811688:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.811690:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.811692:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.811692:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:24.0:1619677447.811694:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:5.0:1619677447.811695:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:24.0:1619677447.811696:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.811697:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.811698:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.811698:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677447.811699:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.811700:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677447.811700:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.811702:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677447.811704:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.811705:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.811706:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.811707:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:5.0:1619677447.811707:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.811708:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.811711:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.811712:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.811712:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:5.0:1619677447.811713:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677447.811714:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.811714:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.0:1619677447.811715:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.811716:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.811717:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.811718:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677447.811718:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.811718:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.811719:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.2:1619677447.811720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.811721:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.811721:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.811722:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.811723:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:4.2:1619677447.811723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.811728:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:8.2:1619677447.811730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.811731:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.811732:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.811734:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.811734:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:1.0:1619677447.811734:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.811736:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677447.811736:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677447.811737:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677447.811737:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.811738:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677447.811738:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.811738:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:17.0:1619677447.811739:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:8.2:1619677447.811740:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.811741:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.811742:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.811742:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.811745:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.811746:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.811747:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.811747:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.811748:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.811756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.811764:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677447.811764:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.811766:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.811767:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677447.811768:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.811770:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.811770:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.811771:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:17.0:1619677447.811772:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.811773:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.811773:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677447.811774:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677447.811774:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.811776:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.811776:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.811777:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.811779:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.811779:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677447.811781:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.811782:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.811782:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:17.0:1619677447.811784:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.811785:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.811785:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:17.0:1619677447.811787:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:30.0:1619677447.811788:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.811788:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.811790:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.811791:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.811791:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.811791:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000400:00000200:18.0:1619677447.811792:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:17.0:1619677447.811792:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:18.0:1619677447.811794:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96723210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.811794:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.811794:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677447.811795:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.811795:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:17.0:1619677447.811795:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:30.0:1619677447.811796:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:18.0:1619677447.811796:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.811796:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000400:00000200:18.0:1619677447.811797:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.811797:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677447.811798:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:18.0:1619677447.811798:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:30.0:1619677447.811799:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.811800:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.811800:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:9.2:1619677447.811800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:18.0:1619677447.811802:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.811803:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.2:1619677447.811805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.811806:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.811807:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:1.0:1619677447.811807:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:18.0:1619677447.811808:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:1.0:1619677447.811809:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:18.0:1619677447.811810:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.811810:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677447.811810:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677447.811810:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:1.0:1619677447.811810:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677447.811811:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.811812:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:20.0:1619677447.811813:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677447.811813:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:8.0:1619677447.811813:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.811814:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.811814:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.811815:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.811816:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.811817:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.811818:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.811820:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.0:1619677447.811820:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.811822:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:9.2:1619677447.811822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:2.0:1619677447.811823:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.811825:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.811826:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.811827:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:24.0:1619677447.811828:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.811831:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.811832:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.811832:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.811833:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.811834:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677447.811914:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.811917:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.811918:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.811920:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.811922:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.811923:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.811925:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.811926:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.811927:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.811928:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.811932:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.811934:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.811936:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.811937:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.811938:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.811940:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.811940:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.811941:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.811942:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.811946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.811954:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.811956:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.811957:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.811959:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.811959:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.811961:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.812049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.812052:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.812054:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.812056:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.812057:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.812059:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.812061:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.812062:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.812063:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.812064:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.812065:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.812069:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.812071:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.812073:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.812074:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.812075:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.812077:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.812077:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.812078:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.812079:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677447.812089:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.812092:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.812093:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.812095:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.812096:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:18.0:1619677447.812097:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677447.812099:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677447.812100:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.812100:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.812103:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.812106:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.812107:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.812108:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.812110:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.812111:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.812112:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.812115:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.812117:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.812120:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.812121:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.812122:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.812123:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.812124:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.812125:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.812126:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.812126:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:12.0:1619677447.812136:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.812137:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:12.0:1619677447.812139:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677447.812139:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:12.0:1619677447.812140:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.812140:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.812141:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:12.0:1619677447.812142:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.812143:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.812144:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.812298:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.812301:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:4.2:1619677447.812301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:30.0:1619677447.812303:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.812305:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.812306:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.812307:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.812308:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.812309:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.812310:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.812312:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.812315:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.812317:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.812319:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.812320:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.812320:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.812322:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.812326:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.812327:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.812327:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677447.812345:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.812347:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.812348:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.812349:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.812351:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.812351:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:13.0:1619677447.812362:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.812364:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.812365:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.812366:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.812368:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.812369:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.812406:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.812408:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.812409:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.812411:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.812414:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.812415:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.812417:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.812418:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.812419:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.812420:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677447.812424:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.812425:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.812428:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.812429:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.812430:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.812431:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.812432:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.812433:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.812433:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.812437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.812444:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.812447:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.812448:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.812448:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.812450:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.812453:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677447.812484:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.812487:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.812487:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.812490:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.812492:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.812493:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.812494:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.812496:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.812497:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.812498:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.812501:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.812502:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.812505:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.812506:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.812507:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.812509:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.812509:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.812510:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.812511:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.812512:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.2:1619677447.812515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.812524:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.0:1619677447.812525:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:5.0:1619677447.812526:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:8.0:1619677447.812527:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:5.0:1619677447.812527:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.812528:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.812529:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:5.0:1619677447.812529:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.812530:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.812532:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.812555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.812562:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.812565:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.812567:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.812569:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.812569:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.812571:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.812572:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.812573:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.812575:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.812576:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.812579:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.812581:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.812584:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.812585:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.812585:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.812587:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.812588:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.812588:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.812589:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.812593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.812601:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.812604:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.812605:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.812606:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.812607:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.812609:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:6.0:1619677447.812635:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.812638:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.812639:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.812641:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.812643:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.812644:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.812646:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.812647:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.812649:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.812649:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.812653:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.812655:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.812658:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.812659:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.812660:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.812662:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.812662:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.812663:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.812663:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677447.812675:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.812678:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.812679:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.812681:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.812681:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.812684:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.812693:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.812696:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.812697:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.812698:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.812698:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.812699:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677447.812711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.812741:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.812743:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.812744:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.812747:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.812749:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.812751:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.812752:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.812752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:18.0:1619677447.812753:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.812754:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.812755:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:18.0:1619677447.812755:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677447.812758:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.812759:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.812759:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.812760:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.812761:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.812762:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.812764:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677447.812764:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.812765:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.812765:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.812767:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.812768:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.812769:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.812769:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.812773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.812781:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.812783:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.812784:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.812785:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.812787:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.812790:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.812791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.812798:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.812801:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.812801:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677447.812802:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:24.0:1619677447.812803:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.812804:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.812805:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.812806:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.812806:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.812807:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:12.0:1619677447.812808:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677447.812809:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.812809:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677447.812810:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.812811:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.812812:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.812815:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.812816:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.812819:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.812820:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.812820:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.812822:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.812822:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.812823:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.812824:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.812828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.812836:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.812839:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.812839:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.812840:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.812842:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.812845:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:31.0:1619677447.812912:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677447.812915:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677447.812917:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.812919:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677447.812920:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.812922:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677447.812923:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.812924:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.812925:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.812926:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677447.812929:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.812931:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677447.812934:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.812935:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677447.812935:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677447.812937:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677447.812938:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677447.812939:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677447.812939:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677447.812944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.812951:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.812954:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.812955:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.812955:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.812957:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.812960:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:18.0:1619677447.812995:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.812997:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.812998:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.813000:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.813002:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.813004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:18.0:1619677447.813004:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.813005:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.813006:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.813007:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.813008:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.813011:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.813013:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.813016:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.813017:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677447.813017:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.813018:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677447.813020:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:18.0:1619677447.813020:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.813021:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.813021:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.813022:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:30.0:1619677447.813023:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.813024:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.813025:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.813027:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.813028:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.813030:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.813031:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.813032:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.813036:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.813038:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.813041:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.813041:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677447.813042:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:30.0:1619677447.813044:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:30.0:1619677447.813045:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.813046:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.813047:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677447.813052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:12.0:1619677447.813054:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677447.813056:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677447.813057:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.813059:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677447.813060:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.813061:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677447.813061:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677447.813062:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.813062:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.2:1619677447.813063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:13.0:1619677447.813064:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.813066:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.813069:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.813071:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.813073:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.813074:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.813076:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.813077:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677447.813078:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677447.813079:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.813080:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.813081:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.813081:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:5.0:1619677447.813081:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.813082:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677447.813082:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.813083:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677447.813083:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677447.813086:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.813088:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.813091:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.813091:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677447.813092:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.813094:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.813095:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.813096:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.813096:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677447.813100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.813107:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.813110:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.813111:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.813111:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.813113:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.813116:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.813144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.813167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.813175:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.813177:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.813179:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.813182:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.813182:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.813183:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:30.0:1619677447.813183:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:31.0:1619677447.813184:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.813184:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.813185:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.813186:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.813186:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.813187:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677447.813187:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.813189:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.813190:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:31.0:1619677447.813191:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.813192:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677447.813196:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.813196:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677447.813197:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.813198:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677447.813199:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677447.813200:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.813200:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:2.0:1619677447.813200:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:31.0:1619677447.813201:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:18.0:1619677447.813202:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.813203:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677447.813205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:2.0:1619677447.813206:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.813208:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.813209:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.813210:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.813211:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.813212:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.813213:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.813215:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:2.0:1619677447.813215:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.813216:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.813217:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.813218:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.813218:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.813219:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.813220:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.813221:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:2.0:1619677447.813221:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.2:1619677447.813223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.813223:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.813223:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.813224:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.813225:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.813229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.813230:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.813233:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.813234:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.813235:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.813235:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.813236:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677447.813236:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.813238:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.813239:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.813240:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.813241:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.813243:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.813386:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.813389:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.813390:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.813392:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.813394:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.813396:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.813397:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.813399:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.813400:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.813401:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.813404:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.813406:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.813408:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.813409:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.813410:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.813411:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.813412:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.813412:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.813413:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.813418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.813426:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.813429:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.813430:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.813430:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.813432:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.813435:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.813511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.813518:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.813521:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.813522:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.813523:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.813526:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.813528:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.813529:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.813531:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.813532:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.813533:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.813536:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.813537:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.813540:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.813541:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.813542:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.813544:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.813544:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.813545:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.813545:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.813550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.813558:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.813560:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.813561:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.813562:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.813564:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.813566:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.813567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:13.0:1619677447.813570:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.813572:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.813573:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.813575:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.813575:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677447.813577:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.813577:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.813578:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677447.813578:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.813579:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.813580:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.813580:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.813602:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.813604:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.813605:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.813607:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.813609:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.813611:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.813612:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.813613:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.813614:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.813615:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.813618:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.813620:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.813622:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.813623:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.813624:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.813625:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.813625:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.813626:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.813627:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.813631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.813638:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.813639:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.813640:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.813641:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.813642:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.813643:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.813696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.813734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.813742:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.813744:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.813746:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.813746:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.813748:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.813750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.813753:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.813754:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.813756:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.813757:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.813758:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.813758:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.813758:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.813790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.813799:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.813802:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.813804:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.813807:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.813807:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.813808:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.813809:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.813811:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.813812:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.813813:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.813816:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.813818:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.813820:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.813820:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.813821:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.813823:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.813823:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677447.813824:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:30.0:1619677447.813825:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.813826:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:18.0:1619677447.813826:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:30.0:1619677447.813827:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:2.0:1619677447.813827:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.813829:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.813831:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.813832:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:20.2:1619677447.813834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:18.0:1619677447.813834:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.813835:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.813835:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:18.0:1619677447.813836:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.813837:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:13.0:1619677447.813837:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.813838:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.813840:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677447.813841:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677447.813841:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.813841:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.813842:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677447.813842:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.813843:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.813843:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:13.0:1619677447.813843:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677447.813844:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:31.0:1619677447.813845:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.813845:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.813846:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:18.0:1619677447.813846:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.813847:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.813847:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.813847:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677447.813849:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:18.0:1619677447.813849:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:31.0:1619677447.813850:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.813850:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:31.0:1619677447.813851:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.813851:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.813851:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:31.0:1619677447.813852:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.813853:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.813854:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:9.2:1619677447.813855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677447.813857:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.813857:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.813858:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:31.0:1619677447.813859:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.813859:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.813860:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.813861:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677447.813862:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.813862:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677447.813863:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.813863:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.813864:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:31.0:1619677447.813865:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.813865:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:2.0:1619677447.813865:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.813866:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677447.813866:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677447.813867:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:2.0:1619677447.813867:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.813870:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.813872:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (21)++
00000800:00000200:2.0:1619677447.813873:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.5.5@o2ib3
00000800:00000200:18.0:1619677447.813874:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000400:00000200:2.0:1619677447.813876:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.813877:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.813879:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677447.813879:0:19932:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 11720
00000400:00000200:13.0:1619677447.813880:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.813881:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919065782400] (21)++
00000400:00000200:13.0:1619677447.813882:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.813882:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (22)++
00000800:00000200:2.0:1619677447.813882:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (23)--
00000800:00000200:12.0:1619677447.813883:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.813883:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000800:00000200:13.0:1619677447.813885:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.813893:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (22)++
00000800:00000200:18.0:1619677447.813895:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (23)--
00000800:00000200:18.0:1619677447.813896:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000800:00000200:2.0:1619677447.813916:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (22)++
00000800:00000200:2.0:1619677447.813918:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.5.5@o2ib3
00000800:00000200:18.0:1619677447.813919:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (23)--
00000800:00000200:2.0:1619677447.813919:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919065782400] (22)--
00000400:00000200:2.0:1619677447.813920:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.813922:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.813923:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.813925:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.813926:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.813927:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.813929:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.813931:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 11720 bytes in 256 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.813933:0:19932:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 11720
00000800:00000200:2.0:1619677447.813935:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.813935:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.813936:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.813938:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.813938:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (21)++
00000800:00000200:2.0:1619677447.813939:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (22)--
00000800:00000200:2.0:1619677447.813939:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (21)--
00000800:00000200:8.2:1619677447.813944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.813952:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.813955:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.813956:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.2:1619677447.813962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.813970:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.813973:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[0] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.813974:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (23)++
00000800:00000200:20.0:1619677447.813976:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (24)--
00000800:00000200:5.0:1619677447.813977:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.813977:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.2:1619677447.813987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.813994:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.813997:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.813997:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.813999:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.814000:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.814002:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.814039:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.814041:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.814042:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.814044:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.814045:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.814047:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.814048:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.814050:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.814051:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.814052:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.814055:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.814057:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.814060:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.814061:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.814062:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.814064:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.814064:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.814065:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.814066:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.814076:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.814079:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.814080:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.814082:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.814083:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.814085:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.814086:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.814089:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.814089:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.814092:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.814094:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.814095:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:13.0:1619677447.814096:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677447.814097:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.814098:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.814099:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.814100:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:13.0:1619677447.814100:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.814101:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.814101:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.814102:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:18.0:1619677447.814103:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.814103:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.814105:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.814108:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.814108:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.814109:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.814111:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.814111:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.814112:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.814113:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.814117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.814125:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.814127:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.814128:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.814130:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.814130:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.814132:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677447.814135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677447.814142:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.814144:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.814145:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.814146:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.814147:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.814148:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.814273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.814276:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.814278:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.814280:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.814281:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.814283:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.814284:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677447.814285:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.814286:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.814288:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.814289:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677447.814292:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.814299:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.814301:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.814302:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677447.814303:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.814305:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.814305:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.814306:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.814307:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677447.814311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.814319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:1.0:1619677447.814319:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.814322:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.814323:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.814323:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.814325:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:24.0:1619677447.814326:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.814327:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:24.0:1619677447.814329:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.814330:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.814332:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.814334:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.814336:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.814338:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.814339:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.814340:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.814341:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.814344:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.814345:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.814348:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.814349:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.814350:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.814350:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677447.814352:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.814352:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.814352:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.814353:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.814353:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.814354:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.814356:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677447.814357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:6.0:1619677447.814359:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.814360:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.814361:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.814363:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.814364:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.814365:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:6.0:1619677447.814365:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:17.0:1619677447.814367:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:6.0:1619677447.814368:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.814368:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.814369:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.814371:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.814371:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.814373:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.814374:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677447.814374:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.814374:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.814376:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.814376:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.814378:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.814379:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.814379:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.814383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:1.0:1619677447.814389:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677447.814390:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677447.814391:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677447.814392:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:1.0:1619677447.814392:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:22.0:1619677447.814393:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677447.814393:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.814394:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.814394:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:1.0:1619677447.814394:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.814395:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.814395:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.814397:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.814397:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.814399:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:2.0:1619677447.814400:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:1.0:1619677447.814400:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.814401:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.814402:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.814403:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:9.2:1619677447.814405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000400:00000200:1.0:1619677447.814407:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.814408:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.814411:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.814412:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:1.0:1619677447.814412:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.814412:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.814414:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677447.814415:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.814415:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:1.0:1619677447.814415:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.814416:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.814417:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.814417:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:1.0:1619677447.814417:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.814418:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.814418:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.814428:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677447.814428:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677447.814431:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677447.814431:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.814432:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.814432:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677447.814432:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.814433:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677447.814434:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.814437:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.814450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.814459:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.814462:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.814464:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.814466:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.814467:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.814468:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677447.814469:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.814471:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.814471:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.814473:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677447.814476:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.814478:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.814480:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.814480:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.814481:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.814483:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.814483:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.814484:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.814485:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677447.814495:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.814498:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.814498:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.814500:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.814502:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.814503:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677447.814557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.0:1619677447.814561:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677447.814563:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677447.814564:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.814565:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677447.814566:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.814568:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:8.0:1619677447.814568:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.814569:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.814571:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.814571:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.814572:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:5.0:1619677447.814573:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.814575:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.814575:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.814577:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.814577:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.814578:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.814579:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.814580:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:5.0:1619677447.814580:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.814584:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.814585:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.814588:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677447.814588:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.814589:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.814590:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677447.814590:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.814591:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.814592:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.814592:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.814593:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677447.814593:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.814594:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.814595:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.814597:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.814598:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.814598:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.814599:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.814607:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.814608:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.814609:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.814611:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:12.0:1619677447.814611:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.814612:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.814613:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.814613:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.814614:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:13.0:1619677447.814615:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.814615:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.814617:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.814685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.814731:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.814734:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.814735:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.814735:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.814737:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.814740:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.814740:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.814743:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.814744:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.814746:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.814748:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.814750:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.814751:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.814752:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.814753:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.814754:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.814757:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.814759:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.814761:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.814761:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.814762:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.814763:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.814764:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.814765:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.814765:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.814769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.814776:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.814779:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.814779:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.814780:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.814782:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.814783:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.814785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.814793:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.814795:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.814796:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.814797:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.814797:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677447.814798:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:17.0:1619677447.814798:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677447.814801:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.814801:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.814803:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.814804:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.814806:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:2.0:1619677447.814900:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.814903:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.814904:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.814906:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.814909:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.814910:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.814911:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723166] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.814912:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.814913:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.814914:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.814918:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.814920:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.814922:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.814922:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.814923:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.814925:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.814925:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.814926:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.814926:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.814931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.814938:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.814940:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.814940:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.814941:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.814942:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.814943:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.815005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.815008:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.815010:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.815013:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.815013:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.815015:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.815017:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.815018:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.815020:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.815021:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.815022:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.815025:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.815026:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.815030:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.815030:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.815031:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.815033:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.815034:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.815035:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.815036:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.815037:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.815042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677447.815046:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.815048:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677447.815048:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:12.0:1619677447.815049:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.815050:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.815050:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:12.0:1619677447.815051:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.815051:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.815052:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.815054:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.815077:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.815080:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.815082:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.815083:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.815084:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.815086:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.815087:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.815088:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.815089:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.815090:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.815093:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.815095:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.815097:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.815098:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.815099:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.815101:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.815101:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.815103:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.815105:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.815105:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677447.815112:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.815115:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.815116:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.815117:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.815118:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677447.815119:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.815119:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.815121:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.815121:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.815122:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677447.815132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.815140:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.815142:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.815143:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.815145:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.815147:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.815149:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.815150:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.815151:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.815152:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.815153:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.815156:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.815158:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.815161:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.815161:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.815162:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.815163:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.815164:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.815165:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.815166:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.815167:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.815170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677447.815172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.815179:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.815180:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:8.0:1619677447.815181:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.815181:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677447.815182:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677447.815182:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.815182:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.815183:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.815184:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.815185:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.815187:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.815188:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.815188:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.815189:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:17.0:1619677447.815190:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:1.0:1619677447.815190:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.815192:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.815193:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.815194:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.815195:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.815198:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.815200:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.815202:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.815203:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.815204:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.815205:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.815206:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.815206:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.815207:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.815219:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.815221:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.815222:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.815222:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.815224:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.815226:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.815236:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.815238:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.815239:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.815239:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.815240:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.815241:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:7.0:1619677447.815300:0:520:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:20.2:1619677447.815486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.815492:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.815494:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.815497:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.815497:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.815499:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.815501:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.815502:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.815504:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.815505:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.815506:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.815514:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.815516:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.815518:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.815519:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.815520:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.815521:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.815522:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.815523:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.815523:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.815536:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.815539:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.815540:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.815540:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.815542:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.815544:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.815555:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.815557:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.815558:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.815560:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.815562:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.815563:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677447.815572:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.815575:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.815576:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.815578:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.815580:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.815582:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.815583:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.815584:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.815585:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.815587:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.815590:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.815591:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.815593:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.815594:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.815595:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.815596:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.815597:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.815597:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.815598:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.815603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.815611:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.815613:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.815613:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.815614:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.815615:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.815616:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.815621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.815628:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.815630:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.815631:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.815631:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.815632:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.815632:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677447.815715:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.815718:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.815720:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.815722:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.815722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677447.815723:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.815724:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677447.815725:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.815726:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.815727:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.815728:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677447.815733:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.815735:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.815738:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.815739:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677447.815740:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677447.815741:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.815742:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.815743:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.815743:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677447.815755:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.815758:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.815758:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.815759:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.815759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677447.815761:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.815764:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.815868:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.815871:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.815872:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.815874:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.815876:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.815878:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677447.815879:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723156] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.815880:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.815881:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.815882:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677447.815886:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.815888:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.815891:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.815892:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677447.815893:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677447.815894:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.815895:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.815895:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.815896:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677447.815907:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.815910:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.815911:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.815913:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.815914:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.815915:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.815928:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.815930:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.815931:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.815933:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.815935:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.815937:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.815938:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.815940:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.815941:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.815942:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.815945:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.815946:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.815949:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.815950:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.815951:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.815953:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.815953:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.815954:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.815954:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677447.816052:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.816054:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.816055:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.816057:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.816059:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.816060:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.816061:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.816063:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.816064:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.816065:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.816068:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.816070:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.816073:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.816074:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.816074:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.816076:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.816077:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.816078:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.816079:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.816079:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.816082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.816089:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:1.0:1619677447.816089:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.816091:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:24.0:1619677447.816092:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.816092:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677447.816092:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677447.816093:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.816093:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677447.816095:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.816099:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677447.816143:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.816145:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.816146:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.816148:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.816149:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.816150:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.816159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:12.0:1619677447.816159:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677447.816161:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677447.816162:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.816162:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.816163:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:20.0:1619677447.816164:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:12.0:1619677447.816164:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.816165:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:20.0:1619677447.816166:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.816167:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.816169:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.816170:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.816172:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.816173:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.816174:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.816175:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.816178:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.816180:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.816183:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.816184:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.816185:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.816187:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.816187:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.816188:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.816189:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.816199:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.816202:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.816203:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.816204:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.816205:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.816207:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.816218:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.816221:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.816222:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.816222:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.816223:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.816224:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.816392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.816399:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.816402:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.816403:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.816404:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.816407:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.816408:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.816409:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.816411:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.816412:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.816412:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.816415:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677447.816417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.0:1619677447.816417:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.816420:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.816421:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.816421:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.816423:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.816423:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.816424:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.816424:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:31.0:1619677447.816426:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.816428:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:8.2:1619677447.816429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677447.816430:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.816433:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.816434:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.816435:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.816436:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.816437:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:31.0:1619677447.816438:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.816438:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.816439:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677447.816440:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.816441:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.816441:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:31.0:1619677447.816443:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.816444:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.816445:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677447.816447:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:31.0:1619677447.816449:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.816450:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.816450:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.816452:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.816453:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.816454:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.816454:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:4.2:1619677447.816454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.2:1619677447.816458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.816466:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.816469:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.816470:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.816472:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.816472:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.816475:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677447.816499:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.816501:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.816502:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.816504:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.816505:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.816506:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677447.816579:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.816582:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.816583:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.816586:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.816586:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.816587:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.816589:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.816590:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.816591:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.816591:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.816594:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.816596:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.816598:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.816599:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.816600:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.816601:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.816602:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.816603:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.816603:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677447.816604:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.816607:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.816608:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.816609:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.816612:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.816614:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677447.816614:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.816615:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.816616:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.816617:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.816617:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.816617:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.816618:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:13.0:1619677447.816620:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.816621:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:1.0:1619677447.816622:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.816625:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.816626:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.816628:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.816629:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.816629:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.816631:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.816632:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.816633:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.816633:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.816644:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.816647:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.816647:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.816649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:12.0:1619677447.816649:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.816650:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677447.816651:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:12.0:1619677447.816652:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677447.816654:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.816656:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.816657:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.816658:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.816660:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677447.816661:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.816663:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.816664:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.816665:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677447.816668:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.816670:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677447.816673:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.816674:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677447.816674:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677447.816676:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677447.816677:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.816678:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.816678:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:13.0:1619677447.816726:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.816728:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.816729:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.816730:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.816732:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.816733:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.816791:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.816794:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.816795:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.816797:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.816800:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.816802:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.816803:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.816804:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.816805:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.816806:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.816810:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.816812:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.816815:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.816816:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.816816:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.816818:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.816818:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.816819:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.816819:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677447.816958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:8.2:1619677447.816996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.817004:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.817006:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.817009:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.817011:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.817012:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.817014:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677447.817016:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677447.817017:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.817018:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.817021:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.817023:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.817025:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.817026:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.817026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:18.0:1619677447.817027:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.817028:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.817029:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.817032:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.817034:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.817036:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.817037:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.817037:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.817039:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.817039:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.817040:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.817041:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.817042:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.817045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.817053:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.817054:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.817055:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.817056:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.817056:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.817056:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.817058:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.817058:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:2.0:1619677447.817058:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.817059:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:2.0:1619677447.817061:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677447.817062:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677447.817063:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677447.817065:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : ACK - routed
00000800:00000200:8.2:1619677447.817067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:24.0:1619677447.817068:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.817070:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677447.817071:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.817073:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.817074:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.817075:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:20.0:1619677447.817076:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:24.0:1619677447.817078:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.817078:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.817079:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.817080:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:20.0:1619677447.817081:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.817082:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.817082:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.817083:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:24.0:1619677447.817083:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677447.817084:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:24.0:1619677447.817085:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:24.0:1619677447.817086:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677447.817087:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed9d3400] (22)++
00000800:00000200:24.0:1619677447.817088:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (23)--
00000800:00000200:24.0:1619677447.817089:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:30.0:1619677447.817097:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.817100:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.817101:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.817102:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:8.0:1619677447.817102:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)++
00000800:00000200:31.0:1619677447.817104:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:8.0:1619677447.817104:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed9d3400] (23)--
00000800:00000200:30.0:1619677447.817105:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:8.0:1619677447.817105:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677447.817106:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:31.0:1619677447.817117:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.817120:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.817121:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.817122:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677447.817122:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.817124:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:17.0:1619677447.817143:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.817146:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.817147:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.817149:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.817152:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.817153:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.817154:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.817156:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.817157:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.817158:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.817162:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.817164:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.817166:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.817167:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.817167:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.817169:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.817170:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.817170:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.817171:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.817183:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.817185:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.817186:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.817186:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.817188:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.817191:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.817193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.817201:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.817203:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.817206:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.817207:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.817208:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.817210:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.817211:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.817212:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.817213:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.817214:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.817218:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.817220:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.817223:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.817224:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.817224:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.817226:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.817227:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.817228:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.817229:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.817232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.817240:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.817243:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.817244:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.817245:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.817246:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.817248:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677447.817264:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.817267:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.817267:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.817269:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.817271:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.817273:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.817274:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.817275:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.817276:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.817277:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.817280:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.817282:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.817285:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.817285:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.817286:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.817288:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.817288:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.817289:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.817290:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.817291:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.817303:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.817304:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.817305:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.817306:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:31.0:1619677447.817413:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.817416:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.817418:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.817420:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.817421:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.817423:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677447.817424:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.817425:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.817426:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.817427:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677447.817431:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.817433:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677447.817436:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.817437:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677447.817437:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677447.817439:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677447.817440:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.817441:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.817441:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677447.817453:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.817456:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.817457:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.817457:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.817459:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.817461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.817462:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677447.817478:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.817481:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.817481:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.817485:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.817488:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.817489:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677447.817491:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.817492:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.817493:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.817494:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677447.817497:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.817499:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677447.817502:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.817502:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677447.817503:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677447.817505:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677447.817505:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.817506:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.817508:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.817508:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:30.0:1619677447.817509:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.817511:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.817512:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.817514:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.817515:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.817516:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:6.0:1619677447.817519:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677447.817519:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:6.0:1619677447.817521:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677447.817521:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:22.0:1619677447.817522:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677447.817522:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:6.0:1619677447.817523:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.817523:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:6.0:1619677447.817525:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:31.0:1619677447.817527:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.817527:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677447.817531:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.817532:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.817532:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.817533:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.817534:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.817562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.817565:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.817568:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.817571:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.817571:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.817573:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.817575:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.817577:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.817578:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.817579:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.817580:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.817583:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.817585:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.817588:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.817589:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.817590:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.817592:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.817592:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.817593:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.817594:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677447.817605:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.817608:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.817609:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.817610:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.817612:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.817613:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.817704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:9.2:1619677447.817743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.817750:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.817753:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.817754:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.817754:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.817757:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.817760:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.817762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.817768:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677447.817770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677447.817771:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.817772:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.817772:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.817773:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.817774:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677447.817778:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.2:1619677447.817780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:31.0:1619677447.817781:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.817783:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.817784:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.817786:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.817786:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:31.0:1619677447.817787:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.817789:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.817789:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:31.0:1619677447.817790:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.817790:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:31.0:1619677447.817792:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.817793:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677447.817793:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.817795:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.817796:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677447.817797:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.817798:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.817799:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:18.0:1619677447.817799:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.817801:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.817802:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.817802:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:22.0:1619677447.817802:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:18.0:1619677447.817802:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677447.817803:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.817805:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.817805:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:22.0:1619677447.817805:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:18.0:1619677447.817805:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.817805:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:31.0:1619677447.817807:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.817808:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000400:00000200:22.0:1619677447.817808:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.817808:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677447.817809:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:22.0:1619677447.817810:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.817810:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.817810:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:22.0:1619677447.817811:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:18.0:1619677447.817811:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:22.0:1619677447.817813:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96723129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.817813:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.817813:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:22.0:1619677447.817814:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.817814:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.817814:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:22.0:1619677447.817815:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677447.817815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:22.0:1619677447.817816:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:12.0:1619677447.817818:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:9.2:1619677447.817818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:22.0:1619677447.817820:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.817821:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.817821:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.817822:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677447.817823:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:12.0:1619677447.817823:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.817824:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.817825:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:22.0:1619677447.817825:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677447.817826:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:22.0:1619677447.817826:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677447.817826:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.817826:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:30.0:1619677447.817827:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.817827:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.817828:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.817828:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677447.817828:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:22.0:1619677447.817829:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677447.817829:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.817830:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677447.817830:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.817831:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.817833:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.2:1619677447.817835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.817843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.817843:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.817846:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.817847:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.817847:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.817849:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.817851:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.817884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677447.817887:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.817890:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.817890:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.817892:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.817893:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.817894:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.817958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677447.817965:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.817967:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.817968:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.817970:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.817972:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.817974:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.817975:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.817976:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.817977:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.817978:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.817981:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.817982:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.817985:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.817986:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.817986:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.817988:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.817989:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.817989:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.817990:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677447.817994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.818002:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.2:1619677447.818006:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677447.818008:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.818009:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.818010:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.818011:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677447.818011:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.818014:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:5.0:1619677447.818014:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.818015:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.818016:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.818019:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.818021:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.818022:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.818023:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.818024:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.818025:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.818028:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.818030:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.818033:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.818034:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.818035:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.818037:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.818037:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.818038:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.818039:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.818049:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.818052:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.818053:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.818055:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.818056:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.818058:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.818177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.818184:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.818187:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.818189:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.818190:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.818191:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.818193:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.818194:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.818195:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.818196:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.818197:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.818201:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.818202:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.818205:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.818206:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.818206:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.818208:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.818208:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.818210:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.818211:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.818212:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677447.818218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677447.818220:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.818222:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.818223:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.818224:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.818225:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677447.818226:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.818227:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:13.0:1619677447.818227:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.818228:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.818229:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677447.818248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.818258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.818266:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.818269:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677447.818273:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.818274:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.818276:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.818278:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677447.818279:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.818281:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.818281:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.818283:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677447.818286:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.818288:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677447.818290:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.818291:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.818291:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.818292:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.818292:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.818292:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677447.818300:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.818301:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677447.818302:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.818302:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.818302:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677447.818302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677447.818303:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.818304:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677447.818305:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.818305:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.818305:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.818306:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.818307:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677447.818308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.0:1619677447.818312:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.818315:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:8.0:1619677447.818315:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:20.0:1619677447.818316:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.818318:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.818319:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.818319:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:5.0:1619677447.818319:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.2:1619677447.818321:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.818324:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.818325:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.818327:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.818327:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.818328:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.818329:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000400:00000200:8.0:1619677447.818329:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.818330:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.818331:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:8.0:1619677447.818331:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.818332:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.818334:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.818335:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.818336:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.818339:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.818341:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.818344:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.818344:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.818345:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.818347:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.818347:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.818348:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.818349:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.818353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.818361:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.818363:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.818364:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.818365:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.818367:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.818369:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.818625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677447.818629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.2:1619677447.818630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:18.0:1619677447.818631:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.818635:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:17.0:1619677447.818635:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677447.818635:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.818637:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.818637:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677447.818638:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:18.0:1619677447.818639:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.818639:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677447.818640:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:6.0:1619677447.818640:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.818641:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:17.0:1619677447.818641:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.818642:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.818644:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.818644:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.818644:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.818645:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.818646:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.818646:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.818647:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.818648:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.818649:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.818649:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.818650:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.818651:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.818652:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.818652:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.818653:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.818655:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.818657:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.818660:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.818660:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.818662:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.818663:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677447.818665:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.818666:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.818666:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.818667:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.818667:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.818667:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:18.0:1619677447.818668:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.818668:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:18.0:1619677447.818669:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677447.818669:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (23)++
00000800:00000200:17.0:1619677447.818669:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.818670:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677447.818670:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:18.0:1619677447.818671:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.818671:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677447.818671:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677447.818672:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677447.818672:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677447.818672:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677447.818673:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677447.818673:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.818673:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:18.0:1619677447.818674:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:6.0:1619677447.818674:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677447.818675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (22)++
00000800:00000200:21.2:1619677447.818678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.2:1619677447.818678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677447.818683:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.2:1619677447.818683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677447.818684:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:1.0:1619677447.818684:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000800:00000200:22.0:1619677447.818686:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:17.0:1619677447.818686:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.818686:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000800:00000200:30.0:1619677447.818687:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:22.0:1619677447.818687:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677447.818687:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:1.0:1619677447.818687:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.818688:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.818688:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677447.818689:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.818689:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000400:00000200:1.0:1619677447.818689:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.818690:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.818690:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000800:00000200:31.0:1619677447.818691:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:17.0:1619677447.818691:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.818692:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.818693:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.818693:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.818694:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:1.0:1619677447.818694:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.818696:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.818697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.818704:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.818707:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.818708:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.818708:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.818709:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.818709:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:9.2:1619677447.818724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677447.818727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677447.818732:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.818734:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.818736:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.818736:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.818738:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.818741:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.818793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.818838:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.818841:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.818842:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.818842:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.818844:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.818847:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.818858:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.818860:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.818861:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.818861:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.818862:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.818863:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677447.818897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677447.818903:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.818906:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.818906:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.818908:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.818910:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.818912:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.818913:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.818915:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.818915:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.818917:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.818920:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.818922:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.818924:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.818925:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.818926:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.818928:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.818928:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.818929:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.818929:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677447.819208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.819214:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.819216:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677447.819218:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677447.819219:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.819221:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.819223:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.819224:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.819225:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.819226:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.819227:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.819231:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.819233:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.819235:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.819236:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.819237:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.819239:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.819240:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.819241:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.819242:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.819243:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.2:1619677447.819248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.819251:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:31.0:1619677447.819252:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677447.819253:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.819254:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.819254:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677447.819255:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.819256:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.819258:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:30.0:1619677447.819259:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.819261:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.819263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.819268:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.819271:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.819272:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.819274:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.819277:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.819278:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.819279:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.819281:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.819282:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.819283:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.819286:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.819288:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.819290:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.819291:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.819291:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.819299:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:9.2:1619677447.819300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (22)++
00000800:00000200:4.2:1619677447.819301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:2.0:1619677447.819301:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.819302:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.819304:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677447.819305:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.819307:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.819307:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.819309:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:2.0:1619677447.819309:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.819310:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.819311:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677447.819312:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.819313:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.819315:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.819318:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.819320:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.819321:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.819322:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.819323:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.819324:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.819328:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.819330:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.819332:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.819333:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.819334:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.819336:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.819336:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.819336:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.2:1619677447.819336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:24.0:1619677447.819337:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.819341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677447.819344:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.819347:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:5.2:1619677447.819348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677447.819349:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.819352:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.819352:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.819353:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:17.0:1619677447.819353:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:30.0:1619677447.819354:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723112] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.819355:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.819355:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.819355:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677447.819356:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.819357:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.819357:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:30.0:1619677447.819358:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:17.0:1619677447.819359:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.819360:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.819362:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.819363:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.819363:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.819364:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:17.0:1619677447.819364:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.819365:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.819365:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.819366:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.819367:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.819368:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.819368:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:1.0:1619677447.819368:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.819369:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.819370:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.819370:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:1.0:1619677447.819370:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.819371:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.819372:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:1.0:1619677447.819373:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.819375:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.819378:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.819379:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.819379:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.819381:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.819382:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:1.0:1619677447.819382:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.819382:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.819383:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.819385:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.819386:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.819387:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.819389:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.819390:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.819391:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.819393:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.819394:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.819396:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.819397:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.819398:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.819414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.819416:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.819419:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.819421:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.819422:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.819424:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.819426:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677447.819427:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.819428:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.819429:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.819430:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677447.819433:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.819435:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677447.819437:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.819438:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.819439:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.819440:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.819441:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.819442:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.819443:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.819444:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677447.819447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677447.819448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677447.819454:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:5.0:1619677447.819454:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:17.0:1619677447.819457:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:21.2:1619677447.819458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:17.0:1619677447.819458:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.819458:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:1.0:1619677447.819458:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.819459:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:17.0:1619677447.819460:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.819460:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677447.819463:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677447.819464:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.819467:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.819468:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.819470:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.819472:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.819473:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677447.819474:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.819476:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.819477:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.819478:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677447.819482:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.819483:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677447.819486:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.819487:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677447.819488:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677447.819490:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677447.819490:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.819491:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.819491:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677447.819508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:9.2:1619677447.819543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.819551:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.819553:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.819554:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.819555:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.819557:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:9.2:1619677447.819560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.819560:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.819567:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.819570:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.819571:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.819571:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.819572:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.819573:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.819869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.819875:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.819878:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.819879:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.819881:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.819884:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.819885:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.819886:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.819888:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.819889:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.819890:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.819893:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.819894:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.819897:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.819898:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.819899:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.819901:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.819901:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.819902:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.819902:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.819914:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.819917:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.819918:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.819919:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.819921:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.819922:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.819931:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.819934:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.819936:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.819937:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.819937:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.819938:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677447.820057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.820063:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.820066:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.820067:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.820069:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.820072:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.820074:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.820075:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.820076:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.820077:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.820078:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:4.2:1619677447.820081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:22.0:1619677447.820082:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.820083:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.820086:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.820087:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.820087:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677447.820088:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.820089:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.820090:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.820090:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677447.820090:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:22.0:1619677447.820091:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.820092:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.820093:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677447.820095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:8.0:1619677447.820096:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.820098:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.820099:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.820100:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.820101:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.820102:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.820103:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.820105:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:8.0:1619677447.820106:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.820106:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.820107:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:8.0:1619677447.820108:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.820109:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.820111:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.820112:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.820112:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.820112:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:8.0:1619677447.820114:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.820115:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.820116:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.820117:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.820118:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.2:1619677447.820118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.2:1619677447.820120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.820123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.820128:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.820130:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677447.820131:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677447.820132:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677447.820132:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.820132:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677447.820133:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.820134:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677447.820135:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.820138:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677447.820162:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.820165:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.820165:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.820167:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.820169:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.820169:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:4.2:1619677447.820284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.820335:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.820338:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.820339:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.820339:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.820341:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.820344:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677447.820370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677447.820376:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677447.820379:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677447.820380:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677447.820382:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.820384:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.820386:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677447.820387:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.820388:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.820389:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.820391:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677447.820394:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.820396:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.820397:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.820398:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.820399:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.820400:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.820401:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677447.820401:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677447.820402:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:9.2:1619677447.820406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.820413:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.820415:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.820416:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.820417:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.820417:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.820419:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.820570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.820578:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.820581:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:5.2:1619677447.820581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.0:1619677447.820582:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.820584:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.820587:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.820587:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677447.820588:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.820589:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.820590:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.820591:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677447.820591:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.820592:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.820593:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.820594:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.820597:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.820599:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.820600:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.820602:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677447.820602:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:8.0:1619677447.820603:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.820603:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:1.0:1619677447.820603:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.820605:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.820605:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.820605:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.820606:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.820606:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677447.820606:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.820607:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.820609:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677447.820611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.820611:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.820614:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.820615:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.820616:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.820617:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.820618:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.820619:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.820620:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677447.820621:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.820623:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.820623:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.820623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677447.820624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:1.0:1619677447.820624:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.820625:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.820626:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.820627:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.820627:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.820628:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677447.820629:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:17.0:1619677447.820629:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.820629:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.820630:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:22.0:1619677447.820632:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677447.820633:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677447.820635:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.820637:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.820639:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:22.0:1619677447.820640:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.820641:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.820642:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.820643:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677447.820647:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.820649:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.820651:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.820652:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:22.0:1619677447.820653:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.820654:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677447.820655:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.820656:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677447.820656:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677447.820661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.820668:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.820671:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.820672:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.820672:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.820674:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.820677:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.820752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.820755:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.820757:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.820759:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.820760:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.820762:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.820764:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677447.820765:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.820767:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.820768:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.820769:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677447.820772:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.820774:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677447.820777:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.820778:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677447.820779:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677447.820781:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677447.820782:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.820783:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.820783:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677447.820787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.820794:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.820797:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.820798:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.820798:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.820801:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.820804:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.820809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677447.820815:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.820817:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.820818:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.820819:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.820819:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.820820:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677447.820889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.820897:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.820900:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.820902:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.820904:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.820905:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.820906:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.820907:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.820908:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.820910:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.820910:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.820914:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.820916:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.820918:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.820919:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.820920:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.820922:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.820922:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.820923:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.820923:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677447.820935:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.820937:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.820938:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.820939:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.820941:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.820942:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.820942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.820983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677447.820986:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.820988:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.820989:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.820990:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.820992:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.820994:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.821005:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.821008:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.821009:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.821009:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.821010:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.821011:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.821018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:13.0:1619677447.821058:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.821061:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.821061:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.821063:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.821065:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.821066:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677447.821070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.821076:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.821079:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.821080:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.821082:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.821083:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.821085:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.821086:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.821087:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.821088:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.821089:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.821093:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.821094:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.821097:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.821098:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.821099:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.821101:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.821102:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.821103:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.821104:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.821104:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677447.821109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677447.821113:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677447.821115:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677447.821116:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.821117:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.821117:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.821117:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.821118:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677447.821119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:13.0:1619677447.821119:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.821120:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.821121:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.821125:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.821127:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.821130:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.821130:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.821132:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.821134:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.821136:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.821137:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.821138:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.821139:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.821143:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.821144:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.821148:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.821149:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.821149:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.821151:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.821152:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.821152:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.821153:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677447.821164:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.821166:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.821167:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.821169:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.821170:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.821172:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.821252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.821260:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.821262:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.821265:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.821266:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.821267:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.821268:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.821269:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.821271:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.821272:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.821273:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.821276:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.821278:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.821281:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.821282:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.821282:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.821284:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.821284:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.821285:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.821286:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677447.821291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.821299:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:12.0:1619677447.821299:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.821301:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:24.0:1619677447.821302:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:12.0:1619677447.821302:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.821303:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:12.0:1619677447.821304:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.821305:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.821306:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.821307:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677447.821308:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.821309:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.821310:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.821312:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.821312:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.821313:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:21.2:1619677447.821315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:24.0:1619677447.821317:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.821319:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.821322:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.821323:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.821323:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.821324:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.821325:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.821326:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.821326:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.821327:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.821327:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.821328:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677447.821329:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.821329:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.821332:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:8.0:1619677447.821335:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677447.821336:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.821338:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.821338:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.821339:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.821340:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.821342:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:8.0:1619677447.821342:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.821343:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.821344:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.821344:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.821346:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.821347:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.821347:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.821348:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.821349:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.0:1619677447.821349:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.821350:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.821351:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.821352:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677447.821354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.821362:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677447.821363:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:21.2:1619677447.821364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:1.0:1619677447.821364:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.821365:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677447.821365:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677447.821365:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000400:00000200:12.0:1619677447.821366:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.821366:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.821367:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:12.0:1619677447.821368:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.821369:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677447.821370:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:12.0:1619677447.821371:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.821373:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.821374:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.821376:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.821378:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:6.0:1619677447.821378:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.821380:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:13.0:1619677447.821382:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:6.0:1619677447.821382:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723089] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.821383:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.821384:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:6.0:1619677447.821384:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.821385:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:6.0:1619677447.821385:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.821386:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:6.0:1619677447.821386:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.821389:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.821391:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.821393:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.821394:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.821395:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.821397:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.821397:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.821397:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.821398:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.821403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.821411:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.821413:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.821414:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.821416:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.821417:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.821418:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677447.821423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677447.821431:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.821434:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.821435:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.821436:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.821437:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.821438:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.821541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677447.821587:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.821590:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.821590:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.821592:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.821594:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.821595:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.821719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.2:1619677447.821750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.821756:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.821758:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:13.0:1619677447.821760:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.0:1619677447.821761:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.821761:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.821763:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:5.0:1619677447.821763:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.821764:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.821766:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.821766:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:12.0:1619677447.821767:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:5.0:1619677447.821767:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.821768:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:5.0:1619677447.821769:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.821770:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.821771:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.821774:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.821776:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.821778:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.821779:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.821779:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.821781:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.821782:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.821782:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.821783:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.821795:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.821797:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.821798:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.821798:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.821800:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.821803:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677447.821830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:21.2:1619677447.821834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:2.0:1619677447.821836:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.821839:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.821840:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.821842:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.821844:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.821846:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.821848:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.821849:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.821850:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.821851:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.2:1619677447.821852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:2.0:1619677447.821855:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677447.821858:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677447.821858:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.821860:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.821861:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:2.0:1619677447.821861:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:1.0:1619677447.821861:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677447.821862:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.821863:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:17.0:1619677447.821864:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.821864:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.2:1619677447.821865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677447.821865:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.821865:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:17.0:1619677447.821867:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.821868:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.821870:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96723083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.821871:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.821872:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.821873:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677447.821873:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.821876:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:6.0:1619677447.821876:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:17.0:1619677447.821877:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677447.821879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:6.0:1619677447.821879:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677447.821880:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.821880:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.821880:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677447.821880:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:6.0:1619677447.821880:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.821882:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.821882:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:17.0:1619677447.821883:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677447.821884:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:17.0:1619677447.821884:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.821885:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677447.821885:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:30.0:1619677447.821886:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96723082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.821887:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.821887:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.821887:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677447.821888:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.821888:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.821889:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677447.821890:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.821893:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.821895:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.821899:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.821900:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.821900:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677447.821900:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677447.821902:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.821903:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677447.821903:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:30.0:1619677447.821904:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:12.0:1619677447.821904:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.821905:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.821906:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:12.0:1619677447.821906:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.821907:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.821909:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.2:1619677447.821910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677447.821911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.821920:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677447.821920:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.821922:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677447.821923:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.821923:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677447.821923:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.821924:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677447.821925:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.821926:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.821928:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677447.821958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.821965:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.821968:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.821970:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.821972:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.821974:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.821975:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677447.821977:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.821978:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.821979:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.821980:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677447.821983:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.821984:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677447.821988:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.821988:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677447.821989:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677447.821991:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.821991:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.821992:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.821993:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677447.822004:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.822007:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.822008:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.822009:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.822011:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.822014:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.822022:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.822025:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.822026:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.822026:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.822027:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.822028:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.822063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.2:1619677447.822099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.822107:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.822110:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.822111:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.822113:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.822113:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.822116:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677447.822213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.822218:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.822221:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.822221:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.822223:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.822226:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.822227:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.822229:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.822230:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.822231:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.822232:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.822234:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.822236:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.822239:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.822239:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.822240:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.822242:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.822242:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.822243:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.822243:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.822248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.822255:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.822258:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.822258:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.822260:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.822261:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.822262:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677447.822358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.822364:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.822367:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.822368:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.822370:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.822372:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.822374:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.822375:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.822377:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.822378:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.822379:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.822382:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.822383:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.822386:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.822387:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.822387:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:4.2:1619677447.822389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:1.0:1619677447.822389:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.822389:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.822390:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.822390:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677447.822397:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.822400:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.822401:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.822403:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.822404:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:24.0:1619677447.822405:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.822406:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:6.0:1619677447.822406:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.822407:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.822407:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.822408:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.822410:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.822410:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.822410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677447.822411:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.822412:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:6.0:1619677447.822412:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:24.0:1619677447.822416:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.822417:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.822420:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.822421:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:22.0:1619677447.822421:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:24.0:1619677447.822422:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.822424:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.822424:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.822424:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.822424:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:24.0:1619677447.822425:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677447.822425:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:24.0:1619677447.822426:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677447.822426:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.822427:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.2:1619677447.822430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.822437:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.822440:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.822441:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.822441:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.822443:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:20.2:1619677447.822445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677447.822446:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.822449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:30.0:1619677447.822453:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.822456:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:6.0:1619677447.822456:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:1.0:1619677447.822456:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:30.0:1619677447.822459:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.822459:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:1.0:1619677447.822459:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:22.0:1619677447.822460:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677447.822460:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:6.0:1619677447.822460:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.822460:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:31.0:1619677447.822461:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:1.0:1619677447.822461:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.822462:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:30.0:1619677447.822463:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.822463:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.822465:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.822466:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.822466:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:30.0:1619677447.822467:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.822468:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.822469:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.822473:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.822474:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.822477:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.822478:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.822478:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.822480:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.822481:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.822482:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.822482:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.822486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.822494:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.822497:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.822498:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.822500:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.822500:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.822502:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677447.822518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.822525:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.822528:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.822530:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.822532:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.822533:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.822535:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677447.822536:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.822537:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.822539:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.822539:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677447.822543:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.822544:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677447.822547:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.822547:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677447.822548:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677447.822549:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.822550:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.822550:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.822551:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677447.822618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677447.822624:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.822627:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.822627:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.822630:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.822632:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.822633:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.822634:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.822636:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.822637:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.822638:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.822641:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.822643:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.822645:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.822645:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.822646:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.822647:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.822648:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.822649:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.822649:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.822653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.822661:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.822662:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.822663:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.822664:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.822665:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.822666:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.822724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.822764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.2:1619677447.822780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.822786:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.822789:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.822789:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.822792:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.822794:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.822795:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.822796:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.822797:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.822799:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.822800:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.822803:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.822805:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.822807:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.822808:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.822809:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677447.822809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:17.0:1619677447.822810:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.822811:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.822811:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.822812:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.822824:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.822826:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.822827:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.822827:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.822830:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.822832:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.822849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677447.822852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677447.822858:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.822861:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.822862:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.822863:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.822865:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.822868:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677447.822964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.822971:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.822974:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.822977:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.822978:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.822979:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.822980:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677447.822982:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.822983:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.822984:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.822985:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.2:1619677447.822986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:30.0:1619677447.822988:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.822990:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.822993:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.822994:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.822994:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677447.822994:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677447.822995:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:30.0:1619677447.822996:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.822996:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:31.0:1619677447.822997:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.822998:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:31.0:1619677447.822999:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.822999:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.823000:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.823001:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:30.0:1619677447.823001:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.823002:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.823003:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.823004:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.823005:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:21.2:1619677447.823005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:31.0:1619677447.823008:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.823009:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:6.0:1619677447.823010:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:31.0:1619677447.823011:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.823012:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:6.0:1619677447.823012:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677447.823013:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.823013:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.823013:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:22.0:1619677447.823013:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.823014:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.823014:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:6.0:1619677447.823014:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.823015:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.823015:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.823015:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677447.823016:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:6.0:1619677447.823016:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.823019:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.823028:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.823030:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.823031:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.823033:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.823035:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.823036:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677447.823136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.823142:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.823145:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.823145:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.823147:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.823149:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.823150:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.823152:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.823153:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.823154:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.823155:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.823157:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.823159:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.823161:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.823162:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.823162:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.823164:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.823164:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.823165:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.823166:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.823167:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.823170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677447.823173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:4.2:1619677447.823176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.823177:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.823178:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.823179:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.823179:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.823179:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.823180:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.823181:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.823181:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.823182:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.823182:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.823189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.823197:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.823199:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.823200:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.823201:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.823202:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.823202:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.823221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677447.823224:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.823227:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:1.0:1619677447.823227:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677447.823228:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.823228:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677447.823230:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:22.0:1619677447.823231:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.823231:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.823233:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.823234:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:1.0:1619677447.823235:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.823237:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.823238:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.823240:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.823241:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.823242:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.823245:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.823247:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.823250:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.823251:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.823252:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.823253:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.823254:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.823255:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.823256:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.823257:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.823259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677447.823262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677447.823266:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677447.823267:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.823269:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.0:1619677447.823269:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.823270:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:17.0:1619677447.823270:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.823271:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677447.823271:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.823273:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.823276:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677447.823381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:21.2:1619677447.823387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.823393:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.823396:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.823397:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.823400:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.823402:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.823404:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.823405:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.823407:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.823408:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.823409:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.823415:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.823417:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.823420:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.823421:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.823421:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.823423:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.823424:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.823425:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:8.2:1619677447.823426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677447.823426:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.823427:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677447.823429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.823436:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.823438:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.823438:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.823439:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.823439:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.823439:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.823441:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.823442:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:21.2:1619677447.823445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677447.823445:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.823446:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.823446:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.823447:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677447.823451:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.823453:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.823454:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.823454:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677447.823479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.823488:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.823491:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.823493:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.823495:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.823495:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.823496:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677447.823498:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.823499:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.823500:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.823501:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677447.823504:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.823506:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.823508:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.823509:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.823510:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.823511:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.823512:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.823513:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.823513:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677447.823524:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.823527:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.823528:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.823529:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.823531:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.823531:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.2:1619677447.823639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.823642:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.823644:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.823647:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.823647:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.823649:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.823650:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.823651:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.823653:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.823654:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.823655:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.823658:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.823660:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.823663:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.823664:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.823665:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.823667:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.823668:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.823668:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.823669:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.2:1619677447.823680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:12.0:1619677447.823680:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.823682:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.823683:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.823685:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.823686:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:13.0:1619677447.823687:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.823688:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.823689:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.823690:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.823692:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.823694:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.823696:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.823697:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.823698:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.823699:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:2.0:1619677447.823699:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.823700:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:13.0:1619677447.823701:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.823702:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:21.2:1619677447.823703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:13.0:1619677447.823703:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:2.0:1619677447.823703:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.823704:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.823705:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.823705:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.823708:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.823709:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.823709:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677447.823710:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677447.823712:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.823712:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.823713:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.823713:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:30.0:1619677447.823714:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.823716:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.823717:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.823718:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.823718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:30.0:1619677447.823720:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.823722:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.823723:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.823724:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.823724:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:30.0:1619677447.823725:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.823728:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.823728:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:30.0:1619677447.823731:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.823731:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.823734:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.823734:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.823734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677447.823736:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.823737:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:18.0:1619677447.823737:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:30.0:1619677447.823738:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677447.823738:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.823739:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:18.0:1619677447.823739:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.823740:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.823741:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.823741:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:18.0:1619677447.823741:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.823742:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.823746:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.823748:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.823750:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.823751:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:12.0:1619677447.823751:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677447.823752:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.823753:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:12.0:1619677447.823753:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677447.823754:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:12.0:1619677447.823754:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.823755:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.823755:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:12.0:1619677447.823755:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.823757:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.823758:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.823759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.823766:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.823769:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.823770:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.823771:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.823772:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.823774:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.823781:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.823783:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.823784:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.823785:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.823787:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.823789:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677447.823824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.823830:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.823832:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.823833:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.823835:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.823838:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.823839:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.823840:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.823842:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.823843:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.823844:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.823847:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.823849:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.823851:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.823851:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.823852:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.823853:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.823854:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.823854:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.823855:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.823860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.823867:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.823869:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.823869:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.823870:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.823871:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.823872:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677447.824102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.824108:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.824111:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.824111:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.824113:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.824115:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.824117:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.824118:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.824119:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.824121:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.824122:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.824125:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.824127:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.824129:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.824130:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:4.2:1619677447.824130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:6.0:1619677447.824131:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.824133:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.824133:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.824134:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.824134:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.824137:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.824140:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.824141:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.824142:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677447.824143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:24.0:1619677447.824145:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.824147:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:24.0:1619677447.824147:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.824148:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.824149:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:24.0:1619677447.824149:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.824150:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.824150:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.824151:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:31.0:1619677447.824152:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.824152:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:24.0:1619677447.824152:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.824155:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677447.824155:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.824156:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:24.0:1619677447.824156:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.824157:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.824158:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.824158:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:24.0:1619677447.824158:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:30.0:1619677447.824159:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.824161:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.824161:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677447.824162:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.824162:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:30.0:1619677447.824163:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.824163:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.824164:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:4.2:1619677447.824164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:30.0:1619677447.824165:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.824165:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.824166:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:30.0:1619677447.824167:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.824167:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.824168:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677447.824169:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.824170:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.2:1619677447.824170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677447.824171:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.824172:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.824173:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.824173:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:4.2:1619677447.824173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677447.824174:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677447.824179:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677447.824180:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.824181:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.824182:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.824182:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.824183:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.824183:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677447.824184:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677447.824184:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.824185:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.824187:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.824188:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.824188:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:13.0:1619677447.824190:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.824190:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.824192:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.824212:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.824214:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.824215:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.824216:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.824218:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.824221:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.824223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:6.0:1619677447.824231:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.824234:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.824235:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.824235:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.824236:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.824237:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.824246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.824251:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677447.824252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677447.824254:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.824256:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.824258:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677447.824259:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:5.0:1619677447.824259:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.824261:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677447.824262:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:5.0:1619677447.824262:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.824264:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.824265:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.824266:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.824266:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.824267:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.824268:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.824270:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677447.824271:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.824272:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.824273:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.824274:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.824274:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.824275:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:5.0:1619677447.824277:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.824278:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:31.0:1619677447.824279:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677447.824280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:5.0:1619677447.824280:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.824281:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677447.824282:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.824283:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.824283:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677447.824284:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.824285:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677447.824285:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.824286:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:31.0:1619677447.824287:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.824287:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677447.824287:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:24.0:1619677447.824288:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:31.0:1619677447.824289:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:24.0:1619677447.824290:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:31.0:1619677447.824291:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.824291:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.0:1619677447.824291:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.2:1619677447.824292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677447.824297:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.2:1619677447.824298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:24.0:1619677447.824300:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.824303:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:20.0:1619677447.824303:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:24.0:1619677447.824304:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.824304:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.824304:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:30.0:1619677447.824305:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.824305:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.824305:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.824306:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677447.824306:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:31.0:1619677447.824307:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:24.0:1619677447.824307:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96723048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.824307:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.824307:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:12.0:1619677447.824307:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.824308:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:30.0:1619677447.824308:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.824309:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:24.0:1619677447.824309:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677447.824309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677447.824310:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:24.0:1619677447.824310:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.824311:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000400:00000200:24.0:1619677447.824311:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.824311:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.824312:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.824312:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677447.824313:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.824313:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.824314:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:24.0:1619677447.824314:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.824315:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:17.0:1619677447.824315:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677447.824316:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.824318:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.824318:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:24.0:1619677447.824319:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.2:1619677447.824319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:1.0:1619677447.824319:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677447.824320:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.824321:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:17.0:1619677447.824321:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.824322:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.824323:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000400:00000200:17.0:1619677447.824323:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.824324:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000400:00000200:17.0:1619677447.824324:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:24.0:1619677447.824325:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:17.0:1619677447.824325:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.824326:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:17.0:1619677447.824327:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.824328:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.824329:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:17.0:1619677447.824329:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:2.0:1619677447.824329:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:21.2:1619677447.824330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000400:00000200:18.0:1619677447.824333:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.824334:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.824334:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:18.0:1619677447.824335:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.824336:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:17.0:1619677447.824336:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:13.0:1619677447.824336:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:18.0:1619677447.824337:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96723046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.824337:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.824337:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:18.0:1619677447.824339:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.824339:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:17.0:1619677447.824340:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677447.824340:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.824340:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.824340:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:18.0:1619677447.824341:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.824341:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.824341:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.0:1619677447.824341:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:18.0:1619677447.824342:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:13.0:1619677447.824342:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.824343:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.824344:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.824344:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677447.824345:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.824345:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677447.824347:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.824349:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.824350:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.824350:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.824351:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.824352:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.824352:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.824353:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.824354:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.824354:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.824355:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677447.824356:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:12.0:1619677447.824356:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.824356:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.824358:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:12.0:1619677447.824358:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.824358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677447.824359:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.824361:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.824362:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.824364:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:2.0:1619677447.824365:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.824368:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.824368:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.824369:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.824370:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.824372:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:2.0:1619677447.824372:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677447.824375:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.824376:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.824377:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.824378:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.824379:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.824416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677447.824422:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.824424:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.824425:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.824427:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.824428:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.824429:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.824431:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.824433:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.824434:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.824435:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.824437:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.824439:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.824440:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.824440:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.824441:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.824442:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.824443:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.824443:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.824444:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.824448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.824454:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.824457:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.824457:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.824458:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.824459:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.824461:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.824712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.824718:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.824721:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.824721:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.824724:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677447.824726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:1.0:1619677447.824726:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.824728:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.824729:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.824730:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.824731:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.824732:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.824733:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.824736:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:1.0:1619677447.824736:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.824738:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.824739:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.824739:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.824741:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.824742:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.824742:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:30.0:1619677447.824743:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677447.824743:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:30.0:1619677447.824744:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96723043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.824745:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.824745:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.824745:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677447.824746:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.824746:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677447.824747:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.824747:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677447.824751:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.824752:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.824755:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.824756:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.824756:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.824758:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.824758:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.824758:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.824759:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.824759:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.824760:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.824760:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:31.0:1619677447.824761:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.824761:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.824764:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:13.0:1619677447.824770:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.824772:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.824773:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.824774:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.824776:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.824777:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.824796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.824798:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.824801:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.824803:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.824804:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.824806:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.824808:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.824809:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.824810:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.824811:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.824812:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.824815:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.824817:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:21.2:1619677447.824820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:20.0:1619677447.824820:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.824821:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.824822:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.824824:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.824824:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.824825:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.824826:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:24.0:1619677447.824828:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.824831:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.824831:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.824833:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.824835:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.824837:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:12.0:1619677447.824837:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:24.0:1619677447.824838:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96723041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.824839:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:24.0:1619677447.824840:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677447.824840:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.824841:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.824842:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:12.0:1619677447.824842:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.824843:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677447.824845:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.824847:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.824848:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.824849:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.824850:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.824851:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.824852:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.824853:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.824853:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.824854:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.824864:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.824866:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.824867:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.824868:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.824870:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.824871:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.824941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.824947:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.824950:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.824950:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.824952:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.824955:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.824956:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.824957:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.824959:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.824960:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.824961:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.824965:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.824966:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.824969:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.824970:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.824971:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.824972:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.824973:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.824974:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.824974:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677447.824985:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.824988:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.824989:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.824991:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.824992:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.824994:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.825048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.2:1619677447.825087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.825095:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.825098:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.825099:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.825099:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.825101:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.825104:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.825114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.825119:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.825122:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.825124:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.825124:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.825127:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.825129:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677447.825130:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.825131:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.825132:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.825133:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677447.825136:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.825138:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.825140:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.825141:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677447.825142:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677447.825144:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677447.825144:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.825145:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.825146:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:5.2:1619677447.825287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.825292:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.825298:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.825300:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.825302:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.825304:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.825306:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.825307:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.825308:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.825309:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.825310:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.825313:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.825315:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.825317:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.825318:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.825319:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.825321:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.825321:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.825323:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.825324:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.825324:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677447.825330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677447.825335:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677447.825336:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.825338:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677447.825338:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.825339:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.825339:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677447.825339:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.825340:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677447.825341:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.825344:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677447.825431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677447.825437:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.825440:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.825441:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.825443:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.825445:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.825446:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.825447:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.825449:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.825450:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.825451:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.825454:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.825456:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.825460:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.825460:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.825461:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.825463:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.825463:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.825464:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.825465:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.825466:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.2:1619677447.825471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.825477:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.825478:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.825479:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.825480:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.2:1619677447.825495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.825501:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.825504:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.825504:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.825506:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.825509:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.825511:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.825512:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.825513:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.825514:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.825515:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.825517:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.825519:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.825521:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.825521:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.825522:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.825523:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.825524:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.825525:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.825525:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.825541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.825550:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.825552:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.825553:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.825553:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.825555:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.825557:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.825563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.825569:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.825572:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.825573:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.825573:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.825574:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:4.2:1619677447.825574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.825575:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677447.825621:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.825623:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.825624:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.825624:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.825627:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.825630:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.825673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.825680:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.825683:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.825684:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.825685:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.825688:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.825689:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.825691:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.825692:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.825693:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.825694:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.825697:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.825698:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.825701:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.825702:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.825702:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.825704:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.825705:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.825705:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.825706:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.825710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.825718:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.825720:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.825721:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.825722:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.825724:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.825726:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.825917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.825926:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.825929:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.825931:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.825933:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.825933:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.825935:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.825936:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.825937:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.825938:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.825939:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.825942:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.825944:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.825946:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.825947:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.825948:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677447.825948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:31.0:1619677447.825949:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.825950:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.825951:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.825951:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677447.825963:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.825965:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.825966:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.825968:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.825969:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.825970:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.825979:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.825982:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.825982:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.825983:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.825984:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.825984:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.825988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677447.825990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:2.0:1619677447.825995:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.825998:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.825999:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.826000:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.826001:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.826005:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.826007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.826014:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.826016:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.826017:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.826018:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.826018:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.826019:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.826257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677447.826264:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677447.826267:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677447.826268:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677447.826270:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677447.826272:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.826273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:24.0:1619677447.826274:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677447.826275:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.826276:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.826277:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.826278:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677447.826281:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677447.826283:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677447.826286:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.826287:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677447.826287:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677447.826289:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677447.826289:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677447.826291:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677447.826292:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677447.826293:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:5.2:1619677447.826299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677447.826306:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:6.0:1619677447.826306:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677447.826308:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677447.826309:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:6.0:1619677447.826309:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.826310:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:8.0:1619677447.826310:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000400:00000200:6.0:1619677447.826311:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.826313:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.826315:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.2:1619677447.826317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677447.826320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677447.826325:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.826328:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.826329:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.826330:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.826332:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.826335:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.826336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.2:1619677447.826337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.826344:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.826347:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.826348:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.826348:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.826349:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.826350:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:9.2:1619677447.826375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.826378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.826383:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.826386:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.826387:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.826387:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.826390:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.826393:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.826405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.826412:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.826415:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.826415:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.826417:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.826420:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.826421:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.826422:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.826424:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.826425:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.826425:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.826428:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.826430:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.826433:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.826433:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.826434:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.826435:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.826436:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.826437:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.826437:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.2:1619677447.826438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.2:1619677447.826441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.826446:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.826448:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.826450:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:17.0:1619677447.826451:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.826452:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.826453:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.826454:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.826455:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.826457:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.826458:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677447.826459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:17.0:1619677447.826459:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.826460:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.826461:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.826464:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.826465:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:1.0:1619677447.826465:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.826466:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.826467:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:1.0:1619677447.826467:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.826468:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.826468:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.826470:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.826472:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.826472:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:18.0:1619677447.826473:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.826473:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:18.0:1619677447.826474:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677447.826475:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.826475:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.826476:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.826476:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677447.826477:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.826477:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:18.0:1619677447.826478:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.826479:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:21.2:1619677447.826482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:18.0:1619677447.826482:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.826484:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.826487:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.826488:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677447.826488:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:18.0:1619677447.826488:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677447.826489:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:22.0:1619677447.826491:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:18.0:1619677447.826491:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (25)--
00000800:00000200:12.0:1619677447.826491:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (25)--
00000800:00000200:6.0:1619677447.826491:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677447.826492:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.826492:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:12.0:1619677447.826492:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.826493:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:22.0:1619677447.826494:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677447.826495:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.826496:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000400:00000200:22.0:1619677447.826497:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.826497:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677447.826498:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.826498:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:22.0:1619677447.826499:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:12.0:1619677447.826499:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:22.0:1619677447.826500:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.826501:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.826502:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.826503:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.826503:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:22.0:1619677447.826504:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.826508:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.826509:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.826512:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.826513:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.826514:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.826516:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.826516:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.826517:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.826518:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677447.826537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.826540:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.826542:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.826545:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.826545:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.826547:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.826549:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.826550:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.826552:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.826553:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.826554:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.826557:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.826559:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.826562:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.826562:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.826563:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.826565:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.826565:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.826566:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.826566:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.826578:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.826581:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.826582:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.826582:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.826584:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.826586:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.826839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.826886:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.826889:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.826890:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.826890:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.826892:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.826895:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.826905:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.826907:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.826908:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.826909:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.826910:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.826910:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.827032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.827039:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.827042:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.827043:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.827045:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.827048:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.827049:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.827050:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.827052:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.827053:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.827053:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.827057:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.827058:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.827061:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.827062:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.827062:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.827064:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.827064:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.827065:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.827066:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.827070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677447.827070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.827078:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.827081:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.827082:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.827082:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.827085:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.827087:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677447.827123:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.827126:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.827127:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.827128:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.827130:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.827131:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.827213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.827221:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.827223:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.827225:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.827226:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.827227:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.827228:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677447.827229:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.827230:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.827231:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.827232:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677447.827235:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.827236:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677447.827239:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.827240:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.827241:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677447.827242:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.827243:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.827244:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.827244:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.827249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.827257:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.827259:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.827261:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.827262:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.827262:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.827266:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.827296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677447.827304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.827310:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.827312:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.827313:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.827316:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.827318:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.827320:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.827321:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96723017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.827323:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.827324:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.827325:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.827329:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.827331:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.827334:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.827335:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.827335:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.827337:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.827338:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.827338:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.827339:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.2:1619677447.827346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.2:1619677447.827347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.827351:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677447.827351:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:31.0:1619677447.827353:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.2:1619677447.827353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:31.0:1619677447.827354:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.827354:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.827355:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.827355:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:31.0:1619677447.827356:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.827356:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677447.827357:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677447.827358:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:22.0:1619677447.827359:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.827359:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:31.0:1619677447.827360:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.827361:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.827361:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677447.827363:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677447.827363:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677447.827363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:20.0:1619677447.827363:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.827363:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677447.827364:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.827365:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.827365:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.827366:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:20.0:1619677447.827366:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.827367:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.827367:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:31.0:1619677447.827368:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:20.0:1619677447.827368:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677447.827369:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.827369:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:20.0:1619677447.827370:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.827371:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.827372:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677447.827372:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.827372:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:30.0:1619677447.827373:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.827374:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677447.827374:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.827375:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677447.827376:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.827380:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.827381:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.827383:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677447.827384:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:6.0:1619677447.827385:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.827386:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677447.827386:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:30.0:1619677447.827387:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677447.827387:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.827388:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677447.827388:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.827388:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:6.0:1619677447.827388:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96723013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.827389:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:6.0:1619677447.827389:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.827390:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677447.827390:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.827390:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.827391:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677447.827391:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677447.827392:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:20.0:1619677447.827392:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.827392:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:6.0:1619677447.827392:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:30.0:1619677447.827393:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677447.827395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677447.827395:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.827397:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:21.2:1619677447.827399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677447.827400:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.827401:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.827401:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677447.827403:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677447.827404:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.827404:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677447.827406:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:6.0:1619677447.827406:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677447.827406:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:22.0:1619677447.827407:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.827407:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000400:00000200:2.0:1619677447.827407:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.827408:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677447.827408:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:31.0:1619677447.827409:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:22.0:1619677447.827409:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:2.0:1619677447.827409:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.827410:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677447.827410:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.827410:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677447.827411:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:22.0:1619677447.827412:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:21.2:1619677447.827413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677447.827413:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.827415:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.827416:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.827418:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.827420:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.827420:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.827421:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.827423:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.827424:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.827424:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.827425:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.827425:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.827598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.827638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.827643:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:21.2:1619677447.827644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:6.0:1619677447.827645:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.827646:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.827647:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.827649:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.827650:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:6.0:1619677447.827652:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677447.827653:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.827653:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.827656:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.827658:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.827659:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677447.827661:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.827662:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.827663:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.827664:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677447.827667:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.827669:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.827672:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.827672:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:18.0:1619677447.827673:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677447.827675:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.827675:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.827676:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.827676:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677447.827688:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.827691:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.827692:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.827694:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.827696:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.827697:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.827771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.827778:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.827781:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.827782:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.827785:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.827787:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.827788:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.827790:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96723010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.827792:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.827793:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.827794:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.827797:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.827799:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.827802:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.827803:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.827803:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.827805:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.827805:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.827806:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.827807:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.827808:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.827811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.827813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.827819:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677447.827820:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.827822:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.827823:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.827823:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.827823:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677447.827823:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.827824:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.827825:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.827828:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.827856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677447.827863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.827869:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.827871:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.827872:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.2:1619677447.827874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:17.0:1619677447.827874:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.827877:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.827879:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677447.827880:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.827881:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677447.827882:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.827883:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.827884:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:31.0:1619677447.827885:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:30.0:1619677447.827888:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.827889:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.827890:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.827892:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.827892:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:31.0:1619677447.827894:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.827895:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96723007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.827895:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.827896:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.827896:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000400:00000200:31.0:1619677447.827897:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.827897:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:31.0:1619677447.827898:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.827899:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.827899:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.827900:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.827900:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677447.827902:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.827904:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:9.2:1619677447.827905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677447.827906:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.827907:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.827908:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677447.827908:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:31.0:1619677447.827909:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.827910:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.827911:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:22.0:1619677447.827911:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.827911:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677447.827912:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:24.0:1619677447.827912:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:22.0:1619677447.827912:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.827915:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:22.0:1619677447.827915:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.827915:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.827916:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.827917:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:24.0:1619677447.827918:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.827921:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677447.827921:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.827924:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.827925:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.827926:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.827928:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.827929:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.827941:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.827943:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.827944:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:21.2:1619677447.827945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:12.0:1619677447.827945:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.827945:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.827946:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.827952:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.827955:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.827958:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.827959:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.827959:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.827961:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677447.827962:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96723006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.827964:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.827965:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.827966:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677447.827969:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.827971:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677447.827974:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.827975:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677447.827975:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677447.827977:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.827977:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.827978:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.827979:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677447.827979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:6.0:1619677447.827990:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.827993:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.827994:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.827994:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.827997:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.828000:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677447.828019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.828026:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.828029:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.828030:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.828030:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.828032:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.828036:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.828176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.828181:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:4.2:1619677447.828182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.828184:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.828185:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.828187:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.828189:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.828191:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.828192:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.828193:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.828194:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.828196:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.828199:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.828201:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.828204:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.828205:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.828206:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.828208:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.828208:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.828209:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.828210:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.828211:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677447.828216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677447.828220:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.828221:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:12.0:1619677447.828223:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.828223:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.828224:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.828224:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:12.0:1619677447.828225:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.828225:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677447.828227:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.828228:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:12.0:1619677447.828228:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.828230:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.828231:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.828233:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.828234:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.828235:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:13.0:1619677447.828237:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.828240:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.828241:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.828242:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.828242:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.828244:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.828268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.828273:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.828276:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.828277:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.828279:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.828282:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.828283:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.828285:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96723002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.828286:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.828287:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.828288:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.828292:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.828297:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.828300:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.828301:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.828302:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.828304:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.828304:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.828305:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.828307:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.828307:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.828310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677447.828313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677447.828317:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677447.828319:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677447.828320:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.828321:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.828321:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:8.0:1619677447.828321:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.828322:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.828323:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677447.828323:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.828326:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677447.828363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:8.2:1619677447.828402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.828410:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.828413:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.828414:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.828415:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677447.828416:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.828418:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.828455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.828471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.828474:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.828477:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.2:1619677447.828478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:20.0:1619677447.828479:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.828481:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.828482:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.828483:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:18.0:1619677447.828484:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:20.0:1619677447.828485:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.828486:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.828487:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.828488:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.828488:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:20.0:1619677447.828489:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.828492:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677447.828493:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.828494:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.828495:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.2:1619677447.828495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:18.0:1619677447.828496:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.828497:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96722998] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.828498:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.828499:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.828500:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.828500:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677447.828500:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.828501:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677447.828501:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677447.828502:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.828502:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.828503:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.828504:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677447.828504:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:22.0:1619677447.828505:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677447.828505:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000400:00000200:18.0:1619677447.828505:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677447.828505:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677447.828506:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.828506:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677447.828506:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:1.0:1619677447.828507:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.828508:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.828508:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.828510:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.828511:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.828511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.828511:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (21)++
00000400:00000200:1.0:1619677447.828511:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.828512:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:1.0:1619677447.828512:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:18.0:1619677447.828514:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:1.0:1619677447.828514:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96722997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677447.828515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677447.828515:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:12.0:1619677447.828515:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677447.828515:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.828516:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:1.0:1619677447.828516:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.828517:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.828517:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:1.0:1619677447.828517:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:12.0:1619677447.828518:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677447.828518:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:12.0:1619677447.828519:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.828521:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.828521:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:1.0:1619677447.828521:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677447.828522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.0:1619677447.828522:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677447.828523:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.828523:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677447.828524:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677447.828524:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.828524:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677447.828525:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.828526:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677447.828527:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:6.0:1619677447.828527:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:1.0:1619677447.828527:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.828528:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:1.0:1619677447.828528:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:30.0:1619677447.828529:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.828529:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.828529:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.828530:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677447.828530:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:1.0:1619677447.828530:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.828531:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.828531:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.828531:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:31.0:1619677447.828532:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.828532:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.828533:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677447.828534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:6.0:1619677447.828534:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:30.0:1619677447.828535:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677447.828536:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96722996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.828537:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:30.0:1619677447.828538:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.828539:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.828540:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677447.828542:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:30.0:1619677447.828543:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.828544:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:30.0:1619677447.828545:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:17.0:1619677447.828545:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.828546:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.828547:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.828549:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.828549:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:17.0:1619677447.828549:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677447.828550:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677447.828552:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.828552:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.828553:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.828554:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677447.828566:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.828569:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.828569:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.828570:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.828572:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.828576:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.828577:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.828579:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.828580:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.828580:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.828581:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.828582:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.828648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.828656:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.828659:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.828661:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.828663:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.828664:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.828666:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.828667:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.828668:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.828669:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.828670:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.828673:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.828675:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.828677:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.828678:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.828678:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.828680:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.828681:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.828682:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.828684:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.828684:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677447.828690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677447.828692:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.828694:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.828695:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.828696:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.828698:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:12.0:1619677447.828698:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.828700:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:13.0:1619677447.828700:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.828701:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.828702:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.2:1619677447.828771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.2:1619677447.828775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.828781:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.828784:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.828784:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.828787:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.828789:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.828791:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.828793:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.828795:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.828797:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.828798:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.828804:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.828806:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.828809:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.828810:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.828811:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.828813:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.828814:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.828815:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.828815:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.828821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.828826:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677447.828829:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:12.0:1619677447.828829:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.828830:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.828831:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:12.0:1619677447.828831:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.828833:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.828833:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.828833:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:12.0:1619677447.828834:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:18.0:1619677447.828835:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.828838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.828841:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.828843:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.828845:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.828846:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.828846:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.828847:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.828848:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.828894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.828900:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.828902:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.828903:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.828905:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.828908:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.828910:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.828911:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.828913:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.828914:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.828915:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.828918:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.828920:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.828921:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.828922:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.828923:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.828924:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.828924:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.828925:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.828925:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.828930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.828938:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.2:1619677447.828940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:1.0:1619677447.828940:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.828940:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.828941:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.828942:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.828944:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.828946:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:8.2:1619677447.828947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.828949:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.828950:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.828951:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.828954:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.828955:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677447.828956:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.828957:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.828958:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.828959:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.828959:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000400:00000200:2.0:1619677447.828959:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.828959:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.828960:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:2.0:1619677447.828960:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.828961:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.828964:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.828966:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.828970:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.828970:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.828971:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.828973:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.828974:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.828974:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.828975:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677447.828985:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.828988:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.828989:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.828991:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.828992:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.828994:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.829068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.829075:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.829078:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.829080:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.829082:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.829083:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.829084:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677447.829085:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.829087:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.829088:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.829089:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677447.829092:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.829094:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677447.829096:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.829097:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677447.829098:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677447.829099:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.829100:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.829101:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.829101:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.829113:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.829116:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.829117:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.829117:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.829119:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.829122:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.829135:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.829138:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.829138:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.829139:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.829140:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:21.2:1619677447.829140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:22.0:1619677447.829141:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677447.829153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.829158:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.829161:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.829161:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.829164:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.829167:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.829169:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.829170:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.829172:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.829173:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.829174:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.829177:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.829179:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.2:1619677447.829181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.829182:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.829183:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677447.829184:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (23)++
00000800:00000200:5.2:1619677447.829184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:6.0:1619677447.829186:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:6.0:1619677447.829186:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.829187:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.829188:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.829189:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)++
00000800:00000200:1.0:1619677447.829189:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677447.829191:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (24)--
00000400:00000200:20.0:1619677447.829192:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.829192:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.829193:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677447.829195:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.829195:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)++
00000800:00000200:5.0:1619677447.829196:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (24)--
00000400:00000200:1.0:1619677447.829196:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.829197:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.829197:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.829198:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.829199:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.829199:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.829199:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.829201:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.829202:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:1.0:1619677447.829202:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.829203:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.829204:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.2:1619677447.829206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000400:00000200:1.0:1619677447.829208:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.829210:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.829213:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:1.0:1619677447.829213:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.829214:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.829215:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.829216:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677447.829216:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.829217:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:1.0:1619677447.829217:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677447.829218:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.829218:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.829219:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677447.829219:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.829220:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.829220:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:4.2:1619677447.829222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677447.829225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677447.829229:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677447.829231:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677447.829233:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:17.0:1619677447.829233:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:31.0:1619677447.829234:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.829234:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.829234:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677447.829236:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.829236:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.829239:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677447.829266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.829270:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.829273:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.829274:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.829274:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.829276:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.829279:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.829323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.829331:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.829334:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.829335:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.829336:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.829340:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.829341:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.829342:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.829344:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.829344:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.829345:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.829349:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.829350:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.829353:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.829353:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.829354:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.829356:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.829356:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.829357:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.829357:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.829362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677447.829369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:17.0:1619677447.829370:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677447.829371:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677447.829373:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677447.829374:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:17.0:1619677447.829374:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.829374:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:20.0:1619677447.829377:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677447.829377:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.829377:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.829380:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.829380:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:20.0:1619677447.829382:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.829383:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.829384:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.829385:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.829386:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.829390:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.829392:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.829394:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.829395:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.829396:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.829398:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.829399:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.829399:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.829400:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677447.829405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.829413:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.829416:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.829417:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.829417:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.829419:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.829422:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.829429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.829435:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.829437:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.829438:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.829440:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.829443:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.829444:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.829445:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.829447:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.829448:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.829449:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.829452:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.829453:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.829455:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.829456:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.829456:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.829458:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.829458:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.829459:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.829459:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.829463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.829470:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.829472:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.829473:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.829473:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.829474:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.829476:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.829563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.2:1619677447.829605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677447.829610:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.829612:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.829613:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.829614:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.829617:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.829618:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.829629:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.829632:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.829632:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.829633:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.829633:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.829634:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.829654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.829662:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.829664:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.829666:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.829669:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.829669:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.829670:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.829671:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.829672:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677447.829672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:30.0:1619677447.829673:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.829674:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.829677:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.829678:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677447.829679:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.829681:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.829681:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:30.0:1619677447.829682:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:22.0:1619677447.829682:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677447.829683:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:6.0:1619677447.829683:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.829684:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.829684:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677447.829685:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.829686:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.829687:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:6.0:1619677447.829687:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677447.829688:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:6.0:1619677447.829688:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.829689:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.829690:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.829691:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:21.2:1619677447.829693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677447.829694:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.829695:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677447.829696:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:12.0:1619677447.829698:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.829698:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.829699:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.829699:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:12.0:1619677447.829700:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.829700:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677447.829701:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677447.829702:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677447.829702:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.829703:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:12.0:1619677447.829703:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.829703:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.829703:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:31.0:1619677447.829704:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677447.829704:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677447.829705:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677447.829715:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.829717:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.829718:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.829720:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.829721:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.829723:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.829804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.2:1619677447.829809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:1.0:1619677447.829810:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.829813:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.829814:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677447.829816:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:1.0:1619677447.829816:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.829818:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.829819:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:1.0:1619677447.829819:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:24.0:1619677447.829820:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:1.0:1619677447.829821:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.829823:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.829823:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.829824:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.829825:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:8.0:1619677447.829826:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.829828:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.829829:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.829830:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96722978] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.829831:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.829832:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.829832:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:8.0:1619677447.829833:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.829834:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.829835:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.829836:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:8.0:1619677447.829837:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.829837:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.829838:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677447.829839:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.829839:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.829839:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677447.829841:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.829842:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.829843:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.829844:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.829845:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.829845:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.829846:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.829851:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.829854:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.829855:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.829855:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:13.0:1619677447.829856:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:22.0:1619677447.829857:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.829858:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:22.0:1619677447.829859:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:13.0:1619677447.829860:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.829862:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.829862:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.829864:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.830128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:12.0:1619677447.830171:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.830174:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.830174:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.830176:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.830178:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.830179:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.830278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677447.830283:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.830286:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.830287:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.830288:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.830291:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.830293:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677447.830300:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.830301:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.830302:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.830303:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677447.830306:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.830308:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.830309:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.830310:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.830311:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.830312:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.830313:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.830314:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.830314:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.830319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677447.830326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:18.0:1619677447.830326:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.830328:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.830329:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.830330:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.830331:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.830332:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:6.0:1619677447.830332:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.830335:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.830336:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.830338:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.830340:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.830341:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.830343:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.830344:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.830345:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.830346:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.830349:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.830351:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.830353:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.830354:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.830355:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.830356:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.830357:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.830357:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.830358:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.830363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.830371:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.830373:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.830374:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.830376:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.830377:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.2:1619677447.830378:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:5.0:1619677447.830383:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:24.0:1619677447.830385:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677447.830388:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677447.830389:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677447.830391:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677447.830393:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.830395:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677447.830396:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.830397:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.830398:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.830399:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677447.830403:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677447.830404:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677447.830407:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.830408:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677447.830408:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677447.830410:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677447.830410:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677447.830411:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677447.830412:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:22.0:1619677447.830424:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.830426:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.830427:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.830428:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.830430:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.830432:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.830639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.830647:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.830650:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.830652:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.830654:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.830655:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.830656:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.830657:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.830658:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.830659:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.830660:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.830663:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.830665:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.830667:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.830667:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.830668:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.830669:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.830670:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.830671:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.830671:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.2:1619677447.830679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.830682:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677447.830683:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.0:1619677447.830685:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:13.0:1619677447.830685:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.830686:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.830687:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677447.830688:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.830688:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.830690:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.830690:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.830691:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:20.0:1619677447.830692:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.830693:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.830695:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.830696:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.830697:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.830700:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.830702:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.830705:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.830706:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.830707:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.830709:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.830709:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.830710:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.830711:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.830721:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.830724:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.830725:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.830726:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.830727:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.830729:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.830742:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.830745:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.830746:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.830747:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.830747:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.830749:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.830858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.830864:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.830867:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.830868:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.830869:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.830872:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.830874:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.830875:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.830876:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.830877:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.830878:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.830881:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.830883:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.830885:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.830886:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.830886:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.830888:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.830889:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.830889:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.830890:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.830895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.830903:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.830906:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.830907:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.830907:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.830909:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.830911:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.830956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.830961:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.830964:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.830966:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.830966:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.830968:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.830970:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.830971:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.830973:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.830974:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.830975:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.830979:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.830981:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.830984:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.830984:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.830985:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.830987:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.830988:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.830989:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.830991:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.830991:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.2:1619677447.830996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.830999:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:31.0:1619677447.831000:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677447.831001:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.831002:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677447.831003:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.831003:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.831003:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:31.0:1619677447.831005:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:30.0:1619677447.831006:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.831008:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.831018:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.831021:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.831022:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.831022:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.831023:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.831024:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.831221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677447.831227:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.831230:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.831231:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.831233:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.831235:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.831237:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.831238:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.831240:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.831241:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.831242:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.831245:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.831247:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.831250:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.831251:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.831252:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.831254:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.831254:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.831255:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.831255:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.2:1619677447.831272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:4.2:1619677447.831286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.831293:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.831302:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.831303:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.831306:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.831309:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.831310:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.831311:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96722967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.831313:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.831313:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.831314:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.831317:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.831319:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.831322:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.831323:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.2:1619677447.831324:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (22)++
00000800:00000200:8.0:1619677447.831328:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (23)++
00000800:00000200:8.0:1619677447.831329:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (24)--
00000800:00000200:8.0:1619677447.831330:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.831331:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.831331:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.831333:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (23)++
00000800:00000200:17.0:1619677447.831336:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (24)++
00000800:00000200:1.0:1619677447.831336:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (24)--
00000800:00000200:17.0:1619677447.831337:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (24)--
00000400:00000200:1.0:1619677447.831337:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.831338:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.831339:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.831340:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.831343:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.831344:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.831344:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.831414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.831423:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.831426:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.831428:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.831430:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.831430:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.831431:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.831432:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.831434:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.831435:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.831436:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.831439:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.831440:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.831443:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.831443:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.831444:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.831446:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.831446:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.831447:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.831448:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677447.831460:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.831462:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.831463:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.831464:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.831466:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.831467:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677447.831482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.831485:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.831488:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.831490:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.831492:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.831493:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.831494:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.831496:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.831497:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.831498:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.831499:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.831502:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.831504:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.831507:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.831508:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.831509:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.831511:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.831512:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:4.2:1619677447.831512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.831513:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.831513:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.831524:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.831527:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.831528:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.831530:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.831532:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.831533:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677447.831554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.831562:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.831565:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.831566:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.831567:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.831568:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.831571:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677447.831574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.831582:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.831585:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.831586:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.831586:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.831587:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.831588:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677447.831791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.831799:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.831801:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.831804:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.831805:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.831806:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.831807:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677447.831809:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.831810:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.831811:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.831812:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677447.831815:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.831816:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677447.831818:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.831819:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677447.831820:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677447.831821:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.831822:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.831823:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.831823:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677447.831840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677447.831847:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.831850:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.831851:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.831853:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.831856:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.831858:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.831859:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.831860:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.831861:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.831862:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.831865:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.831867:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.831869:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.831870:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.831871:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.831872:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.831873:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.831874:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.831875:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.831875:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677447.831879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677447.831881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.831887:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.831888:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.831888:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.831888:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.831889:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.831889:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.831890:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.831891:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.831891:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.831892:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:21.2:1619677447.831895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:6.0:1619677447.831901:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.831904:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677447.831905:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677447.831907:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.831909:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.831911:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.831912:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.831913:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.831914:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.831915:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.831919:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.831920:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.831922:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.831923:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.831924:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.831925:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.831926:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.831927:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677447.831927:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677447.831932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.831939:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.831942:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.831943:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.831943:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.831945:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.831947:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.831955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.831962:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.831964:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.831965:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.831966:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.831967:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.831967:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677447.832020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.832028:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.832031:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.832033:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.832035:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.832036:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.832037:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677447.832038:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.832040:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.832041:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.832042:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677447.832045:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.832047:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.832049:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.832050:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:4.2:1619677447.832050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677447.832051:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.832052:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.832053:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.832053:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.832054:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677447.832065:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.832067:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.832068:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.832070:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.832071:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:21.2:1619677447.832072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:31.0:1619677447.832073:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:22.0:1619677447.832078:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.832080:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.832081:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.832083:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.832086:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.832087:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.832088:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677447.832088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:22.0:1619677447.832090:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.832091:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.832091:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:22.0:1619677447.832093:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:20.0:1619677447.832093:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.832096:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.832096:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.832097:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677447.832097:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:6.0:1619677447.832098:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.832099:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.832099:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.832101:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.832101:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.832102:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:20.0:1619677447.832103:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96722957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.832104:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.832105:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.832106:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.832106:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.832106:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.832107:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000400:00000200:20.0:1619677447.832107:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.832108:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.832110:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000400:00000200:20.0:1619677447.832110:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.832111:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.832112:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:20.0:1619677447.832112:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.832113:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.832114:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677447.832115:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.832116:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.832117:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.832118:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:21.2:1619677447.832119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677447.832119:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.832120:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.832120:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.832124:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.832126:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.832127:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.832128:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677447.832131:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.832134:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.832135:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.832136:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.832137:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.832139:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.832190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:5.0:1619677447.832196:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:5.0:1619677447.832198:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:20.0:1619677447.832201:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:5.0:1619677447.832201:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.832204:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.832205:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:20.2:1619677447.832206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:5.0:1619677447.832207:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.832208:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.832209:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.832210:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:5.0:1619677447.832213:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.832215:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:5.0:1619677447.832218:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.832220:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:5.0:1619677447.832220:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:5.0:1619677447.832222:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:5.0:1619677447.832223:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:5.0:1619677447.832224:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:5.0:1619677447.832224:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:5.2:1619677447.832232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.832237:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.832241:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.832241:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.832244:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.832246:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.832247:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.832248:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677447.832248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:17.0:1619677447.832249:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.832251:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.832252:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.832256:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.832257:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.832258:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.832258:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:5.2:1619677447.832259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:1.0:1619677447.832259:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.832261:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677447.832261:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:17.0:1619677447.832262:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.832263:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.832264:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.832264:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.832265:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.832266:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.832266:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.832267:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.832272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:22.0:1619677447.832278:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677447.832279:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677447.832280:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.832281:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.832282:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.832282:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.832283:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.832283:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000400:00000200:22.0:1619677447.832284:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.832284:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.832284:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:22.0:1619677447.832288:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.832315:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.832317:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.832318:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.832319:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.832321:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.832322:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.832363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.832370:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.832374:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.832375:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.832377:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.832380:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.832381:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.832382:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.832384:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.832385:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.832386:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.832389:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.832390:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.832393:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.832394:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.832395:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.832397:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.832397:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.832398:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.832398:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.832402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.832410:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.832413:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.832414:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677447.832415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:1.0:1619677447.832415:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.832416:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.832419:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.832420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.832422:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:4.2:1619677447.832422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.832425:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.832425:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.832427:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.832427:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:18.0:1619677447.832429:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.832430:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.832431:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:17.0:1619677447.832431:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.832431:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000400:00000200:18.0:1619677447.832432:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.832432:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.832432:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:18.0:1619677447.832434:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.832435:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.832436:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.832440:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.832443:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.832446:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.832446:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.832447:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.832449:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.832449:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.832450:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.832450:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677447.832455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.832462:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.832464:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.832465:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.832467:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.832469:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.832470:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:6.0:1619677447.832473:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.832476:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.832477:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.832477:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.832479:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.832482:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.832491:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.832494:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.832495:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.832495:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.832496:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.832497:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677447.832571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.832580:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.832583:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.832585:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.832587:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.832587:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.832589:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677447.832590:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722949] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.832591:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.832592:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.832593:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677447.832597:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.832598:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677447.832601:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.832602:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677447.832602:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677447.832604:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.832604:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.832605:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.832606:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677447.832607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.832615:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.832618:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.832620:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.832621:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.832622:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.832623:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677447.832624:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.832625:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.832626:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.832627:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677447.832631:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.832632:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677447.832635:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.832636:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.832637:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677447.832639:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.832639:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.832640:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.832642:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.832642:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677447.832644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677447.832648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677447.832652:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.832654:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:31.0:1619677447.832656:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:5.0:1619677447.832656:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.832657:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.832657:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.832659:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.832660:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677447.832660:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677447.832661:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:4.2:1619677447.832668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.832695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.832698:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.832700:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.832702:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.832703:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.832705:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.832707:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:4.2:1619677447.832707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677447.832709:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.832710:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.832711:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:20.0:1619677447.832712:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.832713:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677447.832714:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.832715:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.832715:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.832716:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.832717:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.832718:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677447.832720:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.832721:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.832722:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.832723:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.832725:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.832726:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.832727:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.832728:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.832729:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.832734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677447.832737:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.832739:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677447.832740:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.832741:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.832742:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:12.0:1619677447.832743:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.832743:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.832743:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.832744:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.832746:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.832769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.832775:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.832778:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.832779:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.832781:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.832783:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.832784:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.832786:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722945] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.832787:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.832788:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.832789:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.832793:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.832794:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.832796:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.832797:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.832798:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.832799:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.832799:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.832800:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.832800:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.832805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.832812:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.832814:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.832815:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.832815:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.832817:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.832818:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677447.832837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677447.832842:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.832845:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.832846:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.832847:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.832849:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.832851:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677447.832852:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.832853:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.832854:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.832855:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677447.832858:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.832860:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.832863:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.832863:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677447.832864:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677447.832866:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.832866:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.832867:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.832867:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677447.832878:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.832881:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.832882:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.832884:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.832885:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.832887:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.832901:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.832904:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.832905:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.832905:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.832906:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.832908:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.832918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677447.832927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.832932:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.832935:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.832936:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.832939:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.832942:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.832944:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.832946:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.832947:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.832948:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677447.832948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:22.0:1619677447.832949:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.832953:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.832953:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677447.832955:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677447.832956:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.832957:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.832959:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.832960:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.832960:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:17.0:1619677447.832960:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.832962:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000400:00000200:17.0:1619677447.832962:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.832963:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.832963:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:17.0:1619677447.832963:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:5.2:1619677447.832963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677447.832964:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:17.0:1619677447.832965:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96722941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.832966:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.832967:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:17.0:1619677447.832967:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.832968:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.832970:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.832971:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.832972:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.832972:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.832974:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.832974:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.832975:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677447.832977:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.832978:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.832979:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.832981:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.832981:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.832982:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.832983:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.832984:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677447.832989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677447.832993:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677447.832994:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677447.832996:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677447.832996:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:13.0:1619677447.832997:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.832997:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677447.832998:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.832998:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.832999:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.833001:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.833005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677447.833008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677447.833013:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.833016:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.833017:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.833017:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.833019:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.833023:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.833025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.833032:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.833035:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.833036:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.833036:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.833037:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.833037:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.833083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.833091:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.833094:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.833094:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.833096:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.833100:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.833101:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.833102:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.833104:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.833105:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.833105:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.833109:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.833110:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.833113:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.833114:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.833114:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.833116:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.833116:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.833118:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.833119:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.833119:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.833121:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677447.833125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.833129:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677447.833132:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677447.833132:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.833133:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.833133:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.833134:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.833135:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677447.833135:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.833136:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.833138:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677447.833217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677447.833223:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.833225:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.833226:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.833228:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.833230:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.833232:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677447.833233:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.833234:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.833235:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.833236:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677447.833240:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.833242:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:5.2:1619677447.833242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677447.833245:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.833245:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677447.833246:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677447.833248:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677447.833248:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677447.833248:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.833249:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.833249:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677447.833251:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.833252:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.833253:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.833256:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.833257:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.833259:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.833260:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.833261:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.833262:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.833265:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.833268:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.833271:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.833272:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.833273:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.833275:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.833275:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.833276:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.833277:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.833281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.833287:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:21.2:1619677447.833290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677447.833290:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.833291:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.833291:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.833300:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:8.0:1619677447.833300:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.833301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:30.0:1619677447.833303:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.0:1619677447.833304:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:30.0:1619677447.833305:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.833305:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.833307:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.833307:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:31.0:1619677447.833308:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.833308:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.833308:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000400:00000200:30.0:1619677447.833309:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:8.0:1619677447.833309:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.833310:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:30.0:1619677447.833311:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.833312:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.833313:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.833314:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.833317:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.833319:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.833321:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.833322:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.833323:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.833324:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.833325:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.833326:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.833326:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.833337:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.2:1619677447.833338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677447.833339:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.833340:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.833342:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.833344:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.833345:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.833345:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.833347:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.833349:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.833351:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.833354:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.833355:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.833357:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.833358:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.833359:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.833360:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.833363:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.833364:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.833366:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.833367:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.833367:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.833369:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.833369:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.833371:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.833372:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.833372:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.833375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677447.833378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.833383:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.833384:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.833384:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.833384:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.833385:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:18.0:1619677447.833386:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.833386:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.833387:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.833387:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.833388:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:4.2:1619677447.833407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677447.833430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.833436:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.833439:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.833440:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.833442:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.833444:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.833446:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.833447:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.833448:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.833449:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.833450:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.833454:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.833456:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.833459:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.833460:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.833461:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.833463:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.833463:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.833464:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.833465:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.833475:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.833478:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.833478:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.833480:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.833481:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.833483:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.833493:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.833496:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.833497:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.833498:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.833498:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.833499:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.833515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677447.833522:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.833525:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.833526:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.833528:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.833530:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.833532:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.833533:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.833534:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.833535:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.833536:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.833540:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.833542:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.833545:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.833545:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.833546:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.833548:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.833549:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.833549:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.833550:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.2:1619677447.833551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:9.2:1619677447.833554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677447.833559:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.833561:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:2.0:1619677447.833562:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:31.0:1619677447.833563:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.833564:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:30.0:1619677447.833565:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.833566:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.833566:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.833566:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.833568:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677447.833569:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.833570:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.833571:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.833572:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.833572:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:31.0:1619677447.833573:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.833576:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.833577:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.833580:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.833581:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.833581:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.833583:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.833584:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.833585:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.833585:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.833595:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.833597:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.833598:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.833600:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.833602:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.833603:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.833664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677447.833678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.833684:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.833686:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.833687:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.833689:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.833692:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.833694:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.833695:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.833696:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.833697:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.833698:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677447.833701:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677447.833702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.833703:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.833706:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.833707:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.833707:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.833709:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.833709:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.833710:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677447.833710:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.833711:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677447.833713:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.833714:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.833714:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:9.2:1619677447.833715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:18.0:1619677447.833716:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.2:1619677447.833717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:18.0:1619677447.833719:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.0:1619677447.833721:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.833724:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.833724:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.833725:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.833728:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.833731:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.2:1619677447.833739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.833744:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.833747:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.833748:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.833750:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.833751:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.833753:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.833754:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.833756:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.833757:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.833758:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.833761:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.833763:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677447.833764:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.833766:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677447.833766:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.833767:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677447.833767:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:6.0:1619677447.833768:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.833768:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.833769:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:6.0:1619677447.833770:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.833770:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.833771:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.833771:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:6.0:1619677447.833773:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.833782:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:4.2:1619677447.833782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:13.0:1619677447.833785:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.833786:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.833787:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.833788:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.833789:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677447.833790:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.833791:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.833792:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.833794:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.833797:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.833798:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.833799:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.833801:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.833802:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.833803:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.833806:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.833807:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.2:1619677447.833808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:24.0:1619677447.833810:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.833811:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.833812:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.833813:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.833814:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.833814:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677447.833815:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677447.833815:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:30.0:1619677447.833818:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.2:1619677447.833818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:8.2:1619677447.833819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:30.0:1619677447.833820:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.833822:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.833822:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.833823:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677447.833824:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:30.0:1619677447.833825:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.833826:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.833827:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.833827:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:5.0:1619677447.833827:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:30.0:1619677447.833828:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.833829:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.833829:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.833830:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.833831:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.833831:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:30.0:1619677447.833832:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.833832:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.833834:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.833834:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:30.0:1619677447.833835:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.833836:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.833837:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:5.0:1619677447.833837:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.833838:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.833839:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.833839:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677447.833839:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.833840:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677447.833841:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.833841:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.833843:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:5.0:1619677447.833843:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.833844:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.833845:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.0:1619677447.833845:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.833848:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.833849:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.833850:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677447.833851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677447.833852:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:12.0:1619677447.833853:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.833854:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.833855:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:12.0:1619677447.833856:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.833856:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.833856:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:12.0:1619677447.833858:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.833858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677447.833859:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677447.833860:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.833861:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:12.0:1619677447.833861:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.833862:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.833863:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677447.833866:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.833869:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.833870:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.833870:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.833872:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.833876:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.833877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.833885:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.833888:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.833889:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.833889:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.833890:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.833891:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.833948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.833953:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.833956:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.833957:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677447.833957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000400:00000200:1.0:1619677447.833959:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.833961:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.833962:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.833964:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722923] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.833965:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000400:00000200:1.0:1619677447.833965:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.833966:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.833967:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.833968:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:12.0:1619677447.833971:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:1.0:1619677447.833971:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.833973:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:13.0:1619677447.833975:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.833977:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.833977:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:13.0:1619677447.833978:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.833978:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:13.0:1619677447.833979:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:1.0:1619677447.833980:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.833980:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:13.0:1619677447.833981:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.833981:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:13.0:1619677447.833982:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.833982:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000400:00000200:13.0:1619677447.833983:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.833983:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677447.833984:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:13.0:1619677447.833988:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677447.833988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677447.833990:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677447.833992:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677447.833993:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677447.833993:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.833993:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.833994:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677447.833994:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:12.0:1619677447.833994:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.833995:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.833996:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677447.833996:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:12.0:1619677447.833996:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.833997:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.833997:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677447.833998:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.833999:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677447.833999:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.834000:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677447.834001:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:9.2:1619677447.834002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677447.834006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:18.0:1619677447.834011:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.834013:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:12.0:1619677447.834014:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:2.0:1619677447.834014:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.834015:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.834016:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000400:00000200:18.0:1619677447.834017:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.834017:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677447.834018:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:18.0:1619677447.834020:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.834083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.834123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677447.834127:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.834129:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.834130:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.834132:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.834133:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.834135:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.834151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.834156:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.834159:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.834160:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.834162:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.834165:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.834166:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.834168:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.834169:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.834170:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.834171:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.834175:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.834176:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.834179:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.834180:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.834180:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.834182:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.834182:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.834183:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.834184:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.834196:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.834198:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.834199:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.834200:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.834202:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.834204:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677447.834210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677447.834215:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677447.834217:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677447.834218:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.834219:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.834219:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:30.0:1619677447.834220:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:6.0:1619677447.834220:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.834220:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:30.0:1619677447.834222:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.834224:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.834224:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.834226:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677447.834227:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.834228:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.834229:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.834230:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677447.834233:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.834235:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.834238:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.834239:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677447.834239:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677447.834241:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.834242:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.834243:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.834243:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677447.834254:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.834257:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.834258:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.834258:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.834260:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.834263:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.834344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.834350:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.834353:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.834354:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.834355:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.834358:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.834359:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.834360:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.834361:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.834362:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.834363:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.834366:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.834368:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.834369:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.834370:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.834371:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.834372:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.834373:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.834373:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.834374:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.834381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.834389:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.834391:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.834391:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.834392:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.834393:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.834395:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.834471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677447.834576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.834582:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.834585:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.834586:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.834588:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.834590:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.834592:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.834594:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722916] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.834595:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.834596:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.834597:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.834600:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.834602:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.834605:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.834606:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.834606:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.834608:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.834609:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.834609:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.834610:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677447.834622:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.834625:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.834626:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.834627:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.834629:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.834630:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.834631:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.834631:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:31.0:1619677447.834632:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.834632:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.834633:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.834666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.834672:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.834674:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.834675:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.834677:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.834679:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.834680:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.834681:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.834683:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.834684:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.834685:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.834689:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.834691:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.834693:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.834694:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.834695:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.834697:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.834697:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.834698:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.834698:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.834710:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.834713:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.834714:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677447.834714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:31.0:1619677447.834715:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.834717:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.834718:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.0:1619677447.834722:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.834725:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.834725:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677447.834727:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.834730:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.834732:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.834733:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.834734:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.834735:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.834736:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.834740:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.834741:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.834744:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.834745:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.834746:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.834748:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.834749:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:5.2:1619677447.834749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.0:1619677447.834750:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.834751:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.834752:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677447.834754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677447.834757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:2.0:1619677447.834762:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677447.834764:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:2.0:1619677447.834765:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:24.0:1619677447.834766:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:18.0:1619677447.834766:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.834766:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.834767:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.834768:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:2.0:1619677447.834769:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.834772:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.834849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:20.2:1619677447.834852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.0:1619677447.834857:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:30.0:1619677447.834860:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:8.0:1619677447.834860:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677447.834861:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:30.0:1619677447.834862:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:8.0:1619677447.834863:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.834865:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.834866:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:8.0:1619677447.834866:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.834867:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.834868:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677447.834869:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:8.0:1619677447.834869:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.834870:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.834870:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.834871:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.834872:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.834872:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677447.834873:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.834874:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:8.0:1619677447.834876:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.834877:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.834878:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677447.834879:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.0:1619677447.834881:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.834882:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.834882:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:8.0:1619677447.834882:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677447.834883:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.834884:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.0:1619677447.834884:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:8.0:1619677447.834885:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:30.0:1619677447.834886:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.834886:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:8.0:1619677447.834886:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:30.0:1619677447.834887:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:8.0:1619677447.834887:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:30.0:1619677447.834888:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.0:1619677447.834888:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:9.2:1619677447.834890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.2:1619677447.834892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:4.2:1619677447.834893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (21)++
00000800:00000200:18.0:1619677447.834898:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:24.0:1619677447.834900:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:20.0:1619677447.834900:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677447.834900:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.834901:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.834902:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.834903:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:20.0:1619677447.834903:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:24.0:1619677447.834904:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:20.0:1619677447.834904:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.834904:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.834904:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000400:00000200:20.0:1619677447.834906:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.834907:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677447.834907:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.834909:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677447.834925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677447.834932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677447.834933:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.834935:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.834937:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.834939:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.834940:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.834942:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.834944:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.834945:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.834946:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.834947:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.834951:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.834953:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.834956:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.834956:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.834957:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.834958:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.834959:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.834960:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.834960:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.834972:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.834974:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.834975:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.834976:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.834978:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:12.0:1619677447.834979:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.834980:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.834981:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.834982:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.834982:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.834984:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.834987:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677447.835142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.835148:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.835150:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.835151:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.835153:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677447.835155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:6.0:1619677447.835155:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.835157:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.835158:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722908] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.835159:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.835160:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.835161:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.835165:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.835168:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.835171:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.835171:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.835172:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.835174:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.835174:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.835175:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.835176:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.835180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.835188:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.835190:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.835191:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.835193:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.835193:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.835195:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:9.2:1619677447.835196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.2:1619677447.835199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.835204:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677447.835206:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.835206:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.835207:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.835208:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.835209:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.835210:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000400:00000200:2.0:1619677447.835210:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.835211:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.835212:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.835213:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677447.835214:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.835274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.835280:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.835283:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.835284:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.835286:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.835289:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.835290:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.835292:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.835299:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.835300:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.835301:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.835304:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.835305:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.835308:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.835309:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.835310:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.835311:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.835312:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.835312:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.835313:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.835317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.835326:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.835329:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.835329:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.835330:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.835332:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.835335:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.835462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.835468:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.835470:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.835471:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.835473:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.835475:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.835477:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.835478:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.835479:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.835480:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.835481:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.835484:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.835486:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.2:1619677447.835486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:17.0:1619677447.835488:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.835489:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.835490:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.835492:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.835492:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.835494:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.835494:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.835495:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677447.835500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.835505:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:30.0:1619677447.835506:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:1.0:1619677447.835507:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.835508:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677447.835509:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:1.0:1619677447.835509:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677447.835510:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.835512:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.835514:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.835515:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.835622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.835629:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.835631:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.835632:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.835634:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.835636:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.835638:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.835639:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.835640:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.835642:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.835642:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.835645:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.835647:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.835650:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.835651:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.835652:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.835654:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.835654:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.835656:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.835657:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.835657:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:21.2:1619677447.835662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677447.835667:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:6.0:1619677447.835667:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.835669:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:31.0:1619677447.835670:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:6.0:1619677447.835670:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:31.0:1619677447.835671:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.835671:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:31.0:1619677447.835673:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.835673:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.835676:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.835751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677447.835758:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.835761:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.835761:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.835764:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.835766:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.835768:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.835769:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.835770:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.835771:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.835772:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.835775:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.835777:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.835779:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.835780:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.835780:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.835782:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.835783:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.835784:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.835784:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.835788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.835796:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.835798:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.835799:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.835799:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.835801:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.835803:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.835811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.835818:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.835820:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.835821:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.835821:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.835822:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.835822:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677447.835906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677447.835912:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.835914:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.835915:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.835916:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.835918:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.835919:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.835920:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.835922:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.835922:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.835923:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.835926:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.835928:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.835929:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.835930:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.835930:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.835932:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.835932:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.835933:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.835933:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.835937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.835944:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.835946:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.835947:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.835947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:18.0:1619677447.835948:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.835948:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.835950:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.835986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677447.836000:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.836003:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.836003:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.836005:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.836007:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.836008:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677447.836072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.836079:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.836082:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.836084:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.836086:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.836086:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.836087:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.836088:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.836089:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.836090:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.836091:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.836093:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.836095:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.836097:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.836098:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.836098:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.836100:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.836100:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.836101:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.836102:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.836113:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.836115:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.836116:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.836117:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.836119:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.836120:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.836133:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.836136:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.836137:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.836137:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.836138:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.836139:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.836310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.836314:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.836316:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.836318:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.836320:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.836321:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.836322:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.836324:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.836325:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.836326:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.836327:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.836330:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.836332:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.836334:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.836335:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.836336:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.836337:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.836338:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.836339:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.836339:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.836352:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.836355:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.836356:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.836356:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.836358:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.836360:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.836365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:6.0:1619677447.836369:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.836371:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:22.0:1619677447.836372:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.836372:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.836373:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.836374:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.836374:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.836374:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.836375:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.836376:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.836378:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.836380:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.836381:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.836383:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.836383:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.836385:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.836388:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.836389:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.836392:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.836392:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.836393:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677447.836395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677447.836395:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.836395:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.836396:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.836397:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.836402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.836403:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.836406:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.836408:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.836410:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.836410:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677447.836410:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:31.0:1619677447.836412:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.836413:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.836413:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.836414:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.836414:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:31.0:1619677447.836415:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.836416:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.836417:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:17.0:1619677447.836417:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.836420:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:31.0:1619677447.836421:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.836423:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.836426:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.836426:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.836427:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.836429:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.836429:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.836430:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.836431:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677447.836451:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.836453:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.836454:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.836455:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.836457:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.836458:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677447.836470:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.836472:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.836473:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.836474:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677447.836474:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.836475:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677447.836578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.836585:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.836587:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.836589:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.836590:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.836593:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.836594:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.836596:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722895] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.836597:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.836598:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.836599:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.836602:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.836603:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.836606:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.836607:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.836608:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.836609:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.836610:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.836611:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.836611:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.836615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.836623:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:21.2:1619677447.836625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:1.0:1619677447.836625:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.836627:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.836627:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.836629:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.836631:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:24.0:1619677447.836632:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.836635:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.2:1619677447.836636:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000400:00000200:24.0:1619677447.836637:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.836640:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.836640:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.836642:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.836644:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.836644:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:24.0:1619677447.836645:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.836646:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.836647:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677447.836647:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.836647:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.836648:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.836648:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.836649:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:24.0:1619677447.836650:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.836651:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.836654:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.836655:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.836656:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.836658:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.836658:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.836659:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.836660:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677447.836673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.836682:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.836684:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.836685:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.836687:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.836687:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.836690:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.836714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.836717:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.836719:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.836722:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.836722:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.836724:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.836725:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.836726:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.836728:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.836729:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.836730:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.836734:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.836735:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.836738:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.836739:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.836740:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.836742:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.836742:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.836743:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.836744:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677447.836755:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.836758:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.836759:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.836761:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.836762:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.836764:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.836772:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.836775:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.836776:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.836777:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.836777:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.836778:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.836793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.836801:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.836804:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.836806:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.836808:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.836808:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.836809:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:20.2:1619677447.836809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:31.0:1619677447.836810:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.836812:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.836813:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.836814:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:5.0:1619677447.836814:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:31.0:1619677447.836817:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.836817:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:31.0:1619677447.836819:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677447.836819:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.2:1619677447.836819:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677447.836822:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.836822:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.836823:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:5.0:1619677447.836824:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.836825:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.836825:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.836827:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:5.0:1619677447.836827:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.836828:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000400:00000200:5.0:1619677447.836828:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:31.0:1619677447.836829:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:5.0:1619677447.836830:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.836831:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.836833:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677447.836834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:5.0:1619677447.836834:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:12.0:1619677447.836838:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:5.0:1619677447.836838:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.836840:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:5.0:1619677447.836840:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:12.0:1619677447.836841:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.836842:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:12.0:1619677447.836843:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.836844:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:5.0:1619677447.836844:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.836844:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677447.836845:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:13.0:1619677447.836845:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677447.836845:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.836846:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677447.836847:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.836848:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.836849:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.836849:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:12.0:1619677447.836850:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.836857:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.836860:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.836861:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677447.836862:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:13.0:1619677447.836862:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.836863:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.836864:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:13.0:1619677447.836864:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:30.0:1619677447.836865:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.836865:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:30.0:1619677447.836867:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.836867:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677447.836868:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:12.0:1619677447.836868:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.836869:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.836870:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:13.0:1619677447.836871:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.836872:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.836942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.2:1619677447.836989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.836996:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.836999:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.837000:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.837002:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.837002:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.837005:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677447.837011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.837019:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.837022:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.837023:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.837023:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.837024:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.837025:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677447.837095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.837100:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.837103:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.837103:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.837106:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677447.837108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:22.0:1619677447.837109:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.837110:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.837111:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.837113:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.837113:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:22.0:1619677447.837114:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.837115:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:5.0:1619677447.837116:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.837118:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:22.0:1619677447.837119:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.837119:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.837121:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.837122:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:5.0:1619677447.837123:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.837124:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.837125:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.837126:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.837126:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:5.0:1619677447.837127:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.837128:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.837128:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:5.0:1619677447.837128:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.837128:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.837129:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.837130:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:5.0:1619677447.837132:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.837134:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.837137:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.837138:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.837138:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.837140:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.837141:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.837141:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.837142:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677447.837153:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.837156:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.837156:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.837158:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.837160:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.837161:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.837236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.837243:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.837246:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.837248:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.837250:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.837250:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.837251:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.837252:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.837254:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677447.837254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:30.0:1619677447.837255:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.837256:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.837258:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.837260:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.837261:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.837262:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.837263:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677447.837263:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:31.0:1619677447.837264:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.837264:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.837266:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:31.0:1619677447.837267:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.837267:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.837267:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.837268:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:30.0:1619677447.837268:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:31.0:1619677447.837269:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.837269:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.837271:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.837271:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.837272:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677447.837275:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.837276:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:13.0:1619677447.837278:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:31.0:1619677447.837279:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.837279:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677447.837280:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:13.0:1619677447.837280:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:31.0:1619677447.837281:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:13.0:1619677447.837281:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.837282:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:13.0:1619677447.837282:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.837283:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677447.837284:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.837285:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.837285:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677447.837286:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677447.837291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.837301:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.837303:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.837304:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.837305:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677447.837414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677447.837421:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.837424:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.837425:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.837427:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.837429:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.837431:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.837432:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.837433:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.837434:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.837435:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.837438:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.837440:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.837443:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.837444:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.837444:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.837446:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.837447:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.837448:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.837449:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.837449:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.2:1619677447.837455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.837460:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.837462:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.837463:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.837464:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.2:1619677447.837480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.837486:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.837489:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.837490:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.837492:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.837495:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.837496:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.837497:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.837499:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.837500:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.837501:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.837504:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.837506:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.837508:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.837509:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.837509:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.837511:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.837511:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.837512:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.837512:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677447.837517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.2:1619677447.837518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.837525:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.837527:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.837528:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.837529:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.837530:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.837532:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.837542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.837549:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.837552:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.837553:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.837555:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.837558:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.837559:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.837560:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96722881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.837561:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.837562:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677447.837562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:8.0:1619677447.837563:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.837567:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.837567:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677447.837568:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677447.837570:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.837571:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.837571:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677447.837571:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.837572:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.837572:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:6.0:1619677447.837573:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.837574:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.837575:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.837576:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677447.837576:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677447.837577:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.837578:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.837581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677447.837583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.837588:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677447.837591:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.837591:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.837592:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.837592:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.837593:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.837594:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.837594:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.837594:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.837597:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677447.837633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.837639:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.837642:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.837642:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.837644:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.837647:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.837649:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.837650:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.837651:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.837653:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.837654:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.837657:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677447.837658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677447.837658:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.837660:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.837661:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.837662:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.837662:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.837663:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:18.0:1619677447.837664:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.837664:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:20.0:1619677447.837665:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.837665:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.837665:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.837666:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.837668:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.837669:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:8.2:1619677447.837670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:20.0:1619677447.837671:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.837672:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.837673:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.837674:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.837677:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.837678:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.837679:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.837679:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.837680:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.837682:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677447.837682:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.837683:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.837683:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.837684:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.837685:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.837685:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.837685:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.837687:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.837688:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.837689:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.837694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677447.837698:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677447.837699:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:22.0:1619677447.837700:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.837701:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.837701:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:6.0:1619677447.837702:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.837702:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.837703:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:22.0:1619677447.837704:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.837706:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.837737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.837744:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.837747:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.837749:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.837751:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.837752:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.837753:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.837754:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.837756:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.837757:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.837758:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.837761:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.837762:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.837765:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.837766:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.837766:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.837768:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.837768:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.837769:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.837770:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.837781:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.837783:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.837784:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.837786:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.837788:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.837789:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.837942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.837947:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.837950:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.837952:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.837952:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.837955:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.837957:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677447.837958:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.837959:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.837960:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.837961:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.837964:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.837966:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.837968:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.837969:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.837970:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.837971:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.837972:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.837972:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.837973:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677447.837977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.837985:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.837987:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.837988:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.837988:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.837991:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677447.837993:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.838029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:21.2:1619677447.838041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:5.2:1619677447.838047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.0:1619677447.838048:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.838051:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.838052:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:1.0:1619677447.838053:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:24.0:1619677447.838054:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.838055:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.838056:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677447.838057:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.838058:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.838059:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.838060:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96722875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.838060:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.838062:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.838062:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.838063:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.838063:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722874] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.838064:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677447.838065:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.838066:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.838068:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.838070:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677447.838072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:1.0:1619677447.838072:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.838073:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:13.0:1619677447.838075:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677447.838075:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.838077:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.838077:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.838077:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.838078:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:13.0:1619677447.838078:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.838078:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.838078:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.838079:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:13.0:1619677447.838080:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:1.0:1619677447.838080:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.838080:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677447.838081:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.838081:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:1.0:1619677447.838081:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677447.838082:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.838082:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.838082:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677447.838083:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.838083:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677447.838086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:21.2:1619677447.838090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:17.0:1619677447.838094:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.838096:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:6.0:1619677447.838096:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:17.0:1619677447.838097:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.838097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677447.838097:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:6.0:1619677447.838098:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:17.0:1619677447.838099:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:5.2:1619677447.838099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677447.838100:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677447.838101:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:6.0:1619677447.838102:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.838104:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:6.0:1619677447.838104:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.838106:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:8.2:1619677447.838106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000400:00000200:6.0:1619677447.838106:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677447.838107:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.838107:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.838108:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.838110:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.838110:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.838110:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:6.0:1619677447.838111:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:20.0:1619677447.838113:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.838113:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:6.0:1619677447.838115:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.838116:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.838117:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.838117:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:6.0:1619677447.838118:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:1.0:1619677447.838118:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.838119:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:6.0:1619677447.838121:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.838121:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.838122:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.838124:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.838124:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.838125:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.838126:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.838130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.838137:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.838140:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.838141:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.838141:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.838142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677447.838144:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.838148:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677447.838149:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.838151:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.838151:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.838151:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.838153:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.838156:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.838191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:21.2:1619677447.838195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677447.838198:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.838200:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.838201:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:4.2:1619677447.838202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.838203:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:18.0:1619677447.838204:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.838205:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.838206:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.838208:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.838209:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.838210:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.838211:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.838213:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.838214:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.838215:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.838217:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.838218:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.838218:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.838220:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.838221:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.838221:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.838224:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.838226:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.838228:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.838229:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.838229:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.838229:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.838229:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.838230:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.838231:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.838231:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.838232:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.838232:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.838233:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (23)++
00000800:00000200:18.0:1619677447.838234:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (23)++
00000800:00000200:18.0:1619677447.838235:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (24)--
00000800:00000200:2.0:1619677447.838235:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (24)--
00000800:00000200:18.0:1619677447.838236:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.838236:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.838236:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677447.838238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.2:1619677447.838238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.838242:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.2:1619677447.838243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.2:1619677447.838244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.838246:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.838247:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.838247:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:18.0:1619677447.838248:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.838248:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.838249:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.838250:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.838250:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.838250:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:5.0:1619677447.838251:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677447.838251:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.838252:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.838252:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.838253:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677447.838253:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.838254:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:8.2:1619677447.838255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:5.0:1619677447.838255:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.838256:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.838258:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.838259:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.838260:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.838263:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.838264:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.838265:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.838265:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.838268:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.838321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.2:1619677447.838327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677447.838329:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.838332:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.838333:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.838335:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.838336:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:8.0:1619677447.838338:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.838339:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:8.0:1619677447.838340:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.838341:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.838342:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.838343:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.838344:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.838344:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:8.0:1619677447.838344:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.838345:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677447.838346:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.838347:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.838349:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.838349:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.838350:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.838350:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.838351:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:31.0:1619677447.838353:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.838354:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.838355:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.838355:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.838355:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.838357:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.838357:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677447.838358:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.838358:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.838358:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.838358:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:31.0:1619677447.838359:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.838360:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.838361:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.838362:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.838362:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677447.838363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.838371:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.838374:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:12.0:1619677447.838374:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:17.0:1619677447.838375:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.838375:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.838377:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.838377:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.838377:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.838379:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.838380:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:13.0:1619677447.838381:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.838382:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.838459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.838465:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.838468:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.838469:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.838470:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.838472:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.838474:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.838475:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.838476:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.838477:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.838478:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.838481:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.838483:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.838485:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.838486:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.838486:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.838488:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.838488:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.838489:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.838489:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.838494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.838500:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.838502:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.838502:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.838504:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.838505:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.838506:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.838508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.838515:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.838518:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.838520:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.838522:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.838522:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.838524:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.838525:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.838526:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.838527:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.838528:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.838531:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.838533:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.838535:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.838536:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.838537:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.838538:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.838539:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.838540:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.838540:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677447.838550:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.838553:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.838553:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.838555:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.838557:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.838558:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.838629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677447.838663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.838669:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.838672:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.838674:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:22.0:1619677447.838675:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:18.0:1619677447.838675:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.838677:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.838678:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.838679:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.838679:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:6.0:1619677447.838679:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:18.0:1619677447.838680:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.838681:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.838682:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.838683:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.838684:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:18.0:1619677447.838684:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.838687:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.838689:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.838691:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.838692:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.838692:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.838694:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.838695:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.838695:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.838696:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.838700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.838707:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.838709:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.838710:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.838711:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.838712:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.838713:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.838720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:21.2:1619677447.838725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:18.0:1619677447.838728:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.838730:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.838731:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.838731:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.838732:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:18.0:1619677447.838732:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.838732:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677447.838736:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.838737:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.838739:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.838742:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.838744:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.838746:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.838747:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.838748:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.838749:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.838753:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.838755:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.838758:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.838759:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.838760:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.838762:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.838763:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.838763:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.838764:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.838768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.838775:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.838778:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.838779:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.838779:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.838781:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.838784:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.838827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.838833:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.838836:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.838836:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.838838:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677447.838840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:1.0:1619677447.838840:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.838841:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.838842:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.838843:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.838845:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.838845:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.838849:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.838851:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.838854:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.838855:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.838856:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.838858:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.838858:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.838859:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.838860:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.838861:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.838863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677447.838865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677447.838868:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.838870:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:26.0:1619677447.838871:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.838871:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.838871:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:26.0:1619677447.838872:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.838872:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.838873:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.838873:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:26.0:1619677447.838874:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677447.838874:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:8.0:1619677447.838874:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.838874:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677447.838875:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.838875:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677447.838875:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677447.838876:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.2:1619677447.838880:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.838884:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:1.0:1619677447.838888:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.838890:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.838891:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.838891:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.838893:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.838897:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677447.838920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.838928:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.838930:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.838933:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.838934:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.838935:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.838936:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.838938:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.838939:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.838940:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.838941:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.838944:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.838945:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.838948:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.838948:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.838949:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.838950:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.838951:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.838952:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.838952:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.838963:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.838965:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.838966:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.838967:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.838969:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.838970:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.838971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.838978:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.838981:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:13.0:1619677447.838981:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.838982:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.838983:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.838984:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.838985:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.838985:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:24.0:1619677447.838986:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.838986:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677447.838987:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:13.0:1619677447.838987:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677447.838988:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.838990:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.838990:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.838991:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.838994:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.838996:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.838999:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.838999:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.839000:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.839001:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.839002:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.839003:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.839003:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.839007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.839016:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.839018:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.839019:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.839020:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.839022:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.839024:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.839027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.839035:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.839037:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.839038:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.839038:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.839040:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.839040:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677447.839072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.839077:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.839079:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.839080:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.839082:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.839085:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.839086:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.839087:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.839089:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.839090:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.839091:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.839094:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.839095:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.839097:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.839098:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.839099:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.839100:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.839100:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.839101:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.839101:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.839106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.839112:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.839114:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.839115:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.839116:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.839116:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.839117:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677447.839265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.839276:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677447.839312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677447.839314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677447.839320:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.2:1619677447.839321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677447.839322:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:18.0:1619677447.839323:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.839323:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.839324:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.839325:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677447.839325:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.839326:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:18.0:1619677447.839326:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.839327:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.839329:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.839330:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.839331:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.839333:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.839334:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.839336:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.839337:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.839339:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.839340:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.839341:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.839344:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.839345:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.839348:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.839349:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.839350:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.839352:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.839352:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.839353:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.839353:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.839365:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.839367:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.839368:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.839370:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.839372:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.839373:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.839376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.839381:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.839384:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.839385:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.839386:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.839386:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677447.839387:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:12.0:1619677447.839387:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.839422:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.839424:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.839425:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.839426:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.839429:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.839429:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:21.2:1619677447.839510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.2:1619677447.839514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:30.0:1619677447.839518:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677447.839520:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677447.839521:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.0:1619677447.839523:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:30.0:1619677447.839524:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.839525:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677447.839525:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.839526:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.839527:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.839528:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.839528:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.839529:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.839530:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677447.839531:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.839531:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.839532:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.839533:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.839534:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.839535:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.839536:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677447.839540:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.839541:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.839542:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.839543:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.839544:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.839545:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.839546:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.839546:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.839547:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.839547:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677447.839548:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677447.839548:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.839549:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.839550:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677447.839550:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.839550:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.2:1619677447.839553:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.0:1619677447.839555:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.839557:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.839557:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:1.0:1619677447.839559:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677447.839560:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677447.839561:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.2:1619677447.839562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677447.839562:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677447.839562:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.839563:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.839564:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677447.839565:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000400:00000200:12.0:1619677447.839565:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.839565:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.839566:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:13.0:1619677447.839566:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:31.0:1619677447.839567:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.839567:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.839568:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:12.0:1619677447.839568:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677447.839568:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.839569:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:1.0:1619677447.839569:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677447.839570:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.839570:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:1.0:1619677447.839571:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.839572:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.839573:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:1.0:1619677447.839573:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.839574:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677447.839578:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.839580:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677447.839582:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.839582:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.839583:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.839584:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:30.0:1619677447.839585:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.839586:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.839586:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:1.0:1619677447.839586:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.839586:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677447.839587:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.839587:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.839588:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.839589:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:9.2:1619677447.839592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.839599:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.839602:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.839603:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.839603:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.839605:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.839608:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.839611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677447.839617:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.839620:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677447.839620:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.839621:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.839621:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.839622:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.2:1619677447.839762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.839765:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.839768:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.839770:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.839771:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.839773:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.839775:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677447.839776:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.839777:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.839778:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.839779:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677447.839783:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.839786:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:4.2:1619677447.839787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.839788:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.839789:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.839790:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.839792:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.839793:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.839793:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677447.839794:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677447.839794:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677447.839797:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:8.2:1619677447.839798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:26.0:1619677447.839800:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.839802:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.839804:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.839805:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.839806:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:26.0:1619677447.839807:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.839807:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.839808:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.839809:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.839810:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.839810:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.839812:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:26.0:1619677447.839813:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.839815:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:26.0:1619677447.839816:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.839818:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.839819:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.839819:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.839821:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.839821:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.839822:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.839824:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.839824:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.2:1619677447.839826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:4.2:1619677447.839830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.839832:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677447.839834:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677447.839835:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.839835:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:31.0:1619677447.839836:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.839837:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.839837:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.839839:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.2:1619677447.839839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677447.839840:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:6.0:1619677447.839840:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.839841:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:31.0:1619677447.839842:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:6.0:1619677447.839842:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677447.839843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677447.839844:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:6.0:1619677447.839844:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.839845:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.839846:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:6.0:1619677447.839846:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.839849:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677447.839850:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.839851:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.839852:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.839852:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.839853:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.839853:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:26.0:1619677447.839854:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:1.0:1619677447.839855:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.839856:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.839856:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.839857:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:26.0:1619677447.839858:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.839858:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:1.0:1619677447.839858:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.839859:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:26.0:1619677447.839860:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.839860:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:1.0:1619677447.839860:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.839861:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:6.0:1619677447.839861:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.839861:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:1.0:1619677447.839862:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.839864:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677447.839865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:1.0:1619677447.839865:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.839866:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.839867:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.839869:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677447.839872:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677447.839872:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.839873:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677447.839873:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:17.0:1619677447.839874:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:26.0:1619677447.839875:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677447.839875:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677447.839875:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.839875:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.839876:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:17.0:1619677447.839876:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.839877:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:17.0:1619677447.839878:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.839879:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.839880:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.839880:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.839881:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.839883:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.839883:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.839883:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.839885:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.839886:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:30.0:1619677447.839887:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:1.0:1619677447.839887:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677447.839889:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.839891:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.839891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677447.839892:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.839894:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.839895:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.839895:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:17.0:1619677447.839897:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677447.839898:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.839899:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.839899:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.839900:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.839900:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677447.839901:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.839902:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.839904:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.840011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:26.0:1619677447.840021:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:9.2:1619677447.840049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677447.840053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:2.0:1619677447.840056:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.840059:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677447.840060:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:18.0:1619677447.840060:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.840060:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.840062:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.840063:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000400:00000200:26.0:1619677447.840065:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.840066:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:26.0:1619677447.840069:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.840070:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677447.840071:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.840072:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.840073:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.840074:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677447.840077:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.840078:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677447.840080:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.840081:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677447.840081:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677447.840083:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677447.840083:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677447.840084:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:26.0:1619677447.840085:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:13.0:1619677447.840095:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.840098:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.840099:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.840100:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.840102:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.840102:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:21.2:1619677447.840139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.840146:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.840149:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.840151:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.840153:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.840153:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.840154:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.840155:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.840157:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.840158:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.840158:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.840161:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.840163:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.840165:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.840166:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.840167:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.840168:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.840169:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.840170:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.840172:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.840172:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677447.840178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677447.840180:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.840182:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.840182:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.840184:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.840186:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677447.840186:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.840187:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.840188:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.840189:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.840190:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677447.840320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.2:1619677447.840321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677447.840326:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.840329:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677447.840330:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:1.0:1619677447.840330:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677447.840332:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:17.0:1619677447.840332:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.840334:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.840334:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.840336:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.840336:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:30.0:1619677447.840337:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:17.0:1619677447.840337:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722842] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.840338:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677447.840339:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.840340:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722842] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.840340:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677447.840341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:31.0:1619677447.840342:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.840342:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677447.840344:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.840346:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677447.840346:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.840350:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.840350:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677447.840352:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677447.840352:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.840353:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677447.840353:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.840354:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.840355:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677447.840356:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.840356:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.840356:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677447.840357:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:17.0:1619677447.840357:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.840358:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.840359:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.840360:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.840361:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.840363:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.840363:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677447.840366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677447.840369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:22.0:1619677447.840370:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.840373:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.840374:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.840374:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:5.0:1619677447.840375:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:22.0:1619677447.840376:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.840377:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:5.0:1619677447.840377:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677447.840379:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677447.840379:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:30.0:1619677447.840380:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.840381:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677447.840381:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677447.840382:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:21.2:1619677447.840382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:26.0:1619677447.840385:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:5.0:1619677447.840386:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.840387:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:5.0:1619677447.840387:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:26.0:1619677447.840388:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.840388:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:26.0:1619677447.840390:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.840391:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.840391:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:5.0:1619677447.840391:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:26.0:1619677447.840392:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.840393:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677447.840394:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:6.0:1619677447.840395:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.840398:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.840399:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.840401:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.840403:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.840405:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.840406:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:4.2:1619677447.840407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:6.0:1619677447.840411:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.840414:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677447.840414:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:24.0:1619677447.840417:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677447.840417:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.840418:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:6.0:1619677447.840418:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.840419:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:8.0:1619677447.840420:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.840420:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.840421:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:8.0:1619677447.840422:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.840422:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.840423:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.840424:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677447.840426:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677447.840428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:8.0:1619677447.840429:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.840430:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96722837] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677447.840431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:8.0:1619677447.840432:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.840433:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:8.0:1619677447.840433:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.840434:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:12.0:1619677447.840434:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:8.0:1619677447.840434:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.840436:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677447.840437:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:20.0:1619677447.840437:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.840437:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:8.0:1619677447.840437:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.840438:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677447.840438:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:12.0:1619677447.840438:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.840439:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:20.0:1619677447.840439:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.840439:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.2:1619677447.840440:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677447.840441:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:20.0:1619677447.840441:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:12.0:1619677447.840441:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.840442:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:13.0:1619677447.840442:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.840442:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.840443:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.840443:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:26.0:1619677447.840444:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.840444:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:8.0:1619677447.840444:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.840444:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.840445:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.840445:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.840446:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.0:1619677447.840446:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.840446:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677447.840447:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.840447:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.840448:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:26.0:1619677447.840449:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677447.840451:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677447.840451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677447.840452:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.840453:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.840454:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677447.840458:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.840459:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:1.0:1619677447.840460:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:26.0:1619677447.840461:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.840462:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677447.840462:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:1.0:1619677447.840462:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.840463:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:26.0:1619677447.840464:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677447.840464:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:1.0:1619677447.840464:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.840465:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.840466:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:1.0:1619677447.840466:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677447.840469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.840472:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.840473:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.840475:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677447.840476:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:1.0:1619677447.840476:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.840477:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.840477:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.840478:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677447.840479:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.840480:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.840481:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.840483:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.840484:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677447.840672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:26.0:1619677447.840682:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:21.2:1619677447.840700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677447.840707:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.840709:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.840710:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:26.0:1619677447.840711:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677447.840712:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:24.0:1619677447.840712:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.840713:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.840714:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:21.2:1619677447.840714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:24.0:1619677447.840715:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.840716:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:24.0:1619677447.840716:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.840717:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.840718:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.840719:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.840720:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:24.0:1619677447.840720:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.2:1619677447.840722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:31.0:1619677447.840723:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:24.0:1619677447.840724:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.840725:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.840725:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.840727:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.840728:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.840728:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.840729:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:24.0:1619677447.840729:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.840729:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677447.840730:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.840730:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.840731:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:31.0:1619677447.840732:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.840733:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.840733:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:24.0:1619677447.840733:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000400:00000200:31.0:1619677447.840734:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.840734:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.840734:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:31.0:1619677447.840737:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.840739:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:21.2:1619677447.840740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:31.0:1619677447.840742:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.840742:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677447.840743:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.840743:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:31.0:1619677447.840744:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (25)--
00000800:00000200:31.0:1619677447.840745:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677447.840745:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:31.0:1619677447.840746:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.840746:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:13.0:1619677447.840746:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.840747:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:13.0:1619677447.840748:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.840749:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:8.0:1619677447.840749:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:13.0:1619677447.840750:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677447.840750:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:8.0:1619677447.840750:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677447.840751:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.840751:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.840751:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:12.0:1619677447.840753:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.840754:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677447.840761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.2:1619677447.840765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:13.0:1619677447.840769:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.840771:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677447.840772:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:13.0:1619677447.840772:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.840773:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.840773:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.840773:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677447.840774:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:13.0:1619677447.840774:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.840776:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:26.0:1619677447.840777:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.840777:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.840779:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.840779:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.840779:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:26.0:1619677447.840781:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:26.0:1619677447.840782:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.840782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:26.0:1619677447.840783:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677447.840784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:26.0:1619677447.840785:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.840786:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:5.0:1619677447.840787:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:26.0:1619677447.840791:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.840793:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:26.0:1619677447.840795:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.840796:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:26.0:1619677447.840797:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:26.0:1619677447.840799:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:26.0:1619677447.840803:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677447.840804:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:26.0:1619677447.840805:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:9.2:1619677447.840805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:26.0:1619677447.840806:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.840807:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:26.0:1619677447.840809:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677447.840811:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.840813:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:18.0:1619677447.840813:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:26.0:1619677447.840814:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96722829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.840816:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.840816:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:26.0:1619677447.840817:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.840817:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:18.0:1619677447.840817:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.840817:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.840820:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:26.0:1619677447.840823:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.840824:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:26.0:1619677447.840825:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677447.840826:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.840828:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:26.0:1619677447.840828:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.840829:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677447.840829:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:30.0:1619677447.840830:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.840831:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.840831:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677447.840831:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:31.0:1619677447.840832:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.840832:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677447.840833:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:30.0:1619677447.840834:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.2:1619677447.840837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.840845:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.840847:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.840848:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.840849:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.840851:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:17.0:1619677447.840854:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677447.840870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.840876:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.840878:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.840879:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.840881:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.840883:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.840885:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.840886:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.840887:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.840889:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.840889:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.840893:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.840895:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.840898:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.840899:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.840899:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.840901:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.840902:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.840902:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.840903:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677447.840915:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.840917:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.840918:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.840920:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.840921:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.840923:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.840933:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.840936:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.840937:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.840937:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.840938:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.840939:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.840983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:26.0:1619677447.840994:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:8.2:1619677447.841022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677447.841025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (21)++
00000800:00000200:1.0:1619677447.841030:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.841032:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:26.0:1619677447.841033:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (22)++
00000800:00000200:17.0:1619677447.841033:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.841034:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.841035:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f787000] (23)--
00000800:00000200:26.0:1619677447.841036:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:1.0:1619677447.841036:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.841037:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:1.0:1619677447.841039:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.841246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:5.2:1619677447.841250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.841259:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.841261:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677447.841263:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.841274:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.841276:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677447.841277:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.841279:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.841279:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.841280:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677447.841284:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.841286:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677447.841290:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.841291:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677447.841291:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:21.2:1619677447.841299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:26.0:1619677447.841300:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677447.841301:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.841302:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.841303:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677447.841307:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:22.0:1619677447.841309:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:31.0:1619677447.841310:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677447.841312:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.841312:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.841313:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.841313:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:31.0:1619677447.841314:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.841315:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:22.0:1619677447.841315:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.841316:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677447.841317:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.841319:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.841319:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:31.0:1619677447.841320:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.841321:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677447.841324:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.841326:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677447.841329:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.841329:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677447.841330:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677447.841332:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677447.841332:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677447.841334:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:31.0:1619677447.841335:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:31.0:1619677447.841336:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:9.2:1619677447.841338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.841346:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:21.2:1619677447.841346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:24.0:1619677447.841349:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.841350:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.841350:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.841352:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.841354:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:24.0:1619677447.841354:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:30.0:1619677447.841357:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:30.0:1619677447.841357:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677447.841358:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:5.2:1619677447.841519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:26.0:1619677447.841531:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:21.2:1619677447.841556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.841564:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.841566:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.841566:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677447.841567:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.841568:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.841568:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.841569:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.841570:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.841572:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.841573:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.841575:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.841576:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.841577:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677447.841577:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.841579:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.841580:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:30.0:1619677447.841580:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.841581:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.841583:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.841584:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.841585:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.841586:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.841587:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.841587:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.841588:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.841599:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.841602:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.841602:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.841604:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.841606:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.841607:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.841664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677447.841671:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.841674:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.841675:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.841676:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.841679:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.841680:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.841681:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.841683:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.841684:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.841685:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.841688:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.841690:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.841693:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.841694:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.841695:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.841697:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.841697:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.841698:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.841698:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677447.841709:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.841712:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.841713:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.841714:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.841716:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.841717:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.841780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.841788:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.841790:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677447.841793:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.841796:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.841797:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.841798:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.841799:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.841800:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.841801:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677447.841805:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.841806:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.841809:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.841810:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.841810:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677447.841812:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.841812:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.841813:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.841814:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677447.841826:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.841829:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.841830:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.841830:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.841832:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.2:1619677447.841833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:6.0:1619677447.841835:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.841836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677447.841839:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.841842:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.841843:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677447.841844:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:22.0:1619677447.841844:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:17.0:1619677447.841844:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.841847:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677447.841847:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.841848:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:6.0:1619677447.841848:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.841849:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000400:00000200:17.0:1619677447.841849:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:22.0:1619677447.841850:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.841850:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:31.0:1619677447.841851:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.841851:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:17.0:1619677447.841851:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.841853:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.841854:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.841854:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.841855:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677447.841856:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.841858:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.841859:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.841859:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.841860:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.841862:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:17.0:1619677447.841862:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.841864:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.841865:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:31.0:1619677447.841866:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.841866:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.841867:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677447.841868:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:17.0:1619677447.841868:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.841869:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.841869:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.841870:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.841871:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.841872:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.841873:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.841874:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.841875:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.841876:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:6.0:1619677447.841881:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.841884:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:30.0:1619677447.841885:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:22.0:1619677447.841885:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.841885:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.841887:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.841888:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.841888:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.841889:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:30.0:1619677447.841890:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.841891:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.841893:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677447.841904:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.841907:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.841908:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.841908:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677447.841909:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.841909:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677447.842027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.842032:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.842036:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.842036:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.842038:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.842041:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.842043:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.842044:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.842046:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.842047:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.842048:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.842051:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.842053:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.842056:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.842056:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.842057:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.842058:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.842059:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.842060:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.842060:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.842065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677447.842073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677447.842073:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.842075:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677447.842076:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677447.842076:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.842076:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.842078:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:1.0:1619677447.842079:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.842080:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.842081:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677447.842082:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.842083:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.842085:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.842086:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.842087:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.842088:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.842089:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.842093:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.842095:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.842098:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.842099:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.842100:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.842102:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.842102:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.842103:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.842104:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.842114:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.842117:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.842118:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.842119:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.842120:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.842122:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.842178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.842183:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.842186:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.842187:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.842189:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.842191:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.842193:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.842194:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.842196:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.842197:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.842198:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.842201:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.842203:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.842206:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.842206:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.842207:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.842209:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.842209:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.842210:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.842210:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:26.0:1619677447.842224:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:5.2:1619677447.842240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:26.0:1619677447.842247:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677447.842249:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000400:00000200:26.0:1619677447.842252:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.842254:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.842256:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:26.0:1619677447.842257:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.842258:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.842259:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.842260:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:26.0:1619677447.842263:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.842265:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677447.842266:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.842267:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:26.0:1619677447.842267:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677447.842269:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677447.842269:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677447.842270:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:26.0:1619677447.842270:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:5.2:1619677447.842277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:4.2:1619677447.842279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677447.842286:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:8.0:1619677447.842286:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677447.842288:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:8.0:1619677447.842289:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.842290:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:26.0:1619677447.842291:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.842292:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.842292:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.842299:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677447.842300:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.842300:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.842301:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.842302:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.842302:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677447.842303:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:8.0:1619677447.842303:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677447.842303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:8.0:1619677447.842304:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.842305:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.842306:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677447.842307:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.842309:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.842310:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.2:1619677447.842310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.0:1619677447.842311:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677447.842312:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.842313:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.842313:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.842314:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:8.0:1619677447.842314:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.842315:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:8.0:1619677447.842315:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.842316:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677447.842317:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f787000] (22)++
00000800:00000200:8.0:1619677447.842317:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:26.0:1619677447.842318:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (23)--
00000800:00000200:8.0:1619677447.842318:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677447.842319:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677447.842319:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:8.0:1619677447.842319:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:26.0:1619677447.842320:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677447.842320:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:31.0:1619677447.842321:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:8.0:1619677447.842321:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.842323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677447.842324:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677447.842324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (21)++
00000400:00000200:31.0:1619677447.842326:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.842326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677447.842327:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.842328:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.842329:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96722810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.842329:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.842329:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.842330:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.842331:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.842332:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.842332:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.842332:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:31.0:1619677447.842333:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.842333:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:26.0:1619677447.842334:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.842335:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (22)++
00000800:00000200:21.2:1619677447.842335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:17.0:1619677447.842335:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:26.0:1619677447.842336:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f787000] (23)--
00000800:00000200:24.0:1619677447.842336:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677447.842336:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.842336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.842336:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:31.0:1619677447.842337:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.842337:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:24.0:1619677447.842337:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677447.842338:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.842339:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677447.842339:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:24.0:1619677447.842339:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.842339:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:31.0:1619677447.842341:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.842341:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:17.0:1619677447.842341:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677447.842342:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.842342:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677447.842342:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677447.842342:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.842343:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:31.0:1619677447.842344:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677447.842344:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:31.0:1619677447.842345:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:8.0:1619677447.842345:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.842346:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.842346:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:26.0:1619677447.842347:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.842348:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.842349:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.842350:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:5.2:1619677447.842350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:26.0:1619677447.842351:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.842351:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96722809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.842352:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96722809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.842353:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.842354:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.842354:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.842355:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677447.842355:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677447.842356:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.842356:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:26.0:1619677447.842357:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:13.0:1619677447.842357:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.842358:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.842358:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.842359:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:8.0:1619677447.842359:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677447.842360:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.842361:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.842361:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677447.842362:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:12.0:1619677447.842362:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:26.0:1619677447.842363:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677447.842363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:13.0:1619677447.842363:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677447.842365:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:12.0:1619677447.842365:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.842365:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.842365:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.842366:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.842368:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.842368:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.842368:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:26.0:1619677447.842369:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.842369:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.842369:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:2.0:1619677447.842369:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:26.0:1619677447.842370:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677447.842371:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.842371:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:26.0:1619677447.842372:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.842373:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:18.0:1619677447.842373:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:13.0:1619677447.842373:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677447.842374:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:9.2:1619677447.842374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677447.842374:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.842375:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:13.0:1619677447.842376:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.842376:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.842377:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:2.0:1619677447.842377:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.842378:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.2:1619677447.842378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:13.0:1619677447.842379:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677447.842379:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:4.2:1619677447.842380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677447.842381:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96722807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.842382:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.842384:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:2.0:1619677447.842384:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.842385:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.842385:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.842386:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.842386:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:18.0:1619677447.842388:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.842389:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:2.0:1619677447.842389:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.842391:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:5.0:1619677447.842391:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.842391:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.842392:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:5.0:1619677447.842392:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.842393:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.842394:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.842395:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.842395:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.842396:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:2.0:1619677447.842396:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:9.2:1619677447.842397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.842398:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.842399:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.2:1619677447.842400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.842400:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.842401:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677447.842403:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.842404:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.842405:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.842405:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.842406:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.842406:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.842408:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.842410:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.842411:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.842412:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:6.0:1619677447.842412:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.842413:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:6.0:1619677447.842414:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677447.842414:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:6.0:1619677447.842415:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.842416:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.842418:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.842420:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.842552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.842555:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.842557:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.842559:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.842560:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.842562:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.842563:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.842565:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.842566:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.842567:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.842568:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.842571:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.842573:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.842575:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.842576:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:21.2:1619677447.842577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:20.0:1619677447.842577:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.842579:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.842579:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.842580:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.842580:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.842585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.842588:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:18.0:1619677447.842593:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.842596:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.842597:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.842597:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.842600:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.842602:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.842607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.842612:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.842615:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.842616:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.842618:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.842620:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:2.0:1619677447.842620:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.842621:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:26.0:1619677447.842622:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:2.0:1619677447.842622:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.842623:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.842624:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.842625:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.842625:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.842626:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677447.842627:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:2.0:1619677447.842630:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.842632:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.842634:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.842635:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.842636:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.842637:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.842638:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.842639:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.842639:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:26.0:1619677447.842643:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:8.2:1619677447.842643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.842651:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.842654:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.842655:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.842656:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.842657:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.842659:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677447.842661:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.842667:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.842669:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.842670:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.842672:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.842674:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.842676:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.842677:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.842678:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.842679:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.842680:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.842684:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.842686:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.842688:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.842689:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.842689:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.842691:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.842691:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.842692:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.842693:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.842705:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.842707:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.842708:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.842708:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.842710:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.842712:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.842721:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.842724:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.842724:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.842725:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.842726:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.842727:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.842802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.842809:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.842812:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:20.2:1619677447.842814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:26.0:1619677447.842815:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.842816:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.842818:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.842819:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.842820:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.842821:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.842822:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677447.842822:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.842825:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:26.0:1619677447.842827:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.842828:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.842829:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677447.842829:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:30.0:1619677447.842830:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.842831:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:26.0:1619677447.842832:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677447.842833:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.842833:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.842833:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:30.0:1619677447.842834:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.842835:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.842835:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.842835:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677447.842836:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:26.0:1619677447.842836:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.842837:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:30.0:1619677447.842839:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.842841:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677447.842844:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.842844:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.842845:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677447.842847:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.842847:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.842848:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:22.0:1619677447.842848:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:30.0:1619677447.842849:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:22.0:1619677447.842851:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.842852:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.842852:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.2:1619677447.842853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:22.0:1619677447.842854:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.842857:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.842860:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.842863:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.842864:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.842866:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.842866:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.842869:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.842931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.842936:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.842938:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.842941:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.842941:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.842943:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.842945:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.842946:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.842948:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.842949:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.842950:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.842954:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.842955:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.842958:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.842959:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.842960:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.842962:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.842962:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.842963:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.842964:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677447.842975:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.842978:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.842979:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.842981:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.842981:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:4.2:1619677447.842983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677447.842984:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:24.0:1619677447.842990:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.842993:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.842994:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.842996:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.842998:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.843000:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.843001:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.843002:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.843003:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.843004:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.843007:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677447.843008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:24.0:1619677447.843009:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677447.843011:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:24.0:1619677447.843012:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.843012:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.843013:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.843013:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:24.0:1619677447.843015:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.843015:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:20.0:1619677447.843015:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.843016:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.843016:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.0:1619677447.843016:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.843018:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.843020:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.843021:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677447.843021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:20.0:1619677447.843022:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.843023:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.843024:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.843028:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.843029:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677447.843029:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.843031:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:20.0:1619677447.843032:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.843032:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:1.0:1619677447.843032:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.843033:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:17.0:1619677447.843033:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.843035:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.843035:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:17.0:1619677447.843035:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.843036:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.843037:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:17.0:1619677447.843037:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.0:1619677447.843038:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677447.843041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.2:1619677447.843043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677447.843047:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677447.843049:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677447.843049:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677447.843050:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.843051:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:1.0:1619677447.843051:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.843052:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:6.0:1619677447.843052:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677447.843052:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677447.843052:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:22.0:1619677447.843053:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.843053:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.843053:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.0:1619677447.843054:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:6.0:1619677447.843055:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.843057:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677447.843100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677447.843106:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.843109:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.843110:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.843112:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.843114:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.843115:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.843117:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.843118:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.843119:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.843120:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.843123:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.843125:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.843127:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.843128:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.843129:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.843130:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.843131:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.843132:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.843132:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.843144:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.843147:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.843147:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.843148:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.843150:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.843152:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.843161:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.843164:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.843164:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.843165:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.843166:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.843166:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.843171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.843177:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.843179:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.843180:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.843182:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.843184:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.843186:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.843187:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.843188:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.843189:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.843190:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.2:1619677447.843192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:2.0:1619677447.843193:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.843194:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.843196:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.843196:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.843197:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.843198:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.843199:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:30.0:1619677447.843200:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677447.843200:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.843201:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.843201:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:30.0:1619677447.843203:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:9.2:1619677447.843204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677447.843205:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.843207:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.843207:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.843207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000400:00000200:30.0:1619677447.843209:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.843210:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.843211:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.843212:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.843213:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.843213:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.843214:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.843214:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:18.0:1619677447.843215:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.843215:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.843216:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:30.0:1619677447.843217:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.843217:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.843218:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.843218:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:30.0:1619677447.843219:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.843220:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677447.843222:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.843223:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.843224:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.843226:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.843226:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.843227:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.843229:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.843229:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:4.2:1619677447.843231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.2:1619677447.843232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677447.843236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677447.843240:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:26.0:1619677447.843241:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677447.843243:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677447.843244:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:5.0:1619677447.843244:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.843245:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.843245:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.843246:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.843247:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.843248:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677447.843248:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677447.843268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.843276:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677447.843277:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:31.0:1619677447.843278:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.843280:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.843280:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.843281:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.843281:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.843282:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.843283:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.843284:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.843285:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.843286:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.843287:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.843287:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:31.0:1619677447.843288:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.843289:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.843293:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.843300:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.843302:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.843303:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.843304:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.843306:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.843306:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.843307:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.843308:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.843320:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.843322:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.843323:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.843325:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.843327:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.843327:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:21.2:1619677447.843394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677447.843402:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.843405:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.843406:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.843408:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.843410:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.843412:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.843413:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.843414:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.843415:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.843416:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.843419:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.843421:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.843424:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.843425:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.843425:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.843427:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.843428:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.843429:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.843429:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.843433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.843441:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.843444:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.843446:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.843446:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.843448:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.843451:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.843686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.843692:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.843694:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.843695:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.843697:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.843698:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.843700:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.843701:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.843702:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.843704:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.843705:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.843708:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.843710:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.843713:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.843714:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.2:1619677447.843714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677447.843715:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.843717:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.843717:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.843718:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.843718:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677447.843720:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.843723:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:4.2:1619677447.843724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:1.0:1619677447.843724:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.843726:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.843728:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.843729:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:12.0:1619677447.843729:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:17.0:1619677447.843731:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.843731:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:24.0:1619677447.843732:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:17.0:1619677447.843732:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677447.843732:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.843733:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.843734:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.843735:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:12.0:1619677447.843735:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.843735:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677447.843736:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:17.0:1619677447.843739:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.843740:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.843742:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.843743:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.843744:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677447.843745:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:17.0:1619677447.843746:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.843746:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:24.0:1619677447.843747:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96722789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.843747:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677447.843748:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.843749:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.843749:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.843749:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:24.0:1619677447.843750:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.843751:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.843752:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.843752:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677447.843754:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.843756:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.2:1619677447.843757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677447.843759:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.843760:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.843760:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677447.843761:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677447.843762:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.843763:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.843763:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677447.843764:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.843764:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.843764:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.843765:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.843765:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.843766:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.843766:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677447.843767:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.843768:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.843769:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677447.843769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.843776:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.843779:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.843780:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.843780:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.843782:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:21.2:1619677447.843784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:1.0:1619677447.843785:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.843788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677447.843792:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677447.843795:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:17.0:1619677447.843796:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677447.843797:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.843798:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677447.843799:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.843799:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.843799:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677447.843800:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:17.0:1619677447.843800:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000400:00000200:30.0:1619677447.843801:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:17.0:1619677447.843801:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:30.0:1619677447.843802:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.843803:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.843804:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.843805:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:30.0:1619677447.843809:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.843810:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:30.0:1619677447.843813:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.843813:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:30.0:1619677447.843814:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:30.0:1619677447.843815:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:30.0:1619677447.843816:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677447.843817:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677447.843817:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:26.0:1619677447.843828:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677447.843830:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677447.843831:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.843832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:26.0:1619677447.843833:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.843835:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:31.0:1619677447.843840:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.843843:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.843845:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.843846:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.843847:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.843847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:31.0:1619677447.843848:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.843849:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.843851:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.843851:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.843852:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.843855:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.843856:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677447.843858:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677447.843859:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.843860:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.843860:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.843862:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.843862:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.843863:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.843864:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:4.2:1619677447.843871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.843874:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.843876:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.843877:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.843878:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.843879:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.843880:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:12.0:1619677447.843880:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.843881:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:26.0:1619677447.843882:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.843885:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.843887:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:9.2:1619677447.843887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677447.843888:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.843889:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.843890:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.843891:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677447.843894:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.843895:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:12.0:1619677447.843895:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677447.843895:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677447.843898:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.843898:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.843898:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677447.843899:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.843899:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.843899:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:12.0:1619677447.843899:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.843899:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:2.0:1619677447.843899:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.843900:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677447.843901:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:12.0:1619677447.843901:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:2.0:1619677447.843901:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.843902:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.843903:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.843903:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677447.843905:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.843907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.843915:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.843916:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677447.843917:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.843918:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.843918:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.843919:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677447.843919:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.843920:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.843920:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:6.0:1619677447.843921:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.843923:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.843925:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677447.844024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.844031:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.844034:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.844036:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677447.844037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:31.0:1619677447.844038:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.844038:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.844039:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.844040:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.844042:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.844043:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.844043:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:30.0:1619677447.844044:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.844046:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.844046:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:30.0:1619677447.844049:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.844050:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.844052:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:22.0:1619677447.844053:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.844054:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677447.844054:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:30.0:1619677447.844055:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.844056:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677447.844056:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722783] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.844057:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.844058:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:22.0:1619677447.844058:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.844059:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:22.0:1619677447.844059:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.844060:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:30.0:1619677447.844061:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.844062:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677447.844063:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.844065:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.844067:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677447.844067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677447.844068:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.844069:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:12.0:1619677447.844069:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:22.0:1619677447.844070:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.844071:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:12.0:1619677447.844071:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:22.0:1619677447.844072:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:12.0:1619677447.844072:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.844073:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:12.0:1619677447.844074:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.844075:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677447.844076:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.844076:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:9.2:1619677447.844076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677447.844077:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.844078:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.844079:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677447.844084:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.844086:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.844087:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.844088:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:12.0:1619677447.844089:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:2.0:1619677447.844089:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.844092:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.844092:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:12.0:1619677447.844093:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.844094:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.844094:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.844095:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.844095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.844103:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.844106:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.844106:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.844107:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.844108:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.844109:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677447.844189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.844195:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.844197:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.844198:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.844200:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.844203:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.844204:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.844205:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.844207:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.844207:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.844209:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.844212:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.844214:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.844216:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.844216:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.844217:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.844218:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.844219:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.844219:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.844220:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.844224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.844231:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.844233:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.844234:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.844235:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.844236:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.844237:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.844259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.844262:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.844265:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.844267:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.844268:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.844269:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.844271:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.844273:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.844274:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.844275:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.844276:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.844280:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.844281:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.844284:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.844285:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.844286:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.844288:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.844288:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.844289:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.844290:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677447.844302:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.844305:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.844306:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.844308:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.844309:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.844311:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.844336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677447.844342:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.844344:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.844346:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.844347:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.844350:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.844352:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.844353:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.844355:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.844356:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.844356:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677447.844360:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.844361:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.844364:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.844365:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.844366:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.844367:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.844368:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.844369:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.844369:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:4.2:1619677447.844372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.844382:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.844384:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.844385:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.844387:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.844396:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.844397:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.844398:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.844400:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.844401:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.844402:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.844402:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.844404:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677447.844418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.2:1619677447.844422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:6.0:1619677447.844423:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:26.0:1619677447.844425:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677447.844426:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:26.0:1619677447.844427:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677447.844427:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.844427:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.844428:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677447.844428:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677447.844429:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677447.844429:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.844430:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:6.0:1619677447.844432:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677447.844433:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.844433:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.844436:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.844437:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.844438:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.844440:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.844441:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.844442:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.844445:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.844447:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.844449:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.844450:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.844451:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.844453:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.844453:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.844454:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.844454:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.844459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.844466:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.844469:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.844470:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.844470:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.844472:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.844475:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.844485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.844492:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.844494:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.844495:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.844497:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677447.844498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:1.0:1619677447.844498:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.844500:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.844501:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722777] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.844502:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.844504:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.844504:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.844507:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.844509:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677447.844509:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.844513:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.844513:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.844514:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.844516:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.844516:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.844517:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.844518:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.844529:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.844532:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.844533:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.844535:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.844536:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.844537:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.844554:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.844556:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.844557:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.844559:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.844560:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.844561:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677447.844561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.844569:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.844571:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.844572:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.844574:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.844577:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.844579:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.844580:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722775] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.844581:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.844582:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677447.844583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:8.0:1619677447.844583:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.844587:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.844588:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677447.844591:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677447.844591:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.844592:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.844593:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677447.844594:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.0:1619677447.844594:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.844594:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.844595:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.844595:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:31.0:1619677447.844596:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.844598:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.844598:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.844599:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:8.2:1619677447.844600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677447.844601:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.844602:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.844603:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.844603:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.844606:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.844608:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.844608:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:31.0:1619677447.844610:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.844611:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.844611:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.844611:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.844611:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.844612:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.844613:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.844613:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.844614:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677447.844614:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.844615:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677447.844617:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:13.0:1619677447.844625:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.844628:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.844628:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.844630:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.844631:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.844632:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.844683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.844689:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.844692:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.844693:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.844694:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.844697:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.844698:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.844699:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722773] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.844700:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.844702:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.844703:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.844705:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.844707:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.844709:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.844710:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.844711:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.844712:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.844713:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.844714:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.844714:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.844726:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.844729:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.844730:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.844730:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.844732:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.844735:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.844806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.844813:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.844816:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677447.844819:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.844821:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.844823:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.844824:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.844826:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.844827:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.844828:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.844832:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.844833:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.844836:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.844837:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.844837:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.844839:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.844839:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.844840:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.844840:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677447.844851:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.844854:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.844854:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.844856:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.844858:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.844859:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.844859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677447.844869:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:5.2:1619677447.844894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:9.2:1619677447.844895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677447.844900:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.844902:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:2.0:1619677447.844903:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677447.844903:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.844905:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.844906:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.844908:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:17.0:1619677447.844908:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.844908:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.844909:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677447.844910:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.844911:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.844912:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.844913:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.844914:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677447.844914:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:17.0:1619677447.844917:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.844919:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.844921:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.844922:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.844922:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.844924:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.844924:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.844926:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.844927:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.844927:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677447.844932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677447.844937:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677447.844938:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677447.844940:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677447.844940:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:6.0:1619677447.844941:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.844941:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.844942:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.844942:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677447.844944:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.844946:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.844955:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.844957:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.844958:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.844958:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.844960:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.844960:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677447.844968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677447.844973:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.844976:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.844977:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.844979:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.844981:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.844983:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.844984:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.844985:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.844986:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.844987:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:5.2:1619677447.844987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:18.0:1619677447.844991:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.844993:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.844994:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677447.844996:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.844996:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.844997:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:18.0:1619677447.844997:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.844999:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.844999:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:26.0:1619677447.845000:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.845000:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.845001:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:26.0:1619677447.845002:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.845004:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677447.845005:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.845007:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.845008:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.845010:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677447.845014:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.845016:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677447.845019:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.845020:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677447.845021:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677447.845023:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677447.845024:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.845025:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.845025:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677447.845035:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.845037:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.845038:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.845040:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.845042:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.845043:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677447.845047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.845055:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.845057:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.845059:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.845061:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.845062:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.845063:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.845064:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.845065:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.845066:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.845067:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.845070:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.845071:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.845074:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.845074:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.845075:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.845076:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.845077:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.845078:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.845078:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.845088:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.845090:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.845091:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.845092:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.845094:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.845095:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.845117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.845123:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.845125:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.845126:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.845128:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.845130:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.845132:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.845133:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.845135:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.845136:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.845137:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.845140:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.845142:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.845144:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.845145:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.845146:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.845148:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.845149:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.845150:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.845151:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.845152:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677447.845154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677447.845157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677447.845161:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:22.0:1619677447.845162:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:20.0:1619677447.845163:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677447.845164:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.845165:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677447.845165:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.845165:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:20.0:1619677447.845166:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.845167:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.845169:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677447.845214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.845220:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.845222:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.845223:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.845225:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.845227:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.845229:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.845230:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.845231:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.845232:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.845233:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.845237:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.845238:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.845241:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.845242:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.845243:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.845244:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.845245:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.845245:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.845246:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.845250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.845257:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.845260:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.845260:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.845261:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.845263:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.845266:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677447.845339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677447.845350:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.2:1619677447.845379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.845388:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.845390:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.845391:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.845393:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.845394:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.845396:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677447.845405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677447.845412:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.845415:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.845416:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.845417:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.845418:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:4.2:1619677447.845419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.845420:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:26.0:1619677447.845427:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.845429:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:5.2:1619677447.845431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677447.845432:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.845435:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.845436:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:1.0:1619677447.845437:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677447.845438:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722763] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.845439:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.845440:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.845440:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:26.0:1619677447.845441:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:17.0:1619677447.845441:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.845444:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.845446:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.845447:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.845448:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:1.0:1619677447.845449:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.845450:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.845451:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.845452:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:1.0:1619677447.845452:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677447.845453:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677447.845453:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.845453:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:26.0:1619677447.845455:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.845455:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.845456:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.845457:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:1.0:1619677447.845457:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.845459:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:9.2:1619677447.845462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.845462:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.845462:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.845463:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.845465:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.845465:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.845466:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.845466:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.845470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:2.0:1619677447.845470:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.845472:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.845473:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.845474:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.845476:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:2.0:1619677447.845476:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.845479:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:2.0:1619677447.845479:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677447.845480:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.845480:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.845482:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.845486:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677447.845627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677447.845632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677447.845636:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.845638:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:24.0:1619677447.845639:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:30.0:1619677447.845641:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.845642:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.845642:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.845642:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677447.845643:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.845644:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.845645:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.845645:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.845646:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677447.845646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:24.0:1619677447.845647:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.845648:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.845648:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:30.0:1619677447.845649:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.845649:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722760] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.845651:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.845652:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.845652:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:30.0:1619677447.845653:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.845653:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:30.0:1619677447.845655:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.845655:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.845656:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677447.845658:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.845659:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.845660:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677447.845660:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.845660:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.845661:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.845662:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.845663:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677447.845663:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:22.0:1619677447.845663:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.845664:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:22.0:1619677447.845664:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:24.0:1619677447.845665:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.845666:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:22.0:1619677447.845666:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96722759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.845667:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:22.0:1619677447.845667:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.845668:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:22.0:1619677447.845668:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.845669:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:22.0:1619677447.845669:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.845670:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.845671:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.845672:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677447.845672:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.845674:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677447.845674:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.2:1619677447.845674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:12.0:1619677447.845676:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:22.0:1619677447.845677:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677447.845677:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.845677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.845678:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.845678:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:12.0:1619677447.845679:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.845680:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.845680:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.845681:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677447.845681:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677447.845682:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.845682:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.845683:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.845685:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:8.2:1619677447.845686:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:17.0:1619677447.845687:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.845687:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.845688:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:17.0:1619677447.845689:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.845689:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677447.845690:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:1.0:1619677447.845690:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677447.845691:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677447.845691:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.845692:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.0:1619677447.845692:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.845693:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.845694:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.845696:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.845726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:4.2:1619677447.845735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.845738:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.845739:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.845740:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677447.845743:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677447.845747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677447.845753:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:26.0:1619677447.845754:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.845756:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:26.0:1619677447.845757:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:2.0:1619677447.845757:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:26.0:1619677447.845759:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.845760:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.845761:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.845762:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.845763:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.845763:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.845765:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.845766:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96722756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.845768:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.845769:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.845770:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.845771:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:26.0:1619677447.845772:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.2:1619677447.845773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:18.0:1619677447.845775:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.845776:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.845777:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.845777:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677447.845778:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677447.845780:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.845780:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.845781:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677447.845781:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.845781:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677447.845782:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.845782:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:18.0:1619677447.845783:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.845784:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.845785:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.845786:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.845787:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:8.2:1619677447.845790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677447.845792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:6.0:1619677447.845793:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.845796:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.845797:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.845797:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:20.0:1619677447.845798:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:6.0:1619677447.845798:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.845799:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:20.0:1619677447.845800:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:6.0:1619677447.845800:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.845800:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:20.0:1619677447.845801:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.845801:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:20.0:1619677447.845803:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.845803:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.845804:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.845806:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.845812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677447.845820:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677447.845823:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677447.845825:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.845826:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677447.845828:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.845829:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.845831:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.845832:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.845833:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.845834:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.845837:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.845839:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.845843:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.845844:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.845844:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.845846:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.845847:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677447.845848:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677447.845848:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:8.2:1619677447.845852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.845860:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.845863:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.845864:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.845866:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.845866:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.845869:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677447.845882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919065782400] (20)++
00000800:00000200:18.0:1619677447.845888:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (21)++
00000800:00000200:18.0:1619677447.845891:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:2.0:1619677447.845891:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000400:00000200:18.0:1619677447.845894:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.845896:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.845898:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.845899:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.845900:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.845901:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.845902:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.845906:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.845907:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.845909:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.845910:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.845911:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.845912:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.845913:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (21)++
00000800:00000200:18.0:1619677447.845913:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (22)--
00000800:00000200:18.0:1619677447.845914:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (21)--
00000800:00000200:5.2:1619677447.845917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919065782400] (20)++
00000800:00000200:9.2:1619677447.845919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.845924:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (21)++
00000800:00000200:2.0:1619677447.845926:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:18.0:1619677447.845928:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:2.0:1619677447.845928:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.845929:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.845929:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.845929:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.845931:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.845931:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:2.0:1619677447.845932:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722753] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.845933:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.845934:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.845935:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:18.0:1619677447.845936:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.845937:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (22)++
00000800:00000200:18.0:1619677447.845938:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000400:00000200:2.0:1619677447.845939:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.845940:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.845940:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:18.0:1619677447.845941:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.845943:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:2.0:1619677447.845943:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.845944:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.845944:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:2.0:1619677447.845944:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:18.0:1619677447.845946:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.845946:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:18.0:1619677447.845947:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.845947:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:2.0:1619677447.845948:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (23)++
00000800:00000200:2.0:1619677447.845949:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (24)--
00000800:00000200:2.0:1619677447.845949:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (23)--
00000800:00000200:2.0:1619677447.845950:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677447.845951:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.845952:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.845952:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.845953:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:2.0:1619677447.845953:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.845953:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.2:1619677447.845954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677447.845954:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000800:00000200:2.0:1619677447.845955:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.845957:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.845958:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:18.0:1619677447.845958:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:18.0:1619677447.845960:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:18.0:1619677447.845961:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (21)++
00000800:00000200:18.0:1619677447.845961:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919065782400] (22)++
00000800:00000200:17.0:1619677447.845961:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:5.2:1619677447.845961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919065782400] (22)++
00000800:00000200:18.0:1619677447.845963:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (24)--
00000800:00000200:18.0:1619677447.845964:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (23)--
00000800:00000200:17.0:1619677447.845964:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:18.0:1619677447.845965:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (22)++
00000400:00000200:17.0:1619677447.845965:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.845965:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:9.2:1619677447.845966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:18.0:1619677447.845967:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000400:00000200:17.0:1619677447.845967:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.845967:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (23)++
00000800:00000200:2.0:1619677447.845968:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919065782400] (24)--
00000400:00000200:18.0:1619677447.845969:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.845969:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:2.0:1619677447.845969:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (23)--
00000800:00000200:2.0:1619677447.845970:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000400:00000200:18.0:1619677447.845972:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.845973:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:18.0:1619677447.845973:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:5.2:1619677447.845973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.0:1619677447.845975:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:18.0:1619677447.845975:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.845976:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677447.845976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000400:00000200:24.0:1619677447.845977:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.845977:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.845978:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:5.2:1619677447.845979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919065782400] (21)++
00000400:00000200:24.0:1619677447.845980:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.845981:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677447.845982:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.845982:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.845983:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:24.0:1619677447.845985:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:18.0:1619677447.845985:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.845985:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (22)++
00000800:00000200:17.0:1619677447.845986:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:18.0:1619677447.845987:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.845987:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000400:00000200:1.0:1619677447.845987:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.845988:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:17.0:1619677447.845988:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:18.0:1619677447.845989:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:17.0:1619677447.845989:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.845990:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000400:00000200:2.0:1619677447.845990:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.845990:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.845991:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:17.0:1619677447.845991:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:18.0:1619677447.845992:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (23)++
00000800:00000200:17.0:1619677447.845992:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.845992:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:18.0:1619677447.845993:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (24)--
00000800:00000200:18.0:1619677447.845993:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (23)--
00000800:00000200:17.0:1619677447.845993:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:2.0:1619677447.845993:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.845994:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.845995:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (22)++
00000800:00000200:18.0:1619677447.845996:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000400:00000200:2.0:1619677447.845996:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.845996:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677447.845997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:2.0:1619677447.845997:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.845998:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.845999:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.845999:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677447.846001:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.846002:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.846005:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.846006:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.846008:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.846008:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.846008:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:18.0:1619677447.846010:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:2.0:1619677447.846011:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.846011:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.846012:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96722748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.846012:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.846012:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:18.0:1619677447.846013:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.846014:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.846014:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:18.0:1619677447.846015:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:1.0:1619677447.846015:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.846016:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.846016:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677447.846018:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.846019:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:18.0:1619677447.846020:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.846020:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677447.846021:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677447.846022:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677447.846023:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (23)++
00000800:00000200:2.0:1619677447.846024:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (24)--
00000800:00000200:18.0:1619677447.846025:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.846025:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (23)--
00000800:00000200:18.0:1619677447.846026:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677447.846026:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677447.846026:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (22)++
00000800:00000200:30.0:1619677447.846027:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:2.0:1619677447.846027:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:18.0:1619677447.846029:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677447.846029:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (23)++
00000800:00000200:30.0:1619677447.846030:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:18.0:1619677447.846030:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (24)--
00000800:00000200:18.0:1619677447.846030:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (23)--
00000400:00000200:2.0:1619677447.846030:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.846031:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.846031:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.846032:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:31.0:1619677447.846033:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:18.0:1619677447.846033:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:30.0:1619677447.846034:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.846034:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.846034:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:22.0:1619677447.846035:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:2.0:1619677447.846036:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:508, d:32, s:-96722747] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.846037:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.846038:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:2.0:1619677447.846038:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.846039:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.846039:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:2.0:1619677447.846039:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.846040:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.846041:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:22.0:1619677447.846041:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.846042:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:18.0:1619677447.846042:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.846043:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (22)++
00000800:00000200:26.0:1619677447.846044:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:22.0:1619677447.846044:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677447.846045:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.846045:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.846045:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:18.0:1619677447.846045:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000400:00000200:2.0:1619677447.846045:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.846046:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.846047:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:18.0:1619677447.846048:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.846048:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677447.846052:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.846053:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.846053:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:2.0:1619677447.846053:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:18.0:1619677447.846054:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677447.846056:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.846056:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:18.0:1619677447.846057:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.846058:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.846059:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.846060:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (23)++
00000800:00000200:2.0:1619677447.846062:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919065782400] (24)++
00000400:00000200:18.0:1619677447.846063:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.846063:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (25)--
00000800:00000200:2.0:1619677447.846064:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (24)--
00000800:00000200:18.0:1619677447.846065:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.846065:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677447.846066:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (23)++
00000800:00000200:18.0:1619677447.846068:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.846068:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:31.0:1619677447.846069:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:18.0:1619677447.846069:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:18.0:1619677447.846070:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:2.0:1619677447.846070:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.846071:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:2.0:1619677447.846071:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.846072:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:31.0:1619677447.846073:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.846073:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.846074:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96722745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.846075:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.846075:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:2.0:1619677447.846076:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.846077:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.846078:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.846079:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (24)++
00000800:00000200:18.0:1619677447.846080:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (25)--
00000800:00000200:18.0:1619677447.846080:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (24)--
00000800:00000200:31.0:1619677447.846081:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:18.0:1619677447.846082:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (23)++
00000400:00000200:2.0:1619677447.846082:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.846083:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:12.0:1619677447.846084:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677447.846084:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.846086:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.846087:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:2.0:1619677447.846087:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.846087:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:18.0:1619677447.846088:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.846088:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.846088:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:18.0:1619677447.846090:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:2.0:1619677447.846090:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:13.0:1619677447.846091:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.846091:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.846092:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96722744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.846094:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.846094:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.846095:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677447.846096:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (24)++
00000800:00000200:2.0:1619677447.846097:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (25)--
00000800:00000200:12.0:1619677447.846098:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.846098:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (24)--
00000400:00000200:18.0:1619677447.846099:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.846099:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (23)++
00000800:00000200:2.0:1619677447.846100:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919065782400] (24)--
00000800:00000200:18.0:1619677447.846101:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677447.846101:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (23)--
00000800:00000200:2.0:1619677447.846101:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (22)++
00000800:00000200:26.0:1619677447.846102:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:18.0:1619677447.846103:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.846103:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:18.0:1619677447.846104:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677447.846105:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:18.0:1619677447.846105:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:2.0:1619677447.846105:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.846106:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.846107:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:26.0:1619677447.846108:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.846108:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (23)++
00000800:00000200:18.0:1619677447.846108:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (24)--
00000400:00000200:2.0:1619677447.846108:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.846109:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (23)--
00000800:00000200:18.0:1619677447.846110:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (22)++
00000800:00000200:18.0:1619677447.846111:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000400:00000200:2.0:1619677447.846111:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.846112:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.846113:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.846114:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.846115:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677447.846118:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:18.0:1619677447.846119:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.846119:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.846120:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677447.846120:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.846122:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.846123:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:6.0:1619677447.846123:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.846124:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.846124:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.846125:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96722742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.846125:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677447.846126:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.846126:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:18.0:1619677447.846127:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.846127:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677447.846128:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.846129:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:2.0:1619677447.846129:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677447.846131:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.846131:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (23)++
00000400:00000200:18.0:1619677447.846133:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.846133:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919065782400] (24)++
00000800:00000200:18.0:1619677447.846134:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:2.0:1619677447.846134:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (25)--
00000800:00000200:2.0:1619677447.846135:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (24)--
00000800:00000200:2.0:1619677447.846136:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (23)++
00000800:00000200:18.0:1619677447.846137:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.846137:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:18.0:1619677447.846138:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:18.0:1619677447.846139:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:2.0:1619677447.846139:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.846141:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:13.0:1619677447.846141:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677447.846142:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (24)++
00000800:00000200:18.0:1619677447.846142:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (25)--
00000400:00000200:2.0:1619677447.846142:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.846143:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (24)--
00000800:00000200:18.0:1619677447.846143:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (23)++
00000400:00000200:2.0:1619677447.846143:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:13.0:1619677447.846144:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677447.846145:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000400:00000200:13.0:1619677447.846145:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.846145:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96722741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.846146:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677447.846147:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.846147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:12.0:1619677447.846148:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677447.846148:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.846149:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.846152:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.846152:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.846153:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:2.0:1619677447.846154:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.846156:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.846156:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:2.0:1619677447.846156:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:24.0:1619677447.846157:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:18.0:1619677447.846157:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677447.846157:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.846159:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96722740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.846161:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.846161:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:8.0:1619677447.846161:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.846162:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.846163:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.846164:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:12.0:1619677447.846164:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.846165:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.846165:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:9.2:1619677447.846165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:12.0:1619677447.846166:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.846167:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.846168:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:2.0:1619677447.846168:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.846168:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:18.0:1619677447.846169:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.846169:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:2.0:1619677447.846170:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.846171:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677447.846171:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.846172:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.846172:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:2.0:1619677447.846172:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (24)++
00000800:00000200:8.0:1619677447.846173:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:2.0:1619677447.846173:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (25)--
00000800:00000200:18.0:1619677447.846174:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.846174:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (24)--
00000800:00000200:18.0:1619677447.846175:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:8.0:1619677447.846175:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:2.0:1619677447.846175:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (23)++
00000800:00000200:18.0:1619677447.846176:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:2.0:1619677447.846176:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919065782400] (24)--
00000800:00000200:2.0:1619677447.846177:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (23)--
00000800:00000200:18.0:1619677447.846178:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:2.0:1619677447.846178:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000800:00000200:18.0:1619677447.846179:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (22)++
00000800:00000200:18.0:1619677447.846180:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (22)--
00000800:00000200:18.0:1619677447.846180:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (21)--
00000800:00000200:26.0:1619677447.846186:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.846190:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.846193:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.846194:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.846196:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.846197:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.846199:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:26.0:1619677447.846200:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.846203:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.846204:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.846205:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.846206:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677447.846252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.846260:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.846262:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.846265:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.846266:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.846267:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.846269:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.846270:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.846271:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.846272:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.846273:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:21.2:1619677447.846273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:31.0:1619677447.846276:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.846278:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.846279:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.846281:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:31.0:1619677447.846282:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.846282:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:31.0:1619677447.846283:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.846284:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:22.0:1619677447.846284:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.846285:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.846286:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.846287:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:22.0:1619677447.846287:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.846288:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:22.0:1619677447.846288:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.846290:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.846291:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.846292:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677447.846292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:22.0:1619677447.846299:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.846301:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:22.0:1619677447.846303:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.846303:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.846304:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.846305:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:21.2:1619677447.846305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677447.846305:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.846307:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.846308:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677447.846308:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.846309:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.846309:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677447.846311:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.846311:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.846312:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.846312:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677447.846313:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.846316:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.2:1619677447.846317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677447.846319:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.846321:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.846321:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.846323:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677447.846324:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722737] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.846325:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.846326:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.846326:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:30.0:1619677447.846327:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.846328:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.846329:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.846330:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.846330:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.846331:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.846332:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.846334:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677447.846335:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677447.846335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677447.846336:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677447.846337:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677447.846339:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.846339:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.846341:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.846342:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.846343:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677447.846346:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.2:1619677447.846349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677447.846353:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.846356:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.846356:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:31.0:1619677447.846357:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:6.0:1619677447.846357:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.846359:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.846360:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:6.0:1619677447.846360:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.846361:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.846363:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.846375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.846380:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.846381:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.846382:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.846384:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.846385:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.846386:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677447.846453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677447.846459:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.846462:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.846463:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.846465:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.846468:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.846469:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.846471:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.846472:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.846473:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.846474:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.846477:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.846479:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.846481:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.846482:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.846483:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.846485:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.846485:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.846486:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.846486:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677447.846491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.846499:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.846501:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.846502:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.846503:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.846505:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.846505:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.2:1619677447.846618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.846623:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.846626:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.846628:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.846628:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.846630:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.846632:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.846633:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.846634:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.846636:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.846636:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.846640:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.846642:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.846646:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.846646:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.846647:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.846649:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.846650:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.846650:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.846651:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677447.846662:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.846665:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.846665:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.846667:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.846668:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.846671:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.846821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.846827:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.846830:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.846830:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.846833:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.846835:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.846836:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.846838:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.846839:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.846840:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.846841:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.846844:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.846846:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:21.2:1619677447.846847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:1.0:1619677447.846848:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.846849:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.846850:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.846851:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.846852:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.846853:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:18.0:1619677447.846854:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677447.846855:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.846855:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677447.846856:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.846857:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.846859:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:5.2:1619677447.846860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677447.846862:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.846863:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.846864:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.846865:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:18.0:1619677447.846865:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.846866:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.846866:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:18.0:1619677447.846867:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.846868:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677447.846868:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:22.0:1619677447.846869:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.846869:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.846869:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.846870:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:18.0:1619677447.846871:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:22.0:1619677447.846872:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.846873:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.846874:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677447.846876:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.846876:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.846877:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.846879:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.846879:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.846880:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.846881:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677447.846882:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.846885:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.846886:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.846887:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.846888:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.846889:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.846890:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.846892:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.846892:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.846892:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.846894:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.846897:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.846928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:4.2:1619677447.846933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.846936:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.846938:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.846939:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.846941:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677447.846941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:24.0:1619677447.846944:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.846945:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:24.0:1619677447.846945:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:26.0:1619677447.846947:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:24.0:1619677447.846947:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.846948:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:24.0:1619677447.846948:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.846949:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.846950:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.846955:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.846957:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.846957:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:26.0:1619677447.846959:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.846960:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677447.846961:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:24.0:1619677447.846961:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.846961:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677447.846962:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96722729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.846963:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.846963:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.846963:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677447.846964:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.846964:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.846964:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:26.0:1619677447.846965:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.2:1619677447.846967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:26.0:1619677447.846969:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677447.846969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677447.846970:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677447.846973:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.846974:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677447.846974:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677447.846975:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677447.846976:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677447.846977:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:1.0:1619677447.846977:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:30.0:1619677447.846978:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:26.0:1619677447.846978:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.846979:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:1.0:1619677447.846979:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:30.0:1619677447.846980:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.846980:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.846981:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.846982:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.846983:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.846983:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.846984:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677447.846985:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96722728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.846986:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:30.0:1619677447.846987:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.846988:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.846988:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:6.0:1619677447.846988:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:30.0:1619677447.846991:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.846991:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.846991:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.846992:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.846992:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.846993:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.846994:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:6.0:1619677447.846994:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.846995:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.846996:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.846997:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.846997:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:6.0:1619677447.846998:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.846999:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:22.0:1619677447.846999:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.846999:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677447.847000:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.847000:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:22.0:1619677447.847000:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:30.0:1619677447.847001:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:22.0:1619677447.847001:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.847001:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.847002:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.847002:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.847003:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677447.847022:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.847025:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.847026:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.847027:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.847029:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.847030:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.847045:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.847048:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.847049:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.847050:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677447.847050:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.847051:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677447.847122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.847125:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.847127:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.847130:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.847130:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.847133:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.847134:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677447.847136:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.847137:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.847138:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.847139:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677447.847143:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.847145:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.847147:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.847148:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677447.847149:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.847150:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.847151:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.847151:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.847152:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677447.847156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.847164:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.847167:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.847168:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.847168:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.847170:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.847172:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.847176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.847183:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.847185:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.847186:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.847187:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.2:1619677447.847188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:1.0:1619677447.847188:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.847188:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.847194:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.847197:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.847197:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.847200:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.847202:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.847203:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.847205:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.847206:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.847207:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.847208:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.847212:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.847213:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.847216:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.847217:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.847217:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.847219:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.847219:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.847220:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.847220:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.847225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.847232:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.847234:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.847235:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.847235:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.847237:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.847239:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677447.847327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677447.847334:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.847337:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.847338:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.847340:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.847343:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.847344:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677447.847346:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.847347:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.847348:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.847349:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677447.847352:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.847354:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.847357:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.847358:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.847358:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.847360:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.847361:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.847362:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.847362:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.847374:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.847376:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.847377:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.847378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:12.0:1619677447.847379:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.847381:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.847382:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.847384:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.847387:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.847389:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.847389:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.847390:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:5.0:1619677447.847392:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.847393:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:5.0:1619677447.847393:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:12.0:1619677447.847394:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.847395:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.847395:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:5.0:1619677447.847395:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.847396:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.847397:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:5.0:1619677447.847397:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.847398:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.847402:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.847404:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.847407:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.847407:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.847408:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.847410:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.847410:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.847411:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.847412:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677447.847423:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.847426:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.847427:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.847429:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.847430:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.847431:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.847482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.847489:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.847491:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.847493:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.847496:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.847496:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.847497:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677447.847498:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.847499:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.847500:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.847501:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677447.847504:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.847506:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677447.847508:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.847509:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677447.847510:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677447.847511:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.847512:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.847513:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.847513:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677447.847528:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.2:1619677447.847598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677447.847605:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677447.847608:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677447.847610:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.847612:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677447.847613:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.847615:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:13.0:1619677447.847616:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722722] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.847617:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677447.847618:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677447.847619:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:13.0:1619677447.847622:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.847624:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:13.0:1619677447.847626:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.847627:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:13.0:1619677447.847627:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:13.0:1619677447.847629:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:13.0:1619677447.847630:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677447.847631:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677447.847631:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:20.2:1619677447.847638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.847643:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.847644:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.847645:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.847646:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.847647:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.847648:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.847648:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677447.847649:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:30.0:1619677447.847650:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.847651:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.847652:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.847653:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677447.847655:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.847656:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.847657:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.847657:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677447.847660:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.847662:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677447.847665:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.847666:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.847666:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677447.847668:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.847669:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.847670:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.847672:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.847672:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677447.847678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:8.2:1619677447.847685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677447.847686:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:5.2:1619677447.847687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:31.0:1619677447.847688:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.847688:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.847689:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677447.847692:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.847694:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:18.0:1619677447.847694:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:20.0:1619677447.847695:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.847697:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.847697:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:20.0:1619677447.847698:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.847698:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:18.0:1619677447.847700:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677447.847701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:18.0:1619677447.847702:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.847704:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:18.0:1619677447.847704:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:20.0:1619677447.847705:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000400:00000200:18.0:1619677447.847705:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.847707:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.847708:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000400:00000200:18.0:1619677447.847708:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.847709:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000400:00000200:18.0:1619677447.847709:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.847709:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.847710:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:18.0:1619677447.847710:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.847713:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.847715:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.847716:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.847717:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.847718:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.847719:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.847720:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.847720:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.847721:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.847725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.847732:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.847734:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.847735:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.847736:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.847737:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.847738:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.847754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.847761:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.847764:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677447.847766:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.847769:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.847771:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.847772:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.847773:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.847774:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.847775:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677447.847779:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.847780:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.847783:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.847784:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.847785:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677447.847786:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.847787:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.847788:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.847788:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677447.847796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677447.847801:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:1.0:1619677447.847802:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677447.847804:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:1.0:1619677447.847804:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:22.0:1619677447.847805:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.847805:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677447.847806:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.847807:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.847808:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.847810:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.847811:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:1.0:1619677447.847812:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.847813:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.847814:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.847815:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.847816:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.847820:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.847822:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.847824:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.847825:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.847826:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.847827:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.847828:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.847829:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.847830:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.847841:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.847844:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.847844:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.847845:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.847847:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.847848:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677447.847904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677447.847910:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.847913:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.847913:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.847916:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.847917:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.847919:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.847920:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.847921:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.847923:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.847924:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.847927:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.847929:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.847931:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.847932:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.847933:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.847935:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.847935:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.847936:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.847936:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677447.847951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.847959:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.847962:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.847963:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.847965:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.847965:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.847967:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677447.847988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.847998:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677447.848035:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.848038:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.848039:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.848039:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.848041:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.848044:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.848047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.848055:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.848058:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.848060:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.848062:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.848063:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677447.848063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:31.0:1619677447.848064:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.848065:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.848067:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.848068:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.848068:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:31.0:1619677447.848069:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:6.0:1619677447.848071:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.848072:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:31.0:1619677447.848074:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.848075:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.848076:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:6.0:1619677447.848077:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.848078:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.848079:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:6.0:1619677447.848079:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:31.0:1619677447.848080:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:6.0:1619677447.848080:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.848081:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.848082:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:6.0:1619677447.848082:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.848083:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:6.0:1619677447.848083:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.848084:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:6.0:1619677447.848084:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.848088:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.848089:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.848093:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.848093:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.848094:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.848096:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.848097:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.848098:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.848099:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677447.848102:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.848105:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.848105:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.848107:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.848108:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677447.848108:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:30.0:1619677447.848110:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:26.0:1619677447.848111:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677447.848112:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.848114:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.848117:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677447.848136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.848141:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.848144:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.848145:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.848147:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.848150:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.848151:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:4.2:1619677447.848152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677447.848153:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.848154:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.848156:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.848156:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.848159:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.848161:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677447.848163:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677447.848163:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.848164:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.848165:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.848166:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.848167:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.848168:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:10.0:1619677447.848169:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:2.0:1619677447.848169:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.848170:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:10.0:1619677447.848171:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722712] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.848172:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.848173:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677447.848173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:10.0:1619677447.848174:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.2:1619677447.848175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000400:00000200:10.0:1619677447.848177:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.848179:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.848180:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:17.0:1619677447.848180:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677447.848182:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:17.0:1619677447.848182:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:10.0:1619677447.848182:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.848183:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:10.0:1619677447.848183:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.848183:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.848184:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:17.0:1619677447.848184:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.848184:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:10.0:1619677447.848185:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.848185:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677447.848186:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.848186:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.848186:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677447.848188:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677447.848198:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.848201:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.848202:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.848203:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.848205:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.848207:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.848210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.848213:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.848216:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.848218:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.848219:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.848221:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.848222:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.848223:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.848225:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.848226:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.848227:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.848230:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.848232:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.848234:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.848235:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.848236:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.848237:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.848238:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.848239:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.848240:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.848241:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.848246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677447.848250:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677447.848252:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:6.0:1619677447.848253:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.848254:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.848254:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.848254:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.848255:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.848255:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:6.0:1619677447.848256:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.848258:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.848405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.2:1619677447.848410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:24.0:1619677447.848413:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.2:1619677447.848414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:24.0:1619677447.848415:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.848416:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677447.848416:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:24.0:1619677447.848419:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.848419:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.848419:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:31.0:1619677447.848422:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:24.0:1619677447.848422:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.848422:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677447.848424:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.848424:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.848425:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:24.0:1619677447.848426:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.848426:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677447.848428:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.848428:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.848428:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.848429:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:2.0:1619677447.848429:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.848431:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.848431:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.848431:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.848432:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.848432:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677447.848433:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.848435:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96722708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.848438:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.848438:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.848438:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000400:00000200:31.0:1619677447.848439:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.848440:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.848441:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.848442:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.848444:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.848445:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.848445:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.848445:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:24.0:1619677447.848446:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.848446:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:24.0:1619677447.848447:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.848448:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.848448:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:24.0:1619677447.848449:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.848449:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677447.848450:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.848450:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677447.848450:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:31.0:1619677447.848451:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677447.848451:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.848451:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677447.848451:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:31.0:1619677447.848452:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677447.848452:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:31.0:1619677447.848453:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.848454:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.848455:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.848455:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:9.2:1619677447.848455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.2:1619677447.848455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677447.848459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677447.848463:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677447.848464:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.848465:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.848466:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.848466:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:1.0:1619677447.848466:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.848467:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677447.848467:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677447.848467:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.848467:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:1.0:1619677447.848467:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.848468:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:12.0:1619677447.848469:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:2.0:1619677447.848469:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.848469:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:12.0:1619677447.848470:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.848470:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.848471:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:12.0:1619677447.848472:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.848473:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:13.0:1619677447.848474:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.848475:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677447.848479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.848486:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.848489:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.848491:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.848493:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.848493:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.848494:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677447.848495:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.848497:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.848498:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.848499:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677447.848502:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.848504:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677447.848507:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677447.848507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:30.0:1619677447.848508:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677447.848508:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677447.848510:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.848510:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.848511:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.848512:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677447.848515:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.848517:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677447.848518:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677447.848520:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677447.848523:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.848524:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:22.0:1619677447.848524:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:26.0:1619677447.848526:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.848526:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:26.0:1619677447.848527:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.848528:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.848528:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.848528:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:26.0:1619677447.848529:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677447.848530:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:26.0:1619677447.848533:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:22.0:1619677447.848534:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:26.0:1619677447.848535:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677447.848537:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.848538:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677447.848538:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677447.848540:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677447.848540:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.848541:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677447.848542:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:6.0:1619677447.848543:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.848546:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.848546:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.848547:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.848548:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.848548:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.848552:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.2:1619677447.848552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:13.0:1619677447.848554:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.848555:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.848557:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.848557:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677447.848558:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.848559:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.848561:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.848562:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.848563:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.848566:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.848567:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.848568:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.848570:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.848571:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.848572:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.848575:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.848577:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.848580:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.848581:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.848582:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.848584:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.848584:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.848585:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.848585:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.848596:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.848599:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.848600:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.848601:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.848602:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.848604:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.848659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677447.848666:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677447.848669:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677447.848670:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677447.848672:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.848674:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.848676:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677447.848677:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.848679:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.848679:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.848681:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677447.848684:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.848685:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677447.848689:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.848689:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677447.848690:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677447.848692:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677447.848692:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677447.848693:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677447.848694:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677447.848695:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:21.2:1619677447.848701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:8.2:1619677447.848701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677447.848707:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677447.848709:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:5.0:1619677447.848709:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:26.0:1619677447.848710:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677447.848711:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:5.2:1619677447.848713:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677447.848715:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.848716:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.848717:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.848718:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.848719:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.848721:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677447.848722:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.848722:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.848724:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.848727:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.848729:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.848730:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.848731:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.848732:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.848733:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.848737:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.848738:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.848742:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.848742:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.848743:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.848745:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.848746:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.848746:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.848747:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.848757:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.848760:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.848761:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.848763:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.848763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.848764:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.848766:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677447.848770:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.848772:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.848773:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.848775:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.848778:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.848780:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.848781:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722702] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.848782:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.848783:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.848784:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677447.848788:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.848790:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677447.848793:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.848793:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.848794:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.848796:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.848796:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.848797:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677447.848798:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.848799:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677447.848804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.848810:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677447.848811:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677447.848812:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677447.848813:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677447.848813:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677447.848814:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677447.848814:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.848815:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677447.848816:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.848819:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.848828:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.848831:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.848832:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.848832:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.848833:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.848834:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.848980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.848983:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.848986:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.848988:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.848989:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.848990:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.848992:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.848993:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.848994:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.848996:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.848996:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.849000:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.849002:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.2:1619677447.849002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677447.849005:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.849006:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.849007:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.849009:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.849009:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.849010:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.849010:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:10.0:1619677447.849010:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.849012:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.849013:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.849016:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.849019:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.849020:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:12.0:1619677447.849021:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677447.849021:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.849023:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:10.0:1619677447.849023:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677447.849024:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.849024:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.849025:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677447.849026:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.849028:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677447.849028:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.849030:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677447.849031:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.849034:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.849034:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677447.849035:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.849037:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.849038:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.849038:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:13.0:1619677447.849039:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677447.849039:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677447.849042:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.849043:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:9.2:1619677447.849043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.849044:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.849045:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.849046:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.849051:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.849054:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.849055:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.849055:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.849057:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.849061:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.849172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.849179:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.849181:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.849182:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.849184:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.849187:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.849189:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.849190:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.849191:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.849192:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.849193:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.849196:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.849198:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.849201:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.849201:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.849202:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.849203:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.849204:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.849204:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.849205:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.849210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.849217:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.849220:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.849221:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.849221:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.849224:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.849226:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.849230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.849237:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.849240:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.849240:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.849241:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.849242:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.849242:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.849245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.849250:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.849253:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677447.849255:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677447.849256:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.849258:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.849260:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.849261:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.849263:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.849264:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.849265:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.849268:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.849270:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.849273:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.849274:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.849274:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.849276:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.849277:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.849278:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.849280:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.849280:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.2:1619677447.849285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.849288:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:31.0:1619677447.849289:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677447.849290:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:31.0:1619677447.849292:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677447.849292:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.849292:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677447.849299:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:31.0:1619677447.849301:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.849301:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677447.849302:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:30.0:1619677447.849302:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:30.0:1619677447.849303:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:30.0:1619677447.849304:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.849304:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.849306:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.849317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.849325:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.849327:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.849329:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.849331:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.849332:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.849333:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.849334:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.849336:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.849337:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.849338:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.849341:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.849343:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.849346:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.849347:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.849347:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.849349:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.849350:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.849351:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.849351:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.849447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.849455:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.849458:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.849459:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.849460:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.849461:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.849464:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677447.849498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.2:1619677447.849500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:1.0:1619677447.849504:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.849507:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.849508:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677447.849508:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:1.0:1619677447.849510:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.849511:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.849512:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.849512:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000400:00000200:1.0:1619677447.849512:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.849513:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.849514:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:1.0:1619677447.849514:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.849515:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.849516:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.849517:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.849518:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.849522:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.849524:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.849527:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.849528:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.849529:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.849531:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.849531:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.849532:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.849533:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.849545:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.849547:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.849548:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.849550:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.849552:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.849553:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677447.849593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.849601:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.849604:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.849605:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.849607:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.849610:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.849612:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677447.849613:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.849614:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.849615:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677447.849615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:26.0:1619677447.849616:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677447.849619:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.849621:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:22.0:1619677447.849621:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.849624:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:26.0:1619677447.849625:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.849625:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677447.849626:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677447.849626:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:22.0:1619677447.849627:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.849628:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677447.849628:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.849629:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:22.0:1619677447.849630:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.849631:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677447.849631:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677447.849631:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.849632:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.849633:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.849635:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.849635:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:5.2:1619677447.849637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:22.0:1619677447.849639:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677447.849640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:13.0:1619677447.849640:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:22.0:1619677447.849641:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.849643:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.849643:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:22.0:1619677447.849644:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:13.0:1619677447.849644:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.849645:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:13.0:1619677447.849645:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.849645:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677447.849647:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.849647:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:12.0:1619677447.849647:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677447.849647:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677447.849647:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677447.849648:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677447.849648:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.849649:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:10.0:1619677447.849649:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677447.849650:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.849650:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:31.0:1619677447.849651:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.849653:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677447.849653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677447.849655:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677447.849655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:31.0:1619677447.849656:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.849658:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.849659:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.849660:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.849660:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677447.849660:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:31.0:1619677447.849661:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.849662:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677447.849662:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:2.0:1619677447.849663:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:6.0:1619677447.849664:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.849664:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:2.0:1619677447.849664:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.849665:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:31.0:1619677447.849666:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.849666:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.849668:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.849669:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:31.0:1619677447.849671:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.849672:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.849672:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.849674:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.849675:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.849676:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.849676:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.849688:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.849690:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.849691:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.849692:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.849694:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.849695:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.849794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.849801:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.849803:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.849804:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.849806:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.849809:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.849810:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677447.849812:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.849813:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.849814:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.849815:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677447.849818:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.849820:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677447.849822:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.849823:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677447.849824:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677447.849825:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677447.849826:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.849826:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.849827:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677447.849839:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.849842:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.849842:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.849843:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.849845:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.849848:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.849902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677447.849909:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.849912:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677447.849913:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677447.849915:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677447.849917:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.849919:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677447.849920:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.849921:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.849922:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.849923:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677447.849927:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677447.849928:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677447.849931:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.849931:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677447.849932:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677447.849933:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677447.849934:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.849935:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677447.849936:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677447.849937:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677447.849939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677447.849942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:17.0:1619677447.849947:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:26.0:1619677447.849950:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:17.0:1619677447.849950:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.849951:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.849951:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677447.849952:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677447.849953:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:17.0:1619677447.849953:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:26.0:1619677447.849954:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:17.0:1619677447.849956:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.849991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.849998:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.850001:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.850002:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.850003:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.850006:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.850008:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.850009:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.850010:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.850011:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.850012:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.850015:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.850016:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.850019:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.850020:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.850021:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.850022:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.850023:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.850024:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.850025:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.850026:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.850028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.850031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.850035:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677447.850038:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.850038:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.850039:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.850039:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.850040:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.850041:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677447.850041:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.850042:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.850044:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.850085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.850090:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.850093:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.850094:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.850096:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.850098:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.850099:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.850101:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.850102:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677447.850102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:17.0:1619677447.850103:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.850104:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.850107:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.850109:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677447.850109:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677447.850112:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.850112:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.850113:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677447.850113:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.850114:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.850116:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:10.0:1619677447.850116:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.850117:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.850118:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:10.0:1619677447.850118:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.850119:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.850120:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677447.850120:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677447.850121:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722688] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.850123:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.850124:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677447.850124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677447.850125:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677447.850128:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.850129:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:10.0:1619677447.850130:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:1.0:1619677447.850130:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677447.850131:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.850132:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.850132:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.850132:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:13.0:1619677447.850133:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.850133:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677447.850133:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:1.0:1619677447.850133:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.850134:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.850135:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677447.850135:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677447.850135:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677447.850136:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677447.850136:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.850137:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677447.850147:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.850149:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.850150:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.850151:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:30.0:1619677447.850152:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.850153:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.850154:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:12.0:1619677447.850154:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677447.850155:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:12.0:1619677447.850155:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.850155:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.850157:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677447.850248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.850255:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.850258:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.850260:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.850262:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.850262:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.850263:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.850264:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.850266:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.850266:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.850267:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.850270:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.850272:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.850274:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.850275:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.850275:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.850277:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.850277:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.850279:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.850280:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.850281:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677447.850286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677447.850289:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.850291:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.850292:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.850297:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677447.850301:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.850302:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677447.850303:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.850304:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.850305:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.850306:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677447.850364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677447.850371:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677447.850374:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677447.850376:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.850378:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677447.850378:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.850380:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:12.0:1619677447.850381:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677447.850383:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677447.850383:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677447.850384:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:12.0:1619677447.850387:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.850389:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:12.0:1619677447.850391:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.850392:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:12.0:1619677447.850392:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:12.0:1619677447.850394:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:12.0:1619677447.850394:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677447.850395:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677447.850395:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:30.0:1619677447.850407:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.850409:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:5.2:1619677447.850409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677447.850410:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.850411:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.850413:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.850414:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677447.850415:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.850418:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.850419:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.850421:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.850424:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.850425:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.850426:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.850428:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.850429:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.850430:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.850433:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.850435:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.850438:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.850439:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.850439:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.850441:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.850442:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.850442:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.850442:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.850447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.850454:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.850457:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.850458:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.850458:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.850461:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.850463:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.850497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.850504:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.850507:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.850508:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.850510:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.850512:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.850514:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.850515:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.850516:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.850517:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.850518:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.850522:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.850524:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.850526:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.850527:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.850527:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.850529:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.850529:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.850530:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.850530:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677447.850541:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.850543:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.850544:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.850545:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.850547:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.850549:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.850569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677447.850576:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.850579:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.850579:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.850581:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.850584:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.850586:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.850587:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.850589:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.850589:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.850591:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.850594:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.850596:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.850599:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.850600:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.850600:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.850602:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.850603:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.850604:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.850604:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:10.0:1619677447.850614:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.850617:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.850618:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.850618:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.850620:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.850623:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677447.850640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.850647:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.850650:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.850652:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.850654:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.850654:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.850656:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677447.850657:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722682] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.850659:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.850660:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.850661:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:5.2:1619677447.850661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:31.0:1619677447.850664:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.850666:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:1.0:1619677447.850667:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.850669:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677447.850670:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.850670:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:17.0:1619677447.850670:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677447.850671:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:1.0:1619677447.850672:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.850673:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677447.850674:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.850675:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.850675:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:1.0:1619677447.850675:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.850677:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.850678:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677447.850679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:1.0:1619677447.850679:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.850680:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.850681:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.850684:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.850686:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:1.0:1619677447.850686:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.850689:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:1.0:1619677447.850689:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.850690:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.850690:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.850690:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.850691:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:2.0:1619677447.850692:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.850692:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.850693:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.850694:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.850694:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677447.850696:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.850700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:13.0:1619677447.850706:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:18.0:1619677447.850708:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:13.0:1619677447.850709:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.850710:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.850711:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.850711:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.850712:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000400:00000200:13.0:1619677447.850712:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.850713:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:12.0:1619677447.850713:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.850714:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:13.0:1619677447.850715:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677447.850746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.850754:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.850757:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.850757:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.850759:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.850762:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.850763:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677447.850765:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.850766:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.850767:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.850768:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677447.850771:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.850773:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677447.850776:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.850777:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677447.850777:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677447.850779:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677447.850779:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.850780:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.850780:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677447.850791:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.850794:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.850794:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.850796:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.850797:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.850798:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677447.850881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.850887:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.850890:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.850890:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.850892:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.850894:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.850895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:6.0:1619677447.850896:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.850897:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.850898:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.850899:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.850900:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:4.2:1619677447.850900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:10.0:1619677447.850902:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.850904:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:6.0:1619677447.850904:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.850905:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677447.850906:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677447.850907:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677447.850908:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.850909:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.850910:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:10.0:1619677447.850910:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.850910:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.850911:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:10.0:1619677447.850912:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:6.0:1619677447.850912:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:8.0:1619677447.850913:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.850914:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.850914:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:10.0:1619677447.850915:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.850915:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.850915:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677447.850916:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.850916:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.850916:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677447.850917:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:8.0:1619677447.850917:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.850918:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96722677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.850919:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.850921:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.850921:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.850921:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677447.850924:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:8.0:1619677447.850925:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.850927:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:10.0:1619677447.850927:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.850927:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677447.850928:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:10.0:1619677447.850929:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (24)++
00000800:00000200:8.0:1619677447.850930:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.850931:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (25)--
00000800:00000200:8.0:1619677447.850931:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677447.850932:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:10.0:1619677447.850932:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.850932:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677447.850932:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677447.850933:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.850933:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.0:1619677447.850933:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677447.850934:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.850934:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.850934:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.850935:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.850935:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:30.0:1619677447.850939:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.850940:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:8.2:1619677447.850940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677447.850941:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.850942:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.850943:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677447.850943:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.850944:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677447.850948:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.850950:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.850951:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.850952:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.850952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:17.0:1619677447.850955:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.850958:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.850959:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.850960:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.850960:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677447.850961:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:1.0:1619677447.850962:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.850964:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.850964:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.850965:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:1.0:1619677447.850965:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.850966:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:1.0:1619677447.850966:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:31.0:1619677447.850967:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:1.0:1619677447.850968:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.850969:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.850970:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.850971:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.850975:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.850976:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.850979:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.850980:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.850981:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.850983:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.850983:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.850984:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.850985:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.850995:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.850998:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.850999:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.851001:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.851002:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.851003:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.851018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.851026:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.851028:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.851031:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.851032:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.851033:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.851035:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.851036:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.851037:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.851038:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.851039:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.851043:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.851044:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.851047:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.851048:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.851048:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.851050:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.851050:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.851051:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.851052:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.851062:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.851064:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.851065:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.851067:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.851068:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.851069:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.851187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.851193:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.851196:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.851196:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.851199:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.851201:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.851203:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.851204:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.851206:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.851207:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.851208:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.851211:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.851213:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.851216:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.851216:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.851217:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.851219:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.851219:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.851221:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.851222:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.851222:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.851225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.2:1619677447.851227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677447.851231:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:1.0:1619677447.851232:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677447.851234:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677447.851234:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:24.0:1619677447.851235:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.851235:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677447.851235:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.851236:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677447.851237:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.851241:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.2:1619677447.851370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:22.0:1619677447.851376:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.851379:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677447.851379:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677447.851381:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.851384:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.851386:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677447.851387:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.851388:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.851389:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.851390:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677447.851394:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.851395:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677447.851398:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.851399:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:22.0:1619677447.851399:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677447.851401:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:22.0:1619677447.851402:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.851403:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:22.0:1619677447.851404:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:22.0:1619677447.851404:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:8.2:1619677447.851408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677447.851410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677447.851415:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000800:00000200:1.0:1619677447.851416:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:6.0:1619677447.851418:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:6.0:1619677447.851418:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677447.851419:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:1.0:1619677447.851419:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.851420:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.851420:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.851422:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.851425:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.851428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.851436:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.851438:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:20.2:1619677447.851439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:10.0:1619677447.851439:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.851441:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.851444:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.851445:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:5.0:1619677447.851445:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:26.0:1619677447.851446:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.851448:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.851448:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:26.0:1619677447.851450:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.851451:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.851451:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.851451:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.851455:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.851455:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.851456:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:26.0:1619677447.851457:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.851458:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.851459:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.851460:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.851461:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.851462:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:5.0:1619677447.851462:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:26.0:1619677447.851463:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.851465:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.851465:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677447.851465:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.851466:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.851467:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:5.0:1619677447.851467:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:26.0:1619677447.851468:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677447.851471:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677447.851472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.851472:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677447.851472:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.0:1619677447.851474:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677447.851474:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:4.2:1619677447.851474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677447.851475:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.851476:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677447.851480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:2.0:1619677447.851480:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677447.851481:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677447.851483:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:2.0:1619677447.851483:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.851484:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.851484:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677447.851484:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.851486:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.0:1619677447.851486:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:2.0:1619677447.851486:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.851489:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:2.0:1619677447.851489:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677447.851490:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.851490:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.851492:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.851495:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677447.851518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.851526:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.851529:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.851531:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.851533:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.851534:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.851535:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677447.851536:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.851538:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.851539:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.851540:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677447.851543:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.851545:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677447.851548:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.851549:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.851550:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677447.851552:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.851552:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.851553:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:5.2:1619677447.851553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677447.851555:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.851556:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677447.851558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677447.851559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677447.851559:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.851561:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.851562:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.2:1619677447.851564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677447.851564:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.851566:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.851568:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677447.851568:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.851569:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.851570:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:17.0:1619677447.851570:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.851571:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:31.0:1619677447.851572:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.851572:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.851572:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:17.0:1619677447.851572:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.851572:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.851573:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677447.851573:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:30.0:1619677447.851574:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677447.851574:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.851575:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677447.851576:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.851576:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.851577:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677447.851578:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:31.0:1619677447.851579:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.851581:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.851581:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.851582:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:17.0:1619677447.851582:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:31.0:1619677447.851583:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.851583:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:31.0:1619677447.851585:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.851585:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.851585:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677447.851586:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.851586:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677447.851587:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.851587:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677447.851590:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.851592:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677447.851595:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.851596:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677447.851597:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677447.851598:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.851598:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.851599:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.851599:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.851600:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.851600:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.851601:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:30.0:1619677447.851601:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.851602:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.851604:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:22.0:1619677447.851612:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.851615:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.851615:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.851616:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.851618:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.851622:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677447.851825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677447.851831:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.851833:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:21.2:1619677447.851834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:2.0:1619677447.851834:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.851836:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.851839:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.851840:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:18.0:1619677447.851841:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.851842:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722667] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.851843:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:18.0:1619677447.851843:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.851843:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:18.0:1619677447.851845:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.851846:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:6.0:1619677447.851846:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.851849:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.851849:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.851850:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:18.0:1619677447.851851:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:6.0:1619677447.851852:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.851853:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677447.851853:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.851854:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:6.0:1619677447.851854:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.851855:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677447.851855:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.851856:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.851857:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.851858:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.851858:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:6.0:1619677447.851859:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.851860:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.2:1619677447.851863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.851863:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.851864:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.851864:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.851866:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.851866:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.851867:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.851868:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.851869:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.2:1619677447.851872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.851872:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.851872:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.851874:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.851874:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.2:1619677447.851875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677447.851876:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.851882:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.851882:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.851883:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.851884:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.851884:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.851885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:17.0:1619677447.851886:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.851886:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.851890:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.851891:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.851892:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:17.0:1619677447.851892:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:1.0:1619677447.851892:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677447.851893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:1.0:1619677447.851893:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.851894:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.851895:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.851895:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.851896:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (21)++
00000400:00000200:17.0:1619677447.851896:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.851897:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000400:00000200:17.0:1619677447.851897:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.851898:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:17.0:1619677447.851898:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.851900:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:17.0:1619677447.851901:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.851903:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:17.0:1619677447.851904:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.851904:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.851906:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.851907:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.851908:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.851908:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.851909:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:24.0:1619677447.851910:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:17.0:1619677447.851910:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:24.0:1619677447.851911:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96722664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.851911:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.851911:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.851912:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677447.851913:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.851914:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.851915:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.851918:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.851920:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.851923:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.851923:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:24.0:1619677447.851924:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.851925:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:12.0:1619677447.851926:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:24.0:1619677447.851927:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.851927:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:12.0:1619677447.851927:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.851928:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.851929:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000400:00000200:12.0:1619677447.851929:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.851930:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677447.851931:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.851932:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.2:1619677447.851932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677447.851936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677447.851942:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.0:1619677447.851943:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.851945:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:5.0:1619677447.851945:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:8.0:1619677447.851946:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:5.0:1619677447.851946:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.851947:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:5.0:1619677447.851947:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.851948:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.851950:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.851960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.851963:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.851965:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.851967:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.851968:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.851970:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.851972:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:4.2:1619677447.851972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:20.0:1619677447.851975:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.851976:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.851977:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.851978:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.851979:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.851982:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:20.0:1619677447.851982:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.851983:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677447.851984:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.851985:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.851988:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.851988:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.851989:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.851989:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:26.0:1619677447.851990:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.851991:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.851991:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:26.0:1619677447.851992:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.851992:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677447.851993:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.851993:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.851993:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:26.0:1619677447.851994:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.851997:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.851999:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.852001:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.852002:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:26.0:1619677447.852003:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:31.0:1619677447.852005:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (24)++
00000800:00000200:26.0:1619677447.852005:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:26.0:1619677447.852006:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.852006:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.852007:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677447.852008:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (24)--
00000400:00000200:31.0:1619677447.852008:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.852010:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.852011:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:30.0:1619677447.852012:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:31.0:1619677447.852013:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.852013:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.852014:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.852015:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.852016:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.852034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:31.0:1619677447.852035:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677447.852037:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.852038:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.852039:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.852039:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.852040:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:1.0:1619677447.852040:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:20.2:1619677447.852041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677447.852043:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677447.852044:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677447.852046:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.852049:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:1.0:1619677447.852049:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.852051:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:31.0:1619677447.852052:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:1.0:1619677447.852052:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.852054:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.852056:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.852056:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:1.0:1619677447.852056:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.852057:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:31.0:1619677447.852059:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.852060:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.852061:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.852062:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.852062:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:31.0:1619677447.852064:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.852065:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.852066:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:1.0:1619677447.852066:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.852066:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:1.0:1619677447.852067:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:31.0:1619677447.852069:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.852069:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:1.0:1619677447.852069:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677447.852070:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:31.0:1619677447.852071:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:1.0:1619677447.852071:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:31.0:1619677447.852074:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.852074:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.852075:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.852077:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.852077:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.852078:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.852079:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:10.0:1619677447.852081:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677447.852083:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677447.852084:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677447.852085:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.852087:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.852089:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:10.0:1619677447.852089:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:30.0:1619677447.852091:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.852092:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.852093:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.852095:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.852096:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:26.0:1619677447.852100:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677447.852103:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677447.852104:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677447.852104:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677447.852105:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677447.852105:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:4.2:1619677447.852126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.852133:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.852136:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.852137:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.852139:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.852142:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.852143:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.852144:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.852146:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.852147:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.852148:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.852151:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.852153:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.2:1619677447.852154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:24.0:1619677447.852156:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.852157:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.852157:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.852159:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.852159:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.852160:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.852160:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677447.852160:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.852163:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.852164:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677447.852165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:2.0:1619677447.852166:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.852168:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.852169:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.852171:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.852172:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.852173:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:2.0:1619677447.852173:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.852174:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677447.852175:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.852176:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.852177:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.852177:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.852179:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.852179:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.852182:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.852183:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:2.0:1619677447.852183:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.852183:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.852185:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.852185:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.852186:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.852187:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.852191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.852200:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.852202:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.852203:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.852205:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.852206:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.852208:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.852444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.852452:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.852455:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.852456:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.852458:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.852460:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.852461:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:21.2:1619677447.852462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:10.0:1619677447.852463:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.852464:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.852465:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.852466:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677447.852469:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.852470:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677447.852471:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677447.852472:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:10.0:1619677447.852473:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.852474:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.852474:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:21.2:1619677447.852475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:10.0:1619677447.852475:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677447.852476:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.852476:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677447.852477:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.852477:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677447.852477:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.852477:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:31.0:1619677447.852478:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677447.852480:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.852481:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.852482:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.852482:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:8.0:1619677447.852483:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:31.0:1619677447.852486:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.852486:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.852487:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:31.0:1619677447.852488:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677447.852488:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:8.0:1619677447.852489:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.852490:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.852490:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.852490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677447.852491:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.852491:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.852492:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:8.0:1619677447.852492:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.852493:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:26.0:1619677447.852493:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.852493:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:31.0:1619677447.852494:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677447.852494:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:8.0:1619677447.852494:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96722655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.852495:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.852496:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677447.852496:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.852496:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677447.852497:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.852497:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677447.852498:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.852498:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677447.852499:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.852499:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:8.0:1619677447.852502:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.852503:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677447.852504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677447.852505:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:1.0:1619677447.852506:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.852508:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.0:1619677447.852508:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677447.852508:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:8.0:1619677447.852509:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:1.0:1619677447.852509:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96722654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.852510:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:8.0:1619677447.852510:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:1.0:1619677447.852510:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.852511:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:1.0:1619677447.852511:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.852512:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:22.0:1619677447.852512:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.852512:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.852512:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:6.0:1619677447.852512:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:1.0:1619677447.852512:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:10.0:1619677447.852513:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:8.0:1619677447.852513:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:30.0:1619677447.852514:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:26.0:1619677447.852514:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:8.0:1619677447.852514:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:30.0:1619677447.852515:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677447.852515:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.852515:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.852516:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:22.0:1619677447.852516:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.852516:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:8.2:1619677447.852517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:1.0:1619677447.852517:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.852519:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.852520:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677447.852522:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.852522:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.852523:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.0:1619677447.852525:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:1.0:1619677447.852525:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.852526:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.852526:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.852527:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677447.852528:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.852528:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.852530:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.852532:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.852533:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677447.852534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:8.2:1619677447.852536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:22.0:1619677447.852540:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.852540:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.852541:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.852542:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.852542:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.852544:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.852544:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000400:00000200:22.0:1619677447.852545:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.852547:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:5.0:1619677447.852547:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:22.0:1619677447.852548:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.852548:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.0:1619677447.852548:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.852548:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:6.0:1619677447.852549:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:5.0:1619677447.852549:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:22.0:1619677447.852550:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:20.0:1619677447.852550:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:22.0:1619677447.852551:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.852551:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.852552:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.852553:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.852553:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.852554:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.852554:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.852555:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:22.0:1619677447.852556:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.852559:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.852561:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.852563:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.852564:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.852565:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.852566:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.852567:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.852568:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.852568:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.852574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.852581:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.852583:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.852584:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.852586:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.852587:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.852588:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.852624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.852632:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.852634:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.852636:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.852638:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.852639:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.852640:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677447.852641:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.852643:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.852644:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.852644:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677447.852647:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.852649:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.852651:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.852652:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.852652:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.852654:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.852654:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.852656:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.852656:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677447.852666:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.852669:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.852669:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.852671:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.852672:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.852674:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677447.852677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.852680:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.852682:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.852684:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.852685:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677447.852685:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677447.852687:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677447.852687:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.852688:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000400:00000200:20.0:1619677447.852688:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677447.852689:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677447.852689:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:20.0:1619677447.852690:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.852691:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:20.0:1619677447.852691:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.852692:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.852693:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.852696:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.852698:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.852701:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.852701:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.852702:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.852704:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.852704:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.852705:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.852706:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.852717:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.852720:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.852721:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.852721:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.852723:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.852725:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.852735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.852742:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.852745:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.852746:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.852747:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.852749:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.852750:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.852751:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.852753:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.852754:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.852755:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.852758:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.852760:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.852762:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.852763:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.852764:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.852765:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.852766:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.852766:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.852767:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677447.852777:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.852780:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.852781:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.852782:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.852784:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.852785:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.852807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.852813:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.852816:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.852817:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.852819:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.852822:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.852823:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.852825:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.852826:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.852827:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.852828:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.852831:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.852832:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.852836:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.852836:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.852837:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.852839:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.852840:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.852840:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.852841:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677447.852853:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.852856:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.852857:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.852859:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.852860:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.852862:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.852871:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.852873:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.852874:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.852875:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.852876:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.852877:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.853008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.853016:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.853018:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.853019:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.853021:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.853023:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.853025:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:21.2:1619677447.853026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:10.0:1619677447.853026:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.853028:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.853029:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.853030:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.853033:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:10.0:1619677447.853033:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.853035:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.853036:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.853037:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:10.0:1619677447.853038:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.853039:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.853039:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677447.853040:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:24.0:1619677447.853042:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.853042:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.853042:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:24.0:1619677447.853043:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677447.853043:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.853043:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:24.0:1619677447.853044:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96722647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.853045:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.853046:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.853047:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:9.2:1619677447.853048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:24.0:1619677447.853050:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.853051:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:4.2:1619677447.853051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.853054:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.853054:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.853055:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.853055:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677447.853056:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.853057:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:26.0:1619677447.853058:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677447.853058:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.853058:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:2.0:1619677447.853058:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.853059:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.853059:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.853061:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.853062:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677447.853062:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:26.0:1619677447.853064:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.853065:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:26.0:1619677447.853068:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.853069:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:13.0:1619677447.853069:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:26.0:1619677447.853070:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.853071:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:26.0:1619677447.853072:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677447.853072:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.853073:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677447.853073:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:26.0:1619677447.853074:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:12.0:1619677447.853075:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.853076:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677447.853077:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.853079:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.853082:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.853083:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.853083:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677447.853085:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.853085:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.853086:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.853088:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.853088:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677447.853094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.853099:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677447.853101:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677447.853102:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.853102:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.853103:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.853103:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.853104:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677447.853105:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.853105:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677447.853108:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677447.853237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.853245:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.2:1619677447.853246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677447.853247:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.853249:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.853251:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.853251:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.0:1619677447.853252:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677447.853253:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.853254:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.853255:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:31.0:1619677447.853256:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.853257:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.853257:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677447.853258:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.853258:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.853260:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.853262:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.853262:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.853263:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.853265:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.853265:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.853266:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.853267:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677447.853268:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.853268:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.853269:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.853270:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.853271:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:5.0:1619677447.853271:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.853272:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.853272:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677447.853273:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.853275:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.853276:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.853277:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.853279:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.853279:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.853281:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.853282:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.853282:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677447.853283:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.853285:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.853286:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.853287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:12.0:1619677447.853288:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.853289:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.853290:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677447.853290:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.853292:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:6.0:1619677447.853292:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.853297:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677447.853298:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677447.853298:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.853299:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.853300:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.853302:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.853304:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.853312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:4.2:1619677447.853314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:18.0:1619677447.853318:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.853321:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.853321:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.0:1619677447.853322:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:18.0:1619677447.853323:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.853325:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.853326:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:18.0:1619677447.853326:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.853327:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.853328:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.853329:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.853330:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.853331:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.853331:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.853333:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.853333:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.853335:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96722642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.853336:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.853337:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.853338:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.853339:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.853339:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.853342:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.853343:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.853343:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677447.853343:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.853345:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.853345:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:8.0:1619677447.853345:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.853347:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.853348:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.853348:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.0:1619677447.853348:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.853348:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.853349:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:9.2:1619677447.853351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.853351:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.853351:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.853352:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.853352:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.2:1619677447.853354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:8.2:1619677447.853357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.853360:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.853361:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.853361:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.853361:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.853362:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.853362:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.853363:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.853363:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.853364:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677447.853365:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677447.853365:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.853368:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.853369:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.853369:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.853371:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.853374:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.853382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.853389:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.853391:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.853392:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.853393:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.853393:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.853394:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.853444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.853453:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.853456:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.853458:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.853460:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.853461:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.853462:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.853463:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.853465:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.853466:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.853467:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:4.2:1619677447.853467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:30.0:1619677447.853470:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.853472:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.853474:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.853475:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.853475:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677447.853475:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677447.853477:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.853477:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.853478:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677447.853478:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.853478:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677447.853479:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:26.0:1619677447.853480:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.853483:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.853485:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677447.853486:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.853488:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.853488:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.853489:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:13.0:1619677447.853490:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:26.0:1619677447.853492:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.853492:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.853493:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.853494:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:13.0:1619677447.853494:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.853496:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677447.853497:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.853497:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677447.853498:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677447.853499:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677447.853500:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677447.853501:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.853501:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.853502:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:13.0:1619677447.853505:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.2:1619677447.853506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:13.0:1619677447.853508:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.853508:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.853509:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.853510:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.853511:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.853514:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.853517:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.853518:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.853519:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.853520:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.853522:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677447.853538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.853543:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.853546:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.853547:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.853549:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.853551:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.853552:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.853553:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677447.853553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:22.0:1619677447.853555:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.853556:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.853557:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:24.0:1619677447.853560:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:22.0:1619677447.853560:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.853562:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:24.0:1619677447.853563:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.853564:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677447.853565:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.853566:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.853566:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.853567:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:24.0:1619677447.853569:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.853569:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.853569:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.853570:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:24.0:1619677447.853571:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:22.0:1619677447.853571:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000400:00000200:24.0:1619677447.853572:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96722638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.853572:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:24.0:1619677447.853574:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.853574:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.853576:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:21.2:1619677447.853577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:24.0:1619677447.853579:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.853581:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.853582:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:6.0:1619677447.853582:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:24.0:1619677447.853584:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.853584:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677447.853584:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:6.0:1619677447.853584:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:26.0:1619677447.853585:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:24.0:1619677447.853585:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.853585:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677447.853586:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:10.0:1619677447.853586:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.853586:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.853587:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.853587:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.853588:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:10.0:1619677447.853588:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.853591:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:9.2:1619677447.853592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.853601:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.853604:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.853604:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.853605:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.853607:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.853610:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.853765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.853771:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.853774:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.853775:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.853776:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.853779:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.853780:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.853782:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.853783:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.853784:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.853785:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.853788:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.853790:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.853793:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.853794:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.853794:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.853796:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.853797:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.853798:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.853799:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.853800:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677447.853805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677447.853808:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677447.853810:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677447.853811:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.853812:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.853812:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.853813:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677447.853814:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.853814:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.853815:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.853817:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.853833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.853841:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.853844:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.853845:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.853846:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.853849:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.853850:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677447.853851:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.853853:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.853854:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.853855:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677447.853858:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.853859:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677447.853862:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.853863:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677447.853864:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.853866:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.853866:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.853868:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677447.853869:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677447.853869:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677447.853872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677447.853875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677447.853879:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677447.853882:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:2.0:1619677447.853882:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.853883:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.853883:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.853884:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677447.853885:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:2.0:1619677447.853885:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.853886:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677447.853888:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.853889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.853896:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.853899:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.853900:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.853900:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.853901:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.853901:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.853938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.853944:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.853946:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.853947:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.853948:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.853950:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.853952:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.853953:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.853954:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.853955:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.853956:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.853959:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.853961:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:21.2:1619677447.853963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677447.853964:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.853965:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.853966:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.853968:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.853968:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.853969:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.853969:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:31.0:1619677447.853971:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.853974:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.853976:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.853978:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.853978:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.853980:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:13.0:1619677447.853980:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:31.0:1619677447.853981:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.853982:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.853982:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:31.0:1619677447.853983:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677447.853983:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.853984:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:13.0:1619677447.853985:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.853986:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:31.0:1619677447.853988:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.853990:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.853991:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.853992:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.853993:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.853994:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677447.853995:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677447.853996:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.853997:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.853998:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.853998:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.854001:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.854002:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677447.854002:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677447.854003:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.854004:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.854004:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.854005:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.854005:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.854007:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.854008:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.854022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677447.854029:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.854032:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677447.854033:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677447.854035:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.854037:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.854039:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677447.854040:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.854041:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.854042:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.854043:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677447.854046:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.854048:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677447.854050:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.854051:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677447.854052:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677447.854053:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677447.854054:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.854055:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677447.854055:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677447.854060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.854067:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.854070:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.854071:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.854071:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.854073:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.854076:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.854080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.854086:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.854089:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.854090:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.854090:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.854091:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.854091:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677447.854229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677447.854234:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677447.854237:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677447.854238:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677447.854240:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.854242:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.854244:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677447.854245:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.854246:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.854247:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.854248:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677447.854252:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.854253:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677447.854256:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.854257:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:17.0:1619677447.854257:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677447.854259:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677447.854259:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677447.854260:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677447.854260:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:22.0:1619677447.854272:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.854275:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.854276:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.854276:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.854278:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.854280:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.854290:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.854293:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.854298:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.854300:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.854300:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.854300:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.854436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.854444:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.854447:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.854448:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.854450:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.854452:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.854454:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677447.854455:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.854457:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.854458:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.854459:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677447.854462:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.854464:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677447.854466:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.854467:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677447.854468:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677447.854469:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677447.854470:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.854471:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.854471:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:4.2:1619677447.854484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677447.854485:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.854487:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.854488:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.854488:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.854490:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.854492:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.854493:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.854495:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.854496:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.854497:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.854501:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.854502:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.854503:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.854504:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.854505:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.854506:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.854509:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.854511:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.854513:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.854514:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.854515:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.854516:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.854517:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.854518:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.854519:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.854520:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.854522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.854526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.854530:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.854533:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.854533:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.854534:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.854534:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.854535:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.854535:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677447.854536:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.854537:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.854539:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.854609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.854615:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.854618:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.854619:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.854621:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.854624:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.854625:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677447.854626:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722629] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.854628:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.854629:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.854629:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677447.854633:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.854634:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677447.854636:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.854637:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.854637:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677447.854639:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.854639:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.2:1619677447.854640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:10.0:1619677447.854640:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.854640:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.0:1619677447.854647:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.854649:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.854650:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:30.0:1619677447.854652:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:8.0:1619677447.854652:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.854654:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:8.0:1619677447.854654:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.854655:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.854656:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.854657:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.854658:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96722628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.854659:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.854659:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:8.0:1619677447.854659:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.854660:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.854661:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677447.854664:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.854665:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.854668:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.854668:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.854669:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.854670:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.854671:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.854672:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.854673:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.854674:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:21.2:1619677447.854679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:13.0:1619677447.854683:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:24.0:1619677447.854686:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:13.0:1619677447.854686:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.854686:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.854687:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.854688:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:13.0:1619677447.854688:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.854689:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.854690:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.854691:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.854719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.854726:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.854729:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.854731:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.854733:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.854733:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.854735:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.854736:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.854737:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.854738:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.854739:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.854742:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.854744:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.854747:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.854748:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.854748:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.854750:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.854751:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.854752:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.854754:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.854754:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677447.854756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677447.854760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.854768:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.2:1619677447.854768:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:30.0:1619677447.854770:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.854771:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677447.854771:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677447.854772:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677447.854773:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.854774:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.854774:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:20.0:1619677447.854775:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.854776:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.854778:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.854779:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.854780:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.854781:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.854781:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.854783:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.854784:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.854785:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.854786:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.854787:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.854791:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.854792:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.854795:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.854796:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.854797:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.854799:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.854799:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.854800:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.854800:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.854811:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.854814:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.854815:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.854817:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.854818:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.854820:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.854827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677447.854830:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677447.854833:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677447.854835:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.854836:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677447.854838:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.854840:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:20.0:1619677447.854841:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.854842:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.854843:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.854844:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:20.0:1619677447.854848:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.854850:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677447.854853:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.854854:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:20.0:1619677447.854854:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677447.854856:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677447.854857:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677447.854858:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677447.854858:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:12.0:1619677447.854868:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.854872:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.854872:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.854874:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.854875:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.854877:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677447.854892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.854900:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.854903:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.854905:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.854907:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.854908:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.854909:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.854910:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.854912:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.854913:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.854914:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.2:1619677447.854916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677447.854917:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.854919:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.854921:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.854922:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.854922:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.854922:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677447.854924:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.854924:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.854925:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677447.854925:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.854925:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677447.854926:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:17.0:1619677447.854927:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.854930:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.854931:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.854932:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.854934:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.854935:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.854935:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:17.0:1619677447.854936:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.854938:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.854938:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.854939:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677447.854941:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.854942:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.854942:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.854943:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.854945:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.854946:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.854946:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.854948:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.854949:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.854949:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.854950:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.854960:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.854963:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.854964:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.854965:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.854967:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.854968:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.854978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.854984:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.854986:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.854989:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.854989:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.854992:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.854993:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677447.854995:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722622] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.854996:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.854997:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.854998:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.855002:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.855004:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.855006:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.855007:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.855007:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.855009:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.855010:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.855010:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.855011:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677447.855015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.855023:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.855025:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.855026:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.855026:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.855028:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.855031:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.855056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.855064:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.855066:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.855067:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.855069:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.855072:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.855073:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677447.855075:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.855076:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.855077:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.855078:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677447.855081:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.855083:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677447.855085:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.855086:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677447.855086:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677447.855088:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677447.855088:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.855089:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.855089:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677447.855100:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.855103:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.855103:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.855105:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.855107:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.855108:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.855118:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.855120:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.855121:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.855122:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.855122:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.855124:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677447.855181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677447.855187:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.855190:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.855191:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.855193:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.855196:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.855197:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.855199:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.855200:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.855201:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.855202:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.855205:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.855207:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.855209:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.855210:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.855211:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.855212:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.855213:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.855213:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.855214:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.855219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.855225:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.855228:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.855229:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.855229:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.855231:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.855232:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.855246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.855253:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.855256:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.855257:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.855259:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.855261:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.855263:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.855264:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.855265:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.855267:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.855268:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.855271:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.855273:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.855275:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.855276:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.855277:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.855278:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.855279:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.855280:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.855280:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.855284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.855292:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.855299:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.855300:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.855301:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.855303:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.855306:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.855447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.855452:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.855455:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.855455:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.855457:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.855460:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.855461:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.855462:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.855463:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.855464:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677447.855465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:1.0:1619677447.855465:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.855468:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.855470:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677447.855470:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.855472:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.855473:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:1.0:1619677447.855473:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:22.0:1619677447.855474:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677447.855474:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.855475:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.855476:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.855476:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.855476:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.855477:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677447.855478:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.855479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:6.0:1619677447.855480:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.855482:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.855483:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.855484:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.855485:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:31.0:1619677447.855487:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:6.0:1619677447.855489:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.855490:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:22.0:1619677447.855491:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:31.0:1619677447.855492:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.855492:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.855492:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.855493:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.855494:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.855495:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.855495:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.855495:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.855496:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.855496:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:31.0:1619677447.855497:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.855498:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722616] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.855498:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.855499:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.855499:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.855499:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:31.0:1619677447.855500:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.855500:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:31.0:1619677447.855501:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.855501:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:6.0:1619677447.855501:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:31.0:1619677447.855505:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.855506:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:9.2:1619677447.855508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677447.855509:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.855510:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.855510:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:21.2:1619677447.855511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.855512:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.855512:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:5.2:1619677447.855512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677447.855513:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.855515:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:18.0:1619677447.855515:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:31.0:1619677447.855517:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.855518:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:18.0:1619677447.855518:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.855519:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.855519:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:31.0:1619677447.855520:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.855520:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677447.855521:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.855521:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677447.855522:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.855523:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:10.0:1619677447.855523:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:18.0:1619677447.855524:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:31.0:1619677447.855525:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.855525:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:13.0:1619677447.855525:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:31.0:1619677447.855527:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.855527:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:10.0:1619677447.855527:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.855528:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.855528:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:13.0:1619677447.855528:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.855529:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.855530:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:10.0:1619677447.855531:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677447.855532:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.855532:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.855533:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:10.0:1619677447.855533:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.855534:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:10.0:1619677447.855534:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.855535:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.855536:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677447.855538:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.855538:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677447.855539:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:10.0:1619677447.855539:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.855540:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.855541:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677447.855541:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677447.855543:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.855544:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:10.0:1619677447.855544:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.855545:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677447.855545:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677447.855546:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677447.855547:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677447.855548:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.855548:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.855549:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677447.855550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677447.855557:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:30.0:1619677447.855558:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677447.855559:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:30.0:1619677447.855561:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.855561:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677447.855561:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.855562:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.855562:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:22.0:1619677447.855563:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.855564:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.855565:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.855566:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.855567:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.855568:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.855569:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.855570:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677447.855582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677447.855587:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.855590:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.855591:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.855593:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.855595:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.855597:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677447.855598:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.855599:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.855601:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.855601:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677447.855605:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.855607:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.855610:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.855611:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677447.855612:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677447.855613:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.855614:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.855615:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.855616:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.855617:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.2:1619677447.855622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:12.0:1619677447.855626:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:18.0:1619677447.855627:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.855629:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:12.0:1619677447.855629:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:18.0:1619677447.855630:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:12.0:1619677447.855630:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.855631:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.855631:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677447.855633:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.855634:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.855735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.855742:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.855744:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.855745:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.855747:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.855750:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.855751:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.855752:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.855754:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.855755:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.855756:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.855759:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.855760:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.855763:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.855764:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.855764:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.855766:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.855766:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.855767:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.855767:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677447.855778:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.855780:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.855781:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.855782:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.855784:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.855786:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.855869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.855874:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.855877:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.855878:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.855880:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.855882:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.855884:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.855885:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.855886:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.855887:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.855888:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.855891:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.855893:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.855895:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.855896:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.855896:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.855898:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.855899:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.855899:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.855900:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.855912:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.855914:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.855915:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.855915:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.855917:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.855919:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.855928:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.855930:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.855931:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.855931:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.855932:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.855933:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677447.856020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.856027:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.856030:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.856033:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.856034:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.856036:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.856038:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.856039:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.856040:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.856041:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.856042:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.856045:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.856046:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:21.2:1619677447.856046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677447.856049:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.856050:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.856050:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.856052:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.856052:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.856053:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.856053:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677447.856054:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677447.856054:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:31.0:1619677447.856055:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677447.856056:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.856058:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.856060:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.856061:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.856062:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.856063:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.856064:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.856067:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.856069:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.856072:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.856072:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.856073:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.856074:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:13.0:1619677447.856074:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677447.856075:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677447.856076:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677447.856076:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:13.0:1619677447.856076:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.856077:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.856079:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.856080:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.856081:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677447.856081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.856089:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.856092:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.856093:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.856095:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.856096:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:12.0:1619677447.856097:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.856098:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677447.856100:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:8.2:1619677447.856100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:12.0:1619677447.856101:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.856101:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.856102:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.856103:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.856108:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.856111:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.856112:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.856113:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.856114:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.856115:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.856186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.856191:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.856193:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.2:1619677447.856196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:20.0:1619677447.856196:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.856196:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.856199:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.856201:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:6.0:1619677447.856202:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.856205:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.856205:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:5.0:1619677447.856206:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677447.856207:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.856208:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:6.0:1619677447.856208:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.856209:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.856210:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.856210:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677447.856211:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:6.0:1619677447.856211:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:5.2:1619677447.856211:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677447.856212:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677447.856213:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.856214:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677447.856215:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.856215:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.856216:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.856217:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.856217:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677447.856218:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.856218:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677447.856220:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677447.856220:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:1.0:1619677447.856220:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:5.0:1619677447.856221:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:10.0:1619677447.856222:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96722606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.856224:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.856224:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677447.856225:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.856225:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.856225:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.856226:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:5.0:1619677447.856226:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677447.856226:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:1.0:1619677447.856227:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.856228:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.856228:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677447.856229:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.856229:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000400:00000200:1.0:1619677447.856229:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:5.0:1619677447.856230:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:6.0:1619677447.856231:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677447.856231:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96722605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.856232:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.856232:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.856233:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:1.0:1619677447.856233:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677447.856234:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.856234:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.856235:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:1.0:1619677447.856235:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:10.0:1619677447.856237:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.856237:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677447.856238:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:1.0:1619677447.856239:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.856240:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:10.0:1619677447.856240:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677447.856240:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677447.856240:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.856241:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677447.856241:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677447.856242:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677447.856242:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677447.856243:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:10.0:1619677447.856243:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:1.0:1619677447.856243:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.856244:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677447.856244:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.856244:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:31.0:1619677447.856245:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.856246:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:1.0:1619677447.856246:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.856247:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:1.0:1619677447.856247:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.856248:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677447.856249:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.856249:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.2:1619677447.856249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677447.856249:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:30.0:1619677447.856250:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:1.0:1619677447.856250:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677447.856251:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677447.856251:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.856252:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:30.0:1619677447.856253:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.856254:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:5.2:1619677447.856254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677447.856255:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:12.0:1619677447.856255:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.856256:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:12.0:1619677447.856257:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.856258:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677447.856259:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677447.856259:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:13.0:1619677447.856259:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677447.856260:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677447.856260:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677447.856260:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.856262:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677447.856262:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:22.0:1619677447.856263:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.856263:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.856264:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677447.856265:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.856265:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677447.856266:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.856267:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.856268:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677447.856268:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677447.856269:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.856270:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.856271:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677447.856324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.856332:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.856334:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.856335:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.856337:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.856340:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.856341:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.856343:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.856344:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.856345:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.856346:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.856349:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.856351:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.856355:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.856356:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.856356:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.856358:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.856358:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.856359:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.856359:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.856364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.856372:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.856375:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.856376:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.856376:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.856378:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.856381:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.856382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.856389:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.856392:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.856392:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.856393:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.856393:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.856394:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677447.856474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677447.856481:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.856483:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.856484:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.856486:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.856489:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.856490:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.856492:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.856493:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.856494:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.856495:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.856498:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.856500:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.856502:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.856503:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.856504:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.856506:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.856506:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.856507:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.856507:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:21.2:1619677447.856516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:6.0:1619677447.856521:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:4.2:1619677447.856522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677447.856523:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:6.0:1619677447.856523:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.856524:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.856524:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.856525:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.856526:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:6.0:1619677447.856527:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:10.0:1619677447.856529:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:8.0:1619677447.856529:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.856529:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:8.0:1619677447.856531:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.856532:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.856533:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:8.0:1619677447.856533:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.856534:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722602] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.856535:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.856535:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.856536:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.856537:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.856538:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677447.856539:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.856540:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.856541:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677447.856542:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.856543:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.856544:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.856544:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:10.0:1619677447.856547:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.856547:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.856548:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.856548:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.856549:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.856550:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.856550:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.856551:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:21.2:1619677447.856552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677447.856552:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.856552:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:10.0:1619677447.856553:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.856553:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.856555:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.856555:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.856556:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:9.2:1619677447.856556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.856557:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.856558:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677447.856560:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.856562:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:4.2:1619677447.856563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.856563:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:30.0:1619677447.856564:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.856566:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:31.0:1619677447.856567:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.856567:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.856567:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.856568:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:18.0:1619677447.856568:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677447.856569:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:30.0:1619677447.856570:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96722600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.856570:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:2.0:1619677447.856570:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.856572:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677447.856572:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677447.856572:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.856572:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:30.0:1619677447.856573:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.856573:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677447.856573:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.856574:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677447.856574:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:30.0:1619677447.856575:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677447.856575:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.856579:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.856579:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677447.856580:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677447.856584:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.856584:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:30.0:1619677447.856585:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677447.856587:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677447.856587:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.856588:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.856589:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677447.856593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.856600:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.856603:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.856604:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.856604:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.856607:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.856610:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.856640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.856647:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.856650:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.856652:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.856654:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.856655:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.856656:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.856658:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.856659:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.856660:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.856661:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.856664:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.856666:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.856669:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.856670:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.856671:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.856672:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.856673:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.856674:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.856674:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.856678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.856686:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.856689:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.856690:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.856692:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.856693:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.856695:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677447.856708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.856713:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.856716:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.856717:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.856719:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.856721:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.856723:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.856724:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.856725:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.856726:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.856727:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.856731:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.856733:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.856736:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.856736:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.856737:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.856739:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.856739:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.856740:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.856740:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.856751:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.856754:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.856755:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.856757:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.856758:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.856760:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.856769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.856775:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.856778:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.856780:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.856780:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.856783:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.856784:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677447.856786:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.856787:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.856788:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.856789:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677447.856793:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.856795:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.856797:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.856798:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677447.856798:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.856800:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.856800:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.856801:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.856802:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677447.856806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.856813:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.856816:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.856817:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.856817:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.856819:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.856821:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677447.856919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:5.2:1619677447.856921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:6.0:1619677447.856926:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677447.856927:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677447.856928:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.856929:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677447.856930:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.856931:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:6.0:1619677447.856932:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.856933:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.856935:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.856936:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.856936:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.856938:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.856938:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.856939:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.856940:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.856941:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.856941:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.856942:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.856943:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.856944:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.856949:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.856949:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.856951:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.856953:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.856954:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.856955:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.856955:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.856956:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.856956:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.856957:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.856958:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.856959:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.856959:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677447.856959:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677447.856960:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:6.0:1619677447.856960:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.856960:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677447.856961:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.856964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.2:1619677447.856968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:1.0:1619677447.856971:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.856974:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677447.856975:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:17.0:1619677447.856975:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.856975:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.856977:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.856978:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.856978:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.856979:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:20.0:1619677447.856980:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.856981:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.856983:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.857034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.857041:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.857044:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.857045:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.857047:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.857050:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.857052:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677447.857053:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.857054:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.857055:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.857057:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677447.857060:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.857062:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677447.857064:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.857065:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.857066:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677447.857067:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.857067:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.857068:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.857069:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677447.857080:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.857082:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.857083:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.857084:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.857086:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.857087:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.857141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.857148:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.857151:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.857153:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.857155:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.857156:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.857157:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677447.857158:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.857160:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.857161:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.857162:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677447.857165:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.857166:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677447.857169:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.857169:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677447.857170:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677447.857171:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.857172:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.857172:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.857173:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677447.857182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1400] (20)++
00000800:00000200:26.0:1619677447.857183:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.857185:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.857186:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.857186:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:17.0:1619677447.857187:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1400] (21)++
00000400:00000200:26.0:1619677447.857188:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.857190:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:17.0:1619677447.857190:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.16@o2ib3
00000800:00000200:1.0:1619677447.857191:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (22)--
00000400:00000200:17.0:1619677447.857193:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.1.16@o2ib3 : GET - for me
00000400:00000200:17.0:1619677447.857196:0:19930:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.1.16@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:17.0:1619677447.857198:0:19930:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.1.16@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:17.0:1619677447.857200:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1400] (21)++
00000800:00000200:17.0:1619677447.857201:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1400] (22)--
00000400:00000200:17.0:1619677447.857202:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.1.16@o2ib3 local destination
00000400:00000200:17.0:1619677447.857207:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.1.16@o2ib3(10.51.1.16@o2ib3:10.51.1.16@o2ib3) : REPLY try# 0
00000800:00000200:17.0:1619677447.857209:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.1.16@o2ib3
00000800:00000200:17.0:1619677447.857211:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918ce37f3d80] -> 10.51.1.16@o2ib3 (2) version: 12
00000800:00000200:17.0:1619677447.857212:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918cfdbb1400] (21)++
00000800:00000200:17.0:1619677447.857212:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdbb1400] (22)++
00000800:00000200:17.0:1619677447.857214:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918cfdbb1400] (23)--
00000800:00000200:17.0:1619677447.857214:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (22)--
00000800:00000200:21.2:1619677447.857219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1400] (21)++
00000800:00000200:1.0:1619677447.857224:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1400] (22)++
00000800:00000200:1.0:1619677447.857226:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdbb1400] (23)--
00000800:00000200:17.0:1619677447.857227:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (22)--
00000400:00000200:1.0:1619677447.857227:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.857229:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.1.16@o2ib3: REPLY: OK
00000800:00000200:1.0:1619677447.857230:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (21)--
00000800:00000200:20.2:1619677447.857236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.857243:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.857246:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.857249:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.857250:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.857251:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.857252:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.857254:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.857255:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677447.857255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:31.0:1619677447.857256:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.857257:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.857260:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.857262:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677447.857262:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677447.857264:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.857265:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:24.0:1619677447.857265:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.857265:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677447.857266:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.857267:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.857267:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.857268:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:24.0:1619677447.857268:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.857269:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:24.0:1619677447.857270:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.857272:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.857273:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96722591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.857274:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.857275:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.857276:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.857279:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.857280:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.857283:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.857284:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.857285:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.857286:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.857287:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.857287:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.857288:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:30.0:1619677447.857289:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.857292:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:8.2:1619677447.857292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677447.857298:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.857299:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677447.857299:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.857300:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:30.0:1619677447.857301:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:17.0:1619677447.857302:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.857303:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.857304:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.857306:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.857309:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.857366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.857372:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.857375:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.857377:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.857378:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.857379:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.857381:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.857383:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.857384:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.857385:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.857386:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.857389:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.857391:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.857394:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.857395:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.857396:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.857398:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.857398:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.857399:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.857399:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677447.857411:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.857414:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.857415:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.857416:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.857418:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.857419:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.857422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.857427:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.857430:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:4.2:1619677447.857430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.857430:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.857433:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.857435:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.857436:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:21.2:1619677447.857437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:10.0:1619677447.857438:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677447.857438:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.857439:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677447.857441:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.857441:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:1.0:1619677447.857441:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.857442:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:8.0:1619677447.857444:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:10.0:1619677447.857445:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.857446:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:1.0:1619677447.857446:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.857447:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:10.0:1619677447.857448:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.857449:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:10.0:1619677447.857450:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677447.857450:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.857451:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.857452:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.857452:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677447.857453:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.857453:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:10.0:1619677447.857454:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.857454:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:1.0:1619677447.857454:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.857455:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:10.0:1619677447.857456:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677447.857456:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96722587] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.857456:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.857456:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.857457:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677447.857458:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.857458:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.857460:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.857460:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:10.0:1619677447.857463:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:8.0:1619677447.857464:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.857465:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677447.857466:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.857466:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.857467:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.857469:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677447.857469:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.857469:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677447.857469:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.857469:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677447.857470:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677447.857470:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677447.857471:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.0:1619677447.857471:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:22.0:1619677447.857472:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:8.0:1619677447.857472:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:22.0:1619677447.857473:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.857473:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.857473:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677447.857473:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.857475:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.2:1619677447.857478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677447.857479:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.857481:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.857483:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.857483:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.857484:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.857485:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:22.0:1619677447.857486:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.857488:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.857488:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:21.2:1619677447.857489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:20.0:1619677447.857489:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.857490:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.857493:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.857493:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:6.0:1619677447.857497:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.857499:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.857501:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.857502:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677447.857502:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.857504:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.857505:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.857505:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.857505:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.857506:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.857507:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:6.0:1619677447.857507:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.857508:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.857510:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.857511:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.857512:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.857515:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.857517:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.857520:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.857521:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.857521:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.857523:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.857524:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.857525:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.857526:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.857526:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:21.2:1619677447.857532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677447.857535:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:22.0:1619677447.857537:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:26.0:1619677447.857538:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677447.857539:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.857539:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:10.0:1619677447.857539:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:22.0:1619677447.857540:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:26.0:1619677447.857541:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.857541:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:26.0:1619677447.857545:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677447.857698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.857704:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.857707:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.857708:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.857710:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.857713:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.857714:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.857715:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.857717:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.857718:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.857719:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.857722:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.857723:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.857725:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.857726:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.857727:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.857728:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.857729:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.857729:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.857730:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.857734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.857737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.857741:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.857743:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.857744:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.857744:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.857745:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.857746:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.857746:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.857747:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677447.857747:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:10.0:1619677447.857749:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.857752:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.857753:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677447.857754:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.857756:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.857757:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.857758:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677447.857761:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.857763:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.857766:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.857767:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677447.857768:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.857770:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.857770:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.857771:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.857771:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677447.857776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677447.857783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677447.857783:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.857786:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.857787:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.857787:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.857789:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.857791:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677447.857792:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677447.857793:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.857795:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677447.857795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:31.0:1619677447.857797:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.857797:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.857798:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.857799:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.857801:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.857802:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.857802:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.857802:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.857805:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.857805:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:30.0:1619677447.857806:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.857806:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.857807:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677447.857808:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.857808:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677447.857810:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.857811:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.857811:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.857813:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.857813:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.857815:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.857816:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.857817:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677447.857822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677447.857826:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.857829:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:31.0:1619677447.857830:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:13.0:1619677447.857830:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.857831:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.857832:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.857833:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.857834:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.857834:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.857835:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.857846:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.857849:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.857849:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.857850:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.857850:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.857852:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677447.857857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.857864:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.857866:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.857868:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.857870:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.857871:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.857872:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677447.857873:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.857874:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.857875:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.857876:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677447.857879:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.857881:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677447.857884:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.857885:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677447.857885:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677447.857887:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677447.857888:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.857888:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.857890:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.857891:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677447.857896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677447.857901:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.857903:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:30.0:1619677447.857904:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677447.857904:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.857905:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.857906:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.857907:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:6.0:1619677447.857907:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.857908:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.857910:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677447.858010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.858016:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.858019:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.858019:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.858021:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.858024:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.858025:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.858026:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.858028:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.858029:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.858030:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.858033:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.858034:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.858037:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.858038:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.858039:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.858041:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.858041:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.858042:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.858042:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.858046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.858053:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.858056:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.858057:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.858057:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.858059:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.858062:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.2:1619677447.858078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.858084:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.858086:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.858088:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.858089:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.858091:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.858093:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.858094:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.858095:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.858096:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.858097:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.858100:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.858102:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.858105:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.858105:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.858106:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.858108:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.858108:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.858110:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.858111:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.858111:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.858116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.858119:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.858121:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:22.0:1619677447.858122:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.858122:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.858123:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.858124:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.858125:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.858126:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.858127:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.858129:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.858151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.858158:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.858161:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.858161:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.858163:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.858166:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.858168:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677447.858169:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.858171:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.858171:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.858173:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677447.858176:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.858178:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677447.858180:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.858181:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677447.858182:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.858184:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.858184:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.858186:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.858187:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.858187:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677447.858190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.858193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.858197:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677447.858200:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:2.0:1619677447.858200:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.858201:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.858202:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.858202:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.858203:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677447.858203:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677447.858204:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:21.2:1619677447.858206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:2.0:1619677447.858207:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.0:1619677447.858213:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.858216:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.858217:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.858218:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.858221:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.858222:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.858224:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722578] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.858225:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.858226:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.858227:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.858230:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.858231:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.858234:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.858235:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.858236:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.858237:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.858238:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.858239:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.858239:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677447.858243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.858251:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.858254:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.858254:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.858256:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.2:1619677447.858258:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677447.858259:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.858260:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:10.0:1619677447.858266:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.858269:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.858269:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677447.858271:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.858274:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.858275:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677447.858277:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.858278:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.858279:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.858280:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677447.858282:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.858284:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677447.858286:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.858287:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677447.858288:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677447.858289:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.858290:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.858290:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.858290:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677447.858304:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:21.2:1619677447.858306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:13.0:1619677447.858306:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.858307:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.858308:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.858311:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.858313:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.858313:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.858315:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.858316:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.858318:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.858321:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.858322:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.858323:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.858325:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.858326:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.858327:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.858330:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.858332:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.2:1619677447.858333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:6.0:1619677447.858334:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.858335:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.858336:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.858338:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.858338:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.858339:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.858339:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:26.0:1619677447.858341:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.858344:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:8.2:1619677447.858344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677447.858345:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677447.858347:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.858349:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.858351:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677447.858352:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.858352:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:26.0:1619677447.858353:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.858354:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.858354:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.858355:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.858356:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677447.858357:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.858358:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.858359:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:26.0:1619677447.858360:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.858362:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677447.858365:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.858365:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677447.858366:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677447.858368:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677447.858369:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.858369:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677447.858370:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677447.858374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.858382:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.858385:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.858386:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.858386:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.858388:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.858391:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677447.858480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.858485:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.858487:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.858488:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.858490:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.858492:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.858494:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677447.858495:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.858497:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.858497:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.858499:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677447.858502:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.858504:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677447.858507:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.858508:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677447.858509:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677447.858510:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.858511:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.858512:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.858513:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.858513:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677447.858518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677447.858524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:1.0:1619677447.858524:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.858525:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.858526:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.858527:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677447.858531:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.858533:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.858534:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.858536:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677447.858537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:18.0:1619677447.858538:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.858539:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.858541:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722573] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.858542:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.858543:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:18.0:1619677447.858543:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.858544:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.858546:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:18.0:1619677447.858547:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.858547:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.858548:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.858550:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.858551:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.858552:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.858553:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:18.0:1619677447.858553:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.858553:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.858555:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.858555:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.858557:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:9.2:1619677447.858557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.2:1619677447.858558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677447.858558:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.858559:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677447.858562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677447.858563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:30.0:1619677447.858567:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.858569:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:2.0:1619677447.858569:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:17.0:1619677447.858570:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677447.858570:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000400:00000200:30.0:1619677447.858571:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.858571:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.858571:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677447.858572:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:2.0:1619677447.858572:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:17.0:1619677447.858573:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.858573:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:31.0:1619677447.858574:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.858574:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.858574:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.858574:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:1.0:1619677447.858574:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.858575:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000400:00000200:30.0:1619677447.858576:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:8.0:1619677447.858576:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:17.0:1619677447.858577:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.858578:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722572] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.858579:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.858580:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.858580:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:30.0:1619677447.858582:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.858585:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.858587:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.858589:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.858590:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.858590:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.858592:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.858592:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.858593:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.858594:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.858604:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.858606:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:4.2:1619677447.858606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:12.0:1619677447.858607:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.858608:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.858610:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.858612:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677447.858613:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.858616:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.858617:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.858618:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.858621:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.858622:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.858623:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.858625:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.858626:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.858627:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.858630:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.858631:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.858634:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.858635:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.858636:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.858638:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.858638:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.858639:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.858641:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.858641:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.858643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677447.858647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.858651:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.858653:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.858654:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.858654:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:1.0:1619677447.858655:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.858656:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.858657:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677447.858657:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.858658:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.858660:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.858666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.858673:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.858676:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.858676:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.858677:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.858678:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.858678:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677447.858679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.858687:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.858689:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.858690:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677447.858691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:10.0:1619677447.858692:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677447.858696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:10.0:1619677447.858696:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.858698:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:26.0:1619677447.858699:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:10.0:1619677447.858699:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.858700:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677447.858701:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.858702:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.858703:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677447.858703:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677447.858705:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.858706:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677447.858707:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.858708:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.858709:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.858710:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.858710:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677447.858710:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677447.858711:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.858713:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.858713:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.858713:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.858714:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677447.858714:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677447.858715:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.858717:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.858717:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677447.858717:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.858718:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:31.0:1619677447.858719:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677447.858719:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.858719:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677447.858721:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677447.858721:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:31.0:1619677447.858722:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.858722:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.858723:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677447.858724:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677447.858724:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677447.858725:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.858725:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.858726:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677447.858727:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.858727:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677447.858727:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677447.858728:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.858729:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677447.858729:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677447.858730:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677447.858730:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.858730:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677447.858731:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677447.858731:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677447.858732:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677447.858733:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:8.2:1619677447.858735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677447.858736:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.858737:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.858738:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.858740:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.858741:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:13.0:1619677447.858742:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.858743:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:13.0:1619677447.858744:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.858745:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.858746:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.858746:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.858747:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.858748:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677447.858748:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.858749:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:13.0:1619677447.858749:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677447.858752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.858754:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.858756:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.858758:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.858758:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.858759:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.858760:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.858761:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677447.859032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.859038:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.859041:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.859041:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.859043:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.859046:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.859047:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.859049:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.859050:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.859051:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.859052:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.859056:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.859057:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.859061:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.859061:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.859062:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677447.859063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:1.0:1619677447.859064:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.859064:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.859065:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.859065:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677447.859071:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.859073:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677447.859074:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:12.0:1619677447.859076:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677447.859076:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.859079:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:10.0:1619677447.859079:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.859080:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.859081:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:10.0:1619677447.859081:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:13.0:1619677447.859083:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677447.859083:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.859084:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.859085:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677447.859085:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.859086:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:21.2:1619677447.859087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:10.0:1619677447.859089:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.859091:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.859093:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677447.859094:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.859095:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.859096:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:10.0:1619677447.859096:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677447.859097:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.859097:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:10.0:1619677447.859098:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.859098:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:18.0:1619677447.859099:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.859099:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:18.0:1619677447.859101:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.859103:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:8.2:1619677447.859103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:18.0:1619677447.859104:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.859105:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.859106:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.859107:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:5.2:1619677447.859107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677447.859111:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:18.0:1619677447.859111:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.859113:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:18.0:1619677447.859113:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677447.859114:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:20.0:1619677447.859115:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.859116:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.859116:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677447.859117:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:20.0:1619677447.859117:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.859117:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (23)++
00000800:00000200:5.0:1619677447.859117:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:10.0:1619677447.859118:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677447.859119:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (23)--
00000800:00000200:18.0:1619677447.859119:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:26.0:1619677447.859120:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.859120:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.859120:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.859121:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:26.0:1619677447.859123:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.859124:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677447.859125:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677447.859125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:26.0:1619677447.859127:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.859128:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.859129:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677447.859132:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.859132:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677447.859134:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677447.859135:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677447.859136:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677447.859136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000400:00000200:5.0:1619677447.859136:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.859137:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677447.859138:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677447.859138:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677447.859139:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:5.0:1619677447.859139:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.859140:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.859140:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.859141:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.0:1619677447.859142:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:2.0:1619677447.859142:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677447.859145:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677447.859146:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677447.859148:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.859150:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.859151:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:2.0:1619677447.859152:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.859153:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.859154:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.859154:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.859154:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.859156:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.859157:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677447.859158:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.859158:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:2.0:1619677447.859161:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.859163:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:2.0:1619677447.859163:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.859166:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.859167:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.859167:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.859169:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.859170:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:2.0:1619677447.859170:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677447.859170:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677447.859171:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:30.0:1619677447.859173:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.859174:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.859175:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677447.859176:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.859177:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:10.0:1619677447.859181:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.859184:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677447.859184:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.859185:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.859187:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.859190:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677447.859263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.859265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.859268:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:26.0:1619677447.859270:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677447.859271:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:26.0:1619677447.859273:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:20.0:1619677447.859273:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.859273:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677447.859274:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:26.0:1619677447.859276:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.859276:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.859278:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.859278:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677447.859280:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.859280:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.859281:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.859282:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.859282:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.859284:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.859284:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677447.859285:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.859287:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.2:1619677447.859287:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:26.0:1619677447.859292:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.859292:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.859292:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.859299:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.859301:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.859302:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.859302:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.859304:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.859305:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677447.859305:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:26.0:1619677447.859306:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.859306:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:26.0:1619677447.859307:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.859307:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677447.859307:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96722560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.859308:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:5.0:1619677447.859308:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.859309:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:5.0:1619677447.859309:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.859310:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.859310:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677447.859310:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.859311:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.859311:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677447.859311:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.859312:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.859313:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000400:00000200:5.0:1619677447.859313:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.859314:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.859315:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677447.859315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.859316:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.859318:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677447.859319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.859319:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.859319:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.859321:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.859321:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.859322:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.859322:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.859322:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.859323:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677447.859324:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677447.859324:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:17.0:1619677447.859324:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677447.859325:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677447.859326:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:31.0:1619677447.859327:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.859328:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.859328:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.859328:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:31.0:1619677447.859331:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.859331:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:17.0:1619677447.859331:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677447.859331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.859335:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:22.0:1619677447.859335:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677447.859335:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.859336:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.859337:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.859338:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.859339:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.859339:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677447.859339:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.859340:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.859340:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000400:00000200:22.0:1619677447.859341:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.859341:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677447.859343:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.859343:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677447.859346:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.859347:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.859348:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.859349:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.859350:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.859513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.859521:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.859523:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.859524:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.859526:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.859528:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.859530:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677447.859531:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.859532:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.859533:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.859534:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677447.859537:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.859539:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.859540:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.859541:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677447.859542:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677447.859543:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.859544:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.859545:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677447.859546:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677447.859547:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677447.859552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.859557:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.859558:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.859559:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.859561:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.859561:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677447.859562:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:21.2:1619677447.859562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677447.859562:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677447.859563:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677447.859563:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.859564:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677447.859569:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.859572:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.859574:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.859576:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.859576:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.859577:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677447.859578:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.859580:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.859580:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.859581:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677447.859585:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.859586:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677447.859589:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.859590:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677447.859591:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677447.859592:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677447.859593:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.859594:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.859594:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.859605:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.859608:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.859609:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.859609:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.859612:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.2:1619677447.859614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:6.0:1619677447.859617:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.859618:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.859620:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.859620:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677447.859621:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.859621:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.859622:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.859622:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677447.859623:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.859624:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.859625:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.859628:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.859629:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.859631:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.859632:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.859633:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.859634:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.2:1619677447.859636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:1.0:1619677447.859637:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.859639:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.859642:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:1.0:1619677447.859642:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.859643:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.859643:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.859645:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:1.0:1619677447.859645:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.859646:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677447.859646:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.859646:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.859647:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:18.0:1619677447.859648:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.859651:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.859652:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.859654:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.859655:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.859656:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.859657:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677447.859659:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:18.0:1619677447.859660:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.859661:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677447.859662:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:13.0:1619677447.859662:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.859664:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.859665:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.859665:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.859666:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677447.859666:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.859667:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.859668:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.859668:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.859669:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.859669:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677447.859673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.2:1619677447.859674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677447.859676:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.859679:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.859680:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677447.859681:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677447.859681:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.859681:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.859682:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:12.0:1619677447.859683:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.859684:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:17.0:1619677447.859684:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.859685:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.859686:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.859686:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677447.859688:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:17.0:1619677447.859688:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.859689:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.859690:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:30.0:1619677447.859691:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.859692:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.859693:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.859694:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.859695:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.859699:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.859701:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.859703:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.859704:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.859704:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.859706:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.859707:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.859708:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.859708:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677447.859718:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.859720:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.859721:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.859723:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.859724:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.859726:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.859770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.2:1619677447.859774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:24.0:1619677447.859777:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.859780:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.859780:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677447.859781:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000400:00000200:24.0:1619677447.859782:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.859783:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677447.859784:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677447.859785:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.859785:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.859786:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677447.859787:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.859787:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.859789:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.859789:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.859790:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.859791:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.859792:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.859792:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677447.859793:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.859794:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.859795:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.859797:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:26.0:1619677447.859799:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.859800:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.859801:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.859801:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.859801:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.859803:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677447.859804:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.859804:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.859804:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677447.859805:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677447.859805:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677447.859805:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:26.0:1619677447.859807:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (23)--
00000800:00000200:26.0:1619677447.859807:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677447.859808:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677447.859808:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677447.859809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.859816:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (23)++
00000800:00000200:17.0:1619677447.859819:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (24)++
00000800:00000200:1.0:1619677447.859819:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (24)--
00000800:00000200:17.0:1619677447.859820:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (24)--
00000400:00000200:1.0:1619677447.859820:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.859821:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.859823:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.859823:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.859826:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.859827:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.859828:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677447.860232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.2:1619677447.860235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:10.0:1619677447.860241:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.860243:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.860244:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677447.860245:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000400:00000200:10.0:1619677447.860246:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.860248:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.860249:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.860250:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.860251:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677447.860252:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.860254:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.860255:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.860256:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677447.860259:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.860260:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:10.0:1619677447.860260:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:26.0:1619677447.860261:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.860263:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.860263:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.860264:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:10.0:1619677447.860266:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.860266:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677447.860267:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:26.0:1619677447.860268:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.860269:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.860269:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677447.860270:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677447.860271:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677447.860271:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.860272:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677447.860273:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:26.0:1619677447.860274:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.860275:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677447.860275:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677447.860278:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677447.860278:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677447.860279:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:26.0:1619677447.860280:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:26.0:1619677447.860281:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.860282:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.860283:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.860283:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:8.2:1619677447.860283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:26.0:1619677447.860284:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.860285:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:17.0:1619677447.860291:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.860294:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:10.0:1619677447.860300:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:17.0:1619677447.860301:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.860302:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.860303:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.860304:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:17.0:1619677447.860304:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.860304:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.860305:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.860305:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:17.0:1619677447.860307:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677447.860317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677447.860323:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.860326:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.860327:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.860329:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.860332:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.860333:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.860334:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.860336:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.860337:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.860338:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.860341:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.860343:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.860345:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.860346:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.860346:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.860348:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.860348:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.860350:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.860351:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.860351:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.2:1619677447.860357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.860362:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:22.0:1619677447.860363:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677447.860365:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.860365:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677447.860366:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:18.0:1619677447.860366:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:22.0:1619677447.860367:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.860368:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.860369:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.860372:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.860398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.860404:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.860407:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.860408:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.860410:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.860413:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.860414:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.860416:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.860417:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.860418:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.860419:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.860422:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.860424:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.860426:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.860427:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.860428:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.860430:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.860430:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.860431:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.860431:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.860443:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.860445:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.860446:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:20.2:1619677447.860446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:6.0:1619677447.860447:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.860448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:6.0:1619677447.860449:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.860451:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677447.860456:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677447.860456:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677447.860459:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:26.0:1619677447.860460:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.860461:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677447.860462:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.860464:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677447.860464:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.860465:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.860466:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.860467:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.860468:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.860469:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677447.860470:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.860470:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.860471:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.860472:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677447.860472:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.860473:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.860474:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677447.860476:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.860479:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:26.0:1619677447.860479:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.860481:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677447.860482:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.860483:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.860484:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677447.860484:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.860485:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.860485:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677447.860486:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.860486:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.860487:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.860487:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677447.860487:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.860487:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.860488:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.860489:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677447.860492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.860500:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.860501:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.860503:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677447.860504:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:20.0:1619677447.860504:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.860504:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.860505:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.860507:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.860507:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:22.0:1619677447.860508:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.860509:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.860510:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.860511:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.860512:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.860512:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.860513:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677447.860514:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.860516:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.860518:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.860519:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.860519:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.860520:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.860521:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677447.860565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677447.860570:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.860573:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.860574:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.860576:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.860578:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.860580:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.860581:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.860582:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.860584:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.860584:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.860588:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.860590:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.860593:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.860594:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.860595:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.860597:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.860597:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.860598:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.860599:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677447.860611:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.860614:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.860615:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.860617:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.860618:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.860619:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.860703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.860710:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.860713:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.860715:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.860716:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.860717:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.860719:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:30.0:1619677447.860720:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.860721:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.860723:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.860723:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677447.860727:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.860728:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677447.860731:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.860732:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:30.0:1619677447.860732:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677447.860734:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677447.860735:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.860736:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.860736:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677447.860740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.860748:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.860751:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.860752:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.860752:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.860754:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.860757:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.860760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.860767:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.860769:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.860770:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.860770:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.860771:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.860772:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677447.860873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.860879:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.860881:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.860882:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.2:1619677447.860883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:6.0:1619677447.860884:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.860886:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.860888:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.860889:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.860890:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:6.0:1619677447.860890:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.860891:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677447.860893:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:6.0:1619677447.860893:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.860894:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.860897:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.860897:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677447.860900:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.860900:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677447.860901:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:6.0:1619677447.860903:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677447.860904:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.860904:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:10.0:1619677447.860905:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.860905:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:10.0:1619677447.860906:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.860907:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:6.0:1619677447.860907:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.860907:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.860908:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.860909:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677447.860910:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.860912:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677447.860915:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.860915:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677447.860916:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677447.860917:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677447.860918:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.860919:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.860919:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677447.860920:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.860922:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.860923:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.860925:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.860926:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.860928:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:13.0:1619677447.860930:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.860932:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.860933:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.860934:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.860936:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.860937:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677447.861052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.861057:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.861060:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.861061:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.861063:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.861065:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.861067:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.861068:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.861069:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.861070:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.861071:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.861074:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.861076:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.861078:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.861079:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.861079:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.861081:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.2:1619677447.861082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677447.861082:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.861083:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.861084:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.861084:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677447.861085:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.861087:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:8.2:1619677447.861087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677447.861089:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677447.861090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:20.0:1619677447.861091:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.861093:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.861094:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.861095:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677447.861095:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:20.0:1619677447.861096:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.861096:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.861096:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:20.0:1619677447.861097:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.861097:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:1.0:1619677447.861097:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.861098:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:17.0:1619677447.861098:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.861099:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.861100:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:1.0:1619677447.861100:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.861101:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.861101:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677447.861104:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.861106:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.861107:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.861108:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.861110:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.861111:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.861112:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.861112:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.861117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.861124:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.861127:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.861128:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.861128:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.861130:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.861133:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.861140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677447.861148:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677447.861151:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677447.861153:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677447.861154:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.861157:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.861159:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:8.0:1619677447.861160:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.861162:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.861163:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.861164:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677447.861167:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.861169:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:8.0:1619677447.861171:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.861172:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:8.0:1619677447.861173:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:8.0:1619677447.861174:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:8.0:1619677447.861175:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677447.861176:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677447.861176:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:26.0:1619677447.861186:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677447.861188:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677447.861189:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.861189:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677447.861191:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.861193:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677447.861233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.861240:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.861243:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.861244:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.861246:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.861247:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.861248:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:30.0:1619677447.861249:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.861251:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.861251:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.861252:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677447.861255:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.861257:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677447.861260:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.861260:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:30.0:1619677447.861261:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677447.861263:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677447.861263:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.861264:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.861266:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.861266:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677447.861271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:21.2:1619677447.861272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677447.861279:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:31.0:1619677447.861280:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.861282:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:17.0:1619677447.861282:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.861282:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677447.861283:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677447.861283:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.861284:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:17.0:1619677447.861285:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.861288:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.861344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.861350:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.861352:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677447.861354:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.861355:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.861357:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.861358:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.861360:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.861361:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.861362:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.861363:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.861367:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.861368:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.861371:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.861372:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.861373:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.861374:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.861375:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.861376:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.861376:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:4.2:1619677447.861383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677447.861389:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.861391:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:24.0:1619677447.861391:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:31.0:1619677447.861392:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.861394:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.861394:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.861394:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677447.861396:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.861397:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:24.0:1619677447.861397:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.861399:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.861401:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.861402:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.861403:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.861404:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.861405:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.861409:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.861410:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.861413:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.861414:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.861415:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677447.861415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.0:1619677447.861416:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.861417:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.861417:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.861418:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.861422:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.2:1619677447.861423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.861425:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.861427:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.861427:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.861429:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:1.0:1619677447.861429:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.861431:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.861431:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.861432:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.861433:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.861434:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.861435:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.861436:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.861437:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.861440:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.861442:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.861444:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.861444:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.861445:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.861445:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.861446:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.861448:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.861449:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.861449:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.861450:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.861454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.861460:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:21.2:1619677447.861462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:17.0:1619677447.861462:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.861462:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.861463:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.861464:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.861465:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.861468:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.2:1619677447.861470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.861471:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.861472:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.861473:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.861476:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.861477:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677447.861478:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677447.861479:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.861480:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.861481:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:2.0:1619677447.861481:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.861482:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.861482:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.861482:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000400:00000200:2.0:1619677447.861483:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:1.0:1619677447.861483:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:2.0:1619677447.861486:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.861488:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677447.861491:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.861492:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677447.861492:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677447.861494:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677447.861494:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.861495:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.861496:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677447.861507:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677447.861510:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677447.861510:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677447.861511:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.861513:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:10.0:1619677447.861516:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:26.0:1619677447.861525:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677447.861527:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677447.861528:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677447.861528:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677447.861530:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677447.861530:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677447.861700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:5.2:1619677447.861704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677447.861706:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.861709:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:17.0:1619677447.861710:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677447.861710:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.861712:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.861712:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.861713:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677447.861715:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.861715:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.861716:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.861716:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.861717:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.861719:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.861719:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.861720:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.861720:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.861722:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.861722:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.861723:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.861724:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.861730:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.861731:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.861732:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.861734:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.861736:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.861737:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.861737:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.861738:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.861738:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.861739:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.861740:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.861741:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.861741:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677447.861741:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677447.861742:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677447.861742:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.861742:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.861743:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.861746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677447.861753:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:1.0:1619677447.861754:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:30.0:1619677447.861756:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.861757:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.861757:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.861758:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.861758:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.861760:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677447.861760:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.861761:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.861764:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:1.0:1619677447.861765:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677447.861785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.861792:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.861795:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.861797:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.861799:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.861800:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.861801:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677447.861803:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.861803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:31.0:1619677447.861804:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.861805:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.861806:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677447.861809:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.861810:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677447.861811:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.861812:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.861813:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677447.861815:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677447.861815:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.861816:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677447.861816:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:10.0:1619677447.861817:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.861818:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.861818:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.861819:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677447.861819:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:31.0:1619677447.861820:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:10.0:1619677447.861820:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.861821:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.861822:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.861823:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677447.861827:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.861829:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.861830:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.861832:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677447.861832:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677447.861833:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.861833:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.861834:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:26.0:1619677447.861835:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.861836:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.861837:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.861837:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.861838:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677447.861838:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677447.861840:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.861840:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.2:1619677447.861845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:4.2:1619677447.861847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.861850:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677447.861853:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677447.861853:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.861853:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.861854:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.861855:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.861856:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:6.0:1619677447.861856:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.861857:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.861857:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.861858:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.861859:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677447.861859:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677447.861860:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:26.0:1619677447.861862:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.861863:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677447.861865:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.861866:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.861867:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.861868:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677447.861871:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.861872:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677447.861875:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.861875:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677447.861876:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677447.861877:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677447.861878:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.861878:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.861879:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677447.861891:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.2:1619677447.861891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677447.861893:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.861894:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.861896:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:12.0:1619677447.861897:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.861898:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.861899:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677447.861900:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.861900:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.861902:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.861905:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.861907:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.861908:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.861908:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:18.0:1619677447.861910:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.861910:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.861912:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:13.0:1619677447.861912:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:20.2:1619677447.861913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:13.0:1619677447.861913:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.861913:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.861914:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:18.0:1619677447.861915:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.861915:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677447.861916:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.861917:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.861919:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:18.0:1619677447.861920:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.861921:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.861921:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.861921:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.861922:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677447.861923:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:20.0:1619677447.861924:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.861924:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.861924:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.861925:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:20.0:1619677447.861926:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.861927:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.861928:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677447.861929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:20.0:1619677447.861930:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.861931:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.861934:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.861936:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.861937:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.861938:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.861939:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.861940:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.861941:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:5.0:1619677447.861941:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.861942:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.861943:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.861944:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.861944:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.861945:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.861945:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677447.861945:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677447.861956:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.861959:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.861959:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.861961:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.861962:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.861964:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677447.861974:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677447.861977:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.861978:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.861979:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.861979:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.861981:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.862012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:20.2:1619677447.862015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.0:1619677447.862020:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.862023:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.862023:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:30.0:1619677447.862024:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:8.0:1619677447.862025:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.862026:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.862028:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.862028:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.862030:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:8.0:1619677447.862030:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.862031:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.862031:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.862032:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677447.862033:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.862034:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.862034:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.862035:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.862036:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.862037:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.862038:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677447.862039:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.862041:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.862042:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.862043:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677447.862044:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.862044:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.862045:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.862045:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.862046:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.862046:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:8.0:1619677447.862047:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.862047:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:30.0:1619677447.862048:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.862048:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:8.0:1619677447.862048:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:30.0:1619677447.862050:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:8.0:1619677447.862050:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.862050:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:30.0:1619677447.862052:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.862052:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:21.2:1619677447.862055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:20.2:1619677447.862058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677447.862058:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.862061:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.862061:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.862063:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:13.0:1619677447.862063:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.862064:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.862065:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:24.0:1619677447.862065:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.862065:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677447.862065:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.862065:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:31.0:1619677447.862066:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.862067:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.862067:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:24.0:1619677447.862067:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:30.0:1619677447.862068:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.862068:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.862069:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.862070:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677447.862071:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.2:1619677447.862100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.862106:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.862109:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.862109:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.862111:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.862114:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.862115:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.862117:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.862118:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.862119:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.862120:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.862124:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.862125:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.862128:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.862129:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.862130:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.862132:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.862133:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.862133:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.862134:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.862146:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.862149:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.862150:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.862150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:30.0:1619677447.862151:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.862152:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.862154:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.0:1619677447.862157:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.862160:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.862161:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.862163:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.862165:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.862167:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.862168:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.862170:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.862171:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.862172:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.862176:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.862177:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.862180:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.862181:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.862181:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.862183:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.862183:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.862184:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.862184:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.862189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.862197:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.862200:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.862201:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.862201:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.862203:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.862206:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.862280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.862287:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.862290:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.862291:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.2:1619677447.862297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:10.0:1619677447.862299:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.862302:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.862303:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677447.862304:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.862305:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677447.862306:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.862307:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.862308:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677447.862308:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677447.862311:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.862312:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677447.862313:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.862314:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.862315:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677447.862315:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:31.0:1619677447.862317:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.862318:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677447.862318:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.862319:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677447.862319:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:31.0:1619677447.862320:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677447.862320:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677447.862322:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677447.862322:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677447.862323:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.862323:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.862324:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677447.862325:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.862327:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.862328:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.862328:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.2:1619677447.862328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677447.862330:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.862330:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.862331:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.862332:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677447.862337:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.862340:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.862341:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.862342:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.862343:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677447.862344:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:12.0:1619677447.862346:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677447.862346:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:12.0:1619677447.862347:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.862349:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.862351:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.862352:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.862374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.862380:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.862383:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.862384:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.862386:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.862388:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.862389:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.862391:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.862392:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.862393:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.862394:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.862398:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.862399:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.862402:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.862403:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.862403:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.862405:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.862405:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.862407:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.862408:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.862408:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677447.862411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677447.862413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.862418:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:17.0:1619677447.862419:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677447.862420:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.862421:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677447.862422:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:6.0:1619677447.862422:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:17.0:1619677447.862423:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.862423:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.862425:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.862427:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.862509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.862514:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.862517:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.862518:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.862520:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.862522:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.862524:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:20.2:1619677447.862525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:1.0:1619677447.862525:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.862526:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.862527:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.862528:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.862532:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.862533:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:1.0:1619677447.862534:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.862535:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:1.0:1619677447.862537:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.862538:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677447.862539:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.862539:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.862539:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:30.0:1619677447.862541:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.862541:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.862541:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.862542:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:30.0:1619677447.862543:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:1.0:1619677447.862543:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000400:00000200:30.0:1619677447.862544:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.862544:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677447.862546:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.862547:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.862548:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.2:1619677447.862549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677447.862552:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.862553:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.862554:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677447.862554:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677447.862555:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.862556:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.862556:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:30.0:1619677447.862557:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.862557:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:17.0:1619677447.862557:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677447.862558:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.862558:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677447.862558:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.862560:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.862560:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.862561:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.862561:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.862562:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.862562:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.862566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.862573:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.862576:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.862577:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.862579:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.862580:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.862582:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677447.862648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.862654:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.862657:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.862658:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.862660:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.862662:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.862664:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.862665:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.862667:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.862668:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.862669:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.2:1619677447.862672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:2.0:1619677447.862672:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.862674:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.862677:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677447.862677:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.862678:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.862678:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.862680:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677447.862680:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.862680:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.862681:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.862681:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677447.862682:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.862682:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677447.862685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:5.0:1619677447.862685:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.862687:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.862688:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.862689:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.862690:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.862691:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.862693:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.862694:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:21.2:1619677447.862695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677447.862695:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.862697:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.862697:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.862699:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.862700:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.862700:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.862703:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677447.862703:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.862704:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.862704:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677447.862704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677447.862706:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.0:1619677447.862706:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.862707:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677447.862708:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.862708:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:30.0:1619677447.862709:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.0:1619677447.862709:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000400:00000200:31.0:1619677447.862710:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.862710:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677447.862711:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677447.862712:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.862713:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.862714:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.862714:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677447.862715:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677447.862715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677447.862716:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.862717:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.862718:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.862718:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677447.862718:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:30.0:1619677447.862720:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:26.0:1619677447.862720:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.862720:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:31.0:1619677447.862721:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.862721:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.862721:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:20.0:1619677447.862721:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677447.862722:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.862722:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:26.0:1619677447.862723:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96722515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.862724:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:26.0:1619677447.862724:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.862725:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:26.0:1619677447.862725:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.862726:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.862726:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:26.0:1619677447.862726:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677447.862727:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.862728:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.862729:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:26.0:1619677447.862729:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.862730:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.862731:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.862731:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677447.862731:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677447.862734:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.862735:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677447.862736:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677447.862737:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677447.862738:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.862738:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.862739:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:12.0:1619677447.862741:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.862743:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.2:1619677447.862743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:12.0:1619677447.862744:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677447.862746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:12.0:1619677447.862746:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.862747:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.862748:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.862751:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:24.0:1619677447.862753:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677447.862753:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.862754:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.862756:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:20.0:1619677447.862756:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.862757:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677447.862757:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:24.0:1619677447.862759:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.862759:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677447.862762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000400:00000200:24.0:1619677447.862763:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.862765:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.862766:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.862768:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.862769:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.862770:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.862770:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.862772:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.862773:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.862773:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.862774:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:24.0:1619677447.862775:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.862775:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.862776:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:24.0:1619677447.862778:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.862779:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.862780:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.862782:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.862782:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.862783:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.862784:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.862788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.862796:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.862798:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.862800:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.862800:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.862802:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.862805:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677447.862856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.862861:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.862864:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.862865:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.862867:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.862869:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.862871:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.862872:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.862873:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.862874:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.862875:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.862879:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.862880:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.862883:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.862884:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.862884:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.862886:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.862886:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.862887:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.862887:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.862892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:4.2:1619677447.862895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:5.0:1619677447.862900:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:8.0:1619677447.862902:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.0:1619677447.862902:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.862903:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.862904:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.862905:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677447.862905:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.862905:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.862907:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.862907:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677447.862911:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000400:00000200:8.0:1619677447.862915:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.862916:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.0:1619677447.862918:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:8.0:1619677447.862918:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.862919:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.862920:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.862921:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.862922:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.862922:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000400:00000200:8.0:1619677447.862922:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.862923:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.862924:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:8.0:1619677447.862925:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.862927:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.862929:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.862930:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.862931:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.862933:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.862933:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.862934:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.862935:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.862936:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.862938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.862941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.862946:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.862949:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677447.862949:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.862949:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677447.862950:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677447.862950:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.862951:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.862952:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.862952:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.862955:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.862957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.862965:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.862968:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.862969:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.862969:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.862970:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.862971:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.863111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.863119:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.863121:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.863122:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.863124:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.863127:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.863129:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.863130:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.863131:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.863132:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.863133:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.2:1619677447.863136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:10.0:1619677447.863137:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.863139:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677447.863142:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.863143:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.863144:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.863144:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677447.863145:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.863147:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.863147:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:26.0:1619677447.863148:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.863149:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.863149:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677447.863150:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.863151:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.863152:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677447.863153:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.863155:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.863156:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.863156:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677447.863160:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.863160:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677447.863161:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677447.863163:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.863163:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677447.863164:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677447.863164:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:22.0:1619677447.863164:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.863164:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.863166:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677447.863166:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:6.0:1619677447.863167:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.863168:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677447.863169:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677447.863169:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677447.863169:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.863175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677447.863178:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.863181:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.863182:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.863182:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:30.0:1619677447.863184:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.863184:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:31.0:1619677447.863185:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677447.863185:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677447.863186:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:10.0:1619677447.863186:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677447.863198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.863205:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.863208:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.863210:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.863212:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.863212:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.863214:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.863215:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.863216:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.863218:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.863218:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.863221:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.863223:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.863225:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.863226:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.863226:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.863228:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.863228:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.863229:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.863230:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.863239:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.863242:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.863243:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.863244:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.863246:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.863247:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.863310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.2:1619677447.863312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677447.863313:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.863316:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.863318:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.863320:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677447.863320:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.863321:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.863322:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:30.0:1619677447.863323:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:20.0:1619677447.863324:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.863326:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.863326:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.863327:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677447.863327:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.863328:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677447.863329:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.863332:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.863333:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:30.0:1619677447.863334:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.863335:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.863337:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677447.863338:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.863338:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:30.0:1619677447.863339:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.863339:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:30.0:1619677447.863340:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.863341:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677447.863341:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.863342:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.863342:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:30.0:1619677447.863343:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.863344:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.863346:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.863347:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.863348:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677447.863348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:30.0:1619677447.863349:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.863350:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.863351:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.863353:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.863354:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.863354:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677447.863355:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:31.0:1619677447.863357:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.863358:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.863358:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:21.2:1619677447.863359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677447.863359:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:31.0:1619677447.863360:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.863360:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.863361:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:26.0:1619677447.863361:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:30.0:1619677447.863362:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:12.0:1619677447.863362:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677447.863363:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.863363:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677447.863363:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.863364:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.863364:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:26.0:1619677447.863365:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:12.0:1619677447.863365:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.863366:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677447.863367:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:26.0:1619677447.863368:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.863369:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.863369:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:12.0:1619677447.863369:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.863370:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677447.863373:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677447.863374:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677447.863377:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.863378:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677447.863378:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:12.0:1619677447.863379:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677447.863380:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677447.863380:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.863381:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:12.0:1619677447.863381:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.863382:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677447.863383:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677447.863383:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:12.0:1619677447.863383:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.863384:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.863385:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677447.863387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677447.863389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.863396:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:1.0:1619677447.863396:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:10.0:1619677447.863398:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677447.863399:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:1.0:1619677447.863399:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.863400:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677447.863400:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:1.0:1619677447.863400:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.863402:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:1.0:1619677447.863405:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677447.863469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677447.863477:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.863480:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.863481:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.863482:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.863485:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.863487:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677447.863488:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.863489:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.863490:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.863491:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677447.863494:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.863496:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.863498:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.863499:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.863499:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.863501:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.863501:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:5.2:1619677447.863501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.0:1619677447.863502:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.863502:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:17.0:1619677447.863507:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677447.863509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:17.0:1619677447.863510:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.863510:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.863512:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.863513:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:17.0:1619677447.863515:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.863515:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:13.0:1619677447.863516:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.863516:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.863517:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:2.0:1619677447.863518:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.863519:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677447.863519:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.863519:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.863520:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.863520:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:17.0:1619677447.863521:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.863521:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:2.0:1619677447.863522:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.863523:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677447.863524:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.863525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677447.863526:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.863527:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.863528:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.863529:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:2.0:1619677447.863530:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.863531:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.863532:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677447.863532:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677447.863533:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677447.863533:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:21.2:1619677447.863534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:17.0:1619677447.863534:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677447.863536:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:17.0:1619677447.863536:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.863536:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677447.863536:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677447.863536:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.863537:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.863537:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677447.863538:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:26.0:1619677447.863539:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.863539:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677447.863541:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677447.863542:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.863542:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:2.0:1619677447.863542:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:6.0:1619677447.863543:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677447.863543:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:26.0:1619677447.863544:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:2.0:1619677447.863544:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.863545:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:26.0:1619677447.863546:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96722502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.863546:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.863546:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.863547:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000400:00000200:26.0:1619677447.863548:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.863548:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.863548:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677447.863548:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:26.0:1619677447.863549:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.863550:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677447.863551:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677447.863551:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96722501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.863551:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.863552:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.863553:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677447.863553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000400:00000200:12.0:1619677447.863554:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.863555:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.863555:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:26.0:1619677447.863556:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.863556:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:13.0:1619677447.863557:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677447.863558:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677447.863558:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677447.863559:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:13.0:1619677447.863559:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.863559:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.863560:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:22.0:1619677447.863561:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.863561:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:13.0:1619677447.863561:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.863561:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677447.863562:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.863562:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:26.0:1619677447.863563:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677447.863563:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.863563:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.863563:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.863563:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.863564:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.863564:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.863564:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.863565:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677447.863566:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.863566:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.863566:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.863566:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677447.863567:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677447.863567:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.863567:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677447.863569:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.863569:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677447.863569:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.863570:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.863570:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.863571:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.863572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.863574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677447.863576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.863579:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677447.863581:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:2.0:1619677447.863582:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.863583:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.863583:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.863584:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677447.863584:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:2.0:1619677447.863584:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.863585:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677447.863586:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:2.0:1619677447.863586:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.863587:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.863589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677447.863589:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.863590:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.863592:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677447.863592:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.863593:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.863596:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.863597:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.863597:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.863598:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.863598:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.863606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.863614:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.863616:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.863617:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.863619:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.863622:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.863623:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.863624:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.863626:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.863627:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.863628:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.863631:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.863633:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.863636:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.863637:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.863638:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.863639:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.863640:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.863641:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.863641:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.863645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.863653:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.863656:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.863657:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.863657:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.863659:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.863662:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677447.863710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.863717:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.863720:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.863721:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.863723:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.863726:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.863728:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677447.863729:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.863731:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.863731:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.863732:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677447.863736:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.863737:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677447.863739:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.863740:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677447.863741:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677447.863743:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677447.863743:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.863744:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.863744:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677447.863755:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.863757:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.863758:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.863759:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.863761:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.863762:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.864039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37e800] (20)++
00000800:00000200:10.0:1619677447.864046:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37e800] (21)++
00000800:00000200:26.0:1619677447.864049:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)--
00000800:00000200:10.0:1619677447.864049:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.56@o2ib3
00000400:00000200:10.0:1619677447.864051:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.56@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.864053:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.864055:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677447.864056:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.864057:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.864058:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.864059:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677447.864062:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.864063:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677447.864066:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.864067:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:10.0:1619677447.864068:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:10.0:1619677447.864069:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:10.0:1619677447.864070:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37e800] (21)++
00000800:00000200:10.0:1619677447.864070:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37e800] (22)--
00000800:00000200:10.0:1619677447.864071:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (21)--
00000800:00000200:8.2:1619677447.864091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.864099:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.864102:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:4.2:1619677447.864103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:20.0:1619677447.864104:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.864105:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:20.0:1619677447.864106:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.864109:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:26.0:1619677447.864110:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.864113:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.864114:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.864115:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.864118:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.864120:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.864121:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722497] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677447.864121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677447.864122:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.864123:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.864124:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.864127:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.864127:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677447.864128:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.864130:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.864130:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.864130:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677447.864131:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.864132:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.864133:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:17.0:1619677447.864133:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.864134:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.864134:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.864135:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:17.0:1619677447.864135:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.864136:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:5.2:1619677447.864136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:17.0:1619677447.864138:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.864139:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.864140:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.864141:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:20.2:1619677447.864143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677447.864143:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677447.864145:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.864146:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.864146:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:17.0:1619677447.864147:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677447.864148:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:10.0:1619677447.864148:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.864149:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.864150:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.864150:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.864150:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000400:00000200:10.0:1619677447.864150:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.864151:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677447.864151:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:31.0:1619677447.864152:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:10.0:1619677447.864152:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:30.0:1619677447.864153:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:17.0:1619677447.864153:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000400:00000200:10.0:1619677447.864153:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96722495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.864154:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.864154:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677447.864154:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.864155:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:10.0:1619677447.864155:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.864156:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.864157:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677447.864158:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.864159:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:9.2:1619677447.864159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677447.864160:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:30.0:1619677447.864160:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677447.864160:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.864161:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677447.864161:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96722494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.864162:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:10.0:1619677447.864162:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677447.864163:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.864163:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677447.864163:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.864163:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.864164:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.864165:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.864165:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677447.864165:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.864165:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:10.0:1619677447.864166:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677447.864166:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:30.0:1619677447.864167:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.864168:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.864168:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:30.0:1619677447.864169:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.864169:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:10.0:1619677447.864169:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.864169:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:8.0:1619677447.864169:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.864170:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677447.864171:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677447.864171:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.864172:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.864173:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.864174:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.2:1619677447.864174:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677447.864175:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.864176:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.864176:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.0:1619677447.864179:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:1.0:1619677447.864182:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.864185:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.864186:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.864186:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.864187:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:1.0:1619677447.864188:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.864189:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.864190:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.864191:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:13.0:1619677447.864192:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.864193:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.864194:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.2:1619677447.864194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.864202:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.864205:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.864205:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.864206:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.864207:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.864207:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677447.864321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:4.2:1619677447.864329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.864332:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.864335:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.864337:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:24.0:1619677447.864338:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.864338:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677447.864340:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.864340:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.864342:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.864344:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.864345:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.864346:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.864347:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.864349:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.864351:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.864351:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.864352:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96722492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.864353:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.864354:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.864354:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.864355:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.864359:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677447.864359:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.864360:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.864361:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.864361:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.864363:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.864364:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.864364:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.864365:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.864365:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.864366:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.864366:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.864367:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677447.864368:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.864369:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.2:1619677447.864374:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (22)++
00000800:00000200:8.0:1619677447.864377:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (23)--
00000800:00000200:8.0:1619677447.864377:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.864378:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677447.864378:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.0:1619677447.864379:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.0:1619677447.864381:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:1.0:1619677447.864381:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:5.0:1619677447.864382:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.864383:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.864384:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677447.864384:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.864385:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.864385:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.864386:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:1.0:1619677447.864387:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.864390:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677447.864646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.2:1619677447.864650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677447.864655:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677447.864655:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677447.864658:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:4.2:1619677447.864658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.864659:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.864659:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677447.864662:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.864662:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677447.864663:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.864665:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.864665:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677447.864666:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.864666:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.864668:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.864668:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:17.0:1619677447.864668:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.864669:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.864669:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677447.864670:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.864671:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.864672:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.864673:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.864673:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.864674:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677447.864675:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677447.864675:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.864676:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677447.864678:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96722489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.864680:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.864681:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.864681:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.864682:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.864682:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:17.0:1619677447.864684:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677447.864686:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:26.0:1619677447.864686:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.864687:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.864688:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.864688:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677447.864688:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:31.0:1619677447.864689:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.864689:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:31.0:1619677447.864690:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677447.864690:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.864691:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.864691:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:17.0:1619677447.864691:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:31.0:1619677447.864692:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.864692:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677447.864693:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677447.864693:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:17.0:1619677447.864693:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677447.864694:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677447.864694:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677447.864694:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.864695:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677447.864695:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.864695:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677447.864699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.864706:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:12.0:1619677447.864706:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677447.864708:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.864709:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.864709:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:10.0:1619677447.864709:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:8.0:1619677447.864709:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:12.0:1619677447.864710:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.864711:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:24.0:1619677447.864712:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:12.0:1619677447.864712:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:10.0:1619677447.864712:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.864713:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:13.0:1619677447.864713:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.864714:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677447.864714:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.864715:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:24.0:1619677447.864716:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.864717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:8.0:1619677447.864723:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.864726:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.864727:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.864727:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.864728:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.864728:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677447.864925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677447.864931:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.864934:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.864935:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.864937:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.864940:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.864941:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.864943:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.864944:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.864945:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.864946:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.864949:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.864951:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.864954:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.864955:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.864955:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.864957:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.864958:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.864959:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.864959:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677447.864971:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.864974:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.864975:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.864975:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.864977:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.864980:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.864989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.864991:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.864994:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.864996:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.864997:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.864999:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.865000:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.865001:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.865004:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.865005:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.865006:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.865009:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.865011:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.865013:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.865014:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.865015:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.865016:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.865017:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.865017:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.865018:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.865029:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.865031:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.865032:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.865033:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.865034:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.865037:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.865047:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.865049:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:5.2:1619677447.865049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677447.865050:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.865051:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.865052:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.865052:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:26.0:1619677447.865057:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.865060:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.865061:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.865062:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.865064:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.865066:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677447.865067:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.865068:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.865069:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.865070:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677447.865073:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.865075:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.865076:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.865077:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.865078:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.865079:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.865079:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.865080:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.865080:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677447.865090:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.865091:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.865092:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.865093:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.865094:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.865095:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677447.865098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:26.0:1619677447.865105:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677447.865107:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:10.0:1619677447.865108:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:26.0:1619677447.865109:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.865111:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.865113:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677447.865114:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.865115:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.865116:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.865116:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677447.865120:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677447.865120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:26.0:1619677447.865121:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677447.865124:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.865125:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677447.865125:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.865125:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:26.0:1619677447.865127:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677447.865127:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677447.865128:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:26.0:1619677447.865128:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:5.0:1619677447.865128:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.865130:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.865130:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:8.2:1619677447.865133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:5.0:1619677447.865133:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.865134:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.865136:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.865137:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.865138:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677447.865140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:5.0:1619677447.865140:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.865140:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.865143:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.865144:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:5.0:1619677447.865144:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000400:00000200:1.0:1619677447.865144:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.865146:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677447.865146:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:1.0:1619677447.865147:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.865148:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:5.0:1619677447.865149:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.865150:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677447.865150:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.865150:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.0:1619677447.865151:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:2.0:1619677447.865151:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.865153:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.865153:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.865154:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000400:00000200:2.0:1619677447.865154:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.865155:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:2.0:1619677447.865156:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.865157:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.865158:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.865159:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.865160:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.865164:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.865165:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:2.0:1619677447.865166:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677447.865168:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:2.0:1619677447.865168:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.865168:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:30.0:1619677447.865169:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.865169:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677447.865171:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.865171:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.865171:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:31.0:1619677447.865172:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:2.0:1619677447.865172:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.865172:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:30.0:1619677447.865174:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:9.2:1619677447.865187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.865195:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.865197:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.865197:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.865198:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.865199:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.865201:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.865242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677447.865249:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677447.865252:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677447.865254:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.865256:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677447.865257:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.865258:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.865260:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.865261:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.865262:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.865263:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.865267:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.865268:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.865271:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.865272:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.865272:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.865274:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.865274:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677447.865276:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87b3e5800] (22)++
00000800:00000200:31.0:1619677447.865277:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:31.0:1619677447.865278:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:20.2:1619677447.865284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:13.0:1619677447.865286:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.865289:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.865289:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.865291:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.865292:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)++
00000800:00000200:12.0:1619677447.865292:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.865293:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677447.865301:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:5.2:1619677447.865301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677447.865302:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677447.865303:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:17.0:1619677447.865308:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.865310:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.865311:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.865313:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.865316:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.865317:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.865318:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.865320:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.865321:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.865322:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.865325:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.865327:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.865330:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.865330:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.865331:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.865333:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.865333:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.865335:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.865336:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.865336:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.865339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.2:1619677447.865341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.865346:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677447.865347:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:1.0:1619677447.865348:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.865349:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677447.865350:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677447.865350:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677447.865351:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.865351:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.865353:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.865355:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.865359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.865367:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.865370:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.865371:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.865372:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.865375:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.865377:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.865378:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.865379:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.865380:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.865381:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677447.865385:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.865386:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677447.865389:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.865390:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.865391:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.865392:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.865393:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.865393:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.865394:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677447.865407:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.865410:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.865410:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.865411:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.865413:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.865416:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.865471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.865478:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.865481:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.865483:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.865485:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.865485:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.865487:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.865488:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.865490:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.865491:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.865492:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.865495:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.865497:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.865500:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.865501:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.865502:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.865503:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.865504:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.865506:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.865507:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.865508:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677447.865510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677447.865514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.865522:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677447.865522:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677447.865524:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:20.0:1619677447.865524:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677447.865525:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.865526:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677447.865526:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.865527:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.865528:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.865531:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677447.865670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.865676:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.865679:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.865680:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.865681:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677447.865682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:2.0:1619677447.865683:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.865685:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.865686:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.865688:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.865689:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:2.0:1619677447.865689:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.865690:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677447.865691:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:2.0:1619677447.865695:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.865696:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.865696:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.865697:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:30.0:1619677447.865699:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.865700:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.865700:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:30.0:1619677447.865701:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:2.0:1619677447.865701:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677447.865702:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.865703:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677447.865703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677447.865703:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.865703:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:30.0:1619677447.865704:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.865704:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.865704:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:30.0:1619677447.865705:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677447.865709:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677447.865709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677447.865711:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677447.865713:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.865714:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.865714:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677447.865715:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677447.865716:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.865716:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:30.0:1619677447.865717:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.865718:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677447.865718:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.865718:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:30.0:1619677447.865719:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677447.865720:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.865721:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.865722:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.865722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:17.0:1619677447.865723:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.865723:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.865725:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96722476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.865726:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.865727:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.865728:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.865732:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.865734:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.865735:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.865736:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.865736:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.865737:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.865738:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.865738:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.865740:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.865740:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.865741:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.865741:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.865741:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.865741:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.865742:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677447.865743:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.865743:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.2:1619677447.865746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.865749:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.865750:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.865752:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.865752:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.865754:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.865754:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.865755:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677447.865814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677447.865822:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.865824:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.865825:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.865827:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.865830:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.865832:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.865833:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.865834:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.865835:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.865836:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.865840:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.865841:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.865845:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.865845:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.865846:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.865848:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.865849:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.865850:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.865851:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.865852:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.2:1619677447.865854:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677447.865858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677447.865863:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:24.0:1619677447.865865:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:5.0:1619677447.865865:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.865866:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.865867:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.865868:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:5.0:1619677447.865868:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.865869:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.865870:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677447.865871:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677447.865961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.865968:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.865971:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.865972:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.865974:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.865977:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.865978:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677447.865979:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.865980:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.865981:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.865982:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677447.865985:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.865986:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.865987:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.865988:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.865988:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.865990:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.865990:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.865992:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677447.865993:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677447.865993:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677447.865999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.866004:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.866005:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.866005:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.866007:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:10.0:1619677447.866007:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.866008:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.866009:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677447.866009:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.866010:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677447.866010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:10.0:1619677447.866010:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:6.0:1619677447.866015:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.866018:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.866019:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:10.0:1619677447.866021:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000400:00000200:6.0:1619677447.866021:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.866023:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000400:00000200:6.0:1619677447.866023:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.866024:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.866024:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.866025:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677447.866025:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:6.0:1619677447.866025:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.866026:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.866027:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.866029:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.866029:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.866033:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.866034:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.866037:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.866038:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.866038:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.866040:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.866040:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.866041:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.866042:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.866046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.866054:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.866057:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.866058:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.866058:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.866060:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.866062:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677447.866254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.866259:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.866262:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.866263:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.866265:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.866266:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.866268:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.866269:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.866270:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.866272:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.866273:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.866276:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.866278:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.866281:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.866282:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.866283:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.866285:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.866285:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.866286:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.866286:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.866299:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.866302:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.866303:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.866305:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.866306:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.866308:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.866373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.866381:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.866384:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.866384:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.866386:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.866388:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.866389:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.866391:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.866392:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.866393:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.866394:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677447.866397:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.866399:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.866402:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.866402:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.866403:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677447.866405:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.866405:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.866406:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.866407:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.866408:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677447.866413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.866420:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677447.866420:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677447.866423:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677447.866423:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677447.866424:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.866424:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677447.866424:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677447.866425:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677447.866427:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.866429:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.866475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.866483:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.866486:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.866489:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.866491:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.866491:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.866493:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677447.866494:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.866495:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.866496:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.866497:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677447.866500:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.866502:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677447.866505:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.866506:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677447.866507:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677447.866508:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677447.866509:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.866510:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.866510:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.866515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.866523:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.866526:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.866527:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.866528:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.866531:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.866531:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:21.2:1619677447.866595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.866602:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.866605:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.866607:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.866608:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.866610:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.866611:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677447.866613:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.866614:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.866615:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.866616:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677447.866619:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.866620:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.866623:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.866624:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677447.866625:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677447.866626:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.866627:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.866628:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.866628:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.866640:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.866643:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.866644:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.866644:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.866646:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.866649:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.866681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.866687:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.866690:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.866690:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.866692:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.866695:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.866696:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.866698:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.866699:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.866700:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.866701:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.866704:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.866705:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.866707:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.866708:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.866708:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.866710:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.866710:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.866711:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.866711:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.866716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.866723:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.866725:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.866725:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.866726:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.866727:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.2:1619677447.866729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677447.866729:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677447.866735:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.866738:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.866740:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.866740:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.866743:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.866745:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.866746:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.866747:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.866748:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.866749:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.866752:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.866754:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.866757:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.866758:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.866758:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.866760:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.866761:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.866762:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.866762:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.866773:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.866775:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.866776:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.866778:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.866779:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.866781:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.866794:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.866796:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.866797:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.866798:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.866798:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.866800:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.866854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.866860:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.866863:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.866863:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.866865:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.866868:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.866869:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.866870:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.866872:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.866873:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.866874:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.866877:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.866878:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.866881:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.866881:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.866882:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.866883:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.866884:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.866885:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.866886:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.866886:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677447.866889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677447.866892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:4.2:1619677447.866892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:1.0:1619677447.866897:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677447.866898:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:8.0:1619677447.866899:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.866899:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.866900:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.866900:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.866900:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677447.866900:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.866902:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:6.0:1619677447.866902:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.866903:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677447.866903:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.866906:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.866906:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.866908:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000400:00000200:8.0:1619677447.866913:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.866915:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:17.0:1619677447.866916:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:8.0:1619677447.866916:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.866917:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.866918:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.866919:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.866919:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.866919:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.866920:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.866921:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.866921:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:8.0:1619677447.866923:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.866925:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.866927:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.866928:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.866929:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.866930:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.866931:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.866932:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.866932:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.866936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.866944:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.866946:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.866947:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.866947:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.866950:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.866952:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.867007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.867013:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.867016:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.867017:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.867020:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.867022:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.867023:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.867024:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.867026:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.867027:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.867028:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.867030:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.867032:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.867034:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.867034:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.867035:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.867037:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.867037:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.867038:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.867038:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.867043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.867050:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.867052:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.867053:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.867054:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.867055:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.867056:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.867065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.867073:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.867076:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.867077:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.867079:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.867082:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.867083:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677447.867084:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.867086:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.867087:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.867088:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677447.867091:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.867093:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677447.867096:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.867097:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.867097:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677447.867099:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.867099:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.867100:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.867100:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.2:1619677447.867109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.867113:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.867115:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.867115:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.867117:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.867117:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.867118:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677447.867120:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.867121:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:30.0:1619677447.867121:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.867122:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.867123:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.867125:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.867126:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.867127:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.867128:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.867129:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.867132:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.867133:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.867136:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.867137:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.867138:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.867140:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.867140:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.867141:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.867141:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.867146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.867154:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.867156:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.867157:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.867159:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.867160:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.867162:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677447.867325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.867331:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677447.867333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:17.0:1619677447.867333:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.867334:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.867336:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.867338:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.867338:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.2:1619677447.867339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:17.0:1619677447.867340:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.867341:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.867341:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.867342:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:17.0:1619677447.867343:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.867344:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:17.0:1619677447.867345:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.867346:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677447.867346:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.867347:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:2.0:1619677447.867348:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.867350:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:2.0:1619677447.867350:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677447.867351:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.867351:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.867352:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.867353:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.867354:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.867354:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.867354:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.867356:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677447.867356:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:21.2:1619677447.867357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677447.867357:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.867358:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:5.0:1619677447.867358:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96722459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.867359:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:2.0:1619677447.867360:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.867361:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.867361:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677447.867361:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.867362:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.867363:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000400:00000200:5.0:1619677447.867363:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.867363:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677447.867364:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677447.867364:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:2.0:1619677447.867365:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.867365:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:31.0:1619677447.867366:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677447.867366:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.867368:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.867368:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:31.0:1619677447.867369:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.2:1619677447.867369:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677447.867369:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.867370:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000400:00000200:5.0:1619677447.867371:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.867371:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:30.0:1619677447.867373:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677447.867374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.867374:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:31.0:1619677447.867375:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.867375:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677447.867375:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.867376:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:21.2:1619677447.867376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:1.0:1619677447.867377:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:31.0:1619677447.867378:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.867378:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.867378:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.867378:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677447.867379:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.867380:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677447.867380:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.867380:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:31.0:1619677447.867381:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96722458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.867381:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:18.0:1619677447.867382:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:31.0:1619677447.867383:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.867383:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:5.0:1619677447.867383:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (24)--
00000400:00000200:31.0:1619677447.867384:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.867384:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.867384:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677447.867384:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:31.0:1619677447.867385:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.867385:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.867385:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677447.867385:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.867386:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.867387:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677447.867388:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:18.0:1619677447.867388:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.867389:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.867389:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:2.0:1619677447.867389:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677447.867390:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677447.867391:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.867392:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:30.0:1619677447.867392:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.867393:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.867394:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.867395:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677447.867395:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.867396:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677447.867396:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677447.867397:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.867398:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.867399:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.867401:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.867402:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677447.867409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677447.867410:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.867412:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.867413:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.867413:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.867414:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.867415:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.867415:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.867416:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.867417:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.867417:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.867418:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.867418:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.867418:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677447.867419:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.867420:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677447.867421:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677447.867469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677447.867477:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.867479:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677447.867480:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677447.867482:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.867484:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.867486:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677447.867487:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.867488:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.867489:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.867490:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677447.867494:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.867495:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677447.867498:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.867499:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677447.867499:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677447.867501:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677447.867501:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.867502:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677447.867503:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677447.867508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.867515:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.867518:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.867519:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.867519:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.867522:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.867524:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.867529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.867536:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.867539:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.867540:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.867540:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.867541:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.867541:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.867631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.867638:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.867641:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.867642:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.867643:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.867647:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.867648:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.867649:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.867650:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.867651:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.867652:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677447.867655:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.867657:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.867659:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.867660:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.867660:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677447.867662:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.867663:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.867664:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.867664:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677447.867676:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.867679:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.867680:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.867680:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.867683:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.867685:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.867694:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.867697:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.867697:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.867698:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.867699:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.867700:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.867703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.867711:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.867714:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.867716:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.867717:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.867718:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.867719:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.867721:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.867722:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.867723:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.867724:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.867727:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.867729:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.867732:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.867733:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.867733:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.867735:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.867736:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.867737:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.867737:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.867743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.867750:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.867753:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.867754:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.867755:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.867756:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:21.2:1619677447.867758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:20.0:1619677447.867758:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677447.867763:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.2:1619677447.867763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.867766:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.867767:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.867769:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.867771:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.867771:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:22.0:1619677447.867773:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.867774:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.867774:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.867775:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000400:00000200:22.0:1619677447.867776:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.867776:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.867776:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:22.0:1619677447.867777:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.867777:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:22.0:1619677447.867778:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.867781:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.867783:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.867786:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.867787:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.867788:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.867789:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.867790:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.867790:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.867791:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677447.867802:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677447.867804:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677447.867805:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677447.867806:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.867808:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.867811:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677447.867955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.867960:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.867963:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:20.2:1619677447.867964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:18.0:1619677447.867965:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.867966:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.867968:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.867969:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:2.0:1619677447.867969:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.867971:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.867972:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:2.0:1619677447.867972:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.867974:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:2.0:1619677447.867974:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.867975:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.867975:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.867977:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.867979:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.867979:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.867981:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.867981:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.867982:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.867983:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677447.867984:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.867984:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.867984:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677447.867986:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677447.867986:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.867986:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.867987:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.867988:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:5.0:1619677447.867989:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:9.2:1619677447.867992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677447.867993:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677447.867994:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677447.867994:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.867997:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.867998:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.867998:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.0:1619677447.867998:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.867999:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677447.867999:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.868000:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:26.0:1619677447.868001:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.868001:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:18.0:1619677447.868002:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.868002:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.868002:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000400:00000200:26.0:1619677447.868003:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.868003:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.868004:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.868004:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:26.0:1619677447.868005:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:4.2:1619677447.868005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:26.0:1619677447.868006:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96722451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.868006:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.2:1619677447.868006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677447.868008:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.868008:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:26.0:1619677447.868009:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677447.868009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:9.2:1619677447.868009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:26.0:1619677447.868010:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677447.868013:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.868013:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677447.868013:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:1.0:1619677447.868014:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:26.0:1619677447.868015:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.868015:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:24.0:1619677447.868016:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:20.0:1619677447.868016:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.868016:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:2.0:1619677447.868016:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.868017:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677447.868017:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:26.0:1619677447.868018:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.868018:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.868018:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.868019:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677447.868020:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000400:00000200:24.0:1619677447.868020:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.868020:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677447.868020:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.868021:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (24)--
00000400:00000200:24.0:1619677447.868023:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.868023:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677447.868024:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677447.868024:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:24.0:1619677447.868024:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:26.0:1619677447.868025:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:24.0:1619677447.868025:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96722450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.868025:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.868026:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677447.868027:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:24.0:1619677447.868027:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.868027:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:24.0:1619677447.868028:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.868029:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:9.2:1619677447.868029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:24.0:1619677447.868032:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.868034:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.868037:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.868037:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677447.868038:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.868039:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.868039:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:24.0:1619677447.868040:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.868041:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:18.0:1619677447.868041:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.868041:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.868042:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.868043:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000400:00000200:18.0:1619677447.868043:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.868044:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677447.868046:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.2:1619677447.868046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.868049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.868054:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.868057:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677447.868057:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.868057:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.868058:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.868059:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677447.868060:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.868060:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.868061:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.868062:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.868088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.868095:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.868097:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.868098:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.868100:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.868103:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.868104:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.868105:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.868106:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.868107:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.868108:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.2:1619677447.868109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677447.868111:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.868113:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677447.868114:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677447.868115:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.868116:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.868117:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677447.868117:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.868118:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677447.868118:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.868119:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.868119:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677447.868119:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.868120:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:1.0:1619677447.868122:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.868124:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.868125:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.868126:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.868127:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.868128:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.868132:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:1.0:1619677447.868132:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.868133:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.868135:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.868136:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677447.868136:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677447.868137:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.868137:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.868137:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:6.0:1619677447.868139:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.868139:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.868140:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.868141:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.868141:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.868142:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677447.868152:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.868155:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.868156:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.868158:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.868159:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.868161:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.868279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.868282:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.868284:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.868286:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.868287:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.868289:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.868290:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.868292:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.868297:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.868298:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.868299:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.868303:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.868304:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.868308:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.868308:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.868309:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.868311:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.868312:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.868313:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.868314:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.868314:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.868320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677447.868326:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677447.868326:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.868328:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:30.0:1619677447.868329:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.868330:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.868330:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.868330:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:30.0:1619677447.868332:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.868333:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.868335:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.868393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677447.868401:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.868404:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677447.868405:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677447.868407:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.868410:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.868412:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677447.868413:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.868414:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.868415:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.868416:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677447.868420:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.868421:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677447.868424:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.868425:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677447.868426:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677447.868428:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677447.868428:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.868429:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677447.868429:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677447.868435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.868443:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.868446:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.868447:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.868448:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.868449:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.868452:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677447.868453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.868461:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.868463:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.868464:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.868465:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.868465:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.868466:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677447.868466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.868472:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.868475:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.868475:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.868477:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677447.868479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:2.0:1619677447.868479:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.868480:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.868482:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.868483:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.868484:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.868485:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:24.0:1619677447.868486:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.868489:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:2.0:1619677447.868489:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.868490:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:2.0:1619677447.868491:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:24.0:1619677447.868492:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.868494:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.868495:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.868495:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.868495:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:24.0:1619677447.868497:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:2.0:1619677447.868497:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:24.0:1619677447.868498:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96722444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.868498:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:24.0:1619677447.868499:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.868499:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:24.0:1619677447.868500:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.868500:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.868500:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:24.0:1619677447.868501:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:8.2:1619677447.868504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:24.0:1619677447.868506:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677447.868506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:24.0:1619677447.868508:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.868511:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.868511:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:24.0:1619677447.868512:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677447.868512:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677447.868512:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677447.868513:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:24.0:1619677447.868514:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.868514:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677447.868514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:24.0:1619677447.868515:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.868515:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:20.2:1619677447.868515:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677447.868515:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:24.0:1619677447.868516:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677447.868519:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.868520:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.868521:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.868521:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677447.868521:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.868523:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677447.868524:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:10.0:1619677447.868524:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.868525:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677447.868526:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:13.0:1619677447.868527:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:10.0:1619677447.868528:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.868529:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.868529:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:13.0:1619677447.868530:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.868531:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.868531:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.868532:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.868533:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:10.0:1619677447.868533:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.868534:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:12.0:1619677447.868534:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:10.0:1619677447.868534:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.868535:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.868535:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:10.0:1619677447.868536:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.868537:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.868537:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.868538:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.868538:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.868539:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.868544:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677447.868544:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.868546:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:12.0:1619677447.868546:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677447.868547:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:12.0:1619677447.868548:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.868549:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.868549:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:10.0:1619677447.868549:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.868550:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.868550:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677447.868550:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.868551:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.868551:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677447.868551:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:12.0:1619677447.868552:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677447.868552:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.868553:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677447.868553:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677447.868554:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677447.868554:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.868554:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677447.868555:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.868556:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.868557:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677447.868559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677447.868563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677447.868567:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677447.868568:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.868570:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.868570:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.868570:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.868571:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.868572:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.868573:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:30.0:1619677447.868573:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.868573:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.868574:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:5.0:1619677447.868574:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.868575:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.868576:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677447.868577:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:5.0:1619677447.868577:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677447.868578:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.868578:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677447.868579:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677447.868579:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.868580:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677447.868630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.868636:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.868639:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.868640:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.868642:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.868645:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.868646:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.868647:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.868649:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.868650:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.868651:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.868655:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.868656:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:4.2:1619677447.868656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677447.868659:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.868660:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.868661:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.868662:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:21.2:1619677447.868662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677447.868663:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.868663:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.868664:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.868664:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.868666:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.868667:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677447.868669:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.2:1619677447.868669:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.868671:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.868673:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.868675:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.868676:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.868676:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:8.0:1619677447.868676:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.868678:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.868678:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.868679:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:31.0:1619677447.868680:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.868680:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.868680:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:1.0:1619677447.868680:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:31.0:1619677447.868681:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.868682:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96722440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.868683:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.868683:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.868684:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.868684:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677447.868686:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677447.868687:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.868689:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.868690:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:8.0:1619677447.868690:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.868691:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.868692:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.868692:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.868693:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.868693:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.868694:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.868695:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.868695:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677447.868695:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000400:00000200:8.0:1619677447.868695:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.868696:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.868696:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677447.868697:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677447.868697:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.0:1619677447.868697:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.868698:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677447.868700:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.868700:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.868701:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.868703:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.868704:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.868704:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.868705:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.868707:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:4.2:1619677447.868708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.2:1619677447.868709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677447.868710:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.868710:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.868712:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.868714:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677447.868715:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677447.868715:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.868716:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:26.0:1619677447.868717:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.868718:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677447.868719:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:26.0:1619677447.868720:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.868720:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.868720:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.868722:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.868722:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:26.0:1619677447.868724:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.868725:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.868725:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:26.0:1619677447.868727:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.868728:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.868729:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.868732:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.868734:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.868736:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.868737:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.868737:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.868739:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.868739:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.868741:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.868742:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.868743:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677447.868748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677447.868751:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.868753:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.868754:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.868755:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.868756:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:31.0:1619677447.868757:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.868758:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677447.868758:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.868758:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.868759:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677447.868776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677447.868782:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.868785:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.868786:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.868787:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.868790:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.868791:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677447.868793:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.868794:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.868795:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.868796:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677447.868800:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.868801:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677447.868804:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.868804:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677447.868805:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677447.868807:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677447.868807:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.2:1619677447.868808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677447.868808:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.868808:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677447.868813:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.868816:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.868817:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.868819:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.868820:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:17.0:1619677447.868821:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.868823:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:6.0:1619677447.868823:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.868824:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:17.0:1619677447.868824:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.868824:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.868826:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.868826:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000400:00000200:17.0:1619677447.868827:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.868828:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.868829:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:17.0:1619677447.868832:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.868834:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.868836:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.868837:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.868838:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.868839:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.868840:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.868840:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.868841:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.868853:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.868855:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.868856:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.868856:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.868858:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.868861:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.869176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.869184:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.869186:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.869188:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.2:1619677447.869188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:24.0:1619677447.869189:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.869192:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.869194:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.869195:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.869195:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:24.0:1619677447.869196:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.869197:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677447.869197:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:24.0:1619677447.869198:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:26.0:1619677447.869199:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.869200:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.869202:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677447.869202:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.869204:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:10.0:1619677447.869204:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677447.869205:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.869206:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.869207:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.869207:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:10.0:1619677447.869207:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.869208:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:10.0:1619677447.869208:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.869210:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.869210:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.869210:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.869211:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:10.0:1619677447.869211:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.869213:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677447.869215:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.869216:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677447.869216:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.2:1619677447.869216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677447.869218:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677447.869218:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.869219:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.869220:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:1.0:1619677447.869223:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.869226:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.869227:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.869227:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.869229:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.869230:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.869232:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677447.869232:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:12.0:1619677447.869233:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.869235:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.869236:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.869237:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677447.869325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677447.869332:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.869334:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.869335:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.869337:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.869340:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.869342:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.869343:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.869344:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.869345:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677447.869346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:18.0:1619677447.869347:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.869350:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.869352:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677447.869352:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.869354:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:18.0:1619677447.869355:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.869355:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:1.0:1619677447.869355:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677447.869356:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:17.0:1619677447.869357:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.869358:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.869358:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.869359:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.869360:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:17.0:1619677447.869360:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.869361:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.869362:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.869364:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.869365:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677447.869365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:17.0:1619677447.869366:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.869369:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.869371:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.869373:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677447.869374:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.869375:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:17.0:1619677447.869375:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:20.0:1619677447.869376:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.869376:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677447.869378:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.869378:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.869378:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.869379:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677447.869379:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677447.869380:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677447.869381:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677447.869392:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:4.2:1619677447.869392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.869394:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.869395:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.869397:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.869399:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677447.869400:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677447.869400:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677447.869403:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.869404:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.869405:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.869408:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.869409:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677447.869410:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.869411:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.869412:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.869414:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677447.869417:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.869418:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677447.869421:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.869422:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677447.869423:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.869425:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.869425:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.869426:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.869426:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677447.869431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.869438:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.869441:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.869442:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.869442:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.869444:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.869448:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677447.869489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.869497:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.869500:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.869502:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.869504:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.869504:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.869505:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677447.869507:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.869508:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.869509:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.869510:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677447.869513:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.869515:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677447.869518:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.869518:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677447.869519:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677447.869521:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677447.869521:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.869522:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.869523:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677447.869527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.2:1619677447.869528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.869535:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677447.869535:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.869537:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:30.0:1619677447.869538:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:18.0:1619677447.869538:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.869538:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:30.0:1619677447.869541:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.869541:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.869542:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:18.0:1619677447.869545:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:30.0:1619677447.869546:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.869548:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.869549:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.869550:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.869551:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.869552:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.869555:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.869557:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.869559:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.869560:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.869560:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.869562:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.869562:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.869563:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.869564:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.869574:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.869576:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.869577:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.869578:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.869580:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.869581:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.869617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.869623:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.869625:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.869628:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.869628:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.869630:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.869632:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.869633:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.869634:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.869635:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.869636:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.869640:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.869642:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.869645:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.869646:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.869647:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.869649:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.869649:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.869650:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.869651:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677447.869661:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.869664:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.869665:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.869667:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:30.0:1619677447.869668:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.869670:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.869680:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.869682:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.869683:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.869684:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.869684:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.869686:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.869743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.869748:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.869751:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.869752:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.869754:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.869757:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.869758:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.869760:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.869761:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.869762:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.869763:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.869766:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.869768:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.869771:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.869772:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.869773:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.869775:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.869776:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.869777:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.869778:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.869778:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677447.869783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677447.869787:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677447.869789:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.869790:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:12.0:1619677447.869790:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677447.869791:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677447.869791:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.869792:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677447.869792:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.869793:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.869795:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.869844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.869850:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:4.2:1619677447.869850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:6.0:1619677447.869853:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.869854:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.869856:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.869857:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:6.0:1619677447.869858:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.869860:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:6.0:1619677447.869860:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:24.0:1619677447.869861:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677447.869861:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722426] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.869863:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.869864:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.869864:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.869865:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:8.0:1619677447.869866:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.869868:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677447.869869:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.869870:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96722425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.869871:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.869872:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.869872:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:8.0:1619677447.869874:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:6.0:1619677447.869875:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.869875:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.869876:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:8.0:1619677447.869877:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.869878:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:8.0:1619677447.869879:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677447.869879:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.869879:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.869880:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.869882:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.869883:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.2:1619677447.869885:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.0:1619677447.869888:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.869890:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.869891:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.869891:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.869892:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677447.869893:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.869895:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.869896:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.869896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:20.0:1619677447.869897:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.869899:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.869900:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677447.869904:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.869906:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.869907:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.869907:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.869908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:1.0:1619677447.869910:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.869913:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:10.0:1619677447.869916:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.869918:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.869919:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.869922:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.869925:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.869926:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677447.869927:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.869929:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.869930:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.869930:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677447.869934:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.869936:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677447.869939:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.869939:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677447.869940:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677447.869942:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677447.869942:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.869943:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677447.869944:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677447.869945:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.2:1619677447.869947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677447.869950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677447.869955:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.869957:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:26.0:1619677447.869958:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:1.0:1619677447.869958:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.869959:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.869960:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677447.869961:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:17.0:1619677447.869961:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.869962:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677447.869963:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.870042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.870049:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.870052:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.870053:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.870055:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.870058:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.870059:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677447.870060:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722423] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.870061:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.870063:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.870063:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677447.870066:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.870067:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.870069:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.870069:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677447.870070:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677447.870071:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.870071:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.870072:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.870072:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677447.870083:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.870084:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.870085:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.870086:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677447.870087:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.870088:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677447.870169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.870174:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.870177:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.870178:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.870180:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.870182:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.870184:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.870185:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.870186:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.870187:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.870188:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677447.870192:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.870193:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.870196:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.870197:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.870198:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.870200:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.870200:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.870200:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.870201:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.870206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.870212:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.870215:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.870215:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.870216:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.870218:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.870221:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677447.870261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.870269:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.870271:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.870274:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.870275:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.870276:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.870277:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.870278:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.870280:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.870280:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.870281:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.870284:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.870286:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.870288:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.870289:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.870290:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.870291:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.870292:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.870299:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.870299:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.870306:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.870308:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.870309:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.870310:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:20.2:1619677447.870312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:13.0:1619677447.870313:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.870314:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.870320:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.870323:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:13.0:1619677447.870323:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677447.870325:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.870326:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.870326:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677447.870327:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.870327:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.870327:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.870327:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:30.0:1619677447.870328:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.870329:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.870329:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:30.0:1619677447.870330:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.870331:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.870332:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.870335:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.870336:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.870339:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.870339:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.870340:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.870341:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.870342:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.870342:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.870343:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677447.870353:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.870356:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.870356:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.870358:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.870359:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.870360:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677447.870472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677447.870478:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.870480:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.870482:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.870483:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.870485:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.870487:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.870488:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.870490:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.870490:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.870492:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.870495:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.870496:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.870499:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.870500:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.870500:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.870502:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.870502:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.870503:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.870503:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.2:1619677447.870504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:8.2:1619677447.870509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677447.870512:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677447.870515:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677447.870515:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000400:00000200:10.0:1619677447.870517:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.870517:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:10.0:1619677447.870519:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.870520:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:10.0:1619677447.870521:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677447.870522:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.870523:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000400:00000200:10.0:1619677447.870524:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.870524:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (22)--
00000400:00000200:10.0:1619677447.870525:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.870525:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.870527:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677447.870527:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.870528:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.870530:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:5.0:1619677447.870530:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:10.0:1619677447.870531:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.870532:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.870533:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:26.0:1619677447.870535:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.870536:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677447.870537:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:10.0:1619677447.870537:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.2:1619677447.870538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677447.870538:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:26.0:1619677447.870539:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.870540:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.870540:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677447.870540:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:26.0:1619677447.870541:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.870542:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677447.870543:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:10.0:1619677447.870543:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677447.870544:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677447.870544:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:10.0:1619677447.870545:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:20.0:1619677447.870546:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677447.870546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.870546:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677447.870548:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.870549:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.870550:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:20.0:1619677447.870550:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.870552:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96722416] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.870553:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.870553:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677447.870554:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677447.870554:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:21.2:1619677447.870554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:20.0:1619677447.870554:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.870554:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:20.0:1619677447.870555:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.870556:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677447.870556:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677447.870556:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:26.0:1619677447.870557:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677447.870557:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:26.0:1619677447.870558:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:20.0:1619677447.870558:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.870558:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.870560:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.870560:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:17.0:1619677447.870561:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.870562:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:17.0:1619677447.870563:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:2.0:1619677447.870563:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:4.2:1619677447.870564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.0:1619677447.870565:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677447.870565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000400:00000200:18.0:1619677447.870566:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677447.870567:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.870568:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:18.0:1619677447.870568:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.870570:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:20.0:1619677447.870570:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.870570:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:18.0:1619677447.870570:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:20.0:1619677447.870571:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:18.0:1619677447.870571:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96722415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.870572:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.870573:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:18.0:1619677447.870573:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.870573:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.870573:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:22.0:1619677447.870574:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.870574:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.870574:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:18.0:1619677447.870575:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:8.0:1619677447.870576:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:1.0:1619677447.870576:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.870577:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677447.870577:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.870577:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.870578:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.870579:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000400:00000200:18.0:1619677447.870580:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677447.870580:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:8.0:1619677447.870581:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.870583:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.870584:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:8.0:1619677447.870584:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.870586:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677447.870586:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677447.870586:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:6.0:1619677447.870586:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677447.870587:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677447.870587:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.870588:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.870588:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.870588:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96722414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.870588:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:18.0:1619677447.870589:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000400:00000200:8.0:1619677447.870589:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.870589:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:18.0:1619677447.870590:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:6.0:1619677447.870590:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.870590:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.870591:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.870591:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:8.0:1619677447.870591:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.870591:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677447.870592:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:6.0:1619677447.870592:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.870593:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:8.0:1619677447.870593:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:4.2:1619677447.870594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.870595:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:8.0:1619677447.870596:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677447.870598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.0:1619677447.870598:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.870601:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.870602:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677447.870602:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.870602:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.870603:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:26.0:1619677447.870604:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677447.870604:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:8.0:1619677447.870604:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:26.0:1619677447.870605:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:8.0:1619677447.870605:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677447.870605:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:10.0:1619677447.870606:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.870606:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677447.870606:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:26.0:1619677447.870607:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.870607:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:2.0:1619677447.870607:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677447.870608:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:26.0:1619677447.870609:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000400:00000200:10.0:1619677447.870609:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.870610:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:10.0:1619677447.870612:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677447.870613:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.870613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677447.870614:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.870615:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.870616:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:17.0:1619677447.870618:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:10.0:1619677447.870619:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.870620:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:24.0:1619677447.870620:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677447.870620:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:10.0:1619677447.870620:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.870621:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677447.870621:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.870622:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677447.870622:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.870623:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:24.0:1619677447.870623:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677447.870624:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.870624:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.870625:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677447.870625:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677447.870625:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:17.0:1619677447.870627:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677447.870627:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.870628:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.870628:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.870629:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677447.870638:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.870640:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.870641:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.870642:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677447.870643:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.870644:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677447.870717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.870722:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.870725:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.870726:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.870727:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.870730:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.870732:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.870733:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.870734:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.870735:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.870736:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677447.870740:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.870741:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.870744:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.870745:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.870745:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.870747:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.870748:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.870749:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.870749:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.870753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.870760:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.870762:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.870763:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.870763:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.870765:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.870768:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677447.870773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.870780:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.870783:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.870785:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.870786:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.870787:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.870789:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677447.870790:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722411] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.870791:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.870792:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.870793:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677447.870796:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.870798:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677447.870801:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.870802:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677447.870803:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677447.870804:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677447.870805:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.870806:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.870808:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.870808:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677447.870814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677447.870818:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.870821:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:30.0:1619677447.870822:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677447.870822:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.870822:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.870824:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:6.0:1619677447.870824:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.870825:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.870826:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.870828:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677447.870958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.870965:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.870968:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.870969:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.870970:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.870972:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.870973:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677447.870974:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.870976:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.870977:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.870978:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677447.870981:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.870982:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.870983:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.870984:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677447.870985:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677447.870986:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.870986:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.870987:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.870987:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677447.870997:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.870999:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.871000:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.871000:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677447.871002:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.871003:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677447.871016:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677447.871018:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.871019:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.871019:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.871019:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677447.871020:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677447.871148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677447.871150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677447.871156:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677447.871157:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677447.871158:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677447.871160:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.871160:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:30.0:1619677447.871161:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.871163:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677447.871163:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.871164:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.871165:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.871165:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677447.871166:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677447.871167:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.871168:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.871169:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.871170:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.871170:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.871171:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.871171:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677447.871172:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677447.871176:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.871177:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.871178:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677447.871179:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677447.871180:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.871180:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.871180:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677447.871182:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.871182:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.871182:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677447.871183:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677447.871183:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:30.0:1619677447.871184:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677447.871184:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:30.0:1619677447.871185:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677447.871185:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677447.871186:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.871187:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.871187:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677447.871190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.871195:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.871196:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.871197:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.871198:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:26.0:1619677447.871199:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:10.0:1619677447.871199:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.871200:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.871200:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.871201:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677447.871201:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:10.0:1619677447.871201:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677447.871202:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677447.871202:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.871203:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.871205:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.871208:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677447.871297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677447.871303:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.871306:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.871307:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.871309:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.871312:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.871313:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.871314:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.871316:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.871317:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.871318:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.871321:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.871323:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.871325:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.871326:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.871326:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.871328:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.871329:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.871329:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.871330:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.871335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.871342:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.871344:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.871344:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.871345:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.871346:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.871348:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.871367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.871373:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.871376:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.871377:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.871379:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.871382:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.871384:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.871385:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.871386:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.871387:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.871388:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.871391:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.871393:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.871395:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.871396:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.871396:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.871398:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.871398:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.871399:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.871399:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.871404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.871412:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.871414:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.871414:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.871416:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.871416:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.871418:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677447.871466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.871472:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.871475:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.871475:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.871477:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.871479:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.871481:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.871482:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722405] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.871484:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.871485:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.871486:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.871489:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.871490:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.871493:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.871494:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.871494:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.871496:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.871496:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.871497:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.871499:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.871499:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.871502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677447.871504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.871509:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677447.871510:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677447.871511:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.871512:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677447.871512:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:22.0:1619677447.871513:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:18.0:1619677447.871513:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.871513:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.871516:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.871518:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.871524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.871531:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.871534:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.871535:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.871535:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.871536:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.871537:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677447.871682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.871690:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.871693:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.871695:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.871698:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.871698:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.871700:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677447.871701:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.871703:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.871703:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.871705:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677447.871708:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.871710:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677447.871713:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.871713:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677447.871714:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677447.871716:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.871717:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.871717:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.871718:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677447.871724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.871730:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677447.871730:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677447.871733:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:6.0:1619677447.871733:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677447.871733:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:22.0:1619677447.871734:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.871734:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.871736:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.871736:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.871737:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.871738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:18.0:1619677447.871739:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:6.0:1619677447.871739:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:18.0:1619677447.871740:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.871741:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.871742:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.871743:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677447.871746:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:18.0:1619677447.871747:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.871749:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:18.0:1619677447.871749:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:30.0:1619677447.871751:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.871752:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:18.0:1619677447.871752:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.871752:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.871753:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677447.871754:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.871755:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.871755:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:30.0:1619677447.871756:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:18.0:1619677447.871756:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.871756:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:30.0:1619677447.871757:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.871758:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.871759:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.871760:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677447.871764:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.871765:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677447.871768:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.871769:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.871770:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677447.871772:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.871772:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.871773:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.871774:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.871778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.2:1619677447.871782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.871787:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677447.871789:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677447.871789:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.871790:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.871792:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:5.0:1619677447.871792:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.871793:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:17.0:1619677447.871793:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.871793:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.871795:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.871795:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.871797:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.871798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:21.2:1619677447.871801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:1.0:1619677447.871806:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.871808:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:17.0:1619677447.871809:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.871809:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.871810:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.871811:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:1.0:1619677447.871811:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.871811:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.0:1619677447.871812:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.871814:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.871816:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.871817:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.871819:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.871820:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.871821:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.871822:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.871825:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.871827:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.871829:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.871829:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.871830:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.871831:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.871832:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.871833:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.871833:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.871845:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.871847:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.871848:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.871850:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.871851:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.871852:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.871965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.871974:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.871977:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.871978:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.871979:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.871981:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.871982:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:4.2:1619677447.871982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:10.0:1619677447.871983:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.871985:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.871986:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.871987:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677447.871990:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.871991:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677447.871992:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.871993:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.871994:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:26.0:1619677447.871995:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.871996:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:10.0:1619677447.871997:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677447.871998:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.871998:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:26.0:1619677447.871999:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677447.871999:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:26.0:1619677447.872000:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.872001:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:10.0:1619677447.872001:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677447.872002:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.872003:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.872004:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.872004:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677447.872005:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.872006:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677447.872008:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.872009:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)++
00000800:00000200:26.0:1619677447.872009:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (23)++
00000800:00000200:26.0:1619677447.872010:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (24)--
00000800:00000200:26.0:1619677447.872011:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.872011:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.872012:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677447.872015:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (23)++
00000800:00000200:22.0:1619677447.872018:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000800:00000200:6.0:1619677447.872018:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (24)++
00000400:00000200:22.0:1619677447.872019:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.872019:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000400:00000200:6.0:1619677447.872020:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.872021:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.872022:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.872025:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.872026:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.872026:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.872037:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.872040:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.872040:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.872041:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.872042:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.872042:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.872225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.872228:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.872230:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.872233:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.872234:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.872235:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.872237:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677447.872238:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.872239:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.872240:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.872241:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677447.872245:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.872246:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.872249:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.872249:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677447.872250:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.872252:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.872253:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.872253:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.872254:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677447.872258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.872266:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.2:1619677447.872269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:17.0:1619677447.872269:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.872270:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.872270:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.872272:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677447.872274:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.872276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677447.872276:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:21.2:1619677447.872279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:10.0:1619677447.872279:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677447.872280:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677447.872282:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.872283:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677447.872284:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677447.872284:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.872286:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:5.0:1619677447.872286:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:22.0:1619677447.872287:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:20.0:1619677447.872288:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:10.0:1619677447.872288:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.872288:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:21.2:1619677447.872290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:10.0:1619677447.872290:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.872291:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.872291:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.872292:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.872299:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.872300:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.872300:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.872301:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:22.0:1619677447.872302:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.872303:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.872304:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677447.872304:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000400:00000200:22.0:1619677447.872305:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.872306:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677447.872307:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:5.0:1619677447.872307:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677447.872308:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.872308:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677447.872308:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.872309:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.872310:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.872311:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.872311:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.872312:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:5.0:1619677447.872312:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.872313:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:22.0:1619677447.872313:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.872313:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:5.0:1619677447.872313:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677447.872315:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96722394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.872315:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677447.872315:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:31.0:1619677447.872316:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.872316:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677447.872316:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:31.0:1619677447.872317:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677447.872317:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:31.0:1619677447.872319:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:22.0:1619677447.872319:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.872320:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.872321:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.2:1619677447.872321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.872322:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.872323:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.872323:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.872324:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:31.0:1619677447.872326:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.872326:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677447.872328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.872329:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677447.872330:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.872330:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.872331:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677447.872332:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.872332:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.872333:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:20.0:1619677447.872333:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.872333:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.872334:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677447.872334:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677447.872335:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677447.872335:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.872335:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677447.872336:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677447.872336:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.872337:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.872337:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677447.872337:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.872338:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.872338:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.872338:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677447.872338:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677447.872339:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677447.872339:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.872339:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677447.872339:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.872340:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.872341:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000400:00000200:1.0:1619677447.872341:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.872342:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)++
00000800:00000200:5.0:1619677447.872343:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.872344:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:1.0:1619677447.872344:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677447.872345:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:26.0:1619677447.872348:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.872351:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677447.872351:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.872352:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:26.0:1619677447.872352:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.872354:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.872355:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.872355:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.872356:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:30.0:1619677447.872357:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.872358:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.872360:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.872367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.872374:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.872377:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.872378:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.872380:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.872382:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.872384:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.872385:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.872386:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.872388:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.872388:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.872392:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677447.872393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677447.872394:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.872397:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.872398:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677447.872398:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.872399:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.872400:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:17.0:1619677447.872401:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:2.0:1619677447.872401:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.872401:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.872402:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677447.872402:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.872404:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.872406:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.872406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:17.0:1619677447.872407:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.872409:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.872410:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.872411:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.872412:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677447.872414:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:17.0:1619677447.872415:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.872416:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.872417:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.872417:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:20.0:1619677447.872418:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.872420:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.872421:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677447.872421:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.872422:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.872422:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.872424:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.872425:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.872426:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.872427:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.872428:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677447.872433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677447.872438:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677447.872438:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.872440:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:13.0:1619677447.872441:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677447.872441:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677447.872442:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.872442:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677447.872444:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.872445:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.872446:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.872454:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.872457:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.872458:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.872458:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.872459:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.872461:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677447.872494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677447.872500:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.872503:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.872503:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.872505:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.872508:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.872509:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.872510:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.872512:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.872513:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.872514:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.872516:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.872518:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.872520:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.872521:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.872521:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.872522:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.872523:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.872524:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.872525:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.872525:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677447.872528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677447.872531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:4.2:1619677447.872535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.872536:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.872538:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.872538:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.872539:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.872539:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.872539:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.872540:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.872540:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.872541:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677447.872542:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677447.872543:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.872545:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.872546:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.872548:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677447.872550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:10.0:1619677447.872551:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.872552:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.872553:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.872555:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.872556:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.872557:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677447.872557:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.872560:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.872560:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000400:00000200:10.0:1619677447.872560:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.872561:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.872561:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.872562:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677447.872563:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677447.872565:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.872566:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.872567:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.872569:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.872569:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.872570:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.872570:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677447.872582:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.872585:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.872586:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.872587:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.872589:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.872592:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.872605:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.872607:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.872608:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.872608:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.872609:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.872609:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.872680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.872688:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.872691:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.872693:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.872695:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.872695:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.872697:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.872698:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.872700:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.872701:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.872702:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.872705:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.872707:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.872710:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.872711:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.872711:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.872713:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.872714:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.872715:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.872717:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.872718:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677447.872720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677447.872723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677447.872728:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677447.872731:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:5.0:1619677447.872731:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.872732:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.872733:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.872734:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.872734:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677447.872734:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.872736:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677447.872736:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677447.872839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:20.2:1619677447.872844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:6.0:1619677447.872844:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.872847:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677447.872847:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677447.872847:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:20.0:1619677447.872849:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:6.0:1619677447.872849:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.872852:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.872852:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.872854:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.872854:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:5.2:1619677447.872854:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000400:00000200:6.0:1619677447.872855:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.872856:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.872857:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.872857:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.872858:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.872858:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.872859:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.872860:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.872860:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677447.872861:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000400:00000200:20.0:1619677447.872862:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.872864:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:10.0:1619677447.872864:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:6.0:1619677447.872864:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.872867:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.872867:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:20.0:1619677447.872869:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.872870:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.872870:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.872871:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.872872:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:26.0:1619677447.872873:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.872873:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.872873:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:26.0:1619677447.872874:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:20.0:1619677447.872874:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.872874:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.872874:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:26.0:1619677447.872875:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96722386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.872875:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677447.872876:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.872876:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:26.0:1619677447.872877:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.872877:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677447.872878:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.872878:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677447.872879:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:20.0:1619677447.872879:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:26.0:1619677447.872883:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.872885:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677447.872886:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677447.872888:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.872888:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677447.872889:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:10.0:1619677447.872889:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.872890:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:26.0:1619677447.872890:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:10.0:1619677447.872891:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.872892:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:26.0:1619677447.872892:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677447.872893:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.872893:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677447.872894:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75ca00] (22)++
00000400:00000200:30.0:1619677447.872895:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.872895:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:31.0:1619677447.872896:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.872896:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (23)--
00000800:00000200:10.0:1619677447.872896:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:26.0:1619677447.872897:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:30.0:1619677447.872900:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.872902:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:5.2:1619677447.872902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:30.0:1619677447.872903:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.872904:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.872905:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677447.872905:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.872907:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:10.0:1619677447.872909:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)++
00000800:00000200:10.0:1619677447.872911:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75ca00] (23)--
00000800:00000200:26.0:1619677447.872912:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:10.0:1619677447.872912:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:5.2:1619677447.873017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.873022:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.873025:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.873026:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.873028:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.873030:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.873031:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.873033:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.873034:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.873035:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.873036:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.873040:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.873041:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.873045:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.873045:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.873046:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.873048:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.873049:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.873049:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.873050:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677447.873061:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.873063:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.873064:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.873066:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.873067:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.873069:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.873133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.873140:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.873143:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.873144:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.873146:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.873148:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.873149:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.873150:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.873152:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.873153:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677447.873153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677447.873154:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677447.873157:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.873159:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677447.873159:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677447.873162:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:10.0:1619677447.873162:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.873162:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:1.0:1619677447.873162:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677447.873163:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:17.0:1619677447.873165:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.873165:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.873165:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.873166:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.873166:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:17.0:1619677447.873167:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.873169:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.873170:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.873172:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.873172:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.873173:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.873177:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.873178:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677447.873179:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.873181:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.873182:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.873182:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.873182:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.873182:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677447.873183:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:22.0:1619677447.873184:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.873185:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.873186:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677447.873187:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.873187:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.873187:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.873198:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.873201:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.873202:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.873204:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.873204:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.873207:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677447.873211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.873218:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.873221:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.873222:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.873223:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.873226:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.873227:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.873229:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.873230:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.873231:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.873232:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.873235:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.873237:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.873239:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.873240:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.873241:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.873243:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.873243:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.873244:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.873244:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.873249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.873256:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.873259:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.873259:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.873260:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.873262:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.873265:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.873267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.873275:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.873278:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.873280:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.873282:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.873282:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.873284:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.873285:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.873286:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.873287:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.873288:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.873291:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.873293:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.873302:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.873302:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.873303:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.873305:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.873305:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.873306:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.873307:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.873312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.873321:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.873323:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.873324:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.873326:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.873326:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.873329:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677447.873403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.873411:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.873414:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.873414:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.873416:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.873419:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.873421:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677447.873422:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.873423:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.873424:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.873425:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677447.873428:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.873430:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677447.873433:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.873433:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677447.873434:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.873436:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.873437:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.873438:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677447.873439:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677447.873439:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677447.873443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677447.873445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677447.873451:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677447.873452:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677447.873454:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:18.0:1619677447.873455:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:10.0:1619677447.873455:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:18.0:1619677447.873456:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.873456:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677447.873456:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.873458:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.873462:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677447.873519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.873525:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.873529:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.873529:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.873531:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.873534:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.873535:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.873536:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.873538:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.873539:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.873540:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.873544:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.873546:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.873548:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.873549:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.873550:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.873552:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.873552:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.873553:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.873553:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677447.873556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.873559:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.873562:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:26.0:1619677447.873563:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:20.0:1619677447.873564:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.873565:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677447.873566:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:20.0:1619677447.873566:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.873567:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.873567:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:20.0:1619677447.873568:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677447.873569:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.873570:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.873572:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.873573:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.873574:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:20.0:1619677447.873574:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677447.873577:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.873579:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677447.873582:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.873582:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.873583:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677447.873583:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:20.0:1619677447.873585:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.873585:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677447.873586:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677447.873586:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:20.0:1619677447.873587:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:10.0:1619677447.873587:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677447.873587:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:20.0:1619677447.873588:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:10.0:1619677447.873588:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.0:1619677447.873589:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677447.873591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677447.873594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677447.873599:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:5.0:1619677447.873599:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.873601:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:17.0:1619677447.873602:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:5.0:1619677447.873602:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:17.0:1619677447.873603:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.873604:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:1.0:1619677447.873604:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.873605:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.873607:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677447.873608:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:2.0:1619677447.873610:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.873613:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.873613:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.873615:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.873618:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.873619:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.873621:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.873622:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.873623:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.873624:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.873628:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.873629:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.873633:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.873634:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.873634:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.873636:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.873636:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.873638:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.873639:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.873639:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677447.873644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677447.873649:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677447.873650:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:4.2:1619677447.873651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.873652:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:12.0:1619677447.873652:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677447.873653:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:12.0:1619677447.873653:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.873654:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:12.0:1619677447.873655:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.873656:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677447.873658:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:12.0:1619677447.873658:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677447.873660:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.873661:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.873663:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.873666:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.873668:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.873669:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.873670:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.873671:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.873672:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.873675:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.873677:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.873679:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.873680:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.873681:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.873682:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.873683:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.873684:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.873685:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.873685:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677447.873691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677447.873695:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.873698:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.873698:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.873698:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677447.873700:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.873700:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.873700:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.873701:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677447.873702:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.873703:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.873711:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.873714:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.873715:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.873715:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.873716:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.873717:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.873774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677447.873780:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.873783:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.873784:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.873786:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.873789:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.873790:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.873791:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.873793:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.873794:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.873795:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.873798:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.873800:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.873802:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.873803:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.873804:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.873805:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.873805:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.873806:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.873807:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.873819:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.873821:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.873822:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.873822:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.873824:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.873827:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.873956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.873958:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.873961:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.2:1619677447.873963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:20.0:1619677447.873963:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.873966:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.873966:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.873967:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.873969:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.873970:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.873971:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:20.0:1619677447.873971:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.873972:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677447.873974:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:20.2:1619677447.873975:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.0:1619677447.873975:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.873977:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.873980:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.873981:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.873982:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:30.0:1619677447.873983:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677447.873983:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:24.0:1619677447.873984:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96722373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.873985:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.873986:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:24.0:1619677447.873986:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.873987:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.873988:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677447.873988:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:30.0:1619677447.873989:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.873989:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677447.873990:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677447.873990:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677447.873991:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.873991:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.873991:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.873992:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.873992:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:5.2:1619677447.873992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677447.873993:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.873994:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96722372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.873994:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.873995:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.873997:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.873997:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.873997:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677447.873998:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:24.0:1619677447.873998:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677447.873999:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677447.874000:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.874000:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:24.0:1619677447.874001:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.874001:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:17.0:1619677447.874001:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677447.874002:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:24.0:1619677447.874002:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:31.0:1619677447.874004:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.874004:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.874004:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.874005:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.874006:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.874007:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.874008:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.874009:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:1.0:1619677447.874009:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:31.0:1619677447.874010:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.874010:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.874010:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:1.0:1619677447.874010:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96722371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.874011:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:1.0:1619677447.874012:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.874013:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.874013:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677447.874013:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:1.0:1619677447.874013:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.874014:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:1.0:1619677447.874014:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:30.0:1619677447.874015:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677447.874015:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.874016:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.874018:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:5.2:1619677447.874018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:1.0:1619677447.874018:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.874019:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.2:1619677447.874019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:13.0:1619677447.874020:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677447.874021:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.874024:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.874024:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677447.874025:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677447.874025:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.0:1619677447.874027:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:1.0:1619677447.874027:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.874027:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677447.874028:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.874028:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677447.874028:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677447.874029:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:1.0:1619677447.874029:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:5.0:1619677447.874030:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.874031:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.874032:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.874032:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:26.0:1619677447.874033:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.874034:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:5.0:1619677447.874035:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:26.0:1619677447.874036:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677447.874036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000400:00000200:26.0:1619677447.874037:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.874038:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.874039:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677447.874040:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:4.2:1619677447.874041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:26.0:1619677447.874043:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.874043:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.874044:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.874044:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.874045:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.874045:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:6.0:1619677447.874046:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.874048:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677447.874049:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.874049:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:10.0:1619677447.874049:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.874049:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:26.0:1619677447.874050:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.874050:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:26.0:1619677447.874051:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.874051:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677447.874051:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:20.0:1619677447.874052:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:26.0:1619677447.874053:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:10.0:1619677447.874053:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.874054:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.874055:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.874055:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:10.0:1619677447.874056:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.874057:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.874058:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677447.874059:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677447.874059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677447.874061:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.874062:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.874062:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677447.874066:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.874067:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677447.874067:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677447.874069:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.874069:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:10.0:1619677447.874070:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.874070:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:18.0:1619677447.874071:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.874071:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.874072:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.874072:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.874073:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.874073:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:2.0:1619677447.874073:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.874076:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677447.874084:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.874087:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.874088:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.874090:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.874091:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.874092:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677447.874257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.874265:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.874267:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.874268:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.874270:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.874272:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.874274:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.874275:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.874276:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.874277:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.874278:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.874281:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.874283:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.874286:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.874287:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.874287:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.874289:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.874289:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.874290:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.874290:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.874296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677447.874299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677447.874304:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.874307:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:30.0:1619677447.874308:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677447.874308:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.874308:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677447.874311:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:17.0:1619677447.874311:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677447.874312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677447.874314:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.874315:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.874316:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.874316:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.874317:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677447.874318:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.874320:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.874320:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.874320:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.874321:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677447.874322:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.874322:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.874322:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000400:00000200:30.0:1619677447.874323:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.874323:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:30.0:1619677447.874327:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.874328:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.874331:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.874331:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.874332:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.874333:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.874334:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.874335:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.874337:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.874338:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677447.874343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677447.874347:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.874349:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.874350:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.874351:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:12.0:1619677447.874352:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.874353:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.874354:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.2:1619677447.874354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:12.0:1619677447.874354:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.874355:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.874355:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677447.874362:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:12.0:1619677447.874364:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677447.874365:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:12.0:1619677447.874366:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677447.874366:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677447.874367:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.874367:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.874368:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.874369:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:26.0:1619677447.874370:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.874371:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677447.874372:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677447.874373:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.874374:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.874375:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.874376:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677447.874379:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.874380:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677447.874383:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.874384:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677447.874385:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677447.874386:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677447.874387:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.874388:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677447.874389:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677447.874390:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677447.874392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677447.874395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:5.0:1619677447.874401:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:10.0:1619677447.874403:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:5.0:1619677447.874403:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:10.0:1619677447.874404:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:5.0:1619677447.874404:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.874405:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:20.0:1619677447.874406:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677447.874406:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:5.0:1619677447.874406:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.874409:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677447.874446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.874452:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.874455:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.874455:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.874457:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.874460:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.874461:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.874463:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.874464:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.874465:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.874466:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.874469:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.874471:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.874474:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.874474:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.874475:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.874477:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.874477:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.874478:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.874478:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.874491:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.874493:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.874495:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.874495:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.874497:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.874499:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.874503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.874510:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.874513:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.874513:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.874516:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.874518:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.874520:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677447.874521:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.874522:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.874524:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.874525:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677447.874528:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.874530:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677447.874533:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.874534:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677447.874534:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677447.874536:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677447.874536:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.874537:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.874538:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677447.874549:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.874552:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.874553:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.874553:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.874555:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.874558:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677447.874822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.874830:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.874832:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:4.2:1619677447.874832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677447.874833:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.874835:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.874838:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.874839:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677447.874839:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677447.874840:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.874841:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.874842:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:10.0:1619677447.874843:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.874843:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:10.0:1619677447.874844:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.874845:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.874847:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.874848:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.874848:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.874849:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:10.0:1619677447.874850:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.874851:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96722362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.874851:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677447.874851:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677447.874852:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.874853:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.874853:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:10.0:1619677447.874853:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.874853:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.874854:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677447.874854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677447.874855:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:24.0:1619677447.874856:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.874858:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.874859:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677447.874861:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.874861:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.874862:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.874862:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:24.0:1619677447.874863:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.874863:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677447.874864:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.874865:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.874865:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677447.874865:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.874865:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:17.0:1619677447.874868:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.874868:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.874868:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.874869:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:8.2:1619677447.874869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:17.0:1619677447.874870:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96722361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.874871:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.874872:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:17.0:1619677447.874873:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.874874:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.874874:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:17.0:1619677447.874875:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.874878:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:17.0:1619677447.874879:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.874880:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.874881:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.874882:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.874883:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.874886:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.874886:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.874887:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.874887:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.874887:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.874890:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.874890:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.874892:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.874893:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.874893:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677447.874898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677447.874902:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677447.874903:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677447.874904:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.874905:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.874905:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.874906:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677447.874907:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.874907:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.874908:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.874910:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.875046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.875049:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.875051:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.875053:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.875054:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.875056:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.875058:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.875059:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.875061:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.875062:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.875063:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.875065:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.875067:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.875069:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.875070:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.875071:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.875072:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.875073:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.875074:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.875074:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.875086:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.875089:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.875089:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.875090:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.875092:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.875094:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.875195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.875202:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.875205:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.875207:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.875209:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.875210:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.875211:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.875213:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.875214:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.875215:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.875216:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.875219:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.875221:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.875223:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.875224:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.875225:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.875226:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.875226:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.875228:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.875228:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:4.2:1619677447.875237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.875238:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.875240:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.875241:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.875243:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.875244:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:12.0:1619677447.875244:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.875245:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677447.875247:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:5.2:1619677447.875247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677447.875248:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.875250:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.875252:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.875252:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677447.875254:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:17.0:1619677447.875255:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:26.0:1619677447.875256:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.875256:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677447.875258:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.875259:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.875260:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.875260:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.875263:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.875264:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.875264:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:26.0:1619677447.875266:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.875266:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.875267:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.875268:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.875269:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:26.0:1619677447.875270:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.875271:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677447.875271:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677447.875273:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677447.875273:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677447.875273:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.875274:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.875275:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677447.875275:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.875278:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.875279:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.875279:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.875281:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.875282:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.875282:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.875283:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.875287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:6.0:1619677447.875287:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.875289:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.875290:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.875291:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.875293:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.875298:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:6.0:1619677447.875298:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.875301:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.875302:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.875302:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.875303:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:8.0:1619677447.875304:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.875305:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:9.2:1619677447.875306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:6.0:1619677447.875306:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.875307:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.875308:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.875308:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.875309:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.875311:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.875313:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.875313:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677447.875314:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.875315:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.875315:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677447.875340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677447.875346:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.875349:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.875350:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.875352:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.875354:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.875356:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.875358:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.875359:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.875360:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.875361:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.875364:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.875366:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.875369:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.875370:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.875370:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.875372:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.875373:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.875373:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.875374:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:10.0:1619677447.875386:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.875389:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677447.875389:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.875390:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.875392:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.875395:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677447.875458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.875466:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.875469:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.875472:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.875474:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.875474:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.875475:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.875477:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.875478:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.875479:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.875480:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.875483:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.875485:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.2:1619677447.875486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677447.875487:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.875488:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.875489:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.875490:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.875491:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.875491:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.875492:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677447.875492:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.875495:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.875495:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.875497:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.875500:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.875501:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:12.0:1619677447.875503:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677447.875503:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.875504:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.875505:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:1.0:1619677447.875505:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677447.875506:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.875506:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677447.875509:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.875509:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.875510:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.875511:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.875512:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.875514:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.875515:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.875516:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.875517:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.875518:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.875518:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.875519:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.875523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.875531:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.875533:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.875533:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.875534:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.875535:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.875537:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677447.875567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.875573:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.875575:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.875576:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.875578:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.875581:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.875582:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.875584:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.875585:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.875586:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.875587:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.875591:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.875593:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.875595:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.875596:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.875597:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.875598:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.2:1619677447.875598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677447.875599:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.875600:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.875601:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.875602:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677447.875605:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.2:1619677447.875605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:4.2:1619677447.875606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.2:1619677447.875607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677447.875608:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.875609:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.875611:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.875613:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677447.875613:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:26.0:1619677447.875614:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.875615:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:6.0:1619677447.875615:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:5.0:1619677447.875615:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677447.875616:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:26.0:1619677447.875617:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.875617:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.875617:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677447.875617:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.875617:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:26.0:1619677447.875618:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.875618:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.875620:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.875620:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.875620:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:26.0:1619677447.875621:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.875621:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.875623:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:24.0:1619677447.875624:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.875625:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.875626:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:26.0:1619677447.875627:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.875627:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96722351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.875629:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677447.875629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:26.0:1619677447.875630:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.875630:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.875631:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:24.0:1619677447.875631:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677447.875632:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677447.875633:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677447.875634:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.875634:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:24.0:1619677447.875634:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.875635:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677447.875635:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.875635:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:24.0:1619677447.875638:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.875638:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:24.0:1619677447.875639:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.875639:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:24.0:1619677447.875640:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677447.875641:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:24.0:1619677447.875642:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.875642:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.875643:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:2.0:1619677447.875644:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.875645:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.875645:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677447.875645:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:2.0:1619677447.875645:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.875646:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96722350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.875648:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.875648:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.875648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677447.875648:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.875649:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.875650:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:13.0:1619677447.875651:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.875651:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:4.2:1619677447.875651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:2.0:1619677447.875653:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:20.2:1619677447.875654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:13.0:1619677447.875654:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.875656:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677447.875656:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677447.875657:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.875658:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.875658:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.875659:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.875659:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.875660:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.875660:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.875660:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:1.0:1619677447.875660:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.875661:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:2.0:1619677447.875661:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:31.0:1619677447.875662:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677447.875662:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.875662:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677447.875662:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:1.0:1619677447.875662:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.875663:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:31.0:1619677447.875665:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:12.0:1619677447.875665:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.875665:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:12.0:1619677447.875666:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:9.2:1619677447.875666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677447.875667:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.875667:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.875667:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.875669:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:12.0:1619677447.875669:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677447.875670:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.875671:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.875672:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.875673:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:31.0:1619677447.875674:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.875675:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.875676:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.875676:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.875677:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.875677:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:31.0:1619677447.875680:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.875680:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.875682:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677447.875683:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.875685:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.875686:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.875686:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.875688:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.875689:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.875690:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.875692:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.875692:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677447.875698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677447.875700:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.875702:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.875703:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.875705:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.875706:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:12.0:1619677447.875706:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.875707:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.875708:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.875709:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.875710:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:4.2:1619677447.875899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.875906:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.875908:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.875909:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.2:1619677447.875910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677447.875911:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.875913:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.875915:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:17.0:1619677447.875916:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677447.875916:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.875917:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.875918:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:10.0:1619677447.875919:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.875919:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677447.875920:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677447.875922:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.875923:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.875924:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.875925:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677447.875926:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.875926:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:17.0:1619677447.875927:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.875928:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677447.875929:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677447.875930:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:10.0:1619677447.875930:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.875931:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.875932:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:17.0:1619677447.875933:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.875933:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.875934:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:17.0:1619677447.875935:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:10.0:1619677447.875935:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.875936:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677447.875938:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.875939:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677447.875939:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677447.875941:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.875941:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.875942:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:4.2:1619677447.875942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677447.875943:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.875946:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:9.2:1619677447.875947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:26.0:1619677447.875949:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677447.875949:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.875949:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.875950:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.875951:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.875952:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677447.875952:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.875952:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:24.0:1619677447.875953:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:22.0:1619677447.875955:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677447.875956:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.875957:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.875957:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.875959:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.875962:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.875964:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.875967:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.875967:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.875968:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.875969:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.875970:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.876136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.876143:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.876146:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.876147:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.876149:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.876152:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.876153:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677447.876155:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.876156:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.876157:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.876158:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677447.876161:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.876163:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:21.2:1619677447.876164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:26.0:1619677447.876165:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.876166:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.876166:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.876168:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.876168:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.876169:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:2.0:1619677447.876169:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677447.876170:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677447.876171:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:2.0:1619677447.876172:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.876173:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.876175:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677447.876176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:2.0:1619677447.876177:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.876179:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.876180:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.876181:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:2.0:1619677447.876181:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.876182:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677447.876183:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.876183:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.876183:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:26.0:1619677447.876185:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677447.876186:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:10.0:1619677447.876186:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.876187:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.876187:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:2.0:1619677447.876187:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.876188:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677447.876188:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677447.876189:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.876191:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.876192:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.876192:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.876194:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.876194:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.876195:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.876196:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.876200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.876201:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.876204:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.876205:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.876205:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.876206:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.876207:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:18.0:1619677447.876207:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.876209:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.876210:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.876211:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.876212:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.876214:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.876217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:9.2:1619677447.876217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:30.0:1619677447.876224:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:4.2:1619677447.876225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.876225:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677447.876228:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:18.0:1619677447.876228:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.876228:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.876230:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677447.876231:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.876231:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.876231:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.876231:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677447.876232:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677447.876233:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.876235:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:10.0:1619677447.876235:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:30.0:1619677447.876236:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.876236:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:30.0:1619677447.876238:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.876238:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.876239:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.876240:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677447.876240:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.876242:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.876243:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.876244:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677447.876245:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.876246:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:10.0:1619677447.876246:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.876246:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.876249:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.876250:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:10.0:1619677447.876250:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.876251:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.876252:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.876252:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677447.876252:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.876254:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.876254:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:10.0:1619677447.876255:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.876256:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.876256:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.876258:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.876258:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.876259:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.876260:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:12.0:1619677447.876265:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.876267:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.876268:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.876269:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.876271:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:13.0:1619677447.876271:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.876272:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.876275:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.876275:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.876276:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.876278:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.876281:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.876283:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.876286:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.876287:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.876288:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.876288:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.876290:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677447.876307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.876314:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.876317:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.876319:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.876321:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.876322:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.876323:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677447.876324:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.876325:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.876326:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.876327:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677447.876330:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.876332:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677447.876334:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.876335:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677447.876336:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677447.876337:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.876338:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.876339:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.876339:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.876352:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.876355:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.876355:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.876356:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.876358:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.876361:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.876456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.876464:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.876467:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.876468:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.876469:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.876472:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.876474:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.876475:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.876476:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.876477:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.876478:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.876481:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.876482:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.876485:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.876486:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.876486:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.876488:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.876489:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.876489:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.876490:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.876495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.876504:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.876506:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.876507:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.876507:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.876509:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.876512:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.876600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.876607:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.876610:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.876611:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.876612:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.876615:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.876617:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677447.876618:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.876620:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.876620:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.876621:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677447.876624:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.876626:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677447.876629:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.876629:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677447.876630:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.876632:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.876632:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.876633:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.876634:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677447.876638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.876647:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.876649:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.876650:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.876651:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.876653:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.876656:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.876684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.876687:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.876689:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.876691:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.876693:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.876694:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.876695:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677447.876697:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.876698:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.876699:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.876700:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677447.876704:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.876706:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677447.876708:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.876709:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677447.876710:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677447.876712:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677447.876712:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.876714:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.876715:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.876715:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.2:1619677447.876718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:20.2:1619677447.876721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:10.0:1619677447.876724:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:24.0:1619677447.876726:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:26.0:1619677447.876727:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677447.876727:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:5.0:1619677447.876727:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:24.0:1619677447.876728:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:10.0:1619677447.876728:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.876729:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677447.876729:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:10.0:1619677447.876730:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.876730:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:24.0:1619677447.876731:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.876731:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:10.0:1619677447.876733:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:24.0:1619677447.876734:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.876735:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.876737:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.876738:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.876739:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.876740:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.876743:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.876745:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.876747:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.876748:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.876749:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.876750:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.876751:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.876752:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.876753:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.876754:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:21.2:1619677447.876759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.876766:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:12.0:1619677447.876768:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677447.876768:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.876768:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.876770:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677447.876770:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:12.0:1619677447.876771:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.876772:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.876774:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.876775:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.876873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.876880:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.876883:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.876885:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.876887:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.876888:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.876889:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677447.876890:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.876892:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.876892:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.876893:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677447.876897:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.876898:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677447.876901:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.876901:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677447.876902:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677447.876904:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.876904:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.876905:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.876906:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677447.876915:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.876917:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:5.2:1619677447.876917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:26.0:1619677447.876918:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.876918:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677447.876920:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.876922:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:2.0:1619677447.876922:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.876925:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.876926:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.876928:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.876930:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.876931:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.876932:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.876934:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.876935:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.876936:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.876938:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.876940:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.876942:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.876942:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.876943:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.876945:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.876945:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.876946:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.876946:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.876951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.876957:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.876959:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.876959:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.876960:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.876961:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.876962:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677447.876995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.877000:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.877003:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.877004:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.877006:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.877008:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.877009:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.877010:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.877012:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.877013:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.877014:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.877017:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.877019:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.877021:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.877022:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.877022:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.877024:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.877025:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.877025:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.877026:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.877030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.877038:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.877040:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.877041:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.877041:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.877043:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.877046:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.877072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.877080:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.877082:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.877083:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.877085:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.877087:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.877089:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677447.877090:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.877092:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.877093:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.877094:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677447.877097:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.877099:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.877102:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.877102:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677447.877103:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.877105:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.877106:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.877106:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.877107:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677447.877111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.877118:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.877121:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.877121:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.877122:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.877124:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.877127:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.877221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.877226:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.877229:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.877230:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.877231:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.877234:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.877235:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.877237:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.877238:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.877239:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.877240:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.877243:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.877245:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.877247:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.877248:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.877248:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.877250:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.877250:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.877252:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.877253:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.877253:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677447.877256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677447.877259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.877265:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.877265:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.877266:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.877266:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.877266:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.877267:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.877268:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.877268:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.877269:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.877270:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677447.877317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677447.877322:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.877325:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.877326:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.877328:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.877329:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.877331:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.877332:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.877334:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.877335:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677447.877336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000400:00000200:2.0:1619677447.877336:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.877339:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.877341:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.877343:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.877344:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:2.0:1619677447.877344:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.877345:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677447.877347:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:2.0:1619677447.877347:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.877347:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677447.877348:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:2.0:1619677447.877348:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.877348:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:26.0:1619677447.877350:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.877353:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.877353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:26.0:1619677447.877354:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677447.877356:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.877358:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.877359:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.877360:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:5.0:1619677447.877363:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:26.0:1619677447.877364:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.877365:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677447.877365:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.877366:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.877368:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.877368:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:5.0:1619677447.877368:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.877369:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677447.877370:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677447.877370:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.877371:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (23)++
00000800:00000200:5.0:1619677447.877371:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677447.877372:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677447.877372:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:20.0:1619677447.877372:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.877372:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677447.877373:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:20.0:1619677447.877373:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:26.0:1619677447.877374:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:31.0:1619677447.877386:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.877388:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.877389:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.877391:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.877393:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.877394:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677447.877429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.877437:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.877439:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.877441:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.877443:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.877444:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.877445:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.877446:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.877447:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677447.877448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:30.0:1619677447.877449:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.877449:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.877452:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.877454:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.877454:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677447.877456:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.877457:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.877457:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:30.0:1619677447.877458:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.877459:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.877459:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677447.877459:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.877459:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.877461:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.877462:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:5.0:1619677447.877462:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.877463:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:5.0:1619677447.877463:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.877464:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.877466:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.877467:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.877468:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:21.2:1619677447.877469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:5.0:1619677447.877473:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.877475:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.877477:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:5.0:1619677447.877478:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.877479:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:5.0:1619677447.877479:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677447.877480:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.0:1619677447.877480:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677447.877481:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677447.877481:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.877482:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.877483:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.877484:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.877492:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677447.877495:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:12.0:1619677447.877495:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.877495:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.877497:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.877498:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.877499:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.877499:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:31.0:1619677447.877500:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.877500:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.877502:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.877503:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.877637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.877644:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.2:1619677447.877646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:22.0:1619677447.877647:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.877647:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.877650:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.877651:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.877652:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:6.0:1619677447.877653:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.877654:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.877655:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:6.0:1619677447.877656:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.877656:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:6.0:1619677447.877657:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.877658:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677447.877659:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.877661:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.877662:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.877663:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.877664:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.877664:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:18.0:1619677447.877665:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.877666:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.877667:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677447.877667:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.877668:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.877669:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.877670:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000400:00000200:18.0:1619677447.877671:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.877671:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.877671:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.877672:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677447.877673:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.877675:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.877676:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.877676:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.877678:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.877678:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.877679:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.877680:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:10.0:1619677447.877682:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:9.2:1619677447.877684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.877685:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677447.877686:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677447.877686:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.877689:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.877691:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677447.877692:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:2.0:1619677447.877693:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.877694:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.877695:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.877696:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.2:1619677447.877697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:2.0:1619677447.877698:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.877702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677447.877705:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.877708:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:18.0:1619677447.877709:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.877709:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677447.877710:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:4.2:1619677447.877710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.877712:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677447.877713:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.877713:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.877713:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.877713:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.877714:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:26.0:1619677447.877715:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677447.877717:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722326] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.877718:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677447.877718:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:26.0:1619677447.877719:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.877720:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:10.0:1619677447.877720:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677447.877723:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.877724:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000400:00000200:10.0:1619677447.877725:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.877726:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:10.0:1619677447.877727:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.877728:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.877729:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.877730:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.877731:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.877734:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677447.877734:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.877735:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677447.877736:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677447.877736:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.877738:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677447.877739:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.877739:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.877740:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677447.877740:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.877741:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:10.0:1619677447.877741:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677447.877742:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.877743:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.877743:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677447.877744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677447.877745:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677447.877746:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.877746:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677447.877752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677447.877753:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:22.0:1619677447.877756:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.877757:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.877758:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.877759:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:20.0:1619677447.877759:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:26.0:1619677447.877760:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:22.0:1619677447.877760:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.877760:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.877760:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677447.877762:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677447.877762:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.877763:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677447.877763:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:10.0:1619677447.877764:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677447.877766:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.877778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.877786:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.877789:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.877791:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.877793:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.877793:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.877795:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677447.877796:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.877797:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677447.877797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677447.877798:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.877799:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677447.877802:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.877803:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677447.877804:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.877806:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.877806:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:30.0:1619677447.877807:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.877807:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:1.0:1619677447.877807:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677447.877809:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.877809:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677447.877809:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.877810:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.877811:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:17.0:1619677447.877811:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.877812:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.877813:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.877815:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.877816:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.877817:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.877820:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.877821:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:17.0:1619677447.877822:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.877823:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.877824:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.877825:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.877825:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:31.0:1619677447.877826:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.877826:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.877827:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.877828:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:17.0:1619677447.877828:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.877829:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.877829:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.877830:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.877841:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.877844:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.877844:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.877846:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.877847:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.877849:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677447.877859:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677447.877861:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.877862:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.877863:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.877864:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.877865:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.877985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.877992:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.877995:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.877996:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.877998:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.878001:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.878002:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677447.878004:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.878005:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.878006:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.878007:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677447.878010:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.878012:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677447.878015:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.878015:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677447.878016:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677447.878018:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677447.878018:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.878019:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.878019:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677447.878032:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.878034:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.878035:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.878036:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.878038:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.878040:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677447.878074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.878079:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.878082:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.878083:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.878085:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.878086:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.878088:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.878089:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.878090:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.878091:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677447.878092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:2.0:1619677447.878092:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.878096:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.878098:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.878099:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.878100:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:18.0:1619677447.878102:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.878102:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677447.878103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677447.878103:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:18.0:1619677447.878104:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.878104:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:18.0:1619677447.878106:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:2.0:1619677447.878106:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:18.0:1619677447.878107:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.878108:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.878109:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677447.878109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:30.0:1619677447.878110:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677447.878110:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.878111:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.878112:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.878112:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:30.0:1619677447.878113:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.2:1619677447.878113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.878113:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:31.0:1619677447.878116:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.878116:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.878116:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.878116:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:18.0:1619677447.878118:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:30.0:1619677447.878119:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.878119:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677447.878120:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:18.0:1619677447.878120:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.878120:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:18.0:1619677447.878121:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.878121:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.878121:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:1.0:1619677447.878121:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (21)++
00000400:00000200:30.0:1619677447.878122:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96722319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.878122:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.878122:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:30.0:1619677447.878123:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.878123:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:30.0:1619677447.878124:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.878124:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677447.878124:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:30.0:1619677447.878125:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.878125:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000400:00000200:5.0:1619677447.878125:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.878126:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:20.0:1619677447.878127:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.878127:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:5.0:1619677447.878127:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.878129:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677447.878129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677447.878131:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:21.2:1619677447.878132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677447.878132:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677447.878134:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.878134:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.878134:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:30.0:1619677447.878135:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.878135:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.878136:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.878136:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.878137:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.878137:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:5.0:1619677447.878137:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.878137:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:30.0:1619677447.878138:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.878139:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677447.878139:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:2.0:1619677447.878139:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.878140:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.878141:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677447.878149:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.878151:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.878152:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.878154:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.878155:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.878156:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.878263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.878266:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.878269:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.878271:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.878272:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.878274:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.878275:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.878276:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.878278:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.878279:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.878280:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.878283:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.878285:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.878288:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.878289:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.878289:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.878291:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.878292:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.878292:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.878298:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:13.0:1619677447.878305:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.878308:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.878309:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.878310:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677447.878312:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.878314:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.878321:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.878324:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:21.2:1619677447.878325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:12.0:1619677447.878325:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.878325:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.878326:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.878327:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.878331:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.878333:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.878334:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.878337:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.878339:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.878340:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.878342:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.878343:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.878344:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.878345:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.878348:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.878350:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.878354:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.878354:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.878355:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.878357:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.878357:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.878358:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.878358:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677447.878370:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677447.878372:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677447.878373:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:5.2:1619677447.878373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677447.878374:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.878376:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.878378:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:17.0:1619677447.878379:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.878382:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.878383:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.878385:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.878388:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.878389:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.878391:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.878392:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.878393:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.878394:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.878397:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.878399:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.878401:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.878402:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.878403:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.878405:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.878405:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.878405:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.878406:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.878418:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.878421:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.878422:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.878422:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.878424:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.878426:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.878495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.2:1619677447.878496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677447.878502:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:5.0:1619677447.878502:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677447.878505:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:5.0:1619677447.878505:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:6.0:1619677447.878506:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:20.0:1619677447.878507:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:22.0:1619677447.878509:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.878509:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.878512:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.878512:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.878514:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677447.878514:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:4.2:1619677447.878514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:22.0:1619677447.878516:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.878516:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.878518:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.878518:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.878519:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.878519:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.878520:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677447.878521:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.878522:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.878525:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.878525:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677447.878526:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.878527:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.878528:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.878529:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.878530:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.878531:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.878533:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:22.0:1619677447.878533:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.878534:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96722313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.878534:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:22.0:1619677447.878534:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677447.878534:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.878535:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:24.0:1619677447.878536:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.878536:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677447.878537:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.878537:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:22.0:1619677447.878537:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000400:00000200:24.0:1619677447.878538:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:22.0:1619677447.878538:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:5.0:1619677447.878538:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.878538:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677447.878539:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:5.0:1619677447.878539:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.878540:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:24.0:1619677447.878541:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.878543:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.878546:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.878547:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.878547:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677447.878548:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:24.0:1619677447.878549:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.878549:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.878550:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677447.878551:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:24.0:1619677447.878551:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:26.0:1619677447.878552:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.878552:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677447.878554:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.878555:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.2:1619677447.878555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677447.878557:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:12.0:1619677447.878557:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.878558:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.878560:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.878562:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.878562:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:12.0:1619677447.878563:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.878565:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.878566:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.878566:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.878568:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:20.2:1619677447.878570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677447.878571:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:31.0:1619677447.878578:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.878581:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.878584:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.878586:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.878586:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.878587:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.878589:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.878590:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.878591:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.878592:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.878596:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.878597:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.878601:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.878602:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.878602:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.878604:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.878605:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.878606:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.878607:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.878611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.878618:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.878621:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.878622:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.878623:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.878624:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.878627:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677447.878628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.2:1619677447.878636:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.0:1619677447.878640:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.878643:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:10.0:1619677447.878644:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677447.878644:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.878645:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.878645:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.878646:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:10.0:1619677447.878646:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.878647:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.878649:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.878652:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.878653:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677447.878654:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.878656:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.878657:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.878658:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677447.878661:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.878663:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.878665:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.878666:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677447.878667:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.878669:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.878669:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.878670:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.878671:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677447.878675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.878684:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.878687:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.878688:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.878688:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.878690:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.878694:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.878856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.878861:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677447.878862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:17.0:1619677447.878864:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.878865:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.878866:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.878868:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:17.0:1619677447.878869:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.878871:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677447.878871:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:2.0:1619677447.878871:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:17.0:1619677447.878872:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.878874:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.878874:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.878875:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.878876:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677447.878876:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.878878:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.878879:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722309] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.878880:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.878881:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.878882:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.878883:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:2.0:1619677447.878883:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.878886:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.878887:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.878887:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:2.0:1619677447.878887:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.878889:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:2.0:1619677447.878889:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.878890:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.878891:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.878891:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677447.878892:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.878892:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.878893:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.878895:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.878895:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.878896:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.878897:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.878898:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677447.878901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677447.878902:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.2:1619677447.878903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:30.0:1619677447.878905:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.878906:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.878907:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.878908:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.878908:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677447.878909:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:21.2:1619677447.878910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:1.0:1619677447.878910:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:30.0:1619677447.878911:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677447.878911:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.878912:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677447.878912:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.878912:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.878913:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:1.0:1619677447.878914:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.878917:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:1.0:1619677447.878917:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.0:1619677447.878920:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.878921:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.878922:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.878925:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.878927:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.878928:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.878929:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.878930:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.878931:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.878935:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.878937:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.878940:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.878941:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.878941:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.878943:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.878944:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.878945:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.878945:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:21.2:1619677447.878948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.2:1619677447.878949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677447.878956:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.878958:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.0:1619677447.878958:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:31.0:1619677447.878960:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.878960:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.878961:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.878962:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.878963:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.878963:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.878963:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.878966:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:5.0:1619677447.878966:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:31.0:1619677447.878967:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.878968:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.878969:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.878970:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677447.878973:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.878975:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.878978:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.878978:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677447.878979:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.878981:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677447.878981:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.878982:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.878983:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677447.878987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.878994:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.878997:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.878998:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.878998:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.879000:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.879003:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.879005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.879012:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.879015:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.879015:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.879016:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.879017:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.879017:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.879168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:4.2:1619677447.879172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.879178:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.879180:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:21.2:1619677447.879180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:10.0:1619677447.879182:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:26.0:1619677447.879183:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.879183:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677447.879185:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.879185:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:26.0:1619677447.879186:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.879188:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.879188:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:26.0:1619677447.879189:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:22.0:1619677447.879189:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:26.0:1619677447.879191:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.879192:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.879193:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.879195:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.879195:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:6.0:1619677447.879195:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.879196:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:10.0:1619677447.879198:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:6.0:1619677447.879198:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.879200:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.879200:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.879201:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.879201:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.879202:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.879202:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:10.0:1619677447.879203:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.879204:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:10.0:1619677447.879205:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:6.0:1619677447.879206:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.879209:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677447.879210:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677447.879210:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.879211:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677447.879211:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677447.879213:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677447.879213:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677447.879214:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677447.879214:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.879214:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:26.0:1619677447.879215:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:6.0:1619677447.879215:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677447.879216:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.879216:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.879217:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677447.879217:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:6.0:1619677447.879217:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.879217:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677447.879218:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.879218:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.879219:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.879220:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.879220:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:10.0:1619677447.879221:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677447.879221:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.879222:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.879223:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677447.879223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677447.879225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677447.879226:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.879228:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.879229:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.879230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:13.0:1619677447.879231:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.879232:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:12.0:1619677447.879232:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677447.879233:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:13.0:1619677447.879233:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.879234:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:21.2:1619677447.879234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:22.0:1619677447.879235:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677447.879235:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.879236:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.879236:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677447.879236:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677447.879237:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:17.0:1619677447.879238:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.879238:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:22.0:1619677447.879239:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:18.0:1619677447.879240:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:6.0:1619677447.879240:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.879241:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.879241:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.879241:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.879242:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:22.0:1619677447.879243:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.879243:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.879243:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677447.879244:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.879244:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.879245:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:22.0:1619677447.879246:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:1.0:1619677447.879246:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677447.879248:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.879248:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.879249:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677447.879250:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.879251:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.879251:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.879252:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.879252:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677447.879254:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.879255:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677447.879257:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.879259:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677447.879259:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677447.879262:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677447.879262:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.879263:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.879263:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677447.879264:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:1.0:1619677447.879264:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677447.879265:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.879266:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.879266:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.879267:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.879267:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.879268:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677447.879268:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:18.0:1619677447.879269:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677447.879269:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.879270:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.879273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677447.879274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.879280:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677447.879280:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677447.879280:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:17.0:1619677447.879282:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.879283:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.879283:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677447.879283:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:17.0:1619677447.879284:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:2.0:1619677447.879284:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.879285:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:6.0:1619677447.879285:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.879286:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:2.0:1619677447.879286:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000400:00000200:6.0:1619677447.879288:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.879289:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.879290:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.879302:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.879305:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.879306:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.879306:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.879307:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.879308:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677447.879310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1200] (20)++
00000800:00000200:1.0:1619677447.879316:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:1.0:1619677447.879319:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:17.0:1619677447.879320:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:1.0:1619677447.879322:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.879324:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.879326:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:1.0:1619677447.879327:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.879329:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.879329:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.879331:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:1.0:1619677447.879334:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.879336:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:1.0:1619677447.879338:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.879339:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:1.0:1619677447.879340:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:1.0:1619677447.879341:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:1.0:1619677447.879342:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:1.0:1619677447.879343:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdbb1200] (22)++
00000800:00000200:1.0:1619677447.879344:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (23)--
00000800:00000200:1.0:1619677447.879345:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:8.2:1619677447.879348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.2:1619677447.879350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677447.879355:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)++
00000800:00000200:20.0:1619677447.879356:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:17.0:1619677447.879357:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdbb1200] (23)--
00000800:00000200:17.0:1619677447.879357:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:20.0:1619677447.879358:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:17.0:1619677447.879358:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000400:00000200:20.0:1619677447.879359:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.879361:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.879362:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.879363:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:8.2:1619677447.879369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.879377:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.879379:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.879380:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.879381:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677447.879382:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.879383:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:21.2:1619677447.879501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677447.879509:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677447.879512:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677447.879513:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677447.879515:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.879518:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.879519:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677447.879521:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.879522:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.879523:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.879524:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677447.879527:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.879529:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677447.879532:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.879533:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677447.879533:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677447.879535:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677447.879535:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677447.879536:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677447.879537:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:12.0:1619677447.879548:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.879551:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.879551:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.879553:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.879555:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.879556:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.879579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.879586:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.879588:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.879589:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.879590:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.879593:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.879595:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677447.879596:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.879597:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.879598:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.879599:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677447.879602:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.879603:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677447.879605:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.879606:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.879606:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677447.879607:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.879608:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.879608:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.879609:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677447.879621:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.879623:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.879624:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.879625:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.879628:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.879629:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.879634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.2:1619677447.879638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677447.879640:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.879642:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.879643:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.879645:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.879646:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:1.0:1619677447.879647:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.879649:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:1.0:1619677447.879649:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.879650:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.879652:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.879653:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:1.0:1619677447.879653:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.879654:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.879655:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.879655:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677447.879656:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.879658:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.879659:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.879660:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.879661:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.879661:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:30.0:1619677447.879662:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.879664:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.879665:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:30.0:1619677447.879666:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.879666:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.879667:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.879668:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.879668:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.879668:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.879669:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.879671:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.879671:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.879672:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.879674:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.879674:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.879676:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.879677:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.879678:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:6.0:1619677447.879681:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:20.2:1619677447.879683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:6.0:1619677447.879684:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.879685:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:12.0:1619677447.879685:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:6.0:1619677447.879685:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.879687:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.879688:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.879689:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.879689:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677447.879691:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:12.0:1619677447.879691:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.879692:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.879693:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677447.879694:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.879695:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.879696:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:4.2:1619677447.879799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.879806:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.879810:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.879810:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.879813:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.879815:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.879817:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.879818:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.879819:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.879820:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.879821:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.879825:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.879826:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.879829:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.879830:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.879831:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.879832:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.879833:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.879833:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.879834:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.879838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.879845:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.879848:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.879849:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.879849:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.879851:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.879854:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.879876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.879882:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.879884:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.879885:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.879887:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.879889:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.879890:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.879891:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.879892:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.879893:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.879894:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.879897:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.879898:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.879900:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.879901:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.879902:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.879903:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.879903:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.879904:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.879904:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.879909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.879916:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.879918:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.879918:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.879919:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.879920:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.879921:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.879972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.879979:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.879982:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.879982:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.879984:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.879987:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.879988:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.879989:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.879990:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.879991:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.879992:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.879995:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.879997:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.879999:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.880000:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.880001:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.880002:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.880003:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.880003:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.880004:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677447.880015:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.880017:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.880018:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.880019:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.880022:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.880022:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:21.2:1619677447.880037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.880044:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.880047:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.880049:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.880051:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.880051:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.880052:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677447.880053:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.880055:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.880055:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.880056:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677447.880059:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.880061:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677447.880064:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.880065:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677447.880065:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677447.880067:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677447.880067:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.880068:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.880069:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677447.880073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.880082:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.880085:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.880085:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.880086:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.880088:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.880091:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677447.880143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677447.880148:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.880151:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.880152:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.880154:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.880156:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.880157:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.880158:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.880160:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.880161:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.880162:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.880165:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.880166:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.880169:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.880170:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.880170:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.880172:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.880173:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.880173:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.880174:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677447.880178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.880187:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.880189:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.880190:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.880191:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.880192:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.880194:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677447.880197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.880204:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.880206:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.880207:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.880208:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.880209:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.880210:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.880269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.880277:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.2:1619677447.880277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677447.880280:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.880282:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.880282:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677447.880283:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.880285:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.880285:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.880286:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677447.880287:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677447.880288:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.880289:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.880290:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.880290:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.880292:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.880292:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:31.0:1619677447.880298:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677447.880299:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.880301:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.880301:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.880302:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.880303:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:31.0:1619677447.880304:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:1.0:1619677447.880306:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.880308:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.880308:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677447.880309:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677447.880310:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677447.880310:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.880311:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.880311:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677447.880312:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677447.880312:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:1.0:1619677447.880312:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677447.880313:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:1.0:1619677447.880313:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677447.880314:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:1.0:1619677447.880314:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.880314:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.880318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.2:1619677447.880322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.880326:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677447.880329:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:5.0:1619677447.880329:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.880330:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.880331:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:5.0:1619677447.880331:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.880332:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.880333:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:4.2:1619677447.880333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.880333:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.880334:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.880334:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.880336:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:10.0:1619677447.880341:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.880344:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.880345:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.880347:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.880349:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.880351:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677447.880352:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.880353:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.880354:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.880355:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677447.880359:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.880360:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677447.880362:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.880363:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.880364:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677447.880366:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.880366:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.880367:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677447.880368:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.880368:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677447.880374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677447.880379:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:26.0:1619677447.880381:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:30.0:1619677447.880382:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.880383:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.880383:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.880383:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:30.0:1619677447.880384:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.880384:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677447.880386:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.880387:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677447.880395:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677447.880396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677447.880397:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.880398:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.880399:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.880399:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.880399:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677447.880400:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.0:1619677447.880401:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.880404:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.880405:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.880407:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.880408:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.880410:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.880411:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.880412:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.880413:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.880416:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677447.880416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677447.880418:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.880421:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.880421:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.880422:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.880423:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.880424:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677447.880424:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677447.880426:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:26.0:1619677447.880427:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677447.880427:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.880427:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677447.880427:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677447.880430:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677447.880432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:10.0:1619677447.880432:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.880433:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677447.880435:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.880436:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.880437:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:10.0:1619677447.880437:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.880438:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.880438:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:22.0:1619677447.880440:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677447.880440:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:22.0:1619677447.880441:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.880441:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.880441:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677447.880442:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.880442:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:22.0:1619677447.880444:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.880444:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.880446:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.880447:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677447.880447:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677447.880447:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677447.880449:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.880449:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.880450:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677447.880451:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677447.880452:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677447.880455:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.880457:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.880458:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.880458:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.2:1619677447.880458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677447.880459:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.880460:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:26.0:1619677447.880463:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.880464:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.880465:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.880467:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.880467:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677447.880468:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677447.880469:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.880469:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.880470:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677447.880470:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677447.880479:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.880482:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.880482:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.880482:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.880483:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.880484:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677447.880674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.880682:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.880684:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.880686:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.880688:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.880689:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.880690:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677447.880692:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.880693:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.880694:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.880695:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677447.880698:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.880700:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677447.880702:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.880703:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677447.880704:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677447.880705:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677447.880706:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.880707:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.880707:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.880711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.880719:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.880722:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.880723:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.880724:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.880725:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.880727:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.880766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.880770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677447.880773:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.2:1619677447.880774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:26.0:1619677447.880775:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.880776:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677447.880777:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677447.880778:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.880779:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.880779:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677447.880779:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.880780:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:26.0:1619677447.880781:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:22.0:1619677447.880782:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.880782:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:26.0:1619677447.880783:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.880784:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.880785:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.880785:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.880786:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.880787:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:8.0:1619677447.880787:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.880788:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.880788:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.880789:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:8.0:1619677447.880790:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96722284] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.880791:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722284] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.880792:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.880793:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.880793:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.880794:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.880794:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.880796:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.880796:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:8.0:1619677447.880796:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677447.880799:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.880800:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.880801:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.880802:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:22.0:1619677447.880802:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.880802:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.880803:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.880803:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.880804:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677447.880805:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.0:1619677447.880805:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.880808:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.880808:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.880809:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:8.0:1619677447.880809:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.880810:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.0:1619677447.880810:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.880811:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.880812:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677447.880812:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.880813:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677447.880814:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677447.880814:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.880814:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.880814:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.880815:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:30.0:1619677447.880817:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:8.2:1619677447.880817:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677447.880818:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.880820:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.880821:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.880822:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.2:1619677447.880822:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.880822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (23)++
00000800:00000200:8.0:1619677447.880826:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (23)--
00000800:00000200:20.0:1619677447.880827:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:8.0:1619677447.880828:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:20.0:1619677447.880829:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:1.0:1619677447.880829:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:20.0:1619677447.880830:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.880830:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.880830:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.880831:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:20.0:1619677447.880832:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.880832:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.880833:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.880833:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:1.0:1619677447.880833:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.880834:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:1.0:1619677447.880836:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.880839:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.880926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.880931:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.880934:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.880935:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.880937:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.880939:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.880940:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677447.880942:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.880943:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.880944:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.880945:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677447.880949:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.880951:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.2:1619677447.880952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:17.0:1619677447.880954:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.880954:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677447.880955:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677447.880957:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.880957:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.880958:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677447.880959:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.880960:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.880960:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677447.880961:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.880962:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677447.880963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:18.0:1619677447.880964:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677447.880965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677447.880966:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.880968:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.880969:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.880969:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:18.0:1619677447.880970:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.880971:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:18.0:1619677447.880972:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.880972:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:8.0:1619677447.880972:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.880973:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.880973:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.880974:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.880974:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.880975:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677447.880976:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.880977:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.880979:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.0:1619677447.880979:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:18.0:1619677447.880982:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.880983:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.880983:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.880985:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.880986:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.880986:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.880987:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:10.0:1619677447.880998:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.881001:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.881002:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.881002:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.881004:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.881007:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677447.881161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.881168:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.881170:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.881173:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.881174:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.881175:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.881177:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.881178:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.881179:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.881180:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.881181:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.881184:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.881186:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.881189:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.881190:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.881190:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.881192:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.881192:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.881194:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.881195:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.881196:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.2:1619677447.881198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.2:1619677447.881202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677447.881206:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677447.881206:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.881208:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:26.0:1619677447.881209:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:13.0:1619677447.881209:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.881210:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:10.0:1619677447.881210:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:13.0:1619677447.881211:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.881211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677447.881212:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:26.0:1619677447.881212:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.881213:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677447.881213:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.881214:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.881214:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677447.881216:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.881217:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677447.881218:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.881219:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:26.0:1619677447.881220:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677447.881220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:10.0:1619677447.881220:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677447.881221:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.881222:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677447.881224:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.881225:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:26.0:1619677447.881226:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677447.881226:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.881227:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677447.881228:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:5.0:1619677447.881228:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:10.0:1619677447.881230:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.881231:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.881231:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.881232:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.881233:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.881234:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:5.0:1619677447.881234:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.881235:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:26.0:1619677447.881237:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677447.881237:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96722277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.881238:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677447.881239:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677447.881239:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.881240:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.881240:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.881241:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677447.881241:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:26.0:1619677447.881242:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.881242:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.881243:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677447.881243:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677447.881245:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:5.0:1619677447.881245:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.881246:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.881247:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:5.0:1619677447.881247:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.881248:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677447.881250:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677447.881250:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.881250:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677447.881250:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.881250:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677447.881251:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677447.881251:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.881253:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.881253:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.881254:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.881254:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:12.0:1619677447.881255:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677447.881255:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:12.0:1619677447.881256:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.881258:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.881259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.881260:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.881260:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.881264:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.881267:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.881267:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677447.881267:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:6.0:1619677447.881268:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.881269:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:6.0:1619677447.881270:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.881271:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.881271:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.881273:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:2.0:1619677447.881273:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.881276:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.881276:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.881280:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.881280:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.881281:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.881282:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.881283:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.881460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677447.881466:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.881469:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.881469:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.881471:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.881473:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.881475:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677447.881476:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722276] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.881478:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.881479:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.881481:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677447.881484:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.881486:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677447.881488:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.881489:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677447.881490:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677447.881491:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677447.881492:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.881493:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.881494:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.881494:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.2:1619677447.881501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677447.881505:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677447.881506:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:22.0:1619677447.881508:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677447.881508:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:22.0:1619677447.881509:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.881509:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677447.881509:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:22.0:1619677447.881511:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.881511:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.881514:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.881592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677447.881600:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.881603:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677447.881603:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677447.881605:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677447.881608:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.881610:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677447.881611:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.881612:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.881613:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.881614:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677447.881618:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677447.881620:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677447.881622:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.881623:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677447.881624:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.881626:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.881626:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.881627:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677447.881628:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677447.881632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.881640:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.881643:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.881643:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.881644:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.881646:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.881649:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.881651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.881658:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.881660:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.881661:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.881661:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.881662:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.881663:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.881715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.881717:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.881720:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.881722:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.881723:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.881724:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.881726:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.881727:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.881728:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.881730:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.881730:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.881734:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.881736:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.881739:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.881739:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.881740:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.881742:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.881742:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.881743:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.881743:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.881755:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.2:1619677447.881757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:6.0:1619677447.881758:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.881759:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.881759:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.881761:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.881763:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677447.881763:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677447.881766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677447.881766:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.881766:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.881769:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.881771:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.881772:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:30.0:1619677447.881774:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677447.881774:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.881774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:18.0:1619677447.881775:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.881776:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.881777:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:18.0:1619677447.881777:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.881780:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.881781:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677447.881781:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.881781:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677447.881782:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.881783:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:30.0:1619677447.881784:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:26.0:1619677447.881784:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677447.881784:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.881786:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.881787:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.881787:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.881787:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:10.0:1619677447.881787:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.881788:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677447.881789:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.881789:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:18.0:1619677447.881790:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.881790:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:10.0:1619677447.881790:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.881791:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.881791:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:10.0:1619677447.881793:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677447.881794:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677447.881794:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96722271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.881796:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:10.0:1619677447.881796:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677447.881796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:10.0:1619677447.881797:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677447.881797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:10.0:1619677447.881798:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.881799:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.881800:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677447.881800:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677447.881802:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.881803:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677447.881803:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.881804:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.881804:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677447.881805:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.881805:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.881805:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:26.0:1619677447.881806:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677447.881808:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.881808:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.881808:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.881809:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.881810:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.881810:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:5.0:1619677447.881810:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:26.0:1619677447.881811:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.881811:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:26.0:1619677447.881812:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677447.881812:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.881813:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:26.0:1619677447.881814:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96722270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.881816:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.881817:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.881817:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:26.0:1619677447.881818:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677447.881818:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.881819:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677447.881820:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:10.0:1619677447.881820:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677447.881821:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.881821:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.881822:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.881823:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.881823:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.881824:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677447.881825:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677447.881826:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.881826:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.881827:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:26.0:1619677447.881828:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.881828:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.881829:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677447.881829:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:6.0:1619677447.881829:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.881831:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677447.881832:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.881833:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.881833:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677447.881833:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.881835:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.881837:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.881838:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.881838:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.881839:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.881839:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.881843:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.881846:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.881847:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.881848:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.881850:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.881851:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.882099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.882105:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.882108:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.882109:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.882111:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.882113:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.882115:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.882116:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.882117:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.882118:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.882119:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:4.2:1619677447.882122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:22.0:1619677447.882123:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.882125:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.882127:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.882128:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:24.0:1619677447.882129:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.882129:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.882131:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:24.0:1619677447.882132:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:22.0:1619677447.882132:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.882132:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.882133:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.882133:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.882135:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.882137:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.882139:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.882140:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96722268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.882141:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.882142:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.882143:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.882144:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.882147:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:24.0:1619677447.882147:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.882148:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.882149:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:31.0:1619677447.882150:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.882151:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:24.0:1619677447.882152:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.882153:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:24.0:1619677447.882153:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.882153:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.882155:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.882155:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.882156:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.882157:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.882161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.882169:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.882172:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.882173:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.882174:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.882176:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.882179:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.882182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:4.2:1619677447.882190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.882190:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.882192:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.882193:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.882193:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.882194:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.882195:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677447.882197:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.882200:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.882201:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.882202:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.882205:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.882206:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.882207:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.882208:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.882209:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.882210:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677447.882213:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.882215:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677447.882218:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.882218:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.882219:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.882220:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.882221:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.882222:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677447.882224:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.882224:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677447.882229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.882234:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677447.882236:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:21.2:1619677447.882236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677447.882237:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.882238:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.882238:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.882239:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.882239:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.882240:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677447.882241:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.882243:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677447.882244:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677447.882246:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.882248:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.882250:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.882250:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.882252:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677447.882253:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.882254:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.882255:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.882256:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677447.882259:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.882261:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677447.882263:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.882264:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677447.882264:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677447.882266:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.882266:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.882267:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.882267:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677447.882277:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.882279:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.882280:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.882281:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.882282:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.882285:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677447.882377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677447.882382:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.882385:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.882386:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.882388:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677447.882389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:2.0:1619677447.882391:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.882392:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.882394:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.882395:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.882396:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.882397:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:2.0:1619677447.882397:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:31.0:1619677447.882400:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:2.0:1619677447.882401:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.882404:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.882404:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:2.0:1619677447.882404:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:31.0:1619677447.882407:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.882407:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.882408:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:31.0:1619677447.882409:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:2.0:1619677447.882409:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:31.0:1619677447.882410:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.882410:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:31.0:1619677447.882411:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.882411:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.882412:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.882412:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:31.0:1619677447.882413:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.882414:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.882417:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.882419:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.882421:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.882422:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.882423:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.882424:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677447.882424:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:31.0:1619677447.882425:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.882426:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.882426:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677447.882426:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677447.882427:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677447.882428:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.882430:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.882432:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:30.0:1619677447.882436:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.882439:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.882440:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.882441:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.882443:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.882443:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677447.882611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.882618:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.882621:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.882622:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.882623:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.882626:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.882627:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677447.882629:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.882630:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.882631:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.882632:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677447.882635:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.882636:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.882638:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.882639:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677447.882639:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677447.882640:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.882641:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.882642:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.882642:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677447.882651:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.882653:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:5.2:1619677447.882653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:26.0:1619677447.882654:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.882655:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.882657:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677447.882657:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.882658:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.882659:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.882660:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.882661:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.882663:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.882665:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677447.882666:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.882668:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677447.882669:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000400:00000200:10.0:1619677447.882669:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.882670:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677447.882670:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677447.882671:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.882672:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.882673:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.882673:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.882674:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677447.882675:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677447.882678:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.882679:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677447.882679:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677447.882681:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677447.882681:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.882682:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.882682:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.2:1619677447.882690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:22.0:1619677447.882694:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:21.2:1619677447.882694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677447.882697:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:18.0:1619677447.882697:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:22.0:1619677447.882698:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.882698:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677447.882700:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.882700:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677447.882701:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:22.0:1619677447.882701:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.882703:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:18.0:1619677447.882703:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.882704:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:18.0:1619677447.882705:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.882706:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.882707:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677447.882707:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677447.882708:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.882708:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.882709:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.882710:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.882712:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.882712:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.882713:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677447.882714:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.882715:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.882716:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.882717:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.882720:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.882720:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.882722:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.882723:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.882723:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.882723:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677447.882725:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.882725:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.882725:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677447.882726:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.882726:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.882726:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:31.0:1619677447.882727:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677447.882729:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.882729:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.882730:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.882731:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677447.882731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.882738:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.882740:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.882741:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.882742:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677447.882743:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:2.0:1619677447.882743:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.882744:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.882746:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.882747:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.882747:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.882749:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.882752:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.882755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.882757:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677447.882760:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.882760:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.882762:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.882763:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.882763:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677447.882764:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000400:00000200:20.0:1619677447.882764:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.882764:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.882765:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.882766:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:20.0:1619677447.882766:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677447.882767:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.882769:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.882770:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.882771:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677447.882774:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.882776:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677447.882779:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.882779:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.882780:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.882782:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.882782:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.882783:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.882784:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677447.882789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.882797:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.882799:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.882800:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.882800:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.882803:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.882805:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.882808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.882815:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.882817:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.882818:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.882818:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.882819:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.882820:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677447.883039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.883044:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.883046:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.883047:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.883049:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.883052:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.883053:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.883054:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.883056:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.883057:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.883058:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.883061:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.883063:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.883066:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.883066:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.883067:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.883069:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.883069:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.883070:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.883070:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.883082:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:20.2:1619677447.883083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:6.0:1619677447.883085:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.883086:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.883086:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.883088:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.883089:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:6.0:1619677447.883090:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677447.883091:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.883094:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.883094:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.883096:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.883098:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.883099:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.883101:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.883102:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.883103:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.883107:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.883109:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.883111:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.883112:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.883113:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.883115:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.883115:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.883116:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.883117:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:13.0:1619677447.883128:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.883131:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.883132:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.883134:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677447.883135:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.883136:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.883224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.883231:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.883234:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.883235:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.883237:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.883240:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.883241:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.883242:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.883242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:26.0:1619677447.883243:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.883244:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.883246:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677447.883248:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.883249:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677447.883250:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.883252:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:26.0:1619677447.883253:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.883253:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677447.883254:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.883255:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:8.0:1619677447.883255:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.883256:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.883257:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.883257:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:8.0:1619677447.883257:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.883258:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:8.0:1619677447.883259:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.883260:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96722255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.883261:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.883262:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.883263:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.883266:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.883267:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677447.883270:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677447.883270:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.883271:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:22.0:1619677447.883272:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:8.0:1619677447.883272:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.883273:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:6.0:1619677447.883273:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.883274:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.883274:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.883275:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.883275:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677447.883276:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.883279:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.2:1619677447.883279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.883287:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.883289:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.883290:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.883291:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.883297:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.883300:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677447.883453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.883459:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.883461:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.883462:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.883464:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.883467:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.883468:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.883470:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722254] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.883471:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.883472:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.883473:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.883476:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.883478:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.883480:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.883481:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.883481:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.883483:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.883484:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.883484:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.883485:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.883490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.883499:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.883501:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.883502:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.883502:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.883504:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.883507:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:30.0:1619677447.883522:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.883525:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.883526:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.883527:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677447.883528:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.883529:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677447.883532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.883539:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.883542:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.883542:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.883544:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.883547:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.883548:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.883550:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.883551:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.883552:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.883553:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.883557:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.883558:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.883561:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.883562:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.883563:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.883565:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.883566:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.883566:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.883567:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:21.2:1619677447.883574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.883579:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.883580:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.883581:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.883582:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.883583:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.883583:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.883587:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.883587:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.883588:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.883589:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.883590:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.883592:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677447.883593:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.883594:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.883595:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.883596:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677447.883600:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.883601:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.883604:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.883605:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677447.883606:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.883608:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677447.883608:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.883610:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.883611:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.883612:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677447.883618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.883625:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:8.2:1619677447.883625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677447.883627:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.883628:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.883629:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677447.883630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:1.0:1619677447.883632:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677447.883635:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:1.0:1619677447.883635:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.883636:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.883636:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.883638:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.883639:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:1.0:1619677447.883639:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.883641:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.883642:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:2.0:1619677447.883643:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.883644:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.883646:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.883647:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.883648:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.883649:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.883652:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.883654:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.883656:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.883657:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.883657:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.883659:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.883659:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.883660:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.883661:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.883662:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677447.883665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677447.883668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.2:1619677447.883669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:18.0:1619677447.883673:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.883674:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.883675:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.883675:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.883676:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.883676:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.883677:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.883677:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677447.883677:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.883678:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.883678:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:10.0:1619677447.883680:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.883681:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.883683:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.883686:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.883687:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677447.883689:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.883690:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.883691:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.883692:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677447.883695:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.883697:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677447.883699:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.883700:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677447.883701:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677447.883702:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677447.883702:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.883704:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677447.883705:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677447.883705:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677447.883711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677447.883714:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.883717:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.883717:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.883718:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:30.0:1619677447.883719:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.883720:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:31.0:1619677447.883721:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677447.883721:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677447.883722:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:10.0:1619677447.883722:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677447.883814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.883817:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.883820:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.883823:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.883823:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.883825:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.883827:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677447.883828:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.883829:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.883830:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.883831:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677447.883834:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.883836:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677447.883838:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.883839:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.883840:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.883841:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.883842:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.883843:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.883844:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677447.883847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.883855:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.883858:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.883858:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.883859:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.883861:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.883863:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677447.883869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.883874:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.883877:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.883878:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.883880:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.883882:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.883883:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.883884:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.883886:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.883886:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.883887:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.883890:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.883892:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.883895:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.883896:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.883897:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.883899:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.883899:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.883900:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.883901:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.883912:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.883914:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.883915:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.883917:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.883919:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.883920:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.883927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.883935:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.883938:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.883939:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.883942:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.883942:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.883944:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.883945:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722247] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.883946:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.883947:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.883948:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.883951:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677447.883952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:31.0:1619677447.883953:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.883956:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.883957:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.883958:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.883958:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677447.883959:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.883960:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.883961:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:18.0:1619677447.883961:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.883961:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677447.883963:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000400:00000200:18.0:1619677447.883963:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.883964:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:18.0:1619677447.883965:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.883966:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:8.2:1619677447.883966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:18.0:1619677447.883968:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677447.883969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:18.0:1619677447.883969:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.883970:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.883971:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.883974:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.883974:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677447.883976:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.883977:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:5.0:1619677447.883977:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.883978:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.883979:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:18.0:1619677447.883979:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.883980:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677447.883980:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.883980:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:5.0:1619677447.883980:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.883981:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:18.0:1619677447.883981:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.883983:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.883983:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677447.883983:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677447.883984:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.883985:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677447.883995:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.883998:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.883999:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.884001:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.884001:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.884004:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.884075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.884081:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.884084:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.884084:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.884087:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677447.884088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:17.0:1619677447.884089:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.884091:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.884092:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722245] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.884093:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:17.0:1619677447.884093:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.884094:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.884095:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:22.0:1619677447.884096:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.884096:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.884099:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.884100:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.884102:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.884102:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:22.0:1619677447.884103:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.884105:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.884105:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677447.884106:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.884106:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:22.0:1619677447.884107:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.884107:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:22.0:1619677447.884108:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.884109:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:22.0:1619677447.884111:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.884112:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677447.884113:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.884114:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.884115:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677447.884116:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.884116:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.884116:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.884117:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.884119:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.884119:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.2:1619677447.884120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677447.884121:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:12.0:1619677447.884121:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:22.0:1619677447.884122:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.884122:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677447.884123:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.884124:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.884125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:12.0:1619677447.884126:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.884127:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:21.2:1619677447.884128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:12.0:1619677447.884128:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677447.884129:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.884131:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.884132:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.884133:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.884133:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:1.0:1619677447.884134:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677447.884136:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.884137:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677447.884137:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.884137:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.884138:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.884138:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.884139:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:1.0:1619677447.884140:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.884141:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.884142:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.884143:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677447.884143:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:13.0:1619677447.884144:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.884145:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677447.884282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.884288:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677447.884290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:17.0:1619677447.884291:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.884292:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.884298:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.884300:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677447.884301:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.884302:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:26.0:1619677447.884303:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:17.0:1619677447.884304:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.884304:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:17.0:1619677447.884305:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.884307:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.884307:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.884308:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:4.2:1619677447.884308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:26.0:1619677447.884310:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.884312:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677447.884312:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.884314:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.884315:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:26.0:1619677447.884316:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.884316:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677447.884317:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.884318:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677447.884319:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.884319:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:17.0:1619677447.884319:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.884319:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677447.884320:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:24.0:1619677447.884322:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.884322:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.884322:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677447.884323:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.884323:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.884323:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677447.884325:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.884325:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.884326:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.884328:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96722241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.884329:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.884329:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:24.0:1619677447.884329:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677447.884330:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:24.0:1619677447.884330:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.884331:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:26.0:1619677447.884332:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.884333:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.884333:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.884334:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:24.0:1619677447.884334:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.884336:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677447.884336:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.884338:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:9.2:1619677447.884338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.884339:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677447.884339:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.884340:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.884340:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:12.0:1619677447.884341:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.884342:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.884342:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.884343:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.884343:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.884343:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.884344:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.884346:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.2:1619677447.884347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677447.884349:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.884350:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.884350:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.884352:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.884355:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:2.0:1619677447.884356:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.884357:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.884358:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.884359:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.884361:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.884364:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677447.884371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677447.884378:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.884381:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.884381:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677447.884383:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.884386:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.884388:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.884389:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.884390:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.884391:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.884392:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.884395:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.884397:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.884399:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.884400:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.884400:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.884402:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.884402:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.884403:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.884403:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.884414:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.884416:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.884417:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.884418:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.884420:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.884420:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.884431:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.884434:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.884435:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.884435:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.884436:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.884437:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.884494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.884500:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.884503:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.884503:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.884505:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.884508:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.884509:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.884511:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.884512:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.884513:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.884514:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.884518:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.884519:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.884522:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.884523:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.884524:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.884526:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.884526:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.884527:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.884527:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.884538:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.884540:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.884541:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.884543:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.884544:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.884546:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677447.884585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.884591:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.884594:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.884595:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.884597:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.884598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:22.0:1619677447.884600:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.884601:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.884602:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.884604:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.884605:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677447.884605:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677447.884606:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677447.884608:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.884609:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677447.884610:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.884612:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:10.0:1619677447.884613:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.884615:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.884616:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.884616:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:10.0:1619677447.884616:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.884618:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.884618:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677447.884618:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677447.884619:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677447.884619:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.884620:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677447.884620:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.884621:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677447.884622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:10.0:1619677447.884622:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677447.884626:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.884628:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.884631:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677447.884631:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677447.884632:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.884633:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:26.0:1619677447.884633:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677447.884633:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.884633:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:26.0:1619677447.884634:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.884635:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:30.0:1619677447.884636:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.884636:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.884636:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.884637:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.884638:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677447.884639:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.884639:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.884639:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:10.0:1619677447.884639:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:30.0:1619677447.884640:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677447.884641:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.884643:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.884644:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.884645:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677447.884648:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.884648:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:30.0:1619677447.884649:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.2:1619677447.884649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.884651:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.884651:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:30.0:1619677447.884652:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.884652:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:22.0:1619677447.884652:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.884653:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.884654:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.884654:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.884655:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:6.0:1619677447.884655:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.884656:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.884656:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:6.0:1619677447.884658:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677447.884659:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.884661:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.884662:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.884663:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:31.0:1619677447.884664:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.884664:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.884665:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.884665:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.884666:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.884667:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.884668:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.884669:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.884670:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.884671:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:31.0:1619677447.884672:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.884672:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677447.884674:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.884676:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.884677:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.884678:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.884679:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.884680:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.884680:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.884681:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.884691:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.884693:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.884694:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.884696:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.884698:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.884699:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.884740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.884745:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.884748:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.884749:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.884751:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.884754:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.884755:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.884757:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.884758:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.884759:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.884760:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.884763:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.884765:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.884768:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.884769:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.884770:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.884772:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.884773:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.884773:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.884774:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.884784:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.884787:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.884788:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.884790:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.884791:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.884792:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.884808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677447.884813:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.884816:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.884817:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.884819:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.884821:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.884823:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677447.884824:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.884826:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.884827:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.884828:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677447.884831:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.884833:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677447.884836:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.884837:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677447.884838:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677447.884839:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677447.884840:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.884841:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.884841:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677447.884851:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677447.884854:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677447.884855:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677447.884855:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.884857:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:10.0:1619677447.884861:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:26.0:1619677447.884867:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677447.884870:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677447.884870:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677447.884871:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677447.884872:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677447.884872:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677447.884913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.884919:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.884922:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.884923:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.884924:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.884927:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.884929:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.884930:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.884932:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.884933:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.884934:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.884936:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.884938:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.884940:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.884941:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.884941:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.884943:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.884943:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.884944:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.884945:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.884946:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677447.884949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677447.884951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677447.884956:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677447.884957:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.884958:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:17.0:1619677447.884958:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677447.884959:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.884960:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:17.0:1619677447.884960:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.884960:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.884962:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.884964:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.885012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.885020:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.885023:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.885025:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.885026:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.885027:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.885029:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677447.885030:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.885031:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.885032:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.885033:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677447.885036:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.885038:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677447.885041:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.885042:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.885043:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677447.885045:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.885045:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.885046:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.885047:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.885051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.885058:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.885061:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.885062:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.885064:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.885065:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.885067:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677447.885075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:4.2:1619677447.885076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.885080:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.885083:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.885084:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:10.0:1619677447.885084:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.885086:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:2.0:1619677447.885086:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.885087:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677447.885088:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.885089:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.885090:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.885091:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.885092:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.885093:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677447.885093:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.885094:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.885095:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.885096:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677447.885099:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.885100:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.885100:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677447.885101:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:2.0:1619677447.885103:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677447.885105:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.885106:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.885107:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677447.885107:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.885108:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677447.885109:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.885110:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:2.0:1619677447.885110:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.885111:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:2.0:1619677447.885111:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.885112:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.885112:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:10.0:1619677447.885113:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.885114:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.885115:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677447.885116:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.885117:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:12.0:1619677447.885122:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:4.2:1619677447.885122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:12.0:1619677447.885125:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677447.885126:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:12.0:1619677447.885126:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.885128:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:12.0:1619677447.885128:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.885129:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.885129:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:13.0:1619677447.885129:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:31.0:1619677447.885130:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.885130:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677447.885131:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:30.0:1619677447.885132:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.885132:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677447.885133:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677447.885133:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677447.885143:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.885146:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.885147:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.885147:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.885148:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.885149:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.885226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.885231:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.885234:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.885235:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.885237:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.885240:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.885241:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.885242:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.885243:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.885245:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.885245:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.885249:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.885251:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.885252:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.885253:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.885254:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.885255:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.885256:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.885257:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.885258:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.885259:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677447.885261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677447.885264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.885270:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.885270:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.885270:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:17.0:1619677447.885271:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.885271:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.885271:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.885273:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.885273:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.885274:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.885274:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.885280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.885288:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.885290:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.885291:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.885292:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.885292:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.885292:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.885417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.885423:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.885425:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.885427:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.885428:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.885430:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.885432:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.885433:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.885434:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.885435:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.885436:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.885440:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677447.885440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.885442:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.885444:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.885445:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.885446:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.885447:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677447.885448:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.885449:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677447.885450:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:5.0:1619677447.885450:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:26.0:1619677447.885451:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677447.885451:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.885452:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:10.0:1619677447.885453:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.885455:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.885455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677447.885457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:10.0:1619677447.885457:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677447.885458:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.885459:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.885460:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:10.0:1619677447.885460:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.885461:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.885462:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.885463:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.885464:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.885464:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:10.0:1619677447.885465:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.885466:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.885467:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.885467:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677447.885467:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.885469:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.885470:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.885470:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.885471:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677447.885472:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:18.0:1619677447.885473:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677447.885473:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.885473:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.885474:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677447.885474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.885481:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.885484:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.885485:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.885485:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677447.885486:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:2.0:1619677447.885486:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.885487:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677447.885490:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.885490:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.885492:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.885494:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.885495:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.885497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.885505:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.885508:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.885510:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.885512:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.885513:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.885514:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.885516:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.885517:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.885518:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.885519:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.885522:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.885523:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.885526:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.885526:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.885527:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.885528:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.885529:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.885530:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.885530:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677447.885541:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.885544:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.885544:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.885546:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.885548:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.885549:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.885568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677447.885575:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.885578:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677447.885579:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677447.885580:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.885583:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.885585:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677447.885586:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.885587:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.885588:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.885589:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677447.885592:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.885594:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677447.885597:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.885597:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677447.885598:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.885600:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.885600:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.885601:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677447.885602:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677447.885606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.885614:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.885617:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.885618:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.885618:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.885620:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.885623:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677447.885668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677447.885676:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677447.885679:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677447.885681:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.885683:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.885683:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677447.885685:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677447.885686:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.885687:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.885688:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.885689:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677447.885692:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.885694:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677447.885696:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.885697:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:31.0:1619677447.885698:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677447.885700:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677447.885700:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677447.885701:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677447.885702:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677447.885705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.885713:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.885715:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.885717:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.885717:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.885719:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.885722:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677447.885734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677447.885739:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.885741:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.885742:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.885744:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.885747:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.885748:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.885749:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.885751:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.885752:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.885753:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.885756:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.885758:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.885761:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.885761:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.885762:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.885764:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.885764:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.885764:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.885765:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677447.885770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.885777:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.885779:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.885780:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.885782:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.885783:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.885785:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677447.885786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.885793:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.885796:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.885797:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.885798:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.885799:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.885799:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.2:1619677447.885802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.885810:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.885812:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.885813:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.885815:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.885818:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.885819:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677447.885821:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.885822:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.885823:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.885824:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677447.885827:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.885829:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.885831:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.885831:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677447.885832:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677447.885833:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.885834:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.885835:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.885835:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677447.885845:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.885847:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.885848:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.885848:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677447.885849:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.885851:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677447.885861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.885868:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.885870:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.885871:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.885872:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.885873:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.885875:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677447.885876:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.885877:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.885878:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.885879:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677447.885882:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.885883:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677447.885885:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.885886:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677447.885887:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677447.885889:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677447.885889:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.885890:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.885890:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677447.885903:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.885905:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.885906:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.885907:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.885909:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.885912:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677447.885925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:20.2:1619677447.885926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:6.0:1619677447.885929:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.885932:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:5.0:1619677447.885932:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:22.0:1619677447.885933:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.0:1619677447.885935:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:6.0:1619677447.885936:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.885937:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.885937:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.885938:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.885940:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:5.0:1619677447.885940:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.885941:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.885942:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:6.0:1619677447.885943:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.885943:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.885944:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.885945:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:5.0:1619677447.885945:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.885946:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.885947:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:6.0:1619677447.885953:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.885953:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.885956:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:6.0:1619677447.885957:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.885959:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.885960:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.885960:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:6.0:1619677447.885961:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677447.885961:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:6.0:1619677447.885962:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.885963:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.885963:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:6.0:1619677447.885964:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677447.885964:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:6.0:1619677447.885965:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.885966:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.885966:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.885967:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.885968:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677447.885969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:9.2:1619677447.885970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.2:1619677447.885975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:24.0:1619677447.885977:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.885979:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:20.0:1619677447.885980:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:8.0:1619677447.885980:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.885981:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.885981:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.885982:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.885982:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000400:00000200:24.0:1619677447.885983:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.885983:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.885983:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.885984:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.885985:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.885986:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:24.0:1619677447.885987:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.2:1619677447.885988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.885991:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:9.2:1619677447.885991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677447.885992:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.885994:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.885995:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.885995:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.885996:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.885997:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.0:1619677447.885997:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.886000:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.886000:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.886001:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.886002:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.886002:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.886051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.886058:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.886061:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.886061:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.886063:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.886066:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.886067:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.886068:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.886070:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.886071:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.886072:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.886075:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.886077:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.886080:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.886081:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.886081:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.886083:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.886084:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.886085:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.886086:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.886087:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.886089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.886092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.886097:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677447.886099:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677447.886100:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.886101:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.886101:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:1.0:1619677447.886101:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.886102:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677447.886102:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677447.886103:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.2:1619677447.886104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677447.886106:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.886111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677447.886112:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.886114:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.886117:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.886118:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:1.0:1619677447.886118:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677447.886119:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.886121:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:1.0:1619677447.886121:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.886122:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.886122:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677447.886123:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.886123:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000400:00000200:30.0:1619677447.886124:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.886124:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:30.0:1619677447.886125:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.886126:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.886129:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.886131:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.886134:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.886134:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.886135:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.886136:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.886137:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.886138:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.886139:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.886140:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677447.886146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677447.886148:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.886151:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.886152:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.886153:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.886154:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:12.0:1619677447.886155:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.886156:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:13.0:1619677447.886156:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677447.886157:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.886158:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.886167:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.886170:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.886170:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.886171:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.886172:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.886173:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677447.886183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.886190:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.886193:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.886195:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.886197:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.886197:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.886198:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677447.886200:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.886201:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.886202:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.886203:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677447.886205:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.886207:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677447.886210:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.886211:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677447.886211:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677447.886213:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677447.886214:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.886214:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.886215:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677447.886219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.886226:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.886229:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.886230:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.886231:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.886232:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.886236:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.886241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.886248:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.886251:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.886251:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.886252:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.886253:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.886253:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.886276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.886283:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.886286:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.886287:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.886289:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.886292:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.886299:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677447.886301:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.886302:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.886303:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.886304:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677447.886307:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.886308:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677447.886311:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.886312:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677447.886312:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677447.886314:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677447.886314:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.886315:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.886316:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677447.886327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677447.886329:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.886332:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.886333:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.886333:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677447.886333:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.886335:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.886336:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.886337:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.886338:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:17.0:1619677447.886339:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.886341:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.886342:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.886343:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.886345:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.886346:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.886347:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.886347:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.886350:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:17.0:1619677447.886350:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.886350:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.886351:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.886352:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.886353:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.886353:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:17.0:1619677447.886355:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.886356:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.886357:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.886358:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.886359:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.886360:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.886360:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.886372:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.886374:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.886375:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.886375:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.886377:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.886380:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677447.886541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677447.886547:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.886550:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.886551:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.886553:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.886556:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.886557:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.886558:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.886560:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.886561:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.886562:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.886565:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.886567:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.886570:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.886570:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.886571:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.886573:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.886573:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.886574:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.886575:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677447.886579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677447.886582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.886586:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.886588:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.886588:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:20.0:1619677447.886589:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.886590:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.886591:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.886592:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.886593:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.886595:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:5.0:1619677447.886595:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.886596:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.886597:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.886598:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.886600:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.886600:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.886602:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.886605:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.886607:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.886609:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.886610:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.886611:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.886612:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.886613:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.886614:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.886614:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.886626:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.886628:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:21.2:1619677447.886629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:6.0:1619677447.886629:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.886630:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.886631:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.886633:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677447.886634:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.886637:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.886638:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.886640:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.886643:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.886644:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.886645:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.886647:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.886648:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.886649:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.886652:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.886653:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.886655:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.886656:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.886657:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.886658:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.886659:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.886660:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.886660:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.886671:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.886674:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.886675:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.886675:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.886677:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.886680:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.886691:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.886694:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.886695:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.886695:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.886696:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.886696:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.886741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.886747:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.886750:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.886751:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.886752:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.886755:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.886756:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.886758:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.886759:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.886760:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.886760:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.886763:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.886764:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.886766:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.886766:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.886767:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.886768:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.886769:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.886769:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.886770:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.886774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.886781:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.886783:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.886783:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.886784:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.886785:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.886787:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.886847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.886854:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.886857:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.886858:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.886860:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.886863:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.886864:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677447.886866:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.886867:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.886868:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.886869:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677447.886872:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.886874:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677447.886877:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.886878:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677447.886878:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677447.886880:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677447.886880:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.886881:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.886881:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677447.886893:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.886895:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.886896:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.886898:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.886900:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.886901:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677447.886925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677447.886931:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677447.886934:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677447.886934:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:1.0:1619677447.886937:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.886939:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.886940:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.886942:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.886943:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.886944:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.886945:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:4.2:1619677447.886949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677447.886949:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.886950:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:21.2:1619677447.886953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677447.886954:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.886954:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.886955:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677447.886956:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677447.886957:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.886957:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677447.886958:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:26.0:1619677447.886959:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:1.0:1619677447.886959:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:10.0:1619677447.886960:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677447.886961:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677447.886962:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.886964:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677447.886964:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.886966:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677447.886967:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.886968:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.886969:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677447.886969:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.886969:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:30.0:1619677447.886970:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.886970:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.886971:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677447.886972:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:13.0:1619677447.886972:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:30.0:1619677447.886973:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96722205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.886973:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.886974:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.886975:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.886975:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.886976:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.886976:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:30.0:1619677447.886977:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677447.886977:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:13.0:1619677447.886978:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:30.0:1619677447.886980:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.886981:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.886982:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677447.886982:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.886982:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677447.886984:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.886984:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.886984:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677447.886985:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.886985:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677447.886986:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:30.0:1619677447.886987:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:26.0:1619677447.886987:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:30.0:1619677447.886988:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.886988:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677447.886988:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677447.886989:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.886989:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.886992:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.886992:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677447.886993:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677447.886993:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:4.2:1619677447.886993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:12.0:1619677447.886995:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.886996:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.886996:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.886996:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.886997:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:12.0:1619677447.886998:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.887000:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:12.0:1619677447.887000:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677447.887000:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677447.887001:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.887002:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.887002:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.887003:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677447.887004:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.887004:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677447.887006:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.887154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.887159:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.887162:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.887163:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.887165:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.887168:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.887170:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677447.887171:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.887173:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.887174:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.887175:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677447.887178:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.887180:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677447.887183:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.887184:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677447.887184:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677447.887187:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.887187:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.887188:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.887188:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.887193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.887199:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.887202:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.887203:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.887203:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.887205:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.887208:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677447.887220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677447.887227:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677447.887230:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677447.887231:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677447.887233:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.887236:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.887237:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677447.887238:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.887240:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.887241:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.887242:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677447.887245:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.887247:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677447.887250:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.887251:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677447.887251:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.887253:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.887254:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677447.887255:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677447.887255:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677447.887260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.887267:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.887269:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.887271:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.887271:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.887273:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.887276:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.887280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.887287:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.887289:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.887290:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.887291:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.887292:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.887292:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677447.887339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.887347:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.887350:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.887352:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.887354:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.887354:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.887355:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.887357:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.887358:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.887359:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.887360:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.887363:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.887364:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.887366:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.887367:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.887368:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.887369:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.887370:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.887371:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.887372:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.887373:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677447.887378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677447.887383:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.887385:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:30.0:1619677447.887386:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:13.0:1619677447.887386:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.887388:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:13.0:1619677447.887388:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.887389:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.887390:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.887390:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.887391:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.887393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.887400:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.887403:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.887404:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.887406:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.887408:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.887410:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.887411:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.887412:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.887413:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.887414:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677447.887416:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.887418:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677447.887421:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.887421:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.887422:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.887424:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.887424:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.887425:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.887425:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677447.887438:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.887441:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.887441:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.887442:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.887444:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.887447:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.887535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.887541:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.887543:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.887545:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.887546:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.887548:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.887549:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.887551:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.887552:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.887553:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.887554:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.887557:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.887559:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.887561:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.887562:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.887562:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.887564:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.887564:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.887565:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.887565:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.887570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.887577:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.887578:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.887579:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.887580:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.887580:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.887581:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677447.887666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.887672:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.887674:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.887675:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.887677:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.887679:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.887681:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.887682:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.887683:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.887684:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.887685:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.2:1619677447.887687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:22.0:1619677447.887688:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.887690:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.887690:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677447.887693:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.887693:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:22.0:1619677447.887694:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.887695:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:20.0:1619677447.887695:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.887696:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677447.887697:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.887697:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:20.0:1619677447.887697:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.887698:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.887699:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:20.0:1619677447.887699:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677447.887700:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.887701:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.887702:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.887703:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677447.887707:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.887708:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677447.887710:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677447.887711:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.887711:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.887712:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677447.887713:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.887714:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.887714:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:30.0:1619677447.887715:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.887715:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.887716:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.887716:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677447.887717:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.2:1619677447.887718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677447.887719:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.2:1619677447.887720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677447.887726:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.887728:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:1.0:1619677447.887728:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:10.0:1619677447.887729:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.887731:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.887731:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.887732:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.887732:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.887734:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.887735:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:26.0:1619677447.887736:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677447.887737:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.887737:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:26.0:1619677447.887738:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.887739:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.887740:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677447.887744:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.887746:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677447.887748:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.887749:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677447.887750:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677447.887751:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677447.887752:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.887753:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677447.887754:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677447.887755:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677447.887760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677447.887763:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.887766:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.887767:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.887768:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.887768:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:13.0:1619677447.887770:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.887770:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677447.887771:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677447.887771:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677447.887771:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:21.2:1619677447.887851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677447.887857:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.887860:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.887861:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.887863:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.887865:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.887866:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.887867:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.887868:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.887869:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.887870:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.887873:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.887875:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.887877:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.887878:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.887879:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.887881:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.887881:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.887882:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.887883:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.887884:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:8.2:1619677447.887887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677447.887890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.887895:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:5.0:1619677447.887896:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:18.0:1619677447.887897:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.887898:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677447.887898:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:18.0:1619677447.887899:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:5.0:1619677447.887899:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.887901:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.887902:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:21.2:1619677447.887904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.0:1619677447.887905:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677447.887912:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.887915:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.887917:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.887919:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.887919:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.887921:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.887922:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.887923:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.887924:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.887925:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.887928:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.887930:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.887932:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.887933:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.887934:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.887935:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.887936:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.887937:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.887937:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.887947:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.887950:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.887950:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.887952:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:20.2:1619677447.887954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:13.0:1619677447.887954:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.887954:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:4.2:1619677447.887957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677447.887962:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:24.0:1619677447.887964:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677447.887965:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.887967:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.887967:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.887968:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677447.887969:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.887969:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.887969:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677447.887970:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.887971:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:12.0:1619677447.887971:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.887972:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.887972:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.887972:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.887973:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.887973:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:30.0:1619677447.887974:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.887974:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:12.0:1619677447.887974:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:30.0:1619677447.887975:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.887975:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.887977:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677447.887977:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.887978:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.887979:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677447.887980:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.887983:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:24.0:1619677447.887983:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.887985:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.887985:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677447.887985:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677447.887986:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.887988:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.887988:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:24.0:1619677447.887988:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.887989:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:24.0:1619677447.887989:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677447.887990:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:24.0:1619677447.887990:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.887991:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.887992:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.887992:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.887993:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.887997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677447.888000:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.888002:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.888002:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.888004:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.888005:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:12.0:1619677447.888006:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.888007:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:13.0:1619677447.888007:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:17.0:1619677447.888008:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.888008:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.888010:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.888013:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.888138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.888145:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.888148:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.888149:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.888151:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.888153:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.888155:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677447.888156:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.888157:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.888158:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.888159:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677447.888162:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.888164:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.888167:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.888168:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677447.888168:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.888170:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.888171:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.888171:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.888172:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677447.888177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.888184:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.888187:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.888188:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.888188:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.888190:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.888194:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.888238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.888244:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.888247:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.888248:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.888250:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.888252:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.888254:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.888255:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.888256:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.888257:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.888258:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.888262:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.888264:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.888267:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.888267:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.888268:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.888270:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.888270:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.888271:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.888271:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.888282:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.888285:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.888286:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.888288:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.888290:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.888290:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677447.888315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.888323:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.2:1619677447.888324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677447.888326:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.888326:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.888329:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.888332:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677447.888332:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.888333:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677447.888334:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677447.888334:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677447.888334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38a600] (20)++
00000400:00000200:26.0:1619677447.888336:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.888337:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.888337:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.888338:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.888339:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.888339:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.888340:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677447.888341:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38a600] (21)++
00000400:00000200:31.0:1619677447.888342:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.888342:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.888343:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.888343:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.16@o2ib3
00000400:00000200:31.0:1619677447.888344:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.888345:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677447.888345:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.888347:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.16@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.888348:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.888349:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.888349:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.888349:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677447.888349:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677447.888351:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677447.888351:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677447.888352:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.888352:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000800:00000200:26.0:1619677447.888352:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:30.0:1619677447.888353:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96722188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.888353:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:30.0:1619677447.888354:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.888355:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.888356:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000800:00000200:31.0:1619677447.888359:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.888359:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677447.888360:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000400:00000200:30.0:1619677447.888360:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.888361:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:31.0:1619677447.888362:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.888362:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:31.0:1619677447.888363:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.888363:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.888364:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:31.0:1619677447.888365:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38a600] (22)--
00000800:00000200:30.0:1619677447.888365:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f90d380] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.888365:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e49d37c00] (20)++
00000800:00000200:30.0:1619677447.888366:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e49d37c00] (21)++
00000800:00000200:13.0:1619677447.888367:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:30.0:1619677447.888368:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e49d37c00] (22)--
00000400:00000200:13.0:1619677447.888368:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.888369:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38a600] (21)++
00000800:00000200:30.0:1619677447.888369:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38a600] (22)--
00000400:00000200:13.0:1619677447.888369:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.888370:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38a600] (21)--
00000800:00000200:13.0:1619677447.888371:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.888372:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.2:1619677447.888374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (21)++
00000800:00000200:12.0:1619677447.888374:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.888375:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.888377:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.888378:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)++
00000800:00000200:13.0:1619677447.888378:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.888380:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.888381:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e49d37c00] (23)--
00000400:00000200:24.0:1619677447.888382:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.888382:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)--
00000400:00000200:24.0:1619677447.888384:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.888386:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)--
00000800:00000200:24.2:1619677447.888397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (20)++
00000800:00000200:24.0:1619677447.888399:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)++
00000800:00000200:24.0:1619677447.888402:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677447.888403:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e49d37c00] (22)++
00000800:00000200:8.0:1619677447.888403:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)--
00000800:00000200:24.0:1619677447.888404:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e49d37c00] (22)--
00000800:00000200:24.0:1619677447.888404:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)--
00000800:00000200:20.2:1619677447.888569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.888572:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.888575:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.888577:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.888578:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.888579:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.888581:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.888582:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.888584:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.888585:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.888586:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.888589:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.888591:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.888594:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.888595:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.888596:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.888598:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.888598:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.888599:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.888599:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677447.888610:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.888613:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.888614:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.888616:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.888617:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.888619:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677447.888661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.888668:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.888670:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.888671:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.888673:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.888675:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.888677:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.888678:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.888679:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.888680:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.888681:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677447.888684:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.888686:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677447.888689:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.888689:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.888690:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.888692:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.888692:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.888693:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.888693:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677447.888706:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.888709:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.888709:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.888710:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.888712:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.888715:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677447.888721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.888728:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.888731:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:4.2:1619677447.888731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:30.0:1619677447.888733:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.888735:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.888735:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.888737:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677447.888738:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.888738:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:30.0:1619677447.888740:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.888741:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.888741:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:30.0:1619677447.888742:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.888742:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.888744:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.888746:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.888747:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.888747:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.888748:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:5.2:1619677447.888748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:8.0:1619677447.888749:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96722184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.888750:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.888751:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:8.0:1619677447.888751:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.888752:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:8.0:1619677447.888752:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.888753:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.888753:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677447.888753:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677447.888754:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.888755:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.888755:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677447.888756:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:8.0:1619677447.888757:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.888757:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677447.888759:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.888763:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.888765:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.888766:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.888767:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677447.888767:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (23)++
00000400:00000200:17.0:1619677447.888768:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.888769:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (24)--
00000800:00000200:1.0:1619677447.888769:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (24)++
00000400:00000200:17.0:1619677447.888770:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:8.0:1619677447.888770:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.888770:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000800:00000200:8.0:1619677447.888771:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.888771:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677447.888772:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96722183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.888772:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.888773:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.888774:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.888775:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.888776:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.888779:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.888780:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677447.888781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:1.0:1619677447.888781:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.888782:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.888782:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.888783:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.888785:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.888786:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.888786:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.888787:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677447.888787:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.888787:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.888788:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:18.0:1619677447.888790:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:17.0:1619677447.888790:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:2.0:1619677447.888790:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677447.888791:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.888792:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:18.0:1619677447.888793:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.888793:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.888794:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677447.888795:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.888796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:18.0:1619677447.888797:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.888798:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677447.888798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677447.888799:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.888800:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.888801:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.888803:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:18.0:1619677447.888804:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.888804:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677447.888805:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:18.0:1619677447.888805:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.888806:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.888806:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:24.0:1619677447.888807:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.888807:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.888808:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677447.888809:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.888809:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677447.888809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:18.0:1619677447.888810:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.888811:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.888813:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:18.0:1619677447.888813:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.888813:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.888814:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677447.888814:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:18.0:1619677447.888815:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677447.888817:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677447.888818:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677447.888820:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.888823:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.888824:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.888825:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.888825:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:17.0:1619677447.888827:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.888828:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.888828:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:17.0:1619677447.888829:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:13.0:1619677447.888829:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.888832:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.888832:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:17.0:1619677447.888833:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.888835:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.888837:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.888839:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.888840:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.888840:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.888842:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.888843:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677447.888843:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677447.888844:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:12.0:1619677447.888854:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.888857:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.888858:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.888860:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.888861:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.888862:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.888871:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.888874:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.888875:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.888875:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.888876:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.888877:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.888913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.888918:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.888921:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.888922:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.888924:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.888927:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.888929:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.888929:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.888931:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.888932:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.888933:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.888936:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.888938:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.888941:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.888942:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.888942:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.888944:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.888945:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.888946:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.888947:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.888948:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.888950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677447.888953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.888958:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677447.888958:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:6.0:1619677447.888960:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.888961:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.888961:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677447.888961:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.888962:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.888963:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.888964:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.888967:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677447.888971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.888976:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.888979:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.888980:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.888982:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.888985:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.888986:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.888987:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.888989:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.888990:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.888991:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677447.888995:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.888996:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.888999:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.889000:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.889001:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.889002:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.889003:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.889003:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.889004:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.889008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.889014:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:5.2:1619677447.889015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.0:1619677447.889017:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.889018:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.889018:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.889020:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.889022:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677447.889023:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:26.0:1619677447.889025:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.889026:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.889028:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.889031:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.889032:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677447.889033:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.889035:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.889036:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.889037:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677447.889040:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.889042:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677447.889044:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.889045:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677447.889045:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677447.889046:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677447.889047:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.889048:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.889048:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677447.889059:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.889061:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.889062:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.889064:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.889065:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.889066:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677447.889145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.889151:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.889154:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.889156:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.889156:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.889159:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.889161:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677447.889162:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.889163:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.889164:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.889165:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.889169:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.889171:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.889173:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.889174:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.889175:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.889177:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.889177:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.889178:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.889179:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677447.889183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.889191:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.889193:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.889194:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.889194:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.889196:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.889199:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.889203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:4.2:1619677447.889208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.889210:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.889213:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.889214:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.889214:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:10.0:1619677447.889215:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677447.889215:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.889216:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677447.889217:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.889218:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.889220:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.889223:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.889224:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677447.889225:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.889226:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.889227:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.889228:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677447.889231:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.889232:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.889234:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.889235:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677447.889235:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677447.889236:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.889237:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.889238:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677447.889239:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.889240:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677447.889245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.889250:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.889251:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.889252:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.889253:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.889255:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.889255:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.889256:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677447.889256:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.889257:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677447.889257:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.2:1619677447.889292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.889301:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.889304:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.889306:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.889308:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.889309:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.889311:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677447.889312:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.889313:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.889314:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.889315:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677447.889318:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.889319:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.889321:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.889322:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.889322:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.889324:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.889324:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.889325:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.889325:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.2:1619677447.889334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677447.889336:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.889339:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.889339:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.889340:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677447.889341:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:21.2:1619677447.889341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:31.0:1619677447.889343:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:17.0:1619677447.889343:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:30.0:1619677447.889344:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:1.0:1619677447.889344:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.889346:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.889347:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:17.0:1619677447.889348:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.889350:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:17.0:1619677447.889350:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.889351:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.889351:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:17.0:1619677447.889353:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.889354:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.889354:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.889355:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:31.0:1619677447.889356:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000400:00000200:18.0:1619677447.889356:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.889358:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:31.0:1619677447.889359:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.889360:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000400:00000200:18.0:1619677447.889360:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.889360:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.889361:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677447.889361:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:18.0:1619677447.889361:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.889362:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.889363:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:17.0:1619677447.889363:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:18.0:1619677447.889364:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:17.0:1619677447.889365:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.889366:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.889367:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:18.0:1619677447.889368:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.889369:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.889369:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.889370:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677447.889370:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.889370:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677447.889373:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.889374:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.889374:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677447.889375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.2:1619677447.889375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:18.0:1619677447.889376:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.889377:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.889377:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.889378:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:30.0:1619677447.889383:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677447.889383:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.889385:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:30.0:1619677447.889386:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:1.0:1619677447.889386:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.889387:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677447.889389:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.889389:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.889389:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:1.0:1619677447.889389:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.889392:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.889392:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677447.889392:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:1.0:1619677447.889392:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:10.0:1619677447.889393:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.889394:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677447.889395:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.889395:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.889397:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.889398:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.889399:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:10.0:1619677447.889399:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:30.0:1619677447.889402:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.889404:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677447.889407:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.889408:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677447.889409:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677447.889411:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.889411:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.889412:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.889414:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.889414:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677447.889420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677447.889425:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:31.0:1619677447.889427:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:6.0:1619677447.889427:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.889428:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.889429:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.889430:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.889431:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.889431:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:6.0:1619677447.889431:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.889434:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.889512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677447.889519:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.889522:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677447.889523:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677447.889525:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.889527:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.889529:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677447.889530:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.889531:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.889532:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.889533:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677447.889536:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.889538:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677447.889541:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.889542:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677447.889542:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677447.889544:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677447.889544:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:5.2:1619677447.889544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677447.889545:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677447.889546:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677447.889546:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:4.2:1619677447.889546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:17.0:1619677447.889550:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.2:1619677447.889551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677447.889552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:17.0:1619677447.889553:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:24.0:1619677447.889555:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:17.0:1619677447.889555:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.889555:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.889557:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.889557:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677447.889558:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.889558:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677447.889558:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.889559:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.889560:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:10.0:1619677447.889560:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:17.0:1619677447.889561:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.889562:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.889562:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677447.889562:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:1.0:1619677447.889562:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.889563:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677447.889563:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:17.0:1619677447.889564:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.889565:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.889565:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.889568:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.889570:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96722169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.889570:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.889571:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.889572:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.889572:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.889573:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.889575:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.889576:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.889577:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (23)++
00000400:00000200:24.0:1619677447.889578:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.889578:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (24)--
00000800:00000200:1.0:1619677447.889579:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:24.0:1619677447.889580:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.889580:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.889581:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.889581:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.889581:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.889583:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.889584:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.2:1619677447.889584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.889585:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677447.889587:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (24)--
00000800:00000200:24.0:1619677447.889588:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.889589:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.889590:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.889593:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (23)++
00000800:00000200:17.0:1619677447.889596:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (24)++
00000800:00000200:1.0:1619677447.889596:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (24)--
00000800:00000200:17.0:1619677447.889597:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (24)--
00000400:00000200:1.0:1619677447.889597:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.889598:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.889599:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.889599:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.889601:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.889602:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.889603:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.889604:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.889604:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.889604:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.889605:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.889606:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677447.889810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.889816:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.889819:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.889819:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.889821:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.889823:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.889825:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.889826:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.889827:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.889829:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.889830:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.889833:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.889835:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.889837:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.889838:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.889839:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.889840:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.889841:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.889841:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.889842:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.889847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.889854:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.889856:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.889858:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.889858:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.889860:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.889862:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.889870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.889877:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.889880:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.889881:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.889883:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.889886:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.889887:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.889889:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.889890:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.889891:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.889892:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.889896:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.889897:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.889900:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.889901:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.889901:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.889903:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.889903:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.889904:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.889905:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677447.889910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:31.0:1619677447.889916:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:18.0:1619677447.889916:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677447.889918:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.889919:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.889919:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.889920:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:31.0:1619677447.889922:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.889922:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:18.0:1619677447.889923:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677447.889925:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:18.0:1619677447.889926:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.889927:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.889928:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722166] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.889929:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.889931:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.889931:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677447.889933:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:18.0:1619677447.889935:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:30.0:1619677447.889936:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.889937:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.889937:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:18.0:1619677447.889937:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677447.889938:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:18.0:1619677447.889938:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.889939:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:30.0:1619677447.889940:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677447.889940:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.889941:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.889941:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.889942:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.889943:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.889947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.889954:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.889956:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.889957:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.889958:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.889959:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.889961:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.889978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.889984:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.889987:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.889988:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.889990:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.889992:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.889993:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.889994:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.889995:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.889996:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.889997:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.890000:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.890002:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.890005:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.890006:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.890006:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.890008:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.890009:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.890010:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.890012:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.890012:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677447.890017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677447.890021:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.2:1619677447.890022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:1.0:1619677447.890022:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.890025:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.890025:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.890026:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.890027:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677447.890028:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000400:00000200:12.0:1619677447.890028:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.890030:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000400:00000200:12.0:1619677447.890030:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.890031:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.890031:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677447.890033:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.890033:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:18.0:1619677447.890036:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.890037:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:18.0:1619677447.890039:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.890040:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.890041:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.890042:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:18.0:1619677447.890046:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.890047:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:18.0:1619677447.890050:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.890050:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:18.0:1619677447.890051:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:18.0:1619677447.890053:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:18.0:1619677447.890053:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677447.890054:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677447.890055:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:8.2:1619677447.890059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.890067:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.890069:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.890070:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.890071:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.890073:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.890075:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.890139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.890147:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.890149:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.890152:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.890153:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.890154:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.890156:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.890157:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.890159:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.890160:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.890161:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.890164:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.890166:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.890168:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.890169:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.890170:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.890171:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.890172:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.890173:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.890173:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677447.890183:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.890185:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.890186:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.890188:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.890189:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.890190:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.890202:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.890204:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.890205:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.890206:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.890206:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.890207:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677447.890283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.890290:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.890294:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.890301:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677447.890304:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.890306:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.890307:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677447.890309:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.890310:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.890311:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.890312:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677447.890315:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.890317:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677447.890319:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.890320:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677447.890321:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677447.890323:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677447.890323:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.890324:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.890324:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:4.2:1619677447.890325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.2:1619677447.890329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.890336:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.890338:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:8.0:1619677447.890339:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.890340:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.890341:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.890341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.890341:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.890342:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.890342:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.890344:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.890344:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.890346:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:1.0:1619677447.890347:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:26.0:1619677447.890348:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:8.0:1619677447.890348:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.890349:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.890350:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.890351:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.890351:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:8.0:1619677447.890351:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677447.890354:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.890355:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.890357:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.890357:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:26.0:1619677447.890358:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.890359:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.890359:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677447.890360:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.890360:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:26.0:1619677447.890361:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.890361:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677447.890362:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.0:1619677447.890363:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.890363:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.890364:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:26.0:1619677447.890365:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.890366:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.890366:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677447.890367:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.890369:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677447.890369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677447.890370:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.890370:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.890372:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.890372:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:4.2:1619677447.890372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677447.890373:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.890373:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677447.890376:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.890379:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677447.890379:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.890380:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.890380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:1.0:1619677447.890380:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.890381:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.890382:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677447.890382:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.890383:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.890385:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677447.890386:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:2.0:1619677447.890386:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:30.0:1619677447.890389:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:2.0:1619677447.890389:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:30.0:1619677447.890390:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.890391:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677447.890393:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677447.890393:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.890394:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.890395:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:2.0:1619677447.890396:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.890398:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.890399:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.890401:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.890402:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.890403:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.890406:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.890408:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.890411:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.890411:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.890412:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.890414:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.890414:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.890415:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.890416:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.890417:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677447.890419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677447.890422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:1.0:1619677447.890427:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677447.890428:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:1.0:1619677447.890429:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677447.890430:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.890430:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:1.0:1619677447.890430:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.890431:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.890432:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:1.0:1619677447.890432:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.890435:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.890525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.890532:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.890535:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.890537:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.890538:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.890539:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.890540:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.890541:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.890542:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.890543:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.890544:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.890547:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.890549:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.890551:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.890552:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.890552:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.890554:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.890554:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.890556:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.890557:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.890558:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677447.890563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677447.890567:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.890569:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.890570:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.890571:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:13.0:1619677447.890572:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.890573:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.890574:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:12.0:1619677447.890574:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.890575:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677447.890575:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677447.890595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.890601:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.890603:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.890604:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.890606:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.890609:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.890610:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.890612:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.890613:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.890614:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.890615:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.890618:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.890620:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.890624:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.890625:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.890625:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.890627:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.890628:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.890628:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.890629:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.890640:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.890642:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.890643:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.890645:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.890646:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.890648:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.890809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.890812:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.890814:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.890817:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.890818:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.890819:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.890821:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.890822:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722156] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.890824:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.890825:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.890826:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.890829:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.890831:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.890835:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.890835:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.890836:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.890838:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.890838:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.890840:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.890842:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:4.2:1619677447.890842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.890843:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.890847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677447.890849:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677447.890850:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677447.890852:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.890853:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:13.0:1619677447.890853:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677447.890853:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:13.0:1619677447.890854:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.890855:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:13.0:1619677447.890856:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:10.0:1619677447.890856:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.890856:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677447.890857:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.890857:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:10.0:1619677447.890858:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.890859:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677447.890859:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677447.890861:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.890862:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.890863:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.890864:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677447.890867:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.890869:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677447.890871:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.890872:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.890873:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677447.890874:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.890874:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.890875:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.890876:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677447.890886:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.890889:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.890889:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.890891:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.890893:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.890894:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.890905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:20.2:1619677447.890906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677447.890911:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:31.0:1619677447.890914:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677447.890914:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.890915:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:31.0:1619677447.890917:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:2.0:1619677447.890917:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.890920:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.890920:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.890921:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.890922:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.890922:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677447.890924:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677447.890924:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.890926:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.890926:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.890927:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.890928:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.890928:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677447.890929:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.890930:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677447.890932:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.890935:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.890935:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:21.2:1619677447.890937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38a600] (20)++
00000800:00000200:2.0:1619677447.890937:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.890938:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.890938:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.890938:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677447.890940:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.890940:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.890940:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.890940:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:31.0:1619677447.890941:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677447.890941:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:31.0:1619677447.890942:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.890943:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.890944:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.890944:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38a600] (21)++
00000800:00000200:31.0:1619677447.890945:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.890945:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677447.890946:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.16@o2ib3
00000800:00000200:9.2:1619677447.890946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677447.890947:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38a600] (22)--
00000400:00000200:30.0:1619677447.890948:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.16@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.890950:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.890952:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:30.0:1619677447.890953:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96722152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.890953:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:30.0:1619677447.890954:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.890955:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.890955:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:30.0:1619677447.890956:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:18.0:1619677447.890956:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.890956:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677447.890957:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:12.0:1619677447.890958:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:18.0:1619677447.890959:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:12.0:1619677447.890959:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.890962:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.890962:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.890962:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.890962:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.890964:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.890965:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:9.2:1619677447.890965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:30.0:1619677447.890967:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f90d380] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.890968:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e49d37c00] (20)++
00000800:00000200:30.0:1619677447.890969:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e49d37c00] (21)++
00000800:00000200:30.0:1619677447.890970:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e49d37c00] (22)--
00000800:00000200:30.0:1619677447.890971:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38a600] (21)++
00000800:00000200:30.0:1619677447.890972:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38a600] (22)--
00000800:00000200:30.0:1619677447.890972:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38a600] (21)--
00000800:00000200:2.0:1619677447.890972:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.890975:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:24.2:1619677447.890976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (21)++
00000800:00000200:18.0:1619677447.890976:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.890976:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.890977:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.890977:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677447.890979:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)++
00000800:00000200:24.0:1619677447.890982:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e49d37c00] (23)--
00000400:00000200:24.0:1619677447.890983:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.890983:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)--
00000400:00000200:24.0:1619677447.890985:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.890988:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)--
00000800:00000200:21.2:1619677447.891005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.891011:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.891014:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.891015:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.891017:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.891019:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.891021:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.891022:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.891023:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.891024:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.891025:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.891028:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.891030:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.891033:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.891034:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.891034:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.891036:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.891037:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.891037:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.891038:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:26.0:1619677447.891048:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677447.891051:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677447.891051:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677447.891052:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.891054:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.891057:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677447.891182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.891188:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.891190:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.891191:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.891193:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.891196:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.891198:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.891199:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.891200:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.891201:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.891202:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.891205:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.891207:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.891210:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.891211:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.891212:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.891214:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.891214:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.891215:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.891215:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.891226:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.891229:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.891230:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.891232:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.891233:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.891234:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.891245:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.891247:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.891248:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.891249:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.891249:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.891251:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.891268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677447.891270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.891276:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.891278:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.891279:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.891280:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677447.891281:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.891282:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.891284:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.891285:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.891286:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677447.891288:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722149] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.891289:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.891290:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.891291:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677447.891297:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677447.891299:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.891301:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.891302:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.891303:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677447.891306:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677447.891306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:10.0:1619677447.891306:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.891308:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.891308:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677447.891310:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.891311:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677447.891311:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.891312:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677447.891312:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:5.0:1619677447.891312:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:26.0:1619677447.891313:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.891313:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677447.891314:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677447.891315:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (23)++
00000800:00000200:10.0:1619677447.891315:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:5.0:1619677447.891315:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:10.0:1619677447.891316:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677447.891316:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (23)++
00000800:00000200:26.0:1619677447.891317:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (24)--
00000800:00000200:26.0:1619677447.891317:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (23)--
00000400:00000200:5.0:1619677447.891317:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677447.891318:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:10.0:1619677447.891318:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (24)--
00000800:00000200:26.0:1619677447.891319:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.891319:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (23)--
00000400:00000200:5.0:1619677447.891320:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.891324:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677447.891326:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:5.0:1619677447.891328:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:4.2:1619677447.891328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.891330:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:5.0:1619677447.891330:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96722147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.891331:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:5.0:1619677447.891331:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677447.891332:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:10.0:1619677447.891332:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.891332:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.891333:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677447.891333:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:5.0:1619677447.891333:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.891334:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:10.0:1619677447.891334:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.891335:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.891336:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.891336:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:5.0:1619677447.891336:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677447.891337:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677447.891337:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:10.0:1619677447.891337:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677447.891338:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677447.891338:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.891338:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677447.891338:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:5.0:1619677447.891338:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:22.0:1619677447.891341:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.891341:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.891341:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.891343:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.891344:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677447.891345:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.891345:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:26.0:1619677447.891346:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677447.891346:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.891347:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:26.0:1619677447.891350:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.891350:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.891351:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:9.2:1619677447.891351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.891352:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.891352:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:18.0:1619677447.891359:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.891362:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.891363:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.891364:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.891365:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.891368:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.891514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.891522:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.891525:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.891527:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.891530:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.891530:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.891531:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.891533:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.891534:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.891535:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.891536:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.891540:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.891541:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.891544:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.891545:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.891546:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.891548:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.891548:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.891549:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.891550:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.891555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.891562:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.891565:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.891566:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.891568:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.891569:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.891571:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677447.891591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.891598:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.891601:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.891604:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.891605:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.891606:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.891608:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677447.891609:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.891611:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.891611:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.891612:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677447.891616:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.891617:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677447.891619:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.891620:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677447.891620:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677447.891622:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.891622:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.891623:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.891624:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677447.891633:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.891636:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.891636:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.891637:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.891638:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.891641:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677447.891745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677447.891753:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.891756:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.891758:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.891759:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.891762:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.891763:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.891764:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.891766:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.891766:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.891767:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.891770:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.891772:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.891776:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.891776:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.891777:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.891779:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.891779:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.891780:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.891780:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:21.2:1619677447.891782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:9.2:1619677447.891785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.891788:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.891791:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.891791:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:2.0:1619677447.891792:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:6.0:1619677447.891793:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.891795:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.891795:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.891796:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.891797:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:6.0:1619677447.891797:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.891798:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.891799:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.891801:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.891801:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.891802:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:6.0:1619677447.891803:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.891806:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.891808:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.891810:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.891811:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.891812:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.891814:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.891814:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.891815:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.891816:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.891820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.891828:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.891830:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.891831:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.891833:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.891834:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.891835:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677447.891862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.891868:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.891871:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.891872:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.891874:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677447.891876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:18.0:1619677447.891876:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.891877:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.891878:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.891879:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:18.0:1619677447.891880:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.891881:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:18.0:1619677447.891881:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.891882:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677447.891885:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.891885:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:18.0:1619677447.891886:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.891889:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:20.0:1619677447.891890:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.891891:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.891892:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:18.0:1619677447.891892:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:20.0:1619677447.891893:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.891893:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:20.0:1619677447.891894:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.891894:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:20.0:1619677447.891895:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.891895:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:20.0:1619677447.891896:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.891896:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.891896:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:20.0:1619677447.891899:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677447.891900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.891901:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.891904:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.891904:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.891905:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.891907:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.891907:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.891908:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.891908:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.891909:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677447.891909:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.891910:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.891911:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.891911:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.891914:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677447.891921:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.891923:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.891924:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.891924:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.891926:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.891928:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.891937:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.891940:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.891941:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.891941:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.891942:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.891942:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.892154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.892160:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.892163:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.892164:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.892166:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.892169:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.892171:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677447.892172:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.892173:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.892175:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.892176:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677447.892179:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.892181:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677447.892184:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.892185:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677447.892185:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677447.892187:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.892187:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.892188:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.892189:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.892193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.892200:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.892202:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.892203:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.892204:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.892206:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.892209:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.2:1619677447.892397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.2:1619677447.892398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677447.892404:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677447.892406:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677447.892407:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.892408:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:26.0:1619677447.892409:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.892410:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:2.0:1619677447.892411:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.892413:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.892413:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.892415:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.892415:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.892416:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.892417:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677447.892419:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.892419:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.892420:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.892421:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.892421:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677447.892422:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.892423:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677447.892425:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.892427:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:26.0:1619677447.892428:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.892429:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.892430:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.892430:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.892431:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.892432:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.892433:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.892433:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.892433:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:26.0:1619677447.892434:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677447.892434:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677447.892434:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:26.0:1619677447.892436:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677447.892436:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.892437:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.892438:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677447.892439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.892447:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.892449:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.892450:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.892451:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677447.892451:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.892452:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.892454:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.892454:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.892455:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.892457:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.892458:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.892460:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677447.892584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.892593:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.892595:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.892598:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.892600:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.892600:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.892601:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.892603:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.892604:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.892605:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.892606:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.892609:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.892611:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.892613:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.892613:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.892614:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.892615:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.892616:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.892617:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.892617:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.892627:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.892629:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.892630:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.892631:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.892633:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.892634:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.892708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.892716:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.892718:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.892720:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.892721:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.892724:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.892725:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.892726:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.892728:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.892728:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.892729:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.892732:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.892734:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.892736:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.892737:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.892738:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.892740:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.892740:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.892740:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.2:1619677447.892741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:8.0:1619677447.892741:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677447.892746:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.2:1619677447.892747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677447.892749:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.892750:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.892751:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.892753:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.892754:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:1.0:1619677447.892755:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:2.0:1619677447.892756:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.892757:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.892758:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.892758:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.892759:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:2.0:1619677447.892759:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677447.892759:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.892761:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.892762:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677447.892764:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.892765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:2.0:1619677447.892765:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.2:1619677447.892767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:2.0:1619677447.892767:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.892768:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.892768:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.892770:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.892770:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677447.892772:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677447.892772:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.892773:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.892773:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:10.0:1619677447.892774:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:17.0:1619677447.892775:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.892776:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.892776:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.892777:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677447.892777:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677447.892778:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:21.2:1619677447.892778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677447.892778:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:10.0:1619677447.892780:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677447.892783:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.892784:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677447.892784:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:10.0:1619677447.892784:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677447.892785:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.892786:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:18.0:1619677447.892786:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.892787:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:10.0:1619677447.892787:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.892788:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.892788:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.892788:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677447.892788:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:10.0:1619677447.892789:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677447.892791:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:21.2:1619677447.892792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:10.0:1619677447.892794:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:22.0:1619677447.892796:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677447.892796:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.892797:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677447.892798:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.892799:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:10.0:1619677447.892799:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677447.892799:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:10.0:1619677447.892800:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:22.0:1619677447.892802:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.892802:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:22.0:1619677447.892803:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.892803:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.892804:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677447.892804:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:22.0:1619677447.892805:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.892806:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.892808:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677447.892808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:22.0:1619677447.892809:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.892810:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.892814:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.892816:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677447.892816:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:22.0:1619677447.892819:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.892819:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:18.0:1619677447.892819:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:22.0:1619677447.892820:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:2.0:1619677447.892820:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.892821:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.892822:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.892823:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:18.0:1619677447.892823:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.892824:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.892825:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.892825:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677447.892825:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.892829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:21.2:1619677447.892831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677447.892836:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:6.0:1619677447.892836:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677447.892836:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.892838:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.892839:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677447.892839:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677447.892840:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:6.0:1619677447.892840:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.892840:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000400:00000200:30.0:1619677447.892841:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.892841:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.2:1619677447.892842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677447.892842:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.892842:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677447.892843:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677447.892843:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.892846:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.0:1619677447.892846:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.892848:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.892850:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.892851:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.892853:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.892854:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677447.892856:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.892857:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.892858:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.892859:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677447.892862:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.892864:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677447.892866:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.892867:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.892868:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.892869:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.892870:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.892871:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.892871:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677447.892875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.892883:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.892885:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.892886:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.892887:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.892889:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.892891:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677447.892948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.2:1619677447.892954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:31.0:1619677447.892956:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.892959:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.892961:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.892961:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:31.0:1619677447.892963:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.892963:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677447.892964:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:31.0:1619677447.892965:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:8.0:1619677447.892965:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:31.0:1619677447.892966:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.892967:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.892968:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.892969:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.892970:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.892970:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.892972:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677447.892974:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.892974:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96722130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.892975:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.892976:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.892976:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.892977:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677447.892978:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.892979:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.892980:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.892981:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677447.892981:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.892982:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677447.892982:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677447.892983:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.892983:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677447.892985:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.892986:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.892987:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.892988:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.892989:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.892990:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.892991:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.892992:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677447.892993:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.2:1619677447.892994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:13.0:1619677447.892996:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.892996:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677447.892998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000400:00000200:13.0:1619677447.892998:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.892999:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.893000:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.893004:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.0:1619677447.893005:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:5.0:1619677447.893006:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:8.0:1619677447.893007:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:5.0:1619677447.893007:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.893008:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677447.893009:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.893009:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:5.0:1619677447.893009:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.893012:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:12.0:1619677447.893015:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.893018:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.893019:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.893019:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.893020:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.893021:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.893049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.893055:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.893058:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.893059:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.893061:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.893063:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.893064:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.893066:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.893067:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.893068:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.893069:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.893072:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.893074:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.893076:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.893077:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.893078:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.893080:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.893080:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.893081:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.893081:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.893085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.893093:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.893095:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.893096:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.893096:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.893098:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.893099:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.893116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.893124:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.893127:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.893127:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.893129:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.893132:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.893134:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677447.893135:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.893136:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.893137:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.893138:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677447.893142:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.893144:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677447.893146:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.893147:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677447.893148:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677447.893149:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677447.893150:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.893150:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.893151:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677447.893162:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.893164:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.893165:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.893167:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.893169:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.893170:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677447.893175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919078fb5400] (20)++
00000800:00000200:18.0:1619677447.893180:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919078fb5400] (21)++
00000800:00000200:18.0:1619677447.893183:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.6@o2ib3
00000800:00000200:2.0:1619677447.893184:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (22)--
00000400:00000200:18.0:1619677447.893186:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.6@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.893188:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.893190:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:18.0:1619677447.893191:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.893192:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.893193:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.893194:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677447.893197:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.893199:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677447.893201:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.893202:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:18.0:1619677447.893203:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:18.0:1619677447.893204:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:18.0:1619677447.893205:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919078fb5400] (21)++
00000800:00000200:18.0:1619677447.893206:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919078fb5400] (22)++
00000800:00000200:18.0:1619677447.893207:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919078fb5400] (23)--
00000800:00000200:18.0:1619677447.893208:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (22)--
00000800:00000200:8.2:1619677447.893211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:21.2:1619677447.893213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919078fb5400] (21)++
00000800:00000200:20.0:1619677447.893218:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:2.0:1619677447.893218:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919078fb5400] (22)++
00000800:00000200:2.0:1619677447.893219:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919078fb5400] (23)--
00000800:00000200:2.0:1619677447.893220:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (22)--
00000800:00000200:20.0:1619677447.893221:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:2.0:1619677447.893221:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919078fb5400] (21)++
00000400:00000200:20.0:1619677447.893222:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.893222:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (22)--
00000800:00000200:2.0:1619677447.893222:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.6@o2ib3
00000400:00000200:20.0:1619677447.893223:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.893225:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:2.0:1619677447.893225:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.6@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.893226:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:2.0:1619677447.893228:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.893229:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.893230:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.893232:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.893233:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.893234:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.893237:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.893238:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.893240:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.893241:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.893241:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.2:1619677447.893243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677447.893243:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.893243:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919078fb5400] (21)++
00000800:00000200:2.0:1619677447.893244:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919078fb5400] (22)--
00000800:00000200:2.0:1619677447.893245:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (21)--
00000800:00000200:4.2:1619677447.893246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.893249:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.893252:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:10.0:1619677447.893253:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677447.893253:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.893254:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677447.893255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919078fb5400] (20)++
00000800:00000200:22.0:1619677447.893256:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677447.893256:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.893257:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677447.893257:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.893259:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:17.0:1619677447.893259:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677447.893259:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.893260:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.893260:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.893260:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.893261:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919078fb5400] (21)++
00000400:00000200:10.0:1619677447.893261:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.893262:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.893263:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.893264:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.6@o2ib3
00000400:00000200:17.0:1619677447.893264:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.893264:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:2.0:1619677447.893265:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (22)--
00000800:00000200:22.0:1619677447.893266:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:17.0:1619677447.893266:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677447.893266:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.893267:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.6@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.893268:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.893269:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.893270:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.893271:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677447.893271:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.893272:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.893272:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.893274:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.893275:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.893275:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:18.0:1619677447.893277:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:17.0:1619677447.893278:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677447.893278:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.893279:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:18.0:1619677447.893280:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.893280:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677447.893281:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677447.893282:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.893282:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.893283:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677447.893284:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.893285:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.893285:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677447.893285:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.893286:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.893286:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.893287:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:10.0:1619677447.893287:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.893288:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677447.893289:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.893289:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677447.893290:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677447.893290:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919078fb5400] (21)++
00000800:00000200:10.0:1619677447.893290:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.2:1619677447.893290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.893291:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919078fb5400] (22)--
00000800:00000200:10.0:1619677447.893291:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:18.0:1619677447.893292:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (21)--
00000800:00000200:13.0:1619677447.893296:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677447.893299:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.893302:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.893303:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.893303:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677447.893304:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.893304:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.893304:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:6.0:1619677447.893305:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:13.0:1619677447.893306:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.893308:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:6.0:1619677447.893308:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.893309:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:13.0:1619677447.893309:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:6.0:1619677447.893310:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.893311:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677447.893312:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.893312:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:6.0:1619677447.893312:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:26.0:1619677447.893314:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:21.2:1619677447.893315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677447.893315:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:26.0:1619677447.893318:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:31.0:1619677447.893323:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.893326:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.893328:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.893330:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.893330:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677447.893330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919078fb5400] (20)++
00000400:00000200:31.0:1619677447.893332:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677447.893333:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.893334:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.893335:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.893335:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919078fb5400] (21)++
00000400:00000200:31.0:1619677447.893336:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.893338:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.6@o2ib3
00000800:00000200:18.0:1619677447.893339:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (22)--
00000400:00000200:31.0:1619677447.893341:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.893342:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.6@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.893344:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677447.893346:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677447.893346:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.893347:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:2.0:1619677447.893347:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:31.0:1619677447.893348:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:2.0:1619677447.893348:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.893350:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.893350:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:2.0:1619677447.893350:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.893351:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.893352:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:2.0:1619677447.893352:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677447.893353:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.893354:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:2.0:1619677447.893355:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.893357:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.2:1619677447.893359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677447.893360:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.893360:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.893361:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677447.893363:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677447.893363:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.893363:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919078fb5400] (21)++
00000800:00000200:2.0:1619677447.893364:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919078fb5400] (22)--
00000800:00000200:2.0:1619677447.893365:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (21)--
00000800:00000200:22.0:1619677447.893366:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:30.0:1619677447.893367:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:22.0:1619677447.893367:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.893367:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.893370:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.893370:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677447.893370:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.2:1619677447.893370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677447.893371:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677447.893373:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677447.893378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919078fb5400] (20)++
00000800:00000200:5.0:1619677447.893380:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:6.0:1619677447.893381:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.893382:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.2:1619677447.893383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:5.0:1619677447.893383:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.893384:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.893384:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.893385:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919078fb5400] (21)++
00000800:00000200:6.0:1619677447.893385:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000400:00000200:5.0:1619677447.893385:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.893386:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.893387:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677447.893388:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.6@o2ib3
00000800:00000200:5.0:1619677447.893388:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.893388:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (22)--
00000800:00000200:5.0:1619677447.893389:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000400:00000200:18.0:1619677447.893390:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.6@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.893391:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677447.893391:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.893392:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.893392:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.893393:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000400:00000200:18.0:1619677447.893393:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.893393:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (24)--
00000800:00000200:5.0:1619677447.893394:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.893395:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:20.0:1619677447.893395:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:18.0:1619677447.893395:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.893397:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.893398:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:18.0:1619677447.893398:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.893399:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.893400:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.893400:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.893402:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.893404:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677447.893404:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.893406:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.893407:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.893408:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.893409:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.893410:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.893410:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.893411:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.893411:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677447.893413:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.893413:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.893414:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919078fb5400] (21)++
00000800:00000200:30.0:1619677447.893415:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.893415:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919078fb5400] (22)++
00000800:00000200:18.0:1619677447.893416:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919078fb5400] (23)--
00000800:00000200:18.0:1619677447.893417:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (22)--
00000800:00000200:30.0:1619677447.893418:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.893419:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.893420:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.893421:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:21.2:1619677447.893421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919078fb5400] (21)++
00000800:00000200:20.2:1619677447.893421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:30.0:1619677447.893422:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.893423:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.893423:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.893425:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.893427:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:2.0:1619677447.893427:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919078fb5400] (22)++
00000800:00000200:12.0:1619677447.893428:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.893429:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.893429:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919078fb5400] (23)--
00000800:00000200:5.0:1619677447.893430:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:2.0:1619677447.893430:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (22)--
00000400:00000200:12.0:1619677447.893431:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.893431:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919078fb5400] (21)++
00000800:00000200:20.0:1619677447.893432:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.893432:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.893433:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.893433:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.6@o2ib3
00000800:00000200:18.0:1619677447.893434:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (22)--
00000800:00000200:12.0:1619677447.893434:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.893435:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:5.0:1619677447.893435:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.893436:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:13.0:1619677447.893437:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:2.0:1619677447.893437:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.6@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677447.893438:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.893438:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.893440:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.893440:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.893441:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.893441:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:5.0:1619677447.893441:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.893442:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.893442:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.893443:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.893444:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:2.0:1619677447.893445:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.893446:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.893447:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.893448:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.893449:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.893451:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.893452:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.893453:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.893454:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.893454:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.893456:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.893456:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:2.0:1619677447.893456:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.893457:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:2.0:1619677447.893457:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.893457:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677447.893458:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:2.0:1619677447.893459:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:2.0:1619677447.893460:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919078fb5400] (21)++
00000800:00000200:2.0:1619677447.893460:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919078fb5400] (22)--
00000800:00000200:2.0:1619677447.893461:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (21)--
00000800:00000200:8.2:1619677447.893462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.893470:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000800:00000200:17.0:1619677447.893472:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (24)++
00000800:00000200:1.0:1619677447.893472:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000800:00000200:17.0:1619677447.893473:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000400:00000200:1.0:1619677447.893473:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.893474:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.893475:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.893476:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.893478:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.893479:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.893480:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.893485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.893492:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.893495:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.893495:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.893495:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.893496:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.893497:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.893503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.893510:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.893513:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.893514:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.893516:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.893519:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.893520:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677447.893521:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.893523:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.893524:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.893525:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677447.893528:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.893530:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677447.893533:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.893533:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677447.893534:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677447.893535:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677447.893536:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.893537:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.893537:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677447.893539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919078fb5400] (20)++
00000800:00000200:18.0:1619677447.893545:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919078fb5400] (21)++
00000800:00000200:18.0:1619677447.893547:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.6@o2ib3
00000800:00000200:31.0:1619677447.893548:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:2.0:1619677447.893548:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (22)--
00000400:00000200:18.0:1619677447.893550:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.6@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.893551:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.893551:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.893553:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.893554:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.893555:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:18.0:1619677447.893555:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:31.0:1619677447.893556:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:18.0:1619677447.893556:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.893557:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.893558:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.893559:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677447.893563:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.893565:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677447.893568:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.893568:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:18.0:1619677447.893569:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:18.0:1619677447.893571:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:18.0:1619677447.893571:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919078fb5400] (21)++
00000800:00000200:18.0:1619677447.893572:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919078fb5400] (22)--
00000800:00000200:18.0:1619677447.893572:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (21)--
00000800:00000200:9.2:1619677447.893577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.893584:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.893587:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.893588:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.893588:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.893590:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.893593:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677447.893598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919078fb5400] (20)++
00000800:00000200:2.0:1619677447.893603:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919078fb5400] (21)++
00000800:00000200:2.0:1619677447.893606:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.6@o2ib3
00000800:00000200:18.0:1619677447.893607:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (22)--
00000400:00000200:2.0:1619677447.893608:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.6@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.893611:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.893613:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677447.893614:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.893615:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.893617:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.893617:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677447.893620:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.893622:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677447.893625:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.893626:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:2.0:1619677447.893626:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:2.0:1619677447.893628:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:2.0:1619677447.893629:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919078fb5400] (21)++
00000800:00000200:2.0:1619677447.893629:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919078fb5400] (22)--
00000800:00000200:20.2:1619677447.893630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677447.893630:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (21)--
00000800:00000200:20.0:1619677447.893633:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.893636:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.893638:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.893639:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677447.893641:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:20.0:1619677447.893641:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.893642:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:30.0:1619677447.893643:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:20.0:1619677447.893643:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.893644:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677447.893644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919078fb5400] (20)++
00000400:00000200:20.0:1619677447.893645:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.893646:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.893646:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.893647:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:20.0:1619677447.893647:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:30.0:1619677447.893650:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677447.893650:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919078fb5400] (21)++
00000400:00000200:20.0:1619677447.893651:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.893653:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:18.0:1619677447.893653:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.6@o2ib3
00000800:00000200:2.0:1619677447.893654:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (22)--
00000800:00000200:20.0:1619677447.893656:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.893656:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.6@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.893657:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.893658:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677447.893659:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.893659:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:18.0:1619677447.893659:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.893660:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:18.0:1619677447.893660:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:20.0:1619677447.893661:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.893662:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677447.893662:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722112] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.893663:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.893663:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.893664:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:18.0:1619677447.893664:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.893664:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.893665:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.893666:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:18.0:1619677447.893666:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:8.2:1619677447.893666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:18.0:1619677447.893669:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.893671:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.893672:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.893673:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:1.0:1619677447.893673:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:18.0:1619677447.893674:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.893676:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.893676:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919078fb5400] (21)++
00000800:00000200:1.0:1619677447.893676:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.893677:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.893677:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.893678:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919078fb5400] (22)++
00000800:00000200:21.2:1619677447.893679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919078fb5400] (23)++
00000800:00000200:18.0:1619677447.893679:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919078fb5400] (23)--
00000400:00000200:1.0:1619677447.893679:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.893681:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677447.893682:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (23)--
00000800:00000200:9.2:1619677447.893683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.893685:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919078fb5400] (22)++
00000800:00000200:18.0:1619677447.893687:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.6@o2ib3
00000800:00000200:2.0:1619677447.893687:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:18.0:1619677447.893688:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.6@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.893688:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.893689:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.893691:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.893691:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.893692:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.893693:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:2.0:1619677447.893693:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919078fb5400] (23)++
00000400:00000200:18.0:1619677447.893694:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.893694:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919078fb5400] (24)--
00000800:00000200:2.0:1619677447.893694:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (23)--
00000800:00000200:2.0:1619677447.893695:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:18.0:1619677447.893696:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.893697:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.893697:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (22)--
00000400:00000200:18.0:1619677447.893698:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.893701:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.893702:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.893704:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.893705:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.893705:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.893707:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.893708:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919078fb5400] (21)++
00000800:00000200:18.0:1619677447.893708:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919078fb5400] (22)--
00000800:00000200:18.0:1619677447.893708:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (21)--
00000800:00000200:8.2:1619677447.893713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.893720:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.893723:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.893724:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.893724:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.893726:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.893728:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.893761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677447.893764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677447.893768:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677447.893771:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677447.893771:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.893772:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677447.893774:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677447.893774:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.893776:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.893776:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.893778:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.893778:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.893778:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677447.893779:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.893780:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.893781:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.893781:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.893782:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.893783:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.893784:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.893784:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677447.893785:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677447.893788:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.893789:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.893791:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677447.893791:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677447.893794:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.893795:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677447.893795:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.893795:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:30.0:1619677447.893796:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677447.893796:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677447.893798:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.893798:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677447.893798:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677447.893799:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677447.893799:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677447.893800:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677447.893800:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.893800:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677447.893804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.893811:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.893812:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:12.0:1619677447.893814:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.893815:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.893815:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.893816:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:12.0:1619677447.893816:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.893816:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.893818:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.2:1619677447.893818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:2.0:1619677447.893818:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.893819:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.893822:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.893824:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.893827:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.893827:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.893829:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.893832:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.893833:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.893834:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.893836:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.893837:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.893838:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677447.893841:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.893843:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.893846:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.893847:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.893848:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.893849:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.893850:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.893851:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.893851:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.893855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.893862:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.893864:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.893865:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.893866:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.893868:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.893871:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677447.893911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677447.893917:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.893920:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.893921:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.893923:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.893925:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.893926:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.893928:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.893929:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.893930:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.893931:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.893934:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.893936:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.893938:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.893939:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.893940:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.893941:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.893942:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.893943:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.893944:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.893945:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:8.2:1619677447.893947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677447.893950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677447.893956:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677447.893956:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.893957:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.893958:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677447.893959:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.893959:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.893959:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:5.0:1619677447.893961:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.893962:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.893964:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.893982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:5.2:1619677447.893987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:24.0:1619677447.893989:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.893992:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.893993:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.893994:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.893995:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:24.0:1619677447.893997:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.893997:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.893998:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:24.0:1619677447.893999:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.894001:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.894001:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.894002:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.894003:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.894003:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.894004:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677447.894005:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677447.894006:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.894008:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677447.894008:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.894009:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.894010:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.894011:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:10.0:1619677447.894013:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.894014:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.894014:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.894015:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.894015:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.894017:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677447.894017:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.894018:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.894018:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:10.0:1619677447.894018:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.894019:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677447.894019:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.894020:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.894021:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.894022:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677447.894023:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:8.2:1619677447.894023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677447.894024:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677447.894027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677447.894029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677447.894031:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.894034:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.894034:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.894035:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.894035:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677447.894036:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:17.0:1619677447.894037:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.894037:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.894038:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.894038:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.894039:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677447.894040:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677447.894040:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:26.0:1619677447.894041:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:2.0:1619677447.894041:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.894044:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.894098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.894104:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.894106:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.894107:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.894109:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.894111:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.894113:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.894114:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.894115:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.894116:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.894117:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.894121:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.894123:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.894126:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.894127:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.894127:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.894129:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.894130:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.894130:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.894131:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677447.894142:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.894145:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.894145:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.894147:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.894148:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.894150:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.894165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.894173:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.894175:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.894178:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.894179:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.894180:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.894181:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.894182:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.894184:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.894185:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.894186:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.894189:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.894191:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.894193:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.894194:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.894195:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.894196:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.894197:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.894198:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.894200:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.894200:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677447.894206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677447.894208:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.894210:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.894211:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.894212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:12.0:1619677447.894213:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.894214:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:13.0:1619677447.894214:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.894215:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.894216:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.894217:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.894218:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:10.0:1619677447.894219:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.894222:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.894222:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677447.894224:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.894227:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.894228:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677447.894229:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.894230:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:10.0:1619677447.894231:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.894231:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.894232:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.894233:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:10.0:1619677447.894233:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:13.0:1619677447.894234:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.894235:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.894236:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:10.0:1619677447.894236:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.894238:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.894241:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.894241:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677447.894242:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.894244:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.894245:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.894245:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.894246:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677447.894250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.894257:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.894260:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.894261:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.894261:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.894264:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.894266:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.894271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.894278:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.894280:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.894281:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.894281:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.894283:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.894283:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.894285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.894290:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.894294:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.894298:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.894301:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.894303:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.894305:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.894306:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.894307:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.894308:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.894310:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677447.894313:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.894315:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.894318:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.894318:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.894319:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.894321:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.894321:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.894322:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.894322:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.894327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.894334:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.894337:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.894338:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.894338:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.894340:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.894343:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.894344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677447.894350:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.894353:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677447.894353:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.894354:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.894354:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.894355:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.2:1619677447.894540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.894546:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.894548:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.894548:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.894550:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677447.894551:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.894552:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.894554:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.894555:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.894556:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.894557:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.894558:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.894560:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.894560:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.894563:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.894564:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.894565:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.894566:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.894566:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.894568:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.894568:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.894572:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677447.894572:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.894573:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.894573:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.894573:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.894576:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.894577:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)++
00000800:00000200:1.0:1619677447.894577:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.894578:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (24)--
00000800:00000200:1.0:1619677447.894578:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.894579:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.894579:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.894580:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.894581:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.894582:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.894583:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.894584:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.894585:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677447.894589:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.2:1619677447.894589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677447.894592:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.894593:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.894594:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:30.0:1619677447.894595:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.894595:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677447.894597:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.894597:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:12.0:1619677447.894597:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:30.0:1619677447.894598:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677447.894598:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677447.894598:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.894599:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677447.894600:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.894603:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.894603:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.894611:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.894614:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.894615:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.894616:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.894616:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.894618:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.894624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.894631:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.894634:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.894635:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.894637:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.894640:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.894641:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677447.894642:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.894644:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.894645:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.894646:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677447.894649:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.894650:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.894652:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.894653:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.894654:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.894655:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.894655:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.894656:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.894657:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.894658:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677447.894663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.894668:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.894670:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.894670:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.894671:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.894672:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:10.0:1619677447.894672:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.894673:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.894674:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677447.894674:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.894675:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677447.894797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.894803:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.894806:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.894806:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.894809:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.894811:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.894813:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.894814:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.894815:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.894816:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.894817:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.894821:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.894822:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.894825:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.894826:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.894827:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.894829:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.894829:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.894830:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.894830:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.894835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.894842:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.894845:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.894846:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.894847:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.894848:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.894850:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.894854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.894861:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.894864:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.894865:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.894865:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.894866:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.894867:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.894954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.894961:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.894964:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.894965:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.894967:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.894970:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.894971:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.894973:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.894974:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.894975:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.894976:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.894979:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.894981:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.894984:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.894985:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.894986:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.894987:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.894988:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.894989:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.894989:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.894993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677447.894999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:17.0:1619677447.895001:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.895004:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.895005:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.895005:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.895006:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:17.0:1619677447.895007:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.895009:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.895010:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:17.0:1619677447.895011:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:24.0:1619677447.895012:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.895015:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.895015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677447.895016:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.895018:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.895019:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.895020:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.895021:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:26.0:1619677447.895023:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:24.0:1619677447.895024:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.895025:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:24.0:1619677447.895025:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.895026:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.895028:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.895028:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677447.895029:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.895029:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.895029:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677447.895030:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677447.895031:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.895031:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:26.0:1619677447.895032:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.895032:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:26.0:1619677447.895033:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.895033:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.895033:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:26.0:1619677447.895034:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677447.895036:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.895038:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:9.2:1619677447.895038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.895039:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.895040:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.895040:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.895041:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.895042:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.895042:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.895043:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677447.895045:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.895048:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.895049:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.895049:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.895051:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.895053:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:2.0:1619677447.895054:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677447.895055:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.895056:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.895056:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.895058:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.895059:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677447.895074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.895082:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.895085:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.895087:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.895089:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.895089:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.895090:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677447.895092:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.895093:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.895094:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.895095:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.2:1619677447.895098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:31.0:1619677447.895099:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.895100:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.895103:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.895104:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677447.895104:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.895104:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677447.895106:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677447.895107:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677447.895107:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677447.895108:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.895108:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677447.895108:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.895110:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677447.895112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:1.0:1619677447.895112:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.895114:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.895115:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.895117:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.895118:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.895119:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.895121:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.895122:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.895123:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.895124:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.895124:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.895125:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.895128:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.895129:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.895129:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.895130:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.895130:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.895132:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.895133:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.895134:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.895135:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.895145:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.895148:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.895149:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.895150:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.895151:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.2:1619677447.895152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:12.0:1619677447.895153:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.895160:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.895163:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.895165:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.895167:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.895168:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.895169:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.895171:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.895172:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.895173:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.895174:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.895177:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.895178:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.895182:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.895182:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.895183:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.895185:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.895186:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.895187:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.895187:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.895192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.895199:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.895202:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.895203:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.895205:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.895206:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.895208:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677447.895212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677447.895220:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.895223:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.895224:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.895224:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.895225:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.895226:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.895248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677447.895255:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.895258:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677447.895259:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677447.895260:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.895262:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.895264:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677447.895265:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.895267:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.895268:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.895269:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677447.895272:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.895273:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677447.895277:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.895277:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677447.895278:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677447.895280:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677447.895280:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.895281:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677447.895281:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677447.895286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.895293:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.895300:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.895300:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.895301:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.895302:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.895305:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.895320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.895327:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.895330:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.895331:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.895332:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.895336:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.895337:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677447.895339:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722089] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.895340:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.895340:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.895341:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677447.895344:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.895346:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677447.895348:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.895348:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.895349:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677447.895350:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.895351:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.895351:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.895352:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677447.895364:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.895367:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.895367:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.895369:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.895371:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.895372:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.895383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.895390:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.895393:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.895394:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.895395:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.895397:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.895398:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677447.895399:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.895401:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.895402:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.895402:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677447.895405:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.895406:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.895407:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.895408:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.895408:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.895410:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.895410:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.895411:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.895411:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677447.895421:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.895423:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.895423:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.895424:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.895425:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.895427:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677447.895439:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.895441:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.895442:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.895442:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.895443:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.895443:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677447.895453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.895459:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.895462:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.895463:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.895465:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.895467:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.895468:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.895470:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.895471:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.895472:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.895473:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.895476:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.895478:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.895481:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.895481:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.895482:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.2:1619677447.895483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:6.0:1619677447.895484:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.895485:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.895485:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.895486:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677447.895489:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:9.2:1619677447.895490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:17.0:1619677447.895492:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.895492:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.895495:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.895498:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:17.0:1619677447.895498:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.895499:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:18.0:1619677447.895500:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.895501:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.895501:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.895502:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.895502:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:17.0:1619677447.895503:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.895504:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.895504:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.895507:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:17.0:1619677447.895508:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.895510:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.895514:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.895514:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.895515:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.895517:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.895517:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.895518:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.895519:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:21.2:1619677447.895527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:13.0:1619677447.895530:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.895532:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:13.0:1619677447.895533:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.895534:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.895535:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.895536:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:13.0:1619677447.895536:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.895537:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677447.895538:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:13.0:1619677447.895539:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677447.895540:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.895541:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.895543:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.895544:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.895545:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.895546:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.895549:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.895551:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.895553:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.895554:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.895555:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.895557:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.895557:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.895558:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.895559:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.895570:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.895573:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.895574:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.895574:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.895576:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.895578:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.895669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.895672:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.895675:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.895677:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.895678:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.895679:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.895681:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.895682:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.895683:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.895684:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.895685:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.895689:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.895691:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.895693:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.895694:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.895695:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.895697:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.895697:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.895699:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.895700:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.895700:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:9.2:1619677447.895703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677447.895705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:18.0:1619677447.895710:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677447.895711:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:18.0:1619677447.895713:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:5.0:1619677447.895713:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:18.0:1619677447.895714:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.895714:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:2.0:1619677447.895714:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:5.0:1619677447.895715:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:18.0:1619677447.895717:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.895720:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.895746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.895754:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.895757:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.895760:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.895761:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.895762:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.895764:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.895765:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.895765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:30.0:1619677447.895767:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.895768:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.895769:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.895772:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.895772:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677447.895774:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677447.895774:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.895775:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677447.895777:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677447.895777:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.895778:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.895778:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:10.0:1619677447.895779:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.895780:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:10.0:1619677447.895780:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677447.895781:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677447.895781:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.895782:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.895782:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:10.0:1619677447.895782:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.895783:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.895784:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677447.895787:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677447.895787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677447.895788:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677447.895791:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.895791:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.895792:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677447.895793:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.895794:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.895795:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.895795:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677447.895795:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.895798:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.895798:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.895800:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.895801:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.895804:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677447.895806:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.895808:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.895809:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.895811:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.895812:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.895814:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.895857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.895863:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.895865:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.895866:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.895868:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.895871:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.895873:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.895874:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.895875:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.895876:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.895877:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.895880:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.895882:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.895885:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.895886:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.895886:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.895888:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.895889:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.895889:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.895890:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:21.2:1619677447.895898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:6.0:1619677447.895902:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.895904:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677447.895904:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:22.0:1619677447.895905:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.895905:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.895906:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:6.0:1619677447.895907:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.895908:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.895909:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.895910:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:2.0:1619677447.895912:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.895914:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.895915:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.895918:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.895919:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.895920:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.895923:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.895924:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:22.0:1619677447.895925:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677447.895927:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.895927:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:22.0:1619677447.895928:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.895928:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677447.895928:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677447.895929:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.895930:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677447.895930:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:22.0:1619677447.895931:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.895931:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.895931:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.895932:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677447.895947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.895954:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.895957:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.895957:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.895959:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.895960:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.895961:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677447.896058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.896063:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.896066:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.896067:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.896069:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.896071:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.896072:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.896074:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.896075:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.896076:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.896077:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.896080:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.896082:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.896084:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.896085:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.896086:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.896088:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.896088:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.896089:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.896089:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.896093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.896101:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.896104:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.896104:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.896106:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.896107:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.896108:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677447.896167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.2:1619677447.896170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.896176:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.896177:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677447.896179:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:30.0:1619677447.896179:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.896180:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.896182:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.896184:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.896184:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677447.896185:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.896187:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.896187:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.896189:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677447.896191:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.896192:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.896192:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.896194:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.896194:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:31.0:1619677447.896195:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.896196:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.896197:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.896200:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.896200:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:31.0:1619677447.896201:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.896201:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677447.896202:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.896203:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.896203:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.896204:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.896205:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.896205:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.896206:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.896206:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677447.896207:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.896207:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.896209:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.896209:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.896210:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.896211:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:4.2:1619677447.896212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677447.896214:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:8.2:1619677447.896215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677447.896217:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.896218:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.896218:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677447.896219:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.896222:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:8.0:1619677447.896222:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677447.896223:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.896225:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:8.0:1619677447.896225:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:20.0:1619677447.896226:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.896227:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677447.896228:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:20.0:1619677447.896229:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.896229:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.896231:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677447.896233:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000400:00000200:8.0:1619677447.896238:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.896240:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.896241:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.896241:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:8.0:1619677447.896242:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.896243:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:8.0:1619677447.896244:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.896244:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.896244:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.896245:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.896246:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.896247:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:8.0:1619677447.896248:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.896249:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.896252:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.896253:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.896254:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.896255:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.896256:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.896257:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.896259:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.896259:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.896261:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677447.896265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.896269:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:4.2:1619677447.896271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.896272:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677447.896272:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.896273:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.896273:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.896274:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.896275:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677447.896275:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.896275:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.896278:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:10.0:1619677447.896278:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677447.896280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677447.896281:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.896282:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.896284:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.896286:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.896287:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:1.0:1619677447.896287:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:10.0:1619677447.896288:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.896289:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.896290:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.896291:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:10.0:1619677447.896291:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.896291:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000400:00000200:10.0:1619677447.896292:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.896292:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.896299:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:10.0:1619677447.896301:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.896303:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677447.896305:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.896306:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.896306:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677447.896308:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.896308:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.896309:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677447.896310:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.896311:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677447.896318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677447.896322:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.896324:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.896325:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.896325:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:30.0:1619677447.896327:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.896327:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.896328:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.896328:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677447.896329:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.896330:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677447.896339:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677447.896341:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.896342:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.896343:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.896343:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.896344:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.896362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.896367:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.896370:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.896371:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.896373:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.896375:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.896377:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.896378:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.896379:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.896381:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.896382:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.896385:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.896387:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.896390:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677447.896390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:17.0:1619677447.896391:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.896392:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.896394:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.896394:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.896396:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677447.896396:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.896397:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.896397:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677447.896400:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.896400:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677447.896402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677447.896403:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.896405:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.896407:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.896408:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.896408:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677447.896408:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:18.0:1619677447.896409:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.896410:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.896410:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:18.0:1619677447.896411:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:12.0:1619677447.896411:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.896411:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.896411:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.896412:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677447.896414:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.896415:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.896416:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.896418:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:12.0:1619677447.896418:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677447.896420:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.896421:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.896422:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.896424:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.896424:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.896425:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.896426:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.896427:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677447.896430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677447.896432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.896437:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:5.0:1619677447.896439:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677447.896439:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.896440:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.896441:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.896442:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.896443:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.896444:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.896445:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.896447:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677447.896610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.896617:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.896620:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.896621:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.896623:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.896625:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.896627:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677447.896628:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.896629:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.896630:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.896631:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677447.896634:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.896636:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.896638:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.896639:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.896639:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.896641:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.896641:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.896642:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.896642:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677447.896652:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.896654:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.896654:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.896655:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.896656:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.896657:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677447.896680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.896683:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.896686:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.896688:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.896689:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.896691:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.896692:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.896694:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.896695:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.896696:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.896697:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.896700:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.896702:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.896705:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.896706:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.896707:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.896709:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.896710:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.896711:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.896712:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.896713:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.896717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677447.896721:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.896723:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677447.896724:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.896725:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.896725:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:13.0:1619677447.896726:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.896726:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.896727:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.896728:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.896729:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.896732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.2:1619677447.896736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919874bc4600] (20)++
00000800:00000200:31.0:1619677447.896739:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.896742:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677447.896742:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc4600] (21)++
00000400:00000200:31.0:1619677447.896744:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.896745:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.64@o2ib3
00000800:00000200:30.0:1619677447.896746:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677447.896746:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (22)--
00000400:00000200:31.0:1619677447.896747:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.896748:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.896749:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677447.896750:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.896751:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.896752:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.896753:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.896754:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.896754:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.896755:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:22.0:1619677447.896755:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.896757:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.896758:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.896761:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.896763:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:22.0:1619677447.896764:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.896766:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.896767:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.896768:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677447.896768:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:22.0:1619677447.896769:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.896770:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:22.0:1619677447.896770:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.896771:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677447.896771:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.896773:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677447.896773:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:22.0:1619677447.896773:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919874bc4600] (21)++
00000800:00000200:31.0:1619677447.896774:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.896775:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677447.896775:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919874bc4600] (22)++
00000800:00000200:22.0:1619677447.896776:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919874bc4600] (23)--
00000800:00000200:22.0:1619677447.896776:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (22)--
00000800:00000200:26.0:1619677447.896781:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:21.2:1619677447.896781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677447.896784:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.896785:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.896785:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.896786:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:21.2:1619677447.896786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919874bc4600] (21)++
00000400:00000200:26.0:1619677447.896787:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.896788:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677447.896789:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677447.896789:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.896789:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677447.896789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:31.0:1619677447.896790:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.896791:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.896791:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.896792:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:6.0:1619677447.896792:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc4600] (22)++
00000800:00000200:31.0:1619677447.896793:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677447.896793:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:6.0:1619677447.896793:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919874bc4600] (23)--
00000800:00000200:6.0:1619677447.896794:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (22)--
00000800:00000200:22.0:1619677447.896795:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.896795:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (21)--
00000800:00000200:6.0:1619677447.896796:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.896797:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:22.0:1619677447.896799:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.896802:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.896804:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.896805:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.896806:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.896807:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.896808:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.896811:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.896813:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.896816:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.896817:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.896817:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.896819:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.896820:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.896820:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.896822:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.896822:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:21.2:1619677447.896827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677447.896832:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:6.0:1619677447.896832:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.896834:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:31.0:1619677447.896835:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:6.0:1619677447.896835:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.896835:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:31.0:1619677447.896836:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.896838:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.896839:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.896841:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.896961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677447.896968:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.896971:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.896972:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.2:1619677447.896973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:24.0:1619677447.896974:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.896976:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.896977:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.896979:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.896980:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.896981:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:24.0:1619677447.896981:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.896982:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.896984:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.896986:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.896986:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.896987:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:24.0:1619677447.896988:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.896989:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.896991:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:24.0:1619677447.896991:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677447.896992:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.896992:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.896992:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677447.896994:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.896994:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.896994:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:30.0:1619677447.896995:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.896995:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:24.0:1619677447.896995:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.896995:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:30.0:1619677447.896998:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.897000:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.897003:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.897004:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.897005:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677447.897005:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677447.897006:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.897007:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.897008:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:12.0:1619677447.897008:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.897008:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.897010:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.897010:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:12.0:1619677447.897010:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.897012:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.2:1619677447.897012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677447.897013:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.897016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677447.897021:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.897023:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677447.897024:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:5.0:1619677447.897024:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.897026:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:13.0:1619677447.897026:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677447.897026:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.897027:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.897027:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.897028:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677447.897029:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:13.0:1619677447.897030:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.897030:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.897031:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.897032:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.897033:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.897179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.897185:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.897188:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.897189:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.897191:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.897193:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.897195:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.897196:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.897197:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.897198:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.897199:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.897202:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.897204:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.897206:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.897207:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.897207:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.897209:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.897210:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.897210:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.897211:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.897215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.897223:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.897226:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.897227:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.897227:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.897229:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.897231:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.897234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.897241:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.897244:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.897244:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.897245:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.897246:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.897246:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.897399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.897406:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.897409:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.897410:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.897412:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.897414:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.897416:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.897417:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.897418:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.897419:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.897420:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.897423:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.897425:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.897428:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.897429:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.897430:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.897431:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.897432:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.897432:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.897433:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.897437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677447.897442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.897445:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677447.897445:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677447.897447:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:17.0:1619677447.897448:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.897449:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.897449:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:20.0:1619677447.897450:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.897451:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.897451:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.897452:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.897454:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:17.0:1619677447.897454:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:20.0:1619677447.897455:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.897457:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.897458:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.897459:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.897462:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.897464:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.897467:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.897468:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.897469:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.897471:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.897471:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.897472:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.897473:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.897485:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.897487:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.897488:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.897490:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.897492:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.897493:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.897504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677447.897510:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677447.897513:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677447.897514:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677447.897516:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.897519:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.897520:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677447.897521:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.897523:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.897524:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.897525:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677447.897529:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.897531:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677447.897534:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.897535:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:2.0:1619677447.897535:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:2.0:1619677447.897538:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:2.0:1619677447.897538:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677447.897539:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677447.897540:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677447.897540:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:21.2:1619677447.897546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:31.0:1619677447.897551:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:18.0:1619677447.897551:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:31.0:1619677447.897553:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:18.0:1619677447.897553:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677447.897554:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:31.0:1619677447.897555:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.897555:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:31.0:1619677447.897556:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.897558:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.897560:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.897568:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.897571:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.897572:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.897573:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677447.897573:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.897575:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677447.897695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.897701:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.897703:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.897706:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.897706:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.897709:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.897711:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677447.897712:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.897713:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.897714:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.897715:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.897719:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.897720:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.897723:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.897724:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.897724:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.897726:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.897727:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.897728:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.897728:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677447.897732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677447.897738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:1.0:1619677447.897739:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.897742:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.897743:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.897743:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:1.0:1619677447.897743:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.897745:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.897746:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.897747:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:1.0:1619677447.897747:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:2.0:1619677447.897749:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.897752:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.897753:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.897755:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.897756:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.897757:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.897758:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.897761:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.897763:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.897765:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.897766:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.897767:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.897768:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.897769:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.897769:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.897770:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.897774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.897782:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.897783:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.897784:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.897785:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.897786:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.897788:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.897843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.897850:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.897853:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.897854:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.2:1619677447.897856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:10.0:1619677447.897856:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.897858:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.897860:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.897861:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.897862:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677447.897862:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.897863:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.897864:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.897865:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.897866:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677447.897868:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.897870:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:22.0:1619677447.897871:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.897873:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.897873:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:22.0:1619677447.897874:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.897874:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:22.0:1619677447.897875:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.897876:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.897876:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.897877:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677447.897877:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.897877:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.897878:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677447.897879:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.897880:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.897884:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.897886:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.897890:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.897891:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.897891:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.897892:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.897893:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.897894:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:6.0:1619677447.897894:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.897895:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.897895:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.897896:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:6.0:1619677447.897896:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.897897:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.897900:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677447.897906:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.897910:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.897910:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.897912:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.897913:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.897915:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.897947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.897954:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.897957:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.897959:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.897960:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.897962:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.897963:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677447.897965:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.897966:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.897967:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.897968:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677447.897972:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.897974:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677447.897977:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.897977:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677447.897978:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677447.897980:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677447.897981:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.897981:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.897982:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677447.897986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.897993:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.897996:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.897997:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.897997:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.897999:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.898002:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677447.898106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.898112:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.898114:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.898115:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.898117:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.898120:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.898122:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677447.898123:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.898124:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.898125:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.898126:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677447.898130:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.898132:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677447.898135:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.898135:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677447.898136:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677447.898138:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.898138:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.898139:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.898139:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.898144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677447.898149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:24.0:1619677447.898150:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.898153:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.898154:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.898154:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:26.0:1619677447.898156:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:24.0:1619677447.898157:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.898159:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:24.0:1619677447.898159:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:10.0:1619677447.898160:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.898162:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.898164:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.898164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000400:00000200:26.0:1619677447.898166:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677447.898167:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.898169:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.898169:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.898171:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.898171:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.898173:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000400:00000200:26.0:1619677447.898174:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.898174:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.898174:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:26.0:1619677447.898176:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.898176:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.898176:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:26.0:1619677447.898179:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.898180:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677447.898181:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.898183:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.898183:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.898184:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.898185:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677447.898189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.898197:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.898200:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.898200:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.898201:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.898203:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.898206:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.898207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.2:1619677447.898214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677447.898214:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.898216:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.898217:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.898217:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.898217:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677447.898219:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677447.898219:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.898219:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:20.0:1619677447.898222:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.898223:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.898225:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.898226:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.898228:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.898229:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.898230:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.898231:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.898235:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.898237:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.898240:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.898241:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.898241:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.898243:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.898244:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.898245:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.898245:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677447.898256:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.898259:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.898260:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.898262:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.898263:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.898265:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.898323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677447.898331:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.898334:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677447.898335:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677447.898337:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.898340:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.898341:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677447.898342:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.898344:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.898345:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.898346:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677447.898349:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.898351:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:21.2:1619677447.898352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677447.898354:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.898354:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677447.898355:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677447.898357:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.898357:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.898358:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677447.898358:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:30.0:1619677447.898360:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.898363:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.898365:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.898367:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.898368:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.898369:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677447.898370:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.898371:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.898371:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:30.0:1619677447.898372:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.898373:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:13.0:1619677447.898373:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.898374:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.898376:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.898377:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.898378:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.898379:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:13.0:1619677447.898379:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.898381:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.898382:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677447.898383:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677447.898384:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.898385:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.898386:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.898386:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677447.898393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:22.0:1619677447.898398:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677447.898399:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:22.0:1619677447.898401:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.898402:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.898402:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:22.0:1619677447.898403:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.898405:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.898405:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:5.0:1619677447.898405:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.898408:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.898409:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:5.0:1619677447.898410:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:5.0:1619677447.898411:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.898412:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.898413:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.898414:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:5.0:1619677447.898418:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.898420:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.898421:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.898422:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677447.898423:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.898424:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.898425:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:5.0:1619677447.898426:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708600] (22)++
00000800:00000200:5.0:1619677447.898427:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (23)--
00000800:00000200:5.0:1619677447.898428:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:8.2:1619677447.898431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677447.898434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:5.0:1619677447.898441:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.898442:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (22)++
00000800:00000200:5.0:1619677447.898442:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.898442:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.898443:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708600] (23)--
00000800:00000200:20.0:1619677447.898443:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:5.0:1619677447.898444:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.898445:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.898446:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.898447:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:4.2:1619677447.898451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.898458:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.898460:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.898461:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.898463:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.898466:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.898468:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677447.898469:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.898470:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.898471:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.898472:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677447.898475:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.898477:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677447.898480:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.898481:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677447.898481:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677447.898483:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677447.898484:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.898484:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.898485:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677447.898494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:6.0:1619677447.898497:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.898499:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.898500:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:24.0:1619677447.898501:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:6.0:1619677447.898501:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.898503:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.898504:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.898505:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:6.0:1619677447.898506:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:24.0:1619677447.898507:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.898509:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.898511:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.898512:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.898513:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.898514:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.898515:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.898518:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.898519:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.898521:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.898522:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.898523:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.898524:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.898525:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.898525:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.898526:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.2:1619677447.898528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.898535:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677447.898537:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677447.898538:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.898538:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.898540:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.898540:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.898540:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.898542:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.898542:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.898544:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:13.0:1619677447.898544:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:18.0:1619677447.898545:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.898545:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:18.0:1619677447.898546:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.898547:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.898548:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.898551:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.898553:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.898555:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.898556:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.898557:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.898558:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.898559:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.898559:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.898560:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.898564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.898571:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.898573:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.898574:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.898575:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.898575:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.898577:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.898633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.898640:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.898643:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.898644:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.898646:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.898648:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.898650:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.898651:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.898652:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.898653:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.898654:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.898658:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.898660:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.898663:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.898663:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.898664:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.898666:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.898666:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.898667:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.898667:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.898672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.898680:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.898683:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.898683:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.898684:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.898686:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.898689:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.898692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.898700:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.898702:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.898703:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.898705:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.898708:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.898710:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677447.898711:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.898712:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.898713:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.898714:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677447.898718:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.898719:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677447.898722:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.898722:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677447.898723:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677447.898724:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677447.898725:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.898726:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677447.898727:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677447.898728:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677447.898734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677447.898736:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.898739:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.898740:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.898741:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:13.0:1619677447.898741:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.898743:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:12.0:1619677447.898743:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677447.898744:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:13.0:1619677447.898744:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677447.898745:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.2:1619677447.898780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.2:1619677447.898781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677447.898786:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677447.898789:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677447.898789:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.898790:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.898791:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.898792:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.898794:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.898794:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.898796:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677447.898796:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.898797:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.898797:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.898799:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:17.0:1619677447.898799:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.898800:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.898801:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.898801:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.898802:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.898803:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.898805:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:17.0:1619677447.898806:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.898808:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.898809:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.898811:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.898812:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:17.0:1619677447.898812:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.898813:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.898814:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.898815:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.898815:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:17.0:1619677447.898815:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677447.898816:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:17.0:1619677447.898816:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.898816:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.898818:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677447.898818:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.898819:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.898820:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677447.898824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.2:1619677447.898825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:12.0:1619677447.898827:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.898830:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.898830:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.898832:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.898833:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677447.898833:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:13.0:1619677447.898834:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.898835:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.898836:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:18.0:1619677447.898836:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.898837:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.898837:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:30.0:1619677447.898839:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.898840:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:18.0:1619677447.898840:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.898842:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.898844:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:30.0:1619677447.898845:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.898846:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.898848:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.898849:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.898850:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.898853:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.898855:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.898857:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.898858:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.898858:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.898860:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.898860:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.898861:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.898862:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677447.898872:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.898874:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.898875:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.898877:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.898878:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.898879:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.898888:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.898891:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.898892:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.898892:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.898893:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.898895:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677447.899087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677447.899093:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.899096:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.899097:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.899099:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.899101:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.899102:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:21.2:1619677447.899104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000400:00000200:2.0:1619677447.899104:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.899105:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.899107:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.899108:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.899111:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677447.899112:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000400:00000200:2.0:1619677447.899112:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.899114:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.899114:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.899116:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.899116:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677447.899117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677447.899117:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:18.0:1619677447.899118:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:2.0:1619677447.899118:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677447.899119:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.899120:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.899120:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.899121:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.899122:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:2.0:1619677447.899123:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:26.0:1619677447.899124:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677447.899124:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.899125:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:18.0:1619677447.899126:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.899126:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:26.0:1619677447.899127:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:9.2:1619677447.899127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.899128:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:10.0:1619677447.899128:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.899130:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.899132:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.899132:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.899133:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000400:00000200:26.0:1619677447.899134:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:18.0:1619677447.899134:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:26.0:1619677447.899135:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96722040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.899135:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:26.0:1619677447.899136:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.899136:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677447.899136:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:26.0:1619677447.899137:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.899137:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:2.0:1619677447.899137:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.899138:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.899138:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677447.899138:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:18.0:1619677447.899139:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.899139:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.899141:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:26.0:1619677447.899142:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.899145:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677447.899148:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.899149:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677447.899149:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677447.899150:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677447.899151:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677447.899152:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677447.899152:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:10.0:1619677447.899152:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.899153:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.899154:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:10.0:1619677447.899154:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.899155:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677447.899158:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:13.0:1619677447.899164:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.899166:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.899167:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.899169:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.899170:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.899171:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.899183:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.899186:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.899186:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.899187:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.899187:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.899188:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.899191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.899199:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.899201:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.899204:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.899205:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.899206:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.899207:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.899209:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.899210:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.899211:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.899212:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.899215:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.899216:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.2:1619677447.899217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677447.899218:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.899219:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.899220:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.899221:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.899222:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.899223:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:1.0:1619677447.899223:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677447.899224:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.899225:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677447.899225:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.899226:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.899228:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677447.899231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677447.899231:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.899232:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:12.0:1619677447.899233:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677447.899233:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.899234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.899235:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:1.0:1619677447.899235:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677447.899236:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.899236:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.899236:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:30.0:1619677447.899238:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:12.0:1619677447.899238:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.899240:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677447.899240:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:1.0:1619677447.899240:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.899241:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:12.0:1619677447.899241:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.899242:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.899243:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677447.899243:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:1.0:1619677447.899243:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677447.899244:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677447.899246:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677447.899247:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.899247:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.899247:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:26.0:1619677447.899249:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.899249:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.899250:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677447.899251:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:1.0:1619677447.899251:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.899251:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677447.899252:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.899253:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.899254:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.899255:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:9.2:1619677447.899255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:21.2:1619677447.899256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000400:00000200:26.0:1619677447.899259:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.899260:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.899261:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:24.0:1619677447.899262:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:26.0:1619677447.899263:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.899264:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677447.899264:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:26.0:1619677447.899265:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677447.899265:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.899265:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:6.0:1619677447.899265:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:26.0:1619677447.899266:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:24.0:1619677447.899266:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.899267:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.899267:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677447.899267:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.899268:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:24.0:1619677447.899268:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.899270:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.899271:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:22.0:1619677447.899271:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:22.0:1619677447.899272:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.899274:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.899275:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.899277:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:22.0:1619677447.899281:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.899281:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.899283:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.899284:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:6.0:1619677447.899284:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.899286:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.899288:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.899288:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:22.0:1619677447.899289:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:6.0:1619677447.899289:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.899291:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.899292:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:22.0:1619677447.899292:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.899292:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.899293:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:9.2:1619677447.899298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:6.0:1619677447.899299:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.899300:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.899301:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:6.0:1619677447.899301:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.899303:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677447.899303:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:8.0:1619677447.899305:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.899308:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677447.899309:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.899310:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:21.2:1619677447.899310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (21)++
00000800:00000200:5.2:1619677447.899310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:8.0:1619677447.899311:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.899315:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677447.899315:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:18.0:1619677447.899317:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677447.899318:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:6.0:1619677447.899319:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:18.0:1619677447.899320:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:6.0:1619677447.899320:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:2.0:1619677447.899321:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.899323:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.899326:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.899327:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.899328:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.899329:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.899331:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.899332:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.899335:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.899336:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.899338:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.899339:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.899339:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.899341:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.899341:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.899342:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.899343:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.899347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.899354:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.899357:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.899357:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.899358:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.899359:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.899360:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.899556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.899564:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.899567:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.899569:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.899571:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.899571:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.899573:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677447.899574:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.899575:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.899577:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.899578:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677447.899581:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.899583:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677447.899586:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.899586:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.899587:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677447.899589:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.899589:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.899590:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.899591:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.899595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.899604:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.899607:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.899608:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.899609:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.899610:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.899612:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.899645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:5.2:1619677447.899649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.0:1619677447.899652:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.899655:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677447.899656:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.899657:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.899659:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.899659:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.899659:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677447.899662:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.899662:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.899664:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.899665:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.899665:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.899667:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.899667:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.899668:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.899668:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.899669:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.899670:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.899671:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.899672:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.899675:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.899677:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.899680:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.899682:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.899683:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.899684:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.899685:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.899685:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.899686:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.899686:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.899687:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (23)++
00000800:00000200:17.0:1619677447.899688:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677447.899689:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.899689:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677447.899690:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.899690:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.899690:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677447.899691:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.899692:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.899692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.2:1619677447.899695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.899702:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000800:00000200:17.0:1619677447.899703:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.899703:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.899703:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000800:00000200:17.0:1619677447.899704:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.899704:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.899705:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000800:00000200:17.0:1619677447.899706:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000400:00000200:1.0:1619677447.899706:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.899707:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.899708:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.899708:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.899709:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.899710:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.899711:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.899711:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.899712:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.899712:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.899713:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.899713:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.899926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.899934:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.899937:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.899938:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.2:1619677447.899940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:10.0:1619677447.899940:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.899943:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.899944:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.899945:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.899946:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:10.0:1619677447.899947:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.899947:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.899949:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:10.0:1619677447.899949:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.899949:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.899954:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.899955:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.899956:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.899957:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:10.0:1619677447.899958:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.899959:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.899960:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.899961:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.899962:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.899962:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.899962:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.899963:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.899965:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:18.0:1619677447.899966:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.899967:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677447.899968:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.899968:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677447.899970:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.899970:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677447.899971:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.899972:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.899972:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.899974:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.899974:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.899975:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.899976:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:4.2:1619677447.899976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.899979:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.899983:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.899983:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.899984:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:22.0:1619677447.899984:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.899986:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677447.899986:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.899987:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.899987:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.899988:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677447.899989:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:22.0:1619677447.899989:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677447.899989:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.899990:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.899992:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.899995:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677447.900010:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.900013:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.900014:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.900014:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.900015:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.900015:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677447.900030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.900035:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.900038:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.900039:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.900041:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.900043:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.900045:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.900046:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.900047:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.900048:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.900049:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:5.2:1619677447.900050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:6.0:1619677447.900053:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.900055:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677447.900055:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.900057:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.900058:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:6.0:1619677447.900058:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.900058:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:1.0:1619677447.900059:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.900060:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.900060:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.900060:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.900061:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:17.0:1619677447.900062:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.900062:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:17.0:1619677447.900064:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.900065:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.900066:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677447.900066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:17.0:1619677447.900067:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.900068:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.900071:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.900073:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.900073:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:17.0:1619677447.900076:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.900076:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:17.0:1619677447.900077:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.900077:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:2.0:1619677447.900077:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.900078:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:17.0:1619677447.900079:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.900079:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677447.900079:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.900080:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.900081:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677447.900082:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.900084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.900092:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.900094:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:13.0:1619677447.900094:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677447.900095:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.900095:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.900096:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.900097:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:13.0:1619677447.900097:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.900098:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.900099:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.900101:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.900102:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.900153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.900161:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.900164:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.900166:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.900168:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.900169:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.900171:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.900172:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.900173:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.900174:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.900175:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.900179:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.900181:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.900184:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.900184:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.900185:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.900187:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.900188:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.900188:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.900189:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.900193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.900201:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:21.2:1619677447.900204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677447.900204:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.900204:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.900206:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.900210:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.900211:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.900212:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677447.900214:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.900215:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.0:1619677447.900215:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.900216:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.900216:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000400:00000200:30.0:1619677447.900217:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.900217:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677447.900219:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.900220:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.900222:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677447.900223:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.900224:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.900225:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.900226:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677447.900229:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.900231:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677447.900234:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.900234:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677447.900235:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677447.900237:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677447.900237:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.900238:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.900239:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677447.900243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.900250:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.900253:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.900254:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.900254:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.900256:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.900259:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677447.900276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.900281:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.900284:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.900285:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.900287:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.900290:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.900291:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.900297:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.900298:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.900299:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.900300:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.900303:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.900304:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.900307:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.900308:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.900308:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.900310:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.900310:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.900311:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.900312:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677447.900322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.900325:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.900328:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.900329:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.900329:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677447.900330:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677447.900331:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.900333:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.900334:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677447.900334:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:10.0:1619677447.900336:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.900338:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.900340:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677447.900341:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.900342:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.900343:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.900344:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677447.900347:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.900349:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.900351:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.900351:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677447.900352:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677447.900353:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.900354:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.900355:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.900355:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677447.900365:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.900367:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.900368:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.900368:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677447.900369:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.900371:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677447.900410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.900413:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.900416:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.900418:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.900419:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.900420:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.900422:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677447.900423:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.900424:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.900425:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.900426:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677447.900430:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.900431:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677447.900434:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.900435:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677447.900436:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677447.900438:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677447.900439:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.900439:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.900440:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677447.900451:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.900454:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.900455:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.900457:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.900459:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.900460:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677447.900548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.900555:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.900558:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.900558:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677447.900560:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.900562:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.900563:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677447.900564:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677447.900565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:10.0:1619677447.900566:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.900567:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.900568:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677447.900572:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.900573:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.900574:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.900574:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.900576:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.900577:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677447.900578:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.900578:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.900579:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:26.0:1619677447.900580:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677447.900581:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.900581:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:26.0:1619677447.900582:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.900583:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.900584:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677447.900585:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.900585:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.900586:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677447.900587:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.900588:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.900589:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677447.900591:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.900592:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.900593:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:26.0:1619677447.900593:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677447.900594:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677447.900594:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677447.900595:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:26.0:1619677447.900595:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.900595:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:30.0:1619677447.900596:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.900597:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.900599:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.900600:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:12.0:1619677447.900607:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.900609:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.900610:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.900611:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:12.0:1619677447.900612:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.900613:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.900614:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:12.0:1619677447.900614:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.900615:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.900615:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:13.0:1619677447.900615:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.900616:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.900650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.900657:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.900660:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.900660:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.900662:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.900665:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.900666:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.900668:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.900669:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.900670:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.900671:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:21.2:1619677447.900674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:17.0:1619677447.900674:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.900676:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.900678:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.900678:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.900679:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.900679:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677447.900680:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.900681:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.900681:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.900682:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.900682:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.900683:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.900685:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677447.900687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:6.0:1619677447.900687:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.900688:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.900690:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.900691:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.900692:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.900693:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:1.0:1619677447.900695:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:6.0:1619677447.900696:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.900697:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.900698:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.900698:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:1.0:1619677447.900698:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.900700:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.900701:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.900702:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:1.0:1619677447.900702:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677447.900703:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.900704:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.900705:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.900705:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.900706:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.900710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.900719:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.900721:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.900722:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.900722:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.900725:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.900727:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677447.900745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.900752:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.900755:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.900757:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.900759:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.900760:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.900762:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.900763:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.900765:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.900766:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.900767:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.900770:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.900772:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.900775:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.900776:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.900777:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.900779:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.900779:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.900781:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.900783:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.900783:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677447.900785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677447.900789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677447.900793:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.900795:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.900796:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.900797:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677447.900798:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.900798:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.900799:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.900800:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.900801:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677447.900801:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.900834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677447.900842:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.900844:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677447.900845:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677447.900847:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.900850:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.900851:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677447.900852:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.900852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:10.0:1619677447.900854:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.900855:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.900857:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677447.900859:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.900860:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677447.900861:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.900862:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.900863:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:26.0:1619677447.900864:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.900866:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.900867:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96722016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.900868:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677447.900869:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677447.900869:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:26.0:1619677447.900870:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677447.900870:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:26.0:1619677447.900871:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677447.900872:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677447.900872:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.900874:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:26.0:1619677447.900875:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.900875:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677447.900876:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677447.900877:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677447.900877:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677447.900878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677447.900879:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.900880:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.900880:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.900882:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.900882:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.2:1619677447.900882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.900883:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.900885:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677447.900886:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677447.900888:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677447.900888:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677447.900889:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:20.0:1619677447.900889:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.900892:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.900893:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.900894:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:20.0:1619677447.900894:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.900895:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.900897:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677447.900897:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:30.0:1619677447.900898:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.900899:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.900901:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.900902:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.900995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.901003:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.901006:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.901008:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.901010:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.901011:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.901012:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677447.901013:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.901015:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.901016:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.901017:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677447.901020:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.901021:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677447.901025:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.901025:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677447.901026:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677447.901028:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677447.901028:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.901029:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.901030:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677447.901041:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.901044:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.901045:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.901046:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.901048:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.901050:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.901076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.901083:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.901086:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.901087:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.901088:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.901092:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.901093:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.901094:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96722014] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.901096:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677447.901096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:8.0:1619677447.901097:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.901098:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.901101:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.901102:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677447.901102:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.901105:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:8.0:1619677447.901105:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.901105:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677447.901106:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.901106:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:17.0:1619677447.901107:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.901108:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.901108:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:17.0:1619677447.901109:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.901109:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:17.0:1619677447.901111:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:8.0:1619677447.901111:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.901111:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677447.901112:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.901114:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.901114:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677447.901114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:17.0:1619677447.901115:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:4.2:1619677447.901117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:17.0:1619677447.901120:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.901122:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.901123:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.901124:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677447.901125:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.901125:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677447.901126:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.901126:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677447.901126:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:1.0:1619677447.901126:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.901127:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.901127:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:1.0:1619677447.901128:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.901130:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.901130:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.901131:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.901132:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.901132:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.901133:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.901133:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677447.901138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677447.901142:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:4.2:1619677447.901142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677447.901144:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:1.0:1619677447.901144:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:30.0:1619677447.901145:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.901146:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.901146:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677447.901147:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.901147:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.901148:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.901149:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677447.901150:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677447.901152:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.901152:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.901154:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.901157:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.901158:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677447.901159:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.901161:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.901162:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.901162:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677447.901166:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.901167:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677447.901170:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.901171:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677447.901172:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.901174:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.901174:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.901175:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.901177:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.901177:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677447.901180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.901182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.901188:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677447.901189:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:2.0:1619677447.901191:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.901192:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.901192:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.901192:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677447.901192:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.901194:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:2.0:1619677447.901194:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.901197:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.901199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.901207:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.901209:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.901210:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.901210:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.901211:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.901212:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677447.901296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677447.901302:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.901304:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:21.2:1619677447.901306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:18.0:1619677447.901306:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.901307:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.901310:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.901311:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:6.0:1619677447.901312:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677447.901312:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.901313:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.901315:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.2:1619677447.901315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:6.0:1619677447.901315:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:2.0:1619677447.901315:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.901316:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:6.0:1619677447.901318:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.901320:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.901320:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.901322:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:2.0:1619677447.901322:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677447.901323:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:6.0:1619677447.901323:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96722010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.901325:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.901325:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.901326:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:6.0:1619677447.901326:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.901326:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.901327:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:6.0:1619677447.901328:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:31.0:1619677447.901329:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.901329:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.901329:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:30.0:1619677447.901330:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:2.0:1619677447.901330:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.901331:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000400:00000200:31.0:1619677447.901332:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.901332:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.901332:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:31.0:1619677447.901334:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:6.0:1619677447.901334:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:31.0:1619677447.901335:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96722009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.901336:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677447.901337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:6.0:1619677447.901337:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.901338:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.901338:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000400:00000200:31.0:1619677447.901339:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:6.0:1619677447.901339:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.901340:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.901341:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:31.0:1619677447.901342:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.901342:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:26.0:1619677447.901343:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:18.0:1619677447.901343:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:6.0:1619677447.901343:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.901343:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:31.0:1619677447.901344:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.901345:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:26.0:1619677447.901346:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:18.0:1619677447.901346:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:31.0:1619677447.901347:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.901347:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:26.0:1619677447.901347:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.901347:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.901347:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:31.0:1619677447.901348:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:21.2:1619677447.901348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677447.901349:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:26.0:1619677447.901349:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.901349:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:31.0:1619677447.901350:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.901351:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.901351:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:10.0:1619677447.901351:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677447.901352:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.901353:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677447.901354:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.901354:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:10.0:1619677447.901354:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.901356:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.901357:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:10.0:1619677447.901357:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:6.0:1619677447.901358:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677447.901362:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.901365:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.901366:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.901367:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.901370:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.901371:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677447.901376:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677447.901378:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677447.901379:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677447.901379:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677447.901380:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677447.901381:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677447.901552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.901558:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.901561:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.901562:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.901564:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.901567:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.901569:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.901570:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.901571:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.901572:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.901573:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.901576:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.901578:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.901581:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.901581:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.901582:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.901584:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.901584:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.901585:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.901585:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.901597:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.901600:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.901601:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.901601:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.901603:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.901606:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.901617:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677447.901617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:6.0:1619677447.901619:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.901620:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.901621:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.901622:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.901622:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677447.901623:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.901626:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.901626:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.901629:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.901632:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.901633:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.901635:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.901636:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.901637:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.901638:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.901642:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.901643:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.901646:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.901647:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.901647:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.901649:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.901649:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.901650:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.901651:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.901652:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677447.901657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:8.2:1619677447.901658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.901662:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.901664:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.901664:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.901665:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677447.901665:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.901668:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.901669:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.901669:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.901671:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.901673:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.901674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.901681:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.901683:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.901684:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.901685:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.901686:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.901686:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.901686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.901693:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.901696:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.901697:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.901699:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.901701:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.901703:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677447.901704:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.901705:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.901706:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.901707:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677447.901710:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.901712:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.901713:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.901714:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677447.901715:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677447.901716:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.901717:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.901717:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.901718:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677447.901728:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.901729:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.901730:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.901731:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677447.901732:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.901733:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677447.901745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677447.901751:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.901753:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.901754:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.901756:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.901758:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.901760:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.901761:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.901763:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.901763:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.901765:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.901768:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.901770:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.901772:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.901773:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.901773:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.901775:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.901775:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.901776:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.901777:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.901789:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.901791:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.901792:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.901792:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.901794:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.901796:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.902079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.902087:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.902089:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.902090:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.902092:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.902094:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.902095:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.902096:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.902098:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.902099:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.902100:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677447.902103:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.902105:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677447.902108:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.902109:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.902110:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.902111:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.902112:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.902112:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.902113:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677447.902125:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.2:1619677447.902126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:22.0:1619677447.902128:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:21.2:1619677447.902128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:6.0:1619677447.902128:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.902129:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.902131:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.902133:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:6.0:1619677447.902134:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677447.902134:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677447.902135:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:18.0:1619677447.902137:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.902137:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:20.0:1619677447.902138:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.902138:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.902140:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.902141:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.902143:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.902143:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.902144:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.902145:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677447.902146:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.902146:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96722003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.902147:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.902148:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.902149:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.902149:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.902150:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677447.902154:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000400:00000200:2.0:1619677447.902155:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.902156:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.902158:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.902159:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.902159:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.902160:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.902160:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677447.902161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677447.902161:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.902162:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.902162:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.902163:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.902163:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.902163:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.902164:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:2.0:1619677447.902164:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.902164:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.0:1619677447.902165:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:5.0:1619677447.902165:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677447.902165:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.902166:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:20.0:1619677447.902167:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677447.902168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677447.902170:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.902172:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:8.2:1619677447.902172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:20.0:1619677447.902173:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96722001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.902174:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.902175:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.902175:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:20.0:1619677447.902176:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.902178:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.902179:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.902179:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.902180:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.902180:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.902181:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:1.0:1619677447.902181:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:20.0:1619677447.902182:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.902182:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.902184:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.902184:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677447.902185:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.902186:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.902186:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.902187:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.902187:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.902189:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.902189:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.902190:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677447.902190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.902192:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677447.902194:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.902196:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.902197:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.902197:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.902198:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.902198:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:12.0:1619677447.902201:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.902204:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.902205:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.902207:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.902208:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.902210:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.902221:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.902224:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.902225:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.902225:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.902226:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.902227:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.902245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.902253:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.902255:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.902257:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.902259:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.902259:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.902261:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.902262:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96722000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.902264:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.902265:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.902266:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.902269:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.902271:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.902273:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.902274:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.902275:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.902276:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.902277:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.902278:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.902278:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.902288:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.902290:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.902291:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.902292:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.902294:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.902300:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.902329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.902335:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.902338:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.902339:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.2:1619677447.902339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:22.0:1619677447.902341:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.902343:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.902345:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:17.0:1619677447.902345:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677447.902346:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.902348:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.902348:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.902348:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677447.902350:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.902351:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677447.902352:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.902354:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.902355:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.902356:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:22.0:1619677447.902357:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:17.0:1619677447.902357:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721998] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.902358:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.902360:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677447.902360:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.902361:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.902361:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:17.0:1619677447.902361:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:22.0:1619677447.902363:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.902364:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.902364:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.902365:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:17.0:1619677447.902365:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.902366:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.902370:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.902370:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.2:1619677447.902370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677447.902371:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.902373:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.902373:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.902374:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.902375:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677447.902378:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.2:1619677447.902381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677447.902381:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.902381:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.902383:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.902386:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677447.902386:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.902387:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677447.902389:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677447.902389:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.902390:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.902391:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:13.0:1619677447.902392:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.902393:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:31.0:1619677447.902394:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.902395:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:31.0:1619677447.902396:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.902396:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.902397:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.902399:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.902400:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.902401:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.902402:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.902406:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.902407:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.902410:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.902411:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.902412:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677447.902412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:31.0:1619677447.902414:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.902414:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.902415:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.902416:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:18.0:1619677447.902419:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:8.2:1619677447.902419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.902422:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.902422:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.902424:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.902427:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:18.0:1619677447.902427:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.902428:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:20.0:1619677447.902429:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:18.0:1619677447.902429:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.902430:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.902431:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.902432:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.902433:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.902433:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:18.0:1619677447.902434:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.902438:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.902439:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677447.902440:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.2:1619677447.902440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.902442:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.902443:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.902444:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677447.902446:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:18.0:1619677447.902446:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.902447:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.902447:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.902448:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.902451:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.902452:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:5.0:1619677447.902453:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:5.0:1619677447.902453:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.902454:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.902457:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.902458:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.902460:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.902460:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.902462:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677447.902701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.902709:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.902711:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.902712:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.902715:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.902717:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.902719:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677447.902720:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.902721:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.902722:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.902723:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677447.902727:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.902729:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677447.902732:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.902732:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677447.902733:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.902735:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.902736:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.902737:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677447.902738:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677447.902739:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677447.902741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677447.902744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677447.902749:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.902751:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:10.0:1619677447.902752:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:18.0:1619677447.902753:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.902753:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:2.0:1619677447.902753:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.902754:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:2.0:1619677447.902755:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.902756:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677447.902758:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.902848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.902854:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.902856:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.902858:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.902859:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.902861:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.902863:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.902865:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.902866:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.902867:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.902868:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.902871:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.902873:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.902876:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.902877:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.902877:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.902879:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.902880:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.902881:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.902883:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.902883:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.902888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.902891:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677447.902892:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677447.902893:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.902894:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677447.902894:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677447.902895:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:12.0:1619677447.902895:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.902897:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.902898:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.902900:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.902930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.902938:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.902940:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.902942:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.902943:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.902946:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.902947:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.902949:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.902950:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.902951:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.902952:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.902955:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.902957:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.902960:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677447.902961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:8.0:1619677447.902961:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.902961:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.902963:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.902963:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.902964:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.902965:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677447.902967:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:8.2:1619677447.902969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.902970:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.902970:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.902972:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.902975:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.902976:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.902977:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.902977:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:18.0:1619677447.902979:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.902979:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:18.0:1619677447.902980:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.902980:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:18.0:1619677447.902981:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677447.902981:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.902983:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.902984:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.902986:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677447.902987:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.902989:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.902989:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.902990:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.902992:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.902992:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.902993:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.902994:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.902998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.903004:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.903006:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.903007:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.903008:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.903009:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.903010:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677447.903106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.903113:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.903116:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.903118:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.903119:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.903120:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.903122:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677447.903123:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.903124:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.903125:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.903126:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677447.903129:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.903131:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677447.903133:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.903134:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677447.903135:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677447.903137:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.903137:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.903139:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.903140:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.903141:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677447.903146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677447.903151:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:31.0:1619677447.903154:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677447.903154:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.903154:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.903155:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.903157:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.903157:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:6.0:1619677447.903157:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.903159:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.903160:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.903165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677447.903170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677447.903172:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.903175:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:17.0:1619677447.903175:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677447.903175:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.903178:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.903179:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.903179:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677447.903181:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.903181:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.903182:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677447.903184:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.903184:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.903185:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677447.903186:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.903186:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.903187:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.903188:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.903188:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.903189:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.903191:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677447.903194:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.903195:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.903197:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.903198:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677447.903200:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.903200:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.903201:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.903201:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677447.903202:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.903202:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.903203:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677447.903204:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677447.903204:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677447.903204:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677447.903205:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677447.903205:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.903205:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677447.903206:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677447.903209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677447.903217:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677447.903217:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.903219:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:22.0:1619677447.903220:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677447.903220:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.903220:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.903221:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.903221:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:1.0:1619677447.903222:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.903224:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.903225:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:22.0:1619677447.903227:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677447.903268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.903273:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.903276:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.903277:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.903279:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.903281:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.903283:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.903284:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.903285:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.903286:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.903287:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.903291:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.903292:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.903300:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.903301:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.903302:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.903303:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.903304:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.903304:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.903305:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677447.903318:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.903321:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.903322:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.903323:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.903325:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.903327:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.903347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.903355:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.903358:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.903359:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.903361:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.903364:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.903365:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677447.903366:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.903368:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.903369:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.903370:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677447.903373:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.903375:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677447.903377:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.903378:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677447.903379:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677447.903380:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677447.903380:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.903381:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.903381:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.2:1619677447.903386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:30.0:1619677447.903392:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:18.0:1619677447.903393:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:30.0:1619677447.903394:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.903395:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.903396:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:30.0:1619677447.903397:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.903397:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677447.903399:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.903399:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:18.0:1619677447.903399:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.903401:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.903402:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.903404:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.903405:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.903406:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.903407:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.903410:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.903412:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.903414:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.903415:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.903415:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677447.903417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677447.903417:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.903417:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.903418:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677447.903419:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.903419:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677447.903422:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:9.2:1619677447.903423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677447.903424:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.903425:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.903427:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.903429:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:2.0:1619677447.903429:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:20.0:1619677447.903430:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.903431:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.903431:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:20.0:1619677447.903432:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.903432:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.903433:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.903434:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.903434:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.903436:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.903436:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.903438:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.903441:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.903441:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.903442:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.903444:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.903444:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.903445:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.903446:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.903457:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.903460:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.903461:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.903461:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.903463:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.903466:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.903478:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.903480:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.903481:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.903481:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.903482:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.903483:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.903494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677447.903501:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.903504:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677447.903504:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677447.903506:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.903509:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.903510:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677447.903512:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.903513:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677447.903513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:26.0:1619677447.903514:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.903515:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677447.903518:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.903520:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677447.903520:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677447.903523:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677447.903523:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.903524:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:31.0:1619677447.903525:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.903525:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677447.903527:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.903527:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677447.903527:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.903527:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.903528:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677447.903528:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:31.0:1619677447.903529:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.903530:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.903531:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.903532:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677447.903532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677447.903533:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.903536:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.903538:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.903540:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.903540:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677447.903541:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.903541:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.903543:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.903543:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677447.903543:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:31.0:1619677447.903544:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677447.903544:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.903544:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677447.903545:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:18.0:1619677447.903546:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.903550:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.903556:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.903558:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.903559:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.903561:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.903561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:12.0:1619677447.903563:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.903564:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.903568:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.903570:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.903571:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.903571:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.903572:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.903573:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677447.903776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677447.903782:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.903784:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.903785:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.903787:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.903789:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.903790:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.903791:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.903793:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.903794:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.903795:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.903798:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.903800:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.903802:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.903803:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.903803:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.903805:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.903806:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.903806:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.903807:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677447.903812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.903820:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.903823:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.903824:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.903825:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.903826:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.903828:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.903864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.903871:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.903874:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.903875:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.903877:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.903880:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.903881:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.903883:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.903884:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.903885:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.903886:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.903889:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.903891:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.2:1619677447.903891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:24.0:1619677447.903894:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.903894:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.903895:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.903897:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.903897:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.903898:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.903898:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:30.0:1619677447.903899:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.903901:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:8.2:1619677447.903903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:30.0:1619677447.903904:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.903905:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.903906:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.903907:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677447.903908:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.903909:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.903910:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.903911:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:17.0:1619677447.903911:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:30.0:1619677447.903914:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.903914:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.903915:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.903915:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677447.903916:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:17.0:1619677447.903917:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.903918:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.903919:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.903919:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:17.0:1619677447.903919:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677447.903921:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.903921:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.903922:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.903923:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677447.903933:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.903935:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.903936:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.903938:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.903939:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.903940:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677447.903941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.903947:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.903949:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.903950:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.903952:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.903954:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.903955:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.903957:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.903958:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.903959:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.903960:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.903963:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.903964:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.903966:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.903967:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.903968:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.903969:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.903969:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.903971:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:21.2:1619677447.903972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:1.0:1619677447.903972:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.903973:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677447.903975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677447.903978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677447.903980:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677447.903983:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000800:00000200:17.0:1619677447.903983:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.903984:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:30.0:1619677447.903985:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.903985:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.903985:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.903985:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.903986:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677447.903987:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677447.903987:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.903987:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.903987:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.903988:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677447.903989:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:17.0:1619677447.903989:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:30.0:1619677447.903990:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721978] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.903992:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.903993:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.903994:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677447.903997:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.903998:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.904001:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.904002:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677447.904003:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677447.904004:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.904005:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677447.904006:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677447.904006:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:22.0:1619677447.904018:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.904021:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.904022:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.904023:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.904025:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.904028:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.904066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.904073:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.904076:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.904077:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.904079:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.904082:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.904083:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677447.904084:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.904086:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.904087:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.904087:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677447.904091:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.904092:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.904095:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.904096:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677447.904097:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.904099:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.904099:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.904100:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.904100:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677447.904103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:9.2:1619677447.904105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677447.904110:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.904112:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:2.0:1619677447.904112:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:31.0:1619677447.904114:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.904115:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.904116:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.904116:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.904117:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.904117:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:2.0:1619677447.904118:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.904119:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.904120:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.904122:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.904122:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:31.0:1619677447.904123:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.904124:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.904127:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.904129:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.904131:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.904132:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.904132:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.904134:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.904135:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.904136:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.904136:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.904146:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.904149:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.904150:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.904151:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.904153:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.904154:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.904166:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.904168:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.904169:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.904169:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.904170:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.904171:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.904276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.904282:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.904285:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.904285:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.904287:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.904290:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.904291:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.904292:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.904299:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.904299:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.904301:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.904304:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.904306:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.904309:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.904310:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.904311:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.904312:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.904313:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.904314:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.904314:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:4.2:1619677447.904317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.904325:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677447.904326:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677447.904328:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:13.0:1619677447.904329:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.904329:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.904329:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:13.0:1619677447.904331:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:26.0:1619677447.904332:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.904333:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:26.0:1619677447.904335:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.904335:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677447.904336:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.904337:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.904339:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.904340:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.904341:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677447.904344:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.904345:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.904348:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.904349:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.904350:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677447.904351:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.904352:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.904352:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.904353:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677447.904366:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.904368:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.904369:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.904369:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.904372:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.904374:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677447.904458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:5.2:1619677447.904458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677447.904464:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677447.904465:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677447.904468:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:1.0:1619677447.904468:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:18.0:1619677447.904469:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677447.904469:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:2.0:1619677447.904473:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677447.904473:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.904475:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.904476:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.904476:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677447.904478:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677447.904478:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.904479:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.904480:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.904481:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.904481:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.904482:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.904483:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677447.904484:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677447.904488:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.904489:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677447.904491:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.904492:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677447.904494:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.904495:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.904495:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677447.904496:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:1.0:1619677447.904496:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677447.904497:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677447.904498:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.904499:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677447.904499:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677447.904500:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677447.904500:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677447.904501:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:1.0:1619677447.904501:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677447.904502:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.904503:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.2:1619677447.904508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:12.0:1619677447.904511:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:18.0:1619677447.904514:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:12.0:1619677447.904514:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.904515:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.904516:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677447.904516:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:12.0:1619677447.904516:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.904517:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677447.904518:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:18.0:1619677447.904518:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677447.904518:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:22.0:1619677447.904519:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.904520:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.904520:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.904521:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.904524:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.904663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.904672:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.904675:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.904677:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.904679:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.904679:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.904680:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677447.904682:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.904683:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.904684:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.904685:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677447.904689:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.904690:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.904693:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.904693:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.904694:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.904696:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.904696:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.904698:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.904699:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.904700:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677447.904705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.904710:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:4.2:1619677447.904710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677447.904712:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.904712:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.904714:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.904714:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.904715:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.904715:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.904716:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.904717:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:8.0:1619677447.904717:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677447.904718:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.0:1619677447.904720:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.904721:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.904723:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.904726:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000400:00000200:8.0:1619677447.904726:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.904728:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677447.904728:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:30.0:1619677447.904729:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000400:00000200:8.0:1619677447.904729:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.904730:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677447.904730:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:8.0:1619677447.904730:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.904731:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:8.0:1619677447.904731:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.904732:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.904735:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.904736:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.904739:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.904740:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.904741:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.904742:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.904743:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.904743:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.904744:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.904748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:21.2:1619677447.904755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677447.904756:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.904759:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.904760:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.904760:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677447.904762:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677447.904762:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.904765:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677447.904765:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:31.0:1619677447.904767:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677447.904768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000400:00000200:31.0:1619677447.904769:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.904769:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.904770:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677447.904772:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.904773:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.904774:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.904775:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:1.0:1619677447.904775:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677447.904778:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.904778:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.904778:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.904779:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:31.0:1619677447.904780:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:1.0:1619677447.904780:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.904780:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:31.0:1619677447.904783:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.904784:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677447.904785:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677447.904786:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677447.904787:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.904788:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.904789:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677447.904793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677447.904795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:2.0:1619677447.904801:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:24.0:1619677447.904802:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:2.0:1619677447.904803:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.904804:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.904805:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:18.0:1619677447.904805:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:8.0:1619677447.904805:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:2.0:1619677447.904807:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.904808:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.904810:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:24.0:1619677447.904811:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.904813:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.904814:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.904816:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.904817:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.904818:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.904821:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.904822:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.904825:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.904826:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.904827:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.904828:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.904829:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.904829:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.904830:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.904831:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.2:1619677447.904835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677447.904836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677447.904842:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.0:1619677447.904843:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:20.0:1619677447.904845:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:8.0:1619677447.904845:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.904846:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:20.0:1619677447.904846:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.904846:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:20.0:1619677447.904847:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.904849:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.904850:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677447.904891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:5.2:1619677447.904892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677447.904897:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:17.0:1619677447.904898:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.904901:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:17.0:1619677447.904901:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:2.0:1619677447.904901:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:1.0:1619677447.904901:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677447.904905:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.904906:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.904909:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.904909:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.904911:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.904911:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.904913:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.904913:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.904915:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.904915:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.904916:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.904917:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.904918:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.904918:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.904923:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677447.904923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:18.0:1619677447.904924:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.904925:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.904927:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.904927:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.904928:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.904929:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.904930:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.904930:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.904930:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.904931:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:18.0:1619677447.904931:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.904931:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677447.904931:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677447.904932:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677447.904933:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:18.0:1619677447.904934:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:10.0:1619677447.904934:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.904935:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677447.904935:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677447.904935:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:10.0:1619677447.904936:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677447.904936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:10.0:1619677447.904939:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.904939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:10.0:1619677447.904941:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.904942:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96721965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.904944:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.904944:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:10.0:1619677447.904945:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.904945:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:10.0:1619677447.904946:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677447.904946:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.904947:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.904948:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.904948:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.904949:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.904949:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.904949:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:10.0:1619677447.904950:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.904952:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.904952:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677447.904953:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.904955:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:10.0:1619677447.904956:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.904957:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.904958:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.904959:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.904960:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.904961:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677447.904962:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.904963:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677447.904968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.904973:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677447.904975:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677447.904976:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677447.904977:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677447.904977:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.904977:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.904978:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.904978:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677447.904980:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.904982:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.904996:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677447.904997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:22.0:1619677447.904998:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.904999:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.905000:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.905000:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.905001:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.905003:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.905005:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.905008:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.905008:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.905010:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.905011:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677447.905013:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.905014:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.905015:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.905016:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677447.905020:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.905022:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677447.905025:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.905026:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677447.905026:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677447.905028:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677447.905029:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.905030:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.905031:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.905032:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.2:1619677447.905037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.905039:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:30.0:1619677447.905041:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:20.0:1619677447.905042:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.905042:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.905043:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677447.905044:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.905044:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.905046:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677447.905047:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.905050:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677447.905073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.905080:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.905083:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.905083:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.905086:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.905089:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.905091:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677447.905092:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.905093:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.905094:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.905095:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677447.905098:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.905100:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677447.905102:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.905103:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677447.905104:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677447.905106:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677447.905106:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.905107:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.905107:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677447.905119:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.905121:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.905122:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.905124:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.905125:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.905126:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.905138:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.905141:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.905142:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.905142:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.905143:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.905144:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677447.905272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.905278:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.905280:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.905281:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.905283:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.905285:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.905286:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.905288:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.905289:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.905290:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.905291:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:21.2:1619677447.905293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:17.0:1619677447.905299:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.905299:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677447.905300:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:22.0:1619677447.905302:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.905302:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:17.0:1619677447.905303:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.905304:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.905305:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:6.0:1619677447.905305:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.905307:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.905307:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:6.0:1619677447.905307:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.905308:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.905309:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677447.905309:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.905310:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.905311:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.905312:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.905313:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.905316:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.905318:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677447.905320:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:6.0:1619677447.905320:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.905321:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.2:1619677447.905322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:6.0:1619677447.905322:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677447.905323:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:6.0:1619677447.905323:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:13.0:1619677447.905324:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.905324:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.905325:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.905325:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:13.0:1619677447.905326:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.905327:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.905328:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.2:1619677447.905330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.905331:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.905333:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.905336:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.905338:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.905338:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:1.0:1619677447.905338:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:31.0:1619677447.905340:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677447.905341:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.905341:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.905342:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.905342:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.905343:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.905344:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.905345:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677447.905345:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.905347:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:31.0:1619677447.905349:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677447.905350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677447.905351:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677447.905354:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.905355:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677447.905356:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677447.905357:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677447.905357:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.905358:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.905359:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.905360:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:17.0:1619677447.905360:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.905361:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.905361:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.905362:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.905363:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.905364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.905371:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:4.2:1619677447.905374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.905375:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.905376:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.905378:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.905378:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.905381:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:10.0:1619677447.905382:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.905385:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.905386:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.905388:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.905390:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.905392:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677447.905393:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.905394:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.905395:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.905396:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677447.905400:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.905401:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677447.905403:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677447.905404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677447.905404:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.905405:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677447.905406:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.905407:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.905407:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.905408:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677447.905411:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.905414:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.905416:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.905418:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.905418:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677447.905418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:31.0:1619677447.905419:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.905420:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.905420:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677447.905421:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.905421:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.905422:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.905423:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.905423:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.905424:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.905424:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:30.0:1619677447.905427:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.905427:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.905428:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:31.0:1619677447.905430:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.905430:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:2.0:1619677447.905430:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.905431:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677447.905433:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.905433:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.905433:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.905433:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677447.905434:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.905434:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.905434:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:2.0:1619677447.905434:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.905435:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.905435:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.905436:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:2.0:1619677447.905436:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:30.0:1619677447.905437:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:2.0:1619677447.905437:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.905438:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.905439:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:2.0:1619677447.905439:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.905440:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:2.0:1619677447.905440:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677447.905444:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677447.905445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677447.905446:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.905447:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:2.0:1619677447.905449:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.905450:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:2.0:1619677447.905450:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (23)++
00000800:00000200:2.0:1619677447.905451:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (24)++
00000400:00000200:26.0:1619677447.905452:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.905452:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (24)--
00000800:00000200:31.0:1619677447.905453:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:26.0:1619677447.905454:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.905454:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:2.0:1619677447.905454:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:31.0:1619677447.905455:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:2.0:1619677447.905455:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:31.0:1619677447.905456:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677447.905456:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:31.0:1619677447.905457:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677447.905457:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.905457:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.2:1619677447.905462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677447.905465:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:18.0:1619677447.905467:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:10.0:1619677447.905468:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.905469:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)++
00000800:00000200:18.0:1619677447.905469:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:10.0:1619677447.905469:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.905470:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.905470:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:26.0:1619677447.905471:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)++
00000400:00000200:10.0:1619677447.905471:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.905471:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:26.0:1619677447.905472:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (24)--
00000800:00000200:26.0:1619677447.905473:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.905474:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677447.905474:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.2:1619677447.905582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.905587:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.905590:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.905591:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.905593:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.905595:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.905597:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.905598:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.905600:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.905601:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.905602:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.905605:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.905606:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.905609:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.905610:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.905611:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.905612:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.905613:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.905614:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.905615:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.905616:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677447.905621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677447.905625:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677447.905626:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677447.905627:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677447.905628:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.905628:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677447.905628:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.905629:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677447.905630:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.905631:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.905633:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.905679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.905687:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.905689:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.905690:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.905692:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.905695:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.905696:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.905698:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.905699:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.905700:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.905701:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.905704:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.905706:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.905709:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.905710:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.905711:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.905712:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.905713:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.905714:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.905715:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.905716:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.905718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677447.905721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.905726:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:20.2:1619677447.905728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.0:1619677447.905728:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.905729:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.905730:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.905730:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677447.905730:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.905731:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.905732:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.905732:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.905736:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:1.0:1619677447.905736:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:31.0:1619677447.905739:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.905741:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.905743:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.905744:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.905745:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677447.905747:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.905748:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.905749:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.905750:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677447.905754:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.905755:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677447.905758:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.905759:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677447.905760:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677447.905762:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677447.905762:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.905763:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.905764:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.905768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.905776:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.905779:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.905780:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.905782:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.905782:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.905784:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677447.905786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:4.2:1619677447.905793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.905795:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.905798:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.905799:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.905800:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.905801:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677447.905801:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677447.905802:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:10.0:1619677447.905803:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.905804:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.905806:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.905808:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.905809:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677447.905811:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.905812:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.905813:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.905814:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677447.905817:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.905819:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.905822:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.905823:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677447.905823:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.905825:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.905826:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.905827:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.905827:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677447.905831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.905839:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.905842:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.905843:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.905843:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.905845:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.905848:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677447.905918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.905924:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.905927:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.905927:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.905929:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.905932:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.905933:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:20.2:1619677447.905933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:22.0:1619677447.905935:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.905936:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.905936:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:22.0:1619677447.905937:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.905938:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:20.0:1619677447.905939:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.905942:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.905942:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:22.0:1619677447.905943:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.905944:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.905945:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:20.0:1619677447.905946:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.905947:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.905948:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.905949:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.905949:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:20.0:1619677447.905949:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.905950:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.905951:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000400:00000200:20.0:1619677447.905951:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.905952:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.905953:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.905954:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000400:00000200:20.0:1619677447.905954:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.905955:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677447.905956:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:9.2:1619677447.905957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.905959:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677447.905960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677447.905960:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.905960:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.905962:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.905963:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.905964:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677447.905964:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:20.0:1619677447.905965:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.905965:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.905967:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:2.0:1619677447.905967:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:6.0:1619677447.905968:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677447.905968:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.905969:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.905969:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:2.0:1619677447.905970:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.905973:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:13.0:1619677447.905975:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:9.2:1619677447.905975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:13.0:1619677447.905977:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.905978:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.905978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:13.0:1619677447.905980:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.905981:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.905982:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:13.0:1619677447.905983:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.905984:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.905985:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.905986:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.905987:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:17.0:1619677447.905987:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:18.0:1619677447.905988:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.905988:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:1.0:1619677447.905988:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.905990:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.905992:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.905993:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.905995:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.905996:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.905997:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.905998:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.906002:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.906004:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.906006:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.906007:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.906008:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.906010:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.906010:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.906011:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.906011:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:21.2:1619677447.906015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:9.2:1619677447.906016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.906022:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:30.0:1619677447.906023:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677447.906025:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:5.2:1619677447.906025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677447.906026:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.0:1619677447.906026:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.906027:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.906028:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.906029:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:24.0:1619677447.906029:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.906031:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.906032:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:30.0:1619677447.906033:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:26.0:1619677447.906033:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:9.2:1619677447.906033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000400:00000200:30.0:1619677447.906034:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721949] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.906035:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:30.0:1619677447.906036:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677447.906036:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:30.0:1619677447.906037:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.906039:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677447.906039:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.906039:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.906042:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677447.906043:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.906043:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.906043:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.906044:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:30.0:1619677447.906045:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.0:1619677447.906045:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:26.0:1619677447.906046:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.906047:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677447.906047:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677447.906048:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:26.0:1619677447.906048:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.906049:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677447.906050:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677447.906050:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.906051:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.906051:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677447.906051:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.906051:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.906052:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:26.0:1619677447.906054:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.906056:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677447.906059:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.906060:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677447.906061:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677447.906063:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677447.906063:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.906063:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.906064:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.906065:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:10.0:1619677447.906065:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.906066:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000400:00000200:10.0:1619677447.906066:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.906067:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.906068:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.906069:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677447.906072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677447.906075:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.906077:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.906078:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.906079:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:12.0:1619677447.906079:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.906081:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:13.0:1619677447.906081:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677447.906082:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:13.0:1619677447.906082:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.906082:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.906083:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.906083:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.906083:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677447.906084:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.906085:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.906138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677447.906144:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.906147:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.906148:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.906149:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.906153:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.906154:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.906156:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.906157:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.906158:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.906159:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.906162:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.906164:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.906168:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.906168:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.906170:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.906171:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.906172:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.906173:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.906173:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677447.906184:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.906187:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.906188:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.906190:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.906191:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.906193:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.906266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677447.906272:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:4.2:1619677447.906272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.906274:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.906275:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.906277:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:26.0:1619677447.906279:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:18.0:1619677447.906280:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.906281:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:26.0:1619677447.906282:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.906282:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677447.906283:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.906285:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.906285:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.906286:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677447.906287:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.906289:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.906290:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:26.0:1619677447.906291:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.906292:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721945] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.906292:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:26.0:1619677447.906299:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.906300:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.906300:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677447.906301:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:18.0:1619677447.906301:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.906301:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.906303:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.906303:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.906304:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.906304:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:26.0:1619677447.906305:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.906306:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.906309:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.906310:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.906310:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.906312:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.906313:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.906313:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.906314:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677447.906317:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.906320:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.906321:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.906321:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.906323:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:31.0:1619677447.906325:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677447.906326:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677447.906328:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.906328:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.906330:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.906332:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.906333:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.906450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.906455:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.906458:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.906459:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.906461:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.906463:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.906464:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.906466:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.906467:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.906468:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.906469:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.906473:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.906475:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.906478:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.906479:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.906479:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.906481:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.906482:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.906482:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.906483:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677447.906493:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677447.906496:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677447.906497:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677447.906497:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.906499:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.906502:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677447.906546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.906552:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.906555:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.906556:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.906558:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.906561:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.906562:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.906564:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.906565:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.906566:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.906567:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.2:1619677447.906568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e855c000] (20)++
00000400:00000200:1.0:1619677447.906571:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.906572:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.0:1619677447.906575:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e855c000] (21)++
00000800:00000200:1.0:1619677447.906575:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.906576:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.906576:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.0:1619677447.906578:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.52@o2ib3
00000800:00000200:1.0:1619677447.906578:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.906579:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.906579:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677447.906580:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (22)--
00000400:00000200:5.0:1619677447.906580:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.52@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.906580:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:5.0:1619677447.906583:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.906585:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:5.0:1619677447.906586:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677447.906587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:5.0:1619677447.906587:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.906589:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.906589:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:20.0:1619677447.906590:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:22.0:1619677447.906591:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:20.0:1619677447.906592:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:22.0:1619677447.906594:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:5.0:1619677447.906594:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.52@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.906595:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.906595:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.906595:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.906598:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.906598:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.906598:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:22.0:1619677447.906600:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:20.0:1619677447.906600:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677447.906601:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.906602:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.906603:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.906604:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:5.0:1619677447.906607:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.906608:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.906608:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677447.906609:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677447.906610:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:5.0:1619677447.906611:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677447.906611:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e855c000] (21)++
00000800:00000200:5.0:1619677447.906613:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e855c000] (22)--
00000800:00000200:5.0:1619677447.906613:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (21)--
00000800:00000200:20.0:1619677447.906614:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.906614:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677447.906615:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (22)++
00000800:00000200:5.0:1619677447.906615:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.906617:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (23)--
00000800:00000200:20.0:1619677447.906617:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:9.2:1619677447.906617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (23)++
00000800:00000200:20.2:1619677447.906619:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e855c000] (20)++
00000800:00000200:20.0:1619677447.906622:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.906622:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677447.906623:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e855c000] (21)++
00000800:00000200:24.0:1619677447.906624:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (23)++
00000800:00000200:20.0:1619677447.906625:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.52@o2ib3
00000800:00000200:5.0:1619677447.906625:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (22)--
00000800:00000200:24.0:1619677447.906627:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (24)--
00000400:00000200:20.0:1619677447.906627:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.52@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.906628:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.906628:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (24)++
00000400:00000200:20.0:1619677447.906629:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.906629:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (24)--
00000400:00000200:8.0:1619677447.906629:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.906631:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.906632:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677447.906632:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.906633:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.906635:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.906636:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.906636:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.906639:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:20.0:1619677447.906640:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.52@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.906640:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.906641:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.0:1619677447.906642:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:5.2:1619677447.906642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:20.0:1619677447.906645:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.906645:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:20.0:1619677447.906646:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677447.906647:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677447.906648:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677447.906648:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e855c000] (21)++
00000800:00000200:20.0:1619677447.906649:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197e855c000] (22)++
00000800:00000200:2.0:1619677447.906650:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:20.0:1619677447.906651:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e855c000] (23)--
00000800:00000200:18.0:1619677447.906651:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677447.906652:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (22)--
00000400:00000200:2.0:1619677447.906653:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677447.906656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e855c000] (21)++
00000400:00000200:2.0:1619677447.906656:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.906657:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.906659:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.906660:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.906661:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:2.0:1619677447.906661:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.906662:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e855c000] (22)++
00000400:00000200:2.0:1619677447.906662:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:13.0:1619677447.906663:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:5.0:1619677447.906664:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197e855c000] (23)--
00000400:00000200:13.0:1619677447.906665:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.906665:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (22)--
00000800:00000200:5.0:1619677447.906666:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (21)--
00000400:00000200:2.0:1619677447.906666:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677447.906667:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.906668:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.906668:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:13.0:1619677447.906670:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.906671:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.906672:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.906673:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.906674:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:21.2:1619677447.906675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:2.0:1619677447.906675:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.906677:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.906678:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.906678:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677447.906681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677447.906683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:8.0:1619677447.906687:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:18.0:1619677447.906690:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:8.0:1619677447.906690:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.906691:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677447.906691:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677447.906692:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.906693:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.906693:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:8.0:1619677447.906693:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677447.906694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e855c000] (20)++
00000800:00000200:5.0:1619677447.906694:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:8.0:1619677447.906695:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.906695:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.906697:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.906698:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:8.0:1619677447.906698:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:20.0:1619677447.906699:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e855c000] (21)++
00000400:00000200:8.0:1619677447.906699:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.906700:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.906700:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.906701:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.52@o2ib3
00000800:00000200:5.0:1619677447.906701:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (22)--
00000400:00000200:8.0:1619677447.906702:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.906704:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.52@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.906704:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:20.0:1619677447.906707:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.906707:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.906709:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:8.0:1619677447.906709:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:20.0:1619677447.906710:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.906712:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.906712:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.906714:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.906714:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.906714:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:20.0:1619677447.906715:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.0:1619677447.906716:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.906716:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.906717:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:20.0:1619677447.906718:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.52@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.906718:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677447.906720:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.906723:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.906724:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.906725:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.906726:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.906727:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e855c000] (21)++
00000800:00000200:20.0:1619677447.906728:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e855c000] (22)--
00000800:00000200:12.0:1619677447.906728:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.0:1619677447.906729:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (21)--
00000800:00000200:12.0:1619677447.906731:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.906732:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.906734:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.906735:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.906736:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.906740:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.906742:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.906743:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.906745:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.906746:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.906748:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.906773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.906780:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.2:1619677447.906781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677447.906783:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.906785:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.906787:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.906787:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.906789:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:26.0:1619677447.906789:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677447.906790:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.906791:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:31.0:1619677447.906792:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677447.906792:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:31.0:1619677447.906793:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.906794:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677447.906795:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.906797:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.906798:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.906799:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677447.906800:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:26.0:1619677447.906800:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.906802:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.906802:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.906803:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.906804:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:26.0:1619677447.906804:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.906805:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.906807:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.906807:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677447.906807:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.906808:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.906809:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677447.906809:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677447.906811:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.906812:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677447.906813:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677447.906814:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677447.906815:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.906815:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.906816:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677447.906818:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.906821:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.906822:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.906823:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:20.2:1619677447.906824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e855c000] (20)++
00000800:00000200:13.0:1619677447.906825:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.906826:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.906827:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.906830:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.906830:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.906830:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e855c000] (21)++
00000400:00000200:13.0:1619677447.906832:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.906832:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.52@o2ib3
00000800:00000200:13.0:1619677447.906833:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.906835:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (22)--
00000800:00000200:12.0:1619677447.906835:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677447.906835:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.52@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.906837:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.906839:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677447.906840:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.906842:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:5.0:1619677447.906842:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.906843:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677447.906844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000400:00000200:5.0:1619677447.906844:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:13.0:1619677447.906845:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677447.906846:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:13.0:1619677447.906846:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.906847:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.906847:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677447.906848:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:5.0:1619677447.906848:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.52@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.906849:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.906850:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677447.906853:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677447.906853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.906855:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:5.0:1619677447.906855:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677447.906855:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677447.906857:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677447.906858:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e855c000] (21)++
00000400:00000200:20.0:1619677447.906859:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.906859:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e855c000] (22)--
00000800:00000200:5.0:1619677447.906859:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (21)--
00000800:00000200:5.0:1619677447.906860:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677447.906862:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.906862:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.906863:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:20.0:1619677447.906863:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:10.0:1619677447.906864:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.906865:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:20.0:1619677447.906865:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.906867:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.906867:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.906869:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677447.906869:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.906870:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:20.0:1619677447.906870:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677447.906871:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.906872:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.906873:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:20.0:1619677447.906873:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677447.906873:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677447.906874:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.906874:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96721932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.906876:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677447.906876:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.906876:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:10.0:1619677447.906876:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.906878:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677447.906878:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.906879:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.0:1619677447.906879:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000400:00000200:10.0:1619677447.906879:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.906880:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:26.0:1619677447.906881:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:20.0:1619677447.906881:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000400:00000200:26.0:1619677447.906882:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96721931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.906882:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677447.906883:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677447.906883:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000400:00000200:26.0:1619677447.906884:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.906884:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.906885:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.906886:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.906886:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.2:1619677447.906887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:10.0:1619677447.906890:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677447.906891:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677447.906891:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677447.906892:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.906892:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.906894:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.906895:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.906895:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.906895:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:5.0:1619677447.906895:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:26.0:1619677447.906896:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677447.906897:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677447.906897:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677447.906897:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (23)++
00000800:00000200:5.0:1619677447.906897:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:26.0:1619677447.906898:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677447.906898:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (24)--
00000400:00000200:5.0:1619677447.906898:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.906899:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677447.906899:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.906900:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:10.0:1619677447.906900:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677447.906900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (22)++
00000400:00000200:5.0:1619677447.906900:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.906901:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.906902:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:4.2:1619677447.906904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.906908:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:18.0:1619677447.906911:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:2.0:1619677447.906911:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000800:00000200:18.0:1619677447.906912:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000800:00000200:10.0:1619677447.906912:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:2.0:1619677447.906912:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.906913:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.906914:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:2.0:1619677447.906914:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.906915:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:10.0:1619677447.906915:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.906916:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677447.906918:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.906919:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.906919:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.906923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:21.2:1619677447.906927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:18.0:1619677447.906930:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.906934:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677447.906935:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:18.0:1619677447.906935:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.906935:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.906936:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.906936:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677447.906937:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677447.906939:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.906941:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.906941:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677447.906943:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.906944:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.906945:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.906947:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.906948:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.906951:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.906953:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.2:1619677447.906955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e855c000] (20)++
00000800:00000200:31.0:1619677447.906956:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.906956:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.906957:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677447.906957:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e855c000] (21)++
00000800:00000200:31.0:1619677447.906958:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.906959:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677447.906960:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677447.906960:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:20.0:1619677447.906960:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.52@o2ib3
00000400:00000200:20.0:1619677447.906962:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.52@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.906963:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (22)--
00000400:00000200:20.0:1619677447.906965:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.906966:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677447.906967:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.906969:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.906970:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:20.0:1619677447.906970:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.906971:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:30.0:1619677447.906973:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.906974:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.906974:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.52@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.906976:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.906976:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677447.906977:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.906979:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677447.906980:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e855c000] (21)++
00000800:00000200:20.0:1619677447.906983:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.906984:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677447.906984:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677447.906985:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e855c000] (22)++
00000800:00000200:20.0:1619677447.906986:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.906987:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e855c000] (23)++
00000800:00000200:5.0:1619677447.906987:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.52@o2ib3
00000800:00000200:20.0:1619677447.906988:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e855c000] (24)--
00000800:00000200:20.0:1619677447.906989:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (23)--
00000400:00000200:5.0:1619677447.906989:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.52@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.906990:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (22)--
00000400:00000200:5.0:1619677447.906991:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.906992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:5.0:1619677447.906993:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677447.906994:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.906996:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.906997:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.906998:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.906999:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:5.0:1619677447.907001:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.52@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.907002:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:5.0:1619677447.907003:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:2.0:1619677447.907003:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.907004:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.907005:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.907006:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.907007:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677447.907007:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677447.907007:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677447.907009:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677447.907010:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e855c000] (21)++
00000800:00000200:5.0:1619677447.907011:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197e855c000] (22)++
00000800:00000200:5.0:1619677447.907012:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e855c000] (23)--
00000800:00000200:5.0:1619677447.907013:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (22)--
00000800:00000200:20.2:1619677447.907018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e855c000] (21)++
00000800:00000200:20.0:1619677447.907020:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e855c000] (22)++
00000800:00000200:10.0:1619677447.907021:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:20.0:1619677447.907023:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197e855c000] (23)--
00000800:00000200:20.0:1619677447.907023:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (22)--
00000800:00000200:26.0:1619677447.907024:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677447.907024:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (21)--
00000800:00000200:10.0:1619677447.907024:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.907025:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.907027:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.907031:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677447.907042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e855c000] (20)++
00000800:00000200:20.0:1619677447.907045:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e855c000] (21)++
00000800:00000200:20.0:1619677447.907047:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.52@o2ib3
00000800:00000200:5.2:1619677447.907049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:20.0:1619677447.907050:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.52@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.907052:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.907054:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:5.0:1619677447.907054:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (22)--
00000400:00000200:20.0:1619677447.907055:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.907055:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:20.0:1619677447.907057:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.907058:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.907058:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:20.0:1619677447.907059:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:1.0:1619677447.907059:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.907062:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.907063:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.52@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.907064:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.907066:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:17.0:1619677447.907066:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.907067:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.907069:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677447.907069:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.907070:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:17.0:1619677447.907070:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.907071:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:17.0:1619677447.907071:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677447.907073:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677447.907073:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e855c000] (21)++
00000800:00000200:20.0:1619677447.907074:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e855c000] (22)--
00000400:00000200:17.0:1619677447.907074:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.907075:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (21)--
00000800:00000200:17.0:1619677447.907076:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.907079:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.907079:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.907080:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.907081:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.907082:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.907083:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.907083:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.907085:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.907088:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.907089:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.907091:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.907092:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.907094:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:6.0:1619677447.907095:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.907098:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.907099:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.907099:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.907101:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.907102:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:6.0:1619677447.907103:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:30.0:1619677447.907105:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.907106:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.907106:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677447.907107:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.907109:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:22.0:1619677447.907113:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.907116:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.907117:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.907117:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.907118:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.907119:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.907241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.907248:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.907251:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.907252:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.907254:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.907257:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.907258:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.907260:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.907261:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.907262:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.907263:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.907267:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.907269:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.907272:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.907272:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.907273:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.907275:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.907275:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.907276:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.907276:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677447.907287:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.907289:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.907290:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.907291:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.907293:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.907300:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.907389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.907394:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.907397:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.907398:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.907400:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.907402:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.907404:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.907406:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.907407:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.907408:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.907409:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.907412:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.907414:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.907418:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.907418:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.907419:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.907421:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.907422:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.907422:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.907423:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677447.907435:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.907438:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.907439:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.907441:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.907442:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.907443:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.907444:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:30.0:1619677447.907444:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.907445:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.907445:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.907446:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.907516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677447.907524:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677447.907527:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677447.907529:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.907531:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677447.907531:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.907533:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677447.907534:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721923] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677447.907536:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677447.907537:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677447.907538:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677447.907541:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.907543:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677447.907546:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.907547:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:12.0:1619677447.907548:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677447.907550:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677447.907550:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677447.907551:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677447.907551:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677447.907556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.907565:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.907568:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.907569:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.907569:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.907572:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.907575:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.907618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.907626:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.907629:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.907630:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.907631:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.907634:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.907635:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677447.907636:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.907638:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.907639:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.907640:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677447.907643:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.907645:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677447.907647:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.907648:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677447.907648:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677447.907650:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677447.907650:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.907651:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.907652:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677447.907662:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.907665:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.907666:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.907667:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.907669:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.907670:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677447.907878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.907884:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.907886:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.907887:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.907889:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.907891:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.907893:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.907894:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.907895:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.907896:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.907897:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.907900:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.907902:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.907904:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.907905:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.907905:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.907906:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.907907:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.907908:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.907909:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.907910:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677447.907912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677447.907915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:4.2:1619677447.907916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.907921:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.907922:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.907922:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.907922:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.907922:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.907923:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677447.907923:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.907924:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.907924:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.907925:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.907925:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:26.0:1619677447.907926:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.907927:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.907928:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677447.907928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:26.0:1619677447.907931:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.907932:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.907934:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.907934:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677447.907935:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.907936:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.907936:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:26.0:1619677447.907937:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.907939:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.907939:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.907942:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.907942:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.907943:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.907944:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.907945:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.907946:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.907947:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:5.0:1619677447.907947:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677447.907948:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:5.0:1619677447.907948:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.907949:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:5.0:1619677447.907949:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.907950:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.907951:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.907951:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:5.0:1619677447.907953:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.907954:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.907957:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.907958:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.907959:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.907961:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.907961:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.907962:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677447.907963:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677447.907963:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.907966:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.907967:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.907967:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.907969:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.2:1619677447.907970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:22.0:1619677447.907972:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677447.907973:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.907976:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:18.0:1619677447.907976:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:31.0:1619677447.907977:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.907979:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.907979:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:30.0:1619677447.907980:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:2.0:1619677447.907980:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677447.907982:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:18.0:1619677447.907982:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.907984:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.907986:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.907987:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.907988:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.907989:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.907990:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:21.2:1619677447.907991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:18.0:1619677447.907994:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.907995:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677447.907998:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677447.907998:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.907998:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.907999:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677447.908000:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:18.0:1619677447.908001:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.908001:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:30.0:1619677447.908002:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.908002:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.908002:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:31.0:1619677447.908004:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.908004:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.908006:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.908007:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677447.908007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677447.908008:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.908009:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.908010:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.908014:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.908014:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:30.0:1619677447.908016:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.908016:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.908017:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.908018:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677447.908019:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.908020:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:2.0:1619677447.908020:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.908021:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.908021:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677447.908022:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.908023:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.2:1619677447.908023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:30.0:1619677447.908024:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.908024:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677447.908029:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.908031:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.908032:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.908034:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.908034:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:18.0:1619677447.908035:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.908037:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:13.0:1619677447.908037:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:18.0:1619677447.908038:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721916] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.908038:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.908039:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.908040:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.908040:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:18.0:1619677447.908041:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.908041:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:18.0:1619677447.908042:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.908045:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.908047:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.908048:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.908049:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.908049:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.908051:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.908051:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.908052:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.908053:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.908057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.908064:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.908065:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.908066:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.908067:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.908068:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.908070:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677447.908153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677447.908160:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677447.908163:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677447.908163:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677447.908166:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.908169:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.908170:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677447.908171:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.908173:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.908174:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.908175:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677447.908178:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.908180:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677447.908182:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.908183:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677447.908184:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677447.908185:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677447.908186:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677447.908187:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677447.908188:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677447.908189:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:21.2:1619677447.908194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:31.0:1619677447.908197:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.908200:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.908201:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.908201:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:31.0:1619677447.908202:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.908203:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:30.0:1619677447.908204:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677447.908204:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:21.2:1619677447.908204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:31.0:1619677447.908205:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:26.0:1619677447.908206:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:6.0:1619677447.908209:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.908212:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.908213:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.908214:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.908217:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.908219:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.908220:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.908221:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.908222:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.908223:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.908227:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677447.908227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:6.0:1619677447.908229:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.908232:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.908232:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.908233:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:24.0:1619677447.908234:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677447.908235:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.908236:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.908236:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677447.908237:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:6.0:1619677447.908237:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677447.908238:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.908240:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.908242:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.908244:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.908245:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.908246:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.908247:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.908248:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:24.0:1619677447.908248:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:30.0:1619677447.908250:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.908251:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.908251:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.908253:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.908253:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677447.908254:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.908255:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:24.0:1619677447.908256:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.908257:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.908258:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.908259:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.908260:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.908260:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.908261:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677447.908264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.2:1619677447.908265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677447.908272:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:1.0:1619677447.908273:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:31.0:1619677447.908274:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.908276:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.908276:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.908277:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.908277:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.908278:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.908279:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.908280:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677447.908280:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.908281:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.908283:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.908283:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:31.0:1619677447.908284:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.908285:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.908288:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.908289:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.908293:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.908293:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.908298:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.908300:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.908301:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.908302:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.908303:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.908304:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677447.908307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677447.908311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677447.908319:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677447.908320:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677447.908321:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677447.908322:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:20.0:1619677447.908322:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.908323:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.908324:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677447.908324:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.908326:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.908327:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677447.908373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.908381:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.908383:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.908384:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.908386:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.908389:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.908390:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677447.908391:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.908392:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.908393:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.908394:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677447.908397:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.908399:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677447.908401:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.908402:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677447.908402:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677447.908404:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677447.908404:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.908405:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.908405:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677447.908416:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.908419:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.908419:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.908421:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.908423:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.908424:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677447.908487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.908493:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.908496:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.908498:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.908498:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.908501:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.908503:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.908504:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.908505:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.908506:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.908507:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.908511:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.908513:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.908516:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.908517:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.908517:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.908519:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.908520:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.908521:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.908522:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.908523:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.908528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.908531:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677447.908532:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.908534:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.2:1619677447.908534:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:31.0:1619677447.908536:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.908536:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (23)--
00000400:00000200:31.0:1619677447.908537:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.908537:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.908538:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677447.908539:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.908540:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677447.908541:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:5.2:1619677447.908542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:26.0:1619677447.908544:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677447.908544:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677447.908546:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677447.908548:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.908548:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:26.0:1619677447.908550:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:18.0:1619677447.908551:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:26.0:1619677447.908552:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.908552:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:26.0:1619677447.908554:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.908554:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.908555:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.908556:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.908557:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.908558:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.908559:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721908] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.908561:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000400:00000200:18.0:1619677447.908561:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.908563:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.908564:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677447.908565:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677447.908568:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.908568:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.908569:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677447.908570:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.908570:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677447.908572:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677447.908572:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.908572:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:18.0:1619677447.908572:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.908573:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:18.0:1619677447.908573:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.908573:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.908575:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.908575:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.908576:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.908577:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.908577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.2:1619677447.908582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.908585:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.908588:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:17.0:1619677447.908589:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:5.0:1619677447.908589:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.908590:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677447.908591:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677447.908592:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.908593:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.908593:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.908593:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.908595:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.908597:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.908625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.908630:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.908633:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.908634:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.908636:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.908637:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.908639:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.908640:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.908642:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.908643:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.908644:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.908647:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.908649:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.908652:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.908653:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.908654:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.908656:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.908656:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.908657:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.908657:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.908669:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.908672:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.908673:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.908674:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.908676:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.908677:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.908747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677447.908750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677447.908753:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.908756:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:30.0:1619677447.908757:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677447.908757:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677447.908759:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677447.908759:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.908761:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.908761:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.908763:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677447.908763:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677447.908764:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.908764:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.908766:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.908766:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.908768:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.908768:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.908769:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.908769:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677447.908771:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.908772:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677447.908773:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.908775:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:30.0:1619677447.908777:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.908778:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.908779:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677447.908779:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.908779:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.908781:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.908782:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.908782:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.908782:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.908782:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677447.908783:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677447.908783:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677447.908785:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (23)--
00000800:00000200:30.0:1619677447.908785:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.908786:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.908787:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677447.908795:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:22.0:1619677447.908798:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:6.0:1619677447.908798:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (24)++
00000800:00000200:6.0:1619677447.908799:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000400:00000200:22.0:1619677447.908800:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.908800:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.908802:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.908802:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.908806:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.908807:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.908807:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.908810:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.908810:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.908811:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.908812:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.908812:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.909023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.909026:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.909029:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.909031:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.909032:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.909034:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.909035:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.909037:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.909038:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.909039:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.909040:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.909044:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.909045:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.909049:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.909050:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.909051:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.909053:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.909054:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.909054:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.909055:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.909065:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.909068:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.909069:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.909071:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.909072:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.909074:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.909078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.909083:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677447.909085:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.909086:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:12.0:1619677447.909087:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.909087:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.909089:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.909089:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.909089:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.909090:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:17.0:1619677447.909092:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.909092:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:17.0:1619677447.909093:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.909094:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.909096:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.909097:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.909098:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.909101:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.909103:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.909105:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.909105:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.909106:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.909108:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.909108:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.909109:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.909110:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.909114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.909121:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.909123:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.909124:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.909124:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.909126:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.909127:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677447.909144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.909152:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.909155:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.909157:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.909159:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.909160:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.909161:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.909162:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.909164:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.909164:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.909166:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.909169:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.909170:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.909173:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.909174:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.909174:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.909176:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.909176:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.909178:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.909179:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.909180:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677447.909186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677447.909188:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.909190:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.909191:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.909193:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:13.0:1619677447.909193:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.909194:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.909195:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:13.0:1619677447.909195:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.909196:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.909197:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677447.909271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677447.909278:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.909281:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.909282:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.909284:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.909286:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.909288:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.909289:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677447.909290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:8.0:1619677447.909291:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.909292:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.909298:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.909299:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:8.0:1619677447.909301:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.909302:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.909303:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.0:1619677447.909303:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.909304:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:8.0:1619677447.909306:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.909307:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.909307:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677447.909307:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.909309:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.909309:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:2.0:1619677447.909309:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:8.0:1619677447.909310:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:2.0:1619677447.909310:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.909311:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:2.0:1619677447.909311:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.909312:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.909313:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:9.2:1619677447.909316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677447.909317:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.909319:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.909322:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.909323:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.909325:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:2.0:1619677447.909325:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000800:00000200:5.2:1619677447.909326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.909327:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (25)--
00000800:00000200:2.0:1619677447.909327:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (25)--
00000400:00000200:18.0:1619677447.909328:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.909328:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.909329:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.909329:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:18.0:1619677447.909330:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.909331:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:2.0:1619677447.909331:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000400:00000200:2.0:1619677447.909332:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.909333:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.909333:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.909334:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.909335:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.909336:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.909336:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.909337:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.909338:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.909340:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.909342:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.909343:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.909344:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.909345:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.909346:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:4.2:1619677447.909346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:18.0:1619677447.909349:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677447.909350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.909351:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.909354:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677447.909354:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.909355:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.909356:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.909357:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.909357:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677447.909358:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.909359:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.909359:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:24.0:1619677447.909360:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.909360:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.909360:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677447.909360:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.909361:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000400:00000200:24.0:1619677447.909362:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.909362:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.909362:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.909363:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:24.0:1619677447.909364:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.909365:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.909366:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.909367:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.909368:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.909371:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.909371:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677447.909373:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677447.909374:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.909375:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.909376:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.909377:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:12.0:1619677447.909377:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.909378:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:13.0:1619677447.909378:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.909379:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:12.0:1619677447.909379:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677447.909380:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.909380:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.909381:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.909385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.909394:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.909396:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.909397:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.909398:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.909400:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.909402:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.909405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.909412:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.909415:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.909416:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.909416:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.909417:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.909417:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677447.909542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.909550:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.909553:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.909555:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.909557:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.909558:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.909559:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677447.909560:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.909561:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.909562:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.909563:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677447.909566:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.909568:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677447.909571:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.909571:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677447.909572:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677447.909574:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677447.909574:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.909575:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.909575:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677447.909580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.909587:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.909590:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.909591:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.909591:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.909593:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.909596:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.909598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.909605:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.909608:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.909608:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.909609:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.909610:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.909610:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.909647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.909653:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.909655:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.909656:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.909658:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.909659:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.909661:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677447.909662:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.909663:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.909664:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.909665:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677447.909669:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.909670:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677447.909673:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.909674:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677447.909675:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677447.909677:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.909677:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.909678:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.909678:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.909683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.909689:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:20.2:1619677447.909690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.0:1619677447.909692:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.909693:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.909693:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.909695:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.909698:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:8.0:1619677447.909698:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:31.0:1619677447.909701:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.909703:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.909705:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.909706:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.909707:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677447.909709:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721895] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.909710:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.909711:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.909712:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677447.909715:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.909717:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677447.909720:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.909721:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677447.909722:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677447.909723:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677447.909724:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.909725:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.909725:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.909730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.909738:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.909741:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.909742:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.909743:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.909744:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.909747:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.909819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.909827:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.909830:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.909831:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.909833:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.909836:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.909837:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.909839:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.909840:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.909841:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.909841:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677447.909845:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.909846:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.909849:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.909850:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.909850:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677447.909852:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.909852:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.909854:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.909855:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.909856:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677447.909861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.909867:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677447.909868:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677447.909870:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.909870:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677447.909870:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677447.909871:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677447.909871:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.909872:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677447.909873:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.909876:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.909934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.909937:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.909940:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.909942:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677447.909944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.0:1619677447.909944:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.909945:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.909946:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.909947:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.909949:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.909950:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.909951:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677447.909951:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677447.909954:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:20.0:1619677447.909955:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.909957:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.909957:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677447.909958:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.909959:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.909960:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677447.909961:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.0:1619677447.909961:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:30.0:1619677447.909962:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.909962:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.909963:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:30.0:1619677447.909964:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.909964:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:30.0:1619677447.909965:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.909965:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.909965:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:30.0:1619677447.909966:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677447.909969:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.909970:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.2:1619677447.909970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677447.909973:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.909974:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677447.909975:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677447.909976:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677447.909977:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.909978:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.909978:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677447.909978:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.2:1619677447.909979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677447.909981:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.909982:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.909982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.909982:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.909984:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.909987:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.909987:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.909989:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.909992:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:1.0:1619677447.909992:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.909993:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.909995:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.909995:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.909996:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.909997:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.909998:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.910000:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.910001:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.910002:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.910005:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.910006:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.910006:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.910007:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.910008:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.910011:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.910012:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.910013:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.910014:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.910015:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.910016:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.910017:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.910018:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677447.910023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677447.910027:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677447.910029:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677447.910030:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:17.0:1619677447.910031:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.910031:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677447.910031:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.910032:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677447.910032:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.910033:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.910035:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677447.910065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.910073:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.910075:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.910076:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.910078:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.910081:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.910082:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677447.910083:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.910084:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.910085:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.910086:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677447.910089:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677447.910089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:26.0:1619677447.910091:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677447.910093:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.910094:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677447.910095:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677447.910096:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677447.910096:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677447.910097:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677447.910098:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:31.0:1619677447.910099:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:26.0:1619677447.910099:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677447.910099:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:31.0:1619677447.910101:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.910103:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.910104:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.910105:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:5.2:1619677447.910105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677447.910106:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.910107:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.910108:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.910109:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:12.0:1619677447.910109:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.910111:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:31.0:1619677447.910112:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677447.910112:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.910112:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:12.0:1619677447.910113:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.910114:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677447.910114:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:13.0:1619677447.910115:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677447.910115:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:12.0:1619677447.910116:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677447.910116:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677447.910117:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.910118:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677447.910118:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677447.910120:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677447.910121:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.910121:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.910122:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.910126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677447.910130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:5.0:1619677447.910134:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:24.0:1619677447.910137:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.0:1619677447.910137:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.910138:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.910139:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.910140:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:20.0:1619677447.910140:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.910141:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677447.910142:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:24.0:1619677447.910143:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.910145:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.910147:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.910148:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.910150:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.910151:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.910152:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.910155:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.910156:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.910159:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.910160:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.910160:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.910162:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.910163:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.910163:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.910164:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.910168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.910176:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.910178:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.910179:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.910180:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.910182:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.910185:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.910391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.910397:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.910399:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.910400:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.910402:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.910405:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.910406:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.910408:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.910409:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.910410:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.910411:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.910413:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.910415:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.910418:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.910419:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.910420:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.910422:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.910422:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.910423:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.910423:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.910436:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.910439:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.910440:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.910441:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.910443:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.910444:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.910475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.910482:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.910485:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.910486:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.910488:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.910490:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.910492:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.910493:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.910494:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.910495:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.910496:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.910499:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.910501:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.910503:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.910504:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.910505:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.910506:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.910507:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.910507:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.910508:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.2:1619677447.910517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:21.2:1619677447.910520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:30.0:1619677447.910521:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.910523:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.910523:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.910525:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:30.0:1619677447.910525:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.910527:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:30.0:1619677447.910527:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.910527:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677447.910528:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:31.0:1619677447.910529:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.910530:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:22.0:1619677447.910530:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.910530:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:31.0:1619677447.910532:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.910533:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:22.0:1619677447.910533:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.910535:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.910535:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.910536:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.910537:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677447.910538:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.910538:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.910539:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.910540:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:22.0:1619677447.910540:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.910542:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.910542:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.910543:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.910543:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:22.0:1619677447.910543:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677447.910544:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.910544:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.910545:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677447.910547:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:22.0:1619677447.910547:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.910549:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.910550:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:22.0:1619677447.910550:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.910551:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.910552:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.910553:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.910553:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:22.0:1619677447.910553:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.910553:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677447.910554:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:22.0:1619677447.910554:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.910556:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.910556:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.910558:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.910559:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.910559:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677447.910563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677447.910564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:12.0:1619677447.910564:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.910566:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.910567:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.910569:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.910570:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:12.0:1619677447.910571:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.910571:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:13.0:1619677447.910572:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.910572:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.910572:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.910573:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677447.910573:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.910574:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.910575:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.910576:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.910579:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:12.0:1619677447.910585:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.910588:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.910588:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.910589:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.910590:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.910591:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677447.910643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.910651:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.910653:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.910655:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.910657:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.910657:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.910659:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677447.910660:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.910662:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.910663:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.910664:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677447.910667:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.910669:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677447.910672:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.910673:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677447.910673:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677447.910675:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.910675:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.910676:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.910677:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677447.910687:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.910689:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.910690:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.910690:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.910692:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.910694:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677447.910704:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.910706:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.910707:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.910707:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.910708:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.910709:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677447.910772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.910778:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.910781:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.910782:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.910783:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.910786:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.910787:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.910788:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.910790:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.910791:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.910792:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.910795:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.910797:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.910800:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.910801:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.910802:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.910803:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.910804:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.910804:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.910805:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.910816:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.910819:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.910819:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.910821:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.910822:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.910824:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.910834:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.910836:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.910837:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.910838:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.910838:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.910840:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.910914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.910921:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.910923:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.910924:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.2:1619677447.910926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:10.0:1619677447.910926:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.910928:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.910930:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677447.910931:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.910932:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.910932:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:10.0:1619677447.910933:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.910934:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.910935:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.910936:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.910939:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.910940:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.910942:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.910943:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 280 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.910944:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.910945:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.910947:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677447.910947:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677447.910948:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:2.0:1619677447.910948:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677447.910949:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:2.0:1619677447.910949:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:10.0:1619677447.910950:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677447.910951:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.910952:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.910952:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:2.0:1619677447.910952:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.910954:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.2:1619677447.910956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.910957:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.910958:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.910958:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.910960:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.910961:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.910961:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.910962:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677447.910964:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:8.2:1619677447.910966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.910967:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.910967:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.910970:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.910971:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.910972:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.910973:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.910975:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.910976:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.910978:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.910978:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.910981:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677447.910982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:8.2:1619677447.910987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.910990:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.910992:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.910993:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.910994:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.910994:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.910995:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.910996:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.910996:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.910997:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.910997:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.910998:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.910998:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.911026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.911033:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:4.2:1619677447.911035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677447.911036:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.911037:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.911039:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.911042:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.911042:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677447.911043:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677447.911045:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.911045:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.911045:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:26.0:1619677447.911046:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.911048:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.911048:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.911049:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.911051:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.911052:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.911052:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.911053:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.911054:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 280 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.911055:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.911056:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.911057:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677447.911057:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:26.0:1619677447.911058:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677447.911059:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677447.911060:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:8.0:1619677447.911060:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.911061:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.911061:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677447.911061:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677447.911062:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.0:1619677447.911064:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.911065:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.2:1619677447.911066:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (22)++
00000800:00000200:8.0:1619677447.911070:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (23)++
00000800:00000200:8.0:1619677447.911071:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (24)--
00000800:00000200:8.0:1619677447.911072:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.911073:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.911074:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:1.0:1619677447.911074:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (23)++
00000800:00000200:8.0:1619677447.911075:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677447.911077:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (24)--
00000800:00000200:17.0:1619677447.911078:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (24)++
00000400:00000200:1.0:1619677447.911078:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.911079:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (24)--
00000400:00000200:17.0:1619677447.911080:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.911080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:17.0:1619677447.911081:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.911081:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.911084:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.911085:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.911086:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:24.0:1619677447.911087:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.911090:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.911091:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.911092:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677447.911100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.911108:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.911111:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.911113:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.911115:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.911116:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.911117:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.911119:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.911120:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.911121:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.911122:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.911125:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.911127:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.911129:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.911130:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.911130:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.911132:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.911132:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:4.2:1619677447.911133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677447.911134:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.911136:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.911136:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677447.911140:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.2:1619677447.911142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677447.911143:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.911144:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:13.0:1619677447.911144:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:21.2:1619677447.911145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:13.0:1619677447.911146:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:10.0:1619677447.911146:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677447.911147:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.911148:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:10.0:1619677447.911148:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.911149:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:12.0:1619677447.911150:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.911151:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:13.0:1619677447.911151:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:10.0:1619677447.911151:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.911152:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:10.0:1619677447.911152:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.911153:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:10.0:1619677447.911153:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.911153:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.911154:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.911155:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677447.911155:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677447.911156:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677447.911157:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.911159:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:31.0:1619677447.911160:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677447.911161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:10.0:1619677447.911161:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.911162:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.911162:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.911162:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:31.0:1619677447.911164:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677447.911164:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.911164:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677447.911165:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.911166:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.911166:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677447.911166:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677447.911167:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677447.911167:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.911167:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.2:1619677447.911167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:31.0:1619677447.911168:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.911169:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.911170:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677447.911172:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677447.911173:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677447.911173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677447.911174:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.911175:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677447.911176:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.911176:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:22.0:1619677447.911176:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.911176:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677447.911177:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.911178:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677447.911178:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.911178:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.911179:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.911179:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677447.911179:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677447.911180:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.911180:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:22.0:1619677447.911180:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96721874] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.911181:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:17.0:1619677447.911181:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.911182:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.911182:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.911182:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677447.911182:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.911183:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.911183:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677447.911183:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677447.911183:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.911183:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:10.0:1619677447.911184:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677447.911185:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:22.0:1619677447.911185:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:17.0:1619677447.911185:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.911186:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677447.911187:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.911188:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.911190:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.911190:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:21.2:1619677447.911191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677447.911193:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:17.0:1619677447.911194:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.911195:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.911196:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:17.0:1619677447.911196:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:22.0:1619677447.911197:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677447.911198:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677447.911198:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.911199:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677447.911199:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.911200:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.911200:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677447.911200:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.911200:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.911201:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.911201:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.911202:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677447.911202:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.911203:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.911203:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677447.911203:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677447.911204:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.911205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:12.0:1619677447.911206:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.911207:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.911209:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.911211:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.911212:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:13.0:1619677447.911212:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.911213:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:18.0:1619677447.911215:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.911215:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.911216:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.911216:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.911216:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.911218:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.911219:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.911219:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.911221:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:12.0:1619677447.911222:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.911246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.911249:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.911252:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.911254:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.911255:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.911257:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.911258:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:20.0:1619677447.911262:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677447.911263:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.911263:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:20.0:1619677447.911264:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.911265:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.911266:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.911267:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.911270:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.911271:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000400:00000200:5.0:1619677447.911273:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.911274:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.911274:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.911276:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.911277:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.911278:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.911279:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.911282:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.911283:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677447.911283:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677447.911283:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.911285:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.911285:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.911286:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.911287:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.911288:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677447.911289:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.911289:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.911290:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.2:1619677447.911291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.911291:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.911293:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.2:1619677447.911300:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677447.911300:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.911302:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.911303:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.911303:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.911304:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.911305:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.911305:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677447.911306:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:26.0:1619677447.911308:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677447.911309:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677447.911311:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:8.2:1619677447.911311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:10.0:1619677447.911312:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.911313:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.911316:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.911316:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:26.0:1619677447.911317:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:17.0:1619677447.911318:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677447.911319:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.911319:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:26.0:1619677447.911320:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677447.911320:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.911321:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677447.911322:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677447.911322:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.911322:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:26.0:1619677447.911323:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:17.0:1619677447.911323:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677447.911323:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.911324:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.911324:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:13.0:1619677447.911325:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677447.911326:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.911328:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.911330:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.911331:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.911331:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.911333:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.911333:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.911334:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.911334:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677447.911344:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.911346:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.911347:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.911347:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.911349:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.911350:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677447.911543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677447.911551:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677447.911554:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677447.911556:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.911558:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.911559:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677447.911560:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677447.911561:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721869] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.911562:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.911563:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.911564:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677447.911567:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.911569:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677447.911572:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.911573:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:30.0:1619677447.911573:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677447.911575:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677447.911576:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677447.911577:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677447.911579:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677447.911579:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677447.911582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677447.911586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:2.0:1619677447.911590:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.911592:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:31.0:1619677447.911594:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:18.0:1619677447.911594:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.911594:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.911596:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:2.0:1619677447.911596:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.911597:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677447.911598:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:2.0:1619677447.911599:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677447.911619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677447.911626:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.911629:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.911630:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.911631:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.911634:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.911635:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.911637:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.911638:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.911639:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.911640:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.911643:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.911644:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.911647:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.911648:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.911648:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.911650:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.911650:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.911651:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.911652:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.911672:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.911674:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.911675:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.911676:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.911679:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.911679:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.2:1619677447.911684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.911690:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.911693:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.911693:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.911695:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.911698:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.911699:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.911701:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.911702:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.911703:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.911704:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.911707:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.911709:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.911712:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.911713:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.911713:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.911715:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.911715:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.911717:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.911718:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.911719:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677447.911721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677447.911724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677447.911729:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677447.911729:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.911731:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.911731:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677447.911732:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:2.0:1619677447.911732:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:20.0:1619677447.911733:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.911734:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.911736:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.911737:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677447.911805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.911811:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.911814:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.911814:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.911816:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.911818:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.911820:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.911821:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.911822:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.911823:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.911824:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.911828:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.911830:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.911832:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.911833:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.911834:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.911835:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.911836:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.911837:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.911837:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.911842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.911850:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.911852:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.911853:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.911853:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.911855:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.911857:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.911867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.911874:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.911877:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.911878:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.911880:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.911882:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.911883:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.911884:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.911886:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.911887:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.911888:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677447.911891:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.911893:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.911896:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.911897:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.911897:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677447.911899:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.911899:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.911900:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.911900:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677447.911913:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.911916:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.911916:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.911917:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.911919:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.911922:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.912021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.912029:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.912032:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.912034:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.912036:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.912037:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.912038:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.912040:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.912041:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.912042:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.912043:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:21.2:1619677447.912043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:31.0:1619677447.912046:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.912047:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.912049:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.912050:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.912051:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.912051:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.912052:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.912052:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677447.912053:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.912054:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.912054:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.912055:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677447.912056:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.912056:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.912057:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.912058:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.912060:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.912060:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.912061:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.912064:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.912064:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677447.912065:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.912067:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.912067:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:30.0:1619677447.912068:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:12.0:1619677447.912068:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.912069:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:12.0:1619677447.912069:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.912070:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.912071:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677447.912071:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.912072:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.912072:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.912072:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.912093:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.912095:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.912096:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.912097:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.912099:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.912100:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.912112:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.912114:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.912115:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.912116:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.912116:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.912117:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.912219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677447.912226:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.912228:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.912229:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.912231:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.912233:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.912235:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.912236:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.912237:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.912238:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.912239:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.912241:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.912243:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.912246:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.912247:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.912248:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.912249:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.912250:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.912251:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.912252:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.912253:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.2:1619677447.912255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677447.912259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677447.912265:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.0:1619677447.912266:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:5.0:1619677447.912267:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:8.0:1619677447.912268:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:5.0:1619677447.912268:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.912269:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677447.912270:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.912270:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:5.0:1619677447.912270:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.912273:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677447.912425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.912433:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.912436:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.912437:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.912439:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.912442:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.912443:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677447.912444:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.912446:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.912446:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.912447:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677447.912450:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.912452:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677447.912454:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.912455:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677447.912456:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677447.912457:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.912457:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.912458:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.912458:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677447.912470:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.912473:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.912473:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.912475:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.912477:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.912478:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.912499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.912506:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.912509:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.912510:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.912512:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.912514:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.912515:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.912516:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.912518:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.912519:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.912519:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677447.912523:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.912524:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.912527:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.912528:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.912528:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677447.912530:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.912531:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.912531:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.912531:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677447.912544:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.912547:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.912548:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.912548:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.912550:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.912553:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.912557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.912563:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.912566:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.912567:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.912569:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.912572:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.912573:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.912575:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.912576:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.912577:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.912578:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.2:1619677447.912580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000400:00000200:1.0:1619677447.912582:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.912584:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.912586:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:1.0:1619677447.912587:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.912587:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.912588:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:1.0:1619677447.912588:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.912589:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:1.0:1619677447.912590:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.912590:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677447.912591:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.912592:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:22.0:1619677447.912593:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.912593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:1.0:1619677447.912593:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.912593:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677447.912595:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:22.0:1619677447.912596:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.912597:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.912598:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677447.912598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677447.912599:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.912600:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.912602:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677447.912603:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:22.0:1619677447.912603:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.912603:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677447.912604:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677447.912605:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677447.912605:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.912606:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677447.912606:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.912607:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.912607:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.912607:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.912608:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.912609:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.912609:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:22.0:1619677447.912609:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677447.912609:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.912610:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:30.0:1619677447.912611:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.912611:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677447.912612:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000400:00000200:10.0:1619677447.912612:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.912613:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:22.0:1619677447.912613:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677447.912613:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000400:00000200:10.0:1619677447.912613:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.912614:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.912615:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:9.2:1619677447.912617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677447.912618:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.912620:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677447.912623:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.912624:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677447.912625:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.912625:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677447.912626:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677447.912627:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:2.0:1619677447.912627:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:10.0:1619677447.912628:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:2.0:1619677447.912628:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.912629:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.912629:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677447.912630:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:2.0:1619677447.912631:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.2:1619677447.912633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.912633:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.912636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.912643:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677447.912644:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677447.912645:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677447.912646:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:20.0:1619677447.912646:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677447.912647:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:20.0:1619677447.912647:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.912649:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.912650:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.912652:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677447.912686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.912691:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.912694:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.912695:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.912697:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.912699:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.912701:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.912702:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.912703:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.912704:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.912705:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:21.2:1619677447.912706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:6.0:1619677447.912708:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.912710:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.912712:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:6.0:1619677447.912713:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.912714:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:6.0:1619677447.912714:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.912714:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.912715:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677447.912716:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.912716:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:18.0:1619677447.912717:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.912717:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.912718:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:18.0:1619677447.912719:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.912720:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.912722:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677447.912722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677447.912723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:18.0:1619677447.912724:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.912725:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677447.912726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:18.0:1619677447.912726:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.912729:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:18.0:1619677447.912729:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.912730:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.912731:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677447.912731:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.912731:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.912732:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.912732:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.912733:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.912733:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:24.0:1619677447.912734:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:20.0:1619677447.912734:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.912734:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.912735:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.912735:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677447.912736:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.912736:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.912736:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.0:1619677447.912737:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677447.912737:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:5.0:1619677447.912737:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.912738:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.912739:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.912740:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.912741:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.912741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677447.912742:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.912743:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.912744:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.912746:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.912746:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.912747:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.912748:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.912748:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.912748:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:24.0:1619677447.912749:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.912750:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.912750:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.912751:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.912751:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.912753:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.912754:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.912755:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.912756:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.912756:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677447.912758:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:9.2:1619677447.912760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.912760:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.912761:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.912761:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677447.912762:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.912762:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.912763:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.912763:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.912764:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:24.0:1619677447.912765:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.912765:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677447.912765:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:24.0:1619677447.912766:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677447.912766:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:24.0:1619677447.912767:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:18.0:1619677447.912767:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.912770:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:8.2:1619677447.912770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.912771:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.912771:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.912772:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.912773:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.912776:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677447.912778:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:13.0:1619677447.912779:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677447.912780:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:13.0:1619677447.912780:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.912781:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.912782:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.912783:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.912783:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.912784:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.912786:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677447.912786:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.912970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.912976:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.912979:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.912980:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.912982:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.912985:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.912986:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.912987:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.912989:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.912990:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.912991:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.912994:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.912996:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.912999:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.913000:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.913001:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.913003:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.913003:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.913004:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.913004:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.2:1619677447.913013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677447.913017:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.913019:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.913021:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.913021:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:30.0:1619677447.913022:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.913023:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.913023:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.913024:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677447.913025:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:10.0:1619677447.913026:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.913029:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.913031:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:4.2:1619677447.913031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:10.0:1619677447.913032:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.913034:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.913035:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.913036:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:26.0:1619677447.913039:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.913040:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677447.913041:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.913043:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.913043:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:26.0:1619677447.913046:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.913046:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677447.913047:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:10.0:1619677447.913047:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:26.0:1619677447.913048:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.913048:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677447.913050:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.913050:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.913051:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.913051:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:26.0:1619677447.913054:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.913055:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.913056:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.913056:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.913057:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.913057:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677447.913057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.913058:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.913058:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677447.913058:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.913059:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.913060:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.913061:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.913061:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677447.913065:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.913067:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.913068:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.913069:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.913071:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:2.0:1619677447.913071:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.913073:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.913074:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.913074:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.913074:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:10.0:1619677447.913076:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.913078:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677447.913303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677447.913309:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677447.913312:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677447.913313:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677447.913315:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677447.913318:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.913320:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.913321:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.913322:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.913323:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.913324:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.913327:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677447.913328:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.913330:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.913331:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.913331:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.913333:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.913333:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677447.913334:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677447.913334:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:8.2:1619677447.913339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.913347:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.913349:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.913350:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.913351:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.913351:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:1.0:1619677447.913353:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677447.913394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.913399:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.913402:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.913403:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.913404:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.913406:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.913407:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.913408:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.913410:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.913410:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.913411:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.913414:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.913415:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.913418:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.913419:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.913419:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.913421:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.913421:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:4.2:1619677447.913421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.913422:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.913422:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677447.913429:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.913432:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.913433:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.913434:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.913436:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.913436:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:26.0:1619677447.913437:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677447.913438:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.913438:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.913439:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.913439:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:26.0:1619677447.913440:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.913441:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.913442:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:22.0:1619677447.913442:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.913444:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:26.0:1619677447.913445:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.913447:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.913449:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.913450:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.913450:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.913452:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.913452:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.913453:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.913455:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.913455:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677447.913461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.913466:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.913467:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.913468:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.913469:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.913470:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:10.0:1619677447.913470:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.913471:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.913471:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.913472:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677447.913472:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677447.913515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677447.913522:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.913525:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.913526:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.913528:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.913530:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.913532:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.913533:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677447.913534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:8.0:1619677447.913534:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.913535:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.913536:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.913539:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.913539:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677447.913541:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:22.0:1619677447.913542:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.913542:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:20.2:1619677447.913543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.0:1619677447.913544:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.913545:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.913545:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:6.0:1619677447.913545:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.913547:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.913547:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:6.0:1619677447.913547:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.913548:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:6.0:1619677447.913548:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:5.0:1619677447.913548:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677447.913549:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:6.0:1619677447.913549:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.913551:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.913551:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:6.0:1619677447.913552:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677447.913553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:6.0:1619677447.913553:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:5.0:1619677447.913554:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.913555:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.913557:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:6.0:1619677447.913557:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.913558:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.913559:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.913559:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:5.0:1619677447.913560:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.913561:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.913562:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677447.913562:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:6.0:1619677447.913563:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.913563:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:5.0:1619677447.913563:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96721844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.913564:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.913565:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.913565:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:5.0:1619677447.913565:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.913566:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:21.2:1619677447.913567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677447.913567:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000400:00000200:5.0:1619677447.913567:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.913568:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677447.913570:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.913571:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000400:00000200:5.0:1619677447.913572:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677447.913573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677447.913573:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.913574:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.913574:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.913574:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.913575:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677447.913576:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677447.913576:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.913577:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.913578:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677447.913578:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:5.0:1619677447.913578:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.913579:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:31.0:1619677447.913581:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.913581:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.913581:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.0:1619677447.913581:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.913581:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677447.913582:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:31.0:1619677447.913583:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.913583:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677447.913583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.913583:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.913584:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000400:00000200:31.0:1619677447.913585:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.913586:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96721843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.913586:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677447.913587:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.913587:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:31.0:1619677447.913588:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.913589:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.2:1619677447.913589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.913589:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.913590:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.913592:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.913593:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.913593:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.913594:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.913594:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.913594:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677447.913596:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.913597:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677447.913597:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:31.0:1619677447.913598:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677447.913598:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.913598:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677447.913598:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.913599:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.913599:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677447.913600:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677447.913600:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.913601:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.913602:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677447.913602:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.913602:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677447.913603:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.913604:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677447.913609:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.913612:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.913613:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677447.913613:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:26.0:1619677447.913615:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.913616:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.913616:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.913616:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:12.0:1619677447.913617:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677447.913617:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000400:00000200:26.0:1619677447.913618:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:6.0:1619677447.913618:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.913618:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:26.0:1619677447.913619:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721842] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.913619:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:26.0:1619677447.913620:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677447.913620:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.913621:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.913622:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677447.913622:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.913624:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.913625:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677447.913626:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.913627:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677447.913630:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.913631:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677447.913632:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.913634:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.913634:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.913635:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677447.913636:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677447.913637:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677447.913640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677447.913642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677447.913649:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677447.913649:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677447.913651:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:18.0:1619677447.913652:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:10.0:1619677447.913652:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:2.0:1619677447.913652:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677447.913653:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:18.0:1619677447.913653:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.913655:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.913658:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.913721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.913727:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.913730:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.913730:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.913733:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.913734:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.913736:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.913737:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.913738:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.913739:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.913740:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.913743:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.913745:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.913747:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.913747:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.913748:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.913749:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.2:1619677447.913750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677447.913750:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.913750:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.913751:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.913755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.913756:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.913759:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.913759:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.913761:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.913763:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:1.0:1619677447.913764:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.913765:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:1.0:1619677447.913765:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677447.913766:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.913766:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:1.0:1619677447.913766:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.913768:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.913768:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.913769:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.913770:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:1.0:1619677447.913770:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.913774:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.913776:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.913779:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.913779:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.913780:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.913782:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.913782:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.913783:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.913784:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.913799:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.913802:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.913803:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.913804:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.913805:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.913807:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.913807:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.913808:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:31.0:1619677447.913808:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.913809:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.913810:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.914010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.914017:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.914020:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.914022:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.914024:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.914024:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.914025:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677447.914027:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.914028:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.914029:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.914030:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677447.914033:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.914035:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.914037:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.914038:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.914039:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.914040:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.914040:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.914041:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:4.2:1619677447.914041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677447.914042:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:24.0:1619677447.914048:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.914051:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:31.0:1619677447.914052:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:8.0:1619677447.914052:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677447.914054:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:24.0:1619677447.914054:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.914055:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.914056:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.914056:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.914058:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:24.0:1619677447.914058:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:30.0:1619677447.914059:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:24.0:1619677447.914059:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.914060:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.914061:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.914062:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.914065:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.914067:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.914070:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.914071:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.914071:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677447.914072:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:24.0:1619677447.914073:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677447.914074:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:24.0:1619677447.914074:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.914074:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677447.914075:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:24.0:1619677447.914075:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:31.0:1619677447.914076:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677447.914076:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.914077:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:8.2:1619677447.914079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.914086:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:4.2:1619677447.914087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.914089:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.914090:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.914090:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.914092:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.914094:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677447.914095:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.914096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677447.914097:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.914098:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.914100:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.914103:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.914104:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:1.0:1619677447.914104:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:10.0:1619677447.914105:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721837] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.914106:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.914107:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:10.0:1619677447.914107:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.914107:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000400:00000200:10.0:1619677447.914108:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.914108:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000400:00000200:10.0:1619677447.914109:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677447.914109:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:10.0:1619677447.914112:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.914114:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677447.914116:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.914117:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.914118:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.914119:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.914120:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.914121:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.914121:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677447.914133:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.914136:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.914137:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.914138:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.914140:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.914142:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.914144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.914147:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.914150:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.914152:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.914153:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.914154:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.914156:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.914157:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.914159:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.914160:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.914161:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.914164:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.914166:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.914169:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.914170:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.914170:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.914172:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.914173:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.914174:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.914174:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677447.914185:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.914188:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.914188:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.914190:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.914191:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.914193:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.914313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.914319:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.914322:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.914323:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.914325:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.914327:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.914329:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.914330:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.914331:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.914332:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.914333:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.914337:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.914339:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.914342:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.914343:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.914344:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.914345:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.914346:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.914346:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.914347:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:21.2:1619677447.914354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:12.0:1619677447.914360:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677447.914362:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677447.914362:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.914363:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.914365:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:12.0:1619677447.914365:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.914367:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.914367:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.914368:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.914369:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.914369:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.914371:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.914372:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.914373:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.914375:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.914376:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.914376:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677447.914379:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.914379:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.914380:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.914380:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.914381:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.914382:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.914382:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.914384:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.914385:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.914385:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.914387:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.914387:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.914388:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.914389:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.914398:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.914400:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.914401:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.914403:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.914404:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.914405:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.914435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.914442:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.914444:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.914445:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.914447:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.914450:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.914451:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.914452:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.914454:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.914455:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.914456:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.914458:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.914460:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.914462:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.914462:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.914463:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.914465:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.914465:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.914467:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.914468:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.914468:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677447.914472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677447.914474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.914480:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.914481:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.914481:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.914481:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.914482:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.914482:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.914483:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.914483:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.914484:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.914485:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.914578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:21.2:1619677447.914582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:31.0:1619677447.914587:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.914589:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:8.0:1619677447.914589:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:31.0:1619677447.914592:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.914592:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:30.0:1619677447.914593:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:24.0:1619677447.914593:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:31.0:1619677447.914594:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.914595:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.914596:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.914597:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.914598:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.914599:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.914599:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677447.914600:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.914601:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677447.914601:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.914602:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.914604:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.914605:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.914605:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677447.914608:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:8.0:1619677447.914609:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.914610:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.914610:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677447.914611:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.914611:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.914612:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.914613:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:8.0:1619677447.914613:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.914614:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.914614:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.0:1619677447.914614:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.914615:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.914616:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.914617:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.914618:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.914618:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.914623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677447.914624:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.914627:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.914628:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.914629:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.914631:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:18.0:1619677447.914631:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:30.0:1619677447.914632:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677447.914634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677447.914634:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.914635:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.914635:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.914637:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.914640:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677447.914642:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:9.2:1619677447.914642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:31.0:1619677447.914645:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.914647:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.914648:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.914649:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.914649:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677447.914650:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677447.914652:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721830] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.914652:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677447.914653:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.914653:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.914653:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000400:00000200:31.0:1619677447.914654:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.914654:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000400:00000200:31.0:1619677447.914655:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.914655:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:31.0:1619677447.914658:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.914660:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677447.914663:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.914664:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677447.914665:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677447.914666:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.914667:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.914668:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.914670:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.914670:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677447.914676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677447.914680:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:30.0:1619677447.914683:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:6.0:1619677447.914683:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.914684:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.914684:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.914686:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.914687:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.914687:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:6.0:1619677447.914687:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.914690:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.914817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.914822:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.914825:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677447.914827:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.914828:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.914831:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.914832:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:5.0:1619677447.914834:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.914835:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.914836:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.914837:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:5.0:1619677447.914840:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.914842:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:5.0:1619677447.914846:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.914847:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677447.914847:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.0:1619677447.914849:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677447.914850:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.914851:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.914853:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.914853:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:9.2:1619677447.914855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.2:1619677447.914858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.2:1619677447.914858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:20.0:1619677447.914861:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.914863:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:24.0:1619677447.914864:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:20.0:1619677447.914864:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.914865:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:24.0:1619677447.914866:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.914867:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.914867:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677447.914868:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000400:00000200:24.0:1619677447.914869:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.914871:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.914873:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.914874:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:8.0:1619677447.914874:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.914875:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677447.914876:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677447.914877:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.914879:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.914879:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.914880:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:9.2:1619677447.914882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000400:00000200:8.0:1619677447.914883:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.914886:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677447.914888:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.914889:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.914890:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.914890:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.914890:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677447.914891:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.914891:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677447.914892:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.914893:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677447.914893:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.914894:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677447.914895:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677447.914895:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:22.0:1619677447.914907:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.914910:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.914911:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.914911:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.914913:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.914916:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.914924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677447.914925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.2:1619677447.914926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:20.0:1619677447.914926:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.914929:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.914931:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.914931:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.914932:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677447.914933:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:20.0:1619677447.914934:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.914934:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.914935:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677447.914936:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:18.0:1619677447.914936:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.914936:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:20.0:1619677447.914937:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.914938:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.914939:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.914940:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.914940:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677447.914940:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.914942:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:18.0:1619677447.914942:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.914942:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.914944:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.914944:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.914946:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.914946:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.914947:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.914948:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.914948:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.914949:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.914950:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:20.0:1619677447.914954:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.914954:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.914956:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677447.914957:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.914958:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:18.0:1619677447.914959:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.0:1619677447.914960:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.914961:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.914961:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.914962:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:18.0:1619677447.914962:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.914962:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.914963:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.914963:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.914964:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:18.0:1619677447.914964:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.0:1619677447.914965:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.914965:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.914965:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.914966:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.914966:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.914966:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677447.914966:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.914967:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677447.914968:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:17.0:1619677447.914968:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677447.914969:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.914969:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677447.914970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:21.2:1619677447.914975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677447.914978:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:6.0:1619677447.914979:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677447.914980:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:1.0:1619677447.914980:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.914981:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:6.0:1619677447.914981:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:1.0:1619677447.914981:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.914982:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:12.0:1619677447.914982:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677447.914982:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.914982:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.914983:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:1.0:1619677447.914983:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.914984:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677447.914985:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.914985:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.914985:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:1.0:1619677447.914986:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:12.0:1619677447.914987:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.914988:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677447.914988:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.914991:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.915050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.915057:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:21.2:1619677447.915059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677447.915060:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.915061:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.915063:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.915065:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.915067:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677447.915067:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.915068:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.915070:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:18.0:1619677447.915070:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.915071:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.915072:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.915074:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.915074:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.915076:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.915077:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.915078:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.915079:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677447.915080:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.915081:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677447.915082:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.915082:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.915082:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677447.915083:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.915084:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.915084:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.915085:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.915085:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.915086:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:30.0:1619677447.915087:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.915089:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.915092:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.915093:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677447.915094:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677447.915095:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.915096:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.915097:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.915097:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.915097:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.915100:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.915101:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.915102:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.915104:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.915105:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.915109:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.915112:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.915113:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.915113:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.915115:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.915118:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.915127:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.915130:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.915130:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.915131:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.915132:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.915133:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.915320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.915329:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.915332:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.915334:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.915336:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.915337:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.915339:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.915340:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.915341:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677447.915341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:31.0:1619677447.915342:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.915343:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.915346:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.915347:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677447.915348:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.915350:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.915350:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677447.915351:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.915351:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.915352:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.915353:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:17.0:1619677447.915353:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.915354:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.915355:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:17.0:1619677447.915356:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.915357:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000400:00000200:17.0:1619677447.915357:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:31.0:1619677447.915358:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:17.0:1619677447.915359:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.915360:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.915361:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.915362:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.2:1619677447.915364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677447.915366:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.915367:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:12.0:1619677447.915367:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.915369:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:17.0:1619677447.915370:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.915370:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:12.0:1619677447.915370:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.915371:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677447.915372:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:12.0:1619677447.915372:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.915373:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.915373:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677447.915374:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:17.0:1619677447.915374:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677447.915374:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.915375:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677447.915375:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.915375:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.915376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677447.915377:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:24.0:1619677447.915384:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.915387:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:22.0:1619677447.915387:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.0:1619677447.915387:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.915389:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.915389:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677447.915390:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.915391:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.915391:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:4.2:1619677447.915391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677447.915392:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.915392:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677447.915393:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677447.915393:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.915393:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.915394:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.915394:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:24.0:1619677447.915395:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.915396:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.915396:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.915396:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:24.0:1619677447.915397:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.915398:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:24.0:1619677447.915398:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:26.0:1619677447.915401:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:24.0:1619677447.915401:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.915402:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677447.915403:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:26.0:1619677447.915404:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.915406:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677447.915407:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.915407:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.915408:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:26.0:1619677447.915409:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:24.0:1619677447.915409:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:26.0:1619677447.915410:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.915410:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.915410:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:26.0:1619677447.915411:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.915411:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:26.0:1619677447.915412:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.915413:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.2:1619677447.915415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:26.0:1619677447.915416:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.915418:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.915420:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.915421:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.915421:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.915423:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.915423:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677447.915423:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:26.0:1619677447.915424:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.915424:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677447.915425:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.915426:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.915427:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.915429:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.915432:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677447.915435:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.915438:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.915438:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.915440:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.915442:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.915442:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.915447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.915455:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.915458:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.915460:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.915462:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.915462:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.915463:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677447.915464:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.915466:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.915467:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.915468:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677447.915471:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.915472:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677447.915475:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.915476:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677447.915477:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677447.915478:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677447.915479:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.915480:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.915480:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.915492:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.915495:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.915495:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.915496:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.915498:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.915503:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.915504:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.915506:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.915507:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.915507:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.915508:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.915509:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677447.915523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.915530:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.915533:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.915534:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.915536:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.915538:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.915539:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677447.915541:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.915542:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.915543:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.915544:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677447.915547:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.915548:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677447.915551:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.915551:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677447.915552:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677447.915553:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677447.915554:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.915554:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.915555:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677447.915565:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.915568:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.915569:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.915570:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.915572:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.915573:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677447.915638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.915644:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.915646:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.915647:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.915649:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.915651:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.915652:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.915654:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.915655:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.915656:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.915657:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.915660:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.915662:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.915665:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.915666:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.915666:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.915668:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.915668:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.915669:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.915669:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.915681:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.915683:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.915684:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.915686:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.915687:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.915689:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.915805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.915811:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.915813:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.915816:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.915816:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.915818:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.915820:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.915821:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.915823:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.915825:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.915825:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.915829:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.915831:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.915834:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.915835:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.915836:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.915837:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.915838:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.915839:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.915839:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.915850:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.915852:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:21.2:1619677447.915853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:12.0:1619677447.915853:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.915855:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.915857:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677447.915858:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:12.0:1619677447.915858:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677447.915861:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.915862:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.915864:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.915866:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.915867:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.915868:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.915868:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677447.915870:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.915871:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.915872:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:13.0:1619677447.915872:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.915872:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.915873:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.915874:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.915875:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:22.0:1619677447.915876:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.915878:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.915880:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.915881:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.915882:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.915884:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.915884:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.915885:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.915886:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.915890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.915898:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.915900:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.915901:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.915901:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.915903:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.915906:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.915912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.915919:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.915922:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.915923:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.915923:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.915924:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.915924:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677447.915950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:6.0:1619677447.915955:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677447.915958:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677447.915958:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:4.2:1619677447.915958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:6.0:1619677447.915960:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.915962:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.915963:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:26.0:1619677447.915965:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677447.915965:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.915966:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677447.915968:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.915968:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677447.915968:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.915969:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.915973:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.915974:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.915976:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:26.0:1619677447.915977:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.915979:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:6.0:1619677447.915979:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.915979:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:26.0:1619677447.915980:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.915980:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.915981:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:26.0:1619677447.915982:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.915982:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000400:00000200:26.0:1619677447.915983:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.915983:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000400:00000200:26.0:1619677447.915984:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.915984:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000400:00000200:26.0:1619677447.915987:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677447.915988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677447.915989:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.915991:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.915992:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.915993:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.915994:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.915995:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.915995:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677447.915995:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:26.0:1619677447.915996:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677447.915997:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.915998:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.916000:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.916001:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.916003:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677447.916007:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.916010:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.916010:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.916011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000400:00000200:30.0:1619677447.916012:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.916014:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.916015:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.916017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:5.0:1619677447.916018:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.916021:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.916022:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.916023:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.916023:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.916024:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:10.0:1619677447.916025:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:5.2:1619677447.916026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677447.916027:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677447.916028:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677447.916030:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677447.916032:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.916032:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.916034:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.916035:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677447.916035:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677447.916037:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.916038:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677447.916039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:10.0:1619677447.916039:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.916039:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.916040:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.916042:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.916045:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:31.0:1619677447.916047:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677447.916047:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.916047:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.916049:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.916050:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:10.0:1619677447.916050:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.916051:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677447.916051:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:1.0:1619677447.916051:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.916052:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.916053:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.916053:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677447.916053:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677447.916053:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:1.0:1619677447.916053:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:10.0:1619677447.916054:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677447.916055:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:31.0:1619677447.916057:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.916057:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.916059:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:8.2:1619677447.916059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.916059:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.916060:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96721809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.916062:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.916063:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.916063:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.916064:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.916064:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.916065:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.916067:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.916067:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:31.0:1619677447.916068:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.916068:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.916069:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:31.0:1619677447.916070:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.916070:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.916070:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.916071:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000400:00000200:17.0:1619677447.916072:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:1.0:1619677447.916072:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677447.916073:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.916073:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677447.916074:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:31.0:1619677447.916075:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:31.0:1619677447.916076:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:17.0:1619677447.916076:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677447.916076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677447.916077:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.916078:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.916079:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677447.916080:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677447.916082:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:1.0:1619677447.916082:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:13.0:1619677447.916083:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.916084:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.916084:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.916085:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677447.916086:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:1.0:1619677447.916086:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.916087:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677447.916087:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677447.916088:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.916088:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.916089:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.916091:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.916101:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.916104:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.916105:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.916105:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.916106:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.916108:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677447.916130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.916137:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.916140:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.916142:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.916144:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.916144:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.916146:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677447.916147:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.916148:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.916149:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677447.916149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000400:00000200:30.0:1619677447.916150:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677447.916153:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.916155:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677447.916155:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677447.916157:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:30.0:1619677447.916158:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.916159:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677447.916159:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:30.0:1619677447.916160:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:2.0:1619677447.916160:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.916161:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.916162:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.916163:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.916163:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:2.0:1619677447.916163:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.916164:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677447.916166:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.916167:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.916168:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.916169:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677447.916173:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.916174:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.916175:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677447.916177:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.916178:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677447.916178:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000400:00000200:22.0:1619677447.916179:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.916179:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:5.2:1619677447.916179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919874bc7000] (20)++
00000800:00000200:2.0:1619677447.916179:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677447.916180:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677447.916181:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000400:00000200:22.0:1619677447.916182:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.916182:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677447.916182:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:22.0:1619677447.916185:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.916186:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc7000] (21)++
00000800:00000200:6.0:1619677447.916188:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.1@o2ib3
00000800:00000200:22.0:1619677447.916189:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc7000] (22)--
00000400:00000200:6.0:1619677447.916191:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.4.1@o2ib3 : GET - for me
00000800:00000200:26.0:1619677447.916193:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:6.0:1619677447.916194:0:19937:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.4.1@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000800:00000200:26.0:1619677447.916195:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677447.916196:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.916196:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:6.0:1619677447.916196:0:19937:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.4.1@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:6.0:1619677447.916198:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919874bc7000] (21)++
00000400:00000200:26.0:1619677447.916199:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.916199:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919874bc7000] (22)--
00000800:00000200:26.0:1619677447.916201:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:6.0:1619677447.916201:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.4.1@o2ib3 local destination
00000800:00000200:21.2:1619677447.916203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:6.0:1619677447.916203:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91986aff6e80] -> 10.51.4.1@o2ib3 (2) version: 12
00000800:00000200:6.0:1619677447.916204:0:19937:0:(o2iblnd.c:1251:kiblnd_query()) peer_ni 10.51.4.1@o2ib3 ffff91986aff6e80, alive 0 secs ago
00000400:00000200:6.0:1619677447.916208:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.4.1@o2ib3(10.51.4.1@o2ib3:10.51.4.1@o2ib3) : REPLY try# 0
00000800:00000200:18.0:1619677447.916209:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:6.0:1619677447.916210:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.4.1@o2ib3
00000800:00000200:18.0:1619677447.916212:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.916212:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:6.0:1619677447.916213:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91986aff6e80] -> 10.51.4.1@o2ib3 (2) version: 12
00000800:00000200:6.0:1619677447.916213:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919874bc7000] (21)++
00000800:00000200:6.0:1619677447.916214:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919874bc7000] (22)++
00000400:00000200:18.0:1619677447.916215:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.916215:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919874bc7000] (23)--
00000800:00000200:6.0:1619677447.916216:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc7000] (22)--
00000400:00000200:18.0:1619677447.916218:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.916219:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.916221:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677447.916221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919874bc7000] (21)++
00000400:00000200:18.0:1619677447.916222:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.916223:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.916224:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.916226:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc7000] (22)++
00000800:00000200:22.0:1619677447.916227:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919874bc7000] (23)--
00000400:00000200:18.0:1619677447.916227:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.916228:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.916228:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677447.916229:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc7000] (22)--
00000400:00000200:22.0:1619677447.916231:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.4.1@o2ib3: REPLY: OK
00000800:00000200:18.0:1619677447.916231:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.916232:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.916232:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.916234:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc7000] (21)--
00000800:00000200:18.0:1619677447.916234:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.916234:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.916236:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.916237:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.916237:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677447.916240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677447.916242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.916248:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.916249:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.916249:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.916250:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.916250:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.916250:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.916251:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.916252:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:2.0:1619677447.916252:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.916253:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.916352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.916360:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.916363:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.916363:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677447.916365:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.916368:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.916370:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677447.916371:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721805] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.916372:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.916373:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.916374:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677447.916378:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.916380:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677447.916382:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.916383:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677447.916384:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677447.916385:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677447.916385:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.916386:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.916387:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:4.2:1619677447.916394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677447.916399:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.916401:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:26.0:1619677447.916401:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677447.916402:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.916404:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.916404:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.916405:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677447.916406:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:26.0:1619677447.916406:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.916407:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677447.916408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:26.0:1619677447.916409:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.916410:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.916411:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.916413:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.916414:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.916414:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.916416:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677447.916417:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.916418:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:26.0:1619677447.916419:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677447.916420:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.916421:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.916422:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.916423:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.916423:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:8.0:1619677447.916424:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.916425:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.916425:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.916426:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:8.0:1619677447.916426:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:26.0:1619677447.916427:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000400:00000200:8.0:1619677447.916427:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.916428:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:8.0:1619677447.916428:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.916429:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.916430:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.916433:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677447.916433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677447.916435:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.916438:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.916438:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.916439:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677447.916439:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677447.916440:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:8.0:1619677447.916441:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.916441:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.916441:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677447.916442:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677447.916442:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:8.0:1619677447.916442:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.916442:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677447.916443:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677447.916443:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.916444:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677447.916445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:6.0:1619677447.916445:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.2:1619677447.916446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677447.916449:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.916450:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.916451:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.916452:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:17.0:1619677447.916454:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:6.0:1619677447.916454:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.916456:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.916457:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.916458:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.916458:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:1.0:1619677447.916458:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:6.0:1619677447.916459:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.916460:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.916461:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.916461:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.916463:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:17.0:1619677447.916464:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.916464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000400:00000200:6.0:1619677447.916466:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.916468:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.916471:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.916471:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677447.916472:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.916472:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:1.0:1619677447.916474:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.916475:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:6.0:1619677447.916475:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:1.0:1619677447.916475:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:6.0:1619677447.916476:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.916476:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.916477:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677447.916477:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.916478:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.916479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677447.916486:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:30.0:1619677447.916487:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677447.916489:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:30.0:1619677447.916490:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:26.0:1619677447.916490:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677447.916490:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.916493:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.916493:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.916494:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.916496:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.916496:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:30.0:1619677447.916497:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.916498:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.916499:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.916501:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.916502:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.916505:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.916507:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.916509:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.916510:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.916511:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.916513:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.916513:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.916514:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.916515:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.916527:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.916529:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.916530:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.916531:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.916533:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.916534:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.916799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.916806:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.916809:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.916810:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.916811:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.916814:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.916815:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.916816:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.916817:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.916818:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.916819:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677447.916822:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.916824:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.916826:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.916827:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.916828:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677447.916829:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.916830:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.916830:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.916831:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677447.916843:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.916846:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.916847:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.916847:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.916849:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.916852:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.916870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.916876:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.916879:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.916882:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.916882:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.916885:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.916886:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.916887:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.916889:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.916890:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.916891:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.916894:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.916896:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.916899:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.916900:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.916901:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.916903:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.916903:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.916905:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.916906:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.916907:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.916912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.916915:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677447.916915:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677447.916917:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.916917:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.916918:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.916918:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.916919:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.916920:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.916922:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.916923:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.916963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.916970:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.916973:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.916973:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.916975:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.916977:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.916978:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.916980:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.916981:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.916982:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.916983:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.916987:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.916989:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.916991:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.916992:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.916993:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.916994:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.916995:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.916996:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.916997:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.916997:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677447.917001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677447.917002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:21.2:1619677447.917003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677447.917008:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677447.917008:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.917010:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.917010:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677447.917011:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677447.917011:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.917011:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.917011:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:20.0:1619677447.917013:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.917014:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.0:1619677447.917014:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:31.0:1619677447.917016:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.917016:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677447.917018:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.917019:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.917020:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.917022:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.917023:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.917024:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.917025:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.917029:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.917030:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.917033:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.917034:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.917034:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.917036:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.917037:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.917038:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.917040:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.917040:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677447.917046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677447.917048:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.917050:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.917051:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.917053:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.917054:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677447.917054:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.917055:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.917056:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.917057:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.917058:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677447.917103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.917111:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.917113:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.917114:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.917116:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.917119:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.917120:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677447.917121:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.917122:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.917123:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.917124:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677447.917127:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677447.917129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:10.0:1619677447.917129:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677447.917131:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.917132:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677447.917132:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677447.917134:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677447.917134:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677447.917134:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677447.917135:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.917135:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677447.917137:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.917137:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.917140:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.917142:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.917143:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.917144:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.917146:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:6.0:1619677447.917146:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.917147:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.917148:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:6.0:1619677447.917148:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677447.917149:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.917151:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.917152:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:6.0:1619677447.917152:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.917154:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:6.0:1619677447.917154:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.917157:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.917158:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.917158:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.917160:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.917161:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.917161:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.917162:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.917166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.2:1619677447.917170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:18.0:1619677447.917174:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:17.0:1619677447.917176:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:18.0:1619677447.917177:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.917177:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.917178:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.917179:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677447.917179:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:18.0:1619677447.917180:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.917182:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.917183:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:17.0:1619677447.917184:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.917186:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677447.917187:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.917188:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.917190:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.917191:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677447.917194:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.917196:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677447.917199:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.917200:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:17.0:1619677447.917201:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:17.0:1619677447.917203:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:17.0:1619677447.917203:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677447.917204:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677447.917205:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:26.0:1619677447.917215:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.917218:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.917219:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.917219:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.917221:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.917224:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677447.917233:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.917236:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.917236:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.917237:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.917238:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.917238:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677447.917275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.917282:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.917285:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.917287:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.917289:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.917289:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.917291:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.917292:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.917298:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.917299:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.917300:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.917304:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.917306:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:21.2:1619677447.917306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:31.0:1619677447.917308:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.917309:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.917310:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.917311:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.917312:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.917312:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:2.0:1619677447.917312:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:31.0:1619677447.917313:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:2.0:1619677447.917315:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.917316:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.917318:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.917320:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.917322:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.917323:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.917325:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:2.0:1619677447.917325:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.917326:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.917327:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:2.0:1619677447.917327:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:12.0:1619677447.917328:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.917330:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.917330:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.917332:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.917332:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:12.0:1619677447.917333:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677447.917334:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.917335:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.917336:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.917337:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.917338:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.917339:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.917339:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677447.917344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.917352:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.917355:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.917356:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.917358:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.917359:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.917360:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.917399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.917406:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.917409:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.917409:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.917411:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.917414:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.917416:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.917417:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.917418:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.917419:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.917420:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677447.917424:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.917425:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.917428:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.917429:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.917430:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677447.917431:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.917432:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.917433:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.917433:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677447.917445:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.917448:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.917449:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.917450:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.917452:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.917454:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677447.917744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677447.917750:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.917753:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.917754:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.917756:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.917758:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.917759:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.917760:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.917762:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.917763:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.917764:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.917767:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.917769:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.917770:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.917771:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.917772:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.917773:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.917773:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.917774:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.917774:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677447.917779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.917786:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.917788:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.917789:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.917789:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.917791:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.917792:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.917810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.917816:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.917819:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.917820:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.917822:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.917825:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.917827:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.917828:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.917829:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.917830:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.917831:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.917835:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.917837:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.917840:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.917840:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.917841:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.917843:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.917843:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.917844:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.917844:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.917859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:4.2:1619677447.917863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.917870:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.917872:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.917872:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:24.0:1619677447.917874:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.917874:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677447.917876:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.917877:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.917880:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.917880:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.917881:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677447.917882:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.917882:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.917883:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.917885:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.917886:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.917887:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.917890:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.917892:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.917895:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.917896:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.917897:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.917898:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.917899:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.917900:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.917901:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.917902:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.917904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.917907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.917911:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.917914:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677447.917914:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.917915:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.917915:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.917916:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.917917:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677447.917917:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.917918:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.917920:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677447.917928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:21.2:1619677447.917931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:6.0:1619677447.917933:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.917936:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.917937:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677447.917938:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:6.0:1619677447.917939:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.917941:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:6.0:1619677447.917941:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.917942:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:6.0:1619677447.917943:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:8.0:1619677447.917945:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.917945:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.917946:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.917947:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.917948:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.917948:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:8.0:1619677447.917949:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.917951:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.917952:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.917953:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.917953:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.917954:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677447.917955:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:8.0:1619677447.917958:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.917959:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.917959:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:8.0:1619677447.917960:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677447.917960:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.0:1619677447.917962:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.917962:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.917962:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677447.917963:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.917964:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677447.917964:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.917965:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:8.0:1619677447.917966:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.917966:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:6.0:1619677447.917966:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677447.917967:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.917968:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.917969:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:21.2:1619677447.917970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:21.2:1619677447.917974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:22.0:1619677447.917975:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:30.0:1619677447.917976:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677447.917977:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.917978:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677447.917979:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:22.0:1619677447.917979:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677447.917979:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:30.0:1619677447.917980:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.917981:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.917981:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:13.0:1619677447.917981:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.917982:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.917983:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:24.0:1619677447.917983:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:13.0:1619677447.917983:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.917984:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:24.0:1619677447.917984:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.917985:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.917985:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.917986:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.917993:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677447.917995:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.917996:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.917997:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.917997:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.917998:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677447.918112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.918119:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.918122:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.918122:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677447.918124:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.918127:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.918129:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677447.918130:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.918131:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.918132:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.918133:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677447.918137:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.918139:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.918142:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.918143:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677447.918144:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.918145:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.918146:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.918147:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.918147:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677447.918152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.918159:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.918162:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:5.2:1619677447.918162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677447.918163:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.918164:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677447.918166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:18.0:1619677447.918166:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.918169:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:26.0:1619677447.918171:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677447.918174:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677447.918174:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.918174:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677447.918177:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677447.918177:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.918179:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.918179:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.918180:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.918181:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.918181:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677447.918182:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.918183:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677447.918184:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.918184:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.918185:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.918186:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.918187:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.918187:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.918188:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:26.0:1619677447.918191:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.918192:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.918192:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.918194:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:26.0:1619677447.918195:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.918196:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677447.918196:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.918197:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.918198:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677447.918198:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:26.0:1619677447.918198:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677447.918198:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.918199:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:31.0:1619677447.918200:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:26.0:1619677447.918200:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:31.0:1619677447.918201:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.918201:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677447.918201:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677447.918202:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677447.918204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.2:1619677447.918206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677447.918207:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:9.2:1619677447.918207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.918209:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:12.0:1619677447.918210:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:20.0:1619677447.918212:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.918212:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.918213:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.918213:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:12.0:1619677447.918214:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.918214:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:20.0:1619677447.918215:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.918215:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:20.0:1619677447.918216:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:13.0:1619677447.918216:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.918216:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677447.918216:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.918217:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677447.918217:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:20.0:1619677447.918218:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.918218:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:20.0:1619677447.918219:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.918219:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.918219:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.918220:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.918222:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677447.918222:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.918226:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.918226:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.0:1619677447.918228:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677447.918231:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.918231:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677447.918232:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:13.0:1619677447.918232:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.918234:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677447.918235:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:13.0:1619677447.918235:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.918235:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677447.918236:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:13.0:1619677447.918236:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677447.918237:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:13.0:1619677447.918237:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.918238:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677447.918246:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.918249:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.918249:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.918250:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.918252:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.918255:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677447.918302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.2:1619677447.918307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:1.0:1619677447.918308:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.918311:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.918312:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.918314:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.918315:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:1.0:1619677447.918316:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.918318:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:26.0:1619677447.918319:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677447.918319:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:1.0:1619677447.918320:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.918322:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.918324:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677447.918324:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.918325:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677447.918326:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.918328:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.918328:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.918330:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.918331:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.918331:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.918333:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677447.918334:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.918334:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.918334:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677447.918335:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.918336:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.918336:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.918337:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677447.918338:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677447.918338:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677447.918340:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677447.918342:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.918343:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.2:1619677447.918343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677447.918344:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677447.918345:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677447.918345:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.918346:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677447.918347:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:17.0:1619677447.918350:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.918353:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.918353:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.918354:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.918355:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.918357:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:12.0:1619677447.918358:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.918361:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.918362:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.918363:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677447.918366:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.918366:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.2:1619677447.918565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.918571:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.918574:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.918576:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.918577:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.918579:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.918581:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.918582:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.918583:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.918584:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.918586:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.918589:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.918591:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.2:1619677447.918595:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:21.2:1619677447.918598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:5.0:1619677447.918600:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.918601:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:18.0:1619677447.918602:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677447.918602:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.918604:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.918604:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.918605:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:8.0:1619677447.918605:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:2.0:1619677447.918605:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677447.918606:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.918606:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677447.918607:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.918608:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.918609:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:18.0:1619677447.918609:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.918610:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.918611:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.918612:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.918613:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.918614:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.918615:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.918615:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.918616:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.918616:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721777] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.918617:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:8.0:1619677447.918618:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.918619:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.918620:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.918621:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.918621:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.918621:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.918622:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.918623:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.918624:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.918626:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:8.0:1619677447.918626:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.918627:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.918628:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.918628:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.918628:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.918630:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.918631:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.918631:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:8.0:1619677447.918631:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.918632:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.0:1619677447.918632:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677447.918633:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.0:1619677447.918634:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677447.918634:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.918635:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.918636:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.918643:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677447.918645:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677447.918646:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.918648:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.918649:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.918650:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677447.918650:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677447.918651:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.918651:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677447.918653:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.918654:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.918758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.918764:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.918767:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.918768:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.918770:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.918773:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.918775:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.918776:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.918777:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.918778:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.918779:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.2:1619677447.918779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:6.0:1619677447.918783:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.918785:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.918785:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.918787:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.918788:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.918788:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.918789:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.918790:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:1.0:1619677447.918790:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.918792:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.918792:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:1.0:1619677447.918792:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.918793:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.918794:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:1.0:1619677447.918794:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.918795:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721775] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.918796:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.918797:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.918798:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:4.2:1619677447.918799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677447.918802:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.918804:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.918805:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:10.0:1619677447.918806:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677447.918807:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.918807:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:31.0:1619677447.918808:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:1.0:1619677447.918808:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (24)++
00000400:00000200:31.0:1619677447.918809:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.918809:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.918810:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677447.918811:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.918811:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:1.0:1619677447.918811:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677447.918812:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:1.0:1619677447.918812:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677447.918813:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.918813:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.918815:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:10.0:1619677447.918815:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.918816:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.918817:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.918819:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.918820:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.918821:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.918823:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:10.0:1619677447.918824:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.918825:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.918826:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.918826:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:30.0:1619677447.918828:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.918829:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.918829:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.918830:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677447.918830:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.918830:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.918832:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.918832:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.918833:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677447.918834:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.918835:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677447.918840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.918845:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677447.918847:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677447.918848:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677447.918849:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677447.918849:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.918850:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677447.918850:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.918851:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677447.918852:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.918855:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.918864:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.918866:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.918867:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.918868:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.918868:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.918869:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.918920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.918928:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.918930:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.918931:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.918933:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.918936:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.918937:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677447.918939:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721773] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.918940:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.918941:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.918941:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677447.918944:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.918945:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677447.918948:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.918948:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677447.918949:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677447.918950:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.918951:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.918952:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.918952:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677447.918962:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.918965:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.918965:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.918967:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.918969:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.918970:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.918985:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.918987:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.918988:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.918989:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.918989:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.918990:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.919016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.919023:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.919026:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.919027:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.919028:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.919031:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.919032:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.919033:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.919035:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.919035:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.919036:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.919039:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.919041:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.919044:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.919045:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.919045:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.919047:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.919047:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.919048:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.919048:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.2:1619677447.919049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.2:1619677447.919053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.919056:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.919058:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.919060:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.919061:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:30.0:1619677447.919062:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.919063:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.919063:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:31.0:1619677447.919064:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677447.919064:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:31.0:1619677447.919065:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.919065:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.919067:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.919067:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.919068:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.919069:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.919070:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.919071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000400:00000200:31.0:1619677447.919072:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.919074:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.919076:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.919077:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.919077:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.919078:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.919079:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.919079:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.919080:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.919081:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677447.919081:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.919081:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.919082:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.919083:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.919084:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:13.0:1619677447.919091:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.919094:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.919095:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677447.919096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:13.0:1619677447.919097:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.919098:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.919099:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.919103:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.919105:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.919106:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.919108:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.919110:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.919112:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.919113:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.919114:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.919115:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.919116:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.919119:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.919120:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.919123:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.919124:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.919125:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.919126:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.919127:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.919128:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.919128:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.2:1619677447.919129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677447.919129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.2:1619677447.919132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677447.919135:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677447.919137:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677447.919137:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.919138:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677447.919140:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:20.0:1619677447.919140:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:17.0:1619677447.919140:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.919143:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.919143:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.919143:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.919143:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.919144:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.919145:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.919145:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677447.919146:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.919146:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.919147:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:5.0:1619677447.919147:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:17.0:1619677447.919148:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.919149:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.919149:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:17.0:1619677447.919150:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.919151:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.919151:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677447.919152:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.919153:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677447.919155:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.919157:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:30.0:1619677447.919158:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.919159:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.919160:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677447.919160:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.919161:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.919162:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677447.919163:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.919163:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677447.919164:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:17.0:1619677447.919164:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:30.0:1619677447.919165:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677447.919165:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.919166:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677447.919167:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.919167:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:8.2:1619677447.919168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677447.919169:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.919170:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.919171:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.2:1619677447.919171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.2:1619677447.919172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677447.919176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:1.0:1619677447.919177:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.919178:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.919178:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.919178:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.919178:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.919179:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.919180:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.919180:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:17.0:1619677447.919181:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:1.0:1619677447.919181:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.919183:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:1.0:1619677447.919183:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:31.0:1619677447.919184:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:5.0:1619677447.919184:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.919186:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.919186:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.919187:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.919188:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.919188:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677447.919189:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.919289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.919292:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.919298:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.919300:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.919302:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.919305:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.919306:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.919308:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.919309:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.919310:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.919311:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.919315:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.919317:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.919319:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.919320:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.919321:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.919323:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.919323:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.919324:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.919324:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.919330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.919337:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.919340:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.919341:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.919341:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.919343:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.919346:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.919347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.919354:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.919357:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.919358:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.919358:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.919359:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.919359:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.919544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.919549:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.919552:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.919552:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.919554:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.919557:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.919559:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.919560:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.919561:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.919562:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.919563:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.919567:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.919569:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.919572:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.919572:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.919574:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.919576:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.919576:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.919576:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.919577:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.919589:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.919592:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.919593:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.919595:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.919597:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.919598:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.919599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.919606:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.919609:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.919610:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.919611:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.919614:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.919615:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677447.919617:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.919618:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.919619:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.919620:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677447.919623:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.919625:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.919627:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.919628:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.919628:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.919630:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.919630:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.919631:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.919632:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677447.919642:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.919643:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.919644:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.919645:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.919646:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.919647:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677447.919667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.919674:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.919677:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.919679:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.919681:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.919682:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.919683:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.919684:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.919686:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.919686:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.919687:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.2:1619677447.919688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:31.0:1619677447.919690:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.919692:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.919694:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.919695:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.919695:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.919696:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.919697:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.919698:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.919698:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677447.919699:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.919699:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:30.0:1619677447.919699:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.919701:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.919701:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.919703:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677447.919705:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721763] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.919706:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.919707:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.919708:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:13.0:1619677447.919709:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:30.0:1619677447.919711:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.919711:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.919712:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.919713:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:13.0:1619677447.919714:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.919715:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.919716:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.919717:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.919717:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:13.0:1619677447.919717:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.919719:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.919720:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.919721:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.919721:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.919726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.919735:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.919737:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.919738:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.919740:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.919740:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.919743:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677447.919757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.919763:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.919766:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.919767:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.919769:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.919771:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.919772:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.919773:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.919775:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.919776:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.919777:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.919780:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.919782:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.919785:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.919785:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.919786:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.919787:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.919788:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.919789:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.919789:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.919801:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.919804:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.919805:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.919805:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.919807:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.2:1619677447.919807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:21.2:1619677447.919809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:22.0:1619677447.919810:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.919815:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:18.0:1619677447.919815:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.919818:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.919818:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:24.0:1619677447.919819:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677447.919819:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677447.919823:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.919823:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.919826:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.919827:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.919827:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677447.919829:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677447.919829:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.919830:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.919831:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.919832:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.919833:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.919833:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677447.919835:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.919837:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677447.919838:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.919840:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:24.0:1619677447.919841:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.919843:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.919843:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677447.919844:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677447.919844:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677447.919846:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.919846:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.919846:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:24.0:1619677447.919847:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.919847:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:18.0:1619677447.919847:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:24.0:1619677447.919848:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:18.0:1619677447.919848:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677447.919849:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677447.919850:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677447.919851:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677447.919855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:6.0:1619677447.919859:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677447.919861:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:6.0:1619677447.919862:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.919863:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.919863:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:6.0:1619677447.919863:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.919864:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.919865:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:6.0:1619677447.919866:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:8.0:1619677447.919867:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.919869:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.919869:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.919933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.919936:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.919939:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.919941:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.919942:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.919943:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.919945:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.919946:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.919947:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.919948:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.919949:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.919953:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.919955:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.919957:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.919958:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.919958:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.919960:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.919960:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.919961:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.919962:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.919973:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.919976:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.919977:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.919977:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.919979:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.919981:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.920146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.2:1619677447.920150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677447.920151:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.920154:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.920155:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.920157:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.920158:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677447.920159:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.920160:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:31.0:1619677447.920161:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:1.0:1619677447.920162:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.920164:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.920164:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:1.0:1619677447.920164:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.920165:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.920166:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677447.920167:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.920169:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677447.920170:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.920171:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.920173:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.920173:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:31.0:1619677447.920174:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.920175:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:1.0:1619677447.920175:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.920176:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.920177:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.920178:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:31.0:1619677447.920179:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.920179:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.920180:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.920180:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.920181:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677447.920183:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.920184:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677447.920184:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677447.920186:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.920186:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.920187:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.920189:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.920190:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677447.920192:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.920195:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:21.2:1619677447.920195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677447.920195:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.920196:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.920197:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:6.0:1619677447.920198:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.920200:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:6.0:1619677447.920200:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:26.0:1619677447.920201:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.920201:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.920203:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:26.0:1619677447.920203:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.920205:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:26.0:1619677447.920205:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:30.0:1619677447.920206:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.920207:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677447.920243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.920250:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.920252:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.920253:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.920255:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.920256:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.920258:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.920259:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.920260:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.920261:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.920262:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677447.920266:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.920267:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677447.920270:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.920271:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.920271:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.920273:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.920274:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.920274:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.920275:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677447.920286:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.920289:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.920290:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.920290:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.920292:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.2:1619677447.920292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.920301:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:22.0:1619677447.920301:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677447.920303:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.920306:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.920308:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.920308:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.920310:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677447.920311:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.920312:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.920313:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.920314:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677447.920318:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.920319:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677447.920322:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.920323:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.920324:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677447.920325:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.920326:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.920327:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.920328:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.920332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.920341:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.920343:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.920344:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.920346:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.920346:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.920349:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677447.920351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.920358:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.920361:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.920362:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.920363:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.920363:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.920364:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677447.920534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.920542:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.920545:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.920547:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.920548:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.920550:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.920551:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677447.920552:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.920554:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.920555:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.920555:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677447.920559:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.920561:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677447.920563:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.920564:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677447.920564:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677447.920566:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.920566:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.2:1619677447.920566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677447.920567:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.920568:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677447.920572:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.920574:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.920575:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.920577:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.920579:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:17.0:1619677447.920580:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.920581:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:17.0:1619677447.920581:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677447.920582:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.920582:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721753] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.920582:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:17.0:1619677447.920584:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.920585:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.920585:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.920586:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677447.920589:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:17.0:1619677447.920589:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.920590:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:17.0:1619677447.920591:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677447.920592:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.920592:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.920593:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.920594:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.920594:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:17.0:1619677447.920594:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.920595:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.920596:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.920597:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.920598:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.920598:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.920599:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.920611:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.920613:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.920615:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.920615:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.920617:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.920619:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.920630:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.920633:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.920633:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.920634:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.920635:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.920635:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.920755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.920762:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.920765:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.920766:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.920767:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.920769:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.920771:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.920772:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.920773:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.920774:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.920775:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.920779:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.920781:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.920784:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.920784:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.920785:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.920787:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.920787:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.920788:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.920788:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.920793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.920801:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.920803:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.920804:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.920804:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.920807:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.920810:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677447.920866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.920872:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.920874:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.920876:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677447.920876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:2.0:1619677447.920877:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.920880:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.920882:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677447.920882:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.920883:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.920884:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.920885:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:2.0:1619677447.920885:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.920885:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:2.0:1619677447.920886:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677447.920887:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.920890:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.920890:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.920891:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:2.0:1619677447.920891:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:17.0:1619677447.920893:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.920894:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.920894:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.920895:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:17.0:1619677447.920896:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.920896:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:2.0:1619677447.920896:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.920898:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.920899:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.920899:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677447.920900:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.920900:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677447.920902:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.2:1619677447.920904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677447.920905:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.920905:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.920906:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.920908:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.920908:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.920909:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.920910:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:20.2:1619677447.920911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:17.0:1619677447.920911:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677447.920913:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.2:1619677447.920914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:9.2:1619677447.920914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.2:1619677447.920916:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677447.920917:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.920918:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:5.0:1619677447.920918:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.920919:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.920920:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:20.0:1619677447.920921:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.920921:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.920921:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677447.920923:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677447.920923:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:24.0:1619677447.920924:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:20.0:1619677447.920924:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.920924:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:1.0:1619677447.920924:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.920925:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677447.920925:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.920925:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:20.0:1619677447.920926:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.920927:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.920927:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.920927:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.920928:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.920929:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.920929:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.920931:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.920931:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:5.0:1619677447.920934:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.920935:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000400:00000200:5.0:1619677447.920936:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.920938:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.920938:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.920939:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.920940:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.920941:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.920942:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.920945:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.920946:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:5.0:1619677447.920946:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.920947:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.920948:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.920949:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.920949:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.920950:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.920951:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.920952:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:5.0:1619677447.920952:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.920952:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.920953:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.920953:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.920955:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677447.920956:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.920957:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.920959:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.920959:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677447.920963:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:20.2:1619677447.920964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677447.920965:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:20.0:1619677447.920967:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:12.0:1619677447.920967:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.920969:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:12.0:1619677447.920969:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:20.0:1619677447.920970:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677447.920970:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:20.0:1619677447.920971:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.920971:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677447.920972:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.920972:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.920973:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.920973:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.920978:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.920979:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.920981:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.920982:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.920982:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.920983:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.920983:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.921124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.921132:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.921135:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.921137:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.921138:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.921139:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.921141:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677447.921142:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721747] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.921143:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.921144:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.921146:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677447.921149:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.921151:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677447.921153:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.921154:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677447.921155:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677447.921156:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.921157:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.921158:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.921158:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677447.921165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.921171:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.921173:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.921174:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:10.0:1619677447.921174:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.921175:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677447.921176:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:26.0:1619677447.921178:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.921178:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.921179:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:26.0:1619677447.921180:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.921180:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.921182:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677447.921183:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.921185:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.921186:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.921187:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677447.921189:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.921191:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677447.921193:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.921194:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677447.921194:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677447.921196:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677447.921196:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.921197:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.921197:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:4.2:1619677447.921200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.921207:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677447.921208:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:10.0:1619677447.921209:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:31.0:1619677447.921210:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:26.0:1619677447.921210:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677447.921211:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.921211:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.921212:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.921214:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:10.0:1619677447.921214:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.921215:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:10.0:1619677447.921217:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677447.921218:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677447.921219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:10.0:1619677447.921219:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.921220:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.921221:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677447.921224:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.921226:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:10.0:1619677447.921226:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677447.921228:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:24.0:1619677447.921229:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:10.0:1619677447.921229:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.921230:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677447.921230:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.921230:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:30.0:1619677447.921231:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.921232:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000400:00000200:24.0:1619677447.921232:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.921232:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677447.921232:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677447.921233:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677447.921233:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677447.921233:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677447.921234:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:24.0:1619677447.921234:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.921234:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:24.0:1619677447.921236:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677447.921238:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677447.921238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:24.0:1619677447.921240:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.921241:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.921242:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677447.921245:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.921246:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:24.0:1619677447.921247:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.921249:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.921249:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:24.0:1619677447.921250:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677447.921250:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:20.0:1619677447.921250:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.921251:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.921252:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.921252:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.921253:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677447.921253:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.921254:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677447.921255:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:12.0:1619677447.921264:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.921266:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.921267:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.921269:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.921270:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.921271:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.921275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.921283:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.921285:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.921288:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.921289:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.921290:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.921292:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.921297:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677447.921297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:31.0:1619677447.921298:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.921300:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.921301:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.921303:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.921303:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677447.921305:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.921306:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.921307:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677447.921308:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.921308:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.921309:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:1.0:1619677447.921309:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.921311:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.921311:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.921312:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:1.0:1619677447.921312:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.921313:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:1.0:1619677447.921313:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.921314:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.921315:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.921317:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.921317:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677447.921321:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.921322:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:13.0:1619677447.921325:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677447.921325:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.921326:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.921327:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:13.0:1619677447.921328:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677447.921328:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000400:00000200:13.0:1619677447.921329:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.921329:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:13.0:1619677447.921330:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.921330:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.921330:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.921332:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.921333:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677447.921334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.921342:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:12.0:1619677447.921344:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.921345:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.921346:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:12.0:1619677447.921346:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677447.921346:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.921347:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.921348:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.921348:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:8.0:1619677447.921348:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.921350:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.921351:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677447.921396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.921403:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.921405:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.921406:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.921408:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.921411:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.921412:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.921413:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.921415:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.921416:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.921417:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.921420:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.921422:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.921425:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.921426:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.921426:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.921428:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.921429:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.921429:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.921430:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:26.0:1619677447.921441:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677447.921444:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677447.921445:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.921445:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677447.921447:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.921450:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:10.0:1619677447.921458:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677447.921461:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677447.921461:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677447.921462:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677447.921463:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677447.921463:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677447.921691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.921697:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.921699:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.921702:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.921702:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.921704:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.921706:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.2:1619677447.921708:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:5.0:1619677447.921712:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.921713:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.921714:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.921715:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:5.0:1619677447.921715:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677447.921718:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.921719:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:5.0:1619677447.921719:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677447.921720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:26.0:1619677447.921721:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.921721:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.921724:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677447.921725:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.921725:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:26.0:1619677447.921726:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:5.0:1619677447.921726:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:26.0:1619677447.921728:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.921728:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677447.921728:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.921728:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677447.921729:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:5.0:1619677447.921729:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677447.921730:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.921730:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:26.0:1619677447.921731:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.921732:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677447.921735:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.921736:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677447.921737:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.921738:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:10.0:1619677447.921739:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:13.0:1619677447.921740:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677447.921740:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96721738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.921741:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.921742:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.921743:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.921743:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.921743:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:13.0:1619677447.921745:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.921746:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.921746:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677447.921747:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:10.0:1619677447.921747:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.921748:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:13.0:1619677447.921748:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677447.921749:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677447.921749:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.921750:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.921752:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.921752:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677447.921753:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.921753:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.921754:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.921754:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.921756:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.921757:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.921758:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677447.921759:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.921760:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677447.921762:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.921765:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.921765:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.921765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677447.921767:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.921769:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.921770:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:22.0:1619677447.921771:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677447.921773:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677447.921774:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677447.921775:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677447.921775:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.921776:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.921776:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.921777:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677447.921778:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.921782:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.921794:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.921797:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.921798:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.921798:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.921799:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.921799:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.921828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.921836:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.921839:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.921839:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.921841:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677447.921842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:24.0:1619677447.921844:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.921845:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.921846:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721737] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.921848:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.921848:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.921849:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:18.0:1619677447.921849:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.921852:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.921852:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:24.0:1619677447.921854:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677447.921854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:18.0:1619677447.921855:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.921857:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:18.0:1619677447.921858:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.921859:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:24.0:1619677447.921860:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.921860:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.921861:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:18.0:1619677447.921861:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.921861:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:18.0:1619677447.921862:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.921863:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.921863:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:18.0:1619677447.921863:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.921864:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:18.0:1619677447.921864:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677447.921865:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:24.0:1619677447.921865:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677447.921865:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:18.0:1619677447.921869:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677447.921869:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677447.921869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:18.0:1619677447.921872:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:10.0:1619677447.921872:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.921873:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:18.0:1619677447.921874:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.921875:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:10.0:1619677447.921875:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96721735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.921876:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:10.0:1619677447.921876:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.921877:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:10.0:1619677447.921877:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.921878:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.921878:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.921879:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:10.0:1619677447.921879:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.921880:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:17.0:1619677447.921880:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:18.0:1619677447.921881:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677447.921881:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.921881:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.921883:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:10.0:1619677447.921883:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677447.921884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.2:1619677447.921884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677447.921885:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677447.921887:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:10.0:1619677447.921888:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677447.921888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:10.0:1619677447.921889:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677447.921890:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.921892:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:10.0:1619677447.921892:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677447.921892:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:30.0:1619677447.921893:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677447.921893:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:2.0:1619677447.921893:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:10.0:1619677447.921894:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:20.0:1619677447.921895:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677447.921896:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:20.0:1619677447.921896:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.921896:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.921896:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.921897:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:20.0:1619677447.921898:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:8.2:1619677447.921898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:30.0:1619677447.921899:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.921900:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677447.921901:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:30.0:1619677447.921902:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.921902:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677447.921903:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677447.921904:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.921904:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.921905:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.921905:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000400:00000200:30.0:1619677447.921906:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.921906:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.921906:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.921906:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000400:00000200:30.0:1619677447.921907:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.921907:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.921908:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:30.0:1619677447.921909:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.921909:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.921909:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.921911:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677447.921911:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000400:00000200:30.0:1619677447.921913:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.921915:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.921917:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.921918:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.921918:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.921919:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:30.0:1619677447.921920:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.921921:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677447.921921:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:30.0:1619677447.921922:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.921922:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677447.921922:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677447.921924:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677447.921927:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.921928:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677447.921929:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.921931:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.921932:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677447.921932:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.921933:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:12.0:1619677447.921934:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.921935:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.921936:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:12.0:1619677447.921937:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.921938:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.921938:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:12.0:1619677447.921939:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.921941:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.921942:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677447.921942:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677447.921944:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.921944:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677447.921946:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677447.921947:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677447.921948:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:5.2:1619677447.921953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (21)++
00000800:00000200:22.0:1619677447.921958:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:24.0:1619677447.921960:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:22.0:1619677447.921961:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:24.0:1619677447.921962:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:6.0:1619677447.921962:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677447.921963:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:22.0:1619677447.921963:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.921964:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000400:00000200:22.0:1619677447.921965:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.921968:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677447.922006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.922011:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.922013:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.922014:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.922016:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.922019:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.922020:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.922021:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.922022:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.922024:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.922025:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.922028:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.922030:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.922032:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.922033:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.922034:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.922035:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.922036:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.922036:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.922037:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.922053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.922060:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.922063:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.922064:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.922064:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.922066:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.922068:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.922194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.922197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:26.0:1619677447.922202:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.2:1619677447.922203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677447.922205:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677447.922205:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.922205:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677447.922208:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:26.0:1619677447.922208:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.922208:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677447.922209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:26.0:1619677447.922210:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.922211:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.922211:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:30.0:1619677447.922212:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:1.0:1619677447.922212:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677447.922213:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677447.922214:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.922214:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.922215:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.922216:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.922217:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.922217:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677447.922217:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677447.922218:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.922219:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677447.922219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:10.0:1619677447.922219:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:31.0:1619677447.922220:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.922220:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.922221:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.922221:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677447.922221:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.922222:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677447.922223:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.922224:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:17.0:1619677447.922224:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.922224:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.922226:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677447.922227:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677447.922227:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.922228:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.922228:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:10.0:1619677447.922229:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:26.0:1619677447.922230:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:10.0:1619677447.922231:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96721728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.922232:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.922232:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677447.922233:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.922234:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.922235:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.922235:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:10.0:1619677447.922235:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.922236:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:31.0:1619677447.922237:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:10.0:1619677447.922237:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677447.922238:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96721727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.922239:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.922239:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:18.0:1619677447.922240:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.922240:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:18.0:1619677447.922241:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.922242:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.922242:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.922242:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.922243:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677447.922243:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.922243:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677447.922244:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677447.922244:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:17.0:1619677447.922244:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677447.922244:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.922245:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677447.922245:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.922246:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677447.922246:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.922247:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677447.922247:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677447.922248:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:26.0:1619677447.922248:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.922249:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677447.922249:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.922250:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.922250:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677447.922250:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677447.922250:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.922251:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677447.922251:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:26.0:1619677447.922252:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.922252:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:8.2:1619677447.922252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.922254:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.922254:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:18.0:1619677447.922255:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677447.922255:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:10.0:1619677447.922255:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.2:1619677447.922255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677447.922256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677447.922256:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (24)++
00000800:00000200:10.0:1619677447.922256:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677447.922257:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677447.922258:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677447.922259:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (26)--
00000800:00000200:18.0:1619677447.922259:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.922260:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:13.0:1619677447.922260:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (25)--
00000800:00000200:18.0:1619677447.922261:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:13.0:1619677447.922261:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.922261:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:13.0:1619677447.922262:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.922262:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)++
00000800:00000200:12.0:1619677447.922263:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (25)--
00000800:00000200:20.0:1619677447.922264:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:12.0:1619677447.922264:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.922264:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677447.922265:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:12.0:1619677447.922265:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.922266:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:13.0:1619677447.922266:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:30.0:1619677447.922267:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:20.0:1619677447.922267:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.922268:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:13.0:1619677447.922268:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677447.922268:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:5.0:1619677447.922268:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677447.922269:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677447.922269:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.922269:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677447.922269:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:5.0:1619677447.922269:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.922270:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.922272:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:20.0:1619677447.922273:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:13.0:1619677447.922273:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.922273:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.922274:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:20.0:1619677447.922274:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677447.922275:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677447.922275:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677447.922276:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677447.922276:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.0:1619677447.922277:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.922747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677447.922751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:26.0:1619677447.922755:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.922757:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:22.0:1619677447.922757:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677447.922758:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677447.922760:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:26.0:1619677447.922761:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.922761:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:26.0:1619677447.922764:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.922764:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.922766:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.922766:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.922768:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.922768:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677447.922770:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.922770:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.922772:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.922772:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.922774:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.922775:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.922775:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.922780:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.922781:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677447.922782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:22.0:1619677447.922783:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677447.922784:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.922786:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.922787:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.922787:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.922787:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677447.922788:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.922788:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.922788:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.922790:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.922790:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677447.922790:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:26.0:1619677447.922791:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.922791:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677447.922791:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:26.0:1619677447.922792:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677447.922792:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:26.0:1619677447.922793:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677447.922793:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.922793:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677447.922793:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.922796:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.922797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677447.922798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677447.922798:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.922799:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96721724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.922800:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.922802:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.922803:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:2.0:1619677447.922803:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677447.922805:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:6.0:1619677447.922805:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.922805:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677447.922805:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677447.922806:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:2.0:1619677447.922806:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.922808:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:2.0:1619677447.922808:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677447.922808:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:30.0:1619677447.922809:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.922809:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.922809:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.922810:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.922811:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:30.0:1619677447.922812:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.922812:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677447.922813:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:2.0:1619677447.922813:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:1.0:1619677447.922813:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.922814:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:31.0:1619677447.922815:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.922815:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:2.0:1619677447.922815:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.922815:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:1.0:1619677447.922816:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677447.922817:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.922817:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.922817:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:30.0:1619677447.922818:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.922819:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.922820:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:9.2:1619677447.922821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.922828:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.922831:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.922832:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.922832:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.922834:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.922836:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.922894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.922901:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.922904:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.922905:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.922907:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.922910:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.922912:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.922913:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.922915:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.922916:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.922917:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.922920:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.922922:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.922925:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.922925:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.922926:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.922928:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.922928:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.922929:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.922930:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.922930:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.922933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.922936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.922941:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.922943:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677447.922943:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.922944:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.922945:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677447.922945:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.922946:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677447.922946:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.922947:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.922949:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.923018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.923024:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.923026:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.923027:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.923029:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.923031:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.923032:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.923033:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721722] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.923035:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.923036:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.923037:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.923040:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.923042:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.923043:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.923044:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.923045:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.923046:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.923047:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.923047:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.923048:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.923052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.923059:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.923061:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.923062:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.923063:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.923064:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.923065:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.923075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.923083:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.923085:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.923086:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.923086:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.923086:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.923087:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.923200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.923208:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.923211:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.923213:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.923215:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.923216:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.923217:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.923218:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.923220:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.923221:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.923222:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.923225:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.923227:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.923230:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.923231:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.923232:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.923233:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.923234:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.923235:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.923236:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.923240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.923247:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.923250:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.923251:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.923253:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.923254:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.923256:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677447.923265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.923271:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.923273:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.923274:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.923276:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.923279:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.923280:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.923281:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.923282:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.923284:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.923285:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.923288:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.923290:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.923292:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.923293:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.923298:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.923299:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.923300:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.923300:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.923301:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.923306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.923314:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.923317:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.923318:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.923318:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.923320:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.923322:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677447.923369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.923376:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.923379:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.923382:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.923384:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.923384:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.923386:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.923387:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.923389:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.923390:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.923391:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.923394:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.923396:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.923399:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.923399:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.923400:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.923402:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.923402:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.923404:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.923405:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.923406:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677447.923411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.2:1619677447.923414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:12.0:1619677447.923416:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.923418:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677447.923419:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:12.0:1619677447.923419:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.923420:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.923420:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677447.923421:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.923422:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677447.923422:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.923423:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677447.923423:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:12.0:1619677447.923424:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.923425:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677447.923426:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:5.2:1619677447.923426:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:12.0:1619677447.923428:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.923428:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.923429:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.923430:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677447.923430:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.923432:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.923433:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:5.0:1619677447.923434:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.923435:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.923436:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.923437:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:5.0:1619677447.923437:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.923438:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.923440:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677447.923441:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.923444:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.923444:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677447.923445:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677447.923446:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677447.923447:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.923448:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.923449:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677447.923450:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.923450:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:10.0:1619677447.923451:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.923451:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.923453:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.923453:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:10.0:1619677447.923454:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.923454:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.923455:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:10.0:1619677447.923456:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677447.923459:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.923459:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677447.923460:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677447.923462:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.923462:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.923463:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.923463:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677447.923465:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.923468:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.923469:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.923470:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.923472:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.923473:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:12.0:1619677447.923473:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.923476:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.923476:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.923478:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.923479:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.923480:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.923621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:4.2:1619677447.923623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.923627:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677447.923630:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.923630:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.923631:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:26.0:1619677447.923632:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.923633:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677447.923633:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.923635:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.923636:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.923637:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677447.923638:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.923638:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.923640:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.923640:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.923642:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.923642:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.923643:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677447.923644:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.923645:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.923646:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.923652:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.923653:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.923654:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.923656:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.923658:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.923659:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.923659:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.923659:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.923660:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.923661:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.923661:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.923662:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677447.923663:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.923663:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.923663:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:26.0:1619677447.923664:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677447.923664:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:26.0:1619677447.923665:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677447.923665:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.2:1619677447.923670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677447.923673:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:5.2:1619677447.923674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677447.923675:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:10.0:1619677447.923676:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.923677:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.923677:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:10.0:1619677447.923677:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.923678:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.923679:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:10.0:1619677447.923679:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.923679:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:1.0:1619677447.923680:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677447.923682:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:1.0:1619677447.923682:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:22.0:1619677447.923683:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677447.923683:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677447.923683:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:6.0:1619677447.923683:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.923686:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.923686:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.923688:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.923689:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:1.0:1619677447.923690:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.923691:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.923693:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.923694:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.923695:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.923698:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.923700:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.923703:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.923704:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.923704:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.923706:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.923707:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.923709:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.923710:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.923711:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.923712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677447.923715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677447.923719:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:21.2:1619677447.923720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:17.0:1619677447.923720:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677447.923721:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.923722:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.923722:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.923723:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677447.923723:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.923724:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677447.923725:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.923725:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.0:1619677447.923728:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:2.0:1619677447.923728:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.923729:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.923731:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:5.2:1619677447.923732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:2.0:1619677447.923734:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.923736:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677447.923737:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.923738:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:2.0:1619677447.923739:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.923740:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.923740:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.923741:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677447.923744:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.923745:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000400:00000200:18.0:1619677447.923746:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.923747:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:18.0:1619677447.923748:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.923749:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721712] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.923750:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.923751:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.923751:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.923754:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.923756:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.923758:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.923758:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.923759:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.923759:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677447.923759:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677447.923760:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.923761:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.923762:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.923762:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677447.923762:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.923763:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.923763:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.923764:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.923764:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677447.923765:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.923768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.923775:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:22.0:1619677447.923777:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677447.923777:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.923777:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.923779:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:18.0:1619677447.923779:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.923779:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.923780:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.923780:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677447.923781:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:22.0:1619677447.923782:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.923785:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.923844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.923851:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.923854:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.923855:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.923856:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.923859:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.923861:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.923862:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.923864:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.923865:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.923866:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.923869:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.923871:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.923874:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.923875:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.923876:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.923877:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.923878:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.923878:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.923879:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.923883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.923891:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.923894:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.923895:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.923895:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.923897:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.923900:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.924033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0e00] (20)++
00000800:00000200:26.0:1619677447.924040:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0e00] (21)++
00000800:00000200:26.0:1619677447.924043:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.4.66@o2ib3
00000800:00000200:10.0:1619677447.924044:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0e00] (22)--
00000400:00000200:26.0:1619677447.924046:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.66@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.924050:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.924051:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677447.924052:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.924054:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.924055:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.924056:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677447.924059:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.66@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.924061:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677447.924063:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.924064:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677447.924064:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677447.924066:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677447.924066:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0e00] (21)++
00000800:00000200:26.0:1619677447.924067:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0e00] (22)--
00000800:00000200:26.0:1619677447.924068:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0e00] (21)--
00000800:00000200:12.0:1619677447.924078:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.924080:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.924081:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.924082:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:5.2:1619677447.924083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0e00] (20)++
00000800:00000200:13.0:1619677447.924084:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.924085:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677447.924090:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0e00] (21)++
00000800:00000200:10.0:1619677447.924092:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.66@o2ib3
00000800:00000200:26.0:1619677447.924093:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0e00] (22)--
00000400:00000200:10.0:1619677447.924095:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.66@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.924097:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:10.0:1619677447.924097:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.924099:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:13.0:1619677447.924100:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677447.924100:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.924101:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:10.0:1619677447.924101:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.924102:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.924102:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:10.0:1619677447.924102:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.924103:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:13.0:1619677447.924104:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:10.0:1619677447.924106:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.66@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.924108:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677447.924111:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.924112:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677447.924112:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677447.924114:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677447.924114:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0e00] (21)++
00000800:00000200:10.0:1619677447.924115:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0e00] (22)--
00000800:00000200:10.0:1619677447.924116:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0e00] (21)--
00000800:00000200:8.2:1619677447.924119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.924128:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.924130:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.924131:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.924131:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.924134:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.924136:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.924140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.924147:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.924150:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.2:1619677447.924150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0e00] (20)++
00000800:00000200:1.0:1619677447.924150:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.924151:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.924152:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.924152:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:26.0:1619677447.924157:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0e00] (21)++
00000800:00000200:26.0:1619677447.924160:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.66@o2ib3
00000800:00000200:10.0:1619677447.924161:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0e00] (22)--
00000400:00000200:26.0:1619677447.924162:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.51@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.66@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.924165:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.924166:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.51@o2ib7 local destination
00000400:00000200:26.0:1619677447.924168:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.924169:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677447.924169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:26.0:1619677447.924170:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.924170:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.51@o2ib7
00000400:00000200:26.0:1619677447.924174:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.66@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.51@o2ib7(10.0.10.51@o2ib7:10.0.10.51@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.924175:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.51@o2ib7
00000800:00000200:10.0:1619677447.924177:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.924178:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494880] -> 10.0.10.51@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.924178:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea3be00] (20)++
00000800:00000200:10.0:1619677447.924178:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.924179:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea3be00] (21)++
00000400:00000200:10.0:1619677447.924180:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.924181:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea3be00] (22)--
00000800:00000200:26.0:1619677447.924181:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0e00] (21)++
00000800:00000200:26.0:1619677447.924182:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0e00] (22)--
00000400:00000200:10.0:1619677447.924182:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.924183:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0e00] (21)--
00000800:00000200:26.0:1619677447.924184:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.924184:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677447.924185:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.924186:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.924187:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.924188:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677447.924192:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.924193:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea3be00] (22)++
00000800:00000200:26.0:1619677447.924194:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea3be00] (23)--
00000400:00000200:26.0:1619677447.924194:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.924194:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:26.0:1619677447.924196:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.51@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.924196:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.924197:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea3be00] (22)--
00000800:00000200:10.0:1619677447.924197:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677447.924198:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea3be00] (21)--
00000800:00000200:10.0:1619677447.924198:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677447.924199:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.924200:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.924201:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.924201:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.2:1619677447.924205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0e00] (20)++
00000800:00000200:12.0:1619677447.924212:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677447.924213:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0e00] (21)++
00000800:00000200:12.0:1619677447.924214:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:26.0:1619677447.924215:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.66@o2ib3
00000400:00000200:12.0:1619677447.924215:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.924216:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0e00] (22)--
00000400:00000200:12.0:1619677447.924217:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:26.0:1619677447.924218:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.66@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.924218:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.924219:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677447.924220:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.924222:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:26.0:1619677447.924223:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.924224:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.924225:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.924226:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:26.0:1619677447.924228:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.66@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.924230:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677447.924231:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.924231:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:26.0:1619677447.924232:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677447.924233:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677447.924234:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0e00] (21)++
00000800:00000200:26.0:1619677447.924235:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0e00] (22)++
00000800:00000200:26.0:1619677447.924236:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0e00] (23)--
00000800:00000200:26.0:1619677447.924237:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0e00] (22)--
00000800:00000200:5.2:1619677447.924243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0e00] (21)++
00000800:00000200:10.0:1619677447.924248:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677447.924249:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677447.924249:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.924251:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0e00] (22)++
00000800:00000200:26.0:1619677447.924252:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.66@o2ib3
00000400:00000200:10.0:1619677447.924252:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.924253:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677447.924254:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.66@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.924254:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:10.0:1619677447.924254:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0e00] (23)++
00000800:00000200:10.0:1619677447.924255:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0e00] (24)--
00000400:00000200:26.0:1619677447.924256:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.924256:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0e00] (23)--
00000400:00000200:26.0:1619677447.924257:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677447.924257:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0e00] (22)--
00000400:00000200:26.0:1619677447.924258:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677447.924258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:26.0:1619677447.924260:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.924261:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.924262:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.924262:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.924264:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:26.0:1619677447.924265:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.66@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.924267:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.924267:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.924268:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677447.924269:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.924269:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.924270:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677447.924271:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677447.924271:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:26.0:1619677447.924272:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677447.924272:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0e00] (21)++
00000400:00000200:20.0:1619677447.924272:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.924273:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0e00] (22)--
00000800:00000200:26.0:1619677447.924274:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0e00] (21)--
00000400:00000200:20.0:1619677447.924274:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.924275:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.924276:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.924279:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.924281:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.924283:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.924284:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.924284:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.924284:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677447.924286:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677447.924286:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677447.924287:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.924287:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:13.0:1619677447.924287:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.924288:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:13.0:1619677447.924289:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.924290:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.924291:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677447.924292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.924301:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.924304:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.924305:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.924305:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.924307:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.924310:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.924312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0e00] (20)++
00000800:00000200:10.0:1619677447.924320:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0e00] (21)++
00000800:00000200:10.0:1619677447.924323:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.66@o2ib3
00000800:00000200:26.0:1619677447.924324:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0e00] (22)--
00000400:00000200:10.0:1619677447.924325:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.66@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.924328:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.924330:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677447.924331:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.924332:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.924333:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.924334:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677447.924338:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.66@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.924340:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.924343:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.924343:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677447.924344:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.924346:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.2:1619677447.924346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0e00] (21)++
00000800:00000200:10.0:1619677447.924347:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0e00] (21)++
00000800:00000200:10.0:1619677447.924348:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0e00] (23)--
00000800:00000200:10.0:1619677447.924350:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0e00] (22)--
00000800:00000200:10.0:1619677447.924351:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0e00] (21)++
00000800:00000200:9.2:1619677447.924352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.924353:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.66@o2ib3
00000800:00000200:26.0:1619677447.924354:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0e00] (22)--
00000400:00000200:10.0:1619677447.924355:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.66@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.924357:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.924359:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:2.0:1619677447.924359:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:10.0:1619677447.924360:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721702] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.924361:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.924362:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.924362:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.924363:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:10.0:1619677447.924363:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.924363:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.924365:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:10.0:1619677447.924368:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.66@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.924369:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677447.924370:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677447.924372:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.924373:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677447.924374:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.2:1619677447.924374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0e00] (21)++
00000800:00000200:10.0:1619677447.924375:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677447.924376:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0e00] (22)++
00000800:00000200:10.0:1619677447.924377:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0e00] (23)--
00000800:00000200:10.0:1619677447.924378:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0e00] (22)--
00000800:00000200:10.0:1619677447.924379:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0e00] (21)++
00000800:00000200:10.0:1619677447.924381:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.66@o2ib3
00000800:00000200:26.0:1619677447.924382:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0e00] (22)--
00000800:00000200:8.2:1619677447.924382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:10.0:1619677447.924383:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.66@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.924386:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.924387:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677447.924388:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.924389:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677447.924389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:10.0:1619677447.924390:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.924390:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:10.0:1619677447.924391:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.924393:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:10.0:1619677447.924394:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.66@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.924394:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.924396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:5.0:1619677447.924396:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.924397:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677447.924397:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677447.924397:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.924398:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:5.0:1619677447.924399:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:10.0:1619677447.924400:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677447.924400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000400:00000200:26.0:1619677447.924401:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.924401:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677447.924401:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677447.924403:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.924403:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0e00] (21)++
00000400:00000200:26.0:1619677447.924404:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.924404:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677447.924405:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:10.0:1619677447.924405:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0e00] (22)++
00000800:00000200:10.0:1619677447.924406:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0e00] (23)--
00000400:00000200:26.0:1619677447.924407:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.924407:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.924407:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0e00] (22)--
00000800:00000200:2.0:1619677447.924407:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:26.0:1619677447.924408:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.924408:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.924408:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000400:00000200:26.0:1619677447.924409:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.924409:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.924409:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.924410:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.2:1619677447.924410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0e00] (21)++
00000400:00000200:26.0:1619677447.924411:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.924411:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.924412:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.924413:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677447.924413:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677447.924414:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:10.0:1619677447.924414:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0e00] (22)++
00000400:00000200:26.0:1619677447.924415:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.924416:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.66@o2ib3
00000800:00000200:26.0:1619677447.924418:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:10.0:1619677447.924418:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.66@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.924420:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.924421:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:10.0:1619677447.924421:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.924422:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677447.924422:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:26.0:1619677447.924424:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:10.0:1619677447.924424:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96721699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.924425:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.924426:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.924427:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:26.0:1619677447.924428:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.924429:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.924430:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:9.2:1619677447.924430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.924431:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.924431:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.66@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.924432:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.924433:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677447.924433:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:26.0:1619677447.924434:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.924435:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677447.924436:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.924436:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:4.2:1619677447.924436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.924437:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000800:00000200:26.0:1619677447.924438:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.924438:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (24)++
00000800:00000200:26.0:1619677447.924439:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0e00] (23)++
00000800:00000200:26.0:1619677447.924440:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0e00] (24)--
00000800:00000200:10.0:1619677447.924440:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (25)--
00000800:00000200:10.0:1619677447.924440:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0e00] (24)++
00000800:00000200:10.0:1619677447.924441:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0e00] (25)--
00000800:00000200:10.0:1619677447.924441:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0e00] (23)--
00000800:00000200:2.0:1619677447.924441:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000800:00000200:26.0:1619677447.924442:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0e00] (22)--
00000800:00000200:18.0:1619677447.924442:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:10.0:1619677447.924442:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:2.0:1619677447.924442:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.924443:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.924444:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.924444:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0e00] (21)--
00000400:00000200:2.0:1619677447.924444:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.924445:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.924445:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677447.924445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.2:1619677447.924448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677447.924449:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.924450:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.924452:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.924453:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.924453:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.924454:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677447.924454:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.924456:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.924457:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.924458:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.924459:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.924462:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.924464:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677447.924465:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.924466:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.924467:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.924468:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:5.2:1619677447.924468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0e00] (20)++
00000400:00000200:17.0:1619677447.924471:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.924473:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:26.0:1619677447.924476:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0e00] (21)++
00000800:00000200:17.0:1619677447.924476:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.924476:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677447.924477:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:26.0:1619677447.924479:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.66@o2ib3
00000800:00000200:17.0:1619677447.924479:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.924479:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.924480:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677447.924480:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0e00] (22)--
00000800:00000200:17.0:1619677447.924481:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677447.924482:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.66@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677447.924483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:26.0:1619677447.924485:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.924485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:26.0:1619677447.924486:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677447.924487:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.924489:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.924489:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:26.0:1619677447.924490:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.924491:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:8.0:1619677447.924491:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:6.0:1619677447.924491:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.924492:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677447.924494:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.924495:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.924495:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.924495:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.924496:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.66@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.924497:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.924498:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.924499:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:6.0:1619677447.924499:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.924500:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.924501:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.924501:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:26.0:1619677447.924502:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677447.924502:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:6.0:1619677447.924502:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.924503:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.924504:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:6.0:1619677447.924504:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677447.924505:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0e00] (21)++
00000800:00000200:26.0:1619677447.924505:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0e00] (22)--
00000800:00000200:9.2:1619677447.924505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:26.0:1619677447.924506:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0e00] (21)--
00000400:00000200:6.0:1619677447.924507:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.924509:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:24.0:1619677447.924511:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:6.0:1619677447.924512:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.924513:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.924513:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:24.0:1619677447.924514:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.924515:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.924515:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:6.0:1619677447.924515:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.924515:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.924516:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677447.924517:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.924517:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677447.924517:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677447.924518:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.924520:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:20.2:1619677447.924520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.2:1619677447.924521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677447.924522:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.924523:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.924525:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.924526:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.924527:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677447.924529:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677447.924530:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:20.0:1619677447.924531:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:30.0:1619677447.924532:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.924532:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.924534:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677447.924534:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.924535:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.924535:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:30.0:1619677447.924536:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:20.0:1619677447.924536:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:30.0:1619677447.924538:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.924539:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.924540:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.924541:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677447.924544:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.924546:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.924548:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.924549:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.924550:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.924551:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.924552:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.924552:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.924553:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677447.924563:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.924565:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.924566:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.924568:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.924569:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.924570:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677447.924572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.924579:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.924582:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.924584:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.924586:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.924586:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.924587:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.924588:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.924589:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.924590:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.924591:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.924594:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.924595:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.924597:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.924598:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.924598:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.924600:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.924600:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.924601:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.924602:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.924613:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.924616:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.924616:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.924618:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.924620:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.924621:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.924630:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.924632:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.924633:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.924634:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.924634:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.924636:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.924680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677447.924686:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.924689:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.924690:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.924692:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.924694:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.924696:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.924697:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.924698:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.924699:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.924700:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.924704:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.924706:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.924708:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.924709:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.924710:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.924712:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.924712:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.924713:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.924713:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677447.924718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.924726:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.924729:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.924729:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.924731:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.924732:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.924734:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677447.924757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677447.924765:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677447.924768:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677447.924770:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.924771:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677447.924772:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.924774:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677447.924775:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.924776:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.924777:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.924779:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677447.924782:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.924784:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.2:1619677447.924784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:31.0:1619677447.924786:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.924787:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677447.924788:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677447.924789:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:22.0:1619677447.924789:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:31.0:1619677447.924790:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677447.924791:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:22.0:1619677447.924792:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677447.924792:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:31.0:1619677447.924793:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:31.0:1619677447.924793:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:22.0:1619677447.924794:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677447.924795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:22.0:1619677447.924797:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.924798:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:21.2:1619677447.924799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (21)++
00000400:00000200:22.0:1619677447.924800:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.924801:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.924801:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:22.0:1619677447.924802:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.924803:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.924803:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.924804:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.924804:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.924806:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)++
00000400:00000200:22.0:1619677447.924806:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.924808:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.924808:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.924809:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:30.0:1619677447.924809:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:8.0:1619677447.924810:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:30.0:1619677447.924811:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:22.0:1619677447.924812:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.924813:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:22.0:1619677447.924814:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677447.924816:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677447.924816:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677447.924817:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677447.924818:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:12.0:1619677447.924828:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.924831:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.2:1619677447.924832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:12.0:1619677447.924832:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.924833:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.924834:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.924835:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:12.0:1619677447.924836:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677447.924838:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.924840:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.924841:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.924843:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.924844:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.924845:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.924847:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.924848:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.924849:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.924852:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.924854:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.924856:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.924857:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.924858:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.924860:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.924860:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.924861:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.924862:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677447.924866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.924873:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.924875:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.924877:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.924877:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.924878:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.924882:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.924883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.924888:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.924891:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.924892:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.924894:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.924896:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.924897:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.924898:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.924900:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.924901:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.924902:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.924905:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.924906:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.924910:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.924910:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.924911:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.924913:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.924914:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.924914:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.924915:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677447.924926:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.924928:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.924929:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.924931:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.924932:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.924934:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.924999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.925006:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.925009:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.925011:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.925013:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.925013:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.925015:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677447.925016:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721688] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.925017:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.925018:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.925019:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677447.925023:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.925025:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677447.925028:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.925029:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677447.925029:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677447.925031:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.925032:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.925033:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.925033:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677447.925045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:6.0:1619677447.925045:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.925048:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.925048:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.925049:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.925051:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.925053:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:6.0:1619677447.925054:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677447.925056:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.925058:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.925060:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.925061:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.925062:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.925063:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.925064:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.925065:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.925066:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.925069:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.925070:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.925072:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.925073:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.925073:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.925075:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.925075:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.925076:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.925076:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677447.925087:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.925089:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.925090:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.925091:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.925093:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.925094:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677447.925107:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.925110:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.925111:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.925111:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677447.925112:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.925113:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677447.925145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677447.925151:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.2:1619677447.925152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677447.925154:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.925155:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.925157:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.925157:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677447.925160:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.925160:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.925161:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:2.0:1619677447.925162:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.925163:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.925164:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.925165:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.925166:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.925167:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677447.925167:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.925169:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.925170:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.925171:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.925172:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.925173:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.925174:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:1.0:1619677447.925174:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677447.925177:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.925178:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.925178:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:1.0:1619677447.925178:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.925180:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677447.925180:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677447.925181:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.925182:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.925182:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:1.0:1619677447.925183:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.925184:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.925184:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:8.2:1619677447.925186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677447.925186:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.925186:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.925187:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.925188:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677447.925195:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.925197:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.925198:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.925199:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:20.0:1619677447.925200:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.925200:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.925202:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.925202:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:5.0:1619677447.925202:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:22.0:1619677447.925203:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.925205:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.925207:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.925220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.925227:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.925230:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.925231:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.925233:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.925235:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.925236:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.925238:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.925239:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.925240:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.925241:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677447.925245:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.925247:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677447.925250:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.925251:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.925251:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.925253:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.925253:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.925254:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.925255:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677447.925266:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.925269:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.925270:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.925270:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.925273:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.925276:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.925341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.925344:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.925347:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.925349:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.925350:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.925352:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.925353:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.925355:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.925356:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.925357:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.925358:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.925361:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.925363:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.925366:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.925367:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.925368:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.925370:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.925370:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.925372:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.925373:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.925373:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.925378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677447.925384:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.925384:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.925386:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:12.0:1619677447.925387:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677447.925387:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:12.0:1619677447.925388:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.925388:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:12.0:1619677447.925390:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.925392:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.925393:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.925430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.925438:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.925440:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.925443:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.925445:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.925445:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.925446:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677447.925448:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721682] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.925449:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.925450:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.925451:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677447.925454:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.925456:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677447.925459:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.925460:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677447.925461:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677447.925462:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.925463:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.925464:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.925464:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677447.925477:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.925480:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.925480:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.925481:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.925483:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.925486:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.925495:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.925497:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.925498:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.925498:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.925499:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.925500:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.925514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.925522:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.925524:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.925526:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.925528:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.925529:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.925530:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.925532:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.925533:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.925534:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.925535:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.925538:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.925540:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.925543:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.925544:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.925544:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.925546:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.925547:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.925548:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.925550:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.925550:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677447.925553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677447.925556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677447.925561:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.925563:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677447.925564:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:5.0:1619677447.925564:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.925566:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:5.0:1619677447.925566:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.925567:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677447.925567:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677447.925568:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677447.925569:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.925569:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.925570:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:20.0:1619677447.925570:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.925571:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.925572:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.925643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.925651:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.925653:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.925654:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.925656:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.925659:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.925660:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.925661:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.925662:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.925663:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.925664:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.925668:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.925669:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.925673:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.925673:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.925674:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.925676:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.925676:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.925677:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.925677:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.925682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.925690:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.925693:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.925693:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.925694:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.925696:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.925699:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.925766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677447.925773:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.925776:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677447.925777:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677447.925779:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.925782:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.925783:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677447.925785:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.925786:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.925787:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.925788:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677447.925791:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.925793:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677447.925796:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.925797:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677447.925798:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.925800:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.925800:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.925801:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677447.925803:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677447.925803:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:4.2:1619677447.925804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677447.925806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677447.925809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.925813:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677447.925814:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677447.925815:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:18.0:1619677447.925817:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:10.0:1619677447.925817:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.925818:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:18.0:1619677447.925818:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.925818:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.925819:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677447.925820:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:18.0:1619677447.925820:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.925821:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.925821:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.925823:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.2:1619677447.925823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:10.0:1619677447.925823:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.925825:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.925826:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677447.925827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677447.925828:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.925828:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677447.925830:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677447.925831:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:10.0:1619677447.925832:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.925833:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:18.0:1619677447.925833:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.925834:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.925834:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.925835:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677447.925836:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.925837:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677447.925837:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.925837:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.925838:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.925838:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677447.925838:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.925838:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:31.0:1619677447.925840:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.925840:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:31.0:1619677447.925841:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:10.0:1619677447.925841:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677447.925842:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.925842:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.925842:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:31.0:1619677447.925844:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.925845:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.925846:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677447.925849:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.925850:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677447.925852:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.925853:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677447.925853:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:22.0:1619677447.925854:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:31.0:1619677447.925855:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.925856:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.925857:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677447.925857:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.925857:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.925858:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:22.0:1619677447.925858:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.925859:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677447.925860:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.925863:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677447.925865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677447.925867:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.925869:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:21.2:1619677447.925869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:10.0:1619677447.925870:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.925871:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.925872:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:26.0:1619677447.925873:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.925874:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:30.0:1619677447.925875:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.925876:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677447.925876:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:30.0:1619677447.925877:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677447.925877:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.925878:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.925879:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.925881:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.925883:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:10.0:1619677447.925884:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000400:00000200:2.0:1619677447.925884:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.925885:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.925886:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677447.925887:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.925888:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.925888:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000400:00000200:2.0:1619677447.925888:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:10.0:1619677447.925889:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.925890:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:2.0:1619677447.925891:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.925893:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.925896:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.925897:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.925898:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.925899:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.925900:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.925900:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.925901:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677447.925905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.925914:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.925916:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.925917:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.925919:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.925920:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.925922:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677447.925931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.925937:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.925939:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.925940:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.925942:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.925944:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.925946:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.925947:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.925949:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.925950:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.925951:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.925954:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.925956:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.925959:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.925960:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.925961:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.925963:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.925963:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.925964:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.925964:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.925977:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.925981:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.925982:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.925983:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.925984:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.925986:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.925996:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.925999:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.926000:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.926000:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.926001:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.926002:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.926104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.926110:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.926112:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.926114:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.926115:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.926117:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.926118:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.926120:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.926121:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.926122:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.926123:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.926126:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.926127:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.926130:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.926132:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.926132:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.926134:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.926135:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.926135:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.926136:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.926147:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.926149:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.926150:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.926151:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.926153:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.926154:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.926235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677447.926241:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.926244:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.926244:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.926246:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.926249:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.926250:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.926252:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.926253:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.926254:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.926255:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.926258:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.926260:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.926262:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.926263:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.926264:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.926265:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.926266:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.926267:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.926268:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.926268:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.2:1619677447.926274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:6.0:1619677447.926279:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677447.926279:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.926281:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.926281:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677447.926282:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677447.926282:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.926283:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.926283:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.926285:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677447.926287:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.926317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677447.926322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:17.0:1619677447.926323:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.926326:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:5.2:1619677447.926326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:1.0:1619677447.926327:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.926329:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.926331:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677447.926332:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.926333:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:26.0:1619677447.926334:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:17.0:1619677447.926334:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677447.926334:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:17.0:1619677447.926335:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.926336:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.926337:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677447.926337:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.926338:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.926338:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.926340:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.926340:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.926341:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.926343:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677447.926343:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.926344:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.926345:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.926345:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:17.0:1619677447.926346:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:26.0:1619677447.926347:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.926347:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.926349:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.926349:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677447.926349:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.926350:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.926350:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:2.0:1619677447.926350:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.926351:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:21.2:1619677447.926351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677447.926352:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677447.926352:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677447.926357:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.926357:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.926359:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677447.926359:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677447.926360:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677447.926360:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.926361:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:30.0:1619677447.926362:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677447.926362:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.926362:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.926362:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677447.926363:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677447.926364:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.926364:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677447.926364:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:2.0:1619677447.926364:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.926365:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677447.926366:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:13.0:1619677447.926366:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677447.926367:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.926367:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.926367:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.2:1619677447.926367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677447.926367:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.926367:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:26.0:1619677447.926368:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:30.0:1619677447.926369:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:26.0:1619677447.926369:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677447.926369:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677447.926369:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:30.0:1619677447.926370:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96721669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.926370:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.926370:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:30.0:1619677447.926371:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.926372:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677447.926372:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.926373:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.926373:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:4.2:1619677447.926373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.926373:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677447.926374:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677447.926375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.926376:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:30.0:1619677447.926377:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.926377:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677447.926378:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:1.0:1619677447.926378:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.926379:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677447.926379:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.926379:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:31.0:1619677447.926380:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.926380:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677447.926382:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.926382:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.926383:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.926383:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.926383:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:10.0:1619677447.926383:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:1.0:1619677447.926383:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:31.0:1619677447.926384:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677447.926384:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677447.926384:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.926385:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.926385:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.926385:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.926386:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677447.926386:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.926387:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:10.0:1619677447.926387:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.926389:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:10.0:1619677447.926390:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.926391:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:8.2:1619677447.926391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000400:00000200:10.0:1619677447.926393:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.926394:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.926395:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.926396:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677447.926396:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:1.0:1619677447.926398:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677447.926399:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.926399:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.926399:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.926402:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:12.0:1619677447.926402:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.926402:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677447.926402:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677447.926403:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.926403:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677447.926404:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.926404:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.926404:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677447.926405:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.926406:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677447.926406:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677447.926408:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677447.926409:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.926409:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.926410:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677447.926414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.926422:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.926424:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.926425:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.926427:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.926428:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.926430:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677447.926433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.926440:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.926443:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.926444:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.926445:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.926445:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.926447:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677447.926505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.926511:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.926514:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.926515:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.926517:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.926519:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.926521:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.926522:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721667] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.926524:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.926525:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.926526:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.926529:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.926530:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.926533:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.926534:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.926535:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.926536:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.926537:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.926538:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.926538:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:26.0:1619677447.926559:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677447.926562:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677447.926563:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.926563:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677447.926565:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.926568:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:4.2:1619677447.926664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.926671:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.926674:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.926674:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677447.926676:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.926679:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.926680:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677447.926681:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.926683:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.926684:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.926685:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677447.926688:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.926690:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677447.926692:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.926693:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.926694:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677447.926695:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.926696:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.926697:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677447.926698:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.926699:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677447.926704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677447.926708:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.926710:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.926711:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.926711:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677447.926712:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.926713:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:30.0:1619677447.926714:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.926714:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.926715:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677447.926716:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.926820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.926826:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.926829:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.926830:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.926832:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.926834:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.926836:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.926837:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.926838:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.926839:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677447.926840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:17.0:1619677447.926840:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.926844:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.926846:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.926847:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677447.926849:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.926850:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677447.926850:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.926850:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:30.0:1619677447.926852:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.926852:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.926853:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677447.926854:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.926854:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.926854:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.926854:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677447.926855:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.926857:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.926858:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.926859:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.926860:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.926864:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.926865:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.926865:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677447.926868:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.926868:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.926868:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.926869:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.926870:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:13.0:1619677447.926870:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.926871:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.926872:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677447.926872:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.926873:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.926873:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.926873:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.926883:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.926885:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.926886:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.926887:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.926889:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.926890:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.926902:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.926904:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.926905:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.926906:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.926906:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.926907:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.926924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.926931:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.926934:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.926935:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.926937:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.926939:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.926940:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677447.926941:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.926942:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.926943:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.926944:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677447.926947:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.926949:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677447.926951:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.926951:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677447.926952:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677447.926953:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677447.926954:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.926955:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677447.926956:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677447.926956:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:21.2:1619677447.926957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:5.2:1619677447.926962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.2:1619677447.926963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.0:1619677447.926964:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.926966:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:31.0:1619677447.926968:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:24.0:1619677447.926968:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:31.0:1619677447.926969:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:26.0:1619677447.926969:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677447.926969:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.926970:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.926971:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.926971:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:30.0:1619677447.926972:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677447.926972:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:8.0:1619677447.926972:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.926973:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.926973:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:26.0:1619677447.926973:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677447.926974:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:8.0:1619677447.926974:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.926975:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.926975:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.926976:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:8.0:1619677447.926976:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.926977:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.926977:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.926978:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.926979:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.926980:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.926982:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.926982:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.926983:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.926984:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.926985:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.926987:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.926987:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.926988:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.926989:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.926989:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.926990:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.926991:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.926991:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:30.0:1619677447.926992:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.926992:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.0:1619677447.926992:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:30.0:1619677447.926994:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:8.0:1619677447.926994:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.926994:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:30.0:1619677447.926995:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.926995:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.926996:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:21.2:1619677447.927000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:13.0:1619677447.927002:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677447.927004:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677447.927005:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.927007:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:13.0:1619677447.927007:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.927008:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:24.0:1619677447.927009:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:13.0:1619677447.927009:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:24.0:1619677447.927010:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.927010:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677447.927010:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.927011:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.927011:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.927012:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.927014:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.927091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.927097:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.927100:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.927102:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.927102:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.927105:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.927106:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.927108:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.927109:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.927110:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.927111:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.927115:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.927117:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.927120:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.927121:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.927122:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.927123:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.927124:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.927125:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.927126:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.927127:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.2:1619677447.927131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:20.2:1619677447.927132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.927135:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:31.0:1619677447.927136:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677447.927137:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:18.0:1619677447.927137:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677447.927138:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677447.927139:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677447.927139:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:31.0:1619677447.927140:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.927140:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.927140:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:31.0:1619677447.927142:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:30.0:1619677447.927142:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:18.0:1619677447.927143:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.927145:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:18.0:1619677447.927145:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.927146:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.927148:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.927149:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.927150:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677447.927151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:18.0:1619677447.927152:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.927155:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.927157:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677447.927158:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677447.927159:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.927160:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.927161:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677447.927161:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.927162:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677447.927162:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.927163:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.927164:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:10.0:1619677447.927164:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.927165:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.927166:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:10.0:1619677447.927166:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.927168:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.927169:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677447.927169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677447.927170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000400:00000200:10.0:1619677447.927171:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.927172:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.927173:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677447.927176:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.927176:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:1.0:1619677447.927176:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677447.927177:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:10.0:1619677447.927178:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.927178:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.927179:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677447.927179:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.927180:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.927180:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.927182:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677447.927183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:10.0:1619677447.927183:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.927183:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:1.0:1619677447.927183:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.927185:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.927185:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677447.927185:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677447.927186:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.927187:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:18.0:1619677447.927189:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677447.927192:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677447.927192:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677447.927195:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.927197:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.927198:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.927199:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.927199:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:18.0:1619677447.927201:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.927201:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.927202:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:18.0:1619677447.927202:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.927203:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.927203:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.927205:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.2:1619677447.927206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:18.0:1619677447.927207:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.927209:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.927209:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677447.927211:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.927212:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.927212:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.927213:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.927214:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.927215:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:17.0:1619677447.927215:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.927215:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677447.927216:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677447.927216:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:17.0:1619677447.927217:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.927220:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.927220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:17.0:1619677447.927221:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.927222:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.927225:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.927226:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.927227:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.927229:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.927230:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.927231:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.927231:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.927233:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.927234:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.927235:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.927236:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.927237:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.927237:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.927238:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.927240:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.927240:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.927241:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.927241:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.927245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.927252:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.927254:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.927254:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.927255:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.927256:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.927257:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.927335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.927339:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.927341:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.927343:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.927345:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.927346:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.927348:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.927349:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.927350:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.927352:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.927353:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.927355:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.927357:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.927360:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.927361:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.927362:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.927364:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.927364:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.927365:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.927366:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.927378:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.927381:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.927381:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.927383:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.927385:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.927386:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.927500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.927508:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.927511:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.927512:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.927514:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.927517:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.927518:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677447.927519:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.927521:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.927522:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.927523:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677447.927525:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.927527:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.927529:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.927530:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.927530:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.927532:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.927532:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.927533:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.927533:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.2:1619677447.927540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677447.927543:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.927545:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:17.0:1619677447.927546:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677447.927546:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.927547:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.927547:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.927549:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:10.0:1619677447.927549:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:1.0:1619677447.927549:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.927551:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.927553:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.927555:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.927556:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.927557:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.927558:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.927559:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.927562:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.927564:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.927567:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.927568:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.927568:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.927570:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.927571:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.927571:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.927572:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.927583:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.927585:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.927586:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.927588:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.927589:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.927591:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.927599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.927601:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.927603:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.927604:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.927605:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.927605:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677447.927606:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677447.927607:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677447.927609:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.927609:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.927611:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.927613:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.927615:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.927616:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.927617:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.927618:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.927619:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.927623:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.927624:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.927627:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.927628:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.927629:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.927630:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.927631:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.927631:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.927632:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677447.927644:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.927646:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.927647:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.927648:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.927651:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.927651:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.927661:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.927663:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.927664:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.927665:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.927665:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.927666:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.927727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.927735:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.927738:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.927740:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.927742:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.927742:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.927743:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.927744:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.927745:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.927746:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.927747:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.927749:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.927751:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.927753:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.927754:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.927755:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.927756:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.927757:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.927758:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.927758:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.927769:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.927771:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.927772:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.927773:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.927775:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.927776:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677447.927884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.927891:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.927893:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.927894:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.927896:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.927899:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.927900:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.927901:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.927902:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.927903:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.927904:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:21.2:1619677447.927905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:22.0:1619677447.927907:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.927909:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.927911:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.927911:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:22.0:1619677447.927912:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.927913:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.927914:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:22.0:1619677447.927915:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.927915:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677447.927915:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:22.0:1619677447.927916:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.927917:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000400:00000200:2.0:1619677447.927917:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.927918:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677447.927919:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.927921:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:8.2:1619677447.927922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677447.927922:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677447.927923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677447.927923:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.927924:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.927925:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:6.0:1619677447.927928:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:2.0:1619677447.927928:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.927929:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.927930:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677447.927930:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.927930:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677447.927931:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677447.927932:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:2.0:1619677447.927933:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677447.927933:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.927934:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.927934:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677447.927934:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:1.0:1619677447.927935:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.927937:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677447.927937:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.927938:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:1.0:1619677447.927938:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.927939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.927939:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.927940:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.2:1619677447.927945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677447.927947:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.927949:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.927950:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:12.0:1619677447.927950:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677447.927950:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.927951:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:17.0:1619677447.927951:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.927952:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677447.927953:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:12.0:1619677447.927953:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:18.0:1619677447.927954:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:12.0:1619677447.927954:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.927955:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:12.0:1619677447.927956:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.927957:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.927959:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.928034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677447.928042:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.928044:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677447.928045:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677447.928047:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.928050:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.928051:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677447.928052:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.928054:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.928055:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.928056:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677447.928059:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.928060:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.2:1619677447.928062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677447.928063:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.928064:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677447.928064:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677447.928066:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.928066:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.928067:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677447.928067:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:1.0:1619677447.928068:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.928071:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.928072:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:4.2:1619677447.928073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:1.0:1619677447.928073:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.928076:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.928078:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677447.928078:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.928079:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.928080:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677447.928080:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:1.0:1619677447.928080:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677447.928081:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.928081:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.928083:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:1.0:1619677447.928083:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.928084:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:13.0:1619677447.928084:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.928084:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.928087:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677447.928087:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.928088:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.928090:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.928090:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.928092:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:1.0:1619677447.928092:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677447.928093:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.928093:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.928094:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:8.0:1619677447.928095:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.928095:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.928095:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.928095:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677447.928097:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.928097:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.928098:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.928099:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677447.928101:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677447.928104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677447.928105:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.928107:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.928109:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:8.0:1619677447.928109:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.928110:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (23)++
00000800:00000200:1.0:1619677447.928110:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.928111:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (24)--
00000800:00000200:1.0:1619677447.928111:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.0:1619677447.928112:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (24)++
00000800:00000200:1.0:1619677447.928112:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.928113:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.928114:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (24)--
00000800:00000200:1.0:1619677447.928114:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (24)--
00000400:00000200:17.0:1619677447.928115:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.928115:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.928115:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677447.928116:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.928116:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.928118:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.2:1619677447.928120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.928127:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.928130:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.928131:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.928131:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.928133:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.928135:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677447.928497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.928503:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.928505:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.928506:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.928508:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.928511:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.928512:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.928514:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.928515:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.928516:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.928517:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.928521:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.928523:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.928526:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.928527:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.928527:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.928529:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.928530:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.928530:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.928531:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677447.928544:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.2:1619677447.928544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677447.928547:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.928548:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.928550:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.928551:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677447.928552:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.928553:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677447.928554:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.928554:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.928556:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.928558:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.928559:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.928560:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.928562:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.928563:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.928564:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.928567:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.928569:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.928571:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.928572:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.928572:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.928574:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.928574:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.928575:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.928575:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.928580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.928588:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.928590:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.928590:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.928591:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.928592:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.928593:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.928600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.928608:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.928610:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.928612:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.928613:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.928616:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.928617:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.928618:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.928620:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.928621:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.928622:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.928625:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.928627:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.928630:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.928631:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.928631:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.928633:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.928633:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.928634:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.928636:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.928636:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.928639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677447.928642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:4.2:1619677447.928643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.928646:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.928649:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:26.0:1619677447.928650:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677447.928650:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.928650:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.928650:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.928652:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.928652:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:26.0:1619677447.928653:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:8.0:1619677447.928653:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:10.0:1619677447.928654:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677447.928654:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:26.0:1619677447.928655:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.928655:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:26.0:1619677447.928658:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.928659:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677447.928660:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.928662:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.928663:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.928664:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677447.928667:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.928668:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677447.928671:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.928672:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677447.928673:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.928675:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.928675:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.928676:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.928676:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677447.928681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.928689:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.928692:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.928693:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.928694:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.928695:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.928699:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.928700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.928707:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.928710:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.928711:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.928712:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.928715:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.928717:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.928718:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.928719:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.928720:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.928721:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677447.928724:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.928725:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677447.928728:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.928729:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.928729:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.928731:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.928731:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.928732:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.928732:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677447.928744:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.928747:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.928748:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.928749:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.928751:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.928754:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.928762:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.928764:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.928765:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.928766:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.928767:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.928767:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677447.928769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.928776:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.928779:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.928781:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.928783:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.928784:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.928785:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.928787:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.928788:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.928789:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.928790:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.928793:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.928795:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.2:1619677447.928796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:30.0:1619677447.928797:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.928798:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.928798:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.928800:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.928800:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.928801:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.928802:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677447.928802:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.928805:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.928807:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.928808:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.2:1619677447.928812:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677447.928813:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:21.2:1619677447.928814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:12.0:1619677447.928815:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:5.0:1619677447.928815:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.928816:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.928818:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.928818:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:18.0:1619677447.928819:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:5.0:1619677447.928819:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.928820:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677447.928820:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.928820:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:13.0:1619677447.928821:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.928822:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:5.0:1619677447.928822:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.928822:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:5.0:1619677447.928823:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677447.928825:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.928825:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.928827:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.928828:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.928829:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.928829:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:2.0:1619677447.928831:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.928832:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.928833:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.928833:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.928834:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:2.0:1619677447.928834:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.928835:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.928836:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:2.0:1619677447.928836:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.928837:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.928837:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:18.0:1619677447.928838:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677447.928838:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677447.928839:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96721637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.928840:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.928841:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.928842:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.928842:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.928843:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.928847:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.928847:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.928848:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677447.928849:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:18.0:1619677447.928849:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.928849:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677447.928851:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:2.0:1619677447.928851:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:31.0:1619677447.928852:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.928852:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.928852:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.928853:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.928853:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:31.0:1619677447.928854:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.928854:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.928854:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:30.0:1619677447.928855:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:18.0:1619677447.928855:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.928856:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:31.0:1619677447.928857:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677447.928857:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.928858:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.928859:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.928860:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:26.0:1619677447.928863:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.928866:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677447.928866:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677447.928867:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.928869:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.928869:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677447.928872:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:12.0:1619677447.928872:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.928873:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.928875:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.928876:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.928877:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.928885:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.928888:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.928889:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.928890:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.928890:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.928892:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.929101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677447.929104:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677447.929106:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677447.929109:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.929110:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677447.929111:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.929113:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677447.929114:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.929116:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.929117:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.929118:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:5.2:1619677447.929120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:20.0:1619677447.929121:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.929123:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677447.929126:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.929126:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677447.929127:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677447.929128:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:1.0:1619677447.929129:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.0:1619677447.929130:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677447.929130:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:17.0:1619677447.929130:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.929131:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.929132:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:20.0:1619677447.929133:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:20.0:1619677447.929133:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:1.0:1619677447.929133:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.929135:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:20.2:1619677447.929137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:1.0:1619677447.929137:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.929138:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.929139:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.929140:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.2:1619677447.929141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:10.0:1619677447.929142:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:1.0:1619677447.929143:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.929144:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677447.929144:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:1.0:1619677447.929144:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677447.929145:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.929145:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.929146:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000800:00000200:30.0:1619677447.929148:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:10.0:1619677447.929148:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.929148:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000800:00000200:1.0:1619677447.929148:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.929149:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:1.0:1619677447.929149:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.929149:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.0:1619677447.929150:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:31.0:1619677447.929151:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.929151:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.929151:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:1.0:1619677447.929151:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.929152:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.929152:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677447.929153:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.929153:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677447.929154:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677447.929155:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.929157:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.929158:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.929159:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677447.929162:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.929164:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677447.929165:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:30.0:1619677447.929166:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.929166:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.929167:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:6.0:1619677447.929167:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.929168:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.929168:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.929169:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.929169:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.929170:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:6.0:1619677447.929170:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.929171:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:6.0:1619677447.929173:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677447.929180:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.929183:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.929184:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.929186:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.929187:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.929188:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:22.0:1619677447.929188:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:21.2:1619677447.929190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:22.0:1619677447.929191:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.929191:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.929192:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.929193:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.929194:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.929196:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.929199:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.929200:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.929202:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.929205:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.929206:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677447.929208:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.929209:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.929210:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.929211:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677447.929214:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.929216:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677447.929219:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.929220:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677447.929220:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677447.929222:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677447.929223:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.929223:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.929224:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677447.929234:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677447.929237:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677447.929238:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.929238:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677447.929240:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.929243:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677447.929252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:4.2:1619677447.929252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:6.0:1619677447.929258:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677447.929259:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677447.929260:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.929261:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677447.929262:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.929263:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677447.929263:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.929265:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.929266:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.929267:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:24.0:1619677447.929268:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.929269:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.929270:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.929271:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.929271:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.929272:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.929273:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.929273:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:24.0:1619677447.929274:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.929275:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677447.929278:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.929280:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.929281:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:24.0:1619677447.929283:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677447.929285:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.929286:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.929286:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:24.0:1619677447.929287:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:6.0:1619677447.929287:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:24.0:1619677447.929288:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677447.929289:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:24.0:1619677447.929290:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:6.0:1619677447.929290:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677447.929291:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677447.929291:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677447.929292:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677447.929292:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677447.929293:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.929297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677447.929306:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677447.929306:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:30.0:1619677447.929309:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:17.0:1619677447.929309:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:30.0:1619677447.929310:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.929310:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.929311:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.929312:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.929313:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:17.0:1619677447.929313:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.929316:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677447.929316:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.929323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.929330:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.929333:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.929334:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.929334:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.929335:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.929336:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677447.929548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.929555:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.929558:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.929560:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.929562:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.929562:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.929564:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677447.929565:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.929566:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.929567:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.929568:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677447.929572:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.929573:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677447.929576:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.929577:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677447.929577:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677447.929579:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.929580:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.929581:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.929583:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.929584:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677447.929589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677447.929592:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.929594:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.929595:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.929595:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.929597:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:10.0:1619677447.929597:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.929599:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.929600:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677447.929600:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:30.0:1619677447.929601:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677447.929611:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.929613:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.929614:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.929614:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.929615:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.929616:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677447.929653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.929659:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.929662:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.929664:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.929664:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.929667:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.929668:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.929670:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721629] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.929671:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.929672:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.929673:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.929677:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.929678:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.929682:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.929682:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.929683:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.929685:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.929686:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.929686:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.929688:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.929688:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.929693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.929696:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.929698:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:12.0:1619677447.929698:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677447.929699:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.929699:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.929701:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.929702:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.929704:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.929705:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.929706:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.929733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.929739:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.929741:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.929742:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.929744:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.929746:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.929747:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.929749:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.929750:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.929751:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.929752:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.929756:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.929758:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.929760:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.929761:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.929762:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.929763:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.929764:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.929764:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.929765:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.929777:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.929779:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.929780:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.929781:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.929783:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.929783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.929785:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:10.0:1619677447.929790:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.929793:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.929794:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.929796:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.929799:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.929800:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677447.929801:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.929803:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.929804:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.929805:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677447.929808:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.929810:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.2:1619677447.929812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677447.929812:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.929813:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.929813:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677447.929815:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.929816:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.929817:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677447.929818:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.929818:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677447.929820:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.929822:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.929824:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.929824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677447.929826:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.929827:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.929828:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:30.0:1619677447.929828:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:31.0:1619677447.929829:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.929829:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.929830:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.929830:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677447.929831:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677447.929832:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.929832:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.929832:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:4.2:1619677447.929832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677447.929833:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.929833:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.929834:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677447.929835:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.929836:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:30.0:1619677447.929836:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.929837:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.929839:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677447.929841:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.929841:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.929841:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:30.0:1619677447.929842:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.929843:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677447.929844:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:8.0:1619677447.929844:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.929845:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.929846:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.929846:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:8.0:1619677447.929846:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.929847:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.929847:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.929848:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.929848:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.929849:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.929849:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.929850:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.2:1619677447.929850:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:8.0:1619677447.929853:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.929854:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.929855:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677447.929856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:8.0:1619677447.929856:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.929857:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.929860:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.929862:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.0:1619677447.929862:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677447.929863:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677447.929864:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.929865:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.929865:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.929865:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677447.929866:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:20.0:1619677447.929866:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.929866:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677447.929867:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.0:1619677447.929867:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.929867:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.929868:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.0:1619677447.929868:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.929868:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677447.929869:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.0:1619677447.929869:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.929873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.929881:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.929884:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.929885:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.929885:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.929887:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.2:1619677447.929888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677447.929890:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677447.929896:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.929899:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.929899:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677447.929901:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.929904:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.929905:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677447.929907:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.929908:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.929909:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.929910:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677447.929913:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.929914:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677447.929917:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.929917:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677447.929918:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677447.929920:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.929920:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.929921:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.929921:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677447.929941:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.929943:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.929944:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.929944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:13.0:1619677447.929945:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.929947:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.929949:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.929951:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.929954:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.929955:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.929957:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.929960:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.929962:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.929963:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.929964:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.929965:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.929966:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.929969:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.929971:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.929974:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.929975:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.929976:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.929978:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.929978:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.929979:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.929979:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.930000:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.930003:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.930004:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.930005:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.930007:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.930008:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.930275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.930282:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.930285:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.930286:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.930288:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.930290:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.930291:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677447.930292:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721622] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677447.930293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:26.0:1619677447.930300:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.930301:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.930301:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:26.0:1619677447.930302:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.930303:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.930304:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:26.0:1619677447.930306:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.930306:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.930308:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.930309:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.930310:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.930311:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:24.0:1619677447.930311:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:26.0:1619677447.930312:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:24.0:1619677447.930312:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.930313:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677447.930314:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.930314:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:24.0:1619677447.930314:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.930315:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:24.0:1619677447.930315:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:26.0:1619677447.930316:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:24.0:1619677447.930319:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677447.930320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:24.0:1619677447.930321:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:9.2:1619677447.930321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.930324:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.930324:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677447.930325:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000800:00000200:5.0:1619677447.930326:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:24.0:1619677447.930327:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (24)--
00000800:00000200:24.0:1619677447.930327:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.930328:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677447.930328:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:24.0:1619677447.930329:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:2.0:1619677447.930329:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:20.0:1619677447.930331:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.930331:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:2.0:1619677447.930332:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000800:00000200:18.0:1619677447.930333:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000400:00000200:5.0:1619677447.930333:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.930333:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.930334:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000400:00000200:18.0:1619677447.930335:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.930335:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.930336:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.930337:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96721620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.930338:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.930339:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.930340:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.930341:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:5.0:1619677447.930341:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.930341:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.930342:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:5.0:1619677447.930345:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.930346:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.930348:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.930349:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.930350:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.930351:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.930352:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.930353:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.930354:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677447.930358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.930366:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.930368:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.930369:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.930369:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.930371:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677447.930373:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677447.930411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.930419:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.930422:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.930424:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.930426:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.930427:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.930428:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677447.930429:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.930431:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.930432:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.930433:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677447.930436:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.930438:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677447.930441:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.930442:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677447.930443:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677447.930444:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.930445:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.930446:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.930446:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.930459:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.930462:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.930463:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.930463:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.930465:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.930468:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.930471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.930474:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.930477:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.930479:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.930480:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.930481:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.930483:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.930484:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.930486:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.930487:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.930488:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.930491:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.930492:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.930495:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.930496:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.930497:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.930499:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.930500:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.930501:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.930501:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.930524:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.930527:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.930527:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.930529:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.930531:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.930532:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.930540:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.930543:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.930544:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.930545:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.930545:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.930546:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.930583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.930588:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.930591:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.930592:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.930594:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.930596:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.930598:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.930599:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.930600:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.930601:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.930602:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.930606:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.930608:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.930611:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.930612:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.930613:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.930615:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.930615:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.930616:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.930617:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.930618:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677447.930623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677447.930627:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677447.930628:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677447.930630:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:17.0:1619677447.930630:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.930631:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677447.930632:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.930632:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677447.930633:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.930634:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.930637:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.930734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.930741:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.930744:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.930745:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.930747:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.930750:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.930751:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677447.930753:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721616] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.930754:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.930755:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.930756:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677447.930758:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.930760:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677447.930762:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.930763:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677447.930764:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677447.930765:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.930766:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.930767:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677447.930768:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677447.930768:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677447.930774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.930781:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677447.930783:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677447.930784:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.930785:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677447.930788:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.930790:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.930791:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.930792:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.930794:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.930795:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.930888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.930895:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.930897:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.930898:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.930899:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.930902:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.930904:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677447.930905:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.930906:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.930907:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.930907:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677447.930910:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.930912:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677447.930915:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.930916:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677447.930916:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677447.930918:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677447.930919:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.930919:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.930920:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677447.930924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.930933:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.930936:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.930937:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.930939:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.930939:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.930943:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677447.930982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677447.930989:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677447.930992:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677447.930993:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677447.930994:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.930997:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.930999:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.931000:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.931001:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.931002:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.931003:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.931006:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.931007:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.931009:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.931010:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.931010:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.931011:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.931012:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677447.931013:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677447.931013:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:30.0:1619677447.931024:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.931027:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.931027:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.931029:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.931030:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.931031:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.931061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677447.931062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:20.0:1619677447.931064:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.931066:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.931068:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.931068:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:5.0:1619677447.931069:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.931071:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.931071:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677447.931072:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:20.0:1619677447.931073:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.931074:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.931075:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.931076:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.931077:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.931078:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677447.931078:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.931080:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677447.931081:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.931082:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.931083:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.931084:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.931085:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:1.0:1619677447.931085:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.931088:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.931088:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.931089:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:1.0:1619677447.931089:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677447.931091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677447.931091:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.931091:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677447.931091:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.931092:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.931093:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677447.931094:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.931095:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677447.931095:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (22)++
00000800:00000200:1.0:1619677447.931098:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (23)--
00000800:00000200:1.0:1619677447.931098:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:31.0:1619677447.931099:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677447.931099:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677447.931100:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677447.931101:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:31.0:1619677447.931102:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.931104:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.931104:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:30.0:1619677447.931106:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.931107:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.931107:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000400:00000200:22.0:1619677447.931108:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.931108:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (24)++
00000800:00000200:5.2:1619677447.931108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:31.0:1619677447.931109:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:6.0:1619677447.931109:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000400:00000200:6.0:1619677447.931109:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.931110:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96721611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.931110:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.931112:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.931112:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.931113:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:31.0:1619677447.931114:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.931114:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:6.0:1619677447.931114:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:31.0:1619677447.931115:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.931115:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.931117:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677447.931118:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:31.0:1619677447.931119:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.931119:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:31.0:1619677447.931120:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677447.931124:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.931125:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677447.931125:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677447.931127:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677447.931128:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.931129:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.931129:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.931141:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.931144:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.931144:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.931145:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.931147:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.931149:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.931229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677447.931237:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.931240:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677447.931241:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677447.931243:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.931245:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.931247:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677447.931248:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.931249:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.931250:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.931251:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677447.931254:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.931256:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677447.931259:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.931260:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677447.931260:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677447.931262:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677447.931262:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.931263:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677447.931264:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677447.931268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.931276:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.931279:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.931280:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.931280:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.931282:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.931286:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.931325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.931333:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.931336:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.931337:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.931339:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.931342:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.931343:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.931344:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.931345:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.931346:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.931347:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.931351:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.931352:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.931354:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.931354:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.931355:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.931356:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.931357:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.931358:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.931358:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677447.931370:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.931373:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.931373:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.931375:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.931377:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.931378:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.931450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677447.931457:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.931460:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.931461:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.931463:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.931466:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.931467:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.931469:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677447.931470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:8.0:1619677447.931470:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.931471:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.931471:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.931474:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.931476:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677447.931476:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:22.0:1619677447.931479:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:8.0:1619677447.931479:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.931480:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.931480:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:6.0:1619677447.931480:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.931482:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.931482:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.931483:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.931483:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.931484:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.2:1619677447.931484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:22.0:1619677447.931485:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.931486:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.931488:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677447.931488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:22.0:1619677447.931489:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.931490:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.931490:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677447.931491:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:17.0:1619677447.931493:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.931493:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677447.931496:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.931496:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.931496:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:22.0:1619677447.931499:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:17.0:1619677447.931499:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.931499:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.931500:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.931502:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.931502:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:17.0:1619677447.931502:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.931502:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.931503:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000400:00000200:17.0:1619677447.931503:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96721606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.931504:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:17.0:1619677447.931504:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.931505:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.931506:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.931506:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:17.0:1619677447.931506:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.931506:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677447.931507:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.931507:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:17.0:1619677447.931510:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.931511:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:21.2:1619677447.931513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677447.931515:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.931516:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.931516:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.931518:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.931518:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:17.0:1619677447.931519:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.931519:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677447.931520:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677447.931520:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677447.931520:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677447.931521:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677447.931522:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.931523:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677447.931524:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.931525:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.931527:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677447.931527:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:30.0:1619677447.931528:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.931530:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677447.931531:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.931532:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.931532:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:30.0:1619677447.931533:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.931534:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:13.0:1619677447.931534:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.931535:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.931537:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000400:00000200:13.0:1619677447.931537:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.931538:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.931539:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677447.931540:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:13.0:1619677447.931540:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677447.931540:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:30.0:1619677447.931541:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.931541:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677447.931541:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677447.931542:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:30.0:1619677447.931543:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.931544:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677447.931545:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677447.931547:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.931547:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.931548:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.931549:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.931560:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.931563:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.931564:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.931565:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.931567:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.931570:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.931673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.931681:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.931684:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.931686:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.931687:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.931689:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.931690:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.931692:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.931693:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.931694:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.931695:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.931698:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.931699:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.931702:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.931703:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.931704:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.931705:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.931706:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:4.2:1619677447.931706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677447.931707:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.931707:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.931711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677447.931713:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.2:1619677447.931715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:26.0:1619677447.931716:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.931716:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677447.931718:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.931719:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:10.0:1619677447.931720:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.931722:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.931722:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.931722:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:10.0:1619677447.931722:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677447.931723:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.931724:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.931725:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000400:00000200:10.0:1619677447.931725:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.931725:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:5.0:1619677447.931726:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.931727:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:10.0:1619677447.931727:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.931729:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.931729:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.931732:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.931732:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.931733:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:10.0:1619677447.931734:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.931735:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721602] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.931736:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.931737:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677447.931737:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.931738:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:10.0:1619677447.931738:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677447.931739:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:18.0:1619677447.931741:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.931741:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677447.931741:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.931742:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:18.0:1619677447.931743:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677447.931744:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.931744:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677447.931746:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.931747:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:8.2:1619677447.931747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.931748:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.931750:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.931750:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:4.2:1619677447.931750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677447.931752:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677447.931752:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677447.931753:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:5.0:1619677447.931754:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.2:1619677447.931756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677447.931757:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:5.0:1619677447.931757:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:4.2:1619677447.931757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677447.931759:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:21.2:1619677447.931759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:5.0:1619677447.931759:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.931760:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.931761:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:5.0:1619677447.931761:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.931762:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.931764:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677447.931764:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.931764:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677447.931765:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:2.0:1619677447.931765:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000400:00000200:20.0:1619677447.931766:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.931766:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:8.2:1619677447.931767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.931767:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:24.0:1619677447.931768:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:20.0:1619677447.931768:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.931768:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677447.931769:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.0:1619677447.931771:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:24.0:1619677447.931772:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.931772:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.931773:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677447.931774:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.931775:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.931775:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.931775:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.931776:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.931776:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:24.0:1619677447.931777:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.931778:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.931780:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.931781:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.931782:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.931785:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.931787:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.931790:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.931791:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.931791:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.931793:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.931793:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.931794:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.931795:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.931799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.931807:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.931810:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.931811:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.931811:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.931813:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677447.931816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.931820:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.931822:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.931824:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.931824:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.931825:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.931826:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.931826:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.932032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.932038:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.932040:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.932043:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.932043:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.932045:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.932047:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677447.932048:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.932049:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.932050:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.932052:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.932055:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.932056:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.932059:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.932060:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.932060:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.932062:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.932063:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.932064:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.2:1619677447.932066:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.2:1619677447.932068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.932069:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677447.932074:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.932076:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.932077:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.932078:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.932079:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.932079:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.932081:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.932081:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.932084:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.932085:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.932087:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.932087:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.932089:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.932090:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.932091:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.932092:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:17.0:1619677447.932093:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.932095:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.932098:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.932099:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.932100:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.932101:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.932102:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.932102:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.932103:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677447.932105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.932112:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:12.0:1619677447.932114:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677447.932115:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.932116:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:12.0:1619677447.932117:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:10.0:1619677447.932117:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677447.932118:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.932119:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.932120:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677447.932120:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.932121:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:12.0:1619677447.932122:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.932123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:10.0:1619677447.932123:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.932124:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.932125:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.932126:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677447.932129:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.932130:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677447.932130:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677447.932132:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.932133:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:10.0:1619677447.932133:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.932134:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:30.0:1619677447.932135:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.932135:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.932136:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.932136:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677447.932137:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.932137:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.932137:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:30.0:1619677447.932139:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.932140:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677447.932140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:30.0:1619677447.932141:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.932142:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.932143:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.932146:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.932147:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.932147:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:31.0:1619677447.932148:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.932149:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.932149:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.932150:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.932150:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.932150:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:31.0:1619677447.932151:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.932151:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677447.932151:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:30.0:1619677447.932152:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.932153:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.932153:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.932153:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:8.0:1619677447.932153:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.932154:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677447.932155:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:8.0:1619677447.932156:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.932157:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.932158:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.932160:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.932161:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.932162:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.932164:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:8.0:1619677447.932165:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.932166:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:8.0:1619677447.932166:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.932168:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:13.0:1619677447.932168:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.932169:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.932169:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.932169:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.932170:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:12.0:1619677447.932170:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.932170:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.932171:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.932172:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.932172:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:13.0:1619677447.932172:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.932172:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.932172:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.932173:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:30.0:1619677447.932174:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.0:1619677447.932174:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.932183:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.932186:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.932186:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.932188:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.932190:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.932191:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.932203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:13.0:1619677447.932204:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.932207:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.932208:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.932208:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.932209:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.932210:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677447.932210:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.932212:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.932214:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.932216:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.932217:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.932218:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677447.932219:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.932220:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.932221:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.932222:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677447.932226:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.932228:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677447.932230:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.932231:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677447.932232:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677447.932233:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677447.932234:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.932235:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.932237:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.932238:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677447.932239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677447.932243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677447.932247:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.932250:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:30.0:1619677447.932251:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:2.0:1619677447.932251:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.932252:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.932253:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:21.2:1619677447.932253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:30.0:1619677447.932254:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677447.932254:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.932255:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677447.932258:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:18.0:1619677447.932259:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.932261:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.932262:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.932264:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.932266:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.932268:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677447.932269:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.932270:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.932271:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.932272:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677447.932275:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.932277:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677447.932279:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.932280:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677447.932281:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677447.932282:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677447.932283:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.932284:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.932284:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.932297:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.932300:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.932301:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.932301:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.932303:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.932305:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.932319:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.932322:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.932322:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.932322:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.932323:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.932324:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677447.932434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.932440:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.932443:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.932444:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.932446:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.932449:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.932451:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.932452:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.932453:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.932454:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.932455:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.932459:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.932461:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.932464:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.932465:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.932465:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.932467:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.932467:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.932468:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.932468:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.932480:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.932483:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.932483:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.932485:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.932487:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.932488:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.932530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677447.932533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.932540:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.932542:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.932542:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.932543:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677447.932545:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.932546:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.932548:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.932549:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677447.932551:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.932552:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.932553:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.932554:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677447.932557:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.932559:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:10.0:1619677447.932559:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.932560:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:26.0:1619677447.932561:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677447.932561:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.932562:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.932563:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:10.0:1619677447.932563:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.932563:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.932564:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.932565:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.932567:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.932568:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677447.932568:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.932569:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.932570:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.932570:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.932571:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677447.932571:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.932572:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.932572:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (23)++
00000800:00000200:10.0:1619677447.932573:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (24)--
00000800:00000200:26.0:1619677447.932574:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)++
00000800:00000200:26.0:1619677447.932574:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (24)--
00000800:00000200:10.0:1619677447.932574:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.932574:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.932575:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.932575:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:26.0:1619677447.932576:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.932576:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.932578:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.932584:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.932586:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.932587:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.932588:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.932588:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.932590:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677447.932636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677447.932642:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.932645:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.932646:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.932648:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.932650:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.932652:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.932653:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.932654:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.932655:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.932656:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.932659:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.932661:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.932663:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.932664:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.932665:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.932667:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.932667:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.932668:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.932668:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677447.932674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.932681:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.932684:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.932684:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.932686:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.932688:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.932689:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677447.932783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.932788:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.932791:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.932792:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.932794:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.932797:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.932798:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.932800:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.932801:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.932802:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.932803:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.932806:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.932808:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:5.2:1619677447.932809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:17.0:1619677447.932811:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.932812:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.932813:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.932814:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:18.0:1619677447.932815:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677447.932815:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.932816:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.932817:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:18.0:1619677447.932818:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:17.0:1619677447.932818:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677447.932818:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.932820:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677447.932820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.2:1619677447.932822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677447.932823:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.932824:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.932825:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.932826:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.932827:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:18.0:1619677447.932827:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.932828:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677447.932828:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677447.932830:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.932830:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677447.932830:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:24.0:1619677447.932831:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.932831:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.932831:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.932831:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677447.932834:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.932834:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.932836:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.932837:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:24.0:1619677447.932838:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:18.0:1619677447.932838:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.932840:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.932840:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.932841:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.932842:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.932846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.932854:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.932857:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.932858:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.932860:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.932860:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.932862:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.932871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.932879:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.932882:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.932884:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.932886:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.932887:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.932888:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.932890:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721587] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.932891:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.932892:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.932893:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.932897:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.932899:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.932902:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.932903:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.932904:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.932905:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.932906:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.932907:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.932908:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.932911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.932919:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.932921:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.932922:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.932924:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.932925:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.932927:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677447.932931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677447.932939:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.932942:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.932943:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.932944:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.932944:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.932945:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677447.933066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677447.933071:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.933074:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.933076:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.933077:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.933080:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.933081:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.933082:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.933084:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.933085:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.933086:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.933089:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.933091:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.933093:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.933094:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.933095:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.933096:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.933097:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.933098:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.933098:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677447.933102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.933110:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.933112:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.933113:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.933115:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.933116:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.933117:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677447.933145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.933153:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.933156:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.933158:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.933160:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.933160:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.933162:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.933163:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.933165:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.933166:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.933167:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.933170:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.933172:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.933174:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.933175:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.933176:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.933177:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.933178:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.933179:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.933179:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677447.933188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677447.933190:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.933192:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.933193:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.933193:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:13.0:1619677447.933195:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.933196:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.933196:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.933197:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677447.933198:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677447.933199:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.933202:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.933203:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.933204:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.933206:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.933207:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.933208:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677447.933211:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.933213:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.933215:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.933216:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.933217:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.933219:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.933219:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.933219:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.933220:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.933224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.933230:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.933233:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.933234:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.933234:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.933236:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.933239:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.933243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677447.933249:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.933252:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677447.933252:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.933253:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.933254:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.933255:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677447.933260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.933266:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.933268:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.933269:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.2:1619677447.933270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:22.0:1619677447.933271:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.933272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:22.0:1619677447.933274:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.933276:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:26.0:1619677447.933277:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:22.0:1619677447.933277:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.933278:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.933279:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677447.933280:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:22.0:1619677447.933280:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677447.933280:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677447.933281:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.0:1619677447.933282:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.933283:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:26.0:1619677447.933284:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.933286:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.933286:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.933287:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:8.0:1619677447.933287:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.933289:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.933289:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:8.0:1619677447.933289:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.933291:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.933291:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677447.933292:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.933293:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677447.933293:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.933294:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:26.0:1619677447.933300:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:22.0:1619677447.933300:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:8.0:1619677447.933300:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.933301:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.933302:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.933302:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:8.0:1619677447.933302:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.933303:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:26.0:1619677447.933304:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.933304:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.933305:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677447.933306:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:8.0:1619677447.933306:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.933308:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.933308:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677447.933309:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.933309:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.933311:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.933311:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.2:1619677447.933311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677447.933311:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.933312:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677447.933312:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:26.0:1619677447.933313:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677447.933313:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.933315:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.933315:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677447.933316:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:8.0:1619677447.933316:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677447.933316:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:8.0:1619677447.933318:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.933318:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677447.933318:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:30.0:1619677447.933319:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:6.0:1619677447.933319:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:30.0:1619677447.933320:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.933320:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.933320:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677447.933322:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.933322:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:31.0:1619677447.933323:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.933325:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677447.933325:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.933325:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.933325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677447.933326:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.933327:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.933328:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:10.0:1619677447.933329:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:17.0:1619677447.933331:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677447.933332:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:17.0:1619677447.933332:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.933332:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.933334:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677447.933334:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.933335:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.933336:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.933337:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.933357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.933360:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.933363:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:5.2:1619677447.933363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:20.0:1619677447.933365:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.933368:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.933369:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:18.0:1619677447.933369:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:20.0:1619677447.933370:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.933370:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.933371:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.933372:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.933373:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.933373:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.933373:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:20.0:1619677447.933374:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677447.933377:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.933378:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.933379:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.933379:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.933381:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:18.0:1619677447.933381:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.933381:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.933382:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.933383:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.933384:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.933384:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.933385:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.933386:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.933386:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.933387:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.933388:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.933391:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.933391:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.933392:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.933393:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:18.0:1619677447.933393:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.933394:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.933395:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:18.0:1619677447.933395:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.933396:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)++
00000800:00000200:18.0:1619677447.933396:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.933396:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.933397:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (24)--
00000800:00000200:18.0:1619677447.933397:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.933398:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (23)--
00000800:00000200:18.0:1619677447.933398:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.933398:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677447.933399:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677447.933400:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:5.0:1619677447.933400:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.933400:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.933401:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.933401:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677447.933401:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.933403:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.933404:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:9.2:1619677447.933405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677447.933408:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.2:1619677447.933408:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677447.933411:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:5.0:1619677447.933411:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:31.0:1619677447.933412:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.933412:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.933413:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677447.933413:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:31.0:1619677447.933414:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.933414:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.933414:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:30.0:1619677447.933415:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:21.2:1619677447.933415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:18.0:1619677447.933415:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:2.0:1619677447.933415:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.933416:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677447.933417:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677447.933417:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:12.0:1619677447.933417:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:2.0:1619677447.933417:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.2:1619677447.933418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.933419:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677447.933419:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677447.933420:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677447.933421:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:12.0:1619677447.933421:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.933422:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.933422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.933422:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677447.933423:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.933424:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677447.933424:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.933425:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.0:1619677447.933425:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.933426:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.933426:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677447.933428:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.933430:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677447.933431:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.933432:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:18.0:1619677447.933433:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.933433:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.933433:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.933434:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.933435:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.933435:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:8.0:1619677447.933435:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.933436:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.933437:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.933441:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.933442:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.933445:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.933446:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.933447:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.933448:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.933449:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.933450:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.933451:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.933452:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.2:1619677447.933454:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677447.933457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677447.933461:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.933464:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:24.0:1619677447.933465:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:20.0:1619677447.933465:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.933466:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.933467:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.933467:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677447.933468:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.933469:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677447.933469:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.933534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.933541:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.933544:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.933545:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.933546:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.933548:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.933550:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.933551:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.933552:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.933553:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.933554:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677447.933557:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.933559:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.933562:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.933563:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.933563:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677447.933565:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.933566:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.933566:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.933567:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677447.933579:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.933582:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.933582:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.933583:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.933585:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.933588:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.933597:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.933599:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.933600:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.933600:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.933601:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.933602:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677447.933675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.933683:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.933685:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.933687:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.933689:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.933690:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.933691:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677447.933693:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.933694:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.933695:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.933696:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677447.933699:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.933700:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677447.933703:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.933704:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677447.933704:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677447.933706:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.933707:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.933708:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.933708:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.933719:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.933722:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.933723:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.933723:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.933725:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.933729:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677447.933746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.933751:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.933754:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.933754:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.933757:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.933759:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.933761:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.933762:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.933763:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.933764:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.933765:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.933768:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.933770:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.933772:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.933773:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.933774:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.933776:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.933776:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.933777:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.933777:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.2:1619677447.933802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:9.2:1619677447.933805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:1.0:1619677447.933808:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.933811:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.933811:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:2.0:1619677447.933812:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:1.0:1619677447.933813:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.933814:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.933816:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.933816:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.933816:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.933818:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:2.0:1619677447.933819:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.933819:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721573] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.933821:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:1.0:1619677447.933821:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.933822:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.933823:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:9.2:1619677447.933825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000400:00000200:1.0:1619677447.933827:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.933828:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.933831:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.933832:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:1.0:1619677447.933832:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.933833:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:18.0:1619677447.933835:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.933835:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:1.0:1619677447.933835:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.933835:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.933836:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:1.0:1619677447.933836:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.2:1619677447.933837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677447.933837:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.933837:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:1.0:1619677447.933837:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.933841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:31.0:1619677447.933845:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:8.0:1619677447.933847:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:31.0:1619677447.933848:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.933850:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.933850:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.933851:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.933851:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.933852:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.933853:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.933853:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.933855:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.933856:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721572] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.933856:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:31.0:1619677447.933857:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.933858:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.933859:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.933862:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.933864:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.933866:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.933867:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.933868:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.933869:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.933869:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.933871:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.933873:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.933873:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677447.933879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.933883:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.933884:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.933885:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.933887:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:30.0:1619677447.933887:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.933888:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.933888:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.933889:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.933890:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.933891:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677447.933901:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.933904:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.933905:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.933905:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677447.933906:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.933907:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677447.934152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.934158:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.934161:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.934163:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.934163:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.934166:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.934168:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.934169:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.934171:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.934172:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.934173:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.934176:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.934178:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.934181:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.934181:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.934182:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.934184:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.934184:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.934185:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.934186:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677447.934195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.2:1619677447.934196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:31.0:1619677447.934197:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.934199:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.934200:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.934202:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.934202:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677447.934203:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:24.0:1619677447.934203:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677447.934205:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:2.0:1619677447.934205:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:24.0:1619677447.934206:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:18.0:1619677447.934206:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.0:1619677447.934206:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677447.934209:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677447.934211:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.934211:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.934213:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.934213:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:24.0:1619677447.934215:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:2.0:1619677447.934215:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.934217:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.934217:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.934218:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.934219:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.934219:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:24.0:1619677447.934220:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.934221:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:2.0:1619677447.934224:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000400:00000200:24.0:1619677447.934226:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.934226:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:24.0:1619677447.934228:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:2.0:1619677447.934229:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.934230:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:24.0:1619677447.934231:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.934231:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:24.0:1619677447.934232:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.934233:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677447.934233:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677447.934233:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:24.0:1619677447.934234:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:2.0:1619677447.934234:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:24.0:1619677447.934235:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677447.934235:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677447.934236:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.934237:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.934240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677447.934245:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677447.934247:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677447.934248:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:17.0:1619677447.934249:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:10.0:1619677447.934249:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.934251:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:17.0:1619677447.934252:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.934253:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.934253:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677447.934254:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:17.0:1619677447.934255:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:5.2:1619677447.934255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:17.0:1619677447.934258:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:26.0:1619677447.934263:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.934266:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.934266:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.934268:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.934271:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.934272:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677447.934274:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.934275:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.934276:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.934277:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677447.934281:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.934282:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677447.934285:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.934286:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677447.934287:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.934289:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.934289:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.934290:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.934290:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677447.934297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.934305:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.934308:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.934308:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.934309:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.934311:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.934315:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677447.934351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.934357:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.934360:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.934360:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.934362:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.934365:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.934366:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.934368:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.934369:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.934370:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.934371:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.2:1619677447.934371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:21.2:1619677447.934372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:6.0:1619677447.934375:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.934376:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.934376:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677447.934379:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.934379:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:30.0:1619677447.934380:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677447.934380:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:6.0:1619677447.934380:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.934381:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.934382:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:2.0:1619677447.934382:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.934383:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:6.0:1619677447.934383:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.934384:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.934384:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:30.0:1619677447.934385:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.934385:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.934386:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:31.0:1619677447.934388:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.934388:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.934388:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.934389:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:8.2:1619677447.934389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677447.934390:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.934390:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.934391:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.934392:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.934392:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.934393:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.934395:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677447.934396:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.934398:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:30.0:1619677447.934399:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.934399:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677447.934400:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.934401:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.934401:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.934402:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677447.934402:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.934402:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.934403:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:1.0:1619677447.934403:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.934405:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.934405:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.934405:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:30.0:1619677447.934406:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677447.934406:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.934406:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677447.934406:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677447.934407:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677447.934409:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.934409:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.934410:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.934411:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677447.934411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.934419:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.934422:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.934423:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.934424:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:20.0:1619677447.934424:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.934425:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.934426:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:20.0:1619677447.934427:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:6.0:1619677447.934427:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.934428:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.934430:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.934433:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.934459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.934467:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.934469:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.934472:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.934473:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.934475:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.934476:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.934478:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.934479:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.934480:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.934481:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.934484:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.934486:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.934489:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.934490:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.934491:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.934493:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.934493:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.934494:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.934495:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.934499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.934507:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.934509:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.934510:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.934512:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.934512:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.934515:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.934538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.934546:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.934549:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.934549:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677447.934551:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.934554:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.934556:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677447.934557:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.934558:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.934559:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.934560:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677447.934564:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.934565:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677447.934567:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.934568:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.934569:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677447.934570:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.934571:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.934572:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677447.934573:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.934574:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677447.934579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677447.934583:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.934585:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.934586:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.934586:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:30.0:1619677447.934587:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.934588:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:31.0:1619677447.934589:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.934589:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677447.934590:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677447.934590:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677447.934600:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677447.934602:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.934603:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.934604:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.934604:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.934605:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.934804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.934810:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.934813:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.934814:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.934816:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.934818:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.934820:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.934821:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.934822:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.934823:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.934825:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.934828:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.934829:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.934832:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.934833:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.934833:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.934835:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.934835:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.934836:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.934836:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.934849:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.934852:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.934852:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.934853:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.934855:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.934857:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.934858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.2:1619677447.934863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:8.0:1619677447.934866:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.934868:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:18.0:1619677447.934869:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:24.0:1619677447.934870:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677447.934872:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:8.0:1619677447.934872:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.934872:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.934875:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.934876:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.934877:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.934878:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.934879:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:8.0:1619677447.934879:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.934880:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.934881:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.934882:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.934883:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.934883:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.934884:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.934885:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:8.0:1619677447.934888:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.934890:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.934891:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.934892:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.0:1619677447.934894:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.934895:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.934895:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.934896:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:8.0:1619677447.934896:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.934897:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:8.0:1619677447.934897:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.934898:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677447.934899:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:8.0:1619677447.934899:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.934899:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677447.934900:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.934900:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.934901:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.934902:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:8.2:1619677447.934903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677447.934907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677447.934911:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:1.0:1619677447.934911:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677447.934912:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:10.0:1619677447.934914:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:1.0:1619677447.934914:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:26.0:1619677447.934915:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:17.0:1619677447.934915:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:2.0:1619677447.934915:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:10.0:1619677447.934916:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.934916:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:1.0:1619677447.934916:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.934917:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:1.0:1619677447.934919:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:10.0:1619677447.934920:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:8.2:1619677447.934921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.934924:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:10.0:1619677447.934926:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:1.0:1619677447.934926:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677447.934927:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:1.0:1619677447.934928:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677447.934929:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:17.0:1619677447.934929:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:10.0:1619677447.934930:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:1.0:1619677447.934930:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:10.0:1619677447.934931:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (21)++
00000800:00000200:1.0:1619677447.934931:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:10.0:1619677447.934932:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:1.0:1619677447.934932:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677447.934933:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677447.934963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.934966:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.934968:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.934971:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.934972:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.934974:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.934976:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:20.0:1619677447.934980:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.934982:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.934983:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.934984:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.934985:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:20.0:1619677447.934985:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.934987:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:20.0:1619677447.934989:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.934990:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.934991:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677447.934992:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.934993:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.934995:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:20.0:1619677447.934995:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.934996:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.934996:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:31.0:1619677447.934997:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.934997:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677447.934998:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.934999:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.934999:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.934999:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.935000:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.935001:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:31.0:1619677447.935003:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.935004:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.935008:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.935008:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.935009:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.935011:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.935011:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.935012:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.935013:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677447.935013:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.935015:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.935016:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.935017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:13.0:1619677447.935018:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.935020:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.935021:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.935022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.935025:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.935026:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.935027:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:5.0:1619677447.935027:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.935027:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.935029:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677447.935029:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.935030:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.935032:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.935033:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:20.0:1619677447.935034:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.935034:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.935034:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.935034:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.935035:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:20.0:1619677447.935036:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:12.0:1619677447.935036:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.935036:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.935037:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.935038:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.935039:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.935040:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677447.935044:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.935045:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:21.2:1619677447.935047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677447.935049:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.935050:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677447.935050:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677447.935052:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677447.935053:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.935054:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.935054:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677447.935055:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.935058:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.935060:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.935061:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.935062:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.935064:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:26.0:1619677447.935064:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:30.0:1619677447.935065:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.935066:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:21.2:1619677447.935066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:30.0:1619677447.935067:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.935067:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.935067:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:30.0:1619677447.935068:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.935069:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677447.935070:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.935072:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:30.0:1619677447.935073:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.935074:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:30.0:1619677447.935075:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677447.935075:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677447.935076:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:22.0:1619677447.935077:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.935078:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.935078:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677447.935079:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:22.0:1619677447.935079:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.935080:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.935081:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:22.0:1619677447.935081:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.935082:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.935083:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.935084:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:22.0:1619677447.935084:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.935085:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677447.935085:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.935086:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.935089:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677447.935090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677447.935091:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677447.935092:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:22.0:1619677447.935093:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.935094:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.935094:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677447.935094:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.935095:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.935095:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.935096:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.935097:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.935097:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677447.935097:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.935098:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677447.935098:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677447.935100:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:10.0:1619677447.935100:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677447.935101:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.935102:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677447.935102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.935110:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.935112:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.935113:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.935115:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.935115:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.935117:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677447.935122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677447.935130:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.935132:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.935133:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.935134:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.935135:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.935136:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677447.935220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.935228:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.935231:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.935232:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.935234:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.935236:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.935237:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677447.935238:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.935240:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.935241:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.935242:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677447.935245:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.935247:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677447.935249:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.935250:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677447.935251:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677447.935252:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677447.935253:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.935254:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677447.935255:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677447.935255:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677447.935261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677447.935265:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.935267:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.935268:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.935268:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:31.0:1619677447.935269:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.935270:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677447.935270:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677447.935271:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677447.935272:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677447.935273:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677447.935344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.935351:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.935354:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.935354:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.935356:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.935359:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.935361:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.935362:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677447.935363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:18.0:1619677447.935364:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.935365:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.935366:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.935369:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.935370:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:18.0:1619677447.935371:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677447.935373:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677447.935374:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:18.0:1619677447.935375:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.935376:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:26.0:1619677447.935377:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.935377:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.935378:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:26.0:1619677447.935379:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.935379:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.935379:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.935380:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:26.0:1619677447.935381:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677447.935382:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.935383:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.935384:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.935385:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677447.935388:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.935390:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677447.935393:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.935393:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677447.935394:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677447.935394:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:13.0:1619677447.935395:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:26.0:1619677447.935396:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:13.0:1619677447.935396:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.935397:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.935397:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:13.0:1619677447.935397:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.935398:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677447.935399:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.935400:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677447.935403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.935410:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.935412:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.935413:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.935415:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.935416:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.935418:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677447.935473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.935478:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.935481:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.935482:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.935484:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.935486:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.935488:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677447.935489:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.935490:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.935491:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.935492:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677447.935495:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.935497:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677447.935500:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.935500:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677447.935501:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677447.935503:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.935504:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.935505:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:4.2:1619677447.935505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:17.0:1619677447.935506:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.935506:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.935509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677447.935512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677447.935513:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.935515:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.935517:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:1.0:1619677447.935517:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:24.0:1619677447.935518:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.935519:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:1.0:1619677447.935519:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:8.0:1619677447.935520:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.935520:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.935521:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677447.935522:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.935522:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.935523:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.935524:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.935526:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.935527:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.935528:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.935532:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.935534:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.935534:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.935536:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.935537:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.935537:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677447.935538:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.935539:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.935540:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.935541:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.935542:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.935542:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.935548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.935555:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.935557:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.935558:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.935559:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.935559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:17.0:1619677447.935560:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.935563:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:10.0:1619677447.935566:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.935569:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.935569:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677447.935571:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.935574:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.935576:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.935576:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.935578:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.935579:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.935580:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677447.935583:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.935584:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677447.935587:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.935588:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.935588:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.935590:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.935591:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.935591:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.935592:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677447.935604:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.935607:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.935608:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.935608:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.935610:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.935613:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.935876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.935882:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.935885:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.935885:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.935888:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.935890:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.935891:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.935893:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.935894:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.935895:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.935896:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.935899:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.935901:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.935904:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.935905:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.935905:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.935907:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.935908:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.935908:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.935909:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677447.935913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:9.2:1619677447.935913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.935920:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:5.0:1619677447.935920:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.935922:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:8.0:1619677447.935923:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.935924:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.935924:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.935925:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:8.0:1619677447.935926:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:5.2:1619677447.935926:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:5.0:1619677447.935929:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.935930:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:5.0:1619677447.935931:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.935933:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:5.0:1619677447.935933:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.935934:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721547] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.935936:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:5.0:1619677447.935936:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677447.935937:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:5.0:1619677447.935937:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.935939:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677447.935940:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.935942:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.935943:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.935945:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:5.0:1619677447.935945:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:26.0:1619677447.935946:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.935948:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.935948:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677447.935949:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.935949:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:26.0:1619677447.935950:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:5.0:1619677447.935950:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.935952:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.935953:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.935953:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677447.935954:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.935954:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677447.935955:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677447.935958:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.935958:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677447.935959:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677447.935960:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677447.935961:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.935961:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.935962:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677447.935964:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.935967:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.935968:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.935969:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.935971:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.935972:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.935974:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.935977:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.935977:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.935979:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.935981:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.935982:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677447.936163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.936170:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.936173:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.936174:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.936176:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.936178:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.936180:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677447.936181:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.936182:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.936183:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.936184:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677447.936187:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.936189:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677447.936191:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.936191:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.936192:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677447.936193:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.936194:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.936194:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.936195:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677447.936205:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.936208:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.936208:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.936210:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.936212:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.936213:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.936228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677447.936235:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.936238:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.936238:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.936240:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.936243:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.936244:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.936245:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.936247:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.936248:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.936249:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.936252:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.936254:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.936256:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.936257:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.936258:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.936260:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.936260:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.936261:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.936261:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.936266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.936273:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.936276:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.936277:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.936277:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.936279:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.936282:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.936313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.936319:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.936321:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.936322:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.936324:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.936327:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.936328:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.936329:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.936331:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.936332:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.936333:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.2:1619677447.936334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:17.0:1619677447.936336:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.936338:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.936340:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677447.936340:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.936341:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.936342:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.936343:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:17.0:1619677447.936343:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.936344:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.936344:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.936344:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677447.936345:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677447.936346:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.936348:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.936349:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.936351:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.936352:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.936354:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.936355:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.936359:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:22.0:1619677447.936360:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.936360:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.936361:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.936362:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.936363:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.936366:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.936366:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.936366:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.936367:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.936367:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.936369:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.936370:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.936370:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.936370:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.936371:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.936371:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.936371:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.936372:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.936372:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.936373:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:21.2:1619677447.936378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677447.936382:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:6.0:1619677447.936383:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:26.0:1619677447.936385:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:6.0:1619677447.936385:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:26.0:1619677447.936386:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.936386:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:6.0:1619677447.936386:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.936387:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:26.0:1619677447.936388:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.936391:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677447.936440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.936448:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.936450:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.936452:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.936454:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.936455:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.936456:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677447.936457:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.936458:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.936459:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.936460:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677447.936463:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.936465:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.936468:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.936468:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677447.936469:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677447.936471:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.936471:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.936472:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.936473:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677447.936484:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.936487:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.936488:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.936489:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.936491:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.936494:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.936588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.936596:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.936599:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.936599:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677447.936601:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.936603:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.936604:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677447.936606:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.936607:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.936608:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.936609:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677447.936612:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.936614:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677447.936616:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.936617:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.936617:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677447.936619:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.936619:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.936620:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.936620:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677447.936631:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.936634:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.936634:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.936636:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.936638:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.936639:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.936702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.936707:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.936710:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.936711:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.936713:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.936715:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.936717:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.936718:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.936720:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.936721:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.936722:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.936724:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.936726:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.936729:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.936730:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.936731:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.936733:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.936733:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.936734:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.936734:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.936746:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.936749:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.936751:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.936753:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.936753:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.936756:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.936939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677447.936942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677447.936948:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.936950:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.936950:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000400:00000200:26.0:1619677447.936952:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.936952:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677447.936954:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.936955:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.936957:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.936958:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.936959:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.936960:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.936962:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.936963:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677447.936965:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677447.936966:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.936967:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677447.936967:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.936968:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677447.936968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:26.0:1619677447.936969:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:10.0:1619677447.936969:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677447.936972:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.936973:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:10.0:1619677447.936973:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.936976:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.936976:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:10.0:1619677447.936976:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.936977:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.936977:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:1.0:1619677447.936977:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:26.0:1619677447.936978:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.936978:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:17.0:1619677447.936979:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.936980:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.936980:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.936981:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677447.936981:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677447.936982:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (23)++
00000400:00000200:17.0:1619677447.936982:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.936982:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000400:00000200:17.0:1619677447.936983:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:10.0:1619677447.936983:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677447.936984:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (24)--
00000800:00000200:10.0:1619677447.936984:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (24)--
00000800:00000200:26.0:1619677447.936985:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677447.936985:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96721536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.936985:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:17.0:1619677447.936986:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.936987:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.936988:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:4.2:1619677447.936990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677447.936991:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.936993:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677447.936995:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:17.0:1619677447.936996:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.936997:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.936997:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.936997:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.936998:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677447.936998:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677447.936999:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677447.936999:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:26.0:1619677447.937000:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.937000:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:10.0:1619677447.937000:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.937001:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677447.937001:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677447.937002:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:22.0:1619677447.937002:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:17.0:1619677447.937002:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000400:00000200:6.0:1619677447.937002:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.937003:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677447.937003:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:10.0:1619677447.937003:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677447.937004:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677447.937004:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.937007:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.937008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:12.0:1619677447.937012:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.937014:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677447.937014:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000400:00000200:12.0:1619677447.937015:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.937016:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000400:00000200:12.0:1619677447.937017:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.937017:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:13.0:1619677447.937018:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.937018:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:12.0:1619677447.937020:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.937034:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.937037:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.937038:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.937038:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.937039:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.937040:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677447.937073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.937080:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.937083:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.937085:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.937087:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.937088:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.937088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677447.937089:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.937091:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.937092:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.937093:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.937094:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.937094:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.937097:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.937097:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677447.937098:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.937100:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.937100:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.937102:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.937103:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:17.0:1619677447.937103:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.937104:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:17.0:1619677447.937104:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:30.0:1619677447.937105:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.937106:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677447.937106:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.937107:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.937107:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:17.0:1619677447.937107:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.937108:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.937109:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.937112:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.937113:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.937116:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.937116:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.937117:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:12.0:1619677447.937117:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677447.937119:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.937119:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677447.937119:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677447.937120:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:12.0:1619677447.937120:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.937121:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.937122:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677447.937122:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.937124:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.937125:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.937126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677447.937132:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677447.937132:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.937134:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677447.937135:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677447.937135:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677447.937136:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.937136:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677447.937136:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.937136:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677447.937138:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.937138:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.937139:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:21.2:1619677447.937140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:13.0:1619677447.937140:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:22.0:1619677447.937141:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677447.937141:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.937142:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.937147:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.937150:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.937151:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.937153:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.937156:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.937157:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.937158:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.937160:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.937161:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.937162:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.937165:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.937167:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.937168:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.937169:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.937169:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.937171:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.937171:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.937172:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.937173:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.937174:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:9.2:1619677447.937177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677447.937180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.937185:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.937186:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.937186:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.937187:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:2.0:1619677447.937187:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.937188:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.937189:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.937189:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.937191:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677447.937191:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.937193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.937198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.937201:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:10.0:1619677447.937201:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.937204:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677447.937204:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:10.0:1619677447.937204:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677447.937207:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.937207:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.937208:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.937210:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.937211:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.937213:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677447.937213:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.937215:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.937215:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.937216:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.937217:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.937218:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.937218:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.937219:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677447.937219:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.937225:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677447.937225:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.937227:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.937228:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677447.937231:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677447.937232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:20.0:1619677447.937232:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.937232:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.937233:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.937233:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677447.937233:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.937235:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.937235:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.937236:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:10.0:1619677447.937236:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677447.937238:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:10.0:1619677447.937238:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.937238:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677447.937239:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:31.0:1619677447.937240:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:20.0:1619677447.937240:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:9.2:1619677447.937241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677447.937243:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677447.937245:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677447.937245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677447.937247:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677447.937248:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.937248:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:31.0:1619677447.937249:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:5.0:1619677447.937250:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000400:00000200:31.0:1619677447.937251:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96721530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.937251:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:31.0:1619677447.937252:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.937252:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.937252:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.937253:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.937253:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.937253:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:31.0:1619677447.937255:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:5.0:1619677447.937255:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:2.0:1619677447.937255:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.937256:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.937258:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:31.0:1619677447.937259:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677447.937259:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.937259:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.937260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:31.0:1619677447.937261:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:12.0:1619677447.937261:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.937262:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.937264:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677447.937264:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.937265:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677447.937266:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677447.937268:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677447.937268:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:18.0:1619677447.937268:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677447.937269:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677447.937270:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:18.0:1619677447.937271:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.937271:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.937272:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.937273:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.937273:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.937281:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.937284:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.937285:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.937285:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.937287:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.937290:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677447.937300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677447.937308:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.937310:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.937311:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.937313:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.937316:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.937318:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677447.937319:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.937321:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.937322:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.937323:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.937326:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.937328:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.937330:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.937331:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677447.937331:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.937333:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.937333:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.937334:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.937334:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.2:1619677447.937337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.937343:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.937346:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:13.0:1619677447.937347:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677447.937348:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.937348:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.937349:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.937350:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.937350:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.937351:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.937352:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:12.0:1619677447.937353:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677447.937353:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.937355:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677447.937355:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.937356:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.937357:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.937360:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.937362:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.937364:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.937365:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.937366:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.937367:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.937368:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.937369:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.937370:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.937371:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.937375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.937378:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:22.0:1619677447.937381:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.937381:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.937381:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.937382:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.937384:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.937385:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.937385:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.937387:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.937389:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.937397:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677447.937400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:6.0:1619677447.937400:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.937401:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.937401:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.937402:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.937403:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677447.937408:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.937411:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.937413:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.937415:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.937416:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.937417:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.937418:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.937419:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.937420:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.937421:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.937424:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.937426:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.937428:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.937429:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.937430:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.937431:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.937432:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.937433:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.937433:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.937445:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.937447:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.937448:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.937450:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.937451:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.937452:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677447.937492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.937498:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.937501:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.937502:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.937504:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.937507:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.937508:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.937510:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.937511:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.937512:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.937513:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.937516:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.937518:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.937520:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.937520:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.937521:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.937522:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.937523:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.937523:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.937524:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.937528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.937536:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.937538:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.937538:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.937539:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.937540:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.937541:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.937548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.937555:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.937557:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.937558:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.937558:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.937559:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.937559:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.937696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.937704:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.937707:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.937707:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.937709:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.937712:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.937714:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.937715:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.937717:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.937718:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.937719:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677447.937722:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.937723:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.937726:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.937727:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.937727:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677447.937729:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.937730:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.937731:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.937731:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677447.937743:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.937746:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.937747:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.937748:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.937750:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.937753:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677447.937848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.937855:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.937858:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.937860:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.937862:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.937862:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.937864:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677447.937865:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.937866:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.937867:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677447.937867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:31.0:1619677447.937868:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677447.937872:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.937873:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677447.937874:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677447.937876:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.937877:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677447.937877:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677447.937877:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.937878:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677447.937879:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.937879:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:24.0:1619677447.937879:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.937880:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.937881:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:24.0:1619677447.937881:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.937883:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.937884:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.937885:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.937886:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.937887:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.937890:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.937891:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.937892:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:24.0:1619677447.937894:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.937895:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.937895:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:24.0:1619677447.937896:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.937896:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.937896:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.937897:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.937898:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:6.0:1619677447.937898:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.937899:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.937900:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.937901:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677447.937901:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.2:1619677447.937903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.937906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.937911:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677447.937914:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.937914:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.937915:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.937915:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.937916:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.937916:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.2:1619677447.937917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:1.0:1619677447.937917:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.937918:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.937921:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.937922:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.937923:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.937924:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.937926:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.937927:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.937929:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677447.937929:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.937931:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.937932:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.937933:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.937935:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.937937:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677447.937939:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.937940:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.937940:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.937941:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.937942:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.937942:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.937943:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.937947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.937955:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.937957:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.937957:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.937959:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.937959:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.937960:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677447.938009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677447.938015:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.938018:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.938019:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.938021:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.938023:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.938025:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.938026:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.938027:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.938028:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.938030:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.938033:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.938035:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.938038:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.938039:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.938040:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677447.938041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677447.938042:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.938042:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.938043:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.938043:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:10.0:1619677447.938048:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.938051:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.938052:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:13.0:1619677447.938054:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677447.938054:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.938057:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:10.0:1619677447.938057:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.938058:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.938059:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:10.0:1619677447.938059:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:12.0:1619677447.938060:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677447.938060:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.938062:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.938062:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.938063:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677447.938064:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677447.938067:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.938069:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677447.938071:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.938072:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677447.938073:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677447.938075:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677447.938075:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.938076:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.938076:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677447.938086:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.938089:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.938089:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.938091:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.938093:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.938093:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.938184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.938191:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.2:1619677447.938191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:26.0:1619677447.938194:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.938195:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.938196:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.938199:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.938200:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677447.938200:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.938201:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.938203:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:26.0:1619677447.938203:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.938205:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.938206:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.938206:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.938206:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.938209:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.938211:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:30.0:1619677447.938212:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.938213:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.938213:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:30.0:1619677447.938214:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:26.0:1619677447.938214:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:30.0:1619677447.938215:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.938215:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677447.938216:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677447.938216:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.938216:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:30.0:1619677447.938217:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.938217:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:30.0:1619677447.938218:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.938221:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.938222:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.938224:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.938225:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.938226:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.938227:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.938227:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.938228:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.938228:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:31.0:1619677447.938229:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.938229:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:31.0:1619677447.938230:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.938230:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.938231:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.938232:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.938233:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.938237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677447.938239:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.938242:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.938242:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.938244:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:13.0:1619677447.938244:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.938246:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.938247:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.938247:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:13.0:1619677447.938247:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.938248:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.2:1619677447.938494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.938500:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.938503:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.938504:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.938505:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.938507:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.938509:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.938510:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.938512:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.938513:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.938514:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677447.938517:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.938519:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.938521:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.938522:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.938523:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.938524:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.938525:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.938526:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.938526:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.938531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.938538:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.938541:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.938542:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.938542:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.938545:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.938547:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677447.938609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677447.938616:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.938619:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.938620:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.938622:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.938624:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.938626:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677447.938627:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.938628:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.938629:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.938630:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:21.2:1619677447.938633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:8.0:1619677447.938634:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.938635:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.938638:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.938639:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.938640:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.938641:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677447.938641:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.938642:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.938642:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.938642:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:30.0:1619677447.938644:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.938646:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.938648:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.938648:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.938650:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.938651:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.938652:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.938653:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.938654:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.938657:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.938658:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677447.938659:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.938660:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:30.0:1619677447.938661:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.938661:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:12.0:1619677447.938661:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.938662:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:12.0:1619677447.938662:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.938664:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.938664:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677447.938664:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.938665:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677447.938665:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.938666:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.938667:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677447.938673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677447.938676:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.938677:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.938678:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.938678:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:13.0:1619677447.938679:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.938680:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677447.938680:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.938680:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.938681:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.938681:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.938682:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.938683:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677447.938683:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.938684:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.938684:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.938685:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.938694:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.938697:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.938697:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.938698:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.938698:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.938699:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.938797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.938804:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.938806:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.938807:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.2:1619677447.938808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:22.0:1619677447.938809:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.938812:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.938813:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.938815:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.938816:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677447.938816:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:22.0:1619677447.938817:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.938818:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677447.938819:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.938820:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677447.938823:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677447.938823:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.938825:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677447.938825:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.938826:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677447.938828:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.938829:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677447.938829:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.938830:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.938830:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:10.0:1619677447.938830:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.938831:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:22.0:1619677447.938832:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.938833:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.938834:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.938834:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677447.938835:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.938836:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.938838:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.938839:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677447.938839:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677447.938841:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.938841:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.938842:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.938843:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677447.938845:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.938847:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.938848:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.938850:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.2:1619677447.938850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:30.0:1619677447.938851:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.938852:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:31.0:1619677447.938853:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677447.938854:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.938855:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.938855:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.938856:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:26.0:1619677447.938857:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.938858:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:18.0:1619677447.938859:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.938860:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:30.0:1619677447.938861:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:18.0:1619677447.938862:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.938864:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.938865:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:18.0:1619677447.938865:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.938866:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:18.0:1619677447.938866:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:30.0:1619677447.938867:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.938868:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:18.0:1619677447.938868:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.938869:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.938870:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.938871:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.938872:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.938875:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:18.0:1619677447.938875:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.938875:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.938876:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.938876:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.938877:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:18.0:1619677447.938878:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.938879:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.938880:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.938881:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.938882:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.938883:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.938883:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.938884:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.938889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.938896:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.938898:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.938898:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.938899:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.938900:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.938902:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.938989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.938996:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.938999:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.938999:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.939001:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.939003:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.939005:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.939006:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.939007:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.939008:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.939009:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.939012:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.939014:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.939016:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.939017:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.939018:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.939020:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.939020:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.939021:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.939022:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.939023:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677447.939028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677447.939032:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.939035:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.939035:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.939035:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677447.939037:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.939037:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:31.0:1619677447.939038:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.939038:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677447.939039:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677447.939039:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:4.2:1619677447.939042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.939049:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.939051:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.939052:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.939054:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.939056:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.939057:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.939059:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.939060:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.939061:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.939062:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.939065:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.939067:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.939069:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.939070:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.939070:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.939072:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.939073:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.939073:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.939074:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.2:1619677447.939076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.2:1619677447.939078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.939084:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.939085:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.939088:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:17.0:1619677447.939089:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.939089:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.939091:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.939092:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.939093:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.939093:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.939095:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.939096:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.939097:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.939098:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:20.2:1619677447.939099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.939102:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:17.0:1619677447.939102:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.939104:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.939105:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:17.0:1619677447.939105:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.939105:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677447.939107:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:1.0:1619677447.939107:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.939108:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.939108:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.939109:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.939109:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.939109:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:20.0:1619677447.939111:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:17.0:1619677447.939111:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000400:00000200:20.0:1619677447.939112:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.939112:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.939113:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:20.0:1619677447.939114:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.939114:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000400:00000200:20.0:1619677447.939115:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.939115:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677447.939116:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:9.2:1619677447.939117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:20.0:1619677447.939119:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:5.2:1619677447.939119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677447.939121:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.0:1619677447.939123:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:20.0:1619677447.939124:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.939125:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677447.939125:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:1.0:1619677447.939125:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677447.939126:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.939127:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:20.0:1619677447.939127:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:8.0:1619677447.939127:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.939128:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.939128:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:1.0:1619677447.939128:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.939128:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677447.939129:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:8.0:1619677447.939129:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.939130:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677447.939132:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:26.0:1619677447.939139:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.939142:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677447.939142:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.939143:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.939145:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:26.0:1619677447.939147:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677447.939193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.939199:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.939202:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.939204:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.939204:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.939206:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.939208:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.939209:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.939211:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677447.939212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:5.0:1619677447.939212:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.939213:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.939218:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:5.0:1619677447.939218:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.939219:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.939221:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.939222:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677447.939223:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.939224:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.939224:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.939225:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677447.939226:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.939227:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.939227:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:18.0:1619677447.939228:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:5.0:1619677447.939228:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.939228:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677447.939229:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.939230:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.939231:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677447.939232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:18.0:1619677447.939233:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.939236:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.939238:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677447.939240:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677447.939240:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.939241:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677447.939242:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:21.2:1619677447.939242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:18.0:1619677447.939242:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.939243:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:31.0:1619677447.939244:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.939244:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.939244:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.939245:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:30.0:1619677447.939246:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.939247:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.939247:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:31.0:1619677447.939248:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:8.2:1619677447.939249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677447.939250:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96721505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.939250:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:31.0:1619677447.939251:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.939251:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:31.0:1619677447.939252:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.939254:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:6.0:1619677447.939254:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.939257:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.939257:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:6.0:1619677447.939257:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.939259:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:6.0:1619677447.939259:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.0:1619677447.939260:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:6.0:1619677447.939260:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96721504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.939261:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.939262:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.939263:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:20.0:1619677447.939263:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.939263:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.939263:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677447.939264:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:6.0:1619677447.939264:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.939265:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:6.0:1619677447.939265:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.939266:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677447.939266:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677447.939267:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.939268:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:6.0:1619677447.939268:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.939270:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.939273:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.939274:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.939274:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.2:1619677447.939276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:6.0:1619677447.939276:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677447.939277:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:6.0:1619677447.939277:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.939278:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.939278:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677447.939280:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.939281:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.939282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677447.939283:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.939284:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677447.939284:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677447.939285:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:12.0:1619677447.939286:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.939287:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.939288:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:12.0:1619677447.939289:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.939289:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.939289:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.939289:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.939290:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.939290:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:12.0:1619677447.939291:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.939292:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:13.0:1619677447.939293:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.939294:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.939299:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.939301:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:17.0:1619677447.939302:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.939305:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.939324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677447.939332:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.939335:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:5.2:1619677447.939336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:26.0:1619677447.939337:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677447.939338:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.939341:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.939342:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:2.0:1619677447.939342:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:10.0:1619677447.939344:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.939345:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.939345:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.939346:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:10.0:1619677447.939346:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.939348:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.939349:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.939351:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.939353:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.939354:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.939356:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:10.0:1619677447.939357:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.939358:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677447.939358:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677447.939358:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.939359:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677447.939360:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.939360:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677447.939361:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:2.0:1619677447.939361:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:10.0:1619677447.939362:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677447.939363:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677447.939364:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:2.0:1619677447.939364:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.939366:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:9.2:1619677447.939367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.939368:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677447.939369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:2.0:1619677447.939369:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.939370:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.939372:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.939373:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.939374:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.939376:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.939376:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:26.0:1619677447.939377:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:18.0:1619677447.939377:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677447.939379:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:8.2:1619677447.939379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677447.939380:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:18.0:1619677447.939380:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.939381:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.939381:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677447.939382:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:5.2:1619677447.939382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000400:00000200:18.0:1619677447.939383:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.939387:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.939388:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.939389:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.939389:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.939390:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.939390:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.939392:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.939393:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.939395:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.939395:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.939397:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677447.939450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.939459:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.939462:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.939464:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.939466:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.939467:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.939468:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677447.939469:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.939471:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.939472:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.939473:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677447.939476:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.939478:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677447.939481:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.939482:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677447.939482:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677447.939484:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.939484:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.939485:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.939486:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677447.939496:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.939499:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.939500:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.939500:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.939502:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.939504:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677447.939602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677447.939607:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.939610:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.939611:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.939613:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.939615:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.939616:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.939618:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.939619:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.939620:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.939621:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.939624:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.939626:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.939628:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.939629:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.939630:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.939632:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.939632:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.939633:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.939634:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.939646:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.939649:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.939650:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.939650:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.939653:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.939655:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677447.939710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.939715:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.939717:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.939718:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.2:1619677447.939720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:6.0:1619677447.939720:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.939722:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.939723:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.939724:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.939725:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.939726:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.939727:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.939728:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.939731:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:6.0:1619677447.939732:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.939734:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:6.0:1619677447.939734:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.939735:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.939735:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.939737:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677447.939737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:31.0:1619677447.939738:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.939738:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.939738:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.939739:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:30.0:1619677447.939740:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.939741:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.939741:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677447.939742:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.939742:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.939742:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:30.0:1619677447.939743:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.939743:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677447.939743:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:30.0:1619677447.939744:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.939746:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.939746:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.939746:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:31.0:1619677447.939749:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677447.939749:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.939750:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.939751:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96721497] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.939752:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.939753:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.939753:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.939753:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.939754:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.939755:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.939756:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96721496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.939758:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.939758:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:18.0:1619677447.939758:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.939759:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:18.0:1619677447.939759:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.939760:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.939761:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:18.0:1619677447.939761:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.939762:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (23)++
00000800:00000200:31.0:1619677447.939763:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677447.939763:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (24)--
00000800:00000200:30.0:1619677447.939763:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.939765:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:18.0:1619677447.939765:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.939766:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.939766:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.939767:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.939767:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.2:1619677447.939767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.939768:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677447.939768:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.939768:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.939770:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.939770:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.939771:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.939771:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.939771:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.939771:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677447.939772:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:18.0:1619677447.939772:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (23)++
00000800:00000200:30.0:1619677447.939773:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.939774:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677447.939774:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:18.0:1619677447.939774:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677447.939775:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)++
00000800:00000200:18.0:1619677447.939775:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.939776:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677447.939777:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (24)--
00000800:00000200:8.2:1619677447.939777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:20.0:1619677447.939779:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.939780:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.939781:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)++
00000800:00000200:20.0:1619677447.939783:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.939783:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (24)--
00000400:00000200:5.0:1619677447.939783:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.939785:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:5.0:1619677447.939785:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.939786:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.939787:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.939787:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.939788:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:20.0:1619677447.939789:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.939790:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.939791:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677447.939794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677447.939801:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.939804:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.939805:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.939806:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.939806:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.939808:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.939844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677447.939845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:4.2:1619677447.939851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:5.0:1619677447.939852:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677447.939853:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677447.939854:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:26.0:1619677447.939856:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.939857:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:5.0:1619677447.939857:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.939858:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677447.939858:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677447.939859:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.939860:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.939862:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.939862:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:5.0:1619677447.939862:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:8.0:1619677447.939863:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:26.0:1619677447.939864:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677447.939864:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.939866:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.939866:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.939866:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.939867:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.939868:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.939868:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.939869:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.939869:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677447.939870:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677447.939870:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.939872:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.939874:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.939875:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.939876:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.939876:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.939877:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.939878:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677447.939879:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:24.0:1619677447.939881:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.939881:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.939882:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.939882:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677447.939883:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:24.0:1619677447.939883:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.939883:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677447.939884:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677447.939886:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:24.0:1619677447.939886:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.939886:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.939886:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677447.939887:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677447.939887:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.939887:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.939887:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677447.939888:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:5.0:1619677447.939888:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677447.939889:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:24.0:1619677447.939889:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:26.0:1619677447.939890:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677447.939890:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.939890:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.939891:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.939896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677447.939896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677447.939898:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.939900:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.939901:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.939903:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.939903:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:13.0:1619677447.939904:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677447.939905:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677447.939905:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:1.0:1619677447.939905:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:30.0:1619677447.939906:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:10.0:1619677447.939906:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.939907:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677447.939907:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.939907:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.939907:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.939909:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:13.0:1619677447.939909:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.939909:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.939911:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:1.0:1619677447.939911:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.939912:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677447.939914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.939917:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677447.939918:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:1.0:1619677447.939919:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.939922:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:17.0:1619677447.939922:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.939922:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.939923:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.939923:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:1.0:1619677447.939923:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.939924:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:1.0:1619677447.939925:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.939925:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677447.939926:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677447.940026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.940033:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.940036:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.940038:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.940040:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.940041:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.940042:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677447.940043:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.940045:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.940046:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.940047:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677447.940050:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.940052:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677447.940054:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.940054:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677447.940055:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677447.940056:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.940057:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.940058:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.940058:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677447.940068:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.940071:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.940071:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.940071:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.940073:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.940075:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677447.940150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.940158:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:21.2:1619677447.940159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:30.0:1619677447.940160:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.940162:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.940164:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:2.0:1619677447.940164:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:30.0:1619677447.940165:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.940166:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:2.0:1619677447.940167:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:30.0:1619677447.940168:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.940169:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.940169:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:30.0:1619677447.940170:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.940171:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.940172:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.940174:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.940176:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.940176:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.940177:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.940178:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.940179:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.940180:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.940180:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:30.0:1619677447.940182:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.940183:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.940184:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:2.0:1619677447.940185:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.940186:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.940186:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677447.940187:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677447.940188:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.940189:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.940190:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:2.0:1619677447.940190:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.940190:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.940191:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.2:1619677447.940192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.940193:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.940193:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.940194:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.940195:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677447.940196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:8.2:1619677447.940199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.940203:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677447.940204:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677447.940205:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.940206:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.940207:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.940207:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677447.940207:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:31.0:1619677447.940208:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677447.940208:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.2:1619677447.940209:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:5.0:1619677447.940209:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.940211:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.940213:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.940213:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.940215:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677447.940216:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677447.940216:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.940217:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:26.0:1619677447.940219:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677447.940220:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677447.940221:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677447.940224:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.940225:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677447.940227:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.940228:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.940229:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.940230:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677447.940234:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677447.940235:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677447.940238:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.940239:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677447.940240:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677447.940241:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677447.940242:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.940243:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677447.940243:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677447.940247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.940254:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.940257:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.940258:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.940259:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.940260:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.940262:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677447.940323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.940328:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.940331:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.940332:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.940334:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.940337:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.940339:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.940340:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.940342:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.940342:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.940344:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.940347:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.940349:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.940351:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.940352:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.940353:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.940354:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.940355:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.940356:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.940357:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.940357:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677447.940361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677447.940363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.2:1619677447.940367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:6.0:1619677447.940368:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:17.0:1619677447.940369:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677447.940370:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.940371:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.940371:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677447.940372:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.940373:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.940373:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.940374:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:17.0:1619677447.940375:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.940376:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:17.0:1619677447.940377:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:5.0:1619677447.940378:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.940379:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.940381:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.940383:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.940384:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.940385:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.940386:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.940388:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.940391:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.940392:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.940395:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.940396:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.940397:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.940398:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.2:1619677447.940400:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.0:1619677447.940403:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.940404:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.940404:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677447.940405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.940410:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.940413:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.940414:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:18.0:1619677447.940415:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.940416:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.940417:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.940418:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.940419:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.940420:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.940421:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.940422:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.940423:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.940424:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.940427:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.940428:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.940429:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.940430:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.940430:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.940434:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.940435:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.940436:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.940437:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.940438:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.940438:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.940439:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.940451:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.940454:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.940455:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.940456:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.940458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.940459:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.940460:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.940463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:10.0:1619677447.940466:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.940469:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.940469:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:8.0:1619677447.940470:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677447.940472:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.940473:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.940474:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:10.0:1619677447.940474:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.940476:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677447.940476:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.940477:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.940478:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.940479:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.940479:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.940480:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677447.940481:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.940482:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.940483:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.940484:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.940484:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.940485:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:10.0:1619677447.940486:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:8.0:1619677447.940488:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.940489:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.940490:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677447.940490:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677447.940491:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.940493:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.940493:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677447.940493:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.940494:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.940494:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.0:1619677447.940494:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.940494:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.940496:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.940496:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.940497:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.940498:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.940502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677447.940506:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.2:1619677447.940507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677447.940509:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.940510:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.940510:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.940512:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.940512:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.940513:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.940514:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:22.0:1619677447.940515:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.940515:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.940515:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.940517:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.940517:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.940521:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.940521:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.940522:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:17.0:1619677447.940523:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:1.0:1619677447.940523:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.940524:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.940527:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.940527:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.940527:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:17.0:1619677447.940529:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.940530:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.940531:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.940531:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000400:00000200:17.0:1619677447.940532:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.940533:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.940533:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.940535:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.940537:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.940538:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.940539:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.940541:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.940541:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.940542:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.940543:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.940554:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.940557:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.940558:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.940558:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.940560:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.940562:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677447.940603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.940610:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.940614:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.940614:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.940616:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.940619:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.940621:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677447.940622:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.940624:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.940625:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.940626:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677447.940629:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.940631:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677447.940634:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.940635:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677447.940636:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.940637:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.940638:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.940639:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.940639:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677447.940643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.940651:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.940653:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.940655:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.940655:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.940657:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.940660:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.940662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.940669:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.940672:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.940673:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.940673:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.940674:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.940675:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.940953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.940962:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.940964:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.940966:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.940969:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.940969:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.940971:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.940972:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.940973:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.940974:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.940975:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.940979:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.940981:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.940984:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.940984:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.2:1619677447.940984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677447.940985:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.940987:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.940988:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.940989:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.940989:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:1.0:1619677447.940989:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.940992:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.940993:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677447.940993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:1.0:1619677447.940995:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.940997:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.940999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:1.0:1619677447.940999:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.941000:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.941001:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677447.941002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000400:00000200:1.0:1619677447.941002:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.941003:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.941004:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.941006:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.941007:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.941007:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.941008:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:5.0:1619677447.941008:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:20.0:1619677447.941009:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.941010:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:20.0:1619677447.941011:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.941011:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:1.0:1619677447.941011:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.941013:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677447.941014:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:17.0:1619677447.941014:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.941014:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.941014:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.941016:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.941016:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:17.0:1619677447.941017:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.941017:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.941019:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:5.0:1619677447.941019:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:1.0:1619677447.941019:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:17.0:1619677447.941021:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.941021:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.941023:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.941024:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.941025:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.941026:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.941027:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:5.0:1619677447.941028:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.941029:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.941030:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.941030:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.941031:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:5.0:1619677447.941033:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.941034:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.941036:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.941037:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:5.0:1619677447.941038:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.941039:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.941040:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.941040:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.941041:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677447.941042:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.0:1619677447.941042:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.941042:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.941043:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:17.0:1619677447.941044:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.941044:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:5.0:1619677447.941044:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677447.941045:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677447.941046:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.2:1619677447.941048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677447.941049:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:9.2:1619677447.941049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:30.0:1619677447.941053:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.941054:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.941055:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.941055:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677447.941056:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:24.0:1619677447.941056:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:1.0:1619677447.941058:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:30.0:1619677447.941059:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:24.0:1619677447.941059:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:17.0:1619677447.941059:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.941059:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.941060:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.941060:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:4.2:1619677447.941061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:1.0:1619677447.941061:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.941062:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.941064:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:24.0:1619677447.941066:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677447.941068:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.941069:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.941070:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677447.941072:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677447.941072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000400:00000200:24.0:1619677447.941075:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.941077:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.941078:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.941079:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000400:00000200:24.0:1619677447.941080:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.941080:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.941081:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000400:00000200:24.0:1619677447.941082:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.941082:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.941082:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.941083:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.941083:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:24.0:1619677447.941087:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.941089:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.941092:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.941093:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.941093:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.941095:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.941095:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.941097:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.941098:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.941098:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.941101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677447.941104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.941109:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.941111:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:8.0:1619677447.941111:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.941112:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.941113:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.941113:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.941114:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677447.941115:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.941115:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.941118:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.941206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.941213:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.941215:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.941216:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.941218:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.941222:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.941223:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.941224:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.941225:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.941226:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.941227:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677447.941231:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.941232:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677447.941235:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.941236:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.941237:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.941238:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.941239:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.941240:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.941240:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677447.941252:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.941255:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.941256:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.941256:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.941259:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.941261:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.941262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677447.941270:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677447.941273:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677447.941275:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.941276:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677447.941278:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.941280:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:13.0:1619677447.941281:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.941282:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677447.941283:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677447.941284:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:13.0:1619677447.941287:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.941289:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677447.941292:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.941293:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:13.0:1619677447.941297:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:13.0:1619677447.941299:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:13.0:1619677447.941300:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677447.941301:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677447.941303:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677447.941303:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:8.2:1619677447.941306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677447.941310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:1.0:1619677447.941315:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.941317:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.941318:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:12.0:1619677447.941318:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000400:00000200:1.0:1619677447.941319:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.941321:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000400:00000200:1.0:1619677447.941321:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.941322:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677447.941322:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:21.2:1619677447.941324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:1.0:1619677447.941324:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.941325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:22.0:1619677447.941331:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677447.941333:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677447.941334:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.941335:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677447.941336:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677447.941337:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.941337:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.941338:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.941339:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.941339:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:22.0:1619677447.941340:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.941342:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.941343:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.941345:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.941346:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.941347:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.941351:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.941352:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.941355:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.941356:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.941357:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.941359:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.941359:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.941360:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.941360:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:26.0:1619677447.941372:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677447.941375:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677447.941375:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677447.941376:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.941378:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.941381:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677447.941394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677447.941401:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.941404:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.941405:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.941407:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.941409:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.941411:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.941412:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.941414:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.941415:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.941416:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.941419:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.941421:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.941424:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.941424:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.941425:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.941427:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.941427:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.941429:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.941430:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.941430:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677447.941433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677447.941436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:2.0:1619677447.941441:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.0:1619677447.941443:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:2.0:1619677447.941444:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.941445:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.941445:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:2.0:1619677447.941445:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.941446:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.941447:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:2.0:1619677447.941447:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.2:1619677447.941451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677447.941451:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.941456:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.941459:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.941459:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.941461:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.941464:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.941465:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.941467:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.941468:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.941469:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.941470:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.941473:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.941475:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.941478:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.941478:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.941479:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.941481:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.941481:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.941482:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.941482:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.941494:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.941497:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.941498:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.941498:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.941500:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.941502:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.941512:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.941514:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.941515:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.941516:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.941516:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.941517:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677447.941536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.941543:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.941546:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.941547:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.941548:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.941551:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.941553:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677447.941554:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.941555:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.941556:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.941557:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677447.941561:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.941562:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677447.941565:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.941566:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677447.941567:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677447.941568:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.941569:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.941569:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.941570:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677447.941580:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.941583:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.941583:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.941585:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.941587:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.941588:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.941597:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.941600:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.941601:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.941601:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.941602:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.941604:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.941632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.941639:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.941642:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.941643:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.941644:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.941647:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.941648:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.941649:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.941651:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.941652:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.941652:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677447.941656:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.941657:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.941660:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.941660:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.941661:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677447.941662:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.941663:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.941664:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.941664:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677447.941676:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.941679:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.941680:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.941680:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.941682:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.941685:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.941818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.941823:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.941826:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.941827:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.941828:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.941831:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.941833:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.941834:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.941835:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.941836:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.941838:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.941841:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.941843:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.941846:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.941847:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.941847:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.941849:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.941850:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.941851:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.941852:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.941853:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677447.941858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677447.941861:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677447.941863:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677447.941864:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.941865:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.941865:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:12.0:1619677447.941866:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.941866:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.941867:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.941868:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.941869:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.941936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.941943:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.941946:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.941947:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.941949:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.941951:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.941953:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.941954:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.941955:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.941956:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.941957:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677447.941961:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.941962:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677447.941965:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.941966:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.941967:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.941968:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.941969:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.941970:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677447.941971:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.941971:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677447.941977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.941982:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677447.941984:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677447.941985:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.941985:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.941986:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677447.941986:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.941987:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.941988:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677447.941988:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.941991:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677447.942003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.942011:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.942014:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.942016:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.942018:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.942019:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.942020:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.942022:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.942023:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.942024:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.942025:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.942028:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.942029:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.942031:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.942032:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.942033:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.942034:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.942035:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.942036:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.942036:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.942046:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.942048:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.942049:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.942051:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.942052:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.942054:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.942096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.942102:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.942105:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.942105:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.942107:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.942110:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.942112:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.942113:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.942115:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.942116:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.942117:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.942120:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.942122:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.942124:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.942125:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.942126:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.942127:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.942128:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.942128:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.942129:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.942141:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.942143:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.942144:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.942144:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.942146:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.942149:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.942327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.942335:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.942338:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.942338:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677447.942340:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.942343:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.942345:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677447.942346:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.942347:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.942348:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.942350:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677447.942353:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.942355:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.942358:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.942359:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677447.942359:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.942361:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.942362:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.942362:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.942363:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677447.942368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.942376:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.942379:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.942380:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.942380:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.942383:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.942386:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.942400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.942406:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.942409:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.942409:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.942411:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.942414:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.942416:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.942417:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.942418:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.942419:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.942420:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.942424:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.942426:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.942428:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.942429:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.942430:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.942432:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.942432:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.942432:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.942433:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.942445:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.942448:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.942449:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.942450:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.942452:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.942453:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.942678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.942686:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.942689:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.942691:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.942693:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.942693:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.942695:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.942696:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.942698:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.942698:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.942700:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.942703:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.942705:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.942707:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.942708:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.942709:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.942710:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.942711:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.942712:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.942712:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677447.942722:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.942724:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.942725:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.942726:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.942728:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.942729:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677447.942788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.942796:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.942798:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.942800:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.942802:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.942802:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.942804:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.942805:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.942807:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.942808:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.942809:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.942812:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.942814:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.942816:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.942816:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.942817:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.942818:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.942819:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.942820:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.942822:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.942822:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677447.942828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677447.942830:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.942832:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.942833:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.942834:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.942835:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677447.942836:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.942837:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.942838:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.942838:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.942839:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.942845:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.942848:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.942848:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.942849:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.942849:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.942850:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.942862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197eb55c400] (20)++
00000800:00000200:20.0:1619677447.942868:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197eb55c400] (21)++
00000800:00000200:20.0:1619677447.942871:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.57@o2ib3
00000400:00000200:20.0:1619677447.942873:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.57@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.942874:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c400] (22)--
00000400:00000200:20.0:1619677447.942875:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.942877:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677447.942878:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.942880:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.942881:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.942882:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677447.942885:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.942887:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677447.942890:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.942890:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677447.942891:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677447.942892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197eb55c400] (21)++
00000800:00000200:20.0:1619677447.942893:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677447.942894:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197eb55c400] (22)++
00000800:00000200:20.0:1619677447.942895:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197eb55c400] (23)++
00000800:00000200:5.0:1619677447.942896:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197eb55c400] (23)++
00000800:00000200:5.0:1619677447.942897:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.57@o2ib3
00000800:00000200:4.2:1619677447.942897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.942898:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197eb55c400] (25)--
00000800:00000200:20.0:1619677447.942898:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c400] (24)--
00000800:00000200:20.0:1619677447.942900:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197eb55c400] (23)++
00000400:00000200:5.0:1619677447.942900:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.57@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.942902:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.57@o2ib3
00000400:00000200:5.0:1619677447.942902:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.942904:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.942905:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.57@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.942905:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.942906:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677447.942907:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677447.942908:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.942908:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:5.0:1619677447.942908:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677447.942909:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:5.0:1619677447.942909:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677447.942910:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.942912:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.942912:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.942914:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677447.942914:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.942914:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.942915:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.942915:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96721459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.942917:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.942917:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.942918:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 352 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:26.0:1619677447.942919:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96721459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.942919:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.942920:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677447.942921:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.942923:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.942923:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.942923:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:26.0:1619677447.942924:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.942924:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.942924:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.942926:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.942927:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677447.942928:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677447.942928:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197eb55c400] (24)++
00000800:00000200:5.0:1619677447.942929:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197eb55c400] (25)--
00000400:00000200:26.0:1619677447.942930:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.942930:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677447.942930:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c400] (24)--
00000800:00000200:20.0:1619677447.942931:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.942931:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677447.942931:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197eb55c400] (23)++
00000800:00000200:20.0:1619677447.942932:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:20.0:1619677447.942932:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677447.942932:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197eb55c400] (24)--
00000800:00000200:26.0:1619677447.942933:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677447.942933:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.942933:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c400] (23)--
00000800:00000200:20.0:1619677447.942934:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677447.942934:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c400] (22)--
00000800:00000200:20.0:1619677447.942935:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197eb55c400] (21)++
00000800:00000200:10.0:1619677447.942935:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:20.0:1619677447.942936:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197eb55c400] (22)--
00000800:00000200:26.0:1619677447.942937:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.942937:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c400] (21)--
00000800:00000200:10.0:1619677447.942937:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.942938:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677447.942938:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.942939:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (23)++
00000800:00000200:10.0:1619677447.942939:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:22.0:1619677447.942940:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (24)++
00000800:00000200:26.0:1619677447.942942:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (25)--
00000800:00000200:26.0:1619677447.942943:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.942943:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:26.0:1619677447.942944:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677447.942944:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (24)--
00000800:00000200:26.0:1619677447.942945:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677447.942945:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.942946:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.942946:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.942949:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:22.0:1619677447.942950:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:13.0:1619677447.942950:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.942952:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:13.0:1619677447.942952:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.942952:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677447.942954:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.942955:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.942955:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.942955:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:21.2:1619677447.942957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197eb55c400] (20)++
00000800:00000200:6.0:1619677447.942957:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677447.942958:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.942958:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (23)++
00000800:00000200:6.0:1619677447.942959:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (24)--
00000800:00000200:6.0:1619677447.942959:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.942960:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.942961:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.942962:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197eb55c400] (21)++
00000800:00000200:5.0:1619677447.942964:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.57@o2ib3
00000800:00000200:20.0:1619677447.942967:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c400] (22)--
00000400:00000200:5.0:1619677447.942967:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.57@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.942969:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.942971:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.942972:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.942974:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.942975:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.942976:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.942979:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.942981:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.942984:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.942985:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.942985:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677447.942988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197eb55c400] (21)++
00000800:00000200:5.0:1619677447.942988:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.942989:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197eb55c400] (21)++
00000800:00000200:5.0:1619677447.942990:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197eb55c400] (23)--
00000800:00000200:5.0:1619677447.942990:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c400] (22)--
00000800:00000200:5.0:1619677447.942991:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197eb55c400] (21)++
00000800:00000200:5.0:1619677447.942993:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.57@o2ib3
00000800:00000200:21.2:1619677447.942994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:20.0:1619677447.942994:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c400] (22)--
00000400:00000200:5.0:1619677447.942996:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.57@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.942998:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.942999:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:12.0:1619677447.943000:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.943000:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:5.0:1619677447.943001:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.943002:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.943003:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:5.0:1619677447.943003:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.943004:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:12.0:1619677447.943004:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.943004:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:2.0:1619677447.943004:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:13.0:1619677447.943006:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.943006:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.943008:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.943008:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:12.0:1619677447.943010:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.943011:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:2.0:1619677447.943011:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.943012:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677447.943013:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.943014:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.943015:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:5.0:1619677447.943015:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:2.0:1619677447.943015:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.943016:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.943017:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000400:00000200:2.0:1619677447.943017:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.943018:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197eb55c400] (21)++
00000800:00000200:5.0:1619677447.943019:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197eb55c400] (22)++
00000800:00000200:5.0:1619677447.943020:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197eb55c400] (23)--
00000800:00000200:5.0:1619677447.943021:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c400] (22)--
00000400:00000200:2.0:1619677447.943021:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.943022:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.2:1619677447.943025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197eb55c400] (21)++
00000800:00000200:2.0:1619677447.943025:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.943025:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.943026:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677447.943028:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.943028:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:10.0:1619677447.943029:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:2.0:1619677447.943029:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.943029:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.0:1619677447.943031:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197eb55c400] (22)++
00000800:00000200:26.0:1619677447.943032:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677447.943032:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:20.0:1619677447.943033:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197eb55c400] (23)--
00000400:00000200:10.0:1619677447.943033:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.943034:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c400] (22)--
00000800:00000200:20.0:1619677447.943035:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c400] (21)--
00000400:00000200:10.0:1619677447.943035:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.943038:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:22.0:1619677447.943041:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.943044:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.943044:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.943045:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677447.943046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197eb55c400] (20)++
00000400:00000200:22.0:1619677447.943047:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.943049:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:26.0:1619677447.943050:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:20.0:1619677447.943051:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197eb55c400] (21)++
00000800:00000200:20.0:1619677447.943053:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.57@o2ib3
00000800:00000200:26.0:1619677447.943054:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677447.943054:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677447.943055:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (21)++
00000800:00000200:5.2:1619677447.943055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677447.943056:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677447.943056:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:20.0:1619677447.943056:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.57@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.943059:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.943060:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:5.0:1619677447.943060:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c400] (22)--
00000800:00000200:17.0:1619677447.943061:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:20.0:1619677447.943062:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.943063:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.943064:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.943064:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:20.0:1619677447.943065:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.943065:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.943067:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.943069:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.943070:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.943071:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.943072:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.943073:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.943074:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.943075:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:17.0:1619677447.943075:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.943075:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.943076:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:17.0:1619677447.943077:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677447.943078:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.943078:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197eb55c400] (21)++
00000800:00000200:20.0:1619677447.943079:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197eb55c400] (22)--
00000800:00000200:20.0:1619677447.943080:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c400] (21)--
00000400:00000200:17.0:1619677447.943080:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.943082:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.943085:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.943086:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.943086:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.943088:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.943089:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.943089:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677447.943090:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677447.943090:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.943093:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.943094:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.943096:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.943097:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.943099:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:13.0:1619677447.943100:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.943103:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.943104:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677447.943105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197eb55c400] (20)++
00000400:00000200:13.0:1619677447.943105:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.943107:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.943108:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:13.0:1619677447.943108:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.943109:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197eb55c400] (21)++
00000800:00000200:31.0:1619677447.943111:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.943112:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.943113:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.943113:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:21.2:1619677447.943113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:5.0:1619677447.943113:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.57@o2ib3
00000800:00000200:31.0:1619677447.943115:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.0:1619677447.943115:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c400] (22)--
00000400:00000200:5.0:1619677447.943116:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.57@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.943119:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:5.0:1619677447.943119:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.943120:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:18.0:1619677447.943122:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:5.0:1619677447.943122:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.943122:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:5.0:1619677447.943124:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.943125:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.943126:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.943127:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.943129:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.943131:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677447.943131:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.943132:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677447.943133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197eb55c400] (21)++
00000400:00000200:18.0:1619677447.943134:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.943135:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.943135:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.943136:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:20.0:1619677447.943138:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197eb55c400] (22)++
00000800:00000200:5.0:1619677447.943139:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.943139:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.943140:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.57@o2ib3
00000800:00000200:5.0:1619677447.943140:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677447.943141:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.943142:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.57@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.943142:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.943143:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.943143:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197eb55c400] (23)++
00000800:00000200:5.0:1619677447.943144:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197eb55c400] (24)--
00000400:00000200:20.0:1619677447.943145:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.943145:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c400] (23)--
00000800:00000200:5.0:1619677447.943145:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c400] (22)--
00000400:00000200:20.0:1619677447.943146:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:18.0:1619677447.943147:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.943148:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96721450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.943148:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.943148:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:20.0:1619677447.943149:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.943150:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.943150:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:20.0:1619677447.943151:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.943151:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.943152:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.943153:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.943154:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:12.0:1619677447.943154:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:20.0:1619677447.943155:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.943157:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677447.943157:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.943158:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677447.943159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.0:1619677447.943160:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677447.943160:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.943161:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.943161:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677447.943161:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.943162:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:20.0:1619677447.943163:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.943164:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197eb55c400] (21)++
00000800:00000200:12.0:1619677447.943164:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.943164:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:26.0:1619677447.943165:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677447.943165:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197eb55c400] (22)--
00000800:00000200:10.0:1619677447.943165:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:20.0:1619677447.943166:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c400] (21)--
00000400:00000200:10.0:1619677447.943166:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.943167:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:10.0:1619677447.943168:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.943168:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.943169:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.943171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.943172:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:18.0:1619677447.943179:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.943181:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.943182:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.943182:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:21.2:1619677447.943184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197eb55c400] (20)++
00000400:00000200:18.0:1619677447.943184:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.943188:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677447.943188:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197eb55c400] (21)++
00000800:00000200:5.0:1619677447.943190:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.57@o2ib3
00000400:00000200:5.0:1619677447.943192:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.57@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.943193:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c400] (22)--
00000400:00000200:5.0:1619677447.943195:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.943196:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:5.0:1619677447.943198:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.943199:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.943200:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.943201:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:4.2:1619677447.943203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:5.0:1619677447.943204:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.943206:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.0:1619677447.943209:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.943209:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:5.0:1619677447.943210:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:24.0:1619677447.943211:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.0:1619677447.943212:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:5.0:1619677447.943212:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197eb55c400] (21)++
00000800:00000200:24.0:1619677447.943214:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:5.0:1619677447.943214:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197eb55c400] (22)++
00000800:00000200:8.0:1619677447.943215:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677447.943215:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197eb55c400] (23)--
00000800:00000200:5.0:1619677447.943216:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c400] (22)--
00000400:00000200:24.0:1619677447.943217:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677447.943219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197eb55c400] (21)++
00000400:00000200:24.0:1619677447.943220:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.943221:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.943223:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.943224:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.943224:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197eb55c400] (22)++
00000400:00000200:24.0:1619677447.943225:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.943225:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:24.0:1619677447.943226:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.943227:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.57@o2ib3
00000800:00000200:6.0:1619677447.943228:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.943229:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.943229:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.943230:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197eb55c400] (23)++
00000400:00000200:20.0:1619677447.943231:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.57@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.943231:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197eb55c400] (24)--
00000400:00000200:24.0:1619677447.943232:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.943232:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.943232:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c400] (23)--
00000800:00000200:5.0:1619677447.943232:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c400] (22)--
00000400:00000200:20.0:1619677447.943233:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.943234:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.943235:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:20.0:1619677447.943236:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.943237:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.943238:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.943238:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:20.0:1619677447.943238:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.943239:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677447.943240:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.943241:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.943241:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:20.0:1619677447.943241:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677447.943242:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.943243:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:20.0:1619677447.943244:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677447.943246:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677447.943247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.943250:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.943252:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.943253:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.943254:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.943254:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.943255:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.943255:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:5.0:1619677447.943256:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.943257:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.943257:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197eb55c400] (21)++
00000800:00000200:1.0:1619677447.943257:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677447.943258:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197eb55c400] (22)--
00000800:00000200:17.0:1619677447.943258:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:5.0:1619677447.943258:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.943259:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c400] (21)--
00000400:00000200:1.0:1619677447.943259:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.943260:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.943260:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.943261:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96721446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.943262:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.943263:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.943264:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.943265:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677447.943265:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:5.0:1619677447.943268:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.943270:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677447.943270:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.943272:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.943273:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.943273:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.943273:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.943274:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:30.0:1619677447.943275:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.943276:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.943276:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.943277:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677447.943278:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.0:1619677447.943278:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.943279:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.943280:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.943285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.943288:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.943290:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:22.0:1619677447.943291:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.943291:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.943292:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.943296:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.943299:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.943300:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.943302:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.943305:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677447.943470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.943479:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.943482:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.943484:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.943486:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.943487:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.943488:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.943489:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.943490:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677447.943490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:31.0:1619677447.943492:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.943492:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.943495:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.943497:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.943497:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.943499:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677447.943500:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.943500:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.943501:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677447.943501:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.943503:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:30.0:1619677447.943503:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.943504:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.943505:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.943505:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:30.0:1619677447.943505:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.943506:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.943507:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.943507:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.943508:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.943509:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.943512:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.943514:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.943516:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.943517:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.943518:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.943518:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677447.943519:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.943520:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.943520:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:13.0:1619677447.943520:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:30.0:1619677447.943521:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:13.0:1619677447.943521:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.943523:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.943526:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.943527:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.943528:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.943530:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.943531:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.943533:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.943535:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.943535:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.943550:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.943552:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.943553:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.943554:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.943554:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.943555:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677447.943695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.943703:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.943706:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.943708:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.943709:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.943712:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.943713:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677447.943714:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.943716:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.943717:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.943718:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677447.943721:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.943723:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677447.943725:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.943726:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677447.943726:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:4.2:1619677447.943727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.943728:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677447.943728:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.943729:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.943730:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677447.943731:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.943733:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.943734:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.943735:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.943738:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.943739:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677447.943740:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.943740:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:26.0:1619677447.943741:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.943742:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.943742:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:26.0:1619677447.943743:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:12.0:1619677447.943743:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.943744:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:26.0:1619677447.943746:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.943747:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677447.943748:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:12.0:1619677447.943748:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677447.943750:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.943750:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.943751:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.943752:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.943752:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.943753:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.943754:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677447.943763:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.943765:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.943766:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.943766:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.943768:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.943769:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677447.943834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.943840:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.943844:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.943845:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.943846:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.943850:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.943851:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.943852:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.943854:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.943855:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.943856:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.943859:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.943861:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.943863:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.943864:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.943865:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.943866:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.943867:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.943867:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.943868:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.943873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.943881:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.943883:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.943884:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.943886:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.943887:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.943889:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677447.943893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.2:1619677447.943896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.943900:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.943901:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.943901:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.943902:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.943903:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.943903:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.943904:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.943905:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:5.0:1619677447.943905:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.943906:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.943907:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.943909:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.943910:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.943911:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.943913:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.943913:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.943917:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.943919:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.943921:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.943922:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.943923:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.943925:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.943925:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.943926:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.943927:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677447.943932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.943940:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.2:1619677447.943941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677447.943943:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.943943:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.943944:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.943946:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.943948:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:9.2:1619677447.943948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.943951:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677447.943951:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.943952:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677447.943953:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:17.0:1619677447.943953:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.943955:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.943956:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000400:00000200:17.0:1619677447.943956:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.943956:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.943957:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.943957:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:17.0:1619677447.943957:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.943959:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.943960:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.943961:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.943962:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.943965:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.943966:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.943968:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.943969:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.943969:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.943971:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.943971:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.943973:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.943974:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.943975:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677447.943977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677447.943980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.943986:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.943986:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.943986:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.943987:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:1.0:1619677447.943987:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.943988:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.943989:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.943989:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.943990:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.943990:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.944164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.944167:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.944170:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.944172:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.944173:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.944174:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.944176:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.944177:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.944178:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.944179:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.944180:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.944184:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.944186:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.944189:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.944190:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.944191:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.944192:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.944193:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.944194:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.944194:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.944205:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.944208:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.944209:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.944211:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.944211:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.944214:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.944227:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.944230:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.944231:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.944231:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.944232:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.944233:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.944311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.944319:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.944321:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.944324:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.944326:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.944326:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.944327:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677447.944329:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.944330:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.944331:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.944332:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677447.944335:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.944336:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677447.944339:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.944340:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677447.944340:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677447.944341:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.944342:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.944343:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.944343:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677447.944355:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.944358:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.944359:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.944359:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677447.944361:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.944363:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677447.944364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677447.944372:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.944374:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677447.944375:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677447.944377:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677447.944379:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.944381:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677447.944382:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.944383:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.944384:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.944385:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677447.944387:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677447.944389:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677447.944392:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.944393:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677447.944393:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677447.944395:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677447.944395:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.944396:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677447.944396:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677447.944401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677447.944407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:17.0:1619677447.944410:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.944413:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.944413:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.944414:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.944415:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677447.944416:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:26.0:1619677447.944417:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.944418:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677447.944419:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:26.0:1619677447.944421:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.944423:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.944425:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677447.944426:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.944427:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.944428:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.944429:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677447.944432:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.944434:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677447.944436:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.944437:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677447.944438:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.944440:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.944440:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.944441:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677447.944443:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677447.944443:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677447.944446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677447.944448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677447.944454:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677447.944456:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:2.0:1619677447.944457:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.944458:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.944458:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:2.0:1619677447.944458:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.944459:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.944460:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:2.0:1619677447.944460:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.944463:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.944499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.944505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:26.0:1619677447.944506:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.944508:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.944509:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.944511:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.944513:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677447.944514:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.944515:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:30.0:1619677447.944516:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:26.0:1619677447.944517:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.944519:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.944519:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.944520:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677447.944520:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.944521:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.944521:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677447.944523:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.944524:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.944524:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.944526:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677447.944526:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:30.0:1619677447.944527:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.944527:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677447.944528:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677447.944528:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.944528:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.944529:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.944530:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677447.944531:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.944531:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:21.2:1619677447.944531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:30.0:1619677447.944532:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677447.944532:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.944533:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677447.944535:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.944536:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.944537:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.944538:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:8.0:1619677447.944538:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:4.2:1619677447.944538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677447.944539:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.944540:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.944542:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.944542:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.0:1619677447.944542:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.944543:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:10.0:1619677447.944543:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.944544:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.944545:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.944545:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677447.944547:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:10.0:1619677447.944547:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.944548:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.944548:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.2:1619677447.944548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677447.944549:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677447.944549:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.944550:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:8.0:1619677447.944552:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.944553:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:8.0:1619677447.944555:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.944556:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:8.0:1619677447.944556:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:5.0:1619677447.944556:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:8.0:1619677447.944557:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.944557:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.944558:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.944559:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677447.944559:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:8.0:1619677447.944559:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.944559:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.944560:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677447.944560:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000400:00000200:8.0:1619677447.944560:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.944561:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:26.0:1619677447.944563:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.944563:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677447.944563:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:26.0:1619677447.944564:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.944565:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.944565:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:8.0:1619677447.944565:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.944567:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.944570:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.944570:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.944571:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.944572:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.944573:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.944574:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.944574:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.944585:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.944588:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.944589:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.944590:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.944592:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.944593:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677447.944609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.944615:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.944619:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.944619:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.944622:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.944624:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.944626:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.944627:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.944629:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.944630:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.944631:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677447.944634:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.944636:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677447.944639:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.944640:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.944640:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.944642:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.944643:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.944643:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.944644:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.944648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.944656:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.944658:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.944659:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.944659:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.944661:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.944664:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677447.944790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.944795:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.944798:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.944798:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.944801:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.944803:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.944804:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.944805:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.944807:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.944808:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.944809:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.944812:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.944814:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.944816:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.944817:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.944818:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.944820:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.944820:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.944821:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.944822:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.944823:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677447.944826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677447.944828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677447.944833:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677447.944833:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.944835:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:20.0:1619677447.944836:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:18.0:1619677447.944836:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:20.0:1619677447.944837:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.944837:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:20.0:1619677447.944838:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.944840:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.944841:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677447.944852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.944859:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:4.2:1619677447.944860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677447.944862:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.944864:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677447.944865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677447.944866:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.944867:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.944867:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:31.0:1619677447.944869:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677447.944870:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.944870:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.944871:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:31.0:1619677447.944872:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677447.944872:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677447.944874:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.944874:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.944875:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.944876:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:24.0:1619677447.944876:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.944876:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:24.0:1619677447.944878:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:31.0:1619677447.944880:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.944880:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.944882:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.944883:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:24.0:1619677447.944883:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.944884:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677447.944884:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.944885:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677447.944885:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.944886:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677447.944887:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:10.0:1619677447.944887:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:31.0:1619677447.944888:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.944888:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:24.0:1619677447.944888:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677447.944888:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96721427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.944889:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677447.944889:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.944890:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677447.944890:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:10.0:1619677447.944890:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.944891:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.944894:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.944895:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.944895:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:10.0:1619677447.944895:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.944897:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.944897:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677447.944897:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.944898:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.944898:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:26.0:1619677447.944901:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.944901:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.944902:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677447.944902:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:26.0:1619677447.944903:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.944903:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.2:1619677447.944903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:26.0:1619677447.944905:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.944905:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.944907:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.944907:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.944908:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.944908:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677447.944909:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:1.0:1619677447.944910:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:9.2:1619677447.944911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.944913:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.944914:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.944914:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.944916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:1.0:1619677447.944916:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.944919:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677447.944919:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.0:1619677447.944922:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677447.944922:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.944923:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.944923:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.944924:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:2.0:1619677447.944926:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.944927:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.944927:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.944930:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.944930:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:5.0:1619677447.944932:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.944933:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721426] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.944934:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.944935:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.944936:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.944940:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.944942:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.944945:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.944946:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.944946:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.944948:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.944949:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.944950:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.944950:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.944961:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.944963:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.944964:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.944966:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.944967:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.944968:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.945036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.945043:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.945046:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.945049:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.945050:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.945051:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.945052:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677447.945054:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.945055:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.945056:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.945057:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677447.945061:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.945062:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.945065:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.945065:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.945066:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.945068:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.945068:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.945069:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.945070:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.2:1619677447.945074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677447.945080:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:17.0:1619677447.945080:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677447.945083:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:17.0:1619677447.945083:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:31.0:1619677447.945084:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.945084:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677447.945086:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.945086:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.945087:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.945088:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:17.0:1619677447.945088:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.945090:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.945091:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.945093:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.945093:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.945095:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.945098:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.945100:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.945102:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.945103:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.945103:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.945105:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.945105:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.945106:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.945106:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.945119:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.945121:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.945122:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.945123:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.945125:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.945127:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677447.945217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.945222:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.945224:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.945225:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.945227:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.945229:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.945230:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.945231:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721423] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.945233:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.945234:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.945235:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.945239:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.945240:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.945244:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.945244:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.945245:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.945247:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.945248:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.945248:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.945249:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677447.945266:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.945269:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.945270:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.945272:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677447.945272:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.945275:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.945276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.945279:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.945281:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.945284:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.945285:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.945287:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.945288:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.945290:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.945291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:20.0:1619677447.945292:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677447.945300:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677447.945301:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.945302:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677447.945303:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.945304:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677447.945306:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677447.945307:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.945309:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677447.945311:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.945312:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.945313:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:10.0:1619677447.945313:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677447.945314:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:10.0:1619677447.945314:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.945315:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.945316:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.945316:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:10.0:1619677447.945316:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.945317:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000400:00000200:10.0:1619677447.945317:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.945319:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.945319:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:10.0:1619677447.945321:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.945322:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.2:1619677447.945324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:10.0:1619677447.945326:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.945327:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.945327:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.945329:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.945329:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.945330:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677447.945330:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:10.0:1619677447.945331:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677447.945332:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.945333:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.945334:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677447.945341:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.945344:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.945344:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.945345:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:31.0:1619677447.945346:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.945348:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.945348:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.945349:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.945350:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:22.0:1619677447.945350:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.945352:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.945356:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.945358:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.945360:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.945360:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.945361:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.945362:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.945363:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.945472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:21.2:1619677447.945476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:30.0:1619677447.945479:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.945482:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.945484:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.945484:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:31.0:1619677447.945486:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.0:1619677447.945486:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:30.0:1619677447.945487:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.945487:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:30.0:1619677447.945488:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.945490:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.945490:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.945491:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.945492:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.945493:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:8.0:1619677447.945493:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.945495:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.945496:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.945497:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.945498:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.945499:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.2:1619677447.945499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:8.0:1619677447.945499:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.945499:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:5.2:1619677447.945500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677447.945502:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.945503:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:8.0:1619677447.945503:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.945504:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.945505:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:8.0:1619677447.945505:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677447.945506:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.945507:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.945507:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677447.945507:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677447.945508:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677447.945508:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.945508:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.945509:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.945510:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:26.0:1619677447.945511:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.945511:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.2:1619677447.945512:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:5.0:1619677447.945513:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.945514:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.945514:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.0:1619677447.945514:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.945515:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.945515:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677447.945516:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:8.0:1619677447.945516:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:5.0:1619677447.945516:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.945517:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.945517:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677447.945518:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:26.0:1619677447.945519:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677447.945519:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.945519:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96721418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.945520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:26.0:1619677447.945521:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96721418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677447.945521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677447.945521:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.945522:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.945522:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.945523:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.945523:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677447.945524:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.945525:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:20.0:1619677447.945527:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.945527:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.945528:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677447.945528:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677447.945529:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677447.945530:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:26.0:1619677447.945531:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.945531:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:18.0:1619677447.945532:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:8.0:1619677447.945532:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677447.945533:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:18.0:1619677447.945533:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.945533:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.945533:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:24.0:1619677447.945534:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.945534:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.945534:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677447.945535:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.945536:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.945536:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677447.945537:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677447.945537:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:24.0:1619677447.945537:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.945537:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.945538:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677447.945538:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677447.945539:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:24.0:1619677447.945539:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:5.0:1619677447.945539:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677447.945540:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.945540:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:24.0:1619677447.945540:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96721416] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.945541:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:24.0:1619677447.945542:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.945543:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.945544:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.945547:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.945549:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677447.945549:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677447.945552:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:24.0:1619677447.945552:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.945552:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:12.0:1619677447.945552:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:24.0:1619677447.945553:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:12.0:1619677447.945553:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.945554:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677447.945555:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:24.0:1619677447.945555:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.945555:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:12.0:1619677447.945555:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.945556:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.945556:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:31.0:1619677447.945557:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.945557:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.945557:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.945559:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.945560:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:8.2:1619677447.945560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.945569:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.945572:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.945572:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.945573:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.945575:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.945578:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.945582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.945589:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.945591:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.945592:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.945592:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.945593:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.945593:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677447.945608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.945613:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.945616:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.945617:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.945619:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.945622:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.945623:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.945625:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677447.945625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:6.0:1619677447.945626:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.945627:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.945628:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:1.0:1619677447.945631:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:6.0:1619677447.945632:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.945633:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.945634:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.945634:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:1.0:1619677447.945636:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.945637:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.945638:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.945638:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:1.0:1619677447.945638:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.945640:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000400:00000200:1.0:1619677447.945640:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:6.0:1619677447.945641:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:1.0:1619677447.945641:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.945642:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:1.0:1619677447.945642:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.945643:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:5.2:1619677447.945643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:1.0:1619677447.945643:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.945644:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677447.945644:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677447.945648:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677447.945649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677447.945649:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:1.0:1619677447.945650:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:2.0:1619677447.945651:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:10.0:1619677447.945652:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:1.0:1619677447.945652:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.945653:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677447.945653:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.945653:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677447.945654:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:2.0:1619677447.945654:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.945654:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677447.945655:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:1.0:1619677447.945655:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677447.945656:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:22.0:1619677447.945656:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:10.0:1619677447.945656:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.945656:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.945656:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.945657:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677447.945657:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.945658:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:2.0:1619677447.945659:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:10.0:1619677447.945660:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:8.2:1619677447.945660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:2.0:1619677447.945660:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96721413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.945662:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677447.945663:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:2.0:1619677447.945663:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.945664:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.945667:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:2.0:1619677447.945667:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.945669:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.945670:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.945671:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.945672:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.945672:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.945673:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.945673:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:17.0:1619677447.945674:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.945675:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:2.0:1619677447.945675:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.945676:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.945676:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.945677:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.945681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.945689:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.945691:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.945692:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.945692:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.945694:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.945696:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.945701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.945708:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.945710:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.945711:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.945712:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.945713:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.945713:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677447.945991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.945996:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.945999:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.946000:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.946002:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.946005:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.946006:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.946008:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.946009:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.946010:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.946011:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.946015:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.946017:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.946020:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.946021:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.946021:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.946023:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.946024:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.946024:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.946025:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677447.946037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.946044:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.946046:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:30.0:1619677447.946047:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:21.2:1619677447.946047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677447.946048:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.946049:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:30.0:1619677447.946050:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:8.0:1619677447.946050:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.946051:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.946051:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:31.0:1619677447.946053:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677447.946053:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.946053:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.946053:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.946055:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:8.0:1619677447.946055:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:21.2:1619677447.946056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:8.0:1619677447.946056:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721411] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.946058:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.946059:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.946060:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.946062:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.946062:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:24.0:1619677447.946063:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.946064:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.946065:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.946065:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.946065:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:31.0:1619677447.946067:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:24.0:1619677447.946067:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.946067:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.946068:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.946069:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677447.946070:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:24.0:1619677447.946070:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677447.946071:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.946071:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:18.0:1619677447.946071:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.946072:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.946073:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:18.0:1619677447.946073:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:8.0:1619677447.946074:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.946075:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.946075:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96721409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.946075:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:18.0:1619677447.946076:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.946076:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.946078:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.946078:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.946078:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:8.0:1619677447.946078:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.2:1619677447.946079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.0:1619677447.946080:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.946081:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.946082:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.946083:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.946083:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:18.0:1619677447.946083:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:8.2:1619677447.946084:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.946085:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.946085:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:24.0:1619677447.946086:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.946087:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:18.0:1619677447.946087:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.946088:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:18.0:1619677447.946088:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677447.946088:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:8.0:1619677447.946089:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:4.2:1619677447.946089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (23)++
00000800:00000200:24.0:1619677447.946090:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:18.0:1619677447.946090:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.946090:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677447.946091:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (24)++
00000800:00000200:18.0:1619677447.946091:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.2:1619677447.946091:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.946092:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (25)--
00000800:00000200:18.0:1619677447.946092:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.946092:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677447.946092:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:24.0:1619677447.946093:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (24)--
00000800:00000200:24.0:1619677447.946094:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677447.946095:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:8.0:1619677447.946095:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.946095:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.946097:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.946097:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.946098:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.946099:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:1.0:1619677447.946099:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.946100:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.946101:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96721408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.946102:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:1.0:1619677447.946102:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.946103:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:20.0:1619677447.946103:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.946103:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.946104:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:20.0:1619677447.946105:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.946106:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:5.0:1619677447.946106:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:22.0:1619677447.946107:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.946107:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.946110:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000400:00000200:1.0:1619677447.946110:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.946112:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677447.946112:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.946113:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.946114:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.946114:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.946115:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.946116:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677447.946116:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.946117:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.946119:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.946120:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.946121:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.946122:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.946122:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.946124:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.946125:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:9.2:1619677447.946125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:6.0:1619677447.946125:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.946126:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.946127:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.946132:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.946135:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677447.946136:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.946137:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.946139:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.946142:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.946145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:8.0:1619677447.946151:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.946153:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.946154:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.946154:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.946155:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.946156:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677447.946165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.946172:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.946175:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.946175:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.946177:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.946180:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.946181:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.946182:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.946184:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.946185:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.946186:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.946189:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.946191:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.946194:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.946194:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.946195:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.946197:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.946197:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.946198:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.946198:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677447.946221:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.946223:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.946224:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.946225:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.946227:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.946228:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.946396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.946402:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.946405:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.946406:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.946408:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.946410:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.946412:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.946413:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.946415:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.946416:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.946417:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.946420:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.946421:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.946424:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.946425:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.946425:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.946427:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.946428:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.946429:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.946430:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.946430:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677447.946436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677447.946441:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677447.946442:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677447.946444:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677447.946444:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:13.0:1619677447.946445:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.946445:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.946446:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677447.946447:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.946448:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.946450:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.946468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:20.2:1619677447.946471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:22.0:1619677447.946473:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.946476:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.946477:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.946479:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.946480:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:22.0:1619677447.946482:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.946483:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:22.0:1619677447.946483:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.946485:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721405] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.946486:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.946487:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:22.0:1619677447.946487:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.946487:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.946489:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.946489:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:31.0:1619677447.946492:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.946493:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.946493:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.946495:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.946496:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.946496:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:31.0:1619677447.946497:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.946499:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.946500:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.946500:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:31.0:1619677447.946501:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.946502:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:31.0:1619677447.946503:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.946503:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.946504:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.946504:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677447.946506:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.946507:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.946508:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.946510:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.946510:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.946511:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.946512:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:10.0:1619677447.946515:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:8.2:1619677447.946516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677447.946518:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677447.946519:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677447.946519:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.946521:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.946524:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.0:1619677447.946526:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.946528:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.946529:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.946531:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.946531:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.946534:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677447.946535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:4.2:1619677447.946543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.0:1619677447.946544:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.946548:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.946549:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.946549:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:24.0:1619677447.946550:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677447.946550:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.946552:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:24.0:1619677447.946553:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.946554:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.946556:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.946559:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.946560:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.946561:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.946563:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.946564:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.946565:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.946568:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.946570:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.946573:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.946574:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.946574:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.946576:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.946576:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.946577:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.946577:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.946582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677447.946583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677447.946589:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677447.946590:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.946592:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:2.0:1619677447.946592:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.946593:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:17.0:1619677447.946594:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.946594:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:2.0:1619677447.946595:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.946596:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.946598:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.946600:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:2.0:1619677447.946600:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.946601:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.946602:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.946604:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.946604:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.946608:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.946610:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.946613:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.946614:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.946615:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.946616:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.946617:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.946617:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.946618:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677447.946629:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.946632:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.946632:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.946634:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.946635:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.946637:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.946647:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.946649:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.946650:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.946651:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.946652:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.946653:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.946704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.946711:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.946714:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.946716:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.946718:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.946719:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.946719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677447.946720:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.946721:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.946723:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.946724:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.946725:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.946725:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677447.946726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:1.0:1619677447.946727:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.946728:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677447.946729:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.946730:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.946731:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.946732:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:1.0:1619677447.946733:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.946734:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.946734:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.946734:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:30.0:1619677447.946735:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.946735:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:1.0:1619677447.946735:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:30.0:1619677447.946736:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.946737:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677447.946737:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.946738:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.946738:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.946739:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:1.0:1619677447.946739:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.946740:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:1.0:1619677447.946740:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:30.0:1619677447.946741:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677447.946741:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.946742:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.946744:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96721399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.946745:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677447.946746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677447.946746:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.946747:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.946747:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:18.0:1619677447.946748:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:12.0:1619677447.946748:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.946750:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677447.946750:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677447.946751:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.946751:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.946751:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:18.0:1619677447.946752:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677447.946753:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.946753:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.946753:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677447.946754:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677447.946754:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.946754:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677447.946755:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:12.0:1619677447.946755:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.946755:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.946756:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.946757:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677447.946757:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.946758:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677447.946758:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.946759:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.946761:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.946761:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677447.946762:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677447.946762:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:6.0:1619677447.946766:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:4.2:1619677447.946768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677447.946769:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.946770:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.946772:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.946773:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.946774:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.946775:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.946775:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.946776:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.946776:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:22.0:1619677447.946777:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.946777:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.946778:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:22.0:1619677447.946779:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677447.946779:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.2:1619677447.946779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:26.0:1619677447.946781:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.946783:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.946784:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:26.0:1619677447.946785:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677447.946786:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.946787:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:26.0:1619677447.946788:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.946788:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:26.0:1619677447.946790:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.946791:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.946791:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.946794:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.946794:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.946796:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.946796:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.946797:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.946798:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.946799:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.946801:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.946801:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:2.0:1619677447.946801:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677447.946802:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677447.946804:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:2.0:1619677447.946804:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.946805:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.946805:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.946806:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677447.946806:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.946809:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.946809:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.2:1619677447.946810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.946810:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.946812:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.946812:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.946813:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.946813:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.946817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.946818:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.946820:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.946821:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.946823:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.946824:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677447.946825:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677447.946826:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.946827:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.946828:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.946829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:1.0:1619677447.946829:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.946830:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.946832:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677447.946836:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.946839:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.946840:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.946841:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.946842:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.946843:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677447.946971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.946977:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.946980:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.946981:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.946983:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.946985:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.946986:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.946988:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.946989:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.946990:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.946991:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.946994:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.946996:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.946999:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.947000:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.947001:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.947003:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.947003:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.947003:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.947004:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.947009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.947015:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.947018:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.947019:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.947019:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.947022:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.947025:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677447.947058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:6.0:1619677447.947063:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.947066:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677447.947067:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677447.947069:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.947072:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.947073:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:6.0:1619677447.947074:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.947076:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.947077:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.947078:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:6.0:1619677447.947081:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.947083:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677447.947087:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.947087:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:6.0:1619677447.947088:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:6.0:1619677447.947090:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:6.0:1619677447.947091:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.947091:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677447.947092:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:31.0:1619677447.947104:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.947106:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.947107:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.947109:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.947110:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.947112:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677447.947177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.947184:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.947187:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.947187:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677447.947190:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.947192:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.947193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:10.0:1619677447.947194:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677447.947195:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.947196:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.947197:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.947198:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.947200:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677447.947201:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.947202:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:10.0:1619677447.947203:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.947203:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.947205:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.947206:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.947207:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.947207:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677447.947208:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:24.0:1619677447.947209:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.947210:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.947210:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:24.0:1619677447.947211:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677447.947211:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:24.0:1619677447.947212:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677447.947212:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:24.0:1619677447.947213:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677447.947213:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.947214:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677447.947216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:24.0:1619677447.947217:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.947218:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:4.2:1619677447.947219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677447.947222:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.947222:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.947223:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.947225:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.947225:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.0:1619677447.947225:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677447.947226:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:24.0:1619677447.947226:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.947226:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.0:1619677447.947227:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677447.947228:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:5.0:1619677447.947228:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.947229:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.947230:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677447.947230:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.2:1619677447.947230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677447.947230:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.947233:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.947239:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.947241:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.947242:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.947243:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.947245:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.947248:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677447.947263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677447.947270:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.947273:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.947274:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.947276:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.947279:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.947280:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.947281:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.947282:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.947283:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.947284:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.947287:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.947288:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.947291:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.947292:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.947293:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.947299:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.947300:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.947301:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.947302:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.947303:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.2:1619677447.947305:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677447.947309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677447.947313:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.947315:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.947316:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.947317:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:20.0:1619677447.947318:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.947319:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.947319:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677447.947320:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.947321:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677447.947321:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677447.947341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.947347:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.947350:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.947351:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.947353:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.947356:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.947357:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.947359:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.947360:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.947361:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.947362:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.947366:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.947367:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.947370:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.947371:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.947372:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.947373:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.947374:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.947374:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.947375:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.947388:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.947391:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.947392:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.947393:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.947395:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.947396:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.947461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.947469:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.947471:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.947472:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.947474:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.947477:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.947479:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677447.947480:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.947481:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.947482:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.947483:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677447.947486:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.947487:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:21.2:1619677447.947490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677447.947490:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.947490:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677447.947491:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677447.947493:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.947493:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.947494:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.947494:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677447.947497:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.947500:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.947502:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.947504:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.947504:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677447.947505:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:31.0:1619677447.947506:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677447.947507:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.947508:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.947508:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.947508:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.947509:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.947510:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:13.0:1619677447.947510:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.947512:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.947513:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677447.947514:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.947515:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677447.947519:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.947520:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677447.947520:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677447.947522:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.947523:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.947523:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.947524:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677447.947541:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.947544:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.947545:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.947546:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.947547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1200] (20)++
00000400:00000200:22.0:1619677447.947548:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.947552:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677447.947553:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677447.947556:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:1.0:1619677447.947557:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:17.0:1619677447.947559:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.947562:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.947563:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:17.0:1619677447.947565:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.947566:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.947567:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.947568:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:17.0:1619677447.947572:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.947573:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 984 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677447.947576:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677447.947576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677447.947577:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:17.0:1619677447.947578:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:17.0:1619677447.947579:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:17.0:1619677447.947580:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677447.947580:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:17.0:1619677447.947581:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:26.0:1619677447.947583:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677447.947585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:26.0:1619677447.947586:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.947587:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.947589:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.947591:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.947593:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.947594:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.947594:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:26.0:1619677447.947595:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.947597:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.947597:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.947597:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.947598:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.947600:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677447.947600:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:26.0:1619677447.947602:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.947603:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:26.0:1619677447.947605:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.947608:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.947608:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.947609:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677447.947611:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.947611:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.947612:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.947613:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677447.947635:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.947638:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.947638:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.947639:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.947641:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.947644:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.947756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:5.2:1619677447.947759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.947763:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677447.947765:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.947765:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677447.947766:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.947768:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.947769:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.947771:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.947772:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.947773:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677447.947774:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.947775:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.947777:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.947778:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677447.947780:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.947781:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.947782:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.947782:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.947783:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.947784:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.947784:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677447.947787:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677447.947788:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.947788:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:1.0:1619677447.947789:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677447.947790:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.947791:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677447.947792:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677447.947793:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677447.947794:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.947794:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677447.947795:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.947795:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:17.0:1619677447.947796:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.2:1619677447.947797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677447.947797:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.947798:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.947799:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.947800:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.947801:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.947802:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677447.947805:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.947807:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.2:1619677447.947807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:20.0:1619677447.947808:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.947810:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:20.0:1619677447.947810:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.947812:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677447.947812:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:30.0:1619677447.947813:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.947814:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.947814:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.947814:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.947815:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677447.947816:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.947816:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.947817:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.947820:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.947932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.947940:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.947943:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.947945:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.947947:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.947948:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.947950:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677447.947951:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.947952:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.947953:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.947954:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677447.947958:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.947960:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.947963:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.947964:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677447.947964:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.947966:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677447.947966:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.947967:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.947968:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677447.947972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.947980:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.947982:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.947983:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.947983:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.947986:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.947988:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.947996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.948003:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.948006:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.948006:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.948007:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.948008:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.948008:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.948016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.948023:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.948026:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.948028:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.948030:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.948031:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.948032:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.948034:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.948035:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.948036:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.948037:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.948040:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.948042:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.948044:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.948045:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.948045:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.948046:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.948047:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.948049:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.948050:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.948051:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677447.948056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677447.948058:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.948061:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.948062:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.948063:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.948064:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:13.0:1619677447.948065:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.948066:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:12.0:1619677447.948066:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677447.948067:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.948068:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:4.2:1619677447.948109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.948116:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.948118:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.948119:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.948121:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.948124:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.948126:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677447.948127:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.948128:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.948129:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.948130:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677447.948133:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.948135:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.948138:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.948138:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677447.948139:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.948141:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.948141:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.948142:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.948143:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677447.948147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.948155:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.948158:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.948159:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.948159:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.948161:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.948163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.948166:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.948168:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.948170:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.948171:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.948171:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.948172:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.948172:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677447.948336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.948344:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.948347:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.948349:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.948351:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.948352:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.948353:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677447.948354:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.948356:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.948357:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.948357:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:21.2:1619677447.948357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:31.0:1619677447.948360:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.948362:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.948364:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677447.948365:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.948366:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677447.948366:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.948366:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.948367:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:31.0:1619677447.948368:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677447.948368:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.948369:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:2.0:1619677447.948369:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.948370:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:2.0:1619677447.948372:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.948373:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:8.2:1619677447.948374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677447.948374:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.948376:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.948377:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.948378:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.948381:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.948383:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677447.948383:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.2:1619677447.948385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:17.0:1619677447.948386:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:2.0:1619677447.948386:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.948386:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:1.0:1619677447.948386:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.948387:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.948388:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:17.0:1619677447.948389:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.948390:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.948390:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.948392:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677447.948392:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.948392:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.948393:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.948393:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.948394:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.948395:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.948396:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.948398:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677447.948399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:18.0:1619677447.948400:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.948402:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.948403:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.948404:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.948405:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:18.0:1619677447.948405:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.948405:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.948406:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.948406:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677447.948408:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677447.948408:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:22.0:1619677447.948409:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.948409:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:18.0:1619677447.948410:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.948411:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.948412:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:22.0:1619677447.948413:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677447.948415:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.948416:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.948416:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.948418:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.948419:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.948420:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.948420:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:6.0:1619677447.948425:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.948427:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.948428:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.948428:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.948429:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.948430:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:26.0:1619677447.948431:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.948434:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.948435:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.948435:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677447.948437:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.948440:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677447.948606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.948614:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.948617:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.948619:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.948620:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.948621:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.948623:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.948624:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.948625:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.948626:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.948627:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.948630:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.948632:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.948634:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.948635:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.948636:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.948637:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.948638:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.948639:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.948639:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677447.948650:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.948653:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.948653:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.948655:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.948657:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.948658:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.948668:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.948671:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.948671:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.948672:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.948672:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.948673:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677447.948676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:8.0:1619677447.948683:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677447.948686:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677447.948687:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677447.948689:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.948692:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.948693:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:8.0:1619677447.948694:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.948696:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.948697:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.948697:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:8.0:1619677447.948701:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.948702:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.0:1619677447.948704:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.948705:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:8.0:1619677447.948706:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:8.0:1619677447.948707:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:8.0:1619677447.948708:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677447.948709:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677447.948709:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:10.0:1619677447.948720:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.948723:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.948723:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.948724:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.948725:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.948727:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677447.948752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.948758:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.948760:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.948763:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.948763:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.948766:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.948767:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.948769:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.948770:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.948771:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.948772:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.948775:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.948777:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.948780:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.948780:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.948781:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.948783:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.948784:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.948785:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.948786:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677447.948790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.948797:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:21.2:1619677447.948799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677447.948800:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.948801:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.948801:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.948803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:2.0:1619677447.948803:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.948806:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677447.948806:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.948808:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677447.948809:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677447.948811:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:31.0:1619677447.948812:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.948812:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677447.948813:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.948813:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677447.948814:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.948815:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:31.0:1619677447.948816:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.948816:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.948818:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.948818:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677447.948820:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.948820:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.948821:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:31.0:1619677447.948822:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.948822:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.948824:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.948825:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.948825:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.948825:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.948825:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.948829:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.948831:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.948832:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.948832:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.948833:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:20.0:1619677447.948834:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677447.948835:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.948835:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96721373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.948836:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.948837:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.948837:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.948838:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.948838:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677447.948839:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.948839:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:18.0:1619677447.948839:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677447.948840:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.948841:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.948842:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.948842:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677447.948842:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:18.0:1619677447.948842:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.948843:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677447.948844:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677447.948844:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:18.0:1619677447.948844:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.948845:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677447.948846:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.948846:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677447.948846:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.948847:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.948848:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.948849:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.948849:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.2:1619677447.948850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.948850:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.948851:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:21.2:1619677447.948852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677447.948855:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:9.2:1619677447.948855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.948857:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:12.0:1619677447.948857:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.948857:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:22.0:1619677447.948858:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.948859:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.948859:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:12.0:1619677447.948860:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677447.948860:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677447.948861:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:22.0:1619677447.948861:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:12.0:1619677447.948861:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.948861:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677447.948862:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:30.0:1619677447.948863:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:22.0:1619677447.948863:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:12.0:1619677447.948863:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.948864:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677447.948864:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.948865:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.948865:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.948865:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.948866:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.948867:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.948868:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.948870:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.948975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:5.2:1619677447.948978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.948984:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.948986:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677447.948987:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:26.0:1619677447.948988:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.948988:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677447.948991:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.948991:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.948993:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677447.948995:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.948996:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.948997:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.948998:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677447.948999:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.949001:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677447.949002:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.949003:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677447.949004:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677447.949005:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:10.0:1619677447.949005:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.949006:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677447.949009:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.949011:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.949013:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.949013:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.949014:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677447.949014:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.949015:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677447.949015:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677447.949016:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.949017:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677447.949017:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677447.949018:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677447.949019:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677447.949019:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (23)++
00000800:00000200:26.0:1619677447.949020:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677447.949020:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (24)--
00000800:00000200:26.0:1619677447.949021:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:10.0:1619677447.949021:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (23)--
00000800:00000200:8.2:1619677447.949023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677447.949025:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.949026:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:5.2:1619677447.949027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.949028:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.949028:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.949030:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.949030:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.949031:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677447.949032:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677447.949034:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:5.0:1619677447.949034:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:10.0:1619677447.949035:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.949036:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.0:1619677447.949036:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.949037:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.949039:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.949039:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.949042:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:26.0:1619677447.949046:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.949049:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.949050:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.949050:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.949051:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.949051:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677447.949123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.949129:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.949131:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.949132:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.949134:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.949137:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.949139:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.949140:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.949141:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.949142:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.949143:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.949146:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.949148:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.949151:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.949152:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.949153:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.949155:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.949155:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.949156:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.949156:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.949167:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.949170:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.949171:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.949172:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.949174:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.949175:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.949257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.949258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677447.949264:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677447.949266:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677447.949267:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.949268:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677447.949269:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:10.0:1619677447.949270:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.949272:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.949273:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:10.0:1619677447.949273:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.949274:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.949275:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677447.949276:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677447.949277:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.949277:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.949279:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.949279:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.949281:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.949281:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.949283:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677447.949283:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677447.949289:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677447.949289:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.949292:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677447.949292:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.949301:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.949302:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.949302:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677447.949302:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.949303:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.949303:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677447.949304:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.949305:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.949305:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677447.949305:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.949306:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:10.0:1619677447.949306:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.949306:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.949307:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677447.949311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.949319:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:6.0:1619677447.949320:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.949321:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.949322:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.949323:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.949323:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:20.0:1619677447.949324:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.949324:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.949325:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:6.0:1619677447.949326:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.949327:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:6.0:1619677447.949329:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.2:1619677447.949331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.949338:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.949341:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.949342:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.949343:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.949343:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.949344:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677447.949405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.2:1619677447.949407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:26.0:1619677447.949412:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677447.949412:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.949414:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:26.0:1619677447.949415:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.949416:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677447.949417:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.949417:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.949419:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.949419:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.949421:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.949422:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.949423:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677447.949423:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.949425:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.949425:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.949426:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.949427:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.949427:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677447.949428:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.949429:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677447.949432:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.949434:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.949434:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.949436:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677447.949437:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.949438:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.949438:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677447.949439:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.949440:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677447.949440:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677447.949440:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.949441:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677447.949442:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677447.949442:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677447.949442:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.949442:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677447.949443:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.949444:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677447.949454:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.949456:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677447.949457:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:22.0:1619677447.949457:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.949457:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.949460:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.949460:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.949460:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.949461:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:31.0:1619677447.949462:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.949464:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.949465:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.949472:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.949475:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.949476:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.949477:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677447.949477:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.949478:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677447.949668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.949675:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.949678:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.949679:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.949681:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.949683:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.949684:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.949685:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.949687:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.949688:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.949689:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.949692:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.949694:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.949697:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.949698:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.949698:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.949700:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.949700:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.949701:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.949701:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.949706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.949714:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.949717:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.949718:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.949718:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.949720:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.949723:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677447.949777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.949783:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.949786:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.949787:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.949789:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.949791:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.949793:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.949794:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.949795:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.949796:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.949797:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.949801:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.949803:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.949805:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.949806:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.949807:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.949808:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.949809:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.949810:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.949811:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.949811:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677447.949814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677447.949815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.2:1619677447.949819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677447.949822:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:10.0:1619677447.949823:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677447.949824:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.949825:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.949825:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:10.0:1619677447.949826:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.949827:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:20.0:1619677447.949827:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.949827:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:5.0:1619677447.949828:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.949828:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:10.0:1619677447.949829:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.949829:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677447.949830:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:10.0:1619677447.949831:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.949833:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677447.949834:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.949835:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.949836:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.949837:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677447.949841:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.949842:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677447.949845:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.949845:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677447.949846:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677447.949848:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677447.949848:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.949849:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.949850:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677447.949860:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.949862:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.949863:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.949864:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.949866:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.949867:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677447.949883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.2:1619677447.949888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:30.0:1619677447.949890:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.2:1619677447.949891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:30.0:1619677447.949893:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:6.0:1619677447.949894:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:30.0:1619677447.949895:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.949896:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:5.0:1619677447.949896:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677447.949897:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.949897:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.949897:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.0:1619677447.949899:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:30.0:1619677447.949900:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.949900:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.949902:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.949902:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.949902:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.949903:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.949904:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.949904:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.949905:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.949906:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:20.2:1619677447.949906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:6.0:1619677447.949906:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677447.949907:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.949907:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677447.949908:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:5.0:1619677447.949909:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.949910:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:6.0:1619677447.949910:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.949911:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.949912:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.949912:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.949913:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677447.949914:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.949914:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.949917:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.949918:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:20.0:1619677447.949920:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.949920:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.949921:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.949922:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677447.949922:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:20.0:1619677447.949922:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.949922:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.949923:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96721359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.949923:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.949924:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677447.949924:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677447.949925:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.949925:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677447.949925:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.949926:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:20.0:1619677447.949926:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.949927:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.949928:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.949928:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677447.949929:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (22)++
00000800:00000200:9.2:1619677447.949930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (23)++
00000800:00000200:5.0:1619677447.949930:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.949931:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.949931:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (23)--
00000800:00000200:5.0:1619677447.949931:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:6.0:1619677447.949932:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677447.949932:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677447.949933:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.949933:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.949934:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.949934:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.949935:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.949935:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.949936:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.949936:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677447.949937:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.949937:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.949938:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.949938:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (23)++
00000800:00000200:20.0:1619677447.949939:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.949940:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:18.0:1619677447.949940:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (24)--
00000800:00000200:20.0:1619677447.949941:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000400:00000200:18.0:1619677447.949941:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.949941:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (24)++
00000800:00000200:20.0:1619677447.949942:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:2.0:1619677447.949942:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (24)--
00000400:00000200:2.0:1619677447.949942:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.949944:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.949944:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.2:1619677447.949945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677447.949947:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.949947:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.949948:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.949949:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:12.0:1619677447.949950:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.949951:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.949953:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.949953:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:13.0:1619677447.949954:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.949956:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.949956:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.949957:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.949957:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.949958:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.949961:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.949963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.949966:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.949969:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677447.949970:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677447.949970:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.949971:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.949972:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677447.949973:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:13.0:1619677447.949973:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677447.949973:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.949976:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.949978:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.949980:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.949981:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.949982:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.949984:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.949985:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677447.949988:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.949990:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.949993:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.949994:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.949995:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677447.949996:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.949997:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.949998:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.949999:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.950000:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677447.950006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.950010:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.950013:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677447.950013:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:22.0:1619677447.950014:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.950014:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.950015:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.950015:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677447.950016:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.950017:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677447.950019:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.950027:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.950030:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.950030:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.950031:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.950032:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.950032:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.950043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.950049:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.950051:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.950052:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.950054:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.950057:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.950058:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.950060:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.950061:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.950062:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.950063:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.950067:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.950068:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.950071:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.950072:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.950072:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.950074:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.950074:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.950075:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.950075:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677447.950082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:22.0:1619677447.950087:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.0:1619677447.950088:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:22.0:1619677447.950090:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:5.0:1619677447.950090:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:22.0:1619677447.950091:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.950091:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:5.0:1619677447.950092:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.950093:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:22.0:1619677447.950094:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.950095:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.950096:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:5.0:1619677447.950096:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:5.0:1619677447.950097:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.950099:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.950100:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.950101:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:5.0:1619677447.950104:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.950106:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:5.0:1619677447.950109:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.950110:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:5.0:1619677447.950110:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:5.0:1619677447.950112:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:5.0:1619677447.950113:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:5.0:1619677447.950114:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:5.0:1619677447.950114:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:26.0:1619677447.950124:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677447.950127:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677447.950127:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677447.950128:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.950130:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.950133:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677447.950266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:21.2:1619677447.950273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:31.0:1619677447.950274:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.950277:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.950279:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.950280:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:30.0:1619677447.950281:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.950282:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.950283:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:31.0:1619677447.950284:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:24.0:1619677447.950284:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:31.0:1619677447.950285:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.950286:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.950286:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.950287:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.950289:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677447.950289:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.950291:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677447.950293:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.950298:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.950299:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.950299:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.950300:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.950302:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.950304:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677447.950304:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.950305:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.950305:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:8.0:1619677447.950305:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.950307:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.950307:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.950308:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:8.0:1619677447.950308:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.950308:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.950309:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.0:1619677447.950309:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.950311:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.950311:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.950312:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.950313:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:30.0:1619677447.950321:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.950323:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.950324:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.950324:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:30.0:1619677447.950325:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.950327:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:12.0:1619677447.950327:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:30.0:1619677447.950328:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:12.0:1619677447.950328:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677447.950329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:12.0:1619677447.950329:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.950331:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.950332:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677447.950335:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.950338:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.950338:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.950340:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.950343:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.950344:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.950345:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.950347:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.950348:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.950349:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.950353:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.950354:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.950356:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.950357:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.950358:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.950359:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.950360:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.950361:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.950362:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.950362:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677447.950366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677447.950367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.2:1619677447.950371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677447.950372:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.950374:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:20.0:1619677447.950374:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:22.0:1619677447.950375:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.950376:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.950376:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.950377:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.950378:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.950379:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677447.950380:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677447.950381:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:10.0:1619677447.950382:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.950383:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.950385:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.950388:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.950389:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677447.950391:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.950392:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.950393:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.950394:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677447.950398:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.950400:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.950403:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.950403:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677447.950404:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.950406:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.950406:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.950407:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.950407:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677447.950412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.950420:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.950423:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.950424:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.950424:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.950426:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.950430:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.950504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.950512:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.950514:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.950515:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.950517:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.950520:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.950521:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677447.950522:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.950523:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.950524:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.950525:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677447.950527:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.950529:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.950530:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.950531:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.950531:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.950533:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.950533:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.950534:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.950534:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677447.950545:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.950547:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.950547:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.950548:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.950549:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.950550:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677447.950642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.950648:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.950651:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.950652:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.950654:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.950656:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.950658:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.950659:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.950660:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.950661:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.950662:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.950665:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.950667:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.950669:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.950670:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:21.2:1619677447.950671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:1.0:1619677447.950671:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.950672:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.950673:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.950673:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.950674:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.950676:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.950679:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.950681:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.950682:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.950683:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.950684:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.950685:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.950685:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.950687:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.950687:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.950688:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.950690:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.950691:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.950692:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.950695:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.950696:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:6.0:1619677447.950696:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.950698:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.950701:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.950702:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.950703:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.950705:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.950706:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.950706:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.950707:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:26.0:1619677447.950717:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677447.950720:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677447.950721:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.950721:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677447.950723:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.950726:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:10.0:1619677447.950735:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677447.950737:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677447.950738:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677447.950738:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677447.950740:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677447.950740:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677447.950761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:22.0:1619677447.950766:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.950769:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677447.950770:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677447.950772:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.950775:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.950776:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:22.0:1619677447.950777:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.950778:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.950780:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.950780:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677447.950783:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.950785:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.950786:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.950787:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677447.950787:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.950789:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.950789:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677447.950790:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677447.950791:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:6.0:1619677447.950801:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.950803:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.950804:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.950805:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.950806:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.950807:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.950975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.950983:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.950985:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.950986:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.950988:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.950991:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.950992:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.950993:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.950994:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.950995:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.950996:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.950999:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.951001:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.951004:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.951005:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.951005:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.951007:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.951007:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.951009:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.951010:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.951010:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.951013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.951016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.951021:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.951023:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677447.951023:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.951024:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.951025:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.951025:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.951026:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677447.951027:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.951027:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.951030:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677447.951043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.951049:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.951051:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.951052:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.951054:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.951056:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.951058:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.951059:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.951060:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.951062:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.951062:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.951066:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.951067:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.951070:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.951071:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.951071:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.951073:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.951073:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.951074:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.951074:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.951079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.951087:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.951090:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.951091:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.951091:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.951093:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.951096:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.951097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.2:1619677447.951102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:17.0:1619677447.951104:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.951107:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.951107:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677447.951108:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.951108:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.951109:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.951109:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.951110:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.951111:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.951113:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.951116:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.951117:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.951119:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.951120:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.951121:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.951122:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.951125:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.951127:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:21.2:1619677447.951130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677447.951130:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.951131:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.951131:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677447.951134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677447.951134:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.951135:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.951136:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.951136:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.951136:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677447.951137:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.951137:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.951139:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:1.0:1619677447.951140:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677447.951141:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.951142:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:4.2:1619677447.951142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:18.0:1619677447.951143:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.951143:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.951143:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:18.0:1619677447.951145:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.951146:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.951146:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.951147:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:13.0:1619677447.951147:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677447.951149:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677447.951149:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:1.0:1619677447.951149:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.951150:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.951150:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.951151:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677447.951152:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:1.0:1619677447.951152:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96721343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.951153:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:13.0:1619677447.951153:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.951153:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.951153:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677447.951154:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.951155:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677447.951155:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.951156:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.951156:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.951156:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677447.951156:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677447.951157:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677447.951158:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.951158:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.951158:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.951159:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:26.0:1619677447.951160:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:18.0:1619677447.951160:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:1.0:1619677447.951160:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.951162:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96721342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.951163:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:26.0:1619677447.951164:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677447.951164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677447.951165:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.951165:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.951166:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:26.0:1619677447.951167:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.951167:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.951168:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.951169:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677447.951170:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.951170:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:2.0:1619677447.951171:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677447.951171:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:26.0:1619677447.951172:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.951172:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677447.951173:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677447.951174:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677447.951174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677447.951174:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.951175:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:18.0:1619677447.951175:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.951176:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:2.0:1619677447.951176:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:5.2:1619677447.951177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677447.951178:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.951178:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.951179:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.951179:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677447.951179:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.951180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:17.0:1619677447.951183:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.951183:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.951184:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.951184:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.951184:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.951185:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.951186:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.951186:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.951187:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.951187:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677447.951188:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677447.951190:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677447.951191:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:18.0:1619677447.951191:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.951191:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.951192:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.951193:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677447.951194:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.951195:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.951197:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.951198:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.951199:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.951208:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.951211:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.951212:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.951212:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.951213:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.951214:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.951457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.951465:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.951468:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.951470:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.951472:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.951472:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.951474:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.951475:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.951476:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.951477:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.951478:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.951482:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.951483:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.951486:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.951487:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.951488:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.951490:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.951490:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.951492:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.951493:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.951494:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677447.951497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677447.951500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:4.2:1619677447.951503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.951506:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.951508:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677447.951509:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:5.0:1619677447.951509:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.951511:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:10.0:1619677447.951511:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677447.951511:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.951512:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677447.951512:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.951513:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:10.0:1619677447.951514:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:5.0:1619677447.951514:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:26.0:1619677447.951515:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677447.951515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000400:00000200:10.0:1619677447.951516:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.951519:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.951520:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677447.951522:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.951523:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:10.0:1619677447.951523:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.951524:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.951525:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.951526:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.951527:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.951528:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.951529:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:10.0:1619677447.951529:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.951529:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:10.0:1619677447.951530:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677447.951533:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.951534:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.951534:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677447.951536:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.951536:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.951537:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.951537:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677447.951549:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.951552:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:21.2:1619677447.951552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:30.0:1619677447.951553:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.951555:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:21.2:1619677447.951555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:30.0:1619677447.951557:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.951558:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.951559:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677447.951561:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677447.951562:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.951564:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.951564:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.951565:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.2:1619677447.951565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677447.951566:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:6.0:1619677447.951567:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.951568:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.951569:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:6.0:1619677447.951570:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.951571:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.951571:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:1.0:1619677447.951571:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677447.951573:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.951573:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.951574:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.951574:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.951575:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677447.951575:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.951577:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:6.0:1619677447.951577:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.951577:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.951578:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677447.951579:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.951580:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677447.951582:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.951582:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.951583:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.951584:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:1.0:1619677447.951585:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.951586:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:30.0:1619677447.951588:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677447.951588:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.951589:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677447.951589:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:1.0:1619677447.951590:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.951591:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.951591:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677447.951591:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:30.0:1619677447.951592:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.951593:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677447.951593:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.951595:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.951596:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:1.0:1619677447.951596:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.951597:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:1.0:1619677447.951597:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.951598:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.951599:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:1.0:1619677447.951599:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.951600:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677447.951600:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677447.951601:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677447.951601:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677447.951602:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:6.0:1619677447.951602:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677447.951602:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677447.951604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:26.0:1619677447.951605:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.951606:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.951607:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:10.0:1619677447.951607:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677447.951608:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.951608:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:20.0:1619677447.951609:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:10.0:1619677447.951610:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677447.951611:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:20.0:1619677447.951611:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677447.951611:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.951612:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677447.951613:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.951613:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.951614:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.951615:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:20.0:1619677447.951616:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:10.0:1619677447.951616:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:22.0:1619677447.951617:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:20.0:1619677447.951617:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.951618:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.951618:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.951619:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.951619:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.951621:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.951621:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.951625:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:22.0:1619677447.951626:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.951627:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.0:1619677447.951627:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.951630:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677447.951630:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.951631:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:20.0:1619677447.951631:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.951631:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.951631:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.951632:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.951632:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677447.951633:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.951634:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.951635:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.951636:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677447.951646:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.951649:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.951650:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.951652:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:30.0:1619677447.951652:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.951655:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.951656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677447.951663:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.951666:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.951667:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.951669:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.951672:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.951673:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.951675:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.951676:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.951677:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.951678:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.951681:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.951683:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.951686:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.951687:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.951688:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.951690:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.951690:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.951691:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.951691:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.951696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.951704:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.951706:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.951708:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.951708:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.951710:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.951713:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.951809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.951816:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.951819:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.951820:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.951821:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.951824:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.951826:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.951827:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.951828:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.951829:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.951830:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.951833:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.951834:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.951837:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.951837:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.951838:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.951839:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.951840:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.951841:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.951841:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.951846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.951853:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.951856:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.951857:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.951857:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.951859:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.951862:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.951907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.951914:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.951917:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.951918:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.951919:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.951921:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.951923:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.951924:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677447.951924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677447.951925:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.951926:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.951927:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.951930:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.951930:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677447.951932:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.951932:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.951933:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677447.951934:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.951935:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:1.0:1619677447.951935:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.951936:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.951937:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.951938:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677447.951938:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.951939:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:1.0:1619677447.951939:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:26.0:1619677447.951940:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000400:00000200:1.0:1619677447.951940:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.951941:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677447.951942:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.951943:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.951944:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:4.2:1619677447.951946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677447.951947:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.951949:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.951950:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.951952:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:1.0:1619677447.951952:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677447.951953:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.951953:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:1.0:1619677447.951953:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.951954:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677447.951955:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.951955:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:1.0:1619677447.951955:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.951956:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.951956:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677447.951956:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677447.951957:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677447.951957:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.951957:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.951958:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:12.0:1619677447.951969:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.951971:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.951972:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.951974:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.951976:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.951977:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.952150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.952158:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.952161:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.952163:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677447.952164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:31.0:1619677447.952165:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.952165:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.952166:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.952168:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.952169:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.952170:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.952170:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677447.952171:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.952172:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.952175:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.952175:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.952176:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.952179:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.952179:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.952180:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677447.952182:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677447.952182:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.952183:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.952183:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677447.952183:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.952184:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.952185:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.952186:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.952186:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.2:1619677447.952186:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677447.952187:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:5.0:1619677447.952189:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.952191:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677447.952192:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.952194:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:5.0:1619677447.952194:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.952195:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.952197:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.952197:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.952197:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.952198:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:5.0:1619677447.952199:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (24)++
00000400:00000200:17.0:1619677447.952200:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.952202:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:5.0:1619677447.952202:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (25)--
00000400:00000200:17.0:1619677447.952203:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96721329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.952203:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (25)--
00000800:00000200:12.0:1619677447.952203:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:5.0:1619677447.952203:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:17.0:1619677447.952204:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677447.952204:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.952204:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:17.0:1619677447.952205:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.952205:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677447.952207:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:13.0:1619677447.952207:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.952210:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.952210:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.952211:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677447.952212:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:13.0:1619677447.952213:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677447.952214:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677447.952214:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.952214:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.952215:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.952216:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:13.0:1619677447.952216:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.952218:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.952218:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677447.952218:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.952219:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.952219:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.952231:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.952234:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.952235:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.952235:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.952236:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:6.0:1619677447.952237:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.952239:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.952240:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.952240:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.952241:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.952242:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.952243:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.952293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.952301:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.952303:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.952304:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.952306:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.952309:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.952310:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.952311:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.952312:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.952314:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.952314:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.952318:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.952320:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.952322:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.952323:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.952324:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.952325:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.952326:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.952326:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.952327:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677447.952339:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.952341:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.952342:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.952343:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.952345:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.952346:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677447.952349:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.952350:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.952350:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.952351:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.952352:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.952379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.952387:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.952389:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.952391:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.952393:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.952393:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.952395:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.952396:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.952397:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.952398:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.952399:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.952402:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.952404:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.952406:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.952406:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:4.2:1619677447.952406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677447.952407:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.952409:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.952409:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.952410:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.952410:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:24.0:1619677447.952414:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.952416:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.952417:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.952419:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.952421:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:24.0:1619677447.952422:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.952423:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.952424:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721326] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.952424:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:24.0:1619677447.952425:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677447.952425:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.952426:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677447.952426:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.952427:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677447.952428:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.952429:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:24.0:1619677447.952431:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.952432:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.952435:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.952436:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.952436:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.952438:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.952438:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.952439:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.952439:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.952444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.952452:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.952455:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.952456:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.952456:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.952458:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.952461:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.952485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.952490:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.952493:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.952494:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.952496:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.952498:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.952499:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.952500:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.952502:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.952503:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.952504:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.952507:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.952509:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.952512:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.952513:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.952514:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.952516:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.952516:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.952517:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.952518:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.952519:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677447.952524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677447.952529:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677447.952529:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677447.952531:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677447.952531:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:13.0:1619677447.952532:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.952532:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.952533:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677447.952534:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.952535:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.952537:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.952566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677447.952572:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.952576:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.952576:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.952578:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.952580:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.952582:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.952583:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.952585:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.952585:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.952587:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.952590:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.952592:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.952595:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.952596:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.952597:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.952599:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.952599:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.952600:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.952600:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677447.952611:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.952614:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.952615:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.952616:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.952618:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.952619:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.952687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.952695:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.952698:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.952698:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677447.952701:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.952703:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.952705:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677447.952706:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.952707:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.952708:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.952709:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677447.952713:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.952714:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677447.952717:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.952718:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677447.952719:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677447.952721:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677447.952721:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.952722:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.952722:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677447.952727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.952735:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.952737:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.952738:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.952740:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.952742:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.952743:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677447.952803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.952809:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.952812:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.952813:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.952815:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677447.952818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:2.0:1619677447.952818:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.952819:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:5.2:1619677447.952820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:2.0:1619677447.952821:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.952822:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.952823:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.952824:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:26.0:1619677447.952826:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:2.0:1619677447.952827:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.952828:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:2.0:1619677447.952828:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.952829:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.952831:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:2.0:1619677447.952831:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.952831:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:26.0:1619677447.952832:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.952832:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677447.952833:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.952834:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.952835:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.952835:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:26.0:1619677447.952836:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.952836:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.952836:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:26.0:1619677447.952838:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677447.952839:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677447.952840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677447.952841:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.952842:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.952843:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677447.952845:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677447.952846:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96721320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.952847:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677447.952847:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.952848:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:10.0:1619677447.952848:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.952849:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:10.0:1619677447.952849:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.952850:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.952851:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.952852:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:10.0:1619677447.952853:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.952854:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.952854:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.952855:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677447.952855:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677447.952856:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.952856:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:26.0:1619677447.952858:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677447.952858:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677447.952858:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.952859:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677447.952860:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (23)++
00000800:00000200:10.0:1619677447.952860:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677447.952861:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (24)--
00000800:00000200:10.0:1619677447.952861:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677447.952862:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677447.952862:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677447.952863:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677447.952863:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (23)++
00000800:00000200:8.2:1619677447.952863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677447.952864:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (24)--
00000800:00000200:10.0:1619677447.952864:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (24)--
00000800:00000200:10.0:1619677447.952865:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.2:1619677447.952868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:5.2:1619677447.952871:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677447.952874:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.952876:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677447.952876:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677447.952877:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:13.0:1619677447.952877:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.952878:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677447.952879:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:13.0:1619677447.952879:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.952879:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677447.952880:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:5.0:1619677447.952880:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.952881:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:12.0:1619677447.952881:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.952881:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:20.0:1619677447.952882:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:13.0:1619677447.952882:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677447.952882:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.952883:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.952883:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.0:1619677447.952885:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677447.952892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.952897:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.952900:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.952901:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.952903:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.952906:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.952907:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677447.952909:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.952910:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.952911:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.952912:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.952916:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.952917:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.952920:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.952921:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677447.952922:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.952924:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.952924:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.952925:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.952925:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:21.2:1619677447.952927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.952935:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677447.952936:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677447.952938:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:12.0:1619677447.952939:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:30.0:1619677447.952940:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677447.952940:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.952941:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.952943:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.952943:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.952943:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:30.0:1619677447.952944:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:12.0:1619677447.952944:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:30.0:1619677447.952945:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.952947:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.952948:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.952949:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.952952:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.952954:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.952954:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.952956:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.952956:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.952957:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.952957:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677447.952958:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.952959:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (25)--
00000800:00000200:13.0:1619677447.952959:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:30.0:1619677447.952960:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:30.0:1619677447.952960:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677447.952961:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.952962:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.952963:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.952964:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677447.952969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677447.952971:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.952973:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.952974:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.952976:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.952977:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677447.952977:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.952978:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.952979:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.952980:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.952981:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677447.953085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.953092:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.953095:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.953096:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.953098:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.953100:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.953102:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.953103:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.953104:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.953105:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.953106:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.953109:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.953111:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.953113:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.953114:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:21.2:1619677447.953115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:8.0:1619677447.953115:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.953116:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.953117:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.953117:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.953118:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677447.953120:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:8.2:1619677447.953122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.953123:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.953124:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.953126:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.953128:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.953129:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.953130:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.953131:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:2.0:1619677447.953131:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.953133:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:2.0:1619677447.953133:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.953134:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.953134:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:1.0:1619677447.953134:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.953137:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.953138:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.953140:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.953140:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.2:1619677447.953142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.953143:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.953144:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.953144:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.953146:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.953146:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.953148:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.953149:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:1.0:1619677447.953149:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.953150:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:8.2:1619677447.953152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677447.953152:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.953153:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.953153:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.953154:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:21.2:1619677447.953155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:1.0:1619677447.953155:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677447.953160:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:18.0:1619677447.953160:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:20.0:1619677447.953162:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:18.0:1619677447.953162:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:20.0:1619677447.953163:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.953163:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.953164:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:20.0:1619677447.953165:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.953166:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.953167:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.953291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.953301:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.953304:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.953307:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.953309:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.953309:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.953311:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677447.953312:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.953314:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.953315:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.953316:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677447.953320:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.953321:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677447.953324:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.953325:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:4.2:1619677447.953325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:31.0:1619677447.953326:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677447.953328:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677447.953329:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.953329:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.953330:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:24.0:1619677447.953334:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.2:1619677447.953335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.953337:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000400:00000200:24.0:1619677447.953340:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.953341:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677447.953343:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.953343:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:24.0:1619677447.953344:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.953346:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.953346:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:24.0:1619677447.953347:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.953347:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.953348:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.953349:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:20.0:1619677447.953349:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.953349:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.953352:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:24.0:1619677447.953353:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677447.953353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.953355:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.953358:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.953359:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.953360:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.953361:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.953361:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.953362:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677447.953363:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677447.953363:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:20.0:1619677447.953363:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.953364:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.953365:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.953366:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.2:1619677447.953367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.953367:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.953375:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.953377:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.953378:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.953379:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.953381:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.953384:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.953387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677447.953394:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.953396:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.953397:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.953397:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.953398:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.953399:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677447.953433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.953438:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.953440:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.953441:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.953443:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.953445:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677447.953446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:17.0:1619677447.953447:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677447.953447:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.953449:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.953450:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.953451:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:18.0:1619677447.953452:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:17.0:1619677447.953454:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.953455:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:17.0:1619677447.953456:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677447.953456:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.953457:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:17.0:1619677447.953459:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.953460:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.953460:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000400:00000200:18.0:1619677447.953461:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:17.0:1619677447.953461:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:18.0:1619677447.953462:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.953462:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.953463:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.953463:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677447.953464:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.953464:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:18.0:1619677447.953465:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.953466:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677447.953469:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:9.2:1619677447.953469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:18.0:1619677447.953471:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677447.953474:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.953475:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677447.953475:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:8.0:1619677447.953475:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:18.0:1619677447.953477:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677447.953478:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.953478:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.0:1619677447.953478:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.953479:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:18.0:1619677447.953479:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:8.0:1619677447.953480:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.953482:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.953485:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:26.0:1619677447.953490:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677447.953492:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677447.953493:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677447.953494:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.953496:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:26.0:1619677447.953499:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677447.953578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.953585:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.953588:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.953590:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.953592:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.953593:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.953595:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677447.953596:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.953597:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.953598:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.953599:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677447.953603:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.953605:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.953608:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.953609:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677447.953609:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677447.953611:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.953611:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.953612:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.953613:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677447.953625:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.953628:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.953628:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.953629:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.953631:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.953635:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.953789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.953797:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.953799:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.953802:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.953803:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.953804:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.953806:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.953807:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.953808:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.953809:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.953810:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:20.2:1619677447.953813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:31.0:1619677447.953814:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.953815:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677447.953816:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677447.953817:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.953818:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.953818:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677447.953818:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:31.0:1619677447.953820:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:20.0:1619677447.953820:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.953821:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.953822:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.953822:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677447.953822:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.953823:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.953825:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677447.953826:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721309] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.953827:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.953828:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.953829:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:30.0:1619677447.953832:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:20.0:1619677447.953832:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.953833:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677447.953835:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.953836:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.953836:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677447.953837:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.953837:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.953838:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677447.953839:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677447.953839:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677447.953840:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.0:1619677447.953840:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.953842:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.953843:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.953844:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677447.953846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677447.953848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.953854:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677447.953854:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:5.0:1619677447.953857:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:1.0:1619677447.953857:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.953858:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.953858:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:1.0:1619677447.953858:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.953859:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677447.953860:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.953863:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677447.953915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.953920:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.953924:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.953924:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.953927:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.953929:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.953931:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.953932:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.953934:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.953934:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.953936:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.953938:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.953940:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.953941:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.953942:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.953943:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.953944:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.953945:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.953946:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.953946:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:21.2:1619677447.953951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:9.2:1619677447.953951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.953958:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:18.0:1619677447.953958:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.2:1619677447.953960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:24.0:1619677447.953961:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:18.0:1619677447.953961:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.953961:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.953961:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.953962:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:18.0:1619677447.953963:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.953964:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.953965:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677447.953966:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:24.0:1619677447.953967:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.953969:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677447.953969:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:24.0:1619677447.953971:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.953971:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.953972:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.953973:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.953974:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.953974:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.953975:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.953977:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677447.953978:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.953979:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.953980:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.953981:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.953982:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.953982:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:24.0:1619677447.953985:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.953986:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:5.0:1619677447.953986:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.953987:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.953988:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.953989:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677447.953989:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677447.953990:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.953990:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677447.953992:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.953993:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677447.953994:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677447.953995:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677447.953996:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.953997:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.953999:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.953999:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:13.0:1619677447.954002:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.2:1619677447.954004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:13.0:1619677447.954004:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.954005:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.954007:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000400:00000200:13.0:1619677447.954007:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.954008:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:20.0:1619677447.954009:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:12.0:1619677447.954009:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.954010:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:13.0:1619677447.954010:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.954011:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:20.0:1619677447.954011:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:31.0:1619677447.954012:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.954013:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.954015:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.954016:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677447.954023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677447.954028:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.954030:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:8.0:1619677447.954030:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677447.954031:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.954032:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677447.954033:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:8.0:1619677447.954033:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:30.0:1619677447.954034:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:24.0:1619677447.954034:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.954036:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.954038:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.954040:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.954041:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.954042:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.954043:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.954044:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.954048:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.954050:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.954053:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.954053:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.954054:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677447.954055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.0:1619677447.954056:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.954056:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.954058:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.954059:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.954059:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.954061:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677447.954063:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.954065:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:4.2:1619677447.954065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677447.954066:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.954068:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.954069:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:10.0:1619677447.954070:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.954072:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677447.954072:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:10.0:1619677447.954072:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:17.0:1619677447.954073:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.954073:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.954073:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.954074:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.954074:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:10.0:1619677447.954074:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.954076:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677447.954076:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:10.0:1619677447.954076:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.954077:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:17.0:1619677447.954079:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:10.0:1619677447.954080:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.954082:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677447.954084:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.954085:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677447.954086:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677447.954087:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677447.954088:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.954088:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.954089:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677447.954100:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.954102:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.954103:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.954104:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.954106:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.954107:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677447.954185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.954193:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.954195:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.954197:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.954199:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.954200:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.954200:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677447.954202:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.954203:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.954204:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.954205:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677447.954208:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.954209:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.954212:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.954213:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677447.954213:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677447.954215:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.954216:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.954216:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.954217:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.954229:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.954232:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.954232:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.954233:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.954235:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.954237:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.954356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.954364:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.954367:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.954369:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.954371:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.954372:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.954373:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.954374:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.954375:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.954376:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.954377:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.954380:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.954382:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.954384:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.954385:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.954385:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.954387:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.954387:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.954388:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.954390:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.954390:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677447.954397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.954401:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.954403:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.954403:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.954405:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:30.0:1619677447.954405:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.954406:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.954406:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.954407:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.954408:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.954409:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:4.2:1619677447.954488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.954495:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.954498:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.954499:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.954500:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.954503:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.954504:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.954505:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.954507:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.954508:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.954508:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.2:1619677447.954510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:26.0:1619677447.954511:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.954513:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.954513:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.954515:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:26.0:1619677447.954516:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.954517:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.954517:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:20.0:1619677447.954518:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.954519:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.954519:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677447.954520:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.954520:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.954521:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:20.0:1619677447.954521:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.954522:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.954523:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.954525:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.954526:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.954527:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.954530:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.954531:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.954533:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.954534:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.954535:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.954536:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677447.954536:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.954536:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.954537:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.954538:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.954538:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:22.0:1619677447.954539:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.954539:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.954540:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.954542:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.954550:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.954553:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677447.954554:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677447.954554:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.954554:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.954555:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.954556:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677447.954557:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.954558:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.954560:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.954561:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.954563:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.954609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.954617:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.954620:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.954620:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677447.954622:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.954625:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.954626:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677447.954627:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.954629:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.954630:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.954631:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677447.954634:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.954635:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677447.954638:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.954638:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677447.954639:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677447.954640:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677447.954641:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.954641:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.954642:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677447.954652:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.954655:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.954655:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.954657:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.954659:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.954660:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677447.954881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677447.954888:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.954891:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.954892:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.954894:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.954897:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.954898:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677447.954899:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.954901:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.954901:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.954902:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677447.954905:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.954907:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.954909:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.954910:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.954910:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.954912:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.954912:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.954913:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.954914:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.954915:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:21.2:1619677447.954920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:12.0:1619677447.954924:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:24.0:1619677447.954927:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:12.0:1619677447.954927:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.954927:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.954929:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:12.0:1619677447.954929:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.954930:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.954931:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677447.954931:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.954932:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677447.954942:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677447.954944:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.954945:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.954946:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.954946:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.954947:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.954967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.954972:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.954975:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.954977:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.954978:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.954980:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.954982:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.954983:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.954985:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677447.954986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:5.0:1619677447.954986:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.954987:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.954991:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.954993:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677447.954994:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677447.954995:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.954996:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677447.954997:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.0:1619677447.954997:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.954998:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:31.0:1619677447.954999:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.954999:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.955000:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.955000:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:31.0:1619677447.955001:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.955001:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.955003:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677447.955004:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.955005:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.955007:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.955007:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677447.955011:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.955011:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:31.0:1619677447.955012:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.955014:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677447.955015:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677447.955015:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.955015:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.955016:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677447.955017:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.955018:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:22.0:1619677447.955018:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.955019:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.955020:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677447.955020:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677447.955021:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677447.955025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.955033:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.955036:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.955036:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.955037:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.955039:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.955042:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.955063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677447.955069:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677447.955072:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677447.955073:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677447.955075:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.955077:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.955079:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677447.955080:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.955082:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.955083:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.955084:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677447.955088:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.955089:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677447.955092:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.955093:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:2.0:1619677447.955094:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:2.0:1619677447.955095:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:2.0:1619677447.955096:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677447.955096:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677447.955097:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677447.955101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.955108:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.955110:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.955112:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.955112:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.955114:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.955117:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.955127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.955134:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.955137:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.955138:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.955140:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.955142:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.955143:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.955144:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.955146:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.955147:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.955148:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677447.955151:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.955153:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.2:1619677447.955155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:26.0:1619677447.955156:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.955157:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.955157:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677447.955159:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.955159:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677447.955160:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:26.0:1619677447.955161:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.955162:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.955162:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.2:1619677447.955163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:18.0:1619677447.955163:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.955164:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.955166:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.955168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:18.0:1619677447.955169:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.955170:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:18.0:1619677447.955171:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.955172:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.955172:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.955174:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:18.0:1619677447.955174:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.955175:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:18.0:1619677447.955175:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677447.955175:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:18.0:1619677447.955176:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.955176:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677447.955176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:22.0:1619677447.955177:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.955177:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.955178:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677447.955179:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.955179:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.955180:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:18.0:1619677447.955181:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.955181:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:6.0:1619677447.955181:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.955182:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96721292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.955183:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.955184:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677447.955184:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.955184:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677447.955184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:6.0:1619677447.955186:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.955187:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677447.955187:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:18.0:1619677447.955188:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:10.0:1619677447.955188:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.955189:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.955190:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677447.955190:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677447.955191:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677447.955191:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.955191:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.955192:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:17.0:1619677447.955192:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:10.0:1619677447.955192:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.955192:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.955193:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677447.955193:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:1.0:1619677447.955193:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677447.955194:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:10.0:1619677447.955195:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:6.0:1619677447.955196:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.955197:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.955197:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96721291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677447.955197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677447.955198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677447.955199:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677447.955200:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.955200:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.955200:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:17.0:1619677447.955201:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:6.0:1619677447.955201:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:10.0:1619677447.955202:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.955203:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96721290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.955203:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677447.955203:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.955203:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:17.0:1619677447.955204:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.955204:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677447.955204:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:20.0:1619677447.955205:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677447.955205:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000400:00000200:17.0:1619677447.955206:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.955206:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677447.955206:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:20.0:1619677447.955207:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:17.0:1619677447.955207:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:2.0:1619677447.955207:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:20.0:1619677447.955208:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.955208:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:20.0:1619677447.955210:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:10.0:1619677447.955210:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677447.955211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:17.0:1619677447.955212:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.955212:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677447.955212:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.955214:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677447.955215:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677447.955215:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677447.955216:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.955216:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:22.0:1619677447.955217:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:17.0:1619677447.955217:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.955217:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677447.955218:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:17.0:1619677447.955218:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:30.0:1619677447.955219:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.955219:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:17.0:1619677447.955219:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677447.955220:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677447.955220:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677447.955220:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677447.955220:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:30.0:1619677447.955221:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.955221:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677447.955221:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677447.955222:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.955222:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677447.955222:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.955222:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677447.955223:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.955224:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:9.2:1619677447.955226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.2:1619677447.955226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:18.0:1619677447.955234:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677447.955234:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:18.0:1619677447.955236:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:1.0:1619677447.955237:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:18.0:1619677447.955238:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.955238:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:2.0:1619677447.955238:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:1.0:1619677447.955238:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.955241:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.955241:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.955243:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677447.955245:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.955247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.955253:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.955256:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.955257:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.955257:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.955258:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.955258:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.955585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677447.955586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677447.955592:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677447.955593:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677447.955596:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:17.0:1619677447.955596:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.955596:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677447.955597:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677447.955600:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.955601:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.955602:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.955604:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.955604:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677447.955605:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677447.955606:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.955607:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.955607:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.955609:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.955609:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.955609:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677447.955610:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.955611:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677447.955614:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.955615:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:26.0:1619677447.955618:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.955618:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.955619:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.955619:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677447.955620:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677447.955621:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.955622:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677447.955623:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.955623:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:26.0:1619677447.955624:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:17.0:1619677447.955624:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:26.0:1619677447.955625:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677447.955625:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677447.955626:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.955627:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.955627:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.955628:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677447.955629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.2:1619677447.955631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677447.955635:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:5.0:1619677447.955637:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677447.955637:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677447.955638:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677447.955638:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.955638:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.955640:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.955640:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677447.955640:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:22.0:1619677447.955641:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.955642:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.955642:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.955642:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.955643:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.955643:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:6.0:1619677447.955644:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.955645:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.955647:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:5.0:1619677447.955647:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:22.0:1619677447.955648:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.955648:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.955648:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:5.0:1619677447.955648:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.955649:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.955650:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.955651:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.955655:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.955657:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.955660:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.955661:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.955662:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.955664:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.955664:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.955665:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.955666:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677447.955677:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.955680:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.955681:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.955683:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.955685:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.955686:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.955771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.955776:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.955779:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.955780:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.955782:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.955785:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.955786:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.955787:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.955789:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.955790:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.955791:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.955794:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.955796:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.955799:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.955799:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.2:1619677447.955799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677447.955800:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.955802:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.955802:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.955803:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.955803:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677447.955805:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.955807:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.955808:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677447.955809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:17.0:1619677447.955810:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.955813:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.955814:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.955815:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.955817:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:17.0:1619677447.955817:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.955818:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.955819:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:17.0:1619677447.955819:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:20.0:1619677447.955820:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.955822:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.955823:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.955823:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.955825:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677447.955825:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.955828:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.955829:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.955829:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.955831:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.955831:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.955832:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.955832:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.955844:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.955847:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.955848:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.955848:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.955850:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.2:1619677447.955852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:6.0:1619677447.955852:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677447.955858:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.955861:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.955863:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.955863:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.955866:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.955867:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677447.955869:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721284] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.955870:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.955871:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.955872:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677447.955876:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.955877:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.955880:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.955881:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677447.955882:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677447.955883:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677447.955884:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.955885:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.955886:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:10.0:1619677447.955896:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.955898:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.955899:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.955899:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.955901:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:10.0:1619677447.955904:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677447.955916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.955923:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.955925:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.955926:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.955928:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.955930:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.955932:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677447.955933:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.955934:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.955935:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.955936:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677447.955939:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.955941:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.955943:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.955944:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.955945:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.955946:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.955946:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.955947:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.955948:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677447.955951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.955961:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.955962:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.955962:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.955963:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.955963:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.955965:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:10.0:1619677447.955965:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.955967:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.955968:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.955968:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.955969:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:10.0:1619677447.955969:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677447.955970:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.955972:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.955972:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.955973:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677447.955976:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677447.955977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:10.0:1619677447.955978:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677447.955981:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677447.955981:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.955982:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677447.955983:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.955983:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.955983:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.955984:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.955984:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.955984:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677447.955985:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677447.955985:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.955985:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677447.955987:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:10.0:1619677447.955987:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:2.0:1619677447.955987:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:10.0:1619677447.955988:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677447.955989:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:18.0:1619677447.955990:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677447.955991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677447.955992:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.955994:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:5.2:1619677447.955994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:18.0:1619677447.955996:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.955997:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.955999:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.955999:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:18.0:1619677447.956000:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.956001:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677447.956002:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:2.0:1619677447.956002:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.956004:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677447.956004:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:18.0:1619677447.956004:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.956004:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.956005:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:18.0:1619677447.956006:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.956007:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.956008:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.956009:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.956010:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.956011:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.956013:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.956013:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.956014:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.956015:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:21.2:1619677447.956016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.956022:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.956024:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:10.0:1619677447.956025:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:6.0:1619677447.956025:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.956027:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.956028:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677447.956028:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:22.0:1619677447.956029:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.956029:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.956031:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.956032:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.956032:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.956033:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.956034:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.956035:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677447.956035:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:22.0:1619677447.956039:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.956040:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.956043:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.956043:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.956044:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.956046:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.956046:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.956047:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.956048:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.956052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.956060:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.956062:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.956063:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.956063:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677447.956065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:1.0:1619677447.956066:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.956068:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677447.956072:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.956075:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.956077:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.956079:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.956080:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.956081:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677447.956083:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.956084:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.956085:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.956086:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677447.956089:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.956091:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677447.956094:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.956095:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677447.956095:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677447.956097:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.956098:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.956099:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.956101:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.956101:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677447.956107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677447.956108:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.956111:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677447.956111:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677447.956112:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.956114:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.956115:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:26.0:1619677447.956116:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677447.956117:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.956118:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.956119:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677447.956168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.956173:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.956176:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.956177:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.956179:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.956181:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.956182:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.956184:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.956185:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.956186:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.956187:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.956190:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.956191:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.956193:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.956194:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.956195:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.956196:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.956196:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.956197:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.956197:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.956202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677447.956209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37e800] (20)++
00000800:00000200:1.0:1619677447.956209:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.956211:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.956211:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.956212:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.956213:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.956215:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677447.956216:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37e800] (21)++
00000800:00000200:10.0:1619677447.956219:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.56@o2ib3
00000800:00000200:26.0:1619677447.956220:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)--
00000400:00000200:10.0:1619677447.956222:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.56@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.956225:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.956226:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677447.956228:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.956229:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.956230:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.956231:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677447.956234:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.956236:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677447.956239:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.956240:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:10.0:1619677447.956241:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:10.0:1619677447.956242:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:10.0:1619677447.956243:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37e800] (21)++
00000800:00000200:10.0:1619677447.956243:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37e800] (22)--
00000800:00000200:10.0:1619677447.956244:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (21)--
00000800:00000200:8.2:1619677447.956248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.956256:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.956259:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.956260:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.956261:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.956263:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.956265:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:8.2:1619677447.956269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677447.956277:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.956279:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:5.0:1619677447.956280:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.956281:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677447.956282:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.956283:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:4.2:1619677447.956310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.956318:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.956320:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.956321:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.956323:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.956326:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.956327:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677447.956328:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721276] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.956330:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.956331:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.956332:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677447.956334:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.956336:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.956337:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.956338:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.956338:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.956339:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.956340:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.956341:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.956341:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677447.956351:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.956354:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.956354:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.956355:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.956356:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.956357:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677447.956373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.956381:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.956384:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.956386:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.956388:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.956389:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.956390:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.956391:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.956392:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.956393:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.956394:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.956397:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.956399:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.956401:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.956401:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.956402:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.956403:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.956404:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.956405:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.956405:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677447.956426:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.956428:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.956429:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.956430:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.956432:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.956433:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677447.956449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:5.2:1619677447.956449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:30.0:1619677447.956450:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.956452:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.956453:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.956453:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677447.956454:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.956455:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:6.0:1619677447.956455:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677447.956456:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:22.0:1619677447.956459:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.956459:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:2.0:1619677447.956459:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.956460:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:6.0:1619677447.956463:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.956463:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.956465:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.956466:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677447.956466:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.956468:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.956468:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:6.0:1619677447.956470:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.956470:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.956471:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.956472:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.956473:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.956473:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.956475:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:6.0:1619677447.956477:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.956479:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.956479:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.956482:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.956482:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677447.956483:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.956483:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.956485:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.956485:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.956486:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.956486:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677447.956486:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:6.0:1619677447.956487:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.956487:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.956488:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.956489:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.956490:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.956490:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.956492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.956500:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:12.0:1619677447.956502:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677447.956503:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.956504:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.956504:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677447.956505:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.956506:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.956507:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:12.0:1619677447.956508:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.956509:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.2:1619677447.956509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677447.956511:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.956512:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.956513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.956515:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.956516:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.956517:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.956518:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.956519:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000400:00000200:17.0:1619677447.956520:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.956520:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.956520:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.956521:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.956522:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.956522:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:17.0:1619677447.956524:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.956526:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.956527:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.956528:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.956529:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.956532:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.956534:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.956536:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.956537:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.956538:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.956539:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.956540:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.956540:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.956541:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.956553:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.956555:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.956556:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.956556:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.956558:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.956561:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.956570:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.956573:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.956574:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.956574:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.956575:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.956576:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677447.956655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677447.956660:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.956663:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:4.2:1619677447.956663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.956664:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.956666:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.956669:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.956670:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:18.0:1619677447.956670:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:26.0:1619677447.956672:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:18.0:1619677447.956672:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.956673:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677447.956673:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677447.956675:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.956676:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677447.956677:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.956678:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.956680:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.956680:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:26.0:1619677447.956681:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.956683:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.956683:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:26.0:1619677447.956684:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.956686:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677447.956687:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.956687:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.956687:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.956689:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.956689:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:26.0:1619677447.956690:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.956690:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.956691:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677447.956692:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.956694:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.956694:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.956695:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.956696:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.956697:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.956698:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.956699:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.956700:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677447.956702:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.956705:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:4.2:1619677447.956705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677447.956706:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.956706:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.956708:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.956710:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677447.956710:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.956711:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:20.2:1619677447.956712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:10.0:1619677447.956712:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.956714:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.956716:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.956717:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.956717:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.956717:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677447.956718:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.956718:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.956719:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677447.956720:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677447.956720:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:6.0:1619677447.956721:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:10.0:1619677447.956722:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.956723:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.956724:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:10.0:1619677447.956724:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.956724:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000400:00000200:10.0:1619677447.956725:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:6.0:1619677447.956725:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000400:00000200:10.0:1619677447.956726:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.956726:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:10.0:1619677447.956728:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.956729:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.956731:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:26.0:1619677447.956732:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.956733:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.956734:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.956735:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.956735:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.956736:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.956737:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677447.956737:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.956739:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.956740:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677447.956741:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.956743:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.956744:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.956744:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677447.956745:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.2:1619677447.956748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:9.2:1619677447.956750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677447.956756:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677447.956757:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:30.0:1619677447.956758:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.956760:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.956760:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.956761:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.956761:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.956762:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.956762:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.956764:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.956764:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.956765:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.956767:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.956767:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:30.0:1619677447.956768:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.956769:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.956771:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.956773:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.956775:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.956776:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.956776:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.956778:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.956779:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.956780:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.956780:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.956790:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.956792:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.956793:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.956795:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.956796:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.956797:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.956798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.956804:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.956807:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677447.956807:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.956807:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.956810:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.956811:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.956811:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.956812:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:1.0:1619677447.956812:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.956813:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.956814:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677447.956814:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.956815:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.956816:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.956817:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.956818:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.956821:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.956823:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.956826:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.956826:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.956827:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.956829:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.956829:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.956830:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.956831:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.956832:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677447.956837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.956842:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677447.956843:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677447.956844:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677447.956845:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677447.956845:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677447.956846:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.956846:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.956847:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.956848:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.956850:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677447.957259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.957265:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.957267:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.957268:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.957270:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.957272:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.957273:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.957274:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.957276:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.957277:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.957278:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.957281:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.957283:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.957285:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.957286:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.957287:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.957288:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.957289:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.957290:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.957291:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.957291:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677447.957297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.957305:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.957308:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.957309:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.957309:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.957311:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.957313:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.957316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.957324:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.2:1619677447.957324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.0:1619677447.957327:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.957327:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.957330:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.957330:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:24.0:1619677447.957332:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.957332:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:24.0:1619677447.957333:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:1.0:1619677447.957333:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677447.957335:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.957335:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.957337:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677447.957337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:24.0:1619677447.957338:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.957338:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.957339:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:24.0:1619677447.957340:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.957341:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.957343:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.957344:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.957345:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:31.0:1619677447.957346:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.957349:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:17.0:1619677447.957350:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.957352:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.957352:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.957353:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677447.957353:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:31.0:1619677447.957354:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.957355:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:31.0:1619677447.957356:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677447.957357:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96721263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.957357:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.957357:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677447.957358:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.957358:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.957359:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:31.0:1619677447.957360:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.957360:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.957360:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000400:00000200:31.0:1619677447.957361:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.957361:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.957361:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677447.957362:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:24.0:1619677447.957362:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.957362:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.957362:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677447.957363:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.957363:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:31.0:1619677447.957366:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.957367:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:9.2:1619677447.957367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.2:1619677447.957367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.957370:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.957371:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677447.957372:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677447.957373:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677447.957374:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.957375:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.957375:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677447.957375:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:1.0:1619677447.957375:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.957377:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677447.957378:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.957379:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.957379:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:8.0:1619677447.957379:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.957379:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.957381:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.957383:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.957383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.957384:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677447.957387:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.0:1619677447.957387:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:26.0:1619677447.957390:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.957390:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:21.2:1619677447.957390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:22.0:1619677447.957392:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.957392:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.957393:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:26.0:1619677447.957394:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:22.0:1619677447.957394:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.957394:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677447.957394:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.957395:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.957396:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:26.0:1619677447.957397:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.957398:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:26.0:1619677447.957400:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.957402:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677447.957403:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.957404:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.957405:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.957406:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677447.957410:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.957411:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677447.957414:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.957415:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677447.957416:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677447.957418:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677447.957418:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.957419:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.957420:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677447.957424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.957433:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.957435:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.957436:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.957438:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.957438:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.957441:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677447.957787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677447.957794:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.957797:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.957798:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.957800:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.957802:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.957803:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.957805:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.957806:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.957807:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.957807:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.957810:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.957812:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.957815:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.957816:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.957816:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.957818:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.957819:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.957819:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.957820:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.957824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.957833:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.957835:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.957836:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.957837:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.957839:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.957842:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.957870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.957876:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.957879:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.957879:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.957882:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.957884:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.957886:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.957887:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.957889:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.957890:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.957891:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.957894:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.957896:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.957899:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.957900:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.957900:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.957902:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.957903:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.957903:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.957904:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.957915:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.957918:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.957919:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.957921:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.957922:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.957924:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677447.957935:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677447.957938:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.957939:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.957939:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.957940:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.957941:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.958015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.958017:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.958020:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.958022:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.958023:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.958025:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.958027:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.958028:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.958029:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.958030:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.958031:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.958035:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.958036:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.958039:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.958040:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.958041:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.958043:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.958043:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.958045:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.958046:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.958046:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.958051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677447.958056:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677447.958057:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677447.958059:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677447.958059:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:13.0:1619677447.958060:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.958060:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:13.0:1619677447.958061:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.958061:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.958063:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.958064:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.958096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677447.958103:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.958106:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677447.958106:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677447.958108:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677447.958111:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.958113:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677447.958114:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.958115:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.958116:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.958117:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.2:1619677447.958119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:10.0:1619677447.958120:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677447.958122:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677447.958125:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.958125:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677447.958126:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.958127:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677447.958128:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.958129:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:10.0:1619677447.958129:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.958129:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677447.958130:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:30.0:1619677447.958131:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.958134:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.958134:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.958134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677447.958135:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.958136:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.958138:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.958139:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.958140:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.958143:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.958143:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677447.958144:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:5.0:1619677447.958145:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677447.958146:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677447.958146:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.958147:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.958148:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677447.958148:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.958148:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:30.0:1619677447.958149:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.958150:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.958151:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.958151:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677447.958151:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677447.958161:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.958164:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.958165:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.958166:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.958168:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.958169:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677447.958290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.958299:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.2:1619677447.958299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677447.958302:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.958304:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.958306:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.958306:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.958307:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:26.0:1619677447.958307:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677447.958309:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.958309:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:31.0:1619677447.958310:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677447.958310:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:31.0:1619677447.958311:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.958313:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:26.0:1619677447.958313:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.958316:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.958316:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.958317:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677447.958318:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:26.0:1619677447.958319:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.958320:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.958321:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677447.958321:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.958322:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:26.0:1619677447.958322:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.958323:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677447.958324:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677447.958325:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677447.958325:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.958326:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.958326:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677447.958326:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677447.958329:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.958330:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677447.958330:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:9.2:1619677447.958331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:26.0:1619677447.958332:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677447.958332:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.958332:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.958333:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677447.958339:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.958342:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:5.2:1619677447.958343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.958344:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:13.0:1619677447.958344:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:2.0:1619677447.958344:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.958345:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.958346:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677447.958346:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.958347:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.958348:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.958350:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.958350:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677447.958350:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.958351:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.958352:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:9.2:1619677447.958352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:13.0:1619677447.958353:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:18.0:1619677447.958354:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.958356:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.958357:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721254] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.958358:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000400:00000200:18.0:1619677447.958359:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.958359:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677447.958360:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677447.958360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677447.958360:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000400:00000200:18.0:1619677447.958361:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.958361:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.958362:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.958363:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:18.0:1619677447.958365:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.958366:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.958367:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.958369:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:18.0:1619677447.958370:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.958370:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677447.958371:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.958372:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:1.0:1619677447.958372:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.958373:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.958374:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.958375:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.958375:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:1.0:1619677447.958375:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.958377:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.958378:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.958379:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677447.958380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:1.0:1619677447.958380:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.958381:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.958385:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.958387:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677447.958389:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.958390:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.958390:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677447.958391:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.958391:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.958392:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:17.0:1619677447.958393:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.958394:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.958395:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.958396:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.958396:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.958397:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.958397:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.958400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.958407:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.958409:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.958410:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.958411:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.958413:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.958416:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.958422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.958428:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.958431:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677447.958431:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.958432:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.958433:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.958433:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677447.958436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.958439:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.958441:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.958443:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.958445:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.958446:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.958448:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677447.958449:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.958450:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.958451:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.958452:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677447.958455:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.958457:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.958459:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.958460:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677447.958461:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.958462:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.958463:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.958464:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.958464:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677447.958468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.958477:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.958479:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.958480:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.958481:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.958483:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.958485:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.958489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.958497:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.958499:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.958500:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.958500:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.958501:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.958502:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.958512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.958519:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.958522:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.958523:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.958524:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.958527:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.958528:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.958529:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.958531:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.958532:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.958533:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.958536:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.958538:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.958541:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.958541:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.958542:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.958544:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.958544:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.958545:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.958545:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.958550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.958557:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.958560:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.958561:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.958561:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.958563:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.958566:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.958571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.958578:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.958581:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.958582:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.958582:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.958583:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.958583:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.958592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.958598:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:4.2:1619677447.958598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677447.958600:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.958603:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.958604:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677447.958605:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677447.958605:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.958606:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677447.958608:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:5.0:1619677447.958608:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.958609:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:5.0:1619677447.958610:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.958611:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.958612:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677447.958613:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.958616:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.958616:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677447.958618:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:5.0:1619677447.958618:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677447.958619:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.958620:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.958621:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677447.958622:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.958622:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.958622:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.958623:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.958625:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677447.958626:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.958626:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.958627:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.958627:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:10.0:1619677447.958628:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677447.958630:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.958631:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.958632:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677447.958633:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.958634:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.958634:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.958635:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:12.0:1619677447.958637:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.958640:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.958641:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.958643:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.958644:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.958646:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:12.0:1619677447.958646:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.958648:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.958649:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.958650:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.958652:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.958653:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.958663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.958671:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.958673:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.958676:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.958678:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.958678:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.958680:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.958681:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.958683:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.958684:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.958684:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.958688:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.958690:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.958693:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.958694:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.958694:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.958696:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.958697:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.958698:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.958698:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.958702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.958710:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.958712:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.958713:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.958715:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.958716:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.958718:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677447.958725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677447.958733:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.958735:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.958736:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.958738:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.958741:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.958742:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.958744:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721247] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.958745:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.958746:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.958747:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.958750:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.958752:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.958755:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.958755:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.958756:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.958758:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.958758:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.958759:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.958760:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.958764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.958771:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.958774:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.958775:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.958775:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.958777:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.958780:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.958784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.958790:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.958793:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.958794:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.958794:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.958795:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.958796:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677447.958811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.958818:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.958821:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.958823:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.958825:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.958826:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.958828:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.958829:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.958830:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.958831:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.958832:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.958835:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.958837:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.958839:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.958839:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.958840:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.958842:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.958842:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.958844:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.958846:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.958846:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677447.958852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677447.958854:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.958856:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.958857:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.958858:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.958859:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677447.958860:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.958861:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.958862:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.958863:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.958863:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.958872:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.958875:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.958875:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.958876:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.958877:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.958878:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.959068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.959075:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.959078:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.959080:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.959082:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.959082:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.959084:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.959085:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721245] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.959086:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.959087:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.959088:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.959091:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.959092:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.959094:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.959095:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.959096:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.959097:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.959098:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.959099:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.959100:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.959100:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677447.959106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677447.959109:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.2:1619677447.959109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677447.959111:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.959112:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.959113:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.959114:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:2.0:1619677447.959114:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:13.0:1619677447.959115:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.2:1619677447.959115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:12.0:1619677447.959116:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.959117:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:2.0:1619677447.959117:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:30.0:1619677447.959118:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:18.0:1619677447.959118:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:30.0:1619677447.959120:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:2.0:1619677447.959120:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.959123:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:2.0:1619677447.959123:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.959124:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:26.0:1619677447.959125:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.959126:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:2.0:1619677447.959126:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.959128:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.959129:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.959129:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.959130:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677447.959132:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.959133:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.959134:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677447.959135:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.959136:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:26.0:1619677447.959137:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.959138:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.959138:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677447.959139:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:2.0:1619677447.959140:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.959140:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.959142:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.959142:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:26.0:1619677447.959143:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.959143:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.959143:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:26.0:1619677447.959145:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677447.959148:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.959148:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:8.2:1619677447.959148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677447.959149:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677447.959150:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677447.959151:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.959152:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.959152:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:1.0:1619677447.959156:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.959158:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.959159:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.959159:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.959162:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.959164:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677447.959173:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.959175:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.959176:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.959178:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.959179:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.959180:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677447.959189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.959196:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.959199:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.959200:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.959202:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.959205:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.959206:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.959208:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.959209:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.959210:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.959211:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.959214:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.959216:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.959219:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.959220:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.959220:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.959222:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.959223:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.959223:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.959224:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.959228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.959236:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.959238:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.959239:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.959239:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.959242:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.959244:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.959284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.959290:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.959292:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.959293:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.959300:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.959302:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.959304:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.959305:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.959307:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.959308:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.959309:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.959312:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.959314:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.959317:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.959318:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.959319:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.959320:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.959321:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.959323:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.959324:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.959324:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.959327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677447.959330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677447.959333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:24.0:1619677447.959335:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677447.959336:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677447.959338:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:17.0:1619677447.959338:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.0:1619677447.959338:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.959339:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.959339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677447.959339:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677447.959340:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677447.959340:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677447.959341:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.959343:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.959343:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:24.0:1619677447.959344:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.0:1619677447.959345:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:18.0:1619677447.959346:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.959347:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:18.0:1619677447.959348:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.959350:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:18.0:1619677447.959350:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677447.959350:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.959351:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.959352:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.959353:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.959353:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.959355:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677447.959355:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.959356:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.959358:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.959359:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.959360:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677447.959365:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.959366:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.959368:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.959369:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.959371:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.959372:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.959372:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.959373:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.959373:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.959374:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.959376:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.959376:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.959376:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.959377:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677447.959377:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677447.959378:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677447.959378:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.959379:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677447.959382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:13.0:1619677447.959389:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677447.959390:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:13.0:1619677447.959392:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677447.959393:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:13.0:1619677447.959393:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.959394:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.959396:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.959396:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:20.0:1619677447.959397:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:13.0:1619677447.959397:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.959400:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677447.959400:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.959429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.2:1619677447.959432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.959437:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.959440:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.0:1619677447.959440:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.959442:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.959443:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.959443:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.959444:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677447.959445:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.0:1619677447.959445:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:31.0:1619677447.959446:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.959446:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:31.0:1619677447.959447:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677447.959448:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.959450:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.959451:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.959452:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677447.959456:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.959457:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:4.2:1619677447.959457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677447.959460:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.959461:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677447.959462:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677447.959463:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677447.959464:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.959465:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677447.959465:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677447.959466:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677447.959467:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.959468:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.959470:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677447.959470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:10.0:1619677447.959472:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.959474:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.959475:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.959476:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.959477:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.959477:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:10.0:1619677447.959478:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.959479:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.959481:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.959481:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.959483:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.959483:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.959485:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.959487:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:10.0:1619677447.959488:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.959489:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.959489:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.959491:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.959491:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.959492:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.959492:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677447.959506:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.2:1619677447.959508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:6.0:1619677447.959508:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.959509:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.959510:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.959512:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.959513:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:6.0:1619677447.959515:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677447.959516:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.959518:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.959518:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.959520:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.959522:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677447.959524:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.959525:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.959526:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.959527:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.959531:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.959533:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.959535:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.959536:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.959537:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.959538:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.959539:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.959540:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.959540:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677447.959544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.959552:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.959554:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.959555:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.959555:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.959558:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.959560:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677447.959607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677447.959613:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.959616:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.959617:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.959619:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.959621:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.959622:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677447.959624:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.959625:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.959626:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.959627:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677447.959631:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.959632:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677447.959635:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.959636:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677447.959636:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677447.959638:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677447.959639:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:4.2:1619677447.959639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:18.0:1619677447.959640:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677447.959641:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.959642:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.0:1619677447.959645:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.2:1619677447.959647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.0:1619677447.959648:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.959649:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.959651:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.959652:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677447.959652:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:8.0:1619677447.959653:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.959654:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:22.0:1619677447.959655:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:8.0:1619677447.959655:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:2.0:1619677447.959655:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:22.0:1619677447.959656:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.959656:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.959656:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677447.959657:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:22.0:1619677447.959658:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000400:00000200:8.0:1619677447.959658:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.959659:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.959660:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:8.0:1619677447.959660:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.959664:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.959666:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.959669:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677447.959670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.0:1619677447.959670:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.959670:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.959672:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.959672:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.959674:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.959675:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.959676:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677447.959678:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:8.2:1619677447.959678:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677447.959681:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:4.2:1619677447.959681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:30.0:1619677447.959683:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.959684:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.959685:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.959686:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:30.0:1619677447.959687:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.959688:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.959688:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677447.959688:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:30.0:1619677447.959689:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.959689:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:30.0:1619677447.959690:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.959690:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677447.959690:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.959691:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:30.0:1619677447.959692:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.959692:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677447.959692:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.959695:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.959696:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677447.959697:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.2:1619677447.959698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677447.959700:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.959700:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.959701:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.959702:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.959703:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.959704:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.959704:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:1.0:1619677447.959705:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.959708:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.959709:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.959709:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.959710:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.959710:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:12.0:1619677447.959714:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.959717:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.959717:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.959719:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.959721:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.959721:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677447.959729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.959735:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.959738:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.959738:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.959740:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.959742:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.959744:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.959745:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.959746:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.959748:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.959748:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.959752:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.959754:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.959757:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.959758:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.959759:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.959761:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.959761:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.959762:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.959763:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677447.959774:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.959777:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.959778:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.959779:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.959780:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.959782:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.959801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.959804:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.959806:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.959808:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.959809:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.959811:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.959812:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.959814:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.959815:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.959816:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.959817:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.959820:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.959822:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.959824:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.959825:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.959826:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.959828:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.959829:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.959830:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.959830:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.2:1619677447.959839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677447.959841:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.959843:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.959844:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.959846:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.959847:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.959847:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677447.959849:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677447.959850:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.959850:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.959852:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.959855:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.959856:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677447.959858:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.959859:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.959860:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677447.959860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:26.0:1619677447.959861:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677447.959863:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.959865:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677447.959867:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677447.959867:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.959867:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.959868:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677447.959869:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677447.959869:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.959870:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.959870:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:31.0:1619677447.959871:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.959871:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:31.0:1619677447.959873:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.959873:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.959875:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677447.959876:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.959877:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.959878:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.959879:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677447.959881:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:31.0:1619677447.959882:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.959883:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677447.959883:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.959883:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.959884:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.959885:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.959887:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.959887:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677447.959888:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677447.959888:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677447.959890:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.959890:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.959891:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.959892:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677447.959903:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.959906:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.959907:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.959908:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.959910:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.959913:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.960122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677447.960128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:26.0:1619677447.960129:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.960132:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.960133:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677447.960134:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:26.0:1619677447.960135:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.960136:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.960136:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.960137:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:26.0:1619677447.960138:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.960139:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.960140:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.960141:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.960142:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.960142:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.960143:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.960144:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.960146:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.960147:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.960149:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.960149:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.960150:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677447.960152:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.960154:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.960155:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.960156:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.960156:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677447.960157:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677447.960158:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.960159:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677447.960159:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.960160:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:18.0:1619677447.960160:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677447.960161:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:18.0:1619677447.960161:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677447.960162:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677447.960162:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.960163:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.960164:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.960164:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:4.2:1619677447.960167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.960172:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677447.960174:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:13.0:1619677447.960175:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677447.960175:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.960176:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.960176:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677447.960176:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.960177:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:13.0:1619677447.960178:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:10.0:1619677447.960178:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677447.960178:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:13.0:1619677447.960179:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.960181:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.960181:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677447.960181:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.960184:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.960382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.960387:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.960390:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.960392:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.960393:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.960395:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677447.960396:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:5.0:1619677447.960399:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.960401:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.960402:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.960403:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677447.960403:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:5.0:1619677447.960404:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.960405:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.960406:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:10.0:1619677447.960407:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:5.0:1619677447.960407:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.960408:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:31.0:1619677447.960410:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.960410:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.960413:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.960413:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.960415:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:30.0:1619677447.960416:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.960416:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.960417:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677447.960418:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677447.960419:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.960419:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677447.960420:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.960420:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.960420:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:30.0:1619677447.960421:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.960421:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.960422:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.960422:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.960422:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677447.960423:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677447.960423:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.960423:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.960424:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.960426:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.960426:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677447.960428:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.960429:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.960431:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677447.960431:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.2:1619677447.960431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677447.960432:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.960433:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677447.960434:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.960434:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677447.960434:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:30.0:1619677447.960435:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677447.960435:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.960435:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677447.960436:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.960436:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.960436:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:31.0:1619677447.960437:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.960437:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677447.960437:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:20.0:1619677447.960437:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677447.960438:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.960438:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677447.960438:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677447.960438:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677447.960438:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677447.960439:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.960439:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:31.0:1619677447.960440:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.960440:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.960443:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.960444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.2:1619677447.960445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677447.960449:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.960450:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.960451:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.960453:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:20.0:1619677447.960453:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:10.0:1619677447.960453:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.960454:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.960454:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677447.960455:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.960455:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.960456:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.0:1619677447.960456:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:10.0:1619677447.960456:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:30.0:1619677447.960457:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.960457:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.960458:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.960458:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:20.0:1619677447.960458:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.960459:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.960459:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.960461:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.0:1619677447.960462:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677447.960475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.960483:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.960486:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.960488:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.960490:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.960491:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.960492:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.960494:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.960494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:31.0:1619677447.960495:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.960496:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.960497:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.960500:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.960501:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677447.960502:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.960504:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.960504:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:31.0:1619677447.960505:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.960505:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.960505:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.960506:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.960507:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.960508:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.960508:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.960509:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:8.0:1619677447.960510:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.960511:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.960512:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.960513:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.960514:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.960515:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.960518:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.960519:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677447.960520:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.960522:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677447.960522:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677447.960523:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.960523:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.960523:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:12.0:1619677447.960524:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.960525:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.960525:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.2:1619677447.960525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677447.960526:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.960526:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.960526:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677447.960528:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677447.960531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.960536:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.960538:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677447.960541:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.960541:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.960543:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.960544:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.960544:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.960546:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.960546:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.960547:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.960549:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.960550:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.960551:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.960554:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.960556:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.960557:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.960558:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.960558:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.960559:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:4.2:1619677447.960559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677447.960559:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.960560:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.960562:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.960562:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.960563:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.960564:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677447.960566:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677447.960568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677447.960569:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.960570:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.960571:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.960573:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.960575:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:1.0:1619677447.960575:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:10.0:1619677447.960576:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.960577:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.960577:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.960577:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.960578:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:10.0:1619677447.960578:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.960579:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677447.960580:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.960582:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:10.0:1619677447.960583:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677447.960584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:10.0:1619677447.960585:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677447.960588:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.960589:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.960589:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677447.960590:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:10.0:1619677447.960591:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677447.960592:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:10.0:1619677447.960592:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.960592:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.960593:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677447.960593:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677447.960595:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.960597:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.960598:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.960599:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.960600:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.960601:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.960602:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.960605:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:17.0:1619677447.960605:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.960607:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.960608:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:17.0:1619677447.960608:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677447.960609:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.960609:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.960609:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677447.960610:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:22.0:1619677447.960611:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.960611:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.960612:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677447.960612:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677447.960613:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:22.0:1619677447.960614:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.2:1619677447.960617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.960622:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677447.960624:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677447.960625:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.960626:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.960626:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:1.0:1619677447.960626:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.960627:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.960628:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.960628:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.960628:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.960629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000400:00000200:1.0:1619677447.960629:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.960631:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677447.960637:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677447.960640:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:30.0:1619677447.960642:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.960643:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:30.0:1619677447.960644:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.960646:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.960647:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.960648:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.960649:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.960650:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.960654:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.960656:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.960659:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.960660:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.960660:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.960662:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.960663:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677447.960664:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677447.960664:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:8.2:1619677447.960668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.960676:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.960679:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.960680:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.960682:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.960682:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.960685:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677447.960698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.960704:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.960707:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.960708:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.960711:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.960714:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.960715:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.960717:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.960718:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.960719:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.960720:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.960723:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.960725:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.960727:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.960728:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.960729:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.960730:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.960731:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.960732:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.960733:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.960733:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677447.960737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677447.960738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.960744:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:20.0:1619677447.960745:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677447.960746:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.960746:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677447.960747:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:18.0:1619677447.960747:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:20.0:1619677447.960748:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.960750:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.960750:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.960752:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677447.960995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.2:1619677447.961000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:17.0:1619677447.961001:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.961004:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.961004:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.961006:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:17.0:1619677447.961006:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.961008:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.961009:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:17.0:1619677447.961009:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:6.0:1619677447.961009:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:17.0:1619677447.961011:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.961012:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.961013:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.961014:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.961015:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.961015:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:22.0:1619677447.961016:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.961018:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.961019:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.961020:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.961021:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.961022:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.961022:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.961025:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677447.961026:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.961026:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.961027:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677447.961028:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.961028:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.961029:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.961030:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.961030:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.961031:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.961031:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.961032:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.961034:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.961035:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.961035:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.961036:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.961040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677447.961041:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:21.2:1619677447.961044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:13.0:1619677447.961044:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.961045:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.961046:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.961048:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.961048:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:13.0:1619677447.961049:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677447.961050:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:31.0:1619677447.961051:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677447.961051:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.961051:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.961054:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:1.0:1619677447.961054:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.961056:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:31.0:1619677447.961057:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.961058:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.961060:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.961060:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:31.0:1619677447.961061:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677447.961063:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.961063:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.961064:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.961064:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:31.0:1619677447.961065:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.961065:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.961065:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:31.0:1619677447.961066:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:12.0:1619677447.961066:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:31.0:1619677447.961070:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.961072:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677447.961075:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.961076:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677447.961076:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:4.2:1619677447.961077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677447.961078:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.961078:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.961080:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.961081:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.961082:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.2:1619677447.961087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.2:1619677447.961088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677447.961090:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.961091:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.961091:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.961093:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.961094:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.961094:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.961095:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.961096:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:26.0:1619677447.961097:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.961098:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.961099:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677447.961099:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:30.0:1619677447.961100:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:26.0:1619677447.961100:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.961101:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.961102:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.961103:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.961104:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.961105:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:26.0:1619677447.961107:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.961107:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.961109:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:10.0:1619677447.961109:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.961111:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.961113:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677447.961113:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:26.0:1619677447.961114:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:10.0:1619677447.961114:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.961115:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677447.961116:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.961116:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.961117:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:10.0:1619677447.961117:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:26.0:1619677447.961118:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.961119:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.961120:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.961121:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:10.0:1619677447.961121:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.961122:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677447.961123:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.961123:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:9.2:1619677447.961123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.961126:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.961127:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677447.961127:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677447.961129:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677447.961129:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.961130:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.961130:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677447.961130:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.961134:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.961135:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.961135:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.961137:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.961140:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677447.961140:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677447.961143:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.961144:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.961146:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.961147:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.961149:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677447.961176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:20.2:1619677447.961181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677447.961183:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.961185:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.961186:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677447.961187:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:18.0:1619677447.961188:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.961190:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.961190:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.961192:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:18.0:1619677447.961192:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677447.961192:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.961193:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.961195:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.961195:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.961196:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.961197:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677447.961197:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677447.961199:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.961200:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.961201:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.961201:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.961202:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:18.0:1619677447.961204:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:5.0:1619677447.961206:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.961207:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.961208:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.961208:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677447.961208:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:18.0:1619677447.961210:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.961211:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677447.961211:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.961211:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:18.0:1619677447.961212:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:5.0:1619677447.961212:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:18.0:1619677447.961214:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.961214:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677447.961214:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.961215:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.961216:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.961216:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.2:1619677447.961220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:8.2:1619677447.961221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677447.961223:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.961226:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:2.0:1619677447.961226:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:26.0:1619677447.961227:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.961227:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:17.0:1619677447.961228:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:26.0:1619677447.961229:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.961229:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.961229:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.961230:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677447.961231:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.961232:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.961232:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677447.961233:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:17.0:1619677447.961234:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.961237:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:10.0:1619677447.961240:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.961243:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.961243:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.961244:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.961245:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.961245:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677447.961298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.961306:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.961308:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.961310:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.961313:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.961313:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.961314:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677447.961316:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.961317:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.961318:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.961319:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677447.961323:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.961324:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.961327:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.961327:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.961328:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.961330:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.961330:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.961331:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.961332:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677447.961343:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.961345:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.961346:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.961348:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.961349:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.961350:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677447.961465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.961471:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.961474:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.961475:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.961477:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.961480:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.961482:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.961483:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.961484:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.961485:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.961486:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.961489:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.961491:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.961494:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.961495:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.961495:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.961497:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.961498:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.961498:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.961499:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.2:1619677447.961509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677447.961511:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.961513:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.961514:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.961516:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.961516:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677447.961518:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.961518:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.961519:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677447.961519:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677447.961521:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.961524:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.961525:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.961527:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.961528:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677447.961528:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677447.961529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677447.961529:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.961530:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.961530:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:12.0:1619677447.961531:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.961532:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.961533:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.961534:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677447.961534:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.961536:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677447.961536:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.961538:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.961539:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:1.0:1619677447.961539:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677447.961540:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677447.961540:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:26.0:1619677447.961542:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.961542:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.961542:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.961543:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677447.961544:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.961544:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677447.961546:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.961547:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.961548:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.961549:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.961550:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677447.961554:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.961556:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.961556:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:26.0:1619677447.961558:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.961558:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:26.0:1619677447.961559:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677447.961559:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.961559:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.961560:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:6.0:1619677447.961561:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.961562:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.961562:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.961563:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.961563:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677447.961564:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.961577:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.961579:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.961579:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.961580:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.961581:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.961581:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.961582:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.961582:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.961583:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.961584:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.961584:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.961587:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677447.961855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.961861:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.961863:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.961864:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.961866:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.961868:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.961869:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677447.961870:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.961871:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.961872:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.961873:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677447.961876:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.961878:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.961881:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.961882:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677447.961882:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.961884:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.961885:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.961885:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.961886:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677447.961896:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677447.961899:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677447.961900:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677447.961900:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.961902:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.961905:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677447.961928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:4.2:1619677447.961928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.961933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:17.0:1619677447.961934:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677447.961935:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677447.961936:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:17.0:1619677447.961937:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:26.0:1619677447.961938:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.961938:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677447.961938:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677447.961939:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:17.0:1619677447.961941:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.961942:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.961942:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677447.961943:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.961944:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.961945:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.961946:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.961946:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677447.961948:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677447.961948:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677447.961948:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677447.961949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:17.0:1619677447.961949:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.961950:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.961950:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.961950:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.961952:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.961952:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677447.961953:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.961953:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.961954:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.961954:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.961954:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:26.0:1619677447.961956:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.961957:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.961958:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:17.0:1619677447.961958:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.961959:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.961960:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:2.0:1619677447.961961:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677447.961962:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.961963:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.961964:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.961964:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:26.0:1619677447.961965:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.961965:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:2.0:1619677447.961965:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:26.0:1619677447.961967:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677447.961967:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.961967:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677447.961967:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96721201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.961968:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.961969:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.961969:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677447.961969:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:2.0:1619677447.961969:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677447.961970:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.961970:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000400:00000200:2.0:1619677447.961970:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.961971:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677447.961971:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:2.0:1619677447.961971:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.961972:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (23)++
00000800:00000200:20.0:1619677447.961972:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.961973:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:26.0:1619677447.961974:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.961974:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.0:1619677447.961974:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:26.0:1619677447.961975:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677447.961975:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677447.961976:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.961977:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:21.2:1619677447.961977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.2:1619677447.961977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677447.961977:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.961978:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677447.961978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (22)++
00000800:00000200:13.0:1619677447.961979:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677447.961979:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.961980:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677447.961981:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:13.0:1619677447.961982:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:2.0:1619677447.961982:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677447.961982:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:13.0:1619677447.961983:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.961983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.961983:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.961984:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.961984:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677447.961984:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:13.0:1619677447.961985:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.961985:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677447.961986:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677447.961986:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677447.961986:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (23)++
00000800:00000200:1.0:1619677447.961986:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.961987:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.961989:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.0:1619677447.961989:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (24)--
00000400:00000200:5.0:1619677447.961990:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.961991:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:30.0:1619677447.961992:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677447.961992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:5.0:1619677447.961992:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.961993:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677447.961993:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:5.0:1619677447.961994:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000400:00000200:30.0:1619677447.961995:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.961995:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677447.961995:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:30.0:1619677447.961996:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:5.0:1619677447.961996:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:30.0:1619677447.961997:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96721200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.961997:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.961997:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677447.961998:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677447.961998:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:30.0:1619677447.961999:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.961999:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:5.0:1619677447.961999:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.962000:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.962000:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:6.0:1619677447.962000:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.962001:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:10.0:1619677447.962001:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:30.0:1619677447.962002:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677447.962002:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.962003:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:6.0:1619677447.962003:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:26.0:1619677447.962005:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677447.962005:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (23)++
00000800:00000200:20.0:1619677447.962005:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (24)--
00000400:00000200:30.0:1619677447.962006:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.962006:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:6.0:1619677447.962006:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.962007:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.962008:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.962009:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:30.0:1619677447.962011:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.962011:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:26.0:1619677447.962011:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.962012:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677447.962013:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677447.962013:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:30.0:1619677447.962014:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677447.962014:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.962015:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677447.962015:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.962016:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:26.0:1619677447.962016:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.962017:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677447.962020:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677447.962022:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677447.962026:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.962026:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.962027:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677447.962027:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677447.962027:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.962028:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.962029:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677447.962030:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:10.0:1619677447.962030:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.962031:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677447.962032:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:10.0:1619677447.962032:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.962033:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:8.2:1619677447.962035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.962042:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.962045:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.962046:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.962047:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.962048:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.962051:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677447.962285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677447.962291:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.962297:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.962299:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:18.0:1619677447.962301:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.962304:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.962306:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.962307:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.962308:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.962309:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.962310:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.962314:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.962316:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.962317:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.962318:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.962319:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.962321:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.962321:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.962322:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.962322:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.962328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.962335:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.962337:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.962338:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.962339:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.962340:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.962342:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677447.962388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.962393:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.962396:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.962397:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.962399:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.962401:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.962403:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.962404:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.962406:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.962406:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.962408:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.962411:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.962413:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.962415:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.962416:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.962417:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.962418:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.962419:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.962420:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.962421:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.962421:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677447.962425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677447.962426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.962431:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677447.962433:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:6.0:1619677447.962433:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.962433:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.962434:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677447.962435:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.962436:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.962437:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.962438:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.962441:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677447.962549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.962555:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.962558:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.962559:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.962561:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.962564:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.962565:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.962567:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.962568:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.962569:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.962570:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.962574:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.962575:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.962578:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.962579:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.962579:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.962581:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.962582:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.962582:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.962583:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677447.962585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.962593:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677447.962595:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:10.0:1619677447.962595:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.962596:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677447.962597:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.962598:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.962599:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.962600:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:10.0:1619677447.962601:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.962602:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.962603:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:10.0:1619677447.962603:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677447.962604:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.962605:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.962606:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.962607:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677447.962610:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.962612:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.962614:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.962614:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677447.962615:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677447.962616:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.962617:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.962618:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.962618:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677447.962628:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.962630:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.962630:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.962631:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677447.962632:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.962633:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677447.963060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.963066:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.963069:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.963071:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.963071:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.963073:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.963075:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677447.963076:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.963077:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.963078:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.963079:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677447.963083:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.963085:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677447.963088:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.963089:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677447.963089:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677447.963091:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677447.963092:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.963093:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.963094:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.963095:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.2:1619677447.963100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.963103:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677447.963105:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.963106:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.963107:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677447.963109:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.963111:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.963112:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.963114:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677447.963115:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.963117:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677447.963189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.963195:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.963198:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.963198:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.963201:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.963203:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.963205:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.963206:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.963207:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.963208:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.963209:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.963213:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.963214:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.963217:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.963218:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.963219:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.963221:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.963221:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.963222:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.963222:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.963227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.963234:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.963237:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.963238:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.963238:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.963240:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.963243:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677447.963283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677447.963290:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.963292:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.963293:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.963300:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.963303:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.963305:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.963306:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.963306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:8.0:1619677447.963307:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.963308:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.963309:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.963313:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:8.0:1619677447.963313:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.963315:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.963316:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.963317:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677447.963318:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.963318:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:10.0:1619677447.963319:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.963319:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677447.963320:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.963321:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:10.0:1619677447.963322:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:8.0:1619677447.963322:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.963322:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:10.0:1619677447.963323:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.963323:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:10.0:1619677447.963324:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.963325:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.963326:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:9.2:1619677447.963328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677447.963330:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.963331:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677447.963334:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.963335:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.963335:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:2.0:1619677447.963336:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677447.963337:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.963337:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.963338:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.963338:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677447.963339:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.963340:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.963340:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.963342:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.963345:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677447.963350:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.963352:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.963353:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.963355:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.963357:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.963358:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.963471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.963478:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.963481:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.963483:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.963485:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.963485:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.963487:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677447.963488:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.963489:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.963490:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.963491:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677447.963494:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.963496:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677447.963499:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.963500:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677447.963500:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677447.963502:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677447.963502:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.963503:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.963504:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677447.963508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.963516:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.2:1619677447.963519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677447.963519:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.963520:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.963520:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.963522:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677447.963525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.963527:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677447.963528:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677447.963530:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:17.0:1619677447.963530:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677447.963532:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.963532:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677447.963533:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677447.963533:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.963533:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.963534:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.963534:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:31.0:1619677447.963535:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.963537:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.963538:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.963539:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.963540:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.963541:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.963544:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.963546:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.963548:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.963549:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.963549:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.963551:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.963552:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.963554:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.963555:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.963556:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677447.963561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677447.963564:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.963567:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.963567:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.963569:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:12.0:1619677447.963569:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.963571:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.963572:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.963572:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:13.0:1619677447.963572:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.963573:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:4.2:1619677447.963580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.963587:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.963590:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.963591:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.963593:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.963595:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.963596:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.963598:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.963599:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.963600:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.963601:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.963604:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.963606:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.963608:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.963609:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.963610:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.963611:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.963612:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.963612:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.963613:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.963617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677447.963625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677447.963625:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677447.963628:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677447.963628:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.963629:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.963629:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.963630:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:1.0:1619677447.963631:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.963633:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.963633:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677447.963634:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:20.0:1619677447.963635:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.963637:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.963638:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.963639:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.963640:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.963641:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.963644:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.963646:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.963649:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.963650:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.963650:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.963652:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.963653:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.963654:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.963654:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.963665:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.963668:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.963668:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.963670:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.963671:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.963673:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.963781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.963786:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.963789:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.963790:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.963792:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.963794:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.963796:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677447.963797:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.963799:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.963800:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.963801:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677447.963804:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.963806:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677447.963809:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.963809:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.963810:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677447.963812:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.963813:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.963814:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.963814:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.963825:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.963828:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.963829:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.963830:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.963832:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.963833:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.963878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.963884:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.963887:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.963887:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.963889:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.963891:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.963893:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.963894:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.963895:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.963896:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.963897:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.963901:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.963902:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.963904:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.963905:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.963906:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.963908:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.963908:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.963909:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.963910:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.963914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677447.963920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677447.963921:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.963923:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.963924:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.963925:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.963926:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.963927:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677447.963927:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:26.0:1619677447.963930:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.963931:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.963933:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677447.963933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:26.0:1619677447.963935:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.963937:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677447.963938:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.963939:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.963940:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.963941:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:18.0:1619677447.963941:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.963944:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.963944:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000400:00000200:26.0:1619677447.963945:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.963945:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.963945:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.963946:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:26.0:1619677447.963948:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677447.963950:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.963951:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677447.963952:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677447.963953:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677447.963954:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.963954:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.963955:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677447.963976:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.963979:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.963980:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.963981:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.963983:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.963984:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.963996:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.963998:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.963999:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.964000:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677447.964000:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.964002:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677447.964129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.964136:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.964139:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.964140:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.964141:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.964144:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.964145:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.964147:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.964148:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.964149:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.964150:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677447.964153:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.964154:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677447.964157:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.964158:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.964158:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.964160:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.964161:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.964161:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.964162:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677447.964175:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.964177:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.964178:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.964179:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.964181:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.964184:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.964204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.964210:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.964213:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.964213:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.964216:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.964218:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.964220:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.964221:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.964222:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.964223:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.964224:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677447.964228:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.964229:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.964232:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.964233:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.964233:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.964235:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.964236:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.964237:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.964238:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.964238:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.964241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.2:1619677447.964243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677447.964247:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:17.0:1619677447.964249:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677447.964250:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:17.0:1619677447.964251:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:24.0:1619677447.964252:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.964252:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677447.964252:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.964253:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677447.964254:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.964258:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677447.964274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.964279:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.964282:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.964283:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.964284:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.964287:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.964288:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677447.964289:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.964291:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.964292:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.964292:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677447.964300:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.964302:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.964304:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.964305:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.964306:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.964307:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.964308:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.964308:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.964309:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.964314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.964323:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.964325:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.964326:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.964328:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.964329:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.964330:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677447.964333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677447.964341:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.964343:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.964344:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.964345:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.964345:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.964347:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.964360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:21.2:1619677447.964367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:31.0:1619677447.964368:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.964371:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.964373:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.964374:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:31.0:1619677447.964375:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.964376:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.964377:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:24.0:1619677447.964377:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.964378:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:31.0:1619677447.964379:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.964380:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.964380:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.964381:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.964382:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:24.0:1619677447.964382:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.964384:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.964385:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.964386:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.964387:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.964388:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:24.0:1619677447.964388:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.964389:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677447.964392:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.964392:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.964393:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.964394:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:24.0:1619677447.964394:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677447.964395:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.964396:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.964397:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.964397:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:24.0:1619677447.964397:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.964397:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.964398:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677447.964400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:24.0:1619677447.964401:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.964401:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.2:1619677447.964402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:24.0:1619677447.964403:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.964405:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.964406:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:30.0:1619677447.964407:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:9.2:1619677447.964408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677447.964410:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.0:1619677447.964411:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:30.0:1619677447.964412:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677447.964412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:31.0:1619677447.964414:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.964414:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.964415:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:5.0:1619677447.964415:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:30.0:1619677447.964416:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96721178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.964416:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.964416:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677447.964417:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:30.0:1619677447.964418:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.964418:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.964419:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.964419:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:30.0:1619677447.964420:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.964420:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:2.0:1619677447.964420:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.964421:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.964422:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:5.0:1619677447.964422:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.0:1619677447.964423:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.964424:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:2.0:1619677447.964424:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.964425:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.964427:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677447.964428:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.964431:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.964432:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677447.964432:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677447.964434:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.964435:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.964436:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.964436:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677447.964448:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.964451:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.964452:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.964452:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.964454:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.964457:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.964469:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.964471:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.964472:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.964472:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.964473:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.964474:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.964501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.964504:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.964506:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.964509:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.964509:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.964511:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.964513:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.964514:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.964516:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.964517:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.964518:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.964521:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.964523:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.964526:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.964526:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.964527:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.964529:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.964530:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.964530:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.964531:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677447.964535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.964543:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.964545:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.964546:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.964546:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.964548:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.964551:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.964758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.964764:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.964767:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.964768:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.964770:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.964773:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.964775:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.964776:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.964777:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.964778:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.964779:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.964783:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.964784:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.964787:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.964788:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.964789:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.964791:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.964791:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.964792:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.964793:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.964804:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.964807:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.964808:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.964808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:13.0:1619677447.964810:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.964811:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.964813:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677447.964816:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.964818:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.964819:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.964821:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.964821:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:26.0:1619677447.964824:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.964824:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.964824:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.964825:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.964825:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:26.0:1619677447.964826:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:12.0:1619677447.964826:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:26.0:1619677447.964827:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.964828:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.964829:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.964830:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677447.964834:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.964835:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.964839:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.964839:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.964840:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677447.964842:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.964842:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.964843:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.964845:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.964845:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677447.964851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.964856:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677447.964858:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677447.964859:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677447.964859:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677447.964859:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.964860:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.964860:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.964861:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677447.964862:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.964865:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.964872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.964880:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.964882:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.964883:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.964885:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.964888:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.964889:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.964890:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.964892:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.964893:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.964893:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.964897:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.964898:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.964901:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.964902:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.964902:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.964904:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.964904:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.964905:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.964905:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.964910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.964917:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.964920:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.964921:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.964921:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.964923:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.964926:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.965045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.965053:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.965056:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.965058:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.965059:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.965060:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.965062:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.965063:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.965064:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.965065:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.965066:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.965069:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.965071:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.965073:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.965074:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.965074:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.965076:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.965076:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.965077:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.965078:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.965087:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.965090:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.965091:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.965092:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.965094:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.965095:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677447.965155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:20.2:1619677447.965160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677447.965161:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.965164:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.965165:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677447.965166:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:2.0:1619677447.965166:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.965168:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:2.0:1619677447.965169:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.965170:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.0:1619677447.965171:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.965171:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.965172:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.965173:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.965174:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.965174:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.965175:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.965176:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677447.965177:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.965179:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.965180:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.965180:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.965181:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:2.0:1619677447.965182:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:21.2:1619677447.965184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677447.965185:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.965186:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.965187:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677447.965188:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.965189:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.965189:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677447.965190:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:2.0:1619677447.965190:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.965191:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:30.0:1619677447.965192:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677447.965192:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.965193:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:30.0:1619677447.965195:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.2:1619677447.965195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677447.965195:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:30.0:1619677447.965197:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.965199:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.965199:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.965199:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.965200:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:30.0:1619677447.965201:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:5.0:1619677447.965201:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677447.965202:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96721170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.965202:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.965202:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677447.965203:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677447.965203:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:30.0:1619677447.965204:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.965205:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.965205:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:30.0:1619677447.965206:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.965206:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:17.0:1619677447.965206:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.965206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677447.965207:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677447.965208:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.965209:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.965210:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.965210:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:30.0:1619677447.965211:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.965211:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.965211:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.965213:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677447.965213:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:26.0:1619677447.965214:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.965214:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.965214:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:26.0:1619677447.965215:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:30.0:1619677447.965216:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.965216:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677447.965217:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:26.0:1619677447.965217:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.965218:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:26.0:1619677447.965218:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.965219:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.965220:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.965220:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677447.965220:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:30.0:1619677447.965222:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.965223:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:26.0:1619677447.965223:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.965224:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677447.965225:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.965226:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.965227:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.965227:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:8.2:1619677447.965227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677447.965228:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.965229:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.965229:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:21.2:1619677447.965229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677447.965230:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677447.965233:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677447.965234:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677447.965236:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.965237:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677447.965237:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:31.0:1619677447.965238:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:17.0:1619677447.965238:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000400:00000200:6.0:1619677447.965238:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.965238:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.965239:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677447.965240:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:17.0:1619677447.965240:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:10.0:1619677447.965240:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:6.0:1619677447.965240:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.965241:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.965242:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677447.965242:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.965243:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.965243:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:26.0:1619677447.965244:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.965245:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.965247:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677447.965311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:21.2:1619677447.965316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:8.0:1619677447.965319:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.965322:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.965323:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:18.0:1619677447.965323:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.2:1619677447.965323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:8.0:1619677447.965325:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.965326:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.965326:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:8.0:1619677447.965328:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.965329:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677447.965329:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:1.0:1619677447.965329:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677447.965331:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.965331:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.965331:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.965332:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677447.965333:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677447.965333:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.965335:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.965335:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.965335:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.965336:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.965336:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.965337:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.965337:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.965339:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677447.965339:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.965341:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721166] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.965343:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.965344:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.965345:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:1.0:1619677447.965345:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.965346:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.965349:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:1.0:1619677447.965350:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.965351:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.965351:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.965352:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677447.965353:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677447.965353:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.965354:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677447.965354:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.965355:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.0:1619677447.965355:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:1.0:1619677447.965355:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.965356:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.965356:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677447.965356:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.965356:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.965357:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.965358:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677447.965358:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.965359:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:1.0:1619677447.965359:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677447.965360:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.965361:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:1.0:1619677447.965361:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.965362:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.965365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.2:1619677447.965365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.965369:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.965372:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:17.0:1619677447.965372:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677447.965372:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:22.0:1619677447.965373:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.965374:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.965375:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:17.0:1619677447.965375:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:2.0:1619677447.965375:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:17.0:1619677447.965376:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.965377:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.965377:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.965377:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.965378:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:17.0:1619677447.965379:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.965380:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.965381:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.965383:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677447.965415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.965421:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:4.2:1619677447.965421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677447.965424:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.965424:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.965426:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.965429:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677447.965429:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.965430:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:26.0:1619677447.965432:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.965432:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677447.965432:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.965434:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.965435:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.965436:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.965437:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.965439:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.965440:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.965441:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.965443:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677447.965443:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.965444:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.965445:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.965447:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.965447:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.965448:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.965448:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.965450:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:26.0:1619677447.965451:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.965452:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677447.965453:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.965453:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.965454:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:26.0:1619677447.965456:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.965457:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.965458:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677447.965459:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.965460:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.965461:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.965461:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677447.965464:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.965467:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.965468:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.965470:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.965472:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677447.965472:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:31.0:1619677447.965473:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.965473:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.965474:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.965474:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.965475:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.965475:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:22.0:1619677447.965475:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.965476:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.965477:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.965479:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.965481:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677447.965490:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.965492:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.965493:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.965493:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.965494:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.965495:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.965515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.965523:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.965526:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.965528:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.965530:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.965531:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.965532:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.965534:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.965535:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.965536:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.965537:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.965540:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.965542:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.965544:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.965545:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.965545:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.965547:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.965547:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.965548:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.965549:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677447.965559:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.965561:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.965562:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.965563:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.965565:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.965566:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677447.965668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.965676:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.965678:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.965679:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.965681:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.965684:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.965685:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.965686:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.965688:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.965689:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.965690:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.965693:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.965694:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.965697:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.965698:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.965698:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.965700:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.965700:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.965701:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.965703:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.965703:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.965706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.965709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.965714:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677447.965716:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677447.965717:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.965717:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:1.0:1619677447.965717:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677447.965718:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677447.965718:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.965720:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.965720:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.965723:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.965724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.965732:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.965734:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.965735:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.965735:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.965736:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.965737:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.965778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.965784:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.965787:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.965787:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.965790:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.965792:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.965794:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.965795:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.965796:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.965797:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.965798:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.965802:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.965803:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.965807:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.965808:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.965808:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.965810:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.965811:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.965812:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.965813:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.965813:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677447.965819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:13.0:1619677447.965824:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.965824:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.965826:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.965826:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:13.0:1619677447.965827:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677447.965827:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:13.0:1619677447.965828:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.965829:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.965831:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.965832:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.965847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.965849:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.965852:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.965854:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.965855:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.965856:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.965858:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.965859:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.965861:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.965862:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.965863:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.965866:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.965868:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.965870:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.965871:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.965872:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.965873:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.965874:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.965876:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.965877:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.965877:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.965882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677447.965887:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677447.965888:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:22.0:1619677447.965890:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677447.965890:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:22.0:1619677447.965891:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.965891:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.965891:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:22.0:1619677447.965892:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.965892:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.965895:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.965925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.2:1619677447.965930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677447.965933:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.965936:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.965937:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677447.965938:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677447.965939:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.965941:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677447.965941:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.965943:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.965943:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:30.0:1619677447.965944:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677447.965944:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.965945:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.965946:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677447.965946:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.965947:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.965947:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.965948:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677447.965949:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.965950:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.965951:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677447.965952:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.965954:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:31.0:1619677447.965955:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.965957:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.965958:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.965958:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677447.965959:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.965959:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677447.965960:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:31.0:1619677447.965961:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (23)++
00000800:00000200:10.0:1619677447.965961:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.965962:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677447.965963:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:31.0:1619677447.965963:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677447.965963:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:31.0:1619677447.965964:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677447.965964:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:31.0:1619677447.965965:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677447.965965:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677447.965970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.965975:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)++
00000800:00000200:26.0:1619677447.965977:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (24)--
00000400:00000200:26.0:1619677447.965977:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.965979:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:26.0:1619677447.965980:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.965980:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677447.965981:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.965982:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677447.965982:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.965983:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677447.965983:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.965983:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.965984:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.965985:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677447.965985:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.965986:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)++
00000800:00000200:26.0:1619677447.965986:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (24)--
00000800:00000200:26.0:1619677447.965987:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.965988:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.965988:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677447.966058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:21.2:1619677447.966062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:26.0:1619677447.966065:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.966067:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677447.966068:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:6.0:1619677447.966068:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:26.0:1619677447.966070:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.966071:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.966071:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:26.0:1619677447.966073:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.966073:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.966074:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677447.966075:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.966076:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.966077:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.966077:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.966078:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.966078:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721156] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.966079:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:6.0:1619677447.966080:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.966081:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.966083:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.2:1619677447.966085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677447.966086:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.966088:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:6.0:1619677447.966088:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.966090:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.966091:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.966091:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677447.966092:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677447.966093:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677447.966094:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.966094:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.966094:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:26.0:1619677447.966095:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677447.966095:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:6.0:1619677447.966095:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.966096:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.966097:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:1.0:1619677447.966097:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.966098:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:6.0:1619677447.966098:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.966098:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:1.0:1619677447.966098:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.966099:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:6.0:1619677447.966099:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.966100:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:1.0:1619677447.966100:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:8.2:1619677447.966101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:1.0:1619677447.966101:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96721155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.966102:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.966103:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677447.966104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:1.0:1619677447.966104:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.966107:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.966109:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:1.0:1619677447.966109:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677447.966111:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677447.966111:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:1.0:1619677447.966111:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.966112:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:5.0:1619677447.966112:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.966112:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.966113:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677447.966114:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677447.966114:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677447.966114:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:5.0:1619677447.966114:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.966114:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:30.0:1619677447.966115:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.966115:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:10.0:1619677447.966115:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:1.0:1619677447.966115:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.966115:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.966116:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.966117:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677447.966117:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.966117:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:30.0:1619677447.966120:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.2:1619677447.966120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.966128:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.966130:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.966131:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.966132:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.966133:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.966135:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.966151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.966159:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.966162:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.966164:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.966166:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.966166:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.966168:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.966169:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.966170:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.966171:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.966172:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.966176:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.966177:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.966180:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.966181:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.966181:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.966183:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.966183:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.966184:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.966186:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.966187:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677447.966192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.966196:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.966198:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.966199:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.966200:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:30.0:1619677447.966200:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.966201:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.966201:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.966202:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.966203:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.966204:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:4.2:1619677447.966248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.966255:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.966258:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.966259:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.966261:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.966263:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.966264:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.966266:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.966267:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.966268:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.966269:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.966272:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.966274:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.966276:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.966277:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.966278:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.966279:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.966280:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.966280:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.966281:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.966285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.966293:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.966299:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.966299:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.966301:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.966303:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.966305:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.966311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.966319:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.966322:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.966323:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.966325:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.966328:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.966329:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677447.966330:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.966332:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.966333:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.966334:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677447.966337:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.966338:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677447.966341:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.966342:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677447.966343:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677447.966345:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677447.966345:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.966346:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.966346:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677447.966351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.966359:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.966361:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.966362:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.966364:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.966365:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.966367:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.966453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.966458:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.966461:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.966463:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.966464:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.966465:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.966467:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.966468:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.966469:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.966470:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.966471:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.966475:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.966476:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.966479:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.966479:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.966480:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.966481:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.966482:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.966483:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.966484:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.966496:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.966499:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.966500:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.966500:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.966502:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.966504:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.966680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.2:1619677447.966681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677447.966688:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.966690:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:31.0:1619677447.966691:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677447.966691:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677447.966694:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677447.966694:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.966696:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.966697:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.966698:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.966698:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677447.966698:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677447.966700:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677447.966700:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.966701:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.966702:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.966703:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.966704:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.966705:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.966705:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677447.966706:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677447.966710:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.966711:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.966713:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.966713:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677447.966715:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.966716:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.966716:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.966716:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:31.0:1619677447.966717:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.966717:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677447.966718:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.966719:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677447.966719:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677447.966720:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.966720:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677447.966720:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.966720:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.966721:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677447.966726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.966731:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:18.0:1619677447.966734:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677447.966734:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.966734:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.966735:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.966736:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.966737:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.966738:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.966739:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.966739:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:18.0:1619677447.966741:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.966745:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.966746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.966752:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.966755:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.966756:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.966756:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.966757:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.966758:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.966864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.966870:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.966873:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.966873:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.966875:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.966877:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.966878:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:4.2:1619677447.966878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:17.0:1619677447.966880:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.966881:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.966882:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.966883:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:8.0:1619677447.966885:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:17.0:1619677447.966887:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.966888:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.966889:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677447.966889:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.966891:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.966892:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:8.0:1619677447.966892:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.966893:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.966895:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.966895:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677447.966895:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.966896:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:17.0:1619677447.966897:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:8.0:1619677447.966897:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.966898:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.966898:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677447.966899:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.966899:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.966901:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.2:1619677447.966903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677447.966904:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.966906:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.966908:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.0:1619677447.966909:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.966909:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.966909:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677447.966910:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.966911:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677447.966911:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.0:1619677447.966912:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.966912:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:6.0:1619677447.966912:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.966913:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:8.0:1619677447.966913:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677447.966913:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.966914:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677447.966914:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677447.966915:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.2:1619677447.966917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.966917:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677447.966925:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.966928:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.966929:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.966929:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.966931:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.966934:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.966985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.966993:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.966996:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.966998:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.967000:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.967000:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.967002:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.967003:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.967005:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.967006:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.967007:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.967010:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.967012:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.967015:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.967016:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.967017:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.967018:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.967019:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.967020:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.967020:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.967025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.967033:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.967035:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.967036:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.967038:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.967039:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.967041:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677447.967064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.967070:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.967072:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.967073:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.967075:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.967077:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.967079:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.967080:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.967081:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.967082:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.967083:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.967086:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.967088:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.967091:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.967091:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.967092:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.967094:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.967095:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.967096:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.967097:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.967097:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677447.967100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677447.967102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.967107:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:17.0:1619677447.967108:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677447.967109:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:17.0:1619677447.967110:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:6.0:1619677447.967110:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.967111:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:17.0:1619677447.967112:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.967112:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.967113:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.967116:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.967122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.967129:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.967132:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.967133:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.967133:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.967134:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.967135:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677447.967186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677447.967192:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.967195:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.967196:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.967198:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.967201:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.967202:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.967204:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.967205:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677447.967205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:18.0:1619677447.967206:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.967207:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.967210:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.967212:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677447.967212:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.967215:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:18.0:1619677447.967215:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.967216:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677447.967216:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677447.967217:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:26.0:1619677447.967218:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.967219:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.967219:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.967220:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:26.0:1619677447.967221:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.967221:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:26.0:1619677447.967222:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677447.967224:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.967225:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.967226:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.967227:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677447.967232:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.967232:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.967233:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.967234:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:10.0:1619677447.967234:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.967237:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677447.967237:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.967238:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677447.967239:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.967240:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.967241:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677447.967242:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.967242:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.967243:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.967244:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677447.967248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.967255:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.967257:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.967258:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.967259:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.967261:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.967264:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.967271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.967277:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.967279:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.967282:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.967282:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.967284:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.967285:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677447.967287:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.967288:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.967289:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.967290:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677447.967298:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.967300:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677447.967303:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.967304:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677447.967304:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677447.967306:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677447.967307:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.967307:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.967308:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677447.967320:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.967323:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.967324:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.967326:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.967327:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.967329:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677447.967401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677447.967407:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.967410:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.967411:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.967413:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.967415:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.967416:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677447.967418:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.967419:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.967420:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.967421:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677447.967424:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.967426:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677447.967429:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.967430:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677447.967430:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677447.967432:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677447.967433:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.967434:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.967434:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677447.967445:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677447.967448:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677447.967449:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677447.967449:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.967451:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:10.0:1619677447.967454:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:26.0:1619677447.967463:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677447.967466:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677447.967466:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677447.967467:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677447.967467:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677447.967468:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677447.967564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.967571:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.967574:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.967574:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.967576:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.967578:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.967580:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.967581:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.967583:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.967584:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.967585:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.967588:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.967590:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.967593:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.967594:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.967594:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.967596:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.967596:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.967597:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.967598:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.967610:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.967612:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.967613:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.967615:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.967617:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.967618:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.967714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:4.2:1619677447.967718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.967721:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.967724:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:10.0:1619677447.967725:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677447.967725:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.967727:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.967727:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.967728:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:24.0:1619677447.967729:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.967730:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.967731:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.967732:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.967732:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.967734:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.967734:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.967735:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.967735:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.967736:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677447.967737:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.967738:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.967739:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.967740:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.967742:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:10.0:1619677447.967742:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.967744:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.967745:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.967746:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.967747:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677447.967748:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.967748:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.967749:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.967749:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677447.967749:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.967750:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.967750:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677447.967750:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677447.967751:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.967752:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677447.967753:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.967754:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677447.967755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.2:1619677447.967759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677447.967759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677447.967762:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.967765:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.967765:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677447.967766:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.967767:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:20.0:1619677447.967767:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:1.0:1619677447.967768:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.967770:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:20.0:1619677447.967770:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677447.967771:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677447.967771:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.967771:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:10.0:1619677447.967772:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:20.0:1619677447.967773:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.967773:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.967775:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.967776:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.967776:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (23)++
00000800:00000200:5.0:1619677447.967777:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.967778:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.967779:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677447.967866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.967873:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.967876:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.967878:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.967880:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.967881:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.967882:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677447.967884:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.967885:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.967886:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.967887:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677447.967890:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.967892:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677447.967895:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.967896:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677447.967896:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677447.967897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:30.0:1619677447.967898:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677447.967898:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.967900:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.967900:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677447.967903:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:8.2:1619677447.967905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.967906:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:17.0:1619677447.967909:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.967909:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677447.967911:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.967912:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.967913:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.967913:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.967915:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677447.967915:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.967917:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.967918:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677447.967919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:17.0:1619677447.967919:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.967920:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:1.0:1619677447.967923:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.967924:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.967924:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677447.967925:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677447.967927:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677447.967928:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.967929:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677447.967930:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677447.967931:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.967931:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:17.0:1619677447.967932:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:22.0:1619677447.967934:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.967934:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:17.0:1619677447.967934:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:22.0:1619677447.967935:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:17.0:1619677447.967935:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677447.967936:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:22.0:1619677447.967937:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.967938:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.967939:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.967940:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.967943:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.967944:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677447.967946:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:22.0:1619677447.967947:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.967948:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:22.0:1619677447.967948:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.967948:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677447.967949:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677447.967950:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.967950:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.967951:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.967951:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:26.0:1619677447.967952:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.967952:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:26.0:1619677447.967955:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:8.2:1619677447.967956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.967963:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.967966:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.967967:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.967967:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.967969:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.967972:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.967997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677447.968005:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.968008:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677447.968009:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677447.968010:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677447.968013:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.968014:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677447.968015:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.968016:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.968017:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.968018:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677447.968021:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677447.968023:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677447.968026:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.968027:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677447.968027:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677447.968029:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677447.968029:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.968030:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677447.968031:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677447.968035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.968043:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.968045:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.968046:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.968048:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.968048:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.968050:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677447.968205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:4.2:1619677447.968205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.968211:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:26.0:1619677447.968212:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.968214:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:26.0:1619677447.968215:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:18.0:1619677447.968215:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:10.0:1619677447.968216:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677447.968219:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.968220:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.968222:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.968223:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.968223:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677447.968225:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.968225:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.968227:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.968227:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.968229:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.968229:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.968230:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677447.968231:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.968232:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:5.2:1619677447.968234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:2.0:1619677447.968234:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.968236:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:26.0:1619677447.968239:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.968240:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677447.968241:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:21.2:1619677447.968242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677447.968242:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.968242:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:17.0:1619677447.968243:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:2.0:1619677447.968243:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.968244:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.968244:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677447.968245:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677447.968246:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000400:00000200:17.0:1619677447.968246:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.968246:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.2:1619677447.968247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677447.968247:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.968248:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.968248:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:26.0:1619677447.968249:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677447.968249:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677447.968249:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.968249:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:26.0:1619677447.968250:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677447.968250:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:17.0:1619677447.968250:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:2.0:1619677447.968250:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:26.0:1619677447.968251:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677447.968251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (22)++
00000400:00000200:17.0:1619677447.968252:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96721131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.968252:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:18.0:1619677447.968253:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.968254:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.968255:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:18.0:1619677447.968256:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.968256:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.968257:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:17.0:1619677447.968257:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677447.968259:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.968259:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.968260:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000400:00000200:18.0:1619677447.968261:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96721130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.968261:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.968261:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000400:00000200:5.0:1619677447.968262:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.968262:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.968263:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.968264:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.968264:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:5.0:1619677447.968265:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.968265:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.968266:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677447.968266:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96721129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.968267:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.968268:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.968268:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:5.0:1619677447.968268:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.968270:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.968270:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.968270:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677447.968270:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.968271:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677447.968271:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.968272:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677447.968273:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.968274:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.968274:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.968275:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677447.968275:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.968275:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.968276:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000400:00000200:2.0:1619677447.968276:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.968277:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000800:00000200:5.0:1619677447.968278:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.968278:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.968280:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (24)--
00000800:00000200:18.0:1619677447.968280:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.968281:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677447.968281:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.968282:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677447.968282:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.968282:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.968282:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:6.0:1619677447.968283:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677447.968283:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677447.968284:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:9.2:1619677447.968285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677447.968285:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.968285:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677447.968286:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:5.0:1619677447.968286:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.968287:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.968287:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.968289:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.968292:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677447.968293:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.968297:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.968298:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.968299:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.968299:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:18.0:1619677447.968301:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.968302:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677447.968303:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:13.0:1619677447.968306:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.968306:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677447.968308:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.968309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677447.968311:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.968316:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.968319:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.968320:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.968320:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.968321:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.968322:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677447.968376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.968384:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.968387:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.968389:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.968392:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.968392:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.968394:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.968395:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.968396:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.968397:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.968398:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.968402:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.968404:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.968406:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.968407:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.968408:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.968409:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.968410:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.968411:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.968413:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.968413:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677447.968419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677447.968421:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.968423:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.968424:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.968424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677447.968426:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:12.0:1619677447.968426:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.968427:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.968428:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.968429:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677447.968429:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.968430:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677447.968432:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.968435:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.968435:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677447.968437:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.968440:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.968441:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677447.968441:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:13.0:1619677447.968443:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677447.968443:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.968444:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677447.968444:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.968445:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.968445:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677447.968445:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.968446:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677447.968446:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677447.968449:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.968451:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677447.968453:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.968454:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677447.968455:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677447.968456:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677447.968456:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.968457:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.968458:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677447.968479:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.968481:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.968482:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.968484:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.968485:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.968486:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677447.968501:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.968504:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.968505:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.968505:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677447.968506:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.968507:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677447.968658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.968665:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.968668:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.968669:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.968670:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.968673:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.968674:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.968675:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.968677:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.968677:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.968678:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677447.968681:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.968683:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.968685:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.968686:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.968686:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677447.968688:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.968689:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.968689:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.968690:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677447.968702:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.968704:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.968705:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.968706:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.968708:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.968711:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.968768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.968774:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.968777:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.968778:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.968780:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.968782:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.968783:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677447.968784:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.968786:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.968787:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.968788:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677447.968791:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.968792:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677447.968795:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.968796:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677447.968796:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.968798:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677447.968799:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.968799:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.968800:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.968804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.968811:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.968814:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.968815:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.968815:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.968818:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.968820:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.2:1619677447.968838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.968841:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.968843:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.968846:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.968847:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.968848:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.968850:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.968851:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.968854:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.968855:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.968856:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.968859:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.968861:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.968864:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.968865:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.968866:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.968868:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.968868:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.968869:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.968870:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.968880:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.968883:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.968884:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.968886:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.968887:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.968888:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.968926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.968934:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.968937:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.968939:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.968940:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.968941:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.968942:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.968943:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.968945:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.968946:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.968947:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.968950:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.968951:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.968954:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.968954:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.968955:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.968957:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.968957:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.968958:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.968959:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.968968:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.968971:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.968972:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.968973:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.968975:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.968976:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.968991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677447.968999:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.969002:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.969003:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.969005:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.969008:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.969009:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.969010:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.969011:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.969012:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.969014:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.969018:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.969020:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.969023:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.969024:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.969025:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677447.969025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:24.0:1619677447.969027:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.969027:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.969028:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.969028:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:18.0:1619677447.969031:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:9.2:1619677447.969033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.969034:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:18.0:1619677447.969036:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.969037:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.969038:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.969038:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:18.0:1619677447.969040:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:2.0:1619677447.969040:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.969041:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.969041:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.969043:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.969043:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.969044:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.969045:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.969048:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.969050:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.969052:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.969053:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.969053:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677447.969054:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.969054:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.969056:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.969056:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.969057:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.969058:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.969061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.969068:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.969070:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.969071:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.969072:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.969072:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.969073:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677447.969104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677447.969109:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677447.969112:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677447.969113:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677447.969114:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.969116:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.969117:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.969118:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.969120:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.969121:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.969121:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.969124:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.969126:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.969128:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.969129:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.969130:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677447.969131:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.969132:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677447.969133:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677447.969133:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:12.0:1619677447.969144:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.969147:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.969148:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.969149:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.969150:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.969152:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.969246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.969252:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.969254:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.969257:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.969257:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.969260:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.969261:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677447.969262:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.969264:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.969265:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.969266:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.969270:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.969272:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.969274:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.969275:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.969275:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.969277:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.969277:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.969278:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.969279:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677447.969283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.969291:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.969294:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.969299:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.969300:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.969302:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677447.969305:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.969321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.969329:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.969332:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.969333:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.969335:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677447.969337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:10.0:1619677447.969337:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.969339:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677447.969340:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.969341:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.969342:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.969343:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.969345:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677447.969346:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.969348:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:10.0:1619677447.969348:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.2:1619677447.969349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677447.969350:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677447.969351:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.969351:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677447.969352:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.969353:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.969353:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.969353:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:30.0:1619677447.969354:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:10.0:1619677447.969354:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677447.969354:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677447.969355:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.969355:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.969355:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:30.0:1619677447.969357:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.969357:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:30.0:1619677447.969358:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.969358:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677447.969360:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677447.969361:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.969364:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.969364:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.969365:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:30.0:1619677447.969366:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:13.0:1619677447.969367:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677447.969367:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96721116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.969368:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.969369:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.969370:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.969370:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.969370:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.969371:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.969371:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:1.0:1619677447.969371:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:13.0:1619677447.969372:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.969373:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.969374:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677447.969374:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:1.0:1619677447.969374:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.969375:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.969375:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677447.969375:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.969377:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.2:1619677447.969378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677447.969379:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677447.969380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677447.969380:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.969381:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:12.0:1619677447.969382:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.969383:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.969383:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677447.969385:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677447.969385:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677447.969385:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677447.969386:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677447.969386:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.969386:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677447.969387:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677447.969387:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:12.0:1619677447.969387:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.969388:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.969389:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677447.969390:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.969390:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:9.2:1619677447.969390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:30.0:1619677447.969391:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677447.969391:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.969392:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.969392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:20.0:1619677447.969393:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.969394:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.969395:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96721115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677447.969396:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:31.0:1619677447.969397:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.969397:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:31.0:1619677447.969398:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.969398:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:31.0:1619677447.969399:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.969399:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.969400:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.969400:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:8.0:1619677447.969400:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.969401:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677447.969401:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.969402:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677447.969403:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.969403:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.969405:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.969406:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:31.0:1619677447.969408:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.969408:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.969409:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.969411:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.969411:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.969412:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.969413:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.969422:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.969425:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.969426:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.969427:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:4.2:1619677447.969428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.969429:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.969430:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677447.969435:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.969437:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.969438:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.969440:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.969443:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.969444:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.969445:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.969446:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.969447:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.969448:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677447.969452:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.969453:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.969456:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.969457:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.969457:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677447.969459:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.969459:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.969460:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.969460:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677447.969473:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.969476:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.969477:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.969477:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.969480:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.969484:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.969486:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.969488:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.969489:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.969489:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.969490:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.969490:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677447.969511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677447.969518:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.969521:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.969522:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.969524:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.969527:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.969528:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.969530:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.969531:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.969532:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.969533:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.969536:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.969537:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.969541:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.969541:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.969542:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.969544:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.969545:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.969546:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.969547:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.969548:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.2:1619677447.969553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:13.0:1619677447.969556:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677447.969559:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:13.0:1619677447.969559:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677447.969560:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:13.0:1619677447.969560:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.969561:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.969562:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:13.0:1619677447.969562:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.969563:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.969565:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.969628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.969634:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.969637:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.969637:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.969639:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.969642:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.969643:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.969644:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721112] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.969646:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.969647:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.969648:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.969651:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.969653:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.969656:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.969657:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.969657:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.969660:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.969660:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.969661:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.969661:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677447.969673:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.969675:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.969676:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.969678:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.969679:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.969681:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.969772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.969777:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.969780:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.969781:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.969783:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.969786:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.969787:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.969789:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.969790:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.969791:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.969792:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.969796:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.969797:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.969800:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.969801:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.969802:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.969804:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.969804:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.969805:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.969805:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.969810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.969816:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.969819:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.969819:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677447.969820:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.969822:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.969825:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.969828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677447.969834:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.969836:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.969837:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.969837:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.969838:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.969839:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677447.969846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.969852:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.969854:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.969857:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.969857:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.969859:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.969861:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677447.969862:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.969864:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.969865:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.969866:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677447.969870:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.969871:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.969875:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.969875:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677447.969876:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.969878:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.969879:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.969880:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.969882:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.969882:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.969887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.969890:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677447.969892:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677447.969892:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.969893:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.969893:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677447.969895:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.969897:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.969898:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.969900:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.969902:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.969921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.969929:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.969931:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.969934:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.969935:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.969936:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.969937:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.969939:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.969940:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.969941:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.969942:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.969945:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.969947:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.969951:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.969951:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.969952:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.969954:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.969955:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.969956:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.969956:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.969960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.969968:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.969971:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.969972:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.969974:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.969974:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.969977:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677447.969978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677447.969986:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.969989:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.969990:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.969991:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.969992:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677447.969992:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:21.2:1619677447.970153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677447.970160:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.970163:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.970163:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.970165:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677447.970167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:24.0:1619677447.970168:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.970169:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.970170:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.970172:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.970172:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:24.0:1619677447.970173:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.970174:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677447.970175:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:21.2:1619677447.970176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:1.0:1619677447.970176:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677447.970178:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.970179:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.970180:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:17.0:1619677447.970182:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.970183:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:31.0:1619677447.970184:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677447.970184:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.970185:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.970185:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:17.0:1619677447.970185:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.970187:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:24.0:1619677447.970187:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.970187:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:17.0:1619677447.970187:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.970188:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:17.0:1619677447.970188:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.970189:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:17.0:1619677447.970189:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677447.970190:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.970190:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677447.970193:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.970193:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677447.970193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677447.970195:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:17.0:1619677447.970195:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:31.0:1619677447.970196:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96721106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.970198:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.970198:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.970198:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000400:00000200:31.0:1619677447.970199:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.970199:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:31.0:1619677447.970200:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.970201:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.970201:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.970201:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677447.970202:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.970202:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677447.970203:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:31.0:1619677447.970204:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.970204:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.970206:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:20.0:1619677447.970206:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.970206:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.970207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677447.970209:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.970209:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677447.970210:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:31.0:1619677447.970210:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:31.0:1619677447.970212:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:31.0:1619677447.970213:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677447.970213:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:31.0:1619677447.970214:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677447.970214:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:8.0:1619677447.970216:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.970217:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.970217:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.970218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:8.0:1619677447.970219:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.970223:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677447.970224:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.970226:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.970227:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.970227:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.970228:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.970231:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677447.970237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677447.970243:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.970246:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677447.970246:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.970247:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.970247:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.970248:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677447.970336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.970343:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.970346:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.970348:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.970350:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.970350:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.970352:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.2:1619677447.970352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:30.0:1619677447.970353:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.970354:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.970355:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.970356:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677447.970359:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.970360:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.970361:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:30.0:1619677447.970361:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:31.0:1619677447.970363:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.970364:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.970365:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677447.970365:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:31.0:1619677447.970366:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.970367:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:30.0:1619677447.970367:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:31.0:1619677447.970368:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.970370:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.970370:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.970371:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.970372:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.970373:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.970373:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:31.0:1619677447.970374:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.970375:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.970376:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.970378:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.970378:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.970379:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.970380:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.970381:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.970383:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:6.0:1619677447.970383:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:31.0:1619677447.970384:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.970385:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:6.0:1619677447.970386:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.970387:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.970387:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.970389:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.2:1619677447.970390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:6.0:1619677447.970392:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677447.970393:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.970396:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.970397:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.970398:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:12.0:1619677447.970399:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.970401:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:13.0:1619677447.970401:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.970401:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.970402:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.970402:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.2:1619677447.970657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677447.970660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677447.970663:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.970666:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.970666:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:10.0:1619677447.970668:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:1.0:1619677447.970668:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.970671:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:1.0:1619677447.970671:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.970672:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:1.0:1619677447.970673:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.970674:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.970676:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.970676:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.970678:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.970678:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:10.0:1619677447.970679:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.970681:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677447.970682:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677447.970683:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677447.970684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:10.0:1619677447.970685:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.970685:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:10.0:1619677447.970686:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.970687:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677447.970688:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677447.970688:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.970689:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.970690:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677447.970690:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:10.0:1619677447.970691:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.970691:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.970692:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.970692:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677447.970693:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.970693:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.970694:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:10.0:1619677447.970696:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677447.970696:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.970697:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.2:1619677447.970697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677447.970698:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677447.970699:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.970700:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.970700:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:2.0:1619677447.970701:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:10.0:1619677447.970702:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:2.0:1619677447.970702:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96721101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.970703:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000400:00000200:2.0:1619677447.970703:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677447.970704:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:2.0:1619677447.970704:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.970705:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:2.0:1619677447.970705:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:9.2:1619677447.970706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677447.970708:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.970708:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.970709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:2.0:1619677447.970709:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.970709:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.970710:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.970711:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677447.970712:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677447.970715:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.970715:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.970716:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.970716:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.970717:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:2.0:1619677447.970717:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:18.0:1619677447.970718:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.970719:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:2.0:1619677447.970719:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.970720:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:18.0:1619677447.970720:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.970720:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.2:1619677447.970720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677447.970720:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.970722:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.970723:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.970724:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.970724:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.970725:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677447.970727:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677447.970729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677447.970730:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.970731:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.970731:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677447.970732:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:1.0:1619677447.970732:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.970733:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677447.970734:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:22.0:1619677447.970735:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.970736:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.970736:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677447.970736:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.970737:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:22.0:1619677447.970738:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.970738:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.970741:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.970807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.2:1619677447.970808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677447.970811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677447.970814:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677447.970816:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677447.970816:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.970817:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677447.970818:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677447.970818:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:8.0:1619677447.970820:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.970821:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.970821:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677447.970822:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.970822:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.970823:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.970824:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.970824:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.970825:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677447.970826:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:8.0:1619677447.970826:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.970827:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.970827:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.970828:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:8.0:1619677447.970828:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.970829:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.970829:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.970830:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.970830:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.970830:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677447.970831:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677447.970831:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.970832:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.970833:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:8.0:1619677447.970835:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.970837:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:30.0:1619677447.970838:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.970838:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.970840:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.0:1619677447.970840:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.970841:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.0:1619677447.970841:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677447.970842:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.970842:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677447.970843:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.970843:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.970843:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677447.970844:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.970844:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.970844:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.0:1619677447.970844:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677447.970845:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.970846:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677447.970846:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.970846:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677447.970846:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.970847:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:30.0:1619677447.970848:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:26.0:1619677447.970848:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.970849:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:8.2:1619677447.970849:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677447.970850:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.970850:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677447.970850:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677447.970852:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677447.970853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.2:1619677447.970857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:4.2:1619677447.970857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677447.970858:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.970860:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677447.970860:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.970861:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.970861:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:1.0:1619677447.970861:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677447.970862:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:24.0:1619677447.970862:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.970863:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677447.970863:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.970863:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:31.0:1619677447.970864:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.970864:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.970864:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:10.0:1619677447.970864:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.970865:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:26.0:1619677447.970865:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:30.0:1619677447.970866:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:17.0:1619677447.970866:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:31.0:1619677447.970867:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.970867:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:30.0:1619677447.970868:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677447.970868:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.970868:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.970869:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:26.0:1619677447.970869:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:30.0:1619677447.970870:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.970871:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.970871:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:26.0:1619677447.970871:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.970871:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:30.0:1619677447.970872:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:26.0:1619677447.970872:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:31.0:1619677447.970873:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677447.970873:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677447.970873:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677447.970874:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.970874:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.970875:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677447.970875:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677447.970939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.970944:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.970947:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.970948:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.970949:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.970952:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.970953:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.970954:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.970956:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.970957:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.970958:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.970961:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.970963:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.970966:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.970967:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.970967:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.970969:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.970970:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.970970:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.970971:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:26.0:1619677447.970981:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:20.2:1619677447.970983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:26.0:1619677447.970984:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677447.970985:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.970985:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677447.970988:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.970991:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677447.970991:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:30.0:1619677447.970994:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.970996:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.970997:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.970998:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.970999:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677447.971000:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.971001:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.971002:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.971004:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677447.971007:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.971009:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.971011:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.971012:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.971012:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.971014:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.971014:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.971015:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.971015:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677447.971027:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.971029:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.971030:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.971031:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.971033:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.971034:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677447.971046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677447.971053:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.971056:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677447.971057:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677447.971058:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677447.971059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:10.0:1619677447.971061:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.971062:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677447.971063:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.971065:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.971065:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.971066:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.971066:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.971069:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.971070:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:10.0:1619677447.971070:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.971071:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.971072:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.971074:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.971075:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.971075:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677447.971076:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:8.0:1619677447.971076:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677447.971077:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.971078:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677447.971078:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:8.0:1619677447.971078:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677447.971079:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677447.971079:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:8.0:1619677447.971079:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.971080:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:8.2:1619677447.971084:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:8.0:1619677447.971087:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.971089:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677447.971091:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.971092:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.971092:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.971092:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:8.0:1619677447.971094:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.971094:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:1.0:1619677447.971094:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.971095:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.971096:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000400:00000200:1.0:1619677447.971096:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.971097:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.971097:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:1.0:1619677447.971098:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.971101:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677447.971103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:13.0:1619677447.971106:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.971109:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:24.0:1619677447.971110:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:13.0:1619677447.971110:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.971111:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.971112:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.971112:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.971113:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.971114:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.971114:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677447.971117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.971123:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677447.971124:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677447.971127:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:12.0:1619677447.971127:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:2.0:1619677447.971127:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:12.0:1619677447.971128:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.971129:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:18.0:1619677447.971130:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.971130:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.971131:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:18.0:1619677447.971132:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.971134:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.971136:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.971137:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.971138:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.971139:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.971142:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.971143:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.971145:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.971146:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.971147:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677447.971147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677447.971148:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.971149:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.971149:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.971150:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677447.971153:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.2:1619677447.971154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.971156:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677447.971158:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.971158:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.971159:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:17.0:1619677447.971160:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.971160:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.971161:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.971162:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.971163:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.971163:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.971165:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.971166:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.971167:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.971169:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.971170:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:17.0:1619677447.971171:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.971173:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.971177:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.971178:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.971178:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.971180:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.971181:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.971183:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.971184:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.971184:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677447.971189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677447.971193:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.971195:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677447.971195:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:13.0:1619677447.971196:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.971197:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.971197:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677447.971198:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.971198:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.971199:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.971201:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.971220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.971227:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.971229:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.971230:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.971232:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.971235:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.971236:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.971237:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.971239:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.971240:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.971241:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.971244:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.971245:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.2:1619677447.971246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.971247:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.971249:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.971250:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.971252:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.971253:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.971254:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.971254:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.971255:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677447.971255:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.971256:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.971257:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.971260:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.971261:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677447.971262:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.971263:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.971264:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:10.0:1619677447.971264:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.971265:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677447.971266:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.971267:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.971269:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:10.0:1619677447.971269:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.971271:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.971271:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.971271:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.971274:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.971275:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677447.971276:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.971278:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.971278:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.971279:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.971280:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677447.971284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:9.2:1619677447.971284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.971286:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.971289:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.971291:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.971292:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677447.971292:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.971298:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:20.0:1619677447.971300:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.971300:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.971301:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.971302:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.971303:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721089] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.971303:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.971305:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677447.971305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677447.971306:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.971307:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:2.0:1619677447.971308:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:20.0:1619677447.971310:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.971310:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.971312:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.971313:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677447.971313:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.971313:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.971314:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.971314:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.971315:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.971316:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.971317:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.971319:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.971320:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.971321:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.971322:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677447.971333:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.971336:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.971337:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.971339:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.971341:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.971342:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677447.971431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.971439:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.971442:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.971443:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.971444:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.971447:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.971448:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.971449:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.971451:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.971452:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.971453:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.971456:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.971458:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.971461:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.971462:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.971462:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.971464:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.971464:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.971465:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.971465:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.971470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.971478:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.971481:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.971482:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.971482:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.971484:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.971487:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677447.971502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.971510:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.971513:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.971515:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.971516:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.971517:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.971518:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677447.971519:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.971520:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.971521:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.971522:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677447.971526:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.971527:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677447.971531:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.971531:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677447.971532:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677447.971534:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677447.971534:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.971535:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.971536:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677447.971548:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.971551:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.971551:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.971552:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.971554:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.971557:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.971594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.971602:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.971605:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.971605:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.971607:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.971610:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.971612:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.971613:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.971614:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.971615:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.971616:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.971619:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.971621:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.971623:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.971624:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.971624:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.971626:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.971626:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.971627:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.971627:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677447.971638:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.971640:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.971641:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.971642:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.971644:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.971644:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677447.971716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.971723:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.971726:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.971727:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.971730:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.971730:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.971732:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.971733:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.971734:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.971735:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.971736:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.971739:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.971740:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.971744:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.971744:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.971745:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.971747:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.971747:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.971748:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.971749:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.971753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677447.971755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.971761:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677447.971763:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:1.0:1619677447.971764:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.971765:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677447.971765:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.971766:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.971766:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.971768:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.971768:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.971769:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.971771:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:1.0:1619677447.971771:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.971772:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.971773:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.971774:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.971775:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.971779:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.971780:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.971783:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.971784:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.971785:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.971787:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.971787:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.971788:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.971788:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677447.971797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:30.0:1619677447.971800:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677447.971800:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677447.971803:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677447.971803:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:30.0:1619677447.971804:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.971805:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677447.971806:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:30.0:1619677447.971806:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.971806:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677447.971808:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.971808:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.971809:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:31.0:1619677447.971809:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:30.0:1619677447.971809:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:31.0:1619677447.971810:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:20.0:1619677447.971810:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:30.0:1619677447.971811:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:20.0:1619677447.971811:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.971813:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.971814:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.971815:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677447.971818:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.971820:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677447.971823:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.971824:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677447.971825:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677447.971826:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677447.971827:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.971828:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.2:1619677447.971829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:20.0:1619677447.971830:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.971830:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:9.2:1619677447.971832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.2:1619677447.971835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:6.0:1619677447.971835:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.971838:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:24.0:1619677447.971839:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:22.0:1619677447.971839:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.971840:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.971841:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:5.0:1619677447.971841:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:8.0:1619677447.971842:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.971843:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.971843:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.971843:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.971844:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:6.0:1619677447.971845:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:5.0:1619677447.971845:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:24.0:1619677447.971846:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000400:00000200:6.0:1619677447.971846:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.971847:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.971848:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.971849:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:6.0:1619677447.971849:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.971853:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677447.971853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:6.0:1619677447.971855:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.971858:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.971859:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.971859:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:8.0:1619677447.971860:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677447.971861:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:8.0:1619677447.971862:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:6.0:1619677447.971862:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677447.971863:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677447.971863:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.971864:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.971864:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:8.0:1619677447.971865:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.971868:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677447.971869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:8.0:1619677447.971869:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.971871:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.971872:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677447.971873:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:8.0:1619677447.971873:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.971874:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:8.0:1619677447.971874:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677447.971876:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.971876:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:10.0:1619677447.971876:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:22.0:1619677447.971877:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677447.971877:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.971878:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:8.0:1619677447.971878:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677447.971879:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.971880:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677447.971882:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:8.0:1619677447.971883:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.971883:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.971884:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.971886:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.971886:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.971887:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.971887:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.971892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.971900:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.971903:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.971904:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.971904:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.971906:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.971909:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.971983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.971989:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.971992:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.971993:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.971995:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.971997:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.971999:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.972000:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.972001:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.972002:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.972003:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.972006:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.972008:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.972011:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.972011:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.972012:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.972014:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.972014:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.972015:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.972016:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.972017:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677447.972020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677447.972022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.972028:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.972030:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.972030:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677447.972031:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677447.972031:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677447.972034:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.972037:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.972038:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.972039:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:5.0:1619677447.972039:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.972040:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677447.972042:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.972042:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.0:1619677447.972042:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:26.0:1619677447.972044:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.972046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:26.0:1619677447.972047:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.972049:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677447.972050:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.972052:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.972053:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677447.972054:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:26.0:1619677447.972055:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.972055:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677447.972057:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.972059:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677447.972060:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.972061:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:10.0:1619677447.972062:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677447.972063:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.972064:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.972065:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.972066:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:26.0:1619677447.972069:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677447.972069:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.972070:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677447.972071:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.972071:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677447.972074:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.972075:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.972075:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.972076:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.972076:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677447.972076:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677447.972076:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677447.972078:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.972078:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677447.972079:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.972079:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.972080:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677447.972080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.2:1619677447.972084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.972088:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.972091:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:20.0:1619677447.972092:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:18.0:1619677447.972092:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.972092:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.972094:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:18.0:1619677447.972094:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.972095:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.972097:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.972097:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677447.972098:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.972101:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.972156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.972161:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.972163:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.972166:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.972166:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.972169:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.972170:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.972172:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.972173:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.972174:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.972175:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.972178:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.972180:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.972183:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.972183:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.972184:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.972186:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.972186:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.972187:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.972187:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677447.972199:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.972202:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.972203:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.972203:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.972205:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.972207:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.972414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.972422:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.972425:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.972426:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.972428:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.972431:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.972432:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.972433:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.972434:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.972436:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.972437:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.972440:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.972441:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.972444:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.972445:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.972446:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.972447:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.972448:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.972448:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.972449:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.972453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.972461:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.972464:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.972465:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.972465:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.972468:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.972470:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.972473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.972480:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.972482:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.972483:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.972484:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.972485:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.972485:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.972495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.972500:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.972502:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.972503:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.972505:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.972507:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.972509:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.972510:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.972511:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.972512:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.972513:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.972516:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.972518:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:4.2:1619677447.972519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:1.0:1619677447.972520:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.972521:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.972521:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.972523:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.972523:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.972524:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.972524:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677447.972526:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677447.972529:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677447.972529:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000400:00000200:8.0:1619677447.972531:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.972534:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.972535:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:8.0:1619677447.972537:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.972538:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:8.0:1619677447.972538:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.972539:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.972540:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:22.0:1619677447.972541:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.972542:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.972542:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:8.0:1619677447.972543:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.972544:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.972545:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:22.0:1619677447.972547:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677447.972548:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.972548:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:8.0:1619677447.972549:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.972550:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.972551:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677447.972552:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677447.972552:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.2:1619677447.972556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:6.0:1619677447.972557:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.972560:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.972561:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.972561:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:24.0:1619677447.972562:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:6.0:1619677447.972562:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.972563:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677447.972564:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.972565:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.972566:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.972567:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.972569:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.2:1619677447.972604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.972612:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.972615:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.972617:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.972620:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.972620:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.972621:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.972623:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.972624:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.972625:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.972626:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.972629:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.972630:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.972634:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.972634:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.972635:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.972637:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.972637:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.972638:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.972639:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.972643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.972651:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.972653:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.972654:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.972656:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.972656:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.972659:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.972826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:30.0:1619677447.972834:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677447.972837:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:30.0:1619677447.972839:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.972841:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677447.972842:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.972843:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677447.972845:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.972846:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.972847:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.972848:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677447.972851:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.972853:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677447.972856:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.972857:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.972858:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.972859:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.972860:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677447.972860:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677447.972861:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:8.2:1619677447.972865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.972873:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.972876:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.972877:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.972878:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.972879:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.972881:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677447.972887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:4.2:1619677447.972888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.0:1619677447.972894:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.0:1619677447.972896:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677447.972897:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.972898:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.972898:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:8.0:1619677447.972898:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.972899:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677447.972899:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.972901:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:8.0:1619677447.972901:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.972903:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.972904:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677447.972905:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.972907:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.972908:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.972909:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677447.972912:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.972914:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677447.972916:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.972917:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.972918:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.972919:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.972920:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.972921:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.972922:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.972923:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.972925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677447.972928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677447.972933:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:21.2:1619677447.972934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:24.0:1619677447.972935:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.972936:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677447.972936:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.972936:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.972937:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677447.972937:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.972938:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.2:1619677447.972939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:17.0:1619677447.972939:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.972940:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677447.972942:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:2.0:1619677447.972943:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.972944:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677447.972945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000400:00000200:2.0:1619677447.972945:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677447.972947:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:2.0:1619677447.972948:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.972950:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:2.0:1619677447.972950:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.972952:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.972953:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:2.0:1619677447.972953:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.972953:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677447.972954:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.972955:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.972956:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:2.0:1619677447.972956:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677447.972956:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677447.972957:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.972957:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000400:00000200:31.0:1619677447.972959:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677447.972959:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.972959:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:2.0:1619677447.972960:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000400:00000200:31.0:1619677447.972961:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.972962:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.972963:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.972963:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:31.0:1619677447.972965:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.972965:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.972966:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.2:1619677447.972967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677447.972967:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677447.972969:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.972969:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.972969:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.972970:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:31.0:1619677447.972971:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.972971:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677447.972973:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677447.972974:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.972975:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.972975:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:9.2:1619677447.972975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677447.972976:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677447.972977:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.972977:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677447.972978:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.972978:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.972979:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:17.0:1619677447.972979:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.972981:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.972982:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:17.0:1619677447.972982:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:18.0:1619677447.972984:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:17.0:1619677447.972984:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96721068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.972985:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.972985:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.972986:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.972986:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.972988:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000400:00000200:17.0:1619677447.972988:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.972990:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677447.972990:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:17.0:1619677447.972991:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.972993:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.972993:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.972993:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.972995:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.972997:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.972997:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.972997:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.972998:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677447.972999:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.973001:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.973001:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.973002:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.973002:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.973013:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.973015:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.973016:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.973018:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.973019:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.973021:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.973023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.973029:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:12.0:1619677447.973031:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677447.973032:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.973032:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677447.973034:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.973034:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:22.0:1619677447.973035:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.973035:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.973036:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.973037:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:22.0:1619677447.973038:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.973039:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.973040:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.973042:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.973043:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.973044:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.973047:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.973049:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.973051:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.973052:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.973053:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.973054:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.973055:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.973056:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.973056:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.973061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.973068:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.973071:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.973071:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.973073:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.973074:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.973075:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677447.973264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.973269:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.973272:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.973273:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.973275:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.973278:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.973279:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.973281:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.973282:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.973283:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.973284:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.973287:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.973288:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.973291:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.973291:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.973292:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.973298:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.973298:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.973299:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.973300:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.973301:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677447.973306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677447.973308:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.973311:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.973312:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.973312:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:6.0:1619677447.973312:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.973314:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:6.0:1619677447.973314:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.973315:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.973316:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.973317:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.973317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.973325:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.973327:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.973328:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.973330:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.973333:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.973335:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.973336:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.973338:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.973339:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.973340:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677447.973344:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.973345:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.973348:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677447.973348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677447.973349:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.973350:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677447.973351:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.973352:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.973353:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.973354:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.973355:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677447.973356:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.973358:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.973360:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.973360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677447.973362:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.973363:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.973363:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.973365:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.973365:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:30.0:1619677447.973366:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.973367:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.973367:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.973367:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677447.973367:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:22.0:1619677447.973368:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.973368:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.973369:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.973370:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677447.973371:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.973371:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:30.0:1619677447.973372:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.973374:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677447.973374:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677447.973376:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.973377:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.973378:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.973379:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.973380:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.973381:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.973381:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.973382:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677447.973385:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.973386:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.973386:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.973387:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.973388:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677447.973392:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.2:1619677447.973394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:13.0:1619677447.973395:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.973395:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.973397:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.973399:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.973400:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.973400:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.973403:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.973404:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.973406:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.973409:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.973410:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.973412:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.973413:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.973414:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.973415:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.973418:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.973420:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.973423:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.973424:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.973425:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.973427:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.973427:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.973428:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.973428:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677447.973439:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.973442:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.973443:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.973443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000400:00000200:12.0:1619677447.973445:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.973446:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.973447:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.973449:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677447.973452:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677447.973452:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:1.0:1619677447.973455:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.973457:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.973459:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677447.973460:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.973461:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.973462:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.973463:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677447.973467:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.973468:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677447.973471:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.973471:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:1.0:1619677447.973472:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.973474:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677447.973474:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677447.973475:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677447.973475:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677447.973480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.973483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.2:1619677447.973485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:18.0:1619677447.973487:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.2:1619677447.973490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677447.973490:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:8.0:1619677447.973490:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:18.0:1619677447.973491:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.973491:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.973492:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:20.0:1619677447.973493:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:18.0:1619677447.973493:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.973494:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677447.973495:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:18.0:1619677447.973495:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677447.973496:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677447.973496:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.973499:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.973499:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.973499:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:24.0:1619677447.973501:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.973501:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677447.973502:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.973502:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.973504:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.973504:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.973505:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.973506:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.973506:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.973507:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677447.973508:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.973509:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.973509:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.973512:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.973513:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.973514:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96721059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.973515:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.973516:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.973516:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.973517:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.973517:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.973518:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.973520:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.973521:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.973522:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.973522:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.973523:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.973523:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.973524:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.973524:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.973525:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:24.0:1619677447.973526:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.973526:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.0:1619677447.973526:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.973527:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.0:1619677447.973527:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.973528:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.973529:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.973529:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.973530:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677447.973530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.973530:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677447.973531:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000800:00000200:8.2:1619677447.973532:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.973533:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (24)--
00000800:00000200:24.0:1619677447.973534:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.973535:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.973536:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.973536:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:2.0:1619677447.973538:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:17.0:1619677447.973540:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:2.0:1619677447.973541:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000800:00000200:18.0:1619677447.973542:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:17.0:1619677447.973542:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:2.0:1619677447.973542:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.973543:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000400:00000200:18.0:1619677447.973544:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.973544:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.973544:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.973544:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.973547:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.973548:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.973548:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.973549:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677447.973550:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677447.973552:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.973553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.973560:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.973563:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.973564:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.973564:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.973565:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.973566:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.973650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.973658:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.973661:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.973663:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.973664:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.973665:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.973667:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677447.973668:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.973670:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.973671:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.973672:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677447.973675:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.973677:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677447.973680:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.973681:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677447.973681:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677447.973683:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677447.973684:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.973685:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.973687:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.973687:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677447.973689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677447.973694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.2:1619677447.973694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.973701:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677447.973702:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:30.0:1619677447.973703:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:26.0:1619677447.973704:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:5.0:1619677447.973704:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:30.0:1619677447.973705:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:10.0:1619677447.973705:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:5.0:1619677447.973705:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.973706:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.973707:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:26.0:1619677447.973708:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.973708:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:5.0:1619677447.973708:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.973710:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.973711:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (23)++
00000400:00000200:26.0:1619677447.973712:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677447.973712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:20.0:1619677447.973712:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (24)--
00000800:00000200:20.0:1619677447.973712:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000400:00000200:26.0:1619677447.973713:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:5.0:1619677447.973714:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:26.0:1619677447.973715:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.973715:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:26.0:1619677447.973716:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.973717:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.973717:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:26.0:1619677447.973718:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:22.0:1619677447.973720:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.973721:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:26.0:1619677447.973723:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.973723:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.973725:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:22.0:1619677447.973726:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.973728:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677447.973728:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:26.0:1619677447.973729:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677447.973730:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:22.0:1619677447.973730:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.973731:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677447.973732:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677447.973732:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:22.0:1619677447.973732:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.973733:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677447.973733:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:4.2:1619677447.973733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.973736:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677447.973736:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.973737:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.973738:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677447.973740:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.973740:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677447.973741:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.973741:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:26.0:1619677447.973742:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.973742:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677447.973744:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:26.0:1619677447.973744:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.973744:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.973744:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:26.0:1619677447.973745:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677447.973745:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677447.973746:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:22.0:1619677447.973746:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:30.0:1619677447.973747:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.973747:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96721055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.973749:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:26.0:1619677447.973749:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.973750:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:26.0:1619677447.973750:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677447.973750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677447.973751:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.973753:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:26.0:1619677447.973754:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.973756:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677447.973758:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:26.0:1619677447.973759:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.973760:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:1.0:1619677447.973760:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:26.0:1619677447.973761:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677447.973761:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.973761:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.973762:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.973763:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677447.973763:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.973764:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.973764:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677447.973766:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677447.973782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.2:1619677447.973782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:6.0:1619677447.973786:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677447.973788:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677447.973789:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677447.973790:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677447.973790:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677447.973790:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:6.0:1619677447.973791:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.973791:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677447.973793:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.973794:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:17.0:1619677447.973794:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.973796:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:31.0:1619677447.973797:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.973797:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677447.973797:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.973798:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677447.973800:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.973800:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.973801:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.973802:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.973803:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.973803:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.973804:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.973804:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677447.973806:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.973808:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.973809:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.973811:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:31.0:1619677447.973812:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.973814:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.973815:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.973815:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.973816:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677447.973817:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.973817:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677447.973818:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.973818:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677447.973819:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.973819:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.973819:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.973820:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.973821:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.973822:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.973824:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.973825:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677447.973830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677447.973831:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.973833:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.973834:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.973834:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677447.973835:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:13.0:1619677447.973836:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:22.0:1619677447.973837:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:13.0:1619677447.973837:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.973838:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677447.973839:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:13.0:1619677447.973839:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.973840:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:12.0:1619677447.973840:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.973841:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.973842:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.973842:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.974000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.974006:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.974008:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.974011:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.974011:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.974014:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.974015:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677447.974017:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.974018:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.974019:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.974020:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.974023:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.974025:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.974028:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.974029:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.974029:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.974031:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.974032:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.974032:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.974033:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677447.974037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.974045:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.974047:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.974048:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.974048:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.974051:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.974053:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.974152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.974158:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.974161:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.974161:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.974164:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.974166:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.974168:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677447.974169:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677447.974170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:18.0:1619677447.974170:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.974171:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.974172:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.974176:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677447.974176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677447.974178:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.974181:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.974181:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:18.0:1619677447.974182:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.974182:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677447.974183:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:18.0:1619677447.974184:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677447.974185:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677447.974185:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.974185:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.974186:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:26.0:1619677447.974187:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677447.974187:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677447.974189:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.974190:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.974191:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.974192:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677447.974193:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.974195:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.974196:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677447.974196:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.974197:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677447.974198:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:12.0:1619677447.974199:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:26.0:1619677447.974200:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96721049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677447.974200:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.974201:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000400:00000200:26.0:1619677447.974202:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677447.974202:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:26.0:1619677447.974203:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.974203:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.974203:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:26.0:1619677447.974204:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:12.0:1619677447.974205:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677447.974207:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677447.974208:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.974208:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677447.974209:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.974210:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677447.974211:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.974212:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677447.974213:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.974213:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677447.974214:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.974214:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677447.974215:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.974215:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:10.0:1619677447.974216:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.974217:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:9.2:1619677447.974217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.974218:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677447.974218:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677447.974219:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.974219:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:13.0:1619677447.974220:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.974221:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.974222:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.974222:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.974223:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677447.974224:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.974227:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.974228:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.974228:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.974230:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.974233:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.974242:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.974245:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.974245:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.974246:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.974248:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.974251:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.974276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.974279:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.974282:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.974284:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.974285:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.974286:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.974288:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.974289:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.974290:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.974291:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.974293:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.974300:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.974302:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.974305:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.974305:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.974306:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.974308:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.974309:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.974310:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.974311:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.974312:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:9.2:1619677447.974315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677447.974317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677447.974323:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677447.974323:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:18.0:1619677447.974326:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:5.0:1619677447.974326:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:18.0:1619677447.974327:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.974327:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677447.974327:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.974329:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.974329:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677447.974332:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677447.974362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.974370:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.974373:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.974375:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.974377:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.974378:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.974379:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.974380:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.974381:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.974382:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.974383:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.974386:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.974388:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.974390:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.974391:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.974392:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.974393:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.974394:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.974395:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.974395:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.974406:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.974408:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.974409:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.974410:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.974412:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.974414:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.974422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677447.974427:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.974430:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.974431:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.974432:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.974435:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.974436:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.974438:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.974439:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.974440:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.974441:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677447.974444:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.974446:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.974447:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.974448:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.974449:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.974450:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.974451:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.974451:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.974452:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.974456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.974464:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.974465:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.974466:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.974467:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.974468:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.974469:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.974478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.974484:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.974486:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.974487:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.974488:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.974488:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.974488:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.974614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677447.974623:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677447.974626:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677447.974628:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.974630:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.974630:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677447.974632:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.974633:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.974635:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.974636:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.974637:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.974640:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.974642:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.974646:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.974646:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.974647:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.974649:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.974649:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677447.974650:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677447.974651:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:5.2:1619677447.974654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.2:1619677447.974656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677447.974662:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677447.974663:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:10.0:1619677447.974664:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.974665:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677447.974666:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.974667:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.974667:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.974669:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:10.0:1619677447.974670:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.974670:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:10.0:1619677447.974671:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677447.974672:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:10.0:1619677447.974672:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.974674:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.974675:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.974676:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677447.974679:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.974681:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677447.974683:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.974684:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677447.974684:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677447.974686:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.974686:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.974687:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677447.974689:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677447.974689:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677447.974695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677447.974698:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.974700:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.974701:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.974702:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:12.0:1619677447.974703:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.974704:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:13.0:1619677447.974704:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677447.974705:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:12.0:1619677447.974705:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677447.974707:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.2:1619677447.974728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.974735:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.974737:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.974738:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.974740:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.974742:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.974743:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.974745:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.974746:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.974747:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.974748:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.974751:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.974753:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.974755:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.974756:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.974756:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.974758:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.974758:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.974759:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.974759:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.974764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.974770:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.974772:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677447.974773:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.974774:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.974775:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.974776:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.974879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.974886:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.974888:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.974889:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.974891:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.974893:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.974894:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.974896:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.974897:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.974898:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.974899:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677447.974901:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.974903:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677447.974906:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.974907:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.974907:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.974909:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.974910:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.974910:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.974911:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677447.974933:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.974936:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.974938:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.974938:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.974940:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.974943:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.975185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.975191:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.975193:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.975196:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.975196:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.975198:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.975199:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677447.975201:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.975202:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.975203:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.975204:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.2:1619677447.975207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:5.0:1619677447.975207:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.975209:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.975212:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.975213:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.975214:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.975215:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677447.975215:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.975216:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.975217:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.975217:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677447.975218:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.975220:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.975222:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.975223:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.975224:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.975226:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.975227:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.975228:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.975228:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:30.0:1619677447.975229:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.975230:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.975231:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.975232:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677447.975233:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.975234:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.975234:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.975235:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.975236:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.975237:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.975237:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.975239:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.975239:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.975240:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.975241:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.975251:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.975253:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.975254:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.975255:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.975257:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.975259:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677447.975296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.975302:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.975304:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.975305:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.975307:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.975309:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.975310:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.975312:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.975313:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.975314:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.975315:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.975318:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.975320:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.975323:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.975323:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.975324:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.975326:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.975326:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.975327:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.975327:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.975332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677447.975340:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.975343:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.975344:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.975344:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677447.975346:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.975349:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677447.975427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677447.975435:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677447.975438:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677447.975439:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677447.975441:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.975444:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.975445:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677447.975446:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.975448:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.975449:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.975450:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677447.975453:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.975455:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677447.975458:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.975459:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677447.975459:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677447.975461:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677447.975462:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677447.975462:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677447.975462:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:21.2:1619677447.975468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.2:1619677447.975468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.975475:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:31.0:1619677447.975476:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.975478:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:1.0:1619677447.975478:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.975479:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.975480:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.975481:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.975482:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.975483:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.975483:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.975485:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:1.0:1619677447.975485:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:31.0:1619677447.975486:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.975487:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.975488:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.975489:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677447.975492:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.975494:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677447.975496:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.975496:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677447.975497:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677447.975498:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.975499:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.975500:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.975501:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677447.975521:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.975523:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.975524:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.975524:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.975526:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.975529:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677447.975558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677447.975565:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.975568:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.975568:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.975570:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.975572:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.975573:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.975574:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.975575:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.975576:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677447.975576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:18.0:1619677447.975577:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.975580:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.975582:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677447.975583:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677447.975583:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.975584:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.975584:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.975586:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:18.0:1619677447.975586:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.975586:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.975587:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.975587:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677447.975587:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.975589:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.975592:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.975592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677447.975593:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.975594:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.975595:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.975596:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.975597:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.975598:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:26.0:1619677447.975600:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.975600:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.975601:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.975602:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.975603:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:2.0:1619677447.975603:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:21.2:1619677447.975604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:2.0:1619677447.975605:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:26.0:1619677447.975606:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.975607:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.975608:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677447.975610:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.975610:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.975611:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:8.0:1619677447.975611:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:26.0:1619677447.975612:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.975613:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677447.975614:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.975615:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.975617:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.975618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:8.0:1619677447.975619:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.975621:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.975622:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.975623:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.975624:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.975624:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677447.975625:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:8.0:1619677447.975625:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677447.975626:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.975627:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677447.975627:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.975628:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677447.975628:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.975629:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:8.0:1619677447.975629:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.975630:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.975631:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677447.975632:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.975632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.0:1619677447.975634:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.975635:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.975635:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.975637:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.975637:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.975638:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.975639:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:10.0:1619677447.975640:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677447.975641:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.975642:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.975643:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677447.975643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.975644:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.975644:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.975645:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.975646:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000400:00000200:10.0:1619677447.975646:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.975647:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:10.0:1619677447.975649:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.975650:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:10.0:1619677447.975650:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677447.975651:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.975653:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:10.0:1619677447.975653:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.975653:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.975654:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.975654:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:2.0:1619677447.975654:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.975657:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:10.0:1619677447.975659:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.975660:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677447.975661:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.975662:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.975663:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677447.975663:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677447.975665:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.975666:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.975666:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.975667:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677447.975674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:26.0:1619677447.975676:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.975678:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.975679:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.975679:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677447.975681:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.975682:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677447.975682:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:30.0:1619677447.975685:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.975687:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.975689:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.975689:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.975690:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677447.975691:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.975693:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.975693:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.975694:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677447.975697:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.975699:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.975700:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.975701:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.975701:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.975703:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.975703:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.975704:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.975705:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.2:1619677447.975708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.975714:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677447.975715:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:17.0:1619677447.975716:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677447.975717:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:1.0:1619677447.975717:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677447.975718:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.975719:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.975720:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.975721:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.975722:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:17.0:1619677447.975722:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.975724:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677447.975725:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.975726:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.975727:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.975729:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677447.975732:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.975734:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677447.975737:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.975738:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.975738:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.975740:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.975740:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.975742:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.975743:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.975743:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677447.975746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.2:1619677447.975748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677447.975752:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:1.0:1619677447.975753:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677447.975755:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677447.975755:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.0:1619677447.975756:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.975757:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.975757:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.975757:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677447.975759:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.975762:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677447.975801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.975807:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.975809:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.975810:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.975812:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.975814:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.975816:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.975817:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.975819:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.975820:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.975821:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.975825:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.975826:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.975829:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.975830:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.975831:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.975833:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.975833:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.975834:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.975835:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677447.975846:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.975849:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.975850:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.975851:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.975852:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.975854:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.975895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.975902:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.975905:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.975907:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.975909:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.975909:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.975910:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677447.975912:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.975913:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.975914:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.975915:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677447.975918:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.975920:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677447.975924:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.975924:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677447.975925:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677447.975927:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.975927:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.975928:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.975929:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677447.975940:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.975943:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.975944:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.975944:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.975946:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.975949:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.976014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.976020:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.976023:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.976023:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.976025:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.976028:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.976030:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677447.976031:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.976032:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.976033:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.976034:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.976037:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.976039:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.976042:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.976043:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.976043:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.976045:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.976046:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.976046:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.976047:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.976051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.976057:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.976060:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.976061:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.976061:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.976063:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.976066:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677447.976142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:24.0:1619677447.976149:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677447.976151:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677447.976152:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677447.976154:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.976156:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.976158:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677447.976159:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.976160:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.976161:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.976162:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677447.976165:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.976167:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:4.2:1619677447.976167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.976169:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.976170:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:24.0:1619677447.976171:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:24.0:1619677447.976172:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:24.0:1619677447.976173:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677447.976174:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:10.0:1619677447.976174:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677447.976175:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:24.0:1619677447.976175:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:10.0:1619677447.976176:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.976177:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.976178:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.976180:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.976181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (21)++
00000400:00000200:10.0:1619677447.976182:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.976183:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.976184:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.976185:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:10.0:1619677447.976185:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.976186:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.976187:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.976188:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.976188:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000400:00000200:30.0:1619677447.976189:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:10.0:1619677447.976189:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.976190:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:31.0:1619677447.976191:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677447.976191:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677447.976191:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:30.0:1619677447.976192:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:8.0:1619677447.976192:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:10.0:1619677447.976194:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.976195:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.976195:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.976197:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.976197:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.976198:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.976199:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677447.976204:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.976206:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.976207:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.976208:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677447.976208:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.976209:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:6.0:1619677447.976211:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.976213:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.976214:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.976215:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.976217:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.976220:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.976228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.976231:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.976234:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.976236:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.976237:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.976239:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.976240:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677447.976242:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.976243:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.976244:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.976245:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.976249:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.976250:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677447.976253:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.976254:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.976255:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.976256:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.976257:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.976258:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.976259:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677447.976263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.976270:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.976272:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.976273:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.976274:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.976275:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.976278:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.976370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.976375:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.976378:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.976379:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.976381:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.976384:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.976385:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.976387:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.976388:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.976389:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.976390:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.976393:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.976395:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.976398:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.976399:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.976400:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.976401:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.976402:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677447.976403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:17.0:1619677447.976403:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.976403:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677447.976411:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.976414:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.976415:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677447.976416:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:24.0:1619677447.976417:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.976419:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.976419:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:24.0:1619677447.976420:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:13.0:1619677447.976420:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.976422:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.976422:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.976423:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.976423:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:24.0:1619677447.976424:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.976425:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.976425:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:24.0:1619677447.976429:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.976430:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.976433:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.976433:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677447.976434:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677447.976436:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.976436:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.976437:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.976437:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.2:1619677447.976439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.976447:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677447.976448:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:31.0:1619677447.976450:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:12.0:1619677447.976450:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.976451:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.976451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:31.0:1619677447.976452:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677447.976452:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.976454:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.976454:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:31.0:1619677447.976455:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.976455:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.976456:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.976458:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.976459:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677447.976459:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677447.976460:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.976461:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:26.0:1619677447.976461:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.976462:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:13.0:1619677447.976464:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:31.0:1619677447.976465:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.976465:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.976467:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:13.0:1619677447.976467:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677447.976468:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.976468:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:31.0:1619677447.976469:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677447.976469:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:13.0:1619677447.976469:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.976469:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.976470:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.976470:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:26.0:1619677447.976470:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.976471:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677447.976471:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677447.976472:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.976472:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677447.976472:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.976473:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677447.976473:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677447.976474:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:26.0:1619677447.976476:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.976478:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677447.976481:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.976482:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677447.976482:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677447.976484:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:26.0:1619677447.976484:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.976485:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.976485:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677447.976486:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:26.0:1619677447.976486:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:30.0:1619677447.976487:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.976488:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.976490:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.976491:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:9.2:1619677447.976492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.976500:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.976503:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.976504:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.976504:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.976506:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.976509:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.976520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.976527:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.976530:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.976530:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.976531:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.976532:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.976532:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.976585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.976592:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.976594:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.976595:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.976597:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.976600:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.976601:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677447.976603:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.976604:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.976605:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.976605:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677447.976609:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.976610:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677447.976612:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.976613:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.976613:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677447.976615:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.976616:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.976617:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.976617:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677447.976627:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.976630:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.976630:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.976632:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.976634:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.976635:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.976706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.976713:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.976716:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.976718:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.976720:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.976720:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.976721:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677447.976723:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.976724:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.976725:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.976725:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677447.976729:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.976730:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677447.976734:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.976735:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677447.976735:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677447.976737:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.976738:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.976739:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.976740:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.976741:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677447.976747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677447.976751:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:31.0:1619677447.976754:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677447.976754:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.976755:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.976755:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677447.976756:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.976757:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677447.976757:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.976758:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677447.976760:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677447.976797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.976802:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.976805:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.976806:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.976808:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.976810:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.976811:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.976813:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.976814:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.976815:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.976816:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.976820:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.976821:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.976824:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.976825:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.976826:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.976828:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.976828:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.976829:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.976829:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.976834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.976842:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.976845:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.976846:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.976847:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.976848:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.976850:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.976910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677447.976917:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.976920:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677447.976921:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677447.976923:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677447.976925:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.976926:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:4.2:1619677447.976927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:26.0:1619677447.976928:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.976929:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.976930:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.976931:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.976934:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677447.976935:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677447.976936:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677447.976936:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.976937:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677447.976939:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677447.976939:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.976940:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677447.976940:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:8.0:1619677447.976941:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.976942:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677447.976942:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:8.0:1619677447.976943:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:26.0:1619677447.976944:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:8.0:1619677447.976944:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.976945:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677447.976945:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:8.0:1619677447.976945:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.976946:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.976947:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.976950:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677447.976951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:8.0:1619677447.976951:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677447.976954:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677447.976954:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.976955:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.976955:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677447.976956:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.976957:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.976957:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.976957:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:12.0:1619677447.976958:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:10.0:1619677447.976958:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:8.0:1619677447.976958:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.976958:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677447.976960:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.976960:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:12.0:1619677447.976961:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677447.976961:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677447.976962:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677447.976963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.976971:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.976973:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.976974:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.976975:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.976976:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677447.976977:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.976979:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.976980:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.976980:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677447.976980:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:13.0:1619677447.976981:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.976982:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677447.976983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.976991:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677447.976994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677447.976994:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.976995:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.976995:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.976996:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677447.976996:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:21.2:1619677447.976997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:17.0:1619677447.976997:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677447.976999:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.977001:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.977003:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:24.0:1619677447.977004:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:20.0:1619677447.977004:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.977005:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:24.0:1619677447.977007:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:20.0:1619677447.977007:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.977008:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.977008:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:20.0:1619677447.977010:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.977011:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:24.0:1619677447.977012:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.977015:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.977015:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.977016:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.977018:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721014] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.977018:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:24.0:1619677447.977019:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.977020:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.977021:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.977021:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.977022:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.977023:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.977024:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:24.0:1619677447.977025:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.977026:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:24.0:1619677447.977027:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.977027:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.977028:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:24.0:1619677447.977030:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.977031:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.977032:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.977033:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.977034:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.977035:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.977036:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:31.0:1619677447.977037:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:24.0:1619677447.977037:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:31.0:1619677447.977040:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.977041:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.977042:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:21.2:1619677447.977042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:30.0:1619677447.977044:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.977046:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.2:1619677447.977050:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.977053:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.977056:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.977056:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.977057:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677447.977058:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.977061:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.977062:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.977064:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.977064:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.977067:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677447.977140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.977145:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.977148:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.977149:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.977151:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.977153:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.977155:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.977156:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.977157:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.977159:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.977159:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.977163:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.977165:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.977168:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.977169:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.977169:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.977171:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.977172:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.977172:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.977173:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.977184:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.977187:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.977188:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.977190:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.977191:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.977193:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677447.977201:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677447.977204:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.977204:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.977205:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.977206:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.977207:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677447.977291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.977299:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.977302:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.977303:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.977304:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.977307:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.977308:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677447.977309:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.977311:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.977311:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.977312:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677447.977315:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.977317:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677447.977320:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.977321:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677447.977322:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677447.977323:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677447.977324:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.977325:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.977325:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677447.977341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.977349:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.977352:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.977352:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.977354:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.977355:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.977357:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.977389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.977397:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.977400:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.977402:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.977404:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.977404:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.977406:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677447.977407:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.977409:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.977409:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.977410:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677447.977414:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.977416:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677447.977418:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.977419:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.977420:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677447.977422:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.977422:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.977424:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.977425:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.977426:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.2:1619677447.977427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.2:1619677447.977431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677447.977435:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.977438:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.977438:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:8.2:1619677447.977438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677447.977439:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.977441:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:10.0:1619677447.977441:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.977442:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.977443:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:10.0:1619677447.977443:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.977445:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:5.0:1619677447.977445:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:10.0:1619677447.977446:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.977447:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.977448:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.977448:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:10.0:1619677447.977449:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:5.0:1619677447.977449:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.977451:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.977451:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:10.0:1619677447.977453:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.977454:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:10.0:1619677447.977455:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677447.977457:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.977458:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677447.977459:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677447.977460:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677447.977461:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.977461:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.977462:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677447.977485:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.977487:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.977488:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.977489:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.977492:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.977493:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677447.977569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677447.977575:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.977578:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.977579:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.977581:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.977584:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.977585:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.977586:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.977588:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.977589:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.977590:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677447.977593:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.977595:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.977597:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677447.977598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:2.0:1619677447.977598:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:21.2:1619677447.977599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:2.0:1619677447.977599:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.977601:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.977601:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.977603:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.977604:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.977604:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:6.0:1619677447.977605:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677447.977606:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.2:1619677447.977607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:24.0:1619677447.977609:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:22.0:1619677447.977609:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.977609:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:5.2:1619677447.977609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:8.0:1619677447.977613:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677447.977613:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.977614:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.977615:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677447.977615:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:6.0:1619677447.977616:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.977617:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.977617:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:18.0:1619677447.977617:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:20.0:1619677447.977618:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.977618:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:6.0:1619677447.977618:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:24.0:1619677447.977619:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:2.0:1619677447.977619:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:20.0:1619677447.977620:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.977620:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96721008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.977621:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96721008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.977621:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:6.0:1619677447.977622:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.977623:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.977623:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:6.0:1619677447.977623:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.977624:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:24.0:1619677447.977625:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677447.977626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000400:00000200:24.0:1619677447.977627:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.977628:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.977630:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:24.0:1619677447.977633:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.977633:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.977633:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.977634:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (21)++
00000800:00000200:24.0:1619677447.977635:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.977635:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (23)++
00000800:00000200:5.0:1619677447.977636:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.977637:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:6.0:1619677447.977637:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677447.977637:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:24.0:1619677447.977638:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.977638:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:24.0:1619677447.977639:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.977639:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.977639:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:5.0:1619677447.977639:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.977640:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677447.977640:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.977640:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677447.977641:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.977642:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.977643:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.977643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:24.0:1619677447.977644:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.2:1619677447.977646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.2:1619677447.977648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.977651:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:22.0:1619677447.977652:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.977654:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:5.0:1619677447.977654:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677447.977655:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677447.977655:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:5.0:1619677447.977655:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.977656:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677447.977657:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.977657:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.977658:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.977659:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.977659:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.977660:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:17.0:1619677447.977662:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.977664:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.977683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.977686:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.977689:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.2:1619677447.977691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000400:00000200:20.0:1619677447.977691:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.977693:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.977694:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.977696:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.977697:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.977698:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.977699:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000400:00000200:20.0:1619677447.977699:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.977700:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.977702:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:20.0:1619677447.977704:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.977705:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.977705:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:20.0:1619677447.977706:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.977708:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.977710:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.977711:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:20.0:1619677447.977711:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:31.0:1619677447.977712:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96721005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.977712:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677447.977713:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.977714:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.977714:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.977714:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677447.977715:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:20.0:1619677447.977715:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.977716:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:31.0:1619677447.977719:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.977721:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677447.977723:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.977724:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677447.977725:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677447.977726:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677447.977727:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:12.0:1619677447.977727:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677447.977728:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677447.977728:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:12.0:1619677447.977729:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.977730:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.977732:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.977732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:13.0:1619677447.977733:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.977734:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.977738:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.977741:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.977742:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677447.977742:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.977744:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.977746:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677447.977752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677447.977758:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.977761:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677447.977761:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.977761:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.977762:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.977763:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677447.977862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677447.977869:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.977871:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.977872:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.977874:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.977876:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.977878:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.977879:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.977880:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.977881:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.977882:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.977885:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.977887:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.977890:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.977891:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.977892:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.977893:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.977894:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.977894:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.977895:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677447.977899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.977907:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.977909:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.977910:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.977912:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.977913:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.977915:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677447.977918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.977926:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.977929:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.977930:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.977930:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.977931:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.977932:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677447.977936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.977943:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.977946:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.977948:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.977950:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.977951:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.977952:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.977953:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.977955:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.977956:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.977957:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.977960:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.977962:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.977964:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.977965:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.977965:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.977967:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.977967:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.977968:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.977969:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.977979:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.977982:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.977982:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.977984:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.977986:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.977987:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.978033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.978040:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.978042:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.978043:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.978045:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.978047:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.978049:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.978050:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96721002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.978051:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.978052:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.978053:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677447.978056:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.978057:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677447.978060:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.978061:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677447.978062:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677447.978063:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.978064:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.978065:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.978066:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.978067:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677447.978072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.978077:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677447.978079:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:5.2:1619677447.978079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677447.978080:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.978080:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677447.978081:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677447.978081:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.978082:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.978083:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677447.978083:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.978085:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677447.978086:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677447.978088:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.978089:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.978090:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.978093:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.978094:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.978096:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96721001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.978097:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.978098:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.978099:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.978102:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.978104:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.978106:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.978107:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.978108:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.978109:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.978110:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.978111:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.978112:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.978113:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677447.978118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677447.978123:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677447.978123:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677447.978125:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677447.978125:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:22.0:1619677447.978126:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.978126:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.978126:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.978127:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677447.978128:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.978131:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.978142:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677447.978144:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.978145:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.978146:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.978147:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.978147:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.978289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.2:1619677447.978297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:24.0:1619677447.978299:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.978302:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.978302:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677447.978303:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:24.0:1619677447.978304:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.978306:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:24.0:1619677447.978307:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.978307:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.978309:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677447.978310:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.978311:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96721000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.978311:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.978313:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.978313:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677447.978314:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.978315:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:18.0:1619677447.978315:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.978316:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.978317:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.978319:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677447.978320:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.978322:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677447.978323:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:18.0:1619677447.978324:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677447.978326:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.978326:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.978327:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:18.0:1619677447.978328:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.978328:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:24.0:1619677447.978329:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:18.0:1619677447.978330:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:24.0:1619677447.978331:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:18.0:1619677447.978331:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:24.0:1619677447.978332:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677447.978332:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.978332:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677447.978333:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.978334:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.978336:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677447.978337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.2:1619677447.978337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677447.978341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677447.978344:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:1.0:1619677447.978346:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:2.0:1619677447.978347:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.978347:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.978348:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:18.0:1619677447.978349:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:17.0:1619677447.978349:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.978349:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:2.0:1619677447.978349:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000400:00000200:1.0:1619677447.978349:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.978351:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:2.0:1619677447.978351:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:1.0:1619677447.978351:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.978352:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.978353:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.978355:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.978360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.2:1619677447.978365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.0:1619677447.978367:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.978370:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.978373:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.978373:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.978374:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:5.0:1619677447.978375:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.978377:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:5.0:1619677447.978377:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677447.978379:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720998] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.978380:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.978381:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:5.0:1619677447.978381:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677447.978382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:5.0:1619677447.978382:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.978383:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.978383:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677447.978385:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677447.978387:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.978387:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.978388:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.978389:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.978390:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677447.978390:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:31.0:1619677447.978391:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:30.0:1619677447.978391:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:5.0:1619677447.978393:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677447.978394:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.978394:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677447.978394:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677447.978395:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.978396:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:30.0:1619677447.978397:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.978397:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677447.978398:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:30.0:1619677447.978398:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:5.0:1619677447.978398:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:30.0:1619677447.978399:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.978399:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.978400:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677447.978401:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.978402:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.978402:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:8.2:1619677447.978402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677447.978405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677447.978406:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677447.978406:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.978407:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677447.978407:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677447.978407:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:30.0:1619677447.978408:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.978409:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.978409:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:31.0:1619677447.978410:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677447.978410:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677447.978410:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:31.0:1619677447.978411:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.978411:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677447.978411:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.978411:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677447.978412:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677447.978412:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.978413:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677447.978413:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:30.0:1619677447.978414:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:17.0:1619677447.978414:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.978414:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677447.978415:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:9.2:1619677447.978415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677447.978416:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.978416:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:17.0:1619677447.978416:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.978418:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.978420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.978423:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:8.2:1619677447.978424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:18.0:1619677447.978426:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.978426:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.978427:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.978428:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.978430:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:18.0:1619677447.978430:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.978431:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.978432:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:20.0:1619677447.978433:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.978433:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677447.978434:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:9.2:1619677447.978435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:1.0:1619677447.978435:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:20.0:1619677447.978436:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677447.978436:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.978436:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.978437:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.978438:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:2.0:1619677447.978442:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.978445:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.978446:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.978447:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.978447:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.978448:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677447.978450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.978458:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.978461:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.978462:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.978464:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.978467:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.978469:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677447.978470:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.978471:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.978472:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.978473:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677447.978476:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677447.978476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:26.0:1619677447.978477:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677447.978479:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.978480:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677447.978481:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677447.978482:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677447.978483:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677447.978483:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677447.978484:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677447.978485:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:26.0:1619677447.978486:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677447.978486:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677447.978486:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.978488:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.978490:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.978492:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:5.2:1619677447.978492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:22.0:1619677447.978493:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.978494:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:22.0:1619677447.978495:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.978496:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.978497:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:22.0:1619677447.978497:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.978498:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.978499:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:31.0:1619677447.978500:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.978501:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:22.0:1619677447.978501:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.978501:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677447.978502:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677447.978502:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677447.978503:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:22.0:1619677447.978504:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.978506:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.978507:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.978507:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.978509:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.978510:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.978511:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.978512:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.978515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.978523:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.978526:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.978527:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.978528:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.978529:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.978530:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677447.978664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.978670:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.978672:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.978673:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.978675:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.978677:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.978679:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.978680:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.978682:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.978683:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.978684:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.978687:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.978689:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.978691:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.978692:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.978693:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.978694:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.978695:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.978695:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.978696:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.978708:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.978711:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.978712:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.978712:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.978714:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.978716:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.978770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.978777:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.978780:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.978781:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.978783:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.978785:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.978787:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.978788:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.978789:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.978790:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.978792:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.978795:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.978797:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.978800:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.978800:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.978801:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.978803:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.978803:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.978804:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.978804:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.978809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.978816:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.978819:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.978820:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.978820:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.978822:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.978825:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.978834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677447.978837:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.978840:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.2:1619677447.978841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677447.978842:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.978844:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.978845:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:20.0:1619677447.978846:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.978847:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.978849:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677447.978849:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.978849:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.978850:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.978851:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.978852:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:18.0:1619677447.978852:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.978855:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.978855:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677447.978856:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.978856:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.978856:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.978858:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.978859:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.978860:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.978860:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.978862:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677447.978863:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677447.978863:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677447.978863:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.978864:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.978864:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677447.978865:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.978865:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.978866:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.978866:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.978867:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:18.0:1619677447.978867:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.978868:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.978868:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.978868:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:18.0:1619677447.978868:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677447.978869:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.978873:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.978874:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.978876:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677447.978877:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.978878:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.978879:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:30.0:1619677447.978880:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.978880:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.978880:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677447.978881:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:30.0:1619677447.978882:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (24)++
00000800:00000200:18.0:1619677447.978882:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.978883:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.978883:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:30.0:1619677447.978884:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (25)--
00000800:00000200:18.0:1619677447.978884:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:30.0:1619677447.978885:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677447.978885:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:30.0:1619677447.978886:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.978886:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:12.0:1619677447.978886:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.978887:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.978887:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.978888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.978893:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.978894:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.978895:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.978896:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.978896:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:13.0:1619677447.978897:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.978897:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.978898:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:12.0:1619677447.978899:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677447.978899:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.978900:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.978901:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.978903:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677447.978913:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.978916:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.978917:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.978917:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.978918:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.978919:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.978932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.978938:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.978941:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.978941:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.978943:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.978945:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.978947:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.978948:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.978948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:22.0:1619677447.978949:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.978950:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.978951:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.978955:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.978955:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.978957:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677447.978958:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.978958:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:22.0:1619677447.978959:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.978960:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.978961:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:10.0:1619677447.978961:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.978963:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.978963:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.978964:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.978964:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677447.978964:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.978965:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.978967:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.978968:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.978969:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677447.978969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:10.0:1619677447.978970:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677447.978973:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.978975:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.978976:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:10.0:1619677447.978978:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.978979:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.978979:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.978979:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.978980:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.978981:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.978981:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677447.978982:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677447.978982:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:5.0:1619677447.978982:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.978983:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677447.978984:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677447.978989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677447.978994:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.978996:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677447.978997:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.978998:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677447.978998:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.978999:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000400:00000200:6.0:1619677447.978999:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.979000:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000400:00000200:6.0:1619677447.979000:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.979001:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.979002:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:6.0:1619677447.979003:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.979015:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.979018:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677447.979018:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.979019:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.979020:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.979020:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.979224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.979230:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.979233:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.979234:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.979236:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.979238:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.979239:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.979241:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.979242:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.979243:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.979244:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.979247:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.979249:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.979251:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.979252:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.979253:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.979255:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.979255:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.979256:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.979256:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677447.979263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:6.0:1619677447.979268:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.0:1619677447.979270:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677447.979271:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.979272:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.979272:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.979273:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.979274:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677447.979274:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.979276:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.979276:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:8.0:1619677447.979279:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.979280:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.979281:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.979282:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.979283:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.979284:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.979287:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.979289:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.979292:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.979292:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.979298:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.979300:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.979301:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.979301:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.979302:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.979306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.979314:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.979317:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.979318:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.979318:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.979320:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.979323:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.979353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677447.979360:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.979364:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677447.979365:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677447.979367:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677447.979369:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.979371:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677447.979372:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.979373:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.979374:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.979375:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677447.979379:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677447.979380:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677447.979383:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.979384:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677447.979385:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677447.979386:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677447.979387:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.979387:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677447.979388:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677447.979393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.979401:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.979404:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.979405:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.979406:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677447.979407:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.979409:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.979434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.979441:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.979443:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.979444:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.979446:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.979448:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.979449:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:21.2:1619677447.979451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:10.0:1619677447.979451:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.979452:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.979453:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.979454:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:22.0:1619677447.979456:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:21.2:1619677447.979456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:10.0:1619677447.979456:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.979458:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:22.0:1619677447.979459:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:10.0:1619677447.979459:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.979460:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677447.979460:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:6.0:1619677447.979460:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.979462:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.979462:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677447.979462:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.979462:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.979463:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677447.979464:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.979464:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677447.979465:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:18.0:1619677447.979465:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.979465:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:22.0:1619677447.979467:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.979468:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.979469:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.979470:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.979470:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.979471:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677447.979472:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.979473:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.979474:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:22.0:1619677447.979475:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.979475:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677447.979476:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:18.0:1619677447.979476:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.979477:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.979477:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:18.0:1619677447.979477:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.979478:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677447.979480:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.979480:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.979481:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000400:00000200:18.0:1619677447.979481:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.979482:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:22.0:1619677447.979482:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677447.979484:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677447.979484:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677447.979484:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.979485:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.979486:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677447.979486:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.979487:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.979487:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.979489:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.979489:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.979490:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.979490:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677447.979495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.979496:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677447.979498:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677447.979499:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677447.979499:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.979502:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.979502:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677447.979504:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:2.0:1619677447.979504:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.979505:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.979506:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.979507:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.979509:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:26.0:1619677447.979515:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677447.979518:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677447.979519:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677447.979519:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677447.979520:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677447.979521:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677447.979565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.979573:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.979576:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.979578:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.979580:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.979580:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.979582:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.979583:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.979585:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.979586:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.979587:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.979590:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.979592:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.979595:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.979596:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.979597:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.979599:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.979599:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.979600:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.979601:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.2:1619677447.979602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:8.2:1619677447.979605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.979607:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.979610:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.979611:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677447.979612:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:18.0:1619677447.979613:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.979614:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.979615:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.979615:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.979615:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.979617:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.979618:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.979618:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:18.0:1619677447.979619:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677447.979620:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:18.0:1619677447.979620:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.979621:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677447.979624:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.979626:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.979628:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.979629:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.979629:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.979631:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.979631:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.979632:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.979633:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.979637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.979644:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.979646:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.979647:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.979647:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.979649:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.979650:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.979850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.979857:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.979860:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.979861:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.979862:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677447.979863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677447.979864:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.979866:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677447.979867:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720978] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.979868:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677447.979868:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.979869:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.979870:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:17.0:1619677447.979871:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.979872:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.979874:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.979875:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.979876:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.979878:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:10.0:1619677447.979878:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:17.0:1619677447.979879:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.979880:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677447.979881:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677447.979881:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:17.0:1619677447.979882:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677447.979882:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:17.0:1619677447.979883:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677447.979883:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.979883:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.979884:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.979885:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:17.0:1619677447.979886:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.979888:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:21.2:1619677447.979889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677447.979890:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.979891:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.979891:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.979893:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.979893:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677447.979894:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:17.0:1619677447.979894:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.979894:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.979896:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677447.979896:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.979897:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.979897:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.979899:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677447.979899:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:8.2:1619677447.979899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677447.979900:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:30.0:1619677447.979901:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.979903:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.979904:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.979905:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.979906:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.979906:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:30.0:1619677447.979908:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.979908:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:30.0:1619677447.979909:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.979909:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.979909:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.979911:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.979911:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.979913:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:30.0:1619677447.979914:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677447.979914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:10.0:1619677447.979914:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677447.979916:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677447.979917:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.979917:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.979918:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:30.0:1619677447.979919:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.979919:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.979919:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:8.2:1619677447.979919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677447.979920:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.979920:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677447.979920:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677447.979921:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.979922:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677447.979922:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.979923:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677447.979924:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.979925:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:6.0:1619677447.979925:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.979926:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677447.979927:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:6.0:1619677447.979928:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.979930:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.979930:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:1.0:1619677447.979930:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:21.2:1619677447.979931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677447.979931:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000400:00000200:6.0:1619677447.979931:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.979932:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.979932:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:6.0:1619677447.979932:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.979933:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.979934:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677447.979935:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:12.0:1619677447.979937:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.979937:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.979938:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.979939:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:12.0:1619677447.979939:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.979940:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:31.0:1619677447.979941:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:12.0:1619677447.979941:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.979942:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677447.979942:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.979943:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.979943:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.979944:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.979944:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.979946:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.979947:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.979948:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.979949:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.979950:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:21.2:1619677447.979955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677447.979958:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:22.0:1619677447.979960:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:26.0:1619677447.979961:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677447.979961:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677447.979962:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.979962:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.979962:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.979963:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:26.0:1619677447.979964:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.979967:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677447.979974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1200] (20)++
00000800:00000200:1.0:1619677447.979980:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:4.2:1619677447.979981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:17.0:1619677447.979983:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:1.0:1619677447.979983:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000400:00000200:1.0:1619677447.979986:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.979987:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.979988:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:1.0:1619677447.979989:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:24.0:1619677447.979990:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.979991:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677447.979991:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.979992:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.979993:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.979994:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.979995:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:24.0:1619677447.979997:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.979999:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.980000:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.980001:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96720973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.980002:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:24.0:1619677447.980003:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.980004:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.980005:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.980005:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.980005:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:1.0:1619677447.980006:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:24.0:1619677447.980008:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.980008:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:24.0:1619677447.980010:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.980010:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:1.0:1619677447.980011:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:1.0:1619677447.980011:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:8.2:1619677447.980014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:24.0:1619677447.980015:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.980015:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.980016:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.980018:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.980019:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.980020:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.980020:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.0:1619677447.980023:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:8.2:1619677447.980024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.980025:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.980026:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.980027:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.980028:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.980029:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:17.0:1619677447.980031:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.980034:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.980035:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.980035:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.980037:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.980040:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.980097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.980099:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.980102:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.980104:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.980105:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.980107:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.980108:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677447.980109:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.980111:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.980112:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.980113:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677447.980116:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.980118:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677447.980121:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.980122:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.980122:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.980124:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.980124:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.980125:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.980126:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677447.980130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.980138:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.980140:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.980141:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.980141:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.980143:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.980145:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677447.980260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.980268:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.980271:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.980272:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.980273:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.980275:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.980276:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677447.980277:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.980279:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.980280:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.980280:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677447.980283:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.980285:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677447.980287:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.980288:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677447.980289:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677447.980290:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.980291:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.980292:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.980292:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:4.2:1619677447.980299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.980307:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677447.980308:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:12.0:1619677447.980309:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:26.0:1619677447.980310:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:12.0:1619677447.980310:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.980311:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:12.0:1619677447.980312:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:26.0:1619677447.980314:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.980314:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.980315:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677447.980316:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.980317:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677447.980318:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.980320:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.980321:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.980322:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677447.980325:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.980326:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677447.980329:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.980329:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677447.980330:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.980332:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.980332:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.980334:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.980335:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.980335:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677447.980339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677447.980341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677447.980347:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677447.980348:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:18.0:1619677447.980350:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:10.0:1619677447.980350:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:18.0:1619677447.980351:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.980351:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677447.980351:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677447.980352:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:18.0:1619677447.980353:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677447.980355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.980358:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.980360:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.980362:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.980363:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.980363:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.980363:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.980364:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677447.980390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.980399:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.980402:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.980404:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.980406:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.980407:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.980408:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677447.980409:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.980410:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.980411:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.980412:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.980415:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.980416:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.980419:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.980419:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.980420:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.980421:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677447.980422:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.980423:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.980423:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.980433:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.980436:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.980436:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.980438:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.980440:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.980441:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677447.980484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.980492:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.980494:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.980496:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.980498:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.980498:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.980499:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.980501:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.980502:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.980503:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.980504:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:21.2:1619677447.980504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:30.0:1619677447.980506:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.980508:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.980510:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.980511:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.980511:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.980512:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:30.0:1619677447.980513:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.980513:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.980514:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677447.980514:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:30.0:1619677447.980515:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677447.980516:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.980518:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.980521:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.980522:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:13.0:1619677447.980524:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677447.980524:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96720967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.980525:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.980526:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.980526:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:13.0:1619677447.980527:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.980527:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.980529:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.980530:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.980531:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677447.980531:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677447.980532:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.980535:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.980536:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.980537:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.980539:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.980539:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.980540:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.980541:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.980545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.980553:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.980556:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.980557:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.980557:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.980559:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.980562:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.980626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.980632:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.980635:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.980636:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.980638:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.980640:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.980642:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.980643:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.980644:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.980645:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.980646:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.980650:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.980651:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.980653:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.980654:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.980655:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.980656:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.980656:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.980658:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.980659:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.980660:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677447.980662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677447.980665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.980670:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.980671:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.980671:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.980672:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:1.0:1619677447.980672:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.980673:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.980674:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.980674:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.980675:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.980675:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677447.980855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.980863:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.980866:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.980868:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.980870:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.980871:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.980872:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677447.980873:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.980875:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.980876:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.980877:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677447.980880:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.980882:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677447.980884:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.980885:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677447.980886:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677447.980888:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.980888:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.980889:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.980890:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677447.980894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677447.980900:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:30.0:1619677447.980902:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677447.980902:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.980903:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.980903:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.980904:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:10.0:1619677447.980905:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:30.0:1619677447.980907:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.980908:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677447.980908:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:30.0:1619677447.980910:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.980911:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.980912:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.980912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:30.0:1619677447.980913:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.980914:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.980915:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.2:1619677447.980915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:30.0:1619677447.980918:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.980919:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677447.980920:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677447.980921:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:5.0:1619677447.980921:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677447.980922:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.980922:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677447.980923:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.980923:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677447.980923:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:30.0:1619677447.980925:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.980925:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677447.980925:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.980926:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.980926:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.980927:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:26.0:1619677447.980927:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.980928:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.980929:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677447.980929:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677447.980929:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.980931:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.980931:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677447.980933:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.980934:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677447.980934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:5.0:1619677447.980935:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.980936:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677447.980936:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:5.0:1619677447.980936:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.980937:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677447.980938:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:12.0:1619677447.980939:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.980940:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.980942:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:13.0:1619677447.980943:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.980943:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:26.0:1619677447.980944:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677447.980944:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.980945:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.980945:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.980946:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.980946:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677447.980947:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677447.980948:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677447.980950:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.980951:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.980951:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677447.980952:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677447.980952:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677447.980953:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.980954:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677447.980955:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.980955:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:26.0:1619677447.980956:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677447.980956:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:26.0:1619677447.980957:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.980957:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677447.980958:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.980958:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677447.980961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677447.980964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:31.0:1619677447.980967:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.980969:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.980970:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.980971:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000400:00000200:31.0:1619677447.980972:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.980972:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.980972:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:30.0:1619677447.980973:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677447.980973:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.980973:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.980974:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:5.0:1619677447.980974:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.980975:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.0:1619677447.980976:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.980976:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.980979:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677447.981007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.981015:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.981017:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.981018:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.981020:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.981023:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.981025:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677447.981026:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.981028:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.981029:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.981030:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677447.981033:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.981035:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677447.981037:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.981038:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677447.981039:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677447.981040:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.981041:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.981041:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.981042:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677447.981052:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.981054:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.981055:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.981057:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.981058:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.981059:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.981065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.981071:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677447.981072:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.981073:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.981074:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677447.981075:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.981076:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.981077:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:1.0:1619677447.981077:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.981078:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:1.0:1619677447.981078:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.981079:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677447.981080:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.981081:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.981083:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.981084:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.981085:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.981089:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.981090:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.981094:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.981095:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.981095:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.981097:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.981097:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.981098:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.981099:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.981110:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.981112:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.981113:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.981115:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.981117:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.981118:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.981127:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.981130:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.981131:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.981132:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.981132:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.981134:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.981218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.981221:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.981223:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.981226:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.981226:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.981228:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.981230:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.981231:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.981233:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.981234:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.981235:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.981238:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.981240:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.981243:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.981244:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.981245:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.981246:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.981247:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.981248:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.981249:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677447.981258:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.981261:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.981262:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677447.981263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:12.0:1619677447.981263:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.981264:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.981266:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.981271:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.981274:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.981276:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.981277:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.981278:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.981278:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.981279:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.981281:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.981282:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.981283:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.981284:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.981287:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.981288:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.981289:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.981290:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:30.0:1619677447.981290:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.981291:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.981293:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.981299:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.981300:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:30.0:1619677447.981303:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.981304:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.981304:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.981304:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.981305:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677447.981306:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.981306:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)++
00000800:00000200:30.0:1619677447.981307:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (24)--
00000800:00000200:31.0:1619677447.981308:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.981308:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677447.981308:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.981309:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677447.981309:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.981310:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.981311:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.981312:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.981314:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.981314:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677447.981319:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:21.2:1619677447.981320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677447.981321:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:26.0:1619677447.981322:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:13.0:1619677447.981322:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.981324:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.981325:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.981326:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.981326:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.981326:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:12.0:1619677447.981327:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677447.981328:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.981329:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:26.0:1619677447.981330:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:30.0:1619677447.981331:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.981332:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.981333:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677447.981397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677447.981404:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.981407:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:5.2:1619677447.981407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677447.981408:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.981410:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.981411:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.981413:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:17.0:1619677447.981413:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677447.981414:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.981415:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:18.0:1619677447.981416:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.981416:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677447.981417:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.981418:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677447.981419:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.981421:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.981422:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.981423:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:18.0:1619677447.981424:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:17.0:1619677447.981424:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.981426:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677447.981426:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.981427:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:17.0:1619677447.981427:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677447.981428:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:17.0:1619677447.981428:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.981430:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.981430:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.981431:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:17.0:1619677447.981431:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.981432:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677447.981433:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.981436:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.981437:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.2:1619677447.981437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677447.981438:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.981439:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.981440:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.981440:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.981441:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677447.981445:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.981447:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.981448:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.981450:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.981451:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:13.0:1619677447.981451:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677447.981453:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677447.981454:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.981455:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.981457:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.981458:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.981460:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677447.981465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.981473:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.981475:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.981478:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.981479:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.981480:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.981481:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.981483:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.981484:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.981485:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.981486:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.981489:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.981491:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.981494:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.981494:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.981495:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.981497:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.981497:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.981498:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.981498:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677447.981508:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.981511:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.981512:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.981513:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.981515:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.981516:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677447.981767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.981774:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.2:1619677447.981776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677447.981777:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.981778:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.981779:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.981781:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.981783:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:31.0:1619677447.981784:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677447.981784:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.981785:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.981787:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677447.981787:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.981788:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677447.981789:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.981790:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.981791:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.981791:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.981793:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:10.0:1619677447.981793:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:31.0:1619677447.981794:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.981795:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.981796:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677447.981796:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.981797:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677447.981797:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677447.981798:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677447.981800:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.981800:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677447.981800:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.981801:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.981801:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677447.981802:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677447.981805:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.981805:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.2:1619677447.981805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677447.981806:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.981808:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677447.981808:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.981809:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.981809:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677447.981813:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.2:1619677447.981814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677447.981815:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.981816:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.981818:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.2:1619677447.981818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1200] (20)++
00000800:00000200:20.0:1619677447.981821:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.981822:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677447.981822:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.981824:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.981825:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000400:00000200:1.0:1619677447.981825:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.981826:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000400:00000200:1.0:1619677447.981827:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.981829:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.981831:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.981832:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.981832:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677447.981833:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:17.0:1619677447.981834:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:17.0:1619677447.981836:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.981837:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.981838:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.981839:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:17.0:1619677447.981843:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.981845:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677447.981847:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.981848:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:17.0:1619677447.981849:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:17.0:1619677447.981850:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:17.0:1619677447.981851:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677447.981851:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:17.0:1619677447.981852:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:8.2:1619677447.981856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.981863:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.981866:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.981866:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.981868:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.981869:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.981870:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:21.2:1619677447.981893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.981899:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.981902:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.981902:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.981905:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.981907:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.981909:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.981910:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.981911:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.981912:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.981913:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677447.981917:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.981919:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.2:1619677447.981920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:6.0:1619677447.981922:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.981923:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.981924:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677447.981926:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.981927:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.981927:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677447.981928:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677447.981928:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:26.0:1619677447.981931:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.981931:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.981934:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.981936:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.981938:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:30.0:1619677447.981939:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:26.0:1619677447.981939:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720949] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.981940:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.981941:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677447.981942:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:26.0:1619677447.981942:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677447.981943:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.981945:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677447.981945:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:26.0:1619677447.981947:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.981948:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677447.981949:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677447.981951:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.981952:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677447.981953:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.981955:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.981956:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.981956:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.981957:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677447.981962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.981969:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.981972:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.981973:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.981973:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.981975:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.981978:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.982010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.982016:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.982018:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.982019:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.982021:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.982024:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.982025:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.982027:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.982028:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.982029:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.982030:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.982034:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.982035:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.982039:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.982040:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.982041:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.982042:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.982043:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.982043:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.982044:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.982055:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.982058:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.982059:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.982061:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.982062:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.982064:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.982064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:21.2:1619677447.982069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677447.982073:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.982075:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.982075:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.982077:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:18.0:1619677447.982078:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.982079:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.982081:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.982081:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.982083:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677447.982085:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.982086:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.982087:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.982088:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677447.982090:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.982091:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.982093:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000400:00000200:18.0:1619677447.982094:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.982095:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.982095:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:18.0:1619677447.982096:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.982100:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.982100:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.982101:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677447.982101:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677447.982102:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677447.982104:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.982104:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677447.982105:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.982105:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.982105:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.982106:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (23)++
00000800:00000200:18.0:1619677447.982107:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.982107:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (24)--
00000800:00000200:18.0:1619677447.982108:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.982108:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.982108:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677447.982109:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.982110:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.982110:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:21.2:1619677447.982113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.2:1619677447.982114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677447.982116:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677447.982119:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677447.982119:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:2.0:1619677447.982119:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:10.0:1619677447.982120:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.982121:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:10.0:1619677447.982122:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.982122:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677447.982122:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.982123:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.0:1619677447.982124:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:10.0:1619677447.982125:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:5.0:1619677447.982125:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.982127:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.982127:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.982130:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677447.982148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.982151:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.982153:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.982156:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.982156:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.982158:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.982160:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.982162:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720945] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.982163:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.982164:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.982165:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.982168:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.982170:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.982174:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.982174:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.982175:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.982177:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.982178:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.982179:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.982180:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677447.982190:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.982193:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.982194:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.982195:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.982197:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.982198:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.982289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.982298:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.982300:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.982301:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.982303:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.982306:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.982307:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.982309:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.982310:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.982311:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.982312:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.982316:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.982317:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.982320:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.982321:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.982321:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.982323:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.982323:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.982324:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.982324:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677447.982337:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.982340:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.982340:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.982342:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.982344:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.982345:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.982467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1200] (20)++
00000800:00000200:17.0:1619677447.982474:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677447.982477:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:1.0:1619677447.982477:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:17.0:1619677447.982480:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.982482:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.982484:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677447.982485:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.982486:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.982487:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.982488:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.982491:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.982493:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.982495:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.982495:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.982496:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.982498:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.982498:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677447.982499:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdbb1200] (22)++
00000800:00000200:17.0:1619677447.982501:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (23)--
00000800:00000200:17.0:1619677447.982501:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:8.2:1619677447.982504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677447.982507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677447.982513:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)++
00000800:00000200:1.0:1619677447.982513:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.982513:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.982514:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdbb1200] (23)--
00000400:00000200:1.0:1619677447.982514:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.982515:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:17.0:1619677447.982516:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.982516:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.982517:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:1.0:1619677447.982517:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.982595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.982601:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.982603:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.982604:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.982605:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.982606:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.982608:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677447.982609:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.982610:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.982611:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.982611:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677447.982614:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.982616:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677447.982618:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.982619:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677447.982620:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677447.982622:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677447.982622:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.982623:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.982624:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.982624:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677447.982629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.982634:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:30.0:1619677447.982635:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677447.982636:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:30.0:1619677447.982637:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:17.0:1619677447.982637:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677447.982638:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.982638:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677447.982640:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.982642:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:21.2:1619677447.982642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:30.0:1619677447.982643:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.982644:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.982645:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.982645:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.982645:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.982645:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:31.0:1619677447.982646:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.982647:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677447.982649:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.982652:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.982652:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.982654:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.982657:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.982659:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677447.982660:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.982662:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.982662:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.982664:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677447.982667:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.982668:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677447.982672:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.982672:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:18.0:1619677447.982673:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677447.982675:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677447.982676:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.982676:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677447.982677:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677447.982678:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.2:1619677447.982683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:12.0:1619677447.982688:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677447.982688:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.982690:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:12.0:1619677447.982691:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:2.0:1619677447.982691:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:12.0:1619677447.982692:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.982692:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:12.0:1619677447.982694:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.982696:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.982697:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677447.982697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.982705:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.982707:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.982708:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.982710:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.982712:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.982714:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677447.982715:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.982717:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.982718:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.982719:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677447.982722:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.982724:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677447.982726:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.982727:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677447.982728:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677447.982729:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677447.982729:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.982731:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677447.982732:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677447.982732:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677447.982738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677447.982741:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.982744:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.982745:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.982745:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:31.0:1619677447.982746:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.982747:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677447.982747:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677447.982748:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.982748:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677447.982749:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677447.982759:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.982761:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.982762:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.982763:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677447.982764:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.982765:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677447.982797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.982802:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.982805:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.982807:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.982808:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.982811:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.982812:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.982813:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.982815:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.982816:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.982817:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.982820:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.982822:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.982825:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.982826:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.982826:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.982828:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.982829:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.982830:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.982832:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.982832:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.982837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.982840:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:22.0:1619677447.982842:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.982842:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.982843:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.982843:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.982844:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.982845:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.982846:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.982847:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.982850:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.982868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.982875:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.982878:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.982879:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.982881:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.982883:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.982885:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677447.982886:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.982887:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.982889:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.982889:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.2:1619677447.982892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:10.0:1619677447.982893:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.982894:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:10.0:1619677447.982894:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.0:1619677447.982897:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:10.0:1619677447.982898:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.982899:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.982899:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677447.982899:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.0:1619677447.982900:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:10.0:1619677447.982901:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677447.982901:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677447.982902:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.982902:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:20.0:1619677447.982903:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:10.0:1619677447.982903:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000400:00000200:20.0:1619677447.982904:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.982904:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677447.982906:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.982907:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.982908:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:4.2:1619677447.982909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677447.982913:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.982915:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677447.982915:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:26.0:1619677447.982916:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677447.982917:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:26.0:1619677447.982918:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677447.982918:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:20.0:1619677447.982918:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.982919:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677447.982919:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.982919:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:6.0:1619677447.982919:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.982920:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677447.982921:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:6.0:1619677447.982921:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.982922:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.982922:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.982923:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:6.0:1619677447.982924:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677447.982933:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.982936:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.982937:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.982939:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.982940:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.982941:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677447.982995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.983002:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.983005:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.983006:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.983008:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.983010:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.983012:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.983013:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.983014:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.983015:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677447.983015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:24.0:1619677447.983016:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.983019:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.983021:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677447.983023:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:24.0:1619677447.983024:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.983025:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:24.0:1619677447.983025:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.983025:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:30.0:1619677447.983027:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.983027:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.983027:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677447.983029:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:24.0:1619677447.983029:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:30.0:1619677447.983030:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.983030:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.983030:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:30.0:1619677447.983031:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677447.983032:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677447.983033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:30.0:1619677447.983034:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.983034:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.983035:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:4.2:1619677447.983036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:30.0:1619677447.983038:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.983040:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677447.983041:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:30.0:1619677447.983043:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.983043:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.983043:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:30.0:1619677447.983044:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:17.0:1619677447.983044:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.983044:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.983045:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.983045:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:30.0:1619677447.983046:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.983046:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677447.983047:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:8.0:1619677447.983047:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.983047:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.983048:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.983048:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:1.0:1619677447.983049:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.983053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.983061:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.983063:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.983064:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.983066:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.983066:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.983069:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677447.983073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677447.983080:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.983083:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.983084:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.983084:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.983085:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.983086:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677447.983181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.983187:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677447.983188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:17.0:1619677447.983189:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.983190:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.983192:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.983194:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.983195:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:17.0:1619677447.983196:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:18.0:1619677447.983197:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:17.0:1619677447.983197:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.983198:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677447.983199:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.983200:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.983200:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.983201:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677447.983203:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.983204:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.983205:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:17.0:1619677447.983206:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:18.0:1619677447.983207:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.983208:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.983209:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.983210:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.983210:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.983210:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.983211:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:18.0:1619677447.983213:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.983213:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.983214:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677447.983215:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.983215:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.983216:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677447.983218:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.983218:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.983219:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.983221:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.983221:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.983222:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.983223:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.983224:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677447.983226:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:8.2:1619677447.983227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.983229:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.983229:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.983229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000400:00000200:31.0:1619677447.983231:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.983232:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.983234:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:2.0:1619677447.983235:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:1.0:1619677447.983235:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677447.983237:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:1.0:1619677447.983237:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.983239:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677447.983239:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:1.0:1619677447.983239:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.983240:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:1.0:1619677447.983241:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.983244:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.983245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677447.983252:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.983255:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.983256:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.983256:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.983257:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.983258:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.983264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.983271:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.983274:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.983275:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.983277:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.983280:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.983282:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677447.983283:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.983285:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.983286:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.983287:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677447.983290:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.983292:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677447.983299:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.983299:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.983300:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677447.983301:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.983302:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.983303:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.983303:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677447.983316:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.983318:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.983319:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.983320:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.983323:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.983323:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.2:1619677447.983388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677447.983395:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.983398:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677447.983399:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677447.983401:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.983403:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.983405:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677447.983406:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.983407:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.983408:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.983409:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677447.983413:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.983414:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.2:1619677447.983415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:26.0:1619677447.983417:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.983418:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677447.983418:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677447.983420:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677447.983420:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.983421:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:22.0:1619677447.983421:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677447.983422:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:22.0:1619677447.983424:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.983425:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.983426:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677447.983426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677447.983427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:22.0:1619677447.983429:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.983430:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.983432:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.983433:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.983434:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.983435:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:10.0:1619677447.983435:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677447.983435:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:22.0:1619677447.983438:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.983438:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:5.0:1619677447.983438:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:26.0:1619677447.983439:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:5.0:1619677447.983439:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.983441:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.983441:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.983441:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:10.0:1619677447.983442:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.983443:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.983443:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677447.983444:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.983445:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:10.0:1619677447.983445:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.983447:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.983447:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677447.983447:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:22.0:1619677447.983448:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.983448:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677447.983448:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677447.983448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:10.0:1619677447.983449:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.983450:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.983451:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:20.0:1619677447.983454:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:10.0:1619677447.983454:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.983456:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677447.983457:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.983458:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:20.0:1619677447.983458:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:20.0:1619677447.983459:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:10.0:1619677447.983459:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.983459:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677447.983460:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:10.0:1619677447.983460:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677447.983462:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:10.0:1619677447.983462:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677447.983462:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:20.0:1619677447.983463:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.983463:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.983463:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:20.0:1619677447.983465:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.983466:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.983467:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677447.983469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:30.0:1619677447.983474:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:2.0:1619677447.983474:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:30.0:1619677447.983477:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:2.0:1619677447.983477:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:30.0:1619677447.983478:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.983478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677447.983479:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:30.0:1619677447.983480:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.983481:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:2.0:1619677447.983481:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.983482:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:2.0:1619677447.983484:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.983484:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677447.983486:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677447.983486:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.983487:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:2.0:1619677447.983487:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.983489:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.983490:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.983490:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.983492:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.983493:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.983494:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.983495:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.983496:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.983497:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.983498:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:1.0:1619677447.983498:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.983499:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.983502:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.983503:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.983503:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677447.983503:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.983505:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.983505:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.983506:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.983506:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.983507:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:1.0:1619677447.983507:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.983507:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.983508:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.983509:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.983510:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.983511:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.983511:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.983515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677447.983519:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.983521:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677447.983522:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:13.0:1619677447.983522:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.983524:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:13.0:1619677447.983524:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.983525:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.983526:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677447.983526:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.983527:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.983527:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677447.983529:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:12.0:1619677447.983536:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677447.983539:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.983540:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.983541:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.983542:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.983543:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.983553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.2:1619677447.983559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.0:1619677447.983560:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.983564:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.983566:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.983566:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.983567:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:5.0:1619677447.983569:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.983570:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:5.0:1619677447.983571:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.983572:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.983574:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.983574:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:5.0:1619677447.983574:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.983575:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.983576:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677447.983577:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.983579:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.983580:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.983580:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.983582:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.983582:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.983583:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:31.0:1619677447.983584:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.983586:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.983587:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.983587:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:31.0:1619677447.983588:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.983589:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.983590:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.983590:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.983591:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.983591:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677447.983593:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.983594:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.983594:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.983596:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.983596:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.983597:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.983598:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.983602:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.983605:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.983606:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.983606:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.983608:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677447.983608:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.983610:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:6.0:1619677447.983610:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:13.0:1619677447.983611:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.983613:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.983614:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.983615:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.983774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:4.2:1619677447.983781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677447.983782:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.983784:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.983786:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.983787:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.983788:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:24.0:1619677447.983790:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.983791:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:24.0:1619677447.983792:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:10.0:1619677447.983792:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:24.0:1619677447.983793:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.983794:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.983795:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.983796:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.983797:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.983797:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677447.983798:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.983800:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720923] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.983800:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.983801:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.983802:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.983802:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:26.0:1619677447.983803:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677447.983805:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677447.983806:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.983806:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.983807:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677447.983808:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:24.0:1619677447.983808:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.983809:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.983809:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677447.983810:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.983810:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:26.0:1619677447.983811:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.983812:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.983813:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.983813:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.983814:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.983814:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677447.983817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677447.983822:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:8.2:1619677447.983824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:18.0:1619677447.983825:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677447.983826:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:30.0:1619677447.983827:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:18.0:1619677447.983828:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.983829:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.983830:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.983831:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.983832:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.983832:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:1.0:1619677447.983832:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:31.0:1619677447.983833:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.983834:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:18.0:1619677447.983834:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.983835:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.983835:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677447.983835:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:18.0:1619677447.983836:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.983836:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.983837:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:1.0:1619677447.983838:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:18.0:1619677447.983841:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.983841:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:31.0:1619677447.983843:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:18.0:1619677447.983843:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.2:1619677447.983844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:18.0:1619677447.983845:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.983846:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:18.0:1619677447.983846:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677447.983846:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677447.983847:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.983848:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.983848:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:18.0:1619677447.983848:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.983848:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677447.983849:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:31.0:1619677447.983850:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677447.983850:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:5.2:1619677447.983851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.983853:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:9.2:1619677447.983854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:17.0:1619677447.983856:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.983857:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.983857:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.983858:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.983858:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.983859:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.983860:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677447.983861:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677447.983861:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:1.0:1619677447.983861:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.983862:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.983863:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677447.983864:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.983864:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:18.0:1619677447.983865:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.983866:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.983866:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.983867:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:1.0:1619677447.983867:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.983868:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.983869:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677447.983872:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.983874:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677447.983877:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.983878:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677447.983879:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677447.983880:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677447.983881:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.983881:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.983882:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.983886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677447.983893:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677447.983895:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677447.983896:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677447.983897:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.983899:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.983902:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677447.984044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677447.984051:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.984053:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.984054:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.984056:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.984058:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.984060:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677447.984061:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.984062:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.984063:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.984064:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677447.984067:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.984069:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.2:1619677447.984069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:24.0:1619677447.984072:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.984073:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.984074:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.984074:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:24.0:1619677447.984076:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.984077:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.984077:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:21.2:1619677447.984077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:18.0:1619677447.984077:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:24.0:1619677447.984078:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:2.0:1619677447.984078:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.984080:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.984081:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.984082:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:18.0:1619677447.984083:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677447.984084:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677447.984084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677447.984085:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:18.0:1619677447.984086:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.984086:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:18.0:1619677447.984087:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.984088:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677447.984089:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.984091:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.984091:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.984092:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:20.0:1619677447.984092:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:22.0:1619677447.984094:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.984094:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:18.0:1619677447.984094:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:20.0:1619677447.984095:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.984095:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677447.984096:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.984097:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.984097:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.984097:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.984097:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:22.0:1619677447.984098:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.984098:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677447.984099:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.984100:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.984100:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.984101:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:22.0:1619677447.984102:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.984103:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677447.984104:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:9.2:1619677447.984105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677447.984106:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.984107:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.984107:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.984109:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.984109:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.984110:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.984111:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677447.984112:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.984114:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:8.2:1619677447.984115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:2.0:1619677447.984115:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.984116:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.984117:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.984119:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677447.984123:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.984125:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.984126:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.984128:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.984128:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.984130:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677447.984168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.984175:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.984178:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.984180:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.984182:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.984183:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.984184:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.984186:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.984187:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.984188:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.984189:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.984193:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.984194:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.984197:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.984198:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.984198:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.984200:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.984200:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.984201:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.984202:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677447.984208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:21.2:1619677447.984210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:12.0:1619677447.984212:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.984214:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677447.984215:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677447.984216:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:12.0:1619677447.984216:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.984217:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.984218:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:18.0:1619677447.984219:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:13.0:1619677447.984219:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.984219:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:12.0:1619677447.984220:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677447.984221:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.984222:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.984223:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:31.0:1619677447.984225:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.984226:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.984228:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677447.984228:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677447.984229:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720916] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.984230:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720916] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.984231:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.984232:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.984233:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.984233:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.984234:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.984235:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677447.984240:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.984240:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677447.984242:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677447.984244:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.984245:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.984246:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677447.984247:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.984247:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677447.984248:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677447.984248:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.984249:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.984249:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.984250:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:31.0:1619677447.984251:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677447.984251:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677447.984251:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677447.984252:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.984253:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.984256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:6.0:1619677447.984262:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677447.984264:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:6.0:1619677447.984264:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.984265:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.984265:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.984266:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.984267:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.984268:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:20.0:1619677447.984269:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.984269:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:6.0:1619677447.984271:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677447.984272:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677447.984326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.984332:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.984335:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.984336:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.984338:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.984341:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.984342:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.984344:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.984345:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677447.984345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:17.0:1619677447.984346:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.984347:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.984351:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.984353:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677447.984353:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677447.984356:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.984356:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.984357:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677447.984357:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.984358:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.984360:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.984360:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677447.984360:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.984362:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:10.0:1619677447.984362:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.984363:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000400:00000200:10.0:1619677447.984363:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:17.0:1619677447.984364:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677447.984365:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.984366:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.984367:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.984368:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.2:1619677447.984368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:4.2:1619677447.984370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:10.0:1619677447.984371:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.984373:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:13.0:1619677447.984374:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677447.984374:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677447.984376:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:10.0:1619677447.984376:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.984376:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.984376:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677447.984377:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.984377:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.984377:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.0:1619677447.984377:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.984377:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677447.984379:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.984379:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.984379:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677447.984380:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:10.0:1619677447.984380:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677447.984380:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:12.0:1619677447.984381:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.984381:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:13.0:1619677447.984382:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677447.984383:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.984385:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.984387:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.984388:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96720912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.984389:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.984390:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.984391:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677447.984392:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.984394:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.984395:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.984395:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.984397:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.984397:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677447.984399:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:8.0:1619677447.984399:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.984400:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.0:1619677447.984400:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.984401:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.984403:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.984403:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.984404:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.984404:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.984409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.984416:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.984419:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.984420:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.984420:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.984422:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.984425:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.984603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.984609:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.984611:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.984614:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.984614:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.984616:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.984618:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677447.984619:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.984620:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.984621:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.984622:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677447.984626:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.984628:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.984630:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.984631:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677447.984632:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677447.984634:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677447.984634:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.984635:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.984635:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:26.0:1619677447.984646:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.984648:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.984649:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.984649:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677447.984652:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:26.0:1619677447.984655:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677447.984703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.984709:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.984712:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.984713:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.984715:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.984717:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.984719:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.984720:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.984721:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.984722:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.984723:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.984727:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.984728:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.984731:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.984732:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.984732:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.984734:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.984734:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.984736:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.984737:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.984737:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677447.984740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677447.984742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.984747:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:1.0:1619677447.984748:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677447.984749:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.984750:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677447.984750:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:6.0:1619677447.984751:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:1.0:1619677447.984751:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.984752:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.984754:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.984756:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677447.984907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.2:1619677447.984911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:30.0:1619677447.984915:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.984917:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:2.0:1619677447.984917:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.984920:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:31.0:1619677447.984921:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.984921:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.984921:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:30.0:1619677447.984923:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.984924:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.984925:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677447.984926:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.984926:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.984928:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.984928:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.984929:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.984929:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.984931:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677447.984931:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.984932:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.984933:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677447.984938:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.984939:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.984941:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.984942:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.984944:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.984945:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.984945:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.984946:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.984946:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.984947:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677447.984948:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.984948:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.984948:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:30.0:1619677447.984949:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677447.984949:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.984950:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:30.0:1619677447.984951:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.984952:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.984953:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677447.984955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677447.984958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677447.984963:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:12.0:1619677447.984963:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677447.984966:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:18.0:1619677447.984966:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:12.0:1619677447.984966:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:18.0:1619677447.984967:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.984967:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.984967:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (23)--
00000800:00000200:31.0:1619677447.984969:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:13.0:1619677447.984969:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.984969:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.984970:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677447.984970:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.984971:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.984972:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677447.984975:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:13.0:1619677447.984981:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.984984:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.984985:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.984985:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.984986:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.984987:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.985015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677447.985021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:10.0:1619677447.985022:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.985025:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.985026:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677447.985028:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677447.985028:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.985030:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:10.0:1619677447.985030:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.985031:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:8.0:1619677447.985031:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.2:1619677447.985031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:10.0:1619677447.985033:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.985034:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.985034:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.985035:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.985036:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:24.0:1619677447.985037:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.985037:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:24.0:1619677447.985038:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.985040:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96720906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.985040:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.985041:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:10.0:1619677447.985041:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.985042:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.985044:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677447.985044:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:2.0:1619677447.985044:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.985045:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:10.0:1619677447.985047:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677447.985047:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.985048:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:2.0:1619677447.985048:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.985049:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.985049:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677447.985050:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:2.0:1619677447.985050:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.985051:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677447.985051:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677447.985051:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677447.985052:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:2.0:1619677447.985052:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.985053:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.985054:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.985054:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677447.985055:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.985056:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.985056:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677447.985057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:2.0:1619677447.985057:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.985058:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.985058:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.985059:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.985059:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677447.985059:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:4.2:1619677447.985061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.985061:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.985061:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.985062:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.985063:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677447.985064:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677447.985064:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.985065:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.985065:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:31.0:1619677447.985066:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:22.0:1619677447.985067:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:26.0:1619677447.985068:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:22.0:1619677447.985068:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.985068:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677447.985069:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:9.2:1619677447.985069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.985070:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.985071:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677447.985071:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.985072:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.985072:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677447.985073:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:22.0:1619677447.985074:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.2:1619677447.985075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677447.985076:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.985077:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:31.0:1619677447.985078:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.985079:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.985079:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:31.0:1619677447.985080:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.985080:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.985080:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:31.0:1619677447.985082:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.985082:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:31.0:1619677447.985083:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677447.985083:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.985083:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:18.0:1619677447.985084:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.985086:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:9.2:1619677447.985086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677447.985086:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.985087:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:17.0:1619677447.985087:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.985087:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677447.985087:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:31.0:1619677447.985088:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.985088:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000400:00000200:17.0:1619677447.985089:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.985089:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677447.985090:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:17.0:1619677447.985092:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:31.0:1619677447.985093:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.985093:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:2.0:1619677447.985093:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677447.985094:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.985096:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.985096:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677447.985096:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.985097:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.985097:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677447.985098:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:2.0:1619677447.985098:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.985099:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677447.985100:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.985100:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677447.985106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.985110:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.985112:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.985113:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.985114:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.985115:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:30.0:1619677447.985115:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.985116:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.985117:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.985118:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677447.985118:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677447.985427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:20.2:1619677447.985432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:22.0:1619677447.985434:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.985436:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.985437:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.985439:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.985440:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:22.0:1619677447.985441:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.985443:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:30.0:1619677447.985444:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:22.0:1619677447.985444:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.985446:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.985447:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.985447:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.985448:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.985448:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.985448:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677447.985450:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677447.985452:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.985453:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.985453:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.985454:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.985455:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:30.0:1619677447.985456:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:22.0:1619677447.985458:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677447.985459:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.985459:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.985459:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.985461:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677447.985462:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677447.985462:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677447.985463:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.985463:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.985463:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677447.985464:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.985465:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.985466:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.985467:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.985468:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.985468:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:9.2:1619677447.985468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.985477:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:31.0:1619677447.985479:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:18.0:1619677447.985480:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:31.0:1619677447.985481:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:18.0:1619677447.985481:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.985481:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:31.0:1619677447.985482:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.985483:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.985484:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.985485:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.985486:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:18.0:1619677447.985486:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677447.985496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.985503:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.985506:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.985508:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.985510:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.985510:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.985512:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677447.985513:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.985514:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.985515:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.985516:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677447.985520:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.985521:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.985524:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.985525:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677447.985525:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677447.985527:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.985527:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.985528:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.985529:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.985541:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.985543:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.985544:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.985545:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.985547:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.985550:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.985562:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.985565:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.985566:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.985566:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.985567:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.985568:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.985599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.985602:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.985604:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.985607:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.985607:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.985610:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.985611:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677447.985613:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.985614:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.985615:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.985616:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677447.985619:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.985621:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677447.985624:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.985625:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.985626:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677447.985627:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677447.985628:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.985629:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.985629:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677447.985634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677447.985643:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.985645:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.985646:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.985646:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.985649:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.985650:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677447.985677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.985685:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.985687:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.985688:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.985691:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.985693:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.985695:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677447.985696:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.985697:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.985698:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.985699:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677447.985702:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.985704:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.985706:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.985706:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677447.985707:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677447.985708:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.985709:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.985710:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.985710:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677447.985711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:4.2:1619677447.985718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677447.985719:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.985721:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.985722:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.985722:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.985723:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.985723:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.985724:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.985725:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677447.985726:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.985727:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677447.985728:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677447.985728:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677447.985728:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:26.0:1619677447.985729:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.985729:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:10.0:1619677447.985729:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:26.0:1619677447.985731:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.985731:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.985732:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.985733:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677447.985733:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.985735:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677447.985737:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.985737:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96720897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.985738:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677447.985739:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000400:00000200:8.0:1619677447.985739:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.985740:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677447.985740:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000400:00000200:8.0:1619677447.985740:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:10.0:1619677447.985741:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.985742:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.985743:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.985743:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:8.0:1619677447.985743:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.985744:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677447.985744:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677447.985745:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677447.985745:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677447.985746:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677447.985747:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.985748:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.985748:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677447.985748:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.985749:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677447.985749:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677447.985751:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677447.985751:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.985753:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.985754:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.985755:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.985757:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:12.0:1619677447.985759:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:4.2:1619677447.985760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:12.0:1619677447.985761:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.985762:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.985763:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.985765:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:12.0:1619677447.985766:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677447.985767:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677447.985767:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:13.0:1619677447.985767:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677447.985768:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.985769:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677447.985769:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.985770:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.985770:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677447.985771:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.985774:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.985995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.986003:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.986006:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.986008:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677447.986009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.986010:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.986011:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.986012:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:30.0:1619677447.986012:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:31.0:1619677447.986013:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.986013:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.986015:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.986016:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.986016:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.986019:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.986021:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:30.0:1619677447.986022:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.986023:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.986024:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:30.0:1619677447.986024:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.986025:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:30.0:1619677447.986025:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:31.0:1619677447.986026:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677447.986027:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720895] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.986028:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.986029:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.986030:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:30.0:1619677447.986030:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:31.0:1619677447.986031:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.986032:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:30.0:1619677447.986033:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.986034:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.986034:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677447.986035:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.986036:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677447.986036:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.986037:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.986037:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677447.986037:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:31.0:1619677447.986038:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.986039:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.986040:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.986040:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677447.986040:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.986041:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.986041:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677447.986050:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.986052:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.986053:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.986053:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.986055:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.986058:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677447.986058:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677447.986061:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.986062:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.986062:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677447.986063:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.986064:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677447.986149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.986156:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.986159:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.986160:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.986161:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.986163:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.986165:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677447.986166:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.986167:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.986168:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.986169:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677447.986172:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.986173:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677447.986176:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.986177:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677447.986178:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677447.986180:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677447.986180:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.986180:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.986181:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677447.986186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.986193:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.986196:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.986197:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.986198:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.986200:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.986202:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677447.986225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677447.986230:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.986233:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.986234:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.986235:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.986237:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.986239:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.986240:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.986241:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.986242:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.986243:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.986246:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.986248:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.986251:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.986252:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.986253:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.986255:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.986255:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.986255:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.986256:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:10.0:1619677447.986267:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.986269:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.986270:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.986271:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.986273:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.986276:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677447.986307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.2:1619677447.986314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677447.986319:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.986321:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.986323:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000400:00000200:26.0:1619677447.986324:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.986324:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677447.986327:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.986328:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.986329:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677447.986329:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.986331:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.986332:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.986333:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.986334:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677447.986337:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.986339:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:10.0:1619677447.986339:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677447.986340:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.986341:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.986342:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.986343:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677447.986346:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677447.986346:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.986347:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.986348:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677447.986348:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677447.986349:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.986350:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677447.986351:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677447.986352:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.986352:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.986353:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677447.986353:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677447.986353:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677447.986354:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677447.986355:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677447.986356:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677447.986357:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677447.986359:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677447.986360:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677447.986361:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.986362:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.986362:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.986362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677447.986363:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.986364:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:21.2:1619677447.986365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677447.986369:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677447.986370:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677447.986371:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:5.0:1619677447.986371:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:10.0:1619677447.986372:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677447.986373:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:5.0:1619677447.986374:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.986374:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.986376:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.986377:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.986380:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677447.986406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.986412:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.986415:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.986416:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.986418:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.986420:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.986422:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.986423:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.986424:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.986425:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.986426:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.986429:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.986431:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.986434:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.986434:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.986435:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.986437:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.986437:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.986438:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.986438:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.986451:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.986454:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.986455:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.986455:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.986457:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.986459:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677447.986495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.986503:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.986506:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.986508:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.986510:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.986510:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.986511:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677447.986513:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.986514:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.986515:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.986516:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677447.986520:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.986521:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677447.986524:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.986525:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677447.986526:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677447.986527:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677447.986528:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.986529:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.986529:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677447.986541:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.986543:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.986544:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.986545:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.986547:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.986550:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.986679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.986686:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.986689:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.986690:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.986691:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677447.986691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:26.0:1619677447.986694:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.986696:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677447.986697:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.986698:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677447.986698:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.986699:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.986700:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:30.0:1619677447.986701:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:26.0:1619677447.986704:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.986705:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.986705:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.986706:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:30.0:1619677447.986707:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.986709:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:26.0:1619677447.986709:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677447.986710:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.986710:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:30.0:1619677447.986711:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677447.986711:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:30.0:1619677447.986712:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.986712:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:30.0:1619677447.986713:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:26.0:1619677447.986713:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.986714:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.986714:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.2:1619677447.986715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:30.0:1619677447.986716:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.986717:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677447.986717:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.2:1619677447.986719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677447.986720:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.986720:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:30.0:1619677447.986721:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.986722:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.986723:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:20.0:1619677447.986723:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.986724:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677447.986724:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677447.986725:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.986725:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677447.986725:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.986726:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:5.0:1619677447.986726:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.986727:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.986728:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.986728:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.986729:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.986730:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.986731:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.986732:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.2:1619677447.986734:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:5.0:1619677447.986735:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:12.0:1619677447.986736:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677447.986736:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:12.0:1619677447.986738:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.986739:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.986740:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677447.986741:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.986742:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677447.986742:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:12.0:1619677447.986743:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.986744:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677447.986744:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677447.986745:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:20.0:1619677447.986745:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.986746:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.986746:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:10.0:1619677447.986747:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.986748:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.986749:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:10.0:1619677447.986749:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.986750:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.986750:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:10.0:1619677447.986751:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677447.986752:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.986753:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677447.986754:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:10.0:1619677447.986754:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.986755:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:12.0:1619677447.986757:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677447.986758:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.986758:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677447.986759:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:10.0:1619677447.986759:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677447.986760:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677447.986761:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:10.0:1619677447.986761:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.986764:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:10.0:1619677447.986764:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.986765:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.986765:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677447.986765:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:31.0:1619677447.986766:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.986767:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.986767:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677447.986768:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.986768:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:31.0:1619677447.986769:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677447.986769:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677447.986773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.986781:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.986783:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.986784:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.986786:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677447.986786:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.986788:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677447.986792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677447.986800:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.986803:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677447.986804:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.986805:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.986805:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.986806:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677447.986826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.986832:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.986834:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.986835:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.986837:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.986840:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.986841:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677447.986842:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.986844:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.986845:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.986846:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.986849:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.986851:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.986853:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.986854:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.986855:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.986856:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.986857:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.986858:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.986858:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.986870:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.986873:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.986874:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.986874:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.986876:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.986878:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677447.986937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.986942:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.986945:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.986945:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.986948:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.986950:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.986951:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.986952:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.986954:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.986955:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.986956:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.986959:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.986961:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.986964:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.986965:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.986966:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.986968:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.986968:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.986969:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.986970:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677447.986981:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.986984:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.986985:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.986987:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677447.986987:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.986990:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677447.987018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.2:1619677447.987023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:24.0:1619677447.987025:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.987028:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.987029:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677447.987029:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:24.0:1619677447.987031:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.987032:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:24.0:1619677447.987034:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.987034:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.987035:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677447.987036:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.987037:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.987038:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.987039:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.987040:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:24.0:1619677447.987041:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.987042:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.987042:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.987044:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.987045:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.987047:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:24.0:1619677447.987049:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677447.987050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:24.0:1619677447.987051:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.987051:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677447.987054:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.987054:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:24.0:1619677447.987055:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.987056:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.987056:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.987057:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677447.987057:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:24.0:1619677447.987058:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.987058:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677447.987058:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:24.0:1619677447.987059:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677447.987059:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677447.987060:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:24.0:1619677447.987060:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677447.987060:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:10.0:1619677447.987061:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:2.0:1619677447.987061:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.987062:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:26.0:1619677447.987063:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677447.987064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677447.987066:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.987067:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677447.987068:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.987069:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.987071:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.987071:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677447.987072:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677447.987072:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.987074:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:26.0:1619677447.987075:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.987075:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.987075:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:1.0:1619677447.987075:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.987076:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.987076:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.987077:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.2:1619677447.987077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:1.0:1619677447.987078:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.987079:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:26.0:1619677447.987080:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.987080:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677447.987081:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.987081:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:26.0:1619677447.987083:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677447.987083:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677447.987084:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677447.987084:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677447.987085:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677447.987086:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:8.2:1619677447.987086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:26.0:1619677447.987087:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677447.987087:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:9.2:1619677447.987089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.987089:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.987090:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.987091:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:17.0:1619677447.987092:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.987092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677447.987092:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.987093:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000400:00000200:17.0:1619677447.987094:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:1.0:1619677447.987094:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.987095:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.987095:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.987096:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677447.987097:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:17.0:1619677447.987097:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.987098:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677447.987099:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:18.0:1619677447.987100:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:17.0:1619677447.987100:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:2.0:1619677447.987100:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.987101:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.987102:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677447.987102:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.987103:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:18.0:1619677447.987104:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.987104:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.987106:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:18.0:1619677447.987107:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.987109:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.987110:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677447.987110:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677447.987112:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677447.987113:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.987113:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.987114:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677447.987118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.2:1619677447.987121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:8.0:1619677447.987124:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:2.0:1619677447.987126:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:8.0:1619677447.987127:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677447.987128:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677447.987128:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.987129:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.987130:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:8.0:1619677447.987130:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.987132:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.987134:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:2.0:1619677447.987135:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.987137:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677447.987138:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677447.987139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000400:00000200:2.0:1619677447.987139:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.987140:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.987141:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:24.0:1619677447.987145:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000400:00000200:2.0:1619677447.987145:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.987146:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:24.0:1619677447.987147:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677447.987148:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677447.987148:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677447.987149:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:2.0:1619677447.987149:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.987150:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:2.0:1619677447.987150:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677447.987151:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677447.987153:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677447.987153:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.987153:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677447.987154:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.987159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.987166:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.987169:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.987169:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.987171:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.987172:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.987174:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.987260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.987267:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.987270:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.987271:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.987273:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.987276:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.987277:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677447.987279:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.987280:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.987281:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.987282:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677447.987285:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.987287:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677447.987289:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.987290:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.987290:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677447.987291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677447.987292:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.987292:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.987299:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:5.0:1619677447.987299:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677447.987300:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.987301:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677447.987301:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.987304:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.987305:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.2:1619677447.987306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677447.987307:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:4.2:1619677447.987307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677447.987308:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.987310:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:20.0:1619677447.987310:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.987310:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677447.987311:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.987312:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.987312:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.987314:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:5.0:1619677447.987314:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.987315:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.987315:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:5.0:1619677447.987315:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.987316:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.987316:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:5.0:1619677447.987316:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.987317:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.987317:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677447.987317:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677447.987318:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:20.0:1619677447.987319:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.987320:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677447.987320:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.987320:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:31.0:1619677447.987321:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.987321:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:5.0:1619677447.987321:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.987322:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:20.0:1619677447.987322:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.987323:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.987323:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.987323:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.987324:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:20.0:1619677447.987324:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.987324:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.987325:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.987326:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720874] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.987326:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677447.987328:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.987329:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.987329:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677447.987330:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.987330:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.987331:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.987333:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677447.987334:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)++
00000400:00000200:30.0:1619677447.987335:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.987335:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (24)++
00000400:00000200:20.0:1619677447.987336:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.987337:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (25)--
00000800:00000200:5.0:1619677447.987337:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (24)--
00000800:00000200:30.0:1619677447.987338:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.987339:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677447.987339:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (23)--
00000800:00000200:30.0:1619677447.987341:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.987342:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.2:1619677447.987342:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (22)++
00000800:00000200:30.0:1619677447.987343:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:30.0:1619677447.987345:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:30.0:1619677447.987345:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677447.987346:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.987346:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:30.0:1619677447.987347:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677447.987347:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677447.987348:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:20.0:1619677447.987348:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677447.987348:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (23)++
00000800:00000200:30.0:1619677447.987349:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677447.987349:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677447.987349:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (24)--
00000800:00000200:20.0:1619677447.987350:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677447.987350:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677447.987350:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.987350:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.987350:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.987351:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.987352:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:13.0:1619677447.987352:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:21.2:1619677447.987354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677447.987354:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.987355:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677447.987355:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677447.987356:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.987356:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.987357:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.987359:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677447.987362:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.987364:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.987364:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.987365:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.987366:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677447.987366:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.987367:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.987368:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677447.987369:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:20.2:1619677447.987371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:30.0:1619677447.987372:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.0:1619677447.987377:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.987380:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.987382:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.987382:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.987385:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.987387:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677447.987388:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.987389:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.987390:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.987391:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.987395:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.987397:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.987399:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.987400:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.987400:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.987402:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.987403:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.987403:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.987404:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677447.987408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.987416:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.987419:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.987420:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.987420:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.987422:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677447.987425:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.987467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.987472:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.987475:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.987476:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.987477:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.987480:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.987481:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.987482:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.987484:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.987485:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.987486:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677447.987489:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.987491:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.987493:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.987493:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.987494:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.987496:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.987496:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.987498:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.987499:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.987499:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677447.987502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677447.987504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.987510:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.987511:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.987511:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.987512:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.987512:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.987512:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.987513:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.987513:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.987515:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677447.987515:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677447.987521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.987527:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.987530:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.987531:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.987531:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.987531:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.987532:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677447.987723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677447.987731:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.987733:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.987734:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.987736:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.987739:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.987740:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677447.987742:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.987743:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.987744:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.987745:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677447.987748:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.987750:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677447.987753:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.987754:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.987754:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.987756:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.987757:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.987758:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.987760:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.987760:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677447.987763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677447.987766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:18.0:1619677447.987770:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677447.987773:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:18.0:1619677447.987773:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.987774:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.987774:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.987775:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.987775:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.987776:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:18.0:1619677447.987777:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.987780:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677447.987806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.987814:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.987816:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.987817:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.987819:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.987822:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.987823:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677447.987824:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.987826:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.987827:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.987827:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677447.987831:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.987833:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677447.987835:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.987835:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.987836:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677447.987837:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.987838:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.987839:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.987839:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677447.987851:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.987853:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.987854:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.987855:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.987857:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.987857:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.987865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.987871:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.987873:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.987874:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.987876:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.987878:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.987879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:1.0:1619677447.987879:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.987881:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720869] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.987882:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.987883:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.987884:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:26.0:1619677447.987886:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:1.0:1619677447.987887:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.987889:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.987890:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677447.987890:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:26.0:1619677447.987892:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.987893:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677447.987894:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.987894:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.987894:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:26.0:1619677447.987896:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:1.0:1619677447.987896:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:26.0:1619677447.987897:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.987897:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.987897:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677447.987898:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.987898:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677447.987899:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.987900:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677447.987904:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.987906:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677447.987908:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.987909:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:13.0:1619677447.987909:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:26.0:1619677447.987910:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677447.987911:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:13.0:1619677447.987911:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:26.0:1619677447.987912:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.987912:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:13.0:1619677447.987912:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.987913:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:13.0:1619677447.987914:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.987915:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.987917:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677447.987923:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677447.987926:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.987926:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.987928:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.987930:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.987931:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677447.988172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.988180:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.988183:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.988185:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.988187:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.988187:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.988188:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677447.988189:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.988190:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.988191:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.988192:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.988195:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.988196:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.988199:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.988199:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677447.988200:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677447.988201:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677447.988202:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.988203:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.988203:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677447.988208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677447.988213:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.988215:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:10.0:1619677447.988215:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:12.0:1619677447.988216:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.988218:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.988218:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.988219:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:13.0:1619677447.988219:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677447.988221:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.988222:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677447.988223:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.988224:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677447.988225:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.988227:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.988228:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.988228:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:13.0:1619677447.988231:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677447.988232:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.988233:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.988234:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677447.988234:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:13.0:1619677447.988235:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.988235:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.988236:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677447.988236:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.988237:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.988237:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677447.988239:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.988239:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.988240:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.988240:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677447.988251:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.988253:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.988254:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.988255:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.988257:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.988258:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677447.988284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.988290:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.2:1619677447.988292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677447.988294:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677447.988297:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.988299:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:17.0:1619677447.988300:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:20.0:1619677447.988302:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.988303:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.988303:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.988304:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.988306:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677447.988306:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.988308:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.988308:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.988309:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.988310:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.988310:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.988311:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.988312:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.988313:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.988314:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677447.988316:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.988318:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:17.0:1619677447.988321:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.988322:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.988322:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.988323:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.988323:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.988325:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.988326:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677447.988326:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.988327:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.988327:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677447.988327:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677447.988327:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677447.988329:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677447.988329:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.988330:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.988331:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.988339:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.988342:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677447.988343:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.988344:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.988346:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677447.988347:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677447.988347:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677447.988348:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.988348:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.988349:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.988351:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677447.988385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.988394:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.988396:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.988399:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.988401:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.988401:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.988402:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.988403:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.988405:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.988406:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.988407:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.988410:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.988412:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.988415:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.988416:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.988417:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.988419:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.988420:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.988421:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677447.988423:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677447.988423:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677447.988426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677447.988429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677447.988434:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677447.988437:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:5.0:1619677447.988437:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.988438:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.988439:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:5.0:1619677447.988439:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.988440:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677447.988440:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677447.988441:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677447.988442:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677447.988532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1200] (20)++
00000800:00000200:1.0:1619677447.988538:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:1.0:1619677447.988541:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:17.0:1619677447.988542:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:1.0:1619677447.988544:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.988547:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.988548:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:1.0:1619677447.988549:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.988551:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.988552:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.988553:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:1.0:1619677447.988557:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.988559:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:1.0:1619677447.988561:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.988562:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:1.0:1619677447.988563:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:1.0:1619677447.988564:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:1.0:1619677447.988565:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:1.0:1619677447.988565:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:1.0:1619677447.988566:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:8.2:1619677447.988571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.988578:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.988580:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.988581:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.988583:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.988584:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.988585:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:4.2:1619677447.988602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.988610:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.988612:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.988613:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.988615:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677447.988615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677447.988617:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.988619:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677447.988620:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.988621:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677447.988622:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.988622:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.988623:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:26.0:1619677447.988624:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.988624:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.988627:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677447.988627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:26.0:1619677447.988629:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.988630:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.988631:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:26.0:1619677447.988632:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:4.2:1619677447.988632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:17.0:1619677447.988633:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.988634:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.988635:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.988636:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:10.0:1619677447.988636:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.988637:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.988638:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677447.988638:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677447.988638:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:8.0:1619677447.988638:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677447.988640:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:17.0:1619677447.988640:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.988641:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677447.988641:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:5.2:1619677447.988641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:24.0:1619677447.988642:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:10.0:1619677447.988642:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.988643:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.988643:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677447.988643:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:26.0:1619677447.988644:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.988645:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.988645:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.988646:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677447.988646:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:17.0:1619677447.988647:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677447.988647:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:8.0:1619677447.988647:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.988647:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:10.0:1619677447.988648:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.988649:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677447.988649:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677447.988649:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.988649:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:17.0:1619677447.988650:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677447.988650:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.988651:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677447.988651:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:10.0:1619677447.988651:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.988651:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96720858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.988652:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677447.988652:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.988653:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.988654:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.988655:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:9.2:1619677447.988655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:8.0:1619677447.988655:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.988655:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.988657:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:22.0:1619677447.988658:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:10.0:1619677447.988658:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.988659:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.988659:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:2.0:1619677447.988659:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:508, d:32, s:-96720857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.988660:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.988661:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:2.0:1619677447.988661:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.988662:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:22.0:1619677447.988662:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:2.0:1619677447.988662:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.988663:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.988663:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.988664:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:10.0:1619677447.988664:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.988665:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.988665:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677447.988665:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:22.0:1619677447.988667:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677447.988667:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:8.0:1619677447.988667:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.988667:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:24.0:1619677447.988668:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.988668:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677447.988668:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677447.988668:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677447.988668:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.988669:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.988669:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677447.988670:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.988671:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:8.0:1619677447.988671:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677447.988671:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.988672:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:22.0:1619677447.988672:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.0:1619677447.988672:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677447.988672:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:24.0:1619677447.988673:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677447.988673:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:8.0:1619677447.988673:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677447.988673:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.988674:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677447.988674:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.988675:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.2:1619677447.988676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.988677:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.988677:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.988678:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677447.988679:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:30.0:1619677447.988680:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:2.0:1619677447.988680:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:30.0:1619677447.988683:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:9.2:1619677447.988683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677447.988684:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.988684:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:30.0:1619677447.988685:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:5.2:1619677447.988685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677447.988686:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:1.0:1619677447.988686:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.988687:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677447.988688:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:1.0:1619677447.988688:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.988690:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.988691:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.988692:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.988693:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:1.0:1619677447.988693:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:18.0:1619677447.988694:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.988694:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.988695:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.988696:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.988696:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.988697:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.988697:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.988700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.988707:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.988710:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.988711:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.988711:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.988711:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.988712:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677447.988786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677447.988793:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.988796:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.988797:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.988798:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.988801:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.988803:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.988804:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.988806:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.988807:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.988808:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:20.2:1619677447.988809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:8.0:1619677447.988811:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.988813:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.988816:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.988817:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:8.0:1619677447.988817:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.988818:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.988819:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677447.988820:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677447.988820:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.0:1619677447.988820:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.988821:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.988821:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:26.0:1619677447.988822:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.988825:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.988825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:26.0:1619677447.988826:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677447.988827:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.988829:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.988829:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.988830:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677447.988833:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.988833:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677447.988835:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.988836:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.988837:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.988839:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.988839:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.988839:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.988840:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677447.988840:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677447.988842:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.988842:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:26.0:1619677447.988843:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677447.988844:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677447.988845:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:8.2:1619677447.988845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677447.988846:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677447.988849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677447.988851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677447.988857:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677447.988857:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:10.0:1619677447.988858:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677447.988859:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.988860:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677447.988860:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:10.0:1619677447.988860:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:20.0:1619677447.988861:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677447.988861:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:17.0:1619677447.988861:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.988861:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:1.0:1619677447.988861:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:20.0:1619677447.988863:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:10.0:1619677447.988863:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:17.0:1619677447.988864:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.988867:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.988886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.988892:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.988894:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.988897:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.988897:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677447.988899:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.988900:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677447.988901:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.988903:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.988904:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.988905:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677447.988908:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677447.988910:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.988913:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.988914:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677447.988914:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677447.988916:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677447.988917:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.988918:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.2:1619677447.988921:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.0:1619677447.988923:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.988924:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.2:1619677447.988925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:1.0:1619677447.988926:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677447.988928:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:20.0:1619677447.988928:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:1.0:1619677447.988929:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.0:1619677447.988930:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677447.988930:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:17.0:1619677447.988930:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677447.988931:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:20.0:1619677447.988931:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:10.0:1619677447.988931:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677447.988932:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.988932:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.988934:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000400:00000200:1.0:1619677447.988934:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.988936:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.988937:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.988938:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:1.0:1619677447.988938:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.988940:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.988941:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.988944:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.988945:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:1.0:1619677447.988946:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677447.988948:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.988948:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:1.0:1619677447.988948:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.988949:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (21)++
00000800:00000200:1.0:1619677447.988949:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677447.988950:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:1.0:1619677447.988950:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677447.988951:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:1.0:1619677447.988952:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.988952:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.988954:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.988955:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.988956:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677447.988960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677447.988965:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677447.988966:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677447.988968:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677447.988968:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.988969:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.988969:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677447.988969:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.988970:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677447.988971:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.988973:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677447.989065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.989070:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.989072:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.989073:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.989075:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.989077:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.989079:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677447.989080:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.989081:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.989082:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.989084:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677447.989087:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.989089:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677447.989091:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.989092:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677447.989093:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677447.989095:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.989096:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.989096:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.989096:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.989101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.989108:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.989111:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.989111:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.989113:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.989114:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.989116:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677447.989215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.989221:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.989224:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.989225:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.989227:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.989229:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.989229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:18.0:1619677447.989230:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.989232:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.989233:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.989234:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.989235:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.989235:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.989238:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.989238:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:18.0:1619677447.989239:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.989241:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.2:1619677447.989242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:1.0:1619677447.989242:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.989243:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.989244:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.989244:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:1.0:1619677447.989244:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.989246:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.989246:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:18.0:1619677447.989247:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.989247:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:1.0:1619677447.989247:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.989248:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:1.0:1619677447.989248:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.989249:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:1.0:1619677447.989249:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.989250:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:31.0:1619677447.989252:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:8.2:1619677447.989252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677447.989255:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.989255:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.989256:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:1.0:1619677447.989258:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:31.0:1619677447.989259:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.989260:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:17.0:1619677447.989261:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.989261:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677447.989262:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.989262:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.989262:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000400:00000200:31.0:1619677447.989263:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.989263:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.989263:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:31.0:1619677447.989265:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.989265:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.989265:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000400:00000200:31.0:1619677447.989266:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:1.0:1619677447.989266:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.989267:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.989267:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677447.989268:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:31.0:1619677447.989269:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.989269:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677447.989271:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:9.2:1619677447.989271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677447.989274:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.989275:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677447.989276:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.989277:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:31.0:1619677447.989278:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677447.989279:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.989279:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.989280:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:24.0:1619677447.989280:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677447.989280:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677447.989281:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.989283:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.2:1619677447.989284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.989286:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.0:1619677447.989293:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.989302:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.989303:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.989304:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.989306:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.989308:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677447.989344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.989352:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.989355:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.989356:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.989357:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.989360:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.989362:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677447.989363:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.989365:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.989365:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.989366:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677447.989370:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.989371:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677447.989374:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.989375:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677447.989376:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677447.989377:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677447.989378:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.989379:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.989380:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677447.989381:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677447.989386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.989393:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677447.989393:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677447.989395:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.989395:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677447.989396:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.989397:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677447.989397:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677447.989397:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.989399:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.989402:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677447.989472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.989475:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.989478:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.989480:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.989481:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.989483:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.989484:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:21.2:1619677447.989485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:20.0:1619677447.989486:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.989487:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.989488:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.989489:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:2.0:1619677447.989491:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:20.0:1619677447.989493:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.989493:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:20.0:1619677447.989494:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:18.0:1619677447.989495:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.989496:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.989497:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.989498:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677447.989498:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:2.0:1619677447.989499:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.989500:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:2.0:1619677447.989500:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:20.0:1619677447.989501:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.989501:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:2.0:1619677447.989501:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.989502:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:2.0:1619677447.989503:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.989504:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.989505:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.2:1619677447.989507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:2.0:1619677447.989508:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.989510:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.989512:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.989512:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.989513:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.989514:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677447.989515:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:2.0:1619677447.989515:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.989515:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677447.989516:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.989517:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:17.0:1619677447.989518:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.989519:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.989519:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:9.2:1619677447.989520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:17.0:1619677447.989521:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:21.2:1619677447.989523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677447.989523:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677447.989529:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.989530:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677447.989530:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677447.989531:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.989531:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677447.989532:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.989533:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.989533:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.989534:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677447.989535:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677447.989617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.989622:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.989625:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.989626:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.989627:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.989630:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677447.989630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:22.0:1619677447.989631:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.989632:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.989634:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.989635:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677447.989635:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:22.0:1619677447.989636:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.989638:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:22.0:1619677447.989639:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.989639:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.989641:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:22.0:1619677447.989642:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:2.0:1619677447.989643:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.989644:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.989645:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.989645:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677447.989645:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.989646:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677447.989647:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.989647:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677447.989647:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.989648:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677447.989649:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:2.0:1619677447.989649:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677447.989650:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.989651:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677447.989652:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:8.2:1619677447.989653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677447.989654:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:21.2:1619677447.989655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677447.989656:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.989656:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.989657:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.989658:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.989659:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:6.0:1619677447.989660:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677447.989660:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.989660:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677447.989661:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677447.989663:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:9.2:1619677447.989664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.989664:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677447.989664:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.989665:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.989665:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:1.0:1619677447.989665:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.989667:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.989669:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677447.989671:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.2:1619677447.989671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.989673:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.989674:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.989675:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.989676:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677447.989678:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677447.989678:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.989681:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.989682:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.989682:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.989683:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.989683:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.989800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:21.2:1619677447.989801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677447.989806:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.989809:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:30.0:1619677447.989810:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677447.989810:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677447.989811:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.989813:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:2.0:1619677447.989813:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.989815:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:31.0:1619677447.989817:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.989817:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.989817:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.989818:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.989819:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.989820:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.989820:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677447.989822:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677447.989824:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720842] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.989824:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.989825:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.989826:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.989827:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677447.989827:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677447.989828:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.989829:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677447.989829:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677447.989831:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.989831:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.989831:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.989832:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:30.0:1619677447.989833:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677447.989833:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677447.989836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677447.989837:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.989837:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677447.989838:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677447.989840:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677447.989840:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.989841:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.989842:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677447.989844:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.989846:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.989847:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.989847:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.989849:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.989851:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.989854:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.989857:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.989858:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.989859:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.989861:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.989864:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677447.989912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1200] (20)++
00000800:00000200:1.0:1619677447.989918:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:1.0:1619677447.989920:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:17.0:1619677447.989921:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:1.0:1619677447.989923:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.989926:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.989927:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:1.0:1619677447.989929:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.989930:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.989931:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.989932:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:1.0:1619677447.989936:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.989938:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:1.0:1619677447.989940:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.989941:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:1.0:1619677447.989942:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:1.0:1619677447.989943:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:1.0:1619677447.989944:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:1.0:1619677447.989945:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:1.0:1619677447.989945:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:8.2:1619677447.989950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677447.989957:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677447.989960:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.989960:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.989962:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.989963:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677447.989965:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:8.2:1619677447.989972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677447.989980:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677447.989983:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677447.989984:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.989984:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677447.989985:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.989986:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:21.2:1619677447.990003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677447.990011:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:5.2:1619677447.990013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.0:1619677447.990014:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.990015:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.990016:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.990018:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.990019:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677447.990020:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:17.0:1619677447.990021:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:8.0:1619677447.990021:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.990022:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677447.990023:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.990024:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.990025:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.990026:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677447.990027:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.990029:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:8.0:1619677447.990030:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.990031:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.990033:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.990033:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:17.0:1619677447.990034:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.990035:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.0:1619677447.990037:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.990038:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:17.0:1619677447.990039:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.990039:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.990040:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677447.990041:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.0:1619677447.990041:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.990041:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.990042:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:17.0:1619677447.990044:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.990045:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677447.990046:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.2:1619677447.990046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677447.990048:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677447.990048:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.990049:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.990049:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677447.990054:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.990057:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.990058:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.990059:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.990060:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677447.990060:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.990062:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677447.990063:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.990064:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.990066:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.990067:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.990069:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677447.990078:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677447.990081:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677447.990082:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.990083:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.990083:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.990084:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.990088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.990094:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.990097:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.990097:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677447.990099:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.990102:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.990103:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.990104:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.990106:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.990107:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.990108:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.990111:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.990113:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.990116:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.990116:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.990117:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.990119:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.990119:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.990120:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.990120:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.990125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.990132:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.990135:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.990136:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.990136:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.990138:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.990141:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.990203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.990210:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.990212:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.990213:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.990216:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.990219:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.990220:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.990221:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720837] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.990223:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.990224:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.990225:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.990228:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.990230:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.990233:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.990234:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.990234:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.990236:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.990236:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.990237:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.990238:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.990242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.990250:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.990252:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.990253:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.990253:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.990255:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.990258:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.990484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.990492:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.990494:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.990497:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.990499:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.990500:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.990501:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677447.990503:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.990504:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.990505:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.990506:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677447.990509:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.990511:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.990514:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.990515:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677447.990515:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677447.990517:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.990518:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.990518:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.990519:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.990524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.990532:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.990535:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.990536:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.990538:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.990539:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.990540:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677447.990542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677447.990549:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.990552:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.990553:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.990553:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677447.990554:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.990555:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677447.990632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.990640:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.990642:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.990644:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.990646:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.990647:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.990649:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677447.990650:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.990652:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.990653:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.990654:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677447.990657:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.990659:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677447.990661:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.990662:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677447.990662:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677447.990664:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677447.990664:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.990665:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.990666:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677447.990677:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.990680:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.990681:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.990681:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677447.990683:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.990686:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677447.990783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.990790:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.990792:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.990793:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.990795:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.990797:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.990799:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677447.990799:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.990801:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.990802:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.990803:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677447.990805:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.990807:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.990808:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.990809:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677447.990809:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677447.990811:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677447.990811:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.990812:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.990812:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677447.990823:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.990825:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.990826:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.990827:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.990827:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.990829:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677447.990862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677447.990868:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.990871:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.990872:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677447.990874:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677447.990876:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.990878:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.990879:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.990880:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.990881:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.990882:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.990886:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.990888:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677447.990890:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.990891:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677447.990892:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.990893:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677447.990894:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677447.990894:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.990895:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677447.990900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.990908:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.990910:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.990911:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.990912:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.990913:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.990915:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677447.990950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677447.990956:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.990959:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.990959:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.990961:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.990964:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.990965:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677447.990966:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.990968:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.990968:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.990970:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.990973:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.990975:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677447.990977:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.990978:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677447.990978:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.990980:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677447.990980:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.990981:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677447.990981:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677447.990986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.990994:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.990996:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.990997:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.990998:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.991000:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.991002:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677447.991008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677447.991015:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.991017:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677447.991018:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.991018:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.991019:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.991020:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677447.991033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.991041:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.991044:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.991047:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.991048:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.991049:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.991051:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677447.991052:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.991053:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.991054:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.991055:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677447.991059:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.991061:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677447.991064:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.991065:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677447.991065:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677447.991067:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677447.991067:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.991069:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677447.991070:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677447.991071:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677447.991077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677447.991081:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.991083:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:30.0:1619677447.991084:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677447.991084:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.991085:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677447.991086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:30.0:1619677447.991087:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:6.0:1619677447.991087:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.991088:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.991088:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677447.991090:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:6.0:1619677447.991090:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677447.991092:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677447.991094:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677447.991095:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677447.991097:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.991099:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:20.0:1619677447.991099:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677447.991100:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720830] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.991101:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677447.991102:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000400:00000200:20.0:1619677447.991103:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.991103:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.991104:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000400:00000200:20.0:1619677447.991104:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:22.0:1619677447.991105:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.991105:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:20.0:1619677447.991107:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677447.991109:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677447.991112:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.991113:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677447.991114:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677447.991115:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677447.991116:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.991117:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.991117:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677447.991121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677447.991125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.0:1619677447.991128:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.991131:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677447.991131:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:10.0:1619677447.991132:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:8.0:1619677447.991132:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.991134:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:10.0:1619677447.991135:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.991136:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.991137:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.991137:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:10.0:1619677447.991139:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677447.991140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000400:00000200:10.0:1619677447.991141:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677447.991142:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.991143:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.991144:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.991145:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.991146:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677447.991148:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000400:00000200:10.0:1619677447.991148:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.991149:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677447.991150:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677447.991150:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:10.0:1619677447.991150:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677447.991151:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:10.0:1619677447.991152:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.991153:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677447.991154:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677447.991155:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.991156:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.991156:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.991157:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677447.991168:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.991171:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.991171:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.991171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:13.0:1619677447.991173:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.991175:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.991176:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.991177:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.991180:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.991181:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.991183:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.991185:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.991186:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.991188:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.991189:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.991190:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.991191:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.991194:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.991195:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.991198:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.991199:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.991199:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.991201:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.991202:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.991203:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677447.991204:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.991205:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.991207:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.991210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.991215:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677447.991217:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677447.991218:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677447.991219:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677447.991219:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.991219:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.991220:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.991221:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677447.991222:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.991224:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677447.991249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:6.0:1619677447.991255:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677447.991258:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677447.991258:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000400:00000200:6.0:1619677447.991260:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.991262:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.991264:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.991265:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.991266:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.991268:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.991269:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.991271:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.991273:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.991276:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.991277:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.991277:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.991279:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.991279:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677447.991280:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:6.0:1619677447.991281:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:6.0:1619677447.991282:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:8.2:1619677447.991285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677447.991287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (21)++
00000800:00000200:20.2:1619677447.991288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:22.0:1619677447.991291:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:5.0:1619677447.991294:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677447.991298:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (23)--
00000800:00000200:17.0:1619677447.991299:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677447.991300:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (22)--
00000800:00000200:20.0:1619677447.991300:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.991300:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:22.0:1619677447.991301:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:17.0:1619677447.991302:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:5.0:1619677447.991302:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.991303:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.991303:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:17.0:1619677447.991305:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:5.0:1619677447.991305:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.991305:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.991306:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (23)++
00000400:00000200:5.0:1619677447.991307:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:1.0:1619677447.991307:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (24)--
00000800:00000200:17.0:1619677447.991308:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:5.0:1619677447.991308:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.991308:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:5.0:1619677447.991309:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.991309:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:5.0:1619677447.991310:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.991312:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677447.991315:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.991317:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.991319:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.991320:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677447.991321:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.991323:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.991323:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.991325:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677447.991326:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.991327:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677447.991329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677447.991332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.991335:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.991337:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:17.0:1619677447.991337:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:20.0:1619677447.991338:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.991338:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677447.991340:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.991341:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.991341:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.991342:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.991344:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677447.991349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677447.991357:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.991360:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.991362:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.991365:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.991365:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.991367:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677447.991368:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.991369:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.991370:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.991371:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677447.991374:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.991376:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677447.991379:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.991379:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677447.991380:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.991382:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.991382:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.991383:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.991384:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:4.2:1619677447.991390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677447.991395:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:26.0:1619677447.991397:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677447.991398:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.991399:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.991400:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.991400:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:13.0:1619677447.991401:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.991402:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:26.0:1619677447.991403:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677447.991403:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:26.0:1619677447.991405:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.991406:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677447.991407:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.991409:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.991410:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.991411:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677447.991414:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.991415:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.991417:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.991417:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.991418:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.991419:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.991420:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.991420:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.991421:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677447.991440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677447.991441:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.991443:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.991443:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.991444:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.991445:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.991447:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677447.991448:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.991450:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.991452:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.991454:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.991455:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.991456:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677447.991457:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.991459:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.991459:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.991460:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677447.991463:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.991465:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677447.991468:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.991468:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677447.991469:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677447.991471:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677447.991471:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.991472:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.991473:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.991485:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.991487:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.991488:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.991488:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.991491:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.991494:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677447.991581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677447.991588:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.991591:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677447.991591:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677447.991594:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.991596:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.991597:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677447.991599:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.991600:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.991601:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.991602:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677447.991606:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.991607:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677447.991610:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.991611:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677447.991612:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677447.991613:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677447.991614:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677447.991615:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677447.991615:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.2:1619677447.991633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.991639:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.991641:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677447.991642:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.991642:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:26.0:1619677447.991644:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.991644:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677447.991646:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.991648:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.991648:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.991649:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.991651:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677447.991651:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:26.0:1619677447.991652:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000400:00000200:10.0:1619677447.991652:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.991653:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.991654:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.991654:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677447.991655:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.991655:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.991656:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:10.0:1619677447.991656:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677447.991660:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.991662:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677447.991665:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.991666:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677447.991666:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677447.991668:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.991669:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.991670:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677447.991671:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677447.991671:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677447.991675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677447.991677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677447.991682:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677447.991684:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:2.0:1619677447.991685:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677447.991686:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:18.0:1619677447.991686:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.991686:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.991687:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.991688:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:2.0:1619677447.991689:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.991692:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677447.991693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677447.991701:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677447.991703:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.991704:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.991704:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.991706:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.991706:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677447.991739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.991745:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.991747:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.991748:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.991750:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.991753:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.991754:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.991755:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.991757:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.991758:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.991759:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.991762:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.991764:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.991766:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.991767:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.991768:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.991769:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.991770:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.991770:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.991771:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.991783:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.991786:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.991787:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.991787:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.991789:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.991791:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.991792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.991799:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.991802:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.991802:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677447.991804:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.991807:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.991809:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677447.991810:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.991811:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.991812:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.991813:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677447.991817:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677447.991818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:10.0:1619677447.991819:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677447.991821:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.991822:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.991822:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677447.991824:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677447.991824:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677447.991824:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:10.0:1619677447.991825:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.991826:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677447.991827:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677447.991828:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677447.991829:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.991832:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.991833:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677447.991834:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.991836:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:2.0:1619677447.991836:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.991837:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.991837:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.2:1619677447.991838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:30.0:1619677447.991839:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.991840:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.991840:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000400:00000200:30.0:1619677447.991841:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.991841:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677447.991842:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677447.991843:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.991843:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:30.0:1619677447.991844:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:2.0:1619677447.991844:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.991845:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:20.0:1619677447.991846:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.991846:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677447.991846:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677447.991848:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.991848:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677447.991848:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:20.0:1619677447.991850:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:2.0:1619677447.991850:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:20.0:1619677447.991851:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.991851:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677447.991851:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:20.0:1619677447.991853:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.991854:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677447.991854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677447.991855:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:21.2:1619677447.991857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:20.0:1619677447.991859:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.991860:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677447.991862:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677447.991863:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.991864:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:18.0:1619677447.991864:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.991864:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.991864:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677447.991864:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:20.0:1619677447.991865:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:2.0:1619677447.991865:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.991866:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677447.991866:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.991867:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.991867:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677447.991867:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.0:1619677447.991868:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677447.991868:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677447.991869:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677447.991880:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.991883:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.991884:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.991885:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.991887:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.991888:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.992001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.992008:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.992011:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.992013:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.992014:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.992015:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.992017:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677447.992018:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.992019:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.992020:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.992021:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:4.2:1619677447.992024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:30.0:1619677447.992025:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.992026:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677447.992029:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.992030:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677447.992031:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677447.992031:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677447.992032:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677447.992033:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.992034:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.992034:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677447.992034:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.992035:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.992036:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677447.992039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:8.0:1619677447.992039:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.992041:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.992042:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96720815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.992043:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.992044:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.992045:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677447.992046:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:8.0:1619677447.992048:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.992049:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:8.0:1619677447.992050:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.2:1619677447.992050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:2.0:1619677447.992050:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.992051:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.992051:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:2.0:1619677447.992052:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.992053:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:8.0:1619677447.992053:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.992054:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.992055:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:18.0:1619677447.992056:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.992056:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.992057:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.992057:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677447.992057:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:8.0:1619677447.992058:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677447.992058:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.992059:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.992061:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.992062:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677447.992062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677447.992063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:18.0:1619677447.992063:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.992064:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.992065:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:6.0:1619677447.992068:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:18.0:1619677447.992069:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.992070:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677447.992070:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677447.992071:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.992072:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677447.992073:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.992073:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677447.992074:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:17.0:1619677447.992074:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.992074:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.992074:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.992075:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.992076:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:6.0:1619677447.992076:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.992077:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:17.0:1619677447.992077:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.992078:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.992078:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:6.0:1619677447.992078:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677447.992079:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677447.992080:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:6.0:1619677447.992081:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.992082:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677447.992083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:6.0:1619677447.992083:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:4.2:1619677447.992085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:6.0:1619677447.992087:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.992088:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.2:1619677447.992089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:6.0:1619677447.992092:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.992092:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677447.992093:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.992094:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.992095:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677447.992095:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677447.992095:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:26.0:1619677447.992096:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:6.0:1619677447.992096:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.992097:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677447.992098:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:26.0:1619677447.992099:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.992099:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:5.0:1619677447.992099:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.992101:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677447.992101:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:5.0:1619677447.992101:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:26.0:1619677447.992102:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.992104:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677447.992104:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.992104:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:26.0:1619677447.992106:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.992107:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677447.992107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:26.0:1619677447.992108:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.992109:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677447.992111:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.992112:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:26.0:1619677447.992113:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677447.992113:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:31.0:1619677447.992114:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677447.992114:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.992114:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.992115:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677447.992115:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:31.0:1619677447.992116:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.992116:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:10.0:1619677447.992117:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.992118:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.992119:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.992119:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677447.992120:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.992121:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:30.0:1619677447.992121:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.992122:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.992123:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:26.0:1619677447.992123:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677447.992124:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.992124:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677447.992125:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677447.992125:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.992126:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.992127:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.992127:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677447.992127:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:30.0:1619677447.992128:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:26.0:1619677447.992128:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000400:00000200:30.0:1619677447.992130:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.992130:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (23)++
00000800:00000200:26.0:1619677447.992131:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (24)--
00000800:00000200:10.0:1619677447.992131:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.992132:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677447.992132:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.992132:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677447.992133:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.992133:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:9.2:1619677447.992133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677447.992134:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.992134:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677447.992135:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677447.992135:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677447.992135:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677447.992136:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677447.992136:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.992136:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677447.992137:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.992138:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:4.2:1619677447.992138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677447.992139:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:2.0:1619677447.992140:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:30.0:1619677447.992141:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.992141:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:2.0:1619677447.992143:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.992144:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.992144:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.992145:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:2.0:1619677447.992146:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.992147:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:20.2:1619677447.992147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677447.992148:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677447.992148:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677447.992149:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677447.992150:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.992151:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:12.0:1619677447.992151:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:6.0:1619677447.992151:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.992152:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.992154:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.992154:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.992154:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.992155:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:12.0:1619677447.992156:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.992157:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:22.0:1619677447.992157:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677447.992158:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:13.0:1619677447.992158:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.992158:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677447.992159:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:21.2:1619677447.992183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677447.992190:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677447.992193:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677447.992194:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677447.992196:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.992198:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.992200:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:26.0:1619677447.992201:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.992202:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.992203:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.992204:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:26.0:1619677447.992207:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.992209:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:26.0:1619677447.992212:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.992213:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:26.0:1619677447.992213:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:26.0:1619677447.992215:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:26.0:1619677447.992215:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677447.992216:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677447.992217:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677447.992221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.992229:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.992231:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677447.992232:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.992233:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.992235:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.992238:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677447.992413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677447.992420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677447.992421:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677447.992423:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677447.992423:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.992424:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677447.992426:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:10.0:1619677447.992426:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677447.992428:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.992429:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.992429:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.992431:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.992431:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.992433:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677447.992433:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.992434:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.992434:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.992435:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.992436:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.992437:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.992437:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.992438:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677447.992441:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677447.992442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:10.0:1619677447.992443:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.992445:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.992445:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.992446:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677447.992446:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.992447:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677447.992448:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:24.0:1619677447.992449:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:10.0:1619677447.992449:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.992449:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677447.992450:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.992450:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677447.992451:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:24.0:1619677447.992452:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:20.0:1619677447.992452:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677447.992453:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:8.0:1619677447.992453:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677447.992454:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:24.0:1619677447.992455:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.992455:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.992455:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:24.0:1619677447.992458:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.992459:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677447.992461:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96720806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.992462:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)++
00000400:00000200:24.0:1619677447.992462:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.992463:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.992464:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (24)--
00000400:00000200:24.0:1619677447.992464:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.992465:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.992468:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:24.0:1619677447.992468:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.992469:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:30.0:1619677447.992470:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:24.0:1619677447.992470:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677447.992471:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.992471:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.992472:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.992473:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677447.992473:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:24.0:1619677447.992473:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.992474:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:24.0:1619677447.992474:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677447.992475:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:30.0:1619677447.992475:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:24.0:1619677447.992475:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677447.992476:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:24.0:1619677447.992476:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.992477:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:24.0:1619677447.992477:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.992478:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.992478:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677447.992489:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.992492:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.992493:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.992494:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.992497:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.992498:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677447.992506:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677447.992508:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.992509:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.992510:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.992510:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.992512:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677447.992551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.992558:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.992561:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.992563:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.992565:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.992565:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.992566:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.992567:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720805] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.992569:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.992570:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.992571:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.992573:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.992575:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.992577:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.992578:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.992578:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.992580:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.992581:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.992582:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.992582:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.992592:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.992594:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.992595:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.992596:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.992598:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.992599:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677447.992613:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.992615:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.992616:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.992617:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677447.992617:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.992618:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.992635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.992642:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.992644:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.992645:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.992647:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.992649:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.992651:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.992652:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.992653:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.992654:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.992655:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.2:1619677447.992659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:1.0:1619677447.992659:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.992661:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.992664:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.992665:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.992665:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677447.992666:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:1.0:1619677447.992667:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.992668:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677447.992669:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:1.0:1619677447.992669:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.992670:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.992670:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677447.992671:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.992673:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.992674:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.992675:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:5.2:1619677447.992675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677447.992676:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.992678:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.992679:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.992680:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:13.0:1619677447.992680:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677447.992681:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677447.992682:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:30.0:1619677447.992683:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.992683:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:13.0:1619677447.992683:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.992684:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677447.992685:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.2:1619677447.992685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:17.0:1619677447.992685:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677447.992685:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.992686:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677447.992688:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.992688:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677447.992689:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.992689:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677447.992691:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.992692:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677447.992692:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:30.0:1619677447.992693:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.992693:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677447.992695:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677447.992696:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677447.992697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:26.0:1619677447.992698:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677447.992701:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.992703:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677447.992704:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.992705:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.992706:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.992706:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:26.0:1619677447.992707:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677447.992708:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.992709:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.992711:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.992711:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:26.0:1619677447.992712:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677447.992714:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.2:1619677447.992715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677447.992717:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.992718:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677447.992718:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677447.992719:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.992719:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677447.992720:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677447.992721:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:5.0:1619677447.992721:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:26.0:1619677447.992722:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677447.992722:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:20.0:1619677447.992723:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.992723:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.992723:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677447.992724:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677447.992726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677447.992734:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677447.992736:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677447.992737:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.992739:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677447.992740:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:4.2:1619677447.992741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677447.992744:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:10.0:1619677447.992749:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.992751:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.992752:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.992754:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.992756:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.992758:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677447.992759:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.992761:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.992761:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.992762:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677447.992765:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.992767:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677447.992769:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.992770:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677447.992771:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677447.992772:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677447.992773:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.992774:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.992774:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677447.992778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.992786:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.992788:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.992789:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.992791:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.2:1619677447.992792:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.992794:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.992795:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677447.992796:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.992799:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.992801:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.992801:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.992803:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.992805:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677447.992806:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.992807:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.992808:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.992809:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677447.992813:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.992815:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.992817:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.992818:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677447.992819:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677447.992820:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677447.992821:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.992822:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677447.992822:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677447.992826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.992834:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.992836:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.992837:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.992837:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.992839:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.992842:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.992879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.992887:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.992889:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.992890:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.992892:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.992895:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.992897:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677447.992898:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.992899:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.992900:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.992901:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677447.992904:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.992906:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677447.992909:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.992910:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677447.992910:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.2:1619677447.992911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:26.0:1619677447.992912:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677447.992913:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.992913:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.992914:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.0:1619677447.992914:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.992917:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.992919:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.992920:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.992921:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.992923:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677447.992925:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.992926:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.992926:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:20.0:1619677447.992927:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.992928:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677447.992928:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.992929:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.992930:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.992932:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677447.992932:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.992935:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677447.992935:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677447.992938:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.992939:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677447.992939:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.992941:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.992942:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.992942:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.992943:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.992945:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.992948:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677447.992948:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.992949:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.992950:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.992950:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677447.992953:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.992956:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.992957:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.992959:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.992960:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:4.2:1619677447.992960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:12.0:1619677447.992962:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677447.992967:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.992970:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.992971:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.992973:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.992975:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.992976:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.992978:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.992979:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.992980:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.992981:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.992984:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.992986:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.992989:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.992989:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.992990:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.992992:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.992992:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.992993:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.992993:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.992998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.993005:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.993008:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.993009:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.993009:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.993011:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.993014:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.993088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.993093:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.993095:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.993096:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.993098:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.993101:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.993102:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.993103:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.993104:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.993105:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.993106:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677447.993109:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.993111:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677447.993113:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.993114:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.993115:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.993116:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.993117:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.993117:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.993118:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677447.993130:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.993133:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677447.993133:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677447.993134:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.993136:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.993138:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.993182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.993189:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.993192:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.993193:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.993195:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.993198:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.993199:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.993201:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.993202:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.993203:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.993204:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677447.993208:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.993209:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677447.993213:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.993213:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.993214:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.993216:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.993216:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.993217:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.993217:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677447.993229:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.993232:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.993233:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.993233:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.993235:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.993239:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.993367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677447.993375:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677447.993378:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677447.993381:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677447.993383:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.993383:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677447.993385:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:13.0:1619677447.993386:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677447.993388:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677447.993389:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677447.993390:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:13.0:1619677447.993393:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.993395:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:13.0:1619677447.993399:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.993400:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:13.0:1619677447.993400:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:13.0:1619677447.993403:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:13.0:1619677447.993404:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677447.993404:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677447.993405:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677447.993410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.993418:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.2:1619677447.993421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677447.993421:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.993421:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.993423:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.993426:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.993428:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677447.993430:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.993433:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.993435:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.993437:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.993438:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.993440:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677447.993441:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.993442:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.993443:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.993444:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677447.993447:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.993449:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677447.993452:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.993453:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677447.993454:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677447.993456:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677447.993456:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.993457:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.993458:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.993461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677447.993465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677447.993469:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.993471:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.993472:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.993473:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677447.993474:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.993475:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.993476:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677447.993477:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677447.993479:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.993479:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.993481:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.993483:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677447.993484:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.993485:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.993487:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.993488:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677447.993491:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.993493:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677447.993496:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.993496:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677447.993497:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677447.993499:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677447.993499:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.993501:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677447.993502:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.993503:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677447.993506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677447.993508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677447.993514:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:31.0:1619677447.993516:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:2.0:1619677447.993517:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:31.0:1619677447.993518:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:18.0:1619677447.993518:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677447.993518:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.993519:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.993520:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:2.0:1619677447.993520:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.993523:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677447.993572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.2:1619677447.993576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:17.0:1619677447.993579:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.993581:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.993582:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677447.993583:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:17.0:1619677447.993585:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.993586:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:17.0:1619677447.993587:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.993587:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:17.0:1619677447.993589:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677447.993590:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.993590:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.993591:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.993593:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.993593:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.993594:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677447.993594:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677447.993596:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96720790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.993597:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.993598:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.993599:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.993600:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677447.993600:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677447.993603:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677447.993604:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.993604:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677447.993605:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677447.993607:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677447.993607:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.993607:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.993608:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.993609:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677447.993610:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.993610:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677447.993611:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677447.993613:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677447.993613:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.993614:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.993616:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.993616:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.2:1619677447.993619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:12.0:1619677447.993620:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:21.2:1619677447.993622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:12.0:1619677447.993622:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.993623:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.993625:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.993626:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.993628:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.993628:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.0:1619677447.993629:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677447.993631:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:5.0:1619677447.993631:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.993631:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.993632:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.993633:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:5.0:1619677447.993633:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.993634:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.993636:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677447.993669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677447.993677:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.993680:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677447.993680:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.993683:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.993685:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.993687:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677447.993688:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.993689:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.993691:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.993691:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677447.993695:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.993696:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677447.993699:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.993699:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677447.993700:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677447.993702:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677447.993702:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.993703:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.993703:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677447.993714:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.993716:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.993717:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.993719:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.993720:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.993721:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677447.993911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.993917:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.993919:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.993920:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.993922:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677447.993924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:1.0:1619677447.993925:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.993926:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.993928:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.993929:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677447.993930:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:1.0:1619677447.993930:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.993931:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:24.0:1619677447.993933:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.993934:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677447.993935:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677447.993936:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.993937:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:24.0:1619677447.993939:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.993939:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.993940:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:24.0:1619677447.993941:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:1.0:1619677447.993941:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:24.0:1619677447.993942:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96720787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.993942:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:24.0:1619677447.993943:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.993943:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.993943:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677447.993944:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.993944:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677447.993945:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.993948:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.993950:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.993953:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.993953:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.993954:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.993956:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.993956:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677447.993956:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:24.0:1619677447.993957:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.993957:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677447.993959:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.993960:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.993960:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.993961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:6.0:1619677447.993962:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.993965:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:17.0:1619677447.993970:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.993972:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.993973:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.993974:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.993976:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677447.993978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.993981:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.993983:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.993984:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.993986:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.993986:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.993986:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.993987:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.994003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.994010:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.994013:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.994013:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677447.994015:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.994018:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.994019:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.994020:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.994022:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.994023:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.994024:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677447.994027:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.994028:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677447.994031:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.994032:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.994032:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.994034:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.994034:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.994036:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677447.994037:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.994038:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677447.994043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677447.994048:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677447.994050:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677447.994051:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.994051:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677447.994052:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677447.994052:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.994053:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.994054:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677447.994054:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.994056:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.994061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.994064:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.994067:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.994069:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.994070:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.994072:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.994073:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677447.994074:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.994077:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.994078:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.994079:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677447.994082:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.994084:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677447.994086:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.994087:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677447.994088:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677447.994089:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677447.994090:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.994091:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.994092:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.994093:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677447.994098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677447.994103:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677447.994104:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:6.0:1619677447.994105:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677447.994106:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:22.0:1619677447.994107:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677447.994107:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.994107:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.994108:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:6.0:1619677447.994109:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.994111:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.994121:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.994123:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.994124:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.994124:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.994125:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.994126:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677447.994151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.994156:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.994159:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.994160:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.994162:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.994164:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.994165:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.994167:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.994168:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.994169:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.994170:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.994173:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.994175:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.994177:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.994177:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.994178:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.994179:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.994180:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.994181:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.994182:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.994182:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677447.994185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677447.994188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.994194:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.994195:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.994195:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.994195:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.994195:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.994196:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.994197:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.994197:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.994198:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677447.994199:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677447.994239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.994245:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.994247:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.994248:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.994250:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.994253:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.994254:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.994256:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720783] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.994257:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.994258:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.994259:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.994262:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.994263:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677447.994266:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.994267:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.994268:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677447.994270:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.994270:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.994271:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677447.994272:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677447.994273:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:21.2:1619677447.994278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677447.994282:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:22.0:1619677447.994283:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:26.0:1619677447.994285:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:22.0:1619677447.994285:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:10.0:1619677447.994285:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:22.0:1619677447.994286:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677447.994286:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.994286:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677447.994288:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.994292:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677447.994309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677447.994316:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.994319:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677447.994320:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677447.994322:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.994325:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.994326:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.994327:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.994329:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.994330:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.994331:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.994334:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.994335:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.994338:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.994339:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.994340:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.994341:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.994342:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.994342:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.994343:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.2:1619677447.994346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:8.2:1619677447.994347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.994353:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677447.994355:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:18.0:1619677447.994356:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.994356:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677447.994358:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:18.0:1619677447.994359:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.994359:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.994360:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.994361:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:18.0:1619677447.994361:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.994362:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677447.994363:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:18.0:1619677447.994364:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.994365:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677447.994365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000400:00000200:18.0:1619677447.994366:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.994367:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677447.994371:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.994372:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.994372:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677447.994375:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677447.994376:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:18.0:1619677447.994376:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.994377:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:18.0:1619677447.994377:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677447.994377:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677447.994378:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.994378:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677447.994379:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677447.994380:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.994381:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677447.994382:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677447.994383:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677447.994388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.994393:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:12.0:1619677447.994394:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677447.994395:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:12.0:1619677447.994396:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677447.994396:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:12.0:1619677447.994397:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.994397:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:12.0:1619677447.994399:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.994401:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.994402:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677447.994430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.994437:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.994440:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.994440:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677447.994442:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.994444:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.994446:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.994447:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.994448:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.994449:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.994450:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.994454:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.994456:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.994459:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.994460:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.994460:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.994462:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.994463:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.994463:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.994463:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.994475:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.994477:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.994478:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.994480:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.994481:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.994483:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677447.994604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919078f99600] (20)++
00000800:00000200:20.2:1619677447.994610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677447.994611:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919078f99600] (21)++
00000800:00000200:18.0:1619677447.994614:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.61@o2ib3
00000800:00000200:2.0:1619677447.994614:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078f99600] (22)--
00000400:00000200:18.0:1619677447.994617:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.61@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.994618:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:21.2:1619677447.994620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:18.0:1619677447.994620:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.994621:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:18.0:1619677447.994622:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:18.0:1619677447.994624:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.994625:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.994625:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677447.994626:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:18.0:1619677447.994626:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.994627:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:31.0:1619677447.994628:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.994628:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.994629:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.994630:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.994632:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.994632:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.61@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677447.994633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198623bcc00] (20)++
00000400:00000200:31.0:1619677447.994634:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677447.994634:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000400:00000200:31.0:1619677447.994635:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.994636:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:18.0:1619677447.994638:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f90d380] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677447.994638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:18.0:1619677447.994639:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e49d37c00] (20)++
00000400:00000200:31.0:1619677447.994640:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.994640:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (21)++
00000800:00000200:18.0:1619677447.994640:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e49d37c00] (21)++
00000800:00000200:18.0:1619677447.994642:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e49d37c00] (22)--
00000800:00000200:22.0:1619677447.994643:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.1.10@o2ib3
00000800:00000200:18.0:1619677447.994643:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919078f99600] (21)++
00000800:00000200:18.0:1619677447.994643:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919078f99600] (22)--
00000800:00000200:31.0:1619677447.994644:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:30.0:1619677447.994644:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.994644:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078f99600] (21)--
00000800:00000200:6.0:1619677447.994644:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)--
00000400:00000200:30.0:1619677447.994646:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.994646:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677447.994647:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.994647:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.994648:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:30.0:1619677447.994648:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:24.2:1619677447.994648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (21)++
00000400:00000200:30.0:1619677447.994649:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720777] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.994649:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:22.0:1619677447.994649:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.994649:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677447.994650:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677447.994651:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.994652:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.994652:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)++
00000400:00000200:26.0:1619677447.994653:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.994653:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.994654:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:30.0:1619677447.994654:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:22.0:1619677447.994654:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:31.0:1619677447.994655:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:24.0:1619677447.994655:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e49d37c00] (23)--
00000800:00000200:8.0:1619677447.994655:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (23)--
00000400:00000200:26.0:1619677447.994656:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.994656:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.994656:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.994657:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:26.0:1619677447.994657:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677447.994658:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.994658:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.994659:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:24.0:1619677447.994659:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.994659:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.994660:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:22.0:1619677447.994660:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:31.0:1619677447.994661:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.994661:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.994661:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.994662:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.994662:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)--
00000400:00000200:26.0:1619677447.994663:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677447.994664:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.994666:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.994667:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:22.0:1619677447.994667:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.994668:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677447.994669:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.994670:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:30.0:1619677447.994672:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677447.994672:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:22.0:1619677447.994673:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.994674:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:22.0:1619677447.994674:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:30.0:1619677447.994675:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.994675:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.994676:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677447.994676:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:22.0:1619677447.994676:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:30.0:1619677447.994677:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677447.994677:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677447.994677:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (21)++
00000800:00000200:22.0:1619677447.994678:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (22)--
00000800:00000200:30.0:1619677447.994679:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677447.994679:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677447.994679:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (21)--
00000800:00000200:30.0:1619677447.994680:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677447.994680:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.2:1619677447.994680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677447.994681:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677447.994681:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:30.0:1619677447.994682:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677447.994682:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:8.2:1619677447.994682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:26.0:1619677447.994683:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677447.994686:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677447.994688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677447.994689:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.994689:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677447.994690:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:17.0:1619677447.994691:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.994691:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.0:1619677447.994692:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677447.994693:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.994694:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.994694:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.994694:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.994695:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.994695:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677447.994696:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677447.994696:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.0:1619677447.994696:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.994697:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:17.0:1619677447.994697:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677447.994698:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.994698:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.994699:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677447.994699:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:6.0:1619677447.994699:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.994700:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:17.0:1619677447.994700:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677447.994700:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.994701:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:17.0:1619677447.994701:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.994701:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:6.0:1619677447.994701:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.994703:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:17.0:1619677447.994706:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.994707:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:17.0:1619677447.994708:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.994709:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.994711:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677447.994712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:17.0:1619677447.994712:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677447.994712:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.994713:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.994713:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.994714:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:17.0:1619677447.994715:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677447.994715:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677447.994715:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.994716:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677447.994716:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677447.994717:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677447.994719:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:21.2:1619677447.994719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198623bcc00] (20)++
00000800:00000200:31.0:1619677447.994722:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677447.994724:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.994724:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (21)++
00000800:00000200:30.0:1619677447.994726:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677447.994727:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.994727:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.10@o2ib3
00000800:00000200:12.0:1619677447.994727:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677447.994728:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)--
00000400:00000200:31.0:1619677447.994729:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:12.0:1619677447.994729:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:22.0:1619677447.994730:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677447.994730:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.994731:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720773] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.994732:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.994732:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.994732:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.994733:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.994733:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:31.0:1619677447.994734:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.994734:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:22.0:1619677447.994735:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.994735:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:22.0:1619677447.994737:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.994739:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.994740:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.994740:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677447.994743:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:22.0:1619677447.994745:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677447.994745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198623bcc00] (21)++
00000800:00000200:13.0:1619677447.994745:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.994746:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.994747:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677447.994748:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.994748:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677447.994748:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.994749:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.994749:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677447.994750:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677447.994750:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:13.0:1619677447.994750:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.994751:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677447.994751:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:22.0:1619677447.994751:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677447.994751:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677447.994751:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)++
00000800:00000200:22.0:1619677447.994752:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:22.0:1619677447.994753:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677447.994753:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.10@o2ib3
00000800:00000200:22.0:1619677447.994755:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:6.0:1619677447.994755:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.994756:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (23)++
00000800:00000200:9.2:1619677447.994756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677447.994757:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (24)--
00000400:00000200:6.0:1619677447.994757:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.994758:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (23)--
00000800:00000200:22.0:1619677447.994759:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)--
00000400:00000200:6.0:1619677447.994759:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:6.0:1619677447.994760:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677447.994761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677447.994762:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.994763:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.994764:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677447.994767:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.994767:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:6.0:1619677447.994769:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677447.994769:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.994770:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:2.0:1619677447.994770:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.994771:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.994771:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677447.994771:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677447.994772:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.994772:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.994772:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.994773:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:18.0:1619677447.994774:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.994774:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (21)++
00000800:00000200:6.0:1619677447.994774:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (22)--
00000800:00000200:6.0:1619677447.994775:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (21)--
00000800:00000200:2.0:1619677447.994776:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677447.994777:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.994778:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677447.994778:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.994779:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.994779:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677447.994780:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677447.994781:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677447.994782:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677447.994783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198623bcc00] (20)++
00000800:00000200:22.0:1619677447.994788:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.994789:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (21)++
00000800:00000200:22.0:1619677447.994790:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.994790:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.994791:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.10@o2ib3
00000400:00000200:22.0:1619677447.994793:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.994793:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.994794:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.2:1619677447.994794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:22.0:1619677447.994795:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:6.0:1619677447.994795:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.994796:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)--
00000400:00000200:6.0:1619677447.994797:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:6.0:1619677447.994798:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677447.994800:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:6.0:1619677447.994800:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.994801:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.994801:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:18.0:1619677447.994803:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.994803:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:6.0:1619677447.994806:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677447.994807:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.994809:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:18.0:1619677447.994810:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.994811:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:6.0:1619677447.994811:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.994812:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:6.0:1619677447.994812:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:18.0:1619677447.994813:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.994814:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677447.994814:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:18.0:1619677447.994815:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.994815:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (21)++
00000800:00000200:21.2:1619677447.994816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198623bcc00] (22)++
00000400:00000200:18.0:1619677447.994816:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.994816:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198623bcc00] (22)++
00000800:00000200:6.0:1619677447.994817:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (24)--
00000800:00000200:6.0:1619677447.994819:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (23)--
00000800:00000200:6.0:1619677447.994819:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)++
00000400:00000200:18.0:1619677447.994820:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677447.994820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677447.994821:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.994822:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (23)++
00000800:00000200:22.0:1619677447.994822:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198623bcc00] (24)--
00000800:00000200:6.0:1619677447.994822:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.10@o2ib3
00000800:00000200:22.0:1619677447.994823:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (23)--
00000800:00000200:22.0:1619677447.994824:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)--
00000800:00000200:18.0:1619677447.994824:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.994825:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:6.0:1619677447.994825:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.994826:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.994827:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:6.0:1619677447.994827:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.994828:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.994828:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:6.0:1619677447.994828:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:5.0:1619677447.994828:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:18.0:1619677447.994829:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:6.0:1619677447.994830:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677447.994830:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:6.0:1619677447.994831:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.994831:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.994832:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.994833:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.2:1619677447.994833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677447.994833:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:6.0:1619677447.994834:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677447.994836:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:6.0:1619677447.994837:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.994839:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.994841:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677447.994842:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.994843:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)++
00000800:00000200:17.0:1619677447.994844:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (24)--
00000800:00000200:6.0:1619677447.994844:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (24)++
00000800:00000200:4.2:1619677447.994844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:1.0:1619677447.994844:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000400:00000200:1.0:1619677447.994845:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.994846:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:6.0:1619677447.994847:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (21)++
00000800:00000200:6.0:1619677447.994847:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (22)--
00000400:00000200:1.0:1619677447.994847:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.994848:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (21)--
00000800:00000200:1.0:1619677447.994850:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.994851:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.2:1619677447.994854:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.994855:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.994859:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677447.994861:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.994862:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:21.2:1619677447.994863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198623bcc00] (20)++
00000400:00000200:8.0:1619677447.994863:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.994864:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.994865:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.994865:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:1.0:1619677447.994865:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:8.0:1619677447.994866:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96720767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.994867:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.994868:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (21)++
00000400:00000200:8.0:1619677447.994868:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.994869:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.994870:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:8.0:1619677447.994870:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677447.994871:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.10@o2ib3
00000800:00000200:6.0:1619677447.994872:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)--
00000400:00000200:22.0:1619677447.994875:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.994876:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.994878:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.994878:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.994879:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:22.0:1619677447.994881:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.994882:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677447.994882:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.994882:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:22.0:1619677447.994883:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677447.994883:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:22.0:1619677447.994884:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677447.994885:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:22.0:1619677447.994888:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.994890:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.2:1619677447.994890:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.994893:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.994893:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:8.0:1619677447.994893:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677447.994894:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677447.994895:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677447.994896:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.994896:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:22.0:1619677447.994897:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (21)++
00000800:00000200:22.0:1619677447.994897:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (22)--
00000800:00000200:8.0:1619677447.994897:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677447.994898:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (21)--
00000800:00000200:1.0:1619677447.994899:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:9.2:1619677447.994902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677447.994902:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.994903:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.994903:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.994904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:1.0:1619677447.994905:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.994907:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.994909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.994911:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677447.994913:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677447.994914:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.994916:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.994917:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677447.994918:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.994918:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.994919:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677447.994919:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.994920:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677447.994920:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.994920:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:17.0:1619677447.994921:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.994921:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.994922:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677447.994923:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677447.994924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198623bcc00] (20)++
00000800:00000200:21.2:1619677447.994929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:6.0:1619677447.994929:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (21)++
00000800:00000200:6.0:1619677447.994932:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.10@o2ib3
00000800:00000200:22.0:1619677447.994933:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)--
00000400:00000200:6.0:1619677447.994935:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677447.994937:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:6.0:1619677447.994937:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.994940:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.994940:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:6.0:1619677447.994940:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:6.0:1619677447.994941:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.994943:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.994943:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.994944:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.994945:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677447.994946:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.994948:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:6.0:1619677447.994949:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677447.994950:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96720764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.994951:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.994952:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677447.994952:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:8.0:1619677447.994953:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677447.994955:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.994956:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:8.0:1619677447.994957:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.994957:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677447.994958:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.994959:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677447.994959:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (21)++
00000800:00000200:6.0:1619677447.994959:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (22)--
00000800:00000200:6.0:1619677447.994960:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (21)--
00000800:00000200:21.2:1619677447.994962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198623bcc00] (20)++
00000800:00000200:8.0:1619677447.994962:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.994963:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677447.994964:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677447.994966:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677447.994966:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:22.0:1619677447.994967:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (21)++
00000800:00000200:8.0:1619677447.994967:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.994968:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677447.994969:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.10@o2ib3
00000800:00000200:6.0:1619677447.994970:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)--
00000800:00000200:12.0:1619677447.994971:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:22.0:1619677447.994972:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677447.994972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677447.994973:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:22.0:1619677447.994974:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.994974:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.994976:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:13.0:1619677447.994977:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677447.994977:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.994978:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720763] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.994979:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677447.994979:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:22.0:1619677447.994980:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.994982:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:2.0:1619677447.994982:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.994983:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.994983:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677447.994984:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:2.0:1619677447.994985:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:22.0:1619677447.994986:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.994989:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677447.994989:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677447.994991:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.994992:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:9.2:1619677447.994992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:22.0:1619677447.994993:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.994995:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.994996:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (21)++
00000800:00000200:22.0:1619677447.994997:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198623bcc00] (22)++
00000800:00000200:22.0:1619677447.994998:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (23)--
00000800:00000200:22.0:1619677447.994999:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)--
00000800:00000200:18.0:1619677447.994999:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677447.995002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:18.0:1619677447.995002:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677447.995003:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.995003:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:21.2:1619677447.995004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198623bcc00] (21)++
00000800:00000200:18.0:1619677447.995004:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677447.995005:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.995007:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677447.995008:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677447.995009:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)++
00000800:00000200:22.0:1619677447.995010:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198623bcc00] (23)--
00000800:00000200:6.0:1619677447.995010:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:30.0:1619677447.995011:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:22.0:1619677447.995011:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)--
00000800:00000200:22.0:1619677447.995012:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.995012:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.995013:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.995013:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (21)++
00000800:00000200:31.0:1619677447.995015:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677447.995015:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.995015:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.10@o2ib3
00000400:00000200:6.0:1619677447.995015:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.995017:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.995017:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.995019:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.995020:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.995021:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.995022:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677447.995025:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:6.0:1619677447.995025:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677447.995027:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.995027:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:22.0:1619677447.995029:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:22.0:1619677447.995030:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.995031:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.995032:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.995033:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:6.0:1619677447.995035:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677447.995036:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.995036:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.995037:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.995038:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:6.0:1619677447.995039:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.995040:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.995041:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.995041:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.995041:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677447.995042:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:22.0:1619677447.995042:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:6.0:1619677447.995043:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)++
00000800:00000200:22.0:1619677447.995044:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:6.0:1619677447.995044:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.10@o2ib3
00000800:00000200:22.0:1619677447.995045:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (23)++
00000800:00000200:22.0:1619677447.995045:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (24)--
00000800:00000200:8.2:1619677447.995045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.995046:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (23)--
00000400:00000200:6.0:1619677447.995046:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677447.995047:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)--
00000400:00000200:6.0:1619677447.995048:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.995049:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:8.2:1619677447.995050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:6.0:1619677447.995050:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720760] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.995052:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.995052:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.995052:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:6.0:1619677447.995053:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:1.0:1619677447.995055:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.995056:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:6.0:1619677447.995056:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.995056:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.995057:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:1.0:1619677447.995058:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.995059:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677447.995060:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.995060:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.995061:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677447.995062:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.995062:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677447.995062:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:1.0:1619677447.995062:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677447.995063:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:6.0:1619677447.995064:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677447.995065:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:6.0:1619677447.995065:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (21)++
00000800:00000200:6.0:1619677447.995066:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (22)--
00000800:00000200:6.0:1619677447.995066:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (21)--
00000800:00000200:9.2:1619677447.995070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:4.2:1619677447.995074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677447.995077:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677447.995079:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677447.995081:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:10.0:1619677447.995081:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677447.995081:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.995083:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.995084:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.995085:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677447.995085:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:10.0:1619677447.995087:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677447.995089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677447.995090:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.995091:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677447.995093:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.995094:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677447.995095:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677447.995095:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.995096:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.995098:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.995098:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.995101:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.995101:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.995103:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.995104:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.995106:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677447.995107:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677447.995107:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.995107:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:17.0:1619677447.995108:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677447.995108:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:17.0:1619677447.995110:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.995110:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:10.0:1619677447.995110:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677447.995110:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.995111:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.995112:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:17.0:1619677447.995113:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.995115:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.995117:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.995118:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677447.995118:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.995120:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.995121:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.995121:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.995122:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.995123:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.995126:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:8.2:1619677447.995126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:22.0:1619677447.995127:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.995127:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.995129:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.995132:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677447.995134:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.995136:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677447.995137:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677447.995137:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677447.995139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198623bcc00] (20)++
00000400:00000200:1.0:1619677447.995139:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.995141:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:6.0:1619677447.995143:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.995146:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (21)++
00000800:00000200:6.0:1619677447.995146:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.995147:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.10@o2ib3
00000800:00000200:6.0:1619677447.995147:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.995148:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.995149:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.995150:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.995150:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677447.995151:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)--
00000400:00000200:22.0:1619677447.995152:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677447.995152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000400:00000200:22.0:1619677447.995154:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677447.995155:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.995156:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.995157:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.995158:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.995159:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:22.0:1619677447.995161:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.995162:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677447.995163:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677447.995163:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:22.0:1619677447.995164:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.995164:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677447.995164:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:22.0:1619677447.995166:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.995167:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677447.995167:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677447.995169:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677447.995169:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (21)++
00000800:00000200:22.0:1619677447.995170:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (22)--
00000800:00000200:22.0:1619677447.995171:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (21)--
00000800:00000200:8.2:1619677447.995175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.995183:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.995185:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.995186:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.995186:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.995188:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.995191:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.995239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.995245:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.995247:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.995250:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.995250:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.995252:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.995253:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677447.995255:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.995256:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.995257:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.995258:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677447.995261:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.995263:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.995266:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.995267:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677447.995267:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.995269:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.995270:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.995271:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.995271:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677447.995275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.995282:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.995285:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.995286:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.995286:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.995288:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.995290:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677447.995560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.995566:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.995569:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.995570:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.995572:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.995574:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.995576:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677447.995577:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.995578:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.995579:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.995580:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677447.995583:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.995585:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677447.995587:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.995588:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677447.995588:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677447.995590:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677447.995590:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.995591:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.995591:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.995597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.995605:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.995607:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.995608:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.995608:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.995610:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.995612:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677447.995643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677447.995650:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.995653:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.995654:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.995656:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.995659:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.995660:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677447.995661:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.995663:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.995664:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.995665:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677447.995668:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.995670:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677447.995673:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.995674:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677447.995675:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677447.995677:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677447.995677:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.995678:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.995678:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677447.995684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677447.995691:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.995694:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677447.995695:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677447.995695:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677447.995696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:2.0:1619677447.995697:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.995700:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677447.995702:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.995705:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.995706:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.995708:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.995710:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.995712:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.995713:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720753] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.995714:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.995715:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.995716:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.995720:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677447.995721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677447.995721:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.995724:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.995725:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.995725:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.995727:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.995727:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.995728:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.995728:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677447.995729:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.995732:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.2:1619677447.995733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677447.995734:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.995737:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.995737:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.2:1619677447.995737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:31.0:1619677447.995738:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677447.995740:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677447.995740:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677447.995740:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:31.0:1619677447.995741:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.995742:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677447.995742:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:31.0:1619677447.995743:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:17.0:1619677447.995743:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.995744:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.995744:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:20.0:1619677447.995745:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.995745:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:17.0:1619677447.995747:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:31.0:1619677447.995748:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677447.995749:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.995749:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:20.0:1619677447.995750:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:8.2:1619677447.995750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677447.995751:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:20.0:1619677447.995751:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.995753:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677447.995754:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.995754:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:20.0:1619677447.995754:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677447.995755:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:20.0:1619677447.995755:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677447.995757:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677447.995757:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.995758:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677447.995758:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677447.995759:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:20.0:1619677447.995759:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.995760:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677447.995761:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677447.995761:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.995762:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.995762:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677447.995763:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.995763:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677447.995764:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677447.995765:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677447.995767:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677447.995767:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677447.995768:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677447.995769:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:26.0:1619677447.995770:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.995772:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:9.2:1619677447.995772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677447.995773:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.995773:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677447.995775:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.995778:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:18.0:1619677447.995780:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677447.995783:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677447.995784:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.995784:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677447.995786:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.995788:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677447.995899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677447.995904:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.995907:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.995907:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677447.995909:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677447.995911:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.995913:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677447.995914:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.995915:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.995916:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.995917:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677447.995921:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677447.995922:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677447.995925:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.995926:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677447.995926:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677447.995928:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677447.995929:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677447.995929:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677447.995930:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677447.995941:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.995944:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.995945:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.995945:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.995947:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677447.995950:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.996238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.996245:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.996248:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.996249:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.2:1619677447.996250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677447.996250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677447.996251:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.996253:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.996255:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:17.0:1619677447.996256:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677447.996256:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.996257:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677447.996257:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.996258:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677447.996259:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:10.0:1619677447.996259:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:30.0:1619677447.996260:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:1.0:1619677447.996260:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677447.996264:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.996264:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.996264:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.996265:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677447.996266:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.996267:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.996267:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:10.0:1619677447.996267:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:30.0:1619677447.996269:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677447.996269:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.996271:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.996271:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677447.996271:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677447.996272:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677447.996272:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677447.996272:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:30.0:1619677447.996273:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.996273:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:10.0:1619677447.996274:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677447.996274:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677447.996275:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677447.996275:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677447.996276:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:30.0:1619677447.996277:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677447.996277:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.996281:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.996281:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677447.996284:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677447.996284:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.996284:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677447.996285:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (23)++
00000800:00000200:30.0:1619677447.996287:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.996288:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:17.0:1619677447.996288:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (24)--
00000800:00000200:6.0:1619677447.996288:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (24)++
00000800:00000200:30.0:1619677447.996289:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677447.996289:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.996290:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:30.0:1619677447.996291:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677447.996291:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677447.996291:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677447.996291:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000800:00000200:30.0:1619677447.996292:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677447.996292:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677447.996292:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.996298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.2:1619677447.996299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677447.996300:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:6.0:1619677447.996300:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.996302:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.996302:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:22.0:1619677447.996302:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (23)++
00000800:00000200:22.0:1619677447.996303:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000400:00000200:22.0:1619677447.996304:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.996304:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (24)--
00000800:00000200:1.0:1619677447.996304:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677447.996305:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.996306:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.996306:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:5.0:1619677447.996307:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:1.0:1619677447.996307:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677447.996308:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677447.996308:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677447.996309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677447.996310:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.996310:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.996312:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.996315:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.996316:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677447.996317:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:31.0:1619677447.996318:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677447.996318:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.996318:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677447.996319:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677447.996320:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:20.0:1619677447.996320:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677447.996321:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.996322:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:21.2:1619677447.996404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.996409:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.996412:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.996413:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.996415:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.996418:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.996419:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677447.996420:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677447.996422:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.996423:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.996424:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677447.996427:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677447.996429:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677447.996432:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.996432:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677447.996433:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677447.996435:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677447.996435:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.996436:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.996436:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677447.996441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.996449:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.996451:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.996452:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.996452:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.996455:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.996457:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677447.996459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677447.996465:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.996469:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677447.996470:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677447.996472:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.996475:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.996477:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677447.996478:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.996480:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.996481:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.996482:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677447.996485:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.996487:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677447.996490:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.996491:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677447.996492:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677447.996493:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.996494:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677447.996495:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677447.996495:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.996499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.996507:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.996509:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.996510:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.996511:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.996512:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.996515:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677447.996530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.996537:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.996540:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.996543:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.996544:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.996545:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.996547:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.996548:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.996550:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.996551:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.996551:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677447.996554:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.996556:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677447.996558:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.996559:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.996560:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.996561:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.996562:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.996563:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.996563:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677447.996575:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.996578:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.996579:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.996580:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.996582:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.996583:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.996637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.996645:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.996647:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.996648:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.996650:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.996653:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.996654:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677447.996655:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.996657:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.996657:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.996658:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677447.996661:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.996663:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677447.996665:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.996666:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677447.996667:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677447.996668:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677447.996669:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.996669:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.996670:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677447.996680:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677447.996683:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677447.996684:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.996685:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.996687:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677447.996688:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677447.996693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677447.996700:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.996703:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677447.996703:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677447.996705:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.996708:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.996710:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677447.996711:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.996712:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.996713:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.996714:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677447.996717:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.996718:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677447.996721:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.996721:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677447.996722:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677447.996723:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677447.996724:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.996725:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677447.996726:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677447.996726:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:21.2:1619677447.996732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:12.0:1619677447.996735:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.996737:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.996738:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.996739:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:12.0:1619677447.996740:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.996741:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.996741:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677447.996742:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677447.996743:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677447.996743:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.2:1619677447.996744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.996750:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.996752:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.996754:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677447.996755:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.996757:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.996758:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677447.996759:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.996760:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.996762:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.996762:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677447.996766:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.996767:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677447.996770:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.996771:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677447.996771:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677447.996773:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677447.996773:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.996774:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.996775:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677447.996779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.996787:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.996789:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677447.996790:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.996790:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.996793:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.996795:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677447.996796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.996803:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.996806:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.996807:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.996809:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.996811:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.996812:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677447.996813:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.996814:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.996816:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.996816:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677447.996820:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.996821:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677447.996824:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.996825:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677447.996825:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677447.996827:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.996827:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.996828:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677447.996828:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677447.996839:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.996841:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677447.996842:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.996843:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.996845:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.996846:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677447.996969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677447.996976:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.996978:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677447.996979:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677447.996982:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677447.996984:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.996985:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677447.996987:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677447.996987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:18.0:1619677447.996988:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.996989:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.996990:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677447.996993:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677447.996994:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677447.996994:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677447.996997:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.996998:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:18.0:1619677447.996998:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.996998:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677447.996999:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:26.0:1619677447.997000:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677447.997000:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677447.997001:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677447.997001:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677447.997002:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:26.0:1619677447.997003:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.997004:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677447.997005:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.997007:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.997008:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.997009:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677447.997011:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.997013:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.997013:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677447.997016:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.997016:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677447.997017:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677447.997017:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:22.0:1619677447.997017:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.997017:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677447.997019:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677447.997020:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677447.997020:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:26.0:1619677447.997021:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677447.997022:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677447.997022:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:26.0:1619677447.997023:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677447.997028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.997032:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677447.997035:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677447.997035:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.997036:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.997036:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.997037:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.997038:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677447.997038:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.997039:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677447.997041:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677447.997050:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.997053:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.997054:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677447.997054:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.997055:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.997056:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677447.997080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.2:1619677447.997084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677447.997084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677447.997089:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677447.997090:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677447.997091:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677447.997093:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677447.997093:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.997093:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:31.0:1619677447.997094:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.997095:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:1.0:1619677447.997096:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.997097:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677447.997097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919065782400] (20)++
00000400:00000200:30.0:1619677447.997098:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.997098:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.997099:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677447.997100:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720737] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.997100:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677447.997102:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.997102:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720737] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677447.997103:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (21)++
00000400:00000200:1.0:1619677447.997103:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.997104:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.997105:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.997106:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677447.997106:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000400:00000200:1.0:1619677447.997106:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:18.0:1619677447.997107:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000400:00000200:30.0:1619677447.997108:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.997110:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677447.997110:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.997112:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.997112:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677447.997113:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.997114:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.997114:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.997115:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:1.0:1619677447.997115:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677447.997116:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677447.997116:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.997117:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677447.997117:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.997118:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677447.997119:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.997119:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677447.997120:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677447.997120:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.997120:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677447.997121:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:2.0:1619677447.997121:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:31.0:1619677447.997122:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677447.997122:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:30.0:1619677447.997123:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.997123:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.997123:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677447.997124:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.997124:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:1.0:1619677447.997124:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677447.997125:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677447.997126:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (23)++
00000800:00000200:31.0:1619677447.997127:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (24)--
00000400:00000200:2.0:1619677447.997127:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.997128:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677447.997128:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.997129:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.997129:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677447.997129:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.997129:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:31.0:1619677447.997130:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.997131:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677447.997132:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.997132:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:2.0:1619677447.997132:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.997133:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.2:1619677447.997133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677447.997133:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:21.2:1619677447.997134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677447.997134:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:6.0:1619677447.997135:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677447.997135:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.997136:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677447.997136:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (21)++
00000800:00000200:8.2:1619677447.997137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677447.997137:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919065782400] (22)++
00000800:00000200:20.0:1619677447.997138:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:12.0:1619677447.997138:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677447.997138:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677447.997138:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (23)--
00000800:00000200:22.0:1619677447.997139:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677447.997139:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.997139:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000400:00000200:20.0:1619677447.997141:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677447.997141:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.2:1619677447.997141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:31.0:1619677447.997142:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:12.0:1619677447.997142:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.997142:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:20.0:1619677447.997143:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677447.997144:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:13.0:1619677447.997144:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677447.997144:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.997144:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677447.997144:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919065782400] (21)++
00000800:00000200:31.0:1619677447.997145:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677447.997145:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677447.997146:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:20.0:1619677447.997146:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677447.997146:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677447.997147:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.997148:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.997149:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.997149:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:20.0:1619677447.997150:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677447.997150:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (22)++
00000800:00000200:5.0:1619677447.997151:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:18.0:1619677447.997152:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919065782400] (23)--
00000400:00000200:5.0:1619677447.997152:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.997153:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.997153:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000800:00000200:2.0:1619677447.997153:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (21)--
00000400:00000200:5.0:1619677447.997154:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.2:1619677447.997155:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:5.0:1619677447.997157:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.997158:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677447.997160:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677447.997160:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677447.997160:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.997162:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.997163:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.997163:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677447.997164:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677447.997164:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677447.997165:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677447.997165:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:10.0:1619677447.997166:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:5.0:1619677447.997166:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:26.0:1619677447.997167:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:20.0:1619677447.997167:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677447.997168:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.997169:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677447.997169:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:10.0:1619677447.997169:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677447.997172:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.997173:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677447.997175:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.997176:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.997177:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.997178:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.997179:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677447.997181:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:10.0:1619677447.997181:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000400:00000200:13.0:1619677447.997182:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677447.997183:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:13.0:1619677447.997184:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.997185:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.997185:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.997186:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677447.997187:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.997187:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677447.997188:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677447.997189:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677447.997189:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677447.997190:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:5.2:1619677447.997196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:12.0:1619677447.997197:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.997200:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677447.997201:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677447.997201:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:26.0:1619677447.997203:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677447.997203:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677447.997204:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677447.997206:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:12.0:1619677447.997206:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:10.0:1619677447.997206:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:12.0:1619677447.997207:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.997208:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677447.997209:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000400:00000200:26.0:1619677447.997210:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.997211:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.997211:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677447.997212:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677447.997213:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.997215:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.997216:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.997216:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677447.997219:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.997220:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.997222:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.997222:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.997223:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.997224:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.997225:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677447.997225:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677447.997226:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677447.997235:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.997237:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.997238:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.997239:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.997239:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:21.2:1619677447.997240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:10.0:1619677447.997241:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:22.0:1619677447.997245:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.997248:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677447.997249:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677447.997251:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677447.997254:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677447.997255:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677447.997256:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677447.997258:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677447.997259:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677447.997260:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677447.997263:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677447.997265:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677447.997268:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677447.997269:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677447.997270:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677447.997272:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677447.997273:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677447.997274:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677447.997275:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677447.997276:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:21.2:1619677447.997281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677447.997285:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:6.0:1619677447.997286:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:31.0:1619677447.997288:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:6.0:1619677447.997288:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:31.0:1619677447.997289:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.997289:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677447.997289:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:31.0:1619677447.997291:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.997291:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.997301:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677447.997302:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677447.997304:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677447.997305:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677447.997306:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677447.997306:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677447.997307:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677447.997364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.997372:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.997375:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.997376:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.997377:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677447.997379:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.997380:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677447.997381:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.997383:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.997384:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.997385:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677447.997388:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.997389:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.997391:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677447.997392:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.997392:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677447.997393:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.997394:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.997395:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.997395:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677447.997405:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.997407:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677447.997407:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.997408:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.997409:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677447.997410:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677447.997418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677447.997425:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.997428:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677447.997428:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677447.997431:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677447.997433:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677447.997435:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677447.997436:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677447.997437:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677447.997438:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677447.997439:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677447.997443:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677447.997444:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677447.997447:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677447.997448:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677447.997448:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677447.997450:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677447.997450:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677447.997451:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.997452:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677447.997457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677447.997465:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677447.997467:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677447.997468:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.997470:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.997471:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677447.997472:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677447.997473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.997480:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.997482:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.997483:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.997485:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.997487:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.997489:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677447.997490:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.997492:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.997493:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.997494:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677447.997497:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.997499:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677447.997502:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.997503:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677447.997504:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677447.997505:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677447.997506:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.997507:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.997507:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677447.997511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677447.997520:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677447.997523:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677447.997523:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677447.997524:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.997526:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.997528:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677447.997540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677447.997547:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.997550:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677447.997552:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.997554:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677447.997554:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.997556:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677447.997557:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677447.997558:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.997559:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677447.997560:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677447.997563:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.997565:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677447.997568:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.997569:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677447.997569:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677447.997571:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677447.997572:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677447.997573:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677447.997573:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677447.997585:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.997588:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677447.997588:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677447.997589:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.997591:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.997594:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677447.997618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677447.997626:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.997628:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.997629:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.997631:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677447.997632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:26.0:1619677447.997633:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.997634:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677447.997636:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.997637:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.997638:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.997639:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:10.0:1619677447.997641:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.997642:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677447.997643:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677447.997644:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.997645:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:10.0:1619677447.997647:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.997649:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677447.997649:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:26.0:1619677447.997650:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:10.0:1619677447.997650:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677447.997651:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:10.0:1619677447.997651:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.997652:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677447.997653:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:10.0:1619677447.997653:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677447.997654:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.997655:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.997655:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677447.997657:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.997657:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.997659:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:8.2:1619677447.997659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677447.997662:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.997663:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677447.997663:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677447.997665:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677447.997665:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.997665:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677447.997666:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:10.0:1619677447.997666:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.0:1619677447.997669:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.2:1619677447.997672:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677447.997672:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:20.0:1619677447.997675:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.997677:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.2:1619677447.997677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677447.997679:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:22.0:1619677447.997679:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:20.0:1619677447.997680:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:22.0:1619677447.997681:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:20.0:1619677447.997681:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677447.997682:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:22.0:1619677447.997682:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.997682:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:20.0:1619677447.997683:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:22.0:1619677447.997685:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.997685:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.997685:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677447.997685:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:31.0:1619677447.997686:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677447.997686:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677447.997686:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:31.0:1619677447.997689:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677447.997689:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:31.0:1619677447.997691:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677447.997692:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.997694:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.997695:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.997696:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677447.997699:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.997701:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677447.997704:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.997705:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677447.997705:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677447.997707:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677447.997708:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.997708:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.997709:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677447.997713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677447.997721:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677447.997723:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677447.997724:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.997726:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677447.997726:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.997729:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677447.997744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677447.997747:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.997750:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677447.997752:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.997753:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677447.997755:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.997756:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677447.997758:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720722] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677447.997759:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677447.997760:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677447.997761:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677447.997765:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677447.997767:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677447.997769:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677447.997770:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677447.997771:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677447.997773:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677447.997774:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677447.997775:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677447.997776:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677447.997777:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.2:1619677447.997780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.2:1619677447.997782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677447.997785:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677447.997786:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677447.997787:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677447.997788:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:30.0:1619677447.997789:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:17.0:1619677447.997789:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677447.997790:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.997790:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677447.997790:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.997791:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677447.997791:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677447.997792:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677447.997792:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:1.0:1619677447.997794:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.997795:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:1.0:1619677447.997795:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.997797:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.997798:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.997799:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.997800:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.997803:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.997805:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.997807:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.997807:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.997808:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.997810:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.997810:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.997811:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.997811:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.997816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.997822:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.997824:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.997825:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.997826:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.997827:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.997828:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677447.997927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677447.997932:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677447.997935:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677447.997937:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677447.997937:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.997940:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677447.997941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:5.0:1619677447.997942:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:5.0:1619677447.997944:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.997945:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.997946:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.997947:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:26.0:1619677447.997948:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677447.997950:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.997951:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.997951:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677447.997952:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:26.0:1619677447.997954:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677447.997955:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.997956:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000400:00000200:26.0:1619677447.997957:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.997957:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:26.0:1619677447.997958:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677447.997959:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677447.997959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:5.0:1619677447.997959:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000400:00000200:26.0:1619677447.997960:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677447.997960:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:26.0:1619677447.997962:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.997962:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:5.0:1619677447.997962:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677447.997964:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677447.997964:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:26.0:1619677447.997965:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677447.997965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:26.0:1619677447.997967:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677447.997969:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677447.997969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:8.0:1619677447.997969:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:26.0:1619677447.997970:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677447.997970:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:8.0:1619677447.997971:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:26.0:1619677447.997972:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.997972:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677447.997972:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:26.0:1619677447.997973:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677447.997973:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:26.0:1619677447.997974:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677447.997974:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:8.0:1619677447.997974:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677447.997975:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:20.0:1619677447.997975:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:24.0:1619677447.997976:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677447.997976:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:8.0:1619677447.997977:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.997978:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:8.0:1619677447.997979:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677447.997980:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96720718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.997981:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.997982:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.997983:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:10.0:1619677447.997984:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:24.0:1619677447.997986:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:10.0:1619677447.997986:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:8.0:1619677447.997986:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677447.997987:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.997987:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677447.997988:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677447.997989:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:10.0:1619677447.997989:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.997989:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:21.2:1619677447.997991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677447.997991:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:8.0:1619677447.997992:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.997993:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677447.997993:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677447.997995:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677447.997995:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.997996:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.997996:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:31.0:1619677447.997998:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.998001:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677447.998002:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000400:00000200:31.0:1619677447.998003:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677447.998004:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677447.998005:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.998005:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677447.998005:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677447.998005:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677447.998006:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000400:00000200:31.0:1619677447.998007:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:26.0:1619677447.998007:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:13.0:1619677447.998007:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:31.0:1619677447.998008:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.998010:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.998010:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677447.998010:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.998010:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677447.998012:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:13.0:1619677447.998012:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.998014:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:31.0:1619677447.998015:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.998017:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677447.998018:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677447.998019:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.998020:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677447.998020:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677447.998022:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677447.998022:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.998023:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.998023:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677447.998033:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677447.998036:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677447.998036:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.998038:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.998039:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677447.998040:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.998236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677447.998243:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.2:1619677447.998243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:24.0:1619677447.998246:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677447.998246:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677447.998248:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677447.998249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677447.998250:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:24.0:1619677447.998251:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.998253:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:10.0:1619677447.998253:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.998254:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:24.0:1619677447.998254:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677447.998256:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:24.0:1619677447.998256:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.998256:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.998257:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.998258:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:10.0:1619677447.998258:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.998259:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:10.0:1619677447.998260:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677447.998261:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.998262:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677447.998262:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677447.998263:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:24.0:1619677447.998263:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:30.0:1619677447.998264:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.998264:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.998265:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677447.998265:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.998266:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677447.998266:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677447.998267:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677447.998267:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.998267:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677447.998268:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.998269:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677447.998269:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.998269:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677447.998269:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677447.998270:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:24.0:1619677447.998270:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.998271:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677447.998272:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:30.0:1619677447.998274:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.998275:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677447.998275:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.998275:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:8.2:1619677447.998275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677447.998276:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677447.998278:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.998278:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677447.998278:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677447.998279:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677447.998279:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:30.0:1619677447.998280:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677447.998281:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677447.998281:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677447.998281:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677447.998282:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.998283:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677447.998283:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:1.0:1619677447.998283:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.2:1619677447.998285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.998287:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.998288:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.2:1619677447.998288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.2:1619677447.998288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:1.0:1619677447.998288:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677447.998289:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:1.0:1619677447.998290:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.998291:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:6.0:1619677447.998292:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677447.998293:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677447.998296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677447.998297:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.998299:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677447.998300:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677447.998300:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:20.0:1619677447.998300:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.998301:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.998301:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677447.998302:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:20.0:1619677447.998302:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:1.0:1619677447.998302:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677447.998303:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677447.998303:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.998304:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677447.998304:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677447.998304:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677447.998305:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677447.998305:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677447.998306:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:17.0:1619677447.998306:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:1.0:1619677447.998306:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677447.998307:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677447.998307:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677447.998308:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677447.998308:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.998309:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677447.998309:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.998310:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677447.998310:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:17.0:1619677447.998311:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677447.998315:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.998317:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677447.998320:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.998321:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677447.998321:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677447.998323:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.998323:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.998324:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.998324:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677447.998337:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.998340:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677447.998341:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.998341:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677447.998343:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.998348:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677447.998349:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677447.998351:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677447.998352:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677447.998353:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677447.998353:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.998354:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677447.998461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.998469:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.998472:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677447.998473:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677447.998475:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.998478:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.998479:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677447.998481:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720712] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.998482:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.998483:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.998484:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677447.998487:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.998489:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677447.998492:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.998493:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677447.998494:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677447.998495:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677447.998496:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.998497:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677447.998498:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677447.998498:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677447.998504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677447.998508:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677447.998511:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677447.998511:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677447.998512:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677447.998513:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677447.998513:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.998514:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677447.998515:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677447.998515:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.998518:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677447.998528:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677447.998531:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677447.998532:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677447.998532:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.998533:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677447.998534:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.998689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677447.998695:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.2:1619677447.998698:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677447.998700:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.998701:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677447.998703:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677447.998703:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677447.998706:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.998706:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.998707:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677447.998707:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677447.998709:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.998710:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.998711:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.998712:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.998713:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677447.998713:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.998714:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677447.998716:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.998717:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677447.998717:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.998718:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677447.998719:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:1.0:1619677447.998720:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:5.0:1619677447.998722:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.998723:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677447.998723:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677447.998723:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.998725:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677447.998725:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.0:1619677447.998726:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.998727:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.998727:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677447.998728:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.998729:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677447.998729:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677447.998731:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677447.998731:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:9.2:1619677447.998732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677447.998732:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677447.998733:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677447.998734:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677447.998739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677447.998740:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677447.998743:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677447.998744:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.998745:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677447.998745:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677447.998745:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:2.0:1619677447.998746:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.998747:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677447.998748:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677447.998748:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677447.998749:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677447.998749:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677447.998749:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.998749:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:6.0:1619677447.998751:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677447.998754:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677447.998812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677447.998820:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.998823:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677447.998825:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.998828:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.998828:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677447.998829:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677447.998831:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.998832:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.998833:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.998834:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677447.998838:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.998840:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677447.998843:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.998844:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677447.998844:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677447.998846:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677447.998846:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677447.998847:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677447.998848:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677447.998852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677447.998860:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677447.998863:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677447.998864:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.998864:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677447.998866:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677447.998869:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677447.998917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677447.998924:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677447.998928:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.998928:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677447.998930:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677447.998933:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677447.998935:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677447.998936:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677447.998937:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677447.998939:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677447.998939:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677447.998942:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677447.998944:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677447.998947:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677447.998948:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677447.998949:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677447.998950:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677447.998951:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677447.998951:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677447.998952:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677447.998956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677447.998964:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677447.998966:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677447.998967:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677447.998969:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677447.998970:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677447.998972:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677447.999014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.999021:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.999024:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677447.999025:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.999027:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.113@o2ib7 : GET - for me
00000400:00000200:1.0:1619677447.999029:0:19931:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.113@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:1.0:1619677447.999032:0:19931:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.113@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:1.0:1619677447.999033:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.999034:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.999036:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.999039:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : REPLY try# 0
00000800:00000200:1.0:1619677447.999042:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.999043:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.999044:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.999045:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.999046:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.999047:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.2:1619677447.999052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677447.999055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.999060:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.999061:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677447.999061:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:17.0:1619677447.999062:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.999063:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677447.999064:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: REPLY: OK
00000800:00000200:17.0:1619677447.999065:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677447.999065:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677447.999066:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677447.999067:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.999067:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.999068:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677447.999070:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.999071:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.999072:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.999073:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677447.999076:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.999077:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677447.999079:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.999079:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677447.999080:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677447.999081:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677447.999081:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.999081:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.999082:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677447.999086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677447.999093:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677447.999095:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677447.999095:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677447.999096:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677447.999097:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677447.999098:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.999126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677447.999133:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.999136:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677447.999137:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677447.999139:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677447.999142:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677447.999143:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677447.999145:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677447.999146:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677447.999147:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677447.999148:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677447.999150:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677447.999151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677447.999152:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677447.999155:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677447.999156:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677447.999157:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677447.999159:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677447.999159:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677447.999160:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677447.999160:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677447.999161:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677447.999162:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677447.999162:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677447.999163:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677447.999164:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677447.999165:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677447.999166:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677447.999168:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677447.999168:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677447.999169:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677447.999172:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677447.999172:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:26.0:1619677447.999173:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:13.0:1619677447.999175:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:26.0:1619677447.999176:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677447.999176:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.999177:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:13.0:1619677447.999177:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677447.999178:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677447.999179:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:13.0:1619677447.999179:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677447.999180:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677447.999180:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:12.0:1619677447.999180:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677447.999181:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677447.999193:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:13.0:1619677447.999196:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677447.999197:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677447.999197:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.999198:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677447.999198:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677447.999199:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:22.0:1619677447.999200:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.999200:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677447.999200:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.999201:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677447.999203:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.999221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677447.999228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:30.0:1619677447.999229:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677447.999231:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677447.999233:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677447.999234:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677447.999235:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677447.999236:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.999237:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677447.999238:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:30.0:1619677447.999239:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677447.999240:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.999240:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677447.999241:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.999242:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.999242:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.999243:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677447.999244:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677447.999246:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.999247:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677447.999248:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677447.999248:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.999249:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.999250:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.999253:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677447.999253:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677447.999254:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677447.999254:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.999255:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.999256:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677447.999256:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677447.999257:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677447.999259:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:2.0:1619677447.999259:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677447.999260:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:2.0:1619677447.999260:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677447.999261:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677447.999262:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677447.999263:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677447.999264:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:20.2:1619677447.999265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677447.999265:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677447.999265:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677447.999270:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:5.2:1619677447.999270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677447.999271:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677447.999272:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.999273:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:31.0:1619677447.999274:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677447.999274:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677447.999275:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:13.0:1619677447.999275:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677447.999276:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:18.0:1619677447.999276:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:30.0:1619677447.999277:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:13.0:1619677447.999277:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677447.999278:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:18.0:1619677447.999278:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:13.0:1619677447.999278:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677447.999279:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677447.999280:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:13.0:1619677447.999280:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.999281:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677447.999283:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677447.999288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677447.999289:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677447.999291:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677447.999292:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:24.0:1619677447.999296:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677447.999297:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677447.999298:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677447.999299:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:8.0:1619677447.999301:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677447.999304:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677447.999306:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677447.999308:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677447.999310:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677447.999311:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720702] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677447.999313:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677447.999314:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677447.999315:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677447.999318:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677447.999320:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677447.999323:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677447.999324:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677447.999324:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677447.999326:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677447.999326:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677447.999328:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677447.999329:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677447.999330:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677447.999332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677447.999336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677447.999340:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677447.999342:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677447.999343:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677447.999343:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:1.0:1619677447.999344:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677447.999345:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677447.999346:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677447.999346:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677447.999347:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677447.999348:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677447.999466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677447.999471:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.999474:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677447.999475:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677447.999477:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677447.999478:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677447.999480:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677447.999481:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677447.999482:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677447.999483:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677447.999484:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677447.999487:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677447.999489:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677447.999492:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677447.999493:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677447.999493:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677447.999495:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677447.999495:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677447.999496:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677447.999497:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677447.999509:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677447.999512:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677447.999513:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677447.999515:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677447.999517:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677447.999517:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677447.999559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677447.999565:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.999568:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677447.999569:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677447.999571:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677447.999574:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677447.999575:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677447.999577:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677447.999578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:6.0:1619677447.999578:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677447.999579:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677447.999580:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677447.999583:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677447.999584:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:6.0:1619677447.999585:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677447.999587:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677447.999588:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:6.0:1619677447.999588:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677447.999589:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677447.999589:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:2.0:1619677447.999590:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677447.999591:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677447.999592:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677447.999592:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677447.999593:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:2.0:1619677447.999593:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677447.999595:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677447.999596:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677447.999597:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677447.999598:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677447.999599:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677447.999603:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677447.999604:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:2.0:1619677447.999605:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677447.999606:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677447.999607:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677447.999608:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677447.999608:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677447.999609:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677447.999609:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:10.0:1619677447.999610:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:2.0:1619677447.999611:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677447.999612:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677447.999613:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:2.0:1619677447.999613:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677447.999613:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:26.0:1619677447.999624:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677447.999625:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677447.999626:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677447.999626:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677447.999627:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677447.999627:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677447.999628:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677447.999628:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677447.999629:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:26.0:1619677447.999630:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677447.999630:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:20.2:1619677447.999632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677447.999633:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.0:1619677447.999638:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.999641:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677447.999643:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677447.999643:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677447.999645:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677447.999647:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677447.999648:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677447.999649:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677447.999651:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677447.999651:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677447.999655:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677447.999657:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677447.999660:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677447.999661:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677447.999661:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677447.999663:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677447.999663:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677447.999664:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677447.999665:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677447.999676:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677447.999679:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677447.999680:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677447.999680:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677447.999682:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677447.999684:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677447.999802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677447.999810:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.999813:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677447.999815:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677447.999817:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677447.999817:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677447.999819:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677447.999820:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677447.999821:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677447.999822:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677447.999823:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677447.999827:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677447.999828:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677447.999830:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677447.999831:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677447.999832:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677447.999833:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677447.999834:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677447.999835:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677447.999835:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677447.999845:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677447.999848:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677447.999849:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677447.999850:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677447.999852:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677447.999853:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677447.999871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677447.999877:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.999879:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677447.999880:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677447.999882:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677447.999885:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677447.999886:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677447.999888:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677447.999889:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677447.999890:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677447.999891:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677447.999894:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677447.999896:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677447.999899:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677447.999900:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677447.999900:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677447.999902:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677447.999903:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677447.999904:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677447.999904:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677447.999915:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677447.999918:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677447.999919:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677447.999921:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677447.999922:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677447.999923:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.000055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.000063:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.000065:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.000067:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.000069:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.000070:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.000071:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.000072:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.000073:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.000075:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.000076:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677448.000079:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.000081:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.000083:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.000084:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.000085:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.000087:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.000087:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.000089:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.000090:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.000091:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677448.000093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.000097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.2:1619677448.000098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:18.0:1619677448.000100:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.000103:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.000104:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.000104:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.000105:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:26.0:1619677448.000106:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.000107:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:18.0:1619677448.000107:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.000108:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.000108:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:30.0:1619677448.000109:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.000109:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677448.000110:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:26.0:1619677448.000111:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.000114:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.000115:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.000116:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.000118:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.000118:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.000119:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.000122:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.000124:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.000126:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.000127:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.000127:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.000129:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.000129:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.000130:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.000131:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677448.000137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.000143:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.000145:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:5.2:1619677448.000146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.000147:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.000148:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:2.0:1619677448.000148:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.000149:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:18.0:1619677448.000151:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.000152:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:2.0:1619677448.000152:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.000153:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.000154:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.000155:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.000155:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.000156:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.000157:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677448.000158:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.000158:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:13.0:1619677448.000159:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.000160:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:18.0:1619677448.000161:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.000162:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.000163:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.000164:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677448.000164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:18.0:1619677448.000165:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.000166:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.000166:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.000167:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.000170:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.000171:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:18.0:1619677448.000171:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.000171:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.000172:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.000173:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677448.000174:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:2.0:1619677448.000174:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:8.0:1619677448.000175:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677448.000175:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:24.0:1619677448.000177:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.000177:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.000177:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.000178:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.000178:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.000178:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.000179:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:24.0:1619677448.000180:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.000180:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:24.0:1619677448.000181:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:18.0:1619677448.000181:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.000181:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.000182:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:24.0:1619677448.000183:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96720691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.000183:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:24.0:1619677448.000184:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.000184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000400:00000200:24.0:1619677448.000185:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.000186:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.000186:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:24.0:1619677448.000189:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.000189:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:2.0:1619677448.000189:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:26.0:1619677448.000190:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.000190:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.000191:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.000192:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.000192:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.000192:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.000193:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:24.0:1619677448.000194:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.000194:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:13.0:1619677448.000194:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677448.000195:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.000195:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:24.0:1619677448.000197:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.000197:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677448.000197:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:24.0:1619677448.000198:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.000198:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:13.0:1619677448.000198:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.000200:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.000201:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.000203:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677448.000203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.000210:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.000213:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:12.0:1619677448.000213:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:17.0:1619677448.000214:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.000214:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.000217:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.000217:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.000218:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.000219:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.000219:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.000220:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:12.0:1619677448.000221:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677448.000222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.000230:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677448.000231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:1.0:1619677448.000232:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:20.0:1619677448.000233:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677448.000233:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.000233:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.000234:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.000235:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.0:1619677448.000236:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.000238:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.000239:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.000241:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.000242:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.000244:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.000245:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.000246:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.000247:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.000250:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.000252:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.000255:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.000256:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.000257:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.000259:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.000259:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.000260:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.000261:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:13.0:1619677448.000271:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.000273:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.000274:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.000276:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677448.000277:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.2:1619677448.000278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677448.000279:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677448.000284:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.000287:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.000287:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.000289:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.000292:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.000298:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.000300:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.000301:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.000302:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.000303:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.000306:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.000307:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.000309:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.000310:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.000311:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.000312:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.000312:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.000314:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.000315:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.000315:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.000318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.000321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.000327:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.000328:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.000328:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.000329:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:1.0:1619677448.000329:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.000330:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.000330:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.000331:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.000332:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.000332:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.000373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.000381:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.000384:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.000384:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.000386:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.000389:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.000390:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.000392:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720688] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.000393:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.000394:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.000395:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.000398:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.000400:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.000403:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.000403:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.000404:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.000406:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.000406:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.000407:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.000407:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.000421:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.000423:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.000424:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.000425:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.000427:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.000430:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.000492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.000497:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.000500:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.000500:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.000502:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.000505:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.000506:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.000508:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.000509:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.000510:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.000511:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.000514:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.000516:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.000519:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.000520:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.000521:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.000522:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.000523:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.000523:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.000524:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.000535:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.000537:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.000538:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.000539:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.000541:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.000544:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.000692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.000697:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.000700:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.000701:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.000703:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.000705:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.000706:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.000707:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.000709:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.000710:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.000711:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.000714:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.000716:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.000717:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.000718:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.000719:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.000720:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.000721:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.000721:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.000722:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.000726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.000733:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.000735:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.000736:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.000737:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.000737:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.000738:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.000877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.000883:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.000886:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.000887:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.000889:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.000892:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.000893:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677448.000894:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.000896:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.000897:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.000898:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677448.000901:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.000903:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677448.000906:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677448.000907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:18.0:1619677448.000907:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677448.000907:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677448.000910:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677448.000910:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.000911:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.000911:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677448.000914:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.000917:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.000917:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677448.000920:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.000921:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:8.0:1619677448.000923:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.000924:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.000924:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:8.0:1619677448.000924:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.000925:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.000926:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96720684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.000927:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000400:00000200:8.0:1619677448.000927:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.000929:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.000930:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.000931:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:8.0:1619677448.000933:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.000935:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.000938:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.000939:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.000940:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.000942:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.000942:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.000943:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.000944:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.000948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.000955:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.000958:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.000959:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.000959:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.000960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:2.0:1619677448.000962:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.000965:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.000966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.000966:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.000968:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.000971:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.000971:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.000973:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.000974:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677448.000975:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:18.0:1619677448.000977:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.000977:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.000977:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.000978:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677448.000978:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.000979:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.000980:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.000980:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:5.0:1619677448.000980:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.000984:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.000986:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.000989:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.000990:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.000990:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.000992:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.000993:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.000993:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.000994:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.001006:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.001009:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.001010:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.001012:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.001013:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.001014:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.001078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.001086:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.001089:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.001091:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.001094:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.001094:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.001095:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.001096:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720682] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.001098:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.001099:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.001100:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.001103:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.001105:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.001108:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.001108:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.001109:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.001111:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.001111:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.001112:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.001112:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.001122:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.001125:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.001125:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.001127:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.001128:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.001129:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.001150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.001157:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.001159:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.001160:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.001162:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.001164:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.001166:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.001167:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.001168:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.001169:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.001170:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.001173:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.001174:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.001177:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.001178:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.001178:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.001180:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.001180:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.001181:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.001181:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.001193:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.001196:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.001197:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.001197:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.001199:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.001202:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.001267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.001272:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.001274:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.001275:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.001277:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.001279:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.001281:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.001282:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.001283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:22.0:1619677448.001284:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.001285:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.001286:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:2.0:1619677448.001288:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:22.0:1619677448.001289:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.001291:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677448.001291:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.001292:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.001299:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.001300:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.001301:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.001302:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:2.0:1619677448.001302:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.001304:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.001304:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.001304:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677448.001304:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:22.0:1619677448.001305:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:2.0:1619677448.001305:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.001307:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.001308:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.001309:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.001312:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.001314:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.001317:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.001317:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.001318:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:2.0:1619677448.001318:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677448.001320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:2.0:1619677448.001320:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.001320:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677448.001321:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:2.0:1619677448.001321:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:31.0:1619677448.001322:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.001322:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:31.0:1619677448.001324:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.001325:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.001327:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.2:1619677448.001327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.001328:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.001331:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.001332:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:4.2:1619677448.001332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:26.0:1619677448.001334:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:17.0:1619677448.001335:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:26.0:1619677448.001336:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.001337:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:26.0:1619677448.001338:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677448.001338:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.001338:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:26.0:1619677448.001339:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.001339:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:17.0:1619677448.001340:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.001341:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.001342:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.001342:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:17.0:1619677448.001342:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:26.0:1619677448.001343:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:8.0:1619677448.001343:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.001345:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.001347:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:8.2:1619677448.001347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:26.0:1619677448.001349:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:24.0:1619677448.001350:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.001351:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:26.0:1619677448.001352:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677448.001352:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96720677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.001353:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.001353:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:24.0:1619677448.001354:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.001354:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.001355:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:24.0:1619677448.001355:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.001355:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.001356:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.001356:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.001357:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:17.0:1619677448.001357:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.001357:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.001358:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000400:00000200:24.0:1619677448.001359:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.001359:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.001359:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:24.0:1619677448.001361:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.2:1619677448.001361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:24.0:1619677448.001365:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.001366:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.001366:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.001368:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.001369:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.001369:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.001370:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677448.001370:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.001373:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.001374:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.001374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677448.001376:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.001377:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.001379:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677448.001382:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.001384:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.001385:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.001386:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.001388:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.001390:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.001553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.001560:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.001563:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.001563:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.001565:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.001568:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.001570:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.001571:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.001572:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.001573:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.001574:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.001578:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.001579:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.001582:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.001583:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.001583:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.001585:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.001585:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.001587:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.001588:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.001588:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.001594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.001598:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:26.0:1619677448.001601:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.001601:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.001602:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.001602:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.001603:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.001604:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.001604:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.001605:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.001607:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.001617:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.001619:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.001620:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.001620:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.001621:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.001622:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.001833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.001838:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.001840:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.001841:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.001843:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.001845:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.001847:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.001848:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.001849:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.001850:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.001851:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:21.2:1619677448.001852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:6.0:1619677448.001853:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.001855:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.001857:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.001858:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:4.2:1619677448.001858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.001859:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677448.001859:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.001860:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.001861:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677448.001862:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:6.0:1619677448.001862:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.001863:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000400:00000200:30.0:1619677448.001864:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.001864:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:10.0:1619677448.001865:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.001866:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.001866:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.001868:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677448.001868:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:9.2:1619677448.001868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:26.0:1619677448.001869:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.2:1619677448.001869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:30.0:1619677448.001870:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.001871:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.001872:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.001872:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.001873:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.001874:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:10.0:1619677448.001874:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.001875:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.001876:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677448.001876:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:10.0:1619677448.001876:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.001877:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.001877:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677448.001877:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.001878:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:30.0:1619677448.001879:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.001879:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.001879:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.001880:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.001880:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.001881:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.001882:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.001882:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.001883:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.001884:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.001885:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.001885:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:10.0:1619677448.001885:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.001886:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.001887:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.001887:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.001887:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:9.2:1619677448.001888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:10.0:1619677448.001890:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.001891:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.001891:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.001893:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.001893:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.001894:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.001895:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677448.001896:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.001899:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:8.2:1619677448.001899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.001899:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:12.0:1619677448.001900:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.001900:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.001901:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.001901:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:12.0:1619677448.001902:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.001903:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.001905:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.001907:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.001907:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:12.0:1619677448.001908:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.001910:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.001911:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.001913:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.001913:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.001915:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.001917:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677448.001918:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.001919:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.001920:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.2:1619677448.001920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:13.0:1619677448.001921:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.001922:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677448.001927:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.001929:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.001930:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.001931:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.001932:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.001933:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.001950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.001958:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.001961:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.001963:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.001965:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.001966:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.001967:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.001968:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.001970:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.001971:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.001972:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.001975:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.001977:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.001979:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.001980:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.001980:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.001982:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.001982:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.001983:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.001984:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.2:1619677448.001984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.001990:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.001993:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.001994:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677448.001994:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.001996:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:1.0:1619677448.001996:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677448.001997:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.001997:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.001999:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:13.0:1619677448.002000:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677448.002000:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.002001:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.002002:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:1.0:1619677448.002002:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.002003:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.002004:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.002007:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.002009:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:13.0:1619677448.002012:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.002012:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.002013:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.002013:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:13.0:1619677448.002015:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.002015:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:13.0:1619677448.002016:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677448.002016:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.002017:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.002017:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.002017:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.002018:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.002019:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677448.002029:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.002031:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.002032:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.002032:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.002034:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.002037:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.002108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.002114:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.002116:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.002118:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.002119:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.002121:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.002122:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.002124:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.002125:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.002126:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.002127:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.002131:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.002132:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.002135:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.002136:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.002137:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.002139:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.002139:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.002141:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.002142:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.002143:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.002148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677448.002151:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.002153:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677448.002153:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.002154:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.002155:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.002155:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.002156:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:12.0:1619677448.002156:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.002157:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.2:1619677448.002157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:12.0:1619677448.002158:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.002165:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.002167:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.002168:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.002170:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.002173:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.002174:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.002175:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.002177:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.002178:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.002179:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.002182:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.002183:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.002185:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.002186:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.002187:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.002188:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.002188:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.002189:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.002190:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.002200:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.002202:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.002202:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.002203:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.002204:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.002205:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.002282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.002288:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.002291:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.002292:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.002299:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.002300:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.002301:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.002302:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.002303:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.002304:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.002305:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.002308:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.002310:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.002313:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.002314:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.002314:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.002316:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.002316:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.002317:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.002318:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.002323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.002331:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.002333:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.002334:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.002336:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.002337:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.002339:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.002352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.002358:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.002361:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.002362:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.002364:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.002366:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.002368:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.002369:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720667] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.002370:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.002371:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.002372:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.002375:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.002377:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.002379:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.002380:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.002381:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.002383:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.002383:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.002384:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.002385:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.002389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.002396:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.002399:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.002400:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.002401:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.002402:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.002404:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.002427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.002435:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.002438:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.002440:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.002442:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.002443:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.002444:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.002445:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.002446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:30.0:1619677448.002447:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.002447:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.002449:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.002452:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.002452:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677448.002454:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.002455:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.002456:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677448.002457:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.002458:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:22.0:1619677448.002458:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.002459:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.002460:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.002460:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:4.2:1619677448.002460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677448.002461:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677448.002461:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.002462:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:22.0:1619677448.002462:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.002463:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.002465:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.002466:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.002467:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.002467:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.002470:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677448.002470:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.002470:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:22.0:1619677448.002472:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.002473:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.002474:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:22.0:1619677448.002475:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.002476:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.002476:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677448.002476:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:8.0:1619677448.002476:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.002477:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.002477:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:22.0:1619677448.002478:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.002478:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.002479:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:13.0:1619677448.002479:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.002479:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96720664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.002480:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677448.002480:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:8.0:1619677448.002480:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.002481:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677448.002481:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.002484:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:8.0:1619677448.002487:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.002491:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.002492:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677448.002492:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.2:1619677448.002495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677448.002495:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677448.002495:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.002496:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.002496:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.002496:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.002497:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (22)++
00000400:00000200:17.0:1619677448.002498:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.002499:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677448.002500:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.002500:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677448.002501:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.002501:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:31.0:1619677448.002502:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.002505:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:8.2:1619677448.002505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677448.002507:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.002509:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.002509:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.002511:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.002512:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.002512:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:31.0:1619677448.002513:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.002514:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.002515:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:1.0:1619677448.002515:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.002516:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.002516:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.002518:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.002519:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.002521:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677448.002522:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.002525:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.002526:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.002526:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.002528:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.002529:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.002530:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.002530:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.002534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.002542:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.002544:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.002545:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.002547:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.002548:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.002550:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.002624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.002632:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.002635:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.002636:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.002638:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.002640:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.002642:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.002643:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.002644:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.002645:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.002646:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.002649:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.002650:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.002653:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.002654:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.002654:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.002655:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.002656:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.002657:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.002658:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.002659:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.002665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.2:1619677448.002668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:12.0:1619677448.002668:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.002670:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.002671:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.002672:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.002673:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:13.0:1619677448.002674:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.002674:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.002674:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677448.002675:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.002675:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.002676:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677448.002677:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.002678:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.002680:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.002683:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.002684:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677448.002685:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.002687:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.002688:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.002689:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.002692:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.002693:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.002696:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.002697:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.002697:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.002699:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.002700:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.002700:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.002701:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677448.002705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.002713:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.002715:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.002716:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.002718:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.002719:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.002720:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.002769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.002776:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.002778:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.002779:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.002780:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.002783:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.002785:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677448.002786:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.002787:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.002788:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.002788:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677448.002791:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.002792:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.002793:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.002794:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.002794:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.002796:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.002796:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.2:1619677448.002797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677448.002798:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.002799:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.002799:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677448.002803:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:9.2:1619677448.002803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.002805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677448.002805:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.002806:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.002808:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.002810:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.002811:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:17.0:1619677448.002812:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:2.0:1619677448.002812:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.002813:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:17.0:1619677448.002813:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.002813:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.002814:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:17.0:1619677448.002814:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.002815:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.002816:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:17.0:1619677448.002816:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.002816:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.002817:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:17.0:1619677448.002817:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677448.002818:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:17.0:1619677448.002821:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.002822:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.002825:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.002826:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.002826:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.002828:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.002828:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.002829:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.002830:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.002841:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.002844:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.002845:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:20.2:1619677448.002845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:6.0:1619677448.002845:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.002847:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.002848:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.002849:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677448.002851:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.002853:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.002854:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.002856:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.002857:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.002859:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.002860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677448.002860:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.002861:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.002862:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.002865:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.002867:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.002867:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.002869:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.002870:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677448.002870:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.002871:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677448.002872:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.002873:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.002873:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:30.0:1619677448.002874:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.002874:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.002874:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.002875:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:30.0:1619677448.002876:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.002878:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.002879:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677448.002879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677448.002880:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.002881:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.002885:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.002886:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.002887:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:30.0:1619677448.002889:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.002889:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:30.0:1619677448.002890:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.002890:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677448.002891:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:18.0:1619677448.002891:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.002892:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:18.0:1619677448.002892:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.002893:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.002894:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.002895:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:18.0:1619677448.002895:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677448.002896:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677448.002901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677448.002906:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:31.0:1619677448.002909:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677448.002909:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.002909:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.002910:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.002911:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.002912:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677448.002912:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.002913:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.002915:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.002926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.002933:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.002936:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.002937:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.002938:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.002941:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.002942:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.002943:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.002945:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.002946:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.002946:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.002949:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.002951:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.002954:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.002955:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.002955:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.002957:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.002957:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.002958:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.002958:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.2:1619677448.002962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.002970:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:6.0:1619677448.002970:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:31.0:1619677448.002972:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:6.0:1619677448.002973:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:31.0:1619677448.002974:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.002974:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.002974:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.002976:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.002977:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.002977:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.002979:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:6.0:1619677448.002979:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:31.0:1619677448.002980:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.002982:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.002983:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.002984:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.002987:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.002988:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.002991:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.002992:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.002993:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.002994:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.002995:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.002996:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.002997:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.002998:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.003000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.003003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.003008:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.003010:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677448.003011:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:5.0:1619677448.003011:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.003013:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.003013:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.003014:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.003015:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.003016:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.003016:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.003020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.003027:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.003029:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.003030:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.003031:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.003032:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.003032:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.003108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.003116:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.003119:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.003119:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.003122:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.003125:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.003126:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.003127:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.003128:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.003129:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.003130:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.003133:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.003135:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.003138:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.003139:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.003140:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.003141:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.003142:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.003142:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.003143:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677448.003155:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.003157:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.003158:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.003158:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.003161:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.003164:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.003170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:6.0:1619677448.003172:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.003175:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.003176:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.003176:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.003176:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.003177:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.003177:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677448.003179:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.003180:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.003181:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.003184:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.003186:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.003187:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.003188:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.003189:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.003190:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.003194:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.003195:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.003197:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.003198:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.003198:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.003200:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.003200:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.003202:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.003202:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.003203:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.003206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.003208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.003214:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.003215:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.003215:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.003215:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.003216:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.003216:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.003217:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.003217:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.003218:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677448.003219:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.003225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.003232:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.003235:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.003235:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.003235:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.003236:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.003237:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.003380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677448.003387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.003388:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.003391:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:4.2:1619677448.003392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:30.0:1619677448.003393:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.003395:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.003396:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.003396:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.003397:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:30.0:1619677448.003398:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.003399:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.003399:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677448.003399:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:30.0:1619677448.003401:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.003401:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.003403:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.003403:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.003403:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:30.0:1619677448.003404:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.003404:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.003405:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677448.003406:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.003407:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.003408:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.003408:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.003409:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.003409:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.003410:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.003411:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:24.0:1619677448.003411:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.003412:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96720650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.003414:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.003414:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.003414:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.003415:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.003415:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677448.003415:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.003416:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:24.0:1619677448.003416:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:30.0:1619677448.003417:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.003417:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.003418:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.003419:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.003419:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677448.003419:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.003420:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.003420:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677448.003421:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.003422:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.003422:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.003423:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:24.0:1619677448.003424:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.003424:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.003425:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.003425:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677448.003425:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.003426:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.003427:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.003428:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.003429:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.003430:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677448.003431:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:4.2:1619677448.003431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677448.003432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:13.0:1619677448.003433:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.003434:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.003435:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.003435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677448.003436:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:13.0:1619677448.003437:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.003437:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.003438:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:12.0:1619677448.003438:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.003439:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.003440:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.003440:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:1.0:1619677448.003440:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:10.0:1619677448.003441:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.003442:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.003442:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.0:1619677448.003442:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.003443:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.003444:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.003444:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677448.003444:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.003445:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677448.003445:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677448.003446:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677448.003447:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.003447:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.003449:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.003450:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.003450:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677448.003450:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:13.0:1619677448.003451:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.003453:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.003453:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.003455:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.003456:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.003457:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.003458:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.003458:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.003478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.003484:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.003487:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.003487:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.003490:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.003491:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.003492:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.003494:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.003495:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.003496:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.003497:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.003501:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.003503:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.003506:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.003507:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.003508:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.003510:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.003510:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.003511:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.003511:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.003522:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.003525:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.003526:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.003528:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.003529:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.003531:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.003704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.003710:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.003713:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.003714:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.003716:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.003718:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.003720:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.003721:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.003722:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.003724:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.003725:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.003728:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.003730:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.003733:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.003734:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.003734:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.003737:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.003737:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.003738:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.003738:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:26.0:1619677448.003749:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.003751:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.003752:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.003753:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.003755:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.003756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.003758:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:1.0:1619677448.003761:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.003764:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.003765:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.003767:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000400:00000200:1.0:1619677448.003767:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.003769:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.003770:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.2:1619677448.003770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:10.0:1619677448.003770:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000400:00000200:1.0:1619677448.003770:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.003771:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:1.0:1619677448.003771:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:10.0:1619677448.003772:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:1.0:1619677448.003772:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.003773:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:1.0:1619677448.003774:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.003774:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.003776:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:1.0:1619677448.003776:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.003779:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.003780:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677448.003781:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.003783:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.003783:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.003784:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:1.0:1619677448.003785:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.003786:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.003786:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:20.0:1619677448.003787:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.003787:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677448.003788:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.003788:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:20.0:1619677448.003789:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.003789:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.003789:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.003790:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:20.0:1619677448.003793:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.003794:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.2:1619677448.003794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.003797:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.003797:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677448.003798:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (23)++
00000800:00000200:20.0:1619677448.003800:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (24)--
00000800:00000200:20.0:1619677448.003801:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.003801:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.003802:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677448.003802:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000800:00000200:17.0:1619677448.003804:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000400:00000200:17.0:1619677448.003805:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.003805:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000800:00000200:1.0:1619677448.003806:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000400:00000200:17.0:1619677448.003807:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.003807:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.003809:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677448.003810:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.003811:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.003811:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.003849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677448.003853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.003858:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.003860:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677448.003862:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.003862:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.003863:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677448.003865:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.003865:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.003868:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.003869:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.003870:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.003871:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.003872:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.003874:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.003876:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.003876:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.003877:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.003879:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.003879:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.003879:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.003880:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.003883:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.003883:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.003884:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.003885:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.003885:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.003887:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.003887:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.003888:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.003888:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.003889:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.003889:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.003890:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.003891:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.003891:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.003891:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.003892:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.003892:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.003893:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677448.003900:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.003902:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.003902:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.003902:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.003903:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.003904:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.003904:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.003905:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.003907:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.003908:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.003909:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.003910:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.004019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.004027:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.004030:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.004032:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.004033:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.004034:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.004035:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.004036:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.004038:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.004038:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.004039:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.004042:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.004044:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.004046:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.004046:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.004047:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.004048:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.004049:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.004050:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.004050:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.004061:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.004063:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.004064:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.004065:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.004067:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.004068:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.004068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.004075:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.004077:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.004078:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.004080:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.004083:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.004084:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.004085:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.004086:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.004087:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.004088:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.004091:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.004093:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.004096:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.004097:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.004097:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.004099:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.004099:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.004100:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.004100:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.004105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.004113:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.004116:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.004116:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.004117:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.004119:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.004121:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.004125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.004132:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.004134:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.004135:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.004136:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.004136:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.004137:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.004165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.004171:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.004173:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.004175:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.004176:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.004178:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.004180:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.004181:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.004183:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.004184:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.004185:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.004188:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.004190:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.004193:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.004193:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.004194:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:4.2:1619677448.004195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677448.004196:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.004197:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.004197:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.004198:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677448.004203:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.004206:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.004206:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.004208:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.004209:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:26.0:1619677448.004211:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.004212:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:13.0:1619677448.004212:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.004213:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.004214:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.004215:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.004215:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.004216:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.004216:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:26.0:1619677448.004217:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:13.0:1619677448.004217:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677448.004221:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.004222:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.004225:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.004226:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.004226:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.004228:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.004229:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.004229:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.004230:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.004242:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.004245:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.004245:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.004246:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.004248:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.004251:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.004311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.004317:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.004320:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.004321:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.004323:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.004324:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.004326:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.004327:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.004329:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.004330:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.004331:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.004334:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.004336:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.004339:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.004340:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.004341:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.004342:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.004343:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.004343:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.004344:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.004356:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.004359:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.004360:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.004362:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.004363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:13.0:1619677448.004364:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.004365:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677448.004371:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.004374:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.004375:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.004377:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.004379:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.004381:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.004382:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.004383:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.004385:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.004385:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677448.004389:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.004391:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.004394:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.004395:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.004395:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.004397:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.004397:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.004399:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.004400:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.004400:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.2:1619677448.004403:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.004406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677448.004411:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:24.0:1619677448.004413:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:20.0:1619677448.004413:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.004414:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.004415:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.004416:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:20.0:1619677448.004416:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.004417:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677448.004417:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.004419:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.004419:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.004420:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.004421:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (23)++
00000800:00000200:5.0:1619677448.004421:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.004422:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.004423:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.004424:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.004606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.004613:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.004616:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.004616:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.004618:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.004621:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.004623:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.2:1619677448.004623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:18.0:1619677448.004624:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.004626:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.004626:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.004628:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.004631:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.004631:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677448.004633:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.004634:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.004635:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.2:1619677448.004635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677448.004636:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.004637:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.004638:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.004638:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.004640:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.004640:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.004641:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:10.0:1619677448.004641:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.004642:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:30.0:1619677448.004643:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677448.004643:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:10.0:1619677448.004643:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.004644:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.004645:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.004646:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:10.0:1619677448.004646:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.004648:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.004648:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:5.2:1619677448.004648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677448.004650:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.004650:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.004651:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.004651:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.004653:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.004653:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.004653:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.004653:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:30.0:1619677448.004654:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.004655:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.004655:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:30.0:1619677448.004656:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.004656:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:10.0:1619677448.004656:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677448.004657:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.004657:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677448.004657:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:10.0:1619677448.004658:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677448.004658:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:30.0:1619677448.004659:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677448.004659:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.004659:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.004660:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.004660:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.004661:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.004661:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.004661:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.004663:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.004663:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.004664:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.004665:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677448.004667:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:30.0:1619677448.004667:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.004668:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.004669:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677448.004671:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677448.004673:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677448.004674:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.004676:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.004677:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677448.004678:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677448.004678:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677448.004679:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.004679:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677448.004680:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.004682:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.004696:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.004699:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.004699:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.004700:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.004701:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.004703:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.004779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.004786:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.004788:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.004789:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.004791:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.004794:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.004795:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.004797:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.004798:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.004799:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.004800:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.004804:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.004806:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.004809:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.004810:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.004811:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.004813:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.004813:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.004814:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.004815:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:26.0:1619677448.004825:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.004827:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.004828:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.004829:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.004831:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.004834:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677448.004869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677448.004871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.004876:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.004878:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.004879:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.004881:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.004881:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.004882:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.004883:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.004884:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.004884:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.004885:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.004886:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.004887:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.004888:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.004888:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677448.004889:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.004889:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.004890:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.004890:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.004891:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.004892:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677448.004894:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.004896:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.004896:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.004898:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.004900:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.004901:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677448.004901:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.004901:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.004902:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:31.0:1619677448.004903:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.004903:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.004904:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.004905:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.004905:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.004905:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.004905:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.004906:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.004906:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.004911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677448.004918:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.004920:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:18.0:1619677448.004920:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677448.004921:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.004922:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.004923:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:22.0:1619677448.004924:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.004924:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.004924:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.004926:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.004927:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677448.004930:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.004937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.004942:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.004945:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.004946:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.004948:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.004951:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.004952:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677448.004954:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.004955:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.004956:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.004957:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.004960:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.004962:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.004965:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.004966:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.004966:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.004968:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.004969:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.004970:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.004970:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677448.004974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.004983:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.004985:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.004986:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.004988:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.004988:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.004991:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.005063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.005066:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.005068:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.005071:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.005071:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.005073:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.005074:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677448.005076:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.005077:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.005078:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.005079:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.005082:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.005084:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.005087:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.005088:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.005088:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.005090:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.005091:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.005091:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.005092:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:4.2:1619677448.005095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.2:1619677448.005096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.005103:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.005104:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677448.005106:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:1.0:1619677448.005106:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677448.005107:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677448.005107:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.005107:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.005110:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.005110:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000400:00000200:8.0:1619677448.005112:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.005113:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:1.0:1619677448.005113:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:8.0:1619677448.005115:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720629] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.005116:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.005117:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.005118:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.005122:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.005123:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.2:1619677448.005125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.0:1619677448.005127:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.005127:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.005128:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.005130:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.005130:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.005131:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.005132:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:30.0:1619677448.005133:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:8.2:1619677448.005135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677448.005136:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.005138:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.005140:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.005141:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.005142:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.005143:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.005143:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:30.0:1619677448.005145:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.005145:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:30.0:1619677448.005146:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.005146:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:30.0:1619677448.005147:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677448.005147:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.005149:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.005151:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.005153:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677448.005154:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.005157:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.005158:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.005158:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.005160:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.005161:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.005162:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.005162:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.005166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.005174:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.005177:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.005178:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.005179:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.005180:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.005182:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.005315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.005321:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.005324:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.005325:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.005327:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.005329:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.005330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:1.0:1619677448.005331:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.005332:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.005334:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.005335:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.005336:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:1.0:1619677448.005336:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:18.0:1619677448.005339:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.005340:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:1.0:1619677448.005340:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.005342:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:18.0:1619677448.005343:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.005345:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.005345:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.005346:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.005346:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:18.0:1619677448.005347:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677448.005348:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.005348:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:18.0:1619677448.005349:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.005349:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677448.005350:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.005350:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:18.0:1619677448.005351:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:1.0:1619677448.005351:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.005352:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677448.005354:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.005355:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.2:1619677448.005357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.005358:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.005358:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677448.005359:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.005361:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677448.005361:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.005362:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:17.0:1619677448.005362:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677448.005363:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:18.0:1619677448.005363:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.005364:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:17.0:1619677448.005364:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.005364:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.005365:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:17.0:1619677448.005365:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677448.005366:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.005367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677448.005368:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.005369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:30.0:1619677448.005370:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.005372:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:2.0:1619677448.005374:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:20.0:1619677448.005375:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677448.005376:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.005377:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.0:1619677448.005378:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.005378:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.005378:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:20.0:1619677448.005380:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.005382:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.005383:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:30.0:1619677448.005388:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.005391:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.005392:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.005393:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.005393:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.005394:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.005466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.005472:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.005474:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.005476:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.005477:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.005479:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.005481:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.005482:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.005483:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.005484:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.005485:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.005489:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.005492:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.0:1619677448.005494:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.005497:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.005498:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677448.005498:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.005499:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.005501:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:5.0:1619677448.005501:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.005501:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.005501:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677448.005502:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.005502:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677448.005503:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.005505:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.005507:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.005508:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.005509:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.005510:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.005511:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.005514:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:18.0:1619677448.005515:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.005516:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677448.005517:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:13.0:1619677448.005517:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.005519:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.005520:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.005521:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.005521:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.005522:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.005523:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677448.005524:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.005525:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.005526:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.005526:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677448.005537:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.005540:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.005540:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.005542:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.005543:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.005545:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677448.005554:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.005557:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.005557:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.005558:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.005558:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.005559:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.005620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.005628:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.005631:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.005632:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.005634:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.005638:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.005639:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.005640:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.005642:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.005643:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.005644:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.005647:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.005648:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.005651:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.005652:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.005652:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.005654:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.005654:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.005655:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.005657:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.005657:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.005663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677448.005666:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.005669:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.005669:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.005670:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:13.0:1619677448.005671:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.005672:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.005672:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:12.0:1619677448.005673:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.005674:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.005674:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.2:1619677448.005768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.005773:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.005776:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.005777:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:21.2:1619677448.005778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:1.0:1619677448.005779:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.005782:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.005783:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:6.0:1619677448.005784:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:1.0:1619677448.005784:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720622] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.005786:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.005787:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.005787:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:1.0:1619677448.005787:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.005788:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:6.0:1619677448.005791:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.005793:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.005794:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.005795:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:1.0:1619677448.005795:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:6.0:1619677448.005796:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.005798:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.005798:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.005799:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.005799:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:6.0:1619677448.005800:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:1.0:1619677448.005800:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.005802:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.005802:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.005803:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.005803:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.005804:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.005806:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.005809:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.005809:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.005810:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.005812:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.005812:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.005813:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677448.005814:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:6.0:1619677448.005814:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677448.005817:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.005818:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.005820:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.005821:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:21.2:1619677448.005822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:12.0:1619677448.005823:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.005824:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.005826:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.005827:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.005827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:10.0:1619677448.005827:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:24.0:1619677448.005829:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:26.0:1619677448.005830:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.005832:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:24.0:1619677448.005832:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:2.0:1619677448.005832:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.0:1619677448.005833:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:2.0:1619677448.005835:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:24.0:1619677448.005836:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.005836:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:24.0:1619677448.005838:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.005838:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.005839:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.005841:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.005841:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.005842:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.005842:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.005843:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.005844:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000400:00000200:2.0:1619677448.005844:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.005845:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.005845:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.005847:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.005847:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677448.005847:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.005848:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.005849:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.005849:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.005850:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:4.2:1619677448.005850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677448.005852:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.005852:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.005853:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:10.0:1619677448.005854:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677448.005855:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.005855:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.005856:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.005857:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677448.005857:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.005857:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.005858:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:10.0:1619677448.005858:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.005858:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.005859:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.005859:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.005860:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.005861:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:10.0:1619677448.005861:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.005861:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:24.0:1619677448.005862:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:2.0:1619677448.005862:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:24.0:1619677448.005863:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.2:1619677448.005863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:10.0:1619677448.005863:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:2.0:1619677448.005863:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:10.0:1619677448.005864:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.005866:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677448.005866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.2:1619677448.005866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:10.0:1619677448.005867:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.005869:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.005871:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.005872:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677448.005873:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.005874:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:18.0:1619677448.005874:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:26.0:1619677448.005875:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.005875:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.005876:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:18.0:1619677448.005876:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:26.0:1619677448.005877:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.005877:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.005878:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.005878:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.005878:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.005879:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:26.0:1619677448.005880:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:10.0:1619677448.005880:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.005880:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:20.0:1619677448.005881:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.005881:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.005882:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96720617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.005882:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:18.0:1619677448.005883:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:26.0:1619677448.005884:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.005885:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.005885:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:26.0:1619677448.005886:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.005887:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.005888:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.005889:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000400:00000200:26.0:1619677448.005890:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.005890:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.005892:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.005892:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:31.0:1619677448.005894:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:26.0:1619677448.005895:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677448.005895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.005896:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.005896:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.005897:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.005898:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.005898:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:31.0:1619677448.005899:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.005899:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.005900:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:30.0:1619677448.005901:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.005902:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.005902:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.005903:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.005904:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:8.2:1619677448.005904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.005905:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.005906:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.005907:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.2:1619677448.005908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:5.0:1619677448.005911:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.005914:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:26.0:1619677448.005915:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:5.0:1619677448.005915:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.005917:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:20.0:1619677448.005917:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.005917:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.005918:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.005919:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:5.0:1619677448.005920:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.005962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.005969:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.005972:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.005974:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.005976:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.005977:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.005979:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677448.005980:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720616] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.005981:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.005982:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.005983:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677448.005987:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.005988:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.005991:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.005992:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677448.005992:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.005994:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677448.005994:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.005995:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.005996:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.006007:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.006010:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.006011:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.006012:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.006014:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.006016:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.006181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.006187:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.006189:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.006190:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.006192:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.006196:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.006197:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.006198:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.006200:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.006201:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.006201:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.006204:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.006206:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.006208:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.006209:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.006210:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.006211:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.006212:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.006213:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.006213:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.006225:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.006227:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.006228:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.006228:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.006230:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.006233:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.006369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.006372:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.006375:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.006377:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.006377:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.006379:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.006381:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.006382:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.006384:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.006385:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.006386:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.006389:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.006391:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.006393:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.006394:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.006395:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.006396:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.006397:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.006398:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.006399:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.006400:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.006405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677448.006411:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:21.2:1619677448.006411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.0:1619677448.006412:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:22.0:1619677448.006414:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.006415:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.006415:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.006415:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:22.0:1619677448.006416:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.006416:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.006416:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677448.006419:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677448.006420:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.006423:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.006425:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.006427:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.006427:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.006429:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.006430:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.006431:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.006432:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.006433:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.006436:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.006438:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.006440:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.006441:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.006442:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.006443:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.006444:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.006445:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.006447:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.006447:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677448.006453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.006456:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.006458:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.006459:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.006459:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.006461:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:10.0:1619677448.006461:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.006463:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.006464:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677448.006464:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:30.0:1619677448.006465:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677448.006474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.006481:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.006483:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.006484:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.006486:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.006488:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.006490:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.006491:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.006492:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.006493:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.006494:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.006497:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.006498:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.006500:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.006501:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.006502:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.006503:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.006504:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.006504:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.006504:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.006516:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.006518:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.006519:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.006520:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.006522:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.006523:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.006599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.006606:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.006609:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.006609:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.006612:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.006614:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.006615:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.006616:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.006618:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.006619:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.006620:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.006623:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.006625:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.006628:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.006629:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.006629:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.006631:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.006632:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.006632:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.006633:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.006645:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.006647:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.006648:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.006650:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.006651:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.006653:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.006659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.006666:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.006669:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.006670:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.006672:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.006674:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.006676:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.006677:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.006678:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.006679:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.006680:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.006684:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.006686:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.006688:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.006689:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.006690:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.006692:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.006693:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.006693:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.006694:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.006698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.006706:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.006709:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.006710:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.006711:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.006712:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.006716:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.006720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.006728:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.006730:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.006731:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.006731:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.006732:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.006732:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.006774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.006779:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.006782:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.006782:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.006784:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.006787:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.006789:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.006790:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.006791:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.006792:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.006793:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.006797:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.006799:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.006802:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.006803:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.006804:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.2:1619677448.006804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:6.0:1619677448.006806:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.006806:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.006807:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.006808:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.006808:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677448.006809:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.006812:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.006813:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:21.2:1619677448.006814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677448.006815:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.006816:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.006818:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:31.0:1619677448.006819:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677448.006819:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:2.0:1619677448.006819:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.006820:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.006821:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:2.0:1619677448.006821:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.006822:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:22.0:1619677448.006822:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677448.006822:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.006823:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.006823:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:31.0:1619677448.006825:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.006825:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:2.0:1619677448.006827:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.006828:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:2.0:1619677448.006829:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.006832:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.006833:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.006833:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.006835:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.006836:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.006836:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.006837:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.006842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.006849:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.006852:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.006853:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.006853:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.006855:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.006858:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.006864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.2:1619677448.006868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677448.006871:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.006873:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.006874:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.006874:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:30.0:1619677448.006875:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:1.0:1619677448.006875:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.006876:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677448.006878:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.006880:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.006882:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.006882:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.006884:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.006885:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.006886:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.006887:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.006888:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.006891:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.006893:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.006896:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.006897:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.006897:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.006899:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.006900:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.006900:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.006902:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.006903:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.006905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.006908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.006913:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:31.0:1619677448.006916:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:5.0:1619677448.006916:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.006917:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.006918:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.006919:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.006919:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677448.006919:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.006920:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.006921:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.006925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.2:1619677448.006932:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677448.006936:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:5.0:1619677448.006938:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.006940:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.006940:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.006941:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:5.0:1619677448.006941:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.006942:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.006942:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.006943:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:5.0:1619677448.006943:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:20.0:1619677448.006944:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.006944:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.006946:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.006946:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.006947:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:20.0:1619677448.006948:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.006949:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677448.006950:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.006951:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.006952:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.006952:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.006953:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:20.0:1619677448.006954:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.006955:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.006957:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.006957:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.006958:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.006959:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.006960:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.006963:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.006965:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.006967:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.006968:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.006968:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.006969:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.006969:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.006970:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.006971:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.006971:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.006972:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.006972:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:10.0:1619677448.006972:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.006972:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.006973:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.006974:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.006977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677448.006983:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.006985:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.006985:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:12.0:1619677448.006986:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.006988:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.006988:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:13.0:1619677448.006989:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677448.006989:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.006990:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677448.006991:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:12.0:1619677448.006991:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.006993:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.007013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.007019:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.007022:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.007023:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.007025:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.007027:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.007029:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.007030:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.007031:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.007033:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.007034:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.007037:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.007038:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.007041:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.007042:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.007043:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.007045:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.007045:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.007046:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.007048:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.007048:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:21.2:1619677448.007051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.2:1619677448.007053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.007058:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:31.0:1619677448.007060:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.007060:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677448.007060:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:13.0:1619677448.007061:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.007062:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:31.0:1619677448.007063:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:13.0:1619677448.007063:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.007063:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.007064:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677448.007065:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.007065:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.007066:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.007067:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.007068:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.007069:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.007071:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.007072:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.007073:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.007074:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.007076:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:31.0:1619677448.007077:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.007079:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.007079:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.007080:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.007081:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.007081:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.007082:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.007083:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.007083:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.007083:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.007085:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.007086:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.007086:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.007087:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.007096:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.007099:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.007100:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.007101:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.007103:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.007104:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.007118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.007125:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.007128:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.007128:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.007130:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.007133:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.007135:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.007136:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720602] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.007137:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.007138:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.007139:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.007142:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.007143:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.007146:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.007147:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.007147:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.007149:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.007150:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.007150:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.007151:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.007155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.007162:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.007165:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.007166:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.007167:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.007169:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.007171:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.007193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.007200:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.007203:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.007204:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.007206:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.007209:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.007210:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.007212:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.007213:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.007214:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.007215:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.007218:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.007219:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.007222:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.007223:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.007224:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.007225:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.007226:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.007227:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.007227:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.007231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.007239:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.007241:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.007242:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.007242:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.007244:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.007247:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.007425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.007430:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.007433:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.007434:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.007436:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.007439:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.007441:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.007442:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.007443:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.007444:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.007445:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.007449:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.007450:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.007453:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.007454:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.007454:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.007456:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.007456:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.007457:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.007457:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.007463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.007468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677448.007471:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.007474:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.007475:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.007475:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.007477:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677448.007477:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.007479:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:18.0:1619677448.007480:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:30.0:1619677448.007481:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.007483:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.007484:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.007485:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.007486:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.007487:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.007488:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.007489:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.007492:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.007494:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.007497:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.007497:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.007498:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.007499:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.007500:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.2:1619677448.007500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.007501:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.007502:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677448.007507:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.007509:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.007510:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677448.007512:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677448.007512:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.007514:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.007515:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.007516:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.007516:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:12.0:1619677448.007518:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.007518:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.007519:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.007520:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.007520:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677448.007520:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.007521:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.007525:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.007526:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.007529:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.007530:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.007530:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.007532:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.007533:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.007533:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.007534:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.007547:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.007550:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.007550:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.007551:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.007553:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.007555:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677448.007565:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.007567:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.007568:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.007568:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.007569:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.007570:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.007611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.007617:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.007620:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.007620:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.007622:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.007625:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.007626:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.007627:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.007629:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.007630:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.007631:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.007634:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.007635:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:4.2:1619677448.007636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677448.007638:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.007639:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.007639:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.007641:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.007641:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.007642:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.007642:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:10.0:1619677448.007644:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.007647:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.007648:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.007648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:10.0:1619677448.007650:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.007653:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.007654:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:17.0:1619677448.007655:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:10.0:1619677448.007656:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.007657:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.007658:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:10.0:1619677448.007658:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.007658:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.007659:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.007659:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.007661:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.007663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:10.0:1619677448.007663:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.007664:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677448.007665:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.007668:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.007669:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.007670:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.007671:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.007671:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.007672:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.007672:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.007673:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.007674:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.007676:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.007678:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.007678:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.007680:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.007681:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.007682:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.007683:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.007684:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:22.0:1619677448.007685:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:31.0:1619677448.007687:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.007687:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.007688:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677448.007689:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:22.0:1619677448.007689:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.007691:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.007692:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.007692:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.007693:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677448.007694:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677448.007695:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.007695:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.007696:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.007697:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.007701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.007708:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.007711:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.007711:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.007713:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.007714:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.007716:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.007738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677448.007745:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677448.007748:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677448.007750:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677448.007751:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.007754:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.007756:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677448.007757:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.007758:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.007759:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.007760:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677448.007763:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.007765:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677448.007768:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.007769:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677448.007769:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677448.007771:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677448.007771:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677448.007772:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677448.007773:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:6.0:1619677448.007785:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.007787:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.007788:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.007788:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.007791:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.007794:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.007809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.007816:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.007819:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.007822:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.007823:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.007824:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.007825:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.007827:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.007828:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.007829:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.007830:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.007833:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.007834:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.007836:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.007837:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.007837:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.007839:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.007839:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.007840:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.007841:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.007851:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.007853:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.007854:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.007855:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.007857:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.007858:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.007869:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.007872:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.007873:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.007873:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.007874:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.007875:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.007990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.007996:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.007999:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.008000:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.008001:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.008005:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.008006:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.008007:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.008008:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.008009:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.008010:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.008013:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.008015:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.008017:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.008018:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.008019:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.008020:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.008021:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.008022:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.008023:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.008024:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.008026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.008029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.008034:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.2:1619677448.008036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.0:1619677448.008036:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.008036:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.008037:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.008038:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677448.008038:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.008039:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677448.008039:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.008040:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.008040:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.008041:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.008043:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.008043:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677448.008044:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.008046:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.008047:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.008049:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.008050:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.008051:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.008052:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.008056:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.008057:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.008060:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.008061:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.008062:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.008064:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.008064:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.008065:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.008066:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.2:1619677448.008067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.008073:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.008075:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677448.008076:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:1.0:1619677448.008076:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.008078:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.008079:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.008080:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.008080:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.008082:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677448.008083:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.008083:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:17.0:1619677448.008083:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.008085:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.008086:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:17.0:1619677448.008086:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.008087:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:4.2:1619677448.008088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:17.0:1619677448.008090:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.008092:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.008094:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.008094:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.008095:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.008095:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.008096:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.008096:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.008097:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.008097:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.008098:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.008098:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:21.2:1619677448.008098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:17.0:1619677448.008098:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.008099:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:10.0:1619677448.008099:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.008100:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.008101:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:26.0:1619677448.008101:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.008102:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.2:1619677448.008102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677448.008104:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.008104:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:26.0:1619677448.008106:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.008107:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.008107:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.008107:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:26.0:1619677448.008109:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.008110:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.008110:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.008110:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:26.0:1619677448.008111:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:5.2:1619677448.008111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:22.0:1619677448.008112:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.008112:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:22.0:1619677448.008113:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:1.0:1619677448.008113:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.008114:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:22.0:1619677448.008115:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.008115:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.008117:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.008117:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:26.0:1619677448.008119:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.008119:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.008120:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:10.0:1619677448.008120:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.2:1619677448.008120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:10.0:1619677448.008121:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.008123:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:22.0:1619677448.008125:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.008126:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.008126:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.008127:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:17.0:1619677448.008127:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.008128:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677448.008128:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:10.0:1619677448.008129:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.008130:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:22.0:1619677448.008131:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.008131:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.008131:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.008132:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.008132:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:17.0:1619677448.008132:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (21)++
00000400:00000200:10.0:1619677448.008132:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:17.0:1619677448.008133:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.008133:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:10.0:1619677448.008133:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720587] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.008134:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000400:00000200:10.0:1619677448.008134:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.008135:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677448.008135:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.008136:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.008136:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677448.008136:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.2:1619677448.008136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.008137:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.008139:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.008140:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:9.2:1619677448.008140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:26.0:1619677448.008141:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.008141:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.008142:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.008143:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.008144:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:10.0:1619677448.008146:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.008147:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.008147:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:4.2:1619677448.008147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.008148:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:10.0:1619677448.008148:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677448.008148:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.008149:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.008150:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:10.0:1619677448.008150:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.008151:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.008151:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677448.008152:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:18.0:1619677448.008152:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.008152:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.008153:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.0:1619677448.008153:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:18.0:1619677448.008154:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.008155:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.008156:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.008156:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:9.2:1619677448.008156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.008157:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:18.0:1619677448.008159:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:18.0:1619677448.008161:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.008163:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.008164:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.008164:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.008166:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.008169:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.008274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.008282:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.008285:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.2:1619677448.008285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:30.0:1619677448.008287:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.008289:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.008289:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.008290:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677448.008291:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.008292:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.008292:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.008298:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.008299:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.008300:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.008303:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.008304:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.008306:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.008306:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.008308:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.008309:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.008309:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.008310:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.008310:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.008311:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.008312:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.008312:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.008312:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.008313:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:31.0:1619677448.008315:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.008315:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.008316:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.008317:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.008317:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.008318:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.008319:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.008320:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.008320:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.008322:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.008322:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.008323:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:21.2:1619677448.008323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.008325:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677448.008326:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677448.008328:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.008328:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677448.008329:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:13.0:1619677448.008329:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.008330:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:13.0:1619677448.008330:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.008331:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.008332:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.008333:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677448.008334:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.008337:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.008338:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.008339:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.008341:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.008341:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.008407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.008415:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.008417:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.008418:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.008420:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.008423:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.008425:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.008426:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.008427:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.008428:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.008429:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.008431:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.008433:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.008434:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.008435:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.008436:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.008437:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.008437:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.008438:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.008439:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.008449:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.008450:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.008451:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.008452:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.008453:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.008454:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.008686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.008693:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.008695:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.008696:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.008697:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.008697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:26.0:1619677448.008699:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.008701:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.008702:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.008704:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.008705:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.008705:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.008706:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.008708:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677448.008709:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.008710:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.008711:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.008712:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.008714:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.008714:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:26.0:1619677448.008715:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:10.0:1619677448.008715:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.008716:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:10.0:1619677448.008717:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.008718:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:10.0:1619677448.008718:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.008719:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:10.0:1619677448.008719:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.008720:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.008720:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.008721:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.008722:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.008725:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.008727:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.008728:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.008728:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.008730:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.008730:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.008731:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.008731:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677448.008732:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.008735:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.008736:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.008736:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.008739:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.008742:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:6.0:1619677448.008742:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677448.008744:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.008745:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.008747:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.008748:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.008750:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:22.0:1619677448.008755:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.008758:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.008758:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.008759:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.008760:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.008760:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677448.008761:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.008764:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.008765:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.008766:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.008767:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.008768:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.008777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.008784:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.008787:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.008789:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.008790:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.008791:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.008792:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677448.008793:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.008795:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.008796:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.008796:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677448.008800:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.008801:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.008804:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.008805:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677448.008805:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.008807:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677448.008808:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.008808:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.008809:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677448.008810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.008817:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.008820:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:6.0:1619677448.008820:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.0:1619677448.008821:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.008822:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.008823:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.008824:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.008824:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.008825:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.008827:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.008827:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.008829:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96720580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.008830:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.008830:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:24.0:1619677448.008831:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.008832:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.008835:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.008836:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.008839:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.008839:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.008840:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.008842:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.008842:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.008843:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.008844:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.008848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.008856:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.008859:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.008860:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.008860:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.008862:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.008865:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.008875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.008880:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.008883:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.008883:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.008885:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.008886:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.008888:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.008889:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.008890:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.008891:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.008892:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.008895:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.008897:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.008900:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.008901:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.008902:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.008903:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.008904:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.008904:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.008905:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.008917:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.008919:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.008920:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.008922:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.008923:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.008924:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.008981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.008987:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.008990:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.008991:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.008993:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.008995:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.008996:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677448.008997:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720578] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.008999:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.009000:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.009001:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:4.2:1619677448.009003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677448.009004:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.009006:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.009007:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.009008:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.2:1619677448.009009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:2.0:1619677448.009009:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:26.0:1619677448.009010:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.009010:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.009011:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.009011:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:26.0:1619677448.009012:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:20.0:1619677448.009012:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:2.0:1619677448.009012:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677448.009013:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677448.009015:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:9.2:1619677448.009016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:26.0:1619677448.009017:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.009018:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.009018:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:26.0:1619677448.009019:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.009020:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.009021:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.009022:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.009023:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.009024:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.009024:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.009024:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:26.0:1619677448.009025:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.009025:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:26.0:1619677448.009026:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.009026:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.009026:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.009027:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.009027:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:20.0:1619677448.009029:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.009029:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.009031:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:26.0:1619677448.009033:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.009033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000400:00000200:20.0:1619677448.009034:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.009035:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.009037:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.009038:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.009039:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.009040:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.009040:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.009040:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:26.0:1619677448.009041:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.009041:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.009041:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.009042:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.009043:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677448.009043:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.009043:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.009043:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:26.0:1619677448.009044:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677448.009044:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:2.0:1619677448.009044:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:20.0:1619677448.009045:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:2.0:1619677448.009045:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677448.009046:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:2.0:1619677448.009046:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.0:1619677448.009047:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.2:1619677448.009052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:31.0:1619677448.009054:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.009057:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:10.0:1619677448.009057:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:31.0:1619677448.009058:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.009058:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000400:00000200:31.0:1619677448.009060:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.009060:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:5.0:1619677448.009060:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:30.0:1619677448.009061:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.009061:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:5.0:1619677448.009061:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:10.0:1619677448.009062:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.009062:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677448.009063:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:10.0:1619677448.009064:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:10.0:1619677448.009067:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.009164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.009172:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.009175:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.009177:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.009179:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.009180:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.009181:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.009182:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.009184:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.009185:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.009186:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.009189:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.009191:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.009193:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.009194:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.009195:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.009196:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.009197:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:21.2:1619677448.009197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:30.0:1619677448.009198:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.009198:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:18.0:1619677448.009203:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.009206:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.009207:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:31.0:1619677448.009208:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:18.0:1619677448.009209:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.009211:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.009211:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.009211:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.009212:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.009213:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.009214:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.009215:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.009215:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:18.0:1619677448.009215:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.009217:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.009218:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.009222:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.009224:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.009228:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.009229:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.009229:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.009232:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.009232:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.009233:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.009234:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.2:1619677448.009243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.009246:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.009249:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.009250:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.009251:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:26.0:1619677448.009252:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.009253:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.009255:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.009256:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.009257:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.009258:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.009258:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.009260:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.009260:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.009260:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:26.0:1619677448.009261:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.009261:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720573] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.009262:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.009263:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:10.0:1619677448.009263:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.009264:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.009265:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.009269:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.009271:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.009274:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.009274:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.009275:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.009277:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.009277:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.009278:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.009278:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.009289:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.009292:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.009292:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.009298:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.009299:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.009301:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.009553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.009560:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.009563:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.009564:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.009565:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.009568:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.009569:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.009570:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720572] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.009572:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.009572:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.009573:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.009577:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.009578:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.009581:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.009582:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.009582:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.009584:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.009584:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.009586:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.009587:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.009587:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.009593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.009599:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.009600:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.009601:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677448.009602:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677448.009602:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.009603:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.009603:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.009603:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.009605:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.009608:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.009699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:6.0:1619677448.009704:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677448.009706:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677448.009707:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:6.0:1619677448.009709:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.009711:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.009713:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:6.0:1619677448.009714:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.009715:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.009716:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.009717:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:6.0:1619677448.009720:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.009721:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677448.009724:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.009725:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677448.009725:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.009727:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677448.009728:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677448.009728:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677448.009729:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677448.009733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.009741:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.2:1619677448.009742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.009743:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.009744:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.009745:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.009746:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.009749:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.009750:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677448.009751:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.009752:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.009754:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.009757:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.009758:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.009760:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.009761:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.009762:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.009763:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.009767:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.009769:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.009771:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.009772:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.009773:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.009775:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.009775:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.009776:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.009777:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.009778:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.009783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.009787:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.009788:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677448.009790:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677448.009791:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.009791:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677448.009791:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.009792:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677448.009792:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.009794:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.009795:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.009816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:5.2:1619677448.009817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.009824:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677448.009824:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677448.009827:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:8.0:1619677448.009827:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.009828:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:10.0:1619677448.009828:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.009831:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.009831:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.009832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:26.0:1619677448.009834:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.009834:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.009836:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677448.009836:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:18.0:1619677448.009837:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:26.0:1619677448.009838:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.009838:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.009840:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.009840:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:8.0:1619677448.009840:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.009841:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.009841:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:8.0:1619677448.009842:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.009843:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.2:1619677448.009843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:8.0:1619677448.009843:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.009844:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.009846:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.009848:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.009848:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.009849:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.009849:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.009851:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.009851:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.009851:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.009852:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.009852:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.009853:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.009854:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:26.0:1619677448.009854:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.009855:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677448.009855:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.009856:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677448.009856:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.009856:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677448.009857:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.009857:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:18.0:1619677448.009857:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.009858:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677448.009858:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677448.009858:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:30.0:1619677448.009859:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.009859:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677448.009859:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.009859:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677448.009860:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.009860:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677448.009860:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.009860:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:30.0:1619677448.009861:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:18.0:1619677448.009862:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.009863:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96720566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.009863:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.009864:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.2:1619677448.009864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677448.009865:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.009866:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.009866:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.009866:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677448.009866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:30.0:1619677448.009867:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.009867:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.009868:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.009869:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:12.0:1619677448.009869:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:30.0:1619677448.009870:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.009871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.009872:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:12.0:1619677448.009872:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677448.009872:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:12.0:1619677448.009873:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.009873:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:12.0:1619677448.009874:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.009874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:1.0:1619677448.009874:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:30.0:1619677448.009875:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.009875:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:17.0:1619677448.009875:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:10.0:1619677448.009875:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:30.0:1619677448.009876:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:13.0:1619677448.009876:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.009876:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:1.0:1619677448.009876:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.009877:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:1.0:1619677448.009877:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.009878:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.009878:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.009878:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.009879:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:30.0:1619677448.009880:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.009881:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:18.0:1619677448.009881:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.009881:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:1.0:1619677448.009881:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677448.009882:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:2.0:1619677448.009882:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.009883:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.009883:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.009884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:2.0:1619677448.009884:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.009885:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.009885:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677448.009886:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677448.009887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677448.009889:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.009892:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.009892:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:13.0:1619677448.009893:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.009894:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.009894:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677448.009895:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:13.0:1619677448.009895:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677448.009895:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.009896:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677448.009897:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.009897:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.009898:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.009899:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.009900:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.009900:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.009902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.009909:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.009912:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.009913:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.009914:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.009915:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.009915:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.010029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.010035:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.010037:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.010038:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.010040:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.010043:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.010044:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.010045:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.010047:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.010048:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.010049:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.010052:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.010054:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.010057:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.010058:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.010059:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.010060:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.010061:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.010062:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.010062:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:21.2:1619677448.010071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:10.0:1619677448.010072:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.010075:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.010076:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.010076:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.010076:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:10.0:1619677448.010079:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.010079:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.010080:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:10.0:1619677448.010082:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:2.0:1619677448.010082:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.010084:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.010085:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.010086:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.010088:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.010089:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.010090:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.010094:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.010096:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.010098:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.010099:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.010100:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.010102:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.010102:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.010103:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.010104:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:26.0:1619677448.010114:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.010117:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.010117:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.010118:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.010120:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.010123:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.010249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.010256:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.010258:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.010259:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.010261:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.010263:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.010265:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.010266:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.010267:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.010268:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.010269:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.010272:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.010274:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.010276:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.010277:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.010278:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.010279:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.010280:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.010280:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.010280:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.010291:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.010298:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.010299:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.010300:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.010301:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.010303:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.010395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.010404:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.010406:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.010407:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.010408:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.010411:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.010413:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:5.2:1619677448.010413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677448.010414:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.010415:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.010416:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.010417:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.010419:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.010420:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.010421:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.010422:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.010423:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.010424:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.010425:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.010425:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:17.0:1619677448.010425:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.010427:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.010427:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677448.010427:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.010428:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.2:1619677448.010428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:26.0:1619677448.010429:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:17.0:1619677448.010429:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.010430:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.010431:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.010432:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.010433:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:30.0:1619677448.010436:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.010436:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.010438:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677448.010439:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.010441:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.010441:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.010442:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:6.0:1619677448.010442:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:31.0:1619677448.010443:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677448.010443:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:30.0:1619677448.010444:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.010445:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:17.0:1619677448.010445:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.010445:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.010445:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:30.0:1619677448.010446:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.010446:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.010446:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.010446:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.010446:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.010449:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.010449:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.010450:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677448.010450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:30.0:1619677448.010451:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.010453:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:30.0:1619677448.010455:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.010457:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:24.0:1619677448.010458:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:21.2:1619677448.010458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:30.0:1619677448.010459:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.010460:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:24.0:1619677448.010460:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:30.0:1619677448.010461:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.0:1619677448.010461:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:30.0:1619677448.010462:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:24.0:1619677448.010462:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.010463:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.010464:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:24.0:1619677448.010464:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.010465:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:18.0:1619677448.010465:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:24.0:1619677448.010467:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:18.0:1619677448.010468:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.010468:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677448.010469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:18.0:1619677448.010470:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.010472:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.010474:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.010475:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.010477:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:18.0:1619677448.010477:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.010477:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.010479:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.010480:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.010480:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.010482:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000400:00000200:20.0:1619677448.010483:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.010483:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.010485:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.010486:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677448.010486:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.010487:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.010488:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.010490:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.010490:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.010491:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.010491:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.010496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.010503:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.010504:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.010505:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.010506:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.010507:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.010509:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.010516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.010524:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.010526:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.010527:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.010527:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.010528:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.010529:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.010536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.010543:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.010546:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.010547:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.010548:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.010551:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.010552:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.010554:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.010555:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.010556:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.010557:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.010560:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.010561:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.010565:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.010565:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.010566:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.010568:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.010569:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.010569:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.010570:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.010574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.010582:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.010585:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.010586:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.010586:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.010588:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.010591:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.010756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.010763:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.010766:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.010769:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.010770:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.010771:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.010773:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.010774:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.010775:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.010777:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.010777:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.010781:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.010782:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.010785:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.010785:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.010786:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.010787:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.010788:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.010789:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.010789:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.010799:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.010801:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:5.2:1619677448.010801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677448.010802:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.010803:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.010805:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.010806:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.010807:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:1.0:1619677448.010809:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.010810:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.010812:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.010815:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.010816:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.010817:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.010818:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.010819:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.010820:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.010824:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.010825:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.010828:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.010829:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.010830:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.010831:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.010832:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.010833:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.010833:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.010845:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.010848:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.010849:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.010851:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.010852:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.010853:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.010863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.010869:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.010871:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.010874:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.2:1619677448.010875:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:5.0:1619677448.010877:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.010880:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.010881:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:5.0:1619677448.010881:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.010882:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.010883:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:5.0:1619677448.010884:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677448.010884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677448.010884:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:5.0:1619677448.010886:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.010887:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.010887:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.010889:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.010891:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:26.0:1619677448.010892:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677448.010892:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.010893:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.010894:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:18.0:1619677448.010895:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.010895:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.010895:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.010896:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.010898:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.010898:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.010899:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.010899:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.010899:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.010901:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:26.0:1619677448.010902:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.010902:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.010902:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.010902:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.010903:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:26.0:1619677448.010904:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:18.0:1619677448.010904:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.010905:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.010907:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.010908:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.010908:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.010908:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:26.0:1619677448.010909:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.010909:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.010911:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:26.0:1619677448.010913:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.010914:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677448.010914:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:26.0:1619677448.010915:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.010915:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.010916:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:22.0:1619677448.010917:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.010918:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.010918:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:22.0:1619677448.010918:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.010918:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.010919:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:22.0:1619677448.010920:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.010921:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.010921:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.010922:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677448.010922:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:26.0:1619677448.010923:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:12.0:1619677448.010923:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.010926:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.010927:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.010927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:12.0:1619677448.010929:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.010930:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.010932:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.010934:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.010937:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.010938:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.010938:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.010941:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.010944:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.010977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.010984:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.010986:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.010987:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.010989:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.010992:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.010994:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.010996:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.010997:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.010998:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.010999:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677448.011002:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.011003:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.011006:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.011007:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.011008:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.011009:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.011010:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.011011:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.011012:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.011013:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.2:1619677448.011015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.011017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:21.2:1619677448.011020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:6.0:1619677448.011022:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677448.011024:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:6.0:1619677448.011025:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.011026:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677448.011026:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.011027:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.011028:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:6.0:1619677448.011028:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.011029:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.011030:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.011030:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677448.011030:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.011031:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:6.0:1619677448.011031:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.011032:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:6.0:1619677448.011032:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.011034:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.011035:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677448.011036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000400:00000200:6.0:1619677448.011036:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.011037:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.011040:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.011042:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.011044:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.011044:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:6.0:1619677448.011045:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.011046:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.011046:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.011047:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:6.0:1619677448.011048:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.011048:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677448.011048:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.011049:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.011050:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:6.0:1619677448.011050:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.011051:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.011051:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.011054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.011057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.011062:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:5.0:1619677448.011062:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.011064:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677448.011065:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.011065:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:5.0:1619677448.011065:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.011066:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:5.0:1619677448.011067:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.011068:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.011069:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.011142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.011147:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.011151:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.011151:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.011153:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.011156:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.011158:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.011159:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.011160:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.011161:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.011162:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.011165:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.011167:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.011169:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.011170:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.011171:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.011172:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.011172:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.011173:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.011173:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.011178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.011186:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.011188:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.011188:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.011189:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.011190:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.011191:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.011228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.011236:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.011239:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.011241:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.011242:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.011244:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.011245:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.011247:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.011248:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.011249:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.011250:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.011253:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.011255:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.011258:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.011258:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.011259:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.011261:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.011261:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.011262:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.011263:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.011267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.011274:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.011277:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.011278:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.011279:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.011281:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.011282:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.011322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.011330:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.011333:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.011334:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.011336:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.011339:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.011340:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.011341:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.011343:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.011344:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.011345:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.011348:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.011349:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.011351:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.011352:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.011352:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.011354:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.011354:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.011355:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.011355:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677448.011367:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.011370:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.011370:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.011372:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677448.011374:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.011375:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.011379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.011385:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.011387:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.011389:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.011390:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.011392:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.011394:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677448.011395:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720547] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.011397:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.011397:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.011399:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677448.011402:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.011404:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677448.011407:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.011408:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677448.011409:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677448.011411:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.2:1619677448.011413:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.2:1619677448.011414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.011416:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.011416:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.011417:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.011418:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:5.0:1619677448.011418:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:20.0:1619677448.011419:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.011419:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677448.011421:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:20.0:1619677448.011422:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.011424:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:26.0:1619677448.011424:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:20.0:1619677448.011424:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677448.011424:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:20.0:1619677448.011426:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.011427:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.011428:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.011428:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.011429:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.011429:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.011430:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.011430:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.011431:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:26.0:1619677448.011431:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.011433:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:26.0:1619677448.011434:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:20.0:1619677448.011434:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.011435:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.011437:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.011437:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.011438:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.011439:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677448.011441:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.011442:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.011442:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:26.0:1619677448.011444:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.011444:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.011445:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.011446:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.0:1619677448.011446:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.011447:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:26.0:1619677448.011448:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.011448:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677448.011449:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.011450:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.011451:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.011452:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.011452:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.011453:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677448.011453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677448.011456:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.011458:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677448.011459:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.011460:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.011460:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.011461:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:12.0:1619677448.011462:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.011462:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.011463:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.011464:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.011466:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.011469:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.011470:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.011470:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.011472:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.011475:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.011483:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.011486:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.011486:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.011487:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.011488:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.011488:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.011510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.011517:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.011519:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.011520:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.011522:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.011525:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.011526:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.011528:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.011530:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.011531:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.011532:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.011535:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.011536:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.2:1619677448.011537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:10.0:1619677448.011539:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.011540:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.011541:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.011542:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.011543:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.011543:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.011544:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:24.0:1619677448.011545:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.011547:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.011549:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.011550:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.011553:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.011554:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.011555:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96720543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.011555:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:24.0:1619677448.011557:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.011557:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.011558:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677448.011558:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.011559:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.011559:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.011562:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.011563:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.011565:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677448.011565:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677448.011567:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.011568:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.011569:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.011571:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.011572:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.011572:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.011573:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.011578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.011586:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.011588:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.011589:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.011590:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.011592:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.011595:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.011645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.011651:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.011653:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.011654:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.011656:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.011658:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.011659:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.011661:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.011662:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.011663:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.011664:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.011667:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.011669:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.011671:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.011672:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.011672:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.011674:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.011675:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.011676:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.011677:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.011677:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.011682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:4.2:1619677448.011686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677448.011688:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.011688:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.011690:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.011690:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677448.011691:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:21.2:1619677448.011691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677448.011691:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677448.011692:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.011693:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677448.011693:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.011694:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.011696:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.011696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:8.0:1619677448.011696:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.011697:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677448.011698:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:8.0:1619677448.011699:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.011701:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:8.0:1619677448.011702:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.011702:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000400:00000200:30.0:1619677448.011703:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.011704:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:5.0:1619677448.011704:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:31.0:1619677448.011705:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:8.0:1619677448.011705:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.011706:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.011707:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:30.0:1619677448.011708:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:8.0:1619677448.011708:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.011708:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.011709:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.011709:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.011710:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.011711:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.011711:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.011711:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.011712:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.011712:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:5.0:1619677448.011714:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.011716:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.011716:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.011717:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.011718:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.011719:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.011719:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.011721:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.0:1619677448.011722:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.011723:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.011723:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677448.011723:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.011724:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.011724:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.011725:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:8.0:1619677448.011725:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.011726:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:8.0:1619677448.011726:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.011726:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677448.011726:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.011727:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677448.011727:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:30.0:1619677448.011728:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.011728:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677448.011728:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.011729:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.011729:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.2:1619677448.011731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.011731:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.011731:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:5.0:1619677448.011732:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:5.0:1619677448.011732:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677448.011737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.011738:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:17.0:1619677448.011739:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:26.0:1619677448.011741:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:17.0:1619677448.011741:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:26.0:1619677448.011742:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.011742:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:1.0:1619677448.011742:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.011743:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.011745:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:26.0:1619677448.011746:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.011746:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.011747:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.011748:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.011749:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:20.0:1619677448.011750:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.011750:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677448.011750:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.011752:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.011812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.011818:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.011820:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.011823:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.011823:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.011825:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.011827:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.011828:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.011829:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.011830:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.011831:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.011835:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.011836:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.011839:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.011839:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.011840:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.011841:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.011842:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.011842:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.011843:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677448.011855:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.011857:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.011858:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.011858:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.011861:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.011862:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677448.011871:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.011874:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.011875:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.011875:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.011876:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.011877:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.011974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677448.011980:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.011983:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677448.011983:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677448.011985:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.011987:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.011989:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677448.011990:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.011992:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.011992:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.011994:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677448.011997:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.011999:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.012001:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.012002:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:1.0:1619677448.012003:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677448.012005:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677448.012005:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677448.012006:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677448.012007:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677448.012008:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:6.0:1619677448.012018:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.012021:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.012022:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.012022:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.012024:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.012026:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.012071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:5.2:1619677448.012074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.012078:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677448.012079:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677448.012080:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677448.012080:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.012081:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:17.0:1619677448.012082:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.012082:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.012084:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.012086:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.012087:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.012089:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.012090:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.012091:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.012092:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.012094:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.012096:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.012098:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.012099:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.012100:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.012102:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.012102:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.012103:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.012103:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.012114:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.2:1619677448.012114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:13.0:1619677448.012117:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.012118:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.012119:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:13.0:1619677448.012120:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.012121:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.012122:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677448.012122:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.012123:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.012125:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.012127:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.012129:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.012130:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.012132:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:2.0:1619677448.012132:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.012133:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.012134:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:12.0:1619677448.012135:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.012136:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.012137:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.012138:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:2.0:1619677448.012138:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.012139:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677448.012140:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.012142:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.012143:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.012144:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.012145:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.012146:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.012147:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.012147:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.012151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.012159:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.012161:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.012162:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.012164:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.012164:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.012166:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.012170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.012178:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.012180:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.012181:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.012182:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.012183:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.012184:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.012209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.012214:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.012217:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.012219:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.012219:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.012221:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.012223:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.012224:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.012225:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.012226:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.012228:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.012231:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.012233:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.012236:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.012237:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.012238:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.012240:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.012240:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.012241:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.012242:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677448.012253:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.012255:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.2:1619677448.012255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:31.0:1619677448.012257:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.012258:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.012259:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.012261:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.012261:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.012262:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677448.012263:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.012264:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.012267:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.012268:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.012269:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.012271:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.012272:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.012273:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.012276:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.012278:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.012280:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.012281:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.012282:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.012283:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.012284:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.012284:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.012285:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677448.012301:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.012303:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.012304:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.012306:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.012308:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.012309:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.012317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677448.012325:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.012328:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.012328:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677448.012330:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.012333:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.012334:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.012335:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.012336:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.012337:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.012338:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.012341:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.012343:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.012345:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.012345:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.012346:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.012348:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.012348:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.012349:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.012349:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:31.0:1619677448.012362:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.012364:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.012365:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.012366:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.012368:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.012369:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.012437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.012445:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.012447:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.012449:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.012451:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.012452:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.012453:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.012454:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.012455:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.012456:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.012457:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.012460:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.012461:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.012463:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.012464:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.012465:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.012466:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.012467:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.012468:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.012470:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.012470:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677448.012476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.012478:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.012480:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.012481:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.012483:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.012484:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.012485:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.012486:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:12.0:1619677448.012486:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.012487:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.012488:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.012497:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.012500:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.012501:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.012501:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.012502:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.012503:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.012504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:5.2:1619677448.012507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677448.012510:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.012513:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:17.0:1619677448.012514:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.012514:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.012516:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.012517:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:22.0:1619677448.012518:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.012518:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677448.012520:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:17.0:1619677448.012520:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.012522:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.012523:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.012524:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.012524:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.012525:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:17.0:1619677448.012525:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.012526:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.012528:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.012530:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.012530:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.012531:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.012533:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:17.0:1619677448.012535:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.012537:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.012539:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.012539:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.012540:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:17.0:1619677448.012540:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.012541:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.012541:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.012542:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.012543:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.012543:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.012544:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.012544:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677448.012544:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.012544:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.012545:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.012549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.012557:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:6.0:1619677448.012557:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.012559:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:18.0:1619677448.012560:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:6.0:1619677448.012560:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.012561:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:18.0:1619677448.012561:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.012561:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:6.0:1619677448.012563:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.012564:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.012566:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677448.012568:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.012650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677448.012658:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677448.012661:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677448.012662:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677448.012664:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.012667:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.012668:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677448.012670:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.012671:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.012672:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.012673:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677448.012676:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.012678:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.2:1619677448.012678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:24.0:1619677448.012680:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.012681:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677448.012681:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.012683:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.012683:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:2.0:1619677448.012683:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:24.0:1619677448.012684:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677448.012684:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:2.0:1619677448.012686:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.012687:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677448.012688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:2.0:1619677448.012689:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.012691:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.012693:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:8.0:1619677448.012694:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:2.0:1619677448.012694:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.012696:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:2.0:1619677448.012696:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.012697:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.012697:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.012698:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:2.0:1619677448.012698:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.012699:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.012700:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:2.0:1619677448.012701:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.012703:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.012706:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.012707:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677448.012708:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677448.012710:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677448.012710:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.012711:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.012712:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677448.012722:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:4.2:1619677448.012724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677448.012725:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.012726:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.012728:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.012729:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.012730:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.012731:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.012733:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.012734:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.012736:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.012739:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.012740:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.012741:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.012743:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.012744:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.012745:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.012748:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.012750:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.012753:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.012754:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.012755:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.012756:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.012757:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.012758:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.012759:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.012759:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.012762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.012765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.012771:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:10.0:1619677448.012772:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:20.0:1619677448.012773:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.012774:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.012774:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.012775:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677448.012776:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.012776:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.012776:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.012779:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.012849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.012857:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.012859:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.012861:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.012864:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.012864:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.012865:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.012866:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.012867:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.012868:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.012869:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.012872:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.012874:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.012876:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.012876:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.012877:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.012878:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.012879:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.012880:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.012880:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.012891:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.012893:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.012893:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.012895:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.012897:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.012898:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.013061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677448.013065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677448.013067:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.013070:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.013070:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:26.0:1619677448.013072:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:1.0:1619677448.013072:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.013074:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.013075:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.013076:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:1.0:1619677448.013076:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.013078:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.013079:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.013079:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.013081:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.013081:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.013081:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:26.0:1619677448.013083:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.013085:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.013086:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.013087:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.013088:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.013088:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:26.0:1619677448.013089:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.013092:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.013092:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.013092:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.013093:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:26.0:1619677448.013094:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677448.013095:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:26.0:1619677448.013096:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.013096:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.013096:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.013097:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:1.0:1619677448.013097:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.013098:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.013099:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.013099:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.013100:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.013101:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.013101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.013108:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.013110:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:10.0:1619677448.013111:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:8.0:1619677448.013111:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.013112:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.013113:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.013114:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:24.0:1619677448.013114:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.013114:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.013116:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.013117:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:10.0:1619677448.013117:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.013131:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.013133:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.013134:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.013134:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.013135:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.013135:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.013146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.013151:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.013154:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.013155:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.013157:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.013159:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.013161:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.013163:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.013164:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.013165:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.013166:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.013170:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.013172:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.013174:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677448.013174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677448.013175:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.013176:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.013177:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.013178:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.013178:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.013179:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677448.013182:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.2:1619677448.013183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.013185:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.013187:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.013189:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.013189:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.013191:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:17.0:1619677448.013191:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:31.0:1619677448.013192:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.013193:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.013194:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.013194:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:31.0:1619677448.013195:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.013195:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.013195:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.013197:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.013199:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.013200:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677448.013201:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.2:1619677448.013202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.013204:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.013204:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.013205:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.013207:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.013207:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.013208:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.013209:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677448.013209:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.013212:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.013212:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.013213:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.013214:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.013215:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:12.0:1619677448.013218:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.013221:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:4.2:1619677448.013221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:12.0:1619677448.013222:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.013223:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.013225:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.013225:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.013228:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.013231:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.013231:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.013233:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.013235:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.013236:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.013238:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.013239:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.013240:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.013241:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.013244:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.013245:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.013248:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.013249:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.013250:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.013251:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.013252:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.013253:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.013253:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.013265:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.013268:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.013269:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.013269:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.013271:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.013274:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.013388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.013396:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.013399:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.013401:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.013403:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.013403:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.013404:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.013406:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.013407:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.013408:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.013409:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.013412:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.013414:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.013416:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.013417:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.013417:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.013418:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.013419:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.013420:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.013420:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.013432:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.013434:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.013435:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.013436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:13.0:1619677448.013437:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.013439:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.013439:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677448.013444:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.013447:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.013448:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.013449:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.013451:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.013453:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.013454:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.013455:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.013456:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.013457:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.013460:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.013462:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.013465:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.013465:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.013466:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.013468:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.013468:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.013470:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.013471:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.013471:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.013473:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.013477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.013482:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.013484:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.013485:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.013485:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.013486:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677448.013486:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.013487:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.013488:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677448.013488:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.013491:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.013735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.013743:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.013745:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.013748:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.013749:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.013750:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.013751:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.013752:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.013754:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.013755:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.013756:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.013759:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.013760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.013761:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.2:1619677448.013763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:31.0:1619677448.013764:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.013765:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.013765:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.013767:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.013767:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.013768:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.013769:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.013769:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:18.0:1619677448.013769:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677448.013770:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.013771:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.013771:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.013772:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:18.0:1619677448.013772:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.013772:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:30.0:1619677448.013773:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.013775:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:21.2:1619677448.013776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677448.013776:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.013777:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.013778:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.013779:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.013779:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.013780:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677448.013780:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:22.0:1619677448.013781:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:18.0:1619677448.013782:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.013783:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.013784:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:22.0:1619677448.013784:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:18.0:1619677448.013784:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.013784:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.013785:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677448.013785:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.013785:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.013786:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:30.0:1619677448.013786:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.013787:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.013788:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.013789:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:18.0:1619677448.013791:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.013792:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.013792:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:22.0:1619677448.013792:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677448.013793:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:18.0:1619677448.013793:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.013794:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.013795:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:4.2:1619677448.013795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677448.013796:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.013796:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.013797:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:18.0:1619677448.013797:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.013799:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.013799:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:18.0:1619677448.013799:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.013799:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.013800:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.013801:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:26.0:1619677448.013802:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.013805:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:20.2:1619677448.013805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:8.2:1619677448.013805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677448.013806:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.013808:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.013809:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.013811:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:26.0:1619677448.013811:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.013812:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.013812:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:1.0:1619677448.013813:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:31.0:1619677448.013814:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.013814:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.013814:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:26.0:1619677448.013814:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.013815:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677448.013815:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.013816:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:26.0:1619677448.013816:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.013816:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:30.0:1619677448.013817:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000400:00000200:26.0:1619677448.013817:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.013817:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.013817:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.013818:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.013819:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.013820:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.013820:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.013820:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000400:00000200:1.0:1619677448.013820:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.013821:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000400:00000200:26.0:1619677448.013821:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.013822:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.013823:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:1.0:1619677448.013823:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677448.013824:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.013826:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.013827:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.013828:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.013829:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.013830:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.013831:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.013831:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.013842:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.013844:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.013845:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.013846:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.013848:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.013848:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.013859:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.013861:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.013862:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.013863:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.013863:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.013864:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.013904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.013910:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.013913:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.013913:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.013915:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.013918:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.013919:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.013920:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.013921:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.013922:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.013923:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.013927:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.013929:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.013931:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.013932:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.013932:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.013934:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.013935:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.013935:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.013936:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.013940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.013947:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.013950:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.013951:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.013951:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.013953:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.013956:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.014080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.014087:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.014090:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.014091:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.014093:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.014096:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.014097:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.014099:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.014100:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.014101:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.014102:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.014106:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.014107:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.014110:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.014111:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.014111:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.014113:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.014113:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.014114:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.014114:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677448.014125:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.014127:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.014128:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.014129:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.014131:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.014132:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.014158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.014164:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.014167:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.014168:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.014170:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.014172:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.014174:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.2:1619677448.014174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:17.0:1619677448.014175:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.014176:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.014177:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.014178:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.014181:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.014181:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.014183:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.014184:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.014185:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.014186:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.014187:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:10.0:1619677448.014187:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.014188:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.014190:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.014190:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.014190:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.014191:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:17.0:1619677448.014192:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.014193:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000400:00000200:10.0:1619677448.014193:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.014194:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.014194:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.014195:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.014196:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:5.2:1619677448.014198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.014200:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.014201:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:12.0:1619677448.014202:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.014203:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:10.0:1619677448.014204:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.014205:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.014205:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.014205:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.014205:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:1.0:1619677448.014205:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.014206:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677448.014207:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.014207:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.014207:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677448.014207:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.014208:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:13.0:1619677448.014209:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.014210:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.014210:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.014210:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.014216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.014219:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.014221:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.014222:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.014222:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.014223:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.014223:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:31.0:1619677448.014225:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.014225:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:13.0:1619677448.014225:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.014225:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677448.014226:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:13.0:1619677448.014226:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.014226:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.014227:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:26.0:1619677448.014228:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.014228:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.014229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.014236:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.014239:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.014240:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.014241:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.014243:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.014245:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.014246:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.014247:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.014248:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.014249:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.014252:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.014254:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.014256:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.014257:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.014258:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.014259:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.014260:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.014261:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.014261:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.014273:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.014276:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.014277:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.014277:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.014279:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.014282:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677448.014291:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.014295:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.014300:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.014301:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.014302:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.014302:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.014313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.014319:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:21.2:1619677448.014321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677448.014322:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.014323:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.014325:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.014327:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.014328:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:31.0:1619677448.014329:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677448.014330:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.014332:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:18.0:1619677448.014332:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.014332:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.014334:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.014334:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.014336:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.014337:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.014337:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:31.0:1619677448.014338:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:18.0:1619677448.014339:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.014340:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.014341:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.014342:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.014342:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.014343:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.014343:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.014343:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.014345:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.014345:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:31.0:1619677448.014347:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.014347:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:31.0:1619677448.014348:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.014348:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.014349:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:31.0:1619677448.014352:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.014353:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)++
00000800:00000200:31.0:1619677448.014353:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (23)++
00000800:00000200:21.2:1619677448.014354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677448.014355:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (24)--
00000800:00000200:31.0:1619677448.014356:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.014357:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.014357:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677448.014359:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:22.0:1619677448.014360:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:2.0:1619677448.014361:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.014362:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677448.014363:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:6.0:1619677448.014363:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (24)++
00000800:00000200:2.0:1619677448.014363:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:22.0:1619677448.014364:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.014364:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000400:00000200:6.0:1619677448.014365:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.014366:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:20.2:1619677448.014366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:6.0:1619677448.014367:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.014369:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.014370:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.014371:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677448.014372:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.014375:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.014377:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.014378:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.014380:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.014382:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.014384:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.014385:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.014386:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.014387:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.014390:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.014392:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.014395:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.014396:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.014397:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.014399:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.014399:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.014400:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.014400:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:13.0:1619677448.014412:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.014415:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.014416:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.014418:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.014419:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.014420:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.014454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.014461:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.014464:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.014465:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.014467:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.014469:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.014471:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.014472:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.014473:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.014474:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.014475:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.014478:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.014480:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.014483:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.014483:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.014484:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.014486:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.014486:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.014487:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.014487:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.014492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.014500:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.014502:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.014503:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.014504:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.014506:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.014508:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.014523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.014530:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.014533:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.014535:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.014537:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.014538:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.014539:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.014541:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.014542:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.014543:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.014544:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.014547:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.014549:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.014551:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.014551:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.014552:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.014553:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.014554:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.014555:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.014555:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677448.014565:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.014567:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.014568:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.014570:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.014571:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.014572:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.014614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.014622:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.014624:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.014625:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.014627:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.014630:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.014631:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.014632:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.014634:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.014635:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.014635:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.014638:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.014640:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.014643:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.014644:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.014644:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.014646:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.014647:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.014648:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.014649:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.014649:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.014655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.014660:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.014662:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.014663:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.014664:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.014664:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677448.014664:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.014665:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.014666:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.014667:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.014669:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.014757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.014765:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.014767:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.014768:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.014770:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.014773:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.014774:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.014776:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.014777:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.014778:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.014779:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.014781:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.014783:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.014785:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.014786:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.014786:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.014788:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.014788:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.014789:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.014789:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.014800:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.014802:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.014803:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.014805:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.014806:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.014808:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.014819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.014825:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.014828:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.014829:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.014831:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.014833:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.014834:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.014836:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.014837:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.014838:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.014839:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.014842:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.014844:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.014845:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.014846:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.014847:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.014848:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.014849:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.014849:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.014850:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.014851:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677448.014854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.014857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.014862:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.014863:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.014863:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.014864:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:2.0:1619677448.014864:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.014865:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.014865:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.014866:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.014867:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677448.014867:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.014875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.014882:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.014885:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.014885:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.014886:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.014886:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.014887:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.014944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.014949:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.014951:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.014953:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.014954:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.014955:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.014957:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.014958:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.014959:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.014960:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.014961:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.014964:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.014965:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.014968:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.014969:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.014969:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.014971:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.014971:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.014972:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.014973:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.014973:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.2:1619677448.014979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.014983:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:2.0:1619677448.014984:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:10.0:1619677448.014986:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:2.0:1619677448.014986:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:26.0:1619677448.014987:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.014987:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.014987:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:10.0:1619677448.014989:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.014989:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:10.0:1619677448.014993:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.015149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.015156:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.015158:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.015159:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.015161:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.015163:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.015165:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.015166:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.015167:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.015168:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.015169:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.015173:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.015174:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.015177:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.015177:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.015178:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.015180:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.015180:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.015181:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.015181:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.015192:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.015195:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.015195:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.015197:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.015199:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.015200:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.015252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.015260:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.015262:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.015264:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.015266:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.015266:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.015268:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.015268:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.015270:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.015271:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.015272:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.015275:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.015276:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.015279:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.015280:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.015280:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.015283:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.015283:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.015284:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.015284:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.015289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.015298:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.015301:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.015302:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.015303:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.015304:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.015306:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.015325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.015332:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.015335:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.015336:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.015337:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.015340:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.015342:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.015343:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720497] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.015344:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.015345:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.015346:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.015350:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.015351:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.015354:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.015355:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.015355:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.015356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.015357:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.015358:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.015358:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.015360:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:10.0:1619677448.015361:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.015363:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.015363:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:8.2:1619677448.015364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:10.0:1619677448.015365:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.015367:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.015368:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.015369:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.015370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:10.0:1619677448.015371:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.015371:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.015372:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.015372:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.015375:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:10.0:1619677448.015375:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.015375:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:22.0:1619677448.015376:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.015376:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.015376:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.015378:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.015379:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:10.0:1619677448.015379:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.015380:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.015380:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677448.015380:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.015381:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.015381:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:10.0:1619677448.015382:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.015382:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.015383:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677448.015384:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.015384:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677448.015386:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.015387:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.015388:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.015389:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.015390:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.015394:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.015396:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.015396:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:22.0:1619677448.015398:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.015398:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:22.0:1619677448.015399:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.015399:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:13.0:1619677448.015399:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.015400:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.015401:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.015402:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677448.015403:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677448.015404:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:12.0:1619677448.015404:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677448.015405:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.015405:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677448.015407:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:8.2:1619677448.015407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677448.015408:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.015408:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.015409:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:21.2:1619677448.015411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:12.0:1619677448.015411:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677448.015416:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677448.015417:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.015419:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:5.0:1619677448.015419:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:6.0:1619677448.015420:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:5.0:1619677448.015420:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.015421:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677448.015422:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.015422:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.015424:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.015461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.015467:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.015470:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.015470:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.015473:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.015475:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.015477:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.015478:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.015479:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.015480:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.015481:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.015485:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.015486:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.015489:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.015490:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.015491:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.015492:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.015493:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.015493:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.015494:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.015506:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.015508:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.015509:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.015509:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.015512:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.015514:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.015578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677448.015584:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677448.015586:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677448.015587:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677448.015589:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.015591:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.015593:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.015594:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.015595:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.015596:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.015597:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.015600:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.015601:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.015604:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.015605:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.015605:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.015607:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.015608:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677448.015608:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677448.015609:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:13.0:1619677448.015620:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.015623:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.015624:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.015625:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.015626:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.015629:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677448.015650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.015657:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.015660:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.015661:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.015661:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.015662:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.015663:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.015775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.015782:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:21.2:1619677448.015784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.015785:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:18.0:1619677448.015787:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.015787:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.015788:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.015789:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:18.0:1619677448.015791:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.015792:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.015793:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.015794:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.015796:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.015797:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.015798:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.015800:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.015802:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.015802:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677448.015803:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.015804:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.015805:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.015806:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.015807:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:18.0:1619677448.015811:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.015811:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.015812:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.015813:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.015813:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677448.015815:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.015815:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.015816:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.015816:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677448.015817:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.015817:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677448.015818:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677448.015818:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.015819:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677448.015820:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.015821:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.015821:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677448.015828:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.015830:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:26.0:1619677448.015831:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:12.0:1619677448.015831:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.015833:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.015834:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:13.0:1619677448.015834:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.015834:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.015835:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.015836:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677448.015837:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:26.0:1619677448.015841:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:13.0:1619677448.015847:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.015850:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.015851:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.015851:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.015852:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.015853:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.015885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.015888:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.015891:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.015893:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.015894:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.015896:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.015898:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:20.0:1619677448.015902:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.015903:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.015905:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.015906:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:20.0:1619677448.015906:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.015907:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.015910:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:20.0:1619677448.015911:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.015913:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.015913:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.015914:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.015915:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.015917:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.015918:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:20.0:1619677448.015918:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:30.0:1619677448.015919:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.015919:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677448.015921:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.015921:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:30.0:1619677448.015922:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.015922:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.015922:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677448.015923:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.015923:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:30.0:1619677448.015926:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.015928:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.015931:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.015932:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.015933:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677448.015933:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.015934:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.015935:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.015936:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.015936:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677448.015936:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.015937:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.015939:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.015940:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.2:1619677448.015940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677448.015941:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.015948:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.015950:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.015951:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.015953:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.015953:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.015956:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.015966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.015972:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.015975:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.015976:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.015979:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.015982:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.015983:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.015985:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.015986:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.015987:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.015988:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.015991:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.015993:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.015996:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.015997:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.015997:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.015999:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.016000:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.016000:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.016001:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677448.016012:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.016014:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.016015:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.016017:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.016018:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.016020:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.016078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.016084:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.016087:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.016087:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.016090:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.016093:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.016094:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.016096:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.016097:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.016098:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.016099:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.016102:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.016104:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.016107:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.016108:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.016108:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.016110:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.016111:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.016111:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.016112:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.016123:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.016125:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.016126:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.016128:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.016129:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.016131:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.016162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.016169:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.016172:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.016174:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.016175:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.016176:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.016178:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.016179:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.016180:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.016181:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.016182:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.016185:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.016187:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.016189:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.016190:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.016191:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.016192:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.016193:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.016194:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.016194:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677448.016200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.016205:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.016207:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.016208:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:10.0:1619677448.016208:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.016210:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.016210:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.016212:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.016213:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.016214:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:26.0:1619677448.016215:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.016215:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.016217:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.016218:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.016219:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.016220:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.016221:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.016224:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.016226:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.016229:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.016229:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.016230:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.016232:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.016233:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.016233:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.016233:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.016238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.016245:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.016248:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.016249:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.016250:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.016252:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.016255:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.016276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.016283:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.016286:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.016287:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.016289:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.016292:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.016299:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.016300:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.016302:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.016303:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.016303:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.016306:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.016308:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.016311:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.016312:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.016312:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.016314:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.016315:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.016315:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.016316:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.016320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.016324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:17.0:1619677448.016328:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.016331:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.016332:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.016332:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677448.016332:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.016334:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.016335:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.016335:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.016337:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.016338:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:10.0:1619677448.016340:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.016342:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.016343:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.016344:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.016345:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.016346:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.016349:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.016350:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.016353:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.016353:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.016354:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.016355:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.016356:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.016356:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.016357:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.016369:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.016371:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.016372:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.016373:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.016375:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.016376:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.016382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.016391:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.016393:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:21.2:1619677448.016394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:30.0:1619677448.016395:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.016397:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.016398:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.016399:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.016400:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.016400:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:30.0:1619677448.016401:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.016403:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.016403:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.016403:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:30.0:1619677448.016404:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:22.0:1619677448.016407:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.016408:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.016409:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.016411:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:22.0:1619677448.016411:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.016412:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.016413:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.016414:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.016415:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:22.0:1619677448.016415:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.016416:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:22.0:1619677448.016416:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.016418:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.016418:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:22.0:1619677448.016419:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.016420:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.016421:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:22.0:1619677448.016421:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.016422:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:22.0:1619677448.016424:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677448.016424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677448.016425:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.016426:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.016427:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.016428:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.016428:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.2:1619677448.016428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:22.0:1619677448.016429:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.016433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.016436:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.016437:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:31.0:1619677448.016439:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:20.0:1619677448.016439:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:31.0:1619677448.016440:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.016440:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.016440:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.016441:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:20.0:1619677448.016442:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.016442:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.016443:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.016445:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.016445:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.016446:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677448.016447:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.016448:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.016451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.016459:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.016461:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.016462:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.016463:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.016463:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.016465:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.016508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.016515:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.016518:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.016519:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.016521:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.016524:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.016526:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.016527:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.016528:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.016529:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.016530:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.016534:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.016536:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.016539:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.016539:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.016540:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.016542:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.016543:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.016543:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.016543:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.016548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.016549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.016556:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.016557:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677448.016558:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:18.0:1619677448.016559:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:17.0:1619677448.016560:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.016560:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.016561:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.016561:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.016563:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.016564:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.016565:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:18.0:1619677448.016566:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:1.0:1619677448.016566:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.016567:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.016568:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.016570:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.016573:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.016575:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.016576:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.016577:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.016578:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.016579:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.016579:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.016580:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.016581:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.016585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.016592:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.016594:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.016594:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.016595:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.016596:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.016598:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.016604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.016611:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.016613:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.016614:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.016614:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:4.2:1619677448.016615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677448.016615:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.016616:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677448.016622:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.016625:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.016626:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.016628:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.016631:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.016632:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.016633:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.016635:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.016636:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.016637:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.016640:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.016642:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.016645:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.016646:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.016647:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.016649:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.016649:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.016650:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.016650:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.016654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.016662:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.016664:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.016666:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.016666:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.016668:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.016671:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.016833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.016839:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.016842:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.016842:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.016845:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.016846:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.016848:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.016849:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.016850:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.016852:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.016852:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.016856:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.016858:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.016860:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.016861:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.016861:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.016863:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.016863:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.016865:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.016866:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.016866:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.016871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.016877:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.016877:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.016879:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.016879:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.016880:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677448.016880:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.016881:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.016881:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.016883:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.016885:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.016896:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.016899:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.016899:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.016900:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.016901:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.016901:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.016906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.016909:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.016912:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.016914:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.016915:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.016916:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.016918:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.016919:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.016921:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.016922:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.016923:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.016926:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.016928:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.016931:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.016931:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.016932:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.016934:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.016935:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.016936:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.016937:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.016938:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.016943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677448.016946:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.016948:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677448.016950:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.016950:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.016950:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.016951:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:12.0:1619677448.016952:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.016952:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.016953:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.016955:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.016978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.016987:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.016990:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.016992:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.016993:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.016994:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.016996:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.016997:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.016997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000400:00000200:31.0:1619677448.016998:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.016999:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.017000:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.017004:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.017005:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.017005:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677448.017007:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:31.0:1619677448.017008:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.017008:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:31.0:1619677448.017009:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.017010:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:8.0:1619677448.017010:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677448.017011:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.017012:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.017012:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.017013:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:8.0:1619677448.017013:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.017014:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:20.2:1619677448.017015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:8.0:1619677448.017015:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96720474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.017016:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.017017:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.017017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:8.0:1619677448.017018:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677448.017020:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677448.017022:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677448.017023:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:8.0:1619677448.017023:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.017023:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:13.0:1619677448.017024:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.0:1619677448.017024:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677448.017024:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:18.0:1619677448.017026:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:8.0:1619677448.017026:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.017026:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:21.2:1619677448.017027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:13.0:1619677448.017027:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:8.0:1619677448.017027:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677448.017027:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:2.0:1619677448.017027:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:13.0:1619677448.017028:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.017028:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.017030:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:13.0:1619677448.017030:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.017031:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.017031:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:9.2:1619677448.017031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:13.0:1619677448.017032:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:5.0:1619677448.017032:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.017034:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.017035:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677448.017035:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:5.0:1619677448.017035:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.017036:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.017037:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:5.0:1619677448.017037:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:30.0:1619677448.017038:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:18.0:1619677448.017038:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.017039:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:5.0:1619677448.017039:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.017040:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.017040:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.017041:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677448.017041:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:8.0:1619677448.017041:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:5.0:1619677448.017041:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.017042:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.017043:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.017043:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000400:00000200:5.0:1619677448.017043:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.017045:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.017045:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:30.0:1619677448.017047:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.017049:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.017051:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.017051:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.017051:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.017052:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.017053:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:9.2:1619677448.017053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:18.0:1619677448.017054:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.017055:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.017057:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.017058:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.017059:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.017059:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.0:1619677448.017059:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.0:1619677448.017059:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.017059:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677448.017060:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.017061:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.017061:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:8.0:1619677448.017061:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.017062:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:18.0:1619677448.017062:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.017062:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.0:1619677448.017062:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:5.0:1619677448.017062:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.017063:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.017063:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.0:1619677448.017063:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677448.017063:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677448.017064:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677448.017064:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.0:1619677448.017064:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:30.0:1619677448.017065:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.017065:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677448.017066:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.017067:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.2:1619677448.017067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.017068:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.017069:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.017074:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:20.0:1619677448.017075:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.017077:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:10.0:1619677448.017077:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.017078:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:20.0:1619677448.017078:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.017078:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677448.017079:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.017080:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.017080:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.017080:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:12.0:1619677448.017081:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.017082:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.017083:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:12.0:1619677448.017083:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.017084:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.017085:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.017085:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:13.0:1619677448.017096:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.017099:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.017099:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.017100:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.017100:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.017101:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.017194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.017200:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.017203:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.017204:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.017206:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.017208:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.017210:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.017211:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.017212:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.017213:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.017214:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.017218:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.017220:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.017223:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.017223:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.017224:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.017226:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.017226:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.017227:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.017228:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.017229:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.017234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.017239:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677448.017241:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.017242:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.017243:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.017243:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677448.017243:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.017244:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.017245:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.017245:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.017249:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.017255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.017261:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.017263:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.017264:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.017266:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.017268:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.017270:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.017271:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.017273:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.017274:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.017275:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.017278:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.017280:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.017282:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.017283:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.017284:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.017285:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.017286:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.017286:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.017286:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.017300:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.017303:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.017304:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.017304:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.017306:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.017308:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.017455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:20.2:1619677448.017458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677448.017461:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677448.017464:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.017464:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.017465:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677448.017467:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:2.0:1619677448.017468:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.017469:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.017469:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.017471:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.017472:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.017473:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.017473:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.017475:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677448.017475:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.017477:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.017477:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.017479:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.017479:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.017480:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.017481:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677448.017484:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.017486:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.017486:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.017488:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677448.017489:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.017490:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.017491:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.017491:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.017492:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.017493:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:2.0:1619677448.017493:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.017494:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.017494:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677448.017495:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:2.0:1619677448.017495:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677448.017496:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.017497:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.017498:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.017507:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.017509:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677448.017510:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:12.0:1619677448.017510:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.017512:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.017513:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.017514:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.017514:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:31.0:1619677448.017515:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.017515:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.017518:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.017518:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:4.2:1619677448.017610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.017617:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.017620:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.017621:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.017623:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.017626:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.017627:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.017628:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.017629:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.017630:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.017631:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.017635:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.017637:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.017640:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.017640:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.017641:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.017643:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.017644:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.017644:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.017645:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.017649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.017657:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.017660:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.017661:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.017661:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.017663:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.017667:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.017668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:4.2:1619677448.017669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.017676:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677448.017676:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.017679:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.017680:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:18.0:1619677448.017680:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.017680:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677448.017681:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.017682:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000400:00000200:24.0:1619677448.017683:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.017683:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:24.0:1619677448.017685:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.017687:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.017688:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.017689:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.017690:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.017691:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.017694:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.017696:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.017699:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.017700:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.017700:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.017702:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.017702:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.017703:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.017704:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.017708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.017716:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.017719:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.017720:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.017720:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.017722:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.017725:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.017785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.017791:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.017793:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.017794:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.017796:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.017798:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.017800:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.017801:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.017802:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.017803:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.017804:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.017808:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.017809:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.017812:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.017813:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.017813:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.017815:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.017815:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.017816:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.017817:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.017821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.017829:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.017831:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.017832:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.017832:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.017835:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.017837:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.017853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.2:1619677448.017854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677448.017855:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.017858:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.017860:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.017862:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.017862:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.017864:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.017865:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:20.0:1619677448.017865:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.017867:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.017868:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.017868:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.017870:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.017870:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.017871:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.017872:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.017872:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677448.017873:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.017875:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.017876:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.017877:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.017881:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.017882:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.017884:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.017886:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.017887:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.017888:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.017889:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.017889:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.017890:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.017891:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.017891:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.017891:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.017893:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677448.017893:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.017894:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:20.0:1619677448.017894:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.017894:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677448.017895:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.017895:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.2:1619677448.017903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.017903:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.017905:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.017906:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.017908:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:13.0:1619677448.017908:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.017909:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.017910:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:13.0:1619677448.017910:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677448.017911:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.017912:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:31.0:1619677448.017913:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.017913:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.017913:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.017914:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.017915:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.017916:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677448.017985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.017991:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.017993:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.017994:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.017996:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.017998:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.018000:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.018001:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.018002:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.018003:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.018004:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.018008:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.018010:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.018013:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.018014:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.018014:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.018016:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.018017:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.018017:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.018018:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.018022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.018028:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.018031:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.018032:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.018032:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.018034:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.018038:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.018167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.018174:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.018176:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.018177:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.018179:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.018182:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.018183:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.018184:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.018185:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.018186:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.018187:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.018190:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.018192:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.018195:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.018196:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.018197:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.018199:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.018199:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.018200:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.018200:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.018212:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.018215:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.018216:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.018216:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.018218:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.018222:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677448.018231:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.018233:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.018234:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.018234:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.018235:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.018236:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.018273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.018281:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.018283:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.018284:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.018286:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.018289:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.018290:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.018291:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.018296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:26.0:1619677448.018299:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.018300:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.018301:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:22.0:1619677448.018303:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:26.0:1619677448.018304:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.018305:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:22.0:1619677448.018305:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.018306:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677448.018307:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.018308:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.018309:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:22.0:1619677448.018309:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.018310:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.018311:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:22.0:1619677448.018311:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.018312:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:22.0:1619677448.018312:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:26.0:1619677448.018313:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000400:00000200:22.0:1619677448.018313:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.018314:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677448.018315:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.018316:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.018317:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:5.2:1619677448.018319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:22.0:1619677448.018321:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.018322:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.018325:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.018325:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:10.0:1619677448.018325:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:22.0:1619677448.018326:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:10.0:1619677448.018326:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.018327:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.018328:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677448.018328:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.018328:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.018329:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:22.0:1619677448.018329:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677448.018329:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.018330:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677448.018330:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.018331:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.018332:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.018332:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.018335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:26.0:1619677448.018341:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.018344:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.018344:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:26.0:1619677448.018345:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.018345:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.018346:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.018346:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:26.0:1619677448.018347:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:18.0:1619677448.018347:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.018347:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.018349:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.018352:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.018422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.018430:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.018433:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:21.2:1619677448.018434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:31.0:1619677448.018435:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.018437:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.018438:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.018439:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.018440:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.018441:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.018441:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:31.0:1619677448.018442:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.018442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:24.0:1619677448.018443:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:31.0:1619677448.018444:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:8.0:1619677448.018444:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.018446:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.018447:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.018448:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.018449:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:24.0:1619677448.018449:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.018451:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677448.018451:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:5.0:1619677448.018451:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:31.0:1619677448.018453:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.018453:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:24.0:1619677448.018453:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96720456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.018453:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.018454:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.018454:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.018455:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.018455:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:24.0:1619677448.018455:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.018456:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:24.0:1619677448.018456:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.018457:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.018457:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.018457:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.018458:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677448.018458:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.018459:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.018461:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.018462:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.018463:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.018467:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:5.0:1619677448.018467:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.018468:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.018470:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:5.0:1619677448.018470:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:30.0:1619677448.018471:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.018471:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:20.0:1619677448.018472:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.018473:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.018473:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:31.0:1619677448.018474:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.018474:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.018475:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:24.0:1619677448.018475:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.018475:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.018475:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:24.0:1619677448.018476:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.018476:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:20.0:1619677448.018476:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.018476:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:20.0:1619677448.018477:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.018477:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:24.0:1619677448.018478:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:20.0:1619677448.018478:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.018479:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677448.018479:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)++
00000800:00000200:24.0:1619677448.018480:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:5.0:1619677448.018480:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (24)--
00000800:00000200:5.0:1619677448.018481:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (23)--
00000800:00000200:24.0:1619677448.018482:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000400:00000200:20.0:1619677448.018482:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.018482:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:24.0:1619677448.018483:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677448.018484:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.2:1619677448.018484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.018486:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.018487:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:21.2:1619677448.018488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677448.018488:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.018489:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677448.018490:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.018490:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.018491:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677448.018493:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677448.018493:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.018493:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677448.018494:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.018494:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:31.0:1619677448.018496:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.018496:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.018496:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:30.0:1619677448.018497:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:5.0:1619677448.018497:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.018498:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.018498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:5.0:1619677448.018499:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.018500:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.018502:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:5.0:1619677448.018502:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.018503:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.018504:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (23)--
00000800:00000200:17.0:1619677448.018504:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:5.0:1619677448.018505:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:30.0:1619677448.018506:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.018506:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.018506:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.018507:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:17.0:1619677448.018507:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:5.0:1619677448.018507:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.018507:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:17.0:1619677448.018508:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.018508:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:1.0:1619677448.018508:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677448.018509:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.018509:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.018510:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.018510:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:30.0:1619677448.018511:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.018511:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.018511:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677448.018511:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.018513:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677448.018514:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.018551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.018558:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.018561:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.018562:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.018564:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.018566:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.018567:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.018568:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.018570:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.018571:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.018572:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.018575:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.018577:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.018580:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.018580:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.018581:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.018583:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.018584:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.018584:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.018585:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677448.018587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.018593:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.018595:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.018596:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.018597:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:1.0:1619677448.018598:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.018599:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.018601:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.018601:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.018601:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.018603:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.018603:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.018604:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.018605:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.018606:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:1.0:1619677448.018606:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.018607:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.018611:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.018612:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.018615:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.018616:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.018616:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.018618:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.018619:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.018619:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.018620:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.018624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.018630:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.018633:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.018634:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.018634:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.018637:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.018639:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.018650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.018657:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.018660:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.018662:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.018664:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.018665:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.018666:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.018667:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.018668:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.018669:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.018670:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.018673:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.018675:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.018677:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.018678:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.018679:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.018680:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.018681:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.018682:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.018682:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.018694:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.018696:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.018697:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.018698:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.018700:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.018701:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.018721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.018728:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.018731:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.018732:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.018733:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.018736:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.018737:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.018738:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.018740:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.018741:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.018741:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.018744:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.018746:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.018749:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.018750:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.018750:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.018752:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.018752:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.018753:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.018755:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.018755:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.018758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677448.018760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.018766:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677448.018767:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.018769:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677448.018769:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.018769:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.018770:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.018770:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.018771:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677448.018772:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.018775:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.018945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.018953:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.018955:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.018956:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.018958:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.018961:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.018963:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.018964:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.018965:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.018966:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.018967:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.018970:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.018971:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.018974:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.018975:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.018975:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.018977:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.018977:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.018978:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.018979:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.018980:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.018985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.018990:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677448.018992:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.018993:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.018994:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.018994:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.018995:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.018995:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.018996:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.018996:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.018999:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677448.019008:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.019010:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.019011:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.019012:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.019012:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.019013:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.019055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.019061:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.019064:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.019065:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.019066:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.019069:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.019070:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.019072:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.019073:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.019074:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.019075:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.019078:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.019079:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.019082:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.019082:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.019083:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.019085:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.019085:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.019086:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.019086:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.019091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.019099:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.019101:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.019102:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.019102:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.019104:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.019106:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.019144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.019151:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.019154:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:21.2:1619677448.019154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677448.019154:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.019156:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.019160:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.019161:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677448.019161:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.019162:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.019164:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677448.019164:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.019166:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.019167:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.019167:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.019168:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.019170:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.019170:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.019171:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.019172:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.019172:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:30.0:1619677448.019174:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.019175:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.019175:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.019176:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.019176:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.019177:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677448.019179:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.019179:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.019179:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.019180:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:30.0:1619677448.019181:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.019182:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.019182:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:30.0:1619677448.019183:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.019183:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.019184:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.019185:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:9.2:1619677448.019185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.019186:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.019187:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.019187:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677448.019188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:18.0:1619677448.019193:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.019195:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:18.0:1619677448.019196:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.019196:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.019197:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.019197:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.019197:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.019198:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:18.0:1619677448.019199:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.019199:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677448.019200:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.019200:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.019202:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:12.0:1619677448.019202:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.019204:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.019204:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.019213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:13.0:1619677448.019217:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.019218:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:13.0:1619677448.019220:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.019221:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.019221:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.019221:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.019222:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:13.0:1619677448.019222:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.019223:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677448.019224:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.019226:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.019228:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677448.019229:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.019231:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.019232:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.019233:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.019236:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.019238:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.019241:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.019242:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.019243:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.019244:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.019245:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.019246:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.019246:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677448.019251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.2:1619677448.019257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.019262:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.019262:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677448.019265:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:5.0:1619677448.019265:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.019266:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.019266:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.019268:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:17.0:1619677448.019268:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.019268:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.019270:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:17.0:1619677448.019271:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.019273:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.019274:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.019275:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.019276:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.019277:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.019281:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.019283:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.019286:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.019287:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.019287:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.019289:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.019290:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.019291:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.019292:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.019293:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.019299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.2:1619677448.019304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.019307:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:26.0:1619677448.019308:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.019310:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:1.0:1619677448.019310:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:31.0:1619677448.019311:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.019311:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.019311:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677448.019311:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:31.0:1619677448.019312:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.019312:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.019313:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.019314:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:26.0:1619677448.019314:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.019315:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:26.0:1619677448.019316:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.019317:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.019319:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.019320:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.019321:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.019322:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.019325:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.019327:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.019330:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.019330:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.019331:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.019333:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.019334:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.019334:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.019335:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.019339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.019347:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.019350:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.019351:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.019351:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.019354:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.019357:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.019382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.019390:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.019392:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.019393:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.019395:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.019397:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.019399:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.019400:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.019401:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.019402:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.019403:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.019406:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.019407:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.019410:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.019410:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.019411:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.019413:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.019413:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.019414:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.019414:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.019427:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.019430:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.019430:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.019431:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.019433:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.019436:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.019506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.019511:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.019514:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.019514:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.019516:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.019518:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.019519:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.019520:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.019522:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.019522:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.019523:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.019527:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.019528:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.019531:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.019531:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.019532:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.019534:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.019534:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.019535:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.019536:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.019537:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:21.2:1619677448.019542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.2:1619677448.019543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.019547:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.019549:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.019549:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.019550:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677448.019551:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.019554:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.019555:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.019555:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.019557:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.019559:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.019654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.019662:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.019664:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.019667:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.019668:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.019669:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.019671:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677448.019672:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.019673:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.019674:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.019675:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:21.2:1619677448.019675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000400:00000200:30.0:1619677448.019678:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.019680:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.019682:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:30.0:1619677448.019683:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.019684:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.019684:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.019685:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:30.0:1619677448.019686:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:10.0:1619677448.019686:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:30.0:1619677448.019687:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.019688:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.019688:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:26.0:1619677448.019688:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.019691:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.019692:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:8.2:1619677448.019692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677448.019694:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.019695:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.019696:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.019696:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.019699:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.019699:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:26.0:1619677448.019701:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.019702:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:26.0:1619677448.019703:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.019703:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.019703:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.019703:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.019704:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:1.0:1619677448.019705:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.019706:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.019706:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.019707:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677448.019707:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:1.0:1619677448.019708:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.019709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:21.2:1619677448.019711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:17.0:1619677448.019717:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.019717:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:31.0:1619677448.019718:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677448.019720:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.019720:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:1.0:1619677448.019720:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.019721:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:18.0:1619677448.019721:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:17.0:1619677448.019721:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000400:00000200:31.0:1619677448.019722:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.019722:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.019722:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:2.0:1619677448.019723:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.019724:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.019725:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:2.0:1619677448.019725:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.019726:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:2.0:1619677448.019727:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677448.019728:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.019729:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.019730:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.019731:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.019735:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.019737:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.019740:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.019741:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677448.019741:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677448.019743:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677448.019744:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.019745:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.019746:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.019747:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.2:1619677448.019752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:12.0:1619677448.019756:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:18.0:1619677448.019757:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:12.0:1619677448.019759:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:18.0:1619677448.019760:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.019760:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:12.0:1619677448.019760:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.019761:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:12.0:1619677448.019762:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.019763:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.019764:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677448.019774:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.019777:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.019778:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.019778:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.019779:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.019780:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.019827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.019835:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.019837:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.019840:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.019841:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.019842:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.019844:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.019845:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.019846:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.019847:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.019848:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.019852:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.019853:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.019857:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.019858:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.019858:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.019860:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.019861:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.019862:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.019862:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.019866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.019874:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.019877:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.019877:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.019879:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.019880:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.019882:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.019905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:4.2:1619677448.019905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:5.0:1619677448.019910:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:8.0:1619677448.019912:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.0:1619677448.019913:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.019915:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.0:1619677448.019915:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:5.0:1619677448.019915:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.019916:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.019918:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.019918:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.019920:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677448.019921:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.019922:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.019922:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.019923:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.019924:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.019924:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.019925:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.019926:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.019926:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677448.019928:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.2:1619677448.019930:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:8.0:1619677448.019932:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.019933:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.019934:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.019936:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677448.019936:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677448.019937:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.019938:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.019938:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.019939:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.019939:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.0:1619677448.019940:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.019940:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.0:1619677448.019940:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.019941:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677448.019941:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:18.0:1619677448.019942:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.019942:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.0:1619677448.019942:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677448.019944:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.019944:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:18.0:1619677448.019945:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.019945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:18.0:1619677448.019946:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:5.0:1619677448.019946:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.019947:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.019947:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:18.0:1619677448.019948:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.019949:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.019950:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.019951:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.2:1619677448.019952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:1.0:1619677448.019953:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:18.0:1619677448.019954:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.019955:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:31.0:1619677448.019956:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:18.0:1619677448.019956:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.019956:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677448.019957:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:17.0:1619677448.019957:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.019957:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.019958:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:18.0:1619677448.019958:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.019959:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:20.0:1619677448.019959:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677448.019959:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.019959:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677448.019959:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.019960:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.019961:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:31.0:1619677448.019962:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.019962:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.019962:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:1.0:1619677448.019962:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677448.019963:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:18.0:1619677448.019963:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.019964:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677448.019965:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:9.2:1619677448.019967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.019970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:30.0:1619677448.019975:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:2.0:1619677448.019976:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.019977:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.019977:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:30.0:1619677448.019978:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:18.0:1619677448.019978:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:2.0:1619677448.019978:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.019979:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:18.0:1619677448.019979:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:30.0:1619677448.019980:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000400:00000200:2.0:1619677448.019980:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.019981:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677448.019982:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:18.0:1619677448.019982:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:31.0:1619677448.019983:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:2.0:1619677448.019983:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.020082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.020089:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.020092:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.020093:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.020095:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.020097:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.020099:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.020100:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.020101:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.020102:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.020103:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.020106:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.020108:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.020110:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.020111:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.020112:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.020113:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.020114:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.020115:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.020115:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.020126:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.020128:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.020129:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.020130:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.020133:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.020133:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.2:1619677448.020212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.020217:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.020220:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.020221:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.020223:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.020225:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.020227:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.020228:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.020229:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.020230:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.020231:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.020235:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.020237:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.020238:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.020239:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.020240:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.020241:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.020242:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.020243:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.020243:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.020247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.020255:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.020256:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.020257:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.020258:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.020259:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.020260:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.020265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.020272:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.020274:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.020275:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.020275:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.020276:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.020277:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.020304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677448.020310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677448.020312:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.020314:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.020315:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.020317:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.020318:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.020320:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677448.020320:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.020321:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677448.020323:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.020323:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.020324:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.020325:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.020325:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.020325:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.020326:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677448.020327:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.020328:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.020330:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.020331:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.020332:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.020337:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.020337:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.020339:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:26.0:1619677448.020339:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.020342:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.020342:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.020342:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.020343:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:26.0:1619677448.020343:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.020344:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.020345:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:26.0:1619677448.020345:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.020346:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.020346:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.020347:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.020347:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.020348:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.020350:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.020350:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677448.020352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.020356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.020361:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.020362:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:18.0:1619677448.020362:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:31.0:1619677448.020363:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.020364:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.020364:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:18.0:1619677448.020364:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:30.0:1619677448.020365:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:2.0:1619677448.020365:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.020366:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677448.020366:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.020367:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:21.2:1619677448.020367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:30.0:1619677448.020368:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:18.0:1619677448.020368:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.020369:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677448.020372:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.020374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.020375:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.020376:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.020377:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.020377:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.020379:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000400:00000200:18.0:1619677448.020380:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:2.0:1619677448.020380:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.020380:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.020381:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.020382:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.020382:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:18.0:1619677448.020384:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.020385:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.020386:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.020387:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.020389:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.020392:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.020394:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.020395:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.020396:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.020397:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.020398:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.020399:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.020399:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.020400:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.020404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.020411:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.020412:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.020413:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.020414:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.020415:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.020417:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.020420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.020427:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.020429:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.020430:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.020430:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.020431:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.020432:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.020720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.020726:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.020728:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.020729:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.020731:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.020732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:22.0:1619677448.020734:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.020735:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.020737:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.020737:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677448.020738:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.020739:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.020740:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.020740:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.020741:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.020743:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.020745:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.020745:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.020747:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:1.0:1619677448.020747:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.020748:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.020750:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.020750:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.020751:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:1.0:1619677448.020751:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.020752:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:1.0:1619677448.020752:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.020753:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.020754:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.020755:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.020755:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:1.0:1619677448.020755:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.020757:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.2:1619677448.020760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:1.0:1619677448.020760:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.020761:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.020762:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.020764:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.020764:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.020765:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.020765:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677448.020768:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.020771:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.020772:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.020773:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.020774:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.020776:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677448.020779:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.020782:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.020783:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.020784:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.020786:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.020788:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.020822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.020829:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.020832:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:21.2:1619677448.020832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:10.0:1619677448.020832:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677448.020835:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.020837:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.020839:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:24.0:1619677448.020840:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:10.0:1619677448.020840:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720426] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.020841:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.020842:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:10.0:1619677448.020842:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.020843:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:10.0:1619677448.020844:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:24.0:1619677448.020846:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.020848:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.020849:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.020850:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:10.0:1619677448.020850:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:24.0:1619677448.020851:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96720425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.020853:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.020853:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677448.020854:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.020854:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:24.0:1619677448.020855:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.020855:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.020857:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.020857:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:24.0:1619677448.020858:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.020858:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.020858:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:24.0:1619677448.020859:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.020863:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677448.020863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:24.0:1619677448.020865:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.020866:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.020868:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.020869:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.020870:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.020870:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677448.020871:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.020873:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:9.2:1619677448.020874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677448.020874:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.020876:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.020876:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.020879:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.020881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.020882:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.020885:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.020886:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.020886:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.020888:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:2.0:1619677448.020889:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.020891:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.020892:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.020892:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:2.0:1619677448.020892:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677448.020893:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.020894:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.020951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.020957:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.2:1619677448.020959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677448.020960:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.020961:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.020963:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.020964:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.020966:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677448.020967:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:18.0:1619677448.020967:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.020968:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.020969:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.020970:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:18.0:1619677448.020971:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.020973:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.020973:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.020975:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.020977:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.020977:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.020979:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.020981:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720423] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.020981:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.020982:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.020982:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677448.020983:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.020984:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.020984:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.020984:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:30.0:1619677448.020985:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.020985:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.020986:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:30.0:1619677448.020988:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.020990:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.020993:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.020994:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.020995:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.020996:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:13.0:1619677448.020996:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.020997:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.020998:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:13.0:1619677448.020999:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:30.0:1619677448.021000:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.021000:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:13.0:1619677448.021000:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.021000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:13.0:1619677448.021001:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.021002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677448.021003:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.021004:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.021006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.021007:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.021010:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.021010:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.021010:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:26.0:1619677448.021012:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.021012:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.021013:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.021014:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:26.0:1619677448.021015:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.021015:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.021015:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.021017:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.021017:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677448.021017:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:5.0:1619677448.021017:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:26.0:1619677448.021018:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.021019:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:26.0:1619677448.021019:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.021020:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.021021:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.021025:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.021027:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.021029:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.021030:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.021031:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.021032:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.021033:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.021034:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.021034:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.021046:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.021049:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.021050:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.021050:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.021052:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.021055:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.021120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.021127:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.021130:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.021132:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.021134:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.021135:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.021136:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.021138:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.021139:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.021140:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.021141:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.021144:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.021146:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.021148:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.021148:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.021149:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.021150:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.021151:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.021152:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.021152:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.021162:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.021164:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.021165:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.021165:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.021167:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.021170:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.021311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.021319:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.021322:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.021323:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.021324:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.021327:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.021328:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.021329:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.021331:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.021331:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.021332:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.021335:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.021336:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.021339:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.021340:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.021340:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.021342:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.021342:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.2:1619677448.021342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.021343:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.021343:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677448.021349:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.021351:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.021352:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.021354:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.021355:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:17.0:1619677448.021357:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.021357:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:17.0:1619677448.021358:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:12.0:1619677448.021358:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.021360:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.021360:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.021361:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.021362:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.021362:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.021363:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:17.0:1619677448.021364:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.021367:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.021369:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.021372:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.021373:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:20.2:1619677448.021374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677448.021374:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.021375:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.021376:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.021377:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.021377:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.021382:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.021385:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.021387:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.021388:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:31.0:1619677448.021390:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.021390:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.021390:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:30.0:1619677448.021391:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:13.0:1619677448.021391:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.021393:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677448.021393:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.021394:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.021394:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:30.0:1619677448.021395:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.021396:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.021396:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:30.0:1619677448.021400:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.021402:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.021404:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.021405:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.021405:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.021407:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.021407:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.021408:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.021409:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677448.021419:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.021421:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.021422:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.021423:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.021425:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.021426:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.021426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677448.021432:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677448.021435:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677448.021436:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677448.021438:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.021441:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.021442:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677448.021443:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.021445:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.021446:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.021447:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677448.021450:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.021452:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677448.021455:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.021456:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:1.0:1619677448.021457:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:1.0:1619677448.021458:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:1.0:1619677448.021459:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677448.021460:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677448.021460:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:10.0:1619677448.021470:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.021473:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.021474:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.021474:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.021476:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.021480:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:4.2:1619677448.021596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.021603:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.021605:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.021606:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.021608:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.021610:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.021611:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.021612:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720416] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.021614:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.021615:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.021616:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.2:1619677448.021618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:26.0:1619677448.021619:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.021619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.021621:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.0:1619677448.021621:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.021623:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:17.0:1619677448.021624:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.021625:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.021626:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.021626:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:20.0:1619677448.021626:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.021626:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677448.021627:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:26.0:1619677448.021628:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.021628:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.021628:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677448.021629:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.021630:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:20.0:1619677448.021630:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.021630:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.021631:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.021631:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677448.021632:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.021633:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.021633:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.021634:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677448.021636:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.021636:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677448.021637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677448.021638:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.021638:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.021639:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.021640:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:22.0:1619677448.021642:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:20.0:1619677448.021644:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.021644:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.021645:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:17.0:1619677448.021645:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.021645:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.021646:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.021646:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.021647:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677448.021648:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.021648:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.021649:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.021649:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677448.021651:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677448.021651:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.021651:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.021652:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.021652:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.021652:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.021653:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.021654:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.021654:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.021655:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.021655:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.021656:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677448.021656:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:20.0:1619677448.021657:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677448.021657:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.021658:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.021661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.021664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.021667:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.021670:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677448.021670:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.021671:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.021671:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:13.0:1619677448.021671:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.021671:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.021671:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.021672:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677448.021673:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.021674:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.021675:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677448.021676:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.021676:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.021679:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.021679:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.021728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.021735:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.021738:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.021739:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.021740:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.021743:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.021745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:8.0:1619677448.021745:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.021746:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.021748:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.021748:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.021750:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.021753:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:8.0:1619677448.021753:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.021755:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.0:1619677448.021755:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:31.0:1619677448.021758:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.021758:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.021759:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:30.0:1619677448.021760:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677448.021760:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677448.021761:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.021761:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:31.0:1619677448.021762:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:8.0:1619677448.021762:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.021762:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:31.0:1619677448.021763:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.021763:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:31.0:1619677448.021765:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.021766:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.021767:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:8.2:1619677448.021767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677448.021770:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.021772:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.021774:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:31.0:1619677448.021775:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.021776:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677448.021776:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.021777:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:31.0:1619677448.021778:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:17.0:1619677448.021778:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.021778:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677448.021779:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.021780:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.021780:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:17.0:1619677448.021780:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.021783:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:6.0:1619677448.021792:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.021795:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.021795:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.021796:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.021798:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.021801:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.021912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.021920:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.021923:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.021925:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.021926:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.021927:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.021927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:30.0:1619677448.021929:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.021930:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720411] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.021931:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.021932:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.021933:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.021934:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:21.2:1619677448.021935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:30.0:1619677448.021937:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.021937:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.021938:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:30.0:1619677448.021940:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:22.0:1619677448.021940:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677448.021941:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.021943:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.021943:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.021943:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677448.021944:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.021944:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:2.0:1619677448.021944:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.021945:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:30.0:1619677448.021946:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:22.0:1619677448.021946:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.021947:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.021947:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:2.0:1619677448.021947:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.021948:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:22.0:1619677448.021948:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.021948:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.021949:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.021950:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677448.021951:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.021952:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.021952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:22.0:1619677448.021953:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.021955:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.021955:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.021956:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:2.0:1619677448.021958:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.021960:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.021960:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677448.021961:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.021962:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.021962:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:2.0:1619677448.021962:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.021962:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:5.0:1619677448.021963:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.021964:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.021965:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.021965:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.021965:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:22.0:1619677448.021966:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:5.0:1619677448.021966:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.021966:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.021966:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:22.0:1619677448.021967:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.021969:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677448.021970:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.021970:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.2:1619677448.021970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.021971:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.021971:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677448.021978:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.021980:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.021981:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.021982:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:20.0:1619677448.021983:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.021984:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.021985:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677448.021985:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:31.0:1619677448.021986:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.021988:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.021989:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.021991:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.021994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.022000:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.2:1619677448.022001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.022003:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:18.0:1619677448.022003:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.022004:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:30.0:1619677448.022006:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677448.022006:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.022007:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.022007:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:1.0:1619677448.022007:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.022008:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.022009:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:18.0:1619677448.022009:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.022009:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.022010:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677448.022011:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.022013:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.022013:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.022014:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.022015:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.022015:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.022017:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677448.022017:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.022018:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.022021:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.022022:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.022022:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.022023:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:18.0:1619677448.022024:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:1.0:1619677448.022026:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.022027:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.022028:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:1.0:1619677448.022028:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:18.0:1619677448.022029:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.022031:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.022031:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:1.0:1619677448.022031:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.022032:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.022032:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:1.0:1619677448.022032:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.022033:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.022035:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.022035:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.022036:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.022036:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.022041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:26.0:1619677448.022043:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.022045:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.022046:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.022047:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.022047:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:26.0:1619677448.022049:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.022050:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.022051:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.022051:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:26.0:1619677448.022052:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:24.0:1619677448.022053:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.022056:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.2:1619677448.022069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.022076:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.022079:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.022080:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.022082:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.022084:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.022086:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.022087:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.022088:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.022089:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.022090:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.022094:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.022096:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.022098:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.022099:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.022100:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.022102:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.022102:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.022103:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.022105:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.022105:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.022111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.022114:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.022116:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.022117:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.022118:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:12.0:1619677448.022118:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.022120:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:13.0:1619677448.022120:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.022121:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:12.0:1619677448.022121:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.022122:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.022132:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.022134:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.022135:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.022135:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.022136:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.022137:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.022365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.022372:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.022375:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.022376:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.022377:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.022379:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.022381:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.022382:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720405] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.022383:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.022384:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.022385:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.022389:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.022390:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.022393:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.022393:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.022394:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.022396:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.022397:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.022398:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.022398:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.022403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.022411:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.022414:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.022415:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.022415:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.022417:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.022421:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.022448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.022457:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.022459:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.022461:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.022464:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.022464:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.022466:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.022467:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.022468:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.022468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:31.0:1619677448.022469:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.022470:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.022473:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.022474:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.022475:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.022476:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.022477:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677448.022478:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.022478:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.022478:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.022480:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.022480:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.022480:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.022481:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.022482:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:17.0:1619677448.022483:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.022484:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.022485:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.022487:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.022488:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.022489:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.022492:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.022492:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677448.022493:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:13.0:1619677448.022495:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:17.0:1619677448.022496:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677448.022496:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.022497:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.022497:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:13.0:1619677448.022497:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.022499:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.022499:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.022499:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.022500:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.022501:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.022501:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.022510:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.022513:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.022513:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.022514:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.022515:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677448.022515:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:12.0:1619677448.022516:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677448.022517:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.022519:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.022519:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.022521:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.022523:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.022532:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.022534:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.022535:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.022535:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.022536:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.022537:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.022544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.022551:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.022554:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.022556:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.022558:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.022559:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.022560:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.022561:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.022563:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.022564:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.022565:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.022568:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.022569:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.022572:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.022573:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.022573:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.022575:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.022575:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.022576:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.022577:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.022587:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.022589:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.022590:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.022591:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.022593:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.022593:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.2:1619677448.022627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.022632:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.022635:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.022637:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.022638:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.022640:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.022642:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.022643:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.022644:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.022645:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.022646:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.022650:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.022651:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.022653:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.022655:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.022655:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.022657:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.022657:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.022658:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.022659:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677448.022670:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.022673:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.022674:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.022674:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.022676:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.022678:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.022775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.022782:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.022785:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.022785:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.022787:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.022790:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.022792:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.022793:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.022794:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.022795:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.022796:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.2:1619677448.022797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:26.0:1619677448.022799:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.022800:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.022803:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.022803:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:26.0:1619677448.022804:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.022805:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.022806:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.022806:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:26.0:1619677448.022807:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.022807:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:26.0:1619677448.022808:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.022808:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:18.0:1619677448.022809:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.022811:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.022813:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.022814:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.022816:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.022817:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.022818:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.022820:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:18.0:1619677448.022821:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.022822:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:18.0:1619677448.022822:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.022823:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.022825:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.022825:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.022826:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.022826:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.022827:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.022828:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.022829:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677448.022829:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.022829:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.022830:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677448.022842:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.022844:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.022845:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.022845:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.022847:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.022849:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.022856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.022861:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.022864:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.022865:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.022866:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.022869:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.022870:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.022871:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.022873:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.022874:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.022874:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.022877:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.022879:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.022882:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.022882:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.022883:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.022884:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.022885:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.022886:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.022886:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:10.0:1619677448.022897:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.022899:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.022900:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.022901:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.022903:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.022906:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.022915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:4.2:1619677448.022915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.0:1619677448.022918:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.2:1619677448.022919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1200] (20)++
00000800:00000200:20.0:1619677448.022921:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:8.0:1619677448.022922:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:20.0:1619677448.022923:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.022925:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.022925:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:5.0:1619677448.022925:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:1.0:1619677448.022925:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:24.0:1619677448.022926:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:20.0:1619677448.022928:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:1.0:1619677448.022928:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000400:00000200:20.0:1619677448.022929:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.022929:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:8.0:1619677448.022929:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.022931:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.022932:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.022932:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.022933:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.022934:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.022934:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.022935:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.022937:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.022938:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.022939:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.022940:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.022941:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.022941:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.022941:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:1.0:1619677448.022943:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.022944:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.022944:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:1.0:1619677448.022944:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.022945:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:1.0:1619677448.022945:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.022946:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.022947:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.022951:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.022953:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.022953:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677448.022953:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.022954:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:1.0:1619677448.022954:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 584 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:8.0:1619677448.022960:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.022962:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677448.022962:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.022963:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.022964:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.022965:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.022965:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.022965:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:8.0:1619677448.022966:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.022966:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:17.0:1619677448.022967:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.022967:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (23)++
00000800:00000200:1.0:1619677448.022968:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:8.0:1619677448.022969:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (24)--
00000800:00000200:1.0:1619677448.022969:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677448.022970:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677448.022970:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.022970:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677448.022970:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:8.0:1619677448.022971:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677448.022971:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:17.0:1619677448.022972:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.2:1619677448.022974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:8.2:1619677448.022978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.022982:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.022984:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.022985:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.022986:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.022986:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677448.022987:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:1.0:1619677448.022988:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.022989:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.022989:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:1.0:1619677448.022989:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.022991:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.022994:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.023087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.023092:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.023095:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.023096:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.023097:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.023099:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.023101:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.023102:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.023103:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.023104:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.023105:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.023109:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.023110:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.023113:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.023114:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.023115:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.023117:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.023117:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.023118:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.023118:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677448.023123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.023126:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.023129:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:12.0:1619677448.023129:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:20.0:1619677448.023131:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.023131:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.023132:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.023132:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.023133:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.023134:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.023135:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:13.0:1619677448.023135:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:20.0:1619677448.023136:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.023137:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:20.0:1619677448.023138:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.023138:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.023140:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.023143:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.023145:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.023148:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.023149:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.023149:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.023151:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.023152:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.023152:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.2:1619677448.023153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:20.0:1619677448.023153:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677448.023159:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.023162:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.023162:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:31.0:1619677448.023164:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:22.0:1619677448.023164:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.023167:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:22.0:1619677448.023167:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.023168:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.023169:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677448.023170:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.023170:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:22.0:1619677448.023171:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.023173:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:22.0:1619677448.023173:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677448.023173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:22.0:1619677448.023174:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.023175:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.023178:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.023180:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.023180:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.023182:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:22.0:1619677448.023183:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.023183:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677448.023184:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:26.0:1619677448.023185:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.023185:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.023187:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.023187:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:26.0:1619677448.023188:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.023188:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:26.0:1619677448.023189:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:22.0:1619677448.023189:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:26.0:1619677448.023190:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.023192:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.023193:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.023194:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.023197:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.023199:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.023200:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:26.0:1619677448.023201:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.023202:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:26.0:1619677448.023202:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.023203:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:30.0:1619677448.023204:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.023205:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.023205:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.023205:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.023206:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.023206:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.023207:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.023208:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.2:1619677448.023211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.023219:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.023221:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.023222:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.023224:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.023224:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.023226:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.023297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.023305:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.023307:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.023308:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.023310:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.023313:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.023314:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.023315:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.023317:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.023318:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.023318:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.023321:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.023322:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.023324:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.023325:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.023326:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.023327:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.023327:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.023328:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.023329:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677448.023341:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.023344:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.023344:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.023346:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.023348:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.023348:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.023486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.023492:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.023495:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.023496:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.023498:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.023501:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.023502:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.023503:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.023505:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.023506:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.023507:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.023510:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.023512:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.023515:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.023516:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.023516:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.023518:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.023518:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.023520:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.023521:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.023521:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.023524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.023526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.023531:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677448.023531:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677448.023534:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:17.0:1619677448.023534:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.023534:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677448.023535:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.023536:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.023536:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677448.023538:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.023541:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.023544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.023551:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.023554:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.023555:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.023557:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.023559:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.023561:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.023562:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.023563:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.023564:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.023565:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.023569:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.023571:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.023574:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.023574:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.023575:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.023577:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.023577:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.023578:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.023580:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.023580:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.023586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.023591:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677448.023593:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.023593:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.023594:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.023594:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.023595:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.023596:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.023596:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.023597:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.023600:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.023657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.023665:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.2:1619677448.023665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:31.0:1619677448.023667:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.023669:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.023671:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677448.023671:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:31.0:1619677448.023672:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.023673:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.023674:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.023674:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.023675:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:31.0:1619677448.023676:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.023678:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.023678:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.023679:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.023681:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.023683:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.023683:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.023684:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.023686:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.023687:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.023688:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.023689:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.023690:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.023691:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.023691:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.023693:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.023693:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677448.023693:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.023695:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:18.0:1619677448.023695:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.023696:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.023697:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677448.023698:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.023699:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.023700:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.023701:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.023702:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:21.2:1619677448.023703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.023703:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.023703:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677448.023706:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.023709:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:30.0:1619677448.023710:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:13.0:1619677448.023710:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.023711:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.023713:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.023713:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677448.023713:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.023713:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.2:1619677448.023714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677448.023714:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.023716:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:30.0:1619677448.023717:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.023717:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.023717:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.023718:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677448.023719:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.023722:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:30.0:1619677448.023723:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.023726:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.023727:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.023728:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.023729:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.023730:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.023731:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.023732:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.023732:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000400:00000200:30.0:1619677448.023733:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.023734:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.023735:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.023735:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.023736:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:30.0:1619677448.023737:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.023737:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:30.0:1619677448.023739:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.023741:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.023742:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.023743:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.023744:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.023745:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.023746:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.023747:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.023748:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677448.023753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.023758:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.023760:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.023760:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.023762:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.023762:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.023763:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.023763:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.023764:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.023765:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677448.023766:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:4.2:1619677448.023833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.023840:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.023842:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.023843:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.023845:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.023848:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.023849:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.023850:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.023851:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.023852:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.023853:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.023856:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.023858:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.023860:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.023861:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.023862:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.023863:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.023864:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.023865:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.023866:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.023867:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.023869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677448.023872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.023877:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677448.023879:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.023879:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.023880:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.023881:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677448.023881:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.023882:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.023883:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.023883:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.023884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:1.0:1619677448.023886:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:10.0:1619677448.023892:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.023894:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677448.023895:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677448.023897:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.023900:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.023901:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.023903:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.023904:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.023905:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.023906:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.023909:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.023911:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.023913:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.023914:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.023914:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.023916:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.023916:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.023917:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677448.023918:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677448.023919:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:21.2:1619677448.023925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:12.0:1619677448.023927:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:4.2:1619677448.023927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677448.023930:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.023931:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.023932:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.023933:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:13.0:1619677448.023934:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.023935:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677448.023935:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:12.0:1619677448.023935:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.023937:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.023937:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:26.0:1619677448.023938:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.023938:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.023940:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.023943:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.023945:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.023946:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.023947:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.023948:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.023949:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.023952:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.023953:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.023956:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.023957:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.023957:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.023959:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.023959:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.023960:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.023960:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.023973:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.023976:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.023976:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.023977:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.023978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:22.0:1619677448.023979:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.023980:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677448.023982:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677448.023983:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.023985:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.023986:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.023987:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.023989:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.023990:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.023992:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.023993:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.023994:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.023997:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.023998:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.024000:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.024001:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.024002:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.024003:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.024003:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.024004:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.024004:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677448.024004:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.2:1619677448.024004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677448.024005:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.024005:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.024006:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:6.0:1619677448.024006:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677448.024007:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.024008:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677448.024011:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.2:1619677448.024013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.024014:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.024015:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:13.0:1619677448.024017:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677448.024017:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.024018:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677448.024019:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.024020:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.024020:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.024021:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.024021:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:13.0:1619677448.024022:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.024022:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:5.0:1619677448.024022:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.024023:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.024023:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.024025:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677448.024025:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.024026:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.024027:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.024030:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.024031:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677448.024033:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.024034:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.024035:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:12.0:1619677448.024036:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.024036:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677448.024037:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.024037:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.024038:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.024038:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.024038:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.024039:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.024039:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.024040:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.024044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.024051:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.024054:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.024055:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.024055:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.024058:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.024061:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.024126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.024131:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.024134:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.024135:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.024137:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.024139:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.024141:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.024142:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.024143:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.024145:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.024145:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.024149:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.024151:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.024154:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.024155:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.024155:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.024157:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.024158:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.024158:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.024159:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.024170:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.024173:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.024173:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.024175:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.024176:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.024178:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.024200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.024207:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.024210:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.024213:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.024214:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.024214:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.024216:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.024217:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.024218:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.024219:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.024220:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.024223:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.024225:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.024228:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.024229:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.024230:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.024231:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.024232:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.024233:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.024233:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.024237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.024240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.024245:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677448.024245:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677448.024248:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:18.0:1619677448.024248:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.024249:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.024249:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677448.024249:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:22.0:1619677448.024252:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.024252:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.024253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:22.0:1619677448.024255:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.024257:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:18.0:1619677448.024257:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:22.0:1619677448.024258:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.024258:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.024259:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:22.0:1619677448.024260:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.024260:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:22.0:1619677448.024262:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.024263:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.024264:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677448.024266:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.024267:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.024268:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:22.0:1619677448.024270:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:18.0:1619677448.024270:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.024271:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.024272:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.024273:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.024273:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.024274:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.024275:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.024277:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.024277:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:18.0:1619677448.024277:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.024279:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677448.024279:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.024280:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.024280:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677448.024282:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.024283:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.024284:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.024285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.2:1619677448.024285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:18.0:1619677448.024285:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.024286:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.024287:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.024288:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.024289:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:6.0:1619677448.024290:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:26.0:1619677448.024291:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:6.0:1619677448.024292:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.024296:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (23)--
00000800:00000200:10.0:1619677448.024296:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:21.2:1619677448.024297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677448.024299:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.024300:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.024301:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677448.024301:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:26.0:1619677448.024302:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.024302:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:2.0:1619677448.024302:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:12.0:1619677448.024304:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.024304:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.024305:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.024305:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:26.0:1619677448.024306:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:12.0:1619677448.024306:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.024306:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.024307:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:26.0:1619677448.024308:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000400:00000200:10.0:1619677448.024308:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.024309:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:13.0:1619677448.024309:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.024310:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:12.0:1619677448.024310:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677448.024310:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:26.0:1619677448.024311:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.024311:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.024312:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.024313:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.024313:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.024314:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:10.0:1619677448.024314:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.024315:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.024320:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.024322:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.024324:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.024325:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.024326:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.024328:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.024328:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.024329:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.024329:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677448.024334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.024342:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.024345:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.024347:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.024347:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.024349:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.024352:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.024355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.024362:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.024365:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.024366:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.024366:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.024367:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.024368:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.024371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.024377:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.024380:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.024382:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.024382:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677448.024384:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.024386:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.024387:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.024389:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.024390:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.024390:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.024394:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.024395:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.024398:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.024399:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.024400:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.024402:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.024403:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.024404:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.024404:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:13.0:1619677448.024416:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.024418:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.024419:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.024421:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677448.024423:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.024424:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.024537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.024544:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.024548:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.024549:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.024551:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.024554:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.024555:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.024556:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.024557:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.024559:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.024559:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.024562:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.024564:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.024567:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.024567:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.024568:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.024570:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.024570:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.024571:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.024571:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.024583:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.024587:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.024588:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.024588:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.024590:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.024593:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.024606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677448.024615:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677448.024618:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677448.024620:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.024622:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677448.024623:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.024624:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:5.2:1619677448.024624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:31.0:1619677448.024626:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.024627:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.024628:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.024629:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.024629:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.024632:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:31.0:1619677448.024633:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.024633:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677448.024635:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.024636:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.024638:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.024639:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.024639:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:2.0:1619677448.024639:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.024640:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:31.0:1619677448.024641:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.024641:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677448.024642:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:2.0:1619677448.024642:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.024643:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000400:00000200:2.0:1619677448.024643:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.024644:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.024645:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.024648:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.024650:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.024652:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.024653:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677448.024653:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.024653:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677448.024655:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:2.0:1619677448.024655:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.024655:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:12.0:1619677448.024656:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.024656:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.024657:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:12.0:1619677448.024658:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.024660:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.024660:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677448.024661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.024668:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.024671:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.024671:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.024672:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.024673:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.024675:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.024690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.024697:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.024700:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.024703:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.024704:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.024705:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.024706:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.024708:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.024709:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.024710:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.024711:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.024715:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.024716:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.024719:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.024720:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.024721:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.024722:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.024723:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.024724:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.024725:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.024737:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.024739:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.024740:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.024741:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.024743:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.024746:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.024778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.2:1619677448.024780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.024785:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677448.024786:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677448.024787:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.024788:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677448.024791:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677448.024791:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.024791:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:22.0:1619677448.024793:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.024794:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.024795:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.024796:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.024797:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.024797:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.024798:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677448.024799:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.024799:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.024800:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677448.024801:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.024803:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.024804:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677448.024805:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.024807:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:8.0:1619677448.024809:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.024810:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.024811:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.024811:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677448.024811:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677448.024813:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.024814:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.024814:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677448.024814:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.024814:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:22.0:1619677448.024815:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677448.024815:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.024817:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.024817:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.024817:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.024818:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677448.024819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.2:1619677448.024823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677448.024826:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.024829:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.024830:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.024830:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.024831:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:2.0:1619677448.024832:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.024834:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:1.0:1619677448.024834:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.024836:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.024836:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.024838:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.024841:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.024843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.024850:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.024852:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.024853:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.024853:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.024854:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.024855:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.024950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.024956:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.024958:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.024959:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.024961:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.024963:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.024964:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.024966:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.024967:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.024968:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.024969:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.024972:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.024974:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.024976:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.024976:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.024977:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.024978:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.024979:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.024980:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.024981:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.024982:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:9.2:1619677448.024984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.024987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.024992:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.024993:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.024994:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:2.0:1619677448.024994:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.024995:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.024995:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.024996:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.024996:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.024997:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677448.024998:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.025000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.025008:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.025011:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.025013:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.025015:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.025016:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.025017:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.025019:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.025020:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.025021:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.025022:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.025025:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.025027:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.025029:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.025030:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.025031:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.025032:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.025032:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.025034:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.025034:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.2:1619677448.025036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677448.025044:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:10.0:1619677448.025044:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.025046:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:30.0:1619677448.025047:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.025047:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.025047:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:30.0:1619677448.025049:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.025049:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.025051:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.025052:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:10.0:1619677448.025052:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.025054:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.025055:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.025056:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.025056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677448.025057:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.025058:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.025061:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.025063:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.025063:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.025064:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:1.0:1619677448.025065:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.025066:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.025067:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.025067:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:10.0:1619677448.025067:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.025067:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:31.0:1619677448.025068:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:10.0:1619677448.025068:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677448.025068:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.025069:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.025070:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:10.0:1619677448.025070:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.025070:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.025071:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:1.0:1619677448.025071:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.025072:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:1.0:1619677448.025072:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.025073:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.025075:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677448.025076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677448.025076:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.025077:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.025080:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.025082:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.025084:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677448.025086:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.025086:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.025087:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.025087:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:1.0:1619677448.025087:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677448.025088:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.025089:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.025089:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677448.025090:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.025090:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.025091:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677448.025093:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677448.025102:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.025105:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.025105:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.025107:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.025108:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:21.2:1619677448.025110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:13.0:1619677448.025110:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677448.025117:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.025120:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.025120:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677448.025122:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677448.025123:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.025124:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677448.025125:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.025125:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.025126:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.025126:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677448.025127:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:12.0:1619677448.025127:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677448.025128:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.025130:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.025131:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.025132:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.025135:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.025136:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.025139:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.025140:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.025140:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.025142:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.025142:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.025143:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.025144:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677448.025154:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.025156:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.025157:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.025158:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.025161:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.025162:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.025208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.025213:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.025216:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.025217:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.025218:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.025220:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.025222:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.025223:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.025225:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.025225:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.025227:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.025230:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.025231:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.025234:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.025235:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.025235:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.025237:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.025237:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.025238:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.025239:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.025240:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.025243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.025245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.025250:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:17.0:1619677448.025251:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677448.025252:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:4.2:1619677448.025252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677448.025253:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677448.025253:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677448.025254:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:17.0:1619677448.025254:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.025255:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.025256:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.025259:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.025259:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677448.025262:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:8.2:1619677448.025262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:10.0:1619677448.025263:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.025265:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.025268:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.025269:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:1.0:1619677448.025269:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677448.025271:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.025272:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.025272:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677448.025273:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.025273:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000400:00000200:26.0:1619677448.025274:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.025274:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000400:00000200:26.0:1619677448.025275:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.025275:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:26.0:1619677448.025278:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.025280:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.025282:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.025283:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.025284:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.025285:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.025286:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.025286:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.025287:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.025299:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.025301:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.025302:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.025304:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.025306:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.025307:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.025479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.025485:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.025488:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.025489:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.025491:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.025493:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.025494:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.025496:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.025497:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.025498:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.025499:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.025503:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.025505:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.025507:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.025508:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.025509:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.025510:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.025511:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.025512:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.025512:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.025517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.025526:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.025529:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.025529:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.025530:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.025532:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.025534:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.025641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.025646:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.025648:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.025649:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.025651:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.025654:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.025655:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.025656:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.025657:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.025659:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.025659:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.025663:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.025664:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.025666:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.025667:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.025667:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.025669:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.025669:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.025670:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.025670:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.025675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.025682:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.025684:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.025685:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.025685:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.025686:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.025688:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.025733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.025741:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.025743:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.025746:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.025748:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.025748:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.025750:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677448.025751:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.025752:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.025754:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.025755:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.025758:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.025760:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.025763:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.025764:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.025764:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.025766:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.025767:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.025768:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.025768:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.025772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.025779:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.025782:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.025783:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.025783:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.025785:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.025788:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.025814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677448.025819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:10.0:1619677448.025822:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.025825:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.025825:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:2.0:1619677448.025825:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:10.0:1619677448.025828:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.025828:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.025829:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:10.0:1619677448.025831:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.025831:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.025832:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.025833:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.025834:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.025835:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.025836:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.025837:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.025837:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.025838:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.025838:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.025840:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.025841:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.025843:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.025845:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.025846:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.025848:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.025849:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.025850:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.025850:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.025851:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.025852:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.025852:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.025853:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.025853:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.025854:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.025855:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:2.0:1619677448.025855:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.025855:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:10.0:1619677448.025856:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677448.025856:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.025857:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:4.2:1619677448.025861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.025866:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.025868:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.025870:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677448.025870:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.025871:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.025871:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.025871:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677448.025871:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.025872:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.025873:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.025874:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.025875:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.025876:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:13.0:1619677448.025876:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.025878:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.025879:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.025890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.025893:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.025895:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.025898:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.025898:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.025901:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.025902:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.025904:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.025905:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.025906:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.025907:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.025910:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.025912:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.025915:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.025916:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.025917:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.025919:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.025920:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.025920:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.025921:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.025931:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.025934:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.025935:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.025937:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.025938:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.025939:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.026064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677448.026069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:30.0:1619677448.026072:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.026074:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:10.0:1619677448.026076:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:30.0:1619677448.026077:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.026078:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.026078:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:30.0:1619677448.026079:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.026079:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:30.0:1619677448.026082:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.026082:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:30.0:1619677448.026083:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.026084:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.026085:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.026085:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.026086:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.026086:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.2:1619677448.026086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677448.026087:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.026089:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.026090:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.026090:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.026091:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.026092:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.026092:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.026094:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:10.0:1619677448.026095:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677448.026095:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.026096:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.026097:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.026098:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:17.0:1619677448.026098:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.026098:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.026100:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.026100:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.026100:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.026100:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.026101:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.026101:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.026101:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.026102:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:17.0:1619677448.026102:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.026103:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.026103:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.026103:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:17.0:1619677448.026104:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.026104:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:17.0:1619677448.026105:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.026105:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:17.0:1619677448.026106:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.2:1619677448.026106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:17.0:1619677448.026109:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.026111:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.026113:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.026113:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677448.026114:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:13.0:1619677448.026114:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.026115:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.026116:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.026116:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.026116:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.026116:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.026117:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677448.026118:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677448.026118:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:13.0:1619677448.026118:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.026119:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.026119:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677448.026119:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:13.0:1619677448.026120:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677448.026121:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.2:1619677448.026122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.026122:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677448.026123:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.026124:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.2:1619677448.026126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:1.0:1619677448.026130:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.026131:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.026131:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.026132:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:1.0:1619677448.026132:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.026133:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677448.026133:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.026134:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.026134:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.026135:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677448.026136:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:1.0:1619677448.026136:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677448.026137:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.026137:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.026138:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.026139:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.026219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.026226:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.026229:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.026230:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.026232:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.026234:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.026236:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.026237:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.026239:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.026240:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.026241:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.026244:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.026246:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.026249:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.026250:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.026250:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.026252:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.026253:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.026253:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.026254:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.026258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.026265:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.026268:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.026269:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.026270:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.026272:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.026274:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.026278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.026285:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.026288:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.026289:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.026289:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.026290:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.026290:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.026455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.026461:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.026463:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.026464:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.026466:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.026467:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.026468:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.026470:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.026471:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.026472:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.026473:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.026476:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.026478:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.026480:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.026481:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.026482:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.026484:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.026484:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.026485:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.026485:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.026490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.026497:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.026500:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677448.026501:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677448.026502:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.026504:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.026507:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677448.026509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:8.0:1619677448.026515:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.026518:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.026518:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677448.026518:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.026519:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.026520:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677448.026530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.026538:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.026540:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.026542:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.026545:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.026545:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.026547:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677448.026548:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.026550:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.026551:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.026552:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.026555:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.026557:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.026560:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.026561:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.026561:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.026563:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.026564:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.026565:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.026567:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.026567:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677448.026569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.026573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.026576:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.026579:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.026580:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.026580:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.026581:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:1.0:1619677448.026582:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.026583:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.026584:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.026585:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677448.026586:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.026600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.026608:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.026611:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.026612:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.026613:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.026616:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.026618:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.026619:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.026620:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.026621:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.026622:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.026625:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.026627:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.026629:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.026630:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.026630:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.026632:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.026633:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.026633:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.026634:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.026647:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.026650:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.026650:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.026651:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.026653:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.026656:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677448.026664:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.026667:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.026668:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.026668:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.026669:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.026670:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.026822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.026830:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.026833:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.026835:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.026836:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.026837:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.026839:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.026840:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.026840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677448.026842:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.026843:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.026844:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.026845:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677448.026847:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.026848:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:30.0:1619677448.026849:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.026849:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.026851:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.026852:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.026852:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.026853:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:17.0:1619677448.026854:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.026855:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.026855:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.026855:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:30.0:1619677448.026856:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:17.0:1619677448.026856:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.026857:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:17.0:1619677448.026858:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.026859:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.026860:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.026863:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.026865:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.026867:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.026867:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.026868:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.026869:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.026870:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.026870:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.026871:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.026871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.2:1619677448.026875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.026880:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.026882:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.026883:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.026883:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:20.0:1619677448.026885:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.026885:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.026885:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.026886:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.026886:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.026888:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:1.0:1619677448.026888:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.026890:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.027052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.027059:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.027062:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.027063:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.027065:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.027065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000400:00000200:10.0:1619677448.027068:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.027070:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:17.0:1619677448.027071:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000400:00000200:10.0:1619677448.027071:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.027072:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.027074:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:10.0:1619677448.027074:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.027074:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:10.0:1619677448.027075:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.027077:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.027079:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.027080:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.027081:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.027082:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.027082:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.027084:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.027085:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.027085:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.027086:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.027086:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.027087:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:17.0:1619677448.027089:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.027089:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.027089:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.027090:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.027091:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.027091:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677448.027095:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.027096:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.027096:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.027098:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.027098:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677448.027100:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677448.027101:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677448.027101:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:6.0:1619677448.027102:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.027105:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:5.2:1619677448.027105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677448.027106:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.027106:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.027108:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.027109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:12.0:1619677448.027111:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677448.027111:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:26.0:1619677448.027113:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.027114:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:26.0:1619677448.027115:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:12.0:1619677448.027115:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.027115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:1.0:1619677448.027115:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000400:00000200:12.0:1619677448.027116:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.027116:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677448.027117:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000400:00000200:26.0:1619677448.027118:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.027118:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.027118:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677448.027119:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:12.0:1619677448.027120:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677448.027121:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.027122:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:18.0:1619677448.027122:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:26.0:1619677448.027123:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.027125:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.027125:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.027125:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:26.0:1619677448.027126:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.027127:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.027128:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.027130:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:26.0:1619677448.027131:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.027131:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.027133:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.027134:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:13.0:1619677448.027134:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.027135:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.027135:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677448.027136:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.027136:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.027136:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.027137:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677448.027137:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:18.0:1619677448.027137:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.027137:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:18.0:1619677448.027138:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.027139:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:18.0:1619677448.027139:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.027140:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.027141:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.027142:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.027142:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:18.0:1619677448.027143:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.027144:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.027147:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.027148:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.027148:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.027148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677448.027150:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.027151:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.027151:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:13.0:1619677448.027151:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.027152:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677448.027153:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.027154:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.027155:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:13.0:1619677448.027156:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.027157:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.027157:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.027158:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.027158:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:13.0:1619677448.027159:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.027211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:8.2:1619677448.027213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.027217:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.0:1619677448.027220:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677448.027220:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.027221:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.027222:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.027223:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.027223:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.027224:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.027225:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.027226:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:2.0:1619677448.027226:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.027227:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.027229:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.027229:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.027230:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.027232:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:2.0:1619677448.027233:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.027235:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.027238:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.027239:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.027239:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:8.2:1619677448.027241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.027241:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.027241:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.027242:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.027243:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677448.027248:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.027251:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.027252:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:10.0:1619677448.027253:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:5.0:1619677448.027253:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.027254:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.027255:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:10.0:1619677448.027256:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.027257:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.027258:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.027260:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.027263:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.027313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.027319:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.027321:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.027322:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.027324:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.027326:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.027328:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.027329:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.027330:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.027331:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.027332:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.027335:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.027336:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.027338:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.027338:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.027339:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.027341:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.027341:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.027342:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.027342:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677448.027355:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.027358:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.027359:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.027359:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.027361:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.027364:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.027400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.027405:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.027407:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.027408:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.027410:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.027412:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.027412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:6.0:1619677448.027413:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.027414:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.027416:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.027417:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.027418:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:6.0:1619677448.027418:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.027421:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.027421:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677448.027422:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.027424:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.027424:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.027426:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.027427:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.027427:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.027428:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:17.0:1619677448.027429:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:6.0:1619677448.027429:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:17.0:1619677448.027430:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.027430:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.027430:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:17.0:1619677448.027431:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.027431:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:17.0:1619677448.027432:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.027434:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.027437:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.027439:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.027442:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.027442:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.027443:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.027445:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.027445:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.027446:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.027447:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.027459:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.027461:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.027462:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.027462:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.027464:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.027465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:22.0:1619677448.027467:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677448.027474:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:6.0:1619677448.027475:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.027477:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:22.0:1619677448.027478:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:8.2:1619677448.027478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.027478:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.027479:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000400:00000200:31.0:1619677448.027480:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.027480:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.027480:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:31.0:1619677448.027482:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.027482:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.027484:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.027485:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.027486:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.027486:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:31.0:1619677448.027487:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.027488:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:31.0:1619677448.027489:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:5.0:1619677448.027489:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.027491:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.027491:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.027493:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.027495:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.027497:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677448.027498:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.027499:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.027500:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.027502:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.027503:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.027504:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.027506:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.027507:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677448.027512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:21.2:1619677448.027515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.027521:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.027522:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.027523:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.027524:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.027524:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677448.027525:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677448.027525:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.027527:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.027529:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.027531:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677448.027532:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.027533:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.027534:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.027535:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.027538:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.027539:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.027542:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.027543:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.027543:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.027545:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.027545:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.027546:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.027547:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.027551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.2:1619677448.027555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:1.0:1619677448.027558:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.027561:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677448.027562:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:17.0:1619677448.027562:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.027562:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.027564:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.027565:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.027565:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.027567:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.027567:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677448.027568:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.027570:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.027646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.027654:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.027657:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.027658:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.027659:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.027662:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.027663:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.027665:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.027666:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.027667:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.027668:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.027671:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.027672:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.027675:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.027676:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.027676:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.027678:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.027678:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.027679:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.027680:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.027692:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.027695:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.027695:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.027696:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.027698:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.027700:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.027731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.027736:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.027739:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.027740:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.027742:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.027744:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.027746:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.027747:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.027748:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.027749:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.027750:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.027754:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.027756:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.027758:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.027759:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.027760:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.027762:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.027762:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.027763:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.027763:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:10.0:1619677448.027774:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.027777:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.027777:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.027778:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.027780:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.027783:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.027794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.027800:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.027802:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.027804:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.027805:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.027807:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.027809:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.027810:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.027812:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.027813:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.027814:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.027817:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.027819:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.027821:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.027822:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.027823:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.027824:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.027825:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.027826:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.027826:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677448.027826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.027833:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.027836:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.027836:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.027838:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.027838:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:24.0:1619677448.027840:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.027841:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:24.0:1619677448.027842:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:22.0:1619677448.027842:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.027842:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.027844:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96720332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.027844:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.027846:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.027846:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:24.0:1619677448.027847:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.027848:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.027852:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.027853:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.027856:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.027857:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.027858:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.027859:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.027860:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.027861:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.027861:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.027865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.027873:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.027876:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.027876:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.027877:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.027878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000400:00000200:17.0:1619677448.027879:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.027882:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677448.027886:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677448.027888:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:30.0:1619677448.027890:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.027892:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677448.027893:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.027894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000400:00000200:30.0:1619677448.027895:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.027896:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.027897:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.027898:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.027899:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:22.0:1619677448.027899:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677448.027902:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000400:00000200:30.0:1619677448.027903:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.027903:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:30.0:1619677448.027905:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:22.0:1619677448.027905:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.027908:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.027908:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:22.0:1619677448.027908:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.027909:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:22.0:1619677448.027909:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:30.0:1619677448.027910:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:22.0:1619677448.027910:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.027911:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677448.027912:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87b3e5800] (22)++
00000400:00000200:22.0:1619677448.027912:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.027913:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.027914:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:30.0:1619677448.027914:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:22.0:1619677448.027914:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677448.027917:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.027918:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.2:1619677448.027920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:22.0:1619677448.027921:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.027921:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:22.0:1619677448.027922:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.027923:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:22.0:1619677448.027924:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.027925:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:22.0:1619677448.027925:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000400:00000200:26.0:1619677448.027926:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.027926:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:10.0:1619677448.027926:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:22.0:1619677448.027927:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:31.0:1619677448.027928:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)++
00000400:00000200:26.0:1619677448.027928:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.027928:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:31.0:1619677448.027930:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:9.2:1619677448.027930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.027931:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:26.0:1619677448.027931:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677448.027932:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:20.2:1619677448.027933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (21)++
00000800:00000200:18.0:1619677448.027938:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677448.027938:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000800:00000200:18.0:1619677448.027941:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:6.0:1619677448.027941:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:22.0:1619677448.027942:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000400:00000200:18.0:1619677448.027942:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.027942:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:2.0:1619677448.027942:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.027944:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.027947:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.027952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.027959:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.027962:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.027963:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.027963:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.027964:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.027965:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.027979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.027986:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.027989:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.027990:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.027992:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.027994:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.027996:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.027997:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.027998:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.028000:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.028001:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.028004:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.028006:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.028008:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.028009:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.028010:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.028011:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.028012:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.028012:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.028013:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.028024:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.028027:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.028028:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.028029:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.028032:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.028033:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.028042:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.028044:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.028045:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.028046:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.028046:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.028047:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.028156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.028163:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.028166:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.028167:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.028168:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.028171:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.028172:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.028173:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.028175:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.028176:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.028176:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.028179:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.028180:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.028183:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.028184:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.028184:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.028186:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.028187:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.028188:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.028189:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.028189:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.028192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.028195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.2:1619677448.028199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677448.028202:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.028204:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.028204:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.028205:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677448.028206:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:31.0:1619677448.028207:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.028208:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:31.0:1619677448.028209:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:20.0:1619677448.028209:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.028211:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.028212:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.028212:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.028213:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677448.028214:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:31.0:1619677448.028215:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.028217:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.028218:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.028219:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.028220:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.028221:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.028224:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.028226:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.028229:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.028230:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.028230:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.028232:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.028233:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.028233:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.028234:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.028238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.028245:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.028248:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.028249:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.028251:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.028251:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.028254:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.028347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.028353:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.028356:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.028356:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.028358:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.028361:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.028362:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.028363:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720326] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.028364:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.028365:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.028366:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.028370:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.028371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:1.0:1619677448.028372:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.028374:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.028375:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.028375:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.028377:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677448.028377:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.028378:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.028379:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677448.028380:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.028380:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:1.0:1619677448.028380:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.028381:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677448.028382:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.028385:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.028386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:6.0:1619677448.028387:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.028389:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.028390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:6.0:1619677448.028390:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.028391:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:6.0:1619677448.028391:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.028393:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.028393:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:6.0:1619677448.028393:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.028394:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677448.028394:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677448.028395:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.028395:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677448.028397:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.028398:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:6.0:1619677448.028398:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.028400:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.028400:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:24.0:1619677448.028401:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677448.028401:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677448.028401:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677448.028403:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.028404:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.028405:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:8.0:1619677448.028406:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.028406:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:8.0:1619677448.028408:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:6.0:1619677448.028408:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:8.0:1619677448.028409:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96720324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.028409:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.028410:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:8.0:1619677448.028411:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.028411:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000400:00000200:8.0:1619677448.028412:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.028412:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:8.0:1619677448.028413:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:21.2:1619677448.028416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:8.0:1619677448.028416:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.028418:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677448.028421:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:8.0:1619677448.028421:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.028422:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:8.0:1619677448.028422:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.028422:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677448.028424:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.028424:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677448.028425:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:22.0:1619677448.028425:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677448.028425:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.028425:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:30.0:1619677448.028426:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.028426:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.028426:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:30.0:1619677448.028427:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.028427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677448.028429:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.028430:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:9.2:1619677448.028431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677448.028434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:26.0:1619677448.028435:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.028438:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:20.0:1619677448.028438:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:10.0:1619677448.028438:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677448.028439:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677448.028440:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:26.0:1619677448.028441:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.028441:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:31.0:1619677448.028442:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:2.0:1619677448.028442:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:20.0:1619677448.028443:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.028443:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.028444:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.028445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677448.028445:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.028445:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.028445:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677448.028446:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.028446:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:31.0:1619677448.028447:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.028448:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:26.0:1619677448.028448:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.028448:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:18.0:1619677448.028448:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677448.028449:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:20.0:1619677448.028449:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.028450:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:26.0:1619677448.028450:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.028451:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677448.028451:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.028452:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.028453:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.028454:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677448.028454:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677448.028454:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.028457:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.028458:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.028459:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.028462:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.028462:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.028463:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.028463:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.028465:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.028465:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677448.028465:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.028466:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.028466:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.028466:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.0:1619677448.028468:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.028469:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.028469:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.028470:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (22)++
00000400:00000200:30.0:1619677448.028471:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677448.028472:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.028472:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (24)--
00000800:00000200:20.0:1619677448.028472:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:30.0:1619677448.028473:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.028473:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.028474:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:30.0:1619677448.028475:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.028475:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677448.028477:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (23)++
00000800:00000200:31.0:1619677448.028478:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (24)--
00000400:00000200:31.0:1619677448.028479:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.028479:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.028481:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.028481:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677448.028484:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.028484:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.028484:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.028485:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.028485:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:31.0:1619677448.028486:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.028486:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:31.0:1619677448.028487:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.028488:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677448.028488:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.028489:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.028489:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.028490:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677448.028491:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:6.0:1619677448.028501:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.028504:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.028504:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.028505:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.028507:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.028510:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.028549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.028554:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.028557:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.028559:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.028560:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.028562:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.028564:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.028565:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.028567:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.028568:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.028569:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.028572:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.028574:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.028577:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.028578:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.028579:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.028580:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.028581:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.028582:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.028582:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.028594:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.028596:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.028597:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.028599:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.028601:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.028602:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.028617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.028624:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.028627:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.028630:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.028631:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.028632:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.028633:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.028635:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677448.028635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:30.0:1619677448.028636:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.028637:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.028638:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.028641:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.028642:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.028643:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.028645:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:30.0:1619677448.028646:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.028646:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.028647:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.028647:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:10.0:1619677448.028648:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.028649:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.028649:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.028650:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:10.0:1619677448.028650:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.028651:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:10.0:1619677448.028651:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.028652:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.028653:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.028654:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.028655:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.028659:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.028660:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:13.0:1619677448.028661:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.028663:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.028663:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.028663:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.028664:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.028664:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:13.0:1619677448.028665:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.028666:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:12.0:1619677448.028667:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.028667:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.028667:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:13.0:1619677448.028668:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.028668:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.028672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.028680:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.028682:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.028683:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.028685:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.028686:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.028688:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.028949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.028954:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.028957:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.028958:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.028960:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.028962:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.028964:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.028965:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.028966:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.028967:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.028968:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:4.2:1619677448.028971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677448.028971:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.028973:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.028975:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.028976:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.028976:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.028978:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.028978:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.028979:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.028979:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.028979:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.028981:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.028982:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.028984:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.028987:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.028989:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.028990:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.028991:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.028991:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:26.0:1619677448.028992:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.028993:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.028994:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:21.2:1619677448.028994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:22.0:1619677448.028995:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.028995:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.028997:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.028998:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.029000:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.029000:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677448.029001:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.029003:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.029004:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677448.029004:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.029004:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:31.0:1619677448.029006:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.029006:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.029006:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.029007:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.029008:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.029008:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:31.0:1619677448.029009:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.029010:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.029011:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.029013:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.029014:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.029014:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677448.029018:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.029018:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.2:1619677448.029018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677448.029019:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.029019:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.029020:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.029021:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.029022:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.029023:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.029023:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.029024:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.029024:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.029025:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:10.0:1619677448.029025:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.029026:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.029027:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.029027:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.029028:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.029029:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.029030:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.029032:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.029033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:10.0:1619677448.029034:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.029035:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.029036:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.029037:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.029038:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.029040:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:10.0:1619677448.029041:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.029042:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.029043:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.029044:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:10.0:1619677448.029044:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.029044:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.029045:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.029046:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.029047:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.029047:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:2.0:1619677448.029047:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.029048:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.029049:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677448.029050:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:26.0:1619677448.029058:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.029060:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.029061:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.029061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:10.0:1619677448.029062:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.029063:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.029065:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:18.0:1619677448.029066:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.029069:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.029069:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.029072:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.029074:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.029076:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.029077:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.029078:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.029078:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000400:00000200:18.0:1619677448.029079:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.029080:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.029081:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.029081:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.029082:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.029083:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:18.0:1619677448.029084:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.029084:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:18.0:1619677448.029086:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:4.2:1619677448.029086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:18.0:1619677448.029088:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.029089:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.029090:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.029092:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.029092:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.029093:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:24.0:1619677448.029094:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677448.029094:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:24.0:1619677448.029098:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.029099:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.029100:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.029103:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.029104:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:24.0:1619677448.029105:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:26.0:1619677448.029106:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.029107:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.029107:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96720312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.029107:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:24.0:1619677448.029108:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.029110:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.029110:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.029111:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.029112:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:24.0:1619677448.029115:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.029117:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.029120:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.029121:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.029122:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.029123:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.029124:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.029125:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.029126:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.029127:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.029130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677448.029132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.029137:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.029140:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:8.0:1619677448.029140:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:17.0:1619677448.029141:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.029141:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.029142:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677448.029143:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.029143:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.029144:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677448.029146:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.029274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.029281:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.029285:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.029286:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.2:1619677448.029286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:10.0:1619677448.029287:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.029289:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.029290:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:5.0:1619677448.029291:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:10.0:1619677448.029292:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.029300:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:10.0:1619677448.029300:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.029301:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.029302:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.029302:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.029305:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.029307:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.029308:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.029309:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:5.0:1619677448.029310:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.029311:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.029312:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.029313:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.029313:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:5.0:1619677448.029313:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.029314:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.029315:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:5.0:1619677448.029315:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.029316:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.029317:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.029317:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:5.0:1619677448.029318:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.029320:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.029323:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.029324:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.029324:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.029326:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.029327:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.029328:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.029329:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.029330:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677448.029331:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.029333:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.029334:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:20.2:1619677448.029335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:6.0:1619677448.029335:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.029337:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.029338:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677448.029339:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.2:1619677448.029339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677448.029340:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.029340:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.029341:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677448.029341:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:12.0:1619677448.029342:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.029343:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.029343:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:12.0:1619677448.029344:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.029346:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.029346:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.029347:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:13.0:1619677448.029347:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677448.029347:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:12.0:1619677448.029348:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.029348:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.029348:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:6.0:1619677448.029349:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:17.0:1619677448.029350:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.029351:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.029354:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.029355:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.029356:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720309] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.029357:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.029359:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.029360:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.029363:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.029365:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.029367:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.029368:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.029369:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.029370:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.029371:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.029371:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.029372:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.029384:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.029387:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.029388:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.029388:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.029390:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.029392:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.029687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.029695:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.029697:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.029698:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:4.2:1619677448.029699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:26.0:1619677448.029700:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.029704:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.029706:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.029707:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.029707:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.029708:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677448.029709:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.029710:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.029711:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.029714:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.029715:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.029717:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.029717:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.029718:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.029719:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.029721:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.029722:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.029723:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.029726:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.029727:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:10.0:1619677448.029727:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.029728:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.029729:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.029730:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.029731:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.029732:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.029733:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677448.029733:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.029734:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.029734:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.029735:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.029736:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.029736:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.029736:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.029737:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:8.2:1619677448.029737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.029739:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.029739:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.2:1619677448.029741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:4.2:1619677448.029745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.029749:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:20.0:1619677448.029750:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:26.0:1619677448.029751:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677448.029752:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:22.0:1619677448.029752:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.029752:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:26.0:1619677448.029753:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:20.0:1619677448.029753:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:6.0:1619677448.029753:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.029754:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:20.0:1619677448.029754:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.029755:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.029755:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:26.0:1619677448.029756:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677448.029756:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000400:00000200:6.0:1619677448.029756:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.029756:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:10.0:1619677448.029757:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.029758:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.029759:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677448.029759:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:5.0:1619677448.029759:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:5.0:1619677448.029760:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.029761:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.029763:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.029766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.029771:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.029774:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.029775:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.029777:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.029780:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.029781:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.029782:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.029784:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.029785:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.029786:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.029789:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.029791:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.029793:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.029794:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.029795:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.029797:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.029797:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.029798:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.029799:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.029803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.029811:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.029814:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:20.2:1619677448.029815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:18.0:1619677448.029815:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.029815:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.029817:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.029818:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:2.0:1619677448.029820:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.0:1619677448.029821:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:9.2:1619677448.029821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000400:00000200:20.0:1619677448.029823:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.029824:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:20.0:1619677448.029826:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.029827:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.029828:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.029828:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.029830:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.029831:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.029831:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.2:1619677448.029831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:20.0:1619677448.029832:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:2.0:1619677448.029832:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.029833:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.029834:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.029834:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.029834:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.029835:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.029836:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:17.0:1619677448.029836:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:20.0:1619677448.029837:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.029837:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.029838:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.029839:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.0:1619677448.029840:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.029840:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.029842:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.029842:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.029843:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.029844:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:2.0:1619677448.029844:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:20.0:1619677448.029845:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:17.0:1619677448.029845:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.029845:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.029846:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:20.0:1619677448.029847:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.029847:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:2.0:1619677448.029847:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.029848:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.029848:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.029849:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000400:00000200:17.0:1619677448.029849:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.029849:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.029850:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.029850:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:17.0:1619677448.029851:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.029852:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677448.029854:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.029856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000400:00000200:17.0:1619677448.029856:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.029856:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.029858:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.029859:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677448.029859:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.029860:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.029861:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.029861:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:2.0:1619677448.029861:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677448.029862:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.029862:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.029862:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677448.029863:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:17.0:1619677448.029863:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (24)++
00000800:00000200:2.0:1619677448.029863:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677448.029864:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.029865:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.029865:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677448.029866:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (25)--
00000800:00000200:17.0:1619677448.029866:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.029867:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:17.0:1619677448.029867:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.029867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677448.029868:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.029868:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.029869:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.029870:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.029874:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:18.0:1619677448.029875:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:30.0:1619677448.029876:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.029877:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:18.0:1619677448.029877:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.029878:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.029878:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.029879:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.029879:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.029882:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.029882:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.029884:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677448.029885:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.029906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.029914:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.029916:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.029919:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.029920:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.029921:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.029923:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.029924:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.029925:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.029927:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.029927:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.2:1619677448.029928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:31.0:1619677448.029931:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.029932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.029933:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.029933:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:31.0:1619677448.029936:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.029936:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:31.0:1619677448.029937:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.029937:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:31.0:1619677448.029938:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.029939:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:2.0:1619677448.029939:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.029940:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.029940:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.029941:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.029941:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:2.0:1619677448.029941:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.029942:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:31.0:1619677448.029943:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.029943:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.029944:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.029944:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.029945:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:2.0:1619677448.029946:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.029947:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.029947:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.029948:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.029949:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:21.2:1619677448.029949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:30.0:1619677448.029951:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.029952:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.029953:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.029953:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.029953:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677448.029954:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.029955:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.029956:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:6.0:1619677448.029956:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677448.029957:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.029957:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677448.029957:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.029957:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.029957:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.029958:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677448.029958:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:30.0:1619677448.029959:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.029959:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:6.0:1619677448.029960:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.029961:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.029962:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.029962:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.029963:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.029964:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:30.0:1619677448.029965:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.029965:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:30.0:1619677448.029966:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (21)++
00000800:00000200:6.0:1619677448.029966:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.029967:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.2:1619677448.029967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (22)++
00000800:00000200:6.0:1619677448.029968:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.029970:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (24)--
00000800:00000200:30.0:1619677448.029970:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:21.2:1619677448.029970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:6.0:1619677448.029970:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.029971:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:22.0:1619677448.029971:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.029972:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:6.0:1619677448.029972:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.029973:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.029973:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677448.029976:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:5.0:1619677448.029977:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (23)++
00000800:00000200:18.0:1619677448.029979:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.029979:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.029980:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677448.029980:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (24)--
00000400:00000200:5.0:1619677448.029981:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.029982:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (23)++
00000400:00000200:5.0:1619677448.029982:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.029983:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (24)--
00000400:00000200:20.0:1619677448.029984:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.029986:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.029986:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.029988:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.029989:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.029995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.030002:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.030004:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.030005:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.030006:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.030007:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677448.030007:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.2:1619677448.030128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.030135:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.030138:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.030139:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.030141:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.030145:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.030146:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.030148:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.030149:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.030150:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.030151:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.030154:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.030156:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.030158:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.030159:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.030160:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.030162:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.030162:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.030164:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.030165:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.030165:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677448.030168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.030171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.030176:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.030178:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:2.0:1619677448.030179:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.030180:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.030180:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:2.0:1619677448.030180:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.030181:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.030182:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:2.0:1619677448.030182:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.030186:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.030199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.030206:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.030209:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.030209:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.030211:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.030214:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.030216:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.030217:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.030218:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.030219:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.030220:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.030224:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.030225:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.030228:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.030229:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.030229:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.030231:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.030231:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.030232:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.030233:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.030237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.030245:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.030248:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.030249:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.030249:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.030251:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.030254:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.030259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.2:1619677448.030261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.030266:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.030268:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.030270:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:17.0:1619677448.030271:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.030271:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.030272:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.030272:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.030273:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.030273:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.030274:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.030275:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:1.0:1619677448.030275:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:17.0:1619677448.030277:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.030278:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.030279:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.030280:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.030283:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.030285:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.030286:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.030287:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.030288:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.030289:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:21.2:1619677448.030290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:17.0:1619677448.030290:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.030291:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.030292:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.030292:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.030296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.030298:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.2:1619677448.030299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.030301:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.030302:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.030304:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.030305:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:22.0:1619677448.030306:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.030306:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.030306:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.030307:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:1.0:1619677448.030307:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.030308:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:17.0:1619677448.030308:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.030309:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:22.0:1619677448.030310:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.030310:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:1.0:1619677448.030310:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.030312:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.030312:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:22.0:1619677448.030313:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.030314:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.030317:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.030318:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.030321:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.030321:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.030322:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.030324:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.030324:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.030325:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.030326:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.030330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.030338:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.030340:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.030341:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.030341:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.030343:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.030345:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.030524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:4.2:1619677448.030525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677448.030529:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.030532:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677448.030532:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.030534:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.030535:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:20.0:1619677448.030535:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677448.030536:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:5.0:1619677448.030537:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.030538:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.030539:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.030540:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.030541:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.030542:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.030543:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.030543:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.030544:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.030544:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.030546:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.030547:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.030549:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.030549:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.030551:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.030553:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.030554:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.030555:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.030555:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.030556:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.030558:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.030558:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.030559:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.030559:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.030559:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.030559:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.030560:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677448.030561:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.030561:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.030562:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.030563:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:13.0:1619677448.030573:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.030575:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.030576:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.030576:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:13.0:1619677448.030578:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.030579:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.030580:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:12.0:1619677448.030580:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.030581:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:6.0:1619677448.030581:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.030583:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.030586:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677448.030589:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.030592:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.030593:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.030593:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.030594:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.030595:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.030682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.030688:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.030691:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.030692:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.030694:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.030696:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.030698:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.030699:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.030701:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.030702:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.030703:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.030706:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.030708:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.030711:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.030712:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.030713:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.030715:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.030715:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.030716:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.030716:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677448.030727:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.030730:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.030731:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.030732:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.030733:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.030735:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.030876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.030884:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.030887:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.030889:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.030891:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.030892:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.030893:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.030895:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.030896:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.030897:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.030898:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.030902:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.030903:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.030906:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.030907:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.030908:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.030910:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.030910:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.030911:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.030911:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.030923:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.030926:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.030927:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.030928:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.030930:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.030933:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.030934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.030939:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.030942:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.030943:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.030945:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.030948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:2.0:1619677448.030948:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.030949:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.030951:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.030953:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.030954:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.030955:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677448.030955:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.030956:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.030958:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.030960:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.030960:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.030962:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:2.0:1619677448.030962:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.030963:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.030965:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.030965:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.030966:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:2.0:1619677448.030969:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.030970:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.030970:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.030971:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.030972:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677448.030973:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.030974:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.030975:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.030975:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.030976:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677448.030976:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677448.030977:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.030977:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.030978:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677448.030978:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.030979:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.2:1619677448.030979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.030980:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.030980:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.2:1619677448.030982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:1.0:1619677448.030987:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677448.030988:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:1.0:1619677448.030989:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.030990:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:10.0:1619677448.030990:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:2.0:1619677448.030990:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.030991:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:1.0:1619677448.030991:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.030992:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:1.0:1619677448.030993:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.030994:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.030994:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.030995:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.030995:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.030996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:10.0:1619677448.030997:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:10.0:1619677448.031001:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:6.0:1619677448.031001:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.031004:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.031005:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.031007:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.031009:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.031010:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.031012:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.031013:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.031014:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.031015:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.031018:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.031020:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.031022:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.031022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:6.0:1619677448.031023:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.031024:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.031025:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.031026:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.031026:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677448.031027:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.031027:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677448.031030:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.031031:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.031032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:17.0:1619677448.031033:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.031035:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.031036:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.031038:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.031040:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.031040:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:17.0:1619677448.031041:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.031041:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.031042:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.031043:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.031044:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.031046:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.031047:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.031049:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.031050:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.031050:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.031051:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.031051:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (23)++
00000800:00000200:1.0:1619677448.031051:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.031053:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (24)--
00000800:00000200:17.0:1619677448.031053:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.031053:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (24)++
00000800:00000200:17.0:1619677448.031054:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.031054:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (24)--
00000800:00000200:17.0:1619677448.031055:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.031055:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.031056:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.2:1619677448.031059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.031066:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.031068:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.031068:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.031069:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.031070:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.031071:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.031150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.031157:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.031160:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.031161:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.031162:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.031165:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.031167:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.031168:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.031169:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.031170:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.031171:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.031174:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.031176:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.031178:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.031179:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.031179:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.031181:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.031181:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.031182:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.031183:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:4.2:1619677448.031189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677448.031194:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.031196:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:8.0:1619677448.031196:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:30.0:1619677448.031197:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.031198:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:30.0:1619677448.031199:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.031199:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677448.031200:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:8.0:1619677448.031201:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.031202:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:8.0:1619677448.031204:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.031205:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.031206:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.031208:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.031208:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.031209:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.031212:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.031214:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.0:1619677448.031214:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.031216:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.031217:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.031217:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:8.0:1619677448.031217:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.031217:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.031218:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:8.0:1619677448.031218:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.031219:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.0:1619677448.031219:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.031220:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.031220:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.031221:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.031225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.031232:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.031235:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.031236:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.031237:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.031239:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.031241:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.031437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.031442:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.031445:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.031446:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.031448:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.031449:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.031450:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.031451:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.031453:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.031454:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.031455:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.031458:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.031460:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.031462:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.031463:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.031464:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.031466:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.031466:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.031467:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.031467:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.031480:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.031483:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.031483:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.031485:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.031487:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.031488:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.031517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.031524:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.031527:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.031527:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.031530:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.031532:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.031534:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.031535:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720284] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.031537:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.031538:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.031539:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.031542:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.031544:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.031546:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.031547:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.031548:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.031549:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.031550:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.031550:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.031551:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.031556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.031564:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.031566:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.031567:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.031567:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.031570:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.031572:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.031731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.031736:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.031739:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.031740:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.031742:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.031744:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.031746:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.031747:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.031748:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.031749:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.031750:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.031753:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.031754:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.031757:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.031758:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.031758:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.031760:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.031761:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.031761:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.031762:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:21.2:1619677448.031765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.031773:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.031773:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677448.031776:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:13.0:1619677448.031776:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.031777:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.031778:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.031779:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.031780:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:31.0:1619677448.031781:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.031781:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677448.031782:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677448.031783:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677448.031784:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.031786:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.031787:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.031787:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.031790:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.031792:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.031795:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.031796:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.031796:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.031798:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.031799:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.031799:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.031800:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.031804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.031812:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.031815:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.031816:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.031816:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.031818:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.031821:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.031824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.031831:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.031833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677448.031833:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.031834:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.031834:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.031836:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.031836:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677448.031840:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.031843:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.031844:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.031846:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.031849:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.031850:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.031851:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.031853:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.031854:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.031855:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.031858:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.031860:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.031862:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.031863:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.031864:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.031865:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.031866:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.031866:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.2:1619677448.031866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.031867:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677448.031872:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.031875:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.031875:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.031877:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.031880:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.031881:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:17.0:1619677448.031881:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.031882:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.031883:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:17.0:1619677448.031884:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.031884:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.031885:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.031885:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.031886:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:22.0:1619677448.031887:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.031890:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.031891:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677448.031892:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.031895:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.031896:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.031896:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.031898:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.031898:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.031899:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.031900:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.031911:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.031913:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.031914:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.031916:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.031917:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.031918:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.031940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:30.0:1619677448.031947:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677448.031950:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:30.0:1619677448.031952:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.031954:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:30.0:1619677448.031955:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.031957:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.031958:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.031959:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.031960:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.031962:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.031965:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.031967:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.031970:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.031971:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.031971:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.031973:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.031973:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677448.031975:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871380600] (22)++
00000800:00000200:30.0:1619677448.031977:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (23)--
00000800:00000200:30.0:1619677448.031977:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:9.2:1619677448.031982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.2:1619677448.031983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677448.031990:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (22)++
00000800:00000200:18.0:1619677448.031990:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:31.0:1619677448.031993:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871380600] (23)--
00000800:00000200:18.0:1619677448.031993:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:31.0:1619677448.031994:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:2.0:1619677448.031994:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.031995:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.031996:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000400:00000200:18.0:1619677448.031997:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.032000:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.032066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.032074:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.032077:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.032079:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.032081:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.032082:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.032084:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.032085:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.032087:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.032088:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.032089:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.032092:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.032093:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.032096:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.032097:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.032097:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.032099:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.032100:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.032101:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.032103:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.032104:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677448.032109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677448.032112:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.032114:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.032115:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.032116:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.032117:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:12.0:1619677448.032118:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.032119:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.032120:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.032120:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.032121:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677448.032128:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.032131:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.032132:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.032132:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.032133:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.032134:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.032264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.032270:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.032272:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.032273:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.032275:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.032278:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.032279:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.032280:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.032282:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.032283:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.032284:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.032287:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.032288:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.032290:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.032291:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.032291:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.032293:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.032298:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.2:1619677448.032299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.032300:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.032301:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.032303:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.032304:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.032306:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.032306:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.032307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:1.0:1619677448.032308:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.032309:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.2:1619677448.032309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:17.0:1619677448.032310:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.032310:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.032310:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677448.032311:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.032311:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677448.032315:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:8.2:1619677448.032318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:2.0:1619677448.032318:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.032319:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.032321:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.032323:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.032325:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:17.0:1619677448.032326:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:2.0:1619677448.032326:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720276] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.032327:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.032329:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:2.0:1619677448.032329:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.032329:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.032330:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000400:00000200:2.0:1619677448.032330:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.032331:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.032331:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:2.0:1619677448.032333:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.032335:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.032337:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.032338:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.032339:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.032340:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.032341:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.032341:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.032342:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.032346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.032354:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.032356:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.032357:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.032358:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.032360:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.032361:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.032374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.032381:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.032384:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.032385:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.032387:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.032390:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.032391:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.032393:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.032394:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.032395:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.032396:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.032400:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.032401:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.032404:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.032405:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.032406:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.032408:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.032408:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.032409:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.032410:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.032411:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.032416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.032423:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.032423:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677448.032425:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.032426:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677448.032426:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.032427:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.032427:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.032427:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.032428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000400:00000200:6.0:1619677448.032429:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.032432:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:26.0:1619677448.032436:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.032439:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677448.032440:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677448.032441:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.032444:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.032446:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.032447:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.032448:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.032449:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.032450:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.032453:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.032454:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.032457:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.032458:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.032458:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.032460:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.032460:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.032461:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677448.032461:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677448.032466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.032473:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.032476:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.032477:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.032477:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.032479:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.032482:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.032575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.032581:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.032583:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.032584:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.032586:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.032588:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.032590:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.032591:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.032593:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.032593:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.032594:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.032598:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.032599:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.032602:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.032603:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.032604:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.032606:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.032606:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.032607:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.032608:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.032609:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:21.2:1619677448.032614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.032619:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:30.0:1619677448.032620:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:6.0:1619677448.032621:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.032621:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677448.032622:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:6.0:1619677448.032622:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:30.0:1619677448.032623:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.032625:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.032627:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.032628:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.032674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:5.2:1619677448.032675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.032681:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.032682:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.032683:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.032684:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677448.032685:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.032685:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.032687:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.032689:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.032689:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.032691:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:24.0:1619677448.032692:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.032692:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.032693:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.032694:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.032695:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.032696:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.032696:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:24.0:1619677448.032697:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.032698:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.032699:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.032704:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.032705:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.032707:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:24.0:1619677448.032708:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.032710:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.032711:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.032711:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677448.032712:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.032712:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.032713:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.032714:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.032714:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.032714:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.032715:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.032715:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677448.032716:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677448.032716:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677448.032717:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677448.032719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.2:1619677448.032720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.032727:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:1.0:1619677448.032728:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677448.032730:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677448.032730:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677448.032731:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:17.0:1619677448.032731:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:8.0:1619677448.032732:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.032732:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.032735:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.032735:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.032739:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:1.0:1619677448.032740:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.032773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.032781:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.032784:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.032786:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.032788:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.032789:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.032790:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.032792:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.032793:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.032794:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.032795:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.032798:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.032800:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.032803:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.032804:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.032805:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.032807:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.032807:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.032808:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.032809:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.032813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.032821:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.032824:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.032825:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.032827:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.032827:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.032829:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.032851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.032858:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.032860:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.032861:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.032863:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.032866:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.032867:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.032869:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.032870:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.032871:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.032872:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.032875:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.032877:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.032880:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.032881:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.032881:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.032883:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.032883:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.032884:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.032884:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.032897:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.2:1619677448.032898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:22.0:1619677448.032900:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677448.032900:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.032900:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.032901:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.032903:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:22.0:1619677448.032904:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.032905:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.032906:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677448.032907:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:20.0:1619677448.032908:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.032910:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.032911:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.032912:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.032913:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.032915:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.032915:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.032919:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:20.0:1619677448.032919:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.032919:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.032920:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.032921:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.032921:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.032921:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677448.032924:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.032925:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.032925:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.032927:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.032928:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.032929:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.032929:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.032940:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.032943:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.032944:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.032945:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.032946:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.032949:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.032987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.032995:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.032997:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.032998:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.033000:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.033003:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.033005:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.033006:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.033007:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.033008:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.033009:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:21.2:1619677448.033009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:26.0:1619677448.033012:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.033014:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.033016:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.033016:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.033017:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.033018:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.033019:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677448.033019:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.033020:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.033020:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.033020:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:31.0:1619677448.033021:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.033023:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.033023:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.033025:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.033026:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.033027:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.033028:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.033029:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.033032:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.033032:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677448.033033:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.033034:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677448.033035:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677448.033035:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.033036:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:12.0:1619677448.033036:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.033037:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:31.0:1619677448.033038:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:31.0:1619677448.033039:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.033039:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:31.0:1619677448.033040:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.033040:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.033040:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.033050:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.033052:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.033053:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.033054:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.033056:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.033057:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.033068:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.033070:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.033071:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.033072:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.033072:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.033073:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.033177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.033185:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.033187:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.033189:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.033191:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.033191:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.033193:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.033194:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.033195:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.033196:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.033197:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.033200:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.033201:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.033204:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677448.033204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:30.0:1619677448.033205:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.033206:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.033208:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.033208:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.033209:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.033210:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:18.0:1619677448.033210:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.033213:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:8.2:1619677448.033214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.033214:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.033215:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.033218:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.033219:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.033221:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.033222:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.033222:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:18.0:1619677448.033223:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.033224:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.033224:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.033225:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.033227:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:18.0:1619677448.033227:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.033227:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.033229:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.033231:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.033231:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677448.033232:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.033233:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.033234:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.033235:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.033236:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.033236:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.033241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.033248:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.033250:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.033251:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.033252:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.033253:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.033254:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.033259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.033267:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.033269:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.033270:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.033270:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.033271:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.033272:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.033327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.033334:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.033336:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.033337:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.033339:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.033342:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.033343:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.033345:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.033346:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.033347:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.033348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:17.0:1619677448.033348:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.033351:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.033353:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677448.033354:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677448.033355:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.033356:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677448.033356:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:17.0:1619677448.033357:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677448.033358:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677448.033358:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.033358:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.033359:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:2.0:1619677448.033359:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:17.0:1619677448.033360:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:2.0:1619677448.033362:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.033363:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677448.033364:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.033366:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.033367:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.033368:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677448.033371:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.033372:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.033373:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677448.033374:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.033375:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.033375:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.033376:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.033377:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.033377:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677448.033377:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.033377:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.033378:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.033378:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:22.0:1619677448.033379:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.033382:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677448.033383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.033386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:5.2:1619677448.033390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:18.0:1619677448.033391:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:6.0:1619677448.033391:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.033393:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677448.033393:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:6.0:1619677448.033393:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.033394:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:18.0:1619677448.033394:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.033394:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677448.033394:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.033395:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000400:00000200:18.0:1619677448.033396:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:6.0:1619677448.033396:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.033397:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677448.033397:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.033398:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.033398:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677448.033399:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:10.0:1619677448.033400:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.033401:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677448.033402:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.033403:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.033404:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.033405:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.033406:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.033407:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.033407:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.033408:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:22.0:1619677448.033409:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.033410:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.033412:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.033413:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.033413:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.033414:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:22.0:1619677448.033416:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.033418:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.033420:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.033421:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.033421:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.033421:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:22.0:1619677448.033423:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.033423:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.033423:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:22.0:1619677448.033424:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.033424:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:22.0:1619677448.033425:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:10.0:1619677448.033425:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:22.0:1619677448.033426:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.033426:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.033427:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.033428:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677448.033437:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:4.2:1619677448.033437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.033439:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.033440:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.033442:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.033443:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:30.0:1619677448.033444:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.033444:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:31.0:1619677448.033445:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:26.0:1619677448.033445:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.033447:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.033447:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.033448:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.033449:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.033450:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.033451:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.033451:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.033453:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.033455:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.033456:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.033457:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.033458:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.033459:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.033462:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.033464:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.033466:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.033466:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.033467:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.033468:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.033469:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.033469:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.033470:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.033481:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.033483:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.033484:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.033485:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.033487:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.033488:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.033494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.033499:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.033501:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.033502:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.033502:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.033502:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.033503:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.033503:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677448.033504:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.033505:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.033505:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.033508:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.033509:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.033510:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.033511:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.033512:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.033513:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.033516:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.033518:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.033521:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.033522:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.033522:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.033524:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.033524:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.033525:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.033526:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.033531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.033539:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.033542:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.033543:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.033544:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.033546:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.033547:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.033549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.033557:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.033560:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.033561:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.033561:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.033562:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677448.033562:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.2:1619677448.033671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.033676:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.033679:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.033681:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.033681:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.033683:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.033685:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.033686:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.033688:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.033688:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.033690:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.033693:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.033694:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.033697:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.033698:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.033698:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.033700:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.033701:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.033702:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.033702:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677448.033713:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.033716:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.033717:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.033717:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.033719:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.033721:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.033767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.033774:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.033777:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.033778:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.033780:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.033782:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.033784:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.033785:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.033786:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.033787:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.033788:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.033792:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.033793:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.033796:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.033796:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.033797:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.033799:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.033799:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.033800:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.033800:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.033811:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.033814:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.033814:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.033816:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.033818:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.033819:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.033869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.033877:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.033879:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.033881:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.033883:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.033884:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.033886:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.033887:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.033888:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.033889:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.033890:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.033893:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.033895:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.033898:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.033898:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.033899:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.033901:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.033902:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.033903:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.033905:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.033905:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.033907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.033913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.033919:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677448.033921:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.033921:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.033922:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.033924:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.033924:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.033924:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.033925:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.033925:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.033926:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.034047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.034055:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.2:1619677448.034056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:10.0:1619677448.034057:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.034058:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.034060:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.034062:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.034063:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:10.0:1619677448.034064:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.034065:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720254] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.034066:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:10.0:1619677448.034066:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.034067:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.034067:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:10.0:1619677448.034068:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677448.034069:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.034071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:24.0:1619677448.034072:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.034072:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.034074:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:10.0:1619677448.034074:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:24.0:1619677448.034075:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96720253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.034077:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.034077:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.034077:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.034078:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:24.0:1619677448.034078:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.034078:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:24.0:1619677448.034079:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.034080:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.034080:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.034081:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:10.0:1619677448.034081:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.034081:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:24.0:1619677448.034082:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.034083:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.034085:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677448.034085:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:30.0:1619677448.034086:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.034086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:30.0:1619677448.034087:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.034089:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.034089:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.034090:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.034090:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:30.0:1619677448.034091:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.034091:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677448.034092:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677448.034093:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.034093:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.034094:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000400:00000200:30.0:1619677448.034095:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.034095:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:5.0:1619677448.034095:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:30.0:1619677448.034096:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677448.034096:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:30.0:1619677448.034098:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.034098:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.034099:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:9.2:1619677448.034099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.034100:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.034100:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:30.0:1619677448.034101:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.034101:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.034102:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:5.0:1619677448.034102:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.034103:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:1.0:1619677448.034103:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.034104:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:5.0:1619677448.034104:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.034105:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677448.034106:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677448.034106:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.034107:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.034107:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:1.0:1619677448.034109:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.034110:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:21.2:1619677448.034111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.034111:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.034111:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.034111:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.034113:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677448.034113:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677448.034114:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.034115:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.034116:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:1.0:1619677448.034116:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.034117:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.034117:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.034118:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:2.0:1619677448.034118:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:13.0:1619677448.034119:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.034119:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:12.0:1619677448.034120:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.034121:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.034122:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677448.034122:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677448.034122:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.034123:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677448.034124:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.034127:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.034127:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.034128:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.034130:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.034130:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.034131:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.034132:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.034144:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.034146:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.034147:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.034147:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.034149:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.034151:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.034151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:21.2:1619677448.034155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:17.0:1619677448.034157:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.034160:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677448.034161:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677448.034162:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.034162:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.034164:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:17.0:1619677448.034165:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.034165:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.034166:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:17.0:1619677448.034167:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677448.034168:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.034170:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.034171:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.034171:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677448.034174:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.034176:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.034178:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.034179:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:17.0:1619677448.034179:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.034181:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677448.034181:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.034182:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677448.034183:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:5.2:1619677448.034186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.034194:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.034195:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.034197:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.034197:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677448.034197:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.034198:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.034198:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.034199:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.034201:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.034202:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.034203:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:26.0:1619677448.034204:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.034205:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.034206:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.034207:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.034208:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.034212:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.034213:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.034216:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.034216:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.034217:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.034217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:26.0:1619677448.034218:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.034219:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.034220:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:20.0:1619677448.034220:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:26.0:1619677448.034221:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.034222:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677448.034222:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.034224:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.034226:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.034227:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.034228:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:5.2:1619677448.034228:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:20.0:1619677448.034229:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.034230:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.034231:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.034231:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:20.0:1619677448.034232:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:12.0:1619677448.034233:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.034234:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.034235:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:12.0:1619677448.034236:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.034237:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.034237:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.034237:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:12.0:1619677448.034238:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.034238:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.034239:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.0:1619677448.034239:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.034242:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.034243:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.034243:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.034246:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.034246:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.034247:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.034248:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677448.034252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677448.034256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:18.0:1619677448.034260:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.034263:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:10.0:1619677448.034263:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:2.0:1619677448.034263:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.034264:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.034266:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.034266:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.034267:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:18.0:1619677448.034269:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:10.0:1619677448.034269:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.034271:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.034273:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677448.034274:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720247] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.034276:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.034277:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.034278:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.034281:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.034283:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.034286:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.034287:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.034287:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.034289:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.034289:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.034290:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.034291:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.034296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.034304:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.034307:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.034308:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.034309:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677448.034310:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.034312:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.034323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.034329:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.034332:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.034333:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.034335:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.034338:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.034339:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.034340:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.034341:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.034342:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.034343:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.034346:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.034347:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.034350:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.034351:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.034351:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.034353:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.034354:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.034354:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.034356:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.034356:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.034359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677448.034362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.034367:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677448.034368:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.034370:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:1.0:1619677448.034370:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:24.0:1619677448.034371:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677448.034371:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.034371:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.034372:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.034373:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.034376:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.034457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.034464:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.034467:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.034468:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.034470:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.034472:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.034474:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.034475:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720245] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.034476:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.034477:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.034478:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.034481:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.034483:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.034486:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.034487:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.034487:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.034489:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.034489:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.034490:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.034492:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.034492:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.034495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.034498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.034504:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.034506:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:10.0:1619677448.034506:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:20.0:1619677448.034507:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.034508:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.034508:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.034508:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.034509:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.034510:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.034512:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.034692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.034697:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.034700:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.034701:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.034702:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.034704:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.034706:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.034707:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.034709:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.034709:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.034711:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.034714:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.034716:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.034718:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.034719:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.034720:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.034722:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.034722:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.034724:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.034725:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.034725:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.034728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.034730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.034735:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:1.0:1619677448.034735:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677448.034737:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:1.0:1619677448.034737:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:24.0:1619677448.034738:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:1.0:1619677448.034738:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677448.034739:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.034739:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.034741:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.034744:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.034749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677448.034755:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.034758:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677448.034758:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.034759:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.034760:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.034760:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.034776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.034782:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.034784:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.034785:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:21.2:1619677448.034786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:8.0:1619677448.034787:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.034790:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.034791:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.034793:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.034794:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:8.0:1619677448.034794:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.034795:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.034796:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.034797:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.034800:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.034800:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.034801:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.034802:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.034803:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:31.0:1619677448.034804:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.034805:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.034806:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.034806:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.034806:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:31.0:1619677448.034807:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.034807:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677448.034808:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677448.034809:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.034809:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.034810:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:31.0:1619677448.034811:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.034811:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:31.0:1619677448.034812:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.034815:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.034815:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:9.2:1619677448.034815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.034816:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.034817:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.034818:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.034819:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.034819:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677448.034823:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.034825:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.034827:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.034827:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.034829:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.034831:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.034832:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677448.034833:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.034834:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.034836:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.034837:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.034838:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.034866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.034873:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.034876:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.034876:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.034878:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.034881:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.034882:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.034884:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.034885:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.034886:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.034887:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.034890:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.034891:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.2:1619677448.034894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677448.034894:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.034895:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.034895:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.034897:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.034897:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.034898:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.034898:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.034902:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:8.2:1619677448.034903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.034905:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.034907:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.034909:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.034909:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.034910:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.034912:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.034912:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:30.0:1619677448.034913:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.034914:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.034914:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.034914:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.034915:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.034917:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.034917:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.034918:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.034921:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.034923:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:30.0:1619677448.034924:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.034924:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.034925:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.034927:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.034927:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.034928:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.034929:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.034932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.034940:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.034942:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.034943:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.034945:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.034946:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.034947:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.034955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.034962:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.034965:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.034966:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.034967:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.034967:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.034968:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.035155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.035161:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.035164:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:5.2:1619677448.035164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.035164:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.035166:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.035168:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.035170:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:17.0:1619677448.035170:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677448.035171:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.035172:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.035173:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.035173:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.035173:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677448.035175:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:21.2:1619677448.035176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:17.0:1619677448.035176:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.035179:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.035179:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.035181:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:17.0:1619677448.035181:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:22.0:1619677448.035182:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677448.035183:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.035183:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.035184:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:17.0:1619677448.035184:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.035185:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:18.0:1619677448.035185:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:17.0:1619677448.035185:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.035186:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677448.035186:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677448.035187:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.035187:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677448.035187:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.035188:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:22.0:1619677448.035189:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.035191:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.035191:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.035192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:22.0:1619677448.035193:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:17.0:1619677448.035193:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:22.0:1619677448.035194:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.035196:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.035196:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.035197:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.035197:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:22.0:1619677448.035198:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:17.0:1619677448.035198:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.035199:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.035199:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:17.0:1619677448.035200:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.035200:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.035201:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677448.035201:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:22.0:1619677448.035202:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.035202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:2.0:1619677448.035202:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.035203:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677448.035204:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:2.0:1619677448.035205:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.035207:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.035207:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.035207:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677448.035208:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677448.035208:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677448.035210:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:9.2:1619677448.035210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:22.0:1619677448.035211:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.035211:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.0:1619677448.035211:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:22.0:1619677448.035212:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677448.035212:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.035213:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.035213:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.035215:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:5.0:1619677448.035215:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.035216:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.035216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:13.0:1619677448.035218:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.035218:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.035218:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.035219:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.035221:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677448.035221:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.035221:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.035222:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677448.035223:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.035223:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.035225:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.035225:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.035226:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.035226:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:5.0:1619677448.035226:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.035226:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.035227:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:2.0:1619677448.035227:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.035228:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677448.035229:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.035229:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.035232:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.035232:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677448.035233:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.035233:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.035236:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.035236:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.035238:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.035239:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.035240:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.035244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.035247:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677448.035248:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.035250:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.035251:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.035251:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.035251:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.035252:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.035254:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.035255:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.035257:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.035266:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.035268:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.035269:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.035270:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.035270:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.035271:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.035416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.035424:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.035426:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.035427:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.035429:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.035431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:26.0:1619677448.035432:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.035434:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.035436:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.035437:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.035439:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.035439:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:26.0:1619677448.035440:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.035440:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.035443:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.035444:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.035445:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.035446:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.035447:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.035448:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.035449:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.035450:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.035451:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.035455:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.035455:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.035456:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.035457:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.035457:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.035459:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.035460:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.035461:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.035461:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.035461:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.035462:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.035462:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.035463:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.035465:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.035465:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.035466:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.035466:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.035466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.035472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.2:1619677448.035472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.035474:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.035477:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.035477:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.035478:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.035480:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:18.0:1619677448.035480:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.035481:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.035482:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:31.0:1619677448.035483:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:20.0:1619677448.035483:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.035483:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:31.0:1619677448.035486:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.035486:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.035486:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.035488:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.035489:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:31.0:1619677448.035490:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.035491:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.035492:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.035494:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.035495:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.035496:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.035499:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.035501:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.035503:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.035504:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.035504:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.035506:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.035506:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.035507:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.035508:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.035517:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.035520:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.035521:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.035521:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.035523:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.035525:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.035682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.035689:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.035691:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.035692:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.035694:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.035696:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.035698:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.035699:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.035700:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.035701:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.035702:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677448.035705:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.035707:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.035709:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.035710:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.035711:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.035713:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.035713:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.035714:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.035714:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.035718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.035727:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.035729:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.035730:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.035732:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.035732:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.035735:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.035793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:5.2:1619677448.035797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677448.035801:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:1.0:1619677448.035803:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.035804:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677448.035806:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.035806:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.035807:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.035808:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677448.035809:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.035809:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677448.035811:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:1.0:1619677448.035812:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.035813:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.035813:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:12.0:1619677448.035814:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.035815:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.035816:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.035816:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.035817:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:1.0:1619677448.035818:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.035819:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:12.0:1619677448.035822:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.035824:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.035825:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:1.0:1619677448.035826:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:12.0:1619677448.035828:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.035829:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:1.0:1619677448.035829:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.035830:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:1.0:1619677448.035830:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.035831:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:12.0:1619677448.035832:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:12.0:1619677448.035832:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677448.035833:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:1.0:1619677448.035833:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.035833:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.035834:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:1.0:1619677448.035834:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.035835:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.035838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677448.035842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:2.0:1619677448.035845:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:8.0:1619677448.035848:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:2.0:1619677448.035848:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.035849:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.035849:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.035851:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:2.0:1619677448.035851:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.035852:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.035852:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.035854:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.035855:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:8.0:1619677448.035858:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.035862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:21.2:1619677448.035868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:18.0:1619677448.035869:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.035873:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.035874:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:6.0:1619677448.035874:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677448.035874:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.035875:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.035876:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:22.0:1619677448.035877:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.035877:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.035880:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.035882:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.035884:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.035885:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.035886:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.035887:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.035888:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.035892:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.035894:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.035896:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.035897:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.035898:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.035900:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.035900:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.035901:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.035902:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.035906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.035913:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.035916:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.035917:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.035917:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.035919:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.035921:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.035929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.035937:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.035940:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.035940:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.035942:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.035943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:10.0:1619677448.035945:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.035946:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.035947:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.035949:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.035950:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677448.035950:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.035951:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.035953:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.035953:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:10.0:1619677448.035955:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.035956:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.035957:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.035958:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.035959:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.035960:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96720227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.035960:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.035961:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:24.0:1619677448.035962:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.035962:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:24.0:1619677448.035963:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.035963:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.035964:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.035964:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.035965:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.2:1619677448.035966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677448.035966:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:24.0:1619677448.035967:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.035968:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.2:1619677448.035970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.035971:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.035971:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.035972:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.035974:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677448.035974:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.035974:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.035974:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.035975:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:30.0:1619677448.035976:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.2:1619677448.035977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677448.035977:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:30.0:1619677448.035978:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.035979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.035980:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677448.035980:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:31.0:1619677448.035981:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.035981:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.035981:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.035981:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.035982:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:30.0:1619677448.035983:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677448.035984:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.035985:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.035986:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.035986:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677448.035987:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.035987:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677448.035987:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:30.0:1619677448.035988:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.035989:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:20.0:1619677448.035989:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.035990:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:20.0:1619677448.035991:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.035991:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.035991:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:20.0:1619677448.035992:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.035993:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.035994:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.035995:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.035995:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.035996:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:20.0:1619677448.035996:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.035998:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677448.035999:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.036000:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677448.036000:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:20.0:1619677448.036000:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.036002:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:20.0:1619677448.036002:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.036003:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.036004:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.036004:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677448.036006:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.036007:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.036007:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.036009:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.036010:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.036010:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.036011:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677448.036016:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.036019:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.036019:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.036020:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.036022:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.036022:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.036024:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:22.0:1619677448.036025:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:13.0:1619677448.036025:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.036027:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.036028:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.036030:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.036089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.036095:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.036098:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.036099:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.036101:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.036103:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.036104:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.036106:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.036107:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.036108:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.036109:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.036113:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.036115:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.2:1619677448.036116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.036117:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.036119:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.036120:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.036122:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.036122:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.036123:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.036123:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.036124:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:18.0:1619677448.036125:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.036126:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.036126:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.036126:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.036127:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.036128:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:8.2:1619677448.036128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:18.0:1619677448.036129:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.036130:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.036131:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.036132:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:5.2:1619677448.036132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000400:00000200:18.0:1619677448.036136:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.036137:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677448.036138:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:5.0:1619677448.036139:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677448.036139:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.036140:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.036140:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.036140:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677448.036141:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677448.036142:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:5.0:1619677448.036142:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.036142:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.036144:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000400:00000200:5.0:1619677448.036144:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.036145:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.036145:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.036145:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.036146:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.036147:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:5.0:1619677448.036147:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.036152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:26.0:1619677448.036156:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:2.0:1619677448.036158:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:26.0:1619677448.036159:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.036159:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.036160:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.036160:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.036161:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:26.0:1619677448.036162:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.036162:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677448.036165:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677448.036421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.036429:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.036432:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.036434:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.036436:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.036437:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.036438:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.036439:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.036440:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.036441:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.036442:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:5.2:1619677448.036442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:31.0:1619677448.036446:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.036447:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677448.036448:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.036451:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.036451:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.036451:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677448.036452:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.036453:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:1.0:1619677448.036453:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.036454:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.036455:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.036456:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:1.0:1619677448.036456:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.036458:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:1.0:1619677448.036458:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:31.0:1619677448.036459:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:1.0:1619677448.036459:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.036461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:1.0:1619677448.036461:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.036462:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.036463:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:21.2:1619677448.036465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677448.036466:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.036468:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:8.0:1619677448.036468:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:1.0:1619677448.036468:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.036470:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.036471:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.036471:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.036472:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.036472:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:1.0:1619677448.036472:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.036473:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:1.0:1619677448.036473:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.036473:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:10.0:1619677448.036474:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.036475:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:8.0:1619677448.036475:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.036475:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.036476:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677448.036476:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.036476:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.036477:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.036477:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:1.0:1619677448.036477:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.036478:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.036479:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.036481:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96720220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.036482:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.036483:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.036484:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.036487:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000400:00000200:8.0:1619677448.036487:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.036489:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:8.0:1619677448.036489:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.036490:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.036490:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.036491:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.036491:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:12.0:1619677448.036491:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.0:1619677448.036492:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.036492:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:8.0:1619677448.036493:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (24)++
00000800:00000200:8.0:1619677448.036495:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (25)--
00000800:00000200:12.0:1619677448.036496:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:8.0:1619677448.036496:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.036496:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:12.0:1619677448.036497:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.036497:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677448.036498:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.036498:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.036503:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.036505:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.036507:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.036507:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.036508:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677448.036509:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.036510:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677448.036523:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.036526:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.036527:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.036528:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.036528:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.036530:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.036556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.036563:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.036566:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.036567:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.036568:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.036570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:10.0:1619677448.036571:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.036573:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.036574:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.036575:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.036576:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.036576:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677448.036577:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677448.036579:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.036580:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677448.036582:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.036583:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.036584:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:6.0:1619677448.036585:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.036587:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:10.0:1619677448.036588:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.036588:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:6.0:1619677448.036588:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.036589:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677448.036590:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.036590:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.036591:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.036591:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677448.036591:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.036593:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.036594:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.036594:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.036595:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.036597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.036597:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.036599:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.036600:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:4.2:1619677448.036600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.036601:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.036602:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.036603:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.036604:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677448.036605:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677448.036605:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.036606:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677448.036607:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:18.0:1619677448.036608:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.036609:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:18.0:1619677448.036609:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.036609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.036609:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.036610:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.2:1619677448.036610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.036611:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:18.0:1619677448.036611:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.036615:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677448.036615:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677448.036617:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677448.036618:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.036618:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.036619:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677448.036619:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.036620:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.036621:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.036623:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.036625:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.036815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.036821:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.036824:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.036825:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.036826:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.036829:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.036830:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.036832:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.036833:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.036834:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.036835:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.036838:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.036840:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.036843:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.036844:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.036844:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.036846:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.036847:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.036847:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.036848:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.036852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.036860:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.036863:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.036863:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.036863:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.036866:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.036868:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.036870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.036876:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.036879:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.036880:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.036880:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.036881:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.036882:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.036920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.036923:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.036926:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.036928:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.036929:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.036931:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.036933:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677448.036935:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.036936:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.036937:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.036938:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677448.036941:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.036942:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.036945:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.036946:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.036947:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.036949:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.036949:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.036950:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.036950:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:10.0:1619677448.036961:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.036963:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.036964:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.036965:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.036967:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.036970:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.037062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.037068:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.037070:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.037071:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.037073:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.037076:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.037077:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.037078:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.037079:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.037080:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.037081:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.037084:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.037086:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.037088:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.037089:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.037089:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.037091:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.037091:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.037093:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.037094:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.037094:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.037097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.037099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.037105:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.037106:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.037106:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:17.0:1619677448.037107:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.037107:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.037108:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.037109:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.037109:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.037110:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677448.037110:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677448.037152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677448.037154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:26.0:1619677448.037159:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.037159:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677448.037162:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:18.0:1619677448.037162:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:10.0:1619677448.037162:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677448.037163:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:26.0:1619677448.037166:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.037166:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.037169:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.037169:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.037172:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.037172:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677448.037174:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.037175:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.037177:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.037177:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.037178:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.037179:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.037179:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.037181:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.037185:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.037186:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.037188:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.037189:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.037190:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.037191:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.037192:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.037192:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.037193:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.037194:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.037194:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677448.037194:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677448.037196:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.037196:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:26.0:1619677448.037197:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.037197:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.037197:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:26.0:1619677448.037198:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.037198:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.037200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.037203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677448.037208:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677448.037209:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:6.0:1619677448.037211:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.037211:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:2.0:1619677448.037211:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:5.0:1619677448.037212:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.037212:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.037213:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:22.0:1619677448.037214:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.037214:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:6.0:1619677448.037214:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:5.0:1619677448.037214:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.037215:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.037216:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:6.0:1619677448.037217:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.037220:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.037290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.037298:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.037301:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.037302:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.037304:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.037306:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.037308:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.037309:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.037310:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.037311:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.037312:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.037315:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.037317:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.037320:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.037320:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.037321:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.037323:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.037323:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.037324:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.037324:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.037329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.037337:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.037340:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.037341:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.037341:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.037343:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.037346:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.037367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.037372:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.037375:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.037375:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.037377:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.037378:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.037380:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.037381:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.037382:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.037383:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.037384:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.037388:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.037390:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.037392:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.037393:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.037393:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.037395:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.037396:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.037396:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.037397:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.037411:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.037413:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.037414:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.037414:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.037416:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.037419:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.037436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.037444:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.037447:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.037449:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.037451:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.037452:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.037453:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.037455:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.037456:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.037457:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.037458:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.037462:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.037464:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.037466:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.037466:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.037467:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.037469:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.037469:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.037470:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.037471:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677448.037482:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.037485:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.037485:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.037487:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.037489:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.037490:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.037596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.037603:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.037606:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.037607:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.037609:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.037611:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.037613:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.037614:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.037615:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.037616:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.037618:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.037621:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.037622:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.037625:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.037626:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.037626:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.037628:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.037628:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.037629:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.037630:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.037635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.037636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.037642:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.037643:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.037645:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:6.0:1619677448.037645:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.037646:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:17.0:1619677448.037646:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.037647:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.037647:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.037648:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.037649:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.037650:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:6.0:1619677448.037651:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.037652:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.037654:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.037655:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.037656:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.037660:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.037662:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.037664:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.037665:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.037666:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.037668:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.037668:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.037669:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.037669:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677448.037681:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.037684:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.037685:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.037686:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.037687:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.037689:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.037769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.037776:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.037779:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.037780:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.037782:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.037785:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.037786:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.037787:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.037789:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.037790:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.037791:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.037794:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.037796:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.037799:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.037800:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.037801:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.037803:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.037803:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.037805:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.037806:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.037806:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677448.037809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.037812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:2.0:1619677448.037816:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.2:1619677448.037817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.0:1619677448.037819:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:2.0:1619677448.037819:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.037820:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.037820:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.037821:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.037822:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:2.0:1619677448.037822:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.037823:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:1.0:1619677448.037823:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.037825:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677448.037826:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.037827:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.037827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:1.0:1619677448.037829:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.037831:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.037833:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.037834:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.037835:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677448.037835:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.037836:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.037837:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.037838:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.037838:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.037839:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.037840:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.037841:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:1.0:1619677448.037841:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.037843:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.037845:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.037846:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.037847:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.037848:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.037849:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.037850:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.037850:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.037862:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.037865:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.037866:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.037866:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.037868:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.037870:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677448.037880:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.037883:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.037884:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.037884:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.037885:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.037885:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.037929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.037937:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.037940:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.037941:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.037943:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.037946:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.037948:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.037949:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.037950:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.037951:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.037953:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.037956:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.037958:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.037961:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.037961:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.037962:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.037964:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.037964:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.037965:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.037965:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.037977:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.037980:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.037981:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.037981:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.037983:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.037986:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.038130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.038136:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.038139:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.038139:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.038141:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.038144:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.038145:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.038146:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.038148:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.038149:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.038150:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.038153:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.038154:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.038156:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.038157:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.038158:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.038159:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.038159:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.038160:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.038160:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.038165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.038172:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.038174:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.038175:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.038176:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.038177:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.038178:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.038185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.038192:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.038194:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.038195:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.038195:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.038196:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.038196:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.038207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.038214:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.038217:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.038218:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.038220:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.038223:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.038224:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.038225:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.038227:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.038228:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.038229:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.038232:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.038234:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.2:1619677448.038236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.0:1619677448.038237:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.038238:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.038238:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677448.038240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.0:1619677448.038240:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.038240:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.038241:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.038241:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:26.0:1619677448.038244:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.038246:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.038247:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.038248:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677448.038249:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.038250:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.038252:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.038252:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.038253:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:31.0:1619677448.038254:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.038254:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.038254:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.038256:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.038256:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.038257:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.038257:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.038257:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:8.2:1619677448.038257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677448.038258:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.038259:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.038260:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.038261:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.038264:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.038264:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677448.038264:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:26.0:1619677448.038265:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.038266:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.038266:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.038266:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:1.0:1619677448.038267:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:30.0:1619677448.038268:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.038268:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.038268:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.038268:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.038268:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.038268:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.038269:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.038269:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.038269:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.038270:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:1.0:1619677448.038270:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.038271:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.038272:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.038273:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:1.0:1619677448.038273:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677448.038279:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.038281:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.038282:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.038282:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.038284:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:10.0:1619677448.038284:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.038286:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:10.0:1619677448.038286:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:12.0:1619677448.038287:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.038287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:12.0:1619677448.038288:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.038290:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.038291:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.038294:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.038300:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.038301:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677448.038303:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.038305:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.038306:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.038307:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.038309:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.038310:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.038310:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.038314:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.038315:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.038318:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.038319:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.038319:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.038321:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.038321:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.038322:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.038323:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.038336:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.038339:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.038340:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.038340:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.038342:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.038345:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.038387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.038391:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.038393:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.038395:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.038397:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.038398:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.038399:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.038401:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.038402:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.038403:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.038404:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.038408:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.038410:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.038413:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.038414:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.038415:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.038417:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.038417:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.038418:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.038419:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.2:1619677448.038427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:31.0:1619677448.038430:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.038433:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:2.0:1619677448.038433:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:31.0:1619677448.038434:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.038435:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.038436:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.038437:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:30.0:1619677448.038438:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.038439:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:2.0:1619677448.038439:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.038442:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.038443:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.038445:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.038446:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.038447:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.038448:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.038451:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.038453:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.038456:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.038457:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.038457:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.038459:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.038460:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.038460:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.038461:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677448.038472:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.038475:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.038476:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.038478:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.038479:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.2:1619677448.038479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677448.038481:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677448.038486:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.038488:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.038489:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.038491:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.038493:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.038494:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.038495:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.038497:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.038498:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.038498:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.038501:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.038503:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.038506:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.038506:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.038507:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.038509:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.038509:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.038510:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.038512:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.038512:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.038517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.038522:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.038523:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677448.038524:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.038525:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.038525:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.038525:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.038526:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.038527:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677448.038528:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.038530:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.038669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.038677:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.038679:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.038681:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.038682:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.038685:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.038687:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.038688:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.038689:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.038691:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.038691:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.038695:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.038696:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.038699:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.038699:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.038700:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.038702:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.038702:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.038703:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.038704:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.038713:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.038716:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.038716:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.038718:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.038720:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.038721:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.038733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.038740:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.2:1619677448.038741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:26.0:1619677448.038743:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.038744:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.038745:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.038748:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.038749:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677448.038749:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.038750:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.038751:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:26.0:1619677448.038751:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.038752:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.038754:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.038754:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.038755:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.038756:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.038757:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.038757:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.038758:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.038758:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.038760:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.038760:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.038760:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:30.0:1619677448.038761:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.038761:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:30.0:1619677448.038762:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.038762:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.038762:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.038763:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:30.0:1619677448.038765:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.038765:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.038765:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.038767:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.038770:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.038771:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.038771:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:4.2:1619677448.038771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.038773:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.038773:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.038775:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.038776:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:10.0:1619677448.038776:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:30.0:1619677448.038777:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.038777:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.038778:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.038779:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:10.0:1619677448.038779:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.038780:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:8.2:1619677448.038780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.038781:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.038782:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.038782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.038782:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.038783:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.038788:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677448.038790:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:5.0:1619677448.038791:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.038792:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.038793:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.038793:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.038794:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677448.038794:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.038794:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.038797:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.038901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.038908:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.038911:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.2:1619677448.038912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:31.0:1619677448.038913:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.038915:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.038916:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.038917:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:18.0:1619677448.038918:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:31.0:1619677448.038919:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.038920:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.038921:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.038921:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.038921:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:31.0:1619677448.038923:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.038925:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:31.0:1619677448.038927:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.038928:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.038929:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:31.0:1619677448.038930:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.038930:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.038932:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.038933:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.038933:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:18.0:1619677448.038933:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.038934:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677448.038934:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.038936:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.038936:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.038937:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677448.038937:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:31.0:1619677448.038938:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677448.038939:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.038941:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.038942:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.038942:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.038944:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.038944:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.038945:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.038946:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677448.038948:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:9.2:1619677448.038950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:13.0:1619677448.038951:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.038951:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.038953:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.038954:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.038955:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.038957:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.038959:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.038960:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.038961:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.038962:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.038964:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:12.0:1619677448.038968:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.038970:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.038971:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.038972:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.038972:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.038973:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.039049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.039053:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.039055:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.039057:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.039058:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.039060:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.039062:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.039063:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.039064:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.039065:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.039067:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.039070:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.039072:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.039074:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.039075:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.039076:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.039076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677448.039077:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.039078:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.039079:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.039079:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:26.0:1619677448.039083:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:9.2:1619677448.039084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.039086:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.039087:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.039088:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.039090:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.039091:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:18.0:1619677448.039091:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:26.0:1619677448.039093:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.039093:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:26.0:1619677448.039094:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.039094:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:26.0:1619677448.039095:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.039095:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.039096:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677448.039097:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000400:00000200:26.0:1619677448.039100:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.039100:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:26.0:1619677448.039101:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.039104:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.039104:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.039105:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.039107:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.039107:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.039108:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.039109:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.039110:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.039115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.039118:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.039121:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.039122:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.039122:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:12.0:1619677448.039123:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.039124:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:12.0:1619677448.039125:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.039125:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:13.0:1619677448.039126:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.039126:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:4.2:1619677448.039159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.039166:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.039168:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.039169:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.039171:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.039173:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.039174:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.039175:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.039176:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.039177:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.039178:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.039180:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.039182:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.039183:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.039183:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.039184:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.039185:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.039185:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.039186:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.039186:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.039196:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.039198:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.039198:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.039199:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.039200:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.039201:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.039217:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.039219:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.039220:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.039220:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.039221:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.039221:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.039254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.039260:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.039262:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.039264:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.039265:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.039267:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.039269:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.039270:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.039272:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.039273:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.039274:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.039277:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.039278:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.039280:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.039281:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.039282:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.039283:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.039284:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.039285:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.039286:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.2:1619677448.039292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:21.2:1619677448.039297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:22.0:1619677448.039300:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.039300:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.039302:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.039302:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:22.0:1619677448.039303:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.039304:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:22.0:1619677448.039305:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.039305:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.039307:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:6.0:1619677448.039307:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.039308:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:6.0:1619677448.039309:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:24.0:1619677448.039310:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.039311:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.039312:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.039313:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.039313:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.039315:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:6.0:1619677448.039315:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:24.0:1619677448.039316:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.039316:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.039317:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.039318:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:24.0:1619677448.039319:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.039320:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677448.039320:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.039321:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.039322:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:22.0:1619677448.039322:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.039323:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.039323:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.039323:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.039324:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:24.0:1619677448.039326:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.039327:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.039328:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:6.0:1619677448.039328:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.039328:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.039330:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:24.0:1619677448.039331:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.039331:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:6.0:1619677448.039331:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.039331:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677448.039332:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:6.0:1619677448.039332:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677448.039334:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.039335:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.039335:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.039336:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.2:1619677448.039337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.2:1619677448.039340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.039344:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:30.0:1619677448.039346:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.039347:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:10.0:1619677448.039347:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.039348:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.039348:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.039349:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:20.0:1619677448.039350:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:10.0:1619677448.039350:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.039351:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.039351:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.039353:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.039353:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.039353:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.039353:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.0:1619677448.039354:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:30.0:1619677448.039355:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.039356:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.039356:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:30.0:1619677448.039357:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.039358:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.039359:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.039362:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.039362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.039364:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.039366:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.039367:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.039367:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.039369:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.039369:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.039369:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.039370:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.039371:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.039372:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.039373:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.039374:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.039374:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.039376:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:12.0:1619677448.039381:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.039384:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.039385:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.039387:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.039389:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.039390:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.039402:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.039404:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.039405:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.039406:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.039406:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.039407:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.039465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.039472:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.039475:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.039475:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.039477:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.039478:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.039480:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.039481:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.039482:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.039483:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.039484:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.039487:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.039488:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.039491:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.039491:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.039492:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.039493:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.039494:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.039494:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.039495:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.039506:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.039508:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.039509:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.039510:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.039512:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.039513:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.039548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.039555:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.039557:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.039558:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.039560:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.039562:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.039564:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.039565:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.039566:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.039567:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.039568:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.039571:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.039573:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.039576:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.039576:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.039577:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.039578:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.039579:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.039580:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.039581:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.039582:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.039586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677448.039587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.039594:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.039594:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677448.039595:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.039596:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.039597:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677448.039597:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.039598:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.039598:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.039600:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.039603:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.039683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:4.2:1619677448.039685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.039692:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.2:1619677448.039694:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:10.0:1619677448.039695:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:5.0:1619677448.039696:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:10.0:1619677448.039698:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.039699:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.039699:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.039700:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:5.0:1619677448.039700:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.039700:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000400:00000200:10.0:1619677448.039701:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.039701:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.039702:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:20.0:1619677448.039702:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:10.0:1619677448.039703:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.039704:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.039706:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.039707:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.039708:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.039710:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.039712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:10.0:1619677448.039712:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.039713:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.039713:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.039714:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.039715:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.039717:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.039717:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.039718:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.039719:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.039721:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.039722:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677448.039722:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.039722:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.039723:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:31.0:1619677448.039724:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.039724:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.039725:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:31.0:1619677448.039726:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.039726:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677448.039726:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.039727:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677448.039727:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.039727:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.039728:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.039728:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:31.0:1619677448.039729:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.039729:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.039729:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677448.039730:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.039731:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.039732:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.039732:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.039732:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677448.039733:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:26.0:1619677448.039733:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000400:00000200:31.0:1619677448.039735:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.039736:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.039737:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.039738:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.039738:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:8.2:1619677448.039738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677448.039739:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.039739:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677448.039739:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.039741:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:30.0:1619677448.039741:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.039742:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.039743:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.039743:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.039744:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.039745:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677448.039746:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.039749:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.039749:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.039750:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.039752:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677448.039752:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.039755:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677448.039755:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:12.0:1619677448.039756:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.039757:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.039759:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.039759:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.039786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.039791:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.039794:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.039795:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.039797:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.039799:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.039801:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.039802:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.039803:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.039805:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.039806:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.039809:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.039811:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.039813:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.039814:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.039815:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.039817:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.039817:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.039818:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.039818:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.039823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.039829:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.039832:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.039833:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.039833:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.039835:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.039838:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.2:1619677448.039911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.039918:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.039921:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.039922:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.039924:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.039927:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.039928:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.039929:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.039930:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.039931:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.039932:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.039935:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.039937:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.039939:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.039940:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.039940:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.039942:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.039943:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.039943:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.039944:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.039957:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.039960:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.039960:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.039962:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.039964:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.039965:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.040053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.040059:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.040061:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.040063:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.040064:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.040066:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.040067:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.040069:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.040070:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.040071:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.040072:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.040075:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.040076:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.040078:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.040079:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.040080:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.040081:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.040082:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.040083:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.040084:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.040085:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.040090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.040093:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:22.0:1619677448.040095:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677448.040095:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.040096:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.040096:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677448.040098:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.040099:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.040099:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.040101:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.040103:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.040104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:6.0:1619677448.040109:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677448.040112:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677448.040112:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000400:00000200:6.0:1619677448.040114:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.040117:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.040119:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:6.0:1619677448.040120:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.040121:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.040122:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.040123:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:6.0:1619677448.040127:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.040128:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677448.040131:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.040132:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677448.040133:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.040134:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677448.040135:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677448.040136:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677448.040136:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677448.040140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.040148:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.040151:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.040152:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.040152:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.040154:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.040156:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.040236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.040243:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.040246:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.040247:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.040249:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.040251:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.040253:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.040254:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.040255:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.040257:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.040257:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.040261:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.040263:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.040266:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.040267:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.040268:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.040270:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.040270:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.040271:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.040272:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.040276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.040284:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.040286:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.040287:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.040289:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.040290:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.040292:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.040334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.2:1619677448.040334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee354600] (20)++
00000800:00000200:30.0:1619677448.040343:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677448.040343:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee354600] (21)++
00000800:00000200:20.2:1619677448.040345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.040346:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:24.0:1619677448.040347:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.19@o2ib3
00000800:00000200:8.0:1619677448.040347:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee354600] (22)--
00000800:00000200:5.2:1619677448.040348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677448.040349:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.040350:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.19@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.040351:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.040351:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.040352:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:30.0:1619677448.040353:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:31.0:1619677448.040354:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.040354:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.040354:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.040354:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:24.0:1619677448.040355:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.040356:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.040357:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.040357:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.040357:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:30.0:1619677448.040358:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.040358:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677448.040360:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.040361:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.040362:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.040363:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.040363:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.040364:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.040365:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.040365:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.040366:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:24.0:1619677448.040367:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.040368:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677448.040368:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:24.0:1619677448.040369:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 520 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.040369:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.040370:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.040371:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.040372:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.040372:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.040373:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.040373:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:1.0:1619677448.040373:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.040374:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.040374:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677448.040375:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:24.0:1619677448.040375:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.040376:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee354600] (21)++
00000800:00000200:24.0:1619677448.040377:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee354600] (22)--
00000800:00000200:24.0:1619677448.040378:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee354600] (21)--
00000800:00000200:30.0:1619677448.040379:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.040380:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677448.040380:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:31.0:1619677448.040381:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.040382:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:1.0:1619677448.040382:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.040383:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.040384:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.040384:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677448.040384:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.040385:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677448.040385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.040386:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.040387:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.040387:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.040388:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:1.0:1619677448.040388:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.040389:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677448.040389:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.040389:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.040390:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.040391:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.040391:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:12.0:1619677448.040391:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677448.040391:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.040392:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:12.0:1619677448.040392:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.040392:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.040392:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.040393:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.040393:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677448.040394:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.040395:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.040396:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677448.040396:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:13.0:1619677448.040396:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.2:1619677448.040396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677448.040397:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:10.0:1619677448.040398:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.040399:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.040400:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.040400:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:10.0:1619677448.040401:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.040402:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:22.0:1619677448.040403:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.040404:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:10.0:1619677448.040404:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96720170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.040405:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.040405:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677448.040406:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:10.0:1619677448.040406:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.040407:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.040407:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:30.0:1619677448.040408:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.040408:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.040409:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677448.040410:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.040410:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:30.0:1619677448.040411:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.040411:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:10.0:1619677448.040411:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.040412:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:22.0:1619677448.040413:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.040414:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677448.040414:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.040414:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677448.040415:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:22.0:1619677448.040415:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.040416:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:5.0:1619677448.040416:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.040418:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:10.0:1619677448.040418:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.040418:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:5.0:1619677448.040418:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:10.0:1619677448.040419:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677448.040421:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:10.0:1619677448.040421:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.040421:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.040422:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.040422:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677448.040422:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677448.040423:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:18.0:1619677448.040424:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.040425:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.040426:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:18.0:1619677448.040426:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.040427:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.040427:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:18.0:1619677448.040427:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:31.0:1619677448.040429:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:18.0:1619677448.040429:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.040430:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.040431:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.040432:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.040435:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.040435:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:18.0:1619677448.040437:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677448.040437:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.040438:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.040439:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.040439:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.040440:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.040440:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677448.040441:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.040442:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.040442:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.040443:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.040444:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:6.0:1619677448.040444:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677448.040448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.040456:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.2:1619677448.040457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:2.0:1619677448.040458:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.040458:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.040460:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:18.0:1619677448.040460:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.040461:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.040462:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677448.040463:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677448.040465:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.040466:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677448.040467:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.040470:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677448.040471:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.040472:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.040473:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.040474:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677448.040477:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.040479:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677448.040482:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.040483:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677448.040483:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677448.040485:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677448.040486:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677448.040487:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677448.040487:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:9.2:1619677448.040491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:21.2:1619677448.040496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.040502:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.040504:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.040505:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:8.0:1619677448.040505:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.040507:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677448.040507:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.040509:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.040511:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.040512:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.040512:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677448.040513:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.040514:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.040515:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.040517:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.040518:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.040519:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677448.040522:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.040523:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.040526:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.040527:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.040528:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.040529:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.040530:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.040531:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.040531:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.040535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.040544:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.040546:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.040547:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.040549:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.040549:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.040552:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.040584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677448.040592:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677448.040595:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677448.040597:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.040599:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677448.040600:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.040601:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:30.0:1619677448.040603:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720166] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.040604:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.040605:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.040606:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677448.040609:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.040611:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.2:1619677448.040611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:30.0:1619677448.040614:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.040615:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:30.0:1619677448.040615:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677448.040617:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677448.040617:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677448.040617:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677448.040618:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677448.040619:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677448.040619:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.040620:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.040622:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.040623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:22.0:1619677448.040625:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.040626:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.040627:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.040629:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.040630:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.040631:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:17.0:1619677448.040631:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:22.0:1619677448.040634:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.040634:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.040634:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.040635:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.040636:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:17.0:1619677448.040637:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.040638:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.040639:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.040639:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:17.0:1619677448.040640:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:22.0:1619677448.040641:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.040642:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.040642:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.040643:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.040647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.040655:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.040658:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.040658:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.040659:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.040661:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.040663:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.040760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.2:1619677448.040764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.040767:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.040770:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:1.0:1619677448.040770:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.040771:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677448.040772:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.040773:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.040774:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.040776:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.040777:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.040778:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.040779:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.040781:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.040781:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.040782:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.040783:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.040785:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.040786:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.040786:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.040787:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.040788:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:26.0:1619677448.040793:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.040793:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.040795:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.040796:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:26.0:1619677448.040799:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.040799:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.040800:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:1.0:1619677448.040800:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:26.0:1619677448.040801:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.040801:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:26.0:1619677448.040803:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677448.040803:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:26.0:1619677448.040804:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677448.040804:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.040805:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677448.040805:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.040806:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:1.0:1619677448.040806:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.040809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677448.040813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:2.0:1619677448.040816:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.0:1619677448.040819:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:2.0:1619677448.040819:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.040820:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.040820:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.040821:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.040822:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:2.0:1619677448.040822:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.040823:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.040825:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.040825:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.0:1619677448.040828:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.040829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.040836:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.040839:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.040839:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.040840:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.040841:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.040842:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.040889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677448.040896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677448.040896:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.040899:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.040900:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.040902:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.040903:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677448.040904:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.040905:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677448.040906:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.040907:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.040908:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.040909:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677448.040909:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.040910:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.040911:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.040911:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.040912:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.040914:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.040915:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.040915:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.040916:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.040917:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.040917:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.040920:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.040921:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.040921:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.040922:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.040922:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.040924:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.040925:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.040926:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.040926:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:31.0:1619677448.040927:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.040927:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.040927:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.040928:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.040929:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.040929:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.2:1619677448.040930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.040931:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.040932:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.040933:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:4.2:1619677448.040934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.2:1619677448.040938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.040939:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677448.040941:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:20.0:1619677448.040941:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.040942:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.040943:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.040943:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677448.040943:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.040944:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:20.0:1619677448.040944:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.040945:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.040946:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.040946:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.040947:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677448.040947:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:6.0:1619677448.040947:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.040949:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:6.0:1619677448.040949:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.040951:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.040952:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.040952:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.041176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.041182:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.041185:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.041186:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.041188:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.041190:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.041192:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.041193:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.041195:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.041196:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.041197:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.041200:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.041202:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.041205:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.041206:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.041207:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.041208:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.041209:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677448.041211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:17.0:1619677448.041211:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.041211:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.041212:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677448.041216:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.2:1619677448.041217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.041219:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.041220:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.041222:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.041223:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:22.0:1619677448.041224:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.041224:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.041225:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677448.041226:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:1.0:1619677448.041226:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677448.041227:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.041228:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.041229:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.041230:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677448.041233:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:22.0:1619677448.041234:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.041235:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:22.0:1619677448.041236:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:13.0:1619677448.041236:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.041239:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677448.041239:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.041239:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677448.041240:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.041240:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.041242:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:13.0:1619677448.041242:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677448.041243:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.041244:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.041245:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.041246:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.041248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.041251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.041256:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677448.041256:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.041258:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:17.0:1619677448.041259:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:6.0:1619677448.041259:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:17.0:1619677448.041260:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.041260:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677448.041260:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.041262:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.041265:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.041273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.041282:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.041285:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.041286:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.041286:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.041287:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.041288:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.041310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.041318:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.041321:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.041323:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.041325:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.041326:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.041327:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.041329:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.041330:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.041331:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.041332:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.041335:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.041337:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.041340:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.041341:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.041342:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.041343:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.041344:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.041345:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.041346:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.041359:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.041362:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.041362:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.041363:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.041365:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.041368:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.041458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.041467:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.041469:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.041470:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.041472:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.041475:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.041476:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.041478:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.041479:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.041480:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.041481:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.041484:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.041486:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.041489:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.041490:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.041490:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.041492:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.041492:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.041493:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.041493:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.041498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.041508:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.041510:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.041511:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.041513:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.041513:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.041516:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.041517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.041524:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.041526:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.041527:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.041528:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.041529:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.041530:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.041611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:5.2:1619677448.041615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.041618:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.041620:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.041621:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.041623:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.041623:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.041625:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.041626:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.041628:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.041629:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720156] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.041631:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.041632:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.041634:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.041635:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.041637:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.041638:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.041638:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.041640:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.041640:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.041641:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.041642:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.041645:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.041646:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.041646:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.041647:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.041648:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.041649:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.041650:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.041650:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.041651:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.041651:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.041652:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.041652:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.041653:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677448.041653:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.041654:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.041654:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.041655:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:20.2:1619677448.041657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677448.041657:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.041657:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.041660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.041662:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.2:1619677448.041662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677448.041665:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:6.0:1619677448.041665:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.041666:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.041666:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.041667:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677448.041667:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:31.0:1619677448.041668:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:20.0:1619677448.041669:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:6.0:1619677448.041669:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.041669:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:20.0:1619677448.041670:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.041670:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:31.0:1619677448.041671:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.041671:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.041671:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:30.0:1619677448.041672:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:6.0:1619677448.041672:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:31.0:1619677448.041673:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.041673:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.041674:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:31.0:1619677448.041675:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.041676:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.041678:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.041679:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.041680:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.041681:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:31.0:1619677448.041683:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.041684:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.041685:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.041685:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.041685:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.041686:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.041687:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677448.041688:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.041689:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.041689:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.041691:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.041691:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.041692:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.041693:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.041697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.041705:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.041707:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.041708:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.041710:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.041710:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.041712:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.041735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.041738:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.041740:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.041743:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.041743:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.041745:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.041745:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:20.0:1619677448.041747:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.041748:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.041749:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.041750:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.041751:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.041751:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.041754:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.041755:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677448.041756:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.041757:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.041758:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.041760:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.041762:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.041762:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:20.0:1619677448.041763:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.041764:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:17.0:1619677448.041764:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.041765:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.041766:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.041766:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:17.0:1619677448.041766:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.041767:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:17.0:1619677448.041767:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.041768:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677448.041770:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.041772:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.041775:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.041776:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.041777:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.041779:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.041779:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.041779:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.041780:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.041780:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.041782:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.041783:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.041784:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.041787:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.041788:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.041788:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677448.041791:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.041792:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.041793:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.041795:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.041797:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.041797:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677448.041801:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.041801:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.041802:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.041803:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.041804:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.041808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.041814:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.041817:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.041817:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.041819:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.041822:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.041824:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.041825:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.041827:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.041828:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.041829:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.041832:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.041834:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.041837:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.041838:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.041838:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.041840:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.041841:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.041841:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.041841:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:26.0:1619677448.041853:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.041855:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.041856:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.041857:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.041859:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.041862:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.041875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.041882:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.041885:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.041887:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.041889:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.041890:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.041892:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.041893:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.041894:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.041895:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.041896:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.041900:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.041902:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.041904:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.041905:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.041906:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.041908:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.041908:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.041909:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.041910:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.041921:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.041924:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.041925:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.041925:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.041927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000400:00000200:6.0:1619677448.041927:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.041930:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677448.041932:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.041936:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677448.041936:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000400:00000200:2.0:1619677448.041938:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.041941:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.041942:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677448.041944:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720149] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.041945:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.041946:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.041947:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677448.041950:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.041952:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677448.041954:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.041955:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677448.041956:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677448.041958:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677448.041958:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677448.041960:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677448.041961:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677448.041961:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:21.2:1619677448.041966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:20.2:1619677448.041969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:10.0:1619677448.041970:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:18.0:1619677448.041972:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:10.0:1619677448.041972:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.041973:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677448.041974:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000400:00000200:10.0:1619677448.041974:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.041975:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677448.041975:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:5.0:1619677448.041975:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:10.0:1619677448.041976:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.041978:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:10.0:1619677448.041979:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.0:1619677448.041980:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.041980:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.041983:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.041985:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677448.041986:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.041987:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.041988:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.041989:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677448.041993:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.041994:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677448.041997:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.041998:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.041999:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:5.0:1619677448.041999:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677448.042000:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.042001:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:26.0:1619677448.042002:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.042002:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.042003:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:5.0:1619677448.042003:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:26.0:1619677448.042004:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:5.0:1619677448.042004:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:26.0:1619677448.042005:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.0:1619677448.042005:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.2:1619677448.042009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.042012:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:31.0:1619677448.042013:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:20.0:1619677448.042015:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:31.0:1619677448.042016:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:20.0:1619677448.042016:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.042016:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:31.0:1619677448.042017:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.042019:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.042020:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.042022:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.042042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.042050:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.042052:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.042053:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.042055:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.042058:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.042059:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.042060:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.042062:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.042063:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.042064:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.042067:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.042068:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.042071:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.042072:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.042073:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.042074:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.042075:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.042076:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.042076:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.042088:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.042091:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.042091:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.042092:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.042094:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.042097:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.042134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.042139:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.042141:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.042142:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.042144:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.042146:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.042148:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.042149:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.042150:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.042151:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.042152:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.042155:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.042157:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.042159:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.042160:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.042161:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.042162:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.042163:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.042163:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.042164:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.042169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.042176:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.042179:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.042180:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.042180:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.042182:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.042184:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.042569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.042578:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.042581:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.042583:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.042585:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.042586:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.042587:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.042588:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.042590:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.042591:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.042592:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:5.2:1619677448.042593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:30.0:1619677448.042595:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.042597:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.042599:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.042599:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:30.0:1619677448.042600:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.042601:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.042602:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677448.042602:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:30.0:1619677448.042603:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677448.042603:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:30.0:1619677448.042604:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:2.0:1619677448.042605:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.042606:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.042607:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:2.0:1619677448.042608:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.042609:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.042611:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.042612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:2.0:1619677448.042612:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.042613:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.042614:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.042617:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.042619:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.042621:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:2.0:1619677448.042621:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.042622:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.042623:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:2.0:1619677448.042623:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.042624:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.042625:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:2.0:1619677448.042625:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.042625:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.042626:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.042626:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677448.042627:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.042630:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.042631:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.042632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:12.0:1619677448.042633:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.042635:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.042636:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677448.042639:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.042641:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.042642:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.042644:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.042645:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:13.0:1619677448.042646:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.042647:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:13.0:1619677448.042649:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:5.2:1619677448.042649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677448.042650:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.2:1619677448.042650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:13.0:1619677448.042651:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.042651:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.042652:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677448.042655:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.042657:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677448.042658:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:5.0:1619677448.042659:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.042659:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.042661:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.042661:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.042661:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.042662:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.042663:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:17.0:1619677448.042663:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.042665:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.042666:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.042668:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.042669:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.042670:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.042673:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.042675:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.042678:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.042679:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.042680:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.042682:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.042682:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.042683:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.042683:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:21.2:1619677448.042684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:4.2:1619677448.042688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677448.042691:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.042694:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.042695:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:26.0:1619677448.042696:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:24.0:1619677448.042696:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.042699:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:24.0:1619677448.042699:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.042700:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:24.0:1619677448.042701:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.042702:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.042702:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96720142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.042703:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.042704:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:26.0:1619677448.042705:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.042705:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.042705:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.042707:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:12.0:1619677448.042707:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:26.0:1619677448.042708:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.042708:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.042710:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.042710:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677448.042710:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.042711:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.042711:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:26.0:1619677448.042712:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.042712:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.042713:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677448.042715:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.042716:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.042716:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.042717:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.042717:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.042718:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.042719:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.042719:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:26.0:1619677448.042720:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.042720:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:26.0:1619677448.042721:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.042722:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.042724:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.042724:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.042725:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.042725:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.042730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.042737:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.042740:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.042741:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.042741:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677448.042742:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:18.0:1619677448.042744:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.042744:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.042745:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.042747:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:13.0:1619677448.042747:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.042748:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.042749:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677448.042834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.042841:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.042844:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.042846:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.042847:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.102@o2ib7 : GET - for me
00000400:00000200:2.0:1619677448.042850:0:19932:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.102@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:2.0:1619677448.042852:0:19932:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.102@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:2.0:1619677448.042854:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.042855:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.042857:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.042861:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : REPLY try# 0
00000800:00000200:2.0:1619677448.042862:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.042864:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.042865:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.042865:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.042867:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.042867:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:9.2:1619677448.042873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.042879:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.042881:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.042882:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.042883:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.042884:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: REPLY: OK
00000800:00000200:18.0:1619677448.042885:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.043045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.043051:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.043054:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.043055:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.043057:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.043060:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.043061:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.043062:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.043064:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.043065:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.043066:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.043069:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.043070:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.043073:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.043074:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.043075:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.043076:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.043077:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.043077:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.043078:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.043082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.043090:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.043093:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.043093:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.043094:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.043096:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.043099:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.043103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.043108:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.043111:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.043112:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.043114:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.043116:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.043118:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.043119:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.043120:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.043121:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.043122:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.043126:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.043127:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.043130:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.043131:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.043131:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.043133:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.043134:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.043134:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.043134:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.043147:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.043150:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.043151:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.043151:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.043153:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.043155:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.043217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.043224:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.043227:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.043228:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.043230:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.043233:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.043235:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.043236:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.043237:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.043238:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.043240:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.043243:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.043245:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.043247:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.043248:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.043248:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.043250:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.043250:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.043252:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.043253:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.043253:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.043259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.043262:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.043264:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.043265:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.043266:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677448.043267:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.043268:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:30.0:1619677448.043269:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.043269:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.043270:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.043270:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677448.043291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.043298:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.043301:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.043302:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.043303:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.043306:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.043308:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677448.043309:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.043311:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.043312:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.043313:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.043316:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.043318:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.043321:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.043322:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677448.043323:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677448.043325:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677448.043325:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.043326:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.043326:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677448.043340:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.043342:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.043343:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.043345:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.043347:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.043348:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.043628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.043636:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.043639:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.043640:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.043642:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.043644:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.043646:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.043647:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.043648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:10.0:1619677448.043648:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.043649:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.043650:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.043654:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.043654:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.043656:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677448.043657:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:10.0:1619677448.043658:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.043659:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677448.043659:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:5.0:1619677448.043659:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.043660:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.043661:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.043662:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:5.0:1619677448.043662:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.043663:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.043664:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000400:00000200:5.0:1619677448.043664:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:10.0:1619677448.043665:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:5.0:1619677448.043665:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.043667:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.043668:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.043670:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:5.0:1619677448.043673:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.043674:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.043677:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:26.0:1619677448.043677:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:5.0:1619677448.043677:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.043678:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.043679:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.043679:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.0:1619677448.043679:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:30.0:1619677448.043680:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.043680:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677448.043681:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.043682:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.0:1619677448.043682:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.043683:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.043683:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.043685:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.043686:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.043687:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.043687:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677448.043694:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.043696:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.043697:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.043698:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.043698:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.043699:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.043700:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677448.043701:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.043701:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.043702:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.043703:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.043706:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.043804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.043812:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.043814:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.043816:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.043818:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.043819:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.043820:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.043821:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.043823:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.043824:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.043825:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.043827:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.043829:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.043832:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.043832:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.043833:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.043834:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.043835:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.043836:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.043836:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677448.043847:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.043849:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.043850:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.043851:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.043853:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.043854:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677448.043865:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.043867:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.043868:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.043869:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.043869:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.043870:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.043945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.043952:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.043954:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.043955:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.043957:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.043959:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.043961:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.043962:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.043963:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.043964:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.043965:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.2:1619677448.043968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:6.0:1619677448.043968:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.043970:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.043971:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.043974:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:6.0:1619677448.043974:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.043975:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.043975:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:20.0:1619677448.043976:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:6.0:1619677448.043977:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.043977:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.043978:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.043978:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.043978:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.043979:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:20.0:1619677448.043980:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.043981:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.043983:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.043984:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.043985:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.043988:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:30.0:1619677448.043990:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.043990:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.043992:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677448.043992:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.043993:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.043993:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.043994:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677448.043995:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.043995:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.043996:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.043996:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.043997:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.043997:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677448.043998:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:9.2:1619677448.044002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.044009:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.044012:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.044013:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.044013:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.044016:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.044019:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.044098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.044104:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.044106:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.044107:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.044109:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.044111:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.044113:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.044114:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.044116:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.044117:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.044118:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.044121:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.044123:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.044126:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.044127:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.044127:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.044128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:1.0:1619677448.044129:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.044130:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.044131:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.044132:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.044132:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.044134:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.044137:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.044137:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.044139:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.044142:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:2.0:1619677448.044142:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.044143:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:12.0:1619677448.044145:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:2.0:1619677448.044145:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.044146:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.044146:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.044147:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677448.044148:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.044148:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677448.044149:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:2.0:1619677448.044152:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.044154:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677448.044154:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.044156:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.044157:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.044157:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.044159:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.044160:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.044160:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.044161:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.044166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.044170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.044176:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.044177:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.044178:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.044178:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:21.2:1619677448.044179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:17.0:1619677448.044179:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.044179:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.044180:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:17.0:1619677448.044181:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.044181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677448.044181:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.044184:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677448.044186:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.044188:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.044189:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677448.044191:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.044191:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:31.0:1619677448.044192:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.044193:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.044194:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.044194:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.044196:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.044196:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.044197:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.044198:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677448.044199:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.044199:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.044200:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.044201:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.044201:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.044202:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.044203:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677448.044206:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.044207:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.044208:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.044209:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.044211:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.044212:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.044212:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.044213:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.044213:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.044213:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.044215:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677448.044215:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.044215:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.044216:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677448.044216:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.044217:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.044217:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.044217:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.044227:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.044229:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:21.2:1619677448.044229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:30.0:1619677448.044230:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.044231:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.044232:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:30.0:1619677448.044233:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.2:1619677448.044233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677448.044234:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:22.0:1619677448.044234:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:18.0:1619677448.044235:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:22.0:1619677448.044236:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.044236:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.044238:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.044238:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.044239:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:26.0:1619677448.044241:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677448.044241:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:18.0:1619677448.044241:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.044243:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:30.0:1619677448.044244:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:18.0:1619677448.044244:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.044244:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:18.0:1619677448.044246:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677448.044247:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677448.044247:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.044248:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.044248:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:26.0:1619677448.044248:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.044249:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:18.0:1619677448.044249:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.044250:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:26.0:1619677448.044250:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.044250:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.044251:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.044252:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.044254:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.044255:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.044256:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.044257:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.044257:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.044258:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:5.2:1619677448.044258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee354600] (20)++
00000800:00000200:18.0:1619677448.044261:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.044262:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.044262:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.044263:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.044264:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.044264:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677448.044265:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee354600] (21)++
00000800:00000200:18.0:1619677448.044265:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.044266:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:26.0:1619677448.044267:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.044268:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.044268:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.044268:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.19@o2ib3
00000800:00000200:18.0:1619677448.044268:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.044268:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:8.0:1619677448.044269:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee354600] (22)--
00000800:00000200:26.0:1619677448.044270:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.044270:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.044271:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:24.0:1619677448.044271:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.19@o2ib3 : ACK - routed
00000800:00000200:26.0:1619677448.044272:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:24.0:1619677448.044273:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.044273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000400:00000200:24.0:1619677448.044274:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.044276:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96720125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.044277:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.044278:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.044279:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.044279:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.044279:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:13.0:1619677448.044281:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.044281:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:24.0:1619677448.044282:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : ACK try# 0
00000400:00000200:13.0:1619677448.044282:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.044282:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677448.044282:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.044283:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:24.0:1619677448.044284:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:13.0:1619677448.044284:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.044284:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:24.0:1619677448.044286:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677448.044286:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.044287:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:24.0:1619677448.044287:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677448.044287:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.044287:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.044289:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677448.044289:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677448.044290:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee354600] (21)++
00000800:00000200:12.0:1619677448.044290:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:24.0:1619677448.044291:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee354600] (22)--
00000800:00000200:13.0:1619677448.044291:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.044292:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee354600] (21)--
00000800:00000200:12.0:1619677448.044306:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.044309:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.044309:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.044311:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: ACK: OK
00000800:00000200:13.0:1619677448.044313:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.044313:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.2:1619677448.044314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.044323:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.044325:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.044328:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.044330:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.044331:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.044332:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.044334:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.044335:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.044336:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.044337:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.044340:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.044342:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.044344:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.044345:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.044345:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.044347:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.044347:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.044348:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.044349:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.044359:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.044361:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.044362:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.044363:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.044365:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.044366:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.044384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.044390:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.044393:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.044395:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.044396:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.044398:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.044400:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.044401:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.044402:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.044403:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.044404:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.044407:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.044409:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.044412:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.044413:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.044414:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.044416:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.044416:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.044417:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.044417:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677448.044428:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.044431:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.044432:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.044434:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.044435:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.044437:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.044553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.044561:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.044563:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.044564:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.044566:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.044568:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.044570:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.044571:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.044572:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.044573:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.044574:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.044577:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.044578:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.044581:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.044581:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.044582:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.044583:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.044584:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.044584:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.044585:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.044596:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.044598:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.044599:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.044600:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.044602:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.044603:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.044646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.044653:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.044656:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.044657:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.044658:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.044661:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.044662:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.044663:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.044665:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.044665:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.044666:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.044669:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.044670:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.044673:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.044674:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.044674:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.044675:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.044676:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.044677:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.044677:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.044687:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.044689:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.044690:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.044692:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.044693:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.044694:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.044716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.044723:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.044725:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.044726:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.044728:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.044729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:1.0:1619677448.044731:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.044732:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.044733:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.044735:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:1.0:1619677448.044735:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.044736:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.044737:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.044738:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.044739:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.044741:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677448.044742:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.044744:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.044744:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:18.0:1619677448.044745:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.044746:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.044747:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.044748:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.044748:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:18.0:1619677448.044749:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.044749:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:18.0:1619677448.044750:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.044751:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.044751:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.044752:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677448.044753:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677448.044753:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677448.044755:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.044758:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.044759:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.044759:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.044761:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.044761:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.044762:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.044763:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677448.044763:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.044766:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.044767:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.044768:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.044769:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.044771:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677448.044775:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.044777:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.044778:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.044778:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.044779:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:6.0:1619677448.044780:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:12.0:1619677448.044782:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.044783:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.044783:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:6.0:1619677448.044783:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677448.044784:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:4.2:1619677448.044785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:12.0:1619677448.044786:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677448.044792:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.044795:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.044796:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.044798:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.044800:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.044802:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.044803:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.044804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:8.0:1619677448.044805:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.044806:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.044807:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.044812:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:8.0:1619677448.044812:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.044813:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.044814:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.044815:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.044818:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.044818:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.044819:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.044819:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:24.0:1619677448.044820:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.044821:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96720117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.044821:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:24.0:1619677448.044823:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.044823:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.044824:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.044825:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.2:1619677448.044827:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:24.0:1619677448.044828:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.044829:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.044832:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.044832:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677448.044833:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.044833:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.044834:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.044835:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.044835:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.044837:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.044837:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.044838:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:17.0:1619677448.044838:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.044839:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.044839:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.044840:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.044840:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:17.0:1619677448.044841:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.044843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.044844:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.044846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:8.2:1619677448.044846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.044850:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.044853:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.044854:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.044854:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:2.0:1619677448.044854:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.044854:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.044856:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.044857:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:2.0:1619677448.044857:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.044857:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.044858:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.044858:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677448.044859:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:1.0:1619677448.044859:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.044860:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677448.044861:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.044863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.044866:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.044868:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.044871:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.044872:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.044874:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.044875:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.044876:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.044878:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.044880:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:21.2:1619677448.044883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677448.044883:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.044884:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.044888:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.044889:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.044889:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.044891:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.044891:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:20.0:1619677448.044892:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.044893:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677448.044893:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.044894:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677448.044895:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.044895:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.044896:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.044896:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677448.044897:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.044898:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.044898:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.044899:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000400:00000200:30.0:1619677448.044900:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:20.0:1619677448.044900:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677448.044901:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.044902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677448.044903:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.044904:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.044905:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:20.2:1619677448.044905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.044907:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.044908:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.044908:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.044909:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.044909:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:31.0:1619677448.044910:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.044911:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.044911:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.044911:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:18.0:1619677448.044912:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.044913:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.044913:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:2.0:1619677448.044913:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.044914:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677448.044915:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.044915:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.044915:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:31.0:1619677448.044916:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.044916:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.044917:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.044918:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.044918:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677448.044918:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677448.044919:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.044920:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:9.2:1619677448.044920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:31.0:1619677448.044921:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.044921:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.044922:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.044922:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.044922:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.044923:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.044924:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.044924:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.2:1619677448.044924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.044925:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:4.2:1619677448.044925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677448.044926:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.044927:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.044927:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677448.044929:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.044930:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.044930:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.044931:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.044932:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.044932:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.044934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.044934:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:13.0:1619677448.044935:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.044935:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.044936:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.044937:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.044937:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:13.0:1619677448.044938:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.044939:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.044939:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.044940:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:10.0:1619677448.044940:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.044942:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677448.044942:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.044942:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.044942:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:13.0:1619677448.044943:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.044944:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.044945:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.044945:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677448.044945:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677448.044945:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.044946:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:10.0:1619677448.044946:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.044947:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.044947:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:10.0:1619677448.044951:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.044953:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.044955:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.044955:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.044956:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.044958:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.044958:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.044959:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.044959:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.044970:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.044972:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.044973:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.044975:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.044976:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.044977:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.045033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.045041:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.045043:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.045044:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.045046:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.045048:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.045050:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.045051:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720112] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.045052:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.045053:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.045054:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.045057:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.045058:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.045061:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.045062:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.045063:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.045064:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.045065:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.045066:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.045066:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677448.045078:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.045081:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.045082:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.045082:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.045084:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.045087:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677448.045096:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.045098:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.045099:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.045100:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.045100:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.045101:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.045303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.045310:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.045313:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.045314:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.045316:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.045319:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.045320:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.045321:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.045323:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.045324:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.045324:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.045327:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.045328:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.2:1619677448.045329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.045331:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.045332:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.045332:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.045334:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.045334:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.045335:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.045336:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.045337:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.045337:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.045338:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.045338:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.045340:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.045343:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.045344:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:4.2:1619677448.045344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677448.045346:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.045347:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.045348:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:17.0:1619677448.045348:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.045349:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.045351:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.045351:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.045352:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.045352:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.045352:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.045353:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.045354:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.045354:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.045355:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.045355:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.045357:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.045358:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677448.045358:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.045359:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.045361:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.045361:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.045362:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.045362:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.045368:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.045371:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.045372:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.045372:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.045373:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:12.0:1619677448.045374:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677448.045374:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677448.045377:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.045378:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.045379:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.045381:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.045382:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677448.045390:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.045393:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.045394:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.045395:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.045395:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.045397:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.045405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.045411:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.045414:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.045414:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.045416:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.045419:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.045420:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.045422:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.045423:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.045424:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.045425:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:21.2:1619677448.045428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:18.0:1619677448.045429:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.045430:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.045433:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.045434:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.045435:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.045436:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.045437:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.045437:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.045438:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.045438:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:31.0:1619677448.045439:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.045441:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.045444:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.045444:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.045446:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677448.045447:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.045448:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.045449:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.045449:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:31.0:1619677448.045450:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:12.0:1619677448.045451:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.045452:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.045454:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.045455:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.045455:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.045457:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:12.0:1619677448.045457:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.045460:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.045461:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677448.045461:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.045463:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677448.045464:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.045465:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.045465:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.045480:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.045483:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.045484:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.045484:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:5.2:1619677448.045484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:22.0:1619677448.045486:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.045489:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:10.0:1619677448.045492:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.045494:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.045495:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.045497:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.045500:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.045502:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.045503:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.045504:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.045505:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.045506:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.045509:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.045511:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.045513:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.045514:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.045515:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.045516:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.045517:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.045517:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.045519:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.045519:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.045525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677448.045528:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.045530:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.045531:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.045532:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:13.0:1619677448.045532:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.045533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:26.0:1619677448.045534:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:12.0:1619677448.045534:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.045535:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:13.0:1619677448.045535:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.045536:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.0:1619677448.045536:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.045539:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.045541:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.045542:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.045543:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.045545:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677448.045546:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.045548:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.045549:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:20.0:1619677448.045552:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.045553:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:5.0:1619677448.045554:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.045556:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.045557:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.045558:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.045559:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:5.0:1619677448.045561:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.045562:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677448.045563:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.045564:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000400:00000200:5.0:1619677448.045564:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.045565:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:5.0:1619677448.045565:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.045566:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000400:00000200:5.0:1619677448.045566:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.045567:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.045568:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.045569:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.045570:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:5.0:1619677448.045570:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.045570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677448.045571:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:9.2:1619677448.045572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.0:1619677448.045572:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.045575:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.045575:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.045576:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.045578:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677448.045578:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677448.045579:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:5.0:1619677448.045579:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.045579:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.045580:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:10.0:1619677448.045581:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.045582:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677448.045582:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.045582:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.045583:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.045583:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.045585:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.045586:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.045587:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:24.0:1619677448.045588:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:10.0:1619677448.045588:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.045589:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.045590:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.045591:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:6.0:1619677448.045592:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:10.0:1619677448.045594:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.045594:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.045595:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:9.2:1619677448.045595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000400:00000200:6.0:1619677448.045595:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.045596:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:6.0:1619677448.045597:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.045598:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.045598:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.045599:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.045600:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:6.0:1619677448.045600:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677448.045601:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677448.045601:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:10.0:1619677448.045602:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.045602:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.0:1619677448.045603:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.045604:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.045604:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.045605:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.045606:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:26.0:1619677448.045612:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.045614:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.045615:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.045615:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.045617:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.045618:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.045631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.045637:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.045640:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.045641:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.045643:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.045646:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.045648:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.045649:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.045651:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.045651:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.045653:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.045656:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.045657:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.045660:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.045661:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.045662:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.045664:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.045664:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.045665:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.045665:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677448.045676:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.045679:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.045680:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.045682:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.045683:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.045685:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.045714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.045719:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.045722:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.045723:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.045725:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.045727:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.045729:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.045730:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.045732:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.045733:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.045734:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.045737:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.045739:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.045742:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.045743:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.045743:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.045745:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.045746:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.045746:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.045747:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.045757:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.045759:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.045760:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.045761:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.045763:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.045766:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677448.045785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:5.2:1619677448.045789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.045793:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:1.0:1619677448.045794:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.045795:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.045796:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:17.0:1619677448.045798:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.045798:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:26.0:1619677448.045799:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.045801:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.045801:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.045803:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.045804:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.045804:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.045805:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677448.045806:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.045807:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.045808:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.045809:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677448.045809:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.045810:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.045811:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677448.045815:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000400:00000200:1.0:1619677448.045816:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.045817:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677448.045819:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677448.045820:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.045821:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.045821:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.045822:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.045822:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:26.0:1619677448.045823:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677448.045823:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:26.0:1619677448.045824:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.045825:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:1.0:1619677448.045825:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:26.0:1619677448.045826:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:1.0:1619677448.045826:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.045827:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.045828:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.045829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:9.2:1619677448.045831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.045837:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:20.0:1619677448.045837:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:24.0:1619677448.045840:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:20.0:1619677448.045840:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.045841:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.045841:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.045842:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.045843:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000400:00000200:24.0:1619677448.045844:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.045844:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.045846:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:24.0:1619677448.045847:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677448.046058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.046066:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.046069:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.046072:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.046073:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:4.2:1619677448.046073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:30.0:1619677448.046074:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.046076:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.046077:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.046078:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.046079:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.046080:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.046081:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.046083:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:30.0:1619677448.046084:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.046084:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.046086:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.046087:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.046088:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.046089:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.046089:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:10.0:1619677448.046089:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.046091:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.046091:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677448.046091:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677448.046092:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677448.046092:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.046093:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:10.0:1619677448.046093:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.046094:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.046095:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.046098:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.046100:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.046103:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.046103:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.046104:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.046105:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.046105:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:26.0:1619677448.046106:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.046107:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.046107:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.046109:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.046109:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.046110:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.046110:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.046111:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:6.0:1619677448.046121:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.046124:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.046125:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.046125:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.046127:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.046130:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.046160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.046166:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.046169:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.046170:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.046172:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.046175:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.046176:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.046177:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.046179:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.046180:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.046181:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.046184:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.046185:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.046187:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.046188:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.046189:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.046190:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.046191:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.046192:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.046193:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.046193:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.046196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.046199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.046205:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.046206:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.046206:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.046206:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.046207:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.046207:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.046208:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.046208:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.046209:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.046209:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.046258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.046266:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.046269:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.046271:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.046273:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.046274:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.046275:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.046276:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.046278:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.046279:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.046280:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.046283:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.046285:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.046288:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.046289:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.046290:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.046292:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.046292:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.046299:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:8.2:1619677448.046299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.046301:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.046301:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677448.046307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.046307:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.046310:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.046311:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.046313:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.046315:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.046316:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677448.046317:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.046319:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.046320:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.046321:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:21.2:1619677448.046421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.046427:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.046430:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.046431:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.046433:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.046436:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.046438:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677448.046439:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.046440:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.046441:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.046442:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677448.046446:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.046447:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677448.046450:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.046450:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677448.046451:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.046453:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677448.046453:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.046454:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.046454:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677448.046459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.046466:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.046469:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.046470:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.046471:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.046473:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.046474:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.046494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.046499:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.046503:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.046503:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.046506:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.046508:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.046508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:22.0:1619677448.046510:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.046511:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.046512:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.046513:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.046514:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.046516:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677448.046517:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.046519:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677448.046519:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.046519:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:22.0:1619677448.046521:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.046522:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.046523:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.046523:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.046525:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.046525:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677448.046525:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.046526:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.046527:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000400:00000200:10.0:1619677448.046527:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:22.0:1619677448.046528:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677448.046528:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.046530:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.046531:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.046531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:10.0:1619677448.046532:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.046535:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.046537:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.046538:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:10.0:1619677448.046540:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.046541:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:10.0:1619677448.046541:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.046541:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:17.0:1619677448.046542:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.046542:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:10.0:1619677448.046543:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.046543:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677448.046544:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.046544:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.2:1619677448.046544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:10.0:1619677448.046545:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677448.046547:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.046549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.046550:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.046552:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.046553:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.046555:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.046556:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:2.0:1619677448.046558:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.046559:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:2.0:1619677448.046559:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.046560:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.046560:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.046562:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.046562:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.046562:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.046563:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.046564:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:2.0:1619677448.046564:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.046567:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.046569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.046569:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.046572:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.046573:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.046574:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.046575:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.046576:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.046576:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.046577:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:20.0:1619677448.046578:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.046578:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:20.0:1619677448.046579:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677448.046579:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677448.046580:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.046580:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.046582:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.046584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677448.046587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677448.046590:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:13.0:1619677448.046590:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.046592:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.046593:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:13.0:1619677448.046593:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677448.046593:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:13.0:1619677448.046594:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.046594:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677448.046595:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.046596:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677448.046596:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.046597:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:1.0:1619677448.046598:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.046599:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677448.046600:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.046601:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.046603:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.046604:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.046605:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.046606:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677448.046606:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:12.0:1619677448.046609:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.046610:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677448.046610:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.046611:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.046611:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.046612:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:12.0:1619677448.046613:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.046615:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.046616:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.046616:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.046618:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.046618:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.046619:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.046620:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.046624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:21.2:1619677448.046625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677448.046630:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:6.0:1619677448.046630:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:8.0:1619677448.046633:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:6.0:1619677448.046633:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:24.0:1619677448.046634:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:6.0:1619677448.046634:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.046635:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:8.0:1619677448.046635:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.046637:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.046640:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677448.046644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677448.046650:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.046652:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677448.046653:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.046653:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.046654:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.046654:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677448.046671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.046678:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.046681:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.046683:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.046685:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.046686:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.046687:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.046688:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.046689:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.046690:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.046691:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.046694:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.046696:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.046699:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.046700:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.046700:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.046702:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.046703:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.046704:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.046704:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.046716:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.046718:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.046719:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.046720:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.046722:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.046725:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.046948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.046954:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.046956:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.046957:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.046960:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.046962:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.046963:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677448.046965:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720089] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.046966:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.046967:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.046968:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677448.046971:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.046973:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.046975:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.046976:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677448.046976:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677448.046978:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.046978:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.046979:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.046980:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.046981:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.2:1619677448.046986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.046991:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:22.0:1619677448.046992:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677448.046993:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.046993:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677448.046994:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677448.046994:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:22.0:1619677448.046995:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.046996:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.046997:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.047000:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.047020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fd8000] (20)++
00000800:00000200:5.0:1619677448.047024:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fd8000] (21)++
00000800:00000200:5.0:1619677448.047027:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.1.13@o2ib3
00000800:00000200:20.0:1619677448.047030:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (22)--
00000400:00000200:5.0:1619677448.047030:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.13@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.047032:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.047034:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:5.0:1619677448.047035:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.047036:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.047037:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.047038:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:5.0:1619677448.047041:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.047043:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:5.0:1619677448.047046:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.047046:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677448.047047:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.0:1619677448.047049:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677448.047049:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fd8000] (21)++
00000800:00000200:5.0:1619677448.047050:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fd8000] (22)--
00000800:00000200:5.0:1619677448.047051:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (21)--
00000800:00000200:21.2:1619677448.047053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fd8000] (20)++
00000800:00000200:9.2:1619677448.047055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677448.047059:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fd8000] (21)++
00000800:00000200:20.0:1619677448.047061:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.13@o2ib3
00000800:00000200:8.0:1619677448.047061:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:20.0:1619677448.047063:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.13@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.047064:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.047064:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:5.0:1619677448.047064:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (22)--
00000400:00000200:8.0:1619677448.047065:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.047066:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.047068:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677448.047068:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.047069:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.047070:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.047072:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.047073:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677448.047075:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:20.0:1619677448.047076:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.047078:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.047081:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.047082:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.047082:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.047084:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.047085:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fd8000] (21)++
00000800:00000200:21.2:1619677448.047086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fd8000] (22)++
00000800:00000200:20.0:1619677448.047086:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fd8000] (22)--
00000800:00000200:20.0:1619677448.047087:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (22)--
00000800:00000200:20.0:1619677448.047088:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fd8000] (21)++
00000800:00000200:20.0:1619677448.047090:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.13@o2ib3
00000800:00000200:5.0:1619677448.047091:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (22)--
00000400:00000200:20.0:1619677448.047092:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.13@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.047094:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.047095:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.047096:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.047097:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.047097:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:20.0:1619677448.047098:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.047099:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.047100:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.047101:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:21.2:1619677448.047101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fd8000] (21)++
00000400:00000200:20.0:1619677448.047101:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.047101:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.047103:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:6.0:1619677448.047103:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.047105:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.047105:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677448.047105:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fd8000] (22)++
00000800:00000200:20.0:1619677448.047106:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.047106:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.047107:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.13@o2ib3
00000800:00000200:20.0:1619677448.047108:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.047108:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fd8000] (23)++
00000800:00000200:20.0:1619677448.047109:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fd8000] (24)--
00000400:00000200:5.0:1619677448.047109:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.047110:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (23)--
00000800:00000200:20.0:1619677448.047111:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (22)--
00000400:00000200:5.0:1619677448.047111:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.047113:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:5.0:1619677448.047114:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.047115:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:5.0:1619677448.047115:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.047117:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.047117:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.047118:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.047118:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:22.0:1619677448.047120:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (24)++
00000800:00000200:6.0:1619677448.047120:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:22.0:1619677448.047121:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (24)--
00000800:00000200:22.0:1619677448.047121:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.047121:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.047121:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.047122:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.047123:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.047123:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.047125:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677448.047126:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.047127:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:5.0:1619677448.047127:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:5.0:1619677448.047130:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:5.0:1619677448.047130:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fd8000] (21)++
00000800:00000200:21.2:1619677448.047131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fd8000] (21)++
00000800:00000200:5.0:1619677448.047132:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fd8000] (23)++
00000800:00000200:5.0:1619677448.047134:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fd8000] (24)--
00000800:00000200:5.0:1619677448.047135:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (23)--
00000800:00000200:5.0:1619677448.047135:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fd8000] (22)++
00000800:00000200:20.0:1619677448.047137:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (23)--
00000800:00000200:5.0:1619677448.047137:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.13@o2ib3
00000800:00000200:21.2:1619677448.047139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fd8000] (22)++
00000400:00000200:5.0:1619677448.047139:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.13@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.047141:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:5.0:1619677448.047142:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.047143:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:26.0:1619677448.047144:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:5.0:1619677448.047144:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.047145:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.047145:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fd8000] (23)++
00000800:00000200:10.0:1619677448.047145:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:5.0:1619677448.047145:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.047146:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fd8000] (24)--
00000400:00000200:5.0:1619677448.047146:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.047147:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (23)--
00000400:00000200:5.0:1619677448.047147:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.047148:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.047148:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fd8000] (22)++
00000800:00000200:20.0:1619677448.047149:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.13@o2ib3
00000800:00000200:26.0:1619677448.047151:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:5.0:1619677448.047152:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.047154:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.13@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.047155:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.047156:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.047158:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:5.0:1619677448.047158:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.047159:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.047159:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.047160:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:20.0:1619677448.047161:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.047161:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.047161:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:20.0:1619677448.047162:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.047163:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fd8000] (23)++
00000800:00000200:5.0:1619677448.047164:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fd8000] (24)--
00000800:00000200:5.0:1619677448.047165:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (23)--
00000400:00000200:20.0:1619677448.047166:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.047166:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fd8000] (22)++
00000800:00000200:5.0:1619677448.047167:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.13@o2ib3
00000800:00000200:20.0:1619677448.047168:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.047169:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.047171:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.047171:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.047172:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (23)++
00000400:00000200:5.0:1619677448.047172:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.047173:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:20.0:1619677448.047174:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (24)--
00000400:00000200:5.0:1619677448.047174:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.047175:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)++
00000400:00000200:5.0:1619677448.047176:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.047177:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.047178:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (24)--
00000400:00000200:5.0:1619677448.047178:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677448.047179:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.047179:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fd8000] (23)++
00000800:00000200:20.0:1619677448.047179:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fd8000] (24)--
00000800:00000200:20.0:1619677448.047180:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (23)--
00000400:00000200:31.0:1619677448.047181:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.047181:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:20.0:1619677448.047181:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fd8000] (22)++
00000800:00000200:20.0:1619677448.047182:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.13@o2ib3
00000400:00000200:5.0:1619677448.047182:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.047183:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (24)--
00000400:00000200:30.0:1619677448.047184:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.047185:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:5.0:1619677448.047185:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677448.047186:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:30.0:1619677448.047187:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.047188:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.047188:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.13@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.047188:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.047189:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:31.0:1619677448.047189:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:5.0:1619677448.047189:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:5.0:1619677448.047189:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:30.0:1619677448.047190:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:31.0:1619677448.047191:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:20.0:1619677448.047191:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.047191:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:30.0:1619677448.047192:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.0:1619677448.047192:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fd8000] (23)++
00000400:00000200:20.0:1619677448.047193:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677448.047193:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fd8000] (24)--
00000800:00000200:5.0:1619677448.047193:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (23)--
00000400:00000200:20.0:1619677448.047194:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.047194:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (22)--
00000400:00000200:20.0:1619677448.047196:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.047197:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677448.047197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:20.0:1619677448.047198:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.047201:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.047202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fd8000] (21)++
00000800:00000200:24.0:1619677448.047203:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:20.0:1619677448.047203:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677448.047206:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:20.0:1619677448.047206:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.047206:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:5.0:1619677448.047206:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fd8000] (22)++
00000400:00000200:24.0:1619677448.047207:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.047207:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.047207:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.047207:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.13@o2ib3
00000800:00000200:20.0:1619677448.047209:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677448.047210:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.047210:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fd8000] (23)++
00000400:00000200:5.0:1619677448.047210:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.047211:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fd8000] (24)++
00000800:00000200:20.0:1619677448.047212:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fd8000] (25)--
00000800:00000200:24.0:1619677448.047213:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.0:1619677448.047213:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (24)--
00000800:00000200:20.0:1619677448.047214:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fd8000] (23)++
00000400:00000200:5.0:1619677448.047214:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.047216:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.13@o2ib3
00000400:00000200:5.0:1619677448.047216:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.047217:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.047219:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.047219:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.047220:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.047222:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:20.0:1619677448.047223:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.13@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.047224:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:5.0:1619677448.047225:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.047226:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.047226:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.047227:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.047228:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677448.047229:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:13.0:1619677448.047229:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.047230:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.047231:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.047232:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.047234:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.047237:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.047237:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.047237:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.047238:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.047240:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.047240:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.047240:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.047241:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fd8000] (24)++
00000800:00000200:20.0:1619677448.047242:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.047242:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fd8000] (25)--
00000800:00000200:5.0:1619677448.047242:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (24)--
00000800:00000200:20.0:1619677448.047243:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.047243:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fd8000] (23)++
00000800:00000200:20.0:1619677448.047244:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.047244:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fd8000] (24)--
00000800:00000200:5.0:1619677448.047245:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (23)--
00000800:00000200:20.0:1619677448.047246:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677448.047246:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fd8000] (21)++
00000800:00000200:5.0:1619677448.047246:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (22)--
00000800:00000200:20.0:1619677448.047247:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fd8000] (22)--
00000800:00000200:20.0:1619677448.047248:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (21)--
00000800:00000200:21.2:1619677448.047249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fd8000] (20)++
00000800:00000200:12.0:1619677448.047252:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:5.0:1619677448.047253:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fd8000] (21)++
00000800:00000200:12.0:1619677448.047255:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:5.0:1619677448.047255:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.13@o2ib3
00000400:00000200:12.0:1619677448.047256:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.047258:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (22)--
00000400:00000200:5.0:1619677448.047258:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.13@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.047259:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000400:00000200:12.0:1619677448.047259:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.047260:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677448.047261:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.047261:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.047262:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.2:1619677448.047263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000400:00000200:13.0:1619677448.047263:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.047264:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677448.047264:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:21.2:1619677448.047265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fd8000] (21)++
00000400:00000200:5.0:1619677448.047266:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.047267:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.047268:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.047269:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.047270:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:20.0:1619677448.047270:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fd8000] (22)++
00000800:00000200:13.0:1619677448.047271:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677448.047272:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.13@o2ib3
00000800:00000200:31.0:1619677448.047274:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:5.0:1619677448.047274:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.047276:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.13@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.047277:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:5.0:1619677448.047277:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:20.0:1619677448.047278:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.047279:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.047280:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677448.047281:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.047281:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.047281:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:31.0:1619677448.047282:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.047282:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.047282:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:31.0:1619677448.047284:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677448.047284:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.047284:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:31.0:1619677448.047285:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.047285:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:5.0:1619677448.047286:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fd8000] (23)++
00000400:00000200:31.0:1619677448.047287:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.047287:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fd8000] (24)--
00000800:00000200:5.0:1619677448.047287:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (23)--
00000400:00000200:31.0:1619677448.047288:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.047288:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (22)--
00000400:00000200:31.0:1619677448.047289:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677448.047299:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.047301:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:31.0:1619677448.047302:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.047302:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:20.0:1619677448.047302:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.047304:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.047304:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.047305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fd8000] (21)++
00000800:00000200:20.0:1619677448.047305:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.047306:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.047306:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:20.0:1619677448.047306:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (23)++
00000800:00000200:31.0:1619677448.047308:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.047308:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.047309:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.047309:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (23)--
00000800:00000200:20.0:1619677448.047309:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (24)--
00000800:00000200:20.0:1619677448.047309:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fd8000] (22)++
00000800:00000200:5.0:1619677448.047309:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fd8000] (22)++
00000800:00000200:30.0:1619677448.047310:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.047311:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677448.047311:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fd8000] (24)++
00000800:00000200:5.0:1619677448.047311:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.13@o2ib3
00000800:00000200:31.0:1619677448.047312:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677448.047313:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:20.0:1619677448.047313:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fd8000] (25)--
00000400:00000200:5.0:1619677448.047313:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.13@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.047314:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:20.0:1619677448.047314:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (24)--
00000800:00000200:20.0:1619677448.047315:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (23)--
00000400:00000200:5.0:1619677448.047316:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.047318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:5.0:1619677448.047318:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.047319:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.047321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:5.0:1619677448.047321:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.047322:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.047324:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.047326:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:21.2:1619677448.047326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fd8000] (22)++
00000800:00000200:2.0:1619677448.047326:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:30.0:1619677448.047328:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:5.0:1619677448.047328:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.047329:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.047329:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:31.0:1619677448.047330:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.047330:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677448.047330:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.047330:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.047331:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.047332:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677448.047332:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fd8000] (23)++
00000800:00000200:5.0:1619677448.047333:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.047334:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677448.047334:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fd8000] (24)--
00000800:00000200:5.0:1619677448.047334:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:4.2:1619677448.047334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677448.047334:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.047335:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.047335:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.0:1619677448.047335:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (23)--
00000800:00000200:9.2:1619677448.047335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677448.047335:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.047336:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.047337:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fd8000] (22)++
00000800:00000200:5.0:1619677448.047337:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.047338:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.13@o2ib3
00000800:00000200:5.0:1619677448.047338:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fd8000] (23)++
00000800:00000200:2.0:1619677448.047338:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677448.047339:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fd8000] (24)--
00000400:00000200:31.0:1619677448.047340:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.047340:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.13@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.047340:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (23)--
00000800:00000200:5.0:1619677448.047340:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (22)--
00000800:00000200:2.0:1619677448.047340:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000400:00000200:31.0:1619677448.047342:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677448.047342:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.047342:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677448.047343:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.047343:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.047343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.047343:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.047344:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.047344:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000400:00000200:31.0:1619677448.047345:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.047345:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677448.047345:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:10.0:1619677448.047345:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:2.0:1619677448.047345:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:31.0:1619677448.047346:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.047346:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:31.0:1619677448.047347:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.047347:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.047348:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:20.0:1619677448.047349:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.047349:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.047349:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.047350:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.047350:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:10.0:1619677448.047351:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.047352:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:18.0:1619677448.047352:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.047353:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.047354:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:18.0:1619677448.047355:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677448.047356:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.047356:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.047357:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.047359:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.047359:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.047359:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.047360:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.047360:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.047360:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.047361:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.047361:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.047362:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.047362:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:31.0:1619677448.047363:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.047363:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677448.047365:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677448.047365:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677448.047365:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fd8000] (21)++
00000400:00000200:10.0:1619677448.047365:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.047366:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fd8000] (22)--
00000800:00000200:31.0:1619677448.047367:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:21.2:1619677448.047367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:20.0:1619677448.047367:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (21)--
00000800:00000200:10.0:1619677448.047367:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.047368:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677448.047370:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677448.047371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fd8000] (20)++
00000800:00000200:10.0:1619677448.047371:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.047371:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.2:1619677448.047372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.047373:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.047373:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.047374:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:21.2:1619677448.047374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.047374:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:12.0:1619677448.047375:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.047375:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fd8000] (21)++
00000800:00000200:22.0:1619677448.047376:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:10.0:1619677448.047376:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.047376:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677448.047377:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677448.047378:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.047379:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.047379:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.13@o2ib3
00000800:00000200:17.0:1619677448.047380:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:22.0:1619677448.047381:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677448.047381:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.047382:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:13.0:1619677448.047382:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677448.047382:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.13@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.047382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.047383:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677448.047383:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (22)--
00000400:00000200:22.0:1619677448.047384:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.047384:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.047384:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.047384:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677448.047385:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:5.0:1619677448.047385:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.047386:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677448.047386:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.047387:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.047387:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:5.0:1619677448.047387:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:22.0:1619677448.047388:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:5.0:1619677448.047388:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.047389:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.047389:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.047390:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.047390:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.047391:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.047391:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.047392:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.047392:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:5.0:1619677448.047392:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.047393:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.047393:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.047393:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.047393:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.047394:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.2:1619677448.047395:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677448.047396:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.047397:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.047397:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.047398:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677448.047398:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:5.0:1619677448.047398:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:13.0:1619677448.047399:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.047399:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677448.047401:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.2:1619677448.047402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:5.0:1619677448.047402:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.047403:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677448.047403:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.047404:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.047406:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.047407:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:1.0:1619677448.047407:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.047408:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.047409:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:22.0:1619677448.047410:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.047410:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000400:00000200:1.0:1619677448.047410:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.047411:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:5.0:1619677448.047411:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.047411:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:17.0:1619677448.047412:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.047412:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:1.0:1619677448.047412:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:22.0:1619677448.047413:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.047413:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:22.0:1619677448.047414:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.047414:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.047414:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:1.0:1619677448.047414:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.047415:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fd8000] (21)++
00000800:00000200:5.0:1619677448.047415:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fd8000] (22)--
00000800:00000200:22.0:1619677448.047416:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.047416:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (21)--
00000400:00000200:1.0:1619677448.047416:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.047417:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:1.0:1619677448.047417:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.047418:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677448.047418:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:22.0:1619677448.047419:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.047421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:1.0:1619677448.047422:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.047423:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.2:1619677448.047424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:1.0:1619677448.047426:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.047427:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:21.2:1619677448.047428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fd8000] (20)++
00000800:00000200:1.0:1619677448.047428:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.047429:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677448.047430:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:1.0:1619677448.047430:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.047431:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.047431:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677448.047432:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.047433:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.047434:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.047434:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:9.2:1619677448.047435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.0:1619677448.047435:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.047435:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.047437:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.047437:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.047438:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fd8000] (21)++
00000800:00000200:20.0:1619677448.047439:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.13@o2ib3
00000800:00000200:5.0:1619677448.047440:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:20.0:1619677448.047441:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.13@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.047441:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.0:1619677448.047442:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:5.0:1619677448.047442:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:20.0:1619677448.047443:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.047443:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (22)--
00000400:00000200:20.0:1619677448.047444:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:8.0:1619677448.047444:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.047445:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:20.0:1619677448.047445:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.047446:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.047447:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.047448:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.047449:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677448.047449:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.047451:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:20.0:1619677448.047453:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.047454:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677448.047457:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.047458:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677448.047459:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.047461:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677448.047461:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fd8000] (21)++
00000800:00000200:20.0:1619677448.047462:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fd8000] (22)++
00000800:00000200:20.0:1619677448.047464:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fd8000] (23)--
00000800:00000200:20.0:1619677448.047464:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (22)--
00000800:00000200:21.2:1619677448.047469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fd8000] (21)++
00000800:00000200:30.0:1619677448.047473:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:5.0:1619677448.047473:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fd8000] (22)++
00000800:00000200:30.0:1619677448.047476:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:5.0:1619677448.047476:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fd8000] (23)--
00000400:00000200:30.0:1619677448.047477:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.047477:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (22)--
00000800:00000200:5.0:1619677448.047478:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd8000] (21)--
00000400:00000200:30.0:1619677448.047479:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.047480:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.047482:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677448.047492:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.047495:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.047496:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.047497:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.047497:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.047498:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.047570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.047576:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.047578:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.047581:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.047581:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.047584:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.047585:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677448.047586:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.047588:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.047589:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.047590:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677448.047593:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.047595:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677448.047597:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.047598:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677448.047599:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677448.047600:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677448.047601:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.047602:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.047603:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.047607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.047614:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.047617:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.047618:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.047618:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.047620:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.047622:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.047680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.047685:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.047688:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.047688:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.047690:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.047693:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.047694:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.047695:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.047696:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.047697:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.047698:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.047701:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.047703:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.047705:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.047706:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.047706:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.047708:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.047708:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.047709:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.047710:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.047714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.047722:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.047724:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.047725:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.047727:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.047728:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.047730:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.047848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.047856:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.047859:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.047860:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.047862:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.047863:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.047865:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.047866:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.047867:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.047868:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.047869:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.047873:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.047874:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.047877:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.047878:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.047879:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.047881:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.047881:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.047882:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.047882:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.047886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.047888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.047894:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677448.047895:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677448.047897:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:5.0:1619677448.047897:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677448.047898:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:5.0:1619677448.047899:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.047900:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:6.0:1619677448.047900:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.047901:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.047903:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.047903:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:6.0:1619677448.047904:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.047905:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.047907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000400:00000200:6.0:1619677448.047907:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.047908:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.047909:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.047912:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.047914:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.047915:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.047917:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.047917:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.047918:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.047918:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.047919:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.047919:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.047920:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.047920:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.047920:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.047921:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.0:1619677448.047921:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:6.0:1619677448.047922:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:21.2:1619677448.047926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:8.2:1619677448.047926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.047931:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677448.047934:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677448.047934:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.047935:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:17.0:1619677448.047937:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:2.0:1619677448.047937:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.047938:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.047938:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.047940:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.047940:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.047942:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:17.0:1619677448.047943:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:2.0:1619677448.047943:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.047944:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.047945:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.047946:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.047950:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.047951:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.047955:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.047955:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.047956:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.047958:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.047959:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.047959:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.047960:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:26.0:1619677448.047970:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.047973:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.047974:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.047974:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.047976:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.047979:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.047991:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.047994:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.047995:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.047995:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.047996:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.047997:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.048024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.048031:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.048033:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.048035:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.048036:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.048039:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.048041:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.048042:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.048043:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.048044:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.048045:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.048049:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.048050:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.048053:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.048054:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.048055:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.048056:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:4.2:1619677448.048056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677448.048057:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.048058:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.048058:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677448.048063:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.048066:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.048067:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.048068:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.048070:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:24.0:1619677448.048071:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.048072:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.048073:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96720061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.048073:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.048074:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.048074:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:24.0:1619677448.048075:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.048076:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.048077:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.048077:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.048080:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.048080:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677448.048082:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.048085:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.048086:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.048086:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.048088:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.048088:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.048089:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.048090:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.048094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.048101:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.048104:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.048105:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.048105:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.048108:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.048110:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.048111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.048118:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.048121:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.048122:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.048122:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.048123:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.048123:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.048133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677448.048140:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.048143:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677448.048144:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677448.048146:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.048149:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.048151:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.048152:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.048153:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.048154:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.048155:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.048158:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.048160:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.048162:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.048163:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.048164:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.048165:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.048166:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.048166:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.048167:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:30.0:1619677448.048177:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.048180:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.048181:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.048182:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.048184:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.048185:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.048411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.048417:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.048419:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.048420:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.048422:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.048425:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.048426:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.048427:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.048429:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.048429:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.048431:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.048434:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.048436:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.048438:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.048439:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.048440:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.048441:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.048442:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.048442:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.048443:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.048456:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.048459:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.048460:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.048460:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.048462:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.048464:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.048473:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.048476:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.048476:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.048477:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.048478:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.048478:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.048572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.048581:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.048584:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.048586:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.048588:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.048589:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.048590:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677448.048592:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.048593:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.048594:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.048595:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677448.048599:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.048600:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677448.048604:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.048604:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677448.048605:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.048607:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677448.048608:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.048609:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.048609:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.048621:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.048623:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.048624:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.048624:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.048627:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.048628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:6.0:1619677448.048629:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677448.048636:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.048639:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.2:1619677448.048639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.0:1619677448.048639:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677448.048642:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.048644:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.048645:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.048646:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.048647:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:8.0:1619677448.048648:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.048649:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:8.0:1619677448.048649:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.048650:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677448.048652:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.048653:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:8.0:1619677448.048653:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.048654:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.048655:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:30.0:1619677448.048656:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.048657:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.048658:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.048658:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.048659:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.048659:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:30.0:1619677448.048660:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:8.0:1619677448.048660:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.048662:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.048662:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:5.2:1619677448.048662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000400:00000200:30.0:1619677448.048663:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.048663:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.048663:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:30.0:1619677448.048665:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.048667:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.048667:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.048668:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:9.2:1619677448.048668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.048670:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.048670:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.048670:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:30.0:1619677448.048671:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.048672:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.048672:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:10.0:1619677448.048673:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:26.0:1619677448.048675:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.048676:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:26.0:1619677448.048677:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.048678:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:26.0:1619677448.048679:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:20.2:1619677448.048679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:2.0:1619677448.048679:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:26.0:1619677448.048680:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.048680:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.048682:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:26.0:1619677448.048682:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.048682:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:18.0:1619677448.048682:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.048683:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.048684:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.048685:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:20.0:1619677448.048685:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:31.0:1619677448.048686:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.048686:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:31.0:1619677448.048688:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.048688:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.048688:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:26.0:1619677448.048689:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.048690:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:26.0:1619677448.048691:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:20.0:1619677448.048693:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.048694:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.048695:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:20.0:1619677448.048695:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.048696:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:20.0:1619677448.048696:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:26.0:1619677448.048697:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.048698:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000400:00000200:20.0:1619677448.048698:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.048699:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:26.0:1619677448.048699:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000400:00000200:20.0:1619677448.048699:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.048701:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:20.0:1619677448.048703:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.048704:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:8.2:1619677448.048704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:20.0:1619677448.048708:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.048708:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.048709:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:4.2:1619677448.048710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677448.048711:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.048711:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.048714:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.048715:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.048715:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.048716:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:5.0:1619677448.048716:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.048717:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.048718:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.048718:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677448.048718:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.048719:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.048720:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:5.0:1619677448.048720:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.048721:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677448.048721:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.048722:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.048723:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677448.048724:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:10.0:1619677448.048724:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.048725:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.048725:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.048726:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.048727:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.048727:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.048729:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.048729:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:26.0:1619677448.048731:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:10.0:1619677448.048731:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.048731:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.048732:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:5.0:1619677448.048732:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:26.0:1619677448.048733:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.048733:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.048733:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:10.0:1619677448.048734:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.048735:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.048735:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.048735:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.048735:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.048736:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.048737:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.048738:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:10.0:1619677448.048739:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.048739:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677448.048740:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:26.0:1619677448.048741:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.048742:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.048742:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.048745:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.048746:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:30.0:1619677448.048747:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:10.0:1619677448.048747:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.048749:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:10.0:1619677448.048749:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.048749:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677448.048750:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.048750:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.048751:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:30.0:1619677448.048752:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.048753:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.048755:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:9.2:1619677448.048755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.048762:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.048765:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.048766:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.048766:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.048769:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.048772:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.048943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.048948:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.048951:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.048951:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.048954:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.048956:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.048958:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.048959:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.048960:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.048961:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.048962:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.048966:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.048968:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.048971:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.048971:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.048972:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.048974:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.048975:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.048975:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.048976:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.2:1619677448.048983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677448.048987:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677448.048988:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.048990:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.048991:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.048991:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.048992:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677448.048993:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.048994:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:17.0:1619677448.048995:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.048996:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:17.0:1619677448.048997:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.048998:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.049000:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.049001:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.049002:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.049003:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.049005:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:17.0:1619677448.049006:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.049008:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.049008:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677448.049008:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.049009:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.049010:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.049011:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677448.049011:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.049012:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.049012:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.049014:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.049014:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677448.049015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:17.0:1619677448.049016:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.049017:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.049017:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.049022:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.2:1619677448.049022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.049024:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.049025:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.049027:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.049027:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.049027:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:26.0:1619677448.049030:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.049030:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677448.049030:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:26.0:1619677448.049031:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:6.0:1619677448.049031:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.049031:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677448.049032:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.049032:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677448.049033:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.049034:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.049035:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.049036:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.049037:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:6.0:1619677448.049037:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:26.0:1619677448.049041:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.049043:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.049045:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.049046:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.049047:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.049048:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.049049:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.049050:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.049050:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677448.049059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.049068:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.049070:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.049072:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.049072:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.049073:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.049074:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.049074:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.049076:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.049076:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.049077:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.049078:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.049079:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.049080:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.049082:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:31.0:1619677448.049083:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.049084:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.049084:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.049084:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677448.049087:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.049088:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.049088:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.049090:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.049091:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.049091:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.049092:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.049096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.049104:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.049107:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.049108:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.049110:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.049110:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.049113:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.049220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.049227:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.049230:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.049231:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.049232:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.049235:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.049236:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.049238:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.049239:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.049240:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.049241:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.049244:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.049246:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.049248:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.049248:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.049249:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.049250:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.049251:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.049251:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.049252:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.049262:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.049265:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.049266:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.049267:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.049269:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.049269:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.049315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.049321:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.049324:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.049325:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.049327:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.049329:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.049331:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.049332:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.049333:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.049334:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.049335:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.049339:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.049340:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.049342:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.049343:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.049344:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.049345:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.049346:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.049346:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.049347:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.049352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.049359:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.049361:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.049361:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.049362:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.049363:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.049365:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.049431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.049439:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.049442:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.049442:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.049445:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.049447:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.049449:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.049450:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.049451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:24.0:1619677448.049452:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.049452:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.049454:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.049456:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.049458:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.049459:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:24.0:1619677448.049460:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.049461:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:24.0:1619677448.049461:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.049462:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:31.0:1619677448.049463:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.049463:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.049464:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.049465:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.049465:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:31.0:1619677448.049466:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.049466:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.049467:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.049468:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.049470:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.049470:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.049471:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.049474:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.049476:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.049477:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:31.0:1619677448.049478:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.049479:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677448.049479:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:31.0:1619677448.049480:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (24)++
00000400:00000200:12.0:1619677448.049480:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.049481:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677448.049481:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.049482:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.049483:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:13.0:1619677448.049484:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.049484:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:31.0:1619677448.049485:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.049486:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677448.049491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.049493:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.049495:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.049496:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.049497:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.049499:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:12.0:1619677448.049499:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.049500:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.049502:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.049502:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.049503:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:4.2:1619677448.049550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677448.049552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.049559:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.049561:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.049562:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.049563:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:26.0:1619677448.049565:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.049565:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.049567:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.049568:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.049569:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.049570:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.049570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:26.0:1619677448.049572:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.049573:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.049574:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.049576:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.049577:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.049578:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.049579:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.049580:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.049580:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.049581:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.049584:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.049585:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:10.0:1619677448.049585:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.049586:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.049586:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.049588:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.049589:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.049589:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.049590:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.049590:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.049591:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.049591:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.049592:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.049593:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.049593:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.049594:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.049594:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:9.2:1619677448.049594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.049595:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.049596:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:26.0:1619677448.049597:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.049597:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.049598:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.049599:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.2:1619677448.049599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:26.0:1619677448.049600:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.049602:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:18.0:1619677448.049602:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:26.0:1619677448.049603:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.049603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:26.0:1619677448.049604:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.049605:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.049605:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:26.0:1619677448.049606:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677448.049606:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.049606:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677448.049607:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:18.0:1619677448.049609:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.049610:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.049611:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.049611:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:9.2:1619677448.049611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.049611:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:26.0:1619677448.049612:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.049612:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:17.0:1619677448.049613:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.049613:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.049614:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.049615:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.049615:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:18.0:1619677448.049617:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.049617:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:26.0:1619677448.049618:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.049618:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.049619:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677448.049620:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.049620:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.049621:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.049621:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677448.049621:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.049622:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677448.049622:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.049623:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:18.0:1619677448.049623:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677448.049632:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.049635:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.049636:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.049638:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.049639:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.049641:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.049705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.049710:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.049713:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.049714:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.049716:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.049717:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.049719:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.049720:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.049721:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.049722:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.049723:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.049727:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.049728:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.049731:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.049732:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:20.2:1619677448.049733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677448.049733:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.049735:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677448.049736:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.049736:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.049737:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.049737:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677448.049738:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.049740:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.049741:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.049743:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.049745:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.049746:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.049747:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.049748:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.049748:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:20.0:1619677448.049749:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:13.0:1619677448.049751:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:20.0:1619677448.049752:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677448.049752:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.049754:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:13.0:1619677448.049754:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.049754:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677448.049757:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.049757:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677448.049758:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.049759:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.049761:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.049761:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.049763:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.049764:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.049765:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:9.2:1619677448.049767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677448.049768:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.049770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677448.049771:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.049772:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.049772:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.049773:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.049775:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677448.049775:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677448.049776:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:2.0:1619677448.049777:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:5.0:1619677448.049778:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:2.0:1619677448.049778:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.049779:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.049779:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.049780:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:2.0:1619677448.049781:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.049784:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.049862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.049869:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.049872:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.049874:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.049876:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.049876:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.049878:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.049879:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.049881:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.049882:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.049883:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.049886:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.049888:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.049891:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.049892:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.049893:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.049895:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.049895:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.049896:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.049897:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.049901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.049908:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.049911:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.049912:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.049913:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.049914:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.049916:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.049964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.049971:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.049974:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.049975:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.049977:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.049980:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.049981:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.049982:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.049983:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.049984:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.049985:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.2:1619677448.049987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:10.0:1619677448.049988:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.049990:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.049992:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.049992:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.049993:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.049994:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.049995:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:10.0:1619677448.049995:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.049996:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.049996:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.049997:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.049997:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.049998:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.049999:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677448.050000:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.050002:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.050003:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677448.050004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:18.0:1619677448.050005:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.050005:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.050007:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.050009:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:18.0:1619677448.050010:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.050011:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.050012:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:18.0:1619677448.050012:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.050013:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677448.050013:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.050014:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.050014:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.050015:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:18.0:1619677448.050015:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.050016:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.050016:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.050017:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.050019:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677448.050020:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.050020:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.050021:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.050022:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.050023:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.2:1619677448.050028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:13.0:1619677448.050032:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.050033:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:13.0:1619677448.050035:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.050035:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:13.0:1619677448.050036:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.050036:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.050037:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:13.0:1619677448.050038:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.050038:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.050041:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.050094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.2:1619677448.050099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:10.0:1619677448.050102:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.050105:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677448.050105:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:10.0:1619677448.050105:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:18.0:1619677448.050108:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:10.0:1619677448.050108:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.050108:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:21.2:1619677448.050110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:18.0:1619677448.050111:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.050111:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.050112:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.050113:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.050113:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.050115:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:10.0:1619677448.050115:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.050117:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677448.050117:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.050117:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.050119:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.050119:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.050120:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.050121:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:18.0:1619677448.050121:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.050124:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.050124:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677448.050125:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.050126:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.050127:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.050128:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.050129:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:18.0:1619677448.050129:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.050130:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.050132:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.050132:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.050132:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.050133:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.050133:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:10.0:1619677448.050133:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:31.0:1619677448.050134:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.050134:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677448.050134:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.050136:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:10.0:1619677448.050136:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.050137:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:10.0:1619677448.050137:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677448.050138:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.050138:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:10.0:1619677448.050138:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:31.0:1619677448.050139:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.050139:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:10.0:1619677448.050140:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.050140:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.050142:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677448.050142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.050143:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677448.050144:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:31.0:1619677448.050145:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:5.2:1619677448.050145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.050146:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.050147:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.050147:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.050147:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:17.0:1619677448.050149:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:12.0:1619677448.050150:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:17.0:1619677448.050151:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:12.0:1619677448.050151:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.050152:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.050152:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.050153:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:1.0:1619677448.050153:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.050154:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.050154:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677448.050154:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:26.0:1619677448.050155:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:13.0:1619677448.050155:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.050155:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.050156:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:13.0:1619677448.050156:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.050157:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677448.050157:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:13.0:1619677448.050157:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.050160:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.050215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.050218:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.050220:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.050222:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.050224:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.050225:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.050226:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.050227:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.050229:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.050230:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.050231:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.050235:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.050236:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.050239:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.050240:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.050241:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.050243:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.050243:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.050245:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.050245:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.050256:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.050259:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.050259:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.050261:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.050263:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.050264:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677448.050271:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.050275:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.050275:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.050276:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.050276:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.050277:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.050287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.050294:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.050300:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.050301:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.050304:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.050306:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.050308:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.050309:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.050310:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.050311:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.050312:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.050316:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.050317:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.050320:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.050321:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.050322:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.050323:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.050324:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.050325:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.050325:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.050339:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.050342:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.050343:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.050343:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.050346:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.050349:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.050369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.050377:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.050379:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.050380:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.050382:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.050384:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.050386:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.050387:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.050388:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.050389:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.050390:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.050393:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.050395:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.050398:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.050398:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.050399:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.050401:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.050401:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.050403:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.050404:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.050404:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.050406:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677448.050410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.050415:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677448.050417:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.050418:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.050418:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:24.0:1619677448.050419:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677448.050419:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.050420:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.050421:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.050421:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.050424:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.050456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.050461:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.050463:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.050464:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.050466:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.050469:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.050470:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.050471:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.050473:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.050474:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.050475:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.050478:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.050480:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.050483:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.050483:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.050484:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.050486:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.050487:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.050487:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.050488:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.050500:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.050503:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.050503:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.050505:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.050507:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.050508:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.050526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.2:1619677448.050533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:26.0:1619677448.050534:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.050537:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.050538:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:2.0:1619677448.050539:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:26.0:1619677448.050540:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.050542:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:26.0:1619677448.050543:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.050543:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:26.0:1619677448.050545:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.050546:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.050546:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.050548:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.050549:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.050549:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.050550:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.050550:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.050552:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.050553:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.050554:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.050555:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.050559:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.050560:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.050562:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.050564:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.050565:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.050566:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.050567:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.050567:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.050567:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.050568:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.050568:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.050569:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:26.0:1619677448.050570:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.050570:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.050570:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677448.050571:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.050572:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.050572:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.050576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677448.050582:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677448.050584:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:22.0:1619677448.050585:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.050586:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.050586:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.050587:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:22.0:1619677448.050588:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.050588:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.050588:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677448.050590:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:18.0:1619677448.050590:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.050593:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677448.050599:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.050601:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.050602:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.050603:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.050604:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.050604:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.050623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.050630:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.050633:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.050634:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.050636:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.050639:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.050640:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.050641:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.050643:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.050644:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.050645:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.050648:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.050650:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.050651:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.050652:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.050653:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.050654:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.050655:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.050656:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.050656:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.050666:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.050668:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.050668:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.050669:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.050670:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.050672:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.050738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.050745:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.050748:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.050748:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677448.050750:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.050753:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.050755:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.050756:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.050757:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.050758:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.050759:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:20.2:1619677448.050762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:8.0:1619677448.050762:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.050763:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.050766:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.050767:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.050767:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.050769:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.050769:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677448.050770:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.050770:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.050771:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677448.050772:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.2:1619677448.050773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677448.050774:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677448.050775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:5.0:1619677448.050775:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.050779:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.050780:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:31.0:1619677448.050781:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:5.0:1619677448.050781:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.050783:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:5.0:1619677448.050783:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.050784:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677448.050784:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.050784:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.050785:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.050785:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677448.050787:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.050787:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.050788:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.050789:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.050789:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.050790:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:31.0:1619677448.050791:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:20.0:1619677448.050791:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677448.050791:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:31.0:1619677448.050792:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.050793:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.050794:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:31.0:1619677448.050795:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.050795:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.2:1619677448.050797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.050798:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.050799:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.050799:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:31.0:1619677448.050800:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.050800:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.050800:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.050801:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:31.0:1619677448.050802:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.050802:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677448.050803:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.050803:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.050803:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:20.0:1619677448.050804:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.050804:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.050804:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677448.050805:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.050805:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677448.050806:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:20.0:1619677448.050806:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.050806:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.050807:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.050808:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:20.0:1619677448.050808:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:9.2:1619677448.050808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.050809:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677448.050809:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.050810:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.050810:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:20.0:1619677448.050810:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.050811:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.050811:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.050812:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.050813:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.050814:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.050815:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:2.0:1619677448.050815:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:1.0:1619677448.050815:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.050817:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:21.2:1619677448.050818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:2.0:1619677448.050818:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.050819:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.050819:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.050820:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.050822:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.050822:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.050823:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:20.0:1619677448.050824:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677448.050824:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.050825:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677448.050825:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677448.050825:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:22.0:1619677448.050826:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.050826:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:20.0:1619677448.050827:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.050827:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.050828:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:1.0:1619677448.050828:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:22.0:1619677448.050829:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.050829:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:6.0:1619677448.050829:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.050829:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.050831:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677448.050831:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:1.0:1619677448.050831:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.050832:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.050833:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:20.0:1619677448.050833:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000400:00000200:6.0:1619677448.050833:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.050834:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.050834:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:20.0:1619677448.050835:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000400:00000200:6.0:1619677448.050835:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:1.0:1619677448.050835:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677448.050836:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677448.050836:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:6.0:1619677448.050836:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96720020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.050836:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677448.050837:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.2:1619677448.050837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677448.050838:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.050838:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.050839:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.050839:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.050839:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.050840:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:6.0:1619677448.050840:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:5.2:1619677448.050840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.2:1619677448.050841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677448.050843:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:6.0:1619677448.050845:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.050846:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:17.0:1619677448.050846:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:6.0:1619677448.050846:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:30.0:1619677448.050847:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.050847:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.050848:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.050848:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:1.0:1619677448.050848:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:30.0:1619677448.050849:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.050849:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:31.0:1619677448.050850:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:6.0:1619677448.050850:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.050850:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:1.0:1619677448.050850:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.050851:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.050851:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:5.0:1619677448.050851:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:6.0:1619677448.050852:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:5.0:1619677448.050852:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:1.0:1619677448.050852:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677448.050853:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:1.0:1619677448.050853:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677448.050854:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:6.0:1619677448.050854:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.050855:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.050856:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.050857:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.050858:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:21.2:1619677448.050863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.050866:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:22.0:1619677448.050868:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:10.0:1619677448.050869:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.050870:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.050870:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:10.0:1619677448.050870:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.050871:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.050872:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677448.050872:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.050876:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:4.2:1619677448.051008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.051016:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.051018:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.051019:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.051021:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.051022:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.051024:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.051025:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.051026:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.051027:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.051028:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.2:1619677448.051028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:26.0:1619677448.051031:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.051033:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.051034:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:26.0:1619677448.051036:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.051037:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.051037:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.051037:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.051038:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:26.0:1619677448.051039:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.051039:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.051040:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.051040:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:18.0:1619677448.051040:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.051042:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.051044:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.051045:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.051046:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.051047:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.051048:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.051052:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.051053:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.051053:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.051055:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.051056:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:18.0:1619677448.051056:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.051057:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.051057:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:6.0:1619677448.051057:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.051059:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:6.0:1619677448.051059:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.051060:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.051060:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.051061:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:6.0:1619677448.051062:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677448.051072:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.051075:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.051076:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.051078:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.051080:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.051081:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.051123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:4.2:1619677448.051124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677448.051131:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677448.051131:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677448.051134:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:24.0:1619677448.051134:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.051135:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:31.0:1619677448.051137:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.051137:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:24.0:1619677448.051138:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.051140:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.051141:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.051142:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:31.0:1619677448.051143:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.051143:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.051144:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.051145:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.051146:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.051146:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.051147:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.051148:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.051148:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677448.051151:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.051153:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.051155:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.051156:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.051157:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:31.0:1619677448.051157:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.051157:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.051159:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677448.051160:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677448.051160:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.051161:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:24.0:1619677448.051161:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.051161:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (22)++
00000800:00000200:31.0:1619677448.051162:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677448.051163:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:24.0:1619677448.051163:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (23)--
00000800:00000200:24.0:1619677448.051164:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.051165:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.051165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677448.051166:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.2:1619677448.051171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:1.0:1619677448.051173:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (23)++
00000800:00000200:17.0:1619677448.051176:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (24)++
00000800:00000200:1.0:1619677448.051176:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (24)--
00000800:00000200:17.0:1619677448.051177:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (24)--
00000400:00000200:1.0:1619677448.051177:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.051178:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:17.0:1619677448.051178:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.051179:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.051180:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.051181:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677448.051182:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677448.051183:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:1.0:1619677448.051183:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.051184:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.051185:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.051188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.051196:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.051198:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.051199:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.051199:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.051200:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.051201:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.051247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.2:1619677448.051251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.051255:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677448.051257:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.051258:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.051259:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677448.051259:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.051260:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.051262:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.051263:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.051264:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.051265:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.051266:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.051267:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.051268:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.051269:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.051270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:1.0:1619677448.051270:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.051271:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.051271:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.051272:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.051272:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.051274:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.051277:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677448.051277:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.051279:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.051280:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:1.0:1619677448.051280:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.051282:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.051282:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.051283:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:31.0:1619677448.051284:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.051284:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.051284:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.051285:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.051285:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.051285:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.051286:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.051286:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.051286:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677448.051287:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.051287:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677448.051288:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677448.051288:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.051289:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:30.0:1619677448.051290:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96720013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.051290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:10.0:1619677448.051290:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.051290:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:30.0:1619677448.051291:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677448.051291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:30.0:1619677448.051292:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.051300:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:2.0:1619677448.051301:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:30.0:1619677448.051302:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.051303:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.051304:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.051304:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.051305:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677448.051305:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:13.0:1619677448.051306:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:17.0:1619677448.051307:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.051308:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.051308:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:17.0:1619677448.051309:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:13.0:1619677448.051309:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.051309:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.051311:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.051311:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.051312:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.051313:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.051313:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677448.051313:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:30.0:1619677448.051314:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.051314:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677448.051314:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.051315:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.051316:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677448.051316:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.051317:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:2.0:1619677448.051317:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.051318:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:2.0:1619677448.051318:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.051320:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.051320:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:2.0:1619677448.051321:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.051323:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677448.051326:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.051326:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:21.2:1619677448.051327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.051327:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:12.0:1619677448.051328:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.051329:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677448.051329:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.051330:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.051330:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677448.051331:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.051331:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.051333:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.051334:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.051335:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.051336:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.051338:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.051339:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.051340:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.051343:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.051346:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.051346:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.051347:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.051349:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.051351:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.051367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.051375:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.051377:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.051378:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.051380:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.051382:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.051384:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.051385:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.051386:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.051387:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.051389:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.051392:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.051393:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.051396:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.051397:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.051398:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.051400:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.051400:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.051401:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.051402:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.051403:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.051408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.051413:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.051415:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.051416:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.051416:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:6.0:1619677448.051417:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.051418:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.051418:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.051419:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.051419:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.051422:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.051645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677448.051651:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.051655:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677448.051655:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677448.051657:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.051660:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.051661:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.051663:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.051664:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.051665:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.051666:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.051668:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.051670:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.2:1619677448.051671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:26.0:1619677448.051672:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.051673:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.051674:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.051675:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.051676:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.051677:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677448.051677:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:31.0:1619677448.051679:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.051681:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:8.2:1619677448.051682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:31.0:1619677448.051683:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.051685:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.051686:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.051687:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.051688:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.051689:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.051690:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.051690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:31.0:1619677448.051691:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.051691:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:31.0:1619677448.051694:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.051695:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677448.051696:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:5.0:1619677448.051696:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.051697:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677448.051698:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.051698:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.051699:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.051699:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.051700:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (22)++
00000800:00000200:8.0:1619677448.051700:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:5.0:1619677448.051701:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:24.0:1619677448.051702:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:31.0:1619677448.051703:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.051703:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:8.0:1619677448.051703:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.051704:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.051705:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:8.0:1619677448.051706:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.051709:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:8.0:1619677448.051713:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.051714:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96720008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.051715:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.051716:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.051717:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:8.0:1619677448.051717:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.051719:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.051720:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.051720:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.051722:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.051722:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.051722:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.051725:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.0:1619677448.051725:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.051726:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.051726:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.2:1619677448.051728:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:8.0:1619677448.051732:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.051733:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.051734:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.051735:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.051736:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677448.051736:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.2:1619677448.051738:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.051739:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.051741:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:21.2:1619677448.051742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677448.051742:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.051743:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.051745:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.051747:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:24.0:1619677448.051749:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.051751:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:5.0:1619677448.051751:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:24.0:1619677448.051752:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.051753:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677448.051753:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.051754:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.051755:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:20.0:1619677448.051756:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:5.0:1619677448.051756:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.051758:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000800:00000200:20.0:1619677448.051758:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.051758:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.051759:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.051760:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.051760:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000400:00000200:30.0:1619677448.051761:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.051761:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677448.051762:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:30.0:1619677448.051764:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.051765:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:30.0:1619677448.051767:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.051768:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.051768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:30.0:1619677448.051769:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.051770:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677448.051773:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.051775:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.051775:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.051777:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677448.051778:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.051779:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.051779:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:30.0:1619677448.051779:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677448.051781:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.051781:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677448.051781:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677448.051782:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677448.051782:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000400:00000200:31.0:1619677448.051783:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.051784:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.051784:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.051785:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.051786:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.051787:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:8.2:1619677448.051787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677448.051790:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.051791:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.2:1619677448.051791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677448.051795:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.051795:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.051796:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:1.0:1619677448.051796:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.051797:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.051798:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:1.0:1619677448.051798:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:31.0:1619677448.051799:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.051799:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677448.051799:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.051799:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.051800:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:1.0:1619677448.051801:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.051802:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.051804:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.051804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:1.0:1619677448.051805:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.051806:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677448.051806:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:17.0:1619677448.051807:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96720005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.051807:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.051809:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.051810:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.051811:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.051812:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:17.0:1619677448.051814:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.051815:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.051816:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.051816:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.051816:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.051818:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.051819:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.051820:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.051821:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:17.0:1619677448.051821:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.051822:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.051823:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.051824:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.051824:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.051836:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.051838:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.051839:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.051841:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.051842:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.051844:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.051856:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.051859:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.051860:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.051861:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.051861:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.051862:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.051895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.051900:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.051903:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.051904:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.051906:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.051909:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.051910:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.051911:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.051912:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.051914:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.051914:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.051918:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.051920:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:4.2:1619677448.051921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677448.051921:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.051922:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.051923:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.051924:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.051924:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.051926:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.051927:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.051927:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:10.0:1619677448.051928:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:9.2:1619677448.051930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.051931:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.051931:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:5.2:1619677448.051933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000400:00000200:10.0:1619677448.051934:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.051936:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.051938:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:10.0:1619677448.051938:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.051939:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96720003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.051939:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.051940:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.051940:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.051940:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.051940:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:10.0:1619677448.051941:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.051941:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:10.0:1619677448.051942:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.051942:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.051943:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.051943:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677448.051945:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:10.0:1619677448.051946:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.051948:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:9.2:1619677448.051949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.051951:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.051952:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.051953:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.051954:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.051955:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.051955:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.051956:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.051956:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:18.0:1619677448.051959:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.051959:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:8.2:1619677448.051960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.051960:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.051960:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.051961:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677448.051968:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.051971:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.051972:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.051974:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.051974:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.051977:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.052058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.052064:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.052067:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.052067:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.052069:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.052072:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.052074:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.052075:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96720002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.052076:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.052077:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.052078:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.052082:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.052083:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.052086:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.052086:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.052087:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.052089:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.052090:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.052091:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.052091:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.052095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.052103:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.052105:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.052106:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.052107:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.052109:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677448.052109:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:21.2:1619677448.052114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.052122:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.052125:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.052127:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.052129:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.052130:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.052131:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.052132:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96720001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.052134:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.052135:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.052136:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.052140:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.052141:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.052145:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.052145:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.052146:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.052148:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.052148:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.052149:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.052150:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.052154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.052161:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.052164:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.052165:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.052165:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.052167:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.052170:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.052200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.052208:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.052211:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.052212:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.052213:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.052216:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.052217:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.052219:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96720000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.052220:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.052221:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.052222:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.052225:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.052227:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.052230:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.052230:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.052231:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.052233:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.052233:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.052234:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.052235:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.052239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.052247:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.052250:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.052251:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.052251:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.052253:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.052256:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.052257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.052265:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.052268:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.052269:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.052269:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.052270:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.052271:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.052470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.052478:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.052481:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.052483:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.052485:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.052486:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.052487:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.052488:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.052490:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.052491:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.052492:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677448.052495:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.052496:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.052499:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.052500:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.052501:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.052502:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.052503:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.052504:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.052504:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.052509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.052517:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.052520:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.052522:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.052522:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.052524:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.052527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.052531:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.052532:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.052534:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.052535:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.052535:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.052536:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.052536:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.052571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.052577:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.052580:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.052580:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.052582:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.052584:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.052585:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.052586:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719998] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.052588:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.052589:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.052589:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.052593:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.052595:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.052597:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.052598:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.052599:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.052601:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.052601:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.052602:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.052602:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.052607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.052616:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.052619:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.052619:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.052621:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.052623:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.052624:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.052642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.052648:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.052651:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.052651:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.052654:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.052656:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.052657:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.052659:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.052660:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.052661:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.052662:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.052666:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.052667:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.052670:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.052670:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.052671:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.052673:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.052673:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.052674:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.052674:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.052679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.052686:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.052689:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.052690:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.052690:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.052692:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.052695:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.052731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.052739:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.052742:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.052742:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.052745:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.052747:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.052749:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.052750:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.052752:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.052752:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.052753:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.052756:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.052758:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.052761:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.052761:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.052762:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.052764:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.052765:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.052765:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.052766:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.052773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.052781:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.052783:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.052784:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.052786:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.052787:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.052789:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.052837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.052845:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.052848:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.052850:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.052852:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.052853:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.052855:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677448.052856:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.052857:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.052858:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.052859:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677448.052862:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.052864:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.052867:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.052868:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677448.052868:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.052870:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677448.052870:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.052872:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.052874:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.052874:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677448.052880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677448.052884:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.052886:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:31.0:1619677448.052887:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:6.0:1619677448.052887:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.052888:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.052890:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:22.0:1619677448.052890:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.052891:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.052892:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.052893:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.053005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.053012:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.053015:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.053016:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.053017:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.053020:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.053022:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.053023:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.053024:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.053025:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.053026:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.053029:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.053030:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.2:1619677448.053031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.053033:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.053034:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.053035:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.053036:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.053037:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.053038:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.053038:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677448.053038:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.053040:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.053041:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.053043:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.053046:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.053047:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.053048:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.053050:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:1.0:1619677448.053050:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.053051:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.053052:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.053053:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.053053:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.053054:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.053055:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.053057:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.053057:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.053060:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677448.053060:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.053061:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.053061:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.053063:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.053063:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.053064:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.053065:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.053077:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.053079:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.053080:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.053080:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.053082:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.053084:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677448.053094:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.053097:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.053098:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.053098:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.053099:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.053100:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.053174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.053180:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.053182:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.053185:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.053185:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.053187:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.053189:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.053190:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.053192:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.053193:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.053194:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.2:1619677448.053196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:5.0:1619677448.053197:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.053199:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.053203:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.053203:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.053204:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.053204:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.053206:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:5.0:1619677448.053206:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.053207:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.053208:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.053208:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:31.0:1619677448.053209:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.053210:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.053212:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.053213:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.053214:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.053216:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.053217:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.053218:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.053220:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:31.0:1619677448.053221:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.053223:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.053223:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.053224:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.053225:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.053226:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.053226:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.053227:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.053227:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.053228:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.053229:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.053229:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.053230:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.053231:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677448.053241:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.053243:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.053244:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.053245:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.053247:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.053248:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.053308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.053315:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.053318:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.053318:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.053321:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.053323:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.053324:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.053326:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.053327:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.053328:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.053329:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.053332:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.053334:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.053336:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.053337:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.053338:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.053340:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.053340:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.053341:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.053341:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.053346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.053355:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.053357:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.053358:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.053358:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.053361:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.053363:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.053433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.053438:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.053441:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.053441:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.053443:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.053445:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.053447:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.053448:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.053450:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.053451:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.053452:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.053455:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.053456:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.053459:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.053459:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.053460:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.053462:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.053462:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.053463:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.053464:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.053465:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.053470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.053475:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677448.053476:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.053477:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.053477:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.053478:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.053479:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.053480:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.053480:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.053482:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.053484:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.053579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.053587:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.053590:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.053590:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.053592:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.053596:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.053597:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.053599:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.053600:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.053601:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.053602:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.053605:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.053607:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.053609:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.053610:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.053611:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.053612:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.053613:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.053613:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.053614:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.053625:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.053627:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.053628:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.053630:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.053632:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.053632:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.053643:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.053646:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.053647:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.053647:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.053648:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.053649:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.053768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.053774:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.053777:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.053777:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.053779:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.053782:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.053783:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.053784:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.053786:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.053787:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.053788:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.053791:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.053793:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.053795:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.053796:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.053797:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.053799:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.053799:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.053800:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.053800:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.053813:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.053815:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.053816:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.053818:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.053820:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.053821:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.053864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.053870:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.053873:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.053874:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.053876:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.053878:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.053880:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.053881:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.053882:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.053883:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.053884:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.053888:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.053890:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.053893:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.053894:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.053894:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.053896:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.053897:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.053897:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.053898:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:10.0:1619677448.053908:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.053911:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.053912:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.053912:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.053914:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.053918:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.053957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.053965:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.053968:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.053970:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.053971:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.053972:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.053973:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.053974:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.053976:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.053977:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.053978:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.053981:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.053983:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.053985:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.053986:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.053987:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.053988:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.053989:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.053990:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.053992:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.053992:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677448.053998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.054000:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.054003:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.054003:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.054005:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.054006:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:13.0:1619677448.054007:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.054008:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.054009:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.054009:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.054010:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.2:1619677448.054020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.054022:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.054025:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.054027:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.054028:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.054030:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.054032:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.054033:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.054034:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.054035:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.054036:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.054040:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.054042:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.054045:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.054046:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.054047:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.054048:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.054049:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.054051:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.054052:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.054052:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.054057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.054061:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.054063:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677448.054064:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.054065:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.054065:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:31.0:1619677448.054066:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.054066:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.054067:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.054068:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.054069:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.054156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.054163:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.054166:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.054167:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.054168:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.054171:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.054172:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.054173:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.054175:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.054175:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.054177:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.054180:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.054182:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.054184:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.054185:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.054185:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.054187:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.054187:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.054189:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.054190:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.054190:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.054190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.054195:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677448.054196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.054198:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:30.0:1619677448.054199:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:1.0:1619677448.054199:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.054201:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:17.0:1619677448.054201:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.054202:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.054203:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.054203:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:30.0:1619677448.054204:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.054205:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:17.0:1619677448.054205:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:10.0:1619677448.054205:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:30.0:1619677448.054206:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:10.0:1619677448.054206:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.054207:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:17.0:1619677448.054207:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.054208:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.054209:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.054210:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.054214:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.054215:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.054218:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.054219:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.054220:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.054221:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.054222:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.054223:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.054223:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.054234:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.054237:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.054238:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.054239:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.054240:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.054242:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.054254:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.054257:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.054258:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.054258:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.054259:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.054260:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.054324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.054329:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.054332:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.054333:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.054335:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.054338:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.054339:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.054340:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.054342:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.054343:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.054344:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.054347:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.054349:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.054352:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.054353:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.054354:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.054356:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.054356:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.054358:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.054359:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.054359:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.2:1619677448.054360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:21.2:1619677448.054365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677448.054368:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.054369:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:6.0:1619677448.054370:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:31.0:1619677448.054371:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:10.0:1619677448.054372:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:6.0:1619677448.054372:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:31.0:1619677448.054373:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.054373:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.054373:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.054373:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.054374:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677448.054374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:31.0:1619677448.054375:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.054375:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:10.0:1619677448.054376:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.054377:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.054378:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.054380:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.054380:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:31.0:1619677448.054381:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.054382:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.054382:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.054383:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.054384:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.054385:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000400:00000200:31.0:1619677448.054386:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.054386:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.054387:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.054387:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.054387:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.054388:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.054388:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.054389:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:10.0:1619677448.054389:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.054391:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.054391:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:31.0:1619677448.054392:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.054392:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:10.0:1619677448.054392:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.054394:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.054394:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677448.054394:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.054395:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:10.0:1619677448.054395:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.054396:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:10.0:1619677448.054396:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.054399:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.054401:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.054403:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.054404:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.054404:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.054406:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:13.0:1619677448.054407:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:10.0:1619677448.054407:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.054407:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.054408:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:13.0:1619677448.054410:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.054410:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.054412:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.054414:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.054415:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.054418:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.054420:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.054421:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.054423:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.054425:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.054425:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.054435:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.054438:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.054439:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.054439:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.054440:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.054441:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.054475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.054482:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.054484:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.054487:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.054488:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.054489:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.054490:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.054491:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719978] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.054492:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.054493:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.054494:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.054497:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.054499:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.054501:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.054502:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.054502:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.054503:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.054504:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.054505:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.054505:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.054515:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.054518:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.054518:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.054520:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.054522:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.054523:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.054606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.054613:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.054616:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.054617:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.054619:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.054621:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.054623:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.054624:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.054625:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.054626:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.054627:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.054631:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.054632:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.054636:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.054636:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.054637:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.054639:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.054639:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.054640:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.054640:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.054651:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.054654:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.054655:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.054657:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.054658:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.054659:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.054692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677448.054696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677448.054700:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.054701:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.054702:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:5.0:1619677448.054704:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:31.0:1619677448.054706:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.054706:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677448.054707:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.054707:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.054709:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.054711:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677448.054711:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.054713:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.054713:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.054714:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.054715:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.054717:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.054717:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.054718:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677448.054718:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.054719:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.054725:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.054725:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.054728:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677448.054729:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.054731:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.054732:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677448.054732:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.054733:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677448.054733:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.054734:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.054734:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.054735:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.054735:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677448.054735:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.054736:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.054736:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.054737:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.054738:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.2:1619677448.054741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.054744:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677448.054746:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:20.0:1619677448.054746:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:30.0:1619677448.054748:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.054749:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.054749:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.054749:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:12.0:1619677448.054750:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:30.0:1619677448.054751:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.054751:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.054752:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:12.0:1619677448.054752:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:30.0:1619677448.054753:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:20.0:1619677448.054753:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677448.054754:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.054754:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.054756:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.054756:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.054756:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.054758:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.054759:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:12.0:1619677448.054761:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:20.0:1619677448.054762:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.054764:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.054767:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.054767:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.054768:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.054770:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.054770:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.054771:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.054772:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677448.054777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.054784:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.054787:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.054788:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.054788:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.054790:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.054793:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.054795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.2:1619677448.054796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:26.0:1619677448.054802:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677448.054803:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.054805:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:17.0:1619677448.054806:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:10.0:1619677448.054806:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677448.054806:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.054807:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000400:00000200:26.0:1619677448.054808:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.054808:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.054809:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:26.0:1619677448.054810:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.054812:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.054813:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.054815:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.054815:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.054817:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.054820:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.054822:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:4.2:1619677448.054823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.054825:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.054826:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.054827:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.054829:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.054830:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.054831:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.054831:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.054832:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677448.054833:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.054833:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.054835:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.054837:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.054838:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.054839:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.054841:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.054841:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.054842:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.054842:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:10.0:1619677448.054845:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.054845:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.054846:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.054846:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.054847:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:6.0:1619677448.054848:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.054850:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.054850:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.054851:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.054852:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.054852:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:10.0:1619677448.054853:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.054854:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.054855:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.054856:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.054861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.054865:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.054868:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.054868:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.054869:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.054869:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.054870:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.054871:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.054871:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.054872:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.054874:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.054982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.054989:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.054992:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.054993:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.054995:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.054998:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.054999:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.055001:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.055002:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.055003:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.055004:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.055007:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.055009:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.055012:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.055013:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.055013:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.055015:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.055016:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.055017:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.055017:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.055022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.055029:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.055032:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.055034:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.055034:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.055036:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.055039:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.055144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.055151:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.055154:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.055156:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.055158:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.055158:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.055160:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.055161:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.055162:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.055163:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.055164:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.055167:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.055169:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.055171:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.055172:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.055172:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.055174:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.055175:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.055175:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.055176:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.055186:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.055189:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.055189:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.055189:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.055191:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.055194:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.055204:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.055206:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.055207:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.055207:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.055208:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.055208:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.055217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.055225:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.055227:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.055229:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.055231:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.055231:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.055233:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.055234:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.055235:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.055236:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.055237:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:4.2:1619677448.055239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:30.0:1619677448.055240:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.055242:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.055245:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.055245:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.055246:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.055246:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.055248:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.055248:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.055249:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.055249:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:30.0:1619677448.055250:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.055250:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.055251:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.055254:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.055254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:10.0:1619677448.055255:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.055256:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.055258:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.055259:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.055259:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.055262:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.055262:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:10.0:1619677448.055264:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.2:1619677448.055265:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.055266:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.055267:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:10.0:1619677448.055267:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.055267:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.055269:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.055269:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.055270:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677448.055270:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.055271:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.055272:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:5.0:1619677448.055272:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.055274:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677448.055275:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:26.0:1619677448.055276:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.055276:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.055278:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.055280:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.055282:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:10.0:1619677448.055282:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.055283:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.055284:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:10.0:1619677448.055284:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.055285:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.055285:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.055286:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.055287:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.055288:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:10.0:1619677448.055288:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.055289:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677448.055290:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.055292:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.055293:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:20.2:1619677448.055297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.055300:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.055300:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.055301:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.055301:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677448.055302:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:10.0:1619677448.055302:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.055303:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:20.0:1619677448.055304:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.055306:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.055307:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.055308:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.055310:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.055311:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.055312:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.055313:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.055315:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:20.0:1619677448.055316:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.055317:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:20.0:1619677448.055318:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:13.0:1619677448.055318:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.055320:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.055321:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.055322:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.055322:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.055322:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.055322:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677448.055324:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.055325:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.055326:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.055326:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.055336:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.055339:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.055340:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.055342:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.055343:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.055345:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677448.055356:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.055359:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.055360:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.055360:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.055361:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.055362:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.055362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.055368:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.055370:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.055371:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.055373:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.055376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:17.0:1619677448.055376:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.055377:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.055379:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.055380:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.055381:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.055382:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.055383:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:17.0:1619677448.055385:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.055386:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:17.0:1619677448.055387:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.055387:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.055389:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.055389:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.055390:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.055391:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677448.055392:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.055392:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.055393:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.055394:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:17.0:1619677448.055394:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:26.0:1619677448.055395:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.055395:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.055395:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.055396:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.055397:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.055398:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.2:1619677448.055399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677448.055402:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.055402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.055403:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.055406:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.055407:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.055407:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.055407:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.055408:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.055408:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:26.0:1619677448.055409:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.055409:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:17.0:1619677448.055409:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.055409:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.055409:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.055410:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.055411:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:17.0:1619677448.055411:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.055411:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.055412:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.055413:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:12.0:1619677448.055421:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.055424:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.055425:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.055427:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.055428:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.055429:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.055471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.055478:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.055481:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.055482:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.055483:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.055486:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.055488:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.055489:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.055490:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.055491:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.055492:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.055495:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.055497:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.055499:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.055500:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.055501:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.055503:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.055503:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.055504:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.055504:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.055509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.055517:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.055520:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.055521:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.055521:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.055523:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.055526:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.055539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.055545:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.055548:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.055549:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.055550:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.055553:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.055554:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677448.055555:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.055557:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.055558:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.055559:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677448.055562:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.055563:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677448.055566:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.055567:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677448.055568:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677448.055570:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677448.055570:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.055570:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.055571:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677448.055582:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.055585:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.055585:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.055586:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.055588:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.055591:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:4.2:1619677448.055633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.055640:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.055643:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:4.2:1619677448.055643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677448.055644:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.055646:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.055648:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.055650:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:24.0:1619677448.055650:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.055651:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.055652:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.055653:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.055653:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:24.0:1619677448.055654:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.055656:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.055656:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.055659:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.055659:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.055660:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.055661:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.055661:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.055662:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.055663:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.055664:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677448.055664:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.055665:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.055666:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:26.0:1619677448.055667:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.055667:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.055668:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:24.0:1619677448.055668:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.055668:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.055669:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:26.0:1619677448.055670:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.055671:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.055672:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.055673:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:8.2:1619677448.055673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.055674:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.055675:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.055675:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677448.055681:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.055684:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.055685:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.055685:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677448.055687:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:17.0:1619677448.055688:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.055689:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.055690:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.055690:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.055690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:30.0:1619677448.055692:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.055694:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.055695:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677448.055696:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.055699:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.055700:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.055702:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.055703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:18.0:1619677448.055705:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.055706:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.055708:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.055709:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.055709:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:18.0:1619677448.055710:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.055711:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.055712:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.055713:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:18.0:1619677448.055715:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.055716:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.055718:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.055719:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.055720:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.055721:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:6.0:1619677448.055721:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:18.0:1619677448.055722:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:6.0:1619677448.055722:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.055723:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.055724:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:6.0:1619677448.055724:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.055724:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.055725:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.055725:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:6.0:1619677448.055726:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.055729:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.055730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.055731:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.055734:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.055735:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.055735:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.055737:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:1.0:1619677448.055737:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677448.055738:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.055739:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.055740:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677448.055740:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.055741:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.055741:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.055743:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.055745:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677448.055749:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.055752:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.055753:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.055753:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.055755:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.055758:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677448.055790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.055797:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.055800:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.055802:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.055804:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.055805:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.055806:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.055807:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.055809:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.055810:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.055811:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.055814:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.055816:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.055818:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.055819:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.055819:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.055821:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.055822:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.055822:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.055823:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.055833:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.055836:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.055836:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.055838:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.055839:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.055841:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.055855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.055863:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.055865:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.055866:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.055867:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.055870:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.055871:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:20.2:1619677448.055871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:26.0:1619677448.055872:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.055873:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.055874:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.055875:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:5.0:1619677448.055877:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677448.055878:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.055879:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:26.0:1619677448.055880:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.055882:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.055882:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.055882:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.055883:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.055884:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.055885:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.055885:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:5.0:1619677448.055885:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.055886:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.055887:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:5.0:1619677448.055887:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.055888:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.055889:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.055890:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.055891:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.055895:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.055897:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677448.055897:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.055899:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.055900:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.055900:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.055900:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.055901:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:12.0:1619677448.055902:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.055903:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677448.055903:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.055904:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677448.055904:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.055905:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.055905:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.055916:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.055918:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.055918:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:13.0:1619677448.055919:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.055920:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.055920:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:13.0:1619677448.055921:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.055921:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.055922:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.055923:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.055923:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.055924:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.055978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.055985:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.055987:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.055988:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.055991:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.055993:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.055994:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.055996:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.055997:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.055998:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.055999:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.056002:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.056004:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.056007:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.056008:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.056009:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.056011:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.056011:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.056012:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.056012:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.056017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.056024:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.056027:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.056028:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.056028:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.056030:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.056032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677448.056033:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.056039:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.056042:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.056043:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.056045:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.056047:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.056049:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.056050:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.056051:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.056053:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.056053:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.056057:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.056059:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.056061:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.056062:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.056062:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.056064:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.056064:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.056065:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.056065:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677448.056074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.056076:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.056078:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.056079:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.056079:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677448.056081:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.056082:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.056082:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677448.056083:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:1.0:1619677448.056083:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.056085:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.056087:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.056089:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.056090:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.056091:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.056092:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.056093:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.056097:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.056098:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.056101:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.056102:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.056103:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.056105:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.056105:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.056106:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.056107:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.056117:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.056119:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.056120:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.056122:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.056123:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.056125:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.056341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.056348:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.056351:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.056352:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.056354:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.056357:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.056358:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.056360:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.056361:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.056362:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.056363:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.056366:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.056368:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.056371:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.056371:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.056372:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.056374:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.056374:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.056376:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.056377:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.056377:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.056380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.056384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:4.2:1619677448.056388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677448.056388:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677448.056391:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.056391:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.056392:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.056392:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.056393:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.056394:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677448.056394:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.056395:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677448.056395:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677448.056397:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677448.056398:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.056399:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.056401:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.056403:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.056404:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.056406:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.056407:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.056408:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.056409:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.056412:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.056414:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.056417:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.056417:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.056418:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.056420:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.056420:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.056421:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.056423:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.056423:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.056428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.056435:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.056435:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.056438:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677448.056438:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677448.056438:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.056439:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.056439:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.056440:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.056441:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.056446:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.056448:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.056450:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.056451:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.056451:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.056452:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.056453:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.056478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.056484:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.056487:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.056487:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.056489:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.056492:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.056494:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.056495:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719949] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.056496:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.056497:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.056498:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.056502:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.056504:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.056506:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.056507:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.056508:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.056510:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.056510:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.056511:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.056511:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.056523:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.056526:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:21.2:1619677448.056527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:12.0:1619677448.056527:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.056528:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.056530:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.056532:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677448.056534:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.056537:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.056538:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.056539:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.056542:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.056544:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.056545:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.056546:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.056547:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.056548:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.056552:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.056554:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.056557:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.056558:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.056558:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.056560:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.056560:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.056561:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.056561:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.2:1619677448.056568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.056573:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:13.0:1619677448.056574:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.056576:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.056576:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.056577:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:13.0:1619677448.056577:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.056579:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.056580:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.056580:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677448.056581:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.056582:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.056583:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.056585:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.056586:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.056587:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.056589:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.056590:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:2.0:1619677448.056590:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.056592:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.056592:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.056593:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.056594:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.056594:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.056595:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.056595:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.056595:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.056596:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.056598:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.056598:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.056599:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.056600:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.056604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.056612:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.056615:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.056616:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.056617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677448.056618:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.056621:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.056622:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.056624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.056626:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.056628:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.056631:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.056632:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.056633:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.056633:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.056635:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:20.0:1619677448.056635:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.056636:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.056636:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.056636:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.056637:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:30.0:1619677448.056638:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.056638:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:30.0:1619677448.056639:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.056640:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677448.056643:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.056645:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.056648:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.056649:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677448.056650:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.056651:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677448.056652:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.056653:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.056655:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.056656:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.2:1619677448.056656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.2:1619677448.056661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.056663:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:6.0:1619677448.056665:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.0:1619677448.056667:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:6.0:1619677448.056668:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.056669:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:20.0:1619677448.056669:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:6.0:1619677448.056669:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.056669:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.056670:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:6.0:1619677448.056671:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.056672:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.056673:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.056674:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:5.0:1619677448.056674:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:31.0:1619677448.056675:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.056675:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:5.0:1619677448.056675:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719945] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.056676:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.056677:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.056678:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.056681:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.056682:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.056685:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.056686:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.2:1619677448.056688:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.0:1619677448.056691:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.056693:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.056694:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.056695:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:10.0:1619677448.056696:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677448.056696:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:10.0:1619677448.056699:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.056700:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.056702:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:10.0:1619677448.056702:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.056704:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.056705:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.056705:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.056706:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.056707:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.056708:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.056709:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.056710:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.056711:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.056714:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.056715:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.056716:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.056716:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.056719:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.056720:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.056721:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.056722:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.056723:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.056724:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.056725:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.056725:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.056731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.056734:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.056737:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.056737:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.056738:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:12.0:1619677448.056739:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.056740:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.056741:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:13.0:1619677448.056741:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.056742:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.056742:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.056780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.056787:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.056789:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.056790:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.056792:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.056794:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.056795:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.056797:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.056798:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.056799:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.056799:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.056802:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.056804:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.056806:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.056807:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.056807:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.056809:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.056809:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.056810:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.056811:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.056823:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.056826:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.056826:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.056827:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.056829:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.056832:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.056877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.056883:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.056886:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.056886:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.056889:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.056891:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.056893:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.056894:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.056895:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.056896:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.056897:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.056901:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.056903:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.056906:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.056906:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.056907:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.056909:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.056909:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.056910:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.056910:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.056921:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.056924:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.056925:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.056927:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.056928:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.056929:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.057014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.057020:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.057023:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.057023:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.057025:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.057028:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.057029:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.057031:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.057032:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.057033:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.057034:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.057038:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.057040:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.057043:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.057043:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.057044:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.057046:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.057047:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.057048:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.057049:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.057049:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677448.057054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677448.057058:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.057060:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:12.0:1619677448.057061:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.057062:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.057062:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.057062:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.057063:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:12.0:1619677448.057064:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.057065:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.057066:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.057077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.057084:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.057086:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.057087:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.057089:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.057092:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.057093:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.057095:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.057096:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.057097:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.057098:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.057101:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.057103:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.057106:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.057107:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.057108:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.057109:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.057110:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.057110:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.057111:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.057123:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.057126:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.057127:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.057127:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.057129:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.057131:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.057212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.057219:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.057222:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.057224:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.057226:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.057227:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.057228:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.057230:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.057231:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.057232:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.057233:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.057236:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.057238:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.057241:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.057242:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.057242:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.057244:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.057244:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.057245:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.057246:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.057257:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.057260:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.057261:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.057261:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.057264:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.057267:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.057272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.057280:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.057283:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.057285:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.057287:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.057288:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.057289:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.057291:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.057292:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.057293:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.057298:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.057301:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.057303:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.057306:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.057306:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.057307:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.057309:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.057309:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.057311:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.057313:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.057313:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.057316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.057320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.057327:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677448.057328:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.057330:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677448.057331:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:20.0:1619677448.057331:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.057332:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.057332:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.057332:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.057332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:4.2:1619677448.057334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.0:1619677448.057337:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.057338:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677448.057338:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:5.2:1619677448.057339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.0:1619677448.057341:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.057341:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677448.057343:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:24.0:1619677448.057344:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:17.0:1619677448.057344:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.057344:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.057345:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677448.057346:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677448.057347:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.057348:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.057349:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677448.057349:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.057350:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.057351:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.057352:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.057353:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.057354:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.057354:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.057355:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:24.0:1619677448.057356:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.057358:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.057358:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.057359:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.057360:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.057361:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.057361:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.057363:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:1.0:1619677448.057363:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.057364:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.057365:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.057371:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.057372:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.057375:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.057375:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:24.0:1619677448.057378:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.057378:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.057378:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.057379:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.057379:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:1.0:1619677448.057379:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.057380:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:1.0:1619677448.057380:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.057381:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.057383:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.057383:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:1.0:1619677448.057383:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.057384:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.057384:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677448.057384:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677448.057384:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677448.057385:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677448.057385:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677448.057386:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:1.0:1619677448.057386:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (23)++
00000800:00000200:17.0:1619677448.057387:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (24)--
00000800:00000200:1.0:1619677448.057387:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (24)--
00000800:00000200:1.0:1619677448.057388:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.057389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:9.2:1619677448.057391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.2:1619677448.057393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.057398:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:8.0:1619677448.057398:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:10.0:1619677448.057399:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:17.0:1619677448.057400:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:8.0:1619677448.057400:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677448.057400:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:17.0:1619677448.057401:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.057401:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.057401:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677448.057402:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:10.0:1619677448.057402:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:8.0:1619677448.057402:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.057403:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:17.0:1619677448.057403:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.057403:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.057403:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.057404:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.057405:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.057406:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:10.0:1619677448.057406:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.057409:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:8.0:1619677448.057410:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677448.057454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677448.057461:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.057463:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677448.057464:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677448.057466:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.057468:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.057470:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.057471:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.057473:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.057474:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.057475:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.057478:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.057480:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.057482:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.057483:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.057483:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.057485:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.057486:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.057487:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677448.057488:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677448.057488:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:20.2:1619677448.057490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.2:1619677448.057495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:5.0:1619677448.057497:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677448.057498:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.057500:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:5.0:1619677448.057500:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:13.0:1619677448.057501:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.057502:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:5.0:1619677448.057502:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.057503:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:13.0:1619677448.057503:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.057504:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:13.0:1619677448.057505:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.057505:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:5.0:1619677448.057505:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.057506:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.057506:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:5.0:1619677448.057507:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.057508:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.057509:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.057510:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.057511:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.057515:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.057516:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677448.057517:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677448.057519:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.057520:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677448.057520:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.057520:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677448.057521:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:5.0:1619677448.057521:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677448.057522:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.057523:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677448.057523:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.057524:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.057525:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.057526:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.057527:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:9.2:1619677448.057529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677448.057532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.057535:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.057537:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:18.0:1619677448.057537:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677448.057538:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.057538:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677448.057540:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.057541:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.057541:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.057543:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.057546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.057549:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.057550:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.057552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.057552:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.057553:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.057554:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.057554:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.057555:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.057556:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.057558:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.057559:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.057561:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.057563:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.057565:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.057566:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.057567:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.057568:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.057569:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.057573:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.057574:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.057577:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.057578:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.057578:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.057580:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.057581:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.057582:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.057583:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.057583:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.2:1619677448.057588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.057594:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:6.0:1619677448.057595:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677448.057596:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.057597:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677448.057597:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:18.0:1619677448.057598:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677448.057599:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.057599:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.057600:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.057603:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.057607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677448.057612:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.057614:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.057615:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.057616:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:22.0:1619677448.057616:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.057616:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.057617:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.057617:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677448.057618:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:26.0:1619677448.057619:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.057622:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.057623:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.057625:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.057626:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.057627:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.057628:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.057631:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.057633:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.057636:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.057637:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.057638:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.057639:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.057640:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.057641:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.057641:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.057645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.057654:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.057656:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.057657:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.057659:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.057660:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.057662:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.057668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.057674:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.057677:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.057677:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.057679:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.057682:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.057683:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.057685:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.057686:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.057687:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.057688:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.057691:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.057693:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.057695:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.057696:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.057697:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.057698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:6.0:1619677448.057698:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.057699:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.057699:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.057700:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.057705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:24.0:1619677448.057706:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.057709:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.057711:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.057712:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.057713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:24.0:1619677448.057715:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.057716:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.057717:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96719929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.057717:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:24.0:1619677448.057718:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.057718:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:24.0:1619677448.057720:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.057720:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:24.0:1619677448.057721:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677448.057721:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.057721:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.057722:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677448.057723:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.057723:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.057724:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.057726:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.057727:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.057728:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.057729:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:5.0:1619677448.057730:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:2.0:1619677448.057730:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.057732:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.057732:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.057733:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:2.0:1619677448.057733:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.057734:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:2.0:1619677448.057734:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.057736:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.057737:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:2.0:1619677448.057737:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.057738:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.057739:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:2.0:1619677448.057739:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.057740:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:2.0:1619677448.057741:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677448.057742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.057742:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.057743:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.057744:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:21.2:1619677448.057745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:2.0:1619677448.057745:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.057745:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.057746:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677448.057750:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:9.2:1619677448.057750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.057753:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:8.0:1619677448.057753:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:20.0:1619677448.057754:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.057754:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:20.0:1619677448.057755:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.057755:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677448.057756:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677448.057757:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.0:1619677448.057757:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677448.057758:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677448.057760:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.057761:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.057761:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.2:1619677448.057762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000400:00000200:18.0:1619677448.057763:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.057765:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677448.057770:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.057773:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.057774:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.057774:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.057775:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.057776:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.057791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.057798:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.057801:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.057803:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.057805:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.057806:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.057808:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.057809:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.057811:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.057812:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.057813:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.057816:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.057818:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.057820:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.057821:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.057822:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.057823:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.057824:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.057825:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.057825:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.057835:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.057838:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.057838:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.057840:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.057842:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.057842:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.2:1619677448.058251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.058258:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.058260:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.058261:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.058263:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.058266:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.058268:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.058269:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.058270:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.058271:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.058272:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.058276:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.058278:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.058280:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.058281:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.058282:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.058284:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.058284:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.058285:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.058285:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.058290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:21.2:1619677448.058296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:24.0:1619677448.058298:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.058301:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:22.0:1619677448.058301:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:24.0:1619677448.058302:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.058302:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.058304:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.058304:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.058305:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677448.058307:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:22.0:1619677448.058307:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.058309:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.058311:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.058312:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.058313:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.058314:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.058315:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.058319:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.058321:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.058323:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.058324:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.058325:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.058327:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.058327:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.058328:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.058328:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.058333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.058341:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.058344:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.058345:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.058345:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.058347:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.058349:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.058416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.058424:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:21.2:1619677448.058426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677448.058427:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.058428:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.058430:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.058433:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.058435:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677448.058435:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.058436:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.058438:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677448.058438:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.058439:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.058440:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.058441:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.058442:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.058443:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.058444:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.058445:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.058446:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719923] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.058446:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.058447:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.058448:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.058449:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:10.0:1619677448.058449:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.058450:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.058451:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:30.0:1619677448.058452:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.058452:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.058453:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.058453:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:30.0:1619677448.058454:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:10.0:1619677448.058454:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:30.0:1619677448.058456:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.058457:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.058458:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.2:1619677448.058458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.058460:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.058460:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.058461:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.058461:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.058466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677448.058467:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.058470:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.058471:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.058472:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.058473:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677448.058473:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:5.0:1619677448.058475:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677448.058476:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.058477:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.058477:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.058479:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.058482:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.058487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.058495:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.058498:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.058500:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.058502:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.058503:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.058504:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.058505:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.058507:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.058508:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.058508:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.058511:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.058513:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.058515:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.058516:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.058516:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.058518:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.058518:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.058519:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.058519:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.058530:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.058532:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.058533:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.058534:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.058536:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.058537:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.058617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.058624:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.058626:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.058627:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.058629:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.058632:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.058633:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.058634:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.058636:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.058636:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.058637:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.058640:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.058641:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.058643:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.058644:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.058645:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.058646:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.058647:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.058648:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.058649:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.058649:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.058655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.058659:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.058661:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.058662:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.058662:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677448.058663:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.058664:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.058664:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.058665:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.058666:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.058667:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.058676:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.058678:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.058679:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.058680:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.058681:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.058683:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.058688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.058694:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.058697:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.058698:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.058700:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.058703:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.058704:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.058706:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.058707:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.058708:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.058709:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.058712:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.058714:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.058717:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.058717:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.058718:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.058720:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.058720:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.058721:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.058721:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.058733:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.058736:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.058737:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.058737:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.058739:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.058742:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.058752:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.058755:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.058755:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.058756:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.058757:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.058757:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.058859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.058866:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.058869:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.058870:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.058872:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.058874:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.058876:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.058877:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.058878:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.058879:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.058880:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.058884:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.058885:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.058888:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.058888:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.058889:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.058891:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.058891:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.058892:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.058892:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677448.058897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.058902:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677448.058903:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.058905:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:22.0:1619677448.058905:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.058905:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:31.0:1619677448.058906:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.058908:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.058909:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.058910:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.058911:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:6.0:1619677448.058911:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.058912:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.058913:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.058915:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.058916:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.058917:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.058920:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000400:00000200:6.0:1619677448.058920:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.058922:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.058923:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.058924:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.058924:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.058924:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.058925:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:6.0:1619677448.058925:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.058926:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.058926:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.058928:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.058928:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.058929:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.058930:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.058931:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:21.2:1619677448.058936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.058941:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.058943:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.058943:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.058944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.058945:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677448.058952:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.058955:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.058956:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.058956:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.058958:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.058960:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.058967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.058974:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.058976:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.058977:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.058977:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.058978:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.058979:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.059022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.059027:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.059029:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.059030:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.059032:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.059034:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.059036:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.059036:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.059038:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.059039:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.059040:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.059043:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.059044:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.059047:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.059047:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.059048:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.059050:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.059050:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.059051:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.059051:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677448.059057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:6.0:1619677448.059063:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:30.0:1619677448.059065:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:6.0:1619677448.059066:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:30.0:1619677448.059067:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:22.0:1619677448.059067:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.059067:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.059069:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.059070:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.059071:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.059072:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.059072:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:30.0:1619677448.059073:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.059075:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719916] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.059076:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.059077:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.059078:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.059082:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.059083:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.059086:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.059086:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.059087:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.059089:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.059089:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.059090:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.059091:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677448.059101:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.059103:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.059104:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.059105:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.059107:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.059108:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.059222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.059229:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.059231:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.059232:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.059234:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.059236:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.059238:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.059239:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.059240:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.059241:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.059242:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.2:1619677448.059243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:10.0:1619677448.059245:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.059246:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.059247:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.059248:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:10.0:1619677448.059250:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.059251:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.059251:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.059252:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.059252:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677448.059253:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:20.0:1619677448.059254:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.059254:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.059254:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.059255:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:20.0:1619677448.059256:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.059257:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.059259:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677448.059260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:20.0:1619677448.059261:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.059262:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.059267:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:20.0:1619677448.059267:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.059267:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677448.059268:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.0:1619677448.059270:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.059271:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677448.059271:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.059271:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.059272:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.059272:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.059273:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.059274:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:22.0:1619677448.059275:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.059275:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.059276:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.059276:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.059277:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677448.059278:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677448.059278:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:8.0:1619677448.059278:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.059280:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.059281:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96719913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.059283:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.059284:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.059285:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.059288:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:8.0:1619677448.059289:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.059290:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.059291:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.059292:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.059294:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.059295:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.059297:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:31.0:1619677448.059300:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.059300:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.059302:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.059303:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.0:1619677448.059303:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.059304:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.059304:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.2:1619677448.059304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.059310:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.059313:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.059314:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.059316:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.059318:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.059319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:17.0:1619677448.059320:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.059321:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.059323:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.059324:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.059325:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.059328:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:17.0:1619677448.059329:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.059329:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.059330:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.059331:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.059332:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.059334:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.059335:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.059335:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.059336:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.059336:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677448.059338:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.059339:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.059340:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.059341:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.059342:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.059347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.059352:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677448.059353:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.059354:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.059355:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.059355:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.059356:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.059357:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.059357:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.059359:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.059361:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.059484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.059492:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.059496:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.059496:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.059498:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.059501:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.059503:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.059504:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.059506:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.059507:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.059508:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.059511:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.059513:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.059516:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.059517:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.059518:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.059519:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.059520:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.059521:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.059521:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.059526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.059534:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.059537:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.059538:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.059539:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.059540:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.059544:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.059546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.059554:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.059557:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.059559:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.059561:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.059561:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.059562:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.059563:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.059565:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.059566:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.059567:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.059570:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.059572:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.2:1619677448.059573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:30.0:1619677448.059575:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.059576:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.059576:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.059578:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.059578:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677448.059579:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677448.059580:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.059581:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.059582:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677448.059582:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.059582:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.059584:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.059584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:22.0:1619677448.059587:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.059588:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:21.2:1619677448.059588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:22.0:1619677448.059589:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.059591:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.059592:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.059592:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:22.0:1619677448.059593:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:18.0:1619677448.059594:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:31.0:1619677448.059595:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:22.0:1619677448.059596:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.059596:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.059596:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.059598:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:22.0:1619677448.059598:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:18.0:1619677448.059598:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.059599:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.059600:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.059601:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.059601:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:22.0:1619677448.059602:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:9.2:1619677448.059602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.059603:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.059606:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.059606:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.059607:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.059608:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677448.059609:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.059612:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.059613:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.059613:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.059614:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.059615:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:31.0:1619677448.059619:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.059621:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.059622:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.059623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:31.0:1619677448.059624:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.059626:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.059627:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.059628:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.059629:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.059629:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.059631:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.059633:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.059635:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.059636:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719908] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.059637:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.059639:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.059639:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.059643:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.059644:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.059646:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.059647:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.059648:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.059649:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.059650:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.059651:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.059651:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677448.059661:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.059663:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.059664:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.059665:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.059667:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.059668:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.059791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.059797:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.059799:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.059802:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.059802:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.059805:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.059806:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.059808:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.059809:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.059810:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.059811:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.059814:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.059816:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.059819:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.059820:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.059821:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.059823:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.059823:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.059824:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.059825:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.059836:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.059839:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.059840:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.059842:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.059843:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.059844:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677448.059852:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.059854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677448.059855:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.059856:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.059856:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.059857:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.059857:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677448.059860:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.059862:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.059863:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.059865:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.059868:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.059869:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.059870:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.059872:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.059873:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.059874:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.059877:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.059878:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.059880:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.059881:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.059881:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.059883:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.059883:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.059883:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.059884:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.059889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.059896:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:21.2:1619677448.059897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:1.0:1619677448.059898:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.059899:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.059899:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.059901:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.059902:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677448.059904:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.059907:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.059907:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677448.059909:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.059913:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.059914:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:10.0:1619677448.059915:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.059917:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.059918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:10.0:1619677448.059918:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.059919:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:10.0:1619677448.059922:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.059924:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.059924:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.059925:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.059926:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677448.059927:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:10.0:1619677448.059927:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677448.059927:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:10.0:1619677448.059928:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000400:00000200:18.0:1619677448.059929:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.059929:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.059929:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.059930:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:18.0:1619677448.059932:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.059932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:18.0:1619677448.059933:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.059935:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.059936:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.059937:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.059938:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.059940:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:18.0:1619677448.059942:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.059942:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.059942:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.059943:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.059943:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.059945:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.059945:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.059946:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.059947:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.059948:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.059948:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.059948:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677448.059949:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.059949:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.059949:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:26.0:1619677448.059951:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:18.0:1619677448.059951:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.059952:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.059952:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:26.0:1619677448.059953:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.059953:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.059953:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:26.0:1619677448.059954:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.059955:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.059959:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.059962:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.059964:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.059965:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.059965:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.059966:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.059966:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.059967:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.059968:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.059969:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:10.0:1619677448.059969:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.059970:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.059971:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.059972:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.059972:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.059980:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.059983:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.059983:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.059985:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.059987:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.059987:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.2:1619677448.059989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.059996:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.059999:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:20.2:1619677448.059999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:30.0:1619677448.060001:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.060002:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:31.0:1619677448.060003:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.060003:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.060004:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:30.0:1619677448.060005:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.060007:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.060007:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:30.0:1619677448.060008:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.060008:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:30.0:1619677448.060009:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.060010:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677448.060010:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.060014:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.060016:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:20.0:1619677448.060016:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677448.060018:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.060018:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.060019:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:20.0:1619677448.060019:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.060020:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:20.0:1619677448.060020:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.060021:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.060022:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:20.0:1619677448.060022:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.060025:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:30.0:1619677448.060027:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.060027:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.2:1619677448.060027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.060029:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.060029:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677448.060030:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.060030:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.060031:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.060033:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.2:1619677448.060035:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.060036:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.060037:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.060038:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.060038:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.060038:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.060039:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677448.060040:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.060040:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.060043:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677448.060044:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.060045:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.060046:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.060047:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.060047:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.060048:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677448.060049:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.060049:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.060050:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:30.0:1619677448.060053:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.060062:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.060065:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.060066:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.060067:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.060067:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.060068:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.060087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.060095:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.060098:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.060099:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.060101:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.060103:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.060104:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.060105:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.060107:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.060108:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.060109:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.060112:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.060114:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.060117:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.060118:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.060118:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.060120:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.060120:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.060121:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.060122:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677448.060134:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.060137:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.060137:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.060138:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.060140:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.060143:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.060186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.060188:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.060191:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.060193:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.060194:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.060195:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.060197:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.060198:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.060199:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.060200:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.060201:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:21.2:1619677448.060203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677448.060205:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.060206:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.060210:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.060210:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.060210:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.060211:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.060213:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677448.060213:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.060213:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.060214:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.060214:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:30.0:1619677448.060215:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.060217:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.060218:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.060219:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677448.060220:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.060222:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.060223:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.060224:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:13.0:1619677448.060225:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:30.0:1619677448.060227:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.060228:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:13.0:1619677448.060228:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.060229:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.060230:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.060232:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.060232:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.060233:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677448.060233:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:13.0:1619677448.060233:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.060235:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677448.060235:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.060236:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.060237:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.060249:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.060251:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.060252:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.060253:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.060255:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.060258:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677448.060273:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.060275:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.060276:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.060277:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.060278:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.060278:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.060351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.060357:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.060360:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.060361:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.060363:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.060365:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.060367:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677448.060368:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.060370:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.060371:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.060372:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677448.060375:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.060377:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677448.060379:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.060380:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677448.060381:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677448.060383:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677448.060383:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.060384:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.060385:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677448.060396:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.060399:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.060400:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.060400:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.060402:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:26.0:1619677448.060405:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:10.0:1619677448.060412:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.060415:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.060416:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.060416:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.060417:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.060417:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677448.060581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.060590:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.060592:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.060594:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.060597:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.060597:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.060599:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.060600:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.060601:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.060602:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.060603:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.060607:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.060609:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.060612:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.060613:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.060613:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.060615:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.060616:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.060616:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.060617:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.060622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.060631:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.060634:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.060634:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.060636:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.060636:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.060639:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.060641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.060649:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.060652:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.060653:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.060654:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.060655:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.060656:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.060677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.060684:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.060687:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.060688:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.060689:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.060690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:26.0:1619677448.060692:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.060693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:26.0:1619677448.060694:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.060695:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719895] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.060697:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.060698:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.060699:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:26.0:1619677448.060700:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.060700:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:30.0:1619677448.060701:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.060703:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677448.060703:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.060704:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.060705:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.060706:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:30.0:1619677448.060707:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.060708:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677448.060708:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.060709:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.060710:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.060710:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.060711:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.060712:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:10.0:1619677448.060712:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677448.060713:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.060714:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.060715:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.060716:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:26.0:1619677448.060716:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.060717:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.060717:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:10.0:1619677448.060717:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.060720:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.060720:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.060720:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.060721:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.060722:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:26.0:1619677448.060722:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.060722:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.060723:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.060723:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.060724:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.060724:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.060725:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.060726:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.060726:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.2:1619677448.060727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.060727:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.060727:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.060728:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.060728:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.060728:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:30.0:1619677448.060729:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.060730:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.060730:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:30.0:1619677448.060731:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:10.0:1619677448.060731:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.060733:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:30.0:1619677448.060734:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:9.2:1619677448.060734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:30.0:1619677448.060735:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.060735:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:30.0:1619677448.060736:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.060736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.060737:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.060737:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:30.0:1619677448.060738:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.060738:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:31.0:1619677448.060739:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.060739:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:20.2:1619677448.060739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:26.0:1619677448.060740:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.060740:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.060741:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.0:1619677448.060741:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677448.060742:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:10.0:1619677448.060742:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:18.0:1619677448.060743:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:10.0:1619677448.060743:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96719892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.060744:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:22.0:1619677448.060745:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:10.0:1619677448.060745:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.060746:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.060746:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:10.0:1619677448.060746:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.060746:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:20.0:1619677448.060747:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.060747:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.060747:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:10.0:1619677448.060748:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.060748:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:22.0:1619677448.060749:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.060750:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.060750:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.060752:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:10.0:1619677448.060753:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.060754:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:20.0:1619677448.060756:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677448.060756:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:20.0:1619677448.060757:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.060758:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.060759:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.060759:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:20.0:1619677448.060760:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.060760:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:20.0:1619677448.060761:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.060761:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.060762:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.060763:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:20.0:1619677448.060764:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.060764:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.060765:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:20.0:1619677448.060766:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.2:1619677448.060767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.060769:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.060770:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.2:1619677448.060772:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677448.060775:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.060775:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.060776:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.060777:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.060777:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:5.0:1619677448.060777:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.060779:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677448.060779:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:5.0:1619677448.060779:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677448.060780:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.060780:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677448.060781:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.060782:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:20.0:1619677448.060782:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677448.060782:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.060783:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:20.2:1619677448.060785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677448.060790:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.060791:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677448.060793:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.060793:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.060794:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.060794:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:12.0:1619677448.060795:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.060796:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.060798:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.060799:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.060801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.060806:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.060809:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.060809:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.060811:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.060814:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.060816:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.060817:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.060819:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.060820:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.060821:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.060824:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.060826:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.060829:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.060830:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.060830:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.060832:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.060833:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.060833:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.060833:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:26.0:1619677448.060844:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.060847:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.060848:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.060848:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.060850:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.060853:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677448.060907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.060912:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.060915:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.060916:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.060917:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.060920:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.060922:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.060923:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.060924:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.060925:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.060927:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.060930:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.060932:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.060935:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.060936:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.060936:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.060938:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.060939:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.060940:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.060941:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.060942:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.2:1619677448.060947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:13.0:1619677448.060951:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.060953:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:13.0:1619677448.060953:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.060954:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.060955:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.060955:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:13.0:1619677448.060956:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.060956:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677448.060957:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.060959:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.061063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:4.2:1619677448.061067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677448.061069:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.061072:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.061073:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.061074:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.061074:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677448.061076:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.061077:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.061077:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:17.0:1619677448.061078:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.061079:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.061080:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.061081:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.061082:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.061083:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.061083:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.061084:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.061086:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.061087:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.061088:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.061089:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.061089:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.061090:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.061093:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.061094:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.061095:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.061095:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.061097:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.061097:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.061097:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.061098:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.061099:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.061100:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.061100:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.061101:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.061103:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.061103:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.061104:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.061106:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.061106:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:12.0:1619677448.061109:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.061111:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:4.2:1619677448.061112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:12.0:1619677448.061113:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.061114:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.061115:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677448.061116:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:12.0:1619677448.061117:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.061119:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.061119:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.061120:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.061120:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.061121:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.061121:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.061122:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.061122:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.061125:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677448.061127:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.061130:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.061130:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.061131:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.061131:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.061133:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.061182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.061189:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.061192:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.061194:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.061196:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.061197:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.061198:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677448.061199:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.061201:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.061202:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.061203:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677448.061206:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.061208:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.061211:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.061211:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677448.061212:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.061213:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677448.061214:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.061215:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.061215:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.061227:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.061230:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.061231:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.061231:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.061233:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.061237:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.061257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.061262:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.061265:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.061267:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.061267:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.061270:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.061272:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.061273:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.061275:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.061276:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.061277:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.061280:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.061282:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.061285:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.061285:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.061286:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.061288:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.061288:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.061290:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.061291:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.061291:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677448.061294:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677448.061297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.2:1619677448.061298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:20.0:1619677448.061300:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.061301:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.061302:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:1.0:1619677448.061302:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677448.061304:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677448.061304:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:1.0:1619677448.061305:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.061306:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.061306:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.061308:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.061308:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:1.0:1619677448.061309:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.061311:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.061312:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:18.0:1619677448.061314:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.061315:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.061316:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.061318:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.061319:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.061320:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.061323:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.061325:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.061328:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.061329:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.061329:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.061331:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.061331:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.061332:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.061333:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.061337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.061345:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.061347:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.061348:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.061350:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.061350:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.061353:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.061405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.061413:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.061416:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.061416:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.061419:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.061422:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.061423:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.061424:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.061426:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.061426:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.061427:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.061430:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.061432:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.061435:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.061435:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.061436:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.061438:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.061438:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.061439:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.061440:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677448.061447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.061452:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:22.0:1619677448.061453:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677448.061455:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:22.0:1619677448.061456:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.061457:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.061457:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:6.0:1619677448.061457:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.061459:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.061459:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.061462:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.061463:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:2.0:1619677448.061463:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.061465:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.061466:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.061467:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.061468:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.061472:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.061473:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.061476:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.061477:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.061478:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.061480:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.061480:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.061481:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:4.2:1619677448.061482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:2.0:1619677448.061482:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:24.0:1619677448.061489:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.061492:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:12.0:1619677448.061493:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677448.061493:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.061495:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.061495:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.061496:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.061498:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.061498:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.061500:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:13.0:1619677448.061500:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677448.061501:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.061501:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677448.061502:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.061503:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.061504:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.061508:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.061509:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.061512:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.061513:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.061513:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.061515:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.061515:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.061516:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.061517:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.061521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.061529:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.061531:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.061532:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.061533:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.061535:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.061537:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.061538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.061546:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.061548:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.061549:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.061549:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.061550:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.061551:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.061582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:26.0:1619677448.061589:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677448.061592:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:10.0:1619677448.061593:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:26.0:1619677448.061595:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.061598:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.061600:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:20.2:1619677448.061600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:26.0:1619677448.061601:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.061602:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.061603:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.061603:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677448.061604:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:20.0:1619677448.061606:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.061609:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.061609:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677448.061610:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.061612:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:20.0:1619677448.061612:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.061612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:20.0:1619677448.061614:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.061615:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.061616:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.061616:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.061617:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:20.0:1619677448.061617:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.061617:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:20.0:1619677448.061618:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.061619:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.061619:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000400:00000200:20.0:1619677448.061619:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.061620:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75ca00] (22)++
00000800:00000200:17.0:1619677448.061620:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.061621:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.061622:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (23)--
00000800:00000200:26.0:1619677448.061622:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:20.0:1619677448.061623:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.061624:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.061624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.061625:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.061627:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.061627:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.061628:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.2:1619677448.061628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:20.0:1619677448.061629:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:17.0:1619677448.061629:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:20.0:1619677448.061630:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.061631:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:17.0:1619677448.061631:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96719878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.061631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677448.061632:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.061632:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677448.061633:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.061633:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:17.0:1619677448.061634:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.061635:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.061635:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:10.0:1619677448.061636:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)++
00000400:00000200:1.0:1619677448.061636:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.061637:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75ca00] (23)--
00000800:00000200:10.0:1619677448.061638:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:1.0:1619677448.061638:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.061639:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000400:00000200:17.0:1619677448.061639:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.061640:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677448.061641:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.061642:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.061642:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.061642:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:1.0:1619677448.061643:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:6.0:1619677448.061644:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677448.061645:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.061645:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.061646:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.061647:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.061647:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.061648:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:10.0:1619677448.061648:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.061648:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.061649:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.061649:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.061650:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.061650:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:10.0:1619677448.061651:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:6.0:1619677448.061651:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.061652:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96719877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.061653:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:10.0:1619677448.061654:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.061655:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.061656:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.061659:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.061661:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.061663:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.061664:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677448.061664:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.061665:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677448.061666:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.061666:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:22.0:1619677448.061667:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.061667:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.061667:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.061667:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677448.061668:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.061668:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677448.061668:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.061668:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.061669:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.061670:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.061671:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.061671:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.061672:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677448.061679:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.061681:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.061682:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.061684:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.061685:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.061687:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.061688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.061696:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.061699:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.061702:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.061704:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.061704:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.061706:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.061707:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.061708:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.061709:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.061710:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.061713:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.061714:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.061717:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.061717:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.061718:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.061719:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.061720:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.061721:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.061723:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.061723:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677448.061729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.061731:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.061733:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.061734:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.061735:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.061736:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.061737:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.061738:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.061739:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.061740:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.061741:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677448.061840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.061846:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.061849:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.061849:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.061852:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.061854:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.061856:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.061857:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.061858:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.061859:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.061860:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.061863:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.061865:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.061868:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.061869:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.061870:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.061872:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.061872:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.061873:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.061873:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:4.2:1619677448.061883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677448.061884:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.061887:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.061888:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.061889:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.061890:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:12.0:1619677448.061891:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.061892:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.061893:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.061894:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.061895:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.061898:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.061899:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.061900:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719874] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.061902:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.061903:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.061904:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.061907:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.061908:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.061911:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.061912:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.061912:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.061914:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.061915:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.061915:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.061916:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.061928:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.061931:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.061932:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.061932:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.061934:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.061937:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.062050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.062056:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.062059:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.062060:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.062062:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.062064:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.062066:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.062067:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.062068:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.062069:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.062070:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.062074:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.062075:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.062078:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.062079:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.062080:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.062082:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.062082:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.062084:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.062085:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.062085:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.062090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.062095:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:30.0:1619677448.062096:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677448.062097:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.062097:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.062098:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.062099:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.062099:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.062102:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.062103:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.062104:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.062258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.062263:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.062266:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.062267:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.062269:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.062272:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.062273:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.062274:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.062276:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.062277:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.062278:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.062281:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.062282:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.062285:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.062286:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.062287:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.062289:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.062290:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:4.2:1619677448.062291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677448.062291:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.062292:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.062292:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677448.062299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:10.0:1619677448.062300:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.062303:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.062304:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677448.062305:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:12.0:1619677448.062306:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677448.062306:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.062307:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.062308:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.062309:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677448.062309:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:10.0:1619677448.062309:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.062310:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.062310:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:12.0:1619677448.062312:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.062312:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.062313:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.062314:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.062314:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.062315:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677448.062315:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.062318:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.062320:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:13.0:1619677448.062322:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.062323:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.062323:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.062324:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:13.0:1619677448.062325:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.062326:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.062326:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:10.0:1619677448.062326:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.062326:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677448.062327:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.062327:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.062327:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:13.0:1619677448.062328:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677448.062340:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.062343:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.062344:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.062345:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.062347:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.062349:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.062454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677448.062454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.062462:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677448.062463:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.062465:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.0:1619677448.062465:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.062466:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:31.0:1619677448.062468:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.062469:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.062470:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.062471:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.062472:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.062473:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677448.062473:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:31.0:1619677448.062475:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.062475:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.062476:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.062477:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.062478:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.062478:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.062479:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677448.062479:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677448.062484:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.062484:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.062487:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677448.062487:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.062490:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.062490:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.062491:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.062491:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677448.062491:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.062492:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677448.062493:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.062494:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677448.062494:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.062494:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677448.062495:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.062495:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677448.062496:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.062497:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.062497:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.062500:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677448.062503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677448.062507:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:1.0:1619677448.062508:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677448.062510:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677448.062510:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:1.0:1619677448.062511:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:24.0:1619677448.062512:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:22.0:1619677448.062512:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.062512:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.062512:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.062513:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677448.062513:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.062514:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677448.062515:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.062515:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.062518:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677448.062520:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.062521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.062528:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.062531:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.062531:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.062533:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.062534:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.062534:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.062626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.062632:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.062635:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.062636:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.062638:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.062641:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.062643:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677448.062644:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.062646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:2.0:1619677448.062646:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.062647:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.062648:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.062651:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.062653:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.062653:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.062657:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.062657:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:2.0:1619677448.062657:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.062658:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677448.062658:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:26.0:1619677448.062660:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.062661:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677448.062661:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.062662:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.062662:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:26.0:1619677448.062663:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.062664:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.062665:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.062667:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.062668:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.062669:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.062672:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.062672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:13.0:1619677448.062673:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:26.0:1619677448.062674:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.062676:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.062676:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.062677:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.062677:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.062678:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:13.0:1619677448.062678:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.062679:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.062680:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.062680:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.062680:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.062681:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.062681:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.062681:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.062682:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.062684:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.062686:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.062687:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.062689:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.062690:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.062691:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.062691:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:30.0:1619677448.062692:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.062693:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:12.0:1619677448.062694:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.062695:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.062696:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677448.062696:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.062697:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.062698:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.062699:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.062700:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.062701:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.062702:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.062704:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.062704:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.062705:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.062706:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.062710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.062718:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.062721:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.062722:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.062724:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.062724:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.062726:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.062844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.062851:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.062853:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.062854:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.062856:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.062859:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.062860:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.062861:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.062862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677448.062863:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.062864:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.062864:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.062867:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.062868:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.062869:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.062870:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.062871:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.062873:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.062873:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:1.0:1619677448.062873:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.062874:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.062876:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.062876:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677448.062876:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.062877:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:1.0:1619677448.062877:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:10.0:1619677448.062878:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000400:00000200:1.0:1619677448.062878:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.062879:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677448.062879:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.062880:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.062881:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:4.2:1619677448.062884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677448.062884:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.062886:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.062889:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.062890:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.062890:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.062891:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:1.0:1619677448.062891:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.062892:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.062893:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677448.062893:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677448.062893:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.062893:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.062894:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.062894:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.062894:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677448.062894:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.062896:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.062897:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.062899:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677448.062905:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.062907:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.062908:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.062910:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.062911:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677448.062911:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:13.0:1619677448.062913:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677448.062914:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.062915:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.062915:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.062916:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.062916:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.062959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.062965:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.062968:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.062968:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.062970:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.062973:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.062975:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.062976:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.062977:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.062978:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.062979:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.062982:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.062984:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.062986:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.062987:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.062987:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.062989:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.062989:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.062990:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.062991:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.062995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.063002:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.063005:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.063005:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.063006:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.063007:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.063009:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.063044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.063051:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.063054:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.063056:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.063057:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.063059:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.063060:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.063061:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.063063:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.063064:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.063065:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.063068:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.063070:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.063073:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.063074:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.063075:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.063076:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.063077:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.063078:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.063078:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.063090:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.063093:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.063094:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.063094:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.063096:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.063099:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677448.063108:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.063111:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.063112:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.063112:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.063113:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.063113:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.063218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.063225:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.063228:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.063229:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.063231:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.063234:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.063235:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.063237:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.063238:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.063239:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.063240:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.063243:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.063244:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.2:1619677448.063246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:10.0:1619677448.063247:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.063247:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.063248:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.063249:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.063250:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.063250:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.063251:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.0:1619677448.063253:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677448.063256:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677448.063257:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677448.063258:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.063261:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.063262:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:8.0:1619677448.063262:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.063263:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96719860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.063264:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:8.0:1619677448.063264:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.063265:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.063265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:8.0:1619677448.063265:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.063266:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677448.063267:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.063268:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.063269:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:22.0:1619677448.063270:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:8.0:1619677448.063270:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.063272:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.2:1619677448.063272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677448.063273:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.063273:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677448.063275:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.063276:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.063276:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.063276:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.063278:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.063278:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:22.0:1619677448.063279:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.063279:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677448.063279:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677448.063280:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000400:00000200:22.0:1619677448.063281:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:17.0:1619677448.063281:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.063281:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677448.063282:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.063284:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.063284:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.063284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:22.0:1619677448.063285:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.063286:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677448.063287:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.063288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:22.0:1619677448.063290:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.063290:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.063291:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96719858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.063292:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.063293:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:18.0:1619677448.063294:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:17.0:1619677448.063298:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.063299:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.063299:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677448.063299:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.063300:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.063301:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.063301:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:17.0:1619677448.063301:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.063302:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.063302:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:1.0:1619677448.063302:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.063304:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:18.0:1619677448.063304:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:22.0:1619677448.063305:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.063306:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.063306:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:18.0:1619677448.063306:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.063306:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.063306:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.063307:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:18.0:1619677448.063308:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677448.063309:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96719857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.063309:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:18.0:1619677448.063311:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.063312:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.063312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:18.0:1619677448.063313:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:17.0:1619677448.063313:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.063314:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.063315:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:18.0:1619677448.063316:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677448.063316:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.063317:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.063318:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:17.0:1619677448.063318:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.063318:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677448.063320:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677448.063320:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.063321:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.063322:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677448.063323:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:18.0:1619677448.063323:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.063323:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:20.0:1619677448.063324:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.063324:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.063325:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:20.0:1619677448.063326:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.063328:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.063329:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:9.2:1619677448.063330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677448.063330:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.063332:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:12.0:1619677448.063332:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677448.063332:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.063332:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.063333:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (24)--
00000800:00000200:5.0:1619677448.063334:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.063335:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677448.063335:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.063336:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.063337:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:12.0:1619677448.063338:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.063339:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:13.0:1619677448.063340:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:2.0:1619677448.063340:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.063341:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.063341:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:18.0:1619677448.063342:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:13.0:1619677448.063342:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.063342:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.063343:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:2.0:1619677448.063343:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:13.0:1619677448.063344:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677448.063345:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.063392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.063399:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.063403:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.063404:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.063406:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.063409:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.063411:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.063412:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.063413:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.063414:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.063415:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.063419:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.063420:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.063423:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.063424:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.063424:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.063426:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.063426:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.063427:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.063427:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677448.063439:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.063441:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.063442:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.063443:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.063445:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.2:1619677448.063445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:12.0:1619677448.063446:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.063453:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.063456:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.063457:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.063459:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.063462:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.063463:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.063464:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.063465:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.063466:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.063467:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.063470:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.063471:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.063474:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.063475:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.063475:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.063477:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.063477:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.063478:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.063478:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677448.063492:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.063494:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.063495:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.063496:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.063499:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.063499:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.063588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.063595:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.063598:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.063599:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.063601:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.063603:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.063605:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.063606:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.063607:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.063608:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.063609:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.063612:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.063613:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.063616:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.063617:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.063618:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.063619:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.063620:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.063621:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.063621:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.063626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.063629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.063635:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.063636:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.063637:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.063637:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.063638:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.063640:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.063640:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.063644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:17.0:1619677448.063644:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.063644:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.063645:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:1.0:1619677448.063645:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677448.063646:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.063647:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.063649:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.063650:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.063651:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:31.0:1619677448.063652:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.063654:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.063655:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:17.0:1619677448.063656:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:31.0:1619677448.063657:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.063658:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677448.063659:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.063660:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.063660:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.063660:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:31.0:1619677448.063662:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:17.0:1619677448.063662:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000400:00000200:31.0:1619677448.063663:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.063663:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.063663:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677448.063664:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.063664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677448.063664:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677448.063665:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.063666:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:9.2:1619677448.063668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:31.0:1619677448.063670:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.063671:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.063672:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.063672:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.063674:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.063674:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:31.0:1619677448.063675:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.063676:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.063676:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.063677:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.063677:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:31.0:1619677448.063678:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:30.0:1619677448.063678:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677448.063678:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.063678:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:30.0:1619677448.063680:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96719851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.063681:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.063681:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.063682:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.063682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:30.0:1619677448.063683:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.063684:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.063685:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.063685:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:24.0:1619677448.063685:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.2:1619677448.063685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.063686:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.063687:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.063688:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.063689:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:30.0:1619677448.063691:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.063691:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.063692:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.063692:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.063693:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:5.0:1619677448.063693:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677448.063694:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.063694:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.063695:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677448.063695:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:30.0:1619677448.063696:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677448.063696:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:10.0:1619677448.063697:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.063697:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.063699:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:10.0:1619677448.063699:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677448.063699:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.063701:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96719850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.063702:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.063702:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:10.0:1619677448.063703:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.063704:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.063707:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:6.0:1619677448.063708:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.063709:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:22.0:1619677448.063712:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.063712:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.063712:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:6.0:1619677448.063712:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677448.063713:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:6.0:1619677448.063713:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.063715:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.063715:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:6.0:1619677448.063715:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.063716:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.063716:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:6.0:1619677448.063718:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.2:1619677448.063720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.063728:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.063731:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.063732:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.063733:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.063734:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.063736:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.063759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.063765:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.063767:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.063769:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.063770:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.063771:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.063773:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.063774:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.063775:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.063777:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.063778:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.063781:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.063783:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.063785:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.063786:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.063786:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.063788:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.063789:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.063790:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.063791:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.2:1619677448.063792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:9.2:1619677448.063794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.063799:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.063801:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:18.0:1619677448.063802:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.063802:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677448.063805:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.063805:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.063806:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.063806:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:26.0:1619677448.063807:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.063809:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.063809:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.063811:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.063811:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:26.0:1619677448.063812:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.063813:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.063814:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.063817:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.063817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.063819:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.063823:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.063823:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.063823:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.063824:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.063826:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.063826:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:2.0:1619677448.063826:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.063827:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:18.0:1619677448.063827:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.063827:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.063828:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:2.0:1619677448.063828:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.063829:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.2:1619677448.063832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.063840:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.063842:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.063843:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.063845:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.063846:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.063848:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.063860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.063866:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.063868:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.063869:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.063871:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.063873:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.063875:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.063876:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.063878:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.063879:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.063880:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.063883:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.063885:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.063888:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.063888:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.063889:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.063889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677448.063891:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.063891:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.063893:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.063894:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.063894:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:10.0:1619677448.063896:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677448.063897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.063899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.063899:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.063900:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.063901:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.063904:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.063905:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:1.0:1619677448.063905:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:10.0:1619677448.063906:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.063907:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.063907:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.063907:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677448.063908:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.063908:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677448.063908:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.063909:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:10.0:1619677448.063909:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.063909:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.063910:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.063911:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.063914:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.063916:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.063917:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677448.063919:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.063920:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.063920:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.063922:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.063923:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.063923:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.063924:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677448.063935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677448.063936:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.063939:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.063940:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.063940:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.063941:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677448.063942:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.063944:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:22.0:1619677448.063945:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677448.063945:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.063947:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.063949:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.063951:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.063952:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.063954:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.063955:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.063956:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.063959:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.063961:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.063963:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.063964:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.063965:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.063967:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.063967:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.063969:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.063970:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.063970:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.063972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.063975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.063979:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:1.0:1619677448.063980:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677448.063981:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.063982:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.063982:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:8.0:1619677448.063983:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.063983:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677448.063984:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.063984:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677448.063987:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.063995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677448.064001:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.064003:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.064004:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.064004:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.064005:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.064006:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.064106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.064112:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.064115:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.064116:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.064117:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.064120:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.064121:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.064122:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.064124:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.064125:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.064126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:8.0:1619677448.064126:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.064129:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.064131:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.064133:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.064134:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:8.0:1619677448.064134:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.064135:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.064136:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:8.0:1619677448.064136:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.064137:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:31.0:1619677448.064138:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.064138:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.064138:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:30.0:1619677448.064140:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.064141:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.064143:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.064144:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.064145:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.064146:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.064147:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:13.0:1619677448.064149:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:31.0:1619677448.064150:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.064151:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:13.0:1619677448.064151:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.064152:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.064154:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677448.064154:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.064155:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:12.0:1619677448.064155:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.064156:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:13.0:1619677448.064156:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.064157:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.064158:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.064159:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.064161:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.064161:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.064163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.064167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.064173:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:30.0:1619677448.064175:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:5.0:1619677448.064175:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.064176:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.064177:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.064178:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677448.064178:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.064178:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.064180:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.064181:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.064322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677448.064329:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.064331:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:5.2:1619677448.064331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.064332:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677448.064334:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.064337:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.064338:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:17.0:1619677448.064338:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677448.064340:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719842] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.064341:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.064341:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:18.0:1619677448.064342:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.064342:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677448.064344:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.064344:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.064347:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.064348:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.064349:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:18.0:1619677448.064350:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.064350:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.064352:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.064353:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.064353:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.064354:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:18.0:1619677448.064354:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:17.0:1619677448.064354:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.064356:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:21.2:1619677448.064357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:18.0:1619677448.064357:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:17.0:1619677448.064357:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.064358:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677448.064358:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:17.0:1619677448.064359:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.064362:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.064362:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.064363:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:9.2:1619677448.064363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:6.0:1619677448.064363:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.064365:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.064365:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.064365:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.064366:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677448.064366:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.064366:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.064368:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.064370:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:6.0:1619677448.064370:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.064372:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:24.0:1619677448.064373:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677448.064373:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.064374:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.064374:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96719840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.064375:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.064376:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.064377:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.064378:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677448.064379:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677448.064379:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.064381:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.064382:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.064384:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.064384:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.064386:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.064387:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.064388:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677448.064388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:6.0:1619677448.064390:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.064391:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.064392:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.064394:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:8.0:1619677448.064395:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:6.0:1619677448.064395:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.064396:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.064396:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677448.064397:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677448.064398:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.064398:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.064400:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.064400:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:30.0:1619677448.064408:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.064411:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.064412:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.064413:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.064415:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.064417:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.064425:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.064428:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.064429:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.064430:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.064430:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.064431:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.064466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.064469:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.064471:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:5.2:1619677448.064471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:20.0:1619677448.064474:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.064476:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.064478:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677448.064478:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677448.064479:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:20.0:1619677448.064479:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.064480:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.064482:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:20.0:1619677448.064482:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.064483:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.064483:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.064486:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.064487:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.064489:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.064489:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.064491:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.064493:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.064493:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.064493:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:26.0:1619677448.064494:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.064494:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:26.0:1619677448.064495:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.064496:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677448.064496:0:13335:0:(o2iblnd_cb.c:2075:kiblnd_close_conn_locked()) closing conn to 10.0.10.51@o2ib7
00000400:00000200:26.0:1619677448.064497:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.064497:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677448.064497:0:13335:0:(o2iblnd.c:419:kiblnd_unlink_peer_locked()) peer_ni[ffff91a3b0494880] -> 10.0.10.51@o2ib7 (2)--
00000800:00000200:20.0:1619677448.064498:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.064498:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:26.0:1619677448.064500:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.064501:0:13335:0:(o2iblnd_cb.c:3321:kiblnd_cm_callback()) conn[ffff91a3aea3be00] (20)--
00000800:00000200:26.0:1619677448.064502:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.064505:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.064506:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.064507:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.064509:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.064509:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.064509:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677448.064510:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.064512:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.064512:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:12.0:1619677448.064512:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.064513:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.064514:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.064515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677448.064516:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.064517:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.064518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.064522:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.064525:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:2.0:1619677448.064526:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.064527:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.064527:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:2.0:1619677448.064527:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.064528:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.064529:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:2.0:1619677448.064529:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.064533:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.064567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.064575:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.064578:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.064580:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.064582:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.064583:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.064584:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.064586:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719837] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.064587:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.064588:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.064589:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.064592:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.064594:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.064597:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.064597:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.064598:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.064599:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:21.2:1619677448.064599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677448.064599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.064600:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.064601:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.064602:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.064603:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.064605:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.064606:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.064607:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.064607:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677448.064608:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.064609:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:30.0:1619677448.064610:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:10.0:1619677448.064610:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:30.0:1619677448.064611:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.064612:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.064612:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:30.0:1619677448.064613:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.064613:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.064614:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:13.0:1619677448.064614:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.064615:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.064616:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.064617:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.064618:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.064618:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:12.0:1619677448.064618:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:26.0:1619677448.064619:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96719835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.064620:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.064620:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.064620:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:26.0:1619677448.064621:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.064622:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.064624:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.064624:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677448.064625:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.064626:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:26.0:1619677448.064626:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.064627:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.064627:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.064628:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.064630:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.064630:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.064630:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.064631:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.064632:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (22)++
00000800:00000200:26.0:1619677448.064634:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (24)--
00000800:00000200:26.0:1619677448.064634:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.064635:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.064636:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677448.064636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677448.064640:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (23)++
00000800:00000200:31.0:1619677448.064643:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677448.064643:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000800:00000200:6.0:1619677448.064644:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (24)++
00000400:00000200:22.0:1619677448.064645:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.064645:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000800:00000200:31.0:1619677448.064646:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:6.0:1619677448.064646:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.064647:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677448.064647:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.064648:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:6.0:1619677448.064648:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.064650:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.064651:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677448.064651:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.064661:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.064664:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.064664:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.064665:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.064666:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.064667:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.064694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.064700:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.064703:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.064704:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.064706:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.064708:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.064709:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.064710:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.064712:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.064713:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.064714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:1.0:1619677448.064714:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.064717:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.064719:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677448.064721:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:1.0:1619677448.064722:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.064723:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.064724:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:1.0:1619677448.064724:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.064725:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000400:00000200:31.0:1619677448.064726:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.064726:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.064726:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.064727:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.064728:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677448.064730:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.064731:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.064732:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.064733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:31.0:1619677448.064734:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.064735:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.064736:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677448.064739:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.064740:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:31.0:1619677448.064741:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677448.064743:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:31.0:1619677448.064744:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.064745:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.064745:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:24.0:1619677448.064745:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.064745:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.064746:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.064747:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.064748:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677448.064749:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:24.0:1619677448.064749:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:31.0:1619677448.064750:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677448.064753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.064761:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.064764:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.064765:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.064765:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.064767:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.064771:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.064828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.064834:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.064836:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.064837:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.064839:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.064842:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.064843:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.064844:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.064846:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.064847:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.064848:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.064851:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.064853:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.064855:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.064856:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.064857:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.064859:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.064859:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.064860:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.064860:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.064871:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.064874:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.064875:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.064875:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.064877:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.064880:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677448.064957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.2:1619677448.064960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.064964:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677448.064965:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.064966:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:20.0:1619677448.064967:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.064969:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.064970:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.064972:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.064974:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:5.0:1619677448.064975:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.064976:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.064977:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.064978:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:20.0:1619677448.064980:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.064982:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:26.0:1619677448.064983:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea3be00] (20)++
00000400:00000200:5.0:1619677448.064983:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.064984:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719830] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.064985:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.064985:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.064986:0:19941:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a3aea3be00] (21)--
00000800:00000200:26.0:1619677448.064986:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea3be00] (20)--
00000400:00000200:20.0:1619677448.064987:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.064987:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.064987:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea3be00] (19)--
00000800:00000200:5.0:1619677448.064991:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.064992:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.064992:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:5.0:1619677448.064992:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.064994:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.064994:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677448.064995:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (22)++
00000800:00000200:20.0:1619677448.064996:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.064996:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:5.0:1619677448.064996:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677448.064997:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.064997:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.064997:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:5.0:1619677448.064998:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.064999:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.065000:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.065000:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.065001:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677448.065006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677448.065007:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.065010:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.065011:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.065013:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.065014:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:13.0:1619677448.065014:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.065015:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677448.065017:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.065018:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.065018:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.065020:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.065022:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:13.0:1619677448.065027:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.065030:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.065031:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.065032:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.065032:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.065033:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:7.0:1619677448.065035:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff91a3aea3be00] (18)--
00000800:00000200:21.2:1619677448.065117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.065123:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.065126:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.065127:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.2:1619677448.065127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:2.0:1619677448.065129:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.065131:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.065133:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677448.065134:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.065135:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:2.0:1619677448.065136:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.065137:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.065138:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.065138:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:2.0:1619677448.065138:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.065141:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.065142:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.065144:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.065145:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:2.0:1619677448.065145:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.065146:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.065148:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.065148:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.065149:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.065149:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.065149:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:26.0:1619677448.065150:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.065151:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.065151:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.065152:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.065153:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:26.0:1619677448.065154:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.065155:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.065157:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677448.065157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.065158:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.065159:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.065160:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.065161:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.065162:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.065162:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.0:1619677448.065166:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.065169:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.065170:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.065171:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.065172:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.065172:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.065174:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:5.0:1619677448.065174:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:10.0:1619677448.065175:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.065175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.065176:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.065177:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.065179:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.0:1619677448.065183:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.065186:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.065187:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.065188:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.065189:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.065190:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:26.0:1619677448.065191:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.065194:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.065195:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.065195:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.065196:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:4.2:1619677448.065196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.065199:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.065200:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.065202:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.065204:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.065205:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.065207:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.065209:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.065210:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.065211:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.065212:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.065213:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.065216:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.065218:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.065220:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.065220:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.065221:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.065222:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.065223:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.065223:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.065225:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.065225:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.2:1619677448.065230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:4.2:1619677448.065231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.065234:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.065234:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.065237:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:30.0:1619677448.065238:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.065238:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.065238:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:20.0:1619677448.065239:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.065240:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.065240:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:5.0:1619677448.065240:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.065241:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.065241:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.065242:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:20.0:1619677448.065242:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677448.065242:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.065243:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:20.0:1619677448.065244:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.065245:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.065246:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.065247:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.065250:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.065252:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.065255:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.065256:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.065256:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.065258:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.065258:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.065260:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.065261:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.065262:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.065267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677448.065270:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.065272:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677448.065273:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.065273:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.065274:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:12.0:1619677448.065275:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.065275:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677448.065276:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.065276:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.065278:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.065424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.065432:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.065435:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.065437:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.065440:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.065440:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.065441:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.065443:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.065444:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.065445:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.065446:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.065448:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.065450:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.065452:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.065453:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.065454:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.065456:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.065456:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.065457:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.065458:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677448.065469:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.065471:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.065472:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.065474:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.065476:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.065477:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.065577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.065583:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.065586:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.065587:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.065588:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.065591:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.065592:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.065594:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.065595:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.065596:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.065597:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.065600:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.065602:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.065605:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.065605:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.065606:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.065608:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.065608:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.065609:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.065609:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.065621:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.065624:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.065625:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.065625:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.065627:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.065630:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.065647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.065653:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.065656:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.065657:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.065659:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.065661:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.065662:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.065663:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.065665:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.065666:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.065667:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:4.2:1619677448.065667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.2:1619677448.065669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:18.0:1619677448.065669:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.065671:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.065672:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.065673:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.065674:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.065675:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677448.065676:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.065676:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.065676:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.065677:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.065677:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677448.065678:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.065678:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:30.0:1619677448.065679:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.065682:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.065682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677448.065682:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.065683:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:8.0:1619677448.065684:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.065685:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.065686:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677448.065687:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:8.0:1619677448.065687:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96719822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.065688:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.065689:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.065689:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:30.0:1619677448.065690:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.065690:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.065691:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.065691:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.065691:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:30.0:1619677448.065692:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.065692:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.065693:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.065695:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000400:00000200:30.0:1619677448.065696:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.065697:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.065697:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677448.065698:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.0:1619677448.065700:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.065701:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.065701:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.065702:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.065703:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:8.0:1619677448.065703:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.065704:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677448.065704:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.065705:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677448.065705:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.065706:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677448.065707:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.065707:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.065708:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.065709:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.065713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.065716:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.065718:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.065719:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.065719:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.065721:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.065722:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:26.0:1619677448.065724:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:17.0:1619677448.065724:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.065725:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.065725:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.065728:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.065730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.065733:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.065734:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.065736:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.065737:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.065738:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.065738:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.065739:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.065742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.065749:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.065752:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.065752:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.065754:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.065756:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.065758:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.065759:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.065760:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.065761:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.065762:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.065766:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.065767:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.065770:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.065771:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.065772:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.065773:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.065774:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.065775:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.065775:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.065787:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.065790:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.065791:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.065791:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.065793:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.065796:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.065925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fe8665a00] (20)++
00000800:00000200:24.0:1619677448.065933:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8665a00] (21)++
00000800:00000200:24.0:1619677448.065936:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.5.51@o2ib3
00000800:00000200:8.0:1619677448.065936:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8665a00] (22)--
00000400:00000200:24.0:1619677448.065938:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.51@o2ib3 : REPLY - routed
00000800:00000200:24.0:1619677448.065942:0:19938:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 84734
00000800:00000200:24.0:1619677448.065945:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fe8665a00] (21)++
00000800:00000200:24.0:1619677448.065947:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8665a00] (22)++
00000800:00000200:24.0:1619677448.065948:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8665a00] (23)--
00000800:00000200:24.0:1619677448.065948:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8665a00] (22)--
00000800:00000200:5.2:1619677448.065953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fe8665a00] (21)++
00000800:00000200:8.0:1619677448.065960:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8665a00] (22)++
00000800:00000200:8.0:1619677448.065962:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8665a00] (23)--
00000800:00000200:8.0:1619677448.065963:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8665a00] (22)--
00000800:00000200:5.2:1619677448.065980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fe8665a00] (21)++
00000800:00000200:24.0:1619677448.065987:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8665a00] (22)++
00000800:00000200:24.0:1619677448.065989:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.5.51@o2ib3
00000800:00000200:8.0:1619677448.065990:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8665a00] (23)--
00000800:00000200:24.0:1619677448.065991:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fe8665a00] (23)--
00000400:00000200:24.0:1619677448.065991:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.065993:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.065994:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.065996:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.065997:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.065998:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.066001:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.51@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : REPLY try# 0
00000800:00000200:24.0:1619677448.066002:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 84734 bytes in 256 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.066004:0:19938:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 84734
00000800:00000200:24.0:1619677448.066007:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.066008:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.066008:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.066011:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.066011:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8665a00] (21)++
00000800:00000200:24.0:1619677448.066012:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8665a00] (22)--
00000800:00000200:24.0:1619677448.066012:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8665a00] (21)--
00000800:00000200:9.2:1619677448.066016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.066024:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.066027:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.066028:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:9.2:1619677448.066037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.066044:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.066046:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[2] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.066047:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.066050:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.066054:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.066054:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:9.2:1619677448.066070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677448.066073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677448.066077:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677448.066079:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677448.066080:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.066080:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.066081:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.066083:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: REPLY: OK
00000800:00000200:5.0:1619677448.066083:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.066085:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.066085:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.066086:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:5.0:1619677448.066088:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.066089:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.066091:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.066092:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.066093:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.066095:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:5.0:1619677448.066099:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.066102:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.066103:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677448.066103:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.066105:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.066106:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.0:1619677448.066107:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.066108:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.066108:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:26.0:1619677448.066109:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.066110:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.066111:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.066111:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677448.066112:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.066112:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:26.0:1619677448.066113:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.066115:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.066116:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.066117:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.066118:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.066121:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.066122:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.066123:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677448.066125:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.066125:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:26.0:1619677448.066126:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:13.0:1619677448.066126:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.066127:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.066128:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677448.066128:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.066129:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.066129:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:12.0:1619677448.066129:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.066130:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.066130:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.066140:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.066143:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.066143:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.066145:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.066147:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.066148:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.066149:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.066152:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.066152:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.066153:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.066154:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.066154:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.066172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.066180:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.066183:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.066184:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.066186:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.066188:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.066190:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.066191:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.066192:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.066193:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.066194:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.066197:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.066198:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.066201:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.066202:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.066202:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.066204:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.066204:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.066205:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.066205:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.066218:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.066221:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.066221:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.066222:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.066224:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.066227:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.066405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.066411:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.066415:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.066416:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.066418:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.066421:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.066422:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.066424:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.066425:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.066426:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.066427:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.066430:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.066432:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.066435:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.066436:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.066436:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.066438:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.066439:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.066440:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.066441:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.066442:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.2:1619677448.066443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677448.066447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.066451:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.066452:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.066453:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677448.066454:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:17.0:1619677448.066455:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:12.0:1619677448.066455:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:31.0:1619677448.066456:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.066456:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677448.066456:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.066456:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677448.066458:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.066458:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:12.0:1619677448.066458:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.066459:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:13.0:1619677448.066460:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.066460:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677448.066461:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.066462:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.066463:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.066464:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.066466:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.066468:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.066471:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.066471:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.066472:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.066473:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.066474:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.066475:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.066475:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677448.066485:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.066488:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.066488:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.066490:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.066492:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.066492:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.066529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.066532:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.066534:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.066537:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.066537:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.066540:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.066541:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.066543:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.066544:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.066545:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.066546:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.066549:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.066551:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.066554:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.066555:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.066555:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.066557:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.066558:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.066559:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.066559:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677448.066572:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.066574:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.066575:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.066576:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.066577:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.066580:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677448.066588:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.066590:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.066591:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.066591:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.066592:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.066593:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.066613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.066619:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.066622:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.066623:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.066624:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.066627:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.066629:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.066630:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.066631:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.066632:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.066633:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.066636:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.066638:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.066641:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.066642:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.066643:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.066645:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.066645:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.066646:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.066646:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677448.066658:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.066660:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.066664:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.066664:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.066666:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.066669:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.066735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.066743:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:4.2:1619677448.066744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.066746:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.066748:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.066749:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.066750:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.066751:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.066752:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.066752:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677448.066754:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.066755:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.066755:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:30.0:1619677448.066756:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:10.0:1619677448.066756:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.066758:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.066760:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.066761:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.066762:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:26.0:1619677448.066763:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.066764:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.066765:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.066765:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.066766:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:26.0:1619677448.066766:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.066767:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:26.0:1619677448.066767:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.066769:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.066769:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.066770:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677448.066770:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.066771:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.066772:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.066775:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.066775:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:9.2:1619677448.066775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:26.0:1619677448.066776:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.066778:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.066778:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.066779:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.066779:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:18.0:1619677448.066783:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.066785:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.066786:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.066786:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.066789:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.066792:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677448.066792:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:22.0:1619677448.066795:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.066795:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.066796:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.066798:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.066798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.066800:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677448.066805:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.066808:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.066809:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.066809:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.066810:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.066810:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.066815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677448.066822:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677448.066825:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677448.066827:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.066829:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677448.066830:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.066832:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677448.066834:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.066835:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.066836:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677448.066837:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677448.066841:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.066843:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677448.066845:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.066846:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:12.0:1619677448.066847:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677448.066848:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677448.066849:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677448.066850:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677448.066850:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677448.066855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.066864:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.066866:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.066867:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.066867:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.066870:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.066873:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.066885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.066891:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.066894:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.066894:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.066896:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.066899:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.066901:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.066902:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.066903:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.066904:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.066905:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.066909:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.066911:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.066914:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.066915:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.066916:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.066917:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.066918:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.066919:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.066920:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.066921:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.2:1619677448.066926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.066930:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:2.0:1619677448.066931:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:10.0:1619677448.066932:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.066933:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:2.0:1619677448.066933:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:10.0:1619677448.066934:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.066934:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.066935:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:10.0:1619677448.066936:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.066939:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.067016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.067022:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.067025:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.067025:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.067028:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.067030:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.067032:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.067033:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.067034:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.067035:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.067036:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.067039:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.067040:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.067042:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.067042:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.067043:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.067044:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.067045:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.067045:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.067046:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.067050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.067057:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.067059:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.067059:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.067061:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.067061:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.067063:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.067084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.2:1619677448.067087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.0:1619677448.067091:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.0:1619677448.067093:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677448.067094:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.067095:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677448.067096:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.067098:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:8.0:1619677448.067099:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.067099:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.067102:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.067102:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.067104:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.067104:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.067106:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.067106:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.067108:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.067108:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.067109:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.067109:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.067110:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.067111:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.067114:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.067116:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.067119:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.067120:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.067120:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.067121:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.067121:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.067123:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.067123:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.067124:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.067124:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.0:1619677448.067124:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.067124:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.2:1619677448.067127:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.2:1619677448.067128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.067130:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.067131:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.067132:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.067133:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.067133:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677448.067134:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677448.067136:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:26.0:1619677448.067137:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.067138:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.067139:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.067139:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:26.0:1619677448.067140:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.067140:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.067142:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.067143:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.067144:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:1.0:1619677448.067145:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:26.0:1619677448.067146:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.067147:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677448.067147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:26.0:1619677448.067148:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.067149:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.2:1619677448.067151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:26.0:1619677448.067153:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.067154:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.0:1619677448.067154:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677448.067154:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:20.0:1619677448.067157:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:17.0:1619677448.067157:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.067157:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.067158:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.067158:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.067158:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.067159:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:20.0:1619677448.067160:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677448.067160:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.067160:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:26.0:1619677448.067161:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.067161:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.067162:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.067162:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:26.0:1619677448.067163:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:20.0:1619677448.067163:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.067165:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:5.2:1619677448.067165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:20.0:1619677448.067166:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.067168:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.067169:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.067170:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.067171:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:20.0:1619677448.067173:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677448.067173:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.067175:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.067175:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:20.0:1619677448.067176:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.067176:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.067178:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.067179:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:20.0:1619677448.067179:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.067179:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.067179:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.067180:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677448.067180:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:1.0:1619677448.067181:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:20.0:1619677448.067182:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677448.067182:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:6.0:1619677448.067182:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.067182:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96719802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.067183:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.067184:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:1.0:1619677448.067184:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.067185:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:1.0:1619677448.067185:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.067186:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:9.2:1619677448.067188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:1.0:1619677448.067189:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.067191:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.067193:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.067193:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.067194:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:1.0:1619677448.067194:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.067196:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.067196:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.067197:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.067197:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.067197:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677448.067198:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.067198:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677448.067200:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677448.067203:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.2:1619677448.067212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.067220:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.067222:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.067223:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.067223:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.067225:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.067226:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.067256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.067264:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.067266:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.067269:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.067270:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.067272:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.067274:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677448.067275:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.067276:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.067277:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.067278:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.067282:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.067284:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.067287:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.067287:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.067288:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.067290:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.067290:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.067291:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.067292:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.067297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.067305:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.067308:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.067309:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.067309:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.067311:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.067314:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.067321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.067329:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.067332:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.067333:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.067334:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.067337:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.067339:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.067340:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.067341:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.067342:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.067343:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.067346:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.067348:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.067351:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.067351:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.067352:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.067354:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.067355:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.067356:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.067357:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.067358:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677448.067361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677448.067363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.067370:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:18.0:1619677448.067370:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.067372:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.067373:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677448.067373:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.067374:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:18.0:1619677448.067374:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.067374:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.067376:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.067380:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.067405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.067411:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.067413:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.067414:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.067416:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.067418:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.067419:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.067420:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.067421:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.067422:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.067423:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.067426:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.067428:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.067430:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.067430:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.067431:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.067433:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.067433:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.067434:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.067435:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.067436:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677448.067438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.067441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.067447:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.067447:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.067448:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.067448:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.067448:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.067449:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.067450:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.067450:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.067451:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.067451:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.067457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.067465:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.067467:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.067468:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.067468:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.067468:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.067469:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.067733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.067740:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.067743:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.067743:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.067745:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.067748:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.067750:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.067751:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.067753:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.067753:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.067755:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.067758:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.067759:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.067762:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.067763:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.067764:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.067766:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.067766:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.067767:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.067768:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.067772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.067777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677448.067779:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.067782:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.067783:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.067783:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.067785:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.067786:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.2:1619677448.067788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:2.0:1619677448.067788:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677448.067789:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.2:1619677448.067789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:30.0:1619677448.067791:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.067793:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.067794:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.067795:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:30.0:1619677448.067796:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:26.0:1619677448.067796:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:30.0:1619677448.067797:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.067797:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.067798:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:30.0:1619677448.067799:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.067799:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:30.0:1619677448.067800:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.067800:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:8.0:1619677448.067801:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.067802:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.067802:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.067804:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.067805:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.067805:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.067805:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:30.0:1619677448.067806:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.067807:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:21.2:1619677448.067807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:8.0:1619677448.067807:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96719796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.067808:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.067809:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677448.067809:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.067810:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:26.0:1619677448.067810:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.067810:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.067811:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:26.0:1619677448.067811:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.067811:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.067812:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:26.0:1619677448.067812:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:30.0:1619677448.067813:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677448.067813:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677448.067814:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677448.067815:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.067816:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.067816:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677448.067816:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:26.0:1619677448.067817:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.067817:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677448.067818:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.067819:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:22.0:1619677448.067820:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.067820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.0:1619677448.067821:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.067822:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677448.067822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677448.067822:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.067823:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.067823:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:22.0:1619677448.067823:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.067823:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.067825:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:22.0:1619677448.067825:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:8.0:1619677448.067825:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.067825:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:26.0:1619677448.067826:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:22.0:1619677448.067826:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96719794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.067826:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.067826:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:26.0:1619677448.067827:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.067827:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.0:1619677448.067827:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677448.067827:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:22.0:1619677448.067828:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.067829:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:6.0:1619677448.067829:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677448.067830:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:22.0:1619677448.067830:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677448.067830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677448.067830:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.067830:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677448.067831:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.067833:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:17.0:1619677448.067833:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.067833:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.067834:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.067834:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:17.0:1619677448.067834:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.067836:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.067836:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:6.0:1619677448.067837:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.067838:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:17.0:1619677448.067838:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96719793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.067838:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:17.0:1619677448.067839:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.067840:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:17.0:1619677448.067840:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.067841:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:17.0:1619677448.067841:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677448.067842:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.067842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677448.067842:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677448.067843:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.067844:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.067845:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:18.0:1619677448.067845:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.067846:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.067847:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:17.0:1619677448.067847:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.067848:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677448.067848:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677448.067849:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677448.067850:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.067852:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:8.2:1619677448.067853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677448.067854:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.067854:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.067855:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.067855:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:1.0:1619677448.067855:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.067857:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:1.0:1619677448.067857:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.067859:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.067859:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.067860:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677448.067861:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.067861:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.067862:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:17.0:1619677448.067864:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.067864:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:17.0:1619677448.067865:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000400:00000200:5.0:1619677448.067865:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.067865:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.067866:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.067866:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.0:1619677448.067867:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.067867:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.067870:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:13.0:1619677448.067871:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.067874:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.067874:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.067876:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.067878:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.067880:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.067888:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.067891:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.067892:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.067893:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.067893:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.067895:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.068028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.068035:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.068038:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.068038:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.068040:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.068043:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.068045:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.068046:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.068047:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.068048:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.068049:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.068053:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.068054:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.068057:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.068057:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.068058:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.068060:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.068060:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.068061:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.068062:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.068063:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.068068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677448.068072:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.068074:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.068075:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.068076:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:13.0:1619677448.068076:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.068077:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.068078:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:13.0:1619677448.068078:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.068078:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.068080:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:4.2:1619677448.068142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:5.2:1619677448.068143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:4.2:1619677448.068146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677448.068148:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:24.0:1619677448.068149:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.068152:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677448.068152:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:24.0:1619677448.068153:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:18.0:1619677448.068153:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:10.0:1619677448.068153:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.068156:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.068156:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:2.0:1619677448.068156:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.068157:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.068157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:2.0:1619677448.068158:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.068159:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.068160:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.068160:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.068161:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.068161:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.068162:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.068163:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.068163:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.068163:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.068164:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:2.0:1619677448.068164:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.068165:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.068165:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.068166:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.068167:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.068168:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:24.0:1619677448.068168:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.068169:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.068170:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.068171:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:10.0:1619677448.068171:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.068171:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.068172:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.068174:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.068174:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:31.0:1619677448.068175:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.068177:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:24.0:1619677448.068177:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:10.0:1619677448.068177:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.068179:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96719788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.068179:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.068180:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.068181:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.068181:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.068181:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.068182:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.068182:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.068182:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:31.0:1619677448.068183:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:24.0:1619677448.068183:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:10.0:1619677448.068183:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.068183:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.068184:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677448.068184:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677448.068186:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677448.068186:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (23)--
00000400:00000200:31.0:1619677448.068187:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.068187:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:10.0:1619677448.068187:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.068187:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:24.0:1619677448.068188:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677448.068188:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677448.068189:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.068189:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (23)--
00000800:00000200:8.2:1619677448.068189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.068189:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:10.0:1619677448.068190:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.068190:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.068191:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.068192:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.068192:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:9.2:1619677448.068192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677448.068193:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:4.2:1619677448.068193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677448.068195:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.068195:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.068196:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:21.2:1619677448.068197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677448.068197:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.068198:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.068200:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:18.0:1619677448.068200:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:1.0:1619677448.068200:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:31.0:1619677448.068201:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.068201:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.2:1619677448.068201:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:1.0:1619677448.068201:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.068203:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000800:00000200:31.0:1619677448.068204:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:18.0:1619677448.068204:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.068204:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (24)++
00000400:00000200:1.0:1619677448.068204:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.068205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:8.0:1619677448.068205:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677448.068205:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000400:00000200:2.0:1619677448.068205:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.068206:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677448.068207:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.068207:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:31.0:1619677448.068208:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.068208:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:8.2:1619677448.068208:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000400:00000200:2.0:1619677448.068208:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.068209:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (23)--
00000800:00000200:30.0:1619677448.068210:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:18.0:1619677448.068210:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:31.0:1619677448.068211:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.068211:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677448.068211:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677448.068211:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:31.0:1619677448.068212:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:30.0:1619677448.068212:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.0:1619677448.068212:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677448.068212:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:31.0:1619677448.068213:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.068213:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677448.068214:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:9.2:1619677448.068214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:31.0:1619677448.068215:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.068215:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.068216:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.068216:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.068216:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677448.068217:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.068218:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:17.0:1619677448.068219:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.068220:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:17.0:1619677448.068220:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677448.068220:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.068220:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:1.0:1619677448.068220:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:31.0:1619677448.068221:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.068221:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:17.0:1619677448.068221:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.068221:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:18.0:1619677448.068222:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.068222:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677448.068222:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677448.068223:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.2:1619677448.068224:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677448.068225:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.068225:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.068226:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.068226:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677448.068227:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.068227:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.068227:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677448.068228:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.068229:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.068230:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.068230:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.068231:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.068231:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677448.068233:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.068234:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.068235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:1.0:1619677448.068236:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.068239:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.068241:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:17.0:1619677448.068243:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:1.0:1619677448.068243:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.068244:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.068245:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.068245:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.068246:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.068247:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.068248:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.068250:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.068252:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.068254:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.068255:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677448.068255:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.068256:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.068257:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.068259:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.068259:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.068260:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.068260:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.068264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.068271:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.068273:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.068274:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.068275:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.068276:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.068277:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.068497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fdae00] (20)++
00000800:00000200:20.0:1619677448.068502:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fdae00] (21)++
00000800:00000200:20.0:1619677448.068505:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.38@o2ib3
00000400:00000200:20.0:1619677448.068507:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.38@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.068508:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (22)--
00000400:00000200:20.0:1619677448.068510:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.068511:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.068513:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.068514:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.068515:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.068516:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.068519:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.38@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.068521:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.068523:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.068524:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.068524:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.068526:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.068527:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fdae00] (21)++
00000800:00000200:20.0:1619677448.068528:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fdae00] (22)++
00000800:00000200:20.0:1619677448.068529:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fdae00] (23)--
00000800:00000200:20.0:1619677448.068530:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (22)--
00000800:00000200:21.2:1619677448.068535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fdae00] (21)++
00000800:00000200:5.0:1619677448.068539:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fdae00] (22)++
00000800:00000200:6.0:1619677448.068541:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.068541:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fdae00] (23)--
00000800:00000200:5.0:1619677448.068542:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (22)--
00000800:00000200:6.0:1619677448.068543:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677448.068543:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (21)--
00000400:00000200:6.0:1619677448.068544:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.068545:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.068547:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.068549:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.068587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.068593:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.068596:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.068596:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.068598:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.068600:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.068601:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.068603:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.068605:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.068606:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.068607:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.068610:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.068612:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.068613:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.068614:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.068615:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.068616:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.068617:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.068618:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.068619:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.068620:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.068623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.068626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.068631:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.068632:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.068633:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.068633:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.068633:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.068634:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.068635:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.068635:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.2:1619677448.068636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677448.068636:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677448.068636:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677448.068641:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.068644:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.068646:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.068647:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.068649:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.068650:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.068652:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719783] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.068653:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.068654:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.068655:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.068658:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.068660:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:21.2:1619677448.068664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:5.0:1619677448.068665:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.068666:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.068667:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.068670:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.068670:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.068671:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:2.0:1619677448.068671:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677448.068672:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.068673:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677448.068674:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.068675:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.068677:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.068678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:2.0:1619677448.068679:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.068680:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:20.0:1619677448.068681:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:2.0:1619677448.068682:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.068683:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:12.0:1619677448.068683:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:2.0:1619677448.068683:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.068684:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:2.0:1619677448.068684:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.068685:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.068685:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:2.0:1619677448.068685:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677448.068686:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.068689:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.068690:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.068690:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.068692:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677448.068693:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.068694:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.068695:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.068695:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.068697:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.068698:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.068699:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.068699:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677448.068701:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:9.2:1619677448.068703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677448.068704:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.068704:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.068705:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.068706:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.068707:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677448.068710:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.068713:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.068714:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.068714:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.068716:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.068718:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.068747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.068753:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.068756:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.068756:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.068758:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.068760:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.068762:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.068763:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.068765:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677448.068765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:22.0:1619677448.068766:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.068767:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.068770:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.068772:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:26.0:1619677448.068773:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.068774:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.068775:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:22.0:1619677448.068775:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.068776:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:10.0:1619677448.068776:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.068778:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.068778:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.068778:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.068780:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:26.0:1619677448.068781:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.068781:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.068781:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:26.0:1619677448.068782:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.068784:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.068784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:26.0:1619677448.068785:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.068786:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.068786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:26.0:1619677448.068787:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.068790:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.068791:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677448.068791:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:26.0:1619677448.068792:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677448.068793:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:2.0:1619677448.068793:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:26.0:1619677448.068794:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.068794:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:6.0:1619677448.068794:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677448.068794:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.068795:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:22.0:1619677448.068795:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677448.068795:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:26.0:1619677448.068796:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:2.0:1619677448.068796:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.068797:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.068798:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.068798:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677448.068800:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:10.0:1619677448.068808:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.068810:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.068811:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.068811:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.068813:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.068815:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.068847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.068853:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.068856:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.068856:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.068858:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.068860:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.068862:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677448.068863:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.068864:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.068865:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.068866:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677448.068869:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.068871:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.068873:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.068873:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.068874:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.068875:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.068876:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.068877:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.068878:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.068878:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:9.2:1619677448.068881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.068884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.068889:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.068890:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.068891:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.068891:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.068891:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.068892:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.068893:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.068893:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.068894:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677448.068895:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.068914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.068923:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.068926:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.068928:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.068930:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.068930:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.068932:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.068933:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.068935:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.068936:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.068937:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.068940:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.068942:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.068944:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.068945:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.068945:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.068947:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.068947:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:5.2:1619677448.068947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677448.068948:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.068949:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.068954:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.068957:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.068958:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.068959:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:26.0:1619677448.068959:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.068961:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:26.0:1619677448.068961:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.068962:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.068963:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.068964:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.068964:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719777] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.068965:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.068966:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:26.0:1619677448.068966:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.068967:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.068968:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.068971:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.068973:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.068975:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.068976:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.068976:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.068977:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.068978:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.068978:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.068979:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.068979:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.068980:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:26.0:1619677448.068980:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.068981:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.068981:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.068982:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:12.0:1619677448.068990:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.068992:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.068993:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.068995:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.068996:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.068997:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.069027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.069033:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.069036:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.069037:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.069039:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.069040:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.069042:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.069043:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.069044:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.069045:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.069046:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.069049:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.069051:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.069053:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.069054:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.069054:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.069055:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.069056:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.069056:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.069057:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.069061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.069068:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.069070:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.069071:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.069072:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.069072:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.069073:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.069240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.069246:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.069248:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.069249:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.069251:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.069254:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.069255:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.069257:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719775] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.069258:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.069259:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.069260:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.069263:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.069265:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.069268:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.069269:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.069270:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.069271:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.069272:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.069272:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.069273:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.069277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.069285:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.069288:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.069289:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.069289:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.069291:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.069298:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.069302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:21.2:1619677448.069305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.069311:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.069313:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:30.0:1619677448.069313:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.069314:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.069316:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.069317:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.069318:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.069319:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.069320:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.069322:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.069323:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.069324:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.069325:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.069328:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.069329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:31.0:1619677448.069330:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.069330:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.069331:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719773] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.069332:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:30.0:1619677448.069333:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.069333:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.069334:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:30.0:1619677448.069335:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.069337:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.069337:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677448.069338:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677448.069338:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677448.069339:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:30.0:1619677448.069339:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.069340:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.069341:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.069341:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.069341:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.069342:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000400:00000200:20.0:1619677448.069342:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:31.0:1619677448.069343:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.069343:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.069344:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.069344:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:20.0:1619677448.069344:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96719772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.069345:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.069345:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.069345:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.069346:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677448.069346:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.069347:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.069347:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677448.069347:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.069349:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:20.0:1619677448.069350:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:30.0:1619677448.069351:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.069352:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:20.0:1619677448.069352:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:30.0:1619677448.069353:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.069354:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677448.069355:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.069355:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.069355:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.069356:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.069358:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.0:1619677448.069358:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.069358:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.069359:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:13.0:1619677448.069359:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.069360:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:13.0:1619677448.069362:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.069362:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.069364:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.069364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677448.069366:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.069367:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.069373:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.069376:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.069377:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.069377:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.069379:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677448.069381:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.069384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.069391:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.069393:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.069394:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:21.2:1619677448.069395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:17.0:1619677448.069395:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.069395:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.069396:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:24.0:1619677448.069402:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.069405:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.069406:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.069408:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.069411:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.069412:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.069413:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.069415:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.069416:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.069417:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677448.069420:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.069422:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.069425:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.069425:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.069426:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.069428:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.069428:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.069429:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.069431:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.069431:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.2:1619677448.069434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.069437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677448.069443:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.0:1619677448.069444:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:5.0:1619677448.069445:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:8.0:1619677448.069446:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:5.0:1619677448.069446:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.069447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.0:1619677448.069447:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.069448:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:5.0:1619677448.069448:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.069451:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.069451:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677448.069452:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.069455:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.069457:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.069458:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.069460:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.069462:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.069463:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.069464:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.069465:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.069466:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.069470:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.069472:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.069475:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.069476:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.069477:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.069479:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.069479:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.069480:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.069481:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677448.069484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.069491:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677448.069492:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.069494:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:24.0:1619677448.069494:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:31.0:1619677448.069495:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.069495:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:31.0:1619677448.069498:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.069498:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:24.0:1619677448.069498:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.069500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677448.069501:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:24.0:1619677448.069501:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.069503:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.069504:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.069505:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.069506:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.069506:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:24.0:1619677448.069507:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.069508:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.069509:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677448.069511:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.069512:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.069514:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.069515:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.069516:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.069516:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.069517:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:1.0:1619677448.069517:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.069518:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.069519:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.069519:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:1.0:1619677448.069519:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.069520:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677448.069520:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.069521:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.069521:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:8.2:1619677448.069525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:1.0:1619677448.069525:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.069526:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.069530:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.069531:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.069531:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.069533:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.069534:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.069534:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.069535:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.069535:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.069536:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.069536:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.069537:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.069538:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.069541:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:22.0:1619677448.069547:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.069549:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.069551:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.069551:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.069553:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.069555:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.069561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.069569:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.069572:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.069572:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.069574:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.069577:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.069578:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.069580:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.069581:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.069582:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.069583:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.069587:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.069588:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.069591:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.069592:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.069593:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.069594:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.069595:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.069596:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.069597:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.069598:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.069603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.069608:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.069610:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.069611:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.069611:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.069612:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677448.069612:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.069613:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.069614:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.069614:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.069617:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677448.069628:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.069630:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.069631:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.069631:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.069632:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.069633:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.069688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.069696:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.069699:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:5.2:1619677448.069700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:30.0:1619677448.069701:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.069702:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.069703:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.069704:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.069706:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.069707:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.069708:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.069708:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:30.0:1619677448.069709:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.069711:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.069712:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:30.0:1619677448.069713:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.069714:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.069715:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.069716:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.069716:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.069717:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.069718:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:10.0:1619677448.069718:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:30.0:1619677448.069719:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:10.0:1619677448.069719:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.069720:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677448.069720:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.069721:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.069721:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:10.0:1619677448.069721:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.069722:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.069724:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.069726:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.069727:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.069728:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.069729:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.069730:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.069730:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.069731:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.069731:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677448.069732:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.069734:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.069735:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.069736:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.069738:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.069739:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:26.0:1619677448.069741:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.069743:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.069744:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.069745:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.069746:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.069747:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.069760:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.069762:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.069763:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.069763:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.069764:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.069764:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.069815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.069822:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.069825:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.069826:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.069828:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.069828:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.069830:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.069831:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.069832:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.069833:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.069834:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.069837:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.069838:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.069841:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.069842:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.069842:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.069844:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.069845:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.069846:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.069846:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677448.069848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.069855:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.069857:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.069858:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:6.0:1619677448.069859:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.069860:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.069861:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.069862:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.069863:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.069864:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.069865:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719763] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.069866:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.069867:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.069868:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.069872:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.069872:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.069872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:22.0:1619677448.069873:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.069874:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677448.069874:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.069877:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.069878:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.069878:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677448.069879:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:6.0:1619677448.069880:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.069881:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.069881:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:13.0:1619677448.069882:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:6.0:1619677448.069882:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:13.0:1619677448.069884:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.069885:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:8.2:1619677448.069886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:13.0:1619677448.069888:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.069889:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:13.0:1619677448.069890:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677448.069892:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.069893:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.069894:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:13.0:1619677448.069895:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:1.0:1619677448.069897:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.069898:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:13.0:1619677448.069898:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.069898:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.069900:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:1.0:1619677448.069901:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.069903:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.069903:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:13.0:1619677448.069904:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:13.0:1619677448.069904:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:13.0:1619677448.069906:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:13.0:1619677448.069907:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677448.069909:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677448.069910:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677448.069911:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:5.2:1619677448.069916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:26.0:1619677448.069918:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.069920:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.069921:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.069921:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.069923:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.069924:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:26.0:1619677448.069925:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:12.0:1619677448.069927:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677448.069927:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677448.069928:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677448.069928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.069934:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.069936:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.069937:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.069939:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.069941:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.069943:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.069944:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.069945:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.069946:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.069947:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.069950:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.069952:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.069954:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.069955:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.069956:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.069957:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.069958:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.069959:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.069959:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.069971:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.069973:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.069975:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.069975:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.069977:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.069979:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677448.069994:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.069997:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.069998:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.069998:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.069999:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.070000:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.070008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.070013:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.070016:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.070018:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.070019:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.070021:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.070023:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.070025:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719760] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.070026:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.070027:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.070028:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.070032:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.070034:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.070037:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.070038:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.070039:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.070040:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.070041:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.070042:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.070042:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677448.070053:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.070056:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.070057:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.070059:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.070059:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.070061:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.070105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.070112:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.070114:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.070116:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.070117:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.070119:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.070120:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.070122:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.070123:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.070124:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.070125:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.070129:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.070131:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.070134:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.070135:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.070136:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.070138:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.070138:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.070139:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.070139:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677448.070150:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.070153:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.070154:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.070156:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.070157:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.070159:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.070194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.070201:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.070204:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.070206:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.070208:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.070209:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.070210:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.070211:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.070213:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.070214:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.070215:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.070218:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.070220:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.070223:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.070224:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.070224:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.070226:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.070226:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.070228:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.070229:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.070230:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:4.2:1619677448.070233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677448.070236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677448.070240:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.070240:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.070243:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677448.070243:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:31.0:1619677448.070244:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:26.0:1619677448.070244:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.070244:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.070244:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:10.0:1619677448.070245:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.070246:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.070247:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677448.070247:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.070248:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:10.0:1619677448.070248:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.070249:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:10.0:1619677448.070249:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.070251:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.070252:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.070253:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.070254:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.070257:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.070259:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.070262:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.070263:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.070263:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.070265:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.070265:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.070266:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.070267:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.070278:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.070281:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.070282:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.070282:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.070285:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.070287:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.070290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.070296:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.070299:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.070300:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.070302:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.070304:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.070305:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.070307:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.070308:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.070309:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.070310:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.070314:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.070316:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.070319:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.070320:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.070321:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.070323:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.070323:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.070324:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.070324:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677448.070337:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.070339:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.070340:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.070342:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.070344:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.070345:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.070353:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.070357:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.070358:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.070358:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.070359:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.070360:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.070394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.070401:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.070404:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.070404:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.070406:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.070409:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.070411:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.070412:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.070413:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.070414:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.070415:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.070418:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.070420:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.070423:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.070424:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.070424:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.070426:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.070427:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.070428:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.070428:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.070429:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.070432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.070434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.070439:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677448.070439:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677448.070441:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.070442:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677448.070442:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:1.0:1619677448.070442:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677448.070443:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.070444:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.070446:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.070449:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.2:1619677448.070474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.070482:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.070484:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.070485:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.070487:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.070490:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.070491:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.070493:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.070494:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.070495:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.070496:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.070499:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.070501:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.070502:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.070503:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.070504:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.070505:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.070505:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.070506:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.070506:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.070517:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.070518:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.070519:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.070520:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.070521:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.070522:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.070629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.070635:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.070638:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.070639:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.070640:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.070641:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.070642:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.070643:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719753] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.070645:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.070645:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.070646:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.070649:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.070650:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.070653:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.070654:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.070654:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.070656:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.070657:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.070657:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.070657:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.070670:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.070673:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.070673:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.070674:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.070676:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.070679:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677448.070688:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.070690:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.070691:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.070692:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.070692:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.070693:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.070767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.070773:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.070776:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.070777:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.070779:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.070782:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.070784:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.070785:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.070786:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.070788:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.070789:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:5.2:1619677448.070789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:18.0:1619677448.070792:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.070794:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677448.070795:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.070797:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.070798:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:17.0:1619677448.070798:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.070798:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.070799:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.070801:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.070801:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:17.0:1619677448.070801:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.070802:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.070802:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:17.0:1619677448.070803:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.070805:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.070806:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.070807:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.070808:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.070809:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:10.0:1619677448.070812:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:17.0:1619677448.070813:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.070814:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677448.070815:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.070816:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.070816:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.070817:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.070818:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.070819:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:10.0:1619677448.070819:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.070821:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.070821:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.070822:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.070822:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.070822:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:9.2:1619677448.070826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.070833:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.070835:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.070836:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.070836:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677448.070838:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.070841:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.2:1619677448.070938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:21.2:1619677448.070941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.070946:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.070948:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:5.2:1619677448.070949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f772e00] (20)++
00000400:00000200:31.0:1619677448.070950:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.070950:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.070951:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.070953:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.070954:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.070954:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.070956:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.070957:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.070957:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f772e00] (21)++
00000400:00000200:31.0:1619677448.070958:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.070959:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.070960:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.12.18@o2ib3
00000400:00000200:30.0:1619677448.070961:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.070961:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f772e00] (22)--
00000400:00000200:31.0:1619677448.070963:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.070963:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677448.070964:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.070964:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.18@o2ib3 : REPLY - routed
00000800:00000200:31.0:1619677448.070965:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.070966:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.070966:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.070967:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.070967:0:19941:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 598016
00000800:00000200:31.0:1619677448.070970:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.070971:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:30.0:1619677448.070971:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.070971:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f772e00] (21)++
00000800:00000200:31.0:1619677448.070972:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.070972:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677448.070973:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.070973:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f772e00] (22)++
00000800:00000200:31.0:1619677448.070974:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (22)++
00000800:00000200:26.0:1619677448.070974:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f772e00] (23)--
00000800:00000200:26.0:1619677448.070974:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f772e00] (22)--
00000800:00000200:31.0:1619677448.070975:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (23)++
00000800:00000200:30.0:1619677448.070975:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.070976:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677448.070977:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (24)--
00000800:00000200:31.0:1619677448.070977:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.070977:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.070978:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.070978:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677448.070979:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.070980:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.070980:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.070980:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677448.070980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f772e00] (21)++
00000800:00000200:20.2:1619677448.070983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677448.070986:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:10.0:1619677448.070986:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f772e00] (22)++
00000800:00000200:13.0:1619677448.070989:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.070989:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.070989:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f772e00] (23)--
00000800:00000200:10.0:1619677448.070990:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f772e00] (22)--
00000800:00000200:31.0:1619677448.070991:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:13.0:1619677448.070991:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.070992:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:31.0:1619677448.070993:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:13.0:1619677448.070993:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.070994:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:12.0:1619677448.070994:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.070995:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:22.0:1619677448.070995:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.070995:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.070996:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.070998:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.071001:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.071067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.071073:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.2:1619677448.071074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f772e00] (21)++
00000800:00000200:2.0:1619677448.071076:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.071077:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.071079:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.071081:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f772e00] (22)++
00000400:00000200:2.0:1619677448.071082:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.071084:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.12.18@o2ib3
00000800:00000200:10.0:1619677448.071084:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f772e00] (23)--
00000400:00000200:2.0:1619677448.071084:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.071085:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.071086:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f772e00] (22)--
00000400:00000200:26.0:1619677448.071086:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.071086:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.071088:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.071088:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.071089:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719747] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.071089:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.071092:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.071093:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.071093:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.071094:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.071096:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.071098:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.071099:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : REPLY try# 0
00000800:00000200:2.0:1619677448.071099:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.071100:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.071101:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.071102:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 598016 bytes in 256 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.071102:0:19941:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 598016
00000800:00000200:2.0:1619677448.071102:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.071103:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.071103:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677448.071106:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.071107:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.071107:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:9.2:1619677448.071108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.071109:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.071110:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f772e00] (21)++
00000800:00000200:26.0:1619677448.071110:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f772e00] (22)--
00000800:00000200:26.0:1619677448.071111:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f772e00] (21)--
00000800:00000200:18.0:1619677448.071115:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.071118:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.071118:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.071119:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.071120:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.071120:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:18.0:1619677448.071122:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.071123:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.071124:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.071148:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.071151:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[3] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.071152:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.071155:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.071164:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.071164:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:4.2:1619677448.071225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.071230:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.071232:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.071233:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:10.0:1619677448.071233:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.071234:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.071235:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: REPLY: OK
00000400:00000200:26.0:1619677448.071237:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.071237:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.071238:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.071239:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.071240:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.071241:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.071243:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.071244:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.071245:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.071245:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.071248:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.071250:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.071252:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.071252:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.071253:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.071254:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.071255:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.071256:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.071256:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.071267:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.071269:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.071270:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.071271:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.071274:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.071275:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.071352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677448.071358:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677448.071361:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677448.071362:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677448.071364:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.071367:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.071368:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.071370:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.071371:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.071372:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.071373:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.071376:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.071378:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.071382:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.071382:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.071383:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.071385:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.071385:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677448.071386:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677448.071386:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:13.0:1619677448.071399:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.071402:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.071403:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.071404:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.071406:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.071407:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.071525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.071532:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.071535:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.071536:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.071538:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.071541:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.071542:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.071543:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.071544:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.071545:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.071547:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.071550:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.071552:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.071555:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.071556:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.071556:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.071558:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.071559:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.071559:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.071560:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.071564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.071572:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.071575:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.071575:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.071576:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.071578:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.071581:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.071584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.071587:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.071589:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.071591:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.071593:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.071594:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.071596:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.071597:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.071598:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.071601:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4cc00] (20)++
00000400:00000200:20.0:1619677448.071604:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.071605:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.071607:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4cc00] (21)++
00000400:00000200:20.0:1619677448.071608:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.071610:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.071611:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.1.31@o2ib3
00000800:00000200:6.0:1619677448.071611:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4cc00] (22)--
00000800:00000200:20.0:1619677448.071612:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.071613:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.071613:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:22.0:1619677448.071614:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.31@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.071615:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.071616:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:22.0:1619677448.071617:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.071617:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.071617:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:22.0:1619677448.071619:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:22.0:1619677448.071620:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.071621:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.071622:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.071623:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:22.0:1619677448.071626:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.31@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.071628:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.071632:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.071632:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:22.0:1619677448.071633:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677448.071636:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677448.071637:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4cc00] (21)++
00000800:00000200:22.0:1619677448.071638:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4cc00] (22)--
00000800:00000200:22.0:1619677448.071638:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4cc00] (21)--
00000800:00000200:22.0:1619677448.071640:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.071642:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.071643:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.071643:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.071645:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.071647:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677448.071648:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.071651:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.071652:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.071654:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677448.071654:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.071656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4cc00] (20)++
00000800:00000200:12.0:1619677448.071657:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677448.071661:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4cc00] (21)++
00000800:00000200:6.0:1619677448.071664:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.31@o2ib3
00000800:00000200:22.0:1619677448.071665:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4cc00] (22)--
00000800:00000200:4.2:1619677448.071665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:6.0:1619677448.071666:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.31@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.071669:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.071670:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.071672:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.071673:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677448.071673:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.071674:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.071675:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:6.0:1619677448.071675:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.071676:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.071679:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.071680:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.31@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.071682:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:10.0:1619677448.071683:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.071686:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:6.0:1619677448.071686:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.071687:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.071687:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.071687:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:10.0:1619677448.071689:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.071689:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.071689:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4cc00] (21)++
00000400:00000200:10.0:1619677448.071690:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.071690:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4cc00] (22)--
00000400:00000200:10.0:1619677448.071691:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.071691:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4cc00] (21)--
00000400:00000200:10.0:1619677448.071694:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.071696:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.071700:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.071701:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.071702:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.071702:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.071703:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.071704:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.071704:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.071704:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:26.0:1619677448.071706:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.071706:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:20.2:1619677448.071707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4cc00] (20)++
00000800:00000200:10.0:1619677448.071707:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.071708:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.071709:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:10.0:1619677448.071709:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:4.2:1619677448.071713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.071714:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4cc00] (21)++
00000800:00000200:22.0:1619677448.071716:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.31@o2ib3
00000400:00000200:22.0:1619677448.071719:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.31@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.071719:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.071720:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677448.071721:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:6.0:1619677448.071722:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.071723:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.071723:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.071723:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.071724:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.071724:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.071725:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:22.0:1619677448.071726:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.071727:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.071729:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.071730:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:6.0:1619677448.071733:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.071734:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.31@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.071734:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4cc00] (22)--
00000800:00000200:6.0:1619677448.071735:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677448.071736:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:22.0:1619677448.071739:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.071740:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:22.0:1619677448.071740:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:22.0:1619677448.071742:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:22.0:1619677448.071743:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4cc00] (21)++
00000800:00000200:22.0:1619677448.071744:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4cc00] (22)--
00000800:00000200:22.0:1619677448.071744:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4cc00] (21)--
00000800:00000200:9.2:1619677448.071748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.071754:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.071757:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.071758:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.071758:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677448.071761:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.071764:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.2:1619677448.071793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4cc00] (20)++
00000800:00000200:6.0:1619677448.071798:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4cc00] (21)++
00000800:00000200:22.0:1619677448.071802:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4cc00] (22)--
00000800:00000200:6.0:1619677448.071802:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.31@o2ib3
00000400:00000200:6.0:1619677448.071804:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.31@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.071804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:6.0:1619677448.071807:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.071808:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:6.0:1619677448.071810:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.071811:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.071812:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:6.0:1619677448.071812:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.071813:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.071815:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.071815:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677448.071817:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.31@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.071818:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.071820:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.071821:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.071822:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:6.0:1619677448.071822:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:26.0:1619677448.071823:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.071824:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:6.0:1619677448.071824:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4cc00] (21)++
00000400:00000200:26.0:1619677448.071825:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.071826:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719737] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.071826:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4cc00] (22)++
00000400:00000200:26.0:1619677448.071827:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.071827:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4cc00] (23)--
00000400:00000200:26.0:1619677448.071828:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.071828:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4cc00] (22)--
00000400:00000200:26.0:1619677448.071829:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.071833:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.071835:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.071837:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.071838:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.071839:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.071839:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.071840:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.071841:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.2:1619677448.071841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4cc00] (21)++
00000400:00000200:10.0:1619677448.071841:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.071843:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.071843:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.071844:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.071844:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.2:1619677448.071845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.071846:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:22.0:1619677448.071846:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4cc00] (22)++
00000800:00000200:10.0:1619677448.071846:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.071847:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677448.071848:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4cc00] (23)--
00000800:00000200:10.0:1619677448.071848:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.071850:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.071850:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:22.0:1619677448.071851:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4cc00] (22)--
00000400:00000200:10.0:1619677448.071852:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.071852:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4cc00] (21)--
00000800:00000200:5.2:1619677448.071852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677448.071853:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:10.0:1619677448.071855:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.071856:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.071856:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.071857:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.071857:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.071858:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.071859:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.071860:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:10.0:1619677448.071860:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.071862:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.071862:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.071862:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.071863:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:10.0:1619677448.071863:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677448.071865:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:10.0:1619677448.071868:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.071869:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.071873:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.071873:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.071874:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.071876:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.071877:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.071877:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.071878:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.2:1619677448.071881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:9.2:1619677448.071883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.071889:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.071891:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:30.0:1619677448.071892:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.071894:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.071894:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.071895:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.071895:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.071897:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.071897:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.071898:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.071899:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.071900:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.071901:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.071902:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.071902:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:30.0:1619677448.071903:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.071907:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.071908:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.071911:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.071912:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.071912:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.071914:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.071914:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.071915:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.071916:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.071926:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.071929:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.071929:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.071931:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.071932:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.071933:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.071933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.071939:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.071942:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.071942:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:12.0:1619677448.071945:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677448.071945:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.071947:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.071948:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.071948:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.071949:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.071949:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.2:1619677448.071949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:1.0:1619677448.071949:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:12.0:1619677448.071950:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677448.071950:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.071952:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.071953:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.071954:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.071955:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:1.0:1619677448.071957:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.071958:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.071958:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677448.071959:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.071961:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.071963:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.071963:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.071963:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:18.0:1619677448.071964:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:1.0:1619677448.071964:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677448.071965:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.071966:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.071966:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677448.071967:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.071967:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.071967:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:18.0:1619677448.071968:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.071969:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.2:1619677448.071972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4cc00] (20)++
00000400:00000200:18.0:1619677448.071972:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.071974:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.071976:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.071977:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.071977:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4cc00] (21)++
00000800:00000200:18.0:1619677448.071978:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.071979:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677448.071979:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.071980:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.071980:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:6.0:1619677448.071980:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.31@o2ib3
00000800:00000200:22.0:1619677448.071981:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4cc00] (22)--
00000800:00000200:18.0:1619677448.071981:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677448.071981:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.071982:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.071983:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.31@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677448.071984:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.071985:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:6.0:1619677448.071985:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.071986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677448.071987:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:6.0:1619677448.071987:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:6.0:1619677448.071988:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.071989:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.071990:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.071991:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.071994:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:6.0:1619677448.071995:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.31@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.071996:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:6.0:1619677448.071996:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:17.0:1619677448.071997:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.071997:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.071999:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.072000:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.072000:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.072000:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.072002:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677448.072002:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.072002:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4cc00] (21)++
00000800:00000200:6.0:1619677448.072002:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4cc00] (22)--
00000800:00000200:6.0:1619677448.072003:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4cc00] (21)--
00000800:00000200:22.0:1619677448.072014:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.072016:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.072017:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.072017:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.072019:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.072020:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.072039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:20.2:1619677448.072040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4cc00] (20)++
00000800:00000200:2.0:1619677448.072045:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:6.0:1619677448.072046:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4cc00] (21)++
00000800:00000200:6.0:1619677448.072048:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.31@o2ib3
00000800:00000200:2.0:1619677448.072048:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:22.0:1619677448.072049:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4cc00] (22)--
00000800:00000200:18.0:1619677448.072049:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:6.0:1619677448.072053:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.31@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.072053:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.072055:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.072055:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.072057:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:6.0:1619677448.072058:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677448.072059:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.072060:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.072061:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.072062:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.072063:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.072064:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.072065:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.072065:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.072070:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.072071:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.31@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.072074:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.072074:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677448.072077:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.072077:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.072078:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:6.0:1619677448.072078:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.072078:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.072079:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:6.0:1619677448.072080:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:6.0:1619677448.072081:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4cc00] (21)++
00000800:00000200:6.0:1619677448.072081:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4cc00] (22)--
00000800:00000200:2.0:1619677448.072081:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:6.0:1619677448.072082:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4cc00] (21)--
00000800:00000200:2.0:1619677448.072082:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.072083:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.072084:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.072085:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:8.2:1619677448.072086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.072090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677448.072093:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:20.2:1619677448.072094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:26.0:1619677448.072096:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:18.0:1619677448.072096:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:10.0:1619677448.072096:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:5.0:1619677448.072096:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:26.0:1619677448.072097:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.072099:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.072099:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.072099:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677448.072099:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:18.0:1619677448.072100:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:5.0:1619677448.072100:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.072101:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.072102:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.0:1619677448.072102:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.072102:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.072104:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:5.0:1619677448.072104:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:31.0:1619677448.072106:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.072108:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.072108:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677448.072108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4cc00] (20)++
00000400:00000200:31.0:1619677448.072110:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.072111:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.072113:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.072114:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.072114:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4cc00] (21)++
00000400:00000200:31.0:1619677448.072115:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.072116:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.31@o2ib3
00000800:00000200:6.0:1619677448.072117:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4cc00] (22)--
00000400:00000200:22.0:1619677448.072120:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.31@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.072121:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.072122:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.072123:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.072124:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677448.072126:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.072126:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.072127:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:22.0:1619677448.072127:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.072128:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:22.0:1619677448.072128:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.072129:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.072130:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.072130:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.072131:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.072132:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:22.0:1619677448.072133:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.31@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.072135:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.2:1619677448.072136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.072138:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.072139:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:22.0:1619677448.072139:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.072141:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677448.072142:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4cc00] (21)++
00000800:00000200:22.0:1619677448.072142:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4cc00] (22)--
00000800:00000200:22.0:1619677448.072143:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4cc00] (21)--
00000800:00000200:20.0:1619677448.072144:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.072146:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.072147:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.072149:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.072150:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.072152:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:13.0:1619677448.072154:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.072157:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.072158:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.072159:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.072161:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.072163:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.072211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.072218:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.072221:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.072222:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.072223:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.072226:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.072227:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.072229:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.072230:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.072231:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.072232:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.072235:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.072237:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.072239:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.072240:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.072240:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.072242:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.072242:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.072243:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.072244:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.072254:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.072256:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.072257:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.072259:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.072261:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.072261:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.072289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.072294:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.072300:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.072302:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.072304:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.072307:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.072308:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.072310:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.072311:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.072312:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.072313:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.072316:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.072318:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.072322:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.072322:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.072323:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.072325:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.072326:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.072327:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.072328:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.072329:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.072334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.072337:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.072339:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:12.0:1619677448.072339:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.072340:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.072341:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.072342:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.072343:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.072344:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.072346:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.072347:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.072377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.072383:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.072386:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.072387:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.072389:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.072392:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.072393:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.072394:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.072396:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.072397:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.072398:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.072401:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.072402:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.072404:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.072405:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.072406:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.072407:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.072407:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.072408:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.072408:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.072413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.072420:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.072422:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.072423:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.072423:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.072424:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.072426:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.072431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.072437:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.072439:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.072440:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.072441:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.072441:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.072441:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.072519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.072527:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.072530:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.072531:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.072533:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.072536:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.072537:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.072539:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.072540:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.072541:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.072542:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.072546:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.072547:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.072550:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.072551:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.072551:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.072553:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.072553:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.072554:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.072554:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.072565:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.072568:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.072568:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.072570:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.072571:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.072572:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.072583:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.072585:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.072586:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.072587:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.072587:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.072588:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.072689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.072696:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.072699:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.072700:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.072702:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.072705:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.072706:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:4.2:1619677448.072707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000400:00000200:1.0:1619677448.072707:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.072709:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.072709:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.072711:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.072714:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.072715:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:1.0:1619677448.072715:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677448.072718:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677448.072719:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:1.0:1619677448.072719:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.072719:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.072720:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:8.0:1619677448.072721:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.072722:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.072722:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677448.072723:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.072723:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.072724:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.072725:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677448.072726:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96719722] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.072727:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.072728:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.072729:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677448.072733:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.072734:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677448.072734:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677448.072736:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.072737:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.072737:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.072738:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:12.0:1619677448.072739:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.072739:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:13.0:1619677448.072741:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.072741:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677448.072741:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:8.0:1619677448.072741:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677448.072741:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677448.072742:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:5.2:1619677448.072745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:9.2:1619677448.072747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.072753:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:13.0:1619677448.072753:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.072755:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:13.0:1619677448.072757:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.072757:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.072758:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.072758:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:18.0:1619677448.072759:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.072759:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.072760:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.072760:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.072761:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.072761:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.072762:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.072763:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.072764:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.072765:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.072767:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.072768:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.072771:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.072773:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.072775:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.072775:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.072776:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.072776:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.072777:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.072777:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.072778:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.072778:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.072779:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.072779:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.072780:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.072780:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.072781:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.072782:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.072782:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.072782:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.072784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.072788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.072793:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.072794:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.072794:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.072795:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:2.0:1619677448.072795:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.072796:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.072796:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.072797:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.072797:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677448.072798:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:4.2:1619677448.072817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.072824:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.072827:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.072828:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.2:1619677448.072829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:10.0:1619677448.072830:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.072832:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:10.0:1619677448.072832:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.072833:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:20.0:1619677448.072835:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:10.0:1619677448.072835:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.072837:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.072838:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.072838:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.072838:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:10.0:1619677448.072839:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.072841:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.072842:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.072843:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.072844:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.072846:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.072846:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.072847:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.072848:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.072849:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.072849:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.072850:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:20.0:1619677448.072852:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.072852:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.072852:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.072853:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.072853:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677448.072854:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677448.072857:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.072858:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677448.072859:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.072860:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677448.072861:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.072862:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.072863:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.072864:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677448.072865:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.072868:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.072869:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.072869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000400:00000200:31.0:1619677448.072871:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.072871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677448.072873:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.072874:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.0:1619677448.072874:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:31.0:1619677448.072875:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:5.0:1619677448.072876:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.072877:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677448.072878:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.072878:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.072878:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.072879:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:5.0:1619677448.072880:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:31.0:1619677448.072881:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.072881:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677448.072882:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:5.0:1619677448.072882:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.072884:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:5.0:1619677448.072886:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.072888:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.072890:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.072891:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.072892:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.072893:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.072897:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.072898:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.072901:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.072902:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.072903:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.072904:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.072905:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.072906:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.072907:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.072917:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.072919:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.072920:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.072922:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.072923:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.072924:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.072929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.072937:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.072940:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.072942:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.072944:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.072945:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.072947:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677448.072948:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.072949:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.072950:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.072951:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677448.072954:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.072956:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.072959:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.072960:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677448.072960:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.072962:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677448.072963:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.072964:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.072964:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677448.072973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:6.0:1619677448.072978:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.072980:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677448.072980:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.072981:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000400:00000200:6.0:1619677448.072981:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.072983:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.072984:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.072987:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.072988:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.072989:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:6.0:1619677448.072989:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677448.072990:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677448.072991:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.072992:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.072994:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.072994:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:22.0:1619677448.072997:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.072999:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.073002:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.073003:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:22.0:1619677448.073003:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677448.073005:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677448.073006:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677448.073006:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677448.073007:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:13.0:1619677448.073018:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.073020:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.073021:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.073023:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.073024:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.073026:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.073101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.073107:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.073110:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.073111:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.073112:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.073116:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.073117:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.073118:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.073120:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.073121:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.073122:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.073125:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.073126:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.073128:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.073129:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.073130:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.073131:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.073131:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.073133:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.073134:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.073135:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.073137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.073140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.073146:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.073147:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.073147:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.073147:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.073147:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.073148:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.073149:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.073149:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.073150:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.073150:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.073163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:20.2:1619677448.073168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.073173:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677448.073175:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677448.073177:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.073177:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.073178:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.073180:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.073181:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.073182:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:4.2:1619677448.073182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:31.0:1619677448.073183:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.073185:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.073186:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.073187:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.073189:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.073190:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.073192:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.073192:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:30.0:1619677448.073193:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.073193:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.073195:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.073195:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.073195:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.073196:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:30.0:1619677448.073196:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.073197:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:30.0:1619677448.073197:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.073198:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677448.073198:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.073198:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.073199:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.073199:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:31.0:1619677448.073200:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:31.0:1619677448.073201:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (23)--
00000400:00000200:26.0:1619677448.073201:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96719712] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.073202:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:26.0:1619677448.073202:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.073203:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000400:00000200:30.0:1619677448.073203:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.073203:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.073204:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.073205:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.073205:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.073206:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.073207:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.073208:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.073208:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.073208:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:26.0:1619677448.073208:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.073209:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.073209:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.073209:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.073211:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.073211:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.073211:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.073212:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.073212:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677448.073212:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.073213:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.073213:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.073214:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.073215:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.073216:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.073216:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.073217:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.073217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:9.2:1619677448.073221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.073221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.2:1619677448.073223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:20.0:1619677448.073225:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.073227:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.073228:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.073229:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:20.0:1619677448.073230:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.073230:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677448.073231:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677448.073231:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.073232:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.073232:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.073232:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:20.0:1619677448.073233:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677448.073233:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:24.0:1619677448.073234:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:10.0:1619677448.073234:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.073234:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.073235:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.073235:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:18.0:1619677448.073235:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.073235:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:31.0:1619677448.073236:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.073237:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:10.0:1619677448.073237:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.073238:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:24.0:1619677448.073238:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.073238:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677448.073238:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.073240:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.073241:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.073242:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.073242:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.073243:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:21.2:1619677448.073243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:10.0:1619677448.073243:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677448.073244:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.073245:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:24.0:1619677448.073246:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.073247:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.073248:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.073248:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.073249:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.073250:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677448.073251:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:24.0:1619677448.073252:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.073252:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:10.0:1619677448.073252:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.073252:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.073253:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.073254:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:24.0:1619677448.073255:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.073255:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:6.0:1619677448.073255:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.073256:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.073256:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677448.073258:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.073258:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.073259:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.073260:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677448.073260:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.073261:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96719709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.073262:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.073262:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:6.0:1619677448.073262:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.073263:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.073263:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:6.0:1619677448.073263:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.073264:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.073266:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:9.2:1619677448.073268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677448.073268:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.073269:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.073270:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.073270:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.073272:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.073272:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.073273:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:6.0:1619677448.073273:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.073274:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:6.0:1619677448.073274:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.073276:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677448.073276:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.073276:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.073277:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677448.073278:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:6.0:1619677448.073278:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.073279:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677448.073279:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.073280:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.073282:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.073282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.073284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677448.073285:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677448.073290:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:5.0:1619677448.073291:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:22.0:1619677448.073292:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.073293:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.073293:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677448.073297:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.073298:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.073298:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.073300:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.073302:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.073451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.073459:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.073462:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.073463:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.073465:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.073468:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.073469:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.073471:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.073472:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.073473:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.073474:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.073477:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.073480:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.073483:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.073484:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.073485:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.073487:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.073488:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.073488:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.073489:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.073493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.073501:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:5.2:1619677448.073504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.073504:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.073505:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.073506:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.073507:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:1.0:1619677448.073507:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.073509:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677448.073511:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.073511:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677448.073512:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.073513:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.073513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000400:00000200:17.0:1619677448.073514:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.073516:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.073518:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.073519:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.073520:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.073522:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.073523:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.073524:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.073524:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.073525:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.073526:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.073526:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.073527:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677448.073529:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.073530:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.073530:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.073532:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.073532:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.073533:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.073533:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.073538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.073545:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.073547:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.073547:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.073548:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.073549:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.073550:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.073555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.073561:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.073564:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.073564:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.073565:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.073565:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.073566:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.073828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.073836:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.073839:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.073841:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.073844:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.073844:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.073845:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.073847:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.073848:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.073849:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.073850:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.073854:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.073855:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.073858:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.073859:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.073860:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.073862:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.073862:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.073864:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.073864:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.073868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.073876:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.073878:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.073879:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.073881:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.073882:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.073884:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.073886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.2:1619677448.073894:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.0:1619677448.073897:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.073900:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.073900:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.073901:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.073902:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.073902:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:1.0:1619677448.073902:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.073903:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.073904:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:1.0:1619677448.073905:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.073907:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.073908:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.073909:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.073911:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.073912:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.073913:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.073916:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.073918:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.073920:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.073921:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.073922:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.073923:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.073924:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.073924:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.073925:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.073939:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.073941:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.073942:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.073942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:6.0:1619677448.073942:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.073944:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.073947:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677448.073949:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.073952:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.073952:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.073954:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.073956:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:24.0:1619677448.073957:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.073958:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000400:00000200:24.0:1619677448.073959:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:22.0:1619677448.073959:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:24.0:1619677448.073960:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.073960:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000400:00000200:24.0:1619677448.073961:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.073961:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.073961:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:24.0:1619677448.073962:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.073963:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.073967:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.073969:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.073972:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.073972:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.073973:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.073975:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.073976:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.073977:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.073977:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.073981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.073989:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.2:1619677448.073992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677448.073992:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.073993:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.073993:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.073995:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.073998:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.073998:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677448.074001:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.074003:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.074003:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.074004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:5.0:1619677448.074006:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.074007:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.074009:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.074010:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.074011:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.074012:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677448.074012:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.074015:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.074015:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.074016:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000400:00000200:5.0:1619677448.074016:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.074017:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.074018:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677448.074018:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.074021:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.074022:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.074022:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.074024:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.074025:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.074026:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.074026:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.074037:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.074040:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.074041:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.074043:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.074045:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.074045:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.074069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.074076:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.074079:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.074081:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.074082:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.074083:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.074085:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.074086:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719702] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.074087:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.074088:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.074089:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.074093:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.074094:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.074097:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.074098:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.074098:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.074100:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.074100:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.074102:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.074103:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.074104:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677448.074109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.074111:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.074113:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.074114:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.074116:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.074117:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677448.074117:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.074119:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:13.0:1619677448.074119:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.074120:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.074121:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677448.074142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.074150:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.074152:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.074153:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.074155:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.074157:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.074158:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.074160:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.074161:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.074162:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.074163:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.074166:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.074168:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.074171:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.074172:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.074172:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.074174:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.074174:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.074176:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.074177:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.074177:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.2:1619677448.074180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:4.2:1619677448.074183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.074184:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.074187:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:22.0:1619677448.074188:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:20.0:1619677448.074190:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.074190:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:22.0:1619677448.074191:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:10.0:1619677448.074191:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:22.0:1619677448.074192:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.074192:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.074192:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.074193:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:20.0:1619677448.074194:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:10.0:1619677448.074194:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.074195:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.074195:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.074195:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:20.0:1619677448.074197:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.074197:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.074198:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:20.0:1619677448.074199:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.074202:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.074204:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.074206:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.074207:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.074208:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.074210:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.074211:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.074212:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.074212:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677448.074216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.074224:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.074227:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.074228:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.074228:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.074230:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.074233:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.074285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.074293:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.074303:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:30.0:1619677448.074303:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.074305:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.074307:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.074309:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.074310:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.074311:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.074312:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.074313:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.074317:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.074318:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.074321:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.074322:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.074322:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.074324:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.074324:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.074326:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.074327:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.074327:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:21.2:1619677448.074328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:20.2:1619677448.074333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:6.0:1619677448.074334:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:12.0:1619677448.074338:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:6.0:1619677448.074338:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.074339:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677448.074340:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:6.0:1619677448.074340:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.074341:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:12.0:1619677448.074341:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.074343:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.074343:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.074344:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:6.0:1619677448.074344:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:30.0:1619677448.074345:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:13.0:1619677448.074345:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.074346:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677448.074346:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:6.0:1619677448.074346:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.074347:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.074348:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.074349:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.074353:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.074355:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.074357:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.074357:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.074358:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.074360:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.074360:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.074361:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.074362:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.074366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.074375:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.074377:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.074378:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.074380:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.074380:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.074382:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.074414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.074421:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.074425:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.074426:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.074427:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.074430:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.074431:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.074432:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.074434:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.074435:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.074436:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.074439:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.074441:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.074443:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.074443:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.074444:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.074445:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.074446:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.074447:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.074447:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.074451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.074458:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.074461:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.074461:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.074462:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.074463:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.074464:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.074553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:4.2:1619677448.074553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677448.074557:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.074559:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:26.0:1619677448.074561:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677448.074561:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.074562:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677448.074564:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:20.0:1619677448.074564:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.074565:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677448.074566:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.074568:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.074568:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.074570:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.074571:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.074572:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.074572:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.074573:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.074575:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.074576:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.074577:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.074578:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.074579:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.074579:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.074582:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.074583:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.074583:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.074584:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.074585:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.074586:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.074586:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.074587:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.074587:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677448.074588:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.074588:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.074588:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677448.074590:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.074590:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.074591:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.074592:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:13.0:1619677448.074598:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.074601:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.074601:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.074601:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:13.0:1619677448.074603:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.074603:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:12.0:1619677448.074604:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.074604:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.074605:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.074605:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.074606:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.074607:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:12.0:1619677448.074615:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.074618:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.074619:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.074619:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.074620:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.074621:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.074622:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.074625:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.074625:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.074626:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.074627:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.074627:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.074629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.074636:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.074639:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.074640:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.074642:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.074645:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.074647:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.074648:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.074649:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.074650:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.074651:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.074654:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.074655:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.074658:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.074659:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.074660:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.074662:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.074663:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.074663:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.074664:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.074668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.074677:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.074681:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.074682:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.074682:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.074684:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.074687:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.074730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.2:1619677448.074738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:10.0:1619677448.074738:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.074741:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.074742:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.074743:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.074744:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677448.074745:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.074747:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:10.0:1619677448.074747:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:6.0:1619677448.074747:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677448.074749:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.074750:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.074751:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.074752:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.074753:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.074753:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.2:1619677448.074754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:22.0:1619677448.074755:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.074757:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.074758:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.074758:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.074759:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.074761:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.074761:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.074762:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.074764:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.074765:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:22.0:1619677448.074765:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.074765:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.074765:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.074767:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:30.0:1619677448.074768:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.074768:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.074768:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.074768:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.074769:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.074769:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:30.0:1619677448.074770:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.074770:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.074771:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:30.0:1619677448.074772:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:22.0:1619677448.074772:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677448.074773:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96719691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.074773:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:9.2:1619677448.074773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677448.074774:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.074774:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:30.0:1619677448.074775:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.074775:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.074775:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:30.0:1619677448.074776:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.074780:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.074780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.074780:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:30.0:1619677448.074781:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.074783:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.074783:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:30.0:1619677448.074784:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:2.0:1619677448.074784:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.074785:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:21.2:1619677448.074785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677448.074785:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677448.074786:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:2.0:1619677448.074786:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.074787:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677448.074787:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:30.0:1619677448.074788:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.074788:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.074789:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.2:1619677448.074790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:17.0:1619677448.074790:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:2.0:1619677448.074790:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:17.0:1619677448.074791:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.074791:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.074792:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677448.074794:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.074795:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:30.0:1619677448.074795:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.074795:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.074796:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677448.074797:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.074797:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.074797:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.074797:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:30.0:1619677448.074799:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:2.0:1619677448.074799:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:31.0:1619677448.074800:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.074800:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:30.0:1619677448.074801:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.074803:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.074803:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:30.0:1619677448.074804:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.074805:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.074805:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.074807:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:18.0:1619677448.074807:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:31.0:1619677448.074808:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:18.0:1619677448.074809:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.074810:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.074810:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.074811:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.074813:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.074813:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:30.0:1619677448.074815:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.074816:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.074816:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.074817:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:18.0:1619677448.074817:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:30.0:1619677448.074818:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.074819:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.074819:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677448.074819:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677448.074822:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.074823:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677448.074824:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677448.074826:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677448.074826:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.074827:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.074828:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677448.074829:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.074831:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.074833:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.074833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:26.0:1619677448.074834:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.074835:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.074837:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.074838:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.074838:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.074838:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.074839:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.074840:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677448.074841:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000400:00000200:26.0:1619677448.074843:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677448.074844:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.074845:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.074845:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:10.0:1619677448.074846:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.074847:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.074848:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719688] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.074850:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.074851:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.074852:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.074856:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.074858:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.074858:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.074859:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.074860:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.074861:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.074861:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.074861:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.074862:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.074862:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:26.0:1619677448.074863:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.074865:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.074866:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.074867:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677448.074868:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.074868:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:20.2:1619677448.074874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:13.0:1619677448.074877:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.074879:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.074880:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.074881:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:13.0:1619677448.074882:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677448.074883:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.074883:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.074883:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:13.0:1619677448.074884:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.074885:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:5.2:1619677448.074934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.074940:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.074943:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.074943:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.074945:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.074947:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.074948:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.074949:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.074950:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.074951:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.074952:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.074955:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.074957:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.074959:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.074960:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.074960:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.074962:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.074962:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.074964:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.074965:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.074965:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.074970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.074975:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.074976:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.074977:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.074978:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.074978:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677448.074979:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.074979:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.074980:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.074982:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.074984:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.075132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.075139:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.075142:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.075143:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.075145:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.075148:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.075149:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.075150:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.075152:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.075153:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.075154:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.075157:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.075159:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.075161:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.075161:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.075162:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.075163:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.075164:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.075165:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.075165:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.075175:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.075176:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.075177:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.075178:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.075179:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.075180:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.075190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.075195:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.075198:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.075199:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.075201:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.075203:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.075204:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.075206:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.075207:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.075208:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.075209:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.075213:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.075214:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:21.2:1619677448.075216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677448.075217:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.075218:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.075218:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.075220:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.075221:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.075221:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.075222:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677448.075224:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.075226:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.075229:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.075230:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.075231:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.075232:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:31.0:1619677448.075233:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677448.075234:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.075235:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.075235:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.075235:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:31.0:1619677448.075236:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.075236:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.075238:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677448.075238:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.075241:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:31.0:1619677448.075242:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.075244:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.2:1619677448.075244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677448.075247:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.075248:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.075248:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.075250:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.075250:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.075251:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677448.075251:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.075252:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.075253:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.075254:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.075256:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.075256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:10.0:1619677448.075257:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.075258:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.075260:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.075261:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.075262:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.075263:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.075264:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:10.0:1619677448.075266:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.075267:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.075268:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:10.0:1619677448.075268:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.075268:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.075270:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.075270:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.075271:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.075271:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.075273:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.075273:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677448.075273:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677448.075274:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.075276:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.075277:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.075282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.075285:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.075288:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.075288:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.075289:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:12.0:1619677448.075290:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.075291:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.075291:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:13.0:1619677448.075292:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.075293:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.075293:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.075319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.075324:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.075327:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.075329:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.075330:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.075332:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.075334:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.075335:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719682] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.075336:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.075338:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.075339:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.075342:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.075344:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.075347:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.075348:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.075348:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.075350:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.075351:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.075353:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.075354:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.075354:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:9.2:1619677448.075357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677448.075359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.075362:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.075364:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.075365:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.075366:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677448.075366:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.075368:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.075369:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.075370:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.075372:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.075374:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.075378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.075385:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.075388:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.075389:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.075389:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.075390:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.075390:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.075525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.075533:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.075536:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.075537:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.075539:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.075541:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.075543:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.075544:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.075546:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.075546:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.075548:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.075551:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.075553:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.075556:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.075557:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.075557:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.075559:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.075559:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.075560:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.075560:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.075573:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.075575:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.075577:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.075577:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.075579:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.075582:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677448.075591:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.075593:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.075594:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.075595:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.075596:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.075596:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.075622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.075628:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.075630:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.075632:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.075633:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.075636:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.075638:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.075639:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.075640:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.075641:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.075642:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.075646:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.075647:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.075651:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.075651:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.075652:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.075653:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.075654:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.075655:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.075655:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:26.0:1619677448.075666:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.075668:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.075669:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.075670:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.075672:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.075674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:26.0:1619677448.075675:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:6.0:1619677448.075680:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.075683:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.075683:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.075685:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.075688:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.075689:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.075691:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.075692:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.075693:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.075694:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:5.2:1619677448.075696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:6.0:1619677448.075697:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.075699:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677448.075702:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.075702:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.075703:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.075704:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:17.0:1619677448.075705:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:6.0:1619677448.075706:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.075706:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677448.075706:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.075707:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:17.0:1619677448.075708:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.075708:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.075709:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:17.0:1619677448.075710:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.075711:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.075713:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.075714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:17.0:1619677448.075714:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.075715:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.075716:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:10.0:1619677448.075717:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:22.0:1619677448.075719:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:17.0:1619677448.075720:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.075720:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.075721:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:22.0:1619677448.075721:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:17.0:1619677448.075721:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:10.0:1619677448.075721:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.075722:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.075723:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677448.075723:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.075724:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.075725:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.075726:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:10.0:1619677448.075726:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:17.0:1619677448.075728:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.075728:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.075729:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.075730:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.075734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.075740:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.075743:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.075744:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.075744:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.075746:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.075750:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.075753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677448.075759:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.075761:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.075762:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.075762:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.075763:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.075764:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.075820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.075829:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.075831:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.075834:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.075836:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.075836:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.075838:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.075839:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.075840:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.075841:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.075842:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.075845:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.075845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677448.075847:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.075850:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.075851:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.075851:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.075853:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.075853:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.075853:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.075854:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.075855:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.075855:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.075857:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.075858:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.075860:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.075861:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.075862:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.075864:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.075865:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.075865:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:6.0:1619677448.075866:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:26.0:1619677448.075868:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.075869:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677448.075869:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.075870:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.075871:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.075871:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.075872:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.075872:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:6.0:1619677448.075873:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.075874:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.075874:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.075875:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.075875:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677448.075876:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677448.075885:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.075887:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.075888:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.075888:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.075890:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.075892:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.075903:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.075905:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.075906:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.075906:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.075907:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.075907:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.076167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.076173:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.076175:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.076176:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.076178:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.076181:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.076182:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.076184:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.076185:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.076186:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.076187:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.076190:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.076192:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.076195:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.076196:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.076196:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.076198:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.076198:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.076199:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.076199:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.076212:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.076214:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.076215:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.076215:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.076218:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.076220:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.076228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.076235:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.076238:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.076238:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.076240:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.076242:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.076244:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.076245:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.076246:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.076247:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.076248:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.076252:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.076253:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.076256:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.076257:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.076257:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.076259:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.076259:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.076260:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.076261:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.076262:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.076267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.076274:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.076276:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.076277:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.076278:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:26.0:1619677448.076278:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.076281:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.076282:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.076283:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.076285:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.076286:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.076309:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.076312:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.076313:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.076313:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.076314:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.076315:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.076329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.076336:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.076339:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.076340:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.076342:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.076345:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.076346:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.076347:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.076349:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.076350:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.076351:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.076354:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.076356:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.076357:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.076358:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.076359:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.076360:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.076361:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.076362:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.076362:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.076366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.076374:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.076376:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.076377:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.076377:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.076378:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.076381:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.076385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.076390:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.076393:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.076393:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.076395:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.076397:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.076398:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.076399:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.076401:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.076401:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.076402:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.2:1619677448.076403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:18.0:1619677448.076405:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.076406:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.076409:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.076410:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677448.076410:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.076410:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.076412:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.076412:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:30.0:1619677448.076413:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:18.0:1619677448.076413:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.076413:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:30.0:1619677448.076415:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.076417:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.076418:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.076420:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.076421:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.076422:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.076423:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.076424:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.076425:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:30.0:1619677448.076428:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.076428:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:30.0:1619677448.076429:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:13.0:1619677448.076429:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.076430:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.076432:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.076432:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.076433:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.076433:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.076433:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.076435:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.076435:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.076436:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.076437:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677448.076447:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.076449:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.076449:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.076451:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.076453:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.076454:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.076513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.076521:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.076524:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.076525:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.076526:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.076529:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.076531:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677448.076532:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.076533:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.076534:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.076535:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677448.076538:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.076539:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677448.076542:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.076542:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677448.076543:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677448.076545:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677448.076545:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.076547:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.076548:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.076548:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:21.2:1619677448.076554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:13.0:1619677448.076556:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.076558:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.076559:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.076560:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.076561:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:12.0:1619677448.076562:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677448.076562:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:13.0:1619677448.076563:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677448.076563:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.076564:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677448.076574:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.076576:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.076577:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.076577:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.076578:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.076579:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.076634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.076642:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.076644:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.076645:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.076647:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.076649:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.076650:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.076652:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.076653:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.076654:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.076655:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.076658:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.076659:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.076662:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.076663:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.076663:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.076665:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.076666:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.076666:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.076667:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.076680:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.076682:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.076683:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.076684:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.076686:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.076689:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.076698:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.076700:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.076701:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.076702:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.076702:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.076703:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.076749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.076755:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.076758:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.076759:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.076761:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.076763:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.076765:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.076766:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.076767:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.076768:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.076769:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.076773:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.076774:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.076777:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.076778:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.076778:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.076780:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.076781:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.076781:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.076782:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.076786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.076793:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.076795:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.076796:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677448.076796:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.076798:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.076801:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677448.076894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.076901:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.076903:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.076904:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.076906:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.076908:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.076909:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.076910:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719667] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.076912:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.076913:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.076914:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.076917:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.076919:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.076922:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.076923:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.076923:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.076925:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.076925:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.076926:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.076928:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.076928:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677448.076934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.2:1619677448.076939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.076945:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.076947:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.076947:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677448.076947:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677448.076948:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677448.076950:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.076951:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.076951:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.076953:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.076956:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.076959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.076966:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.076969:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.076970:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.076972:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.076975:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.076977:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.076978:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.076979:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.076980:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.076981:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.076984:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.076985:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.076988:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.076989:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.076990:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.076992:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.076992:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.076993:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.076993:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.076998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.077005:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.077008:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.077009:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.077010:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.077012:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.077014:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.077022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677448.077029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.077030:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.077033:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.077035:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.077036:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.077037:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.077037:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.077039:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677448.077039:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.077040:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677448.077041:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.077042:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.077042:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.077043:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.077044:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.077045:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.077047:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.077047:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.077048:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.077049:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.077049:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.077050:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.077051:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.077051:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.077052:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.077053:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.077054:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.077054:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677448.077054:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.077056:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:10.0:1619677448.077056:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.077057:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.077058:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677448.077059:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.077059:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.077060:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.077062:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.077062:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.077063:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.077063:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677448.077064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.077066:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.2:1619677448.077067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:12.0:1619677448.077068:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.077069:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.077071:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:12.0:1619677448.077071:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.077073:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.077074:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.077074:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677448.077074:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.077075:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677448.077076:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.077078:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.077079:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.077081:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.077081:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.077084:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.077087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.077094:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.077097:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.077098:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.077099:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.077099:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.077100:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.077132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.077139:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.077142:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.077142:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.077142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:26.0:1619677448.077144:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.077147:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.077147:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:26.0:1619677448.077149:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:2.0:1619677448.077150:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:26.0:1619677448.077151:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.077151:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:26.0:1619677448.077153:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.077154:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.077154:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.077155:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677448.077158:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.077159:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.077160:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.077161:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.077162:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:2.0:1619677448.077163:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.077164:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.077164:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.077165:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:2.0:1619677448.077165:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.077166:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.077167:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.077168:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.077168:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:2.0:1619677448.077168:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.077169:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677448.077170:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.077173:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.077174:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.077175:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.077177:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.077177:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.077178:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:30.0:1619677448.077179:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:2.0:1619677448.077179:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:30.0:1619677448.077182:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.077183:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.077185:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.077186:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.077187:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:12.0:1619677448.077189:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.077192:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.077193:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.077194:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.077196:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.077197:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.077256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.077262:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.077265:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.077266:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.077268:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.077270:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.077271:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.077273:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.077275:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.077276:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.077277:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.077280:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.077282:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.077285:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.077286:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.077287:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.077289:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.077289:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.077291:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.077292:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.077292:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.077298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.077304:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677448.077305:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.077306:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.077307:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.077308:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677448.077308:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677448.077309:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.077311:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.077312:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.077314:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.077399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.077407:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.077410:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.077411:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.077413:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.077415:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.077417:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.077418:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.077419:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.077420:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.077421:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.077425:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.077427:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.077429:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.077430:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.077430:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.077432:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.077432:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.077433:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.077434:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677448.077446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.077453:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.077455:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.077456:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.077458:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.077461:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.077462:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.077463:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96719659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.077465:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.077465:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.077466:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.077469:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.077470:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.077472:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:8.0:1619677448.077473:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.077473:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.077474:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.077475:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.077475:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.077475:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.077476:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.077476:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:31.0:1619677448.077477:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.077477:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:31.0:1619677448.077479:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.077480:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:12.0:1619677448.077487:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.077490:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.077490:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.077492:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.077493:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.077495:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677448.077505:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.077508:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.077509:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.077509:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.077510:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.077511:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.077584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.077587:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.077589:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.077591:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.077593:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.077594:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.077595:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.077597:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.077598:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.077599:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.077600:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.077604:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.077606:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.077609:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.077610:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.077610:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.077612:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.077613:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.077613:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.077614:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:5.2:1619677448.077620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.077627:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.077630:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.077631:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.077633:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.077635:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.077637:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.077638:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.077639:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.077640:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.077641:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.077644:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.077646:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.077647:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:18.0:1619677448.077648:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.077649:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.077649:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.077650:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.077651:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.077651:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.077652:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.077652:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:31.0:1619677448.077653:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.077653:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:30.0:1619677448.077654:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.077655:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.2:1619677448.077657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.077665:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.077668:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.077669:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.077669:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.077671:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.077674:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.077756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.077761:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.077764:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.077766:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.077766:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.077769:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.077770:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.077772:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.077773:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.077774:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.077775:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.077778:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.077780:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.077783:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.077784:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.077784:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.077786:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.077787:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.077788:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.077788:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.077799:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.077801:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.2:1619677448.077801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:12.0:1619677448.077802:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.077804:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.077805:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.077806:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.077809:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.077811:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:5.2:1619677448.077811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:10.0:1619677448.077812:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.077814:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.077816:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.077817:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:13.0:1619677448.077817:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.077817:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000400:00000200:26.0:1619677448.077818:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.077820:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.077820:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000400:00000200:26.0:1619677448.077821:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.077821:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:13.0:1619677448.077821:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.077822:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.077822:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.077822:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.077823:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677448.077823:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.077824:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677448.077826:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.077826:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.077828:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.077828:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:4.2:1619677448.077829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677448.077829:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.077830:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.077831:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.077832:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.077832:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.077833:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.077833:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.077835:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.077836:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.077836:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:2.0:1619677448.077836:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.077837:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.077838:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677448.077838:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.077839:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.077839:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:2.0:1619677448.077841:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.077842:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.077842:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:2.0:1619677448.077842:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677448.077844:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:2.0:1619677448.077844:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000400:00000200:10.0:1619677448.077845:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.077846:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:2.0:1619677448.077846:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:12.0:1619677448.077847:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:10.0:1619677448.077847:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96719653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.077847:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677448.077847:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:10.0:1619677448.077848:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.077849:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.077850:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:10.0:1619677448.077850:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:8.2:1619677448.077850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:12.0:1619677448.077851:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.077853:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.077853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (21)++
00000800:00000200:13.0:1619677448.077854:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:10.0:1619677448.077855:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.077856:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.077857:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.077858:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:1.0:1619677448.077858:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:10.0:1619677448.077860:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.077860:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:18.0:1619677448.077861:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:10.0:1619677448.077861:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677448.077862:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:17.0:1619677448.077862:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.077862:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677448.077862:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000400:00000200:1.0:1619677448.077862:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.077864:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.077864:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677448.077864:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.077866:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:1.0:1619677448.077866:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:10.0:1619677448.077867:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.077867:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.077873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.077878:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:26.0:1619677448.077880:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.077881:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:26.0:1619677448.077882:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677448.077882:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.077882:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.077883:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.077883:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.077885:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.077888:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.078000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.078008:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.078011:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:21.2:1619677448.078011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:30.0:1619677448.078013:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.078014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:31.0:1619677448.078016:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.078016:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.078017:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:30.0:1619677448.078017:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:31.0:1619677448.078019:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.078019:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.078020:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.078020:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:30.0:1619677448.078021:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.078022:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.078023:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.078024:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677448.078026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:31.0:1619677448.078027:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.078027:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.078028:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.078029:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.078029:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.078031:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:30.0:1619677448.078031:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.078031:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:17.0:1619677448.078032:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677448.078033:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.078034:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.078035:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.078035:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.078035:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677448.078036:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.078037:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.078038:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.078039:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.078039:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.078040:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.078043:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.078043:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.078044:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.078044:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:31.0:1619677448.078045:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.078046:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.078046:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.078047:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.078047:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.078048:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.078048:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677448.078049:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.078049:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.078049:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:17.0:1619677448.078049:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:30.0:1619677448.078050:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677448.078050:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:17.0:1619677448.078050:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96719650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.078051:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.078052:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.078052:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:17.0:1619677448.078052:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677448.078052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.078053:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.078053:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:17.0:1619677448.078053:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.078054:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.078054:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000400:00000200:17.0:1619677448.078054:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.078055:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.078055:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677448.078056:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.078057:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677448.078057:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.078057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.078058:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677448.078059:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.078060:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.2:1619677448.078060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677448.078062:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677448.078062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677448.078063:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:17.0:1619677448.078063:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)++
00000400:00000200:20.0:1619677448.078064:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.078064:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (23)++
00000400:00000200:20.0:1619677448.078065:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.078066:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (24)--
00000800:00000200:5.0:1619677448.078066:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:1.0:1619677448.078066:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (24)++
00000800:00000200:17.0:1619677448.078067:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.078067:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:1.0:1619677448.078067:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (25)--
00000800:00000200:17.0:1619677448.078068:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.078068:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.078069:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677448.078069:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677448.078069:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.078070:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:1.0:1619677448.078070:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.078071:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677448.078071:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000800:00000200:17.0:1619677448.078072:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000400:00000200:17.0:1619677448.078072:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.078072:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:1.0:1619677448.078073:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.078074:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.078074:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:1.0:1619677448.078074:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.078075:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.078075:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.078076:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:2.0:1619677448.078077:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.078078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.078081:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:1.0:1619677448.078085:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.078088:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.078088:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.078089:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.078089:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.078090:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.078312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.078318:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.078321:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.078323:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.078324:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.078326:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.078327:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.078329:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.078330:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.078331:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.078332:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.078336:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.078338:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.078341:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.078341:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.078342:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.078344:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.078344:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.078345:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.078346:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.078358:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:21.2:1619677448.078361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:13.0:1619677448.078361:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.078362:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.078364:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.078365:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.078367:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677448.078368:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.078371:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.078372:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.078373:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.078376:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.078378:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.078379:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.078380:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.078381:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.078382:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.078385:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.078387:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.078390:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.078391:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.078391:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.078394:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.078394:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.078395:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.078396:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.078400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.078408:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.078412:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.078413:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.078413:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.078415:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.078418:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.078468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.078474:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.078477:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.078478:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.078480:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.078482:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.078484:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.078485:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.078487:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.078487:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.078489:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.078492:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.078494:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.078497:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.078498:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.078498:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.078500:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.078501:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.078501:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.078502:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.078506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.078514:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.078516:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.078517:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.078518:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.078520:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.078521:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.078529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.078535:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.078538:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.078539:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.078541:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.078542:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.078544:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.078545:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.078546:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.078547:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.078548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:17.0:1619677448.078549:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.078552:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.078554:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.078556:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.078557:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.078558:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.078559:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677448.078559:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.078560:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.078561:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.078561:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.078561:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.078562:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.078563:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.078563:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.078565:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.078566:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.078568:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.078569:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.078570:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.078573:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:30.0:1619677448.078573:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.078574:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.078575:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.078575:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.078577:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.078578:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.078579:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.078579:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.078580:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.078581:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.078581:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.078581:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.078582:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.078583:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.078593:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.078595:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.078596:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.078597:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.078599:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.078600:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.078636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.078644:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.078647:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.078648:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.078650:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.078651:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.078652:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.078653:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.078655:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.078656:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.078656:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:5.2:1619677448.078657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:31.0:1619677448.078660:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.078662:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.078664:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.078665:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.078665:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.078666:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.078667:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.078668:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.078668:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:31.0:1619677448.078669:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:10.0:1619677448.078669:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.078670:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.078671:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.078672:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677448.078673:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.078673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:26.0:1619677448.078675:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.078676:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.078677:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.078677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677448.078678:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.078679:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.078681:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:26.0:1619677448.078682:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.078683:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.078683:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.078684:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.078685:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.078686:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.078686:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.078687:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.078688:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:26.0:1619677448.078688:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.078688:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.078689:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677448.078689:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:30.0:1619677448.078690:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.078690:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.078691:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.078691:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.078693:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.078693:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.078699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.078704:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677448.078706:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677448.078706:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.078707:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.078708:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:6.0:1619677448.078708:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.078709:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.078709:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677448.078710:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.078713:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.078770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.078777:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.078779:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.078780:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.078782:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.078785:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.078787:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.078788:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.078789:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.078790:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.078791:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.078794:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.078796:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.2:1619677448.078799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:8.0:1619677448.078799:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.078799:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.078800:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.078802:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.078802:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.078803:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.078803:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677448.078806:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.078808:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:8.2:1619677448.078808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.078809:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677448.078811:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.078814:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.078815:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.078816:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.078816:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:10.0:1619677448.078818:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.078818:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.078818:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.078819:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:10.0:1619677448.078819:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677448.078820:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.078822:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.078824:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.078825:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:10.0:1619677448.078826:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.078827:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677448.078827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.078828:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.078828:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.078830:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.078830:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.078831:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.078832:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:17.0:1619677448.078834:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.078837:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.078837:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.078838:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.078839:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.078839:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:26.0:1619677448.078841:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.078843:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.078844:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.078845:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.078846:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.078848:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.078874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.078881:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.078883:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.078884:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.078885:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.078887:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.078889:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.078890:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.078891:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.078892:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.078893:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.078896:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.078897:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.078900:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.078901:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.078901:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.2:1619677448.078903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677448.078903:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.078904:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.078904:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677448.078905:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.078905:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677448.078908:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.078910:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.078911:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.078913:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.078915:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:22.0:1619677448.078916:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:20.0:1619677448.078916:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.078917:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.078918:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.078919:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:20.0:1619677448.078919:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677448.078920:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.078920:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.078923:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.078924:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.078926:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677448.078927:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.078930:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.078931:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.078932:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.078934:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.078934:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.078935:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.078936:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.078946:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.078949:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.078950:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.078951:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.078953:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.078954:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.078961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.078968:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.078971:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.078973:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.078975:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.078976:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.078978:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677448.078979:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.078980:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.078981:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.078982:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677448.078986:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.078987:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.078991:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.078991:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677448.078992:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.078994:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677448.078994:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.078995:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.078996:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.079007:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.079010:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.079010:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.079011:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.079013:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.079016:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.079024:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.079026:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.079027:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.079028:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.079029:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.079029:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.079030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.079036:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.079039:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.079040:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.079041:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.079044:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.079045:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.079047:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.079048:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.079049:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.079050:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.079053:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.079055:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.079057:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.079058:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.079058:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.079060:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.079060:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.079061:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.079062:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.079075:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.079078:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.079079:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.079079:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.079081:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.079083:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.079087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:20.2:1619677448.079091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:24.0:1619677448.079093:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.079096:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.079097:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:31.0:1619677448.079099:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:24.0:1619677448.079099:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.079101:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.079102:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:24.0:1619677448.079103:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677448.079105:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.079105:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.079106:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:24.0:1619677448.079106:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.079107:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.079108:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.079108:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677448.079109:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.079111:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.079113:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.079113:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.079114:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.079115:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:24.0:1619677448.079115:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:31.0:1619677448.079119:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.079119:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.079120:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.079120:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.079121:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677448.079122:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.079123:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.079123:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:24.0:1619677448.079123:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:31.0:1619677448.079124:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:24.0:1619677448.079124:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.079124:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:31.0:1619677448.079125:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.079126:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.079127:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.079127:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:9.2:1619677448.079128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.079136:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:30.0:1619677448.079138:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:2.0:1619677448.079139:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.079140:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.079140:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.079141:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.079141:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.079143:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.079144:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677448.079144:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.079147:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677448.079148:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677448.079156:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.079158:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.079159:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.079160:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.079160:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.079161:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.079428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.079436:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.079438:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.079440:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.079443:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.079443:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.079444:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.079445:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.079446:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.079447:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.079448:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.079451:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.079453:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.079454:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.079455:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.079456:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.079457:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.079457:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.079459:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.079461:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.079461:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677448.079467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.079470:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.079473:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:30.0:1619677448.079474:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:26.0:1619677448.079474:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.079474:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.079476:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.079477:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.079478:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.079478:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:30.0:1619677448.079479:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677448.079515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.079522:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.2:1619677448.079523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.079525:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.079527:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.079529:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.079529:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.079529:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677448.079531:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.079532:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.079532:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.079533:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677448.079535:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.079536:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.079536:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.079537:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677448.079539:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.079540:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.079541:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:30.0:1619677448.079542:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:17.0:1619677448.079543:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.079544:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.079545:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.079546:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.079546:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.079547:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.079548:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.079549:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.079550:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.079550:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.079551:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.079551:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:17.0:1619677448.079552:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.079555:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.079556:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.079556:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.2:1619677448.079556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677448.079558:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.079559:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.079560:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.079561:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.079562:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.079566:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.079569:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:12.0:1619677448.079571:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677448.079571:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.079572:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.079572:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:20.0:1619677448.079574:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:12.0:1619677448.079574:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:5.0:1619677448.079574:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.079574:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:20.0:1619677448.079575:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.079575:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677448.079575:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.079575:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.079576:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:20.0:1619677448.079577:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (24)--
00000400:00000200:12.0:1619677448.079577:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.079577:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.079578:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:13.0:1619677448.079579:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677448.079579:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677448.079581:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.079725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.079731:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.079734:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.079734:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.079736:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.079739:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.079740:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.079742:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.079743:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.079744:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.079745:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.079749:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.079751:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.079754:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.079755:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.079755:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.079757:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.079758:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.079758:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.079759:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677448.079770:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.079773:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.079774:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.079776:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.079777:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.079779:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.079787:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.079790:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.079791:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.079791:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.079792:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.079794:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.079828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.079835:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.079838:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.079839:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.079840:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.079843:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.079844:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.079845:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.079847:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.079848:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.079849:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.079852:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.079854:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.079857:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.079858:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.079858:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.079860:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.079860:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.079861:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.079862:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.079873:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:21.2:1619677448.079876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:6.0:1619677448.079876:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.079877:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.079877:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.079879:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.079882:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677448.079883:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.079885:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.079886:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.079888:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.079890:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.079891:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.079893:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719629] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.079894:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.079895:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.079896:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.079899:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.079901:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.079903:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.079904:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.079904:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.079906:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.079906:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.079907:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.079909:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.079909:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.2:1619677448.079914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677448.079919:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677448.079920:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:22.0:1619677448.079922:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677448.079922:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:6.0:1619677448.079923:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677448.079923:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:22.0:1619677448.079924:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.079924:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:22.0:1619677448.079926:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.079929:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.079987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:21.2:1619677448.079991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677448.079994:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.079997:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.079998:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:31.0:1619677448.079999:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:8.0:1619677448.080000:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.080001:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:8.0:1619677448.080003:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.080004:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.080004:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:30.0:1619677448.080005:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.2:1619677448.080005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:31.0:1619677448.080006:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.080006:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.080007:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:8.0:1619677448.080007:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.080008:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:8.0:1619677448.080008:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.080010:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.080010:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:8.0:1619677448.080010:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677448.080012:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.080013:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:31.0:1619677448.080014:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.080014:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:31.0:1619677448.080015:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:8.0:1619677448.080015:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.080017:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.080018:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:8.0:1619677448.080018:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.080020:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96719626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.080021:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.080021:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.080022:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.080022:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.080022:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.080023:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.080024:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:20.0:1619677448.080024:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677448.080025:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.080025:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.080026:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.080027:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:31.0:1619677448.080028:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.080028:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677448.080028:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:31.0:1619677448.080029:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677448.080029:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.0:1619677448.080030:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.080031:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:9.2:1619677448.080031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.080032:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.080033:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.080033:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677448.080033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677448.080033:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.080034:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.080034:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.080036:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.080037:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.080037:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.080038:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677448.080038:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677448.080040:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:18.0:1619677448.080041:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:24.0:1619677448.080042:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:18.0:1619677448.080042:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.080042:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:24.0:1619677448.080043:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.080044:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:18.0:1619677448.080045:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.080045:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:13.0:1619677448.080048:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:9.2:1619677448.080048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.080048:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.080049:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.080049:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.080051:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677448.080051:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:6.0:1619677448.080051:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:13.0:1619677448.080052:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.080053:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:13.0:1619677448.080054:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677448.080054:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.080055:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677448.080055:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677448.080056:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.080056:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.080057:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677448.080057:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677448.080058:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677448.080066:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.080069:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.080070:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.080071:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.080071:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.080073:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.080221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.080228:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.080230:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.080231:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.080234:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.080236:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.080238:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.080239:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.080240:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.080241:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.080242:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.080245:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.080245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:26.0:1619677448.080247:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.080250:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.080250:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.080251:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.080252:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677448.080253:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:26.0:1619677448.080254:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.080254:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.080254:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677448.080255:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.080256:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677448.080256:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.080258:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.080260:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.080261:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.080261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:22.0:1619677448.080262:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.080263:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.080264:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.080268:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.080268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677448.080269:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:22.0:1619677448.080270:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677448.080271:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.080272:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.080273:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.080273:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677448.080273:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677448.080274:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.080275:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677448.080275:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.080276:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.080276:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.080277:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.080278:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.080278:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:2.0:1619677448.080278:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:26.0:1619677448.080279:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.080281:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.080285:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.080287:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.080288:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.080289:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.080290:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.080290:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.080291:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.080291:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.080292:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.080300:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.080303:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.080304:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.080305:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.080306:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:10.0:1619677448.080308:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.080309:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.080310:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.080311:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.080312:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.080312:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.080313:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.080326:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.080329:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.080330:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.080331:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.080333:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.080338:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.080339:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.080341:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.080342:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.080342:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.080343:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.080344:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.080396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.080405:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.080408:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.080410:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.080412:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.080412:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.080414:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.080415:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719622] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.080416:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.080417:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.080418:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.080421:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.080422:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.080425:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.080425:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.080426:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.080427:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.080428:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.080429:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.080429:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677448.080441:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.080444:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.080445:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.080446:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.080448:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.080449:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.080478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.080486:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.080489:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.080490:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.080492:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.080495:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.080496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:8.0:1619677448.080497:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.080498:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.080499:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.080500:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.080501:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.080502:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677448.080504:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.080505:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.080506:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.080507:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677448.080509:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.080510:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.080510:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:1.0:1619677448.080510:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.080512:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.080512:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.080513:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.080513:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.080514:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.080514:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.080515:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.080516:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.080517:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.080518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:1.0:1619677448.080518:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.080522:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.080524:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.080527:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.080527:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.080527:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.080528:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.080528:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:17.0:1619677448.080529:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.080530:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.080531:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.080531:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.080532:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.080533:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677448.080534:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.080534:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.080536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.080544:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.080546:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.080546:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.080547:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.080548:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.080549:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.080643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.080651:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.080655:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.080656:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.080658:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.080661:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.080662:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.080663:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.080665:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.080666:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.080667:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.080670:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.080671:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.080674:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.080675:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.080675:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.080677:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.080677:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.080679:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.080680:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.080680:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.080686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.080691:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.080693:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.080694:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.080694:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.080695:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.080695:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.080696:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.080696:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.080697:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.080698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677448.080700:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.080704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677448.080705:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.080708:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.080709:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.080710:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.080712:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677448.080713:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.080714:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.2:1619677448.080714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:10.0:1619677448.080715:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.080716:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.080717:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.080718:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677448.080718:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.080718:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.080719:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.080719:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.080719:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:31.0:1619677448.080721:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677448.080722:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.080722:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:31.0:1619677448.080723:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.080724:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.080724:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.080725:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.080725:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.080726:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.080727:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:6.0:1619677448.080728:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.080729:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.080730:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.080730:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.080731:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.080732:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.080732:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.080732:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.080733:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.080733:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677448.080733:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.080735:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96719616] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.080736:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.080736:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:6.0:1619677448.080736:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.080737:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:6.0:1619677448.080737:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.080738:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.080738:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:31.0:1619677448.080739:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.080740:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.080740:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:6.0:1619677448.080742:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.080744:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677448.080744:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677448.080746:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.080747:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.080747:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677448.080748:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.080748:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:12.0:1619677448.080750:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677448.080750:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:13.0:1619677448.080751:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.080752:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.080753:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.080753:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.080754:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.080755:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.2:1619677448.080755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:6.0:1619677448.080755:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000400:00000200:22.0:1619677448.080756:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.080756:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.080757:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.080758:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.080761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.080763:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:26.0:1619677448.080764:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:22.0:1619677448.080764:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.080765:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:26.0:1619677448.080766:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:22.0:1619677448.080766:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:10.0:1619677448.080766:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:22.0:1619677448.080767:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.080768:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.080769:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.080769:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.080771:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.080771:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.080774:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.080775:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.080777:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.080778:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.080779:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.080779:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.080781:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.080781:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000400:00000200:26.0:1619677448.080783:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.080783:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.080784:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.080785:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.080785:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.080785:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.080786:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:26.0:1619677448.080789:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.080790:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.080790:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.080792:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.080792:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.080793:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.080794:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.080798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.080806:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.080808:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.080809:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.080809:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.080812:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.080814:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.080827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.080833:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.080835:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.080838:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.080838:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.080841:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.080842:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.080844:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.080845:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.080846:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.080847:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.080851:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.080853:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.080856:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.080857:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.080858:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.080860:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.080860:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.080862:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.080863:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.080864:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.080868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.080871:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:30.0:1619677448.080872:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.080873:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.080874:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677448.080875:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677448.080875:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:30.0:1619677448.080876:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.080878:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.080879:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.080881:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.080929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.080935:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.080938:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.080938:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.080940:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.080942:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.080943:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.080945:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.080946:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.080947:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.080948:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.080950:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.080952:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.080954:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.080955:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.080955:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.080957:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.080957:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.080958:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.080958:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.080963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.080969:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.080971:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.080972:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.080973:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.080973:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.080975:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.080993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.081000:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.081003:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.081004:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.081006:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.081009:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.081011:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.081012:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.081013:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.081014:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.081015:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.081019:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.081020:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.081022:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.081023:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.081024:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.081025:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.081026:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.081026:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.081027:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.081038:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.081041:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.081041:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.081043:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.081045:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.081046:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.081145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.081152:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.081155:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.081156:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.081158:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.081160:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.081162:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.081163:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.081165:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.081165:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.081166:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677448.081169:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.081171:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.081174:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.081175:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.081175:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.081177:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.081177:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.081178:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.081179:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.081183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.081190:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.081193:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.081194:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.081196:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.081197:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.081199:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.081202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.081207:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.081210:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.081211:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.081213:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.081216:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.081218:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.081219:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.081220:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.081221:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.081222:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.081226:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.081227:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.081230:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.081230:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.081231:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.081233:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.081234:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.081234:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.081235:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.081240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.081248:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.081251:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.081252:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.081252:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.081254:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.081256:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.081298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:4.2:1619677448.081303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.081307:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.081309:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:26.0:1619677448.081311:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677448.081312:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.081313:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:31.0:1619677448.081314:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.081314:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:30.0:1619677448.081315:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.081316:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.081316:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.081318:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.081319:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.081319:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.081320:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.081320:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:5.2:1619677448.081321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677448.081322:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.081322:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.081324:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.081325:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.081326:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.081326:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:17.0:1619677448.081328:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.081329:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.081331:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.081331:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.081331:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.081332:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.081332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677448.081333:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.081333:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.081334:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:17.0:1619677448.081334:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.081335:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.081335:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.081335:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.081336:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.081336:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.081337:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:26.0:1619677448.081337:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:17.0:1619677448.081337:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:26.0:1619677448.081338:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:17.0:1619677448.081338:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96719607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.081339:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.081339:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677448.081339:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677448.081339:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.081340:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.081340:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.081341:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:17.0:1619677448.081341:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.081342:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:17.0:1619677448.081342:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.081342:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:20.2:1619677448.081345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677448.081345:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.081346:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.081348:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.081348:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.081349:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:10.0:1619677448.081349:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:17.0:1619677448.081351:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.081351:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96719606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.081352:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:17.0:1619677448.081352:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677448.081352:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:26.0:1619677448.081353:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:17.0:1619677448.081353:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (24)++
00000400:00000200:12.0:1619677448.081353:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.081353:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.081354:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:26.0:1619677448.081354:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.081355:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:10.0:1619677448.081355:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.081356:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:17.0:1619677448.081356:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.081356:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677448.081356:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.081356:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.081357:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677448.081357:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.081357:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.081357:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.081358:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.081359:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:10.0:1619677448.081360:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.081362:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.081362:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.081363:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.081364:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.081365:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.081365:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:13.0:1619677448.081366:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:10.0:1619677448.081366:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.081367:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.081368:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.081368:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.081369:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.081370:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677448.081370:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677448.081371:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.081372:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.081375:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.081376:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.081377:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:13.0:1619677448.081377:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.081379:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:13.0:1619677448.081379:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.081379:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:26.0:1619677448.081380:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.081381:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.081382:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.081383:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.081395:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.081397:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.081398:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.081398:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.081399:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.081399:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.081414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.081421:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.081423:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.081424:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.081426:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.081429:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.081430:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.081432:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.081433:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.081434:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.081435:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.081438:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.081439:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.081442:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.081442:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.081443:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.081445:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.081445:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.081446:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.081446:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677448.081458:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.081461:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.081462:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.081462:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.081464:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.081467:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.081580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.081587:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.081590:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.081592:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.081594:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.081594:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.081596:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.081597:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.081598:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.081600:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.081601:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.081604:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.081606:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.2:1619677448.081607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:31.0:1619677448.081608:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.081609:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.081610:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.081611:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.081612:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.081613:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.081613:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677448.081613:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.081616:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.081617:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.081619:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.081622:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.081623:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:12.0:1619677448.081623:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:18.0:1619677448.081624:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.081625:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:18.0:1619677448.081626:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.081626:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.081627:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.081628:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.081629:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.081629:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.081631:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.081632:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677448.081634:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.081636:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.081637:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.081638:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.081640:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.081640:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.081641:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.081641:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.081646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.081655:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.081657:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.081658:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.081660:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.081661:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.081663:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.081678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.081685:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.081687:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.081688:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.081690:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.081691:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.081693:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.081694:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719602] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.081695:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.081697:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.081697:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.081701:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.081703:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.081706:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.081707:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.081708:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.081709:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.081710:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.081711:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.081711:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.081724:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.081727:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.081727:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.081728:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.081730:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.081736:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.081738:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.081740:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.081741:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.081741:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.081742:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.081742:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.081908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.081914:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.081917:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.081918:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.081920:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.081922:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.081924:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.081925:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.081927:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.081927:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.081929:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.081932:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.081934:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.081936:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.081937:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.081938:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.081940:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.081940:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.081942:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.081943:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.081943:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.081946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.081948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.081952:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677448.081954:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677448.081955:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.081956:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.081956:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.081956:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677448.081956:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.081957:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.081958:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.081961:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.082027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.082034:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.082037:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.082038:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.082040:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.082042:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.082044:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.082045:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.082046:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.082047:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.082048:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.082051:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.082053:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.082055:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.082056:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.082056:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.082058:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.082058:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.082060:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.082061:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.082061:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.082067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.082070:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.082073:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.082074:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.082074:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:30.0:1619677448.082075:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.082076:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.082076:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.082077:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.082077:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.082078:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.082192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.082197:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.082200:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.082201:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.082203:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.082206:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.082207:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.082209:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.082210:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.082211:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.082212:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.082216:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.082218:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.082221:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.082221:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.082222:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.082224:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.082224:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.082225:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.082225:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:10.0:1619677448.082236:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.082239:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.082239:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.082240:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.082242:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.082245:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.082260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.082267:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.082270:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.082271:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.082273:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.082275:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.082276:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.082278:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.082279:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.082280:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.082281:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.082284:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.082286:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.082289:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.082290:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.082291:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.082292:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.082298:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.2:1619677448.082298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.082299:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.082300:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677448.082307:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.082309:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.082310:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.082311:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.082313:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.082316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677448.082316:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677448.082323:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.082326:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.082327:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.082328:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.082331:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.082333:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.082334:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.082335:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.082336:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.082337:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.082340:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.082342:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.082344:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.082345:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.082346:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.082347:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.082348:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.082348:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.082349:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677448.082362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:22.0:1619677448.082363:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.082365:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.082366:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.082367:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.082368:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.082369:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:22.0:1619677448.082371:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677448.082372:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.082372:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.082375:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.082378:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.082379:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.082380:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.082382:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.082383:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.082384:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.082387:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.082389:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.082392:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.082392:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.082393:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.082395:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.082395:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.082396:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.082396:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.082401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.082409:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.082412:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.082412:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.082414:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.082416:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.082417:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.082492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.082500:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.082502:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.082503:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.082505:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.082508:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.082509:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.082511:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.082512:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.082513:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.082514:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.082517:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.082519:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.082521:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.082522:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.082522:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.082524:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.082524:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.082526:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.082527:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.082527:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.082533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677448.082537:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.082539:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:26.0:1619677448.082540:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:13.0:1619677448.082540:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.082541:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.082542:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.082542:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.082543:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.082544:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.082545:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.082559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.082564:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.082567:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.082568:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.082570:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.082572:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.082574:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.082575:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.082577:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.082578:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.082579:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.082582:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.082584:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.082587:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.082588:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.082588:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.082590:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.082591:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.082592:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.082592:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.082603:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.082606:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.082607:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.082609:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.082609:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.082611:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.082845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.082852:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.082855:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.082857:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.082860:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.082860:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.082861:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.082862:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.082864:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.082865:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.082866:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:21.2:1619677448.082867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000400:00000200:31.0:1619677448.082869:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.082871:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.082873:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.082874:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.082874:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:31.0:1619677448.082875:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.082876:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.082877:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.082877:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.082877:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:31.0:1619677448.082878:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.082879:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:10.0:1619677448.082879:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.082880:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677448.082881:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.082883:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.082884:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.082885:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.082886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677448.082886:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.082887:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677448.082887:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:12.0:1619677448.082890:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:10.0:1619677448.082890:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677448.082891:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.082892:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.082892:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.082893:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.082894:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.082895:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:12.0:1619677448.082895:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.082895:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.082895:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:30.0:1619677448.082896:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677448.082896:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.082897:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.082898:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.082898:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.082899:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677448.082901:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677448.082901:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:13.0:1619677448.082904:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:9.2:1619677448.082904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.082907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:13.0:1619677448.082907:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.082907:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.082908:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.082908:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.082909:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.082911:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.082914:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:2.0:1619677448.082914:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.082916:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:18.0:1619677448.082916:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.082916:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.082917:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.082917:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:2.0:1619677448.082918:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.082921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.082923:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.082925:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.082926:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.082927:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.082929:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.082931:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.082932:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.082934:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.082935:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.082936:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.082937:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.082940:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.082942:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.082944:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.082945:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.082946:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.082947:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.082948:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.082948:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.082949:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.082953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.082962:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.082965:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.082966:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.082967:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.082968:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.082970:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.083005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.083007:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.083010:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.083012:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.083013:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.083015:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.083016:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.083018:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.083019:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.083020:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.083021:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.083024:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.083026:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.083029:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.083030:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.083030:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.083032:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.083033:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.083034:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.083035:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.083045:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.083047:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.083048:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.083050:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.083051:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.083053:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.083086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.083093:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.083096:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.083098:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.083100:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.083100:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.083102:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.083103:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.083104:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.083105:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.083106:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.083110:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.083111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677448.083112:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.083115:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.083115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.083116:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.083117:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.083118:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.083119:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.083119:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.083120:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.083120:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.083121:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:1.0:1619677448.083121:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.083122:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677448.083123:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:26.0:1619677448.083124:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.083124:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.083124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:1.0:1619677448.083127:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.083128:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.083129:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.083130:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.083131:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.083131:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677448.083133:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:1.0:1619677448.083133:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.083134:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.083135:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.083135:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:1.0:1619677448.083135:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.083136:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.083137:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.083138:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677448.083138:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.083138:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.083139:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.083141:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:26.0:1619677448.083144:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.083144:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.083147:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.083148:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.083150:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.083151:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.083151:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.083152:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.083152:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:4.2:1619677448.083153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677448.083153:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.083154:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.083155:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.083155:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.083156:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.083157:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677448.083157:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.083158:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:1.0:1619677448.083158:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.083160:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.083161:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.083161:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.083164:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.083166:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.083167:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:6.0:1619677448.083168:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:26.0:1619677448.083169:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96719586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.083170:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.083170:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:26.0:1619677448.083171:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.083171:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:12.0:1619677448.083171:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:26.0:1619677448.083172:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:6.0:1619677448.083172:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.083174:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.083175:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.083175:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.083177:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677448.083177:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.083178:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677448.083179:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:26.0:1619677448.083180:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:12.0:1619677448.083180:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.083183:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.083184:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.083185:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.083187:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.083187:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.083187:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.083188:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.083189:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.083190:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.083190:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.083191:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.083192:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.083192:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.2:1619677448.083192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.083200:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.083202:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.083203:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.083205:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.083206:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.083208:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.083214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.083222:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.083225:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.083226:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.083226:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.083227:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.083228:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.083356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.083362:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.083365:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.083366:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.083368:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.083371:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.083372:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.083373:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.083375:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.083376:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.083377:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.083380:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.083382:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.083385:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.083386:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.083387:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.083389:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.083389:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.083390:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.083390:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:10.0:1619677448.083403:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.083406:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.083406:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.083407:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.083409:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.083412:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.083459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.083465:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.083468:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.083470:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.083470:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677448.083473:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.083475:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:5.0:1619677448.083476:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.083477:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.083478:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.083479:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:5.0:1619677448.083483:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.083485:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:5.0:1619677448.083488:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.083489:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:5.0:1619677448.083489:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:5.0:1619677448.083491:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:5.0:1619677448.083492:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.083493:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.083495:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.083495:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:9.2:1619677448.083497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:20.2:1619677448.083500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.083503:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:8.0:1619677448.083504:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:20.0:1619677448.083505:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.083506:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.083507:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.0:1619677448.083507:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.083508:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677448.083508:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.083509:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:8.0:1619677448.083510:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:8.0:1619677448.083513:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:2.0:1619677448.083515:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.083517:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.083518:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677448.083519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000400:00000200:2.0:1619677448.083520:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.083523:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.083524:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:24.0:1619677448.083525:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000400:00000200:2.0:1619677448.083526:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.083527:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.083529:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677448.083529:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:2.0:1619677448.083529:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.083530:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000400:00000200:2.0:1619677448.083530:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.083531:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.083531:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:2.0:1619677448.083533:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.083535:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.083537:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.083537:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.083538:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.083539:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.083539:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.083541:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.083542:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.083543:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677448.083545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.083548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.083554:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.083554:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.083555:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.083555:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.083555:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.083556:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.083557:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.083557:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.083558:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677448.083559:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.083658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.083664:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.083667:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.083668:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.083670:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.083672:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.083674:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.083675:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.083676:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.083677:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.083678:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.083682:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.083684:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.083686:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.083687:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.083688:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.083689:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.083689:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.083690:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.083690:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.083695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.083703:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.083706:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.083706:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.083708:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.083710:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.083711:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.083713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.2:1619677448.083722:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.2:1619677448.083724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.083725:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.083726:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.083727:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.083728:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:17.0:1619677448.083728:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.083728:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.083728:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.083729:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:20.0:1619677448.083730:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.083730:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677448.083731:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:5.0:1619677448.083731:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677448.083732:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677448.083733:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.083734:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.083735:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.083736:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.083737:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.083737:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.083738:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.083739:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:20.0:1619677448.083740:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677448.083740:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.083742:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.083743:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.083744:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.083745:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:20.0:1619677448.083747:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:17.0:1619677448.083749:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.083750:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.083751:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.083751:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.083751:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:20.0:1619677448.083753:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.083754:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.083754:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677448.083754:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.083755:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677448.083755:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.083756:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.083758:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.083758:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.083759:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.083759:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.083764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:22.0:1619677448.083767:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.083769:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.083770:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.083770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677448.083770:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:6.0:1619677448.083770:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.083772:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.083773:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.083773:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677448.083774:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.083775:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:8.0:1619677448.083776:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.083778:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677448.083779:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:31.0:1619677448.083781:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.083783:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.083786:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.083786:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.083787:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.083789:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.083790:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.083791:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.083792:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.083795:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.083797:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.083800:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.083800:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.083801:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.083802:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.083803:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.083804:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.083804:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677448.083809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.083816:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677448.083816:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.083818:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:30.0:1619677448.083819:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:13.0:1619677448.083819:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.083821:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.083822:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.083823:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:12.0:1619677448.083823:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.083824:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:30.0:1619677448.083825:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.083827:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.083828:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719578] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.083829:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.083830:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.083831:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.083835:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.083836:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.083839:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.083840:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.083841:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.083842:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.083843:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.083844:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.083844:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.083848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.083858:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.083860:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.083861:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.083863:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.083863:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.083866:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.084051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.084058:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.084061:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.084062:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.084064:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.084067:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.084068:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.084070:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.084071:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.084072:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.084073:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.084076:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.084078:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.084080:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.084081:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.084082:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.084084:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.084084:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.084085:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.084085:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:4.2:1619677448.084089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:9.2:1619677448.084090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.084095:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.084098:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:2.0:1619677448.084098:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677448.084099:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.084100:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.084100:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.084102:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:8.0:1619677448.084102:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.084102:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.084104:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.084105:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.084106:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96719576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.084107:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677448.084108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677448.084108:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.084109:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.084112:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:8.0:1619677448.084113:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.084114:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.084115:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.084116:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.084116:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.084117:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.0:1619677448.084118:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.084118:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.084118:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.084119:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.084119:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.0:1619677448.084120:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.084121:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.084122:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.084124:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.084124:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.084126:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.084130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.084134:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.084137:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.084137:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.084138:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.084138:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.084139:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.084139:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.084140:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.084141:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.084144:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.084145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.084153:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.084156:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.084157:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.084157:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.084158:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.084159:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.084322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.084331:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.084333:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.084336:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.084338:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.084339:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.084340:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:20.2:1619677448.084340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:31.0:1619677448.084341:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.084342:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.084343:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.084343:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.084344:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:20.0:1619677448.084346:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.084349:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.084349:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677448.084350:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.084351:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.084352:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:20.0:1619677448.084353:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677448.084354:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.084354:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.084355:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.084355:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:20.0:1619677448.084356:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.084357:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.084357:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:20.0:1619677448.084357:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.084358:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.084358:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.084359:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.084361:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.084363:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.084366:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.084367:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.084367:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.084369:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.084370:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:20.0:1619677448.084370:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.084370:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.084371:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.084373:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.084374:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.084375:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.084376:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.084378:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:12.0:1619677448.084382:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.084385:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.084386:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.084387:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.084389:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.084390:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677448.084400:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.084403:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.084403:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.084404:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.084404:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.084405:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.084490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.084498:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.084501:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.084501:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.084503:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.084506:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.084507:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.084508:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719573] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.084510:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.084511:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.084512:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.084515:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.084516:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.084519:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.084520:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.084521:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.084522:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.084523:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.084523:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.084524:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.084537:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.084539:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:5.2:1619677448.084539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677448.084540:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.084541:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.084543:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.084545:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.084546:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677448.084547:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.084548:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.084550:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.084552:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.084554:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.084555:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719572] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.084557:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.084558:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.084559:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.084562:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.084564:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.084567:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.084568:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.084569:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.084571:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.084571:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.084573:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.084574:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.084575:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.084580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.084583:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677448.084585:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677448.084586:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.084587:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.084587:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.084588:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677448.084589:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.084589:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.084590:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.084592:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.084600:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.084603:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.084604:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.084605:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.084606:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.084607:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.084658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.084666:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.084669:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.084670:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.084672:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.084674:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.084676:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.084677:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.084678:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.084679:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.084680:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.084684:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.084685:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.084688:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.084689:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.084690:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.084692:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.084692:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.084693:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.084693:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.2:1619677448.084693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:9.2:1619677448.084698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.084700:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.084703:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.084704:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.084705:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.084706:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:10.0:1619677448.084708:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.084709:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:10.0:1619677448.084709:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.084710:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.084710:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:10.0:1619677448.084711:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.084712:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.084713:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.084714:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.084714:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.084718:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.084720:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.084720:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.084723:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.084724:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.084725:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.084727:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.084727:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.084728:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.084729:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.084733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.084737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677448.084740:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:21.2:1619677448.084741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.084743:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:30.0:1619677448.084744:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:2.0:1619677448.084744:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.084745:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.084746:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:2.0:1619677448.084746:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.084747:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:18.0:1619677448.084748:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.084749:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:30.0:1619677448.084750:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.084750:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:31.0:1619677448.084751:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.084752:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.084752:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:30.0:1619677448.084753:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677448.084755:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.084755:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.084756:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.084757:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.084758:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.084758:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.084759:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677448.084759:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.084761:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.084762:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.084764:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.084766:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:18.0:1619677448.084769:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:30.0:1619677448.084770:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.084771:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677448.084771:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677448.084771:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.084773:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677448.084774:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.084774:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677448.084774:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.084775:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677448.084775:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.084775:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.084777:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.084777:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.084778:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.084779:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677448.084789:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.084790:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.084790:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:22.0:1619677448.084791:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.084791:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.084792:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.084793:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.084794:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.084797:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.084798:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677448.084798:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.084799:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.084927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:21.2:1619677448.084931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:26.0:1619677448.084934:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:22.0:1619677448.084936:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.084937:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.084937:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:22.0:1619677448.084939:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.084939:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:26.0:1619677448.084941:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:22.0:1619677448.084943:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.084945:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.084945:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.084947:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.084948:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.084949:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.084950:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.084951:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.084951:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.084952:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.084953:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.084954:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.084954:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.084958:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.084960:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.2:1619677448.084961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.084963:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.084963:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.084963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:26.0:1619677448.084964:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.084965:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.084965:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.084966:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:1.0:1619677448.084966:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.084967:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.084968:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.084968:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:22.0:1619677448.084968:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.084968:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.084969:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (22)++
00000800:00000200:1.0:1619677448.084969:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.084970:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.084971:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:22.0:1619677448.084971:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677448.084972:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.084972:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677448.084972:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.084973:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.084973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677448.084974:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:1.0:1619677448.084975:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.084977:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.084977:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677448.084978:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:1.0:1619677448.084978:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96719565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.084980:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.084980:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.084981:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.084982:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:5.0:1619677448.084982:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)++
00000400:00000200:31.0:1619677448.084983:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96719564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.084983:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.084984:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (24)--
00000400:00000200:31.0:1619677448.084985:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.084985:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.084986:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.084987:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.084987:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)++
00000400:00000200:5.0:1619677448.084987:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677448.084988:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (24)--
00000400:00000200:20.0:1619677448.084989:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.084992:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.084992:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.084993:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.084995:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.084996:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.084999:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:1.0:1619677448.084999:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.085000:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.085000:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.085000:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677448.085000:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677448.085001:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:1.0:1619677448.085001:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.085002:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.085003:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:1.0:1619677448.085003:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.085003:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.085004:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:1.0:1619677448.085004:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.085005:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:1.0:1619677448.085005:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.085007:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.085008:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.085009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.085014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.085019:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.085021:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.085022:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.085023:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677448.085024:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:30.0:1619677448.085024:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.085025:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.085026:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677448.085026:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677448.085027:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:20.0:1619677448.085027:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.085028:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.085029:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.085029:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.085029:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.085031:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.085031:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.085032:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:5.0:1619677448.085032:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.085033:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.085034:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.085123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.085130:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.085133:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.085134:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.085136:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.085139:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.085141:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.085142:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.085143:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.085144:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.085145:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.085149:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.085151:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.085154:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.085154:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.085155:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.085157:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.085157:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.085158:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.085160:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.085160:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.085166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.085170:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.085173:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.085173:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.085174:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.085174:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.085175:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.085176:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.085176:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.085177:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.085179:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.085526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:4.2:1619677448.085529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677448.085536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:10.0:1619677448.085536:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.085538:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.085539:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:20.0:1619677448.085539:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:10.0:1619677448.085539:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677448.085542:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.085542:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:10.0:1619677448.085543:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.085544:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.085545:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:10.0:1619677448.085546:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.085547:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.085547:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.085548:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.085549:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677448.085551:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.085551:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.085553:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.085553:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.085555:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.085556:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.085556:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.085557:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.085562:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.085563:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.085565:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677448.085565:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.085567:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96719560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.085567:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.085568:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.085569:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.085569:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:26.0:1619677448.085570:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.085570:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:26.0:1619677448.085571:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.085572:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.085572:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.085573:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.085574:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:10.0:1619677448.085575:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.085576:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.085576:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.085577:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.085579:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.085579:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.085580:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.085581:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.085582:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.085582:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.085583:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.085583:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677448.085584:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.085584:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677448.085585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.085586:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.085586:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.085586:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.085587:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.085587:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.085588:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.085589:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.085589:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.085591:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.085593:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.085594:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:2.0:1619677448.085596:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.085597:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.085597:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.085599:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.085600:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677448.085602:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677448.085603:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.085603:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.085604:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.085606:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.085609:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677448.085617:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.085620:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.085621:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.085621:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.085622:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.085622:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.085636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.085645:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.085648:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.085650:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.085652:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.085653:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.085654:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.085656:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.085656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:31.0:1619677448.085657:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.085658:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.085659:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.085662:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.085664:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.085665:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.085666:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677448.085667:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.085668:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.085668:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:30.0:1619677448.085668:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.085670:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.085670:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:30.0:1619677448.085670:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.085671:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.085672:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:30.0:1619677448.085672:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:31.0:1619677448.085673:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.085673:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.085674:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.085675:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.085676:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.085679:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.085680:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.085683:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.085683:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:22.0:1619677448.085683:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:30.0:1619677448.085684:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.085685:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.085686:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:22.0:1619677448.085686:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.085686:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:30.0:1619677448.085687:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.085687:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:22.0:1619677448.085687:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.085689:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.085692:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.085693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677448.085698:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677448.085699:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.085701:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:17.0:1619677448.085702:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:13.0:1619677448.085702:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.085703:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677448.085705:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.085706:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.085706:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.085708:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:17.0:1619677448.085709:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.085711:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:21.2:1619677448.085712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000400:00000200:17.0:1619677448.085712:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.085714:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.085715:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.085716:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.085718:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:17.0:1619677448.085719:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.085721:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:17.0:1619677448.085721:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677448.085721:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677448.085724:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.085724:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.085725:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:18.0:1619677448.085726:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.085726:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:18.0:1619677448.085727:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:17.0:1619677448.085727:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.085728:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.085728:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677448.085729:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.085729:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:18.0:1619677448.085730:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.085731:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.085732:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.085735:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.085737:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.085740:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.085741:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.085741:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:6.0:1619677448.085741:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:18.0:1619677448.085743:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:6.0:1619677448.085743:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.085744:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.085744:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:6.0:1619677448.085744:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.085745:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677448.085745:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:6.0:1619677448.085747:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.085749:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:26.0:1619677448.085755:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.085758:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.085759:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.085760:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.085762:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.085765:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.085779:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.085781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:26.0:1619677448.085782:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.085782:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.085783:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.085784:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.085784:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:22.0:1619677448.085786:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.085789:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.085790:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.085792:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.085794:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.085795:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.085797:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.085798:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.085799:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.085800:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.085803:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.085804:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.085807:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.085808:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.085808:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.085810:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.085811:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.085812:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.085813:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.085813:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.085817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.085818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.085823:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.085825:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.085825:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677448.085825:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677448.085826:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677448.085827:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.085829:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.085829:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.085830:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.085833:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.085973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:5.0:1619677448.085978:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:5.0:1619677448.085981:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.6.13@o2ib3
00000400:00000200:5.0:1619677448.085983:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.085984:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:5.0:1619677448.085986:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.085988:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:5.0:1619677448.085989:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.085991:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.085991:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.085992:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:5.0:1619677448.085995:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.085997:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:21.2:1619677448.086001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:5.0:1619677448.086001:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f90d380] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.086002:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e49d37c00] (20)++
00000800:00000200:5.0:1619677448.086003:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e49d37c00] (21)++
00000800:00000200:5.0:1619677448.086005:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e49d37c00] (22)--
00000800:00000200:5.0:1619677448.086006:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677448.086007:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (23)++
00000800:00000200:5.0:1619677448.086007:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:5.0:1619677448.086008:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677448.086009:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:5.0:1619677448.086009:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:24.2:1619677448.086011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (21)++
00000400:00000200:20.0:1619677448.086011:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.086013:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.086015:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)++
00000400:00000200:20.0:1619677448.086015:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677448.086016:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.086017:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e49d37c00] (23)--
00000400:00000200:24.0:1619677448.086018:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.086018:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.086018:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)--
00000400:00000200:20.0:1619677448.086019:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.086020:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677448.086022:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.086024:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.086026:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.086029:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)--
00000800:00000200:20.0:1619677448.086029:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.086030:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677448.086031:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677448.086033:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677448.086033:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677448.086034:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677448.086035:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677448.086038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.086045:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.086047:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.086048:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.086048:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.086048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:8.0:1619677448.086050:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.086052:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:26.0:1619677448.086055:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.086058:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.086058:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.086060:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.086063:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.086065:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.086065:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.086067:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.086068:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.086069:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:21.2:1619677448.086069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000400:00000200:26.0:1619677448.086072:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.086073:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:26.0:1619677448.086074:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:21.2:1619677448.086075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677448.086077:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.086077:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:26.0:1619677448.086078:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.086079:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.086079:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:5.0:1619677448.086080:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.086081:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.086081:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.086082:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.086083:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.086083:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:5.0:1619677448.086083:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.086084:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:31.0:1619677448.086086:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:5.0:1619677448.086086:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.086087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677448.086087:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.086088:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.086089:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.086089:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:30.0:1619677448.086090:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.086092:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.086093:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.086094:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:2.0:1619677448.086094:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:31.0:1619677448.086095:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96719550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.086096:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:31.0:1619677448.086097:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.086097:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:31.0:1619677448.086098:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.086098:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.086098:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.086099:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.086100:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.086100:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:5.0:1619677448.086101:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:2.0:1619677448.086101:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.086102:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:5.0:1619677448.086103:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000400:00000200:31.0:1619677448.086104:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.086104:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:5.0:1619677448.086104:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:2.0:1619677448.086104:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.086105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:31.0:1619677448.086106:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.086109:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.086110:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.086110:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.086112:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.086112:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.086112:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.086114:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.086115:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:18.0:1619677448.086115:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.086115:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:31.0:1619677448.086116:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677448.086116:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.086116:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.086117:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.086118:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.086118:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.086119:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.086119:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.086121:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.086122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.086124:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:22.0:1619677448.086125:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.086129:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:21.2:1619677448.086129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:6.0:1619677448.086129:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.086130:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:22.0:1619677448.086130:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.086132:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:22.0:1619677448.086132:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.086133:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.086134:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677448.086134:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:22.0:1619677448.086135:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677448.086137:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677448.086139:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.086141:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:20.0:1619677448.086142:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.086144:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:5.0:1619677448.086144:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677448.086145:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.086146:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.086146:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:20.0:1619677448.086147:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.086148:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.086149:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.086150:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677448.086153:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.086153:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.086155:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.086156:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.086157:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.086158:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.086159:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.086160:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.086160:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:17.0:1619677448.086160:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.086161:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.086162:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:17.0:1619677448.086162:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677448.086163:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677448.086164:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000400:00000200:17.0:1619677448.086165:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.086166:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677448.086167:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:17.0:1619677448.086167:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.086169:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.086170:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:21.2:1619677448.086171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:17.0:1619677448.086171:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.086172:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.086173:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.086174:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.086174:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.086175:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.086176:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:30.0:1619677448.086178:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:5.0:1619677448.086178:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000400:00000200:30.0:1619677448.086179:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.086179:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:30.0:1619677448.086180:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.086180:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:31.0:1619677448.086181:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.086181:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:5.2:1619677448.086182:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677448.086184:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:6.0:1619677448.086186:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.0:1619677448.086186:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:5.0:1619677448.086188:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.086189:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.086189:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.086190:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.086190:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.086191:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.086193:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.086193:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677448.086193:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.086193:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.086194:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719547] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.086195:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:26.0:1619677448.086196:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.086196:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.086197:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.086198:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.086199:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.086201:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677448.086202:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.086203:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.086204:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.086204:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.0:1619677448.086204:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.086205:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.086206:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:22.0:1619677448.086207:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.086207:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.086208:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.0:1619677448.086208:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.086209:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:5.0:1619677448.086209:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:26.0:1619677448.086210:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.086210:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677448.086210:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.086211:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.086212:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.086212:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:5.0:1619677448.086213:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:5.0:1619677448.086213:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:26.0:1619677448.086214:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.086215:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.086215:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.086217:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.086217:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.086218:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.086219:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.086220:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:21.2:1619677448.086223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:12.0:1619677448.086224:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.2:1619677448.086225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.086227:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.086228:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.086229:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000400:00000200:12.0:1619677448.086230:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.086231:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.086232:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:10.0:1619677448.086232:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:12.0:1619677448.086233:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.086233:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:20.0:1619677448.086234:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.086234:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.086235:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:21.2:1619677448.086235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:10.0:1619677448.086235:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.086236:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.086236:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:20.0:1619677448.086237:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.086237:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.086237:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:26.0:1619677448.086238:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:20.0:1619677448.086239:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:10.0:1619677448.086239:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:20.0:1619677448.086240:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.086240:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:20.0:1619677448.086241:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.086242:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.086243:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:6.0:1619677448.086243:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.086244:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.086247:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.086248:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.086250:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:6.0:1619677448.086250:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.086251:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:20.0:1619677448.086253:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.086253:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.086254:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.086255:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:6.0:1619677448.086255:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.086256:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.086257:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.086257:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000400:00000200:6.0:1619677448.086257:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:20.0:1619677448.086258:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677448.086259:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:6.0:1619677448.086260:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.086262:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.086265:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.086266:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.086266:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:26.0:1619677448.086268:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:6.0:1619677448.086268:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.086268:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.086269:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.086269:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:26.0:1619677448.086271:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.086272:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.086272:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.086274:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.086274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:26.0:1619677448.086277:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:2.0:1619677448.086281:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.086284:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.086285:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.086285:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.086287:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.086289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677448.086289:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:10.0:1619677448.086298:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.086301:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.086302:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.086303:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.086305:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.086307:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:5.2:1619677448.086307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000400:00000200:10.0:1619677448.086308:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.086310:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.086311:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.086312:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.086313:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:10.0:1619677448.086316:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.086316:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677448.086317:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:10.0:1619677448.086318:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.086319:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.086320:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.086321:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.086322:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:1.0:1619677448.086322:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.086323:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:10.0:1619677448.086324:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.086325:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.086325:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677448.086325:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.086326:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:1.0:1619677448.086326:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.086327:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.086328:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677448.086331:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.086332:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.086335:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.086336:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:1.0:1619677448.086336:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.086338:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:1.0:1619677448.086338:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677448.086338:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677448.086340:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:31.0:1619677448.086341:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.086341:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.086341:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677448.086341:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:31.0:1619677448.086343:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.086345:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.086346:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.086347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:22.0:1619677448.086352:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677448.086353:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:22.0:1619677448.086354:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.086355:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.086355:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677448.086356:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:6.0:1619677448.086356:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.086357:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.086357:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:22.0:1619677448.086360:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.086437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.086446:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.086449:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.086451:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.086452:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.086453:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.086455:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.086456:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.086458:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.086459:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.086460:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.086463:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.086464:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.086467:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.086468:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.086468:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.086470:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.086470:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.086472:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.086472:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677448.086483:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.086485:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.086486:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.086487:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.086489:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.086490:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677448.086499:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.086501:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.086502:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.086503:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.086503:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.086504:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.086564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.086570:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.086573:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.086575:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.086576:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.086578:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.086580:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.086582:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.086583:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.086584:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.086585:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.086588:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.086590:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.086593:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.086594:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.086594:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.086596:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.086597:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.086598:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.086599:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.086600:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.086605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.086608:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:30.0:1619677448.086609:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.086610:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.086611:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677448.086612:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677448.086612:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:30.0:1619677448.086613:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.086615:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.086616:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.086618:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.086627:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.086630:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.086631:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.086631:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.086632:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.086634:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.086651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.086658:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.086661:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.086662:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.086664:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.086666:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.086667:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.086668:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.086670:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.086671:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.086672:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:21.2:1619677448.086673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:26.0:1619677448.086675:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.086677:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.086679:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.086680:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.086680:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.086681:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.086682:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.086682:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.086683:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677448.086683:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.086684:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.086684:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:30.0:1619677448.086685:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.086687:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.086688:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.086689:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.086690:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677448.086690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677448.086692:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.086693:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.086693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677448.086694:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.086695:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.086696:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:30.0:1619677448.086697:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.086697:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.086698:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:1.0:1619677448.086698:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.086699:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.086699:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.086699:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.086699:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.086700:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.086701:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.086701:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:1.0:1619677448.086701:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.086702:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.086703:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.086704:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.086704:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:1.0:1619677448.086704:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.086706:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.086706:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.086707:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:1.0:1619677448.086707:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.086708:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:1.0:1619677448.086709:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.086710:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.086711:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.086712:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.086713:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.086717:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.086718:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:6.0:1619677448.086720:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:1.0:1619677448.086721:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.086722:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:1.0:1619677448.086722:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:22.0:1619677448.086723:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:1.0:1619677448.086723:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:6.0:1619677448.086724:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.086724:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.086725:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:6.0:1619677448.086726:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.086726:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.086727:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.086727:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.086729:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.086732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.086737:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.086738:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677448.086740:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.086740:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.086741:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.086741:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:22.0:1619677448.086742:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677448.086742:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.086742:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.086743:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.086744:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677448.086744:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.086745:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.086747:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.086748:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.086749:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.086823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:21.2:1619677448.086826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:18.0:1619677448.086830:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:6.0:1619677448.086832:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677448.086833:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.086834:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:22.0:1619677448.086835:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.086835:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:18.0:1619677448.086837:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.086838:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.086840:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.086840:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.086841:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.086843:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.086843:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677448.086845:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.086845:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.086847:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.086847:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.086848:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:6.0:1619677448.086848:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.086850:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677448.086854:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.086855:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.086857:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.086858:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677448.086860:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.086861:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.086862:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.086862:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.086862:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.086863:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:18.0:1619677448.086864:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.086865:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:6.0:1619677448.086865:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.086866:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677448.086867:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:6.0:1619677448.086867:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.086867:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677448.086868:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.086868:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.2:1619677448.086874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:12.0:1619677448.086876:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.086879:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.086879:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:12.0:1619677448.086880:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.086881:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:2.0:1619677448.086881:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:12.0:1619677448.086882:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.086882:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:13.0:1619677448.086883:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.086883:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:31.0:1619677448.086884:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.086885:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.086885:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677448.086887:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.086887:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.086890:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.087000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.087007:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.087010:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.087011:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.087013:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.087016:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.087018:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.087019:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.087020:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.087022:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.087023:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.087026:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.087028:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.087031:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.087032:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.087033:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.087034:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.087035:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.087035:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.2:1619677448.087035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:24.0:1619677448.087036:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.2:1619677448.087038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:8.2:1619677448.087040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.087041:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.0:1619677448.087044:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:18.0:1619677448.087044:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.087044:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.0:1619677448.087046:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:18.0:1619677448.087047:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677448.087049:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.087049:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:1.0:1619677448.087049:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:18.0:1619677448.087050:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.087051:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.087051:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:1.0:1619677448.087051:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.087052:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.087052:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.087053:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677448.087053:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.087054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:20.0:1619677448.087055:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.087055:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.087055:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.087057:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.087057:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.087058:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.087059:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.087059:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:20.0:1619677448.087060:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:2.0:1619677448.087061:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.087063:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:18.0:1619677448.087065:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000400:00000200:20.0:1619677448.087066:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.087067:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.087068:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:2.0:1619677448.087069:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.087070:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:2.0:1619677448.087071:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:18.0:1619677448.087072:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.087073:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.087073:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:2.0:1619677448.087073:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96719531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.087074:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:18.0:1619677448.087074:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.0:1619677448.087075:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:2.0:1619677448.087075:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.087076:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:2.0:1619677448.087076:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.087077:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677448.087077:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677448.087078:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:18.0:1619677448.087078:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:2.0:1619677448.087078:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.087079:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:18.0:1619677448.087080:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.087080:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677448.087081:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.087082:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.087083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:2.0:1619677448.087084:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.087087:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.087088:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.087088:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.0:1619677448.087090:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:2.0:1619677448.087090:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.087091:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677448.087091:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.087091:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.0:1619677448.087092:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:2.0:1619677448.087092:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:24.0:1619677448.087093:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:22.0:1619677448.087093:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.087094:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.087094:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.087094:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:8.2:1619677448.087096:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:22.0:1619677448.087097:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.087099:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.087100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000400:00000200:8.0:1619677448.087101:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.087103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677448.087105:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.0:1619677448.087106:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.087107:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:5.0:1619677448.087107:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.087109:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:5.0:1619677448.087109:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.087111:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677448.087111:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.087112:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.087114:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.087114:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:5.0:1619677448.087114:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:20.0:1619677448.087115:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.087116:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.087116:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:20.0:1619677448.087117:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:5.0:1619677448.087117:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677448.087118:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.087118:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.087120:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.087120:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.087121:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.087121:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.087122:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:10.0:1619677448.087123:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.087124:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.087126:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.087127:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.087129:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:10.0:1619677448.087130:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.087132:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.087132:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.087133:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.087133:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.087135:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.087135:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:10.0:1619677448.087135:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.087136:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.087136:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.087137:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677448.087138:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:10.0:1619677448.087138:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.087139:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:10.0:1619677448.087139:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.087140:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.087140:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.087141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.087143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:5.0:1619677448.087148:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:17.0:1619677448.087149:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677448.087150:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:30.0:1619677448.087151:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.087151:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:17.0:1619677448.087152:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:5.0:1619677448.087152:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:17.0:1619677448.087153:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.087153:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.087154:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.087155:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.087155:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.087157:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.087157:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.087158:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677448.087160:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.087451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.087457:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.087459:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.087460:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.087462:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.087465:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.087466:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.087468:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.087469:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.087470:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.087471:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.087475:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.087477:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.087480:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.087481:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.087482:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.087483:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.087484:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.087484:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.087485:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.087490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.087497:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.087500:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.087501:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.087501:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.087504:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.087506:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.087509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677448.087515:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.087518:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.087518:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677448.087519:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.087520:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.087520:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.2:1619677448.087587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:4.2:1619677448.087593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.087599:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.087601:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.087603:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:26.0:1619677448.087604:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.087604:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677448.087606:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.087606:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.087608:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.087609:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.087611:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.087612:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.087613:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.087614:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.087615:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.087617:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.087617:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.087618:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.087619:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.087619:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.087620:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.087623:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.087624:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.087626:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.087626:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.087627:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.087627:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.087628:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.087628:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.087629:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.087630:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.087630:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.087631:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.087631:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.087631:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.087631:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.087632:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.087632:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677448.087633:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.087640:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.087643:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.087643:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.087644:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.087645:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:10.0:1619677448.087645:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.087646:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677448.087648:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.087648:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.087650:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.087652:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.087653:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:26.0:1619677448.087657:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.087660:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.087660:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.087660:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.087661:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.087661:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.087680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.087687:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.087690:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.087691:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.087693:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.087695:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.087697:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.087698:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.087699:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.087700:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.087701:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.087705:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.087707:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.087710:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.087711:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.087711:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.087713:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.087713:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.087714:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.087714:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677448.087727:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.087729:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.087731:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.087732:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.087734:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.087735:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.087927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.087935:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.087938:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.087940:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.087942:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.087942:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.087944:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.087945:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.087946:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.087947:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.087948:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.087951:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.087953:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.087956:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.087957:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.087957:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.087959:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.087960:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.087961:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.087961:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.087965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.087974:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.087977:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.087978:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.087980:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.087980:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.087983:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.088050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.088056:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.088059:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.088060:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.088062:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.088065:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.088066:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.088068:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.088069:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.088070:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.088071:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.088074:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.088076:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.088078:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.088079:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.088080:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.088082:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.088082:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.088083:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.088083:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.088087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.088094:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.088097:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677448.088097:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677448.088098:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.088100:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.088103:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677448.088132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.088139:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.088142:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.088143:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.088145:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.088147:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.088149:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.088150:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.088151:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.088152:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.088153:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677448.088156:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.088158:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.088161:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.088161:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.088162:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.088164:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.088165:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.088166:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.088167:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.088167:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.2:1619677448.088170:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.2:1619677448.088171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:21.2:1619677448.088173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:2.0:1619677448.088176:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677448.088178:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677448.088179:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:20.0:1619677448.088180:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:18.0:1619677448.088180:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:24.0:1619677448.088181:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:20.0:1619677448.088181:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.088183:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:20.0:1619677448.088183:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.088183:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.088184:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677448.088184:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:2.0:1619677448.088185:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.088186:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677448.088186:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:2.0:1619677448.088187:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.088188:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.088189:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.088190:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.088191:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.088195:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.088196:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.088199:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.088199:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.088200:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.088202:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.088202:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.088203:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.088204:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.088208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.088216:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.088219:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.088219:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.088221:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.088221:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.088224:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.088275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.088278:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.088280:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.088283:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.088283:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.088285:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.088287:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677448.088288:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.088289:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.088290:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.088291:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677448.088300:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.088302:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.088304:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.088305:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.088306:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.088307:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.088308:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.088309:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.088309:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.2:1619677448.088312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.2:1619677448.088313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.088320:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677448.088320:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:10.0:1619677448.088323:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:5.0:1619677448.088323:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.088323:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.088324:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:5.0:1619677448.088325:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.088327:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:10.0:1619677448.088327:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.088328:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:10.0:1619677448.088329:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.088331:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.088332:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.088333:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.088334:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.088335:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.088339:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.088340:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:21.2:1619677448.088342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677448.088343:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.088344:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.088345:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.088346:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.088347:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.088347:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.088348:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.088350:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.088353:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.088355:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.088358:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.088358:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.088359:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.088360:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.088360:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:30.0:1619677448.088361:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.088362:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.088363:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.088365:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:30.0:1619677448.088366:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677448.088366:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.088367:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.088368:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.088369:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.088370:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.088370:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.088371:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.088372:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.088373:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.088374:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:30.0:1619677448.088375:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.088375:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.088375:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:30.0:1619677448.088376:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.088376:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.088376:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.088377:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.088385:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.088388:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.088388:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.088390:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.088392:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.088393:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.088501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677448.088507:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.088510:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677448.088510:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677448.088513:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.088516:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.088517:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677448.088519:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.088520:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.088521:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.088522:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677448.088525:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.088527:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:5.2:1619677448.088529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.088531:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.088532:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:1.0:1619677448.088533:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:17.0:1619677448.088534:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.088535:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.088535:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:1.0:1619677448.088536:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677448.088537:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677448.088538:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.088538:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:1.0:1619677448.088539:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.088540:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.088542:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.088543:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.088544:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.088544:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.088545:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.088547:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:17.0:1619677448.088548:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.088549:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:17.0:1619677448.088549:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.088550:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.088551:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.088551:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.088551:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.088552:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677448.088553:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.088553:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.088554:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.088554:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.088555:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.088556:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:8.2:1619677448.088560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.088565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:1.0:1619677448.088567:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.088569:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.088570:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.088571:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.088571:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:1.0:1619677448.088572:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.088574:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:1.0:1619677448.088574:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:24.0:1619677448.088575:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.088577:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000400:00000200:8.0:1619677448.088581:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.088584:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:8.0:1619677448.088585:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.088586:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:17.0:1619677448.088588:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.088588:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (21)++
00000400:00000200:8.0:1619677448.088588:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.088588:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.088589:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000400:00000200:8.0:1619677448.088589:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.088590:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:8.0:1619677448.088591:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.088591:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.088594:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.088596:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.088599:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.088600:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.088600:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.088602:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.088602:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.088603:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.088604:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.088608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.088616:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.088618:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.088619:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.088619:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.088621:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.088624:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.088629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.088636:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.088638:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.088639:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.088639:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.088640:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.088641:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.088681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.088688:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.088690:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.088691:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.088693:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.088695:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.088697:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.088698:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.088699:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.088700:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.088701:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.088704:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.088706:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.088709:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.088710:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.088710:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.088712:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.088712:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.088713:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.088714:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.088726:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.088730:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.088730:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.088731:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.088733:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.088736:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.088811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.088819:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.088822:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.088824:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.088825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:31.0:1619677448.088826:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.088826:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.088828:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677448.088829:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.088830:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:31.0:1619677448.088831:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.088831:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.088833:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.088833:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.088833:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:31.0:1619677448.088836:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.088838:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:22.0:1619677448.088840:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.088841:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.088842:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677448.088842:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:22.0:1619677448.088842:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.088844:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.088844:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:31.0:1619677448.088845:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:22.0:1619677448.088845:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.088846:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.088846:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:22.0:1619677448.088847:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.088847:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.088849:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.088853:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.088855:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.088858:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.088858:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.088859:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.088859:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.088860:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.088861:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:6.0:1619677448.088861:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.088862:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.088863:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.088863:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.088864:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677448.088865:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:8.2:1619677448.088867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.088867:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677448.088875:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:21.2:1619677448.088876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:20.0:1619677448.088877:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.088878:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.088880:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.088882:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.088882:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677448.088883:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677448.088885:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.088886:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.088888:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.088891:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.088892:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677448.088893:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.088895:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.088896:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.088897:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677448.088900:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.088903:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677448.088905:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.088906:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677448.088907:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.088908:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677448.088909:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.088909:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.088910:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677448.088914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.088922:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.088924:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.088925:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.088927:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.088928:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.088929:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.088936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.088944:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.088946:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.088948:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.088948:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.088949:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.088950:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.088959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.088967:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.088970:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.088970:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.088973:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.088976:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.088977:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.088978:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.088980:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.088981:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.088982:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.2:1619677448.088983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:26.0:1619677448.088985:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.088987:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.088990:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.088990:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.088991:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.088991:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.088993:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:26.0:1619677448.088993:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.088993:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:30.0:1619677448.088995:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.088995:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677448.088996:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.088996:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:31.0:1619677448.088997:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.088997:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.088999:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:8.2:1619677448.088999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.089000:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.089000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677448.089001:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.089002:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.089002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:30.0:1619677448.089003:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.089005:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.089006:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.089007:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677448.089007:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:10.0:1619677448.089009:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:1.0:1619677448.089009:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:30.0:1619677448.089010:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.089010:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.089010:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677448.089011:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.089011:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.089011:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:5.0:1619677448.089011:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.089012:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:30.0:1619677448.089013:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.089013:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.089013:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.089013:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:5.0:1619677448.089013:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.089013:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.089014:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:1.0:1619677448.089014:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.089015:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:30.0:1619677448.089016:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:5.0:1619677448.089016:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677448.089017:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:1.0:1619677448.089017:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.089018:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.089019:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.089020:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.2:1619677448.089022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:8.2:1619677448.089023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000400:00000200:1.0:1619677448.089023:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.089025:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.089027:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.089028:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:1.0:1619677448.089028:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677448.089029:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:1.0:1619677448.089029:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:31.0:1619677448.089030:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.089031:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.089031:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:1.0:1619677448.089031:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.089031:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.089032:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.089032:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:1.0:1619677448.089032:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:30.0:1619677448.089033:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:1.0:1619677448.089033:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:30.0:1619677448.089034:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677448.089034:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.089034:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.089035:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.0:1619677448.089035:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677448.089036:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677448.089036:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.089037:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.089038:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.089040:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.089044:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.089046:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677448.089047:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.089048:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.089048:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677448.089048:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.089049:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677448.089050:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.089050:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.089052:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.089093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.089096:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.089098:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.089101:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.089102:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.089103:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.089105:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.089106:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.089107:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.089108:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.089109:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.089113:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.089115:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.089117:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.089118:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.089119:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.089121:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.089121:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.089122:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.089123:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.089127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.089132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677448.089134:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.089137:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.089138:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.089138:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.089139:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:2.0:1619677448.089140:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.089142:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.089143:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:2.0:1619677448.089143:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:26.0:1619677448.089145:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.089148:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.089149:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.089150:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.089151:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.089152:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.089153:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.089157:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.089159:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.089161:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.089162:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.089162:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.089164:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.089165:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.089165:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.089166:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.089176:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.089179:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.089180:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.089181:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.089183:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.089184:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.089211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.089218:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.089220:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.089221:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:21.2:1619677448.089223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:18.0:1619677448.089223:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.089225:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.089227:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.089228:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.089229:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.089230:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.089231:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:18.0:1619677448.089231:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:24.0:1619677448.089234:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:18.0:1619677448.089235:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.089235:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.089237:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.089238:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:24.0:1619677448.089240:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.089240:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.089241:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:24.0:1619677448.089242:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:18.0:1619677448.089242:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:24.0:1619677448.089243:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96719504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.089243:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:24.0:1619677448.089244:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.089244:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:24.0:1619677448.089245:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.089245:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:24.0:1619677448.089246:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.089246:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.089247:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:24.0:1619677448.089250:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.089252:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.2:1619677448.089252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:24.0:1619677448.089254:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.089255:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.089256:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.089257:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:24.0:1619677448.089259:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.089259:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:2.0:1619677448.089259:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:24.0:1619677448.089260:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.2:1619677448.089260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.089260:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:24.0:1619677448.089261:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:18.0:1619677448.089261:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.089264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.089268:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.089271:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.089271:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.089272:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:5.0:1619677448.089273:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.089274:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.089274:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:5.0:1619677448.089275:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:2.0:1619677448.089275:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.089276:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.089277:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.089281:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.089293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.089301:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.089304:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.089305:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.089307:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.089310:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.089311:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.089312:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.089314:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.089314:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.089315:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.089319:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.089320:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.089323:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.089324:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.089324:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.089326:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.089327:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.089328:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.089329:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.089330:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.089336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.089341:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.089343:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.089344:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.089345:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677448.089345:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:21.2:1619677448.089345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677448.089346:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.089346:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.089347:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.089348:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.089351:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677448.089353:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.089356:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.089358:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.089360:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.089361:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.089362:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.089363:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.089364:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.089365:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.089366:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.089369:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.089370:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.089372:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.089373:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.089374:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.089375:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.089376:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.089377:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.089379:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.089379:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677448.089385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.089387:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:20.2:1619677448.089388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:10.0:1619677448.089389:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.089390:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.089390:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.089391:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:10.0:1619677448.089392:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.089393:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677448.089394:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:10.0:1619677448.089394:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:30.0:1619677448.089395:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.089396:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677448.089396:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:30.0:1619677448.089397:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677448.089397:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.089398:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.089399:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.089401:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.089402:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.089403:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.089404:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.089407:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.089409:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.089412:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.089413:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.089414:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.089415:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.089416:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.089417:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.089418:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677448.089428:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.089431:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.089431:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.089433:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:31.0:1619677448.089434:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.089436:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.089517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.089524:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.089527:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.089527:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.089529:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.089531:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.089533:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.089534:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.089535:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.089536:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.089537:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.089540:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.089542:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.089544:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.089545:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.089545:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.089546:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.089547:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.089548:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.089548:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.089553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.089560:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.089562:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.089563:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.089564:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.089565:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.089566:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.089708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.089715:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.089718:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.089718:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.089720:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.089723:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.089724:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:4.2:1619677448.089724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:18.0:1619677448.089725:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.089727:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.089727:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.089729:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.089731:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:18.0:1619677448.089732:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.089734:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.089734:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.089735:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677448.089736:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.089737:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.089738:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677448.089738:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.089739:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.089740:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.089740:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.089741:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:8.0:1619677448.089741:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.089742:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.089744:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96719498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.089745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677448.089745:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.089746:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.089747:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.089751:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.089752:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.089752:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.089754:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:8.0:1619677448.089755:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.089755:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.089756:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.089756:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.089757:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677448.089757:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.089758:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.089758:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.089759:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677448.089759:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.0:1619677448.089760:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.089764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.089772:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.089775:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.089776:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.089776:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.089778:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.089781:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.089808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.089816:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.089819:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.089821:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.089823:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.089823:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.089824:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.089825:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719497] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.089827:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.089828:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.089829:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.089832:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.089833:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.089835:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.089836:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.089837:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.089838:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.089839:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.089840:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.089840:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.089850:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.089852:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.089853:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.089853:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.089855:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.089858:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.089927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.089936:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.089938:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.089940:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.089942:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.089943:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.089944:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.089946:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.089947:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.089948:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.089948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:30.0:1619677448.089949:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.089951:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.089953:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.089954:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:4.2:1619677448.089954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.089956:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.089957:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.089957:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.089957:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.089957:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:30.0:1619677448.089959:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:18.0:1619677448.089959:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:30.0:1619677448.089960:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.089960:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.089961:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:18.0:1619677448.089961:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.089961:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:18.0:1619677448.089962:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:26.0:1619677448.089964:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677448.089964:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.089964:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:18.0:1619677448.089965:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.089967:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.089968:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.089968:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.089970:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.089971:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000400:00000200:10.0:1619677448.089972:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:18.0:1619677448.089974:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.089974:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96719494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.089975:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.089976:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677448.089976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.089977:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:10.0:1619677448.089977:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.089978:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677448.089978:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.089979:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.089979:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.089980:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.089981:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.089982:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:10.0:1619677448.089982:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.089983:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:9.2:1619677448.089984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.089984:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:10.0:1619677448.089986:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677448.089987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:10.0:1619677448.089987:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.089987:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:10.0:1619677448.089988:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.089989:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:5.0:1619677448.089989:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.089990:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.089990:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.089990:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.089991:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:5.0:1619677448.089991:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.089993:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677448.089994:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677448.089994:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.089995:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:2.0:1619677448.089995:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.089996:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.089997:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.089997:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.089998:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.089999:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677448.089999:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677448.090003:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:9.2:1619677448.090004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:22.0:1619677448.090006:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.090007:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.090007:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.090009:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.090011:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677448.090012:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.090013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:18.0:1619677448.090014:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.090015:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.090015:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.090016:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.090017:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677448.090018:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.090021:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.090022:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.090024:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.090026:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.090027:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.090029:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.090030:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.090031:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.090032:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.090035:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.090037:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.090040:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.090041:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.090041:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.090043:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.090044:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.090044:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.090045:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.090046:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:21.2:1619677448.090051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.090056:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.090058:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.090059:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.090060:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.090060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.090068:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.090070:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.090071:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.090073:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.090074:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.090075:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.090222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.090227:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.090230:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.090232:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.090233:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.090235:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.090237:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.090238:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.090239:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.090241:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.090242:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.090245:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.090247:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.090250:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.090251:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.090251:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.090253:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.090254:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.090255:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.090256:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.090267:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.090270:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.090271:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.090273:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.090274:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.090276:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.090322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.090328:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.090331:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.090332:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.090334:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.090335:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.090337:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.090338:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.090339:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.090340:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.090341:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:5.2:1619677448.090343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:17.0:1619677448.090344:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.090346:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.090348:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.090349:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.090350:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.090351:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.090351:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.090352:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.090353:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:17.0:1619677448.090353:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.090353:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.090354:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.090356:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.090359:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.090361:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.090362:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.090363:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.090364:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.090365:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677448.090367:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:26.0:1619677448.090369:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.090370:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677448.090370:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.090371:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.090371:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.090373:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.090373:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.090374:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.090375:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.090375:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.090375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.090377:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.090377:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.090379:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:21.2:1619677448.090379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:26.0:1619677448.090380:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.090381:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.090382:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.090383:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.090383:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677448.090383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677448.090386:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.090386:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.090386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677448.090387:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:26.0:1619677448.090388:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.090389:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.090390:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:24.0:1619677448.090390:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:22.0:1619677448.090390:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.090390:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.090391:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:8.0:1619677448.090391:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:2.0:1619677448.090391:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:26.0:1619677448.090392:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.090392:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:24.0:1619677448.090393:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.090394:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.090394:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:26.0:1619677448.090395:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.090395:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.090395:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:2.0:1619677448.090395:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.090396:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.090397:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.090397:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.090397:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.090398:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:2.0:1619677448.090398:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.090399:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.090400:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96719488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.090402:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.090402:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.090402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:2.0:1619677448.090402:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:24.0:1619677448.090403:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.090404:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.090405:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:24.0:1619677448.090407:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.090408:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.090409:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677448.090409:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.090410:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.090411:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.090412:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677448.090412:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.090412:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.090413:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677448.090413:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.090413:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.090414:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.090414:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.090415:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:24.0:1619677448.090415:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.090416:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:24.0:1619677448.090416:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.090417:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.090417:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:10.0:1619677448.090418:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.090420:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.090421:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96719487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.090422:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677448.090422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677448.090423:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.090424:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:6.0:1619677448.090426:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:10.0:1619677448.090427:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.090428:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677448.090429:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677448.090429:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.090430:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.090430:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.090432:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:10.0:1619677448.090432:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.090433:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:6.0:1619677448.090433:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.090433:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.090434:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.090434:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:18.0:1619677448.090436:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.090436:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:6.0:1619677448.090436:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677448.090437:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.090437:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.090438:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:18.0:1619677448.090439:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.2:1619677448.090442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677448.090445:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.090448:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.090449:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.090449:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.090450:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.090450:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:22.0:1619677448.090451:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677448.090452:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.090453:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.090455:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.090456:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.090458:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.090557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.090566:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.090569:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.090571:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.090573:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.090574:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.090575:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.090576:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.090578:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.090579:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.090580:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.090583:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.090585:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.090587:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.090588:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.090588:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.090590:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.090591:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.090592:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.090592:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677448.090603:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.090605:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.090606:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.090607:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.090610:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:21.2:1619677448.090611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:13.0:1619677448.090611:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.090619:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.090621:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.090624:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.090625:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.090626:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.090627:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.090628:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.090629:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.090630:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.090631:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.090634:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.090636:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.090638:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.090639:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.090639:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.090641:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.090641:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.090642:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.090643:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.090653:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.090655:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.090656:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.090658:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.090660:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.090661:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.090736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.090742:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.090744:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.090745:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.090747:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.090749:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.090750:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.090752:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.090753:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.090754:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.090755:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.090759:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.090760:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.090763:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.090764:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.090764:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.090766:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.090766:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.090767:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.090767:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.090772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.090780:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.090782:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.090783:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.090785:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.090786:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.090787:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.090787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.2:1619677448.090788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.090796:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.0:1619677448.090796:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677448.090799:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:8.0:1619677448.090799:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.090800:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677448.090800:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.090801:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.090801:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:8.0:1619677448.090802:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.090802:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:8.0:1619677448.090805:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.090806:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.090807:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.090809:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.090810:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.090811:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.090814:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.090816:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.090819:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.090819:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.090820:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.090822:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.090822:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.090824:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.090825:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.090825:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.090827:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.090831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.090835:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.090838:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.090838:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.090839:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.090839:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.090840:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.090841:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677448.090841:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.090842:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677448.090845:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.090866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.090869:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.090872:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.090874:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.090875:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.090877:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.090878:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677448.090880:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.090881:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.090882:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.090883:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.090886:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.090888:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.090890:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.090891:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.090892:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.090893:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.090894:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.090895:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.090895:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677448.090899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.090907:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.090910:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.090910:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.090911:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.090913:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.090914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.090917:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.090918:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.090920:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.090922:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.090922:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.090922:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.090923:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.090950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.090957:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.090960:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.090961:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.090963:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.090966:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.090967:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.090969:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.090970:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.090971:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.090972:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.090975:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.090977:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.090978:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.090979:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.090980:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.090981:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.090981:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.090982:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.090982:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677448.090985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677448.090992:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:31.0:1619677448.090993:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.090994:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:31.0:1619677448.090995:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:10.0:1619677448.090995:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.090996:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:31.0:1619677448.090998:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.090998:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.090999:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.090999:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:31.0:1619677448.091000:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.091002:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.091003:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.091004:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.091005:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.091006:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.091009:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.091011:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.091014:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.091015:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.091016:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.091018:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.091018:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.091019:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.091019:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.091024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.091032:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.091035:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.091036:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.091037:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.091038:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.091040:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.091068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.091070:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.091073:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.091075:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.091076:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.091078:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.091079:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677448.091081:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677448.091081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:20.0:1619677448.091084:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.091085:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.091086:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:26.0:1619677448.091089:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677448.091089:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.091091:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:20.0:1619677448.091091:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:10.0:1619677448.091092:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677448.091093:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.091094:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.091094:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677448.091095:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:26.0:1619677448.091096:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.091096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677448.091097:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:20.0:1619677448.091097:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677448.091097:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677448.091098:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.091099:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.091099:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.091100:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.091100:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677448.091101:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.091102:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.091102:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:9.2:1619677448.091103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:26.0:1619677448.091105:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.091105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677448.091105:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.091105:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:26.0:1619677448.091108:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.091108:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.091110:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.091110:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:1.0:1619677448.091110:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.091111:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.091111:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:26.0:1619677448.091112:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:2.0:1619677448.091112:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:1.0:1619677448.091112:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:5.0:1619677448.091113:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:2.0:1619677448.091113:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.091113:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96719477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.091114:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.091114:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.091114:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677448.091114:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:1.0:1619677448.091114:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.091115:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.091115:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.091115:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677448.091115:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.091116:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.091117:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677448.091119:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:1.0:1619677448.091120:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.091122:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:9.2:1619677448.091125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:1.0:1619677448.091125:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.091125:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677448.091126:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:1.0:1619677448.091126:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.091128:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.091128:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.091129:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:1.0:1619677448.091129:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677448.091130:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.091130:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677448.091131:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.091132:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:18.0:1619677448.091133:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.091134:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677448.091135:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.091136:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.091137:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.091138:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.091138:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:22.0:1619677448.091141:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:31.0:1619677448.091144:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.091144:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.091145:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.091145:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.091147:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.091148:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.091149:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677448.091149:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677448.091150:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.091150:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.091152:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.091281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.091288:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.091291:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677448.091296:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.091300:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.091304:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.091305:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.091307:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.091308:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.091309:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.091310:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.091313:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.091315:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.091319:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.091319:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.091320:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.091322:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.091322:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.091323:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.091324:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.091328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.091337:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.091340:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.091341:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.091341:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.091343:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.091346:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.091367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.091375:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.091378:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.091379:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.091381:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.091383:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.091384:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.091386:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.091387:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.091388:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.091389:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.091393:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.091394:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.091397:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.091397:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.091398:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.091400:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.091400:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.091401:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.091401:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677448.091413:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.091415:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.091416:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.091418:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.091419:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.091420:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.091430:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.091432:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.091433:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.091434:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.091434:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.091436:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.091467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.091474:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.091477:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.091478:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.091479:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.091482:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.091483:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.091484:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.091485:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.091486:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.091487:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.091490:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.091491:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.091493:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.091494:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.091494:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.091496:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.091496:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.091498:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.091499:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.091499:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.091505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.2:1619677448.091507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.091512:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:10.0:1619677448.091512:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:31.0:1619677448.091514:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:10.0:1619677448.091514:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:31.0:1619677448.091515:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.091515:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677448.091516:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.091516:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.091516:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.091517:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677448.091518:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.091519:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:30.0:1619677448.091519:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.091520:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.091522:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.091523:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.091524:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.091526:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.091526:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.091527:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.091530:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.091532:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.091534:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.091534:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.091535:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.091536:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.2:1619677448.091536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.091537:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.091538:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.091540:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.091540:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677448.091542:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.091544:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.091545:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.2:1619677448.091546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.091547:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.091549:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.091549:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:17.0:1619677448.091551:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:12.0:1619677448.091551:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:17.0:1619677448.091552:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.091552:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.091554:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:17.0:1619677448.091554:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.091554:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.091555:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.091556:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:17.0:1619677448.091556:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:13.0:1619677448.091556:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.091557:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:12.0:1619677448.091557:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.091558:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:17.0:1619677448.091560:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.091562:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.091564:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.091565:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.091565:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.091567:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:13.0:1619677448.091567:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.091568:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.091569:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.091570:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:13.0:1619677448.091570:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.091571:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.091571:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.091571:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.091572:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.091573:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.091575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.091581:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677448.091582:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.091584:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.091584:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.091585:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677448.091585:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.091586:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.091586:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.091588:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.091590:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.091765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.091770:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.091773:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.091775:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.091776:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.091778:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.091779:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677448.091781:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.091782:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.091783:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.091784:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677448.091788:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.091790:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677448.091792:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.091793:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677448.091794:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677448.091795:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677448.091796:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.091797:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.091798:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.091802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.091810:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.091813:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.091813:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.091814:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.091816:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.091818:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.091840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.091847:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.091850:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.091850:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.091852:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.091855:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.091856:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.091857:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.091858:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.091858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:26.0:1619677448.091860:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.091860:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.091863:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:26.0:1619677448.091864:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.091866:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.091866:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.091867:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677448.091869:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.091869:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.091870:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.091870:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:22.0:1619677448.091871:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.091872:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.091872:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677448.091872:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:26.0:1619677448.091873:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.091873:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.091873:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.091875:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.091876:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.091877:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.091880:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.091882:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.091884:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677448.091884:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.091885:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.091885:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.091886:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.091887:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.091887:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.091888:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.091888:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:31.0:1619677448.091889:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.091889:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677448.091891:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.091891:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:8.2:1619677448.091894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.091902:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.091904:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.091905:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.091906:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.091908:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.091910:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.091924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.091931:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.091934:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.091936:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.091937:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.091938:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.091940:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.091941:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.091942:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.091943:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.091944:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.091948:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.091949:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.091952:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.091953:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.091953:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.091955:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.091955:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.091957:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.091958:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.091959:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677448.091964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.091967:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.091969:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.091970:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.091971:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.091972:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677448.091973:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.091974:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.091975:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.091975:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.091976:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677448.091982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677448.091987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677448.091988:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.091988:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.091991:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677448.091991:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.091991:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:12.0:1619677448.091992:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.091993:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.091993:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677448.091994:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:1.0:1619677448.091994:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.091995:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.091996:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:1.0:1619677448.091996:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.091997:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:1.0:1619677448.091997:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677448.091999:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.091999:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.092001:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.092001:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.092002:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.092003:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.092003:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677448.092005:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.092006:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.092007:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.092007:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.092009:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.092009:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:2.0:1619677448.092012:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.092012:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.092013:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677448.092014:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.092014:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.092015:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.092016:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.092016:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.092017:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.092017:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677448.092018:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.092019:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.092020:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.092021:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.092022:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.092022:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:6.0:1619677448.092028:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.092031:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.092032:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.092032:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.092034:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677448.092034:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.092036:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677448.092037:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.092038:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.092040:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.092041:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.092043:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.092101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.092108:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.092111:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.092112:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.092114:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.092116:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.092117:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.092118:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.092120:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.092121:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.092122:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.092126:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.092127:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.092130:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.092130:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.092131:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.092132:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.092133:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.092134:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.092134:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677448.092145:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.092147:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.092148:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.092149:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.092151:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.092152:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.092331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.092338:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.092341:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.092342:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.092343:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.092345:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.092347:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.092348:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.092349:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.092350:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.092351:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.092354:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.092355:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.092358:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.092359:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.092359:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.092361:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.092362:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.092362:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.092363:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.092376:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.092379:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.092379:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.092380:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.092382:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.092385:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677448.092398:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.092401:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.092401:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.092402:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.092403:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.092404:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.092623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.092631:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.092634:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.092636:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.092638:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.092639:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.092640:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.092641:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.092642:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.092643:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.092644:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.092647:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.092649:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.092651:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.092652:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.092652:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.092654:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.092654:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:21.2:1619677448.092654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.092655:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.092656:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677448.092658:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.092659:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.092661:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.092662:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.092663:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.092665:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.092666:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.092666:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:31.0:1619677448.092667:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.092668:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.092668:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:12.0:1619677448.092668:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.092669:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.092671:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.092672:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.092673:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.092674:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677448.092674:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.092677:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.092677:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.092678:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.092680:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.092680:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.092681:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.092681:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.092686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.092694:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.092697:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.092697:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.092698:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.092700:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.092703:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.092758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.092765:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.092768:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.092768:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.092770:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.092772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:10.0:1619677448.092773:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.092775:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.092776:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.092777:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.092778:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.092779:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.092780:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.092782:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.092783:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:10.0:1619677448.092783:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.092784:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.092785:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.092788:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.092788:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:8.0:1619677448.092788:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.092789:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677448.092790:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:10.0:1619677448.092791:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.092791:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:8.0:1619677448.092791:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96719460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.092792:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:8.0:1619677448.092792:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.092793:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:8.0:1619677448.092793:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.092794:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:9.2:1619677448.092797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677448.092797:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.092798:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.092801:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.092802:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.092803:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.092804:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.092804:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.0:1619677448.092805:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.092805:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.092806:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677448.092807:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.092808:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.092808:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.092810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677448.092810:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.092814:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677448.092819:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.092821:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.092822:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.092823:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.092825:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.092828:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.092886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677448.092891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:1.0:1619677448.092891:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.092894:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.092895:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.092897:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:1.0:1619677448.092897:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.092899:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.092900:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:1.0:1619677448.092900:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:2.0:1619677448.092901:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:1.0:1619677448.092901:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.092903:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.092903:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.092904:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.092905:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.092906:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.092908:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.092909:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.092910:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.092911:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.092912:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.092912:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.092913:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.092916:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.092916:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.092917:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.092917:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.092918:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.092919:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.092920:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.092920:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.092921:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.092921:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677448.092922:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.092923:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.092924:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.092925:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.092926:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.092926:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.092931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677448.092932:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.092934:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.092935:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.092937:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.092938:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.092939:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:12.0:1619677448.092940:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677448.092941:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.092942:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.092944:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.092945:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.092947:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.093095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.093100:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.093103:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.093103:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.093106:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.093108:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.093109:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.093111:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.093112:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.093113:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.093114:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.093118:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.093120:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.093122:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.093123:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.093124:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.093125:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.093126:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.093126:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.093127:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.093131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.093139:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.093141:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.093142:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.093142:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.093145:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.093147:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.093233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.093238:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.093241:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.093242:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.093244:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.093247:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.093248:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.093250:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.093251:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.093252:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.093253:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.093256:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.093258:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.093260:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.093261:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.093262:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.093263:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.093263:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.093264:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.093265:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677448.093276:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.093279:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.093280:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.093280:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.093282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:6.0:1619677448.093283:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.093285:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677448.093287:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.093290:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.093291:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.093298:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.093300:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.093302:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.093303:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.093304:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.093305:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.093306:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.093309:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.093311:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.093314:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.093314:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.093315:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.093317:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.093317:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.093318:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.093318:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677448.093331:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.093334:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.093335:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.093336:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.093338:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.093339:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.093396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.093402:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.093405:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.093406:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.093407:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.093411:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.093412:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.093413:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.093414:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.093415:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.093416:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.093418:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.093419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677448.093419:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.093421:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.093421:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.093422:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.093423:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.093423:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.093424:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.093425:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:26.0:1619677448.093427:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:9.2:1619677448.093429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.093430:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.093430:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.093433:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.093436:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.093436:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:26.0:1619677448.093437:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.093438:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.093438:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.093439:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.093440:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.093440:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:26.0:1619677448.093441:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.093442:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.093443:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:18.0:1619677448.093444:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:26.0:1619677448.093446:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.093448:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:9.2:1619677448.093450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.093451:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.093452:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.093452:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.093454:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.093455:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.2:1619677448.093455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.093456:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.093457:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.093457:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677448.093457:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.093460:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.093461:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.2:1619677448.093461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.093461:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677448.093461:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.093462:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:4.2:1619677448.093463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.093463:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677448.093464:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.093465:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.093466:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.093469:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:1.0:1619677448.093469:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.093470:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:1.0:1619677448.093471:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677448.093472:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:5.0:1619677448.093472:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:1.0:1619677448.093472:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.093473:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.093473:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:5.0:1619677448.093473:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.093473:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.093474:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.093475:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.093475:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.093475:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.093478:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:1.0:1619677448.093479:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.093481:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.093484:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.093485:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.093485:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.093487:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.093488:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.093488:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.093489:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.093500:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.093502:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.093503:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.093505:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.093506:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.093507:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677448.093515:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.093518:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.093519:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.093519:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.093520:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.093521:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.093704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.093712:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.093715:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.093717:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.093720:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.093720:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.093721:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.093723:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.093724:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.093725:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.093726:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.093730:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.093732:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.093734:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.093735:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.093735:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.093737:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.093737:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.093738:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.093739:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677448.093749:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.093751:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.093752:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.093754:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.093755:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.093756:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.093807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.093812:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.093815:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.093816:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.093818:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.093821:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.093822:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.093823:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.093825:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.093826:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.093827:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.093829:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.093831:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.093833:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.093833:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.093834:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.093836:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.093836:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.093837:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.093838:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.093839:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.093842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.093844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.093850:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.093851:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.093851:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.093851:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.093852:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.093853:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.093853:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.093854:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.093855:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.093855:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.093894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.093897:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.093899:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.093901:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.093903:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.093904:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.093905:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.093906:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.093907:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.093908:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.093909:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.093913:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.093915:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.093917:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.093918:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.093919:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.093920:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.093921:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.093922:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.093922:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.2:1619677448.093933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677448.093935:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.093937:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.093938:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.093939:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:4.2:1619677448.093939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.093940:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:22.0:1619677448.093940:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.093943:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677448.093943:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:30.0:1619677448.093945:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.093946:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677448.093946:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677448.093947:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.093948:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:10.0:1619677448.093949:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:30.0:1619677448.093950:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.093950:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:30.0:1619677448.093952:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.093953:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.093953:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.093954:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.093957:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.093958:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.093959:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.093961:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.093961:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:30.0:1619677448.093962:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.093963:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:10.0:1619677448.093963:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.093964:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.093964:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677448.093964:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.093965:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.093966:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.093966:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:10.0:1619677448.093966:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.093968:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.093970:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.093973:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.093974:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.093974:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (23)++
00000800:00000200:26.0:1619677448.093976:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (24)++
00000800:00000200:10.0:1619677448.093976:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (24)--
00000800:00000200:26.0:1619677448.093977:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (24)--
00000800:00000200:10.0:1619677448.093977:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:26.0:1619677448.093978:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.093978:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.093979:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.093979:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.093980:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)++
00000800:00000200:10.0:1619677448.093980:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (24)--
00000400:00000200:10.0:1619677448.093981:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.093982:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.093983:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.093983:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.093984:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.093992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.093999:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:5.2:1619677448.094001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:24.0:1619677448.094002:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.094002:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.094004:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.094007:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.094008:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:24.0:1619677448.094008:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.094010:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.094011:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:24.0:1619677448.094011:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.094012:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.094012:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:24.0:1619677448.094013:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.094014:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.094016:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.094016:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.094017:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.094018:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.094018:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.094020:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.094020:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.094021:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.094021:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.094021:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.094022:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.094024:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.094024:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:26.0:1619677448.094025:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.094025:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.094025:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:26.0:1619677448.094027:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.094028:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.094029:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.094030:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.094030:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:9.2:1619677448.094030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.094031:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.094031:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.094031:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.094031:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.094032:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.094033:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.094034:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.094035:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.094037:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.094037:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677448.094038:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.094041:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.094042:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.094042:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.2:1619677448.094043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:18.0:1619677448.094044:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.094046:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:18.0:1619677448.094047:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677448.094048:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.094049:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.094050:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:12.0:1619677448.094051:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.094052:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.094052:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:12.0:1619677448.094053:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.094053:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.094054:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.2:1619677448.094076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.094081:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.094084:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.094085:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.094087:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.094088:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.094090:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.094091:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.094093:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.094093:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.094095:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.094097:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.094099:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.094101:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.094101:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:4.2:1619677448.094101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:17.0:1619677448.094102:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.094104:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.094104:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.094104:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.094105:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.094110:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.094113:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.094116:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.094116:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:1.0:1619677448.094117:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:8.0:1619677448.094118:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.094119:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.094120:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.094121:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:8.0:1619677448.094121:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.094122:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.094123:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:1.0:1619677448.094123:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:8.0:1619677448.094124:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.094127:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000400:00000200:8.0:1619677448.094130:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.094131:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.094132:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.094134:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:8.0:1619677448.094135:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.094137:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.094137:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.094138:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.094138:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.094139:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.094139:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.0:1619677448.094140:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.094141:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.094142:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.094143:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.094144:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.094144:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.094145:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.094149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.094156:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.094159:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.094160:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.094160:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.094162:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.094165:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.094395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.094401:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.094404:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.094405:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.094406:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.094409:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.094410:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.094411:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.094413:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.094414:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.094415:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.094418:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.094420:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.094423:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.094424:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.094424:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.094426:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.094427:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.094429:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.094430:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.094430:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:21.2:1619677448.094436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.094440:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:22.0:1619677448.094441:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:26.0:1619677448.094443:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:22.0:1619677448.094443:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:10.0:1619677448.094443:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:22.0:1619677448.094444:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677448.094444:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.094444:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677448.094446:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.094448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.094452:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:10.0:1619677448.094453:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.094455:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.094456:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.094458:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.094459:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.094461:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.094462:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.094463:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.094464:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.094465:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.2:1619677448.094465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.2:1619677448.094468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:10.0:1619677448.094468:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.094470:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.094471:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.094473:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.094474:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:10.0:1619677448.094474:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.094474:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.094475:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:17.0:1619677448.094476:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.094477:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.094477:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:17.0:1619677448.094478:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.094478:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.094479:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.094479:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677448.094480:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:10.0:1619677448.094480:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.094481:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:17.0:1619677448.094482:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.094484:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.094484:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.094485:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.094486:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.094486:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.094488:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.094489:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.094490:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96719439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.094492:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.094492:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.094493:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:26.0:1619677448.094493:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.094494:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.094495:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.094495:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:31.0:1619677448.094496:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:17.0:1619677448.094496:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677448.094497:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.094497:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:31.0:1619677448.094498:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.094498:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.094498:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.094499:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.094499:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.094500:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.094501:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.094501:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.094504:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677448.094504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.094505:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.094505:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.094507:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.094507:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.094508:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.094508:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:1.0:1619677448.094511:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.094513:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.094514:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.094515:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.094516:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.094518:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:13.0:1619677448.094520:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.094522:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.094523:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.094525:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.094527:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.094527:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:21.2:1619677448.094560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.094566:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.094569:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.094570:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.094572:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.094574:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.094576:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.094577:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.094578:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.094580:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.094580:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.094584:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.094586:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.094589:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.094590:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.094590:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.094592:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.094593:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.094594:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.094595:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.094595:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.2:1619677448.094601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.094604:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:18.0:1619677448.094606:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:10.0:1619677448.094607:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.094608:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.094608:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:10.0:1619677448.094608:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.094609:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.094610:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:10.0:1619677448.094610:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.094613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677448.094613:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:30.0:1619677448.094620:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.094623:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.094625:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.094627:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.094627:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.094628:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.094629:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.094631:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.094631:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.094632:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.094635:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.094637:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.094639:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.094640:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.094641:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.094642:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.094643:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.094644:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.094644:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.094654:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.094656:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.094657:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.094659:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.094660:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.094662:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.094680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.094686:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.094689:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.094691:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.094692:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.094694:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.094696:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.094697:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.094698:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.094699:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.094700:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.2:1619677448.094703:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:5.0:1619677448.094708:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.094710:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.094711:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.094714:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.094714:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.0:1619677448.094714:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.094715:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.094715:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:26.0:1619677448.094717:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.094717:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.094718:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:26.0:1619677448.094719:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.094719:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000400:00000200:26.0:1619677448.094720:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:5.0:1619677448.094721:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000400:00000200:26.0:1619677448.094722:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.094722:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:26.0:1619677448.094723:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.094724:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.094725:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.2:1619677448.094726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000400:00000200:26.0:1619677448.094728:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.094729:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.094729:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:31.0:1619677448.094730:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.094732:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.094732:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677448.094733:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.094733:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.094733:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.094733:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.094733:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:26.0:1619677448.094734:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677448.094735:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.094736:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.094736:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.094737:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.094737:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.094737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:31.0:1619677448.094738:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.094738:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677448.094743:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:9.2:1619677448.094744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.094746:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.094749:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.094749:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.094750:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:18.0:1619677448.094751:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:30.0:1619677448.094753:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.094753:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:18.0:1619677448.094753:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.094753:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.094754:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:18.0:1619677448.094754:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.094755:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:2.0:1619677448.094755:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:30.0:1619677448.094756:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:18.0:1619677448.094756:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.094757:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.094758:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.094759:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.094761:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.094764:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.094764:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.094765:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.094766:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.094768:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.094769:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.094770:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.094772:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.094772:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.094773:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.094774:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.094775:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.094778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.094780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677448.094785:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677448.094786:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.094788:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:17.0:1619677448.094788:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677448.094789:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677448.094789:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.094789:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.094790:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:17.0:1619677448.094791:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.094793:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.094798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.094803:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.094806:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.094807:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.094808:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.094811:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.094812:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.094813:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.094815:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.094816:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.094816:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.094819:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.094820:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.094822:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.094823:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.094823:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.094825:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.094825:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.094826:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.094827:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677448.094839:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.094842:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.094842:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.094843:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.094845:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.094847:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.094868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.094875:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.094878:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.094879:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.094881:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.094883:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.094885:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.094886:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.094888:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.094889:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.094890:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.094893:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.094895:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.094898:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.094898:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.094899:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.094900:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.094901:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.094902:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.094904:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.094904:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:21.2:1619677448.094909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:13.0:1619677448.094913:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.094915:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.094916:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.094916:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:13.0:1619677448.094917:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.094918:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:12.0:1619677448.094919:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.094919:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.094920:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:4.2:1619677448.094920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677448.094921:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.094927:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.094929:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.094930:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.094932:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.094935:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.094936:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.094938:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.094939:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.094940:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.094941:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.094944:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.094946:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.094949:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.094950:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.094951:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.094952:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.094953:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.094953:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.094954:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.094966:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.094969:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.094970:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.094970:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.094972:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.094976:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677448.094988:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.094991:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.094991:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.094992:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.094993:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.094993:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.095049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.095055:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.095058:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.095059:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.095061:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.095063:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.095064:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.095065:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.095067:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.095068:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.095069:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.095072:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.095074:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.095077:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.095077:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.095078:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.095080:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.095080:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.095081:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.095081:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.095093:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.095096:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.095097:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.095097:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.095099:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.095101:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.095202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.095209:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.095211:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.095212:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.095214:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.095216:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.095218:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.095219:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.095221:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.095222:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.095223:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.095226:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.095228:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.095230:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.095231:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:21.2:1619677448.095231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:24.0:1619677448.095232:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.095234:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.095234:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.095235:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.095235:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677448.095236:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.095239:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.095239:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.095241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:22.0:1619677448.095242:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.095244:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.095246:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.095247:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.095248:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.095249:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:22.0:1619677448.095250:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.095250:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.095252:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.095253:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.095253:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:22.0:1619677448.095254:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.095255:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.095256:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.095258:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677448.095259:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.095259:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.095260:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.095262:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.095262:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.095263:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.095264:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.095268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.095276:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.095278:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.095279:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.095281:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.095282:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.095283:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.095331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.095338:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.095341:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.2:1619677448.095342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:31.0:1619677448.095343:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.095346:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.095346:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.095347:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677448.095348:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:31.0:1619677448.095349:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.095350:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.095351:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.095351:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:31.0:1619677448.095353:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.095353:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.095354:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.095355:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.095356:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.095357:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.095357:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.095358:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:31.0:1619677448.095359:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.095359:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:5.0:1619677448.095360:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719426] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.095361:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.095362:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:5.0:1619677448.095362:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.095363:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677448.095363:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.095364:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677448.095364:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:31.0:1619677448.095365:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.095366:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.095368:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.095368:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677448.095368:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.095368:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.095370:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.095371:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:20.0:1619677448.095372:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677448.095373:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96719425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.095374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677448.095375:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.095375:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.095377:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:12.0:1619677448.095378:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:20.0:1619677448.095380:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.095380:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.095380:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677448.095381:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.095381:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:30.0:1619677448.095382:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677448.095382:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.095382:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:12.0:1619677448.095383:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.095384:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.095384:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677448.095384:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677448.095385:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:20.0:1619677448.095385:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.095385:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.095385:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:30.0:1619677448.095386:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.095386:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677448.095386:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.095386:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677448.095387:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677448.095387:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.095388:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677448.095389:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.095390:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.095390:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.095390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.2:1619677448.095396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.095401:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.095404:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:1.0:1619677448.095404:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677448.095405:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.095405:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.095406:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.095407:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.095407:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.095408:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.095410:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.095411:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:1.0:1619677448.095413:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.095414:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.095422:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.095424:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.095425:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.095425:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.095427:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.095427:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.095444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.095450:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.095453:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.095453:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.095456:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.095458:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.095460:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.095461:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.095462:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.095463:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.095465:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.095468:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.095470:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.095473:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.095474:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.095474:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.095476:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.095477:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.095477:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.095478:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:26.0:1619677448.095489:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.095492:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.095493:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.095493:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.095496:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.095498:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.095643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.095650:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.095653:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.095653:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.095655:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.095658:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.095659:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.095660:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719423] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.095662:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.095663:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.095664:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.095667:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.095669:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.095671:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.095672:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.095673:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.095674:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.095675:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.095675:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.095676:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.2:1619677448.095684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:30.0:1619677448.095687:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.095689:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.095690:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.095690:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000400:00000200:30.0:1619677448.095691:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.095693:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677448.095693:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:31.0:1619677448.095694:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.095695:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:1.0:1619677448.095695:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677448.095697:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.095699:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.095700:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.095702:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.095703:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.095704:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.095707:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677448.095709:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.095712:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.095713:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.095713:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.095715:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.095716:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677448.095717:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdf63800] (22)++
00000800:00000200:1.0:1619677448.095718:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (23)--
00000800:00000200:1.0:1619677448.095719:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:9.2:1619677448.095721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.095725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (21)++
00000800:00000200:24.0:1619677448.095727:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.095730:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:17.0:1619677448.095730:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)++
00000400:00000200:24.0:1619677448.095731:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.095731:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.095733:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:17.0:1619677448.095733:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdf63800] (23)--
00000800:00000200:17.0:1619677448.095734:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677448.095734:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:24.0:1619677448.095736:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.095747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.095754:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.095757:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.095758:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.095760:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.095762:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.095764:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.095765:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.095766:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.095767:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.095768:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.095772:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.095773:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.095776:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.095777:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.095778:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.095779:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.095780:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.095781:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.095782:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.095782:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.095788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.095793:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677448.095795:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.095796:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.095797:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:10.0:1619677448.095797:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.095798:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.095798:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.095798:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.095800:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.095803:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.095875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.095882:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.095885:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.095887:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.095887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:30.0:1619677448.095889:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.095890:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.095891:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.095892:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.095892:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:31.0:1619677448.095894:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.095895:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.095895:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:31.0:1619677448.095896:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.095896:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.095898:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.095900:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.095900:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.095902:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.095902:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.095903:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.095905:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.095906:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.095907:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.095907:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:22.0:1619677448.095907:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:31.0:1619677448.095908:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.095910:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.095911:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.095911:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:22.0:1619677448.095911:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.095912:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:22.0:1619677448.095913:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.095916:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677448.095916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.095917:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.095918:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.095920:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.095920:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.095921:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.095921:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677448.095925:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.095928:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.095929:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.095931:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.095931:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.095932:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.095934:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:5.0:1619677448.095934:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:26.0:1619677448.095935:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.095935:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.095937:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.095939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.095940:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.0:1619677448.095947:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.095950:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.095951:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.095951:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.095952:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.095953:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:5.0:1619677448.095953:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:26.0:1619677448.095956:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.095956:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.095957:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.095958:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.095958:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677448.095984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.095991:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.095994:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.095996:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.095997:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.095998:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.096000:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.096001:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.096002:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.096003:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.096004:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.096008:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.096009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.096010:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.096012:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.096013:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.096013:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.096014:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.096015:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.096015:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.096016:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.096017:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677448.096017:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.096018:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.096020:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.096023:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.096024:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.096026:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.096027:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.096028:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677448.096028:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.096029:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:13.0:1619677448.096030:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.096031:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.096033:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.096033:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.096034:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.096036:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.096036:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.096038:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.096039:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.096040:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.096042:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.096042:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.096043:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.096044:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.096045:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677448.096046:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.096049:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.096050:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.096050:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.2:1619677448.096050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.096051:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.096052:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677448.096055:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677448.096056:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.2:1619677448.096057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:17.0:1619677448.096058:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.096058:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.096058:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677448.096059:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.096059:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.096060:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.096062:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.096064:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.096064:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:10.0:1619677448.096067:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.096068:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.096070:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.096072:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.096074:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.096075:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719416] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.096077:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.096078:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.096079:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.096082:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.096084:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.096087:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.096088:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.096089:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.096091:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.096091:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.096092:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.096092:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.096097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.096104:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.096107:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.096108:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.096108:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.096111:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.096114:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.096275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.2:1619677448.096277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:20.0:1619677448.096278:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.096280:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.096282:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.096283:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677448.096284:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.096285:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.096286:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.096286:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:20.0:1619677448.096287:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:4.2:1619677448.096288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:20.0:1619677448.096289:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.096290:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.096291:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.096292:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677448.096292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677448.096296:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:18.0:1619677448.096298:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.096299:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677448.096300:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:8.0:1619677448.096300:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:18.0:1619677448.096301:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.096301:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677448.096303:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.096303:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.096304:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:18.0:1619677448.096304:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.096305:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:18.0:1619677448.096305:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.096307:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.096309:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.096310:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.096312:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.096312:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.096312:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.096312:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677448.096314:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.096315:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96719413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.096315:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.096315:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96719413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.096316:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:26.0:1619677448.096317:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.096317:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.096318:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677448.096318:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.096319:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.096319:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.096319:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:8.0:1619677448.096319:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677448.096320:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.096320:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.096320:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.096321:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.096322:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677448.096323:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.096323:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.0:1619677448.096324:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:8.0:1619677448.096324:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.096325:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.096325:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:18.0:1619677448.096325:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.096325:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677448.096326:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:8.0:1619677448.096326:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.096327:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.096327:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.0:1619677448.096329:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.096330:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677448.096330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677448.096330:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:26.0:1619677448.096331:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677448.096331:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.096332:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.096332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.096333:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677448.096333:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:26.0:1619677448.096334:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677448.096334:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677448.096335:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677448.096335:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:26.0:1619677448.096336:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.0:1619677448.096337:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.096337:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677448.096338:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:2.0:1619677448.096338:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:13.0:1619677448.096339:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.2:1619677448.096340:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.096341:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:2.0:1619677448.096341:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:13.0:1619677448.096342:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:5.0:1619677448.096342:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677448.096343:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:13.0:1619677448.096343:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.096343:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677448.096343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:2.0:1619677448.096343:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.096346:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.096346:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.096347:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:22.0:1619677448.096348:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:12.0:1619677448.096348:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:9.2:1619677448.096348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:1.0:1619677448.096348:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:12.0:1619677448.096349:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:24.0:1619677448.096350:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:13.0:1619677448.096350:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.096350:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:22.0:1619677448.096351:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:12.0:1619677448.096351:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677448.096351:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677448.096351:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:22.0:1619677448.096352:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.096352:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677448.096352:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:1.0:1619677448.096352:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.096353:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677448.096353:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.096354:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677448.096354:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:24.0:1619677448.096355:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677448.096356:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.096356:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.096357:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.096357:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (21)++
00000400:00000200:1.0:1619677448.096357:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.096358:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.096359:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677448.096359:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:1.0:1619677448.096360:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.096459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.096468:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.096471:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.096473:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.096475:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.096475:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.096476:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.096478:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719411] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.096479:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.096480:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.096481:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.096485:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.096487:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.096489:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.096490:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.096491:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.096493:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.096493:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.096495:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.096496:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.096497:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.096499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.096502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.096507:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677448.096510:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:5.0:1619677448.096510:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.096511:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.096512:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:5.0:1619677448.096512:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.096513:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677448.096513:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.096515:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.096515:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.096553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.096559:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.096561:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.096562:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.096564:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.096566:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.096567:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.096569:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.096570:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.096571:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.096572:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.096575:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.096576:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:5.2:1619677448.096577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:17.0:1619677448.096579:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.096580:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.096581:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.096583:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.096583:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.096584:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.096584:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.096584:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.096587:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.096588:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.096591:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.096593:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.096595:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:10.0:1619677448.096595:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.096596:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.096597:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:10.0:1619677448.096597:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.096598:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.096598:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.096599:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:12.0:1619677448.096600:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.096601:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.096603:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:10.0:1619677448.096603:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.096605:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.096607:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.096608:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.096609:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.096610:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.096611:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.096612:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.096613:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.096614:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.096619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.096622:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.096625:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.096626:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.096627:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.096627:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:31.0:1619677448.096629:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.096629:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.096629:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.096630:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:10.0:1619677448.096630:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677448.096638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.096643:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.096645:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.096646:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.096646:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.096646:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.096647:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.096648:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.096648:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677448.096649:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.096650:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.096652:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.096653:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.096654:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.096656:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.096657:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.096658:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.096661:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.096662:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.096665:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.096666:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.096667:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.096668:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.096669:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.096670:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.096670:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.096682:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.096685:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.096686:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.096686:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.096688:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.096691:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.096707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677448.096709:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677448.096712:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677448.096714:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.096715:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677448.096716:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.096718:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:20.0:1619677448.096719:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.096721:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.096722:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.096723:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:20.0:1619677448.096726:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.096727:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:20.0:1619677448.096730:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.096731:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:20.0:1619677448.096732:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:20.0:1619677448.096734:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:20.0:1619677448.096734:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677448.096736:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:20.0:1619677448.096737:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:20.0:1619677448.096737:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.2:1619677448.096742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:26.0:1619677448.096746:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.096748:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:5.0:1619677448.096748:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000800:00000200:10.0:1619677448.096749:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.096750:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.096750:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000800:00000200:5.0:1619677448.096751:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:5.0:1619677448.096751:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000400:00000200:26.0:1619677448.096752:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.096755:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677448.096770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.096775:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.096777:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.096779:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.096780:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.096782:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.096784:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000400:00000200:5.0:1619677448.096788:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677448.096789:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.096791:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000400:00000200:5.0:1619677448.096791:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.096792:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.096793:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.096794:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677448.096794:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000400:00000200:5.0:1619677448.096797:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.096799:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.096800:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.096801:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.096802:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.096803:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:5.0:1619677448.096803:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.096803:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:10.0:1619677448.096804:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719405] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.096805:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:10.0:1619677448.096806:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.096806:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.096806:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.096806:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.096807:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:10.0:1619677448.096808:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.096811:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.096811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.096813:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.096816:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.096816:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)++
00000800:00000200:10.0:1619677448.096817:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (23)++
00000800:00000200:10.0:1619677448.096819:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (24)--
00000800:00000200:10.0:1619677448.096819:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:1.0:1619677448.096819:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (23)++
00000800:00000200:10.0:1619677448.096820:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677448.096821:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:1.0:1619677448.096822:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (24)--
00000800:00000200:17.0:1619677448.096823:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (23)++
00000400:00000200:1.0:1619677448.096823:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.096824:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (24)--
00000400:00000200:1.0:1619677448.096825:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.096826:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.096827:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.096829:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.096830:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.096831:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.096903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.096910:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.096913:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.096913:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.096915:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.096918:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.096920:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.096921:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.096922:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.096923:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.096924:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.096927:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.096928:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.096931:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.096931:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.096932:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.096934:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.096934:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.096935:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.096935:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.096947:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.096950:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.096950:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.096951:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.096953:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.096956:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.096966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.096972:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.096975:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.096976:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.096978:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.096980:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.096982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:17.0:1619677448.096982:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.096983:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.096984:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.096985:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.096986:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.096990:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.096990:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.096992:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:17.0:1619677448.096992:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:31.0:1619677448.096995:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.096995:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.096996:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677448.096996:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.096997:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:31.0:1619677448.096998:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.096999:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:31.0:1619677448.097000:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:17.0:1619677448.097000:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.097001:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.097001:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.097001:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677448.097002:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.097003:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.097004:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.097008:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.097009:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.097012:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.097012:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.097013:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.097013:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.097014:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.097014:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.097016:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.097016:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:30.0:1619677448.097016:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.097017:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.097018:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.097019:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.097020:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.2:1619677448.097022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.097029:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.097032:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.097033:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.097035:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.097036:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.097038:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.097177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.097182:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.097184:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.097187:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.097187:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.097189:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.097191:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677448.097192:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.097193:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.097194:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.097195:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677448.097198:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.097200:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677448.097202:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.097203:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677448.097204:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677448.097205:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677448.097206:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.2:1619677448.097208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.0:1619677448.097208:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.097208:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.097216:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.097218:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.097221:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.097222:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.097223:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.097225:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.097226:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.097227:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.097228:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.097229:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.097233:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.097235:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.097238:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.097238:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.097239:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:9.2:1619677448.097239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.097241:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.097241:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.097242:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.097243:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677448.097246:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.097249:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.097250:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.097250:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.097252:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.097254:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677448.097255:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.097256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.097257:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.097258:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.097259:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.097260:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:6.0:1619677448.097260:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.097261:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.097263:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:22.0:1619677448.097264:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.097264:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.097266:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.097269:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.097270:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.097271:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.097272:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.097274:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.097274:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.097275:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.097276:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.097277:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.097278:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.097278:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.097279:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677448.097280:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.097283:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.097284:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.097284:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.097286:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.097287:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.097287:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.097288:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.097292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.097301:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.097304:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.097305:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.097305:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.097307:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.097310:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.097311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.097319:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.097321:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.097322:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.097322:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.097323:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.097324:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.097386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.097391:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.097394:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.097394:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:5.2:1619677448.097395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:1.0:1619677448.097396:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.097399:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.097400:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:2.0:1619677448.097401:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:1.0:1619677448.097401:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.097403:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.097404:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:1.0:1619677448.097404:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.097405:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:1.0:1619677448.097405:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677448.097407:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.097410:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.097410:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.097412:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:1.0:1619677448.097412:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:2.0:1619677448.097413:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.097415:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.097415:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.097416:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.097417:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.097417:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.097417:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.097419:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.097419:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677448.097420:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.097420:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.097420:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677448.097422:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.097424:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.097425:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.097425:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.097427:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.097427:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.097428:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.097429:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677448.097432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:9.2:1619677448.097433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677448.097433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.097438:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.097440:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.097441:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.097441:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.097442:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.097443:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.097443:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677448.097443:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.097444:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.097444:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677448.097445:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:26.0:1619677448.097445:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.097445:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:31.0:1619677448.097446:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:18.0:1619677448.097446:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.097447:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:30.0:1619677448.097447:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.097448:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:10.0:1619677448.097448:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.097449:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.097450:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.097451:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.097451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:30.0:1619677448.097452:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677448.097453:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677448.097454:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.097455:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.097456:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.097456:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.097458:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.097458:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.097458:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677448.097459:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.097459:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677448.097461:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.097462:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.097462:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.097463:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000400:00000200:30.0:1619677448.097464:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.097464:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:10.0:1619677448.097465:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.097467:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.097467:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.097470:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.097471:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.097471:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.097472:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.097472:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.097472:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.097474:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.097474:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677448.097475:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.097475:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.097476:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.097476:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.097477:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:10.0:1619677448.097477:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.097481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.097488:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:20.0:1619677448.097489:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.097491:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:6.0:1619677448.097491:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.097492:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:20.0:1619677448.097492:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.097492:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.097495:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.097495:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:6.0:1619677448.097496:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.097498:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:6.0:1619677448.097499:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.097508:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.097510:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.097511:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.097511:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.097512:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.097513:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.097568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.097573:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.097576:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.097578:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.097579:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.097581:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.097582:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.097584:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.097585:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.097586:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.097587:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.097591:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.097593:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.097596:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.097597:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.097597:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.097599:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.097600:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.097600:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.097601:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.097613:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.097615:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.097616:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.097618:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.097620:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.097621:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.097690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.097697:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.097699:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.097700:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.097702:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.097705:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.097706:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.097707:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.097709:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.097710:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.097711:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.097714:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.097716:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.097719:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.097720:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.097720:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.097722:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.097722:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.097723:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.097723:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.097728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.097736:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.097738:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.097739:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.097740:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.097742:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.097745:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.097815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.097822:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.097825:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.097826:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.097828:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.097831:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.097832:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.097834:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.097835:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.097836:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.097837:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.097840:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.097842:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.097844:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.097845:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.097846:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.097847:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.097848:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.097848:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.097849:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677448.097861:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.097864:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.097864:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.097865:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.097867:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.097870:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.097933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:4.2:1619677448.097935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677448.097939:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.097941:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.097942:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.097942:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677448.097944:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.097945:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.097946:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677448.097946:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.097947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:10.0:1619677448.097947:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.097948:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.097949:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.097951:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.097951:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.097952:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.097953:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:10.0:1619677448.097953:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677448.097953:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:10.0:1619677448.097955:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.097956:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.097957:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:10.0:1619677448.097958:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.097958:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.097959:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.097960:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.097961:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.097961:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.097962:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.097963:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.097964:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.097965:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:10.0:1619677448.097966:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.097966:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.097967:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.097968:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.097968:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:18.0:1619677448.097969:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:1.0:1619677448.097969:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000400:00000200:18.0:1619677448.097970:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96719389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.097970:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.097971:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.097972:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.097972:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.2:1619677448.097972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:18.0:1619677448.097973:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.097973:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.097973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:21.2:1619677448.097974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:18.0:1619677448.097974:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.097975:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.2:1619677448.097977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677448.097978:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.097979:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:10.0:1619677448.097980:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.097981:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.097981:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:9.2:1619677448.097981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.097981:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:17.0:1619677448.097982:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:10.0:1619677448.097982:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:31.0:1619677448.097983:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.097983:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:10.0:1619677448.097983:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677448.097983:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.097984:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.097984:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.097984:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:20.2:1619677448.097985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677448.097985:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000400:00000200:2.0:1619677448.097985:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.097985:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.097985:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.097986:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:18.0:1619677448.097986:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:17.0:1619677448.097987:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.097987:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.097987:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677448.097988:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:4.2:1619677448.097988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677448.097988:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.097989:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96719388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.097990:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.097990:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677448.097991:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:2.0:1619677448.097991:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.097992:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:2.0:1619677448.097992:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.097993:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.097994:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:2.0:1619677448.097994:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.097995:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:18.0:1619677448.097995:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:31.0:1619677448.097996:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96719387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.097996:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.097997:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:31.0:1619677448.097998:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.097998:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.097999:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:18.0:1619677448.097999:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:31.0:1619677448.098000:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.098000:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677448.098000:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.098001:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.098001:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.098001:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.098002:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:18.0:1619677448.098002:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.098003:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.098004:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.098004:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:2.0:1619677448.098004:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.098005:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.098006:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.098006:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.098007:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.098007:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.098008:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.098008:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:30.0:1619677448.098009:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.098009:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.098009:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677448.098010:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96719386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.098010:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.098011:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.098012:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.098012:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.098012:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:30.0:1619677448.098013:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.098013:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:2.0:1619677448.098013:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:30.0:1619677448.098014:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.098014:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.098015:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:31.0:1619677448.098017:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.098017:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.098018:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.098018:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.098019:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.098020:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.098021:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.098022:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.098022:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.098023:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.098023:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.098024:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.098024:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.098024:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.098024:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.098025:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.098026:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.098026:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.098027:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677448.098027:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.098028:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.098029:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.098030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.098031:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.098032:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677448.098033:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:22.0:1619677448.098034:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:13.0:1619677448.098035:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.098036:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.098037:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677448.098037:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:13.0:1619677448.098037:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.098037:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.098038:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.098039:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.098040:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.098040:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677448.098040:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.098040:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.098042:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.098043:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.098334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677448.098340:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677448.098344:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677448.098345:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677448.098346:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.098348:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.098350:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.098351:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.098352:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.098353:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.098354:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.098358:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.098360:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.098363:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.098364:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.098364:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.098366:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.098366:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677448.098367:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677448.098368:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677448.098373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.098380:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.098383:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.098384:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.098384:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.098386:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.098389:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.2:1619677448.098510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.098517:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:4.2:1619677448.098517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677448.098519:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.098520:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.098522:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.098525:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.098525:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:18.0:1619677448.098526:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.098528:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.098528:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.098529:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677448.098530:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.098531:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.098532:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.098532:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.098535:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.098536:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.098537:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.098537:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.098538:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.098540:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.098540:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:10.0:1619677448.098540:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.098541:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677448.098541:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.098542:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:18.0:1619677448.098543:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.098543:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.098544:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.098545:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:10.0:1619677448.098545:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.098547:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:9.2:1619677448.098549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.098550:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.098551:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.098551:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.098553:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.098554:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.098554:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.098555:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677448.098556:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.098558:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:8.2:1619677448.098559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:2.0:1619677448.098559:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.098560:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.098561:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.098563:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677448.098568:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.098570:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.098571:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.098573:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.098574:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.098576:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.098613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.098619:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.098622:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.098622:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:5.2:1619677448.098623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:6.0:1619677448.098624:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.098626:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.098628:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:6.0:1619677448.098628:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.098629:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.098630:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.098631:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:6.0:1619677448.098632:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.098632:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677448.098633:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.098635:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.098637:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.098639:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.098640:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.098641:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.098642:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:6.0:1619677448.098642:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:17.0:1619677448.098643:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.098643:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:17.0:1619677448.098644:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.098645:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.098645:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.098645:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:17.0:1619677448.098646:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:6.0:1619677448.098647:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.098648:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.098648:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:17.0:1619677448.098650:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.098651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.098652:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.2:1619677448.098653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.098655:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.098656:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.098657:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:22.0:1619677448.098658:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:17.0:1619677448.098659:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:22.0:1619677448.098660:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:17.0:1619677448.098660:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.098660:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677448.098661:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677448.098661:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.098661:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:22.0:1619677448.098662:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677448.098662:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:1.0:1619677448.098662:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.098663:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.098664:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.098665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.098667:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.0:1619677448.098671:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.098674:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.098674:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.098675:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.098677:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.098680:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.098683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.2:1619677448.098686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:24.0:1619677448.098689:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.098692:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:26.0:1619677448.098693:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:24.0:1619677448.098693:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.098693:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.098694:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.098694:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:26.0:1619677448.098696:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.098696:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.098699:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.098701:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.098702:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.098704:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.098705:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.098706:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.098707:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.098710:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.098712:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.098715:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.098716:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.098717:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.098718:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.098719:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.098719:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.098720:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.098724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:21.2:1619677448.098727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677448.098732:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:31.0:1619677448.098735:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.098735:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.098736:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.098736:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677448.098738:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:17.0:1619677448.098738:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000400:00000200:31.0:1619677448.098740:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.098741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677448.098742:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:31.0:1619677448.098743:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.098743:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.098744:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:30.0:1619677448.098745:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:31.0:1619677448.098746:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.098746:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.098748:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.098749:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.098750:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.098752:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.098753:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.098754:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.098755:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:31.0:1619677448.098756:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.098757:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.098758:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.098759:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.098759:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.098762:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.098763:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.098763:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:30.0:1619677448.098763:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.098764:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.098765:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.098765:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.098766:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.098766:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.098767:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677448.098767:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.098768:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.098768:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.098769:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.098770:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.098771:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.098773:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.098773:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.098776:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:20.2:1619677448.098779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.098779:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.098780:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.098780:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.098782:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:10.0:1619677448.098782:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.098784:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:12.0:1619677448.098785:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.098785:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.098787:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:12.0:1619677448.098787:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.098789:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.098790:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.098790:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:13.0:1619677448.098790:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.098791:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.098795:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.098798:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.098798:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.098799:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.098800:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.098800:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.098821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.098828:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.098830:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.098831:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.098833:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.098836:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.098837:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.098838:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.098839:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.098840:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.098841:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.098844:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.098845:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.098848:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.098849:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.098849:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.098851:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.098852:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.098852:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.098853:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.098865:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.098868:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.098869:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.098869:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.098871:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.098874:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.098960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.098966:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.098968:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.098969:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.098971:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.098973:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.098975:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.098976:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.098977:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.098978:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.098979:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.098982:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.098984:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.098986:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.098987:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.098988:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.098989:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.098990:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.098990:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.098991:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.099003:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.099005:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.099006:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.099006:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.099008:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.099011:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.099151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:4.2:1619677448.099156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677448.099158:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.099161:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:26.0:1619677448.099162:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677448.099162:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.099163:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.099165:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.099166:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:8.0:1619677448.099167:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.099168:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.099168:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677448.099170:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.099170:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.099171:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.099172:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.099172:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.099173:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.099174:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.099174:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677448.099176:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.099176:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677448.099178:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.099180:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.099180:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.099182:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.099184:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.099185:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.099185:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.099185:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.099186:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.099186:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.099187:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:26.0:1619677448.099188:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.099188:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.099188:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677448.099189:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.099189:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677448.099189:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:26.0:1619677448.099190:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.099193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.2:1619677448.099197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677448.099201:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.099204:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.099205:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.099205:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.099206:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:17.0:1619677448.099207:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.099208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:5.0:1619677448.099209:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:17.0:1619677448.099210:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:5.0:1619677448.099211:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.099212:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:5.0:1619677448.099213:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.099214:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.099215:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (23)++
00000800:00000200:20.0:1619677448.099215:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (24)--
00000800:00000200:8.2:1619677448.099215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.099216:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677448.099216:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (24)--
00000800:00000200:20.0:1619677448.099217:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.099218:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:24.0:1619677448.099219:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.099221:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.099223:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.099223:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:24.0:1619677448.099226:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.099226:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.099227:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.099227:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000400:00000200:24.0:1619677448.099228:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:1.0:1619677448.099228:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.099229:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:24.0:1619677448.099230:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.099231:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.099231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:24.0:1619677448.099233:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.099234:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.099236:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:24.0:1619677448.099237:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.099239:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.099239:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.099239:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.099241:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.099242:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.099243:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:17.0:1619677448.099243:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.099244:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:17.0:1619677448.099245:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:24.0:1619677448.099246:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:17.0:1619677448.099246:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.099247:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.099248:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:17.0:1619677448.099248:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.099248:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.099249:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:17.0:1619677448.099250:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.099252:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.099253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.099254:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.099257:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.099258:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.099258:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677448.099260:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:17.0:1619677448.099260:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.099261:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.099261:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.099262:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677448.099263:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.099264:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.099264:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.099266:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.099270:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677448.099273:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.099275:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.099276:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.099278:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.099280:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.099281:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677448.099290:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.099293:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.099298:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.099299:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.099299:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.099300:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.099428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.099431:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.099434:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.099436:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.099437:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.099439:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.099441:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.099442:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.099443:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.099444:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.099445:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.099449:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.099451:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:4.2:1619677448.099451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677448.099455:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.099456:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.099457:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.099459:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.099459:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.099459:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.099461:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:26.0:1619677448.099462:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677448.099462:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:10.0:1619677448.099462:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:20.0:1619677448.099463:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:10.0:1619677448.099464:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.099466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.2:1619677448.099468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:10.0:1619677448.099468:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.099469:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.099470:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.099472:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.099473:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.099474:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677448.099474:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.099474:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.099475:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677448.099477:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:6.0:1619677448.099477:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677448.099477:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:10.0:1619677448.099478:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.099478:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.099478:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677448.099479:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.099479:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:10.0:1619677448.099480:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:31.0:1619677448.099481:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.099481:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.099482:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677448.099483:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.099484:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.099484:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.099484:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.099484:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:31.0:1619677448.099486:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:10.0:1619677448.099486:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:31.0:1619677448.099487:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.099487:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.099488:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.099488:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677448.099489:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.099489:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000400:00000200:31.0:1619677448.099490:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.099490:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677448.099493:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.099495:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:4.2:1619677448.099495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.099497:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.099498:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.099498:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.099498:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.099500:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.099500:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.099500:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.099500:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.099501:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.099501:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:30.0:1619677448.099502:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.099502:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:31.0:1619677448.099503:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.099504:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.099504:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.099505:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.099508:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.099510:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.099511:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.099512:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.099512:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.099514:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.099516:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.099574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.099579:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.099582:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.099584:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.099585:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.099587:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.099589:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.099590:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.099592:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.099593:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.099594:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.099597:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.099599:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.099602:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.099603:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.099604:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.099606:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.0:1619677448.099611:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.099611:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.099612:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.099613:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:10.0:1619677448.099614:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.099617:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.099618:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.099619:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.099621:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.099623:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677448.099623:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.099624:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.099625:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.099626:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.099627:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.099627:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677448.099629:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.099629:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.099630:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.099632:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.099634:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.099635:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.099636:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.099637:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.099638:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.099639:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.099640:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.099640:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.099641:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.099652:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.099654:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.099655:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.099656:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.099658:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.099658:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:21.2:1619677448.099742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.099748:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.099751:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.099751:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.099754:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.099756:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.099757:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.099759:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.099760:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.099761:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.099762:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.099765:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.099767:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.099770:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.099771:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.099772:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.099774:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.099774:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.099775:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.099775:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:26.0:1619677448.099786:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.099789:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.099789:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.099790:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.099792:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.099794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:26.0:1619677448.099795:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:8.0:1619677448.099801:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:5.2:1619677448.099803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.0:1619677448.099804:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.099805:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.099807:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.099809:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677448.099810:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.099811:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:17.0:1619677448.099812:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.099812:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:8.0:1619677448.099813:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.099814:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.099815:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.099815:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.099816:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.099818:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.099819:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:8.0:1619677448.099821:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.099821:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.099823:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.099824:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.099825:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.099825:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:8.0:1619677448.099828:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.099828:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.099829:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677448.099829:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.099831:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.099831:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:1.0:1619677448.099831:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:8.0:1619677448.099832:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.099833:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:1.0:1619677448.099834:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.099835:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.099835:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:9.2:1619677448.099837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.099837:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.099837:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.099839:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.099840:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.099840:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.099845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.099845:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.099848:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.099849:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:4.2:1619677448.099849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677448.099849:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.099850:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.099851:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:2.0:1619677448.099851:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.099853:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677448.099853:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.099854:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.099854:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677448.099854:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.099855:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677448.099856:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.099856:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677448.099856:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.099857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.099858:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.099859:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677448.099859:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:10.0:1619677448.099861:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.099864:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.099865:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677448.099866:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.099867:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.099868:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.099868:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.099869:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677448.099869:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.099870:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000400:00000200:10.0:1619677448.099870:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.099871:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:10.0:1619677448.099871:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.099875:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.099876:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.099879:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.099880:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.099881:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.099882:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.099883:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.099883:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.099884:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.099896:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.099899:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.099900:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.099900:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.099902:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.099905:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677448.099914:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.099917:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.099918:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.099918:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.099919:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.099920:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.100265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.100272:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.100274:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.100275:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.100277:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.100277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000400:00000200:24.0:1619677448.100280:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.100281:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.100282:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.100283:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.100283:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:24.0:1619677448.100284:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.100285:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.100285:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677448.100286:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:24.0:1619677448.100288:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.100290:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.100291:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.100293:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.100293:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.100300:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677448.100300:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.100301:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:24.0:1619677448.100302:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.100303:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:2.0:1619677448.100303:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.100304:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.100304:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:2.0:1619677448.100304:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.100305:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.100306:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:9.2:1619677448.100309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677448.100310:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.100312:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.100316:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.100317:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.100318:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.100318:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.100319:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.100320:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.100320:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.100320:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677448.100322:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677448.100323:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.100323:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:2.0:1619677448.100324:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.100325:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:26.0:1619677448.100333:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.100335:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.100336:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.100337:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.100339:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.100341:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.100341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.2:1619677448.100342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:5.2:1619677448.100344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:8.0:1619677448.100348:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677448.100349:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.100350:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:24.0:1619677448.100351:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.100351:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:2.0:1619677448.100351:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.100352:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:18.0:1619677448.100354:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.100355:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677448.100356:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.100358:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.100359:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.100360:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.100361:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.100362:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.100363:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.100364:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.100365:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.100366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000400:00000200:18.0:1619677448.100366:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.100367:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.100368:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.100369:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.100370:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.100372:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000400:00000200:18.0:1619677448.100372:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.100372:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677448.100373:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.100375:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:18.0:1619677448.100375:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.100375:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:2.0:1619677448.100375:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.100377:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.100379:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.100379:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:8.0:1619677448.100380:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.100382:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.100383:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.100383:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.100384:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.100384:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.100385:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.100386:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.100386:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:22.0:1619677448.100387:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.100387:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (23)++
00000800:00000200:8.0:1619677448.100387:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.100388:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000400:00000200:22.0:1619677448.100389:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:18.0:1619677448.100389:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (24)--
00000800:00000200:8.0:1619677448.100389:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677448.100389:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.100390:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96719357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.100390:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.100390:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677448.100391:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.100391:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:22.0:1619677448.100392:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.100392:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.100392:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (23)++
00000400:00000200:22.0:1619677448.100393:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.100393:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.100394:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:5.2:1619677448.100394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.100394:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.100395:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:8.0:1619677448.100396:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (24)--
00000800:00000200:2.0:1619677448.100396:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:22.0:1619677448.100397:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.100397:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.100397:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677448.100397:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.0:1619677448.100398:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677448.100398:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:22.0:1619677448.100399:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677448.100400:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.100400:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677448.100400:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000800:00000200:18.0:1619677448.100401:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:21.2:1619677448.100402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677448.100403:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.100403:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (24)++
00000800:00000200:1.0:1619677448.100403:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000800:00000200:22.0:1619677448.100404:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:1.0:1619677448.100404:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.100405:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:17.0:1619677448.100405:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000400:00000200:17.0:1619677448.100405:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.100407:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:17.0:1619677448.100407:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.100407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677448.100407:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:1.0:1619677448.100407:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.100408:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677448.100408:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.100409:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:2.0:1619677448.100409:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:22.0:1619677448.100410:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:6.0:1619677448.100410:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677448.100410:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677448.100410:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.100411:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:6.0:1619677448.100411:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.100412:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677448.100412:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:6.0:1619677448.100413:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:1.0:1619677448.100413:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677448.100414:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.100417:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:6.0:1619677448.100417:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:26.0:1619677448.100418:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.100420:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.100421:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:10.0:1619677448.100422:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.100423:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.100424:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.100424:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.100425:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.100426:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.100426:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.100426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677448.100427:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.100428:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:10.0:1619677448.100428:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.100429:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:22.0:1619677448.100429:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.100429:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:22.0:1619677448.100430:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.100430:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.100431:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.100431:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.100432:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:10.0:1619677448.100432:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.100432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677448.100434:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:10.0:1619677448.100434:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677448.100435:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.100437:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.100438:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.100439:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.100439:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.100439:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.100440:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677448.100441:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:10.0:1619677448.100441:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.100442:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.100442:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677448.100443:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.100443:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.100443:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.100444:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677448.100445:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.100447:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.100447:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.100448:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.100450:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.100451:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.100452:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.100453:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96719354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.100453:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.100453:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.100454:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.100455:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.100455:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.100455:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.100456:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.100456:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:26.0:1619677448.100457:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:17.0:1619677448.100457:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:10.0:1619677448.100457:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.100458:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.100459:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.100459:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.100460:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677448.100461:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.100463:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.100463:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.100464:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.100464:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:8.2:1619677448.100464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.100465:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.100466:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.100468:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.100468:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.100469:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.100469:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677448.100472:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.100474:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.100474:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.100475:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.100477:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.100478:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:10.0:1619677448.100479:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.100481:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.100482:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.100482:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.100484:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.100485:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:8.2:1619677448.100485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.100493:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.100495:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.100496:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.100496:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.100497:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.100498:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:26.0:1619677448.100502:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.100505:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.100505:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.100506:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.100507:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.100508:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.100615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.100623:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.100626:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.100628:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.100629:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.100630:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.100632:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.100633:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.100634:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.100635:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.100636:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.100639:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.100640:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.100643:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.100643:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.100644:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.100645:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.100646:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.100647:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.100648:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.100658:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.100661:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.100662:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.100663:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.100665:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.100666:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.100678:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.100680:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.100681:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.100682:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.100682:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.100683:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.100789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.100796:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.100798:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.100799:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.100801:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.100804:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.100805:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.100806:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.100808:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.100809:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.100809:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.100812:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.100814:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:5.2:1619677448.100814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:24.0:1619677448.100816:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.100817:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.100817:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.100819:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.100819:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.100820:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:20.2:1619677448.100820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:24.0:1619677448.100822:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.100822:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:10.0:1619677448.100822:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.100824:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.100825:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.100827:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.100828:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:21.2:1619677448.100828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000400:00000200:10.0:1619677448.100828:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.100829:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.100830:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.100831:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:12.0:1619677448.100831:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:10.0:1619677448.100832:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.100833:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:10.0:1619677448.100833:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.100834:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677448.100834:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.100834:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:8.0:1619677448.100834:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:30.0:1619677448.100835:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:12.0:1619677448.100835:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.100836:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.100836:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:13.0:1619677448.100837:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.100837:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:10.0:1619677448.100837:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.100837:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:31.0:1619677448.100838:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.100839:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.100839:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.0:1619677448.100839:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:31.0:1619677448.100840:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.100841:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.100841:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.100841:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:31.0:1619677448.100842:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.100842:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.100843:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.100843:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677448.100844:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.100844:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:31.0:1619677448.100846:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.100846:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.100846:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.100848:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.100849:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.100849:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.100851:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.100852:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.100852:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:5.2:1619677448.100852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.100853:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.100856:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.100857:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.100858:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.100860:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.100860:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.100861:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.100862:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.100863:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:13.0:1619677448.100863:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:10.0:1619677448.100863:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.100863:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.100866:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.100866:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.100868:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.100869:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.100870:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.101056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.101062:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.101065:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.101065:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.101068:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.101071:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.101072:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.101073:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.101075:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.101076:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.101077:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.101080:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.101082:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.101085:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.101085:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.101086:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.101088:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.101089:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.101089:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.101090:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.101096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.101103:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.101106:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.101106:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.101108:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.101110:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.101111:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.101117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.101125:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.101128:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.101129:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.101129:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.101130:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.101131:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.101194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.101202:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.101205:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.101207:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.101208:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.101209:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.101210:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.101212:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.101214:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.101215:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.101216:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.101219:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.101221:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.101223:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.101224:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.101225:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.101226:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.101227:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.101228:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.101230:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.101230:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677448.101236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.101238:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.101240:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.101241:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.101243:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:12.0:1619677448.101243:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.101244:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.101245:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.101246:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.101247:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.101248:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677448.101253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677448.101260:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677448.101263:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677448.101264:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677448.101265:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677448.101268:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.101269:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677448.101270:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.101271:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.101272:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.101273:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677448.101276:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677448.101278:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677448.101281:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.101281:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677448.101282:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677448.101284:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677448.101284:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677448.101285:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677448.101286:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677448.101287:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:5.2:1619677448.101292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (21)++
00000800:00000200:22.0:1619677448.101299:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677448.101300:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677448.101301:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:22.0:1619677448.101302:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:8.0:1619677448.101302:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:22.0:1619677448.101303:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.101303:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:6.0:1619677448.101303:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.101305:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.101308:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.101383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.101391:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.101394:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.101395:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.101396:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.101398:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.101399:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.101400:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.101402:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.101403:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.101403:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.101407:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.101408:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.101410:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.101411:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.101411:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.101413:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.101414:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.101414:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.101415:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677448.101427:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.101430:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.101430:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.101432:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.101434:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.101435:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.101461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.101468:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.101471:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.101471:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.101474:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.101476:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.101478:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:4.2:1619677448.101478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:18.0:1619677448.101479:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.101481:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.101482:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.101483:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:24.0:1619677448.101485:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:18.0:1619677448.101486:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.101488:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:18.0:1619677448.101488:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.0:1619677448.101489:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.101491:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.101491:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.101492:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.101493:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:24.0:1619677448.101494:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.101494:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.101495:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:24.0:1619677448.101496:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:18.0:1619677448.101496:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.101496:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:24.0:1619677448.101497:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96719344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.101499:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.101500:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.101501:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.101505:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.101506:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.101506:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:24.0:1619677448.101509:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.101509:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.101510:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:24.0:1619677448.101510:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:10.0:1619677448.101510:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.101511:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.101512:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:10.0:1619677448.101512:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.101513:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.101513:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.101514:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.101515:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.2:1619677448.101515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677448.101515:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:8.2:1619677448.101518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.101520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.0:1619677448.101521:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.101524:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.101526:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.101527:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.101527:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677448.101528:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.0:1619677448.101529:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:5.0:1619677448.101529:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.101530:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:5.0:1619677448.101530:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:1.0:1619677448.101530:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.101531:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.101531:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:5.0:1619677448.101531:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.101531:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.101533:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.101534:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.101535:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.101536:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677448.101540:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.101541:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677448.101542:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.101544:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.101545:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.101546:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.101547:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.101548:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.101549:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.101549:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.101553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.101561:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.101564:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.101564:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.101564:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.101566:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.101569:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.101656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.101662:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.101665:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.101665:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.101668:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.101670:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.101671:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.101673:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.101674:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.101675:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.101676:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.101680:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.101682:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.101685:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.101686:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.101686:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.101688:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.101688:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.101689:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.101690:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677448.101701:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.101703:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.101704:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.101706:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.101707:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.101709:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.101718:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.101721:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.101722:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.101723:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.101723:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.101725:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.101765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.101772:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.101775:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.101777:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.101779:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.101779:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.101781:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.101782:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.101783:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.101784:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.101785:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.101788:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.101790:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.101793:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.101794:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.101795:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.101797:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.101797:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.101798:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:21.2:1619677448.101798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.101801:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677448.101802:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.101803:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.2:1619677448.101804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:31.0:1619677448.101805:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.101806:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.101807:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.101808:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.101809:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.101811:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.101811:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.101811:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:31.0:1619677448.101812:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.101814:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:31.0:1619677448.101815:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.101815:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.101816:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.101817:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.101818:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.101819:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.101820:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.101820:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677448.101821:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.101822:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.101823:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.101824:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.101824:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677448.101827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.101834:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.101836:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:13.0:1619677448.101836:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.101837:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:13.0:1619677448.101838:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:26.0:1619677448.101839:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.101839:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.101840:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.101841:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.101843:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:13.0:1619677448.101843:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:26.0:1619677448.101844:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.101844:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677448.101845:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.101846:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.101847:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.101849:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.101851:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.101853:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.101854:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.101854:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.101856:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.101856:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.101857:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.101857:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.101868:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.101870:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.101871:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.101872:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.101874:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.101875:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.101991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677448.101993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.0:1619677448.101999:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.101999:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.102002:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677448.102002:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.102003:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.102005:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.102006:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.102006:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.102008:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.102009:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.102010:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:5.0:1619677448.102011:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.102011:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.102013:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.102013:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.102015:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.102015:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.102016:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.102016:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:5.0:1619677448.102018:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.102021:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.102023:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:5.0:1619677448.102025:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.102026:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.102027:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.102027:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.102028:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.102030:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.102030:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.102031:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.102031:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.102031:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.102032:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.102032:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677448.102034:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.102035:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.102035:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.102036:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.102044:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.102046:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.102047:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.102047:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.102048:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.102049:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:12.0:1619677448.102049:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.102051:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.102052:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.102053:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.102054:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.102055:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.102063:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.102065:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.102066:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.102067:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.102067:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.102068:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.102080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.102083:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.102085:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.102087:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.102088:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.102090:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.102092:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677448.102093:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.102095:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.102096:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.102097:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677448.102100:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.102102:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.102105:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.102106:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.102106:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.102108:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.102109:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.102110:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.102111:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:10.0:1619677448.102121:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.102123:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.102124:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.102124:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.102127:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.102130:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.102134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.102142:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.102144:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.102145:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.102147:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.102149:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.102151:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.102152:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.102153:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.102154:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.102155:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.102159:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.102160:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.102163:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.102164:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.102165:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.102167:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.102167:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.102168:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.102168:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.102173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.102180:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.102183:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.102184:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.102184:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.102186:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.102190:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.102306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.102312:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.102314:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.102316:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.102317:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.102319:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.102321:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.102322:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.102323:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.102324:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.102325:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.102329:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.102331:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.0:1619677448.102334:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.102337:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.102338:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677448.102338:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.102339:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.102341:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:5.0:1619677448.102341:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.102341:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.102341:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677448.102342:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:18.0:1619677448.102343:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.102343:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677448.102346:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.102347:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.102348:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.102350:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.102351:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.102352:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.102355:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.102355:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.2:1619677448.102356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677448.102356:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677448.102357:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677448.102358:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677448.102358:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.102359:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.102359:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:12.0:1619677448.102360:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.102361:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.102361:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.102362:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:13.0:1619677448.102362:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.102363:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677448.102363:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.102365:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.102367:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:9.2:1619677448.102367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677448.102369:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.102372:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.102372:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.102373:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.102374:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.102374:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:13.0:1619677448.102375:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.102376:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.102376:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:31.0:1619677448.102377:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.102377:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.102378:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.102378:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677448.102378:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.102379:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:2.0:1619677448.102379:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.102380:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.102380:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:31.0:1619677448.102381:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.102381:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677448.102382:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.102383:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.102386:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.102387:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.102388:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.102390:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.102390:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.102391:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:21.2:1619677448.102391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:31.0:1619677448.102392:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:18.0:1619677448.102397:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:8.2:1619677448.102397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.102400:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.102401:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.102403:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.102404:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:18.0:1619677448.102404:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.102405:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677448.102407:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.102407:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.102407:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.102408:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.102409:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.102410:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.102410:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:18.0:1619677448.102411:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.102413:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:18.0:1619677448.102414:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.102416:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.102419:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.102420:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.102420:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.102422:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.102423:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.102424:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.102425:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.102426:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.2:1619677448.102431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:12.0:1619677448.102434:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.102436:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:12.0:1619677448.102437:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.2:1619677448.102437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:12.0:1619677448.102438:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.102438:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.102439:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.102439:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:12.0:1619677448.102440:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.102441:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.102443:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677448.102444:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.102447:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.102447:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.102450:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.102453:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.102454:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.102456:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.102457:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.102458:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.102459:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.102463:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.102465:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.102468:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.102469:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.102470:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.102471:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.102472:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.102473:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.102473:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.102484:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.102487:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.102488:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.102490:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.102490:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.102493:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.102507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677448.102509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677448.102515:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.102517:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.102517:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.102518:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.102519:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677448.102521:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.102522:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.102523:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.102525:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.102525:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.102527:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.102527:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.102528:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.102528:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.102529:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.102530:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.102531:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.102532:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.102532:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.102533:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.102536:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.102538:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.102541:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.102542:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.102542:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.102542:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.102543:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.102545:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.102545:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.102545:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.102546:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.102546:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.102547:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:31.0:1619677448.102548:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677448.102548:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:31.0:1619677448.102549:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.102549:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677448.102549:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.102550:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677448.102554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.102559:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:26.0:1619677448.102561:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.102562:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:6.0:1619677448.102562:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (24)++
00000800:00000200:26.0:1619677448.102563:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677448.102563:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.102564:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677448.102564:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000400:00000200:6.0:1619677448.102564:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.102565:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.102565:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.102566:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.102570:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.102570:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.102571:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.102694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.102700:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.102703:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.102705:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.102705:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.102708:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.102709:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.102711:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.102712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:5.0:1619677448.102714:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.102715:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.102716:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.102717:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.102720:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:5.0:1619677448.102720:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.102721:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.0:1619677448.102722:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:6.0:1619677448.102723:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.102726:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.102726:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.102727:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:5.0:1619677448.102727:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.102727:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:6.0:1619677448.102729:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719326] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.102729:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:6.0:1619677448.102730:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.102730:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:6.0:1619677448.102731:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.102731:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:6.0:1619677448.102732:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.102732:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.102733:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:6.0:1619677448.102736:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.102738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.102738:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.102741:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:6.0:1619677448.102741:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.102742:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677448.102742:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.102742:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.102743:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.102743:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.102744:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.102744:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:6.0:1619677448.102744:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.102745:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:13.0:1619677448.102746:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.102746:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:13.0:1619677448.102747:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.102747:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:12.0:1619677448.102748:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677448.102748:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:13.0:1619677448.102750:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.102753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677448.102756:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677448.102758:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:30.0:1619677448.102759:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.102760:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.102761:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.102761:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:30.0:1619677448.102762:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.102763:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.102763:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677448.102765:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.102826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.102833:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.2:1619677448.102834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.0:1619677448.102836:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.102837:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.102839:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.102839:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:24.0:1619677448.102841:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.102842:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:24.0:1619677448.102843:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:17.0:1619677448.102843:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677448.102844:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.102846:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.102846:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.102847:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.102848:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.102849:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.102852:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.102854:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.102854:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.102855:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.102857:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.102857:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.102858:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.102859:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.102859:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:1.0:1619677448.102859:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677448.102861:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.102861:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.102862:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.102862:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.102863:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.102865:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.2:1619677448.102867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.102868:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.102869:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.102870:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.102872:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.102873:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.102874:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.102875:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.102876:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.102876:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.102877:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.102877:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.102878:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.102880:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.102881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.102883:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677448.102884:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677448.102885:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677448.102885:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.2:1619677448.102886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.102887:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.102888:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.102889:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.102889:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.102889:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677448.102891:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.102892:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.102892:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.102893:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.102893:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.102894:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:1.0:1619677448.102894:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.102894:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.102902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.102905:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.102907:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.102908:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.102909:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.102910:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.102911:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.102911:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.102912:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677448.102913:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.102914:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.102917:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.102918:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.102920:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.102921:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.102922:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.102923:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.102926:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.102928:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.102930:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.102931:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.102931:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.102933:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.102934:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.102935:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.102936:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.102937:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677448.102943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.102945:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.102947:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.102948:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.102949:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.102951:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:13.0:1619677448.102951:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.102952:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.102953:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.102954:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.102955:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.2:1619677448.103006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.103014:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.103017:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.103017:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677448.103019:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.103022:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.103023:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.103025:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.103026:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.103027:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.103027:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.103030:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.103032:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.103035:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.103036:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.103036:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.103038:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.103038:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.2:1619677448.103039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677448.103039:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677448.103041:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.103041:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:20.0:1619677448.103042:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.2:1619677448.103044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.103045:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.2:1619677448.103046:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:5.0:1619677448.103048:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.103051:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.103051:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:26.0:1619677448.103053:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:20.0:1619677448.103054:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.103054:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.103054:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.103055:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:17.0:1619677448.103055:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.103056:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:20.0:1619677448.103056:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:26.0:1619677448.103057:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:20.0:1619677448.103057:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.103057:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:5.2:1619677448.103058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:20.0:1619677448.103059:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.103060:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.103060:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:20.0:1619677448.103061:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677448.103064:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.103064:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677448.103066:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.103067:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.103068:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677448.103069:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.103070:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677448.103070:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:18.0:1619677448.103070:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.103072:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677448.103073:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.103073:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:18.0:1619677448.103073:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.103074:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677448.103074:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.103076:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.103077:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.103078:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677448.103078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:18.0:1619677448.103079:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.103083:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.103085:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.103087:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.103088:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.103088:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.103089:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:2.0:1619677448.103089:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.103090:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677448.103091:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.103092:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.103092:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.103093:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.103094:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677448.103094:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.103095:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.103098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.103105:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.103106:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.103107:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.103108:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.103109:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.103110:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.103118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.103125:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.103127:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.103128:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.103128:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.103129:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.103129:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.103197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.103204:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.103207:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.103209:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.103210:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.103212:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.103213:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.103215:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.103216:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.103217:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.103218:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.103221:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.103221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:5.2:1619677448.103222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677448.103223:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.103225:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.103226:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.103227:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.103228:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:1.0:1619677448.103228:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.103229:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.103229:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677448.103230:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.103231:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677448.103231:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000800:00000200:17.0:1619677448.103231:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.103231:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677448.103232:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677448.103234:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.103235:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.103236:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.103237:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.103238:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677448.103239:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.103239:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:10.0:1619677448.103240:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:30.0:1619677448.103241:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.103241:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.103242:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.103243:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.103243:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.103243:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.103244:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.103244:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.103245:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.103245:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.103246:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.103248:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:30.0:1619677448.103250:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.103250:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.103253:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.103254:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.103256:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.103257:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:1.0:1619677448.103257:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.103258:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.103258:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:1.0:1619677448.103258:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.103259:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.103260:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677448.103261:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:26.0:1619677448.103261:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.103261:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:1.0:1619677448.103261:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.103261:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.103262:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (21)++
00000800:00000200:1.0:1619677448.103262:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.103263:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:26.0:1619677448.103263:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:1.0:1619677448.103263:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.103264:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:26.0:1619677448.103264:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:30.0:1619677448.103265:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:9.2:1619677448.103267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.103270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:18.0:1619677448.103276:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:13.0:1619677448.103276:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:31.0:1619677448.103278:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:13.0:1619677448.103278:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:18.0:1619677448.103279:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:13.0:1619677448.103279:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.103280:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc2e00] (23)--
00000400:00000200:18.0:1619677448.103280:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.103280:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.103281:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:13.0:1619677448.103281:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.103282:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:12.0:1619677448.103282:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:18.0:1619677448.103283:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.103284:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677448.103286:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.103295:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.103303:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.103306:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.103307:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.103309:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.103312:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.103314:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.103315:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.103316:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.103317:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.103318:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.103322:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.103324:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.103327:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.103327:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.103328:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.103330:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.103330:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.103331:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.103332:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.103337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.103345:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.103348:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.103349:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.103351:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.103351:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.103354:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.103541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.103547:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.103550:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.103550:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.103553:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.103556:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.103557:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.103558:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.103560:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.103561:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.103562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:17.0:1619677448.103562:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.103565:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.103567:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:2.0:1619677448.103568:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677448.103569:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.103570:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.103571:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:2.0:1619677448.103571:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.103572:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677448.103572:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.103573:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.103573:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:2.0:1619677448.103573:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:17.0:1619677448.103574:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:2.0:1619677448.103575:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.103577:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.103578:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.103579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:2.0:1619677448.103579:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.103580:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.103581:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.103584:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677448.103585:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:2.0:1619677448.103586:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.103587:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.103588:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:2.0:1619677448.103588:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.103589:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.103589:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.103589:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677448.103591:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.103591:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.103591:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.103592:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.103593:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677448.103595:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677448.103597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.103604:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.2:1619677448.103605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.103606:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.103607:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.103609:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.103609:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.103610:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.103611:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.103611:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.103612:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.103613:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.103615:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.103617:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.103618:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.103619:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.103620:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.103621:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.103624:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.103625:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.103627:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.103628:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.103629:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.103630:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.103631:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.103631:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.103632:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677448.103633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.103641:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.103644:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:4.2:1619677448.103645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:31.0:1619677448.103646:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.103647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:31.0:1619677448.103649:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.103649:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.103650:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.103652:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.103653:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.103653:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.103654:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.103655:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.103655:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:20.0:1619677448.103655:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:10.0:1619677448.103656:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677448.103657:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.103658:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.103659:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.103659:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.103660:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.103661:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.103661:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.103662:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.103663:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677448.103663:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:26.0:1619677448.103664:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.103665:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.103665:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.103666:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.103666:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:26.0:1619677448.103666:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.103667:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.103668:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.103669:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.103670:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.103670:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:26.0:1619677448.103671:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.103672:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.103676:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.103676:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.103677:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.103679:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.103679:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.103680:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.103680:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.103685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.103693:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.103694:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677448.103696:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677448.103696:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677448.103697:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:5.0:1619677448.103697:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.103698:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.103699:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:6.0:1619677448.103701:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.103701:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.103703:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:6.0:1619677448.103704:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.2:1619677448.103705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.103712:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.103713:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.103715:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.103716:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.103716:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.103717:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.103717:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.103717:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677448.103718:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.103718:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.103719:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677448.103719:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.103778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.103786:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.103789:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.103792:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.103793:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.103795:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.103796:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.103797:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.103799:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.103800:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.103801:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.103804:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.103806:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.103808:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.103809:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.103809:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.103811:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.103811:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.103813:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.103815:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.103815:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677448.103821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.103822:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.103825:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.103826:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.103826:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.103828:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:10.0:1619677448.103828:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.103830:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677448.103831:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.103831:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.103832:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677448.103905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.103913:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.103915:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.103916:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.103918:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.103920:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.103922:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.103923:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719309] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.103924:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.103925:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.103926:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.103929:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.103931:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.103934:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.103935:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.103936:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.103937:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.103938:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.103938:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.103939:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.103943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.103951:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.103954:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.103955:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.103956:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.103958:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.103961:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.103993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.103999:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.104001:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.104002:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.104004:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.104007:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.104008:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.104010:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.104011:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.104012:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.104013:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.104016:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.104018:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.104021:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.104021:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.104022:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.104024:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.104024:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.104025:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.104026:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.104038:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.104040:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.104041:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.104043:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.104045:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.104046:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.104149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:4.2:1619677448.104153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.104158:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.104160:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.104161:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.104163:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.104163:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.104165:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.104167:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.104169:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.104170:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.104171:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.104172:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.104173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:10.0:1619677448.104173:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.104175:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.104176:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.104177:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.104177:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.104179:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.104179:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.104180:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.104181:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.104184:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.104184:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.104185:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.104185:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.104186:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.104188:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.104188:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.104188:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.104189:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.104189:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677448.104189:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.104190:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.104191:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (23)--
00000800:00000200:26.0:1619677448.104191:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.104191:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677448.104193:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.104193:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.104194:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.104194:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.104195:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.104195:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.104196:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.104197:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.104199:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.104200:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:4.2:1619677448.104200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:10.0:1619677448.104201:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96719305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.104201:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (23)++
00000400:00000200:10.0:1619677448.104202:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.104203:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.104204:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (24)++
00000400:00000200:10.0:1619677448.104204:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:6.0:1619677448.104204:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000800:00000200:22.0:1619677448.104205:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000400:00000200:6.0:1619677448.104205:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.104206:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.104208:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.104209:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:22.0:1619677448.104209:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.104210:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:10.0:1619677448.104210:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.104211:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.104212:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.104212:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.104213:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.104213:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:6.0:1619677448.104214:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.104217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677448.104217:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.104217:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.104218:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.104220:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.104220:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.104221:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.104222:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:31.0:1619677448.104225:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:8.2:1619677448.104226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.104227:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.104230:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.104232:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.104232:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.104233:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.104234:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.104234:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:31.0:1619677448.104236:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.104236:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:31.0:1619677448.104237:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.104237:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677448.104237:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.104239:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.104240:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:31.0:1619677448.104241:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.104243:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677448.104244:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.104246:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.104246:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.104247:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.104249:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.104249:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.104250:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.104251:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.104261:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.104264:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.104264:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.104266:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.104267:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.104268:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.104298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.104305:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.104308:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.104309:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.104310:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.104313:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.104314:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.104315:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.104317:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.104318:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.104318:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677448.104321:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.104323:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.104326:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.104327:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.104328:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.104329:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.104330:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.104331:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.104331:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.104335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.104336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.0:1619677448.104344:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:31.0:1619677448.104345:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.104347:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.0:1619677448.104347:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.104348:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.104350:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.104350:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.104351:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.104352:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.104353:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.104353:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:31.0:1619677448.104354:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.104356:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.104357:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.104358:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.104359:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.104362:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.104364:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.104366:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.104367:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.104367:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.104369:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.104369:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.104370:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.104371:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.104382:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.104384:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.104385:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.104385:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.104387:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.104390:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.104447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677448.104448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.104454:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.104454:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.104457:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:10.0:1619677448.104457:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:6.0:1619677448.104457:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677448.104458:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.104460:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.104462:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.104464:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.104465:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:10.0:1619677448.104465:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.104467:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.104467:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:22.0:1619677448.104469:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.104469:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.104471:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.104471:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.104472:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:10.0:1619677448.104472:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.104474:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677448.104477:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.104479:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.104480:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:10.0:1619677448.104481:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.104482:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.104483:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.104484:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:10.0:1619677448.104484:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.104485:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.104486:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:10.0:1619677448.104486:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.104487:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.104487:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.104488:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.104488:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.104488:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.104489:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.104490:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.104493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.104500:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:6.0:1619677448.104502:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677448.104503:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.104504:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.104505:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.104505:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.104506:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.104506:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.104507:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.104509:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.104510:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:6.0:1619677448.104512:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.104661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.104664:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.104666:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.104669:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.104670:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.104671:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.104672:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.104674:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.104675:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.104676:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.104677:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.104681:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.104682:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.104685:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.104686:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.104687:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.104689:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.104690:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.104691:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.104692:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.104693:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.2:1619677448.104693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:9.2:1619677448.104695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677448.104698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677448.104698:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.104701:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.104702:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.104703:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677448.104703:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:18.0:1619677448.104705:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:17.0:1619677448.104705:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.104706:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.104706:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.104706:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:2.0:1619677448.104706:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.104707:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:17.0:1619677448.104708:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.104709:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.104709:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.104711:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.104712:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:17.0:1619677448.104712:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.104713:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.104714:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:9.2:1619677448.104714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:17.0:1619677448.104718:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.104719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677448.104720:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677448.104721:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.104723:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.104724:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677448.104724:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:2.0:1619677448.104724:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.104725:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:2.0:1619677448.104725:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.104725:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.104726:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.104726:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677448.104727:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.104727:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.104729:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:17.0:1619677448.104729:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.104730:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:10.0:1619677448.104730:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.104731:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.104732:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.104733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:26.0:1619677448.104735:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.104736:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:5.2:1619677448.104736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.104737:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.104739:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.104739:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:26.0:1619677448.104740:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.104741:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:1.0:1619677448.104741:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677448.104742:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.104743:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.104743:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.104743:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.104744:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.104745:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.104745:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.104745:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.104747:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.104749:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:26.0:1619677448.104750:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.104750:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.104751:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.104753:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.104753:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.104754:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.104754:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.104759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.104767:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.104770:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.104771:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.104772:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.104773:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.104775:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.104777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.104784:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.104787:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.104788:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.104788:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.104789:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.104790:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.104946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.104954:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.104956:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.104959:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.104960:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.104961:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.104963:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.104964:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.104965:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.104966:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.104967:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.104971:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.104973:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.104976:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.104977:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.104977:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.104979:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.104979:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.104980:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.104981:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.104985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.104993:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.104996:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.104997:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.104997:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.104999:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.105002:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.105010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.105016:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.105019:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.105020:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.105020:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.105021:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.105021:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.105025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.105030:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.105032:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.105033:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.105035:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.105037:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.105038:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.105039:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.105040:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.105041:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.105042:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.105045:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.105046:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.105049:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.105049:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.105050:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.105052:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.105052:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.105052:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.105053:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.105057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.105065:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.105066:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.105067:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.105068:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.105068:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.105069:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.105072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.105078:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.105080:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.105082:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.105083:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.105085:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.105086:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.105087:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.105089:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.105090:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.105091:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.105094:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.105096:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.105098:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.105099:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.105100:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.105102:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.105102:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.105103:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.105104:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677448.105116:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.105118:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.105119:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.105119:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.105121:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.105123:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.105139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.105146:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.105149:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.105149:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.105151:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.105154:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.105156:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.105157:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.105158:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.105159:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.105160:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.105163:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.105164:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.105166:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.105166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.105167:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.105167:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.105169:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.105169:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.105170:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.105171:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.105173:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.105174:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.105176:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.105177:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:10.0:1619677448.105177:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.105178:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.105179:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.105179:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.105180:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.105180:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.105181:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.105181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000400:00000200:10.0:1619677448.105183:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.105183:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.105184:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.105186:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.105186:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:24.0:1619677448.105188:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000400:00000200:10.0:1619677448.105188:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.105189:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677448.105190:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:10.0:1619677448.105192:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.105192:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677448.105193:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677448.105193:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.105193:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677448.105195:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.105195:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.105195:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.105196:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:24.0:1619677448.105197:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677448.105198:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96719291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.105198:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.105198:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:24.0:1619677448.105200:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.105201:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.105201:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.2:1619677448.105204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:24.0:1619677448.105205:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677448.105206:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:13.0:1619677448.105208:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677448.105209:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.105209:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677448.105210:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.105211:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:13.0:1619677448.105211:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.105211:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.105212:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677448.105212:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:26.0:1619677448.105213:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:24.0:1619677448.105213:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677448.105213:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000400:00000200:13.0:1619677448.105213:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.105214:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:13.0:1619677448.105215:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.105215:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.105216:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.105225:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:13.0:1619677448.105228:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.105228:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.105229:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.105229:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.105230:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.105231:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.105232:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:6.0:1619677448.105232:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:13.0:1619677448.105233:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.105234:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.105234:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.105361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.105367:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.105370:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.105370:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.105373:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.105373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:17.0:1619677448.105375:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.105377:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.105378:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.105379:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.105380:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.105380:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:17.0:1619677448.105381:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.105384:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.105384:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:17.0:1619677448.105386:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.105388:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:8.0:1619677448.105389:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.105390:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.105391:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.105392:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:8.0:1619677448.105392:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.105393:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:17.0:1619677448.105394:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.105394:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.105394:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677448.105394:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96719289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.105395:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.105396:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.105397:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.105398:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:9.2:1619677448.105400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:8.0:1619677448.105401:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.105403:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.105408:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.105409:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.105409:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.105410:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.105410:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:24.0:1619677448.105411:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.105412:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.105412:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:24.0:1619677448.105413:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.105413:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.105414:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677448.105415:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.105416:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.2:1619677448.105417:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.105426:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.105429:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.105429:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.105430:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.105432:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.105435:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.105571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.105577:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.105579:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.105580:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.105582:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.105584:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.105586:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.105587:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.105588:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.105589:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.105590:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.105593:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.105595:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.105597:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.105598:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.105599:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.105601:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.105601:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.105602:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.105602:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.105607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.105615:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.105617:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.105618:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.105618:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.105621:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.105623:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.105691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.105699:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.105701:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.105702:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.105704:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.105707:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.105709:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.105710:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.105711:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.105712:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.105713:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:5.2:1619677448.105713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:8.0:1619677448.105717:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.105718:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.105721:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677448.105722:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.105722:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.105723:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:8.0:1619677448.105723:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.105724:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677448.105725:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:26.0:1619677448.105726:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.105726:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.105726:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.105727:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:26.0:1619677448.105728:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.105730:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.105731:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.105731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677448.105732:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.105733:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.105734:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.105737:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.105738:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677448.105738:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.105740:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.105741:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:2.0:1619677448.105741:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.105742:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:18.0:1619677448.105742:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.105742:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.105743:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.105744:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.105744:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:2.0:1619677448.105744:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.105745:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677448.105747:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677448.105756:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.105759:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.105760:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.105762:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.105763:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.105764:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.105773:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.105775:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.105776:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.105777:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.105777:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.105778:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.105800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.105806:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.105809:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.105810:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.105812:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.105814:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.105815:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.105817:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.105818:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.105819:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.105820:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.105823:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.105825:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.105828:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.105829:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.105829:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.105831:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.105831:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.105832:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.105832:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.105844:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.105847:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.105848:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.105849:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.105852:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.105852:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.106049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677448.106055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677448.106056:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.106058:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.106058:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.106059:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677448.106061:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.106064:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.106064:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.106064:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:10.0:1619677448.106066:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.106067:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.106067:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.106069:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.106069:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719284] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.106070:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719284] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.106071:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.106072:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.106073:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.106073:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.106075:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.106075:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.106080:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.106081:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.106083:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.106084:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.106086:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.106087:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.106087:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.106088:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.106089:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.106089:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.106091:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.106091:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.106091:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.106092:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.106093:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.106093:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:10.0:1619677448.106093:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.106094:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.106095:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.106100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.106103:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.106105:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.106106:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.106107:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677448.106108:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.106109:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.106109:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:31.0:1619677448.106110:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.106110:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677448.106110:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.106111:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.106113:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.106113:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.106114:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.106116:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.106119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677448.106119:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.106121:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.106123:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.106123:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.106124:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.106125:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.106125:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677448.106126:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.106129:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.106132:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.106133:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.106135:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.106136:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.106137:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.106139:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.106140:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.106141:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.106144:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.106146:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.106148:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.106149:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.106150:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.106152:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.106152:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.106153:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.106154:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.106164:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.106166:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.106167:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.106168:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.106170:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.106171:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.106227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.106234:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.106237:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.106237:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:20.2:1619677448.106238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:10.0:1619677448.106239:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.106242:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.106243:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.106244:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.106245:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.106246:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677448.106246:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.106247:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.106249:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:10.0:1619677448.106250:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.106252:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.106252:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.106253:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.106254:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.106254:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.106254:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.106255:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:31.0:1619677448.106256:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:10.0:1619677448.106256:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:31.0:1619677448.106257:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.106257:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.106257:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.106258:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:31.0:1619677448.106259:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.106260:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.106260:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.106263:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.106265:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.106267:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.106267:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.106268:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.106268:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:31.0:1619677448.106269:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.106269:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:31.0:1619677448.106270:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677448.106270:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.106271:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.106271:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:10.0:1619677448.106271:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.106272:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.106273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:26.0:1619677448.106274:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:18.0:1619677448.106279:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:12.0:1619677448.106281:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:18.0:1619677448.106282:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.106282:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.106284:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.106284:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.106284:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.106286:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.106286:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.106287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:18.0:1619677448.106287:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:13.0:1619677448.106288:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.106288:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:18.0:1619677448.106289:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.106290:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.106290:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000400:00000200:18.0:1619677448.106291:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.106292:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:18.0:1619677448.106292:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.106292:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.106293:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:6.0:1619677448.106300:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.106302:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:10.0:1619677448.106302:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.106302:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.106303:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:22.0:1619677448.106305:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.106305:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.106306:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677448.106307:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:22.0:1619677448.106308:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.106309:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:22.0:1619677448.106310:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:18.0:1619677448.106310:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.106310:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.106310:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:5.2:1619677448.106310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:22.0:1619677448.106311:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.106311:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:13.0:1619677448.106311:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677448.106312:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:13.0:1619677448.106312:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:22.0:1619677448.106313:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.106314:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.106314:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.106314:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:22.0:1619677448.106315:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.106315:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.106316:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:1.0:1619677448.106316:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677448.106319:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.106319:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.106320:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677448.106321:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:1.0:1619677448.106323:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.106324:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.106325:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.106325:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:1.0:1619677448.106325:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.106326:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:22.0:1619677448.106327:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.106327:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:1.0:1619677448.106327:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:26.0:1619677448.106328:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:22.0:1619677448.106328:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677448.106328:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96719277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.106329:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.106329:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:1.0:1619677448.106329:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.106330:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.106330:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.106331:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.106332:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.106334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:1.0:1619677448.106334:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.106335:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:1.0:1619677448.106336:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.106337:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.106338:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.106339:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.106340:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.106341:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.106341:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.106342:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677448.106343:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.106345:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:8.2:1619677448.106346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:5.0:1619677448.106346:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.106348:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.106349:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.106350:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677448.106354:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.106356:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.106357:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.106357:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.106359:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.106361:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.106454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.106457:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.106460:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.106462:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.2:1619677448.106462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:20.0:1619677448.106464:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.106466:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:5.0:1619677448.106466:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:2.0:1619677448.106467:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:20.0:1619677448.106468:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719276] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.106469:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.106470:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.106471:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.106471:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:20.0:1619677448.106472:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677448.106475:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.106476:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.106478:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:2.0:1619677448.106478:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.106479:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.106481:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.106482:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.106482:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:2.0:1619677448.106482:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.106483:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:2.0:1619677448.106484:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.106485:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:2.0:1619677448.106485:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.106486:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.106486:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.106487:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:2.0:1619677448.106488:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.106490:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.106492:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.106493:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.106494:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.106495:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.106496:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.2:1619677448.106497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677448.106497:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:30.0:1619677448.106498:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:2.0:1619677448.106498:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.106499:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:30.0:1619677448.106501:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:8.2:1619677448.106501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677448.106502:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.106502:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677448.106504:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:31.0:1619677448.106505:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.2:1619677448.106505:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:30.0:1619677448.106507:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.0:1619677448.106507:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677448.106509:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677448.106511:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:5.0:1619677448.106511:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677448.106512:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677448.106513:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.106513:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677448.106513:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:5.0:1619677448.106513:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.106513:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.106515:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:1.0:1619677448.106515:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.106516:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.106517:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677448.106518:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:5.0:1619677448.106519:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.106520:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.106521:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.106522:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.106525:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.106527:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.106530:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.106531:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.106531:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.106533:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.106534:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.106535:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.106537:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.106537:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.106542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.106545:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677448.106546:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.106548:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.106549:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677448.106549:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:4.2:1619677448.106549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.0:1619677448.106550:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:13.0:1619677448.106550:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.106551:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.106552:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.106554:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677448.106556:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.106559:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.106560:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.106562:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.106565:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.106566:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.106568:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.106569:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.106570:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.106571:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.106574:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.106575:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.106578:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.106579:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.106579:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.106581:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.106581:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.106582:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.106583:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.106587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.106594:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.106597:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.106598:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.106598:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.106600:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.106603:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.106790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.106797:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.106800:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.106801:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.106803:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.106804:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.106806:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.106807:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.106808:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.106809:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.106810:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.106812:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.106814:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.106816:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.106816:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.106817:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.106818:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.106818:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.106819:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.106820:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677448.106831:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.106833:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.106833:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.106834:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.106835:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.106836:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.106892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.106897:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.106900:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.106901:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.106903:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.106905:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.106906:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.106908:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.106909:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.106910:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.106911:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.106913:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.106915:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.106917:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.106918:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.106919:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.106920:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.106921:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.106922:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.106922:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677448.106926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.106934:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:6.0:1619677448.106934:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:31.0:1619677448.106937:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:6.0:1619677448.106937:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.106938:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.106938:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.106939:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.106941:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.106941:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:6.0:1619677448.106941:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:31.0:1619677448.106943:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:6.0:1619677448.106943:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:31.0:1619677448.106944:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.106946:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.106947:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.106947:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.106950:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.106952:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.106955:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.106956:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.106957:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.106958:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.106959:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.106960:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.106962:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.106962:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.106964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.106970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.106976:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:21.2:1619677448.106977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677448.106979:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.106980:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.106981:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.106983:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:5.2:1619677448.106983:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.106985:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.106985:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677448.106985:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.106986:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:30.0:1619677448.106987:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.106988:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.106989:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:1.0:1619677448.106989:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.106990:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.106992:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.106992:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.106992:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.106995:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.106996:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.106996:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.106998:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.106998:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677448.106999:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.107000:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.107002:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.107003:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.107004:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.107004:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.107005:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.107005:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.107010:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.107011:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.107013:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.107014:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.107015:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.107016:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.107017:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.107018:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.107018:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.107018:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.107018:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.107019:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.107020:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:30.0:1619677448.107021:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677448.107021:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.107021:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.107021:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.107022:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:4.2:1619677448.107023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.107024:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.107024:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677448.107024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.107027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.2:1619677448.107029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.2:1619677448.107031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.107031:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.107032:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677448.107033:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:10.0:1619677448.107033:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.107034:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677448.107034:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:17.0:1619677448.107034:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.107034:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.107034:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.107034:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677448.107035:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.107035:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677448.107037:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.107037:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.107037:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.107037:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.107038:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:21.2:1619677448.107038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:10.0:1619677448.107038:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.107039:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:5.0:1619677448.107039:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:22.0:1619677448.107040:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.107040:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.107040:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677448.107040:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677448.107041:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:10.0:1619677448.107041:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.107041:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:22.0:1619677448.107042:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.107042:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:31.0:1619677448.107043:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677448.107043:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:6.0:1619677448.107043:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:31.0:1619677448.107044:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:20.0:1619677448.107044:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.107044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000400:00000200:10.0:1619677448.107045:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.107046:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.107047:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.107047:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677448.107047:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.107047:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.107048:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.107049:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.107050:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677448.107050:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.107051:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.107051:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:6.0:1619677448.107052:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.107054:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96719265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.107054:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.107055:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.107055:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000400:00000200:20.0:1619677448.107056:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.107056:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.107056:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.107056:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:6.0:1619677448.107057:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.107058:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.107059:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:6.0:1619677448.107059:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.107062:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.107063:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.107065:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.107065:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.107065:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.2:1619677448.107066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:20.0:1619677448.107066:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.107066:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.107067:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677448.107068:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.107068:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.107068:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.107069:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.107069:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.107069:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.107070:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677448.107070:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.107070:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.107070:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677448.107071:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677448.107071:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:6.0:1619677448.107071:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.107072:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677448.107072:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.107073:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677448.107074:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:6.0:1619677448.107074:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:8.2:1619677448.107075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677448.107075:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677448.107075:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.107077:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.107079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:18.0:1619677448.107079:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.107080:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:8.2:1619677448.107080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:18.0:1619677448.107081:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.107082:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96719264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.107083:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:17.0:1619677448.107083:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:31.0:1619677448.107084:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.107084:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.107084:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.107085:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:17.0:1619677448.107085:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:31.0:1619677448.107086:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.107086:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.107086:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.107087:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.107087:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:5.0:1619677448.107087:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:1.0:1619677448.107087:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:22.0:1619677448.107088:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.107089:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677448.107090:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:18.0:1619677448.107090:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.107090:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.107090:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.107090:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.107092:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.107093:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.107093:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.107093:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.107095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.107095:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677448.107096:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.107097:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.107098:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.107100:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.107100:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.107101:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.107101:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:1.0:1619677448.107103:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.107106:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.107107:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.107107:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.107108:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.107109:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:12.0:1619677448.107112:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.107115:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.107116:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.107118:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.107119:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.107121:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.107335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.107342:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.107344:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.107346:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.107348:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.107350:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.107352:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.107353:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.107355:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.107356:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.107357:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.107360:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.107361:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.107364:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.107365:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.107366:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.107367:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.107368:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.107369:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.107370:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.107371:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.107374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.107377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.107381:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.107384:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.107384:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.107385:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.107385:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.107386:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.107387:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.107387:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.107388:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677448.107390:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.107506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.107514:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.107517:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.107520:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.107522:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.107522:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.107524:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.107525:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.107526:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.107527:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.107528:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.107532:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.107533:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.107536:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.107537:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.107538:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.107539:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.107540:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.107541:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.107541:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.107545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.107553:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.107556:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.107557:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.107558:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.107560:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.107561:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.107598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.107604:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.107607:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.107608:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.107610:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.107613:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.107614:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.107616:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.107617:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.107618:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.107619:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.107622:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.107624:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.107626:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.107627:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.107628:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.107629:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.107630:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.107630:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.107631:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.107635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.107643:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.107645:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.107646:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.107646:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.107648:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.107650:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.107755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.107763:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.107766:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.107766:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.2:1619677448.107768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:26.0:1619677448.107769:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.107771:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.107772:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:5.0:1619677448.107773:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677448.107774:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.107775:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.107776:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.107776:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:26.0:1619677448.107778:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.107779:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.107779:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.107782:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.107782:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.107784:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.107784:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677448.107785:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.107786:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.107787:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.107787:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.107788:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.107788:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677448.107788:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:26.0:1619677448.107790:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.107791:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.107792:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:5.0:1619677448.107792:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.107793:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.107794:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.0:1619677448.107794:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677448.107796:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677448.107797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.107797:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.2:1619677448.107799:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677448.107802:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677448.107804:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.107805:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677448.107805:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.107805:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677448.107806:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:5.0:1619677448.107806:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677448.107807:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.107808:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.107808:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:2.0:1619677448.107808:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.107809:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.107810:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.107810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:18.0:1619677448.107811:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.107815:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677448.107816:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.107818:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.107818:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.107819:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.107821:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.107823:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.107921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.107927:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.107930:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.107931:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.107933:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.107935:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.107936:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.107938:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.107939:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.107940:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.107941:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.107944:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.107946:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.107948:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.107949:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.107950:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.107952:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.107952:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.107953:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.107953:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.107968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.107974:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.107977:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.107978:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.107978:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.107980:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.107984:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.108016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.2:1619677448.108018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:31.0:1619677448.108023:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.108025:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:31.0:1619677448.108026:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:10.0:1619677448.108027:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:31.0:1619677448.108028:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.108028:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:30.0:1619677448.108030:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677448.108030:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:31.0:1619677448.108031:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.108032:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.108033:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.108034:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.108034:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677448.108036:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.108036:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.108037:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.108038:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.108038:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.108039:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.108040:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677448.108042:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.108044:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.108044:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.108045:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.108046:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.108047:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.108048:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.108048:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.108049:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:31.0:1619677448.108050:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.108050:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.108050:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.108051:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.108052:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.108052:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.108052:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.108053:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.108054:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677448.108057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677448.108064:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.108065:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:13.0:1619677448.108066:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.108067:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.108068:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.108069:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:13.0:1619677448.108069:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.108069:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.108071:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.108072:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677448.108072:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.108075:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.108078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677448.108081:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.108084:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.108084:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.108085:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677448.108086:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677448.108086:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.108087:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677448.108089:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.108089:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.108090:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.108091:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.108091:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.108104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.108112:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.108115:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.108117:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.108118:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.108119:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.108120:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.108122:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.108123:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.108124:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.108125:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.108128:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.108130:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.108132:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.108133:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.108134:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.108136:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.108136:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.108137:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.108138:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.108142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.108150:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.108152:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.108153:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.108155:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.108156:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.108158:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.108161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.108169:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:4.2:1619677448.108169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677448.108172:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.108173:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.108173:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.108175:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677448.108175:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.108176:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.108179:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.108179:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.108181:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.108184:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.108184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:26.0:1619677448.108186:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.108187:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719254] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.108189:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.108190:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.108190:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:26.0:1619677448.108191:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.108192:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.108193:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:26.0:1619677448.108196:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.108196:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.108198:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.108199:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.2:1619677448.108199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:2.0:1619677448.108200:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.108202:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.108203:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.108204:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.108204:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:2.0:1619677448.108204:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.108206:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.108206:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.108206:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:2.0:1619677448.108206:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.108207:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677448.108207:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.108207:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.108208:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:18.0:1619677448.108210:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.108212:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.108212:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.108213:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:2.0:1619677448.108214:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.108215:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96719252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.108216:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.108217:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.108218:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.108220:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677448.108221:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.108222:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.108222:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:6.0:1619677448.108223:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677448.108223:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.108224:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:18.0:1619677448.108224:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:6.0:1619677448.108225:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.108225:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.108226:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.108227:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.108227:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.108228:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.108228:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.108229:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.108229:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.108230:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:6.0:1619677448.108230:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677448.108230:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.108231:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.108231:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.108232:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.108232:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:21.2:1619677448.108235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:12.0:1619677448.108238:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:2.0:1619677448.108240:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:12.0:1619677448.108241:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677448.108242:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.108242:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:12.0:1619677448.108243:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.108243:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.108244:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677448.108245:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.108246:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.108246:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.108247:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.108247:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677448.108248:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.108251:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.108426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.108432:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.108434:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.108435:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.108437:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.108440:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.108442:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.108443:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.108444:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.108445:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.108446:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.108450:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.108452:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.108454:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.108455:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.108456:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.108457:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.108458:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.108459:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.108459:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677448.108460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677448.108464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:10.0:1619677448.108467:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.108470:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.108471:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677448.108471:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:10.0:1619677448.108473:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.108474:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.108475:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.108475:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:10.0:1619677448.108476:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.108477:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.108478:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.108479:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.108480:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:10.0:1619677448.108480:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.108481:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.108482:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:9.2:1619677448.108482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000400:00000200:10.0:1619677448.108486:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.108487:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.108488:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:10.0:1619677448.108490:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.108491:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:10.0:1619677448.108491:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.108491:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:10.0:1619677448.108492:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.108492:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:10.0:1619677448.108493:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.108493:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.108493:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:10.0:1619677448.108494:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.108495:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.108495:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677448.108496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:8.2:1619677448.108499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.108500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677448.108502:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.108505:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.108505:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.108507:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.108508:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.108508:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:22.0:1619677448.108509:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.108511:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:22.0:1619677448.108511:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:5.0:1619677448.108511:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.108512:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.108513:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.108514:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.108514:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.108515:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.108516:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.108516:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677448.108516:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.108517:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.108518:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:31.0:1619677448.108519:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.108521:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:22.0:1619677448.108521:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.108522:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.108524:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.108524:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.108525:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.108526:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.108527:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.108528:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.108528:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677448.108530:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.108530:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.108532:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.108532:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.108532:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.108533:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677448.108535:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.108536:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:8.2:1619677448.108536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.108537:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.108539:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677448.108539:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.108540:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.108540:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677448.108544:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.108547:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.108548:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.108548:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.108550:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.108552:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.108553:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677448.108555:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.108556:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.108556:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.108559:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.108562:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.108570:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.108573:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.108574:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.108574:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.108575:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.108576:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.108656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.108659:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.108661:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.108663:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.108663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:20.0:1619677448.108665:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.108667:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677448.108668:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719247] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.108668:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.108670:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.108670:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:20.0:1619677448.108671:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.108672:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.108673:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.108673:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:20.0:1619677448.108676:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.108677:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:18.0:1619677448.108679:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.108680:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.108681:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:18.0:1619677448.108681:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.108682:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:18.0:1619677448.108683:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:20.0:1619677448.108684:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677448.108684:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:18.0:1619677448.108684:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.108685:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:18.0:1619677448.108685:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.108686:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677448.108687:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.108688:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:9.2:1619677448.108690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:18.0:1619677448.108691:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.108693:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.108696:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.108697:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.108698:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.108698:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.108700:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.108700:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.108700:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.108701:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.108702:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.108702:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.108703:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677448.108704:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.108705:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:8.2:1619677448.108706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.108714:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.108716:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.108717:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.108719:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.108719:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.108722:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.108722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.108729:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.108732:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.108733:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.108734:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.108737:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.108739:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.108740:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719245] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.108741:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.108742:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.108743:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.108746:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.108747:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.108750:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.108751:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.108752:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.108754:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.108754:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.108755:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.108755:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.108759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.108767:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.108770:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.108770:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.108771:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.108773:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.108776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.108776:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677448.108781:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.108784:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:8.2:1619677448.108784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:4.2:1619677448.108785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677448.108786:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.108787:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.108787:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.108789:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.108790:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.108790:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.108791:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.108791:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:26.0:1619677448.108792:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677448.108792:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:1.0:1619677448.108792:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:17.0:1619677448.108794:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.108795:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:17.0:1619677448.108795:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.108796:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677448.108797:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.108798:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.108799:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.108801:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.108802:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.108803:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.108804:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.108805:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.108807:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.108807:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.108808:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.108808:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.108808:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:26.0:1619677448.108809:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.108810:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.108811:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.108811:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.108812:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677448.108813:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.108815:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.2:1619677448.108815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677448.108818:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.108818:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.108819:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.108821:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.108821:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.108822:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.108822:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.108824:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.108824:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:12.0:1619677448.108824:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677448.108825:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:12.0:1619677448.108826:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:30.0:1619677448.108827:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677448.108827:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.108829:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.108829:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.108829:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.108830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677448.108831:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:13.0:1619677448.108831:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:30.0:1619677448.108832:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.108832:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:30.0:1619677448.108833:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.108834:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.108834:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:30.0:1619677448.108835:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.108837:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.108837:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:30.0:1619677448.108838:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.108838:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.108838:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.108839:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.108840:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.108841:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.108841:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.108841:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.108844:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.108844:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.108844:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677448.108845:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (22)++
00000800:00000200:30.0:1619677448.108847:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.108848:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.108849:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.108849:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.108860:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.108863:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.108864:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.108864:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.108866:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.108869:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677448.108878:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.108881:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.108882:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.108882:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.108883:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.108883:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.108907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37e800] (20)++
00000800:00000200:26.0:1619677448.108914:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37e800] (21)++
00000800:00000200:26.0:1619677448.108917:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.56@o2ib3
00000800:00000200:10.0:1619677448.108917:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)--
00000400:00000200:26.0:1619677448.108919:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.56@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.108922:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.108924:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:26.0:1619677448.108925:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.108927:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.108927:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.108928:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677448.108932:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.108934:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677448.108937:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.108937:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:26.0:1619677448.108938:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:26.0:1619677448.108939:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:26.0:1619677448.108940:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37e800] (21)++
00000800:00000200:26.0:1619677448.108941:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37e800] (22)++
00000800:00000200:26.0:1619677448.108942:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37e800] (23)--
00000800:00000200:26.0:1619677448.108943:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)--
00000800:00000200:8.2:1619677448.108946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:21.2:1619677448.108948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37e800] (21)++
00000800:00000200:20.0:1619677448.108953:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.108955:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:10.0:1619677448.108955:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)++
00000400:00000200:20.0:1619677448.108956:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.108958:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677448.108958:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37e800] (23)--
00000800:00000200:10.0:1619677448.108958:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)--
00000800:00000200:26.0:1619677448.108959:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (21)--
00000800:00000200:5.0:1619677448.108959:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:20.0:1619677448.108963:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.108965:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:31.0:1619677448.108966:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.108969:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.108971:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.108971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000800:00000200:30.0:1619677448.108973:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.108974:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.108976:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.108977:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.108978:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.108979:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.108979:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000400:00000200:31.0:1619677448.108980:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.108981:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.108982:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.108983:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000400:00000200:31.0:1619677448.108984:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.108984:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.108985:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:31.0:1619677448.108986:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.108989:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.108989:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.108990:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.108992:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.108992:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.108994:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.108994:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.108998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.109004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:5.0:1619677448.109008:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677448.109010:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.0:1619677448.109010:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.109011:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.109013:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.109013:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:5.0:1619677448.109013:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.109014:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.109016:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.109016:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:18.0:1619677448.109018:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.109020:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677448.109021:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.109022:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.109024:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.109024:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677448.109028:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.109030:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.109032:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.109032:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677448.109033:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677448.109035:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.109035:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.109036:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.109036:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677448.109048:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.109050:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.109051:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.109052:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.109054:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.109056:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.109113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.109119:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.109122:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.109122:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.109125:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.109127:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.109129:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.109130:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.109131:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.109133:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.109134:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.109137:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.109138:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.109141:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.109142:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.109142:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.109144:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.109144:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.109146:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.109147:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.109147:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.109152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.109157:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.109158:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.109160:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.109160:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.109160:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:4.2:1619677448.109160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677448.109161:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.109161:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.109162:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.109163:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.109166:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:10.0:1619677448.109167:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.109170:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.109170:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.109173:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.109175:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.109177:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.109178:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.109179:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.109181:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.109182:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.109185:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.109186:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.109189:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.109189:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.109190:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.109192:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.109192:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.109193:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.109193:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.2:1619677448.109196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.109199:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.109201:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.109203:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.109204:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.109204:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.109206:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.109207:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:20.0:1619677448.109207:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677448.109208:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.109208:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.109210:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.109210:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.109211:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:20.0:1619677448.109211:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.109212:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.109213:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:20.0:1619677448.109216:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.109217:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.109220:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.109221:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.109221:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.109222:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:31.0:1619677448.109224:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.109225:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:20.0:1619677448.109226:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.109227:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:30.0:1619677448.109227:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.109228:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677448.109228:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677448.109229:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.109229:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.109230:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.109235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.109237:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.109240:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:5.0:1619677448.109240:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:30.0:1619677448.109241:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.109242:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.109242:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:31.0:1619677448.109243:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.109243:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.109244:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.109245:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.109307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.109315:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.109318:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.109320:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.109322:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.109322:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.109324:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677448.109325:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.109327:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.109328:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.109329:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677448.109332:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.109334:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677448.109337:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.109338:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677448.109338:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.109340:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677448.109341:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.109342:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.109343:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.109344:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677448.109350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677448.109354:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.109357:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:30.0:1619677448.109358:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:22.0:1619677448.109358:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.109358:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677448.109361:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.109361:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677448.109361:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.109363:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.109364:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.109365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:26.0:1619677448.109373:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677448.109376:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:10.0:1619677448.109377:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:26.0:1619677448.109379:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.109381:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.109383:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.109384:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.109385:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.109386:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.109387:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.109391:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.109392:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.109394:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.109395:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.109396:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.109397:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.109397:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677448.109398:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:26.0:1619677448.109399:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:30.0:1619677448.109409:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.109412:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.109412:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.109414:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.109416:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.109416:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.109455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.109462:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.109465:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.109466:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.109467:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.109469:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.109471:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.109472:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.109473:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.109474:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.109475:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.109478:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.109480:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.109483:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.109483:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.109484:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.109485:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.109486:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.109486:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.109487:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.109499:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.109501:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.109502:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.109503:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.109504:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.109507:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.109516:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.109519:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.109519:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.109520:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.109521:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.109521:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.109534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.109540:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.109543:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.109543:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.109546:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.109548:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.109549:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.109551:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.109552:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.109553:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.109554:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.109557:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.109559:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.109562:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.109563:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.109563:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.109565:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.109565:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.109566:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.109566:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.109578:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.109581:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.109582:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.109584:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.109585:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.109587:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.109656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.109664:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:20.2:1619677448.109665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:24.0:1619677448.109667:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.109667:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677448.109669:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.109671:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:8.0:1619677448.109673:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.109673:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:8.0:1619677448.109675:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:20.0:1619677448.109676:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:8.0:1619677448.109676:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.109676:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677448.109678:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.109679:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.109679:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.109680:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.109680:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:5.0:1619677448.109682:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.109683:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.109685:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.109685:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.109686:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677448.109688:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.109690:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.109692:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677448.109694:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.109695:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.109695:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.109696:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.109696:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:20.2:1619677448.109697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677448.109697:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.109698:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.109698:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.109699:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677448.109699:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.109700:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677448.109700:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.109701:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.109701:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677448.109704:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:9.2:1619677448.109704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.109707:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:9.2:1619677448.109708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:31.0:1619677448.109710:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.109711:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.109712:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.109712:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:31.0:1619677448.109714:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:24.0:1619677448.109714:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:2.0:1619677448.109714:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:31.0:1619677448.109715:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96719229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.109716:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.109716:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.109716:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.109717:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:31.0:1619677448.109718:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.109718:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.109718:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:2.0:1619677448.109719:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.109720:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677448.109721:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000400:00000200:31.0:1619677448.109723:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.109723:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677448.109725:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677448.109725:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:31.0:1619677448.109728:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.109728:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.109729:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.109731:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.109731:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.109733:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.109734:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.109735:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677448.109740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.109745:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.109746:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.109747:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.109749:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:30.0:1619677448.109749:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.109750:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.109750:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.109751:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.109752:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.109753:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:4.2:1619677448.109870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.109877:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.109879:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.109880:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.109882:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.109885:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.109887:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.109888:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.109889:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.109890:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.109891:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.109894:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.109896:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.109899:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.109900:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.109900:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.109902:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.109902:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.109903:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.109903:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.109915:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.109918:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.109918:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.109919:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.109921:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.109924:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.110020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.110028:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.110031:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.110033:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.110035:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.110035:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.110037:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677448.110039:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.110040:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.110041:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.110042:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677448.110045:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.110046:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677448.110049:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.110050:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.110050:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.110052:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.110052:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.110053:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.110054:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.110058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.110064:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.110066:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.110067:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.110067:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.110069:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.110072:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.110079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677448.110085:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.110087:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.110088:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.110088:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.110089:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.110089:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.2:1619677448.110101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.110109:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.110111:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.110112:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.110114:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.110116:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.110118:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.110119:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.110120:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.110121:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.110122:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.110125:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.110126:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.110128:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.110129:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.110130:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.110132:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.110132:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.110133:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.110133:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.110144:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.110146:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.110147:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.110148:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.110150:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.110151:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.110152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677448.110159:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677448.110161:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677448.110162:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677448.110164:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677448.110166:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.110168:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677448.110169:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.110170:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.110171:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.110172:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677448.110175:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677448.110176:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677448.110177:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.110178:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677448.110178:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.110180:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.110180:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677448.110181:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677448.110181:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677448.110185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.110191:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.110192:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.110193:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.110194:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.110195:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677448.110197:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677448.110308:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.110310:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.110311:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.110313:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.112@o2ib7 : GET - for me
00000400:00000200:22.0:1619677448.110316:0:19936:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.112@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:22.0:1619677448.110318:0:19936:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.112@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:22.0:1619677448.110319:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.110320:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.110322:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:22.0:1619677448.110326:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : REPLY try# 0
00000800:00000200:22.0:1619677448.110327:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.110329:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.110330:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.110331:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.110332:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.110333:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.110345:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.110347:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.110348:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.110349:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.110350:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: REPLY: OK
00000800:00000200:6.0:1619677448.110352:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.110404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.110412:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.110415:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.110416:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.110417:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.110420:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.110422:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.110423:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.110424:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.110425:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.110426:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677448.110429:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.110431:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.110433:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.110434:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.110435:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.110436:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.110437:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.110438:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:5.2:1619677448.110438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.0:1619677448.110439:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.110440:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.2:1619677448.110442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.110445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:17.0:1619677448.110445:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.110447:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.110448:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.110450:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.110451:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:24.0:1619677448.110452:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:20.0:1619677448.110453:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:17.0:1619677448.110453:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.110454:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.110454:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:24.0:1619677448.110455:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:17.0:1619677448.110455:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.110456:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:20.0:1619677448.110456:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.110456:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:17.0:1619677448.110457:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.110457:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:17.0:1619677448.110458:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.110459:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:17.0:1619677448.110459:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.110462:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.110464:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.110466:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.110467:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.110468:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.110469:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.110470:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.110471:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.110471:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.110476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.110484:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.110486:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.110486:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.110487:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.110488:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.110490:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.110660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.110667:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.110670:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.110671:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.110673:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.110676:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.110677:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:4.2:1619677448.110677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677448.110678:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.110679:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.110680:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.110681:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.110684:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:24.0:1619677448.110684:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.110687:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:24.0:1619677448.110687:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.110688:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.110690:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.110690:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.110691:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.110691:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677448.110692:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.110693:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:26.0:1619677448.110694:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:24.0:1619677448.110694:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.110694:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:26.0:1619677448.110695:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.110695:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:26.0:1619677448.110696:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.110697:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.110698:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.2:1619677448.110699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677448.110701:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.110702:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.110705:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.110706:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.110707:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.110707:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:26.0:1619677448.110708:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.110709:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.110709:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:26.0:1619677448.110710:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:17.0:1619677448.110710:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.110710:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:26.0:1619677448.110711:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.110712:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677448.110713:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.110715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.110716:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.110717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.110723:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.110724:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:18.0:1619677448.110726:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:10.0:1619677448.110726:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.110727:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677448.110727:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.110728:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.110729:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:18.0:1619677448.110730:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.110733:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.110764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.110772:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.110775:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.110776:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.110777:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.110780:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.110782:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.110784:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.110785:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.110786:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.110787:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.110790:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.110792:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.110795:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.110795:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.110796:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.110797:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.110798:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.110799:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.110799:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677448.110808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.110813:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.110814:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.110815:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.110816:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.110816:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:22.0:1619677448.110818:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.110819:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.110822:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.110822:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.110823:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.110824:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.110824:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.110825:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.110827:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.110828:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.110829:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.110833:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.110835:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.110837:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.110838:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.110839:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.110840:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.110841:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.110842:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.110842:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.110847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.110855:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.110857:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.110858:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.110859:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.110861:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.110863:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.110881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.110887:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.110890:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.110892:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.110893:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.110895:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.110896:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.110898:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.110899:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.110900:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.110901:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.110905:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.110906:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.110910:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677448.110911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:5.0:1619677448.110911:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.110911:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.110913:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.110914:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.110915:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.110916:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.0:1619677448.110919:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:9.2:1619677448.110920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.110921:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.110922:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.110924:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.110927:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:8.0:1619677448.110928:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.110929:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:2.0:1619677448.110930:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.110931:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:8.0:1619677448.110931:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96719217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.110931:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.110933:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.110933:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.110934:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.110935:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.110935:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:8.0:1619677448.110939:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.110940:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.110943:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.110944:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.110945:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.110946:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.110947:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.110947:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.110948:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.110953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.110960:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.110962:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.110963:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.110965:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.110966:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.110968:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.110971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.110980:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.110981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.0:1619677448.110983:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.110985:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.110986:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.110986:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.110987:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:30.0:1619677448.110990:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.110993:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.110995:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.110997:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.110998:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.110999:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.111000:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.111002:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.111003:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.111004:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.111007:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.111009:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.111011:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.111012:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.111012:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.111014:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.111014:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.111015:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.111016:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.111027:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.111029:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.111030:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.111032:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.111034:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.111035:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677448.111045:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.111047:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.111048:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.111049:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.111049:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.111050:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.111145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.111153:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.111156:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.111158:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.111160:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.111160:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.111161:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.111163:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.111164:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.111165:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.111165:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.111169:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.111171:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.111173:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.111173:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.111174:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.111176:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.111176:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.111177:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.111178:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677448.111188:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.111190:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.111191:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.111193:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.111194:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:4.2:1619677448.111194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677448.111196:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.111201:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.111203:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.111204:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.111206:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.111209:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.111211:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.111212:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.111213:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.111214:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.111215:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.111217:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.111219:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.2:1619677448.111220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.111222:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.111222:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.111223:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.111225:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.111225:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.111225:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.111226:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.111226:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677448.111228:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.111229:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.111231:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.111232:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.111234:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.111235:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.111236:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.111237:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.111238:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:17.0:1619677448.111238:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.111241:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:17.0:1619677448.111241:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.111241:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.111242:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.111244:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.111244:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.111246:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.111247:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.111247:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.111248:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:17.0:1619677448.111249:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.111249:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.111250:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.111250:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.111255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.111262:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.111264:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.111265:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.111265:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.111267:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.111268:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.111296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.111304:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.111307:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.111308:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.111310:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.111313:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.111314:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.111316:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.111317:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.111318:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.111319:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.111322:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.111324:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.111327:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.111327:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.111328:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.111329:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.111330:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.111331:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.111331:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.111343:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.111345:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.111346:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.111348:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.111350:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.111352:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.111494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.111500:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.111503:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.111504:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.111506:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.111508:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.111509:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.111511:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.111512:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.111513:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.111514:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.111517:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.111519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:6.0:1619677448.111519:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.111522:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.111522:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.111523:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.111525:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:5.0:1619677448.111525:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.111526:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.111526:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.111527:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677448.111528:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.111530:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.111530:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.111532:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.111534:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.2:1619677448.111535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000400:00000200:5.0:1619677448.111535:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.111537:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:20.0:1619677448.111537:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000400:00000200:5.0:1619677448.111537:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.111538:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.111539:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:5.0:1619677448.111539:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.111540:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.111541:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.111541:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:20.0:1619677448.111542:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.111544:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.111544:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.111545:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.111546:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677448.111547:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96719209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.111548:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.111548:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.111549:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:20.0:1619677448.111549:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.111550:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677448.111554:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.111555:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.111556:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:20.0:1619677448.111558:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.111558:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.111559:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:20.0:1619677448.111559:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.111559:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:5.0:1619677448.111559:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.111560:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.111560:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:5.0:1619677448.111560:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.111561:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:20.0:1619677448.111562:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.111562:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.0:1619677448.111562:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.111563:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (22)++
00000800:00000200:5.0:1619677448.111563:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.111564:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (23)--
00000800:00000200:5.0:1619677448.111564:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.111565:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677448.111566:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:5.0:1619677448.111567:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677448.111568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:13.0:1619677448.111575:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.111576:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.111578:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:13.0:1619677448.111578:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:20.0:1619677448.111579:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.111579:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.111581:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677448.111582:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.111582:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:13.0:1619677448.111583:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.111584:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.111585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:13.0:1619677448.111586:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677448.111592:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.111595:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.111595:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.111597:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.111600:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.111602:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.111603:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.111604:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.111605:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.111606:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.111610:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.111612:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.111615:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.111616:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.111616:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.111618:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.111619:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.111620:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.111620:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:26.0:1619677448.111630:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.111633:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.111633:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.111634:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.111636:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.111639:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.111694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.111701:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.111704:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.111705:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.111707:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.111708:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.111710:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.111711:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.111712:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.111713:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.111714:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.111718:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.111720:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.111722:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.111722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.111723:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.111724:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.111725:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.111726:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.111727:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:17.0:1619677448.111728:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.111728:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.111729:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.111731:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.111732:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.111734:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.111734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677448.111735:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.111737:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:12.0:1619677448.111737:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:17.0:1619677448.111738:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.111740:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.111740:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.111740:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.111741:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.111742:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:17.0:1619677448.111742:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:12.0:1619677448.111743:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.111744:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.111744:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:13.0:1619677448.111744:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.111745:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:17.0:1619677448.111746:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.111748:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.111749:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677448.111751:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.111752:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.111753:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.111755:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.111755:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.111757:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.111758:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.111758:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.111763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.111767:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677448.111769:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677448.111770:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.111771:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.111771:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.111771:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677448.111772:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.111772:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.111773:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.111776:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.111785:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:4.2:1619677448.111786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677448.111788:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.111789:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.111789:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.111790:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.111791:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.111793:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.111795:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.111796:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.111798:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.111800:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.111802:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.111803:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.111804:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.111805:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.111806:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.111809:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.111811:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.111813:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.111814:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.111815:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.111816:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.111816:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.111817:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.111818:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.111827:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.111829:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.111830:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.111830:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.111831:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.111833:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.111853:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.111856:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.111856:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.111857:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.111857:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.111857:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.111907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.111915:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.111918:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.111921:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.111922:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.111923:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.111924:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.111925:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.111926:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.111927:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.111928:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.111931:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.111933:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.111935:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.111936:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.111936:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.111937:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.111938:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.111939:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.111940:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.2:1619677448.111943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:5.2:1619677448.111947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:13.0:1619677448.111950:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:26.0:1619677448.111951:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:13.0:1619677448.111952:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:2.0:1619677448.111952:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:26.0:1619677448.111953:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:13.0:1619677448.111953:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.111954:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:13.0:1619677448.111955:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.111955:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.111956:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:13.0:1619677448.111957:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:26.0:1619677448.111958:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.111958:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:2.0:1619677448.111959:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.111960:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.111961:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.111962:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.111962:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.111963:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677448.111964:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.111965:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.111966:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.111967:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:2.0:1619677448.111967:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.111968:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.111969:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677448.111972:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.111973:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:2.0:1619677448.111976:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.111977:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.111978:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.111978:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677448.111978:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.111980:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.111980:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.111981:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:2.0:1619677448.111981:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.111981:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:26.0:1619677448.111982:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:2.0:1619677448.111982:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.111984:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.111984:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:8.2:1619677448.111986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677448.111986:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.111987:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.111987:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.111989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.111992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677448.111996:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.111997:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.111997:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677448.111998:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:17.0:1619677448.111999:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.111999:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.111999:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.112000:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.112001:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677448.112001:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.112001:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.112002:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677448.112005:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.112005:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.112006:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677448.112006:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.112009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.112017:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.112019:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.112020:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.112021:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.112022:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.112022:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.112124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.112130:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.2:1619677448.112132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.112133:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.112135:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.112136:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.112137:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.112138:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.112138:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.112140:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.112141:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.112143:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.112144:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.112145:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.112148:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.112149:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.112150:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.112152:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677448.112152:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.112153:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.112154:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.112155:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.112156:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.112159:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.112160:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.112160:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.112161:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.112162:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.112162:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.112163:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.112164:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.112164:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677448.112165:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.112166:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.112166:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.112167:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.2:1619677448.112168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.112169:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.112169:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.112170:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.112171:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677448.112175:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.112178:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.112179:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.112179:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.112181:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677448.112181:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.112184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:13.0:1619677448.112184:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677448.112184:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:13.0:1619677448.112185:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.112186:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.112188:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.112189:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.112189:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.112192:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.112193:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.112195:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.112198:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.112199:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.112200:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.112201:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.112202:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:2.0:1619677448.112203:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.112204:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:12.0:1619677448.112206:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.112207:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.112207:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.112208:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677448.112208:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:13.0:1619677448.112209:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.112210:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.112212:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.112212:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.112213:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.112215:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.112215:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.112216:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.112217:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.112218:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:9.2:1619677448.112221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.112224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.2:1619677448.112224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:18.0:1619677448.112229:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677448.112231:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:18.0:1619677448.112231:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.112231:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:18.0:1619677448.112232:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.112232:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.112232:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.112233:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.112234:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:4.2:1619677448.112234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677448.112234:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677448.112235:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:18.0:1619677448.112236:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.0:1619677448.112236:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677448.112238:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.112240:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.112242:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:10.0:1619677448.112242:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:24.0:1619677448.112243:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.112243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:24.0:1619677448.112244:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.112244:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.112245:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:24.0:1619677448.112246:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.112247:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.112248:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.112250:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:24.0:1619677448.112251:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.112251:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.112252:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:24.0:1619677448.112253:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.112253:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.112254:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:10.0:1619677448.112254:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.112255:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677448.112256:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.112256:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.112256:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.112256:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677448.112257:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:10.0:1619677448.112257:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.112258:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:10.0:1619677448.112258:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:24.0:1619677448.112259:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.112260:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677448.112261:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677448.112261:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000400:00000200:10.0:1619677448.112261:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.112262:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.112264:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.112264:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.2:1619677448.112264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.112265:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.112266:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.112267:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.112268:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.112268:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.112270:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.112272:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:12.0:1619677448.112272:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:10.0:1619677448.112272:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:12.0:1619677448.112274:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:10.0:1619677448.112274:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677448.112275:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.112276:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.112277:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.112277:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.112278:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96719196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.112279:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.112279:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:10.0:1619677448.112280:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.112280:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.112281:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.112284:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.112286:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.112288:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.112288:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.112289:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.112289:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (23)++
00000800:00000200:10.0:1619677448.112289:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (24)++
00000400:00000200:26.0:1619677448.112290:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.112290:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (24)--
00000400:00000200:26.0:1619677448.112291:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.112291:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.112292:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.112293:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.112293:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677448.112300:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.2:1619677448.112303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.112309:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.112312:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:10.0:1619677448.112313:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:1.0:1619677448.112314:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.112315:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.112315:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.112316:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.112318:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.112318:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.112319:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:17.0:1619677448.112319:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:10.0:1619677448.112319:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:17.0:1619677448.112321:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.112322:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.112323:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.112324:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.112327:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.112329:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.112331:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.112331:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.112332:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.112334:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.112334:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.112335:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.112336:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.112341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.112348:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.112350:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.112351:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.112352:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.112353:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.112354:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.112538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677448.112545:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677448.112548:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677448.112549:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677448.112551:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.112554:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.112556:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:22.0:1619677448.112557:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.112558:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.112560:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.112560:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677448.112564:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.112565:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.112568:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.112569:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:22.0:1619677448.112570:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.112572:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677448.112572:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677448.112574:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677448.112575:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677448.112575:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:21.2:1619677448.112581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (21)++
00000800:00000200:13.0:1619677448.112586:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677448.112586:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:6.0:1619677448.112588:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:6.0:1619677448.112588:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:13.0:1619677448.112589:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:6.0:1619677448.112589:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000400:00000200:13.0:1619677448.112590:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.112592:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.112593:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.112595:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.112722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.112729:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.112733:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.112734:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.112736:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.112738:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.112739:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.112740:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.112742:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.112743:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.112744:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.2:1619677448.112746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:8.0:1619677448.112747:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.112749:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.112752:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.112752:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.112753:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.112755:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:8.0:1619677448.112755:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.112755:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.112756:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.112756:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:31.0:1619677448.112757:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.112759:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.112761:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.112762:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.112763:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.112764:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.112765:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.112766:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.112767:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:12.0:1619677448.112767:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.112769:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.112769:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.112771:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677448.112771:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.112773:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.112773:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.112774:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.112776:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.112777:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.112778:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.112780:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.112780:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.112781:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.112782:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.112786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.112795:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.112797:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.112799:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.112800:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.112800:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.112804:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.112826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.112832:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.112836:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.112836:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.112838:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.112840:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.112841:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.112843:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.112844:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.112845:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.112846:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.112849:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.112849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:18.0:1619677448.112851:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.112853:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.112854:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.112855:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.112856:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.112856:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:24.0:1619677448.112857:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677448.112857:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.112857:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:24.0:1619677448.112859:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.112860:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.112862:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.112862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:24.0:1619677448.112864:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.112866:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.112867:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96719190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.112869:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.112869:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.112869:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:24.0:1619677448.112870:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.112871:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.112872:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.112873:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.112874:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.112875:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.112876:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677448.112877:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.112880:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.112881:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.112882:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.112884:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.112884:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.112885:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.112886:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677448.112893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.112900:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677448.112901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.112903:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.112905:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.112906:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.112908:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.112908:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:26.0:1619677448.112909:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.112910:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.112911:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.112911:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:26.0:1619677448.112912:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.112912:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.112912:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:26.0:1619677448.112913:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.112914:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.112916:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.112917:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677448.112918:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.112920:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.112921:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.112921:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.112923:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.112924:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.112925:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.112926:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:21.2:1619677448.112926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677448.112927:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.112932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.112934:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.112935:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677448.112937:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:30.0:1619677448.112937:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.112938:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.112939:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.112939:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677448.112940:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.112940:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.112941:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:10.0:1619677448.112941:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:30.0:1619677448.112942:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.112942:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:30.0:1619677448.112943:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.112943:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:30.0:1619677448.112944:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.112945:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.112948:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.112950:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.112950:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.112951:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.112952:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.112952:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.112952:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.112953:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.112954:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.112955:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.112956:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.112956:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.112976:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.112978:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.112979:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.112980:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.112981:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.112983:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.113152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.113158:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.113161:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.113161:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.113163:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.113165:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.113167:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.113168:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.113169:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.113170:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.113171:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.113175:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.113176:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.113180:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.113180:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.113181:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.113183:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.113184:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.113184:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.113185:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.113189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.113195:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.113198:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.113199:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.113199:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.113201:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.113205:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677448.113335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.113343:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.113346:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.113346:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.113348:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.113350:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.113351:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.113352:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.113354:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.113355:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.113356:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.113359:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.113361:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.113363:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.113364:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.113365:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.113367:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.113367:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.113368:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.113368:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.113382:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.113385:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.113385:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.113386:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.113388:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.113392:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.113413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.113419:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.113422:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.113423:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.113425:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.113427:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.113429:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.113430:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.113431:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.113432:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.113433:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.113436:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.113438:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.113441:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.113442:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.113443:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.113444:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.113445:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.113445:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.113446:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677448.113458:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.113461:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.113462:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.113464:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.113465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:12.0:1619677448.113466:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.113467:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.113470:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.113473:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.113473:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.113476:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.113478:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.113480:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.113481:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.113482:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.113484:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.113485:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.113488:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.113490:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.113493:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.113494:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.113494:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.113496:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.113497:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.113498:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.113499:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.113499:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:21.2:1619677448.113504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.113509:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:31.0:1619677448.113511:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677448.113511:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.113512:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:31.0:1619677448.113513:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:22.0:1619677448.113513:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:31.0:1619677448.113514:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.113514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:31.0:1619677448.113516:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.113518:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.113519:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.113520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677448.113522:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.113524:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.113525:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:30.0:1619677448.113527:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677448.113527:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:30.0:1619677448.113530:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677448.113530:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.113532:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.113533:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.113533:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.113534:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677448.113534:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.113535:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.113535:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.113536:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.113536:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.113537:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.113539:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.113540:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.113540:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000400:00000200:30.0:1619677448.113541:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.113542:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.113544:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.113546:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.113547:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.113548:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.113548:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.113549:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.113549:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.113549:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.113551:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.113551:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.113551:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.113551:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.113552:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:30.0:1619677448.113553:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:26.0:1619677448.113554:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.113554:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:30.0:1619677448.113555:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.113555:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.2:1619677448.113560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:21.2:1619677448.113561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.113565:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.113566:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.113567:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.113568:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:26.0:1619677448.113569:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:10.0:1619677448.113569:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.113570:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.113570:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:31.0:1619677448.113571:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.113571:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677448.113571:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.113571:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.113572:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.113573:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.113573:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677448.113574:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.113574:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.113576:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677448.113578:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.113578:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.113582:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.113585:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.113586:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.113586:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.113587:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.113588:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.113604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.113607:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.113610:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.113612:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.113613:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.113615:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.113616:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.113618:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.113619:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.113620:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.113621:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.113625:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.113627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:20.0:1619677448.113627:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.113630:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.113631:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.113632:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.113633:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:2.0:1619677448.113633:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.0:1619677448.113634:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.113635:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:2.0:1619677448.113635:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.113636:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.0:1619677448.113637:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.113637:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:2.0:1619677448.113638:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.113641:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.113642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:2.0:1619677448.113642:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677448.113643:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.113645:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.113646:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:2.0:1619677448.113646:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.113647:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:5.0:1619677448.113648:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677448.113649:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.113650:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.113650:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:2.0:1619677448.113650:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.113651:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.113651:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:31.0:1619677448.113652:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.113652:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:30.0:1619677448.113653:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.113655:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:2.0:1619677448.113655:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.113656:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677448.113657:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677448.113659:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677448.113659:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.113660:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.113660:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:30.0:1619677448.113663:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.113666:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.113667:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.113667:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.113668:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.2:1619677448.113668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.113669:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677448.113670:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.113673:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.113674:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:17.0:1619677448.113674:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.113674:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.113675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:10.0:1619677448.113676:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:17.0:1619677448.113677:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.113678:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.113680:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.113681:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677448.113682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677448.113682:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677448.113683:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.113685:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677448.113685:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677448.113685:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:17.0:1619677448.113687:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.113688:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.113689:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.113689:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.113690:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.113690:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.113691:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.113692:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:10.0:1619677448.113693:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.113695:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.113696:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.113697:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.113697:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.113697:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.113698:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.113699:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.113699:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.2:1619677448.113699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000400:00000200:10.0:1619677448.113700:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.113701:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.113702:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96719177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.113703:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.113704:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.113704:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:31.0:1619677448.113705:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.113705:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.113705:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.113706:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.113706:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.113706:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:17.0:1619677448.113707:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.113707:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.113708:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.113708:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.113709:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.14.11@o2ib3
00000400:00000200:31.0:1619677448.113710:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.113711:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.113712:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:13.0:1619677448.113712:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : REPLY - routed
00000800:00000200:10.0:1619677448.113712:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:12.0:1619677448.113713:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:10.0:1619677448.113713:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:12.0:1619677448.113714:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.113714:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.113715:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.113715:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.113715:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.113716:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:13.0:1619677448.113716:0:19942:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000800:00000200:12.0:1619677448.113716:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:10.0:1619677448.113716:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.113717:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:12.0:1619677448.113717:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.113718:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:12.0:1619677448.113718:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.113719:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.113720:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.113721:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:13.0:1619677448.113721:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677448.113721:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.113722:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:12.0:1619677448.113722:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677448.113723:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677448.113724:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677448.113724:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:8.2:1619677448.113725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.113728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:6.0:1619677448.113728:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.2:1619677448.113730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:22.0:1619677448.113731:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.113731:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.113732:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.113734:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.113736:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:6.0:1619677448.113737:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677448.113738:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:12.0:1619677448.113738:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:5.0:1619677448.113738:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677448.113740:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:12.0:1619677448.113740:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (23)--
00000800:00000200:30.0:1619677448.113741:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677448.113741:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.113741:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:5.0:1619677448.113741:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.113742:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.113743:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.113743:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.113744:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:13.0:1619677448.113744:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677448.113744:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.113745:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.113746:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.113746:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677448.113747:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:20.0:1619677448.113747:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:5.0:1619677448.113747:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.113748:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.113749:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.113955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.113961:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.113964:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.113964:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.113967:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.113969:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.113971:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.113972:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.113973:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.113974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:18.0:1619677448.113975:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.113976:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.113979:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.113981:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.113982:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.113983:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.113984:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.113985:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:18.0:1619677448.113985:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.113986:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:30.0:1619677448.113987:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.113987:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.113988:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.113988:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:31.0:1619677448.113989:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.113990:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.113991:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:8.2:1619677448.113992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.113993:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.113994:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.113995:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.113996:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.2:1619677448.113998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000400:00000200:30.0:1619677448.113999:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.114001:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.114001:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.114003:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677448.114004:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.114004:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.114005:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.114005:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:20.0:1619677448.114005:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.114005:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:30.0:1619677448.114007:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.114007:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.114007:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.114008:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.114008:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677448.114008:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.14.11@o2ib3
00000800:00000200:30.0:1619677448.114009:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.114010:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000400:00000200:12.0:1619677448.114011:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.114012:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677448.114013:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:12.0:1619677448.114014:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.114015:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.114016:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677448.114017:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.114020:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:12.0:1619677448.114020:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : REPLY try# 0
00000800:00000200:12.0:1619677448.114022:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 1048576 bytes in 256 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:5.2:1619677448.114022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677448.114023:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:12.0:1619677448.114023:0:19943:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000800:00000200:6.0:1619677448.114024:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.114025:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.114027:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.114028:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.114029:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.114029:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:22.0:1619677448.114030:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677448.114030:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.114030:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.114031:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677448.114032:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.114033:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677448.114033:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000400:00000200:1.0:1619677448.114033:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.114034:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000400:00000200:1.0:1619677448.114036:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.114038:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.114039:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.114040:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.114041:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.114042:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:1.0:1619677448.114042:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:13.0:1619677448.114045:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:6.0:1619677448.114045:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.114046:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:1.0:1619677448.114046:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.114047:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:13.0:1619677448.114048:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:6.0:1619677448.114048:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677448.114048:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:13.0:1619677448.114049:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677448.114049:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677448.114051:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.114052:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.114053:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.114054:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.114055:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.114057:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.114058:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.114058:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.114060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.2:1619677448.114063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.114067:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:17.0:1619677448.114068:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677448.114069:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:17.0:1619677448.114070:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.0:1619677448.114070:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677448.114071:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.114071:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.114072:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677448.114073:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.114076:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677448.114097:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.114099:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[3] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.114103:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.114104:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.114118:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.114118:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:4.2:1619677448.114120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:4.2:1619677448.114125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677448.114127:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.114130:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:26.0:1619677448.114131:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677448.114131:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.114132:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.114134:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.114135:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:8.0:1619677448.114135:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.114136:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677448.114137:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.114138:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96719172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.114139:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.114140:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.114140:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.114141:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677448.114141:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677448.114143:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96719171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.114145:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.114145:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.114146:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.114146:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677448.114147:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:26.0:1619677448.114149:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.114150:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.114151:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.2:1619677448.114151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.0:1619677448.114151:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.114151:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.114153:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:26.0:1619677448.114154:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.114154:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677448.114155:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.114155:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677448.114155:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:5.0:1619677448.114156:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.114157:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.114157:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677448.114157:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.114157:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677448.114158:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.114159:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.114159:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.114161:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.114162:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:4.2:1619677448.114163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:5.0:1619677448.114164:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.114165:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677448.114166:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96719170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.114168:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.114169:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.114170:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:5.0:1619677448.114170:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:24.0:1619677448.114172:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.114173:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.114174:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.114174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677448.114174:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.114175:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677448.114178:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.114178:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677448.114179:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677448.114181:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.114182:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.114182:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.114182:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677448.114182:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677448.114183:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677448.114185:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:1.0:1619677448.114185:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:22.0:1619677448.114186:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.114186:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.114186:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:1.0:1619677448.114186:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.114190:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.114190:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.114193:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677448.114194:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:9.2:1619677448.114198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.114205:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.114208:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.114209:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.114209:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.114211:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.114214:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.114221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.114228:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.114231:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.114231:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.114232:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.114233:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.114233:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:12.0:1619677448.114242:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.114245:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.114246:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.114248:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677448.114248:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: REPLY: OK
00000800:00000200:12.0:1619677448.114250:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.114264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.114266:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.114269:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.114271:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.114272:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.114274:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.114275:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:21.2:1619677448.114277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b84000] (20)++
00000400:00000200:20.0:1619677448.114277:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.114278:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.114279:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.114280:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.2:1619677448.114282:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:20.0:1619677448.114284:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.114285:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.114285:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (21)++
00000800:00000200:5.0:1619677448.114286:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.3.1@o2ib3
00000800:00000200:20.0:1619677448.114288:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.114288:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.114288:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.114289:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677448.114289:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.114291:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.114291:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677448.114291:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:5.0:1619677448.114291:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.114292:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:1.0:1619677448.114292:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.114298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677448.114298:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:20.0:1619677448.114299:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.114300:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.114300:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.114301:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)--
00000400:00000200:17.0:1619677448.114301:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.114302:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.114303:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.114304:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.2:1619677448.114305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000400:00000200:17.0:1619677448.114305:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.114307:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.114307:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:17.0:1619677448.114307:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:5.0:1619677448.114307:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.114308:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:1.0:1619677448.114308:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677448.114309:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:17.0:1619677448.114309:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96719167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.114309:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:21.2:1619677448.114310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b84000] (21)++
00000400:00000200:1.0:1619677448.114310:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.114311:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.114311:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:17.0:1619677448.114312:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.114312:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000400:00000200:17.0:1619677448.114313:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:5.0:1619677448.114314:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.114315:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)++
00000800:00000200:5.0:1619677448.114315:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.114315:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.114316:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.114317:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000800:00000200:1.0:1619677448.114317:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.114318:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.114318:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (23)++
00000400:00000200:17.0:1619677448.114319:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.114319:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (24)--
00000800:00000200:1.0:1619677448.114319:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.114320:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.114320:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)--
00000800:00000200:1.0:1619677448.114320:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.114320:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.114321:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:5.0:1619677448.114321:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)--
00000800:00000200:1.0:1619677448.114321:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:20.0:1619677448.114322:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.114324:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:9.2:1619677448.114324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.114324:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:20.0:1619677448.114326:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96719166] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.114326:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.114327:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.114327:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:20.0:1619677448.114328:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.114328:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:20.0:1619677448.114329:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.114330:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.114330:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.114331:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.114331:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677448.114332:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:20.0:1619677448.114333:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.114334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b84000] (21)++
00000800:00000200:20.0:1619677448.114335:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.114335:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.114336:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.114336:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.114338:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.114338:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)++
00000400:00000200:2.0:1619677448.114338:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.114339:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.0:1619677448.114340:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:5.0:1619677448.114340:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000800:00000200:2.0:1619677448.114341:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:5.0:1619677448.114342:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.114343:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:20.0:1619677448.114344:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (23)++
00000800:00000200:20.0:1619677448.114344:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (24)--
00000800:00000200:13.0:1619677448.114344:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000400:00000200:5.0:1619677448.114344:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.114345:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)--
00000800:00000200:20.0:1619677448.114346:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)--
00000800:00000200:13.0:1619677448.114346:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:5.0:1619677448.114346:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:13.0:1619677448.114347:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.114347:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96719165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.114349:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.114350:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.114351:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000400:00000200:5.0:1619677448.114351:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.114352:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:5.0:1619677448.114352:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:12.0:1619677448.114353:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.114354:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.114355:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677448.114355:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.114358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b84000] (21)++
00000800:00000200:12.0:1619677448.114358:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:5.0:1619677448.114358:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.114360:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677448.114363:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)++
00000800:00000200:5.0:1619677448.114363:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.114364:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.114364:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.114365:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000800:00000200:21.2:1619677448.114367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677448.114367:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.114367:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.114368:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (23)++
00000400:00000200:20.0:1619677448.114369:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.114369:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (24)--
00000800:00000200:5.0:1619677448.114370:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)--
00000400:00000200:20.0:1619677448.114371:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:5.0:1619677448.114371:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)++
00000400:00000200:20.0:1619677448.114372:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.114372:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000800:00000200:8.2:1619677448.114373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:20.0:1619677448.114374:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.114375:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.114376:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677448.114376:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.114380:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:5.0:1619677448.114380:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.114381:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.114381:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:31.0:1619677448.114383:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.114383:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.114383:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:5.0:1619677448.114383:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.114383:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.114384:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:5.0:1619677448.114384:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677448.114384:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.114385:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.114385:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96719163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.114387:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.114387:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.114387:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.114388:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.114389:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96719162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.114390:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.114392:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677448.114393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677448.114394:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.114394:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.114395:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.114397:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.114398:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.114398:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:31.0:1619677448.114399:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.114399:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)++
00000800:00000200:5.0:1619677448.114399:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.114400:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (22)++
00000800:00000200:5.0:1619677448.114400:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:1.0:1619677448.114400:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:31.0:1619677448.114401:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.114401:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.114402:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (24)--
00000800:00000200:20.0:1619677448.114403:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (23)++
00000800:00000200:17.0:1619677448.114403:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (24)--
00000800:00000200:5.0:1619677448.114403:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.114403:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (23)++
00000800:00000200:1.0:1619677448.114403:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.114404:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:31.0:1619677448.114405:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.114405:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b84000] (25)++
00000800:00000200:1.0:1619677448.114405:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.114405:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677448.114406:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.114406:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.114407:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (26)--
00000800:00000200:31.0:1619677448.114408:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.114408:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.114408:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (25)--
00000800:00000200:8.2:1619677448.114408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.114408:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b84000] (26)++
00000800:00000200:31.0:1619677448.114409:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.114409:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (25)++
00000800:00000200:5.0:1619677448.114409:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b84000] (25)--
00000800:00000200:31.0:1619677448.114410:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677448.114411:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000800:00000200:5.0:1619677448.114411:0:19935:0:(o2iblnd_cb.c:929:kiblnd_post_tx_locked()) 10.51.3.1@o2ib3(1): redundant or enough NOOP
00000800:00000200:5.0:1619677448.114411:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (25)--
00000800:00000200:8.2:1619677448.114412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:20.0:1619677448.114413:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.114413:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (24)--
00000800:00000200:5.0:1619677448.114414:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)++
00000400:00000200:20.0:1619677448.114415:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.114415:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:20.0:1619677448.114416:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:5.0:1619677448.114416:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b84000] (24)--
00000800:00000200:5.0:1619677448.114416:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)--
00000400:00000200:20.0:1619677448.114417:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96719161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.114417:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.114418:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.114418:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:20.0:1619677448.114419:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.114419:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:1.0:1619677448.114419:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:20.0:1619677448.114420:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.114420:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.114421:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.114421:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.114422:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:5.0:1619677448.114422:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.114423:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.0:1619677448.114424:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:20.0:1619677448.114425:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.114425:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677448.114425:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)++
00000800:00000200:22.0:1619677448.114426:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.114426:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.114426:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000800:00000200:20.0:1619677448.114427:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:6.0:1619677448.114429:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.114429:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.114430:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.114431:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000400:00000200:5.0:1619677448.114431:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.114432:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:6.0:1619677448.114432:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677448.114433:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000400:00000200:5.0:1619677448.114433:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677448.114434:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.114436:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:9.2:1619677448.114439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:5.0:1619677448.114440:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.114441:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.114442:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:20.0:1619677448.114443:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (23)++
00000800:00000200:20.0:1619677448.114443:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (24)--
00000800:00000200:20.0:1619677448.114444:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)--
00000800:00000200:10.0:1619677448.114444:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:5.0:1619677448.114445:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.114446:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:20.0:1619677448.114446:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677448.114447:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)++
00000800:00000200:5.0:1619677448.114447:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:26.0:1619677448.114448:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677448.114448:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:20.0:1619677448.114448:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000800:00000200:10.0:1619677448.114448:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:8.0:1619677448.114449:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677448.114450:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.114450:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.114451:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.114451:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677448.114452:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:24.0:1619677448.114453:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.114453:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.114453:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.114453:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677448.114454:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (23)++
00000400:00000200:20.0:1619677448.114455:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677448.114455:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (24)--
00000800:00000200:5.0:1619677448.114455:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)--
00000400:00000200:20.0:1619677448.114456:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.114456:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.114456:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)++
00000400:00000200:20.0:1619677448.114458:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.114458:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:5.0:1619677448.114458:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000400:00000200:20.0:1619677448.114459:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677448.114459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:10.0:1619677448.114460:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.114461:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677448.114463:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:10.0:1619677448.114463:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.114464:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677448.114464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000400:00000200:5.0:1619677448.114465:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.114466:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.114466:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:18.0:1619677448.114467:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:5.0:1619677448.114467:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.114470:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.114470:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:8.0:1619677448.114470:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000400:00000200:5.0:1619677448.114470:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:2.0:1619677448.114470:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.114471:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.114471:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96719157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.114472:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.114473:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:18.0:1619677448.114473:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.114473:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:10.0:1619677448.114474:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677448.114474:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (21)++
00000400:00000200:5.0:1619677448.114474:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.114475:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000400:00000200:5.0:1619677448.114475:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.114476:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:5.0:1619677448.114476:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.114477:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.114478:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:10.0:1619677448.114478:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677448.114479:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.114479:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.114480:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.114480:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.114480:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:5.0:1619677448.114480:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.114481:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677448.114482:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.114482:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.114482:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.0:1619677448.114482:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.114483:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (23)++
00000800:00000200:20.0:1619677448.114484:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (24)--
00000800:00000200:20.0:1619677448.114484:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)--
00000800:00000200:20.0:1619677448.114485:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)++
00000800:00000200:5.0:1619677448.114485:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.114486:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.114487:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000800:00000200:5.0:1619677448.114487:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:20.0:1619677448.114489:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.114489:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.114489:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (23)++
00000800:00000200:5.0:1619677448.114490:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b84000] (24)++
00000400:00000200:20.0:1619677448.114491:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.114492:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:5.0:1619677448.114492:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (25)--
00000800:00000200:5.0:1619677448.114492:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (24)--
00000400:00000200:20.0:1619677448.114493:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:5.0:1619677448.114493:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)++
00000400:00000200:20.0:1619677448.114494:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96719156] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.114495:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:5.0:1619677448.114495:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000400:00000200:30.0:1619677448.114496:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.114496:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.114496:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:20.0:1619677448.114497:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.114498:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.114498:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:13.0:1619677448.114498:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677448.114499:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:13.0:1619677448.114499:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.114501:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:13.0:1619677448.114501:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.114502:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:21.2:1619677448.114502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:12.0:1619677448.114502:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:31.0:1619677448.114503:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.114503:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.114504:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.114505:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.114505:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677448.114506:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677448.114507:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:5.0:1619677448.114507:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.114508:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:5.0:1619677448.114508:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:20.0:1619677448.114509:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.114510:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.114510:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:20.0:1619677448.114510:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:5.0:1619677448.114510:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.114511:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:6.0:1619677448.114511:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:5.0:1619677448.114512:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.114513:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:5.0:1619677448.114513:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.114514:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.114514:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.114517:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.114518:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.114519:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.114520:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96719154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.114520:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.114521:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.0:1619677448.114521:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (24)++
00000800:00000200:31.0:1619677448.114522:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:22.0:1619677448.114522:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.114522:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (25)--
00000800:00000200:20.0:1619677448.114522:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (24)--
00000800:00000200:5.0:1619677448.114522:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.114523:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.114523:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)++
00000800:00000200:5.0:1619677448.114523:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:22.0:1619677448.114524:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:20.0:1619677448.114524:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b84000] (24)--
00000800:00000200:5.0:1619677448.114524:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.114525:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:20.0:1619677448.114525:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)--
00000800:00000200:5.0:1619677448.114525:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:31.0:1619677448.114526:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.114526:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (23)--
00000800:00000200:5.0:1619677448.114526:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (23)++
00000800:00000200:20.0:1619677448.114527:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)++
00000800:00000200:5.0:1619677448.114527:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (23)--
00000400:00000200:31.0:1619677448.114528:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.114528:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.114528:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)--
00000800:00000200:5.0:1619677448.114528:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)++
00000800:00000200:20.0:1619677448.114529:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000800:00000200:31.0:1619677448.114530:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.0:1619677448.114530:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000800:00000200:22.0:1619677448.114531:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.2:1619677448.114531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:20.0:1619677448.114533:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.114533:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.114534:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.114535:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.114535:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:20.0:1619677448.114536:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.114537:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000400:00000200:5.0:1619677448.114537:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.114538:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.114538:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677448.114538:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:22.0:1619677448.114539:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:20.0:1619677448.114539:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96719153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.114541:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.114542:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.114543:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:9.2:1619677448.114544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:5.0:1619677448.114544:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677448.114546:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000400:00000200:5.0:1619677448.114546:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96719152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.114547:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.114548:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.114548:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.114549:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.114550:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677448.114550:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.114551:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.114552:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:2.0:1619677448.114552:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:30.0:1619677448.114553:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677448.114553:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.114554:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.0:1619677448.114554:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.114554:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677448.114554:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.114554:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.114555:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.114556:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.114556:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677448.114556:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.114557:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (23)++
00000400:00000200:2.0:1619677448.114557:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.114558:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (24)--
00000800:00000200:5.0:1619677448.114558:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.114559:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)--
00000800:00000200:5.0:1619677448.114559:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.114560:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)++
00000800:00000200:5.0:1619677448.114560:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.114560:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.0:1619677448.114561:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000800:00000200:5.0:1619677448.114561:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:9.2:1619677448.114562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.114562:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (23)++
00000400:00000200:20.0:1619677448.114564:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.114564:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b84000] (24)++
00000800:00000200:5.0:1619677448.114565:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (25)--
00000400:00000200:20.0:1619677448.114566:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.114566:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (24)--
00000400:00000200:20.0:1619677448.114567:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:5.0:1619677448.114567:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.114568:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.114568:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.114569:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96719151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.114570:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.114570:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:5.0:1619677448.114570:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.114571:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.114572:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.114572:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.114573:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.114573:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.114574:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.114575:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.114575:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)++
00000400:00000200:18.0:1619677448.114576:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.114577:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:6.0:1619677448.114577:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677448.114577:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000800:00000200:18.0:1619677448.114578:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:6.0:1619677448.114578:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.114579:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:5.0:1619677448.114580:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.114581:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.114582:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.114582:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.114583:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:30.0:1619677448.114584:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.114584:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.114585:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.114586:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.114587:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.114588:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:5.0:1619677448.114588:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.114590:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677448.114591:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.114591:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.114592:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.114593:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.114593:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:5.0:1619677448.114593:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.114594:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.114595:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:30.0:1619677448.114596:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:20.0:1619677448.114596:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (24)++
00000800:00000200:5.0:1619677448.114596:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.114597:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96719149] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.114597:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (25)--
00000800:00000200:20.0:1619677448.114597:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (24)--
00000800:00000200:5.0:1619677448.114598:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.114599:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.114599:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677448.114599:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:30.0:1619677448.114600:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.114600:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)++
00000800:00000200:5.0:1619677448.114600:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:30.0:1619677448.114601:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:20.0:1619677448.114601:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b84000] (24)--
00000800:00000200:20.0:1619677448.114601:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)--
00000800:00000200:5.0:1619677448.114601:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677448.114602:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)++
00000800:00000200:5.0:1619677448.114602:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (23)++
00000800:00000200:5.0:1619677448.114603:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (24)--
00000800:00000200:5.0:1619677448.114603:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)--
00000800:00000200:20.0:1619677448.114604:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000800:00000200:5.0:1619677448.114604:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)--
00000400:00000200:30.0:1619677448.114605:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.114606:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.114607:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:9.2:1619677448.114607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:20.0:1619677448.114608:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.114609:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.114609:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:30.0:1619677448.114610:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:20.0:1619677448.114610:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:30.0:1619677448.114611:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:20.0:1619677448.114611:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96719148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.114612:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:22.0:1619677448.114612:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:20.0:1619677448.114612:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.114613:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:22.0:1619677448.114613:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.114613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b84000] (21)++
00000400:00000200:20.0:1619677448.114613:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.114613:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.114614:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.114614:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:2.0:1619677448.114614:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:20.0:1619677448.114615:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:22.0:1619677448.114616:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.114616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.114617:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:20.0:1619677448.114618:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.114618:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.114618:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.114620:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)++
00000400:00000200:2.0:1619677448.114620:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.114621:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677448.114621:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.114622:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000800:00000200:1.0:1619677448.114622:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.114623:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:31.0:1619677448.114624:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:20.0:1619677448.114624:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.114625:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:20.0:1619677448.114625:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:5.0:1619677448.114625:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.114625:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.114626:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.114627:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:20.0:1619677448.114627:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:31.0:1619677448.114628:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.114628:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (23)++
00000400:00000200:5.0:1619677448.114628:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.114628:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.114629:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (24)--
00000800:00000200:20.0:1619677448.114629:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)--
00000800:00000200:20.0:1619677448.114630:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)--
00000800:00000200:6.0:1619677448.114630:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:5.0:1619677448.114630:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.114631:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.114631:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:1.0:1619677448.114631:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.114632:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96719147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.114633:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677448.114633:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:31.0:1619677448.114634:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:22.0:1619677448.114634:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:5.0:1619677448.114634:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.114634:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.114635:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000400:00000200:5.0:1619677448.114635:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.114636:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.114636:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:5.0:1619677448.114636:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677448.114636:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.114637:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.114639:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:21.2:1619677448.114640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b84000] (21)++
00000400:00000200:5.0:1619677448.114641:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.114643:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.114644:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.114646:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:20.0:1619677448.114646:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)++
00000800:00000200:5.0:1619677448.114646:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.114646:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677448.114647:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000800:00000200:5.0:1619677448.114647:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.114648:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.114649:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:5.0:1619677448.114649:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:22.0:1619677448.114650:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.114650:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.114650:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:5.0:1619677448.114650:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (23)++
00000800:00000200:1.0:1619677448.114650:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.114650:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677448.114651:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b84000] (24)++
00000800:00000200:1.0:1619677448.114651:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:22.0:1619677448.114652:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.114652:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.114653:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (25)--
00000800:00000200:5.0:1619677448.114653:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (24)--
00000800:00000200:1.0:1619677448.114653:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.114653:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.114654:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)--
00000800:00000200:1.0:1619677448.114654:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677448.114655:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:20.0:1619677448.114655:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:1.0:1619677448.114655:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:20.0:1619677448.114656:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96719145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.114656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.114657:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:20.0:1619677448.114657:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.114659:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:21.2:1619677448.114659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b84000] (22)++
00000400:00000200:20.0:1619677448.114659:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677448.114659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:22.0:1619677448.114660:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000400:00000200:20.0:1619677448.114660:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:6.0:1619677448.114660:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.114661:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.114662:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:20.0:1619677448.114663:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.114664:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.114665:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677448.114665:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:24.0:1619677448.114666:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:5.0:1619677448.114666:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.114668:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.114668:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.114669:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:20.0:1619677448.114669:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:24.0:1619677448.114670:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.114670:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:8.0:1619677448.114670:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677448.114670:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.114670:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)++
00000800:00000200:5.0:1619677448.114671:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b84000] (24)--
00000400:00000200:24.0:1619677448.114672:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.114672:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)--
00000800:00000200:20.2:1619677448.114673:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.114673:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677448.114674:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.114674:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)--
00000800:00000200:24.0:1619677448.114675:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.0:1619677448.114675:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (21)++
00000800:00000200:5.0:1619677448.114675:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.114676:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (22)--
00000800:00000200:5.0:1619677448.114676:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.114677:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (21)--
00000800:00000200:20.0:1619677448.114678:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.114678:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.114680:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.114682:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677448.114683:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.114684:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:5.0:1619677448.114684:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.114685:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.114686:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:30.0:1619677448.114687:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.114688:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.114691:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677448.114691:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.114691:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.114693:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677448.114694:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.0:1619677448.114696:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.114697:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.114697:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.114699:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.114699:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.114700:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.114701:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.114705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.114713:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.114715:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.114716:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.114717:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.114719:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.114721:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.114893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.114900:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.114903:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.114904:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.114906:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.114909:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.114911:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.114912:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.114913:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.114914:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.114915:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:21.2:1619677448.114916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:26.0:1619677448.114919:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.114920:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.114923:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.114923:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.114924:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.114925:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.114926:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677448.114927:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.114927:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.114928:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.114928:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.114929:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000400:00000200:31.0:1619677448.114930:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.114930:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.114930:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677448.114931:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677448.114933:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.114933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677448.114934:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.114935:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.114936:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:4.2:1619677448.114936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.114939:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.114940:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.0:1619677448.114941:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:31.0:1619677448.114943:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.114943:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:31.0:1619677448.114944:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677448.114944:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.0:1619677448.114944:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.114945:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.114945:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:31.0:1619677448.114946:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.114946:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.114947:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.114947:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677448.114947:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.114947:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.114948:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677448.114948:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.114950:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:6.0:1619677448.114959:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.114962:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.114963:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.114963:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.114966:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.114968:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.115215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.115222:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.115225:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.115226:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.115227:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.115230:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.115232:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.115233:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.115234:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.115235:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.115236:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.115240:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.115242:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.115244:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.115245:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.115246:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.115248:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.115248:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.115249:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.115249:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.115254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.115262:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.115264:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.115265:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.115266:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.115268:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.115270:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.115292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.115298:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.115301:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.115301:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.115303:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.115306:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.115307:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.115308:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.115310:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.115311:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.115312:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.115315:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.115316:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.115319:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.115320:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.115320:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.115322:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.115322:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.115323:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.115323:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.115329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.115336:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.115338:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.115339:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.115340:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.115342:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.115344:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.115379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:5.2:1619677448.115379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.0:1619677448.115386:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:26.0:1619677448.115387:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677448.115389:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.115389:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:26.0:1619677448.115390:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.115391:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:8.0:1619677448.115392:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.115393:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.115394:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.115396:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.115396:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.115397:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.115397:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.115398:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.115399:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.115399:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.115400:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.115401:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.115401:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.115403:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.115405:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.115406:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.115407:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.115408:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.115410:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.115411:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.115411:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.115411:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.115411:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.115412:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.115413:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.115414:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.115414:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677448.115414:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:26.0:1619677448.115415:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677448.115415:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.115415:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:26.0:1619677448.115416:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.2:1619677448.115419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:12.0:1619677448.115428:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677448.115428:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.115430:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:12.0:1619677448.115431:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.115432:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.115432:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.115434:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.115435:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.115436:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:12.0:1619677448.115436:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.115437:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:8.2:1619677448.115438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:13.0:1619677448.115439:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677448.115439:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.115440:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.115440:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.115441:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677448.115441:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.115443:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.115445:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.115446:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.115446:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.115447:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.115447:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.115534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.115543:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.115545:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.115548:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.115550:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.115550:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.115552:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.115553:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.115554:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.115555:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.115556:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.115560:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.115561:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.115564:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.115565:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.115565:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.115567:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.115567:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.115568:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.115569:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.115573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.115580:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.115583:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.115584:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.115585:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.115586:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.115588:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.115592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.115600:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.115603:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.115604:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.115604:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.115605:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.115606:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.115686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.115691:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.115693:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.115695:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.115696:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.115698:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.115699:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677448.115700:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.115702:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.115703:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.115704:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677448.115707:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.115709:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.115711:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.115712:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.115713:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.115714:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.115715:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.115715:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.115716:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.115720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.115728:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.115731:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.115732:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.115732:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.115734:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.115737:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.115744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.115752:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.115754:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.115757:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.115758:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.115759:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.115761:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677448.115762:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.115764:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.115764:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.115765:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677448.115769:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.115770:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677448.115773:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.115774:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677448.115774:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.115776:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677448.115777:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.115778:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.115780:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.115780:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677448.115786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677448.115790:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:30.0:1619677448.115793:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677448.115793:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.115794:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.115794:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677448.115796:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:22.0:1619677448.115796:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.115797:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.115798:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.115799:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.115825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.115830:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.115832:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.115833:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.115835:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.115837:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.115839:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.115840:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.115841:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.115842:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.115843:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.115846:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.115848:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.115850:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.115851:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.115852:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.115853:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.115854:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.115854:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.115855:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.115859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.115867:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.115869:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.115870:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.115871:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.115873:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.115875:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.115898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677448.115905:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.115907:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677448.115908:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677448.115910:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.115912:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.115914:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.115915:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.115916:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.115917:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.115918:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.115921:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.115922:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.115924:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.115925:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.115926:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.115927:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.115928:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.115928:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.115929:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:26.0:1619677448.115939:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.115941:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.115941:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.115942:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.115943:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.115945:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.116076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.116083:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.116085:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.116086:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.116087:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.116087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677448.116089:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.116090:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.116091:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.116092:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.116093:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.116093:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.116094:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.116095:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.116096:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.116099:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.116099:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.116101:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.116101:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.116102:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.116103:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.116105:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.116105:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.116106:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.116106:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677448.116106:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.116107:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.116108:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.116109:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.116109:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.116110:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:4.2:1619677448.116110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:1.0:1619677448.116110:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.116112:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.2:1619677448.116114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.116114:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.116115:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.116115:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.116117:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.116117:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.116117:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.116118:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.116119:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:24.0:1619677448.116120:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:1.0:1619677448.116120:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677448.116121:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677448.116122:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:24.0:1619677448.116123:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.116123:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.116125:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.116126:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.116126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.116126:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:24.0:1619677448.116127:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.116129:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.116129:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.116130:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96719130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.116131:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:24.0:1619677448.116132:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.116132:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.116132:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:24.0:1619677448.116133:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.116133:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:17.0:1619677448.116133:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.116133:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:24.0:1619677448.116134:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.116134:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.116135:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.116135:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.116136:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677448.116138:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.116140:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.116140:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:9.2:1619677448.116140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.116143:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.116143:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.116144:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.116146:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.116146:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.116147:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677448.116147:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.116148:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677448.116150:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.116151:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.116151:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.2:1619677448.116152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.116152:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.116152:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.116158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677448.116160:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.116162:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.116163:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.116163:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.116165:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.116166:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677448.116167:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677448.116168:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.116169:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.116171:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.116174:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.116174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000400:00000200:26.0:1619677448.116176:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.116177:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.116178:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.116179:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.116180:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.116181:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677448.116183:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.116184:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.116184:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:26.0:1619677448.116185:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.116185:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.116186:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.116187:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677448.116188:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.116189:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.116190:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.116191:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.116192:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.116193:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.116194:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.116195:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.116200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677448.116205:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.116208:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:10.0:1619677448.116208:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677448.116208:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.116209:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.116209:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.116210:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677448.116211:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.116211:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677448.116214:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.116346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.116354:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.116357:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.116359:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.116362:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.116362:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.116363:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.116364:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.116366:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.116367:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.116368:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.116371:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.116372:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.116375:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.116376:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.116377:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.116378:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.116379:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.116380:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.116380:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.116385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.116393:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.116396:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.116397:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.116398:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.116400:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.116401:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.116601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.116609:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.116612:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.116613:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.116615:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.116618:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.116619:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.116621:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.116622:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.116623:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.116624:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.116627:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.116628:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.116631:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.116632:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.116632:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.116633:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.116634:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.116635:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.116635:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.116647:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.116649:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.116650:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.116651:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.116653:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.116654:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.116753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.116759:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.116761:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.116764:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.116764:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.116766:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.116767:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.116769:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.116770:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.116771:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.116772:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.116775:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.116777:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.116780:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.116780:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.116781:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.116783:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.116784:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.116785:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.116786:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.116786:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.116791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.116794:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.116796:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.116797:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677448.116797:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677448.116798:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677448.116799:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.116800:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.116801:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.116802:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.116804:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.116831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.116837:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.116840:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.116840:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.116842:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.116844:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.116845:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.116846:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.116848:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.116849:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.116850:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.116853:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.116855:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.116856:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.116857:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.116858:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.116859:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.116860:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.116860:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.116861:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.116865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.116872:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.116874:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.116875:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.116875:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.116877:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.116877:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.116888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.116893:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.116896:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.116897:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.116899:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.116901:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.116901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:22.0:1619677448.116902:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:5.2:1619677448.116902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:22.0:1619677448.116904:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.116905:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.116906:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.116907:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:8.0:1619677448.116908:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677448.116909:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:22.0:1619677448.116910:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.116911:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.116912:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:10.0:1619677448.116912:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.116913:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677448.116913:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.116915:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.116915:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.116915:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.116916:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.116917:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:10.0:1619677448.116918:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.116918:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.116919:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.116919:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.116920:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677448.116920:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.116920:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:22.0:1619677448.116921:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677448.116921:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.116921:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96719123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.116922:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.116923:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.116924:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.116924:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.116925:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.116925:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:10.0:1619677448.116930:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.116930:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.116932:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.116932:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.116932:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.116933:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.116935:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.116935:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.116936:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.116936:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:26.0:1619677448.116937:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.116937:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.116937:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677448.116938:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.116938:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.116939:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677448.116940:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.116940:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677448.116940:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.116940:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:26.0:1619677448.116941:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.116941:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.116942:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:10.0:1619677448.116942:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.2:1619677448.116944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:9.2:1619677448.116946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.116952:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677448.116954:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677448.116955:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.116956:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.116956:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.116957:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.116958:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.116958:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:1.0:1619677448.116958:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.116961:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.116961:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677448.116964:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.117045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.117052:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.117055:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.117056:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.117058:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.117061:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.117062:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.117063:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.117065:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.117066:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.117067:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.117070:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.117071:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.117073:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.117074:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.117074:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.117076:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.117076:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.117077:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.117077:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.117088:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.117091:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.117091:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.117093:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.117095:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.117096:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.117211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.117216:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:21.2:1619677448.117219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.117219:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.117221:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.117222:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:2.0:1619677448.117222:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.117224:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677448.117225:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.117227:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.117227:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:20.2:1619677448.117228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:2.0:1619677448.117229:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.117230:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.117231:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.117232:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.117235:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.117236:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:18.0:1619677448.117237:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.117237:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.117238:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:31.0:1619677448.117239:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:18.0:1619677448.117240:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.117240:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.117241:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.117241:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677448.117242:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.117242:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.117243:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:18.0:1619677448.117243:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.117243:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.117244:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677448.117245:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.117247:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677448.117248:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000400:00000200:31.0:1619677448.117249:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96719118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.117249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677448.117250:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.117250:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.117251:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.117251:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:31.0:1619677448.117252:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.117253:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.117253:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.117254:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.117254:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.117255:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.117255:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.117256:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000400:00000200:31.0:1619677448.117257:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.117257:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (24)++
00000800:00000200:18.0:1619677448.117258:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (25)--
00000800:00000200:5.2:1619677448.117258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.117258:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (25)--
00000800:00000200:31.0:1619677448.117259:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677448.117259:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.117259:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.117259:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.117260:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:2.0:1619677448.117261:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.117262:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.117262:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:31.0:1619677448.117263:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677448.117263:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.117263:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.117264:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.117264:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.117264:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.117265:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:31.0:1619677448.117266:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.117266:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:9.2:1619677448.117266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.117268:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.117269:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.117270:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.117272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.117274:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.117276:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.117276:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.117278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:18.0:1619677448.117278:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.117278:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.117281:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677448.117283:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:30.0:1619677448.117286:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.117286:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.117286:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.117288:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:20.0:1619677448.117288:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.117289:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.117290:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.117290:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.117291:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.117365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.117373:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.117376:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.117377:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.117379:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.117381:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.117383:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.117384:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.117385:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.117387:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.117387:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.117391:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.117393:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.117396:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.117396:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.117397:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.117399:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.117399:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.117400:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.117401:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.117402:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.117407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.117413:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.117414:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.117416:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.117417:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677448.117417:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.117418:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.117418:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.117418:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.117420:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.117423:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.117437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.117443:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677448.117444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:17.0:1619677448.117447:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.117447:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.117450:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.117451:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677448.117452:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.117454:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:17.0:1619677448.117454:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.117455:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.117455:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.117457:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.117458:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.117459:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.117460:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.117460:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.117461:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.117463:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.117464:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.117465:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.117466:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.117467:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:17.0:1619677448.117467:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.117471:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.117472:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.117472:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.117473:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677448.117473:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.117475:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.117475:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.117476:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.117476:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:17.0:1619677448.117476:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.117476:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.117477:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.117478:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.117479:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.117479:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.117480:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.117488:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.117490:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:10.0:1619677448.117490:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:13.0:1619677448.117491:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.117492:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.117492:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.117493:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:13.0:1619677448.117493:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.117495:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:10.0:1619677448.117495:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.117496:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.117496:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.117535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.117543:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.117546:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.117547:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.117548:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.117551:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.117553:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.117554:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.117555:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.117556:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.117557:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.117560:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.117562:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.117565:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.117566:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.117566:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.117568:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.117569:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.117570:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.117572:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.117572:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677448.117574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.117578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:2.0:1619677448.117582:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.0:1619677448.117584:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:2.0:1619677448.117585:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:8.0:1619677448.117586:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:2.0:1619677448.117586:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.117587:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.117587:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.117589:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:2.0:1619677448.117589:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.117591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:2.0:1619677448.117592:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.117593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.117599:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:18.0:1619677448.117600:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.117602:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.117602:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:18.0:1619677448.117603:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.117603:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.117604:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.117604:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000400:00000200:26.0:1619677448.117605:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.117605:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:26.0:1619677448.117607:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.117608:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.117609:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.117611:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.117612:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.117613:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.117615:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.117617:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.117620:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.117620:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.117621:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.117623:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.117623:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.117624:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.117625:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.117629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.117638:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.117641:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.117642:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.117644:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.117644:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.117647:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.117773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.117781:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.117784:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.117786:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.117788:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.117789:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.117789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:30.0:1619677448.117791:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.117792:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719112] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.117793:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.117794:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.117794:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:30.0:1619677448.117795:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.117797:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.117798:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:30.0:1619677448.117800:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.117801:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.117802:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:22.0:1619677448.117804:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.117805:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.117805:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:22.0:1619677448.117805:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:30.0:1619677448.117806:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677448.117806:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.117807:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:22.0:1619677448.117807:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.117808:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:22.0:1619677448.117808:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.117809:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.117809:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:22.0:1619677448.117809:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.117812:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.117814:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.117816:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.117817:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.117818:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:12.0:1619677448.117819:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:22.0:1619677448.117820:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.117820:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.117821:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.117822:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:12.0:1619677448.117822:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:22.0:1619677448.117823:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:12.0:1619677448.117823:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.117824:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.117825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677448.117826:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.117827:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.117828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.117834:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.117834:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:26.0:1619677448.117836:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.117836:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:6.0:1619677448.117836:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.117837:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677448.117837:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677448.117838:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.117839:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.117839:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.117841:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.117842:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.117844:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.117845:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.117846:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.117847:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.117849:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.117849:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.117851:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.117852:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:10.0:1619677448.117853:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.117855:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.117858:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.117859:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.117859:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.117861:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.117861:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.117862:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.117862:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.117874:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.117877:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.117878:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.117878:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.117880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:6.0:1619677448.117880:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.117883:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:6.0:1619677448.117883:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677448.117885:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.117887:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.117888:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.117890:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.117892:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.117893:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.117894:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.117895:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.117896:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.117900:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.117902:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.117905:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.117906:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.2:1619677448.117907:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.117910:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.117912:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.117912:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.117913:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.117913:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.117914:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:5.0:1619677448.117914:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.117915:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.117916:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.117918:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.117920:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.117921:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.117922:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.117923:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.117924:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:5.0:1619677448.117924:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.117927:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:5.0:1619677448.117928:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.117929:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.117930:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:31.0:1619677448.117931:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.117931:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.117933:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.117933:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:31.0:1619677448.117934:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.0:1619677448.117934:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.117936:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.117937:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.117938:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.117938:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.117943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.117949:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:2.0:1619677448.117951:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:30.0:1619677448.117952:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.117953:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.117953:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:2.0:1619677448.117953:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.117954:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.117955:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:18.0:1619677448.117955:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.117956:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:2.0:1619677448.117957:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.117959:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.118064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677448.118072:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677448.118074:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677448.118077:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.118078:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677448.118079:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.118081:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677448.118082:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.118083:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.118085:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.118085:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677448.118089:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.118090:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677448.118093:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.118094:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:30.0:1619677448.118094:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:30.0:1619677448.118096:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:30.0:1619677448.118096:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677448.118097:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677448.118098:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677448.118101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.118107:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.118110:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.118111:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.118111:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.118113:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.118115:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.118122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677448.118129:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.118131:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.118132:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.118132:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.118133:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.118134:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.118136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.118143:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.118146:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.118147:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.118148:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.118151:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.118153:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.118154:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.118155:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.118156:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.118157:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.118160:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.118161:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.118164:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.118165:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.118166:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.118167:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.118168:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.118168:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.118169:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.118181:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.118183:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.118184:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.118185:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.118187:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.118189:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677448.118207:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.118210:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.118211:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.118211:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.118212:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.118212:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.118274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.118281:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.118283:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:20.2:1619677448.118283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.0:1619677448.118284:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.118286:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.118289:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.118290:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:31.0:1619677448.118291:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:24.0:1619677448.118291:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.118293:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.118300:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:24.0:1619677448.118300:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.118301:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:24.0:1619677448.118302:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.118303:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.118305:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.118305:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.118307:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:24.0:1619677448.118307:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.118308:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.118309:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.118310:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.118310:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.118311:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.118311:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.118311:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.118313:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.118313:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:31.0:1619677448.118314:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.118314:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.118315:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:31.0:1619677448.118316:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.118319:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.118319:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.118320:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.2:1619677448.118320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677448.118322:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:31.0:1619677448.118322:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.118323:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.118323:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677448.118327:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:10.0:1619677448.118328:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677448.118330:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677448.118331:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.118331:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.118332:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:13.0:1619677448.118333:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.118334:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.118335:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677448.118335:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677448.118336:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677448.118336:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.118337:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:10.0:1619677448.118337:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.118338:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.118339:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:12.0:1619677448.118340:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:10.0:1619677448.118340:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.118341:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.118342:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.118343:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.118347:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.118348:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.118351:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.118352:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.118353:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.118355:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.118355:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.118356:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.118357:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.118361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.118369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.118373:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.118375:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.118376:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:18.0:1619677448.118377:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.118377:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:18.0:1619677448.118379:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.118380:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.118382:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.118382:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.118383:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:2.0:1619677448.118383:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:18.0:1619677448.118384:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.118385:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.118386:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.118388:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.118389:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.118392:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.118394:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.118396:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.118397:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.118397:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.118399:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.118399:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.118400:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677448.118401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.118401:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.118405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.118407:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.118410:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.118411:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.118413:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:1.0:1619677448.118413:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.118415:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.118415:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.118416:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.118417:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:1.0:1619677448.118417:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:18.0:1619677448.118418:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.118418:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.118419:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.118420:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:1.0:1619677448.118420:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.118421:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.118424:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.118426:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.118428:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.118428:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.118429:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677448.118430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677448.118431:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.118431:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.118432:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.118432:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.118437:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677448.118437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.118440:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.118440:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.118443:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.118444:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:26.0:1619677448.118446:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.118446:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:26.0:1619677448.118447:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677448.118447:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.118448:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:26.0:1619677448.118449:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.118450:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.118451:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.118451:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:26.0:1619677448.118452:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.118453:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.118456:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.118458:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.2:1619677448.118460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.118461:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.118462:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.118462:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.118464:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.118465:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.118465:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.118466:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677448.118467:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.118470:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.118471:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.118471:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.118472:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.118472:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:22.0:1619677448.118478:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.118481:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.118481:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.118482:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.118484:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.118486:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677448.118495:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.118498:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.118498:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.118499:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.118500:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.118501:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.118689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.118696:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.118699:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.118700:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.118702:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.118704:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.118705:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.118707:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.118708:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.118709:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.118709:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.118712:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.118713:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.118715:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.118715:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.118716:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.118716:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.118717:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.118719:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.118720:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.118720:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.118726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.118730:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.118731:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.118732:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.118734:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.118734:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.118735:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.118736:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.118736:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.118737:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.118737:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.118741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.118748:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.118751:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.118753:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.118755:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.118755:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.118756:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.118757:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.118758:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.118759:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.118760:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.118763:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.118764:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.118767:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.118768:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.118768:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.118769:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.118770:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.118771:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.118771:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677448.118781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:10.0:1619677448.118781:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.118783:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.118784:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.118784:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:10.0:1619677448.118784:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.118786:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:10.0:1619677448.118787:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.118789:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.118789:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:10.0:1619677448.118790:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:20.0:1619677448.118791:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.118793:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677448.118794:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.118796:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.118797:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.118798:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:26.0:1619677448.118800:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000400:00000200:20.0:1619677448.118801:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.118803:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.118804:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:10.0:1619677448.118804:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.118805:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.118806:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.118806:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.0:1619677448.118807:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.118808:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677448.118809:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677448.118810:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677448.118811:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.118812:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.118812:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677448.118816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.118823:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.118825:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.118826:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.118827:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.118829:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.118830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:8.0:1619677448.118831:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:2.0:1619677448.118835:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.118838:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.118839:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.118841:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.118843:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.118845:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.118846:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.118848:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.118849:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.118850:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.118853:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.118855:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.118858:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.118859:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.118859:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.118861:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.118862:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.118863:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.118863:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:10.0:1619677448.118873:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.118876:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.118876:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.118877:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.118879:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.118882:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.118883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677448.118890:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677448.118893:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677448.118895:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.118896:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677448.118897:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.118899:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677448.118900:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.118902:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.118903:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.118904:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677448.118907:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.118909:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677448.118911:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.118912:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.118913:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.118915:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.118915:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677448.118916:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677448.118916:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677448.118920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.118926:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.118929:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.118930:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.118930:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.118932:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.118934:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.2:1619677448.119064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.119071:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.119073:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.119074:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.119076:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.119078:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.119079:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.119080:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.119081:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.119082:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.119083:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.119086:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.119087:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.119089:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.119090:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.119090:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.119091:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.119092:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.119093:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.119094:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.119094:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.119097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.119100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.119106:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.119107:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.119107:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.119108:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.119108:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.119108:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.119109:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.119110:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.119110:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.119112:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.119150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.119157:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.119159:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.119160:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.119162:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.119164:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.119165:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.119167:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.119168:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.119169:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.119170:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.119174:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.119175:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.119178:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.119179:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.119180:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.119181:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.119182:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.119182:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.119183:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.119195:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.119198:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.119198:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.119199:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.119201:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.119204:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.119210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.119216:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.119219:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.119221:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.119222:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.119224:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.119226:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.119227:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.119229:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.119230:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.119231:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.119234:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.119236:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.119238:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.119239:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.119240:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.119241:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.119242:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.119242:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.119243:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.119248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.119255:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.119258:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.119259:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.119259:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.119261:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.119263:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.119265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.119272:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.119275:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.119275:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.119276:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.119277:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.119277:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.119372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.119380:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.119383:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.119385:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.119388:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.119388:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.119389:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.119391:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.119392:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.119393:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.119394:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:21.2:1619677448.119395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:30.0:1619677448.119397:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.119399:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.119401:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677448.119402:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.119402:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.119403:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.119404:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:30.0:1619677448.119405:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.119405:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:22.0:1619677448.119405:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.119406:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.119407:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.119407:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:6.0:1619677448.119409:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.119411:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:8.2:1619677448.119412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:6.0:1619677448.119412:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.119414:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.119415:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.119416:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:20.0:1619677448.119420:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:6.0:1619677448.119420:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.119423:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:6.0:1619677448.119423:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:20.0:1619677448.119424:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.119425:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.119426:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.119426:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677448.119426:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.119427:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677448.119428:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:6.0:1619677448.119429:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.119430:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.119430:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.119431:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.2:1619677448.119436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:9.2:1619677448.119438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.119445:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.119447:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.119448:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.119449:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.119450:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.119451:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.119452:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.119454:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.119456:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.119457:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719089] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.119458:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.119460:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.119461:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.119462:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.119463:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.119464:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000400:00000200:18.0:1619677448.119465:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.119465:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.119466:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.119467:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.119467:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.119467:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.119469:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:18.0:1619677448.119470:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.119471:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.119471:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.119473:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.119473:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.119474:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.119475:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.119481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.119489:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.119492:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.119493:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.119493:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.119495:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.119497:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.119542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.119547:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.119550:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.119551:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.119552:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.119554:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.119555:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.119556:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.119558:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.119558:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.119560:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.119563:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.119565:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:21.2:1619677448.119566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677448.119567:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.119568:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.119569:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.119570:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.119571:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.119571:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.119572:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.119573:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.119576:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.119578:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.119580:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.119581:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.119583:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.119584:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.119584:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:31.0:1619677448.119585:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.119586:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.119586:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:31.0:1619677448.119587:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677448.119587:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.119587:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.119590:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.119592:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.119594:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.119595:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677448.119597:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.119598:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.119598:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.119600:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.119600:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.119601:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.119602:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.119613:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.119615:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.119616:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.119617:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.119619:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.119621:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.119713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.119718:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.119721:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.119722:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.119724:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.119727:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.119729:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677448.119730:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.119731:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.119732:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.119733:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.119737:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.119738:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.119741:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.119742:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.119742:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.119744:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.119744:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.119745:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.119746:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677448.119750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.119759:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.119761:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.119762:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.119764:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.119764:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.119766:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.119924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.119930:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.119933:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.119933:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.119935:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.119937:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.119939:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.119940:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.119941:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.119942:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.119943:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.119945:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.119947:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.119949:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.119949:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.119950:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.119951:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.119952:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.119952:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.119952:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677448.119964:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.2:1619677448.119966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677448.119967:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.119968:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.119968:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.119970:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.119972:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677448.119972:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.119974:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.119975:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.119977:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.119979:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.119981:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.119982:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.119984:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.119985:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.119986:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.119989:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.119991:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:4.2:1619677448.119991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:17.0:1619677448.119994:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.119995:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.119996:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.119998:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.119998:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.119998:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.119999:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.119999:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677448.120001:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.120001:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677448.120003:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.120006:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.120007:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.120008:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96719083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.120010:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.120011:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.120012:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:30.0:1619677448.120014:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:8.0:1619677448.120015:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.120016:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.120017:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.120017:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:30.0:1619677448.120019:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.120020:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:8.0:1619677448.120020:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.120021:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.120022:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677448.120023:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.0:1619677448.120024:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.120024:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.120026:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.120027:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.120027:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.120030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677448.120033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.120038:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677448.120040:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.120040:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.120041:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.120042:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.120042:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677448.120042:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.120044:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.120044:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.120047:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.120066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.120074:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.120077:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.120079:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.120080:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.120081:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.120083:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.120084:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.120085:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.120086:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.120087:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.120090:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.120092:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.120094:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.120094:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.120095:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.120096:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.120097:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.120098:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.120098:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.120108:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.120110:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.120111:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.120113:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.120114:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.120116:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.120362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.120368:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.120371:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.120372:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.120374:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.120376:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.120378:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.120379:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.120380:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.120381:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.120382:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.120386:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.120388:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.120389:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.120390:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.120391:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.120392:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.120393:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.120393:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.120394:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.120399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.120406:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.120408:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.120409:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.120410:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.120410:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.120412:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.120418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.120425:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.120427:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.120428:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.120429:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.120429:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677448.120429:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.2:1619677448.120549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.120557:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.120560:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.120560:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.120562:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.120565:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.120567:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.120568:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.120569:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.120570:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.120571:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.120575:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.120577:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.120579:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.120580:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.120581:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.120582:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.120583:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.120584:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.120585:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.120585:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.120591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.120597:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:26.0:1619677448.120598:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:31.0:1619677448.120599:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.120600:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.120600:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:31.0:1619677448.120601:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.120601:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.120602:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677448.120603:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.120604:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677448.120615:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.120617:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.120618:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.120619:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.120619:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.120620:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.120679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.120686:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.120688:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.120689:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.120691:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.120693:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.120695:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.120695:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.120697:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.120698:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.120699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:10.0:1619677448.120699:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.120701:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.120703:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.120706:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.120706:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.120707:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.120708:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.120709:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.120709:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.120710:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:26.0:1619677448.120711:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.120711:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.2:1619677448.120712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:10.0:1619677448.120712:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.120713:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.120714:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.120716:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.120717:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.120718:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.120719:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.120719:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:26.0:1619677448.120720:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.120721:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:8.0:1619677448.120722:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.120723:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:31.0:1619677448.120724:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.120724:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.120724:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.120725:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.120726:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.120726:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.120728:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:8.0:1619677448.120728:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.120729:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.120729:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.120730:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.120730:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677448.120730:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.120731:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96719077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.120732:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.120733:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:8.0:1619677448.120733:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.120734:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.120734:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:8.0:1619677448.120734:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.120735:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:9.2:1619677448.120738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677448.120738:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.120740:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.120743:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.120743:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.120744:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.120746:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.120746:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:2.0:1619677448.120746:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.0:1619677448.120747:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.120747:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.2:1619677448.120749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.120749:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.120750:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.120750:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.120752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:2.0:1619677448.120752:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.120754:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.120756:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677448.120757:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.120757:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.0:1619677448.120759:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:1.0:1619677448.120759:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.120761:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.120762:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:1.0:1619677448.120762:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677448.120763:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.120763:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.120764:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.120765:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.120766:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:1.0:1619677448.120766:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.120767:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:1.0:1619677448.120767:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.120770:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.120772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:1.0:1619677448.120772:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.120773:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.120774:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.120775:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.120776:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.120777:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.120778:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:5.0:1619677448.120779:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.120779:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.120780:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.120782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.120783:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.120785:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.2:1619677448.120785:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.120789:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.120790:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.120790:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677448.120791:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.120792:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.120792:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:17.0:1619677448.120793:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.120793:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.120794:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.120795:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.120795:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.120796:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677448.120797:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.120820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.120827:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.120830:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.120832:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.120834:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.120834:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.120835:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.120837:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.120838:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.120839:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.120840:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.120843:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.120845:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.120848:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.120848:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.120849:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.120851:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.120851:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.120852:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.120854:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.120855:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677448.120860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.120866:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677448.120868:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.120868:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.120869:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.120870:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.120871:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.120871:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:13.0:1619677448.120871:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.120873:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.120874:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.120884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.120891:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.120893:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.120894:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.120896:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.120899:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.120900:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.120901:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.120902:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.120903:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.120904:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.120907:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.120909:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.120912:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.120912:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.120913:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.120915:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.120915:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.120916:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.120917:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.120922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.120930:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.120933:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.120934:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.120934:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.120937:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.120939:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.120953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.120956:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.120958:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.120961:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.120962:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.120963:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.120964:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677448.120966:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.120967:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.120968:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.120969:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677448.120973:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.120975:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.2:1619677448.120978:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.0:1619677448.120981:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.120982:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677448.120983:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.120985:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.120985:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677448.120985:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.120986:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.120987:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677448.120988:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.120991:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.120991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.120992:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.120993:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.120994:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.120996:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.120997:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.120998:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.120999:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:18.0:1619677448.120999:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.121001:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:31.0:1619677448.121002:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.121002:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.121002:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.121005:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:18.0:1619677448.121005:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.121007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:18.0:1619677448.121007:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:31.0:1619677448.121008:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.121009:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.121009:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.121011:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.121012:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.121013:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:6.0:1619677448.121013:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677448.121014:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.121015:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:6.0:1619677448.121016:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.121017:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.121017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:6.0:1619677448.121019:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.121021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:6.0:1619677448.121021:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.121022:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.121024:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.121025:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.121025:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:6.0:1619677448.121026:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.121027:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:5.0:1619677448.121027:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.121028:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.121029:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.121030:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.121030:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.121031:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.121032:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.121033:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.121034:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:6.0:1619677448.121034:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677448.121034:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:6.0:1619677448.121036:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.121037:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.121037:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.121039:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.121040:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.121041:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.121041:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.121045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.121052:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.121055:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.121056:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.121056:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.121059:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.121061:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.121152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.121159:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.121162:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.121165:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.121166:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.121167:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.121169:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677448.121170:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.121172:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.121173:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.121174:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677448.121177:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.121179:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.121182:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.121183:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677448.121184:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.121186:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677448.121186:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.121187:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.121188:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677448.121194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677448.121199:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.121200:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.121202:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677448.121202:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:22.0:1619677448.121203:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.121203:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.121203:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.121205:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.121206:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.121207:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.121209:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:22.0:1619677448.121210:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:17.0:1619677448.121210:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.121212:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.121213:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.121214:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.121217:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.121219:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.121222:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.121223:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.121224:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.121225:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.121226:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.121227:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.121227:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.121231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.121236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:8.0:1619677448.121237:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.121240:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.121241:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:18.0:1619677448.121241:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:8.0:1619677448.121241:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.121244:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:8.0:1619677448.121244:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.121245:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.121247:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.121248:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:18.0:1619677448.121249:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.121251:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.121252:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.121253:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.121255:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.121256:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.121259:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.121261:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.121264:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.121265:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.121266:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.121267:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.121268:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.121269:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.121269:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677448.121281:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.121283:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.121284:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.121286:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.121287:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.121289:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677448.121304:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.121307:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.121308:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.121308:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.121309:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.121309:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.121386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.121394:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.121397:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.121398:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.121399:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.121402:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.121404:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.121405:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.121407:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.121407:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.121409:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.121412:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.121413:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.121416:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.121417:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.121417:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.121419:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.121419:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.121420:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.121420:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.121432:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.121434:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.121435:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.121436:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.121438:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.121439:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.121507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.121515:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.121517:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.121518:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.121519:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.121523:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.121524:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.121525:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.121527:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.121527:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.121527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:24.0:1619677448.121528:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.121530:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:24.0:1619677448.121531:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.121532:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:24.0:1619677448.121533:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.121535:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.121536:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.121536:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:24.0:1619677448.121537:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.121537:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677448.121537:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.121539:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.121539:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:20.0:1619677448.121539:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:24.0:1619677448.121540:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.121540:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:20.0:1619677448.121540:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.121541:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.121542:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.121543:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:8.2:1619677448.121545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677448.121547:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.121548:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677448.121550:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.121551:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677448.121552:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677448.121553:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:1.0:1619677448.121553:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677448.121554:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677448.121555:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e84c1600] (20)++
00000800:00000200:17.0:1619677448.121556:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.121556:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.121557:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.121559:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:1.0:1619677448.121559:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.121560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677448.121560:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:9.2:1619677448.121560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677448.121561:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.121562:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e84c1600] (21)++
00000800:00000200:1.0:1619677448.121562:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.121565:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.121567:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.72@o2ib3
00000800:00000200:5.0:1619677448.121567:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1600] (22)--
00000800:00000200:2.0:1619677448.121567:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:20.0:1619677448.121568:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.72@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.121568:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677448.121569:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.121569:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.121570:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677448.121570:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:20.0:1619677448.121571:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.121571:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677448.121571:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:2.0:1619677448.121571:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.121573:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:20.0:1619677448.121573:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:2.0:1619677448.121573:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.121575:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.121576:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.121577:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.121577:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677448.121578:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.121578:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:31.0:1619677448.121580:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.121580:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:31.0:1619677448.121582:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.121583:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.121583:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.121586:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.121586:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000400:00000200:31.0:1619677448.121587:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.121588:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:4.2:1619677448.121588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677448.121590:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f90d380] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.121591:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e49d37c00] (20)++
00000400:00000200:31.0:1619677448.121592:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.121592:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e49d37c00] (21)++
00000800:00000200:31.0:1619677448.121594:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677448.121594:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e49d37c00] (22)--
00000800:00000200:26.0:1619677448.121595:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.121595:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e84c1600] (21)++
00000800:00000200:20.0:1619677448.121596:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e84c1600] (22)--
00000800:00000200:20.0:1619677448.121596:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1600] (21)--
00000800:00000200:26.0:1619677448.121597:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:31.0:1619677448.121598:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.121598:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:10.0:1619677448.121598:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.121599:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:26.0:1619677448.121600:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:24.2:1619677448.121600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (21)++
00000800:00000200:31.0:1619677448.121601:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.121602:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.121602:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.121603:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:26.0:1619677448.121603:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.121604:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:24.0:1619677448.121604:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)++
00000400:00000200:26.0:1619677448.121605:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96719062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.121607:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.121607:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.121607:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e49d37c00] (23)--
00000800:00000200:9.2:1619677448.121607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:26.0:1619677448.121608:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:24.0:1619677448.121608:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.121608:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)--
00000400:00000200:24.0:1619677448.121610:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.121612:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.121614:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:24.0:1619677448.121614:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)--
00000800:00000200:18.0:1619677448.121615:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:26.0:1619677448.121617:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.121618:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.121618:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677448.121618:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.121618:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.121619:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.121620:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.121621:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.121621:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677448.121621:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.121622:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:24.2:1619677448.121622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (20)++
00000800:00000200:18.0:1619677448.121624:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:24.0:1619677448.121625:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)++
00000800:00000200:9.2:1619677448.121627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:24.0:1619677448.121628:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677448.121628:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e49d37c00] (22)++
00000800:00000200:8.0:1619677448.121628:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)--
00000800:00000200:24.0:1619677448.121629:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e49d37c00] (22)--
00000800:00000200:24.0:1619677448.121630:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)--
00000800:00000200:2.0:1619677448.121634:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.121637:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677448.121637:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.121638:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.121638:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.121639:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.121640:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677448.121640:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:22.0:1619677448.121641:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.121641:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.121643:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.121646:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.121780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.121787:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.121790:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.121793:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.121794:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.121795:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.121797:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.121798:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.121799:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.121800:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.121801:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.121804:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.121804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:30.0:1619677448.121806:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.121809:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.121810:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.121810:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.121810:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:30.0:1619677448.121812:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.121813:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:21.2:1619677448.121813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:18.0:1619677448.121813:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.121813:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:30.0:1619677448.121814:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.121814:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:18.0:1619677448.121815:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.121818:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:18.0:1619677448.121818:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.121818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:18.0:1619677448.121820:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:22.0:1619677448.121821:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:18.0:1619677448.121821:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.121821:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:18.0:1619677448.121823:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.121824:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.121825:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.121825:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677448.121826:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.121827:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:22.0:1619677448.121829:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.121830:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96719059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.121830:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.121830:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:22.0:1619677448.121831:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.121831:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.121832:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.121832:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.121833:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.121833:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.121834:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.121835:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.121836:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.121837:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.121838:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:22.0:1619677448.121839:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.121839:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.121840:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:5.2:1619677448.121840:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:22.0:1619677448.121841:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.121841:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.121842:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:22.0:1619677448.121843:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.121843:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677448.121844:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.121845:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.121846:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.121847:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:21.2:1619677448.121847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:22.0:1619677448.121848:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.121848:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677448.121849:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677448.121850:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:5.2:1619677448.121851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677448.121852:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:12.0:1619677448.121852:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.121852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.121852:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:13.0:1619677448.121854:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.121855:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.121855:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.121855:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.121856:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:17.0:1619677448.121857:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:13.0:1619677448.121857:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:12.0:1619677448.121857:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:2.0:1619677448.121857:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:12.0:1619677448.121858:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96719058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.121860:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:12.0:1619677448.121860:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.121861:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:1.0:1619677448.121861:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677448.121862:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.121863:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:12.0:1619677448.121863:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677448.121864:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.121864:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.121866:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.121866:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:17.0:1619677448.121867:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.121867:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:17.0:1619677448.121868:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:13.0:1619677448.121868:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677448.121868:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.121869:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:17.0:1619677448.121870:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96719057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.121871:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.121872:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:17.0:1619677448.121872:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.121873:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.121873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000400:00000200:17.0:1619677448.121874:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:12.0:1619677448.121875:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.121876:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677448.121876:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (22)++
00000400:00000200:17.0:1619677448.121877:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.121878:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:17.0:1619677448.121879:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677448.121879:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677448.121880:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677448.121880:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:17.0:1619677448.121881:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.121881:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:17.0:1619677448.121882:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.121882:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.121883:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.121884:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.121884:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.121885:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.121885:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:20.0:1619677448.121886:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (24)++
00000800:00000200:5.0:1619677448.121886:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:20.0:1619677448.121887:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (24)--
00000800:00000200:17.0:1619677448.121887:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677448.121887:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (24)--
00000400:00000200:20.0:1619677448.121888:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.121889:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:20.0:1619677448.121890:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.121890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.121892:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677448.121898:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.121900:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.121901:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.121902:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.121903:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.121905:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.121921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.121928:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.121931:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.121932:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.121934:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.121937:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.121938:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.121939:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.121940:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.121942:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.121943:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:21.2:1619677448.121945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:10.0:1619677448.121946:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.121948:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.121951:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.121952:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.121952:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.121953:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.121954:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.121955:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:10.0:1619677448.121955:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.121956:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.121956:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:31.0:1619677448.121957:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.121959:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.121960:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.121960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677448.121961:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.121962:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.121963:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.121964:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.121965:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.121968:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:31.0:1619677448.121969:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.121970:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.121971:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.121972:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.121973:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.121973:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.121974:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.121975:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:18.0:1619677448.121975:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.121977:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.121977:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.121978:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.121979:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677448.121979:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.121983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.121990:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.121992:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.121993:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.121994:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.121995:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.121998:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.122105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.122112:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.122115:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.122116:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.122118:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.122121:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.122122:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.122124:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.122125:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.122126:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.122127:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.122130:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.122132:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.122134:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.122135:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.122135:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.122137:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.122138:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.122139:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.122140:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.122140:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.122146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.122151:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.122153:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.122153:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.122154:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677448.122155:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677448.122155:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.122156:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.122157:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.122157:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.122160:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677448.122172:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.122172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:22.0:1619677448.122175:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.122176:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.122176:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.122177:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.122177:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677448.122178:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.122181:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.122181:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.122183:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.122186:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.122187:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.122188:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.122190:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.122191:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.122192:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.122195:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.122197:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.122199:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.122199:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.122200:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.122201:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.122202:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.122203:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.122204:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.122204:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:9.2:1619677448.122207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.122210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.122215:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.122216:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.122217:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:2.0:1619677448.122217:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.122218:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.122218:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.122218:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.122219:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.122220:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677448.122221:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.2:1619677448.122266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:20.2:1619677448.122269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:2.0:1619677448.122271:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.122274:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.122275:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677448.122275:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:2.0:1619677448.122276:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.122277:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.122279:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.122279:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.122280:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:2.0:1619677448.122280:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.122281:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.122282:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.122282:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.122283:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.122283:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.122284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:2.0:1619677448.122284:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.122285:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.122286:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:5.0:1619677448.122286:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.122288:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.122288:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.122289:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.122289:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.122292:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.122292:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.122298:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000400:00000200:20.0:1619677448.122300:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.122300:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.122302:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:2.0:1619677448.122302:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.122303:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.122303:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.122304:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96719050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.122305:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.122305:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.122305:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.122306:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:20.0:1619677448.122307:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.122307:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000400:00000200:20.0:1619677448.122308:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:4.2:1619677448.122308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:2.0:1619677448.122308:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677448.122310:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.122312:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.122312:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.2:1619677448.122314:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677448.122315:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677448.122315:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.0:1619677448.122316:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.122317:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.122318:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.122318:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677448.122318:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:5.0:1619677448.122318:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.122319:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677448.122319:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.122320:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.122320:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:8.0:1619677448.122320:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.122320:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.122321:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677448.122321:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.122321:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677448.122322:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:18.0:1619677448.122322:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:5.0:1619677448.122322:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.122323:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677448.122323:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:8.0:1619677448.122323:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.122323:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677448.122324:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.122325:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.122325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677448.122325:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.122326:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96719049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677448.122327:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.122328:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:8.0:1619677448.122328:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.122329:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.122330:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677448.122330:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.122334:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:8.0:1619677448.122334:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.122334:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677448.122336:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.122337:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:6.0:1619677448.122337:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.122338:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:18.0:1619677448.122338:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.122338:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.122338:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.122338:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:8.0:1619677448.122339:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.122340:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:18.0:1619677448.122341:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:8.0:1619677448.122341:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:9.2:1619677448.122342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.122342:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.122342:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677448.122342:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.122343:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677448.122345:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677448.122347:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.2:1619677448.122347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.122349:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.122351:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.122351:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.122352:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.122353:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.122354:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677448.122356:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.122358:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.122359:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.122360:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.122362:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.122365:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.122366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.122373:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.122375:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.122376:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.122377:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.122377:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.122378:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.122413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.2:1619677448.122414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.122420:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.122421:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677448.122423:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:26.0:1619677448.122424:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:6.0:1619677448.122424:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:10.0:1619677448.122425:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677448.122427:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.122428:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.122430:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.122431:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.122431:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.122432:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.122433:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:22.0:1619677448.122434:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.122435:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.122435:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.122436:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.122437:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677448.122438:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.122439:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:22.0:1619677448.122443:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.122444:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.122445:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.122447:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:22.0:1619677448.122449:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.122450:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.122450:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:22.0:1619677448.122450:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:26.0:1619677448.122451:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:22.0:1619677448.122451:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.122452:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.122453:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677448.122453:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.122454:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.122455:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.122456:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677448.122456:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.122457:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.122458:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.122458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.122463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.122465:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:17.0:1619677448.122466:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:10.0:1619677448.122467:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.122467:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.122468:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:6.0:1619677448.122468:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:17.0:1619677448.122469:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:10.0:1619677448.122469:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.122470:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.122470:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:1.0:1619677448.122470:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.122471:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:17.0:1619677448.122472:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.122472:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:6.0:1619677448.122472:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677448.122475:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.122562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.122569:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.122571:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.122572:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.122574:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.122575:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.122577:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.122578:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.122579:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.122580:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.122581:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.122585:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.122587:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.122589:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.122589:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.122590:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.122592:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.122592:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.122593:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.122593:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.122605:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.122607:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.122608:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.122610:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.122612:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.122613:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.122824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:4.2:1619677448.122826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.122832:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.122833:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.122835:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:26.0:1619677448.122836:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.122836:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:31.0:1619677448.122837:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.122839:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:10.0:1619677448.122839:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.122840:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.122841:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.122842:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.122842:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677448.122843:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.122844:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.122845:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.122846:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.122847:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.122847:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.122848:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.122849:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677448.122852:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.122853:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.122855:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.122855:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677448.122857:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.122858:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.122858:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.122859:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.122859:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677448.122860:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.122860:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.122861:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.122861:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.122861:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677448.122862:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:10.0:1619677448.122862:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.122863:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.122863:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:12.0:1619677448.122872:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.122875:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.122875:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.122877:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:12.0:1619677448.122877:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.122879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:13.0:1619677448.122879:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677448.122880:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:12.0:1619677448.122880:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:22.0:1619677448.122881:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.122881:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.122883:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.122886:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677448.122888:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677448.122889:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677448.122889:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.122891:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677448.122892:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.122892:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.122892:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677448.122893:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677448.122893:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000400:00000200:30.0:1619677448.122894:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.122894:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677448.122895:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:30.0:1619677448.122896:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.122897:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.122897:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.122898:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.122898:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:30.0:1619677448.122899:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.122899:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:30.0:1619677448.122900:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:13.0:1619677448.122900:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.122901:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:30.0:1619677448.122903:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.122905:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.122907:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.122907:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.122908:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.122909:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.122910:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.122911:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.122913:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.122913:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.2:1619677448.122917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.2:1619677448.122919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.122920:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.122923:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.122924:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.122924:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.122924:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677448.122926:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.122926:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:31.0:1619677448.122928:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:26.0:1619677448.122929:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.0:1619677448.122929:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.122929:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.122930:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.122931:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:5.0:1619677448.122931:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.122932:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:5.0:1619677448.122933:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.122934:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.122935:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.122936:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.122937:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.122940:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.122942:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.122944:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.122945:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.122945:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.122947:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.122948:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.122949:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.122949:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.122954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.122961:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.122964:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.122965:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.122965:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.122967:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.122970:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.123075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.123082:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.123085:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.123086:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.123087:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.123090:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.123091:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.123092:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.123093:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.123094:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.123095:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.123098:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.123099:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.123101:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.123102:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.123102:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.123104:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.123104:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.123105:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.123107:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.123107:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.123113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.123117:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.123118:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.123119:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.123120:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.123121:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.123122:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.123123:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.123123:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.123123:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.123124:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677448.123131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:10.0:1619677448.123134:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.123136:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:6.0:1619677448.123136:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.123137:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.123137:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.123138:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.123138:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:6.0:1619677448.123139:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.123140:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.123142:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.123144:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.123145:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.123147:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.123148:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.123149:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.123150:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.123153:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.123155:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.123157:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.123158:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.123159:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.123160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677448.123161:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.123162:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.123162:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.123163:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:26.0:1619677448.123167:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.123170:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.123170:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.123172:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.123174:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:26.0:1619677448.123174:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.123176:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:26.0:1619677448.123176:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677448.123177:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.123177:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.123178:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.123179:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.123179:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.123180:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:26.0:1619677448.123180:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.123182:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:26.0:1619677448.123183:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.123185:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.123188:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.123189:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.123190:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.123191:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.123192:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.123192:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.123193:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.123193:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.123193:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.123195:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.123195:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.123196:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.123198:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:9.2:1619677448.123198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.123206:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.123209:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.123209:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.123210:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.123212:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.123215:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.123253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:22.0:1619677448.123258:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677448.123261:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677448.123262:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677448.123264:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.123267:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.123269:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677448.123270:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.123271:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.123272:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.123273:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677448.123277:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.123278:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677448.123281:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.123282:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:22.0:1619677448.123282:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677448.123284:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:22.0:1619677448.123284:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677448.123285:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677448.123285:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:8.2:1619677448.123290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.123299:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.123302:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.123303:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.123303:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.123305:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.123307:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.123315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.123323:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.123326:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.123327:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.123327:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.123328:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.123328:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.123431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.123439:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.123442:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.123442:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677448.123445:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.123447:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.123449:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677448.123450:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.123452:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.123453:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.123454:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.123457:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.123459:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.123462:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.123463:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.123463:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.123465:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.123466:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.123466:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.123467:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.123471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.123479:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.123481:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.123482:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.123484:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.123486:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.123487:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.123508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.123515:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.123517:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.123518:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.123520:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.123522:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.123524:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.123525:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.123526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677448.123527:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.123527:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.123528:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.123531:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.123531:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.123533:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.123534:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:26.0:1619677448.123535:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.123535:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.123536:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.123537:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:1.0:1619677448.123537:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.123538:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:1.0:1619677448.123538:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.123539:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.123540:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:1.0:1619677448.123540:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:26.0:1619677448.123541:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000400:00000200:1.0:1619677448.123541:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.123542:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677448.123543:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.123544:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.123545:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:4.2:1619677448.123547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677448.123548:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.123549:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.123552:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.123553:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.123553:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.123554:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:1.0:1619677448.123554:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.123556:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677448.123556:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677448.123556:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.123556:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.123557:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:10.0:1619677448.123557:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.123557:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.123557:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.123558:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677448.123558:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.123560:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.123563:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.123565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677448.123570:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.123572:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.123573:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.123574:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677448.123575:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677448.123575:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.123577:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677448.123578:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.123578:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:13.0:1619677448.123580:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.123580:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677448.123582:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:13.0:1619677448.123583:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677448.123585:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.123586:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677448.123587:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:13.0:1619677448.123590:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.123592:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:13.0:1619677448.123594:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.123595:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:13.0:1619677448.123596:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:13.0:1619677448.123598:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:13.0:1619677448.123598:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677448.123599:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677448.123599:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677448.123603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.123610:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.123612:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.123613:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.123614:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677448.123616:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.123618:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.2:1619677448.123662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.123668:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.123671:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.123672:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.123674:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.123677:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.123679:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.123680:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.123682:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.123683:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.123684:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.123687:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.123689:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.123691:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.123692:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.123693:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.123695:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.123695:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.123696:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.123696:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.123701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.123708:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.123711:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.123712:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.123712:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.123714:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.123717:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.123738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.123746:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.123748:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.123751:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.123753:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.123753:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.123755:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677448.123756:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.123757:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.123758:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.123759:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.123763:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.123764:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.123767:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.123768:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.123769:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.123771:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.123771:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.123772:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.123773:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.123777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.123784:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.123786:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.123788:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.123788:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.123790:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.123792:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.123901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.123909:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.123912:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.123913:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.123914:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.123917:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.123919:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.123920:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.123921:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.123922:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.123923:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.123926:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.123927:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.123930:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.123931:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.123931:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.123933:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.123933:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.123934:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.123935:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.123946:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:21.2:1619677448.123947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:22.0:1619677448.123949:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.123950:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.123950:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.123953:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.123954:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:22.0:1619677448.123955:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677448.123957:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.123957:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677448.123959:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.123962:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.123963:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.123965:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.123966:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.123967:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.123968:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.123971:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.123973:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.123975:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.123976:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.123977:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.123979:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.123979:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.123980:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.123981:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.123985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.123993:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.123996:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.123996:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.123997:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.123999:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.124002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:18.0:1619677448.124002:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677448.124008:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.124010:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.124013:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.124013:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.124015:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.124017:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.124018:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.124020:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.124021:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.124022:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.124026:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.124027:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.124031:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.124031:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.124032:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.124034:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.124035:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.124036:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.124037:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.124038:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.2:1619677448.124043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.124045:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:30.0:1619677448.124047:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.124048:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.124048:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.124049:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677448.124050:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.124051:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.124053:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.124054:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.124056:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.124327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.124334:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.124337:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.124338:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.124340:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.124343:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.124344:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.124346:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.124347:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.124348:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.124349:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.124352:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.124353:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.124356:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.124357:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.124358:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.124359:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.124360:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.124361:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.124361:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.124374:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.124377:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.124378:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.124378:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.124380:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.124383:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.124398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.124406:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.124409:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.124411:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.124414:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.124414:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.124416:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.124417:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.124418:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.124419:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.124420:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.2:1619677448.124422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:31.0:1619677448.124423:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.124424:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.124425:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.124428:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.124428:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:31.0:1619677448.124429:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.124430:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:20.0:1619677448.124430:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.124431:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.124432:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.124432:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677448.124433:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.124433:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.124433:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.124435:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.124436:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.124437:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.124438:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.124438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:20.0:1619677448.124439:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.124443:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.124444:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.124447:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.124447:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.124448:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.124448:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.124449:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:5.0:1619677448.124449:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.124451:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:5.0:1619677448.124451:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.124452:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.124452:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.124453:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.2:1619677448.124453:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677448.124454:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.124458:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677448.124460:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.124463:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:30.0:1619677448.124464:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:1.0:1619677448.124464:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.124465:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.124466:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.124467:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.124468:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.124469:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.124470:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:17.0:1619677448.124470:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.124471:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.124473:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:17.0:1619677448.124473:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.124474:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.124475:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.124478:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.124480:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.124481:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.124482:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.124483:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.124484:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.124485:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.124486:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.124487:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.124488:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.124490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.124493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.124499:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.124500:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.124500:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.124501:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:1.0:1619677448.124501:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.124502:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.124503:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.124503:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.124503:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677448.124504:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.2:1619677448.124507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.124515:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.124517:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.124518:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.124520:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.124523:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.124524:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:21.2:1619677448.124526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:10.0:1619677448.124526:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.124527:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.124528:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.124529:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.124533:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.124534:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.124534:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.124536:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:10.0:1619677448.124537:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.124538:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.124538:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.124538:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.124540:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677448.124540:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.124540:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677448.124541:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.124541:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:31.0:1619677448.124542:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:10.0:1619677448.124542:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:31.0:1619677448.124544:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.124545:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.124546:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.124547:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.124550:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.124552:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.124552:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.124553:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.124554:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.124554:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.124555:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.124555:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677448.124556:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.124557:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.124558:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.124558:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.124558:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677448.124559:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.124559:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.124563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.124571:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.124574:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.124574:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.124575:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.124577:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.124580:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.124583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.124590:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.124592:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.124593:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.124593:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.124595:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.124595:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.124727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:20.2:1619677448.124730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987eb50600] (20)++
00000800:00000200:2.0:1619677448.124733:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.124736:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:22.0:1619677448.124737:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987eb50600] (21)++
00000800:00000200:18.0:1619677448.124737:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.124738:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.124740:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.15@o2ib3
00000800:00000200:6.0:1619677448.124741:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987eb50600] (22)--
00000400:00000200:2.0:1619677448.124741:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.124743:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.1.15@o2ib3 : GET - for me
00000400:00000200:2.0:1619677448.124743:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.124744:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.124746:0:19936:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.1.15@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:2.0:1619677448.124746:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.124747:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.124748:0:19936:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.1.15@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000400:00000200:2.0:1619677448.124748:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:22.0:1619677448.124749:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987eb50600] (21)++
00000800:00000200:22.0:1619677448.124751:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987eb50600] (22)--
00000400:00000200:2.0:1619677448.124751:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.124753:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.1.15@o2ib3 local destination
00000800:00000200:2.0:1619677448.124753:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.124756:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.124757:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.1.15@o2ib3(10.51.1.15@o2ib3:10.51.1.15@o2ib3) : REPLY try# 0
00000800:00000200:2.0:1619677448.124757:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.124758:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:22.0:1619677448.124759:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.1.15@o2ib3
00000800:00000200:2.0:1619677448.124760:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.124760:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.124761:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:22.0:1619677448.124762:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919876b8f400] -> 10.51.1.15@o2ib3 (2) version: 12
00000800:00000200:22.0:1619677448.124762:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91987eb50600] (21)++
00000800:00000200:2.0:1619677448.124762:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:22.0:1619677448.124763:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987eb50600] (22)++
00000800:00000200:22.0:1619677448.124764:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91987eb50600] (23)--
00000800:00000200:22.0:1619677448.124765:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987eb50600] (22)--
00000800:00000200:20.2:1619677448.124770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987eb50600] (21)++
00000800:00000200:26.0:1619677448.124772:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.124774:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.124775:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:6.0:1619677448.124775:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987eb50600] (22)++
00000400:00000200:26.0:1619677448.124776:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.124777:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987eb50600] (23)--
00000400:00000200:26.0:1619677448.124778:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.124778:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.124779:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987eb50600] (22)--
00000400:00000200:6.0:1619677448.124780:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.1.15@o2ib3: REPLY: OK
00000800:00000200:26.0:1619677448.124782:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:6.0:1619677448.124782:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987eb50600] (21)--
00000800:00000200:20.2:1619677448.124965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.124972:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.124975:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.124978:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.124979:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.124981:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.124982:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.124983:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.124985:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677448.124985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:30.0:1619677448.124986:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.124987:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.124990:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.124992:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.124992:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.124995:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.124995:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:30.0:1619677448.124996:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.124996:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.124997:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.124998:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.124998:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677448.124998:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.125000:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:10.0:1619677448.125000:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.125001:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000400:00000200:10.0:1619677448.125001:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:30.0:1619677448.125002:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:10.0:1619677448.125003:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.125004:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.125005:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.125006:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:20.2:1619677448.125007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677448.125008:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.125010:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.125012:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:10.0:1619677448.125012:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.125012:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.125013:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:10.0:1619677448.125013:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:31.0:1619677448.125014:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.125014:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.125015:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:10.0:1619677448.125015:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.125015:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677448.125016:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.125016:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:10.0:1619677448.125016:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.125017:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.125018:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.125018:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677448.125019:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.125025:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.125027:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.125028:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.125029:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.125030:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.125031:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.125063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.125068:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.125071:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.125072:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.125074:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.125076:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.125077:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.125079:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.125080:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.125081:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.125082:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.125086:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.125088:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.125091:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.125092:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.125092:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.125094:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.125095:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.125095:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.125096:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.125107:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.125110:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.125111:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.125112:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.125114:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.125115:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.125128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.125134:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.125136:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.125137:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.125139:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.125142:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.125143:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.125145:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.125146:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.125147:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.125148:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.125151:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.125153:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.125155:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.125156:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.125156:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.125158:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.125158:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.125159:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.125160:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677448.125171:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.125174:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.125175:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.125175:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.125178:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.125180:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.125235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.125240:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.125242:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.125243:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.125245:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.125247:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.125249:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.125250:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.125251:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.125252:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.125253:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.125257:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.125259:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.125261:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.125262:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.125263:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.125265:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.125265:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.125266:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.125266:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.125271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.125278:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.125280:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.125281:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.125282:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.125284:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.125286:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.125341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.125347:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.125350:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.125350:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.125353:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.125355:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.125357:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.125358:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.125359:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.125360:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.125361:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.125365:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.125367:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.125370:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.125371:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.125371:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:21.2:1619677448.125372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677448.125373:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.125374:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.125374:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.125375:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.125380:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.125382:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.125385:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.125387:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.125387:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.125388:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:22.0:1619677448.125388:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:31.0:1619677448.125390:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.125390:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:31.0:1619677448.125391:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.125391:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.125391:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:31.0:1619677448.125392:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.125393:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.125394:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.125397:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.125399:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.125399:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677448.125402:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.125403:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.125404:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.125405:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.125406:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.125407:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.125408:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.125412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.125420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677448.125420:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.125423:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.125424:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.125424:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.125427:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.125428:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.125430:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677448.125431:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.2:1619677448.125431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677448.125431:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.125433:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.125435:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.125436:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.125438:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719014] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.125439:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677448.125439:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.125440:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.125441:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.125442:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:10.0:1619677448.125445:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.125446:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.125446:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.125447:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:30.0:1619677448.125448:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.125450:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:10.0:1619677448.125450:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.125451:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:30.0:1619677448.125452:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.125452:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:30.0:1619677448.125453:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.125453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:30.0:1619677448.125454:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.125454:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.125454:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677448.125455:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.125455:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.125456:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.125457:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:30.0:1619677448.125458:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.125459:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.125460:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:4.2:1619677448.125462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.125463:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.125463:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.125463:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:30.0:1619677448.125464:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.125464:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.125466:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:24.0:1619677448.125466:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.125467:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.125467:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:6.0:1619677448.125467:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:30.0:1619677448.125468:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:24.0:1619677448.125468:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.125469:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.125469:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:24.0:1619677448.125470:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:22.0:1619677448.125470:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.125471:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677448.125471:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.125472:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:24.0:1619677448.125472:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96719012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.125472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.125473:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.125473:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.125474:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.125476:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.125476:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:24.0:1619677448.125477:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.125480:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.125481:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:24.0:1619677448.125482:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.125484:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.125484:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.125484:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.125485:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.125485:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:20.0:1619677448.125486:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.125487:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.125487:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677448.125488:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.125489:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.125490:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.125490:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677448.125490:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677448.125491:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.125494:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.125494:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.125495:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.125496:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.125496:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.125496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:12.0:1619677448.125500:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.125502:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.125503:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.125503:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:12.0:1619677448.125505:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.125505:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.125506:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677448.125507:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.125507:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677448.125507:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.2:1619677448.125688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.125694:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.125697:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.125697:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.125700:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.125702:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.125704:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.125705:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.125706:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.125707:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.125708:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.125711:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.125713:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.125716:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.125717:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.125718:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.125719:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.125720:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.125721:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.125721:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.125733:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.125735:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.125736:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.125738:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.125739:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.125741:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.125753:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.125756:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.125757:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.125757:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.125758:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.125759:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.125874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:20.2:1619677448.125879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677448.125880:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.125883:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.125884:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.125886:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.125887:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:18.0:1619677448.125889:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.125890:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:18.0:1619677448.125890:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.125892:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.125893:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.125894:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:18.0:1619677448.125894:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.125895:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.125896:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.125896:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.125898:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.125900:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96719009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.125900:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.125901:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.125903:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.125903:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.125905:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.125906:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.125907:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:31.0:1619677448.125908:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.125908:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.125910:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.125911:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.125911:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.125911:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.125912:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:31.0:1619677448.125914:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.125915:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.125915:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.125917:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.125917:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.125918:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.125919:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677448.125924:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.125926:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.125927:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.125929:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.125929:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.2:1619677448.125930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:12.0:1619677448.125930:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.125931:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.125932:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.125932:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.125933:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677448.125934:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.125934:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677448.125935:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.125938:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.125941:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.125941:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.125943:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.125945:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.125946:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.125947:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.125949:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.125950:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.125953:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.125955:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.125958:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.125959:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.125960:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.125961:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.125962:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.125964:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.125965:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.125966:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.125971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.2:1619677448.125972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.2:1619677448.125973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677448.125974:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.125976:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.125977:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.125977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.0:1619677448.125978:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677448.125978:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:13.0:1619677448.125979:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.125980:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.125980:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.125981:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.0:1619677448.125982:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:17.0:1619677448.125982:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.125982:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.125983:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677448.125983:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:24.0:1619677448.125984:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677448.125984:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677448.125984:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.125986:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.125987:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677448.125987:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:1.0:1619677448.125987:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.125988:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.125988:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677448.125989:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.125990:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677448.125990:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.125991:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96719007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.125992:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.125993:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.125993:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.125994:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.125994:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.125995:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.125995:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.125996:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.125997:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677448.125997:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96719005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.125999:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.126000:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.126000:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.126001:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.126003:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.126007:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.126007:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.126010:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677448.126010:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.126012:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.126012:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.126013:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.126013:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.126013:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.126014:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677448.126014:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.126015:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.126016:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.126016:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677448.126016:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.126017:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.126017:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.126017:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:30.0:1619677448.126018:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:24.0:1619677448.126018:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.126018:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:30.0:1619677448.126019:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.126019:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.126020:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:24.0:1619677448.126020:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.126020:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677448.126022:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.126022:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677448.126023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.126026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:4.2:1619677448.126026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:12.0:1619677448.126027:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:21.2:1619677448.126028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.126030:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677448.126031:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.126032:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:12.0:1619677448.126032:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.126033:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677448.126033:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:13.0:1619677448.126034:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:1.0:1619677448.126034:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:17.0:1619677448.126035:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.126035:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677448.126035:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.126035:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:8.0:1619677448.126035:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:1.0:1619677448.126035:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:12.0:1619677448.126036:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.126036:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.126037:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:8.0:1619677448.126037:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.126038:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677448.126038:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:13.0:1619677448.126038:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.126038:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.126039:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.126040:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677448.126040:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.126041:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.126041:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677448.126043:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.126044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677448.126051:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677448.126054:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677448.126056:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.126057:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677448.126059:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.126060:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:12.0:1619677448.126061:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.126062:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.126063:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.126063:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.126064:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.126064:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:31.0:1619677448.126065:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:12.0:1619677448.126065:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.126065:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.126066:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677448.126068:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.126070:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677448.126072:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.126073:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.126074:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.126075:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.126076:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677448.126077:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677448.126077:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677448.126081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.126088:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.126090:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.126091:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.126091:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.126093:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.126096:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.126099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677448.126105:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.126108:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.126108:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.126108:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.126109:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.126110:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.126127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.126134:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.126137:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.126138:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.126140:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.126142:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.126144:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.126145:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.126146:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.126147:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.126148:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.126151:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.126152:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.126154:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.126155:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.126156:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.126157:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.126158:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.126159:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.126159:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677448.126169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677448.126170:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.126173:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.126174:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.126175:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.126176:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.126177:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.126178:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677448.126178:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.126179:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.126181:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.126183:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.126185:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.126186:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.126187:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.126188:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:10.0:1619677448.126188:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.126189:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.126190:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.126191:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.126192:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.126192:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:10.0:1619677448.126192:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.126193:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677448.126194:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.126196:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.126197:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.126197:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.126199:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.126199:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.126200:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.126200:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.2:1619677448.126200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.126206:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.126209:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.126209:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:12.0:1619677448.126211:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:1.0:1619677448.126211:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.126213:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:1.0:1619677448.126213:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.126214:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.126215:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:12.0:1619677448.126216:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.126217:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:1.0:1619677448.126217:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96719001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.126218:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.126219:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:1.0:1619677448.126219:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.126220:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.126224:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.126225:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.126228:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.126229:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.126229:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.126231:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.126231:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.126232:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.126232:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.126233:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.126235:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.126236:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.126237:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.126237:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.126239:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677448.126245:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.126247:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.126248:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.126249:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.126251:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.126253:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.126389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.126397:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.126400:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.126402:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.126405:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.126405:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.126406:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.126407:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96719000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.126409:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.126410:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.126411:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.126414:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.126416:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.126419:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.126419:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.126420:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.126421:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.126422:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.126423:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.126423:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.126435:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.126437:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.126438:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.126440:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.126442:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.126442:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.2:1619677448.126530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.126536:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.126539:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.126540:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.126542:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.126545:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.126546:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.126548:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.126549:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.126550:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.126551:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.126554:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.126556:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.126558:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.126559:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.126559:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.126560:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.126561:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.126562:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.126562:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.126567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.126575:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.126577:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.126578:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.126579:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.126580:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.126581:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.126607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.126613:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.126616:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.126617:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.126619:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.126621:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.126622:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.126624:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718998] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.126625:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.126626:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.126627:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.126631:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.126632:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.126635:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.126636:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.126637:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.126639:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.126639:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.126640:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.126640:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:26.0:1619677448.126651:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.126654:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.126655:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.126655:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.126658:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.126661:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:10.0:1619677448.126670:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.126673:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.126674:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.126674:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.126675:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.126676:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677448.126696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.126703:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.126705:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.126706:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.126708:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.126711:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.126712:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.126713:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.126715:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.126716:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.126717:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.126720:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.126722:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.126724:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.126725:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.126726:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.126728:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.126728:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.126728:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.126729:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677448.126739:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.126742:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.126742:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.126744:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.126746:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.126747:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.126775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.126783:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.126786:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.126788:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.126789:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.126790:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.126791:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.126793:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.126794:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.126795:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.126796:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.126799:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.126801:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.126803:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.126804:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.126804:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.126806:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.126806:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.126807:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.126807:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.126817:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.126819:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.126820:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.126821:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.126822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:12.0:1619677448.126823:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.126824:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.126830:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.126832:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.126833:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.126835:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.126837:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.126838:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.126839:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.126841:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.126841:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.126842:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.126845:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.126847:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.126849:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.126850:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.126851:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.126852:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.126852:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.126853:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.126854:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.126864:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.126867:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.126867:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.126869:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.126871:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.126872:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.126883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.126886:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.126888:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.126890:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.126892:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.126893:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.126895:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.126896:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.126898:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.126899:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.126900:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.126903:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.126905:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.126908:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.126909:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.126909:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.126911:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.126912:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.126913:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.126913:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677448.126915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.126922:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:12.0:1619677448.126923:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.126925:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.126926:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:12.0:1619677448.126926:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.126926:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.126928:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.126928:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.126930:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.126931:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677448.126931:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.126933:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.126934:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.126935:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.126936:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.126937:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.126940:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.126942:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.126945:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.126946:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.126947:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.126949:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.126949:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.126949:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.126950:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.126962:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.126965:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.126966:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.126966:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.126969:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.126971:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677448.126983:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.126986:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.126986:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.126987:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.126988:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.126988:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.127024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.127030:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.127032:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.127033:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.127035:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.127037:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.127038:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.127040:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.127041:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.127042:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.127043:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:5.2:1619677448.127043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:17.0:1619677448.127046:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.127047:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677448.127049:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677448.127050:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.127051:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.127051:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:2.0:1619677448.127052:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.127053:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677448.127053:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.127054:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.127054:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.127055:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:2.0:1619677448.127055:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.127058:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.127059:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:9.2:1619677448.127060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:2.0:1619677448.127060:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.127061:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.127062:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.127063:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.127066:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:2.0:1619677448.127066:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.127067:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.127069:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:2.0:1619677448.127069:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677448.127070:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.127070:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:2.0:1619677448.127070:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.127070:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:24.0:1619677448.127072:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.127072:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.127072:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.127073:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.127074:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:24.0:1619677448.127075:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.127078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.127085:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.127087:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.127088:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.127089:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.127090:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.127092:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.127099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.127106:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.127109:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.127109:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.127110:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.127111:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.127111:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.127235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.127242:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.127245:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.127246:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.127248:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.127251:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.127252:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.127254:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.127255:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.127256:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.127257:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:21.2:1619677448.127258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:26.0:1619677448.127260:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.127262:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.127264:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.127265:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.127266:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.127267:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.127267:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.127268:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.127269:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677448.127269:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.127270:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:21.2:1619677448.127270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:31.0:1619677448.127271:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.127271:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:31.0:1619677448.127273:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.127273:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.127274:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:6.0:1619677448.127275:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:31.0:1619677448.127276:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.127276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677448.127277:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.127278:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.127278:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.127278:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:31.0:1619677448.127279:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:13.0:1619677448.127280:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:6.0:1619677448.127281:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.127282:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.127283:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.127283:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:31.0:1619677448.127284:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.127284:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.127285:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.127285:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:31.0:1619677448.127286:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677448.127286:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.127286:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677448.127286:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:13.0:1619677448.127287:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:6.0:1619677448.127287:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.127288:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677448.127288:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.127289:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.127289:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.127290:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:31.0:1619677448.127290:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:6.0:1619677448.127291:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:31.0:1619677448.127292:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677448.127292:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.127299:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677448.127299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677448.127300:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:6.0:1619677448.127302:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.127304:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.127307:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.127308:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.127308:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:1.0:1619677448.127308:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:6.0:1619677448.127310:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.127311:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677448.127311:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.127312:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:6.0:1619677448.127312:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:1.0:1619677448.127312:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.127313:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.127314:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677448.127314:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.127318:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.127320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.127324:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:22.0:1619677448.127325:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:26.0:1619677448.127326:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:22.0:1619677448.127327:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:10.0:1619677448.127327:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.127328:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.127328:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.127329:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:26.0:1619677448.127330:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.127333:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677448.127378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.127385:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.127388:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.127388:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677448.127390:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.127393:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.127394:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677448.127395:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.127397:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.127397:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.127398:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.127401:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.127403:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:4.2:1619677448.127404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:10.0:1619677448.127406:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.127407:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.127408:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.127409:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.127410:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.127411:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:8.0:1619677448.127411:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677448.127412:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.127412:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:24.0:1619677448.127415:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.127415:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.127418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:8.0:1619677448.127419:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677448.127421:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.127423:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.127425:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:8.0:1619677448.127425:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.127426:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96718986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.127426:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:26.0:1619677448.127427:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.127427:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:8.0:1619677448.127427:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.127428:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:8.0:1619677448.127428:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.127428:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.127428:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.127429:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.127431:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:8.0:1619677448.127432:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.127432:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.127435:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.127436:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:8.0:1619677448.127440:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.127443:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.127444:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.0:1619677448.127444:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.127444:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.127446:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.127446:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677448.127447:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:8.0:1619677448.127447:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677448.127448:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:8.0:1619677448.127448:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677448.127449:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.127449:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.127451:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.127452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.127459:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.127462:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.127463:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.127463:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.127465:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.127468:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.127521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.127529:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.127532:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.127534:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.127535:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.127536:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.127538:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.127539:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.127541:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.127542:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.127543:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.127546:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.127548:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.127551:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.127551:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.127552:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:4.2:1619677448.127553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.127554:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.127554:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.127555:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.127556:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.127559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.127560:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.127563:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.127564:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.127565:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.127568:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.127568:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:10.0:1619677448.127569:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.127570:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.127571:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.127571:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:10.0:1619677448.127572:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.127572:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.127574:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:10.0:1619677448.127574:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.127574:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.127577:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.127578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:10.0:1619677448.127578:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.127580:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.127581:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.127583:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:10.0:1619677448.127583:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.127583:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.127584:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:20.0:1619677448.127586:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.127586:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.127586:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.127586:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.127586:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677448.127587:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:20.0:1619677448.127588:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.127590:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677448.127591:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.127592:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.127593:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.127594:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677448.127598:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.127599:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677448.127600:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.127602:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677448.127602:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.127602:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677448.127603:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677448.127603:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:6.0:1619677448.127603:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.127605:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000400:00000200:6.0:1619677448.127605:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.127606:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.127606:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.127607:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677448.127608:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677448.127611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.127619:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.127622:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.127623:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.127623:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.127625:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.127627:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.127674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:5.2:1619677448.127678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.127681:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.127683:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.127685:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.127686:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.127686:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.127688:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.127688:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.127689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:2.0:1619677448.127690:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677448.127691:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.127692:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.127693:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.127694:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677448.127696:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.127697:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:18.0:1619677448.127697:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:26.0:1619677448.127699:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:18.0:1619677448.127699:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.127699:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.127700:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:18.0:1619677448.127701:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.127701:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:18.0:1619677448.127702:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.127703:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.127704:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.127706:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.127707:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.127708:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.127709:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96718980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.127709:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.127711:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.127711:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.127711:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.127712:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.127712:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.127712:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677448.127712:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:26.0:1619677448.127713:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.127713:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.127715:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.127715:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.127717:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.127719:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.127719:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.127719:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.127719:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.127720:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (23)++
00000800:00000200:2.0:1619677448.127720:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:9.2:1619677448.127721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.127722:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.127722:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.127722:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (24)--
00000800:00000200:2.0:1619677448.127723:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677448.127724:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000800:00000200:18.0:1619677448.127724:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.127725:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (24)++
00000800:00000200:2.0:1619677448.127725:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:26.0:1619677448.127727:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (25)--
00000800:00000200:5.2:1619677448.127727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:26.0:1619677448.127728:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677448.127728:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (25)--
00000800:00000200:10.0:1619677448.127728:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:2.0:1619677448.127728:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.127729:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:18.0:1619677448.127729:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.127730:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.127730:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:2.0:1619677448.127730:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:26.0:1619677448.127731:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:18.0:1619677448.127731:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.127731:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.127731:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.127732:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677448.127733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.127733:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:10.0:1619677448.127734:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:10.0:1619677448.127736:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:18.0:1619677448.127737:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.127739:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.127741:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.127741:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.127742:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.127743:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.127746:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.127752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.127758:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.127760:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.127763:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.127763:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.127765:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.127767:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.127768:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.127770:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.127771:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.127772:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.127775:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.127777:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.127780:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.127781:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.127782:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.127784:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.127784:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.127785:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.127786:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:12.0:1619677448.127797:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.127799:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.127800:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.127802:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.127804:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.127805:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.127825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.127830:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.127833:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.127834:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.127835:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.127838:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.127840:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.127841:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718978] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.127842:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.127843:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.127844:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.127847:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.127849:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.127851:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.127852:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.127853:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.127854:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.127855:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.127855:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.127856:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.127860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.127868:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.127870:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.127871:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.127872:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.127874:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.127875:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:13.0:1619677448.127888:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.127891:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.127892:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.127893:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.127894:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.127895:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.128055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.128062:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.128064:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.128065:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.128067:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.128070:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.128072:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.128073:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.128074:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.128075:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.128076:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.128080:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.128081:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.128085:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.128086:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.128086:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.128088:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.128088:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.128090:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.128091:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.128092:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.128097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.128101:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.128104:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.128105:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.128105:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:6.0:1619677448.128105:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.128107:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677448.128107:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.128108:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.128109:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.128110:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.128148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:20.2:1619677448.128151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:26.0:1619677448.128155:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:5.0:1619677448.128156:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.128158:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677448.128158:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:5.0:1619677448.128159:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:26.0:1619677448.128161:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.128162:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.128162:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.128164:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.128165:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.128165:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.128167:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.128167:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677448.128168:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.128169:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.128170:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.128170:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.128171:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.128172:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:26.0:1619677448.128173:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.128178:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.128179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:5.0:1619677448.128179:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.128182:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.128183:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:26.0:1619677448.128184:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.128185:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.128186:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.128186:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.128186:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.128187:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.128187:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.128187:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:26.0:1619677448.128188:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.128189:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677448.128189:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:5.0:1619677448.128189:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.128190:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.0:1619677448.128190:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.128191:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.128191:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:31.0:1619677448.128193:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.128194:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.128195:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.128195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.2:1619677448.128195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:31.0:1619677448.128197:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.128198:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96718974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.128200:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.128200:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.128201:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.128202:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:13.0:1619677448.128202:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.128203:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:13.0:1619677448.128204:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.128205:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.128205:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:31.0:1619677448.128206:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.128206:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677448.128206:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.128206:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.128207:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.128207:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.128208:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.128209:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677448.128209:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.128209:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.128210:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.128212:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:1.0:1619677448.128212:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.128213:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.128214:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:9.2:1619677448.128214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000400:00000200:1.0:1619677448.128214:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:31.0:1619677448.128215:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:1.0:1619677448.128215:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.128216:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:1.0:1619677448.128216:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.128217:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677448.128217:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.128218:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:1.0:1619677448.128218:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.128220:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.128221:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677448.128221:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000400:00000200:1.0:1619677448.128221:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.128223:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.128224:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.128225:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.128225:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:21.2:1619677448.128226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.128226:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.128226:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:1.0:1619677448.128226:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.128227:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.128228:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:1.0:1619677448.128228:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.128230:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.128230:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.128231:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:1.0:1619677448.128231:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:26.0:1619677448.128232:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.128232:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.128232:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.128233:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.128234:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:10.0:1619677448.128234:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.128236:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.128237:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677448.128237:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:30.0:1619677448.128238:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677448.128238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.128242:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677448.128243:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677448.128244:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:17.0:1619677448.128245:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:12.0:1619677448.128246:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.128247:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.128247:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.128248:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677448.128248:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.128251:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677448.128263:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.128266:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.128267:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.128268:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.128268:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.128269:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.128493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.128500:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.128503:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.128505:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.128508:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.128508:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.128510:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.128511:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.128512:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.128513:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.128514:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.128518:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.128520:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.128523:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.128523:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.128524:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.128526:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.128527:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.128527:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.128529:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.128529:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.128533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.128536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.128544:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.128544:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.128546:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677448.128547:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.128547:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.128547:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.128548:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.128549:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.128550:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.128552:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.128564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.128572:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.128574:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.128575:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.128577:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.128579:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.128581:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.128582:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.128583:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.128584:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.128585:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.128589:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.128591:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.128594:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.128595:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.128595:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.128597:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.128597:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.128598:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.128598:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.128612:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.128615:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.128615:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.128616:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.128618:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.128621:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.128737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:20.2:1619677448.128737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.128743:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:2.0:1619677448.128744:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677448.128747:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:2.0:1619677448.128747:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.128748:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.0:1619677448.128749:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.128750:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.128751:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.128752:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.128754:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677448.128754:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.128756:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.128756:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.128757:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.128758:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.128759:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.128759:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.128760:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.128761:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677448.128762:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.128766:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000400:00000200:2.0:1619677448.128767:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.128769:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677448.128770:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.128772:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.128773:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:2.0:1619677448.128773:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.128774:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:2.0:1619677448.128774:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.128775:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.128776:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677448.128777:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.128777:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:2.0:1619677448.128777:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.128778:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:2.0:1619677448.128778:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.128779:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.128779:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.128783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.128790:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.128792:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:12.0:1619677448.128792:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677448.128793:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.128794:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.128794:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.128795:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.128796:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000400:00000200:12.0:1619677448.128797:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.128799:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677448.128799:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.128800:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.128977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.128983:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.128986:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.128987:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.128989:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.128991:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.128993:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.128994:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.128996:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.128997:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.128999:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.129002:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.129004:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.129007:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.129007:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.129008:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.129010:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.129010:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.129011:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.129011:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.129016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.129024:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.129027:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.129028:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.129028:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.129030:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.129033:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.129034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.129041:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.129044:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.129045:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.129045:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.129046:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.129047:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.129070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.129076:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.129079:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.129079:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.129082:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.129083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:6.0:1619677448.129084:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.129085:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.129087:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.129088:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.129089:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.129090:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677448.129090:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.129093:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.129094:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.129095:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.129096:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.129097:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.129099:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.129100:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.129101:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:6.0:1619677448.129101:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:10.0:1619677448.129102:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.129102:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.129103:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:10.0:1619677448.129104:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.129104:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677448.129105:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.129105:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.129105:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677448.129106:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.129109:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.129111:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.129114:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.129115:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.129116:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:10.0:1619677448.129116:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.129117:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.129118:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.129119:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:10.0:1619677448.129119:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.129119:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:31.0:1619677448.129120:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.129122:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.129123:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:8.2:1619677448.129123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.129125:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.0:1619677448.129131:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.129133:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.129135:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.129136:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.129137:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.129140:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.129210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.129216:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.129219:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.129219:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.129222:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.129225:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.129226:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.129228:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.129229:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.129230:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.129231:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.129234:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.129236:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.129239:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.129239:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.129240:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.129242:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.129242:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.129243:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.129243:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.129248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.129255:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.129258:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.129259:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.129259:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.129261:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.129263:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.129281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.129286:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.129289:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.129290:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.129292:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.129298:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.129299:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.129301:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.129302:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.129303:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.129304:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.129307:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.129309:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.129312:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.129312:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.129313:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.129315:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.129315:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.129315:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.129316:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677448.129329:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.129332:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.129333:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.129335:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.129336:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.129338:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.129520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.129528:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.129530:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:21.2:1619677448.129531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677448.129531:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.129533:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.129537:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.129538:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:30.0:1619677448.129539:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677448.129539:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.129540:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.129541:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.129541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677448.129542:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677448.129543:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.129544:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.129545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677448.129547:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.129547:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.129547:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.129548:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:10.0:1619677448.129549:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:30.0:1619677448.129550:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.129551:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.129551:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:30.0:1619677448.129552:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.129552:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677448.129553:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.129553:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.129554:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:26.0:1619677448.129554:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.129555:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:17.0:1619677448.129555:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.129555:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:26.0:1619677448.129556:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:10.0:1619677448.129556:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.129558:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:30.0:1619677448.129559:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.129559:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:1.0:1619677448.129559:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.129560:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96718961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.129561:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.129562:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:10.0:1619677448.129562:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.129563:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.129563:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.129564:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:1.0:1619677448.129564:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96718960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.129565:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.129565:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677448.129565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.129566:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:26.0:1619677448.129566:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677448.129566:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.129567:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:26.0:1619677448.129567:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:1.0:1619677448.129567:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.129568:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:26.0:1619677448.129568:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:1.0:1619677448.129569:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.129571:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.129573:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.129573:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:1.0:1619677448.129573:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.129574:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.129574:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.129574:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:9.2:1619677448.129574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:1.0:1619677448.129575:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.129576:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:10.0:1619677448.129577:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.129577:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.129578:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:1.0:1619677448.129578:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.129579:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.129579:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:5.0:1619677448.129579:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.129579:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.129580:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:1.0:1619677448.129580:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.129581:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:18.0:1619677448.129582:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:10.0:1619677448.129582:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677448.129582:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677448.129582:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.129583:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677448.129583:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.129583:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.129584:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.2:1619677448.129584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677448.129585:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.129586:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.129586:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.129589:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.129589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.129592:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.129592:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:31.0:1619677448.129594:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:20.0:1619677448.129595:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.129596:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:20.0:1619677448.129596:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.129596:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:17.0:1619677448.129596:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:31.0:1619677448.129597:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.129597:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.129598:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677448.129598:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:31.0:1619677448.129599:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.129599:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.129600:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677448.129601:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:17.0:1619677448.129601:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.129603:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:17.0:1619677448.129604:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.129609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.129615:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.129618:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.129619:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.129619:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.129620:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.129621:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.129675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.129683:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.129686:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.129688:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.129689:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.129691:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.129692:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.129694:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.129695:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.129697:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.129697:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.129701:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.129703:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.129705:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.129706:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.129707:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.129708:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.129709:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.129710:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.129710:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677448.129720:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.129723:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.129723:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.129725:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.129726:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.129727:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.129739:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.129741:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.129742:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.129743:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.129743:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.129745:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.129796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.129801:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.129804:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.129805:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.129807:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.129808:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.129810:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.129811:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.129813:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.129814:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.129815:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.129818:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.129819:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.129822:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.129823:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.129824:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.129825:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.129826:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.129826:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.129827:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.129838:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.129841:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.129841:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.129843:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.129844:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.129846:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.129869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.129875:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.129878:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.129879:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.129880:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.129883:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.129885:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.129886:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.129887:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.129888:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.129889:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.129893:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.129894:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.129896:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.129897:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.129898:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.129899:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.129900:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.129901:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.129902:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.129902:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677448.129905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.129908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.129913:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.129914:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.129915:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.129915:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.129916:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.129916:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.129917:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.129918:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.129918:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.129919:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677448.129991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.129997:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.129999:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.130001:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.130002:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.130004:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.130005:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.130007:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.130008:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.130009:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.130010:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.130014:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.130016:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.130019:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.130020:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.130021:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.130022:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.130023:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.130024:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.130026:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.130026:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.130031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.130034:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:30.0:1619677448.130035:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.130036:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.130037:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.130037:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.130038:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.130039:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.130041:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.130042:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.130044:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.130055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.130062:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.130065:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.130066:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.130068:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.130070:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.130071:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.130072:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.130074:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.130075:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.130076:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.130079:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.130081:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.130083:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.130084:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.130085:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.130086:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.130087:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.130088:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.130089:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.130090:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.130095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.130098:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.130101:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.130101:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.130102:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677448.130103:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.130104:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.130104:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.130105:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.130106:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677448.130106:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.130115:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.130118:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.130119:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.130120:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.130120:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.130122:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.130125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.130133:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.130135:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.130137:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.130139:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.130140:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.130141:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.130142:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.130143:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.130144:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.130145:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.130148:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.130150:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.130153:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.130154:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.130154:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.130156:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.130157:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.130157:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.130158:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.130162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.130169:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.130172:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.130173:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.130174:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.130175:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.130177:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.130184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.130190:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.130193:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.130193:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.130195:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.130198:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.130199:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.130201:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.130202:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.130203:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.130204:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.130208:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.130210:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.130213:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.130214:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.130214:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.130216:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.130217:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.130217:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.130218:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.130222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.130229:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.130231:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677448.130232:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677448.130233:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.130235:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.130238:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.2:1619677448.130310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.130318:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.130320:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.130321:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.130323:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.130325:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.130327:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.130328:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.130329:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677448.130329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:26.0:1619677448.130330:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.130331:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.130334:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.130336:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677448.130337:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677448.130338:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.130339:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:8.0:1619677448.130339:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:26.0:1619677448.130340:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:24.0:1619677448.130340:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677448.130341:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:8.0:1619677448.130341:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.130342:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.130343:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.130343:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:8.0:1619677448.130344:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.130346:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.130347:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96718951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.130348:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.130349:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.130350:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.130352:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.130354:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677448.130356:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:8.0:1619677448.130357:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.130358:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:8.0:1619677448.130358:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.130358:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:30.0:1619677448.130359:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.130360:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.130360:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.130361:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.130361:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.130362:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:30.0:1619677448.130363:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.130364:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:8.2:1619677448.130366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:21.2:1619677448.130370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:17.0:1619677448.130374:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:18.0:1619677448.130376:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677448.130377:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.130378:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.130379:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:1.0:1619677448.130379:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.130380:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.130380:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.130382:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.130383:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.130384:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:18.0:1619677448.130385:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.130386:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.130387:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.130388:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.130389:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.130392:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.130394:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.130397:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.130397:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.130398:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.130400:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.130400:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.130401:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.130402:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677448.130413:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.130416:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.130417:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.130419:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.130420:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.130422:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677448.130434:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.130436:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.130437:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.130438:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.130438:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.130440:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.130442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.130449:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.130452:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.130453:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.130455:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.130457:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.130458:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.130459:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718949] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.130459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677448.130461:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.130462:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.130463:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.130465:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.130465:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.130467:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.130468:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.130469:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.130470:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.130471:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.130471:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:1.0:1619677448.130471:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.130473:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.130473:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.130474:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677448.130474:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.130475:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:1.0:1619677448.130475:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.130476:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.130478:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.130478:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.130480:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.130483:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.130485:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:1.0:1619677448.130485:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.130486:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.130487:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:1.0:1619677448.130487:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:30.0:1619677448.130488:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.130488:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.130489:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:30.0:1619677448.130490:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.130490:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.130491:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.130491:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:30.0:1619677448.130492:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:1.0:1619677448.130492:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.130493:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.130495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.130498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.130504:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.130505:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.130505:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.130505:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:17.0:1619677448.130506:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.130506:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.130507:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.130508:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.130508:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677448.130510:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.130605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.130613:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.130616:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.130618:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.130620:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.130621:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.130623:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.130624:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.130625:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.130626:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.130627:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.130630:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.130632:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.130635:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.130636:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.130637:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.130638:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.130639:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.130639:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.130640:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:21.2:1619677448.130648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.130655:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.2:1619677448.130655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.130657:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.130659:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.130661:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.130661:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.130661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:30.0:1619677448.130663:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.130664:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.130664:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:30.0:1619677448.130665:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.130666:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.130666:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.2:1619677448.130666:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919065782400] (20)++
00000400:00000200:30.0:1619677448.130667:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.130668:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:5.0:1619677448.130670:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.130671:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.130671:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.130672:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.130672:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (21)++
00000400:00000200:5.0:1619677448.130673:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.130674:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.130675:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.130675:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:18.0:1619677448.130676:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000400:00000200:20.0:1619677448.130677:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.130678:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.130679:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718945] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.130680:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.130681:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.130681:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.130681:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.130681:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.130682:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.130682:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:30.0:1619677448.130683:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.130683:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677448.130683:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.130684:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.0:1619677448.130684:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:2.0:1619677448.130684:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.130685:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677448.130686:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:2.0:1619677448.130686:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.130687:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:2.0:1619677448.130687:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.130688:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.130688:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:20.0:1619677448.130691:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.130692:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.130694:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.130694:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:12.0:1619677448.130694:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.130694:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677448.130695:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.130696:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.130697:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:12.0:1619677448.130697:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.130697:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.130697:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677448.130698:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:12.0:1619677448.130698:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.130699:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.130699:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:12.0:1619677448.130699:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.130699:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.130699:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (21)++
00000800:00000200:13.0:1619677448.130700:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.130700:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (22)--
00000800:00000200:2.0:1619677448.130701:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (21)--
00000800:00000200:12.0:1619677448.130702:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677448.130705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:26.0:1619677448.130710:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:18.0:1619677448.130712:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:26.0:1619677448.130713:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.130713:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.130714:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.130715:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.130715:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.130716:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.130716:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.130718:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.130719:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:26.0:1619677448.130720:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.130730:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.130733:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.130734:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.130734:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.130735:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.130736:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.130795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.130800:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.130803:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.130804:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.130806:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.130809:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.130809:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.130811:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.130812:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.130813:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.130814:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.130817:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.130819:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.130820:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.130821:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.130821:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.130822:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.130823:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.130824:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.130824:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.130830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.130831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677448.130837:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677448.130837:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.130839:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.130839:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.130840:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677448.130840:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.130841:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677448.130842:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000400:00000200:1.0:1619677448.130843:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.130844:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:1.0:1619677448.130845:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.130846:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.130848:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.130849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:1.0:1619677448.130849:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.130850:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.130851:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.130854:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.130855:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:2.0:1619677448.130856:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.130857:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.130857:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.130858:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:18.0:1619677448.130859:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.130859:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.130859:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:2.0:1619677448.130860:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677448.130860:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.130861:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677448.130861:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.130862:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677448.130862:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.130866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677448.130869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677448.130873:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.130875:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.130875:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.130876:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.130876:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.130877:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.130879:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:17.0:1619677448.130879:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:10.0:1619677448.130879:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.130881:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.130884:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.130885:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.130886:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.130887:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.130888:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.130890:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.2:1619677448.130891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:26.0:1619677448.130894:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.130896:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.2:1619677448.130897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:26.0:1619677448.130899:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.130899:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.130900:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.130900:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.130901:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:10.0:1619677448.130902:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.130903:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.130903:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.130904:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.130905:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.130905:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:10.0:1619677448.130905:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.130905:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:26.0:1619677448.130906:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:20.0:1619677448.130908:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:10.0:1619677448.130908:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677448.130908:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.130909:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.130911:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.130911:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.130911:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.130912:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677448.130913:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.130914:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.130915:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:5.0:1619677448.130915:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.130916:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.130917:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.130918:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.130919:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.130921:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.130922:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:5.0:1619677448.130923:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.130924:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.130926:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.130927:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677448.130927:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.130930:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.130931:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.130931:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.130932:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.130932:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.130933:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.130935:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.130935:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.130936:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:5.0:1619677448.130936:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.130937:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:5.0:1619677448.130937:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677448.130938:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:5.0:1619677448.130938:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.130947:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.130950:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.130951:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.130951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:30.0:1619677448.130953:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.130954:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.130955:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.0:1619677448.130959:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.130961:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.130962:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.130963:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.130964:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.130967:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.131054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.131062:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.131065:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.131067:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.131069:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.131069:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.131071:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.131072:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.131074:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.131075:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.131076:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.131080:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.131081:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.131084:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.131085:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.131086:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.131088:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.131088:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.131090:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.131091:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.131092:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677448.131097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.2:1619677448.131099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.2:1619677448.131104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.131104:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.131105:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.131107:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:1.0:1619677448.131107:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:30.0:1619677448.131108:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677448.131108:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.131109:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:1.0:1619677448.131110:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.131111:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:1.0:1619677448.131111:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.131113:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:5.0:1619677448.131114:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:1.0:1619677448.131114:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.131115:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.131115:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.131116:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.131117:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.131117:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.131118:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:5.0:1619677448.131119:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.131122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000400:00000200:1.0:1619677448.131122:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.131123:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.131126:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.131127:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.131128:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.131129:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.131130:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.131130:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.131131:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.131131:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677448.131132:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.131132:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.131133:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.131134:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.131135:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677448.131149:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.131151:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.131152:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.131154:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.131155:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.131157:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.131215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.131221:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.131224:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.131224:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.131227:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.131229:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.131230:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.131231:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.131233:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.131234:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.131235:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.131238:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.131240:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.131244:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.131244:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.131245:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.131247:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.131247:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.131248:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.131248:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677448.131259:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.131262:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.131263:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.131264:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.131266:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.131267:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.131286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.131293:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.131300:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677448.131302:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.131303:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.131305:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.131306:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.131308:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.131310:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.131311:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.131312:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.131315:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.131317:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.131319:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.131320:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.131321:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.131322:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.131323:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.131323:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.131324:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677448.131333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:13.0:1619677448.131336:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.131338:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.131339:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.131339:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:13.0:1619677448.131340:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.131342:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.131342:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.131343:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:12.0:1619677448.131343:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677448.131344:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.131348:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.131349:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.131351:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.131352:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.131353:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.131354:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.131357:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.131358:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.131361:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.131362:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.131363:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.131364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:2.0:1619677448.131365:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.131365:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.131366:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.131367:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:22.0:1619677448.131371:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:4.2:1619677448.131373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677448.131374:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.131375:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.131377:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.131379:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:22.0:1619677448.131380:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.131381:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.131382:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.131382:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:10.0:1619677448.131382:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677448.131383:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.131384:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.131384:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:22.0:1619677448.131385:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.131386:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.131388:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:22.0:1619677448.131388:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:10.0:1619677448.131388:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.131390:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.131390:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.131391:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.131393:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:10.0:1619677448.131395:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.131397:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.131397:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:22.0:1619677448.131398:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.131398:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:10.0:1619677448.131399:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.131400:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.131400:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.131401:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.131401:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677448.131401:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.131402:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677448.131402:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.131405:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.131407:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.131411:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.131412:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.131413:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.131414:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.131415:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.131415:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.131415:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:26.0:1619677448.131416:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.131417:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.131417:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:26.0:1619677448.131418:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.131418:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.131423:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:26.0:1619677448.131424:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.131425:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.131426:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.131426:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.131427:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.131429:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.131656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.131662:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.131665:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.131666:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.131668:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.131671:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.131672:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.131674:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.131675:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.131676:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.131677:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.131681:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.131682:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.131685:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.131686:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.131686:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.131688:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.131688:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.131689:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.131689:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.131701:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.131704:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.131705:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.131705:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.131707:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.131710:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.131745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.131753:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.131756:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.131758:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.131760:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.131760:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.131762:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.131763:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.131764:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.131765:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.131766:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.131770:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.131771:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.131773:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.131774:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.131775:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.131777:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.131777:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.131779:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.131781:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.131781:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677448.131787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.131789:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.131791:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.131792:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.131793:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.131794:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677448.131795:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.131796:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.131797:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.131797:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.131798:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.131813:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.131815:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.131816:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.131817:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.131817:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.131818:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.132092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.132099:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.132102:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.132102:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.132104:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.132106:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.132108:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.132109:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.132110:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.132111:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.132112:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.132114:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.132116:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.132118:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.132119:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.132120:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.132121:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.2:1619677448.132122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:10.0:1619677448.132122:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.132123:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.132124:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.132124:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.132130:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:5.2:1619677448.132130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.132133:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:30.0:1619677448.132135:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.132136:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.132137:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677448.132138:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:26.0:1619677448.132138:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:30.0:1619677448.132139:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.132140:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.132140:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.132140:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:30.0:1619677448.132141:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.132141:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:30.0:1619677448.132142:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.132144:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:30.0:1619677448.132145:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.132146:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.132146:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.132147:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.132148:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.132149:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.132150:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:12.0:1619677448.132150:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.132151:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:13.0:1619677448.132151:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.132153:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.132153:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677448.132154:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677448.132155:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:8.2:1619677448.132159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.132168:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.132170:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.132171:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.132173:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.132173:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.132176:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.132179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.132186:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.132189:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.132190:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.132190:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.132191:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.132192:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.132208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.132215:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.132218:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.132218:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.132220:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.132223:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.132224:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.132226:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.132227:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.132228:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.132229:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.132232:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.132233:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.132236:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.132237:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.132237:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.132239:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.132239:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.132240:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.132242:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.132242:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.2:1619677448.132245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:4.2:1619677448.132248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.132251:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.132253:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.132253:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.132254:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.132255:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.132255:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.132256:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:1.0:1619677448.132256:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.132257:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677448.132257:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.132258:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677448.132258:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.132259:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.132259:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.132259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:1.0:1619677448.132260:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.132261:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.132263:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:1.0:1619677448.132263:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.132264:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:1.0:1619677448.132264:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.132265:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677448.132265:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:22.0:1619677448.132266:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:1.0:1619677448.132269:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.132270:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.132272:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:22.0:1619677448.132273:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.132275:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:1.0:1619677448.132275:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.132276:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000400:00000200:22.0:1619677448.132277:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.132277:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:22.0:1619677448.132278:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.132279:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.132279:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.132279:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677448.132280:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:1.0:1619677448.132280:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.132281:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.132282:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677448.132283:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.132285:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.2:1619677448.132285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.132287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.132288:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.132289:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.132289:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.132291:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:24.0:1619677448.132292:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:22.0:1619677448.132292:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.132292:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677448.132293:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:9.2:1619677448.132298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677448.132298:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:1.0:1619677448.132298:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677448.132300:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.132300:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677448.132300:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.132301:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677448.132302:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.132303:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.132305:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.132306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677448.132307:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:24.0:1619677448.132308:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:18.0:1619677448.132309:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.132310:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.132310:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.132311:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:18.0:1619677448.132312:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.132313:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.132314:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.132315:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677448.132316:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.132333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.132338:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.132341:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.132342:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.132344:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.132346:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.132347:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.132348:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.132350:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.132351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:2.0:1619677448.132351:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.132352:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.132355:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.132356:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.132358:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.132359:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.132359:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.132360:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.132361:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.132362:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:2.0:1619677448.132362:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.132362:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.132363:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:31.0:1619677448.132364:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.132367:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.132367:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677448.132367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677448.132369:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677448.132370:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718923] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.132371:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.132372:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.132373:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:18.0:1619677448.132375:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:31.0:1619677448.132376:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.132376:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.132377:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.132378:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677448.132378:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.132379:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.132381:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.132381:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677448.132382:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.132382:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.132384:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.132384:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.132385:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.132386:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.132389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.132395:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:4.2:1619677448.132397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.132398:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.132399:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.132400:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.132401:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:8.0:1619677448.132401:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.132402:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.132404:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:24.0:1619677448.132405:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.132405:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.132408:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.132409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000400:00000200:24.0:1619677448.132410:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.132411:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.132413:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.132414:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.132415:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.132416:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.132417:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677448.132418:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000400:00000200:24.0:1619677448.132419:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.132419:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.132420:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.132421:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.132421:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677448.132424:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.132425:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.132426:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.132427:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.132428:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.132429:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.132430:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.132430:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.132433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.132436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.132442:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677448.132443:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.132444:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.132445:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.132445:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677448.132445:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.132446:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.132447:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.132447:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.132451:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.132731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.132739:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.132742:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.132743:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.132744:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.132748:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.132749:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.132750:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.132751:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.132752:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.132753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:10.0:1619677448.132753:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.132755:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.132757:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.132758:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.132759:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.132759:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.132759:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.132760:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:22.0:1619677448.132761:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:10.0:1619677448.132761:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.132762:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.132762:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677448.132762:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.132764:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.132767:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.132768:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.132769:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.132770:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.132771:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.132772:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:22.0:1619677448.132772:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.132774:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.132775:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.132775:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:22.0:1619677448.132776:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.132777:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.132778:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.132779:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:22.0:1619677448.132781:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.132782:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.132782:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.132784:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.132785:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.132786:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.132786:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.132797:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.132799:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.132800:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.132800:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.132802:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.132805:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:26.0:1619677448.132825:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.132827:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.132828:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.132829:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.132829:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.132830:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.132866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.132872:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.132875:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.132875:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.132877:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.132880:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.132882:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.132883:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.132885:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.132886:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.132887:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.132890:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.132892:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.132895:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.132896:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.132897:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.132898:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.132899:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.132900:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.132900:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677448.132903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:9.2:1619677448.132904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:21.2:1619677448.132905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677448.132906:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.132909:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:24.0:1619677448.132911:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:20.0:1619677448.132911:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.132913:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677448.132913:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:5.0:1619677448.132913:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.132914:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.132914:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.132915:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.132915:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677448.132916:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:24.0:1619677448.132917:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.132917:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.132918:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.132919:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.132920:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677448.132920:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:20.0:1619677448.132920:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.132921:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.132921:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.2:1619677448.132923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:30.0:1619677448.132924:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677448.132925:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.132926:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.132927:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.132928:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.132928:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:30.0:1619677448.132929:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.132929:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.132932:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.132932:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.132932:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:20.0:1619677448.132933:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:30.0:1619677448.132934:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.132934:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.132936:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.132936:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.132936:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:2.0:1619677448.132936:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.132937:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.132937:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:2.0:1619677448.132937:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.132939:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:30.0:1619677448.132940:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.132940:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:2.0:1619677448.132940:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96718916] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.132941:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677448.132941:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.132942:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.132943:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.132943:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:2.0:1619677448.132943:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.132944:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.132944:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:2.0:1619677448.132946:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.132948:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.2:1619677448.132949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.132949:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677448.132950:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.132951:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.132951:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677448.132952:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.132952:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.132953:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.132953:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.132953:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:6.0:1619677448.132955:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.132955:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.132956:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.132956:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677448.132957:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677448.132958:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677448.132959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.132960:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.132961:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.2:1619677448.132961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000400:00000200:1.0:1619677448.132961:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.132963:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.132966:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677448.132967:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.132968:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:8.2:1619677448.132968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.132968:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:18.0:1619677448.132969:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.132969:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.132970:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:22.0:1619677448.132971:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.132971:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.132972:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.132972:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677448.132973:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677448.132974:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.132975:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.132975:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.132976:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677448.132976:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.132977:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677448.132979:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.132980:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.132980:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.132981:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.132981:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.132988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.132996:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.132998:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.132999:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.133001:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.133003:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.133005:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.133006:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.133007:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.133008:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.133009:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.133012:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.133014:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.133017:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.133018:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.133018:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.133020:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.133020:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.133021:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.133022:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.133033:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.133036:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.133037:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.133037:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.133039:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.133042:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.133203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.133208:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.133211:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.133212:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.133214:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.133216:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.133217:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.133219:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.133220:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.133221:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.133222:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.133225:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.133227:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.133229:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.133230:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.133230:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.133232:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.133232:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.133233:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.133234:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.133234:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.2:1619677448.133240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677448.133245:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677448.133245:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.133247:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:22.0:1619677448.133248:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:18.0:1619677448.133248:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:22.0:1619677448.133249:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.133249:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677448.133249:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.133251:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.133255:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.133293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.133303:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:21.2:1619677448.133304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:31.0:1619677448.133306:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.133308:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.133310:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:6.0:1619677448.133310:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:31.0:1619677448.133311:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.133312:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:6.0:1619677448.133313:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:31.0:1619677448.133314:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.133314:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:31.0:1619677448.133316:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.133317:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.133317:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.133318:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:6.0:1619677448.133319:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.133321:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.133323:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.133323:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.133324:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.133325:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.133326:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:6.0:1619677448.133327:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.133329:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.133330:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:6.0:1619677448.133330:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.133331:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.133332:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677448.133332:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.133333:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.133334:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.133334:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:6.0:1619677448.133335:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.133336:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.133337:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.133339:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.133339:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.133340:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.133341:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:21.2:1619677448.133345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:12.0:1619677448.133346:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.2:1619677448.133346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677448.133349:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.133350:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.133352:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.133353:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.133354:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.133354:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.133356:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.0:1619677448.133356:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:30.0:1619677448.133358:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.133358:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677448.133359:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:5.0:1619677448.133359:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.133360:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.133361:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.133362:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:20.0:1619677448.133362:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:30.0:1619677448.133363:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.133363:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:30.0:1619677448.133365:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.133366:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.133367:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677448.133371:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.133372:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677448.133375:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.133376:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:30.0:1619677448.133377:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:30.0:1619677448.133378:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:30.0:1619677448.133379:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.133380:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.133380:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.133384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.133390:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.133393:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.133394:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.133394:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.133396:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.133398:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.133427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.133434:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.133437:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.133438:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.133440:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.133443:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.133444:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.133445:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.133447:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.133448:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.133449:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.133452:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.133453:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.133457:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.133458:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.133458:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.133460:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.133460:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.133461:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.133462:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677448.133470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677448.133474:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.133477:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.133478:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.133478:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.133478:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.133480:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.133481:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.133481:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:22.0:1619677448.133483:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:10.0:1619677448.133483:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.133486:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.133488:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.133489:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.133490:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.133491:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.133492:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.133495:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.133497:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.133499:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.133500:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.133500:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.133502:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.133502:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.133503:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.133504:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.133525:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.133528:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.133528:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.133530:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.133531:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.133532:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.133625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.133628:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.133630:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.133632:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.133634:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.133635:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.133637:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.133638:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718908] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.133639:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.133640:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.133641:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.133645:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.133647:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.133649:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.133650:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.133651:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.133652:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.133653:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.133654:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.133655:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.133656:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:9.2:1619677448.133659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677448.133661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.133667:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:2.0:1619677448.133667:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677448.133669:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.133670:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:2.0:1619677448.133670:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.133671:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.133671:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:2.0:1619677448.133671:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.133673:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.133676:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.133689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.133696:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.133699:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.133700:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.133702:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.133704:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.133706:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.133708:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.133709:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.133710:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.133711:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677448.133714:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.133716:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.133718:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.133719:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.133720:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.133721:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.133722:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.133723:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.133723:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.133727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.133736:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.133738:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.133739:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.133741:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.133741:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.133744:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.133759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677448.133767:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677448.133770:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677448.133772:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.133773:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677448.133774:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.133776:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677448.133777:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.133778:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.133779:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.133780:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677448.133783:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.133783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677448.133785:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.133786:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.133788:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.133788:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.133788:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:31.0:1619677448.133789:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.133791:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:20.0:1619677448.133791:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.133791:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677448.133792:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677448.133793:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677448.133793:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000400:00000200:20.0:1619677448.133793:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.133795:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.133796:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.133797:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677448.133797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677448.133798:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.133799:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.133802:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.133804:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.133805:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677448.133807:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.133808:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.133808:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.133808:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.133809:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677448.133809:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.133811:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.133811:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:18.0:1619677448.133811:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.133812:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.133812:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677448.133814:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.133822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.133829:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.133831:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.133833:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.133833:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:12.0:1619677448.133834:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.133834:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.133835:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677448.133836:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.133837:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.133839:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.133840:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.133842:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.133859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.133866:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.133869:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.133870:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.133872:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.133874:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.133876:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.133877:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.133878:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.133879:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.133880:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.133883:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.133885:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.133888:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.133888:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.133889:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.133891:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.133891:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.133892:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.133892:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.133897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.133905:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.133908:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.133909:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.133909:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.133911:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.133914:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.134052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.2:1619677448.134053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:6.0:1619677448.134058:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:30.0:1619677448.134060:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677448.134061:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677448.134062:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:30.0:1619677448.134063:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:6.0:1619677448.134065:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.134066:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.134066:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.134067:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.134069:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.134069:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:30.0:1619677448.134070:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:6.0:1619677448.134070:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.134072:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.134072:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.134073:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.134074:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.134074:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677448.134076:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.134078:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:6.0:1619677448.134078:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.134082:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.134082:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677448.134084:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.134085:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.134085:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:6.0:1619677448.134086:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677448.134087:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.134087:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677448.134088:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:6.0:1619677448.134088:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677448.134088:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:30.0:1619677448.134089:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.134089:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:30.0:1619677448.134090:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.134091:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.134092:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:8.2:1619677448.134093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677448.134094:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.134094:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677448.134100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.134101:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.134103:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.134104:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.134104:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:4.2:1619677448.134105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:17.0:1619677448.134106:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.134108:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:17.0:1619677448.134108:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:31.0:1619677448.134110:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.134111:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.134112:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.2:1619677448.134112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:13.0:1619677448.134112:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.2:1619677448.134113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:31.0:1619677448.134114:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.134115:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:13.0:1619677448.134115:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677448.134116:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:13.0:1619677448.134116:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.134116:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:13.0:1619677448.134117:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.134119:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:26.0:1619677448.134119:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.134119:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.134120:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677448.134121:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.134121:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.134121:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677448.134123:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.134123:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677448.134123:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:17.0:1619677448.134124:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.134124:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677448.134125:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.134125:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.134125:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000400:00000200:31.0:1619677448.134126:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.134126:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.134126:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.134126:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.134127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:31.0:1619677448.134128:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.134128:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.134129:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.134129:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.134130:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.134133:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.134133:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.134134:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.134135:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677448.134136:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.134136:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.134136:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.134138:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.134139:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.134139:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.134139:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.134140:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.134140:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.134140:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:17.0:1619677448.134140:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.134142:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.134142:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:17.0:1619677448.134142:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:31.0:1619677448.134143:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.134143:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677448.134143:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96718899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.134144:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677448.134144:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.134144:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.134145:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:17.0:1619677448.134145:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.134146:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.134146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:17.0:1619677448.134147:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.134150:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.134152:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.134153:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677448.134155:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677448.134155:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.134156:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:10.0:1619677448.134156:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.134157:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.134157:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:22.0:1619677448.134158:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.134159:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.134159:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)++
00000400:00000200:10.0:1619677448.134159:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.134159:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:17.0:1619677448.134160:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:17.0:1619677448.134160:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.134161:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.134161:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.134162:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.134162:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.134162:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677448.134163:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.134163:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:12.0:1619677448.134165:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.134165:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96718898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.134166:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000400:00000200:10.0:1619677448.134166:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.134167:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677448.134167:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:10.0:1619677448.134167:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.134168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:13.0:1619677448.134168:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.134168:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:10.0:1619677448.134168:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.134170:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:13.0:1619677448.134170:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.134170:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.134171:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677448.134172:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677448.134172:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:10.0:1619677448.134172:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.134173:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.134174:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.134175:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.134176:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.134178:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.134178:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.134179:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:6.0:1619677448.134179:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:26.0:1619677448.134180:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677448.134180:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.134182:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677448.134182:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.134183:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.134183:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.134183:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.134183:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.134184:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677448.134184:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000400:00000200:26.0:1619677448.134185:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:10.0:1619677448.134185:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677448.134185:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:26.0:1619677448.134186:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.134187:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.134188:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.134189:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.134190:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.134193:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:12.0:1619677448.134193:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677448.134195:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677448.134196:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.134197:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.134198:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.134198:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.134199:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:12.0:1619677448.134199:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.134200:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.134201:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.134201:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:12.0:1619677448.134201:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.134202:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.134203:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.134206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:13.0:1619677448.134213:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.134214:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:13.0:1619677448.134216:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.134217:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.134217:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.134218:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:13.0:1619677448.134218:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:1.0:1619677448.134218:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.134219:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.134220:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677448.134220:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:1.0:1619677448.134223:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.134276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.134281:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.134284:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.134285:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.134287:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.134289:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.134290:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.134292:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.134297:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.134299:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.134300:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.134303:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.134305:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.134308:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.134309:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.134309:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.134311:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.134311:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.134312:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.134312:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.134318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.134326:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.134329:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.134330:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.134330:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.134332:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.134334:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.134460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677448.134465:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677448.134469:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677448.134469:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:1.0:1619677448.134471:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.134473:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.134475:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.134476:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718895] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.134477:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.134478:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.134479:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.134483:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.134484:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.134486:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.134487:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.134488:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.134489:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.134489:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677448.134490:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677448.134490:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677448.134495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.134502:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.134503:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.134504:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.134505:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.134506:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.134507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:17.0:1619677448.134507:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:24.0:1619677448.134514:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.134516:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.134517:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.134519:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.134522:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.134523:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.134524:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.134526:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.134527:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.134528:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677448.134531:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.134533:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.134536:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.134537:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.134538:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.134539:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.134540:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.134541:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.134541:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.134545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.134554:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.134556:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.134557:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.134559:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.134560:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.134562:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.134722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.134725:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.134727:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.134729:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.134730:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.134732:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.134734:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.134735:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.134736:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.134737:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.134738:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.134742:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.134744:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.134747:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.134747:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.134748:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.134750:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.134751:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.134752:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.134753:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.134754:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.134760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.2:1619677448.134760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677448.134764:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.134767:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:5.0:1619677448.134767:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:30.0:1619677448.134768:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.134768:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677448.134769:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.134769:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677448.134770:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.134770:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677448.134771:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.134771:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.134771:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:30.0:1619677448.134773:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:10.0:1619677448.134773:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.134776:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.134777:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.134779:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.134780:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.134781:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.134782:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.134785:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.134786:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.134789:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.134790:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.134790:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.134792:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.134792:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.134792:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.134793:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.134794:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:31.0:1619677448.134795:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.134795:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.134796:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.134797:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.134797:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.134798:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.134801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.134806:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677448.134808:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:12.0:1619677448.134808:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.134809:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.134810:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.134811:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:12.0:1619677448.134811:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.134812:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.134813:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.134814:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.134835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.134842:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.134845:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.134845:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.134847:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.134850:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.134851:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.134852:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.134854:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.134855:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.134856:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.134859:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.134860:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.134863:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.134864:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.134864:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.134866:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.134867:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.134867:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.134868:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.134872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.134872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.134879:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.134880:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:26.0:1619677448.134882:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.134882:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:1.0:1619677448.134882:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.134884:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.134884:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.134885:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.134886:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.134888:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.134888:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:10.0:1619677448.134889:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.134891:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.134892:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.134893:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.134894:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.134898:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.134899:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.2:1619677448.134899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.134903:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.134903:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:17.0:1619677448.134904:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.134904:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.134906:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677448.134907:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:10.0:1619677448.134907:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.134907:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.134908:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.134909:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.134909:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677448.134910:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.134911:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.134912:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:9.2:1619677448.134912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:17.0:1619677448.134914:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.134915:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677448.134915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677448.134916:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.134917:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.134920:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.134920:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.134921:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:17.0:1619677448.134922:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.134922:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.134923:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.134924:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677448.134924:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.134924:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.134926:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677448.134926:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.134926:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:2.0:1619677448.134926:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.134927:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.134929:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.134929:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.134930:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.134930:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677448.134931:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.134941:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.134944:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.134945:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.134947:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.134948:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.134950:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.135185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:21.2:1619677448.135191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677448.135191:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.135194:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.135195:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.135197:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.135198:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677448.135199:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.135200:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677448.135201:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:18.0:1619677448.135202:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.135204:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.135205:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.135205:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.135205:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.135206:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.135207:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.135208:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677448.135210:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.135210:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.135211:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.135211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:30.0:1619677448.135212:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.135213:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.135213:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.135216:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.135217:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.135217:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.135218:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.135219:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.135219:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.135220:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:18.0:1619677448.135220:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.135220:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.135221:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:30.0:1619677448.135222:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.135222:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:31.0:1619677448.135223:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.135223:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.135224:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677448.135225:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.135225:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:31.0:1619677448.135227:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.135228:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96718886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.135229:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.135230:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.135231:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:8.2:1619677448.135231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677448.135232:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:31.0:1619677448.135235:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.135235:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677448.135236:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.135236:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:13.0:1619677448.135236:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.135237:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.135237:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:13.0:1619677448.135237:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.135238:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.135239:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.135239:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.135239:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:12.0:1619677448.135239:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.135239:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:13.0:1619677448.135240:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.135241:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:1.0:1619677448.135241:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:31.0:1619677448.135242:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677448.135242:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.135243:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.135243:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:1.0:1619677448.135243:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.135245:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.135248:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677448.135253:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.135256:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.135257:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.135259:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677448.135259:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.135261:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.135261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.135267:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.135270:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.135271:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.135273:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.135275:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.135277:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.135278:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.135279:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.135280:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.135281:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.135284:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.135285:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.135288:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.135289:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.135289:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.135291:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.135291:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.135292:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.135292:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677448.135294:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.2:1619677448.135299:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.135302:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.135305:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.135306:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.135307:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:10.0:1619677448.135308:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.135310:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.135311:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.135311:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.135311:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:10.0:1619677448.135312:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.135313:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.135314:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.135315:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.135316:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.135317:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:10.0:1619677448.135317:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.135320:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.135322:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.135324:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.135325:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.135326:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.135328:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.135329:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.135329:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.135330:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.135335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.135343:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.135346:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.135347:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.135348:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.135349:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.135352:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.135507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.135513:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.135516:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.135517:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.135519:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.135521:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.135523:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.135524:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.135526:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.135527:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.135528:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.135531:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.135533:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.135536:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.135537:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.135538:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.135540:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.135540:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.135542:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.135543:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.135543:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.135548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.135553:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677448.135554:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.135555:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:12.0:1619677448.135555:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:17.0:1619677448.135556:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677448.135556:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.135557:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677448.135558:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.135560:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.135561:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.135593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.135601:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.135603:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.135604:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.2:1619677448.135605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:26.0:1619677448.135606:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.135609:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.135610:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.135611:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.135611:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677448.135613:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.135614:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.135614:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:26.0:1619677448.135615:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.135616:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.135617:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.135620:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.135622:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.135624:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.135624:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.135625:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:26.0:1619677448.135626:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.135627:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:5.0:1619677448.135627:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.135628:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.135628:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.135629:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.135629:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:5.0:1619677448.135629:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.135630:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:5.0:1619677448.135630:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677448.135633:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.135634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.135635:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677448.135638:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.135639:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677448.135639:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677448.135641:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677448.135642:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.135642:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.135643:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.135643:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677448.135644:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.135645:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.135645:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:20.0:1619677448.135647:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.135647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677448.135650:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677448.135655:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.135658:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.135659:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.135659:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.135661:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.135664:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.135667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.135674:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.135677:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.135678:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.135678:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.135679:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.135679:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.135685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.135693:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.135696:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.135698:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.135700:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.135700:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.135702:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:21.2:1619677448.135702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:31.0:1619677448.135703:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.135704:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.135705:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.135707:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.135708:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:31.0:1619677448.135711:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.135711:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.135711:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:31.0:1619677448.135713:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.135714:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.135716:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.135717:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:18.0:1619677448.135717:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.135718:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:18.0:1619677448.135719:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677448.135720:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.135720:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:18.0:1619677448.135720:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.135721:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.135722:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:18.0:1619677448.135722:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.135723:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000400:00000200:18.0:1619677448.135723:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.135724:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.135726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:18.0:1619677448.135727:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.135729:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.2:1619677448.135730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677448.135732:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.135733:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.135733:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.135734:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677448.135735:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.135736:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677448.135736:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:18.0:1619677448.135737:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000400:00000200:5.0:1619677448.135737:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.135738:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:18.0:1619677448.135738:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:20.0:1619677448.135739:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.135739:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:5.0:1619677448.135739:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.135740:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.135741:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.135742:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.135743:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.135744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:13.0:1619677448.135748:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.135749:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:13.0:1619677448.135750:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.135751:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.135751:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.135752:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:13.0:1619677448.135753:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.135753:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677448.135754:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.135755:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.135818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.135826:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.135829:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.135831:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.135833:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.135834:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.135835:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.135837:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.135838:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.135839:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.135840:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.135843:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.135844:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.135847:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.135848:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.135849:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.135850:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.135851:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.135852:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.135853:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.135865:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.135868:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.135869:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.135869:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.135871:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.135875:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.135949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.135956:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.135959:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.135960:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.135962:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.135964:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.135966:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.135967:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.135968:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.135969:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.135970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:10.0:1619677448.135970:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.135973:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.135974:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.2:1619677448.135975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677448.135976:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:10.0:1619677448.135977:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.135977:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.135978:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:18.0:1619677448.135979:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:10.0:1619677448.135979:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.135980:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.135980:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677448.135980:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677448.135980:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677448.135981:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677448.135981:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677448.135983:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:18.0:1619677448.135984:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.135984:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677448.135985:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677448.135986:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.135987:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.135988:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.135989:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.135989:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.135990:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.135991:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:31.0:1619677448.135992:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:1.0:1619677448.135992:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96718875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.135994:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:1.0:1619677448.135994:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.135995:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.135995:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000400:00000200:1.0:1619677448.135995:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.135996:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.135997:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:31.0:1619677448.135998:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.135998:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:18.0:1619677448.135999:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.136000:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:1.0:1619677448.136000:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.136001:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677448.136001:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.136003:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.136003:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677448.136003:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.136004:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.136005:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677448.136005:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.136006:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.136006:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.136008:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:9.2:1619677448.136009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.136009:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.136009:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.136010:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677448.136016:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.136018:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.136019:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.136020:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.136021:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.136022:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677448.136023:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677448.136025:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.136026:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.136026:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.136028:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.136030:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677448.136040:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.136043:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.136044:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.136044:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.136045:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.136046:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.136075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.136083:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.136086:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.136088:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.136090:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.136091:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.136092:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.136094:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718874] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.136095:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.136097:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.136097:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.136101:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.136103:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.136105:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.136106:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.136107:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.136108:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.136109:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.136110:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.136110:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677448.136120:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.136122:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.136123:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.136125:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.136126:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.136128:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677448.136140:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.136142:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.136143:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.136144:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.136144:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.136145:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.136214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.136220:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.136223:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.136224:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.136226:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.136228:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.136230:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.136231:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.136233:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.136233:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.136235:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.2:1619677448.136237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:18.0:1619677448.136238:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.136239:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.2:1619677448.136241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:18.0:1619677448.136242:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.136242:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.136243:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.136243:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.136244:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:18.0:1619677448.136244:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.136245:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677448.136245:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:18.0:1619677448.136246:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.136246:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677448.136246:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.136247:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:17.0:1619677448.136249:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.136250:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.136250:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:17.0:1619677448.136251:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.136251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677448.136252:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.136254:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.136254:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:20.0:1619677448.136256:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.136256:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.136258:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.136259:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.136259:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.136259:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.136260:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:20.0:1619677448.136261:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.136261:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.136261:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.136262:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.2:1619677448.136263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:1.0:1619677448.136263:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.136265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677448.136269:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:20.0:1619677448.136270:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.136270:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.136272:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.136272:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.136273:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:17.0:1619677448.136273:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.0:1619677448.136274:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.136274:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:2.0:1619677448.136275:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.136275:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677448.136276:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.136277:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:20.0:1619677448.136277:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.136277:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.136277:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677448.136278:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.136278:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:2.0:1619677448.136278:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.136279:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:8.2:1619677448.136279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (22)++
00000400:00000200:2.0:1619677448.136279:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.136280:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.136281:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.136281:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:2.0:1619677448.136281:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96718870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.136282:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677448.136282:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.136283:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.136283:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:17.0:1619677448.136283:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:2.0:1619677448.136283:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.136284:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677448.136284:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:2.0:1619677448.136284:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.136285:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677448.136285:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.136286:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:26.0:1619677448.136287:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96718869] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.136287:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000800:00000200:1.0:1619677448.136287:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000800:00000200:17.0:1619677448.136288:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.136288:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (23)++
00000400:00000200:26.0:1619677448.136289:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.136289:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (24)--
00000400:00000200:1.0:1619677448.136289:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.136290:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.136290:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:2.0:1619677448.136290:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.136291:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:26.0:1619677448.136292:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.136292:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.136293:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.136300:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.136302:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677448.136302:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:22.0:1619677448.136303:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.136303:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677448.136303:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.136304:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:26.0:1619677448.136305:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.136305:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677448.136306:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.136307:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.136307:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677448.136308:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677448.136308:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:22.0:1619677448.136309:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677448.136309:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:26.0:1619677448.136311:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.136313:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.136313:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.136315:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.136315:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.136316:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.136317:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.136321:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.136324:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.136324:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.136326:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.136328:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677448.136328:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:12.0:1619677448.136329:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.136331:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.136332:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.136332:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.136334:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.136337:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.136338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:13.0:1619677448.136339:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.136342:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.136342:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.136343:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.136344:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.136345:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.136345:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.136348:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.136350:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.136352:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.136353:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.136354:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:20.2:1619677448.136354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:31.0:1619677448.136355:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.136357:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.136358:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.136359:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.136360:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.136362:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.136363:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:31.0:1619677448.136364:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:5.0:1619677448.136365:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.136366:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677448.136367:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.136368:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677448.136368:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:5.0:1619677448.136368:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.136370:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:5.0:1619677448.136370:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677448.136371:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:5.0:1619677448.136371:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.136372:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.136372:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:5.0:1619677448.136372:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.136373:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.136374:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.136377:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.136379:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.136382:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.136383:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:22.0:1619677448.136384:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.0:1619677448.136384:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.136386:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:5.0:1619677448.136386:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.136386:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.136387:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:5.0:1619677448.136387:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:22.0:1619677448.136388:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.136388:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:22.0:1619677448.136390:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.136392:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677448.136398:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.136401:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.136401:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.136403:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.136404:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.136406:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.136477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.136485:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.136488:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.136490:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.136491:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.136492:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.136493:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.136495:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.136496:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.136497:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.136498:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.136501:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.136503:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.136505:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.136506:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.136506:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.136508:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.136508:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.136509:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.136510:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677448.136519:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.136522:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.136522:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.136524:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.136526:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.136527:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.136535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.136542:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.2:1619677448.136544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.136545:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.136546:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.136547:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.136550:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.136550:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.136551:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:17.0:1619677448.136553:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:10.0:1619677448.136553:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.136553:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.136555:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.136556:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.136556:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.136557:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677448.136559:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.136561:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.136562:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.136562:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.136564:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.136564:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:17.0:1619677448.136565:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.136566:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.136566:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.136567:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.136568:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.136569:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:17.0:1619677448.136570:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.136570:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.136571:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:17.0:1619677448.136572:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.136572:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.136573:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.136575:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.136576:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.136577:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.136578:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:4.2:1619677448.136578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.136579:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.136579:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.136580:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.136584:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.136585:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.136585:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.136587:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:26.0:1619677448.136588:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.136588:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.136589:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.136589:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.136590:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677448.136591:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:26.0:1619677448.136591:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.136594:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.136595:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.136596:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.136598:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.136599:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677448.136601:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.136604:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.136605:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.136605:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.136606:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.136607:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.136669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.136674:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.136677:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.136677:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.136679:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.136682:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.136683:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.136684:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.136686:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.136687:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.136688:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.136692:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.136694:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.136697:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.136698:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.136699:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.136700:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.136701:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.136702:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.136703:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.136704:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:21.2:1619677448.136709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.136712:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.136714:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:22.0:1619677448.136714:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:26.0:1619677448.136716:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.136716:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:10.0:1619677448.136716:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.136717:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:26.0:1619677448.136718:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.136718:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:26.0:1619677448.136721:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.136854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.136860:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.136863:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.136864:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.136866:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.136869:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.136870:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.136871:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.136872:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.136873:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.136874:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.136877:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.136879:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.136880:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677448.136881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677448.136881:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.136882:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.136883:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.136883:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.136885:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.136886:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.136886:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.136889:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.2:1619677448.136890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677448.136892:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.2:1619677448.136893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677448.136894:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.136895:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.136896:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.136897:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:17.0:1619677448.136898:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:30.0:1619677448.136899:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.136899:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.136899:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:30.0:1619677448.136900:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.136900:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.136900:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.136901:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677448.136902:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.136902:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.136902:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:30.0:1619677448.136903:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.136903:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677448.136905:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:30.0:1619677448.136907:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.136909:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.136911:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.136912:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.136913:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.136914:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.136915:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.136916:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.136918:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.136918:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677448.136924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.2:1619677448.136925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:13.0:1619677448.136926:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.136929:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.136930:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.136931:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.136932:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677448.136932:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.136932:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:13.0:1619677448.136933:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.136934:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.136935:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677448.136935:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.136936:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677448.136936:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.136936:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.136937:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.136938:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.137120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:5.2:1619677448.137125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.0:1619677448.137127:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.137130:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:17.0:1619677448.137130:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.137131:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.137133:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.137133:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.137134:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677448.137136:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.137136:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.137138:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.137138:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.137139:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677448.137140:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.137141:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.137142:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.137143:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.137143:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.137144:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.137145:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.137145:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677448.137151:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.137151:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.137153:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677448.137155:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.137157:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.137158:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.137158:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.137158:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677448.137159:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.137160:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.137160:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677448.137161:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.137161:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.137162:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.137162:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.137162:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677448.137163:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.137163:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.137167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677448.137173:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.137175:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:12.0:1619677448.137176:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.137176:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.137177:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.137178:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:12.0:1619677448.137178:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.137179:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.137180:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.137181:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677448.137181:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.137183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:1.0:1619677448.137184:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.0:1619677448.137190:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.137193:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.137194:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.137196:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.137199:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.137201:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.137202:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.137203:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.137204:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.137205:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.137209:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.137211:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.137214:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.137215:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.137215:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.137217:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.137217:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.137219:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.137220:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.137220:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677448.137223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.137226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:18.0:1619677448.137231:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677448.137233:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:18.0:1619677448.137234:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:24.0:1619677448.137235:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:18.0:1619677448.137235:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.137235:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.137236:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.137237:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:18.0:1619677448.137237:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.137241:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.137243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.137251:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.137254:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.137255:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.137255:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.137256:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.137256:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.137303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.137311:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.137314:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.137314:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.137316:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.137318:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.137319:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.137321:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.137322:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.137323:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.137324:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.137327:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.137329:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.137332:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.137333:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.137333:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.137335:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.137336:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.137336:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.137337:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.137351:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.137354:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.137354:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.137355:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.137357:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.137360:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.137408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.2:1619677448.137410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:20.0:1619677448.137416:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:26.0:1619677448.137418:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677448.137419:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:26.0:1619677448.137421:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.137421:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:20.0:1619677448.137422:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.137423:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:26.0:1619677448.137424:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677448.137425:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.137427:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.137427:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.137429:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.137429:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.137431:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.137431:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.137432:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.137433:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.137433:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.137434:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.137436:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:21.2:1619677448.137436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000400:00000200:20.0:1619677448.137438:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.137440:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.137440:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.137442:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.137442:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:20.0:1619677448.137444:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.137445:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677448.137445:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:18.0:1619677448.137445:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:26.0:1619677448.137446:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.137446:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:26.0:1619677448.137447:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.137447:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.137448:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.137448:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.137449:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000400:00000200:18.0:1619677448.137449:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.137450:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.137450:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:26.0:1619677448.137451:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.137452:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.137452:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:18.0:1619677448.137452:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.137453:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.137455:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96718854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.137456:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.137457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:18.0:1619677448.137457:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677448.137457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677448.137458:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.137462:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:18.0:1619677448.137462:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.137464:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:18.0:1619677448.137464:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.137465:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.137466:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:31.0:1619677448.137467:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.137467:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.137467:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.137468:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.137469:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:2.0:1619677448.137469:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.137470:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:18.0:1619677448.137470:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.137470:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.137471:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677448.137472:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677448.137472:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:2.0:1619677448.137472:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000400:00000200:30.0:1619677448.137473:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.137473:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:30.0:1619677448.137475:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.137475:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677448.137475:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:30.0:1619677448.137476:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:18.0:1619677448.137476:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677448.137477:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677448.137477:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:30.0:1619677448.137478:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.137479:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.137480:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.137481:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:21.2:1619677448.137483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:30.0:1619677448.137484:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.137485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:30.0:1619677448.137486:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:22.0:1619677448.137486:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:30.0:1619677448.137489:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.137489:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:30.0:1619677448.137490:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:30.0:1619677448.137490:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:22.0:1619677448.137490:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.137490:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.137490:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677448.137491:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:30.0:1619677448.137492:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000400:00000200:22.0:1619677448.137492:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.137492:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:30.0:1619677448.137493:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.137493:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.137493:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:30.0:1619677448.137494:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.137494:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.137494:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677448.137495:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.137495:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.137498:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.137498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:18.0:1619677448.137500:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.137501:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.137503:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.137504:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.137505:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.137505:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:18.0:1619677448.137506:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.137507:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.137508:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.137508:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.137509:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.137510:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.137512:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.137513:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:18.0:1619677448.137514:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.137515:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.137515:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.137517:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.137518:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.137518:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.137519:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.137523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.137530:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.137532:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.137532:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.137533:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.137534:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.137536:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.137568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.137575:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.137577:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.137578:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.137580:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.137583:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.137584:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.137585:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.137586:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.137587:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.137588:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.137591:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.137593:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.137596:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.137597:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.137597:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.137599:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.137599:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.137600:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.137600:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.137613:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.137615:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.137616:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.137616:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.137618:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.137620:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.137633:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.137636:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.137636:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.137637:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.137638:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.137638:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.137716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.137723:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.137726:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.137727:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.137729:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.137732:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.137733:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.137734:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.137736:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.137737:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.137738:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.137741:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.137743:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.2:1619677448.137746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:10.0:1619677448.137746:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.137746:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.137747:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.137749:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.137749:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.137750:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.137750:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.137752:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.137754:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.137756:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.137757:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:6.0:1619677448.137757:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.137759:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.137760:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.137760:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.137762:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.137762:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.137763:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.137765:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.137766:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.137767:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.137771:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.137773:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.137773:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.137774:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677448.137777:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.137777:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.137778:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.137780:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.137780:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.137781:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.137783:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677448.137784:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.137786:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.137787:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.137787:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.137788:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677448.137788:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.137792:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.137795:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.137795:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.137796:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.137798:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.137801:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.137813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.137820:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.137822:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.137823:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.137825:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.137828:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.137829:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.137830:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.137831:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.137832:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.137833:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.137836:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.137838:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.137839:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.137840:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.137841:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.137842:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.137842:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.137843:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.137844:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677448.137853:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.137855:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.137856:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.137857:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.137857:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.137859:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.137972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.137978:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.137981:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.137981:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.137983:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:4.2:1619677448.137984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:18.0:1619677448.137985:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.137986:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.137988:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.137989:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.137990:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.137991:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.137991:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.137994:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.137994:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:18.0:1619677448.137995:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.137997:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.137997:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.137999:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.137999:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.138000:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.138000:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:10.0:1619677448.138001:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:18.0:1619677448.138002:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:10.0:1619677448.138002:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.138003:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.138003:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:10.0:1619677448.138003:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.138004:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:10.0:1619677448.138004:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.138005:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.138009:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.138010:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.138013:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.138014:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.138015:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (23)++
00000800:00000200:6.0:1619677448.138016:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:10.0:1619677448.138018:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (25)--
00000800:00000200:10.0:1619677448.138019:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.138019:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.138020:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.138020:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:22.0:1619677448.138021:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.138021:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.138023:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677448.138027:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.138029:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.138031:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.138031:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.138032:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.138033:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.138035:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.138368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.138374:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.138377:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.138379:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.138380:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.138382:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.138384:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677448.138385:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.138387:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.138388:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.138389:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:5.2:1619677448.138391:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:5.0:1619677448.138396:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.138398:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.138399:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677448.138400:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.138400:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.138401:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.138402:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.138403:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.0:1619677448.138403:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.138404:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.138404:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677448.138405:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.138405:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:26.0:1619677448.138408:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.138409:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:8.2:1619677448.138410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:26.0:1619677448.138411:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.138412:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.138413:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.138414:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.138416:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.138418:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.138418:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677448.138419:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.138420:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.138420:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677448.138421:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:20.0:1619677448.138421:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.138422:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.138422:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.138423:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:20.0:1619677448.138423:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.138424:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.0:1619677448.138424:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.138425:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:10.0:1619677448.138434:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:21.2:1619677448.138435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:10.0:1619677448.138436:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.138437:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.138438:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.138439:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.138441:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.138441:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:22.0:1619677448.138444:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:8.2:1619677448.138444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:6.0:1619677448.138445:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.138447:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.138449:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.138451:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.138452:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.138453:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.138453:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000400:00000200:22.0:1619677448.138454:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.138455:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:5.0:1619677448.138456:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.138457:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.138458:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.138458:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:22.0:1619677448.138459:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.138460:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677448.138461:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.138464:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.138464:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.138465:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.138467:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.138468:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.138469:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.138469:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:21.2:1619677448.138472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677448.138479:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:31.0:1619677448.138480:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.138482:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677448.138482:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.138482:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.138483:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.138485:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.138486:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.138487:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.138488:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.138489:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:31.0:1619677448.138490:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.138491:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718842] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.138492:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.138493:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.138494:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.138498:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.138500:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.138502:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.138503:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.138503:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.138505:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.138506:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.138507:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.138507:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.138516:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.138518:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.138519:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.138520:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.138521:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.138523:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.138536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.138542:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.138545:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.138546:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.138547:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.138549:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.138550:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.138551:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.138552:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.138553:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.138554:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.138557:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.138558:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.138561:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.138562:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.138563:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.138564:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.138565:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.138566:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.138567:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.138567:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.138573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.138577:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.138580:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.138580:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.138581:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.138581:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.138582:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.138583:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.138583:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.138584:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.138586:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.138620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.138627:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.138630:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.138631:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.138632:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.138635:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.138637:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.138638:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.138639:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.138640:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.138641:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.138644:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.138646:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.138647:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.138648:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.138648:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.138650:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.138650:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.138651:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.138652:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.138653:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677448.138656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.138659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.138665:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.138665:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.138666:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.138666:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.138666:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.138667:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.138668:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.138668:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.138669:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.138669:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:4.2:1619677448.138683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.138690:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.138693:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.138694:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.138696:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.138699:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.138700:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.138702:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.138703:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.138704:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.138706:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.138710:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.138712:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.138715:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.138715:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.138716:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.138718:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.138718:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.138719:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.138721:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.138721:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.138724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.138727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.138732:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677448.138734:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.138735:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.138736:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.138736:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.138737:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677448.138737:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677448.138737:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.138739:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.138742:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.138906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.138913:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.138916:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.138918:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.138919:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.138922:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.138924:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.138925:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.138927:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.138928:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.138929:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.138932:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.138934:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.138936:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.138937:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.138937:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.138939:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.138939:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.138940:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.138941:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.138942:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.138947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.138951:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.138954:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.138955:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.138955:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:30.0:1619677448.138956:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.138957:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.138957:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.138958:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.138958:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:10.0:1619677448.138959:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677448.138970:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.138972:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.138973:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.138974:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.138974:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.138975:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.139012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.139018:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.2:1619677448.139019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:22.0:1619677448.139021:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.139022:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.139024:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.139027:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:22.0:1619677448.139027:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.139028:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:31.0:1619677448.139030:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:22.0:1619677448.139030:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718837] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.139032:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.139033:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.139034:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.139035:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.139035:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677448.139038:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.139038:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.139039:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:22.0:1619677448.139040:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:31.0:1619677448.139041:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.139042:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.139043:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.139043:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.139044:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.139044:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.139044:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.139046:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.139046:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:31.0:1619677448.139047:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.139047:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.139048:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677448.139049:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.139052:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677448.139052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.139053:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.139053:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.139055:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.139055:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.139056:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.139057:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:2.0:1619677448.139059:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.139063:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.139064:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.139064:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.139066:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.139068:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677448.139069:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:13.0:1619677448.139070:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.139071:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.139073:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.139075:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.139076:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.139154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.139162:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.139164:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.139165:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.139167:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.139170:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.139172:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.139173:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.139174:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.139175:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.139176:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677448.139180:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.139182:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.139185:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.139185:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.139186:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.139188:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.139188:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.139189:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.139189:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.139194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.139202:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.139205:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.139206:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.139208:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.139208:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.139211:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.139421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.139426:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.139429:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.139430:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.139432:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.139434:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.139436:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.139437:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.139438:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.139439:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.139440:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.139443:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.139445:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.139448:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.139449:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.139449:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.139451:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.139451:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.139452:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.139452:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.139458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.139465:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.139468:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677448.139468:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677448.139469:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.139471:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.139474:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.2:1619677448.139489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.139493:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.139495:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.139497:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.139499:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.139499:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.139501:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.139502:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.139503:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.139504:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.139505:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.139508:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.139510:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.139512:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.139513:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.139514:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.139515:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.139516:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.139517:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.139519:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.139519:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.139524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.139529:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.139530:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:6.0:1619677448.139531:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.139532:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.139532:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:22.0:1619677448.139533:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.139533:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.139534:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:6.0:1619677448.139535:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.139537:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677448.139553:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.139555:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.139556:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.139556:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.139557:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.139558:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.139573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.139581:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.139583:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.139584:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.139586:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.139589:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.139590:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.139591:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.139593:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.139594:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.139595:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.139598:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.139600:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.139603:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.139603:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.139604:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.139606:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.139606:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.139607:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.139608:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.139612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.139620:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.139623:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.139624:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.139625:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.139626:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.139629:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.139649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.2:1619677448.139650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677448.139657:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.139659:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:30.0:1619677448.139660:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.139660:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.139662:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677448.139662:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.139664:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.139665:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.139666:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.139667:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677448.139667:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.139668:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.139668:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.139669:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.139670:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.139671:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.139671:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.139672:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.139673:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.139673:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.139676:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.139677:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.139680:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.139680:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.139681:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.139681:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.139681:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.139683:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.139683:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.139684:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.139684:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.139684:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.139684:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.139685:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.139686:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:30.0:1619677448.139687:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.139688:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.139688:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.139695:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677448.139698:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677448.139698:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.139700:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.139702:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.139703:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.139703:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677448.139704:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.139704:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677448.139705:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.139710:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.139711:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.139713:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.139713:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.139714:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.139714:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.139715:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.139722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.139729:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.139731:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.139732:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.139734:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.139737:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.139738:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.139739:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.139741:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.139742:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.139743:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.139746:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.139747:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.139750:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.139751:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.139752:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.139753:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.139754:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.139755:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677448.139756:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.139757:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677448.139759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.139763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.139770:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677448.139771:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:10.0:1619677448.139772:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:20.0:1619677448.139773:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:10.0:1619677448.139773:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:20.0:1619677448.139774:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.139774:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:20.0:1619677448.139776:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.139777:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.139779:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.139979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.139984:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.139987:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.139989:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.139990:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.2:1619677448.139991:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:5.0:1619677448.139995:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.139996:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677448.139997:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677448.139998:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.139999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:17.0:1619677448.140000:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.140000:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:5.0:1619677448.140001:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.140003:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.140003:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.140004:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.140006:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.140007:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:5.0:1619677448.140008:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.140008:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:8.0:1619677448.140009:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:1.0:1619677448.140009:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.140010:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677448.140011:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.140011:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.140012:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.140013:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.140013:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:5.0:1619677448.140014:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.140015:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:8.0:1619677448.140016:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.140016:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677448.140017:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.140018:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:5.0:1619677448.140018:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.140018:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:8.0:1619677448.140019:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96718826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.140019:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677448.140019:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:5.0:1619677448.140020:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:8.0:1619677448.140021:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.140021:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.140022:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677448.140023:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.140023:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.140023:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.140025:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.140025:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677448.140026:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.140027:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677448.140028:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.140028:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.140028:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677448.140030:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677448.140031:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.140032:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.140033:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677448.140033:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.2:1619677448.140033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:12.0:1619677448.140034:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.140035:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.140035:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:8.0:1619677448.140035:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.140036:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.140036:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677448.140037:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.140038:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677448.140039:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677448.140039:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:20.2:1619677448.140041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677448.140042:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:13.0:1619677448.140042:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:17.0:1619677448.140043:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677448.140043:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.140043:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677448.140044:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.140045:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677448.140047:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677448.140047:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.140048:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:31.0:1619677448.140049:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677448.140049:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677448.140049:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.140051:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:12.0:1619677448.140051:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:31.0:1619677448.140053:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.140055:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.140055:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.140057:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.140058:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.140059:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.140060:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.140061:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.140062:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:31.0:1619677448.140064:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.140064:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.140065:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:31.0:1619677448.140066:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.140066:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.140066:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.140067:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.140069:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.140069:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.140070:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.140071:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.140072:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.140073:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.140075:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.140075:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677448.140081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677448.140083:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.140085:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.140086:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.140088:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.140089:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:13.0:1619677448.140089:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.140089:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.140091:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.140092:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.140093:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:4.2:1619677448.140105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.140112:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.140115:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.140116:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.140117:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.140121:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.140122:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.140123:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.140125:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.140125:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.140126:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.140129:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.140131:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.140133:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.140134:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.140134:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.140136:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.140137:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.140137:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.140138:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677448.140145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:30.0:1619677448.140148:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:6.0:1619677448.140150:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677448.140151:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.140151:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.140153:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.140153:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.140154:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:31.0:1619677448.140155:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.140156:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:6.0:1619677448.140156:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.140158:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.140160:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.140161:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.140162:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.140163:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.140164:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.140168:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.140170:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.140173:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.140174:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.140174:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.140176:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.140177:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.140178:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.140179:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.140180:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:21.2:1619677448.140185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677448.140189:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677448.140190:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:31.0:1619677448.140192:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:22.0:1619677448.140192:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:31.0:1619677448.140193:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.140193:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.140193:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:31.0:1619677448.140195:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.140196:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.140198:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.140213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.140215:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.140218:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.140220:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.140221:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.140223:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.140225:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.140226:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.140227:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.140228:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.140229:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.140233:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.140234:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.140235:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.140237:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.140238:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.140238:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.140239:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.140239:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.140240:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.140241:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.140241:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.140241:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:31.0:1619677448.140242:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.0:1619677448.140242:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.140243:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:12.0:1619677448.140253:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.140255:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.140256:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.140258:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677448.140260:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.140262:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.140287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.140294:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.140300:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.140302:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.140304:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.140307:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.140308:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.140309:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.140310:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.140312:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.140312:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.140316:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.140318:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.140321:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.140321:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.140322:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.140324:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.140324:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.140325:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.140325:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.140337:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.140339:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.140340:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.140342:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.140344:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.140345:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677448.140354:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.140356:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.140357:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.140357:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.140358:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.140360:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.140461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.140467:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.140470:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.140472:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.140473:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.140475:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.140475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:5.0:1619677448.140476:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.140478:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.140479:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.140480:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.140481:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677448.140482:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:5.0:1619677448.140485:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.140486:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.140486:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677448.140488:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:24.0:1619677448.140489:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.140491:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.140491:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.140491:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.140492:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677448.140493:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.140494:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96718819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.140494:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.140494:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:24.0:1619677448.140495:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.140495:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:24.0:1619677448.140496:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.140496:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:24.0:1619677448.140498:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.140501:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.140503:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.140506:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.140506:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:12.0:1619677448.140506:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677448.140507:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.140507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677448.140508:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:24.0:1619677448.140509:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.140509:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:12.0:1619677448.140509:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.140510:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.140511:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:12.0:1619677448.140511:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.140512:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.140514:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.140514:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677448.140515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:21.2:1619677448.140517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677448.140517:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677448.140519:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.140520:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.140521:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:17.0:1619677448.140523:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.140523:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.140524:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.140525:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677448.140525:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.140526:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.140526:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.140528:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.140529:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:17.0:1619677448.140530:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.140532:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:31.0:1619677448.140533:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.140533:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.140535:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.140537:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.140537:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.140539:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.140539:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.140539:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:30.0:1619677448.140540:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.140541:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.140541:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:30.0:1619677448.140542:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.140542:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.140543:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.140544:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:30.0:1619677448.140546:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.140546:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.140547:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.140548:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.140548:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.140549:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.140551:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677448.140551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:30.0:1619677448.140552:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.140552:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.140554:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.140554:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.140555:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.140557:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.140558:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677448.140558:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.2:1619677448.140561:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.140562:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:21.2:1619677448.140563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677448.140566:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.140567:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.140568:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.140569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.140569:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.140571:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:1.0:1619677448.140571:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.140572:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.140572:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:31.0:1619677448.140573:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:1.0:1619677448.140573:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.140574:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677448.140574:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:31.0:1619677448.140575:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:1.0:1619677448.140575:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.140576:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.2:1619677448.140577:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.140582:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.140582:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.140583:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.140583:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.140584:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.140584:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:1.0:1619677448.140585:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.140586:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.140587:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.140587:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.140588:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.140601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.140609:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:4.2:1619677448.140610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.140611:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.140613:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.140615:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.140616:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.140617:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:26.0:1619677448.140618:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.140619:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.140620:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.140620:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:31.0:1619677448.140621:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.140621:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677448.140623:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.140624:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.140624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:26.0:1619677448.140626:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.140627:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.140628:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.140628:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.140629:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.140630:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.140631:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:31.0:1619677448.140632:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.140632:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.140633:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.140633:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:26.0:1619677448.140633:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.140634:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:31.0:1619677448.140635:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.140635:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:31.0:1619677448.140636:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.140637:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.140637:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:26.0:1619677448.140637:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.140638:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.140640:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:18.0:1619677448.140640:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.140641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:18.0:1619677448.140642:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:26.0:1619677448.140643:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.140643:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96718814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.140644:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.140645:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:18.0:1619677448.140645:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.140646:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:18.0:1619677448.140646:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.140647:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:18.0:1619677448.140647:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.140648:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.140648:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677448.140649:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:18.0:1619677448.140650:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.140652:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:18.0:1619677448.140652:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.140653:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.140655:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.140655:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.140655:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677448.140656:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.140657:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.140658:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677448.140659:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.140660:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:22.0:1619677448.140661:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:18.0:1619677448.140661:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.140661:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:22.0:1619677448.140663:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.140664:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.140665:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.140667:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.140670:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.2:1619677448.140670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:13.0:1619677448.140672:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.140674:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.140675:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.140677:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.140677:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:12.0:1619677448.140678:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.140680:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.140680:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.140681:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.140682:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.140683:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.140684:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.140885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.140892:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.140895:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.140897:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.140898:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.140900:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.140901:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677448.140903:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.140904:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.140905:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.140906:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677448.140909:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.140911:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.140914:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.140915:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.140915:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.140917:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.140918:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.140919:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.140919:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677448.140922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.2:1619677448.140924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.140930:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.140932:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.140933:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:17.0:1619677448.140935:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.140935:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.140936:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.140938:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.140938:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.140940:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.140941:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.140942:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.140943:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.140944:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.140947:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.140948:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.140948:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.140949:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677448.140950:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.140954:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.140955:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.140955:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.140957:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.140957:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.140958:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.140958:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.140963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.140970:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.140972:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677448.140973:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677448.140974:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.140975:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.140978:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.2:1619677448.141023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.141028:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:4.2:1619677448.141030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677448.141031:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.141032:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.141034:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.141036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677448.141037:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677448.141037:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.141038:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:26.0:1619677448.141040:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.141040:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:2.0:1619677448.141040:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.141041:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.141043:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.141044:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:2.0:1619677448.141044:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:10.0:1619677448.141045:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.141046:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:10.0:1619677448.141047:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.141048:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.141049:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.141050:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.141050:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:10.0:1619677448.141051:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.141051:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:10.0:1619677448.141052:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.141053:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.141054:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.141054:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.141054:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.141054:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.141055:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677448.141056:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.141056:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.141056:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.141057:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:31.0:1619677448.141058:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.141058:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.141058:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:31.0:1619677448.141059:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.141060:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.141061:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.141062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677448.141064:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.141064:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.141066:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.141067:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.141067:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.141068:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.141068:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.141069:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.141069:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.141070:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677448.141070:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.141071:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.141071:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.141071:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.141072:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.141072:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:31.0:1619677448.141073:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:10.0:1619677448.141073:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:1.0:1619677448.141073:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.141074:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:10.0:1619677448.141074:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677448.141074:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.141076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:1.0:1619677448.141076:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.141079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.141079:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:12.0:1619677448.141083:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677448.141084:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:12.0:1619677448.141085:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:26.0:1619677448.141086:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:12.0:1619677448.141087:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.141087:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.141087:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.141088:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.141089:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:13.0:1619677448.141089:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677448.141089:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.141090:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.141090:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.141091:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:12.0:1619677448.141091:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.141094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:5.0:1619677448.141094:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677448.141100:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.141103:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.141103:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.141106:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.141108:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.141110:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677448.141111:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.141112:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.141114:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.141115:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677448.141118:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.141120:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677448.141123:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.141124:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677448.141124:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677448.141127:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677448.141127:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.141128:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.141128:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677448.141138:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.141141:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.141142:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.141142:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.141144:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:10.0:1619677448.141148:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:26.0:1619677448.141158:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.141161:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:20.2:1619677448.141161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677448.141161:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.141162:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.141163:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.141163:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.0:1619677448.141164:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.141167:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.141169:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.141170:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.141172:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.141174:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.141175:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.141176:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.141177:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.141178:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.141182:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.141184:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.141187:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.141188:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.141189:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.141190:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.141191:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.141192:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.141192:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.141203:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.141205:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.141206:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.141208:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.141209:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.141211:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.141265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.141273:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.141276:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.141278:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.141280:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.141280:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.141282:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.141283:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.141284:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.141285:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.141286:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.141290:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.141291:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.141299:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.141300:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.141301:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.141303:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.141303:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.141304:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.141305:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.141309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.141316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.141322:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.141322:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.141324:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:17.0:1619677448.141325:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:5.0:1619677448.141325:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.141326:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.141327:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.141327:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.141328:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.141330:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:17.0:1619677448.141331:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.141332:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.141334:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718805] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.141335:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.141336:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.141337:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.141340:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.141342:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.141344:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.141344:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.141345:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.141346:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.141347:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.141347:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.141348:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.141353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.141359:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.141361:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.141362:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.141363:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.141364:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.141365:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.141390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.2:1619677448.141394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:31.0:1619677448.141397:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.141400:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:2.0:1619677448.141400:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:31.0:1619677448.141402:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.141403:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.141404:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:30.0:1619677448.141405:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.141406:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.141406:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.141407:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.141408:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.141410:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.141410:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.141411:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.141412:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.141412:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.141413:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.141415:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.141416:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.141417:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.141418:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.141421:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.141421:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.141423:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.141424:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.141424:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.141425:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.141426:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.141427:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.141428:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677448.141428:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.141428:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.141429:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677448.141429:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.141431:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.141431:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.141432:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.141433:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677448.141440:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.141443:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:10.0:1619677448.141443:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:12.0:1619677448.141444:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.141445:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.141446:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.141447:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:13.0:1619677448.141447:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.141447:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.141448:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677448.141449:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.141452:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:13.0:1619677448.141459:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.141462:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.141462:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.141463:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.141463:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.141464:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:13.0:1619677448.141464:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.141467:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.141468:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.141468:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.141469:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.141470:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.141742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.2:1619677448.141746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677448.141748:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.141751:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.141751:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.141753:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677448.141753:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.141755:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.141756:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:30.0:1619677448.141757:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:17.0:1619677448.141758:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.141760:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.141760:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.141761:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.141761:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.141762:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.141763:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677448.141765:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.141767:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.141767:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.141768:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.141770:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.141770:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:30.0:1619677448.141771:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.141772:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.141772:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.141773:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:30.0:1619677448.141774:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.141774:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.141775:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.141776:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.141776:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.141778:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.141778:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.141780:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.141780:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.2:1619677448.141780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677448.141781:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.141783:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.141783:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.2:1619677448.141783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.141784:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.141785:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677448.141789:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.141790:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.141790:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.141790:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.141791:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.141791:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.141792:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.141793:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:1.0:1619677448.141793:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.141794:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:6.0:1619677448.141797:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.141800:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.141801:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.141801:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.141803:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.141807:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.141849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.141856:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.141859:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.141859:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.141861:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.141864:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.141865:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.141866:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.141868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:10.0:1619677448.141868:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.141868:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.141869:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.141873:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.141874:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.141876:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.141877:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.141878:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:10.0:1619677448.141878:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.141878:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.141879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:31.0:1619677448.141880:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.141880:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.141880:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.141881:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:10.0:1619677448.141881:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.141882:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.141882:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:31.0:1619677448.141883:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.141884:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:31.0:1619677448.141885:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.141886:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.141887:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.141888:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.141888:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:31.0:1619677448.141889:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.141890:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677448.141891:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.141893:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.141893:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:31.0:1619677448.141895:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:20.0:1619677448.141895:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:22.0:1619677448.141896:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:20.0:1619677448.141897:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96718798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.141897:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.141898:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.141898:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:22.0:1619677448.141898:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.141899:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:20.0:1619677448.141899:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.141900:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:22.0:1619677448.141900:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.141900:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.141901:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:20.0:1619677448.141901:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.141902:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.141904:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:22.0:1619677448.141904:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677448.141905:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.141905:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.141906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.141907:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.141909:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.141910:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.2:1619677448.141912:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.141914:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.141915:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.141915:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.141916:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:5.0:1619677448.141916:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.141917:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:5.0:1619677448.141918:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.141919:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.141919:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.141920:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.141920:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:30.0:1619677448.141922:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:5.0:1619677448.141922:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.141923:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677448.141923:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.141923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.141924:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.2:1619677448.141925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:8.2:1619677448.141927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:17.0:1619677448.141930:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677448.141932:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:17.0:1619677448.141933:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:5.0:1619677448.141933:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:17.0:1619677448.141934:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.141934:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:1.0:1619677448.141934:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.141935:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:17.0:1619677448.141936:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.141936:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677448.141937:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.141937:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.141938:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.141939:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:17.0:1619677448.141939:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677448.141940:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.141995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677448.141999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:1.0:1619677448.142001:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.142003:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:18.0:1619677448.142004:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677448.142004:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.142006:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.142007:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.142008:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:1.0:1619677448.142008:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.142010:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677448.142011:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.142011:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.142012:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.142013:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.142014:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.142014:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677448.142015:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.142017:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.142018:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.142019:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.142020:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.142021:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.142021:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:18.0:1619677448.142024:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.142026:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.142027:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.142028:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.142028:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677448.142029:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.142029:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677448.142030:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.142031:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.142031:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.142032:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.142032:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:1.0:1619677448.142032:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.142033:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677448.142033:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677448.142034:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.142037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.142044:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.142045:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.142047:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:6.0:1619677448.142047:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.142048:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:17.0:1619677448.142049:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.142049:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.142049:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:6.0:1619677448.142051:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.142052:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.142053:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:17.0:1619677448.142055:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.142058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.142064:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.142067:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.142068:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.142068:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.142069:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.142070:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.142152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.142160:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.142162:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.142165:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.142166:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.142167:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.142169:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677448.142170:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.142172:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.142173:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.142174:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677448.142177:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.142179:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.142182:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.142183:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.142183:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.142185:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.142186:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.142187:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.142189:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.142189:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677448.142191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.142195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.142198:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.142201:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:31.0:1619677448.142202:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:17.0:1619677448.142202:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.142202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677448.142202:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.142204:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.142205:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.142206:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.142206:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677448.142208:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677448.142209:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677448.142211:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677448.142211:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677448.142214:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.142216:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.142218:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677448.142219:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.142220:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.142221:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.142222:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677448.142225:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.142227:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.142230:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.142230:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:17.0:1619677448.142231:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.142233:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677448.142233:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677448.142234:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677448.142235:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677448.142239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.142242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.142247:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.142249:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.142249:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.142250:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.142251:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677448.142252:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.142253:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.142255:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.142256:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.142256:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.142257:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.142258:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.142259:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.142261:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.142262:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677448.142262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:18.0:1619677448.142263:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.142267:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.142269:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.142270:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.142272:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.142273:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.142273:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.142274:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.142274:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.142274:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.142275:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.142275:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677448.142277:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.142278:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.142279:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.142280:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.142280:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.2:1619677448.142286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677448.142290:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:2.0:1619677448.142291:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:26.0:1619677448.142292:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.142293:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:18.0:1619677448.142299:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (23)--
00000400:00000200:26.0:1619677448.142300:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.142301:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.142301:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:26.0:1619677448.142303:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.142307:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.142331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.142338:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.142340:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:21.2:1619677448.142341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:17.0:1619677448.142341:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.142343:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.142345:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.142347:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:1.0:1619677448.142347:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.142348:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.142349:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.142350:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677448.142350:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.142351:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.142353:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.142353:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677448.142356:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.142356:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.142358:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:1.0:1619677448.142358:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:22.0:1619677448.142359:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.142360:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.142362:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.142362:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.142363:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:1.0:1619677448.142363:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.142363:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.142365:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.142365:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677448.142366:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.142366:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.142367:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.142368:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.142371:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.142372:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.142373:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.142375:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.142375:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.142376:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.142377:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677448.142380:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.142383:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.142383:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.142385:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.142387:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.142387:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:12.0:1619677448.142388:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.142390:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.142390:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.142391:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.142393:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.142396:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:4.2:1619677448.142402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.142409:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.142411:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.142412:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.142414:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.142417:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.142418:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.142419:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.142421:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.142422:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.142423:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.142426:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.142428:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.142431:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.142432:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.142433:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.142434:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.142435:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.142436:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.142436:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.142448:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.142451:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.142451:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.142452:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.142454:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.142457:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.142551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.142558:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.142561:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.142562:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.142564:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.142567:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.142568:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.142569:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.142570:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.142571:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.142572:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.142575:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.142576:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.142578:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.142578:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.142579:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.142580:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.142580:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.142581:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.142581:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677448.142591:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.142593:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.142594:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.142594:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.142595:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.142597:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.142620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.142628:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.142630:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.142632:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.142635:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.142635:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.142636:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.142638:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.142639:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.142640:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.142641:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.142643:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.142645:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.142647:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.142648:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.142648:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.142650:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.142651:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.142651:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.142652:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.142662:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.142664:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.142665:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.142666:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.142668:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.142669:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.142831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:4.2:1619677448.142834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677448.142836:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.142839:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.142840:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.142841:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677448.142842:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.142843:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.142844:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677448.142844:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.142846:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.142847:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.142848:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.142849:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.142849:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.142850:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.142851:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.142851:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.142852:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.142855:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.142855:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.142856:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.142857:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.142858:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.142861:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.142861:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.142862:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.142863:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.142864:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.142864:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.142865:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.142865:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.142866:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.142867:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.142868:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.142868:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.142870:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.142871:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.142871:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.142872:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.142881:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.142883:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.142885:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.142885:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.142886:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.142887:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.142887:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.142889:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.142891:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.142892:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677448.142892:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.142893:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.142895:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.142895:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.142896:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.142897:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.142897:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.143070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.143076:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.143079:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.143080:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.143082:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.143085:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.143086:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.143087:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.143088:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.143089:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.143090:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.143093:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.143095:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.143096:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.143097:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.143097:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.143098:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.143099:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.143100:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.143101:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.143102:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.143107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.143112:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.143113:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.143114:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.143115:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.143116:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:10.0:1619677448.143116:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.143117:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.143117:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.143118:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.143118:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677448.143188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.143194:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677448.143194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:18.0:1619677448.143197:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.143197:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.143200:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.143201:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:18.0:1619677448.143203:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.143204:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:18.0:1619677448.143204:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:10.0:1619677448.143205:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:18.0:1619677448.143206:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.143208:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.143209:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.143209:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.143210:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.143211:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.143212:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.143214:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718783] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.143214:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.143215:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.143217:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.143217:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.143218:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.143218:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.143219:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.143220:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677448.143222:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.143222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:18.0:1619677448.143222:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.143222:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.143223:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.143223:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:26.0:1619677448.143224:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.143227:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.143228:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:9.2:1619677448.143228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.143229:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.143230:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.143231:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.2:1619677448.143231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:26.0:1619677448.143232:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.143232:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.143233:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:10.0:1619677448.143234:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:26.0:1619677448.143235:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:2.0:1619677448.143235:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.143236:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.143237:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.143237:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.143237:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.143238:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.143238:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.143239:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:18.0:1619677448.143240:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.2:1619677448.143240:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:10.0:1619677448.143241:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96718782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.143241:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.143242:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:10.0:1619677448.143242:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.143243:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677448.143243:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:10.0:1619677448.143244:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.143244:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:12.0:1619677448.143246:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:10.0:1619677448.143246:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:12.0:1619677448.143247:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.143247:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677448.143248:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.143249:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:5.0:1619677448.143249:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.143250:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:13.0:1619677448.143250:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.143251:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:5.0:1619677448.143251:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:26.0:1619677448.143252:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:10.0:1619677448.143252:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000400:00000200:5.0:1619677448.143253:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96718781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.143255:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:5.0:1619677448.143255:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.143256:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.143256:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.143257:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:10.0:1619677448.143258:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.143258:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.143259:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.143261:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.143261:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:5.0:1619677448.143261:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.143262:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.143263:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:5.0:1619677448.143263:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677448.143265:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.143266:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677448.143267:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677448.143268:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677448.143269:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.143270:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.143271:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.143272:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677448.143273:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.143275:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:9.2:1619677448.143275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:13.0:1619677448.143276:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.143277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:13.0:1619677448.143277:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677448.143279:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.143280:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677448.143281:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:18.0:1619677448.143282:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:20.0:1619677448.143283:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.143284:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.143285:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677448.143285:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.143286:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.143286:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.143288:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.143290:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.143315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:30.0:1619677448.143324:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677448.143327:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:30.0:1619677448.143329:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.143331:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:30.0:1619677448.143331:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.143332:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.143334:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.143335:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.143336:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.143337:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.143341:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.143343:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.143345:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.143346:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.143347:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.143348:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.143349:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677448.143350:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677448.143350:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:8.2:1619677448.143355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.143363:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.143366:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.143367:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.143368:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.143369:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.143371:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.143393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.143400:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.143403:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.143404:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.143405:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.143409:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.143411:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.143412:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.143413:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.143414:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.143415:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.143418:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.143420:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.143422:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.143423:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.143424:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.143426:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.143426:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.143427:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.143428:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.143433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.143441:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.143443:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.143445:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.143445:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.143447:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.143450:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.143463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.143469:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.143472:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.143472:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.143474:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.143477:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.143479:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.143480:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.143481:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.143482:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.143483:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.143487:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.143489:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.143491:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.143492:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.143493:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.143495:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.143495:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.143496:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.143497:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.143498:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.143501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.143503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.143507:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677448.143508:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677448.143510:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:17.0:1619677448.143510:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.0:1619677448.143510:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.143511:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.143511:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.143512:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677448.143513:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.143517:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.143521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:21.2:1619677448.143522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:8.0:1619677448.143527:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:18.0:1619677448.143528:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:24.0:1619677448.143530:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.143530:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:18.0:1619677448.143531:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:8.0:1619677448.143531:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (21)++
00000800:00000200:2.0:1619677448.143531:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.0:1619677448.143532:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.143532:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:18.0:1619677448.143534:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.143536:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.143537:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677448.143539:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718777] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.143540:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.143541:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.143542:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677448.143545:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.143547:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.143550:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.143550:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677448.143551:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677448.143553:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.143553:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.143554:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.143555:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.143556:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.2:1619677448.143561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:6.0:1619677448.143566:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:4.2:1619677448.143567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677448.143567:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:6.0:1619677448.143569:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677448.143569:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.143569:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677448.143570:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.143570:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.143571:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:6.0:1619677448.143573:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:26.0:1619677448.143575:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.143575:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:26.0:1619677448.143578:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.143579:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.143580:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.143583:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.143585:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:22.0:1619677448.143585:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:26.0:1619677448.143586:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.143587:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.143587:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.143588:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.143588:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.143588:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.143589:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000400:00000200:26.0:1619677448.143590:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.143590:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:26.0:1619677448.143593:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.143594:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.143597:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.143598:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.143599:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.143601:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.143601:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.143602:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.143602:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.143614:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.143616:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.143617:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.143617:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.143619:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.143622:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.143692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.143700:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.143702:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.143703:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.143704:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.143708:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.143709:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.143710:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718775] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.143711:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.143712:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.143713:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.143715:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.143716:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.143717:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.143718:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.143718:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.143720:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.143720:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.143721:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.143721:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677448.143731:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.143733:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.143733:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.143734:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.143735:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.143736:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.143750:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.143752:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.143753:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.143753:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.143753:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.143754:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.143980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.143986:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.143988:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.143989:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.143991:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.143994:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.143995:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.143997:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.143998:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.143999:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.144000:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.144003:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.144005:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.144008:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.144009:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.144009:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.144011:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.144011:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.144012:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.144013:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:26.0:1619677448.144023:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.144025:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.144026:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.144027:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.144029:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.144031:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:4.2:1619677448.144048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.144055:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.144057:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.144058:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.144060:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.144062:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.144063:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.144064:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718773] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.144065:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.144066:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.144067:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.144070:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.144072:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.144073:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.144074:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.144075:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.144076:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.144076:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.144077:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.144077:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.144087:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.144089:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.144089:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.144090:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.144091:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.144092:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.144117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.144124:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.144127:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.144129:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.144130:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.144131:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.144132:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677448.144133:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.144135:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.144136:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.144137:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677448.144140:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.144141:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677448.144144:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.144145:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677448.144146:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.144147:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677448.144148:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.144149:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.144149:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.144160:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.144163:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.144164:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.144164:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.144167:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.144170:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.144232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:5.0:1619677448.144238:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:5.2:1619677448.144241:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677448.144243:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:5.0:1619677448.144245:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:5.0:1619677448.144247:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.144249:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:5.0:1619677448.144249:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.144251:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:26.0:1619677448.144252:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.144252:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:5.0:1619677448.144252:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.144254:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.144255:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.144255:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.144256:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.144256:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677448.144258:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:5.2:1619677448.144259:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677448.144260:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.144261:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.144262:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.144264:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:5.0:1619677448.144265:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.144265:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.144268:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.144268:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:1.0:1619677448.144268:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:10.0:1619677448.144269:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.144270:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.144271:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000400:00000200:1.0:1619677448.144271:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.144272:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:5.0:1619677448.144272:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.144274:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.144274:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:10.0:1619677448.144275:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.144275:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:5.0:1619677448.144275:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000400:00000200:1.0:1619677448.144275:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.144276:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677448.144276:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000400:00000200:1.0:1619677448.144276:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:10.0:1619677448.144278:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.144278:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:1.0:1619677448.144278:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96718769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.144279:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.144279:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:1.0:1619677448.144279:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677448.144280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000400:00000200:1.0:1619677448.144280:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.144281:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.144284:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.144286:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.144287:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:1.0:1619677448.144287:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.144288:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.144289:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:1.0:1619677448.144289:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677448.144290:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.144290:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677448.144291:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677448.144291:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.144291:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677448.144292:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.144292:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677448.144293:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:8.2:1619677448.144299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677448.144300:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:20.0:1619677448.144301:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:13.0:1619677448.144301:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:12.0:1619677448.144301:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.144303:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:12.0:1619677448.144303:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.144304:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677448.144305:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677448.144305:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.144306:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.144307:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:12.0:1619677448.144307:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677448.144309:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.144310:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.144312:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.144312:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.144314:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.144356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.144363:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.144365:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.144366:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.144368:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.144371:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.144372:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.144374:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.144375:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.144376:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.144377:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.144380:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.144382:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.144385:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.144386:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.144386:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.144388:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.144388:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.144389:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.144389:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.144393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.144402:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.144404:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.144405:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.144405:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.144407:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.144410:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.144567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.144575:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.144578:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.144580:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.144583:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.144583:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.144585:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.144586:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.144587:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.144588:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.144589:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.144592:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.144593:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.144596:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.144596:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.144597:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.144598:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.144599:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.144600:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.144600:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.2:1619677448.144601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.144608:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.144611:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:13.0:1619677448.144611:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677448.144611:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.144614:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.144614:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.144614:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.144616:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.144616:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.144617:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.144618:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.144618:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.144618:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:18.0:1619677448.144620:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.144620:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.144622:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.144625:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.144627:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.144628:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.144629:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.144630:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.144631:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.144632:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.144633:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.144634:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.144634:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677448.144637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.144640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.144646:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.144647:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.144647:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.144648:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:2.0:1619677448.144648:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.144649:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.144650:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.144650:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.144651:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677448.144651:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.144755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.144762:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.144765:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.144765:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677448.144767:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.144770:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.144772:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.144773:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.144774:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.144775:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.144776:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.144780:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.144782:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.144784:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.144785:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.144785:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.144787:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.144787:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:4.2:1619677448.144788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677448.144789:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.144790:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.144791:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:26.0:1619677448.144796:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.2:1619677448.144796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:26.0:1619677448.144798:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:12.0:1619677448.144799:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:10.0:1619677448.144799:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.144801:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.144802:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.144802:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.144803:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.144803:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:12.0:1619677448.144804:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.144805:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:24.0:1619677448.144805:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:26.0:1619677448.144806:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.144806:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677448.144806:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.144807:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677448.144807:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:26.0:1619677448.144808:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.144809:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.144810:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.144813:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.144815:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.144817:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.144817:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.144818:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.144820:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.144820:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.144821:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.144821:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.144832:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.144834:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.144835:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.144837:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.144838:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.144840:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.144854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.144860:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.144863:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.144865:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.144865:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.144867:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.144869:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.144870:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718763] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.144871:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.144872:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.144873:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.144877:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.144879:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.144881:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.144882:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.144882:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.144884:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.144884:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.144885:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.144885:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677448.144898:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.144900:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.144901:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.144902:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.144903:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.144905:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.145163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.145170:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.145173:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.145175:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.145177:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.145177:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.145179:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.145180:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.145181:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.145182:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.145183:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.145186:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.145188:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.145190:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.145191:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.145192:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.145193:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.145194:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.145195:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.145195:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.145205:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.145207:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.145208:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.145209:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.145211:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.145212:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.145218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.145225:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.145227:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.145228:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:12.0:1619677448.145228:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.145230:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.145230:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.145231:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.145232:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.145232:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.145233:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.145234:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677448.145234:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.145235:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.145236:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.145237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:10.0:1619677448.145237:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.145238:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.145240:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:10.0:1619677448.145241:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.145243:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:10.0:1619677448.145243:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.145245:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.145246:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.145246:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.145246:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677448.145247:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:20.0:1619677448.145248:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.145248:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:20.0:1619677448.145249:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:10.0:1619677448.145249:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.145250:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:20.0:1619677448.145251:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718760] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.145251:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000400:00000200:20.0:1619677448.145252:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.145252:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.2:1619677448.145253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:20.0:1619677448.145253:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.145254:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:4.2:1619677448.145257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.145258:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:20.0:1619677448.145259:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.145261:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.145261:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.145261:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.145262:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:31.0:1619677448.145263:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677448.145263:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.145264:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.145264:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:20.0:1619677448.145264:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:6.0:1619677448.145264:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.145265:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:31.0:1619677448.145266:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.145266:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:30.0:1619677448.145267:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.145267:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677448.145267:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.145267:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:6.0:1619677448.145267:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.145268:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.145268:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677448.145268:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:6.0:1619677448.145268:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.0:1619677448.145269:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:6.0:1619677448.145270:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.145271:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.145272:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.145273:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.145277:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.145279:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.145281:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.145282:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.145283:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.145283:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.145284:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.145284:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:22.0:1619677448.145285:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.145286:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.145287:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.145287:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.145288:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.145289:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677448.145289:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.145293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.145302:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.145305:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.145306:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.145306:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.145308:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.145311:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.145315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.145323:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.145325:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.145326:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.145327:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.145327:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.145328:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.145354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.145360:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.145362:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.145363:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.145365:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.145366:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.145368:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.145369:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.145371:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.145371:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.145373:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.145376:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.145378:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.145380:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.145381:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.145382:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.145384:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.145384:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.145385:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.145385:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.145391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.145397:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.145400:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.145401:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.145401:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.145403:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.145406:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.2:1619677448.145429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.145437:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.145440:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.145440:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.145442:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.145445:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.145446:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.145447:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.145449:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.145450:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.145451:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.145454:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.145456:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.145459:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.145460:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.145461:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.145462:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.145463:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.145463:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.145464:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677448.145477:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.145479:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.145480:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.145481:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.145483:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.145486:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.145582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.145590:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.145593:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.145595:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.145597:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.145597:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.145599:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.145600:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.145601:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.145602:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.145603:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.145607:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.145608:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.145610:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.145611:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.145612:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.145613:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.145614:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:21.2:1619677448.145614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.145615:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.145616:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.145617:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.145619:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677448.145621:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.145621:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.145622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:30.0:1619677448.145623:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.145624:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.145625:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.145627:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.145627:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.145627:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677448.145628:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.145630:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:30.0:1619677448.145633:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.145633:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.145633:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.145636:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:13.0:1619677448.145636:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:5.0:1619677448.145636:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.145638:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.145638:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:5.0:1619677448.145638:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677448.145639:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:13.0:1619677448.145639:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.145639:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.145639:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96718754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.145640:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.145641:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:5.0:1619677448.145641:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.145642:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:13.0:1619677448.145642:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.145642:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.145643:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:5.0:1619677448.145643:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.145644:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.145644:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677448.145644:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:5.0:1619677448.145647:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.145650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.145650:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.145652:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:5.0:1619677448.145653:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.145654:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.145655:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.145655:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.145655:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.145655:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:26.0:1619677448.145657:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.145657:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.145658:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.145658:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677448.145659:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:5.0:1619677448.145659:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677448.145660:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677448.145661:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.145662:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.145663:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.145669:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.145672:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.145673:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.145675:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.145676:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.145678:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.145746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.145753:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.145756:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.145757:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.145759:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.145761:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.145763:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.145764:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718753] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.145765:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.145766:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.145767:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.145771:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.145773:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.145776:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.145777:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.145777:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.145779:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.145780:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.145781:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.145781:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677448.145792:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.145795:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.145796:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.145798:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.145799:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.145801:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.145880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.145887:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.145889:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.145890:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.145892:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.145895:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.145896:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.145897:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.145898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:10.0:1619677448.145899:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.145899:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.145900:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.145901:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.145903:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:10.0:1619677448.145904:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.145906:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.145906:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:10.0:1619677448.145907:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.145909:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.145909:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.145910:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:20.0:1619677448.145911:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:10.0:1619677448.145911:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:20.0:1619677448.145912:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.145912:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:20.0:1619677448.145913:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.145913:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.145913:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677448.145914:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.145914:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:20.0:1619677448.145915:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.145919:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.145921:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.145924:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.145924:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.145925:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.145925:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (24)++
00000800:00000200:20.0:1619677448.145928:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (25)--
00000800:00000200:31.0:1619677448.145929:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:20.0:1619677448.145929:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.145929:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000400:00000200:31.0:1619677448.145930:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.145930:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677448.145931:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.145932:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.145936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:31.0:1619677448.145937:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.145938:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.145940:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.145941:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.145941:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.145941:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:31.0:1619677448.145943:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.145944:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:31.0:1619677448.145945:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:22.0:1619677448.145945:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.145947:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.145949:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.145951:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.145952:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.145953:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.145954:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.145955:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.145957:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:6.0:1619677448.145959:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.145960:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.145961:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:6.0:1619677448.145961:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.145962:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.145962:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.145964:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:6.0:1619677448.145964:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.145965:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.145966:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.145968:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.145969:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.145970:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.145971:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.145972:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:21.2:1619677448.145977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677448.145980:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677448.145982:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:31.0:1619677448.145983:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.145984:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.145984:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.145984:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.145985:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:31.0:1619677448.145986:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.145987:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.145989:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.146094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.2:1619677448.146099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677448.146102:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.146104:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.146105:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.146106:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677448.146107:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.146109:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677448.146109:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.146111:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.146111:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:31.0:1619677448.146113:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677448.146113:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.146114:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.146114:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.146115:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.146116:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.146117:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.146117:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677448.146119:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.146120:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.146121:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.146122:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.146125:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.146125:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.146127:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.146128:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.146128:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:30.0:1619677448.146129:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.146129:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.146130:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.146130:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.146131:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677448.146132:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.146132:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.146132:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.146133:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.146133:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:30.0:1619677448.146134:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.146134:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.146134:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677448.146138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.146140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.146145:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.146146:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:12.0:1619677448.146147:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:10.0:1619677448.146147:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:12.0:1619677448.146148:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.146148:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:12.0:1619677448.146149:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.146149:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:18.0:1619677448.146150:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.146150:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:2.0:1619677448.146150:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.146151:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.146151:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677448.146152:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.146153:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.146156:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.146158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:9.2:1619677448.146160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:31.0:1619677448.146166:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677448.146167:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.146168:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:18.0:1619677448.146170:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.146170:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:31.0:1619677448.146171:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.146171:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.146172:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:18.0:1619677448.146173:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000400:00000200:31.0:1619677448.146174:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.146174:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.146175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:31.0:1619677448.146176:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:20.2:1619677448.146176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:31.0:1619677448.146177:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718747] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.146178:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.146179:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.146180:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:2.0:1619677448.146181:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.0:1619677448.146182:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.146184:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.146184:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.146185:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:5.0:1619677448.146185:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:31.0:1619677448.146186:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677448.146187:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.146187:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.146189:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.146189:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677448.146190:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.146190:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:5.0:1619677448.146190:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.146192:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.146192:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:5.0:1619677448.146192:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.146192:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.146193:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.146194:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.146194:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:2.0:1619677448.146194:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.146195:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.146195:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.146197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197eb55c600] (20)++
00000400:00000200:5.0:1619677448.146197:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.146197:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.146198:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.146199:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.146199:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.146203:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197eb55c600] (21)++
00000800:00000200:30.0:1619677448.146205:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:20.0:1619677448.146205:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.58@o2ib3
00000400:00000200:5.0:1619677448.146205:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.146205:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:30.0:1619677448.146207:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:2.0:1619677448.146207:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.146208:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.146208:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.2.58@o2ib3 : GET - for me
00000800:00000200:5.0:1619677448.146208:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.146210:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.146211:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677448.146211:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.146211:0:19934:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.2.58@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000800:00000200:2.0:1619677448.146211:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.146212:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.146212:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.146212:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:20.0:1619677448.146213:0:19934:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.2.58@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:5.0:1619677448.146213:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.146213:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.146214:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:2.0:1619677448.146214:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.0:1619677448.146215:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197eb55c600] (22)++
00000800:00000200:5.0:1619677448.146215:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.146215:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.0:1619677448.146216:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197eb55c600] (23)--
00000800:00000200:5.0:1619677448.146216:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.146216:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:20.0:1619677448.146218:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.2.58@o2ib3 local destination
00000400:00000200:20.0:1619677448.146222:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.2.58@o2ib3(10.51.2.58@o2ib3:10.51.2.58@o2ib3) : REPLY try# 0
00000800:00000200:5.0:1619677448.146223:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.146224:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.2.58@o2ib3
00000800:00000200:5.0:1619677448.146225:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.146225:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.146226:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9197e5a60980] -> 10.51.2.58@o2ib3 (2) version: 12
00000800:00000200:20.0:1619677448.146227:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197eb55c600] (22)++
00000800:00000200:6.0:1619677448.146227:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.146227:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c600] (22)--
00000800:00000200:20.0:1619677448.146228:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197eb55c600] (22)++
00000800:00000200:6.0:1619677448.146229:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.146230:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.2:1619677448.146230:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677448.146230:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677448.146231:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.146232:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197eb55c600] (23)--
00000800:00000200:20.0:1619677448.146233:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c600] (22)--
00000800:00000200:13.0:1619677448.146233:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:6.0:1619677448.146233:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:20.0:1619677448.146234:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:13.0:1619677448.146234:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.146235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197eb55c600] (21)++
00000800:00000200:20.0:1619677448.146235:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:13.0:1619677448.146235:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.146235:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677448.146236:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677448.146237:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.146238:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.146238:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.146238:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197eb55c600] (22)++
00000800:00000200:5.0:1619677448.146241:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197eb55c600] (23)--
00000400:00000200:5.0:1619677448.146242:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.146244:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c600] (22)--
00000400:00000200:5.0:1619677448.146244:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.2.58@o2ib3: REPLY: OK
00000800:00000200:5.0:1619677448.146246:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c600] (21)--
00000800:00000200:12.0:1619677448.146249:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.146252:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.146253:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.146253:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.146254:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.146255:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.146326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.146332:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.146335:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.146335:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.146338:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.146341:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.146342:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.146343:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.146345:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.146346:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.146347:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.146350:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.146352:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.146355:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.146356:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.146356:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.146358:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.146359:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.146359:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.146360:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.146365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.146372:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.146375:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.146376:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.146376:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.146378:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.146381:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677448.146487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.146494:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.146497:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.146498:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.146499:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.146501:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.146503:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.146504:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.146505:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.146506:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.146507:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.146511:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.146512:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.146515:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.146516:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.146516:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.146518:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.146518:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.146519:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.146519:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.146524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.146532:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.146535:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.146535:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.146536:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.146538:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.146541:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.146591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:22.0:1619677448.146597:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677448.146600:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677448.146601:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677448.146603:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.146604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:22.0:1619677448.146606:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.146608:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:6.0:1619677448.146609:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:22.0:1619677448.146610:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.146611:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.146611:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:22.0:1619677448.146612:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.146613:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677448.146616:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.146618:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:6.0:1619677448.146619:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.146620:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.146621:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.146622:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:6.0:1619677448.146622:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.146623:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:22.0:1619677448.146624:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.146624:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.146626:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.146626:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.146627:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.146628:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677448.146629:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677448.146631:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000400:00000200:6.0:1619677448.146631:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.146633:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.146633:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.146634:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.146635:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.146635:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.146637:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.146637:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.146638:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.2:1619677448.146638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:6.0:1619677448.146638:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.146638:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.146639:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677448.146641:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.146641:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.146642:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.146642:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677448.146646:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.146649:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.146651:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.146653:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.146653:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.146655:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:30.0:1619677448.146655:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:31.0:1619677448.146656:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.146656:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.146657:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.146657:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.146658:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.146658:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:31.0:1619677448.146659:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.146659:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.146660:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.146661:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.146661:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.2:1619677448.146661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677448.146662:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000400:00000200:31.0:1619677448.146663:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.146663:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.146663:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677448.146664:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.146665:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.146667:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.146668:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.146669:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:1.0:1619677448.146669:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677448.146670:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677448.146670:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.146672:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.146672:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:1.0:1619677448.146672:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.146673:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:1.0:1619677448.146674:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.146675:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.146676:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:1.0:1619677448.146676:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.146677:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.146678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:1.0:1619677448.146678:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.146680:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.146681:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.2:1619677448.146682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:1.0:1619677448.146684:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.146686:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.146689:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.146689:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.146690:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.146690:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:1.0:1619677448.146690:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.146692:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:1.0:1619677448.146692:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.146692:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.146693:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677448.146694:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.146694:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.146694:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.146694:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.146695:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000400:00000200:20.0:1619677448.146696:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.146696:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.146697:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.146699:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.146700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.146704:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677448.146706:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:30.0:1619677448.146707:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.146708:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.146708:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.146709:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.146709:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677448.146710:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.146711:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.146713:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.146798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.146805:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.146808:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.146810:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.146812:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.146813:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.146815:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.146816:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.146817:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.146818:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.146819:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.146822:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.146824:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.146827:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.146828:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.146829:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.146830:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.146831:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.146832:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.146832:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.146844:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.146847:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.146847:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.146848:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.146850:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.146853:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.146946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.146951:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.146954:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.146955:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.146957:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.146960:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.146961:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.146962:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718737] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.146963:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.146964:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.146965:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.146968:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.146969:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.146971:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.146972:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.146973:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.146974:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.146975:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.146975:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.146976:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.146980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.146988:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.146990:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.146991:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.146992:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.146992:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.146994:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.147012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.147020:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.147023:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.147023:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.147026:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.147027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000400:00000200:10.0:1619677448.147028:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.147030:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.147031:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.147032:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.147033:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.147034:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:10.0:1619677448.147034:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.147037:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.147038:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.147038:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.147039:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000400:00000200:10.0:1619677448.147039:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.147040:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:10.0:1619677448.147041:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.147044:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.147045:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.147045:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.147047:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.147048:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.147048:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.147049:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.147061:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.147064:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.147065:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.147066:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.147067:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.147070:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.147128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.147135:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.147138:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.147138:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:21.2:1619677448.147140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:18.0:1619677448.147141:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.147144:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.147146:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:2.0:1619677448.147146:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.147147:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:18.0:1619677448.147148:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.147149:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.147150:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.147151:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.147154:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.147156:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.147156:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.147157:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:18.0:1619677448.147158:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.147158:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.147160:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.147160:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.147161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000400:00000200:2.0:1619677448.147161:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.147164:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.147164:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.147165:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.147166:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.147166:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.147167:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:2.0:1619677448.147168:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.147169:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.147169:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.147169:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.147170:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677448.147170:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:2.0:1619677448.147170:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.147171:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677448.147171:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.147171:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.147172:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:5.0:1619677448.147172:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.147172:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:20.0:1619677448.147173:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:2.0:1619677448.147173:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:5.0:1619677448.147175:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.147176:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:5.0:1619677448.147178:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96718733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.147179:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.147180:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.147181:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.147182:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.147184:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:5.0:1619677448.147184:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677448.147185:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.147187:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.147187:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:12.0:1619677448.147188:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677448.147189:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:5.0:1619677448.147189:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.147190:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.147190:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.147190:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.147191:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:13.0:1619677448.147192:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.147192:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.147192:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.147193:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:12.0:1619677448.147194:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.147194:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708600] (22)++
00000800:00000200:5.0:1619677448.147195:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (23)--
00000800:00000200:5.0:1619677448.147196:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:8.2:1619677448.147198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.147202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677448.147208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:30.0:1619677448.147209:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.147212:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.147214:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.147215:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.147215:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677448.147216:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677448.147216:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.147216:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.147217:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.147217:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (22)++
00000400:00000200:30.0:1619677448.147218:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:20.0:1619677448.147218:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708600] (23)--
00000400:00000200:5.0:1619677448.147218:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.147219:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:30.0:1619677448.147220:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.147220:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:30.0:1619677448.147221:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.147221:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:30.0:1619677448.147222:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.147222:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:5.0:1619677448.147222:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.147223:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.147223:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:30.0:1619677448.147224:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:5.2:1619677448.147228:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677448.147229:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.147231:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:5.0:1619677448.147231:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.147233:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.147233:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.147234:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:5.0:1619677448.147234:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:1.0:1619677448.147234:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.147235:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:5.0:1619677448.147235:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.147236:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.147236:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:30.0:1619677448.147237:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677448.147237:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677448.147237:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.147238:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.147238:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:5.0:1619677448.147239:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.147240:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.147241:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.147242:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.147244:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.147244:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.147246:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96718730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.147247:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.147248:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.147249:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:1.0:1619677448.147249:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.147250:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.147250:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:13.0:1619677448.147251:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:5.0:1619677448.147251:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:13.0:1619677448.147252:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.147252:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.147253:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:13.0:1619677448.147254:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.147254:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:1.0:1619677448.147254:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.147255:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677448.147255:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.147256:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:1.0:1619677448.147256:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:13.0:1619677448.147257:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677448.147259:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.147260:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.147260:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.147262:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.147263:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.147263:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.147264:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.147266:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.147269:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.147270:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.147270:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:31.0:1619677448.147271:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.147273:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:12.0:1619677448.147273:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.147274:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:12.0:1619677448.147274:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.147274:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.147275:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.147276:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677448.147276:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.147279:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.147280:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.147280:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.147282:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:6.0:1619677448.147282:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.147284:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:30.0:1619677448.147285:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.147286:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.147287:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.147287:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.147289:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:22.0:1619677448.147295:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.147301:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.147302:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.147303:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.147303:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.147304:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.147441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.147447:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.147450:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:4.2:1619677448.147450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677448.147451:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.147453:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.147454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:17.0:1619677448.147455:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.147456:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.147457:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.147459:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.147460:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677448.147460:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.147461:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.147462:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.147462:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.147464:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:17.0:1619677448.147465:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.147466:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.147467:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.147467:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.147469:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.147470:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.147470:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:10.0:1619677448.147470:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.147471:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:17.0:1619677448.147471:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:26.0:1619677448.147472:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.147472:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.147473:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.147474:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.147474:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.147474:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677448.147475:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.147476:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.147478:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.147479:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96718727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.147480:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.147481:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.147482:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.147483:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.147483:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.147486:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.147486:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.147487:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.147487:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:26.0:1619677448.147488:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.147488:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.147490:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.147490:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:26.0:1619677448.147491:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.147491:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.147491:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.147491:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:6.0:1619677448.147491:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.147492:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.147492:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.147493:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.147494:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.147494:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.147494:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:6.0:1619677448.147494:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.147495:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.147496:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.147496:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:6.0:1619677448.147497:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.147505:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.147508:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.147508:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.147508:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677448.147508:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.147510:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.147510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:12.0:1619677448.147511:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.147512:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.147513:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:12.0:1619677448.147514:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.147516:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:21.2:1619677448.147517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:12.0:1619677448.147517:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.147518:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.147520:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.147521:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.147523:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.147525:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677448.147525:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.147526:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:31.0:1619677448.147528:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677448.147528:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.147530:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.147531:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.147531:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.147532:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677448.147532:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.147535:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.147536:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.147537:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.147538:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:26.0:1619677448.147539:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.147539:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:31.0:1619677448.147540:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.147540:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:31.0:1619677448.147541:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.147541:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:31.0:1619677448.147542:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.147542:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.147542:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.147543:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.147543:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:31.0:1619677448.147545:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.147547:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677448.147550:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.147551:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677448.147551:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.147553:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677448.147553:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.147553:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.147554:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.147554:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.147555:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:30.0:1619677448.147555:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.147556:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677448.147557:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.147559:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:6.0:1619677448.147566:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.147569:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.147570:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.147570:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.147573:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.147575:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.147643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.147650:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.147653:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.147654:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.147656:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.147659:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.147661:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.147662:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.147663:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.147665:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.147665:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.147669:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.147671:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.147673:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.147674:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.147674:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.147676:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.147676:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.147677:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.147678:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.147683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.147691:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.147693:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.147694:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.147694:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.147697:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.147699:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.147700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677448.147704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:17.0:1619677448.147705:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.147708:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.147708:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.147709:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000400:00000200:17.0:1619677448.147710:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.147712:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.147712:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677448.147713:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:17.0:1619677448.147714:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.147715:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.147715:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.147717:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.147718:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.147718:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.147719:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677448.147720:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677448.147722:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718722] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.147723:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.147724:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.147725:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:2.0:1619677448.147725:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.147726:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.147729:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.147729:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.147730:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.147730:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:2.0:1619677448.147731:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.147732:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.147733:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.147734:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.147735:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:2.0:1619677448.147735:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.147736:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.147736:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:2.0:1619677448.147736:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:2.0:1619677448.147738:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:2.0:1619677448.147739:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677448.147739:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:5.2:1619677448.147740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.147740:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677448.147744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.147745:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:1.0:1619677448.147745:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677448.147748:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:1.0:1619677448.147748:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.147749:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.147749:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677448.147750:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.147751:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:31.0:1619677448.147752:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.147752:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:24.0:1619677448.147754:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:31.0:1619677448.147755:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:24.0:1619677448.147755:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.147755:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.147757:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.147761:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.148007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.148014:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.148017:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.148018:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.148020:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.148023:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.148024:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.148025:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.148026:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.148027:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.148028:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.148032:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.148034:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.148037:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.148038:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.148038:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.148040:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.148040:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.148041:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.148043:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.148043:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.2:1619677448.148044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:4.2:1619677448.148049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.148052:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.148054:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:22.0:1619677448.148054:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:30.0:1619677448.148056:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.148056:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.148057:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677448.148058:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677448.148058:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677448.148058:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.148059:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.148059:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677448.148059:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:30.0:1619677448.148061:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:26.0:1619677448.148061:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:30.0:1619677448.148062:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.148062:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.148064:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.148065:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.148065:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.148065:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:30.0:1619677448.148069:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.148070:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.148072:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.148073:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.148073:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.148075:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.148075:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.148076:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:6.0:1619677448.148076:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.148077:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:6.0:1619677448.148079:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.148080:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.148080:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.148081:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.148082:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677448.148087:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.148090:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.148091:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.148093:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.148094:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.148095:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.148122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.148129:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.148132:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.148133:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.148135:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.148136:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.148137:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.148138:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.148140:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.148141:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.148142:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.2:1619677448.148142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:31.0:1619677448.148144:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.148146:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.148148:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.148148:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.148149:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.148149:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.148151:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.148151:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.148152:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677448.148153:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:17.0:1619677448.148153:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.148154:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:1.0:1619677448.148154:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.148155:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:1.0:1619677448.148157:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.148158:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.148159:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.148160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677448.148161:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.148161:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.148162:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677448.148162:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:12.0:1619677448.148165:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.148165:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.148166:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677448.148167:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.148168:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:1.0:1619677448.148168:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:13.0:1619677448.148169:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.148170:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.148171:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:1.0:1619677448.148171:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.148172:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677448.148172:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.148172:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.148173:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677448.148175:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.148175:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.148176:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.148176:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.148187:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.148190:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.148191:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.148191:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677448.148192:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.148193:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.148194:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:13.0:1619677448.148194:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.148195:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:13.0:1619677448.148195:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.148195:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.148197:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.148358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86fbd5800] (20)++
00000800:00000200:31.0:1619677448.148366:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86fbd5800] (21)++
00000800:00000200:31.0:1619677448.148369:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.18@o2ib3
00000400:00000200:31.0:1619677448.148371:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.148373:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86fbd5800] (22)--
00000400:00000200:31.0:1619677448.148374:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.148375:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.148377:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.148378:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.148379:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.148380:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.148383:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.148385:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 504 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.148389:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.148389:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.148390:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.148392:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.148392:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86fbd5800] (21)++
00000800:00000200:31.0:1619677448.148393:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86fbd5800] (22)--
00000800:00000200:31.0:1619677448.148394:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86fbd5800] (21)--
00000800:00000200:5.2:1619677448.148402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677448.148406:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.148409:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677448.148410:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.148410:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.148411:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.148413:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.148413:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.148414:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677448.148416:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:10.0:1619677448.148416:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.148419:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.148420:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.148422:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.148423:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.148424:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.148425:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:21.2:1619677448.148426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000400:00000200:10.0:1619677448.148428:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.148430:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.148432:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.148432:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:18.0:1619677448.148433:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:10.0:1619677448.148433:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.148435:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.148435:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.148436:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.148436:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:18.0:1619677448.148437:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677448.148438:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677448.148440:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.148442:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.148444:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677448.148445:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.148447:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.148447:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:4.2:1619677448.148447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:18.0:1619677448.148448:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.148449:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:12.0:1619677448.148449:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.148450:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.148453:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677448.148453:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.148454:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677448.148454:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677448.148455:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:12.0:1619677448.148455:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.148457:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.148458:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677448.148459:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.148460:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.148460:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:18.0:1619677448.148460:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:18.0:1619677448.148462:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:26.0:1619677448.148463:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.148463:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.148463:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:26.0:1619677448.148464:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:18.0:1619677448.148464:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:26.0:1619677448.148465:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.148466:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.148468:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.148469:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.148472:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.148474:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.148475:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.148477:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:26.0:1619677448.148477:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.148477:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:30.0:1619677448.148478:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.148478:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:30.0:1619677448.148480:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.148480:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.148480:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.148481:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.148481:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.148481:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.148483:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:6.0:1619677448.148494:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.148496:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.148497:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.148498:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.148500:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.148503:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.148532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.148537:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.148540:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.148541:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.148543:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.148546:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.148547:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.148548:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.148550:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.148551:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.148552:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.148554:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.148556:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.148558:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.148559:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.148559:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.148561:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.148561:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.148562:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.148562:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.148567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.148574:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.148576:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.148577:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.148577:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.148579:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.148580:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.148700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.148705:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.148707:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.148708:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.148709:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.148711:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.148713:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.148714:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718712] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.148715:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.148716:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.148717:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.148720:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.148721:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.148724:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.148725:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.148725:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.148727:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.148728:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.148728:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.148729:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:10.0:1619677448.148740:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.148742:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.148743:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.148744:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.148746:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.148749:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.148845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677448.148851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.148853:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.148855:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.148858:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.148858:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.148859:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.148860:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.148860:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.148861:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677448.148862:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677448.148863:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.148864:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.148865:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.148866:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.148867:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677448.148867:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.148869:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.148870:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.148871:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.148872:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.148873:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.148874:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:26.0:1619677448.148875:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.148877:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.148878:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.148878:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677448.148878:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.148880:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:26.0:1619677448.148880:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.148881:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.148882:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.148882:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.148883:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.148884:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.148884:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.148886:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.148887:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.148887:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.148887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.148888:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.148892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.148896:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.148899:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677448.148900:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:17.0:1619677448.148900:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.148900:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.148903:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:17.0:1619677448.148903:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.148903:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.148904:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.148906:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.148906:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677448.148910:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.148952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.148959:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.148962:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.148963:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.148964:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.148967:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.148968:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.148970:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.148971:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.148972:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.148973:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.148976:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.148977:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.148979:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.148979:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.148980:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.148981:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.148982:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.148982:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.148983:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677448.148987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:26.0:1619677448.148993:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:22.0:1619677448.148993:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.148994:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.148995:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.148996:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:10.0:1619677448.148996:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.148997:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.148997:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677448.148999:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:22.0:1619677448.148999:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.149002:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.149003:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.149004:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.149006:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.149007:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.149008:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.149011:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.149013:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.149013:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.149016:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:22.0:1619677448.149016:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.149016:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:22.0:1619677448.149017:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.149017:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.149017:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.149018:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:22.0:1619677448.149019:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.149019:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:22.0:1619677448.149020:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.149021:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.149022:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:26.0:1619677448.149031:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.149034:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.149035:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.149035:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.149037:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.149040:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.149187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.149193:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.149196:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.149196:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.149199:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.149201:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.149202:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.149204:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.149205:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.149206:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.149207:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.149210:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.149212:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.149213:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.149214:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.149215:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.149216:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.149217:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.149217:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.149218:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.149222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.149229:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.149231:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.149231:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.149232:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.149233:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.149235:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.149240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.149247:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.149249:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.149250:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.149250:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.149250:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.149251:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.149379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:4.2:1619677448.149379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.149387:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677448.149387:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.149390:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:10.0:1619677448.149390:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:8.0:1619677448.149390:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:26.0:1619677448.149391:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.149393:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.149393:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.149396:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.149396:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.149397:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:8.0:1619677448.149398:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.149399:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.149399:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.149400:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.149401:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.149401:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.149402:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:8.0:1619677448.149402:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.149403:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.149407:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.149407:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.149409:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.0:1619677448.149409:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.149411:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.149411:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.149412:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:8.0:1619677448.149412:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.149413:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.149414:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.149414:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677448.149414:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.149415:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.149416:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.0:1619677448.149416:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.149416:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.149417:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.149417:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.149422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.149427:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.149428:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.149429:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.149430:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.149430:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:26.0:1619677448.149431:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.149433:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:2.0:1619677448.149433:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.149434:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.149434:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.149436:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.149439:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.149518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.149522:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.149525:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.149528:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.149529:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.149531:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.149533:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.149534:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.149536:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.149537:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.149538:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.149541:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.149543:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.149547:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.149547:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.149548:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.149550:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.149551:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.149552:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.149552:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.149564:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.149566:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.149567:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.149569:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.149571:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.149572:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.149580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.149586:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.149589:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.149589:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.149592:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.149595:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.149596:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.149597:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.149598:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.149600:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.149601:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.149604:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.149605:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.149607:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.149608:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.149609:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.149610:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.149610:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.149611:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.149611:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.149616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.149623:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.149625:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.149625:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.149626:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.149627:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.149628:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.149639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.149646:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.149649:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.149649:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.149650:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.149650:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.149650:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:4.2:1619677448.149744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.149751:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.149753:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.149754:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.149756:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.149758:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.149759:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.149761:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718702] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.149762:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.149763:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.149764:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.149767:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.149769:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.149772:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.149772:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.149773:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.149775:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.149775:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.149776:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.149777:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.149778:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.149781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.149783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.2:1619677448.149789:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.149794:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:5.0:1619677448.149794:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677448.149795:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.149795:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.149797:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677448.149797:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677448.149798:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.149798:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:5.0:1619677448.149798:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.149799:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677448.149800:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.149800:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.149801:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.149803:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:26.0:1619677448.149805:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.149806:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.149808:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.149809:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.149810:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.149811:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.149814:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.149815:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.149818:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.149818:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.149819:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.149820:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.149821:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.149822:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.149823:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.149824:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.149829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.149832:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.149835:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.149835:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.149836:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:12.0:1619677448.149837:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.149838:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:13.0:1619677448.149839:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.149839:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.149840:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.149840:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.149939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.149946:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.149948:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.149949:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.149951:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.149953:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.149954:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.149955:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.149957:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.149957:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.149958:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.149961:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.149963:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.149964:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.149965:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.149966:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.149967:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.149968:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.149968:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.149969:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.149973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.149980:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.149982:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.149983:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.149984:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.149985:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.149986:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.150070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.150073:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677448.150076:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86fbd5800] (20)++
00000800:00000200:5.0:1619677448.150079:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.150080:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.150082:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.150085:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86fbd5800] (21)++
00000400:00000200:20.0:1619677448.150085:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.150086:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677448.150087:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.13.18@o2ib3
00000400:00000200:20.0:1619677448.150088:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.150090:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.150091:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.18@o2ib3 : REPLY - routed
00000400:00000200:20.0:1619677448.150091:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.150092:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86fbd5800] (22)--
00000400:00000200:20.0:1619677448.150092:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.2:1619677448.150092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb0a00] (20)++
00000800:00000200:30.0:1619677448.150094:0:19944:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000400:00000200:20.0:1619677448.150096:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.150097:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.150098:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86fbd5800] (21)++
00000800:00000200:17.0:1619677448.150098:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb0a00] (21)++
00000800:00000200:30.0:1619677448.150100:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86fbd5800] (22)++
00000800:00000200:20.0:1619677448.150100:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.150101:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86fbd5800] (23)--
00000800:00000200:30.0:1619677448.150101:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86fbd5800] (22)--
00000800:00000200:20.0:1619677448.150101:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.150101:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.19@o2ib3
00000800:00000200:20.0:1619677448.150102:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.150102:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb0a00] (22)--
00000400:00000200:17.0:1619677448.150103:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.15.19@o2ib3 : GET - for me
00000800:00000200:20.0:1619677448.150104:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.150104:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677448.150106:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86fbd5800] (21)++
00000400:00000200:17.0:1619677448.150106:0:19930:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.15.19@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000800:00000200:21.2:1619677448.150108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:17.0:1619677448.150108:0:19930:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.15.19@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:20.0:1619677448.150109:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.150110:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677448.150110:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb0a00] (21)++
00000800:00000200:17.0:1619677448.150111:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb0a00] (22)--
00000400:00000200:17.0:1619677448.150113:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.15.19@o2ib3 local destination
00000800:00000200:31.0:1619677448.150114:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86fbd5800] (22)++
00000800:00000200:24.0:1619677448.150114:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:13.0:1619677448.150116:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677448.150117:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86fbd5800] (23)--
00000400:00000200:17.0:1619677448.150117:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.15.19@o2ib3(10.51.15.19@o2ib3:10.51.15.19@o2ib3) : REPLY try# 0
00000800:00000200:31.0:1619677448.150118:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86fbd5800] (22)--
00000800:00000200:24.0:1619677448.150118:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.150118:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:17.0:1619677448.150119:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.15.19@o2ib3
00000800:00000200:13.0:1619677448.150119:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.150120:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.150121:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.150122:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918ce37f3280] -> 10.51.15.19@o2ib3 (2) version: 12
00000400:00000200:13.0:1619677448.150122:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.150123:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918cfdbb0a00] (21)++
00000800:00000200:17.0:1619677448.150123:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdbb0a00] (22)++
00000800:00000200:12.0:1619677448.150123:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677448.150124:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.150125:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918cfdbb0a00] (23)--
00000800:00000200:17.0:1619677448.150125:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb0a00] (22)--
00000800:00000200:13.0:1619677448.150125:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677448.150126:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.150127:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.150128:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.150130:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.150130:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:5.2:1619677448.150132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb0a00] (21)++
00000400:00000200:24.0:1619677448.150134:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.150136:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677448.150137:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb0a00] (22)++
00000800:00000200:24.0:1619677448.150139:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.150139:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdbb0a00] (23)--
00000800:00000200:24.0:1619677448.150140:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.150140:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:1.0:1619677448.150140:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.150141:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb0a00] (22)--
00000800:00000200:24.0:1619677448.150142:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.150142:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:1.0:1619677448.150142:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.15.19@o2ib3: REPLY: OK
00000800:00000200:24.0:1619677448.150143:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.150144:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:1.0:1619677448.150144:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb0a00] (21)--
00000800:00000200:8.2:1619677448.150148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.150156:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.150158:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.150159:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.150161:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.150162:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.150164:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.150180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.150187:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.150190:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.150192:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.150194:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.150195:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.150196:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.150198:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.150199:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.150200:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.150200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:31.0:1619677448.150201:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.150204:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.150206:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.2:1619677448.150207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:31.0:1619677448.150208:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.150208:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.150209:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.150209:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.150209:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677448.150210:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677448.150211:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.150211:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.150212:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.150212:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:30.0:1619677448.150212:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.150212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:30.0:1619677448.150213:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.150213:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:31.0:1619677448.150214:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.150216:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:5.0:1619677448.150216:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:30.0:1619677448.150217:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.150217:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.150219:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.150219:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.150219:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677448.150220:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:20.0:1619677448.150220:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.150222:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:12.0:1619677448.150222:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.150222:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.150223:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677448.150223:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96718695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.150225:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.150225:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:30.0:1619677448.150226:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.150226:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677448.150226:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.150227:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.150228:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:20.0:1619677448.150228:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:12.0:1619677448.150228:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.150229:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.150229:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.150232:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.150232:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.150233:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96718694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.150235:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.150235:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:12.0:1619677448.150235:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677448.150235:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.150236:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:10.0:1619677448.150236:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.150237:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.150238:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677448.150238:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.150238:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.150239:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.150239:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.150240:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.150240:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677448.150240:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:10.0:1619677448.150241:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.150242:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677448.150243:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:10.0:1619677448.150243:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.0:1619677448.150244:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.150244:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:10.0:1619677448.150246:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.150247:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.150248:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.150249:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:13.0:1619677448.150250:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:10.0:1619677448.150250:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.150250:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.150251:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:13.0:1619677448.150252:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.150253:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.150254:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:13.0:1619677448.150255:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.150256:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.150257:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:13.0:1619677448.150257:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:31.0:1619677448.150258:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.150259:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.150260:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.150262:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:6.0:1619677448.150264:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.150266:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.150267:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.150268:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.150268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:12.0:1619677448.150269:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:6.0:1619677448.150270:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.150272:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.150273:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:12.0:1619677448.150273:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:6.0:1619677448.150273:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:17.0:1619677448.150274:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:12.0:1619677448.150274:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.150274:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.150275:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.150276:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.150277:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.150277:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:17.0:1619677448.150277:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:30.0:1619677448.150278:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:1.0:1619677448.150278:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:30.0:1619677448.150280:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:17.0:1619677448.150280:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.150282:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.150283:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.150285:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.150286:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.150287:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.150288:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.150292:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.150298:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.150301:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.150302:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.150303:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.150305:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.150305:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677448.150306:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677448.150306:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:12.0:1619677448.150318:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.150321:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.150322:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.150323:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.150326:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.150327:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.150369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86fbd5800] (21)++
00000800:00000200:31.0:1619677448.150377:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86fbd5800] (22)++
00000800:00000200:31.0:1619677448.150380:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.13.18@o2ib3
00000800:00000200:31.0:1619677448.150382:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86fbd5800] (23)--
00000400:00000200:31.0:1619677448.150383:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.150385:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:30.0:1619677448.150385:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86fbd5800] (22)--
00000400:00000200:31.0:1619677448.150386:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.150387:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.150388:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.150389:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:31.0:1619677448.150392:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : REPLY try# 0
00000800:00000200:31.0:1619677448.150394:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 1048576 bytes in 256 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:31.0:1619677448.150396:0:19945:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000800:00000200:31.0:1619677448.150403:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.150403:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:31.0:1619677448.150404:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:31.0:1619677448.150406:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:31.0:1619677448.150406:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86fbd5800] (21)++
00000800:00000200:31.0:1619677448.150407:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86fbd5800] (22)--
00000800:00000200:31.0:1619677448.150408:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86fbd5800] (21)--
00000800:00000200:26.0:1619677448.150419:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.150421:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.150422:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.150482:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.150484:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[3] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.150485:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:20.2:1619677448.150487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677448.150489:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:5.0:1619677448.150493:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.150495:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.150497:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.150498:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.150501:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.150502:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.150503:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.150504:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.150504:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:5.0:1619677448.150505:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.150506:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.150507:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.150510:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.150512:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.150515:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.150515:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.150516:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.150518:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.150519:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.150520:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.150521:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.150551:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.150554:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.150555:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.150557:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.150558:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.150559:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.150568:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.150571:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.150572:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.150572:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.150573:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.150574:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.150586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.150593:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.150596:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.150597:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.150599:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.150602:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.150603:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.150604:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96718690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.150606:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.150606:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.150607:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677448.150610:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.150612:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.150615:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.150616:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.150617:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.150618:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.150619:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.150619:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.150620:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.150634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.150642:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.150644:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.150645:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.150647:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.150648:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.150650:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.150655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.150662:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.150664:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.150665:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.150666:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.150667:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677448.150667:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.150672:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.150675:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.150676:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.150677:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.150679:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: REPLY: OK
00000800:00000200:26.0:1619677448.150682:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677448.150716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.150723:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.150725:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.150726:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.150728:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.150731:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.150732:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.150734:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.150735:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.150736:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.150737:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.150741:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.150742:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.150745:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.150746:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.150747:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.150749:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.150749:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.150750:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.150750:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677448.150762:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.150765:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.150766:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.150768:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.150768:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.150771:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.150773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:4.2:1619677448.150777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.150781:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.150783:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:10.0:1619677448.150784:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.150786:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.150786:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:30.0:1619677448.150787:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677448.150787:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677448.150789:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.150789:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.150790:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.150791:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.150792:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718688] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.150793:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.150793:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.150794:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.150794:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718688] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.150795:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.150795:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.150796:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.150797:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.150799:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.150801:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.150803:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.150805:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.150805:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.150805:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.150806:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.150808:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.150808:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.150809:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.150809:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.150810:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:10.0:1619677448.150810:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (22)++
00000800:00000200:10.0:1619677448.150811:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (23)--
00000800:00000200:31.0:1619677448.150812:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.150812:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.150812:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.150812:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.150813:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.150814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.2:1619677448.150818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.150822:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (23)++
00000800:00000200:5.0:1619677448.150824:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (24)--
00000400:00000200:5.0:1619677448.150825:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.150826:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.150827:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (23)++
00000400:00000200:5.0:1619677448.150827:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.150829:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:20.0:1619677448.150829:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (24)--
00000800:00000200:30.0:1619677448.150830:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.150830:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.150830:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:30.0:1619677448.150831:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.150832:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.150832:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.150834:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.150921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86fbd5800] (20)++
00000800:00000200:21.2:1619677448.150926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.150930:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86fbd5800] (21)++
00000800:00000200:30.0:1619677448.150932:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.13.18@o2ib3
00000400:00000200:30.0:1619677448.150934:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.18@o2ib3 : REPLY - routed
00000800:00000200:31.0:1619677448.150935:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.150936:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677448.150937:0:19944:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000400:00000200:31.0:1619677448.150938:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.150941:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.150942:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677448.150943:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86fbd5800] (22)++
00000400:00000200:31.0:1619677448.150944:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.150945:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.150945:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86fbd5800] (23)++
00000400:00000200:31.0:1619677448.150946:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.150946:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86fbd5800] (24)--
00000800:00000200:30.0:1619677448.150946:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86fbd5800] (23)--
00000400:00000200:31.0:1619677448.150947:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.150947:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86fbd5800] (22)--
00000800:00000200:30.0:1619677448.150949:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.150950:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.150951:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.2:1619677448.150952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86fbd5800] (21)++
00000800:00000200:31.0:1619677448.150954:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.150954:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.150955:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.150956:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.150957:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.150957:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.150958:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677448.150960:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86fbd5800] (22)++
00000800:00000200:31.0:1619677448.150962:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86fbd5800] (23)--
00000800:00000200:31.0:1619677448.150963:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86fbd5800] (22)--
00000800:00000200:12.0:1619677448.150968:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.150971:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.150971:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.150973:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.150975:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.150976:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.151049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.151055:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.151057:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.151060:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.151060:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.151062:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.151064:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.151065:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.151067:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.151068:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.151069:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.151073:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.151074:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.151077:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.151078:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.151079:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.151081:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.151082:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.151083:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.151085:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.151085:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.151090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.151093:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677448.151094:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.151096:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.151096:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677448.151097:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677448.151097:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:31.0:1619677448.151098:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.151099:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.151099:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.151101:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.151103:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:1.0:1619677448.151106:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.151109:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.151109:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.151112:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.151113:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:1.0:1619677448.151114:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.151115:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:30.0:1619677448.151116:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.151117:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.151117:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:1.0:1619677448.151117:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.151118:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:1.0:1619677448.151118:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.151119:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:1.0:1619677448.151119:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.151120:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.151124:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.151125:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:5.2:1619677448.151127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677448.151128:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.151129:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.151130:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.151132:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.151132:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.151133:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:26.0:1619677448.151134:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677448.151134:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.151135:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.151137:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.151138:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.151140:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.151140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.151142:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.151144:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:12.0:1619677448.151144:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:26.0:1619677448.151145:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.151146:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677448.151146:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:26.0:1619677448.151147:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.151147:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.151148:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.151148:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:26.0:1619677448.151149:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.151149:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677448.151149:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.151150:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.151151:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.151152:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:26.0:1619677448.151153:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.151154:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.151157:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.151158:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.151159:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.151161:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.151162:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.151163:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.151163:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.151168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.151176:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.151179:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:20.2:1619677448.151180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86fbd5800] (21)++
00000400:00000200:2.0:1619677448.151180:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.151181:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.151182:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.151186:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677448.151188:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86fbd5800] (22)++
00000800:00000200:31.0:1619677448.151191:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.13.18@o2ib3
00000800:00000200:31.0:1619677448.151193:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86fbd5800] (23)--
00000400:00000200:31.0:1619677448.151194:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.151194:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86fbd5800] (22)--
00000400:00000200:31.0:1619677448.151195:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:31.0:1619677448.151197:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718682] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.151198:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.151199:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.151200:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:31.0:1619677448.151203:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : REPLY try# 0
00000800:00000200:31.0:1619677448.151205:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 1048576 bytes in 256 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:31.0:1619677448.151206:0:19945:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000800:00000200:31.0:1619677448.151215:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.151215:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:31.0:1619677448.151216:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:31.0:1619677448.151217:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:31.0:1619677448.151218:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86fbd5800] (21)++
00000800:00000200:31.0:1619677448.151219:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86fbd5800] (22)--
00000800:00000200:31.0:1619677448.151219:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86fbd5800] (21)--
00000800:00000200:20.2:1619677448.151225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.151232:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.151235:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.151237:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.151237:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:31.0:1619677448.151239:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.151239:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.151239:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.151240:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:30.0:1619677448.151241:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.151242:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.151243:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.151244:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.151245:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.151248:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.151249:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.151253:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.151253:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.151254:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.151256:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.151256:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.151257:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.151257:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.151262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.151269:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.151272:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.151273:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.151274:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:20.0:1619677448.151274:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.151275:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.151277:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[1] from 10.0.10.106@o2ib7
00000800:00000200:20.0:1619677448.151278:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:10.0:1619677448.151278:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.151281:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:8.2:1619677448.151282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.151290:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.151292:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.151293:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:5.0:1619677448.151297:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.151298:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.151299:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.151300:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.151300:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.151311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.151319:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.151322:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.151323:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.151325:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.151327:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.151329:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.151330:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96718680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.151331:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.151332:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.151333:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.151336:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.151338:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.151340:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.151341:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.151342:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.151343:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.151343:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.151345:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.151346:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.151347:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:21.2:1619677448.151353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.151359:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.151361:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.151362:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.151363:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:21.2:1619677448.151372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:21.2:1619677448.151377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:13.0:1619677448.151377:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:18.0:1619677448.151378:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:13.0:1619677448.151380:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.151380:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.151381:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:13.0:1619677448.151382:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.151382:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.151384:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:12.0:1619677448.151384:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.151385:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.151385:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:18.0:1619677448.151386:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.151387:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:18.0:1619677448.151388:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.151389:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.151391:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.151391:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.151392:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677448.151392:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.151393:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.151394:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.151396:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.151398:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96718678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.151398:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:31.0:1619677448.151400:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.151401:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.151401:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.151402:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.151403:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.151404:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.151405:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:31.0:1619677448.151406:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.151406:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.151407:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677448.151408:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.151408:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.151409:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.2:1619677448.151409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:31.0:1619677448.151411:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.151412:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.151412:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.151414:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.151415:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.151415:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677448.151416:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.151417:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.151418:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677448.151418:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.151419:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.151421:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.151422:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.151423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:2.0:1619677448.151424:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.151425:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96718677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.151426:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.151427:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.151428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:2.0:1619677448.151428:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:9.2:1619677448.151429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.151431:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677448.151432:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.151433:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.151434:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677448.151434:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.151436:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677448.151436:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.151436:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.151437:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.151438:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.151438:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.151439:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.151439:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677448.151439:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:6.0:1619677448.151439:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.151439:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.151440:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677448.151440:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.151440:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:18.0:1619677448.151441:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.151441:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:2.0:1619677448.151442:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:6.0:1619677448.151443:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:508, d:32, s:-96718676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.151443:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:18.0:1619677448.151444:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.151445:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.151445:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:6.0:1619677448.151446:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.151446:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000400:00000200:6.0:1619677448.151447:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.151447:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.151448:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:8.2:1619677448.151448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.151448:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:6.0:1619677448.151451:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.151452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677448.151453:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.151454:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.151455:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677448.151456:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:10.0:1619677448.151456:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:6.0:1619677448.151456:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.151457:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677448.151457:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.151458:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:6.0:1619677448.151458:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:26.0:1619677448.151459:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:20.0:1619677448.151459:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.151459:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.151460:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:10.0:1619677448.151460:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:18.0:1619677448.151461:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:10.0:1619677448.151461:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.151461:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.151461:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677448.151461:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:20.0:1619677448.151462:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.151462:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:5.0:1619677448.151462:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.151463:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:20.0:1619677448.151464:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:10.0:1619677448.151464:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: REPLY: OK
00000800:00000200:6.0:1619677448.151464:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:10.0:1619677448.151467:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:9.2:1619677448.151467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.151475:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.151478:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.151479:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.151479:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:4.2:1619677448.151481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:18.0:1619677448.151482:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.151484:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:26.0:1619677448.151488:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:9.2:1619677448.151489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:26.0:1619677448.151491:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.151492:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.151494:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.151496:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.151497:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000400:00000200:26.0:1619677448.151498:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.151499:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.151500:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000400:00000200:26.0:1619677448.151501:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.151501:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:26.0:1619677448.151502:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.151502:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000400:00000200:26.0:1619677448.151503:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677448.151503:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.151503:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:26.0:1619677448.151506:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.151508:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.151511:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.151512:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.151512:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.151514:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.151515:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.151515:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.151516:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.151528:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.151531:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.151531:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.151532:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.151534:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.151537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.151539:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.151542:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.151543:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.151543:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.151544:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.151545:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677448.151545:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.151546:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.151546:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.151547:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.151548:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.151650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.151655:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.151658:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.151660:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.151661:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.151663:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.151665:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.151666:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.151667:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.151668:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.151669:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.151672:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.151674:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.151678:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.151678:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.151679:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.151681:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.151682:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.151683:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.151684:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.2:1619677448.151691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677448.151693:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.151696:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.151697:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.151698:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:12.0:1619677448.151699:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.151700:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.151700:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.151701:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677448.151702:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677448.151703:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.151706:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.151707:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.151709:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.151710:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.151711:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.151712:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.151715:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.151716:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.151720:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.151720:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.151721:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.151722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677448.151723:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.151723:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.151724:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.151725:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.151729:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.151732:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.151734:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.151735:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:31.0:1619677448.151736:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.151737:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.151738:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:30.0:1619677448.151739:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:13.0:1619677448.151739:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.151740:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677448.151741:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.151741:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:30.0:1619677448.151742:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.151743:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.151743:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:13.0:1619677448.151744:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:30.0:1619677448.151747:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.151748:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.151751:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.151752:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:12.0:1619677448.151752:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.151753:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.151755:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.151755:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.151755:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.151756:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:12.0:1619677448.151756:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.151756:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:30.0:1619677448.151757:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677448.151757:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.151759:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.2:1619677448.151761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.151768:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.151771:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.151772:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.151774:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.151774:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.151777:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.151823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.151830:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.151833:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.151834:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.151835:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.151837:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.151839:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.151840:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.151841:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.151842:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.151843:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.151846:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.151848:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.151851:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.151851:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.151852:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.151853:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.151854:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.151855:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.151856:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.151856:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.151859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.151862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.151867:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677448.151869:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.151870:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.151871:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.151871:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:1.0:1619677448.151871:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.151872:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.151873:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.151873:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677448.151876:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.151911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.151918:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.151921:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.151922:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.151924:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.151926:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.151928:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.151929:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.151930:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.151931:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.151932:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:5.2:1619677448.151933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677448.151935:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.151937:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.151938:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.151940:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.151940:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.151941:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.151941:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.151942:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.151943:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.151943:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.151944:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:1.0:1619677448.151944:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.151945:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:1.0:1619677448.151946:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.151948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:1.0:1619677448.151948:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.151949:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.151951:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.151952:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.151953:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.2:1619677448.151957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:17.0:1619677448.151957:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:1.0:1619677448.151957:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.151959:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.151960:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.151960:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.151962:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:17.0:1619677448.151962:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:1.0:1619677448.151963:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.151964:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:22.0:1619677448.151965:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:17.0:1619677448.151965:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.151965:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.151966:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677448.151967:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677448.151967:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.151967:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677448.151968:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.151968:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.151969:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677448.151970:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.151971:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.151973:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.151974:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.151975:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.151977:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.151980:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.151981:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.151982:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.151983:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.151983:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.151985:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.151985:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.151986:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.151987:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.151988:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.151989:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.151989:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.151990:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.151991:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.151991:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.151995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.152003:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.152006:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.152006:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.152008:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.152009:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.152010:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.152055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.152062:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.152065:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.152067:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.152069:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.152070:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.152072:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.152073:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718667] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.152075:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.152075:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.152077:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.152080:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.152081:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.152084:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.152085:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.152085:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.152087:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.152087:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.152088:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.152089:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.152099:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.152101:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.152102:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.152104:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.152105:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.152106:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.152139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.2:1619677448.152143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:4.2:1619677448.152145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677448.152149:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.152150:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.152152:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:17.0:1619677448.152152:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.152152:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.152154:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677448.152156:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.152156:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677448.152158:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.152158:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.152159:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.152161:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.152162:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.152162:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.152163:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.152164:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.152164:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.152165:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.152166:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.152166:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.152168:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.152169:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.152170:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.152171:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.152173:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.152173:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.152175:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96718664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.152176:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.152176:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.152177:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:10.0:1619677448.152177:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.152178:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.152178:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.152178:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.152179:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.152179:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.152180:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.152180:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.152181:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.152183:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.152183:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.152184:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.152184:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.152185:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.152187:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:26.0:1619677448.152187:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.152188:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.152188:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.152189:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.152189:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.152190:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.152190:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.152191:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:12.0:1619677448.152192:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:10.0:1619677448.152192:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.152192:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:12.0:1619677448.152194:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:10.0:1619677448.152194:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:12.0:1619677448.152195:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.152196:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.152196:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.152196:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:13.0:1619677448.152198:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.152199:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.152199:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677448.152200:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.152200:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677448.152201:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.152202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677448.152203:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.152205:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.152208:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.152209:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.152209:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.152209:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.152211:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:12.0:1619677448.152211:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.152212:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.152212:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.152212:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.152214:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.152215:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.152215:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:6.0:1619677448.152215:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677448.152216:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.152217:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677448.152223:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.152226:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.152226:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.152227:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.152228:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.152228:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.152505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.152511:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.152514:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.152515:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.152517:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.152520:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.152521:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.152523:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.152524:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.152525:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.152526:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.152530:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.152532:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.152534:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.152535:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.152536:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.152537:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.152538:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.152539:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:21.2:1619677448.152540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:1.0:1619677448.152540:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.152541:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.152546:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.2:1619677448.152546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.152549:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.152549:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:4.2:1619677448.152550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677448.152551:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:18.0:1619677448.152552:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.152553:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:12.0:1619677448.152553:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:18.0:1619677448.152554:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.152554:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.152555:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.152555:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:18.0:1619677448.152556:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:12.0:1619677448.152556:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.152557:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.152558:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:18.0:1619677448.152558:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.152558:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.152559:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.152561:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:18.0:1619677448.152561:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.152561:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.152561:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.152561:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677448.152565:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.152566:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.152568:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.152569:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.152571:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:18.0:1619677448.152571:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.152572:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.152572:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.152572:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:26.0:1619677448.152574:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.152574:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.152574:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.152575:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.152575:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000400:00000200:26.0:1619677448.152576:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.152576:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.152577:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:26.0:1619677448.152579:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.152581:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.2:1619677448.152582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677448.152584:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.152585:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.152585:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.152587:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:13.0:1619677448.152587:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677448.152588:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.152588:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677448.152588:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:26.0:1619677448.152589:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:13.0:1619677448.152590:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.152590:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:13.0:1619677448.152591:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.152591:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:13.0:1619677448.152592:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.152592:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677448.152593:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.152595:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.152601:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.2:1619677448.152602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:6.0:1619677448.152604:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.152605:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.152605:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.152607:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:6.0:1619677448.152607:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.152610:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:6.0:1619677448.152610:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677448.152611:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.152613:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.152616:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.152618:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.152619:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.152620:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.152621:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.152622:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.152625:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.152627:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.152629:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.152630:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.152631:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.152632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677448.152632:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.152633:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.152633:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.152634:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.152639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.152640:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.152643:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.152645:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.152647:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.152647:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.152647:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:30.0:1619677448.152649:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.152650:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.152650:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.152650:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.152652:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.152652:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.152653:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.152654:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.152654:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.152655:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:30.0:1619677448.152658:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.152660:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.152662:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.152663:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.152664:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.152665:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.152666:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.152667:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.152668:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.152677:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.152679:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.152680:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.152682:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.152684:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.152685:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.152704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.152710:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.152712:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.152714:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.152715:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.152718:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.152719:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.152721:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.152722:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.152723:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.152724:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.152728:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.152730:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.152733:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.152734:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.152734:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.152736:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.152737:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.152738:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.152739:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.152740:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.2:1619677448.152745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.152748:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:31.0:1619677448.152749:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.152750:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.152751:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.152751:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677448.152752:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.152753:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.152754:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.152755:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.152757:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.152778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86fbd5800] (20)++
00000800:00000200:30.0:1619677448.152786:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86fbd5800] (21)++
00000800:00000200:5.2:1619677448.152786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.152789:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.13.18@o2ib3
00000400:00000200:30.0:1619677448.152791:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.18@o2ib3 : REPLY - routed
00000800:00000200:31.0:1619677448.152792:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86fbd5800] (22)--
00000800:00000200:1.0:1619677448.152792:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.152794:0:19944:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000800:00000200:1.0:1619677448.152795:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.152796:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.152798:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.152800:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.152801:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86fbd5800] (21)++
00000800:00000200:30.0:1619677448.152802:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86fbd5800] (22)++
00000400:00000200:1.0:1619677448.152802:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677448.152803:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86fbd5800] (23)--
00000800:00000200:30.0:1619677448.152803:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86fbd5800] (22)--
00000400:00000200:1.0:1619677448.152803:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.152805:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.152805:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.152807:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.2:1619677448.152809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86fbd5800] (21)++
00000400:00000200:1.0:1619677448.152810:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.152812:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.152815:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.152816:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677448.152817:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86fbd5800] (22)++
00000800:00000200:1.0:1619677448.152817:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.152819:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86fbd5800] (23)--
00000800:00000200:1.0:1619677448.152819:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.152819:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.152820:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.152820:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.152821:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86fbd5800] (22)--
00000800:00000200:31.0:1619677448.152832:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.152835:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.152836:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.152838:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.152839:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.152841:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.152880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.152888:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.152890:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.152891:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.152893:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.152896:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.152897:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.152898:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.152900:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.152901:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.152902:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.152905:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.152907:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.152909:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.152909:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.152910:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.152911:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.152912:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.152912:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.152913:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677448.152923:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.152924:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.152925:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.152926:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.152927:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.152929:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.152940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.152946:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.152949:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.152950:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.152952:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.152954:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.152956:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.152957:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.152958:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.152959:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.152960:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.152964:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.152965:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.152968:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.152969:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.152969:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.152971:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.152972:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.152973:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.152973:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.152983:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.152986:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.152986:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.152987:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.152989:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.152992:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677448.153055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.153063:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.153066:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.153068:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.153070:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.153070:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.153072:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.153073:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.153075:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.153076:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.153077:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.153080:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.153082:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.153084:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.153085:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.153085:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.153087:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.153087:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.153088:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.153089:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677448.153099:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.153102:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.153102:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.153104:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.153105:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.153106:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.153109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.153116:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.153119:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.153120:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.153121:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.153123:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.153125:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.153126:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.153127:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.153128:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.153129:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.153132:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.153133:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.153135:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.153136:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.153137:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.153138:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.153139:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.153139:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.153140:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.153150:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.153153:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.153153:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.153155:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.153157:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.153157:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:21.2:1619677448.153218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.153226:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.153228:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.153230:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.153232:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.153232:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.153233:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.153234:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.153236:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.153237:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.153237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86fbd5800] (21)++
00000400:00000200:30.0:1619677448.153238:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.153241:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.153243:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.153244:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86fbd5800] (22)++
00000800:00000200:31.0:1619677448.153245:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.13.18@o2ib3
00000800:00000200:30.0:1619677448.153246:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.153246:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.153247:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86fbd5800] (23)--
00000800:00000200:30.0:1619677448.153247:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:31.0:1619677448.153248:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.153249:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:30.0:1619677448.153249:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:31.0:1619677448.153251:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.153252:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.153253:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.153254:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:30.0:1619677448.153255:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.153255:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.153256:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:31.0:1619677448.153257:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : REPLY try# 0
00000800:00000200:31.0:1619677448.153258:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 1048576 bytes in 256 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677448.153258:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86fbd5800] (22)--
00000800:00000200:31.0:1619677448.153260:0:19945:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000800:00000200:6.0:1619677448.153262:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.2:1619677448.153263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:6.0:1619677448.153265:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.153266:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.153266:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.153268:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.153269:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677448.153269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:17.0:1619677448.153271:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:6.0:1619677448.153271:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677448.153272:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.153273:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.153273:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:31.0:1619677448.153274:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:17.0:1619677448.153274:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.153274:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677448.153276:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677448.153277:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86fbd5800] (21)++
00000400:00000200:17.0:1619677448.153277:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.153277:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:31.0:1619677448.153278:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86fbd5800] (22)--
00000800:00000200:31.0:1619677448.153278:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86fbd5800] (21)--
00000800:00000200:18.0:1619677448.153278:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677448.153279:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.153281:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.153281:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.153282:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677448.153282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:17.0:1619677448.153283:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.153284:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.153285:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677448.153286:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.153287:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96718649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.153289:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.153290:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.153290:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.153291:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.153291:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.153292:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.153298:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.153299:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:2.0:1619677448.153300:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.153302:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.153303:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677448.153303:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.153304:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.153305:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.153306:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.153306:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.153306:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.153307:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677448.153307:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.153307:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.153309:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.153310:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.153311:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.153311:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:22.0:1619677448.153320:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.153323:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.153324:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.153324:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.153326:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.153327:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:22.0:1619677448.153329:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677448.153330:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.153330:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.153332:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.153334:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.153335:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677448.153343:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.153345:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.153347:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.153347:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.153348:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.153349:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677448.153350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.153357:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.153361:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.153361:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[1] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.153366:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.153379:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.153379:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.2:1619677448.153726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.153734:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.153738:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.153739:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.153740:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.153742:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: REPLY: OK
00000800:00000200:17.0:1619677448.153745:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.153787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.153794:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.153798:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.153799:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.153801:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.153804:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.153805:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.2:1619677448.153805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:24.0:1619677448.153807:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.153808:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.153808:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:24.0:1619677448.153809:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.153810:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.153811:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.153814:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.153814:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:24.0:1619677448.153815:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.153818:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.153818:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.153820:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:5.2:1619677448.153820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:24.0:1619677448.153821:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.153821:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.153822:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:20.0:1619677448.153822:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.153823:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677448.153823:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.153824:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:20.0:1619677448.153824:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.153825:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.153825:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.153826:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:20.0:1619677448.153828:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.153828:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677448.153830:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.2:1619677448.153830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.153832:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.153832:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.153833:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677448.153833:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677448.153834:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.153835:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:10.0:1619677448.153835:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.153836:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.153837:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:10.0:1619677448.153837:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.153838:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677448.153838:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:10.0:1619677448.153839:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.153840:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96718646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.153841:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677448.153841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.153841:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.153842:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:10.0:1619677448.153842:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.153842:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.153843:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:17.0:1619677448.153844:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:1.0:1619677448.153844:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.153845:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.153846:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.153847:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.153847:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:17.0:1619677448.153848:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.153848:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.153850:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677448.153851:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:10.0:1619677448.153852:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.153853:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.153853:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.153854:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.153855:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.153856:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.153857:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.153858:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.153863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.153869:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.153870:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.153870:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.153872:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:26.0:1619677448.153873:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.153873:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.153873:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.153874:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.153874:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.153875:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677448.153902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.153909:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.153912:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.153913:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.153915:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.153918:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.153919:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.153920:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.153922:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.153923:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.153924:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.153927:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.153929:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.153931:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.153932:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.153933:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.153935:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.153935:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.153936:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.153936:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:4.2:1619677448.153939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.153946:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677448.153948:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.153949:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.153950:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:13.0:1619677448.153951:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.153952:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.153952:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.153953:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.153954:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.153954:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.153955:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:13.0:1619677448.153956:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677448.153956:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.153957:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.153959:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.153959:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.2:1619677448.153959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:10.0:1619677448.153963:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.153965:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677448.153965:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:10.0:1619677448.153967:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.153968:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:2.0:1619677448.153968:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.153969:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:10.0:1619677448.153969:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.153970:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.153971:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.153971:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677448.153971:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.153972:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:2.0:1619677448.153974:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.153975:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.153976:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.153978:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.153979:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.153980:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.153983:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.153984:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:2.0:1619677448.153984:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.153986:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.153987:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.153987:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.153987:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.153987:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677448.153989:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.153989:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.153990:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.153990:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677448.153991:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:2.0:1619677448.153991:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:30.0:1619677448.153993:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:9.2:1619677448.153996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.154004:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677448.154006:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:18.0:1619677448.154006:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.154007:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.154007:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.154008:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.154009:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.154010:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:18.0:1619677448.154010:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.154011:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.154012:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677448.154012:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.154080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.154088:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.154090:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.154092:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.154094:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.154095:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.154097:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.154098:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.154099:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.154100:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.154101:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.154104:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.154106:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.154108:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.154109:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.154109:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.154111:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.154111:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.154113:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.154114:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.154115:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677448.154120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.154122:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.154125:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.154125:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.154127:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.154128:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.154129:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.154130:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:12.0:1619677448.154130:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.154131:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.154132:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677448.154363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.154369:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.154372:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.154373:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.154374:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.154377:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.154379:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.154380:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.154381:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.154382:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.154383:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.154387:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.154388:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.154391:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.154392:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.154393:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.154395:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.154395:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.154396:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.154396:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.154419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.154426:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.154429:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.154429:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.154430:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.154432:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.154435:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.2:1619677448.154572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.154580:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.154583:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.154583:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.154585:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.154588:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.154589:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.154590:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.154592:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.154593:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.154594:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.154597:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.154599:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.154600:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.154601:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.154601:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.154603:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.154603:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.154604:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.154604:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677448.154604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.154611:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.154613:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:10.0:1619677448.154614:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:8.0:1619677448.154614:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.154616:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.154616:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.154617:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.154618:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.154618:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.154619:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.154620:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:10.0:1619677448.154620:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:24.0:1619677448.154621:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.154623:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.154624:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.154625:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.154628:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.154630:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.154632:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.154633:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:24.0:1619677448.154633:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.154634:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.154636:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:24.0:1619677448.154636:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.154636:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:10.0:1619677448.154636:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.154637:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.154637:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:24.0:1619677448.154637:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.154637:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:26.0:1619677448.154638:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:9.2:1619677448.154642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.154650:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.154653:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.154654:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.154654:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.154656:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.154660:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.154681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.154687:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.154690:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.154692:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.154692:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677448.154694:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.154696:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677448.154697:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.154699:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.154700:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.154701:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677448.154705:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.154707:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677448.154710:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.154711:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677448.154712:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677448.154714:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.154714:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.154715:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.154715:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677448.154727:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.154730:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.154731:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.154733:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677448.154735:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.154736:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.154776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.154783:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.154786:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.154786:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.154788:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.154789:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.154791:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.154792:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.154793:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.154794:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.154795:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.154798:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.154800:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.154803:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.154804:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.154804:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.154806:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.154807:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.154808:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.154809:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.154809:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677448.154812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677448.154815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.154820:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.154822:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:18.0:1619677448.154823:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.154824:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:2.0:1619677448.154824:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.154825:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677448.154825:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.154825:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:18.0:1619677448.154827:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.154829:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.154832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.154839:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.154841:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.154843:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.154843:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.154844:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.154844:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.154899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.154905:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.154907:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.154908:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.154910:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.154911:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.154913:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677448.154914:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.154915:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.154916:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.154917:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677448.154919:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.154921:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677448.154924:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.154925:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677448.154925:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677448.154927:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677448.154928:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.154929:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.154930:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.154931:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.2:1619677448.154936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:4.2:1619677448.154938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:26.0:1619677448.154939:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:2.0:1619677448.154941:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:26.0:1619677448.154942:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.154943:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.154943:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:2.0:1619677448.154943:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.154944:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:26.0:1619677448.154945:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:8.0:1619677448.154945:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:2.0:1619677448.154945:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677448.154947:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:26.0:1619677448.154948:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:24.0:1619677448.154948:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677448.154950:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.154953:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.154955:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:8.0:1619677448.154956:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.154957:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.154958:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.154960:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677448.154963:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.154964:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:8.0:1619677448.154967:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.154968:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:8.0:1619677448.154968:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:8.0:1619677448.154970:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:8.0:1619677448.154970:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677448.154971:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677448.154971:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:5.2:1619677448.154980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.154981:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.154983:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.154984:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.154984:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.154986:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.154986:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.154988:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:10.0:1619677448.154988:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:1.0:1619677448.154989:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.154991:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.154994:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.154996:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.154997:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.154998:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.154999:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.155000:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:26.0:1619677448.155002:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000400:00000200:17.0:1619677448.155003:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.155005:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.155006:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:17.0:1619677448.155006:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:10.0:1619677448.155006:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.155007:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.155008:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:17.0:1619677448.155008:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.155009:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.155010:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.155011:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.155012:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.155014:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.155015:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.155015:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.155017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.155020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.155024:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.155026:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:1.0:1619677448.155026:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677448.155027:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.155028:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.155028:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.155029:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.155029:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677448.155030:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.155032:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.155191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.155198:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.155200:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.155201:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.155203:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.155205:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.155206:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.155208:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.155209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:10.0:1619677448.155209:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.155211:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.155211:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.155215:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.155215:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:10.0:1619677448.155217:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.155218:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.155219:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:10.0:1619677448.155220:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.155221:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:8.0:1619677448.155221:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.155222:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677448.155223:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.155224:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.155224:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:8.0:1619677448.155224:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:10.0:1619677448.155225:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.155225:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:8.0:1619677448.155226:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96718632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.155227:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.155228:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.155229:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:9.2:1619677448.155230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677448.155232:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.155234:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.0:1619677448.155236:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.155236:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:8.0:1619677448.155237:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.155238:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.0:1619677448.155238:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:8.0:1619677448.155239:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.155240:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.155240:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:18.0:1619677448.155241:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.155242:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.155242:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.155244:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.155247:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:26.0:1619677448.155250:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.155252:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.155253:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.155253:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.155256:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.155258:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.155365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.155373:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.155376:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.155378:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.155380:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.155380:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.155381:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.155382:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.155384:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.155385:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.155386:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.155389:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.155389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.155390:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.155393:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.155394:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.155394:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.155396:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.155396:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.155397:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677448.155397:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.155398:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:31.0:1619677448.155399:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.155399:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.155400:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.155401:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.155401:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.155403:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:8.2:1619677448.155403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:31.0:1619677448.155404:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.155405:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.155406:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.155407:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.2:1619677448.155407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:31.0:1619677448.155410:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.155411:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.155413:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.155414:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.155414:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.155415:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.155415:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.155415:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677448.155415:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677448.155416:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.155416:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677448.155417:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.155417:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.2:1619677448.155417:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677448.155418:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677448.155419:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.155420:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.155421:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:20.0:1619677448.155422:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.155423:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.155423:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:5.0:1619677448.155424:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:5.0:1619677448.155425:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.155426:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.155427:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.155428:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677448.155428:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.155429:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.155430:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.155433:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.155434:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.155436:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718629] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.155437:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.155438:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.155439:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.155442:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.155443:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.155444:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.155445:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.155446:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.155446:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.155447:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:13.0:1619677448.155448:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.155448:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677448.155450:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:5.0:1619677448.155450:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:5.0:1619677448.155451:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.155451:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677448.155452:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.155452:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.155473:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.155476:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.155477:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.155478:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.155480:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.155481:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677448.155494:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.155497:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.155498:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.155498:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.155499:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.155501:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.155690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.155698:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.155701:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.155702:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.155704:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.155706:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.155708:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.155709:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.155710:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.155711:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.155712:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.155715:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.155717:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.155720:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.155720:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.155721:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.155723:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.155723:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.155724:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.155725:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.155729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.155737:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.155739:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.155740:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.155742:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.155743:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.155745:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.155749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.155757:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.155759:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.155761:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.155762:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.155762:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.155763:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.155771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.155778:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.155781:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.155782:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.155784:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.155786:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.155788:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.155789:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.155791:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.155791:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.155792:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.2:1619677448.155795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:24.0:1619677448.155796:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.155797:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.155800:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.155800:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.155801:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.155802:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.155803:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:24.0:1619677448.155803:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.155804:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.155804:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:30.0:1619677448.155805:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.155807:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.155809:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.155809:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.155811:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.155812:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.155813:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.155814:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.155814:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:30.0:1619677448.155815:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:12.0:1619677448.155817:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.155817:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.155819:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677448.155819:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.155821:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.155821:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.155821:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.155823:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.155824:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.155825:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.155826:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.155827:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.155828:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.155828:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.155833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.155841:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.155844:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.155845:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.155847:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.155847:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.155850:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.155921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.155924:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.155927:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.155929:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.155930:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.155931:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.155933:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.155934:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.155936:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.155937:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.155938:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.155942:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.155943:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.155946:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.155947:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.155948:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.155949:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.155950:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.155951:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.155953:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.155953:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.155958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.155962:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.155964:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677448.155965:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.155966:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.155966:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:31.0:1619677448.155967:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.155967:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.155968:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.155969:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.155970:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.155979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.2:1619677448.155986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677448.155987:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.155989:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.155991:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.155993:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.155994:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.155994:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:30.0:1619677448.155996:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:26.0:1619677448.155996:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:30.0:1619677448.155997:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.155997:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:30.0:1619677448.155999:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.156000:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.156000:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.156001:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.156003:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.156004:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.156004:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.156005:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.156006:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.156007:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.156008:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.156009:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.156009:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.156010:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.156011:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.156012:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:26.0:1619677448.156012:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.156013:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.156013:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.156014:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.156014:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.156017:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.156017:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.156018:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.156020:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.156020:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.156021:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.156022:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.156024:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.156026:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:9.2:1619677448.156026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:13.0:1619677448.156027:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.156028:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.156030:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.156031:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.156034:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.156037:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.156038:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.156039:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.156041:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.156044:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.156059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.156066:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.156068:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.156069:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.156071:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.156074:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.156075:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.156076:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718622] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.156078:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.156079:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.156080:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.156082:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.156084:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.156085:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.156085:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.156086:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.156087:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.156087:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.156088:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.156088:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.156099:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.156101:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.156101:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.156102:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.156103:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.156104:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.156193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:21.2:1619677448.156194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.156200:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677448.156201:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.156204:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:22.0:1619677448.156204:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.156204:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:30.0:1619677448.156208:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:6.0:1619677448.156208:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.156209:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.156211:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.156212:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.156213:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677448.156214:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:6.0:1619677448.156215:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.156216:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.156216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:6.0:1619677448.156217:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.156219:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.156219:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.156220:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.156220:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.156221:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.156223:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.156226:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:6.0:1619677448.156226:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.156227:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:31.0:1619677448.156228:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.156229:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.156231:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.156231:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:6.0:1619677448.156231:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.156232:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:10.0:1619677448.156233:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.156233:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.156234:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.156234:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:31.0:1619677448.156235:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:6.0:1619677448.156235:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.156236:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:10.0:1619677448.156236:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96718619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.156236:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.156236:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:31.0:1619677448.156237:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:10.0:1619677448.156237:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.156237:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677448.156238:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677448.156238:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.156239:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.156239:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:10.0:1619677448.156239:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.2:1619677448.156241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:10.0:1619677448.156243:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.156244:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.2:1619677448.156245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.156247:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.156248:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.156249:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:10.0:1619677448.156249:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.156250:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.156251:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:17.0:1619677448.156252:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:10.0:1619677448.156252:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:17.0:1619677448.156253:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.156253:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:5.0:1619677448.156253:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:1.0:1619677448.156253:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:10.0:1619677448.156254:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:17.0:1619677448.156255:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.156256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.156256:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.156257:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.156258:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.156259:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:17.0:1619677448.156259:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677448.156261:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:5.0:1619677448.156261:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.156263:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677448.156264:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.156264:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.156264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.156264:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.156266:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:17.0:1619677448.156266:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:1.0:1619677448.156266:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.156268:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.156269:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:17.0:1619677448.156269:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677448.156269:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:10.0:1619677448.156270:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:1.0:1619677448.156270:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.156270:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:26.0:1619677448.156271:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:17.0:1619677448.156271:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.156432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.156438:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.156441:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.156442:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.156444:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.156446:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.156447:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.156449:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.156450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:18.0:1619677448.156450:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.156451:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.156452:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.156455:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.156456:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.156458:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.156459:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.156460:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.156460:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.156461:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:18.0:1619677448.156462:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.156462:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:30.0:1619677448.156463:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.156463:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.156464:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:31.0:1619677448.156465:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677448.156465:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:30.0:1619677448.156466:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.156467:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677448.156469:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.156469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.156470:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.156470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000400:00000200:30.0:1619677448.156471:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.156472:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677448.156475:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:30.0:1619677448.156476:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.156476:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677448.156477:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677448.156477:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.156478:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677448.156479:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:2.0:1619677448.156479:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:30.0:1619677448.156480:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.156480:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.156481:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677448.156481:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:20.0:1619677448.156481:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.156483:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:5.0:1619677448.156483:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.156484:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:4.2:1619677448.156484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.156485:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.156485:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677448.156485:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:10.0:1619677448.156492:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.156494:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.156495:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.156497:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.156499:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:10.0:1619677448.156500:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.156501:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:10.0:1619677448.156501:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:6.0:1619677448.156502:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.156503:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.156503:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718616] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.156504:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.156505:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.156505:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.156506:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.156510:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.156511:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:10.0:1619677448.156512:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.156515:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.156515:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.156516:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.156518:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.156518:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.156519:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.156521:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.156521:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.156527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.2:1619677448.156529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:6.0:1619677448.156531:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.156533:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:26.0:1619677448.156534:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.156534:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.156534:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.156534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.156535:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:18.0:1619677448.156535:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:26.0:1619677448.156536:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.156536:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.156537:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:18.0:1619677448.156537:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.156538:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.156540:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:6.0:1619677448.156540:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677448.156540:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677448.156543:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.156543:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.156544:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677448.156545:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677448.156546:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.156546:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.156547:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.156548:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.156548:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.156549:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.156550:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.156552:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.156553:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:1.0:1619677448.156553:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.156554:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000400:00000200:1.0:1619677448.156555:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.156556:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.156556:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.156556:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:1.0:1619677448.156556:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.156557:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.156559:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.156559:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677448.156559:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.156560:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677448.156560:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:1.0:1619677448.156560:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.156562:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.156562:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677448.156562:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.156563:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.156564:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677448.156565:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.156565:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.156566:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.156568:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677448.156569:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.156569:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.2:1619677448.156570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677448.156570:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.156575:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (23)++
00000800:00000200:31.0:1619677448.156577:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:6.0:1619677448.156577:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000800:00000200:22.0:1619677448.156578:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (23)++
00000400:00000200:6.0:1619677448.156578:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.156579:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000800:00000200:31.0:1619677448.156580:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:22.0:1619677448.156580:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.156581:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000400:00000200:22.0:1619677448.156582:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.156583:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.156584:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:6.0:1619677448.156584:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.156585:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:31.0:1619677448.156586:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.156586:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:31.0:1619677448.156588:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.156589:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.156591:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.156592:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.156593:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.156596:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.156598:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.156601:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.156602:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.156603:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.156605:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.156605:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.156606:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.156607:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.156611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.156619:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.156621:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.156622:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.156624:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.156624:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.156626:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.156674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.156678:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.156680:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.156682:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.156683:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.156685:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.156686:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.156688:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.156689:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.156690:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.156691:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:21.2:1619677448.156692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:20.0:1619677448.156695:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.156696:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.156699:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.156700:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:8.0:1619677448.156700:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677448.156701:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.0:1619677448.156702:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.156703:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677448.156703:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.156704:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.156705:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.156705:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:8.0:1619677448.156705:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.156707:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.156709:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.156710:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96718611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.156711:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.156712:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.156713:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.156716:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:8.0:1619677448.156717:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.156718:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:8.0:1619677448.156718:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.156719:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.156721:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.156721:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.156722:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:8.0:1619677448.156722:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.156723:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.156724:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.0:1619677448.156724:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.156725:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.156726:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.156727:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.156728:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:21.2:1619677448.156733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:13.0:1619677448.156737:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:24.0:1619677448.156740:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:13.0:1619677448.156740:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.156740:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.156742:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:13.0:1619677448.156742:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.156743:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.156744:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677448.156744:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.156745:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677448.156756:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.156759:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.156759:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.156760:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.156761:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.156762:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.156925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.156930:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.156933:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.156934:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.156936:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.156937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:22.0:1619677448.156938:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.156940:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.156941:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.156942:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.156943:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.156944:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.156944:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.156947:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.156947:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:22.0:1619677448.156949:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.156951:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.156951:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.156954:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.156955:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.156955:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:22.0:1619677448.156956:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.156956:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:10.0:1619677448.156957:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.156958:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:10.0:1619677448.156958:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.156959:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677448.156959:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.156960:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:10.0:1619677448.156960:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.156961:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.156962:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677448.156963:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.156965:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:21.2:1619677448.156967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.156968:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.156969:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.156970:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.156971:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677448.156972:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:10.0:1619677448.156972:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.156972:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:10.0:1619677448.156973:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.156973:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.156974:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:31.0:1619677448.156975:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:6.0:1619677448.156975:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:31.0:1619677448.156976:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.156976:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:31.0:1619677448.156978:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.156978:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.156981:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:6.0:1619677448.156985:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.156987:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.156988:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.156989:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.156991:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.156993:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.157006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.157014:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.2:1619677448.157015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.157017:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.157017:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.157019:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.157021:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.157022:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.157024:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:17.0:1619677448.157024:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.157024:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.157025:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.157027:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.157028:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.157029:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.157030:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677448.157030:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.157032:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.157033:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.157034:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.157035:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.157037:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:17.0:1619677448.157037:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.157038:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.157040:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.157041:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.157041:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:17.0:1619677448.157041:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.157043:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.157044:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.157044:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.157045:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.157046:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677448.157047:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.157048:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.157048:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:9.2:1619677448.157049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.157050:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.157051:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.157051:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.157052:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677448.157057:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.157060:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.157061:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.157061:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.157063:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:2.0:1619677448.157063:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.157066:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:2.0:1619677448.157066:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:12.0:1619677448.157067:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.157069:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.157070:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.157072:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.2:1619677448.157177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.157186:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.157188:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.157191:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.157191:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.107@o2ib7 : GET - for me
00000400:00000200:5.0:1619677448.157194:0:19935:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.107@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:5.0:1619677448.157196:0:19935:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.107@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:5.0:1619677448.157198:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.157198:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.157200:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677448.157204:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : REPLY try# 0
00000800:00000200:5.0:1619677448.157206:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.157208:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.157208:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.157209:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.157210:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.157211:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.2:1619677448.157216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.157224:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.157226:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.157227:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.157229:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: REPLY: OK
00000800:00000200:5.0:1619677448.157230:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.157231:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.157411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.157416:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.157419:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.157420:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.157422:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.157425:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.157426:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.157427:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.157429:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.157430:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.157431:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.157434:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.157435:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.157438:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.157439:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.157439:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.157441:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.157441:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.157442:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.157443:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.157448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.2:1619677448.157448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.2:1619677448.157450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.157455:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:1.0:1619677448.157455:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.157456:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:5.0:1619677448.157458:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:1.0:1619677448.157458:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:18.0:1619677448.157459:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:17.0:1619677448.157459:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677448.157460:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.157460:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677448.157461:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:1.0:1619677448.157462:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.157463:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.157463:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.157465:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:1.0:1619677448.157465:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.157466:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.157467:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:5.0:1619677448.157468:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.157468:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.157470:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.157470:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.157471:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.157472:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.157472:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:5.0:1619677448.157473:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.157475:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:21.2:1619677448.157476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:1.0:1619677448.157477:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.157480:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:5.0:1619677448.157481:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.157483:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.157483:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.157484:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.157485:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.157485:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.157486:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677448.157487:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.157487:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.157487:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.157488:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.0:1619677448.157488:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.157488:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.157488:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677448.157490:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.157490:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.157491:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.157491:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677448.157492:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.157492:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.157492:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:30.0:1619677448.157494:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677448.157495:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96718603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.157497:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.157497:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677448.157497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:30.0:1619677448.157499:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:13.0:1619677448.157500:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:31.0:1619677448.157502:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:30.0:1619677448.157502:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.157503:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:31.0:1619677448.157504:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.157504:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:13.0:1619677448.157504:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.157504:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.157505:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.157505:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.157507:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.157507:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.157507:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.157507:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:30.0:1619677448.157508:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.157508:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:13.0:1619677448.157508:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.157509:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.157509:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.157510:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.157510:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:10.0:1619677448.157510:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.157511:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.157512:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:10.0:1619677448.157513:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.157514:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.157515:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677448.157515:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.157516:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718602] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.157517:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:10.0:1619677448.157518:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.157518:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.157519:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.157519:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.157520:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.157520:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:21.2:1619677448.157520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.157521:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.157522:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.157523:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:10.0:1619677448.157523:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.157524:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.157525:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:31.0:1619677448.157526:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.157526:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.157527:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.157527:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.157527:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.157528:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.157528:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.157529:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:6.0:1619677448.157529:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.157531:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.157531:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677448.157531:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.157532:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.157532:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.157534:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.157544:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.157546:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.157546:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.157547:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.157548:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.157548:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.157549:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.157549:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.157550:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.157551:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.157551:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.157554:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.157580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.157587:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.157590:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.157592:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.157594:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.157595:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.157597:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.157598:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.157599:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.157600:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.157601:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.157604:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.157606:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.157609:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.157610:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.157611:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.157613:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.157613:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.157614:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.157616:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.157616:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.157619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.157623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.157631:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.157631:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.157633:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677448.157634:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.157634:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.157635:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.157635:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.157637:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.157637:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.157638:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.157639:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:20.0:1619677448.157640:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (24)--
00000800:00000200:5.0:1619677448.157640:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:20.0:1619677448.157641:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677448.157641:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.2:1619677448.157802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.157807:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.157810:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.157811:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.157813:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.157816:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.157817:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.157819:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.157820:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.157821:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.157822:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.157825:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.157827:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.157829:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.157830:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.157831:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.157832:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.157833:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.157834:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.157835:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.157836:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.157841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677448.157843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:1.0:1619677448.157847:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677448.157848:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.157849:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.157849:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677448.157850:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.157850:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677448.157850:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677448.157851:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.157851:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:22.0:1619677448.157853:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.157854:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.157856:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.157857:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:6.0:1619677448.157857:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.157858:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.157859:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:5.2:1619677448.157859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:6.0:1619677448.157860:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.157862:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.157863:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.157864:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.157866:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.157867:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.157868:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.157870:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.157870:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677448.157871:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.157872:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000400:00000200:26.0:1619677448.157873:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.157873:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.157875:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.157875:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:26.0:1619677448.157876:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.157876:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.157876:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:26.0:1619677448.157877:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.157878:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.157880:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.157881:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.157882:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.157885:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.157887:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.157887:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.157889:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.157890:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.157890:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.157891:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.157892:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:10.0:1619677448.157892:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.157894:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.157895:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.157895:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.157896:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.157896:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:26.0:1619677448.157897:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.157898:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.157903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.157906:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.157908:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.157909:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.157909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677448.157910:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:30.0:1619677448.157911:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.157912:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:30.0:1619677448.157913:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.157913:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.157914:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677448.157915:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.157916:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.157918:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.157920:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.157922:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.157922:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.157924:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.157925:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.157926:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.157927:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.157928:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.157931:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.157933:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.157936:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.157937:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.157937:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.157939:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.157939:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.157940:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.157941:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.157945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.157953:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.157956:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.157957:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.157957:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.157959:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.157963:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.158037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.158045:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.158048:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.158048:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.158051:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.158053:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.158055:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.158056:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.158056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677448.158057:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.158058:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.158059:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.158061:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.158062:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.158063:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.158064:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.158065:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.158066:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.158066:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:17.0:1619677448.158067:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.158067:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.158068:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.158069:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.158069:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677448.158070:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.158070:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:17.0:1619677448.158071:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.158073:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.158074:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.158075:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.158076:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.158079:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.158081:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.158082:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677448.158084:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.158085:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:17.0:1619677448.158085:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (23)++
00000400:00000200:31.0:1619677448.158086:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.158086:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (23)++
00000400:00000200:31.0:1619677448.158088:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.158088:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:17.0:1619677448.158088:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:17.0:1619677448.158089:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.158089:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.158090:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.158091:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.158101:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.158103:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.158104:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.158106:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.158107:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.158108:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.158182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.158189:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.158192:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.158193:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.158195:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.158197:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.158199:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.158200:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.158202:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.158203:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.158204:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677448.158207:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.158208:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.158211:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.158212:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.158213:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.158215:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.158215:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.158216:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.158216:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.158220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.158229:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.158232:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.158233:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.158234:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.158235:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.158237:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.158369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677448.158375:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.158378:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:5.2:1619677448.158378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.158380:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677448.158381:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.158381:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.158382:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:18.0:1619677448.158384:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.158385:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.158386:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.158387:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.158389:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.158390:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.158391:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.158393:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.158394:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.158395:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.158396:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.158397:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.158397:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.158398:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.158399:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.158402:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.158402:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.158403:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.158404:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.158404:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.158406:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.158406:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.158407:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677448.158407:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.158407:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677448.158408:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677448.158409:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:2.0:1619677448.158409:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:2.0:1619677448.158409:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.158410:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.158410:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.158411:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.158413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (22)++
00000800:00000200:21.2:1619677448.158414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.158420:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677448.158421:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000800:00000200:22.0:1619677448.158423:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:1.0:1619677448.158423:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000800:00000200:17.0:1619677448.158424:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000800:00000200:6.0:1619677448.158424:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677448.158424:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.158425:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000400:00000200:22.0:1619677448.158426:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.158426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:17.0:1619677448.158426:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.158427:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.158428:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.158430:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.158431:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:17.0:1619677448.158431:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.158431:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:22.0:1619677448.158432:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.158432:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677448.158434:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:22.0:1619677448.158434:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.158435:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.158436:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.158437:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677448.158441:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677448.158441:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.158442:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.158442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677448.158443:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.158445:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.158446:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677448.158446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:31.0:1619677448.158447:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677448.158447:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.158447:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:31.0:1619677448.158448:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.158449:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.158449:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.158449:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.158449:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.158450:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:31.0:1619677448.158451:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.158451:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:31.0:1619677448.158452:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.158452:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:24.0:1619677448.158453:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677448.158453:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.158455:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.158455:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.2:1619677448.158455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:31.0:1619677448.158456:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.158457:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.158458:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.158459:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:24.0:1619677448.158459:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.158460:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96718589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.158461:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.158461:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677448.158462:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.158462:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.158462:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.158463:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.158463:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:26.0:1619677448.158463:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677448.158463:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:24.0:1619677448.158464:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:31.0:1619677448.158465:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.158465:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.158466:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:24.0:1619677448.158466:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96718588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.158466:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.158466:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.158467:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:26.0:1619677448.158467:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.158467:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.158469:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:24.0:1619677448.158469:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.158469:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.158470:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.158470:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.158471:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:26.0:1619677448.158472:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.158473:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.158474:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:24.0:1619677448.158474:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.158475:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.158476:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677448.158476:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.158477:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.158478:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.158479:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677448.158479:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.158479:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:24.0:1619677448.158480:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.158480:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.158482:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.158482:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677448.158482:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:24.0:1619677448.158483:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.158483:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677448.158483:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.158484:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:4.2:1619677448.158484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677448.158485:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.158486:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.2:1619677448.158487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.158488:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.158489:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.158489:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:22.0:1619677448.158490:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.158490:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.158492:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.158494:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677448.158495:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:10.0:1619677448.158495:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.158496:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677448.158496:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677448.158497:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.158498:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:22.0:1619677448.158499:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677448.158499:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.158499:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.158500:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.158500:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.158501:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.158501:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:17.0:1619677448.158501:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.158504:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.158579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677448.158587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.158588:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.158590:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.158593:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.158593:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.158594:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.158595:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.158595:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.158596:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677448.158597:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677448.158599:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.158600:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718587] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.158602:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.158602:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.158603:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.158604:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677448.158605:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677448.158607:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.158608:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.158610:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.158610:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.158611:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.158613:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:1.0:1619677448.158615:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.158616:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.158617:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.158617:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:1.0:1619677448.158617:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.158619:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.158619:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:1.0:1619677448.158619:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.158620:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.158620:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.158621:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.158621:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:1.0:1619677448.158621:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.158622:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.158623:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.158623:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.158625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.2:1619677448.158629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.158633:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.158635:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:17.0:1619677448.158636:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:5.0:1619677448.158636:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.158638:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:17.0:1619677448.158638:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:5.0:1619677448.158638:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.158639:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.158640:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.158641:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.158641:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677448.158643:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.158645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.158653:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.158655:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.158656:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.158656:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.158657:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.158657:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.158711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.158713:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.158716:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.158718:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.158719:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.158721:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.158722:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.158724:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.158725:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.158726:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.158727:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.158731:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.158732:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.158736:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.158737:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.158737:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.158739:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.158740:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.158741:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.158742:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.158752:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.158755:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.158756:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.158758:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.158759:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.158761:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.158794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677448.158799:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.158802:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677448.158803:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677448.158805:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.158807:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.158808:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677448.158809:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.158811:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.158812:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.158813:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677448.158816:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.158818:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.158820:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.158821:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:17.0:1619677448.158822:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.158823:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677448.158824:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.158824:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677448.158825:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:6.0:1619677448.158838:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.158840:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.158842:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.158842:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.158844:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.158844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:6.0:1619677448.158846:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677448.158851:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.158854:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.158855:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.158857:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.158859:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.158861:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.158862:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.158863:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.158865:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.158865:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.158869:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.158871:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.158873:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.158874:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.158875:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.158876:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.158877:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.158878:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.158878:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.158888:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.158891:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.158891:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.158893:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.158895:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.158895:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.2:1619677448.158915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.158921:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.158923:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.158926:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.158926:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.2:1619677448.158930:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:5.0:1619677448.158932:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.158934:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:5.0:1619677448.158935:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.158935:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677448.158936:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.158938:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.158938:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.158939:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677448.158939:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.158942:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.158943:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000400:00000200:1.0:1619677448.158944:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.158946:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.158946:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.158947:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.158949:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.158949:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.158950:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677448.158950:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:1.0:1619677448.158950:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.158951:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.158953:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677448.158953:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.158954:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000400:00000200:1.0:1619677448.158954:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.158955:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:1.0:1619677448.158956:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:9.2:1619677448.158958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.158958:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.158959:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.158960:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.158962:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.158962:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.158963:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.158963:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677448.158965:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.158968:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.158969:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.158969:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.158971:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677448.158974:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677448.158975:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.158978:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.158979:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.158979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:6.0:1619677448.158979:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.158981:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.158983:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677448.158985:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.158987:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677448.158988:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.158989:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.158990:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.158990:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.2:1619677448.159196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.159202:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.159205:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.159205:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.159208:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.159210:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.159212:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.159213:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.159214:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.159215:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.159216:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.159220:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.159221:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.159224:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.159224:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.159225:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.159227:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.159227:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.159228:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.159228:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.159233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.159241:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.159243:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.159244:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.159244:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.159246:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.159248:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.159274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.159279:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.159282:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.159282:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.159284:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.159286:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.159288:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.159289:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.159290:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.159291:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.159292:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.159299:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.159301:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.159302:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.159303:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.159304:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.159306:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.159306:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.159307:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.159308:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.159309:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.159312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.159315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.159321:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.159322:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.159322:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:17.0:1619677448.159323:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.159323:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.159323:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.159324:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.159324:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.159325:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677448.159326:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.159332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.159339:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.159341:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.159342:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.159343:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.159343:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.159344:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.159368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677448.159374:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677448.159377:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677448.159378:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677448.159380:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.159382:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.159383:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677448.159385:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718578] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.159386:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.159387:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.159388:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677448.159391:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.159392:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677448.159395:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.159396:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677448.159397:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677448.159398:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677448.159399:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677448.159400:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677448.159400:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:26.0:1619677448.159412:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.159415:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.159416:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.159416:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.159418:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.159421:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677448.159437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.159445:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.159448:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.159450:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.159453:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.159453:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.159454:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:5.2:1619677448.159454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:31.0:1619677448.159455:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.159456:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.159457:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.159458:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:10.0:1619677448.159461:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677448.159462:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.159463:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:31.0:1619677448.159464:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:26.0:1619677448.159464:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.159466:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.159467:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.159468:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.159468:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:10.0:1619677448.159468:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.159469:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:31.0:1619677448.159470:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.159470:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677448.159470:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.159471:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.159472:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:10.0:1619677448.159472:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.159472:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.159473:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.159476:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.159476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:10.0:1619677448.159478:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.159480:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.159481:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.159482:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677448.159483:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:10.0:1619677448.159483:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.159483:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.159484:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.159485:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:18.0:1619677448.159486:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.159487:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.159487:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.159489:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.159492:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:12.0:1619677448.159496:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.159498:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.159499:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.159501:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.159503:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.159504:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.159573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.159579:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.159581:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.159582:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.159584:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.159587:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.159588:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.159590:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.159591:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.159592:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.159593:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.159596:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.159598:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.159601:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.159601:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.159602:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.159604:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.159604:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.159605:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.159605:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677448.159618:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.159621:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.159622:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.159622:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.159624:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.159626:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.159651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:5.2:1619677448.159653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677448.159658:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:1.0:1619677448.159659:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.159661:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.159661:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:17.0:1619677448.159662:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.159662:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:18.0:1619677448.159665:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.159666:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.159668:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.159668:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.159669:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677448.159671:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677448.159671:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.159672:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.159673:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.159674:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.159674:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.159676:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.159676:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677448.159677:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677448.159681:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.159682:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.159684:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.159685:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677448.159687:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.159687:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:20.2:1619677448.159688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677448.159688:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.159688:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677448.159689:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677448.159690:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.159690:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:20.0:1619677448.159691:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677448.159691:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677448.159691:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.159692:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:1.0:1619677448.159692:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.159692:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677448.159693:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:18.0:1619677448.159693:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.159693:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:20.0:1619677448.159696:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.159697:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677448.159698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:20.0:1619677448.159699:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.159701:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.159702:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96718572] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.159703:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.159704:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:20.0:1619677448.159704:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.159705:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.159706:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:5.0:1619677448.159706:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:22.0:1619677448.159707:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.159707:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:5.0:1619677448.159708:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.159708:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.159709:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.159710:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.159711:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.159712:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.159713:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:22.0:1619677448.159714:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677448.159714:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677448.159716:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.159717:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.159717:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.159719:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.159720:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.159721:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.159721:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677448.159724:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.159726:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.159727:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.159727:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.159728:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.159729:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677448.159732:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.159734:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.159735:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.159737:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.159738:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.159740:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.159749:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.159752:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.159753:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.159754:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.159754:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.159756:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.159768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.159774:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:4.2:1619677448.159775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677448.159777:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.159778:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.159780:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.159782:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677448.159782:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.159783:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.159785:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.159786:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.159786:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677448.159787:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.159788:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.159789:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677448.159790:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.159792:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.159793:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.159794:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.159795:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.159795:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:26.0:1619677448.159797:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.159797:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.159798:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.159798:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:26.0:1619677448.159799:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.159799:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.159800:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.159801:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.159802:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.159802:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:26.0:1619677448.159803:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.159804:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.159807:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677448.159807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.159808:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.159809:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.159811:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.159811:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.159812:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.159812:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677448.159814:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.159817:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.159817:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.159818:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.159820:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.159822:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677448.159824:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.159827:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.159828:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.159828:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.159830:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.159833:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.159932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.159940:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.159942:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.159943:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.159945:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.159948:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.159949:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.159950:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.159951:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.159953:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.159953:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.159956:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.159957:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.159959:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.159959:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.159960:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.159961:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.159962:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.159962:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.159963:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677448.159973:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.159974:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.159975:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.159976:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.159976:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.159978:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.160043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.160050:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.160053:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.160055:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.160057:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.160058:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.160059:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.160060:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.160062:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.160062:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.160063:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.160066:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.160068:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.160070:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.160070:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.160071:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.160072:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.160073:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.160074:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.160075:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.160076:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677448.160082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.160083:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.160086:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.160086:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.160087:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.160088:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.160089:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.160091:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:10.0:1619677448.160091:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677448.160092:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.160093:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.160102:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.160104:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.160105:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.160105:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.160106:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.160106:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.160215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.160222:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.160225:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.160226:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.160228:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.160230:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.160232:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.160233:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.160234:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.160235:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.160236:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.160239:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.160241:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.160243:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.160244:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.160245:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.160246:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.160247:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.160247:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.160248:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.160252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.160261:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.160263:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.160264:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.160265:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.160266:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.160269:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.160382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.160388:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.160391:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.160391:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.160394:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.160396:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.160397:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677448.160398:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.160400:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.160401:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.160402:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677448.160405:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.160407:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677448.160410:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.160410:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677448.160411:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.160413:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677448.160413:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.160414:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.160414:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677448.160420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.2:1619677448.160426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.160433:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.160434:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677448.160435:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.160436:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.160437:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.160437:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.160438:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.160439:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.160440:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:17.0:1619677448.160440:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.160441:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.160442:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677448.160442:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:20.0:1619677448.160443:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000400:00000200:17.0:1619677448.160443:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.160443:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.160444:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (22)--
00000400:00000200:17.0:1619677448.160444:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.160445:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:17.0:1619677448.160446:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.160446:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.160450:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.160452:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.160455:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.160456:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.160457:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.160458:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.160459:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.160459:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.160460:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.160465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.160471:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.160474:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.160475:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.160475:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.160477:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.160480:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677448.160546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.160554:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.160557:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:21.2:1619677448.160558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:31.0:1619677448.160559:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.160561:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.160562:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.160563:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.160564:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.160564:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.160565:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:4.2:1619677448.160565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:31.0:1619677448.160566:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.160567:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.160568:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.160571:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.160572:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.160572:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677448.160573:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.160575:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.160575:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:26.0:1619677448.160575:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.160575:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:30.0:1619677448.160576:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.160578:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.160579:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.160579:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.160580:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.160580:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.160582:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:31.0:1619677448.160583:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.160583:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96718562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.160584:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:30.0:1619677448.160584:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.160585:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:10.0:1619677448.160585:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.160586:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.160586:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.160587:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.160587:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677448.160587:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.160588:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.160589:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.160589:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.160589:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.160590:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.160590:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.160591:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.160591:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.160592:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.160592:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.160593:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.160593:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.160593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.160595:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.160596:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)++
00000800:00000200:10.0:1619677448.160597:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (23)++
00000800:00000200:10.0:1619677448.160599:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (24)--
00000800:00000200:10.0:1619677448.160599:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.160600:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.160600:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.160601:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (23)++
00000800:00000200:5.0:1619677448.160603:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (24)--
00000800:00000200:13.0:1619677448.160604:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:5.0:1619677448.160604:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.160606:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (23)++
00000400:00000200:5.0:1619677448.160606:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.160607:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (24)--
00000800:00000200:13.0:1619677448.160607:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:20.0:1619677448.160608:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.160608:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.160609:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.160610:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:20.0:1619677448.160611:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:13.0:1619677448.160611:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.160612:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.160614:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677448.160614:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.160828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.160835:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.160838:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.160839:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.160841:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.160843:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.160845:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.160846:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.160847:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.160849:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.160849:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.160853:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.160854:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.160857:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.160858:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.160858:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.160860:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.160860:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.160861:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.160861:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.160866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.160874:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.160876:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.160877:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.160878:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.160880:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.160882:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.160938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.2:1619677448.160942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:31.0:1619677448.160946:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.160946:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:31.0:1619677448.160949:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.0:1619677448.160949:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677448.160951:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:31.0:1619677448.160952:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.160952:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:5.0:1619677448.160952:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : ACK - routed
00000400:00000200:31.0:1619677448.160954:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.160956:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.160956:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.160957:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.160957:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.160959:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.160960:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.160961:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.160962:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.160962:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.160963:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.160964:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:31.0:1619677448.160969:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.160969:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000800:00000200:31.0:1619677448.160972:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.160973:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:31.0:1619677448.160974:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.160975:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.160975:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.160976:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.160976:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.160977:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:31.0:1619677448.160978:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.160978:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.160979:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.0:1619677448.160979:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.160979:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:31.0:1619677448.160980:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677448.160980:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.160981:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.160984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:12.0:1619677448.160989:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.160992:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:12.0:1619677448.160992:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.160993:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.160994:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:21.2:1619677448.160995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:20.0:1619677448.160995:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.160995:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.160996:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677448.160997:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:12.0:1619677448.160998:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.160999:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.161000:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:2.0:1619677448.161001:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.161004:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.161005:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.161007:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.161008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677448.161009:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.161010:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.161012:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.161013:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.161014:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.161015:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677448.161015:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.161017:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.161018:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677448.161019:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.161021:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.161023:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.161024:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.161025:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:10.0:1619677448.161026:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.161026:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:10.0:1619677448.161027:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.161028:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.161028:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.161028:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:10.0:1619677448.161030:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.161030:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000400:00000200:10.0:1619677448.161031:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.161031:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000400:00000200:10.0:1619677448.161032:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.161032:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:10.0:1619677448.161035:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.161037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.161038:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.161041:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.161041:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.161042:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:10.0:1619677448.161042:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.161043:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.161043:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:26.0:1619677448.161044:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.161045:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.161045:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:10.0:1619677448.161045:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:26.0:1619677448.161046:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.161046:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:10.0:1619677448.161046:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.161047:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.161048:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.161049:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.161050:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:6.0:1619677448.161059:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.161062:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.161063:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.161063:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.161066:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.161068:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677448.161079:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.161081:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.161082:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.161083:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.161084:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.161084:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.161108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.161114:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.161117:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.161119:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.161120:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.161121:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.161123:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.161125:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.161126:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.161127:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.161128:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.161132:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.161134:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.161136:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.161137:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.161138:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.161140:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.161141:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.161142:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.161143:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.161144:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.161149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.161152:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:30.0:1619677448.161153:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.161154:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.161155:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677448.161156:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677448.161156:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:30.0:1619677448.161157:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.161159:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.161159:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.161162:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.161297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.161305:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.161307:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.161308:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.161310:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.161313:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.161314:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.161315:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.161317:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.161317:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.161319:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.161322:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.161324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677448.161324:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.161327:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.161327:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.161328:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.161330:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.161330:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.161331:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:31.0:1619677448.161333:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.161333:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.161333:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.161335:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.161337:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.161339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.161340:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.161340:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.161341:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.161342:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.161344:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.161344:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.161345:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:6.0:1619677448.161345:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.161346:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677448.161348:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.161348:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677448.161348:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.161349:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677448.161349:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677448.161350:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.161350:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.161350:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.161352:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.161353:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.161354:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.161354:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:6.0:1619677448.161355:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677448.161356:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.161357:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.161358:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.161358:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.161369:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.161371:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.161372:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.161374:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.161375:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.161376:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.161461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.161468:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.161471:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.161472:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.161474:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.161477:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.161478:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.161479:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.161481:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.161482:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.161483:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.161486:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.161487:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.161490:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.161491:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.161491:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.161493:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.161494:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.161495:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.161495:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.161500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.161508:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.161511:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.161512:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.161512:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.161514:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.161518:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.161519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.161526:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.161529:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.161529:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.161530:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.161531:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.161531:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.161640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.161647:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.161650:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.161650:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.161653:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.161655:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.161657:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.161658:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.161659:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.161660:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.161661:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.161664:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.161666:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.2:1619677448.161666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.161667:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.161668:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.161668:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.161670:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.161670:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.161671:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.161671:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677448.161672:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.161675:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.161675:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.161678:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.161680:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.161681:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:17.0:1619677448.161682:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:26.0:1619677448.161683:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:17.0:1619677448.161683:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.161684:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.161684:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.161684:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:17.0:1619677448.161685:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.161686:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.161687:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.161689:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:17.0:1619677448.161690:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.161692:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.161695:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.161696:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.161696:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.161698:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.161698:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.161699:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.161700:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:21.2:1619677448.161708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677448.161712:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.161714:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.161715:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.161715:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677448.161716:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:22.0:1619677448.161718:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.161719:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677448.161720:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:31.0:1619677448.161721:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.161722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:30.0:1619677448.161723:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.161724:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.161725:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677448.161725:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.161726:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.161727:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:31.0:1619677448.161728:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.161729:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.161731:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.161731:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.161731:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.161734:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.161735:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.161736:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:31.0:1619677448.161737:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.161737:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.161739:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.161739:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.161740:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:20.0:1619677448.161740:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.161741:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.161741:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.161742:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.161743:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.161744:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.161744:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:20.0:1619677448.161744:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.161746:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.161748:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.161749:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.161750:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.161751:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.2:1619677448.161753:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:13.0:1619677448.161755:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.161756:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.161757:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.161758:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.161758:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.161759:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.161760:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.161761:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.161762:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.161764:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:13.0:1619677448.161764:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.161765:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:30.0:1619677448.161766:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.161767:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677448.161768:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.161768:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.161768:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.161769:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:30.0:1619677448.161770:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.161771:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.161772:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.161773:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.161773:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:30.0:1619677448.161774:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.161775:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.161779:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.161781:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.161784:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.161785:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.161786:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.161787:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.161788:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.161790:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.161791:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.161792:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.161794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.161798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.161802:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.161804:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.161805:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.161806:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.161807:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.161807:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.161808:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.161809:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.161810:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.161810:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.161816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.161824:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.161827:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.161827:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.161829:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.161831:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.161832:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.161834:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718547] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.161835:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.161836:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.161837:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.161840:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.161841:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.161844:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.161845:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.161845:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.161848:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.161848:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.161849:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.161850:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:5.2:1619677448.161850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:21.2:1619677448.161851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:10.0:1619677448.161851:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677448.161854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.161856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677448.161857:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677448.161858:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:22.0:1619677448.161859:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.161860:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677448.161861:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:26.0:1619677448.161863:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:22.0:1619677448.161863:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.161863:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:22.0:1619677448.161865:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.161865:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.161865:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.161866:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.161867:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677448.161867:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677448.161867:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.161867:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.161868:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677448.161869:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.161869:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.161869:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.161870:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.161871:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.161872:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.161873:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677448.161873:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.161874:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.161876:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.161878:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.161879:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.161880:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:22.0:1619677448.161881:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677448.161881:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:2.0:1619677448.161881:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.161884:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.161884:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.161886:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.161886:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.161886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:2.0:1619677448.161887:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.161888:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:2.0:1619677448.161888:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.161888:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.161889:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677448.161889:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:22.0:1619677448.161890:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677448.161890:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.161891:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:2.0:1619677448.161891:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:22.0:1619677448.161892:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677448.161892:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:2.0:1619677448.161893:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:20.0:1619677448.161894:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.161894:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.161894:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677448.161895:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.161896:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.161896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.161898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:20.0:1619677448.161898:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.161900:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96718544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.161900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677448.161901:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:20.0:1619677448.161901:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.161902:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.161903:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.161904:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:6.0:1619677448.161904:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:31.0:1619677448.161905:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.161906:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:6.0:1619677448.161906:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.161906:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:31.0:1619677448.161907:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.161907:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677448.161907:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677448.161908:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:20.0:1619677448.161908:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.161908:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.161909:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677448.161909:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677448.161910:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.161910:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.161910:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.161910:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:31.0:1619677448.161912:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.161912:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)++
00000400:00000200:5.0:1619677448.161912:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.161913:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.161913:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.161913:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.161914:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.161915:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.0:1619677448.161915:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.161915:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.161916:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.161917:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.161918:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.161918:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.161918:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.161919:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.161919:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.161920:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.161921:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.161921:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.161921:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.161926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677448.161930:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.161932:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:5.0:1619677448.161932:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000400:00000200:30.0:1619677448.161934:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.161934:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.161935:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677448.161936:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.161936:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.161936:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677448.161939:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.161950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.0:1619677448.161954:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.161956:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677448.161959:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.161959:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.161961:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.161963:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677448.161964:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.161966:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.161967:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.161968:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677448.161971:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.161973:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 440 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677448.161976:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.161977:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677448.161978:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677448.161979:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.161980:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.161981:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.161981:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:31.0:1619677448.161992:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.161995:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.161996:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.161997:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.161999:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.162001:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.162080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.162087:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.162089:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.162090:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.162092:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.162095:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.162096:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.162097:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.162099:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.162100:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.162100:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.162103:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.162105:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.162108:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.162108:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.162109:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.162111:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.162111:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.162111:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.162112:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.162124:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.162127:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.162128:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.162128:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.162131:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.162134:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677448.162145:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.162148:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.162149:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.162149:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.162150:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.162152:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.162168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.162174:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.162177:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.162178:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.162179:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.162181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:1.0:1619677448.162182:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.162184:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.162185:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.162186:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:1.0:1619677448.162186:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.162187:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.162188:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.162189:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.162190:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.162193:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.162193:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.162195:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.162196:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.162197:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677448.162198:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.162198:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:18.0:1619677448.162199:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.162199:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677448.162200:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.162201:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.162201:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:18.0:1619677448.162202:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.162202:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.162203:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.162204:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.162205:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677448.162206:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.162207:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.2:1619677448.162209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.162211:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.162211:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.162212:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:18.0:1619677448.162215:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:17.0:1619677448.162215:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677448.162215:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)++
00000800:00000200:18.0:1619677448.162216:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.162216:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.162217:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677448.162217:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.162218:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677448.162218:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:17.0:1619677448.162219:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677448.162219:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.162220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:12.0:1619677448.162221:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.162222:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.162223:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677448.162224:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.162224:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.162225:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677448.162226:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.162227:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.162230:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:13.0:1619677448.162230:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.162231:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.162232:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.162233:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.162234:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:13.0:1619677448.162234:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:31.0:1619677448.162235:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.162235:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.162235:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:31.0:1619677448.162236:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:13.0:1619677448.162236:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677448.162237:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.162238:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.162240:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.162241:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.162244:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.162246:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.162248:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.162249:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.162250:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.162251:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.162252:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.162253:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.162253:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.162263:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.162265:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.162266:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.162268:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.162270:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.162271:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.162486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.162493:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.2:1619677448.162493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:26.0:1619677448.162496:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.162497:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.162498:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.162499:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:26.0:1619677448.162502:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.162502:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.162503:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:26.0:1619677448.162504:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.162505:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.162505:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.162507:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.162508:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.162508:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.162508:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.162510:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.162512:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.162513:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.162514:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.162515:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.162516:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.162516:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.162518:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.162519:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.162520:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:6.0:1619677448.162520:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.162521:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.162522:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.162522:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.162523:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.162523:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.162524:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.162525:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.162526:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.162527:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.162528:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.162529:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.162529:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.162534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.162535:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.162538:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.162538:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.162540:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.162542:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.162543:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.162543:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.162544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677448.162545:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.162546:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.162546:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.162547:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677448.162548:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.162549:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.162551:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.162551:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.162553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.162553:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.162554:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.162555:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.162557:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.162558:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.162559:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.162560:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.162560:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.162563:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.162564:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.162564:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.162564:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.162565:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.162565:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677448.162566:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.162568:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.162569:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.162570:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.162571:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.162572:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.162572:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.162573:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677448.162585:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.162587:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.162588:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.162588:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.162590:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.162592:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677448.162606:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.162609:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.162609:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.162610:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.162611:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.162611:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.162628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.162636:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.162638:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.162639:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.162641:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.162644:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.162645:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.162646:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.162646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677448.162648:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.162649:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.162650:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.162652:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.162653:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.162654:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.162655:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.162655:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.162657:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.162658:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.162659:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.162659:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677448.162659:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.162661:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:1.0:1619677448.162661:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:10.0:1619677448.162662:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:1.0:1619677448.162662:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.162663:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.162663:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:1.0:1619677448.162663:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.162664:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.162665:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:9.2:1619677448.162668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677448.162669:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.162671:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.162673:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.162674:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.162674:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677448.162675:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677448.162676:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.162676:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.162677:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.162678:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:1.0:1619677448.162678:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677448.162679:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.162679:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.162682:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.162684:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677448.162690:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.162693:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.162693:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.162694:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.162696:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.162698:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.162725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.162732:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.162735:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.162736:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.162738:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.162740:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.162742:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.162743:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.162743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:24.0:1619677448.162745:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.162746:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.162747:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.162749:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:24.0:1619677448.162750:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.162752:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.162752:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.162753:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.162754:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:24.0:1619677448.162755:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.162756:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.162756:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:18.0:1619677448.162757:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.162758:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.162758:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:18.0:1619677448.162758:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:24.0:1619677448.162759:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.162760:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:18.0:1619677448.162760:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.162761:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.162762:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.162763:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.162766:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.162768:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:4.2:1619677448.162768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:18.0:1619677448.162769:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.162770:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:12.0:1619677448.162770:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:18.0:1619677448.162771:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.162773:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.162773:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:12.0:1619677448.162773:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:18.0:1619677448.162774:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.162774:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:12.0:1619677448.162774:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.162775:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:12.0:1619677448.162776:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.162777:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.162778:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677448.162778:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.162778:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677448.162778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677448.162780:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.162782:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.162784:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.162785:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96718531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.162785:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:8.0:1619677448.162787:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.162788:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.162788:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:8.0:1619677448.162789:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.162789:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.162790:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677448.162790:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:2.0:1619677448.162792:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:13.0:1619677448.162793:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.162794:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.162794:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:8.0:1619677448.162794:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.162794:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677448.162795:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.162796:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677448.162797:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677448.162799:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.162800:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.162800:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.162802:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.162802:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.162804:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.162805:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.162805:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.162807:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.162811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.162815:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.162818:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.162818:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.162819:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.162819:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.162820:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.162821:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.162822:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677448.162822:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.162824:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.162908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.162915:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.162918:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.162919:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.162920:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.162923:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.162925:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.162926:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.162928:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.162929:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.162930:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.162933:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.162934:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:21.2:1619677448.162936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:26.0:1619677448.162937:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.162938:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.162938:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.162940:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.162940:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.162941:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.162942:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.162943:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.162943:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677448.162944:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.162944:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.162946:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.162949:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.162949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677448.162951:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.162952:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.162954:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:6.0:1619677448.162954:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.162955:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.162956:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:10.0:1619677448.162956:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:22.0:1619677448.162957:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.162957:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:10.0:1619677448.162958:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677448.162959:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.162959:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.162960:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.162961:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.162962:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.162963:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677448.162963:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.162966:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.162967:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.162967:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.162969:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.162970:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.162970:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.162971:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.162975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.162983:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.162986:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.162986:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.162987:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.162989:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.162991:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.163003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.163010:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.163013:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.163014:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.163015:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.163016:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.163016:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.163139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.163145:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.163148:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.163150:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.163150:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.163152:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.163154:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.163156:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.163157:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.163158:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.163159:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.163162:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.163164:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.163167:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.163168:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.163168:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.163170:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.163170:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.163171:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.163172:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677448.163176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.163183:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.163186:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.163187:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.163187:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.163189:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.163191:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.163214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.163221:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.163224:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.163226:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.163228:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.163228:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.163230:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.163231:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.163232:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.163233:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.163234:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.163237:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.163237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677448.163239:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.163241:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.163242:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.163243:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.163243:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.163244:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.163245:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.163245:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677448.163246:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:1.0:1619677448.163246:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.163248:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:17.0:1619677448.163248:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.163249:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677448.163250:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.163252:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.163253:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.163254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677448.163254:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.163256:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.163256:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.163256:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.2:1619677448.163258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:17.0:1619677448.163259:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.163259:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.163260:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.163262:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:17.0:1619677448.163262:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.163262:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.163262:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.163264:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.163265:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:26.0:1619677448.163265:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.163265:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.163266:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.163266:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677448.163266:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.163266:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.163268:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:17.0:1619677448.163268:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.163269:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.163269:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.163269:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.163270:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677448.163271:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.163274:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.163276:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.163277:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.163278:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.163279:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.163280:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:13.0:1619677448.163281:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.163283:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:26.0:1619677448.163284:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677448.163284:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.163286:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:13.0:1619677448.163286:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.163287:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.163288:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.163289:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:13.0:1619677448.163289:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.163290:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.163291:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.163292:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.163292:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.163293:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.163305:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.163308:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.163308:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.163310:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.163313:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.163313:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:21.2:1619677448.163341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.163347:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.163350:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.163351:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.163353:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.163355:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.163357:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.163358:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.163359:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.163360:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.163361:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.163365:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.163367:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:21.2:1619677448.163367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:22.0:1619677448.163370:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.163371:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.163371:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.163373:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:20.0:1619677448.163373:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:22.0:1619677448.163374:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.163374:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.163375:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677448.163375:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.163377:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.163379:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.163380:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.163382:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.163383:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.163384:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.163385:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.163385:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:20.0:1619677448.163386:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:10.0:1619677448.163388:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.163389:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:20.0:1619677448.163389:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.163389:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.163392:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:10.0:1619677448.163392:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.163394:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.163395:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.163396:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:10.0:1619677448.163396:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.0:1619677448.163397:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.163398:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.163399:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677448.163401:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677448.163401:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:8.2:1619677448.163404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:21.2:1619677448.163407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.163413:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.163414:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677448.163414:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.163415:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677448.163415:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.163415:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.163417:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.163417:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.163418:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677448.163419:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.2:1619677448.163428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:4.2:1619677448.163433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.163436:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.163439:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.163441:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.163441:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.163443:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.163443:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.163443:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.163444:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:30.0:1619677448.163445:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.163446:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.163446:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.163448:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.163448:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.163449:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.163450:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.163450:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.163451:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.163452:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.163453:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.163454:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.163455:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.163456:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.163458:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.163459:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.163460:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.163460:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.163461:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.163463:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.163463:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.163463:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.163463:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.163464:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677448.163464:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.163465:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.163466:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.163466:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.163467:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.163467:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.163469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.163476:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.163479:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.163480:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.163480:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:20.0:1619677448.163482:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.163483:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.163483:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677448.163483:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:6.0:1619677448.163484:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.163485:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:6.0:1619677448.163486:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.163490:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.163766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.163772:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.163775:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.163776:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.163779:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.163781:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.163783:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.163784:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.163785:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.163787:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.163788:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.163791:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.163793:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.163795:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.163796:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.163797:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.163798:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.163799:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.163799:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.163800:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.163812:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.163815:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.163816:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.163816:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.163818:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.163820:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.163826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.163830:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.163832:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.163832:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.163833:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.163833:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.163834:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:21.2:1619677448.163835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677448.163835:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.163836:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.163836:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677448.163837:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.163839:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.163841:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:31.0:1619677448.163842:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:22.0:1619677448.163842:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.163843:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.163844:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.163846:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:22.0:1619677448.163846:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:30.0:1619677448.163849:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.163851:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.163851:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.163853:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:31.0:1619677448.163854:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.163856:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677448.163856:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.163857:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.163858:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.163858:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:31.0:1619677448.163859:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.163860:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.163860:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.163860:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:31.0:1619677448.163861:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.163861:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.163862:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.163863:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:31.0:1619677448.163865:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.163866:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.163869:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677448.163869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677448.163870:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.163871:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.163872:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.163872:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:31.0:1619677448.163873:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.163874:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.163874:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.163874:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:26.0:1619677448.163875:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.163875:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.163876:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.163876:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.163877:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.163878:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677448.163878:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.163881:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:6.0:1619677448.163886:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.163889:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.163890:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.163890:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.163891:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000400:00000200:6.0:1619677448.163892:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.163894:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.163894:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.163895:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.163896:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.163896:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:6.0:1619677448.163896:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.163914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.163920:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.163924:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.163924:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.163927:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.163929:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.163930:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.163931:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.163933:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.163934:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.163935:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.163938:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.163940:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.163942:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.163943:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.163943:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.163945:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.163945:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.163946:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.163946:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.163951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.163958:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.163961:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.163961:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.163962:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.163963:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.163965:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.163973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.163980:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.163982:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.163983:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.163983:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.163983:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.163984:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.164100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677448.164104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.164107:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.164109:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.164109:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.164111:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.2:1619677448.164111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677448.164112:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:10.0:1619677448.164112:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.164113:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.164115:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.164115:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.164116:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.164117:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.164117:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:17.0:1619677448.164118:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.164119:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.164119:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:17.0:1619677448.164120:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677448.164120:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.164121:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.164121:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.164122:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.164122:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.164123:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.164124:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.164125:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.164126:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:5.0:1619677448.164127:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.164128:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.164129:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96718514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.164130:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.164130:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.164131:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.164131:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.164133:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:5.0:1619677448.164133:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.164134:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.164134:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.164135:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.164136:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.164137:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:10.0:1619677448.164137:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:5.0:1619677448.164137:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.164138:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:10.0:1619677448.164138:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.164139:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.164139:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.164140:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.164140:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.164140:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677448.164142:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:5.0:1619677448.164142:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.164143:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.164143:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.164143:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.164144:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:9.2:1619677448.164146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.0:1619677448.164146:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.2:1619677448.164148:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.164150:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.164151:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.164151:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.164152:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.164153:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:8.0:1619677448.164153:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:1.0:1619677448.164153:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:30.0:1619677448.164154:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.164155:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:30.0:1619677448.164156:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.164156:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677448.164156:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677448.164157:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.164157:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.164157:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.164158:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.164159:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:12.0:1619677448.164160:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677448.164160:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.164162:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.164163:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.164163:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:12.0:1619677448.164165:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.164167:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.164168:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677448.164169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677448.164175:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.164177:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677448.164178:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.164178:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.164180:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.164180:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677448.164197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.164203:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.164206:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.164207:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.164208:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.164210:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.164212:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.164213:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.164214:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.164216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:2.0:1619677448.164216:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.164216:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.164220:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.164222:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.164224:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677448.164224:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.164225:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.164226:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.164227:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:2.0:1619677448.164227:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.164228:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:31.0:1619677448.164229:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.164229:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:30.0:1619677448.164230:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:2.0:1619677448.164230:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.164231:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:31.0:1619677448.164232:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.164233:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.164235:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.164236:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.164236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:31.0:1619677448.164237:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.164238:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.164241:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.164241:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677448.164242:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:31.0:1619677448.164243:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.164244:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:18.0:1619677448.164244:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:31.0:1619677448.164245:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.164245:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.164245:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677448.164245:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.164246:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677448.164246:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677448.164247:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.164248:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:22.0:1619677448.164248:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:31.0:1619677448.164249:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.164249:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.164250:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:22.0:1619677448.164251:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677448.164261:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.164263:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.164264:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.164266:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.164267:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.164268:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.164425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.164431:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.164434:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.164435:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.164437:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.164440:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.164441:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.164443:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.164444:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.164445:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.164446:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.164449:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.164451:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:21.2:1619677448.164452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:17.0:1619677448.164454:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.164455:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.164455:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.164457:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.164457:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.164458:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.164458:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677448.164459:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.164462:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.164463:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.164465:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.164468:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.164469:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.164470:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96718510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.164472:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.164472:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:8.0:1619677448.164473:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.164474:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:6.0:1619677448.164474:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.164475:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.164476:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.164477:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.164478:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.164480:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:6.0:1619677448.164481:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677448.164484:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.164485:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.164485:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.164487:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.164488:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.164489:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.164490:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.164491:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.2:1619677448.164493:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.164496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677448.164501:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:24.0:1619677448.164503:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:20.0:1619677448.164503:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.164504:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.164505:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.164506:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:20.0:1619677448.164506:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.164507:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677448.164507:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.164509:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.164514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.164522:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.2:1619677448.164522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:30.0:1619677448.164525:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.0:1619677448.164526:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:30.0:1619677448.164527:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.164528:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:31.0:1619677448.164529:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.164529:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.164531:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677448.164531:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.164531:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.164533:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.164534:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.164534:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.164535:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:30.0:1619677448.164536:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.164537:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.164538:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.164539:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.164541:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.164542:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:30.0:1619677448.164544:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.164546:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.164547:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.164549:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.164549:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.164549:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:30.0:1619677448.164550:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.164551:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.164551:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.164552:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.164552:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:30.0:1619677448.164553:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.164553:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677448.164553:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.164554:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.164555:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.164556:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.164556:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.164560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:12.0:1619677448.164564:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.164567:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677448.164568:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:12.0:1619677448.164568:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.164569:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.164570:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.164571:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.164572:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677448.164573:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.164573:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.164574:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.164575:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:8.2:1619677448.164578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000800:00000200:12.0:1619677448.164580:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.164583:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.164584:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.164585:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.164585:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.164586:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:12.0:1619677448.164587:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.164588:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.164589:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.164590:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677448.164591:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.164592:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:21.2:1619677448.164672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fdae00] (20)++
00000800:00000200:5.0:1619677448.164676:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fdae00] (21)++
00000800:00000200:5.0:1619677448.164678:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.38@o2ib3
00000400:00000200:5.0:1619677448.164680:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.38@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.164681:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (22)--
00000400:00000200:5.0:1619677448.164682:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.164684:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:5.0:1619677448.164685:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.164687:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.164688:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.164689:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:5.0:1619677448.164692:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.38@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.164695:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.2:1619677448.164700:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.0:1619677448.164702:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:26.0:1619677448.164703:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677448.164705:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.164706:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:20.2:1619677448.164706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.164706:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677448.164706:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:18.0:1619677448.164707:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:10.0:1619677448.164707:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.0:1619677448.164708:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677448.164708:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fdae00] (21)++
00000800:00000200:20.0:1619677448.164709:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.164709:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fdae00] (22)--
00000800:00000200:20.0:1619677448.164710:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:18.0:1619677448.164710:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:5.0:1619677448.164710:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (21)--
00000800:00000200:2.0:1619677448.164710:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:26.0:1619677448.164711:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.164712:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677448.164714:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.164714:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.164714:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.164714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:26.0:1619677448.164715:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677448.164715:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.164716:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.164717:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.164717:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.164717:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.164718:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.164719:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.164719:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.164719:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:8.0:1619677448.164720:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:1.0:1619677448.164720:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:26.0:1619677448.164721:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.164721:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.164721:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.164721:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677448.164722:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.164723:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.164723:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.164724:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:18.0:1619677448.164724:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.164724:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.164724:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.164725:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.164725:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.164726:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677448.164727:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.164727:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.164728:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.164728:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:8.0:1619677448.164730:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:1.0:1619677448.164730:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96718503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.164731:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.164731:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.164732:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.164734:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.164734:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.164735:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.164735:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:20.0:1619677448.164735:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.164735:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:26.0:1619677448.164736:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.164736:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.164737:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.164738:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.164738:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.164739:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677448.164739:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.164740:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.164740:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:18.0:1619677448.164740:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:1.0:1619677448.164740:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.164741:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677448.164742:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.164742:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.164742:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677448.164743:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.164743:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:20.0:1619677448.164744:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.164746:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:1.0:1619677448.164746:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.164746:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:20.0:1619677448.164747:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:9.2:1619677448.164747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.164747:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:20.0:1619677448.164749:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677448.164749:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:20.0:1619677448.164750:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:1.0:1619677448.164750:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.164751:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677448.164751:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:1.0:1619677448.164751:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.164751:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.164754:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:2.0:1619677448.164754:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.2:1619677448.164755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:13.0:1619677448.164755:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.164755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:2.0:1619677448.164756:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:13.0:1619677448.164757:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.164757:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000400:00000200:2.0:1619677448.164757:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.164758:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677448.164759:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.164759:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677448.164759:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.164759:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.164760:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.164761:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.164761:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:24.0:1619677448.164762:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:2.0:1619677448.164762:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677448.164764:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.164764:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:24.0:1619677448.164765:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:5.0:1619677448.164765:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:24.0:1619677448.164766:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.164766:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:5.0:1619677448.164766:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:24.0:1619677448.164768:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.164768:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:24.0:1619677448.164771:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.0:1619677448.164774:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:5.0:1619677448.164777:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:20.0:1619677448.164779:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:5.0:1619677448.164779:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.164782:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.164784:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677448.164785:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.164786:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.164788:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.164789:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.164792:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.164794:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.164797:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.164798:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677448.164799:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.164800:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.164801:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:5.0:1619677448.164802:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:5.0:1619677448.164803:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:10.0:1619677448.164812:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.164815:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.164816:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.164816:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.164819:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.164821:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.164854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677448.164860:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677448.164863:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677448.164864:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677448.164866:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.164869:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.164870:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.164872:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.164873:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.164874:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.164875:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.164878:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.164880:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.164883:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.164884:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.164884:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.164886:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.164887:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677448.164887:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677448.164888:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:8.2:1619677448.164893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.164900:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.164903:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.164904:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.164904:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.164906:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.164909:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.164983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.164991:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.164994:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.164996:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.164997:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.164998:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.165000:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.165001:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.165002:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.165003:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.165004:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.165007:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.165009:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.165012:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.165013:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.165013:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.165015:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.165015:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.165016:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.165017:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.165026:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.165028:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.165029:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.165029:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.165031:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.165034:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.165046:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.165049:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.165049:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.165050:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.165050:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.165051:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.165128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.2:1619677448.165130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.165137:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.165139:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.165139:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.165140:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:26.0:1619677448.165141:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.165142:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.165142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677448.165143:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.165145:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.165146:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.165147:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.165148:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.165148:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.165149:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.165150:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.165151:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.165151:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.165154:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.165154:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.165155:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.165155:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.165157:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.165157:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.165157:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.165158:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.165158:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.165159:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.165160:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718497] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.165161:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.165162:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.165163:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:1.0:1619677448.165163:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.165164:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677448.165164:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:26.0:1619677448.165165:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.165166:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.165167:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (23)++
00000800:00000200:26.0:1619677448.165168:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (24)--
00000800:00000200:26.0:1619677448.165169:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (23)--
00000400:00000200:1.0:1619677448.165169:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.165170:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.165170:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.165171:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677448.165172:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.165173:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:4.2:1619677448.165174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677448.165174:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.165175:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:10.0:1619677448.165176:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.165176:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:13.0:1619677448.165177:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.165178:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.2:1619677448.165178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677448.165178:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.165178:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.165179:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.165179:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.165180:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.165180:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.165180:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.165181:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:13.0:1619677448.165182:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.165182:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.165183:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:12.0:1619677448.165184:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.165184:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677448.165184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:26.0:1619677448.165185:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677448.165185:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.165185:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.165186:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.165186:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.165187:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.165188:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.165189:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.165189:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.2:1619677448.165189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:26.0:1619677448.165192:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.165194:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:8.0:1619677448.165194:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:26.0:1619677448.165195:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96718496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.165196:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.165196:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:26.0:1619677448.165197:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.165197:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:26.0:1619677448.165198:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:24.0:1619677448.165198:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.165198:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.165199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:17.0:1619677448.165199:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.165200:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.165200:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.165200:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677448.165202:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:26.0:1619677448.165203:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.165203:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.165203:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.0:1619677448.165204:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:26.0:1619677448.165206:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.165206:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:20.0:1619677448.165207:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.165208:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:26.0:1619677448.165209:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.165210:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.165211:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:20.0:1619677448.165211:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.165212:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:20.0:1619677448.165212:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:26.0:1619677448.165213:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677448.165213:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.165214:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.165214:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:20.0:1619677448.165215:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.165216:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.165217:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.165221:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.165222:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.165225:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.165226:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.165226:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.165226:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:20.0:1619677448.165228:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.165228:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:6.0:1619677448.165228:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.165229:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:20.0:1619677448.165229:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.165230:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:6.0:1619677448.165230:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.165232:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.165234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.165235:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677448.165242:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.165245:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.165246:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.165246:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.165248:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.165250:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.165256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.165263:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.165266:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.165266:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.165267:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.165268:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.165268:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.165308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.165315:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.165318:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.165318:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.165320:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.165322:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.165324:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.165325:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.165326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:18.0:1619677448.165327:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.165328:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.165329:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.165333:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.165333:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.165334:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.165335:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.165336:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.165338:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.165339:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.165340:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:2.0:1619677448.165340:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:18.0:1619677448.165341:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:2.0:1619677448.165341:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.165343:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:2.0:1619677448.165343:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.165344:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.165344:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.165345:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.165347:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.165347:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677448.165348:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.165348:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.165350:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.165353:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.165354:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.165354:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.165356:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.165356:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:10.0:1619677448.165357:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:2.0:1619677448.165357:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.165358:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:10.0:1619677448.165359:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.165360:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.165361:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.165363:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.165366:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:12.0:1619677448.165369:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.165372:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.165373:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.165374:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.165376:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.165377:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677448.165387:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.165390:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.165391:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.165391:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.165392:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.165394:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.165434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.165442:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.165445:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.165447:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.165450:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.165450:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.165451:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.165452:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.165454:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.165454:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.165455:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.165458:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.165460:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.165463:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.165464:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.165465:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.165466:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.165467:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.165468:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.165470:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.165470:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.165473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.165476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.165482:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:31.0:1619677448.165484:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:5.0:1619677448.165485:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.165486:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.165487:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.165487:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:5.0:1619677448.165487:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.165488:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.165489:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.165491:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.165492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.165500:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.165502:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.165503:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.165504:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.165505:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.165506:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.165604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.165612:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.165615:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.165617:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.165618:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.165620:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.165621:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.165623:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.165624:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.165625:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.165626:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.165629:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.165631:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.165633:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.165634:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.165635:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.165636:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.165637:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.165638:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.165639:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.165640:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677448.165646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.2:1619677448.165647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:12.0:1619677448.165648:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.165650:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.165651:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.165652:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.165654:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.165655:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:26.0:1619677448.165655:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677448.165655:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.165658:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.165659:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.165659:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.165659:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.165660:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:26.0:1619677448.165661:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.165664:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.165665:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.165667:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.165668:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.165669:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.165669:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.165672:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.165674:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.165676:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.165677:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.165677:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.165679:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.165679:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.165680:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.165680:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677448.165687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.0:1619677448.165691:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:12.0:1619677448.165692:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.165694:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:5.0:1619677448.165694:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:12.0:1619677448.165695:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.165697:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:12.0:1619677448.165697:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.165697:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.165698:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:5.0:1619677448.165699:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.165700:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:5.0:1619677448.165701:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.165702:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.165703:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.165704:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.165705:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.165708:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.165710:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.165712:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.165713:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.165714:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.165715:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.165716:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.165717:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.165717:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.165721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.165728:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.165731:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.165731:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.165733:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.165734:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.165735:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:4.2:1619677448.165797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.165804:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.165807:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.165808:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.165810:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.165813:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.165814:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.165816:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.165817:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.165818:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.165819:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.165822:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.165824:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.165827:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.165828:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.165829:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.165831:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.165831:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.165832:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.165834:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.165834:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677448.165837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677448.165840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.165845:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.165847:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:18.0:1619677448.165848:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.165848:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.165849:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.165849:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677448.165849:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.165850:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:18.0:1619677448.165851:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.165855:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.165881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.165886:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.165889:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.165891:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.165891:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.165894:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.165895:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.165896:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.165898:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.165899:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.165900:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.165903:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.165904:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.165907:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.165908:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.165909:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.165910:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.165911:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.165912:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.165912:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.165924:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.165926:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.165927:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.165929:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.165930:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.165932:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.165946:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.165949:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.165950:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.165951:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.165951:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.165953:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.166192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.166200:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.166202:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.166204:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.166206:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.166207:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.166208:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.166209:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.166211:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.166212:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.166212:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.166215:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.166217:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.166219:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.166220:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.166221:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.166222:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.166223:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.166223:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.166224:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.166234:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.166237:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.166238:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.166239:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.166241:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.166242:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.166252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.166259:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.166262:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.166263:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.166265:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.166267:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.166269:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.166270:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.166271:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.166272:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.166273:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.166276:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.166277:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.166280:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.166280:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.166281:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.166282:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.166283:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.166284:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.166284:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.166294:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.166302:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.166302:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.166304:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.166304:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.166306:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.166373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677448.166381:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677448.166383:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677448.166385:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.166387:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677448.166387:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.166389:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.166390:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.166391:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.166392:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.166393:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.166396:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.166398:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.166400:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.166401:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.166402:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.166403:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.166404:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677448.166405:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:30.0:1619677448.166407:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:30.0:1619677448.166407:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:9.2:1619677448.166410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.166413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:2.0:1619677448.166419:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:31.0:1619677448.166421:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:2.0:1619677448.166421:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.166423:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.166423:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.166424:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc2e00] (23)--
00000400:00000200:2.0:1619677448.166424:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.166425:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677448.166426:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:2.0:1619677448.166428:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.166439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.166444:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.166447:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.166447:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.166449:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.166452:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.166453:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.166454:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.166455:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.166456:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.166457:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.166460:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.166462:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.166465:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.166466:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.166466:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.166468:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.166469:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.166469:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.166470:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.2:1619677448.166473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.166483:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.166484:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.166484:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.166486:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.166487:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.166487:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.166489:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.166490:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.166492:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.166493:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.166494:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.166495:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.166496:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.166497:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.166498:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.166499:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.166499:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.166501:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.166504:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.166505:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.166505:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.166507:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.166507:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.166508:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.166508:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677448.166518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.166521:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.166523:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.166524:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.166525:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.166527:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.166528:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.166528:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677448.166529:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677448.166530:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.166530:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.166532:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.166534:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.166535:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.166536:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.166537:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.166538:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.166541:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.166543:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.166545:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.166545:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.166546:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.166547:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.166548:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.166549:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.166549:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677448.166561:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.166563:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.166564:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.166565:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.166567:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.166568:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.166577:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.166579:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.166580:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.166581:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.166582:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.166584:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.166611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.166618:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.166621:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.166622:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.166624:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.166626:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.166628:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.166629:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.166630:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.166632:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.166632:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.166636:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.166638:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.166640:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.166641:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.166642:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.166644:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.166645:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.166646:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.166647:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.166648:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.166650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.166653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.166657:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677448.166658:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677448.166660:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:17.0:1619677448.166660:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.166661:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677448.166661:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.166662:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.166662:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677448.166664:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.166667:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.2:1619677448.166737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.166742:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.166745:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.166746:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.166748:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.166751:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.166752:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.166753:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.166755:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.166756:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.166757:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:4.2:1619677448.166760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677448.166760:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.166761:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.166764:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.166764:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.166765:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.166767:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.166767:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.166767:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.166769:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:10.0:1619677448.166770:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:2.0:1619677448.166770:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.166770:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:26.0:1619677448.166771:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.166773:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.166775:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.166775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000400:00000200:10.0:1619677448.166777:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.166778:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.166779:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.166780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000400:00000200:10.0:1619677448.166780:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.166781:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:10.0:1619677448.166782:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.166783:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.166784:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.166785:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:10.0:1619677448.166785:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.166786:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:10.0:1619677448.166786:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.166788:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:8.2:1619677448.166788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.166789:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.166790:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:20.0:1619677448.166791:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.166791:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:20.0:1619677448.166793:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.166793:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.166793:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677448.166793:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:10.0:1619677448.166794:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.166794:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:20.0:1619677448.166795:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:10.0:1619677448.166795:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.166795:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.166796:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96718477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.166796:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.166798:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.166798:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.166799:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.166800:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.166804:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.166805:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677448.166807:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.166807:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.166808:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.166808:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677448.166809:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.166809:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:22.0:1619677448.166810:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.166811:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.166811:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:6.0:1619677448.166811:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.166812:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000400:00000200:22.0:1619677448.166813:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.166813:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.166813:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:22.0:1619677448.166817:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.2:1619677448.166817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.166825:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:6.0:1619677448.166826:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.166827:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.166828:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.166829:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.166830:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.166830:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000400:00000200:5.0:1619677448.166830:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.166831:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:6.0:1619677448.166831:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.166831:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:6.0:1619677448.166832:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.166894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677448.166900:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677448.166902:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677448.166903:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677448.166905:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.166907:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.166909:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.166910:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.166911:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.166912:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.166913:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.166917:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.166919:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.166921:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.166922:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.166923:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.166924:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.166925:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677448.166925:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677448.166926:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:8.2:1619677448.166931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.166939:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.166942:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.166943:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.166943:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.166945:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.166947:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.166960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.166965:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.166968:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.166969:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.166971:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.166973:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.166975:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677448.166977:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.166978:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.166979:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.166980:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.166984:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.166985:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.166988:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.166989:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.166989:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.166991:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.166992:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.166992:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.166993:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.166994:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:8.2:1619677448.166997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.166999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.167004:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:20.0:1619677448.167005:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:18.0:1619677448.167006:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:20.0:1619677448.167007:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:18.0:1619677448.167007:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:20.0:1619677448.167008:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.167008:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:20.0:1619677448.167010:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.167011:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.167012:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.167042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.167048:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.167051:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.167051:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.167053:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.167056:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.167057:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.167059:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.167060:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.167061:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.167062:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.167066:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.167067:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.2:1619677448.167068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677448.167070:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.167071:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.167072:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.0:1619677448.167073:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.167074:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.167074:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.167075:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.167075:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677448.167076:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.167078:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.167078:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.167080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:5.0:1619677448.167080:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.167082:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:4.2:1619677448.167082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:5.0:1619677448.167083:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.167085:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.167085:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.167086:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:5.0:1619677448.167087:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.167089:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677448.167089:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.167090:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.167090:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.167090:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.167091:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.167092:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:8.0:1619677448.167092:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.167092:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.167094:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.167094:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.0:1619677448.167096:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.167097:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.167097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677448.167097:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.167097:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:26.0:1619677448.167099:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.167100:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.167100:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.167100:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677448.167101:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.167101:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677448.167102:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.167102:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:26.0:1619677448.167103:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677448.167103:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.167106:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000400:00000200:26.0:1619677448.167107:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.167107:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.167107:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.167108:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:26.0:1619677448.167109:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:24.0:1619677448.167109:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.167109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:26.0:1619677448.167112:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.167112:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.167113:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:26.0:1619677448.167113:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:6.0:1619677448.167114:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.167115:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (25)--
00000800:00000200:26.0:1619677448.167115:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.167116:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:26.0:1619677448.167116:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677448.167117:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.167117:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.167117:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.167117:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:31.0:1619677448.167119:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.167119:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:30.0:1619677448.167120:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (24)--
00000400:00000200:30.0:1619677448.167121:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.167121:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.167122:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:6.0:1619677448.167123:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.167124:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.167124:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.167125:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677448.167126:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:6.0:1619677448.167127:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.167128:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.167129:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.167130:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.167133:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.167135:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.167138:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.167139:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.167140:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.167142:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.167143:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.167143:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.167144:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677448.167146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.167154:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.167156:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:30.0:1619677448.167156:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:31.0:1619677448.167158:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.167158:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.167159:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.167160:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.167161:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.167163:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.167164:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.167166:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.167167:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.167168:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.167168:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:31.0:1619677448.167170:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.167170:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.167171:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.167172:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.167172:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.167173:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.167174:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.167174:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.167175:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.167175:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.167176:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.167177:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.167179:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.167180:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.167180:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.167181:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.167185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.167194:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.167196:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.167197:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.167199:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.167200:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.167202:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.167225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.167232:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.167234:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.167235:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.167237:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.167240:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.167242:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.167243:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.167244:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.167245:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.167246:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.167249:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.167251:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.167254:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.167254:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.167255:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.167257:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.167257:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.167258:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.167258:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.167263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.167270:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.167273:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.167274:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.167274:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.167276:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.167279:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.167376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.167382:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.167384:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.167386:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.167387:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.167389:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.167391:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677448.167392:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.167394:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.167395:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.167396:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.167399:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.167401:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.167404:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.167405:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677448.167405:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.167407:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.167408:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.167408:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.167409:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:10.0:1619677448.167421:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.167424:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.167424:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.167425:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.167427:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.167430:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.167444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.167451:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.167453:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.167454:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.167456:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.167459:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.167460:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.167461:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.167463:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.167463:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.167464:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.167467:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.167469:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.167471:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.167471:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.167472:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.167474:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.167474:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.167475:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.167475:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.167480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.167487:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.167489:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.167490:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.167491:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.167492:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.167493:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.167501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.167508:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.167511:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.167512:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.167513:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.167516:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.167517:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.167518:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.167520:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.167521:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.167522:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.167525:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.167527:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.167530:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.167530:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.167531:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.167532:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.167533:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.167534:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.167535:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.167536:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.167541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.2:1619677448.167545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677448.167546:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.167549:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.167549:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.167550:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:30.0:1619677448.167551:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.167551:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.167552:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.167553:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.167553:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.167554:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.167554:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.167555:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.167555:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.167557:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.167559:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.167560:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.167561:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.167563:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.167564:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.167564:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.167567:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.167569:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.167572:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.167573:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.167573:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.167575:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.167576:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.167576:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.167577:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.167581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.167590:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.167593:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:21.2:1619677448.167594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:18.0:1619677448.167594:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.167594:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.167596:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.167600:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677448.167601:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.167604:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.167604:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.167606:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.167609:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.167611:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.167612:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.167613:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.167614:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.167614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:24.0:1619677448.167615:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.167618:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.167619:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.167622:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:24.0:1619677448.167622:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.167623:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.167624:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.167625:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:24.0:1619677448.167626:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.167626:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:30.0:1619677448.167627:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.167627:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.167628:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:31.0:1619677448.167629:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.167629:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.167631:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.167632:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.167632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677448.167633:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.167634:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.167635:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.167638:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.167639:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.167639:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:30.0:1619677448.167641:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.167641:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:30.0:1619677448.167642:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.167643:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:18.0:1619677448.167643:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.167643:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677448.167644:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.167645:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.167645:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:18.0:1619677448.167645:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.167646:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:18.0:1619677448.167648:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.167654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677448.167658:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.167661:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.167661:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.167661:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:13.0:1619677448.167663:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.167664:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.167665:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:12.0:1619677448.167665:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677448.167665:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.167666:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677448.167667:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677448.167667:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.167743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.167748:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.167751:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.167753:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.167754:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.167756:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.167757:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.167759:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.167760:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.167761:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.167762:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.167765:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.167767:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.167769:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.167770:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.167771:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.167772:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.167773:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.167774:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677448.167775:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677448.167776:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:8.2:1619677448.167778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:21.2:1619677448.167781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.167787:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.167788:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.167789:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677448.167789:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000400:00000200:5.0:1619677448.167789:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.167790:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.167791:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.167791:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.167792:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677448.167793:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:4.2:1619677448.167803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.167810:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.167814:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.167814:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.167817:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.167819:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.167821:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.167822:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.167823:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.167824:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.167825:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.167828:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.167830:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.167833:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.167834:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.167834:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.167836:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.167836:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.167837:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.167837:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.167851:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.167854:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.167855:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.167855:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.2:1619677448.167856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:22.0:1619677448.167857:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.167860:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677448.167861:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.167864:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.167865:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.167867:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.167869:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.167870:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.167871:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.167873:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.167874:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.167875:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.167878:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.167879:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.167882:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.167883:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.167883:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.167885:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.167886:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.167886:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.167887:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.167898:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.167900:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.167901:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.167903:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.167904:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.167906:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.167913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.167920:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.167923:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.167925:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.167927:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.167927:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.167929:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.167930:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.167931:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.167932:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.167933:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.167937:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.167938:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.167941:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.167942:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.167942:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.167944:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.167945:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.167945:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.167946:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677448.167951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:13.0:1619677448.167956:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677448.167956:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677448.167959:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.167959:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.167959:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.167960:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:13.0:1619677448.167962:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.167962:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:6.0:1619677448.167964:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.167965:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:6.0:1619677448.167967:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.167968:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.167969:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.167971:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.167972:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.167973:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.167976:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.167978:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:12.0:1619677448.167980:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.167980:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.167981:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:12.0:1619677448.167982:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.167982:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677448.167983:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.167984:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:6.0:1619677448.167984:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.167984:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:12.0:1619677448.167985:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677448.167985:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677448.167986:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.167986:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.167990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.167998:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.168000:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.168001:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.168003:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.168004:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.168005:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.168007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.168015:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.168018:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.168019:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.168019:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.168020:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.168021:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.168149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.168156:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.168159:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.168160:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.168162:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.168165:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.168167:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.168168:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.168169:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.168170:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.168171:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.168174:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.168176:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.168178:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.168179:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.168179:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.168181:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.168181:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.168182:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.168182:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:21.2:1619677448.168186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.168192:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:13.0:1619677448.168193:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:18.0:1619677448.168195:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.168195:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:13.0:1619677448.168196:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.168196:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.168198:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.168199:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.168199:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:18.0:1619677448.168200:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.168201:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:18.0:1619677448.168202:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.168203:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.168204:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.168206:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.168207:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.168210:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.168212:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.168214:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.168215:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.168216:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.168217:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.168218:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.168219:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.168220:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677448.168231:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.168234:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.168234:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.168235:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.168237:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.168237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677448.168239:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677448.168242:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.168245:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.168246:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.168248:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.168251:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.168252:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.168253:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.168255:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.168256:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.168257:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.168260:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.168261:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.168263:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677448.168264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677448.168264:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.168265:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.168266:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.168266:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.168268:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.168269:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.168269:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.168270:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.2:1619677448.168272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.168274:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.168275:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.168277:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.168280:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.168280:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.168281:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.168282:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:5.0:1619677448.168282:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.168282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677448.168282:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.168283:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677448.168283:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677448.168283:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.168284:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.168285:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:5.0:1619677448.168285:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.168285:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.168286:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.168287:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.168287:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.168288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000400:00000200:5.0:1619677448.168288:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.168289:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.168292:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:5.0:1619677448.168292:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.168293:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.168299:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.168301:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.168302:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.168302:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.168303:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.168303:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000400:00000200:10.0:1619677448.168303:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.168304:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677448.168304:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.168304:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.168305:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677448.168306:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.168307:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:5.0:1619677448.168307:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.168308:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:10.0:1619677448.168309:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.168309:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.168309:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:10.0:1619677448.168310:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.168311:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.168312:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:9.2:1619677448.168314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677448.168316:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.168318:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.168320:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.168321:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.168321:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:2.0:1619677448.168322:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.168323:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.168323:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.168324:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.168324:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677448.168324:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.168326:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.168326:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.168327:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.168330:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677448.168336:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.168339:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.168340:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.168341:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.168343:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.168344:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.168436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677448.168441:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677448.168444:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677448.168445:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677448.168447:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.168449:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.168450:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.168451:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.168453:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.168454:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.168454:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.168457:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.168459:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.168462:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.168463:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.168464:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.168466:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.168466:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677448.168467:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677448.168468:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677448.168468:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:5.2:1619677448.168474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677448.168479:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:12.0:1619677448.168479:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.168481:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677448.168481:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677448.168482:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:12.0:1619677448.168482:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.168483:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.168484:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.168486:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.168488:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.168662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.168670:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.168673:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.168675:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.168677:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.168677:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.168679:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.168680:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.168681:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.168682:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.168683:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.168687:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.168689:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.168691:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.168692:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.168693:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.168695:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.168695:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.168696:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.168696:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.168709:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.168712:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.168712:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.168713:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.168715:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.168718:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.168842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.168847:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.168850:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.168850:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.168852:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.168854:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.168856:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.168857:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.168858:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.168859:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.168860:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.168863:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.168863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:22.0:1619677448.168865:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.168867:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677448.168867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:22.0:1619677448.168868:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.168868:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.168870:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:22.0:1619677448.168870:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.168871:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.168871:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.168872:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:26.0:1619677448.168873:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.168874:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:30.0:1619677448.168875:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677448.168876:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.168877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.168878:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:26.0:1619677448.168878:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.168880:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.168880:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:31.0:1619677448.168881:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677448.168881:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.168882:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.168883:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.168883:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.168885:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.168885:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.168885:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:30.0:1619677448.168886:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.168888:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.168888:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:30.0:1619677448.168889:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.168889:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.168889:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.168889:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.168890:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.168892:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:1.0:1619677448.168892:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.168894:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.168895:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.168895:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677448.168896:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.168896:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.168896:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.168898:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.168898:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.168899:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.168899:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.168899:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.168900:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.168900:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:30.0:1619677448.168901:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.168901:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.168903:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.168904:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:8.2:1619677448.168904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.168905:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677448.168911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677448.168914:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.0:1619677448.168916:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:13.0:1619677448.168916:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.168917:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.168918:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.168919:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.168919:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.168920:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.168920:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.168921:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:21.2:1619677448.168921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000400:00000200:20.0:1619677448.168921:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.168921:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.168922:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.168923:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.168924:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.168925:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677448.168926:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.168929:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677448.168931:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.168931:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677448.168933:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.168935:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.168936:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.168937:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.168938:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.168940:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.168943:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.168945:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.168947:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.168948:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.168949:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.168951:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.168951:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:21.2:1619677448.168952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:5.0:1619677448.168952:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.168953:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.168956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:24.0:1619677448.168961:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.168964:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:20.0:1619677448.168964:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:8.0:1619677448.168965:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.168967:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.168967:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.168967:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.168969:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.168969:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.168970:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:24.0:1619677448.168971:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:20.0:1619677448.168971:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:24.0:1619677448.168972:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.168973:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.168974:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.168975:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:8.2:1619677448.168977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000400:00000200:24.0:1619677448.168980:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.168982:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.168985:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.168985:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.168985:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:24.0:1619677448.168986:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.168987:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:24.0:1619677448.168988:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.168988:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:24.0:1619677448.168989:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.168989:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:20.2:1619677448.168989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.168989:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:24.0:1619677448.168990:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677448.168990:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.168991:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.0:1619677448.168993:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.2:1619677448.168994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.168995:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.168997:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.168999:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.168999:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.169001:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:5.0:1619677448.169001:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677448.169002:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.169002:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.169003:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.169003:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.169004:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.169005:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.169005:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.169008:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.169009:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:5.0:1619677448.169009:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.169011:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.169014:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.169015:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.169016:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.169017:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.169018:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.169019:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.169021:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.169021:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.169026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.169029:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677448.169030:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.169031:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.169032:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677448.169032:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677448.169033:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:12.0:1619677448.169033:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.169035:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.169037:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.169038:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.169093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.169099:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.169102:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.169102:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.169105:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.169107:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.169108:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.169109:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.169110:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.169111:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.169112:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.169115:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.169116:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.169118:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.169119:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.169120:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.169121:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.169121:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.169122:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.169122:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.169127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.169134:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.169136:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.169136:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.169137:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.169138:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.169139:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.169167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.169174:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.169177:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.169179:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.169181:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.169181:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.169183:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.169184:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.169185:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.169186:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.169187:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677448.169190:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.169192:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.169194:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.169195:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.169196:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.169197:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.169198:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.169199:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.169201:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.169202:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677448.169204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.169207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.169211:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.169213:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:30.0:1619677448.169215:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:18.0:1619677448.169215:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.169215:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.169217:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.169218:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.169218:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.169219:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677448.169220:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.169222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677448.169229:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.169232:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677448.169233:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677448.169235:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.169238:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.169239:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.169240:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.169242:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.169243:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.169244:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.169248:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.169249:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.169251:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.169252:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.169253:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.169254:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.169254:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.169255:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.169256:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:30.0:1619677448.169267:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:21.2:1619677448.169268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:30.0:1619677448.169269:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.169270:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.169272:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.169273:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.169273:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677448.169274:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:6.0:1619677448.169276:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.169277:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.169279:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.169282:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.169283:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.169285:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.169286:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.169287:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.169288:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.169291:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.169298:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.169300:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.169301:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.169302:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.169303:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.169304:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.169305:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.169306:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.169306:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.169310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.169312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677448.169317:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:22.0:1619677448.169318:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.169320:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.169320:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677448.169320:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677448.169321:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:20.0:1619677448.169321:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.169322:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.169323:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.169325:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.169356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.169363:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.169367:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.169368:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.169370:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.169373:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.169374:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.169376:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.169377:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.169378:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.169379:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.169382:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.169384:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.2:1619677448.169384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:4.2:1619677448.169386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677448.169387:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.169388:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.169389:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.169390:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.169391:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.169391:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.169392:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:26.0:1619677448.169393:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.169395:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.169395:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.169396:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:8.2:1619677448.169396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677448.169398:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.169399:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.169401:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.169401:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.169402:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.169403:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.169404:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:26.0:1619677448.169405:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.169406:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.169407:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:17.0:1619677448.169407:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.169408:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.169409:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.169410:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.169411:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.169413:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.169413:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:17.0:1619677448.169414:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:10.0:1619677448.169414:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.169415:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.169416:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.169417:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.169419:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.169420:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.169421:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:10.0:1619677448.169421:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.169422:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.169423:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.169424:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.169426:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (23)++
00000800:00000200:10.0:1619677448.169426:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.169427:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (24)--
00000800:00000200:10.0:1619677448.169427:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.169427:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.169428:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.169429:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.169430:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.169430:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.169431:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.169431:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.169432:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.169433:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.169434:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:5.2:1619677448.169434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:10.0:1619677448.169435:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.169437:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.169437:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.169438:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.169439:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.169439:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.169439:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.169440:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677448.169443:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.169446:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.169446:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.169447:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.169449:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.169452:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:26.0:1619677448.169457:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.169460:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.169461:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.169461:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.169462:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.169462:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.169503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.169508:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.169511:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.169512:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.169513:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.169516:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.169518:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.169519:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.169520:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.169521:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.169522:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.169525:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.169526:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.169528:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.169529:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.169530:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.169532:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.169532:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.169533:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.169533:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.169538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.169545:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.169547:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.169547:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.169548:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.169549:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.169551:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.169609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:20.2:1619677448.169613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677448.169616:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.169618:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.169619:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:31.0:1619677448.169621:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:2.0:1619677448.169621:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:21.2:1619677448.169623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.169624:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:2.0:1619677448.169624:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.169625:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677448.169627:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.169628:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.169629:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.169630:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:2.0:1619677448.169630:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.169631:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.169631:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:2.0:1619677448.169631:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.169632:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.169633:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.169635:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.169635:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000400:00000200:31.0:1619677448.169637:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.169637:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.169638:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:31.0:1619677448.169639:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.169641:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.169641:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:30.0:1619677448.169642:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.169642:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:31.0:1619677448.169643:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.169644:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.169644:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677448.169644:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.169645:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:31.0:1619677448.169646:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.169646:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:2.0:1619677448.169646:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:30.0:1619677448.169647:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96718434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.169648:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.169649:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.169650:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.169653:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.169654:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:30.0:1619677448.169654:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.169655:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.169656:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.169656:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.169657:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:22.0:1619677448.169657:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:31.0:1619677448.169658:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.169658:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.169659:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.169659:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:31.0:1619677448.169660:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.169660:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.169660:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677448.169661:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:22.0:1619677448.169661:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.169663:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.169663:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:22.0:1619677448.169663:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:30.0:1619677448.169664:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.169665:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.169666:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677448.169669:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.169671:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.169672:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.169676:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677448.169678:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.169678:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.169680:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.169680:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.169681:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.169682:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:13.0:1619677448.169682:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677448.169683:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000400:00000200:6.0:1619677448.169683:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.169684:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:13.0:1619677448.169684:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677448.169685:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.169686:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:13.0:1619677448.169686:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.169687:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.169687:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677448.169687:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677448.169688:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677448.169688:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.169688:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677448.169699:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.169702:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.169703:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.169703:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.169704:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.169705:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.169877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.169881:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.169884:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.169885:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.169887:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.169889:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.169890:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.169892:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.169893:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.169894:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.169895:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.169899:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.169900:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.169904:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.169904:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.169905:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.169907:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.169908:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.169908:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.169908:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.169919:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.169922:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.169923:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.169923:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.169925:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.169929:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677448.169951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.0:1619677448.169955:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.169958:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677448.169960:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.169960:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.169962:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.169964:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677448.169965:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.169967:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.169968:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.169969:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.169972:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.169974:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 440 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.169976:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677448.169977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.169977:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677448.169978:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.169979:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.169980:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:5.0:1619677448.169980:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.169980:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.169981:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.169982:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.169982:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677448.169983:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.169985:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.169987:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.169988:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.169989:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.169990:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.169991:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:20.0:1619677448.169991:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.169994:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:20.0:1619677448.169994:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.169995:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.169995:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:20.0:1619677448.169996:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.169998:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.169999:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.170000:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.0:1619677448.170000:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.170000:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.170002:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.170003:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.170004:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.170005:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.170015:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.170018:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.170019:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.170021:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.170022:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.170023:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.170057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.170062:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.170065:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.170066:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.170068:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.170070:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.170071:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.170073:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.170074:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.170075:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.170076:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.170079:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.170081:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.170084:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.170085:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.170085:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.170087:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.170088:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.170089:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.170089:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.170100:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.170103:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.170103:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.170105:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.170106:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.170108:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.170116:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.170119:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.170120:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.170121:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.170121:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.170123:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.170149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.170156:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.170159:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.170160:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.170161:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.170164:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.170166:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.170167:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.170168:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.170169:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.170170:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.170173:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.170174:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.170176:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.170177:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.170178:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.170179:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.170179:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.170180:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.170181:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677448.170190:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.170192:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.170193:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.170193:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.170194:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.170195:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.170202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.170210:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.170212:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.170214:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.170216:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.170217:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.170217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:31.0:1619677448.170218:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.170219:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.170221:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.170222:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.170223:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.170225:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:31.0:1619677448.170226:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.170226:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677448.170228:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.170229:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.170230:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.170231:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:30.0:1619677448.170231:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.170232:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.170233:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:30.0:1619677448.170233:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.170234:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.170235:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.170236:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.170237:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.170238:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.170239:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.170239:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:30.0:1619677448.170240:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.170241:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.170242:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.170245:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.170245:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:13.0:1619677448.170245:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.170246:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:13.0:1619677448.170247:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:30.0:1619677448.170248:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.170248:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:13.0:1619677448.170248:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.170249:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:13.0:1619677448.170249:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.170250:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677448.170251:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.170253:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677448.170254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677448.170262:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.170263:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:12.0:1619677448.170265:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.170265:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.170265:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:12.0:1619677448.170266:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.170266:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.170267:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677448.170267:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.170268:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.170269:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.170272:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.170297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.170299:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.170302:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.2:1619677448.170304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:20.0:1619677448.170305:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.170307:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.170307:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.170309:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677448.170310:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718426] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.170310:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:20.0:1619677448.170311:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.170313:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.170313:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.170313:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:20.0:1619677448.170314:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.170317:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.170319:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.170319:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.170321:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.170321:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.170322:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.170323:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.170324:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.170325:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.170325:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:18.0:1619677448.170325:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.170326:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.170327:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.170327:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.170329:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.170330:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.170330:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:18.0:1619677448.170330:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.170331:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.2:1619677448.170335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:18.0:1619677448.170335:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.170335:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.170336:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:8.2:1619677448.170336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.170338:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.170338:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.170339:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.170340:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677448.170341:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.170343:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:17.0:1619677448.170344:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677448.170344:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.170345:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677448.170347:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.170348:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.170348:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.170350:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.170351:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:17.0:1619677448.170352:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677448.170354:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.170355:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.170355:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.170358:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.170361:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.170377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.170384:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.170386:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.170387:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.170389:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.170391:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.170392:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.170394:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.170395:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.170396:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.170397:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.170400:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.170402:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.170406:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.170406:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.170407:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.170409:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.170409:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.170410:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.170411:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.170412:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.170417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.170423:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677448.170424:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.170426:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677448.170426:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.170427:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.170427:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.170427:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.170427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:26.0:1619677448.170428:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.170430:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.170432:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677448.170433:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677448.170435:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.170436:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.170438:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.170441:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.170442:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.170443:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718423] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.170445:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.170446:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.170447:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.170450:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.170452:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.170454:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.170455:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.170456:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.170457:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.170458:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.170458:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.170459:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.170463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.170471:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.170473:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.170473:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.170474:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.170475:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.170477:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.170498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.170504:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.170506:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.170507:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.170509:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.170511:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.170513:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.170514:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.170515:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.170516:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.170517:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.170521:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.170522:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.170525:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.170526:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.170527:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.170528:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.170529:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.170529:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.170530:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677448.170561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:9.2:1619677448.170564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.170570:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677448.170572:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677448.170572:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.170573:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.170575:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.170575:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.170577:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.170578:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.170580:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.170581:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.170583:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.170584:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.170585:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.170586:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.170587:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677448.170588:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:24.0:1619677448.170589:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.170590:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.2:1619677448.170592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:24.0:1619677448.170593:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.170594:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.170595:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.170596:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.170597:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677448.170597:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677448.170598:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:31.0:1619677448.170600:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.2:1619677448.170602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677448.170603:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.170605:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.170607:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.170608:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677448.170608:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.170610:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.170610:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:31.0:1619677448.170611:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.170612:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.170612:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.170612:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.170613:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.170613:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:31.0:1619677448.170616:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.170616:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.170619:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.170619:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:31.0:1619677448.170621:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677448.170621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:31.0:1619677448.170622:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.170622:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.170624:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.170624:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.170625:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.170626:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677448.170628:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.170631:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.170632:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.170632:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.170633:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.170634:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:12.0:1619677448.170637:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.170640:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.170640:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.170642:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.170644:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.170645:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.170822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.170828:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.170830:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:5.2:1619677448.170831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677448.170831:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.170833:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.170836:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.170837:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.170838:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.170838:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677448.170840:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.170841:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:17.0:1619677448.170841:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.170841:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:17.0:1619677448.170842:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.170844:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.170846:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.170847:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.170848:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.170849:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.170850:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.170851:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.170852:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:10.0:1619677448.170852:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.170852:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.170853:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:10.0:1619677448.170854:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.170855:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.170855:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.170857:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:10.0:1619677448.170857:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.170858:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.170859:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.170859:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.170861:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.170862:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.170863:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.170864:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.2:1619677448.170864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.170865:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.170865:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.170866:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.170868:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:1.0:1619677448.170869:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:30.0:1619677448.170870:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.170871:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.170871:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.170872:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.170872:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677448.170873:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.170874:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.170876:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:12.0:1619677448.170876:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.170879:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.170880:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.170881:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.170883:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.170884:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.170918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.170924:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.170927:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.170929:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.170930:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.170932:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.170933:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.170935:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.170936:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.170937:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.170938:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.170942:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.170943:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.170946:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.170947:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.170948:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.170950:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.170951:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.170952:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.170953:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677448.170963:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.170966:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.170967:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.170969:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.170970:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.170972:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.170989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.170996:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.170999:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.170999:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:4.2:1619677448.171001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:26.0:1619677448.171002:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.171005:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.171007:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.171008:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718416] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.171009:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:26.0:1619677448.171010:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.171010:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677448.171012:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.171013:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.171015:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.171016:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000400:00000200:10.0:1619677448.171017:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.171018:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.171019:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.171020:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.171021:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.171022:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.171022:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.171026:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.171026:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.171027:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.171028:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.171028:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.171030:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.171031:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.171031:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.171032:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677448.171032:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.171033:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.171033:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.171034:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.171035:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.171035:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.171035:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.171036:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.171036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.171037:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.2:1619677448.171040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.171047:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:22.0:1619677448.171048:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:20.0:1619677448.171048:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:26.0:1619677448.171050:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.171050:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:20.0:1619677448.171050:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:26.0:1619677448.171051:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:22.0:1619677448.171051:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:20.0:1619677448.171051:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.171052:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.171052:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:20.0:1619677448.171053:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.171054:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:22.0:1619677448.171055:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.171056:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677448.171058:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.2:1619677448.171058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.171065:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.171068:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.171069:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.171070:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.171071:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.171072:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.171156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:21.2:1619677448.171159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677448.171162:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.171165:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.171166:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:30.0:1619677448.171167:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:2.0:1619677448.171168:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.171170:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:2.0:1619677448.171170:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.171172:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:31.0:1619677448.171174:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:2.0:1619677448.171174:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.171175:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.171175:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.171176:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.171177:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.171177:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677448.171178:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.171180:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.171181:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.171182:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.171183:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.171184:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.171184:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.171186:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.171187:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:30.0:1619677448.171188:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.171188:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.171190:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.171190:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.171190:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.171191:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.171192:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:30.0:1619677448.171193:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.171193:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.171194:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.171196:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.171196:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.2:1619677448.171196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:8.2:1619677448.171196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.171197:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.171199:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.171199:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677448.171202:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677448.171203:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:21.2:1619677448.171205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.171205:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:17.0:1619677448.171205:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.171206:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.171206:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:1.0:1619677448.171206:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:18.0:1619677448.171207:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.171209:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.171209:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:18.0:1619677448.171210:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.171211:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:13.0:1619677448.171211:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:18.0:1619677448.171212:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:13.0:1619677448.171212:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.171213:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:18.0:1619677448.171214:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677448.171214:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.171214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.171215:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:18.0:1619677448.171215:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.171215:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.171216:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677448.171216:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.171217:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:18.0:1619677448.171217:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:13.0:1619677448.171217:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:18.0:1619677448.171221:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.171221:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.171223:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.171223:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.171224:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.171225:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.171225:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:1.0:1619677448.171225:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677448.171226:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.171226:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.171226:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677448.171228:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.171228:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.171229:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.171230:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.171234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.171241:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.171243:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.171244:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.171245:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.171246:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.171248:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.171251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:21.2:1619677448.171255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677448.171259:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.171261:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.171262:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.171262:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.171263:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.171263:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.171263:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677448.171264:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:2.0:1619677448.171265:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.171266:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677448.171268:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.171269:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.171270:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718411] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.171271:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.171272:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.171273:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.171276:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.171277:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.171280:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.171281:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.171281:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.171283:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.171283:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677448.171284:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677448.171284:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:21.2:1619677448.171288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.0:1619677448.171292:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:12.0:1619677448.171295:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.171299:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.171300:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:12.0:1619677448.171302:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:5.0:1619677448.171302:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.171303:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.171303:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.171305:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.171305:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.171306:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.171307:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.171308:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677448.171309:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.171310:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.171311:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.171314:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.171316:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.171318:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.171319:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.171320:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.171321:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.171322:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.171323:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.171323:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.171328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.171335:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.171338:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.171339:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.171340:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.171341:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.171343:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.2:1619677448.171534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.171540:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.171543:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.171545:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.171545:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.171547:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.171549:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:5.0:1619677448.171550:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.171551:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.171553:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.171553:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:5.0:1619677448.171557:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.171558:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:5.0:1619677448.171561:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.171561:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:5.0:1619677448.171562:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:5.0:1619677448.171564:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:5.0:1619677448.171564:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.171565:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.171565:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677448.171570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.171576:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.171579:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.171580:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.171581:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.171583:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.171586:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677448.171636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.171643:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.171646:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.171647:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.171649:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.171652:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.171653:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.171655:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.171656:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.171657:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.171658:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.171661:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.171663:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.171665:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.171666:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.171667:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.171669:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.171669:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.171670:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.2:1619677448.171671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677448.171671:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677448.171674:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.171676:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.171679:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.171680:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.171682:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.171683:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:6.0:1619677448.171683:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:20.0:1619677448.171685:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.171686:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.171686:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.171687:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:20.0:1619677448.171687:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.171687:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.171688:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.171689:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.171692:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.171692:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677448.171693:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.171696:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.171697:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.171698:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.171700:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.171700:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.171701:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.171702:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677448.171713:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.171716:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.171717:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.171719:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.171720:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.171722:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.171754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.171761:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.171764:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.171764:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.171767:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.171770:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.171771:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.171772:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.171773:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.171774:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.171775:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.171778:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.171780:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.171782:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.171783:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.171783:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.171785:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.171785:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.171787:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.171788:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.171788:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.171791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.171793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677448.171797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677448.171798:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:17.0:1619677448.171799:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677448.171800:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:17.0:1619677448.171801:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:2.0:1619677448.171801:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677448.171802:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.171802:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677448.171803:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:26.0:1619677448.171804:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677448.171804:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.171806:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677448.171807:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.171808:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.171810:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.171813:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.171814:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.171816:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718405] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.171817:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.171818:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.171819:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.171822:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.171824:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.171826:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.171827:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.171827:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.171829:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.171829:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.171830:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.171830:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.171841:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.171843:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:5.2:1619677448.171843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677448.171844:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.171846:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.171847:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.171848:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:1.0:1619677448.171848:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.171851:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.171852:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.171854:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.171857:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.171858:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:31.0:1619677448.171860:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000400:00000200:1.0:1619677448.171860:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.171861:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.171862:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000400:00000200:1.0:1619677448.171862:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.171863:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000400:00000200:1.0:1619677448.171863:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677448.171864:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.171865:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.171866:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:1.0:1619677448.171866:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.171868:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.171870:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.171871:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.171872:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.171873:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.171874:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.171875:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.171875:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.171889:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.171892:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.171892:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.171893:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.171895:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.171897:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.171938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.171945:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.171948:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.171949:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.171951:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.171953:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.171955:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.171956:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.171957:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.171958:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.171959:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.171962:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.171963:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.171966:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.171966:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.171967:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.171969:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.171969:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.171970:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.171970:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.171980:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.171983:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.171984:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.171985:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.171987:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.171988:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.171998:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.172000:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.172001:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.172002:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.172002:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.172003:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.172017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.172024:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.172027:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.172029:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.172031:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.172031:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.172032:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.172033:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.172034:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.172035:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.172036:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.172038:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.172040:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.172042:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.172042:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.172043:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.172044:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.172045:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.172045:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.172046:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677448.172056:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.172058:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.172059:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.172061:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.172062:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.172063:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.172351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.172358:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.172361:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.172362:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.172364:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.172366:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.172367:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.172369:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.172370:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.172371:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.172372:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.172374:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.172376:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.172378:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.172379:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.172379:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.172381:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.172381:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.172382:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.172384:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.172384:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.172386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677448.172390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.172392:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677448.172395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677448.172395:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.172396:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:4.2:1619677448.172397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.172398:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:17.0:1619677448.172398:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.172400:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:17.0:1619677448.172400:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.172401:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.172401:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:17.0:1619677448.172402:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:10.0:1619677448.172402:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:31.0:1619677448.172403:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.172403:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677448.172403:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.172403:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.172404:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.172405:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.172405:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677448.172405:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:17.0:1619677448.172405:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.172406:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:17.0:1619677448.172406:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.172406:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:31.0:1619677448.172407:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.172407:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677448.172407:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677448.172409:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.172411:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.172411:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.172412:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.172413:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.172414:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:17.0:1619677448.172414:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:10.0:1619677448.172414:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677448.172415:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.172416:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.172416:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.172417:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.172417:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.172417:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:10.0:1619677448.172418:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.172419:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.172419:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.172419:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.172419:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.172420:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.172420:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.172420:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.172421:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677448.172425:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.172426:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.172426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.172428:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.172428:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.172431:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.172431:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.172432:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.172432:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.172433:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.172433:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.172433:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:30.0:1619677448.172434:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.172434:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.172435:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.172435:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.172435:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:30.0:1619677448.172436:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677448.172436:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:10.0:1619677448.172436:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:1.0:1619677448.172436:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.172437:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.172437:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.172438:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677448.172438:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.172440:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.172443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.172447:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.172449:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.172450:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.172450:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677448.172452:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.172452:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:30.0:1619677448.172453:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.172453:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677448.172453:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:31.0:1619677448.172454:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.172454:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.172455:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.172456:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.172457:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.172459:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.172462:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.172470:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.172473:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.172474:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.172474:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.172475:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.172475:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.172643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.0:1619677448.172647:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.172649:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677448.172652:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.172652:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.172655:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.172656:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.172657:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.172659:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.172660:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.172661:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.172664:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.172666:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.172669:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677448.172670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:5.0:1619677448.172670:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.172671:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.172674:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.172674:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.172676:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:18.0:1619677448.172677:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.2:1619677448.172678:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.2:1619677448.172679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:18.0:1619677448.172680:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:5.0:1619677448.172680:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:2.0:1619677448.172680:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.172682:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.172682:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:21.2:1619677448.172683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (21)++
00000800:00000200:17.0:1619677448.172683:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.172684:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:18.0:1619677448.172685:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.172685:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:18.0:1619677448.172686:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:17.0:1619677448.172686:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:5.0:1619677448.172686:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.172687:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:1.0:1619677448.172687:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.172688:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000400:00000200:18.0:1619677448.172688:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.172689:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.172689:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.172690:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:18.0:1619677448.172690:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.172690:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.172691:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.172691:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677448.172692:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:18.0:1619677448.172692:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677448.172692:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.172694:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677448.172696:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000400:00000200:17.0:1619677448.172696:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.172697:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.172698:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:17.0:1619677448.172699:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.172699:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.172700:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.172701:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.172701:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.172703:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:17.0:1619677448.172703:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.172704:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.172704:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.172705:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677448.172705:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.172708:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.172708:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.172709:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:9.2:1619677448.172709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.172711:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.172711:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.172712:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.172712:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677448.172716:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.172718:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.172719:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.172720:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.172721:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.172723:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677448.172725:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.172727:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.172728:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.172728:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.172730:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.172733:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.172759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.172766:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.172769:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.172771:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.172773:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.172774:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.172776:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.172777:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.172778:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.172779:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.172780:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.172784:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.172786:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.172788:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.172789:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.172789:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.172791:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.172791:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.172793:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.172794:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.172795:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677448.172801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.172805:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.172807:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.172807:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.172809:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.172809:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.172810:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.172810:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.172811:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.172812:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.172812:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.172823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.172829:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.172832:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.172833:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.172835:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.172837:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.172839:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.172840:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.172841:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.172842:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.172843:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.172846:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.172848:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.172851:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.172852:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.172852:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.172854:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.172854:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.172855:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.172855:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.172867:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.172870:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.172871:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.172871:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.172873:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.172876:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.173007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.173012:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.173015:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.173016:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.173018:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.173020:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.173022:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.173023:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.173025:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.173026:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.173027:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.173030:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.173031:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.173034:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.173034:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.173035:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.173036:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.173037:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.173038:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.173039:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.173040:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.173045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.173050:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677448.173050:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.173052:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.173053:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.173053:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.173054:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.173054:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.173054:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.173056:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.173058:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.173160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.173166:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.173169:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.173170:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.173172:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.173174:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.173175:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.173176:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.173177:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.173178:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.173179:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.173183:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.173184:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.173186:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.173187:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.173187:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.173188:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.173189:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.2:1619677448.173190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677448.173190:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.173190:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.173195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.173196:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.173199:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.173201:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.173201:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677448.173202:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.173202:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.173204:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:2.0:1619677448.173204:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.173204:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.173205:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.173206:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.173206:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.173207:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.173208:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.173208:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:5.0:1619677448.173209:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677448.173213:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.173214:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.173217:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.173217:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.173218:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.173220:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.173220:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.173222:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.173223:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.173223:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677448.173226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677448.173228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.173231:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.173233:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:1.0:1619677448.173233:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:20.0:1619677448.173234:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.173235:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677448.173236:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.173237:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.173237:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.173239:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.173242:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.173390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.173398:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.173401:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.173403:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.173405:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.173406:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.173407:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.173408:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.173410:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.173411:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.173412:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.173415:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.173416:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.173420:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.173420:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.2:1619677448.173420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.173421:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.173423:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.173423:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.173424:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.173425:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:17.0:1619677448.173426:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.173429:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:8.2:1619677448.173430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:1.0:1619677448.173430:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.173431:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.173434:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.173435:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.173436:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.173438:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:17.0:1619677448.173438:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.173439:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.173440:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:17.0:1619677448.173440:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:20.0:1619677448.173441:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.173443:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.173443:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.173444:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:17.0:1619677448.173445:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.173447:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677448.173449:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.173450:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.173450:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.173452:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.173452:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.173453:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.173454:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.173467:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.173469:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.173470:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.173472:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.173474:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.173475:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.173513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677448.173520:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677448.173523:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677448.173525:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.173527:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677448.173528:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.173529:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:12.0:1619677448.173531:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.173532:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.173533:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677448.173534:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:12.0:1619677448.173537:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.173538:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:12.0:1619677448.173542:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.173543:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:12.0:1619677448.173544:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677448.173545:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:12.0:1619677448.173546:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677448.173547:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677448.173549:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677448.173549:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:8.2:1619677448.173551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.173557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:5.0:1619677448.173563:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:13.0:1619677448.173565:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:5.0:1619677448.173565:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.173567:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.173568:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677448.173568:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:5.0:1619677448.173568:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.173569:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:13.0:1619677448.173570:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.0:1619677448.173572:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.173585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.173590:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.173592:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.173594:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.173595:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.173597:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.173599:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.173600:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.173601:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.173602:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.173603:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.173606:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.173608:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.173611:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.173611:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.173612:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.173613:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.173614:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.173615:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.173615:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.173619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.173627:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.173629:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.173630:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.173631:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.173632:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.173633:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:4.2:1619677448.173676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.173684:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.173687:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.173687:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.173689:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.173692:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.173694:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.173695:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.173697:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.173698:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.173699:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.173702:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.173704:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.173707:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.173707:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.173708:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.173710:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.173711:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.173711:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.173712:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.173716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.173725:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.173727:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.173729:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.173729:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.173731:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.173734:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.173868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.173874:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.173876:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.173877:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.173879:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.173881:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.173883:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.173884:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.173886:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.173887:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.173888:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.173892:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.173894:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.173896:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.173897:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.173898:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.173899:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.173900:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.173900:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.173901:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.173905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677448.173908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.173911:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.173912:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677448.173913:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.173915:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.173915:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.173916:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.173916:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.173917:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.173917:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:1.0:1619677448.173918:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.173919:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677448.173920:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.173921:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.173921:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:20.0:1619677448.173922:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.173923:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677448.173927:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.173928:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677448.173931:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.173931:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677448.173932:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.173934:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677448.173934:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.173935:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.173936:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.173940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.173947:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.173949:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:21.2:1619677448.173950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:17.0:1619677448.173950:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.173950:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.173952:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.173954:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:24.0:1619677448.173958:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.173961:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.173962:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.173963:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.173966:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.173968:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.173969:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.173970:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.173971:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.173972:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.173975:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.173977:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.173980:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.173980:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.173981:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.173983:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.173984:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.173985:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.173986:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.173987:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677448.173989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.173993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:2.0:1619677448.173996:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.173999:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.174000:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.174000:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:2.0:1619677448.174000:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.174001:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.174002:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:2.0:1619677448.174002:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.174004:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:2.0:1619677448.174005:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.174010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.174017:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.174020:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.174021:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.174021:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.174022:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.174023:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.174128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fdae00] (20)++
00000800:00000200:5.0:1619677448.174132:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fdae00] (21)++
00000800:00000200:5.0:1619677448.174134:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.38@o2ib3
00000800:00000200:20.0:1619677448.174137:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (22)--
00000400:00000200:5.0:1619677448.174137:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.38@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677448.174139:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.174141:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.174142:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.174143:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.174145:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.174146:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.174149:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.38@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.174151:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.174153:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.174154:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.174154:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.174156:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.174157:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fdae00] (21)++
00000800:00000200:5.0:1619677448.174158:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fdae00] (22)--
00000800:00000200:5.0:1619677448.174158:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (21)--
00000800:00000200:21.2:1619677448.174170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677448.174170:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.174172:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.174173:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.174173:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.174175:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677448.174177:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677448.174178:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.174180:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.174183:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.174184:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.174185:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.174187:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.174188:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.174190:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.174191:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.174192:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.174195:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.174196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.174197:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.174199:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.174200:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.174201:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.174201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:31.0:1619677448.174203:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.174203:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:4.2:1619677448.174203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.174204:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.174204:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.174205:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677448.174205:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677448.174206:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:2.0:1619677448.174206:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:30.0:1619677448.174207:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.174209:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.174209:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.174210:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:10.0:1619677448.174210:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677448.174212:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:10.0:1619677448.174213:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:30.0:1619677448.174214:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.174214:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677448.174214:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.174215:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:30.0:1619677448.174216:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.174217:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.174217:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:10.0:1619677448.174217:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.174217:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.174218:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.174218:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677448.174219:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.174219:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.174220:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.174221:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.174221:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.174221:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.174222:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.174223:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.174223:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96718378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.174223:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.174224:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.174226:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.174226:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.174228:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.174228:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.174229:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.174229:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.174229:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.174230:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.174230:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.174230:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.174231:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.174232:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.174232:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677448.174232:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.174233:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:13.0:1619677448.174233:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.174233:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.174233:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.174233:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.174234:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677448.174234:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.174234:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.174234:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677448.174235:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.174236:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.174236:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.174237:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.174237:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:10.0:1619677448.174238:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.174238:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.174238:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:10.0:1619677448.174239:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.174240:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.174241:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.174242:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.174242:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.174242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677448.174244:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.174246:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.174247:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.174249:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.174250:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:13.0:1619677448.174250:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.174251:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.174252:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:18.0:1619677448.174252:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.174253:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.174254:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.174255:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:18.0:1619677448.174255:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.174256:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.174256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:26.0:1619677448.174257:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.174257:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.174257:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.174258:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.174259:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.174260:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.174261:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.174262:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.174264:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.174266:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677448.174267:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.174268:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.174269:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.174270:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.174274:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:4.2:1619677448.174274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677448.174275:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.174275:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.174276:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.174277:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.174277:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.174277:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.174279:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:24.0:1619677448.174280:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677448.174280:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.174281:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.174282:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:24.0:1619677448.174283:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:10.0:1619677448.174284:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.174284:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:8.0:1619677448.174284:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:10.0:1619677448.174285:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.174285:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:24.0:1619677448.174286:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.174289:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.174290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:24.0:1619677448.174291:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.174300:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96718376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.174300:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:24.0:1619677448.174301:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.174302:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.174302:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:24.0:1619677448.174303:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.174303:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.174305:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000400:00000200:24.0:1619677448.174307:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.174307:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.174308:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:24.0:1619677448.174309:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.174312:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.174313:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.174314:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.174315:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.174316:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.174317:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.174317:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.174321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.174329:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.174332:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.174333:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.174333:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.174335:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.174338:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.174341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.174349:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.174352:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.174352:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.174353:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.174353:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.174354:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.174468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.174474:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.174477:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.174478:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.174479:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.174482:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.174482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:22.0:1619677448.174483:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.174485:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.174486:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.174487:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.174488:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:8.0:1619677448.174489:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:22.0:1619677448.174491:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.174492:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.174493:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677448.174493:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:8.0:1619677448.174495:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.174496:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.174497:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.174497:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:8.0:1619677448.174498:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.174499:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.174499:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:8.0:1619677448.174499:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:22.0:1619677448.174500:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:8.0:1619677448.174500:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96718374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.174501:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:8.0:1619677448.174502:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.174503:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.174505:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:8.0:1619677448.174509:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.174512:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.174513:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:8.0:1619677448.174514:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.174516:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.174516:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.174517:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.174518:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.174518:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677448.174518:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.174519:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:8.0:1619677448.174520:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.174521:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.174521:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677448.174521:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.0:1619677448.174522:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.174524:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:9.2:1619677448.174527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.174532:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.174534:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.174535:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677448.174535:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677448.174536:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.174537:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.174538:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677448.174538:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.174539:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.174539:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.174542:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.174544:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.174573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.174581:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.174583:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.174586:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.174588:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.174588:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.174589:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.174591:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.174592:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.174593:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.174594:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.174597:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.174599:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.174601:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.174602:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.174602:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.174604:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.174605:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.174606:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.174606:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.174617:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.174619:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.174620:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.174622:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.174624:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.174624:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.174643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.174648:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.174651:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.174653:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.174653:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.174655:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.174657:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.174658:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.174659:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.174659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:20.0:1619677448.174660:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.174661:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.174664:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.174665:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677448.174667:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677448.174668:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.174669:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:20.0:1619677448.174669:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.174669:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:10.0:1619677448.174670:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677448.174671:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.174671:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000400:00000200:26.0:1619677448.174672:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.174672:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.174673:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:26.0:1619677448.174675:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.174676:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.174677:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.174677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000400:00000200:26.0:1619677448.174678:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.174679:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.174680:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.174683:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.174684:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:26.0:1619677448.174685:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.174686:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:26.0:1619677448.174687:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.174687:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.174688:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.174689:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.174689:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.174689:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.174691:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.174691:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677448.174691:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:26.0:1619677448.174692:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.174692:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.2:1619677448.174695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000800:00000200:9.2:1619677448.174697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.174702:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:18.0:1619677448.174704:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677448.174705:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.174706:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.174706:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.174707:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:18.0:1619677448.174707:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.174708:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.174708:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:2.0:1619677448.174708:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.174710:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.174714:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.174740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.174747:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.174750:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.174750:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.174752:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.174755:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.174757:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.174758:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.174759:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.174760:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.174761:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.174764:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.174766:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.174769:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.174769:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.174770:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.174772:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.174772:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.174773:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.174774:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.174775:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.174780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.174785:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:26.0:1619677448.174787:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.174788:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:26.0:1619677448.174789:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677448.174789:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.174789:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.174790:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.174791:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.174791:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.174794:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.174807:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.174809:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.174810:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.174811:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.174811:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.174812:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.174841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.174844:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.174846:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.174849:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.174849:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.174851:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.174852:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677448.174853:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.174855:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.174856:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.174857:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677448.174860:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.174862:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677448.174864:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.174865:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677448.174866:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.174867:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677448.174868:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.174869:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.174869:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.174873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.174881:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.174883:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.174884:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.174885:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.174887:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.174889:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.174903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.174908:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.174911:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.174911:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.174914:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.174916:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.174917:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.174919:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.174920:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.174921:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.174922:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.174925:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.174927:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.174929:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.174930:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.174930:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.174932:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.174932:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.174933:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.174934:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.174935:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.174938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.174940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.174946:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:5.0:1619677448.174946:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.2:1619677448.174947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:22.0:1619677448.174948:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.174948:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.0:1619677448.174948:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677448.174950:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:5.0:1619677448.174950:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.174951:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.174954:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.174955:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677448.174956:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.174958:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:21.2:1619677448.174958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:30.0:1619677448.174961:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.174963:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.174963:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.174964:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.174965:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.174965:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.174966:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.174968:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.174968:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.174970:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.174972:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.174974:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.174974:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.174976:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677448.174976:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.2:1619677448.174976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:31.0:1619677448.174977:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.174978:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.174979:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.174980:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.174983:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.174983:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.174983:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:31.0:1619677448.174984:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.174984:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.174984:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.174986:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.174986:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:24.0:1619677448.174986:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:31.0:1619677448.174987:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.174987:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.174987:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:8.0:1619677448.174987:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:31.0:1619677448.174988:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.174988:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677448.174989:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.174989:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:24.0:1619677448.174989:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.174990:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.174992:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:8.2:1619677448.174992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.174993:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.174993:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:24.0:1619677448.174993:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.174995:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.174996:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96718365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.174998:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.174998:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.174999:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:21.2:1619677448.174999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.175002:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.175002:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:24.0:1619677448.175003:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.175004:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677448.175005:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.175005:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:13.0:1619677448.175006:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.175007:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.175008:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:24.0:1619677448.175008:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.175008:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.175009:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:13.0:1619677448.175009:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.175009:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677448.175009:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.175010:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:24.0:1619677448.175010:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.175011:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.0:1619677448.175011:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:24.0:1619677448.175012:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677448.175013:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677448.175013:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.175013:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677448.175013:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677448.175014:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.175018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677448.175022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.175024:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:2.0:1619677448.175025:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677448.175027:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:2.0:1619677448.175028:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:20.0:1619677448.175029:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.175030:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.175030:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:2.0:1619677448.175030:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.175032:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.175032:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.175033:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:9.2:1619677448.175034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677448.175035:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.175036:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.175037:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.175037:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:20.0:1619677448.175038:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.175038:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.175041:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.175042:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.175042:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.175042:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.175043:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.175044:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.175044:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677448.175047:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.175047:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.175048:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.175050:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.175051:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.175052:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.175052:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677448.175056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.175063:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.175065:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.175066:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.175066:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.175068:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.175071:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.175127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.175135:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.175137:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.175138:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.175140:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.175144:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.175145:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.175146:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.175148:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.175149:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.175150:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.175153:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.175155:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.175158:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.175158:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.175159:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.175161:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.175161:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.175162:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.175162:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.175174:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.175177:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.175178:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.175179:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.175181:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.175183:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.175210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.175216:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.175219:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.175220:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.175222:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.175224:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.175226:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.175227:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.175229:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.175230:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.175231:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.175234:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.175236:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.175239:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.175240:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.175240:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.175242:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.175242:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.175243:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.175244:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.175248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.175254:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.175257:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.175258:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.175258:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.175260:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.175263:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.2:1619677448.175290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.175300:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.175303:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.175303:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.175305:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.175308:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.175310:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.175311:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.175312:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.175313:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.175314:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.175318:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.175319:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.175322:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.175322:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.175323:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.175325:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.175325:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.175326:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.175326:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.175338:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.175341:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.175341:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.175343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:12.0:1619677448.175343:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.175345:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.175346:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.175351:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.175354:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.175356:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.175358:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.175358:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.175359:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.175359:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.175360:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.175360:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.175361:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:30.0:1619677448.175362:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.175362:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.175362:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.175362:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:30.0:1619677448.175363:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.175363:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.175364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:30.0:1619677448.175366:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.175368:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.175370:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.175370:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677448.175371:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.175371:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.175372:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.175373:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677448.175373:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:30.0:1619677448.175374:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.175374:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.175374:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.175375:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.175378:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.175379:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.175380:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.175381:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.175382:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.175383:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:12.0:1619677448.175384:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:22.0:1619677448.175386:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.175386:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.175387:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.175389:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:12.0:1619677448.175389:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.175391:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.175391:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.175391:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677448.175392:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.175393:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.175395:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.175395:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.175396:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.175396:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.175402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.175411:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.175413:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.175414:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.175416:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.175416:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.175418:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.175452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.175458:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.175461:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.175462:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.175464:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.175466:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.175467:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.175469:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.175470:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.175471:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.175472:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.175476:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.175477:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.175481:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.175482:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.175482:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.175484:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.175484:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.175485:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.175486:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677448.175498:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.175500:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.175501:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.175503:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.175505:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.175506:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.175518:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.175521:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.175522:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.175523:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.175523:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.175525:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.175602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.175611:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.175614:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.175616:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.175617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:31.0:1619677448.175618:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.175618:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.175620:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.175621:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.175622:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.175623:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.175624:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.175624:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.2:1619677448.175626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:31.0:1619677448.175627:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.175627:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.175628:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.175629:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.175630:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.175632:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.175632:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.175633:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.175633:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:26.0:1619677448.175633:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:31.0:1619677448.175634:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:26.0:1619677448.175635:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.175635:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677448.175636:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:17.0:1619677448.175636:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.175637:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677448.175637:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.175638:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:1.0:1619677448.175638:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.175639:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.175640:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000400:00000200:26.0:1619677448.175640:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.175641:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:1.0:1619677448.175641:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.175642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.2:1619677448.175643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:1.0:1619677448.175643:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.175644:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.175645:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96718355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.175647:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.2:1619677448.175647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:1.0:1619677448.175647:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.175648:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.175649:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.175650:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.175651:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.175651:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.175651:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.175653:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.175653:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677448.175653:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.175654:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677448.175654:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.175654:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.175655:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.175655:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:1.0:1619677448.175655:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.175656:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:30.0:1619677448.175656:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.175657:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.175658:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.175658:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.175658:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:1.0:1619677448.175658:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.175659:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.175659:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.175660:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:1.0:1619677448.175660:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (23)++
00000400:00000200:30.0:1619677448.175661:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677448.175661:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.175661:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:30.0:1619677448.175662:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96718354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.175663:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (24)++
00000800:00000200:1.0:1619677448.175663:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:31.0:1619677448.175664:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (24)--
00000400:00000200:30.0:1619677448.175664:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.175664:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.175664:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677448.175665:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.175665:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.175665:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677448.175666:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.175667:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.175668:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677448.175670:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.175670:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.175672:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.175672:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.175673:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.175674:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.175675:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.175676:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.175676:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.175677:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.175677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677448.175678:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.175679:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.175679:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.175679:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.175680:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.175680:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.175685:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.2:1619677448.175685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677448.175687:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.175688:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.175690:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.175692:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.175693:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:10.0:1619677448.175693:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:17.0:1619677448.175695:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:10.0:1619677448.175695:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.175696:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.175696:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.175697:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.175697:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.175698:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.175699:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.175702:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.175703:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677448.175704:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.175706:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.175707:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.175707:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.175709:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.175709:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.175710:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.175711:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.175721:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.175723:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.175724:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.175726:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.175727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:12.0:1619677448.175727:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.175728:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677448.175734:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.175737:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.175738:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.175739:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.175742:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.175744:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.175745:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.175746:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.175747:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.175748:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.175751:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.175752:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.175755:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.175756:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.175757:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.175759:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.175759:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.175760:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.175760:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.175765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.175773:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.175775:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.175776:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.175777:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.175779:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.175782:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.175916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.175922:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.175925:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.175925:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.175927:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.175930:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.175932:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.175933:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.175934:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.175935:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.175937:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.175940:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.175942:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.175944:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.175945:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.175946:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.175947:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.175948:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.175949:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.175951:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.175951:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.175953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.2:1619677448.175956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.175959:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:17.0:1619677448.175961:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677448.175962:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.175963:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.175963:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:8.0:1619677448.175963:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.175964:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.175965:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.175965:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.175969:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677448.175970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.0:1619677448.175974:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.175977:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677448.175979:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:9.2:1619677448.175979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000400:00000200:5.0:1619677448.175979:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.175981:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.175983:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.175984:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.175985:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000400:00000200:5.0:1619677448.175986:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.175987:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.175988:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677448.175988:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:5.0:1619677448.175988:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:24.0:1619677448.175990:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.175991:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.175991:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677448.175991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:5.0:1619677448.175993:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.175995:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.175997:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.175998:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:31.0:1619677448.175999:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.175999:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.176001:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.176001:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:31.0:1619677448.176002:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.0:1619677448.176002:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.176003:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:31.0:1619677448.176004:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.176006:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.176007:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.176007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000400:00000200:31.0:1619677448.176008:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.176009:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.176011:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.176012:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.176013:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.176015:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:31.0:1619677448.176016:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.176017:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.176017:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.176018:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.176019:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.176020:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.176020:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:31.0:1619677448.176021:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.176021:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.176022:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:31.0:1619677448.176023:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.176024:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.176025:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.176025:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.176035:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.176038:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.176039:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.176039:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.176041:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.176043:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.176120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:5.2:1619677448.176124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:6.0:1619677448.176126:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.176129:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.176130:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:10.0:1619677448.176132:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:6.0:1619677448.176132:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.176134:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:6.0:1619677448.176134:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.176135:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677448.176136:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.176138:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.176139:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.176140:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.176141:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.176142:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.176142:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.176143:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.176145:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.176146:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.176147:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.176148:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.176149:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:6.0:1619677448.176149:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.176152:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.176153:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.176153:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.176154:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.176155:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677448.176156:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.176156:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:4.2:1619677448.176156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:10.0:1619677448.176157:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.176157:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.176157:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.176158:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.176158:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.176160:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.176160:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.176161:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.176162:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.176163:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677448.176164:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.176166:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.176167:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677448.176168:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.2:1619677448.176168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:8.0:1619677448.176169:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.176171:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:8.0:1619677448.176171:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.176172:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.176173:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:8.0:1619677448.176173:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677448.176174:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.176174:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:8.0:1619677448.176174:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96718346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.176175:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:26.0:1619677448.176175:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.176175:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.176177:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:8.0:1619677448.176177:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.176178:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.176178:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677448.176178:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.176179:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.176180:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.176180:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.176181:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.176181:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:8.0:1619677448.176181:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.176183:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.176186:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.0:1619677448.176186:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.176187:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.176188:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.176189:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:8.0:1619677448.176189:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.176190:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:8.0:1619677448.176190:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677448.176191:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.176191:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:8.0:1619677448.176191:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:31.0:1619677448.176193:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.0:1619677448.176193:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.176193:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.2:1619677448.176196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677448.176196:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.176199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.0:1619677448.176201:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.176204:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677448.176204:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.176206:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:5.0:1619677448.176206:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.176206:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677448.176207:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677448.176207:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.176208:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:5.0:1619677448.176208:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.176208:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.176209:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.176210:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:5.0:1619677448.176210:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.176210:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.176212:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.176213:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.176213:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:5.0:1619677448.176214:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.176215:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.176219:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.176221:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.176224:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.176224:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.176225:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.176227:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.176228:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.176229:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.176229:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.176240:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.176243:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.176244:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.176245:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.176247:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.176248:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.176277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:4.2:1619677448.176281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677448.176283:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.176286:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.176286:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.176288:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.176288:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677448.176291:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.176291:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.176292:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677448.176292:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.176299:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.176299:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.176300:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.176301:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.176302:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.176303:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.176303:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.176305:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.176306:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.176307:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.176308:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.176309:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.176309:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.176311:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.176312:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.176313:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.176314:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:10.0:1619677448.176314:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.176315:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.176315:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.176316:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.176316:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.176318:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.176319:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.176320:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.176321:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:8.2:1619677448.176321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.176322:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.176322:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.176323:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677448.176329:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.176331:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.176332:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.176333:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.176334:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.176335:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:1.0:1619677448.176336:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677448.176338:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.176339:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.176340:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.176340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:30.0:1619677448.176342:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.176343:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677448.176348:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.176350:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.176351:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.176352:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.176353:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.176353:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.176376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.176382:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.176385:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.176386:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.176388:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.176390:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.176391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:2.0:1619677448.176392:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.176394:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.176395:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.176397:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:2.0:1619677448.176397:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.176398:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677448.176398:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.176401:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.176402:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.176403:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.176403:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.176405:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.176405:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.176406:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.176407:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.176408:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.176411:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.176413:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.176415:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.176415:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.176416:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.176416:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.176417:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.176417:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.176419:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.176419:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.176420:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.176420:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.176421:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (23)++
00000800:00000200:2.0:1619677448.176421:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.176422:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677448.176423:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (24)--
00000800:00000200:18.0:1619677448.176423:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.176423:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (24)--
00000800:00000200:9.2:1619677448.176426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.176429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:13.0:1619677448.176434:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.176434:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.176435:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.176436:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:2.0:1619677448.176436:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.176437:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.176437:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:13.0:1619677448.176437:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677448.176438:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:13.0:1619677448.176438:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.176438:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.176440:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677448.176440:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:13.0:1619677448.176441:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.176442:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.176444:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.176620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.176628:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.176631:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.176633:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.176635:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.176636:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.176637:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.176638:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.176640:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.176641:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.176642:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.176645:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.176647:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.176649:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.176650:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.176651:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.176652:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.176653:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.176654:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.176655:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.176656:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677448.176662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.176664:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.176666:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.176667:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.176667:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.176670:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:26.0:1619677448.176670:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.176672:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:26.0:1619677448.176672:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:30.0:1619677448.176673:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.176674:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677448.176685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.176692:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.176695:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.176696:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.176698:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.176700:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.176701:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.176702:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.176703:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.176704:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.176705:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.176708:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.176708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.176710:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.176712:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.176713:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.176713:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.176714:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.176715:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.176716:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.176716:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.176717:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.176717:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.176717:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.176718:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677448.176719:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.176720:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.176722:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:4.2:1619677448.176723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677448.176723:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.176725:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.176726:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.176727:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.176728:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.176730:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:26.0:1619677448.176730:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:1.0:1619677448.176730:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.176731:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.176732:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:1.0:1619677448.176732:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:30.0:1619677448.176733:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.176733:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.176734:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.176734:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.176735:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:1.0:1619677448.176735:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.176736:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.176737:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.176739:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.176739:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.176740:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.176740:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.176752:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.176755:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.176756:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.176757:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.176758:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.176760:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.176776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.176779:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.176781:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.176784:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.176785:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.176787:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.176788:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.176790:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.176791:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.176792:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.176793:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.176796:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.176798:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.176801:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.176802:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.176803:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.176805:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.176805:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.176807:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.176808:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.176808:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.176813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.176818:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.176819:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:30.0:1619677448.176821:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:5.0:1619677448.176821:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.176822:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677448.176823:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.176823:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:30.0:1619677448.176824:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.176825:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.176827:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.176836:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.176839:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.176840:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.176840:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.176841:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.176842:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.176876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.176884:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.176886:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.176887:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.176889:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.176891:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.176893:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.176894:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.176895:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.176896:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.176897:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.176900:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.176902:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.176905:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.176906:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.176906:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.176908:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.176908:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.176909:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.176909:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677448.176922:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.176925:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.176925:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.176926:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.176928:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.176931:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.176932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.176940:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.176943:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.176945:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.176947:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.176947:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.176948:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.176949:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.176951:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.176952:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.176953:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.176956:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.176958:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.176960:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.176960:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.176961:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.176962:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.176963:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.176964:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.176964:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677448.176974:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.176977:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.176977:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.176979:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.176980:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.176981:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677448.176996:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.176999:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.176999:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.177000:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.177001:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.177002:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.177116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.177122:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.177125:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.177127:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.177127:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.177130:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.177132:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:5.0:1619677448.177135:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.177136:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.177137:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677448.177137:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.177139:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.177140:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:5.0:1619677448.177140:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.177141:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677448.177143:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.177145:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.177146:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.177148:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.177148:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:17.0:1619677448.177149:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.177149:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.177150:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.177151:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:5.0:1619677448.177151:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:17.0:1619677448.177152:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.177152:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.177152:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:17.0:1619677448.177153:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.177154:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.177155:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.2:1619677448.177156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:17.0:1619677448.177159:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.177161:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.177164:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.177165:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.177165:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.177166:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.177166:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.177167:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.177167:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.177168:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.177169:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.177169:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.177170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:17.0:1619677448.177170:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677448.177171:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.177172:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.177173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.177175:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.177178:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.177180:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.177181:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:5.0:1619677448.177181:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.177183:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.177183:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.177183:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.177184:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.177185:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.177185:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.177186:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.177187:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.177188:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.177188:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:20.0:1619677448.177189:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.177192:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.177194:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.177196:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.177197:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.177198:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.177199:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.177200:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.177202:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677448.177203:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677448.177204:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:8.2:1619677448.177205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:21.2:1619677448.177209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.177214:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.177215:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677448.177215:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.177216:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000400:00000200:5.0:1619677448.177216:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.177217:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.177218:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.177218:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.177219:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677448.177220:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.2:1619677448.177273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.177279:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.177282:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.177282:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:4.2:1619677448.177284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:18.0:1619677448.177285:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.177287:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.177289:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.177290:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.177291:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:18.0:1619677448.177291:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.177292:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.177299:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.177301:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:18.0:1619677448.177303:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.177304:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.177307:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.177307:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.177308:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:18.0:1619677448.177309:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.177310:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.177312:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.177312:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.177313:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.177313:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:26.0:1619677448.177314:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.177314:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.177315:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.177316:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.177317:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:26.0:1619677448.177318:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.177318:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.177319:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:26.0:1619677448.177320:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:8.2:1619677448.177322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.177323:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.177324:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:5.2:1619677448.177324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:26.0:1619677448.177325:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.177326:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.177327:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.177328:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.177328:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677448.177330:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:20.0:1619677448.177331:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677448.177332:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:20.0:1619677448.177333:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:2.0:1619677448.177333:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:20.0:1619677448.177334:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.177334:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:20.0:1619677448.177336:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.177337:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.177339:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:6.0:1619677448.177342:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.2:1619677448.177343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.177345:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.177345:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.177346:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.177348:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.177351:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677448.177351:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.177354:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.177355:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.177355:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.177356:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.177358:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.177470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.177477:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.177480:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.177481:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.177483:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.177486:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.177488:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.177489:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.177491:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.177491:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.177492:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.177496:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.177497:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.177500:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.177501:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.177501:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.177503:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.177504:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.177505:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.177505:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.177509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.177516:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.177519:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.177519:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.177521:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.177522:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.177524:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.177529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.177536:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.177539:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.177540:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.177541:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.177541:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.177543:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.177566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.177569:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.177571:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.177573:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.177574:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.177575:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.177576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:20.0:1619677448.177577:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677448.177578:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.177579:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.177580:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.177581:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:1.0:1619677448.177582:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:20.0:1619677448.177584:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.177585:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.0:1619677448.177586:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:17.0:1619677448.177586:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.177587:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.177589:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.177589:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.177590:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677448.177591:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:1.0:1619677448.177591:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.177592:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.177593:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677448.177593:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677448.177593:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.177594:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:1.0:1619677448.177594:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.177595:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677448.177595:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:9.2:1619677448.177599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:1.0:1619677448.177599:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.177601:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.177604:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.177604:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.177605:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.177607:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:1.0:1619677448.177607:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.177607:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.177609:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:1.0:1619677448.177609:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:18.0:1619677448.177610:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.177610:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:1.0:1619677448.177610:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.177611:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677448.177612:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.177615:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.177615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.177620:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.177621:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.177623:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:12.0:1619677448.177623:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.177624:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.177624:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.177625:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677448.177626:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.177627:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.177629:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.177655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.177661:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.177663:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.177665:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.177666:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.177669:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.177670:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677448.177671:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718326] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.177673:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.177674:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.177675:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677448.177678:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.177680:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677448.177683:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.177684:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677448.177684:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677448.177686:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677448.177687:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.177688:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.177688:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677448.177699:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.177702:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.177702:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.177703:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.177705:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:26.0:1619677448.177708:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.177946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.177952:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.177954:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.177955:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.177957:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.177960:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.177961:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.177962:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.177963:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.177964:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.177965:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.177968:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.177970:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.177973:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.177974:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.177974:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.177976:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.177977:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.177977:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.177978:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.177989:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.177992:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.177992:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.177994:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.177995:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.177997:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.178007:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.178010:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.178011:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.178011:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.178012:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.178014:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.178055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.178062:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.178065:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.178065:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.178067:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.178070:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.178071:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.178072:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.178074:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.178075:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.178076:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.178079:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.178081:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.178084:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.178085:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.178085:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.178087:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.178087:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.178088:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.178088:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677448.178091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.178100:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:22.0:1619677448.178100:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:30.0:1619677448.178103:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:22.0:1619677448.178103:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.178104:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.178104:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:30.0:1619677448.178105:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.178106:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.178107:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.178107:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.178108:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:22.0:1619677448.178109:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:30.0:1619677448.178110:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.178111:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.178112:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.178113:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.178116:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.178117:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.178119:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.178120:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.178121:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.178122:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.178123:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.178123:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.178124:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677448.178134:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.178136:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.178137:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.178139:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.178140:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.178141:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.178202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.0:1619677448.178207:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.178209:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677448.178211:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.178212:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.178214:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.178215:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.178217:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.178218:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.178219:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.178220:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.178223:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.178225:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.178227:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.178227:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.178228:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.178230:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.178230:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.178231:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.178232:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.178236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.178243:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.178245:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.178246:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.178247:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.178248:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.178249:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.2:1619677448.178363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.2:1619677448.178364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:17.0:1619677448.178369:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.178370:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:17.0:1619677448.178372:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:18.0:1619677448.178373:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:1.0:1619677448.178373:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.178374:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677448.178377:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.178377:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.178380:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.178381:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.178381:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.178382:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:17.0:1619677448.178383:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.178384:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.178385:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.178386:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.178386:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.178387:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.178387:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.178388:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677448.178393:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.178393:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.178395:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.178396:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:17.0:1619677448.178397:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.178397:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.178398:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.178398:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.178399:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.178400:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:17.0:1619677448.178400:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.178400:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.178401:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:17.0:1619677448.178401:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.178402:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:17.0:1619677448.178402:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677448.178403:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677448.178404:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677448.178404:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:8.2:1619677448.178406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:9.2:1619677448.178408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.178411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:1.0:1619677448.178414:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677448.178416:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:1.0:1619677448.178416:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.178417:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.178417:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.178418:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677448.178418:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.178419:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000400:00000200:2.0:1619677448.178419:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.178419:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.178420:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677448.178421:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.178421:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.178421:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677448.178423:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677448.178423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.178423:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.178425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:17.0:1619677448.178430:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.178433:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:17.0:1619677448.178433:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.178434:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.178434:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.178435:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.178436:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.178436:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:17.0:1619677448.178436:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677448.178437:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.178437:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.178438:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.178477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.2:1619677448.178481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677448.178485:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.178486:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.178487:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.0:1619677448.178488:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:31.0:1619677448.178491:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.178491:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.178491:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.178492:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.178493:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.178494:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.178495:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.178496:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.178497:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.178497:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.178499:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.178500:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.178501:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.178501:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.178503:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.178503:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.178509:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.178510:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.178512:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.178513:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.178515:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.178516:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.178516:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.178517:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.178517:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.178518:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.178519:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.178519:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.178520:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.178520:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.178520:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677448.178521:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.0:1619677448.178521:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677448.178522:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.178533:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:12.0:1619677448.178536:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677448.178536:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.178537:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.178537:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.178539:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.178539:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.178539:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:12.0:1619677448.178541:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.178541:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677448.178543:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.178544:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677448.178553:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.178556:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.178556:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.178557:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.178558:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.178558:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.178697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677448.178704:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.178707:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677448.178708:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677448.178710:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.178713:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.178714:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.178716:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.178717:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.178718:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.178719:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.178723:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.178725:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.178728:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.178729:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.178730:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.178731:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.178732:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.178733:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677448.178734:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677448.178735:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.2:1619677448.178737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.178741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:20.0:1619677448.178745:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.178747:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.178748:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.178748:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677448.178749:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000400:00000200:20.0:1619677448.178750:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.178750:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.178751:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:5.0:1619677448.178752:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.178753:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.178822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.178829:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.178832:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.178833:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.178835:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.178837:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.178839:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.178841:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.178842:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.178843:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.178844:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.178847:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.178848:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.178851:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.178852:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.178852:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.178854:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.178855:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.178855:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.178856:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.178860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.178868:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.178870:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.178872:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.178872:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.178874:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.178877:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.178883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.178890:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.178892:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.178893:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.178893:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.178894:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.178895:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.178998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.179005:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.179008:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.179010:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.179012:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.179012:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.179012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:30.0:1619677448.179014:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.179015:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.179017:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.179017:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:30.0:1619677448.179018:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.179019:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.179020:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.179021:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:30.0:1619677448.179023:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.179024:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.179026:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.179026:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.179028:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:30.0:1619677448.179029:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.179030:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:2.0:1619677448.179030:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.179031:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:2.0:1619677448.179031:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.179032:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.179033:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.179033:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:2.0:1619677448.179033:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.179034:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.179035:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:2.0:1619677448.179037:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.179038:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.2:1619677448.179039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.179041:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.179042:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.179043:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.179045:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.179045:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677448.179046:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677448.179046:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.179047:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.179048:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677448.179049:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.179050:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.179051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:5.0:1619677448.179052:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.179053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.0:1619677448.179053:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.179055:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.179055:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677448.179056:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.179057:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.179057:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:20.0:1619677448.179058:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.179058:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:20.0:1619677448.179061:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677448.179061:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.179062:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.179063:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677448.179083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.179089:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.179091:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.179093:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.179094:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.179095:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.179097:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.179098:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.179100:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.179100:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.179102:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.179105:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.179107:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.179109:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.179110:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.179110:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.179112:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.179112:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.179113:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.179114:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677448.179126:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.179129:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.179129:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.179130:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.179132:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.179134:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.179163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.179168:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.179171:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.179172:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.179174:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.179177:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.179178:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.179179:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.179181:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.179182:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.179183:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.179186:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.179187:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.179189:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.179190:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.179190:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.179192:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.179193:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.179193:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.179194:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.179203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.179210:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.179212:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.179213:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.179213:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.179215:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.179216:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.179345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.179350:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.179353:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.179355:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.179356:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.179357:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.179358:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.179360:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.179361:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.179362:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.179363:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.179366:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.179367:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.179369:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.179370:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.179370:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.179372:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.179372:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.179373:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.179374:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.179378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.179386:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.179388:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.179389:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.179390:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.179391:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.179393:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:8.2:1619677448.179400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.179408:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:4.2:1619677448.179408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677448.179411:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.179412:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.179413:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677448.179414:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.179415:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:10.0:1619677448.179416:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.179419:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.179419:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.179421:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.179424:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.179426:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.179427:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.179428:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.179430:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.179430:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.2:1619677448.179434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:10.0:1619677448.179434:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.179435:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.179438:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.179439:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.179440:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.179441:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.179442:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.179442:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.179442:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.179443:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.179445:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:21.2:1619677448.179446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:31.0:1619677448.179447:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.179449:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.179449:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.179451:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.179452:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718309] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.179453:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.179454:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.179454:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:31.0:1619677448.179455:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.179457:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.179457:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:31.0:1619677448.179459:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.179459:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.179460:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.179460:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.179462:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.179462:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.179464:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.179466:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.179466:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:31.0:1619677448.179468:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.179468:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:22.0:1619677448.179468:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.179469:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.179470:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:22.0:1619677448.179470:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.179471:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:22.0:1619677448.179471:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:31.0:1619677448.179472:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:6.0:1619677448.179472:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.179473:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:6.0:1619677448.179473:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:31.0:1619677448.179474:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:22.0:1619677448.179475:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.179475:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.2:1619677448.179476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.179478:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:20.2:1619677448.179480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:22.0:1619677448.179481:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.179482:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.179483:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:5.0:1619677448.179483:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:22.0:1619677448.179485:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.179486:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.179486:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.0:1619677448.179486:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677448.179487:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:5.0:1619677448.179487:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.179488:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.179489:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.179489:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.179490:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.179491:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677448.179491:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677448.179493:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.179497:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.179499:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.179500:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.179501:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.179503:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.179506:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677448.179549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.179556:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.179559:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.179561:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.179563:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.179564:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.179565:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.179567:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.179568:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.179569:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.179570:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.179573:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.179575:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.179578:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.179579:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.179579:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.179581:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.179581:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.179582:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.179583:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.179595:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.179598:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.179598:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.179599:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.179601:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.179604:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.179789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.2:1619677448.179797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba2600] (20)++
00000800:00000200:10.0:1619677448.179797:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.179800:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.179800:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.179802:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.179803:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba2600] (21)++
00000400:00000200:10.0:1619677448.179805:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.179806:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.5.26@o2ib3
00000800:00000200:18.0:1619677448.179807:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba2600] (22)--
00000400:00000200:10.0:1619677448.179807:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.179808:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.179809:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.179810:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.179810:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.26@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.179811:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.179813:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.179815:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.179817:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.179818:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677448.179819:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.179820:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.179821:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:2.0:1619677448.179821:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.179822:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677448.179822:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.179823:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:10.0:1619677448.179824:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.179824:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.179825:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.179825:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:2.0:1619677448.179826:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.26@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.179828:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 976 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:9.2:1619677448.179830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.179831:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.179831:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:4.2:1619677448.179832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677448.179833:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:2.0:1619677448.179835:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:2.0:1619677448.179836:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba2600] (21)++
00000800:00000200:2.0:1619677448.179836:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba2600] (22)--
00000800:00000200:2.0:1619677448.179837:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba2600] (21)--
00000800:00000200:18.0:1619677448.179838:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.179839:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.179840:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.179841:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.179841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:2.0:1619677448.179841:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.179842:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:18.0:1619677448.179843:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.179843:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.179845:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.179847:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677448.179848:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:26.0:1619677448.179849:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.179850:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:26.0:1619677448.179851:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.179851:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.179852:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.179853:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.179854:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.179854:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:26.0:1619677448.179855:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.179856:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.179856:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:26.0:1619677448.179860:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.179862:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.179864:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.179865:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.179866:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.179867:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.179868:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.179869:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.179870:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.179871:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.179877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.179881:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.179883:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.179884:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.179885:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.179885:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.179886:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.179886:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.179887:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.179888:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.179890:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677448.179900:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.179903:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.179904:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.179904:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.179905:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.179905:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.180010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.180018:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.2:1619677448.180020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:31.0:1619677448.180021:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.180023:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.180025:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.180026:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.180026:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:31.0:1619677448.180027:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677448.180029:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.180029:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:31.0:1619677448.180030:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.180031:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.180031:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:31.0:1619677448.180032:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:8.0:1619677448.180032:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.180035:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.180035:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.180037:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:8.0:1619677448.180037:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.180039:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96718302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.180040:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.180041:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677448.180041:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.180042:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677448.180042:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:8.0:1619677448.180042:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.180044:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677448.180045:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:8.0:1619677448.180045:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.180046:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.180046:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677448.180047:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.180050:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.180051:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.180051:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.180053:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.180053:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.180054:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.180056:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.180056:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677448.180058:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.180060:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:8.2:1619677448.180060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.180061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:6.0:1619677448.180061:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.180062:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.180064:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.180066:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677448.180069:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:5.0:1619677448.180069:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:24.0:1619677448.180071:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.180072:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677448.180072:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:24.0:1619677448.180073:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:5.0:1619677448.180073:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.180074:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.180075:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:6.0:1619677448.180076:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.0:1619677448.180077:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677448.180079:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.180079:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.180080:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.180080:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.180081:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.180092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.180095:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.180097:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.180100:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.180100:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.180102:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.180103:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677448.180105:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.180106:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.180107:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.180108:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677448.180112:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.180114:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677448.180117:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.180117:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677448.180118:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.180120:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677448.180120:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.180122:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.180123:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.180124:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.2:1619677448.180129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677448.180133:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:5.0:1619677448.180134:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:30.0:1619677448.180135:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.180136:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.180136:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.180137:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:30.0:1619677448.180138:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.180138:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677448.180139:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.180141:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.180238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.180246:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.180248:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.180249:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.180251:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.180254:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.180255:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.180257:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.180258:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.180259:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.180260:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.180264:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.180265:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.180268:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.180268:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.180269:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.180271:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.180271:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.180272:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.180272:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.180283:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.180285:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.180286:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.180287:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.180289:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.180290:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.180322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677448.180329:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677448.180332:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677448.180333:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677448.180334:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.180337:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.180338:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677448.180340:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.180341:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.180342:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.180342:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677448.180345:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.180346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:8.0:1619677448.180347:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677448.180349:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.180350:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:8.0:1619677448.180350:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:22.0:1619677448.180352:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677448.180352:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:8.0:1619677448.180352:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677448.180353:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:22.0:1619677448.180355:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:8.0:1619677448.180355:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:6.0:1619677448.180355:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677448.180356:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:22.0:1619677448.180357:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.180360:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.180361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (21)++
00000400:00000200:22.0:1619677448.180362:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.180363:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.180365:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.180366:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.180367:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:24.0:1619677448.180368:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:24.0:1619677448.180370:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000400:00000200:22.0:1619677448.180370:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.180371:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:22.0:1619677448.180371:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:24.0:1619677448.180372:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:30.0:1619677448.180373:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:22.0:1619677448.180374:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.180375:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:22.0:1619677448.180375:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000400:00000200:30.0:1619677448.180376:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.180376:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:30.0:1619677448.180378:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.180378:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.180378:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.180379:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:31.0:1619677448.180380:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:22.0:1619677448.180381:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.180381:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677448.180382:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.180387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.180390:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:6.0:1619677448.180392:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:26.0:1619677448.180393:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.180394:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.180394:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.180394:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.180394:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.180395:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.2:1619677448.180395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:26.0:1619677448.180396:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.180400:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:26.0:1619677448.180402:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.180404:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.180405:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.180406:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000400:00000200:26.0:1619677448.180407:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.180408:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.180409:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.180409:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.180410:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.180410:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.180411:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:26.0:1619677448.180412:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.180413:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.180414:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.180415:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.180416:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:5.2:1619677448.180416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677448.180419:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.180421:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.180423:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.180423:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.180424:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.180425:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:1.0:1619677448.180425:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:26.0:1619677448.180426:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:17.0:1619677448.180426:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.180427:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.180428:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:1.0:1619677448.180428:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.180429:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.180430:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:1.0:1619677448.180430:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.180431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000400:00000200:1.0:1619677448.180431:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.180432:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.180433:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.180434:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.180435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:1.0:1619677448.180435:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.180439:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.180440:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:1.0:1619677448.180441:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.180442:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:31.0:1619677448.180443:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:10.0:1619677448.180443:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:1.0:1619677448.180443:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.180444:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677448.180445:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:20.0:1619677448.180445:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:10.0:1619677448.180445:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:5.0:1619677448.180445:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.180445:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:31.0:1619677448.180446:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.180446:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.180447:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:5.0:1619677448.180447:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.180447:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.180447:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.180448:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.180448:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.180448:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.180449:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:5.0:1619677448.180449:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.180450:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.180451:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:5.0:1619677448.180452:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.180452:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.180454:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.180457:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.180459:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677448.180460:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:21.2:1619677448.180460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.0:1619677448.180462:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.180463:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:5.0:1619677448.180463:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000400:00000200:22.0:1619677448.180464:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.180464:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:5.0:1619677448.180464:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:22.0:1619677448.180466:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.180466:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.180467:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.180467:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.180468:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:30.0:1619677448.180469:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677448.180469:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:30.0:1619677448.180472:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.2:1619677448.180472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000400:00000200:30.0:1619677448.180474:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.180476:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.180477:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.180478:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.180479:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.180480:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:30.0:1619677448.180481:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.180482:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.180482:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:30.0:1619677448.180483:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.180483:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.180485:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.180486:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:30.0:1619677448.180487:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.180487:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:30.0:1619677448.180489:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.180491:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.180493:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.180493:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.180495:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.180496:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.180497:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.180499:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.180500:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677448.180505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.180509:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.180511:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.180512:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.180513:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.180514:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.180516:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.180516:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677448.180516:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.180517:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.180518:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677448.180618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.180625:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.180628:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.180628:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.180629:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.180631:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.180632:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.180633:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.180634:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.180635:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.180636:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.180638:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.180639:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.180640:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.180641:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.180642:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.180642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:26.0:1619677448.180644:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.180644:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.180645:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.180646:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.180646:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677448.180648:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:26.0:1619677448.180649:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.180650:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.180651:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:26.0:1619677448.180652:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.180652:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:10.0:1619677448.180652:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.180653:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.180653:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.180653:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.180654:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:2.0:1619677448.180654:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.180655:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.180655:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.180656:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)++
00000800:00000200:10.0:1619677448.180656:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)++
00000800:00000200:26.0:1619677448.180657:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (25)--
00000800:00000200:10.0:1619677448.180657:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (25)--
00000400:00000200:2.0:1619677448.180657:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.180658:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.180658:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.180659:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:2.0:1619677448.180659:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.180660:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96718291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.180661:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.180662:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.180663:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.180666:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:2.0:1619677448.180666:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.180668:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:2.0:1619677448.180668:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.180669:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.180670:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:10.0:1619677448.180670:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.180671:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.180672:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.180672:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.180673:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:2.0:1619677448.180673:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.180674:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.180674:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:31.0:1619677448.180675:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.180675:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.180675:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.180676:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:30.0:1619677448.180677:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:2.0:1619677448.180677:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:31.0:1619677448.180678:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.180679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.180685:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:13.0:1619677448.180687:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.180688:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.180688:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.180690:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.180690:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.180690:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.180692:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.180692:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.180693:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.180693:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000400:00000200:18.0:1619677448.180694:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.180695:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.180696:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.180697:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:18.0:1619677448.180697:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.180698:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.180698:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (21)++
00000400:00000200:18.0:1619677448.180699:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.180699:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.180699:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:13.0:1619677448.180701:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:18.0:1619677448.180703:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.180704:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.2:1619677448.180705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677448.180707:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.180708:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.180708:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.180710:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.180711:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677448.180711:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677448.180712:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.180712:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677448.180713:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.180716:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.180716:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.180718:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.180719:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677448.180721:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.180722:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:5.0:1619677448.180722:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.180723:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.180724:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:5.0:1619677448.180724:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.180725:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.180725:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:5.0:1619677448.180728:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.180729:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.180731:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.180732:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.0:1619677448.180733:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.180734:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.180735:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.180737:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.180737:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.180738:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.180739:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.180740:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677448.180743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677448.180745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.180748:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.180750:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.180750:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677448.180750:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:20.0:1619677448.180751:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677448.180753:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.180754:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.180754:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.180756:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.180758:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.180816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.180823:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.180826:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.180827:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.180829:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.180831:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.180832:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.180834:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.180835:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.180836:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.180837:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.180840:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.180842:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.180844:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.180845:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.180846:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.180847:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.180848:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.180849:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.180849:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.180860:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.180862:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.180863:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.180864:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.180866:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.180867:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677448.180877:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.180879:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.180880:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.180881:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.180881:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.180882:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.180890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.180897:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.180899:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.180901:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.180904:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.180904:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.180905:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.180906:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.180907:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.180908:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.180909:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.180912:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.180913:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.180916:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.180916:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.180917:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.180919:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.180919:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.180920:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.180920:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.180931:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.180933:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.180934:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.180935:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.180937:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.180938:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.180946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:13.0:1619677448.180947:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.180950:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.180950:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.180951:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.180951:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677448.180951:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677448.180952:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677448.180954:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.180954:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.180956:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.180959:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.180961:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.180962:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.180964:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.180965:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.180966:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.180969:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.180971:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.180973:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.180974:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.180975:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.180977:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.180977:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.180978:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.180978:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.180983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.180990:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.180993:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.180994:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.180994:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.180996:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.180999:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.181001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.181008:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.181011:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.181011:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.181014:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.181016:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.181017:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.181019:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.181020:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.181021:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.181022:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.2:1619677448.181024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677448.181025:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.181027:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.181029:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.181030:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.181030:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.181031:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.181032:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.181033:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.181033:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:17.0:1619677448.181033:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.181033:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:26.0:1619677448.181034:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:1.0:1619677448.181035:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.181037:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.181039:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.181040:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718284] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.181041:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677448.181042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:1.0:1619677448.181042:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.181043:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:13.0:1619677448.181044:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677448.181046:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.181047:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.181048:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.181048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677448.181048:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:13.0:1619677448.181049:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.181049:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:12.0:1619677448.181051:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.181051:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.181052:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677448.181052:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:1.0:1619677448.181052:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:24.0:1619677448.181053:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677448.181053:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.181055:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:8.0:1619677448.181055:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.181055:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.181055:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.181056:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:8.0:1619677448.181057:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.181057:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:10.0:1619677448.181058:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:1.0:1619677448.181058:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.181059:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:8.0:1619677448.181059:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.181060:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96718283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.181061:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.181062:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.181063:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.181063:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.181063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677448.181064:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:10.0:1619677448.181065:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.181066:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96718282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.181067:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.181067:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.181068:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.181068:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:10.0:1619677448.181068:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.181069:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677448.181069:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.181070:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677448.181071:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677448.181071:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677448.181072:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.181072:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677448.181072:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.181072:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:10.0:1619677448.181073:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.181074:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.181074:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:10.0:1619677448.181075:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677448.181075:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.181076:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.181077:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:10.0:1619677448.181077:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.181077:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.181077:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:10.0:1619677448.181078:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677448.181078:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677448.181079:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.181080:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.181081:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.181082:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.181083:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.181083:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.181083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.181089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.181090:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.181093:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.181094:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.181094:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:1.0:1619677448.181094:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.181095:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.181097:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.181097:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:1.0:1619677448.181097:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.181098:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.181098:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.181098:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.181099:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677448.181100:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.181100:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.181103:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.181131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.181137:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.181140:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.181140:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.181142:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.181145:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.181146:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677448.181147:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.181149:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.181150:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.181151:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677448.181154:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.181156:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.181157:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.181158:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.181159:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.181160:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.181161:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.181162:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.181162:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.181166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.181173:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.181175:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.181176:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.181177:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.181178:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.181180:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.181181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.181188:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.181190:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.181192:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.181193:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.181196:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.181197:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.181199:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.181200:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.181201:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.181202:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.181205:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.181207:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.181210:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.181211:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.181212:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.181214:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.181214:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.181215:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.181215:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.181220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.181220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:18.0:1619677448.181227:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.181228:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677448.181230:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:10.0:1619677448.181230:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.181231:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:18.0:1619677448.181231:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.181231:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.181234:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.181234:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.181237:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.181238:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:10.0:1619677448.181239:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.181240:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.181241:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.181243:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.181243:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.181246:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.181248:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.181250:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.181250:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.181251:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.181252:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.181253:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.181254:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.181254:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.181272:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.181275:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.181276:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.181277:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.181279:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.181280:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.181415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.181420:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.181423:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.181424:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.181426:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.181428:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.181429:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.181431:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.181432:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.181433:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.181434:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.181438:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.181439:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.2:1619677448.181441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677448.181442:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.181443:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.181444:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.181446:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.181446:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.181447:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.181447:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.181450:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.181452:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.181455:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.181457:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.181457:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.181458:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.181460:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.181460:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:31.0:1619677448.181461:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.181462:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.181462:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:31.0:1619677448.181463:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:12.0:1619677448.181463:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.181465:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.181467:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.181467:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.181469:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.181470:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.181471:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.181472:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.181472:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.181474:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.181474:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.181475:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.181476:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:21.2:1619677448.181480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:13.0:1619677448.181485:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.0:1619677448.181486:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.181488:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:13.0:1619677448.181488:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.181489:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.181491:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.181491:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.181492:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677448.181492:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.181494:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.181494:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:20.0:1619677448.181495:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.181497:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718276] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.181498:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.181499:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.181500:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.181503:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.181503:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.181505:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:12.0:1619677448.181506:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.181507:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.181507:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.0:1619677448.181508:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:12.0:1619677448.181508:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.181508:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677448.181509:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.181510:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:12.0:1619677448.181510:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677448.181511:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.181512:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677448.181513:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677448.181513:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:8.2:1619677448.181517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:21.2:1619677448.181520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.181526:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.181527:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677448.181527:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.181528:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000400:00000200:5.0:1619677448.181528:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.181529:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.181530:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.181530:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.181531:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677448.181532:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:21.2:1619677448.181603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.181610:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.181613:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.181615:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.181617:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.181617:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.181618:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.181620:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.181621:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.181622:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.181623:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.181626:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.181628:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.181630:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.181631:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.181631:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.181633:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:4.2:1619677448.181633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.181634:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.181635:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.181635:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.181643:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.181644:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.181646:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:26.0:1619677448.181647:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.181647:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.181648:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.181649:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.181650:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.181651:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.181652:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.181654:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.181655:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.181655:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.181658:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.181659:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.181659:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.181660:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.181661:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.181664:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.181665:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.181666:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.181667:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.181668:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.181668:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.181669:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.181674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.181675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.181681:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677448.181682:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:22.0:1619677448.181684:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:20.0:1619677448.181684:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.181685:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.181685:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.181686:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.181687:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.181688:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:22.0:1619677448.181689:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.181690:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:20.0:1619677448.181690:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:22.0:1619677448.181692:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.181693:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.181694:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.181695:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.181699:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.181700:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.181703:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.181704:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.181704:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.181706:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.181706:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.181707:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.181708:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.181712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.181720:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.181723:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.181724:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.181724:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.181726:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.181728:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.181766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.181772:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.181775:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.181775:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.181778:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.181780:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.181782:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.181783:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.181784:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.181785:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.181787:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.181790:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.181792:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.181795:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.181795:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.181796:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.181798:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.181798:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.181799:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.181800:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.181804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:21.2:1619677448.181809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.181814:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.181816:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.181817:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:8.0:1619677448.181818:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.181819:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677448.181820:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.181822:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.181824:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.181825:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.181826:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:8.0:1619677448.181826:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:24.0:1619677448.181827:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.181827:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.181829:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.181830:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.181831:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677448.181835:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.181837:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677448.181839:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.181840:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677448.181840:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677448.181842:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677448.181842:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.181843:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.181844:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677448.181854:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.181856:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.181857:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.181859:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.181860:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.181861:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.181869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.181875:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.181878:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.181880:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.181880:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.181882:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.181883:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.181885:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.181886:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.181887:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.181888:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.181892:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.181894:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.181896:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.181897:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.181898:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.181900:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.181900:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.181901:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.181902:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.181912:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.181914:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.181915:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.181917:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.181918:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.181920:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.181975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.181981:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.181984:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.181985:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.181987:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.181989:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.181991:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.181992:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.181993:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.181994:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.181995:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.181999:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.182000:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.182003:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.182004:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.182005:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.182007:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.182007:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.182008:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.182008:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677448.182019:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.182022:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.182023:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.182024:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.182025:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.182027:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.182039:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.182042:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.182043:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.182044:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.182044:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:4.2:1619677448.182044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677448.182046:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.182051:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.182054:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.182055:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.182057:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.182059:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.182061:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.182062:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.182063:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.182064:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.182065:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.182069:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.182069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.182071:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.182074:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.182075:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.182075:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.182075:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.182077:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:10.0:1619677448.182077:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.182077:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.182078:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677448.182078:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.182079:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:17.0:1619677448.182080:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.182083:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.182084:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.182085:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.182087:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.182088:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.182089:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:6.0:1619677448.182091:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:17.0:1619677448.182092:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.182093:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.182094:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:17.0:1619677448.182094:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:6.0:1619677448.182095:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.182096:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.182097:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.182097:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:6.0:1619677448.182097:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.182099:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.182099:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.182100:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.182100:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677448.182101:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.182105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.182112:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.182114:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.182115:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.182116:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.182117:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.182119:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.182248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.182255:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.182259:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.182261:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.182262:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.182263:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.182264:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.182266:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.182267:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.182268:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.182269:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.182272:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.182274:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.182276:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.182277:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.182277:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.182279:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.182279:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.182280:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.182281:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.182291:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.182298:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.182299:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.182300:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677448.182300:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.182302:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.182327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.182333:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.182336:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.182337:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.182338:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.182340:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.182342:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.182343:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.182344:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.182345:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.182346:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.182349:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.182351:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.182354:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.182355:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.182355:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.182357:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.182358:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.182359:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.182360:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.182361:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.182365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.182371:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.182371:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.182373:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.182373:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.182374:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677448.182374:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677448.182375:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.182377:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.182378:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.182379:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.182412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.182420:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.182422:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.182423:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.182425:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.182427:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.182429:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.182430:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.182431:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.182432:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.182433:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.2:1619677448.182436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:8.0:1619677448.182436:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.182438:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.182440:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677448.182440:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.182441:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:20.0:1619677448.182442:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:8.0:1619677448.182442:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.182443:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:20.0:1619677448.182444:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.182444:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.182444:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.182445:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677448.182446:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.182447:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.182448:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.182450:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.182451:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.182452:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.182453:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.182455:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:20.0:1619677448.182456:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.182457:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:20.0:1619677448.182458:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:12.0:1619677448.182458:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.182460:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.182461:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.182462:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.182462:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677448.182463:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.182463:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677448.182464:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.182465:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.182466:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.182467:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677448.182476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677448.182477:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.182479:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.182480:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.182482:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.182483:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.182484:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677448.182484:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.182486:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.182487:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.182489:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.182492:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.182493:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.182494:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.182496:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.182497:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.182498:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.182501:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.182502:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.182505:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.182505:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.182506:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.182507:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.182508:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.182509:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.182509:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.182520:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.182522:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.182523:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.182524:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.182526:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.182527:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.182536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.182543:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.182546:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.182548:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.182549:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.182550:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.182551:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.182552:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.182553:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.182554:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.182555:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.182557:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.182559:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.182561:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.182562:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.182562:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.182564:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.182564:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.182565:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.182566:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.182576:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.182578:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.182579:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.182579:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.182581:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.182583:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.182622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.182629:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.182631:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.182633:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.182634:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.182637:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.182638:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677448.182639:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.182641:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.182642:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.182643:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677448.182646:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.182648:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677448.182651:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.182651:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677448.182652:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677448.182654:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677448.182654:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.2:1619677448.182655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.182655:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.182655:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677448.182661:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677448.182662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:17.0:1619677448.182663:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.182664:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.182665:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:17.0:1619677448.182666:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.182668:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.0:1619677448.182668:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:17.0:1619677448.182669:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.182670:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:5.0:1619677448.182670:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.182671:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.182671:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:17.0:1619677448.182671:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:6.0:1619677448.182671:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:21.2:1619677448.182672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:17.0:1619677448.182673:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.182674:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000400:00000200:17.0:1619677448.182674:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.182675:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.182675:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.182676:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.182677:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:5.0:1619677448.182677:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:22.0:1619677448.182678:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:5.0:1619677448.182678:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.182680:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.182681:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:17.0:1619677448.182681:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.182682:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:5.0:1619677448.182682:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.182683:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:17.0:1619677448.182684:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:22.0:1619677448.182685:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.182687:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.182687:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.182687:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.182688:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:22.0:1619677448.182689:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:17.0:1619677448.182689:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:22.0:1619677448.182690:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96718257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.182690:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677448.182691:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.182691:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.182692:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.182692:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677448.182692:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.182693:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.182693:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000400:00000200:22.0:1619677448.182694:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.182694:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:22.0:1619677448.182695:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.182696:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.182696:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.182697:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.182698:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:22.0:1619677448.182699:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.182701:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.2:1619677448.182702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:12.0:1619677448.182703:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:22.0:1619677448.182704:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.182705:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.182706:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:12.0:1619677448.182706:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.182707:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.182708:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.182708:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.182708:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.182709:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677448.182709:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:12.0:1619677448.182709:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.182710:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.182711:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677448.182712:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.182713:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.182714:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.182716:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.182717:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677448.182718:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.182719:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.182720:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (23)++
00000800:00000200:5.0:1619677448.182720:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:31.0:1619677448.182721:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.182721:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677448.182722:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.182723:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:31.0:1619677448.182724:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.182725:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.182726:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.182728:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.182729:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.182740:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.182743:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.182744:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.182745:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.182745:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.182747:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.182942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.182948:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.182951:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.182951:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.182954:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.182956:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.182958:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.182959:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.182960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:1.0:1619677448.182960:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.182961:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.182962:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.182966:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677448.182966:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.182967:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.182968:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677448.182969:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.182970:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.182971:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.182971:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.182971:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.182972:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:5.0:1619677448.182973:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.182973:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.182973:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677448.182974:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:1.0:1619677448.182974:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:5.0:1619677448.182976:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.182977:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677448.182978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677448.182978:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.182979:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677448.182982:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.182984:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:17.0:1619677448.182985:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677448.182987:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.182987:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:17.0:1619677448.182988:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.182988:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.182988:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:1.0:1619677448.182989:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.182990:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677448.182990:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:17.0:1619677448.182991:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.182991:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.182992:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677448.182993:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:9.2:1619677448.182996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.183003:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.183006:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.183007:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.183007:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.183009:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.183012:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.183015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.183022:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.183024:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.183025:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.183025:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.183026:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.183027:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.183068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.183074:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.183076:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.183077:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.183079:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.183082:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.183083:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.183084:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718254] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.183086:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.183087:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.183088:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.183090:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.183092:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.183094:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.183095:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.183096:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.183097:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.183098:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.183099:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.183100:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.183100:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.183103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.183105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.183111:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:1.0:1619677448.183111:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677448.183113:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.183113:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.183114:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677448.183114:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.183115:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.183115:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.183116:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.183119:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.183123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.183130:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.183133:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.183133:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.183135:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.183138:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.183140:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.183141:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.183142:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.183143:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.183144:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.183147:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.183149:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.183152:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.183153:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.183153:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.183155:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.183155:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.183156:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.183156:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.183161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.183169:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.183171:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.183172:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.183172:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.183174:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.183176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.183179:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.183180:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.183182:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.183183:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.183184:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.183184:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.183185:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.183222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.183230:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.183233:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.183235:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.183236:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.183237:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.183238:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.183240:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.183241:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.183242:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.183243:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.183246:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.183247:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.183250:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.183251:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.183252:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.183254:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.183254:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.183255:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.183256:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.183260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.183267:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.183270:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.183271:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.183273:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.183274:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.183276:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.183281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.183288:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.183291:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.183292:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.183292:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.183294:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.2:1619677448.183299:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.0:1619677448.183303:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:26.0:1619677448.183307:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.183310:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.183310:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.183312:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.183314:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.183316:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.183317:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.183319:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.183320:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.183321:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.183324:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.183326:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.183329:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.183330:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.183330:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.183332:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.183332:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.183333:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.183334:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677448.183347:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.183350:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.183351:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.183351:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.183353:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.183359:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.183361:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.183363:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.183363:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.183363:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.183364:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.183365:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.183409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.2:1619677448.183414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.183417:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.183420:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:1.0:1619677448.183421:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677448.183422:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.183423:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677448.183424:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677448.183424:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677448.183425:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.183427:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.183427:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.183429:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.183429:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.183430:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677448.183431:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.183431:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.183432:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.183433:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.183433:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.183435:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.183436:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677448.183439:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.183440:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.183441:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.183443:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.183443:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677448.183444:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.183444:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.183446:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.183446:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.183447:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.183447:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.183447:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:30.0:1619677448.183448:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:1.0:1619677448.183449:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.183449:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.183450:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:1.0:1619677448.183450:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.183451:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677448.183451:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.183455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:21.2:1619677448.183456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:4.2:1619677448.183458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677448.183460:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.183462:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.183463:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.183463:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:31.0:1619677448.183465:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:13.0:1619677448.183465:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.183465:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677448.183465:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.183466:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.183466:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.183467:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:12.0:1619677448.183467:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.183468:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677448.183468:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.183468:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:8.0:1619677448.183468:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.183469:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.183469:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.183471:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.183472:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:10.0:1619677448.183473:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.183475:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.183476:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.183477:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.183478:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.183479:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.183482:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.183484:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.183487:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.183488:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.183489:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.183490:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.183491:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.183492:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.183493:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.183494:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.183499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.183504:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:26.0:1619677448.183506:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.183507:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:26.0:1619677448.183508:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677448.183508:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.183508:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.183509:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.183510:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.183511:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.183514:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.183538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677448.183543:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.183546:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677448.183547:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677448.183549:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.183552:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.183554:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677448.183555:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718247] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.183556:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.183558:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.183559:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677448.183562:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.183563:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677448.183566:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.183567:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:17.0:1619677448.183568:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:17.0:1619677448.183569:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:17.0:1619677448.183570:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.183571:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677448.183573:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677448.183573:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:20.2:1619677448.183574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.183577:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.183579:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:5.2:1619677448.183579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:20.0:1619677448.183581:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677448.183581:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:20.0:1619677448.183584:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.183584:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.183585:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.183585:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.183585:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:1.0:1619677448.183585:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000400:00000200:20.0:1619677448.183586:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677448.183587:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.183587:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677448.183588:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:10.0:1619677448.183589:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.183589:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:20.0:1619677448.183590:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.183591:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.183592:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.183592:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:20.0:1619677448.183595:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.183597:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.183600:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.183601:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.183602:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:20.0:1619677448.183602:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (22)++
00000800:00000200:20.0:1619677448.183604:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (24)--
00000800:00000200:26.0:1619677448.183606:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.183606:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:10.0:1619677448.183606:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.183607:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)++
00000800:00000200:20.0:1619677448.183607:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.183607:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:26.0:1619677448.183608:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.183608:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.183616:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.183618:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.183619:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.183619:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.183621:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:10.0:1619677448.183624:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.183634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.0:1619677448.183638:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.183641:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677448.183643:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.183643:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.183645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:5.0:1619677448.183645:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.183647:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:5.0:1619677448.183647:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:20.0:1619677448.183648:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.183648:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718245] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.183649:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.183651:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.183651:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.183654:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.183656:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.183657:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.183659:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.183659:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000400:00000200:20.0:1619677448.183660:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.183660:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:20.0:1619677448.183661:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:5.0:1619677448.183661:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000400:00000200:20.0:1619677448.183662:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.183664:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.183665:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.183666:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.183667:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.183667:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:8.2:1619677448.183668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000400:00000200:20.0:1619677448.183669:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.183669:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677448.183671:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.183671:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.183672:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.183673:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.183673:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.183673:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.183674:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.183675:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:5.0:1619677448.183675:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.183676:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.183677:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.183677:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.183678:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.183678:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677448.183680:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.183680:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.183685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.183690:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.183691:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:6.0:1619677448.183692:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677448.183693:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:22.0:1619677448.183694:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.183694:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.183694:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.183695:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:6.0:1619677448.183696:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.183698:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.183760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.183769:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.183771:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.183774:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.183775:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.183776:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.183778:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.183779:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.183780:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.183781:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.183782:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.183785:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.183787:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.183790:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.183791:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.183791:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.183793:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.183794:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.183795:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.183797:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.183797:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.183799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.183803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.183807:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.183810:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:30.0:1619677448.183811:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:5.0:1619677448.183811:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.183812:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.183813:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.183814:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.183815:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677448.183815:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:30.0:1619677448.183816:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.183822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.2:1619677448.183827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677448.183830:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.183833:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.183833:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.183834:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.183835:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.183835:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677448.183835:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.183836:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.183837:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:17.0:1619677448.183838:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.183841:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.183842:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.183844:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.183845:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.183846:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.183847:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.183851:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.183852:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.183855:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.183856:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.183857:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.183859:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.183859:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.183860:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.183861:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.183862:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.183866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.183871:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677448.183872:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677448.183874:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677448.183874:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.183875:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677448.183875:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.183875:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677448.183877:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.183878:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.183880:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.183902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677448.183910:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.183912:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677448.183913:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677448.183915:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.183918:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.183919:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.183921:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.183922:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.183923:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.183924:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.183927:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.183928:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.183931:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.183932:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.183932:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.183934:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.183935:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.183935:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677448.183936:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677448.183940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.183948:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.183951:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.183952:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.183952:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.183955:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.183958:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.183962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.183969:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.183972:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.183972:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.183973:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.183974:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.183974:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.184085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:21.2:1619677448.184090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:10.0:1619677448.184092:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.184095:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.184096:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.184097:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:2.0:1619677448.184097:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.184099:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:10.0:1619677448.184100:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.184101:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:10.0:1619677448.184102:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.184103:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.184104:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.184105:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.184106:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.184106:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.184107:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.184108:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677448.184109:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.184110:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.184112:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.184113:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.184118:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000400:00000200:2.0:1619677448.184118:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.184120:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.184122:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.184123:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.184124:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.184125:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.184125:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.184125:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.184126:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.184126:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.184127:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.184128:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.184128:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.184128:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:10.0:1619677448.184129:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.184130:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677448.184131:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.184132:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677448.184132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.184137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:5.0:1619677448.184140:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.184143:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677448.184144:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:5.0:1619677448.184144:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.184145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677448.184145:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.184146:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.184147:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.184148:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.184148:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:5.0:1619677448.184148:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677448.184153:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.184156:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.184157:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.184157:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.184159:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.184162:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.184166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.184173:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.184176:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.184177:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.184177:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.184178:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.184179:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.184341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.184350:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.184352:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.184353:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.184355:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.184358:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.184359:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.184361:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.184362:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.184363:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.184364:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.184367:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.184368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:26.0:1619677448.184369:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.184372:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.184372:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.184373:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.184374:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.184375:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.184376:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:30.0:1619677448.184377:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.184377:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.184378:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.184380:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.184382:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.184384:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.184384:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.184384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:30.0:1619677448.184386:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.184387:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.184388:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.184388:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:30.0:1619677448.184389:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.184390:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:21.2:1619677448.184390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:13.0:1619677448.184391:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:10.0:1619677448.184391:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:13.0:1619677448.184392:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.184393:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677448.184393:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.184393:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.184394:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.184395:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.2:1619677448.184395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:13.0:1619677448.184396:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.184396:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.184397:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.184398:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.184398:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.184398:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.184399:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677448.184399:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.184399:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.184401:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.184401:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.184401:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:31.0:1619677448.184402:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.184402:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.184403:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.184404:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677448.184404:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.184404:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677448.184405:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.184406:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.184406:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.184407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677448.184408:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.184408:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:13.0:1619677448.184408:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.184410:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.184410:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.184411:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.184411:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.184411:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:20.0:1619677448.184412:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.184412:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:31.0:1619677448.184413:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.184413:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677448.184414:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.184414:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:20.0:1619677448.184415:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.184415:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.184416:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.184417:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.184418:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.184419:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.184421:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.184422:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.184422:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.184423:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677448.184424:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.184424:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.184425:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.184426:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.184426:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.184427:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.184428:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.184428:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.184428:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.184429:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.184430:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677448.184430:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.184430:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.184431:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.184432:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.184439:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.184442:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.184443:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.184445:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:12.0:1619677448.184445:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.184446:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677448.184447:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:12.0:1619677448.184447:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:22.0:1619677448.184448:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.184449:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.184450:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.184452:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677448.184461:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.184464:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.184464:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.184465:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.184466:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.184466:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.184506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.184513:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.184516:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.184517:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.184519:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.184521:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.184523:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.184524:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.184525:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.184526:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.184527:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.184531:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.184532:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.184536:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.184536:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.184537:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.184539:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.184539:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.184539:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.184540:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.184552:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.184555:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.184555:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.184556:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.184558:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.184561:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.184565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.184571:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.184575:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.184575:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.184577:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.184579:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.184581:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.184582:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.184583:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.184584:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.184586:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.184589:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.184591:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.184593:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.184594:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.184594:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.184596:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.184596:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.184597:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.184597:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.184602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.184609:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.184611:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.184612:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.184613:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.184614:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.184615:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.184643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.0:1619677448.184647:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.184649:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677448.184652:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.184652:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.184654:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.184655:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.184656:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.184657:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.184659:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.184659:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.184663:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.184664:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.184666:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.184667:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.184668:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.184669:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.184669:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.184670:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.184671:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.184675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.184682:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.184684:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.184685:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.184687:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.184688:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.184689:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:21.2:1619677448.184954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.184962:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.184965:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.184967:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.184969:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.184969:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.184971:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.184972:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.184973:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.184974:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.184975:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.184979:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.184981:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.184984:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.184985:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.184985:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.184987:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.184988:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.184989:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.184989:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677448.185001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:6.0:1619677448.185001:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.185003:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.185004:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.185005:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.185007:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:6.0:1619677448.185007:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.185010:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:6.0:1619677448.185010:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677448.185010:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.185012:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.185014:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.185016:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.185017:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.185018:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.185019:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.185020:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:21.2:1619677448.185021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:18.0:1619677448.185024:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.185025:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:22.0:1619677448.185026:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.185029:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:18.0:1619677448.185029:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.185029:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677448.185030:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.185031:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:22.0:1619677448.185032:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.185032:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.185033:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.185033:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.185034:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:22.0:1619677448.185035:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.185037:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.185038:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.185040:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.185041:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.185042:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.185045:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.185045:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:22.0:1619677448.185047:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.185047:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.185048:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:22.0:1619677448.185049:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.185049:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.185050:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.185051:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:10.0:1619677448.185051:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.185053:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.185053:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.185054:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:10.0:1619677448.185054:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:22.0:1619677448.185055:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.185056:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.185059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.185061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.185066:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:1.0:1619677448.185067:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677448.185068:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.185069:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.185070:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677448.185070:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.185071:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.185071:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.185073:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.185076:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.185112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.185120:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.185122:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.185123:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.185125:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.185128:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.185129:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.185130:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.185132:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.185133:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.185134:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.185137:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.185138:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.185140:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.185140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:26.0:1619677448.185141:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.185141:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.185143:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.185143:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.185144:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.185144:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.185148:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677448.185150:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677448.185153:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.185154:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:10.0:1619677448.185154:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:13.0:1619677448.185155:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.185156:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.185156:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.185157:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:13.0:1619677448.185157:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:13.0:1619677448.185158:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.185159:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:13.0:1619677448.185160:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.185161:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677448.185161:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.185161:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:13.0:1619677448.185164:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.185166:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677448.185169:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.185170:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677448.185171:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677448.185174:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677448.185174:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677448.185175:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677448.185175:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677448.185180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.185187:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.185190:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.185191:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.185191:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.185194:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.185197:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.185238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.185245:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.185248:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.185249:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.185251:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.185252:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.185253:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.185255:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.185256:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.185257:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.185258:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.185261:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.185262:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.185264:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.185264:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.185265:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.185266:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.185267:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.185267:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.185268:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.185277:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.185279:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.185279:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.185280:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.185282:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.185283:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.185320:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.185322:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.185323:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.185323:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.185323:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.185324:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.185356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.185364:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.185367:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.185369:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.185372:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.185372:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.185373:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.185374:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.185376:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.185377:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.185378:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.185381:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.185382:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.185384:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.185385:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.185385:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.185387:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.185387:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.185388:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.185389:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.185400:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.185403:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.185404:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.185405:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.185407:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.185409:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.185636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.185644:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.185647:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.185647:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.185649:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.185651:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.185652:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.185653:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.185654:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.185655:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.185656:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.185659:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.185661:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.185663:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.185664:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.185665:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.185667:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.185667:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.185668:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.185668:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.185673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.185682:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.185685:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.185686:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.185686:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.185688:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.185691:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.185769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.0:1619677448.185773:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.185776:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677448.185778:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.185779:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.185780:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.185782:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.185783:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.185784:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.185785:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.185786:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.185790:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.185792:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.185794:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.185795:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.185795:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.185797:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.185797:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.185798:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677448.185799:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:5.0:1619677448.185800:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:8.2:1619677448.185803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:21.2:1619677448.185806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.185811:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.185812:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677448.185812:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.185812:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.185813:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677448.185814:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.185814:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.185815:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.185816:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.0:1619677448.185817:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:4.2:1619677448.185851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.185858:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.185860:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.185861:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.185863:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.185866:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.185868:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.185869:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.185870:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.185871:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.185872:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.185875:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.185877:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.185880:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.185881:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.185881:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.185883:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.185884:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.185885:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.185885:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.185889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.185897:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.185900:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.185901:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.185901:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.185903:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.185906:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.185907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.2:1619677448.185907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.185914:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.185915:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.185916:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.185917:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.185918:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.185919:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000400:00000200:17.0:1619677448.185919:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.185919:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.185920:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.185921:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:17.0:1619677448.185922:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.185923:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.185925:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.185926:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.185927:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.185928:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.2:1619677448.185931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:17.0:1619677448.185932:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.185933:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:21.2:1619677448.185935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:17.0:1619677448.185936:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.185937:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.185937:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.185939:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.185939:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.185940:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.185940:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.185941:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677448.185941:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:30.0:1619677448.185942:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:2.0:1619677448.185944:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:30.0:1619677448.185945:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.185945:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:31.0:1619677448.185946:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.2:1619677448.185946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:2.0:1619677448.185947:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.185948:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.185949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:20.0:1619677448.185949:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:30.0:1619677448.185950:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.185950:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.185952:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.185952:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:2.0:1619677448.185952:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.185953:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.185955:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.185955:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.0:1619677448.185955:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:2.0:1619677448.185956:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.185957:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.185957:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:20.0:1619677448.185957:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:6.0:1619677448.185957:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677448.185957:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.185958:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.185958:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:30.0:1619677448.185959:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.185959:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:22.0:1619677448.185960:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.185960:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.185962:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.185962:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.185964:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96718218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.185965:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.185965:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.185966:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.185967:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.185967:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.185968:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.185969:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.185969:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.185969:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:22.0:1619677448.185970:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.185971:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677448.185972:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.185972:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:2.0:1619677448.185972:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.185973:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:20.0:1619677448.185973:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.185973:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.185974:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:6.0:1619677448.185974:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96718217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.185974:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.185975:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.185975:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.185975:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.185976:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677448.185976:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.185976:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.185977:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:30.0:1619677448.185978:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677448.185978:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.185978:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.185978:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:20.0:1619677448.185979:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.185980:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:20.0:1619677448.185980:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.185980:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:30.0:1619677448.185981:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.185981:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:6.0:1619677448.185981:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.185981:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.0:1619677448.185982:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.185983:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.185983:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:6.0:1619677448.185983:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.2:1619677448.185985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:8.2:1619677448.185987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.185987:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.185988:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.185988:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.185989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677448.185990:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677448.185991:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:6.0:1619677448.185991:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.185992:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677448.185992:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:6.0:1619677448.185992:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:22.0:1619677448.185993:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.185993:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:6.0:1619677448.185993:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.185994:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.185994:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:6.0:1619677448.185994:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677448.185994:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:10.0:1619677448.185995:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:6.0:1619677448.185995:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.185996:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.185996:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:10.0:1619677448.185996:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.185997:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:9.2:1619677448.185997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:1.0:1619677448.185997:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:22.0:1619677448.185998:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:1.0:1619677448.185998:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.185999:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:17.0:1619677448.185999:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:10.0:1619677448.185999:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.186000:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.186001:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:1.0:1619677448.186001:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:10.0:1619677448.186002:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:2.0:1619677448.186004:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:1.0:1619677448.186004:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677448.186007:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.186008:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.186008:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.186010:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.186013:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.186034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.186039:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.186042:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.186043:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.186045:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.186047:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.186049:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.186050:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.186051:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.186052:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.186053:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.186056:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.186058:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.186061:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.186062:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.186063:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.186065:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.186065:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.186066:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.186066:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677448.186077:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.186080:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.186081:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.186082:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.186084:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.186085:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.186104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.186112:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.186115:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.186115:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.186117:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.186120:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.186121:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.186122:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.186123:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.186124:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.186125:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.186128:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.186129:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.186131:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.186132:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.186133:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.186134:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.186135:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.186135:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.186136:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.186145:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.186147:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.186148:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.186149:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.186150:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.186151:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.186260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.186267:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.186269:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.186270:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.186271:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.186273:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.186274:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.186275:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.186277:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.186277:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.186278:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.186281:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.186282:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.186285:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.186286:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.186286:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.186288:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.186288:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.186289:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.186291:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.186291:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.186299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.186302:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.186305:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.186306:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677448.186306:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:6.0:1619677448.186307:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.186308:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.186309:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.186309:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.186310:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.186312:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.186313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:26.0:1619677448.186320:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677448.186323:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:10.0:1619677448.186324:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:26.0:1619677448.186326:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.186329:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.186330:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.186332:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.186333:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.186334:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.186335:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.186337:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.186339:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.186342:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.186342:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.186343:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.186345:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.186345:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677448.186346:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:26.0:1619677448.186346:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:8.2:1619677448.186352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.186360:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.186362:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.186363:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.186365:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.186365:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.186368:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.186480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.186486:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.186489:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.186489:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.186492:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.186494:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.186495:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:5.2:1619677448.186495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:17.0:1619677448.186496:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.186498:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.186499:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.186500:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.186503:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.186503:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.186504:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.186506:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.186507:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.186507:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.186507:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.186508:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:10.0:1619677448.186509:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.186510:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.186510:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.186511:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.186511:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:10.0:1619677448.186512:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.186513:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.186515:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.186516:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677448.186516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:10.0:1619677448.186517:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.186518:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.186522:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.186523:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:1.0:1619677448.186523:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.186524:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.186525:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.186526:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.186526:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.186527:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.186528:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:1.0:1619677448.186528:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.186529:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:1.0:1619677448.186529:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:10.0:1619677448.186530:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.186531:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.186532:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.186533:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.186538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.186543:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.186545:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.186546:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.186546:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:31.0:1619677448.186548:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.186548:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.186548:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.186549:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.186550:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.186551:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.186573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677448.186578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:10.0:1619677448.186580:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.186583:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.186584:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677448.186584:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677448.186586:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.186587:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.186587:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677448.186588:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.186590:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.186590:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.186592:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.186593:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.186593:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.186595:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.186595:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.186596:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.186597:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677448.186598:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.186599:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.186600:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.186606:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.186607:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.186608:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.186610:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.186611:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.186612:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.186613:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.186613:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.186614:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.186614:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.186615:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.186616:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.186616:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.186617:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.186617:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.186617:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.186618:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.186618:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.186628:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.186631:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.186632:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.186632:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:30.0:1619677448.186633:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.186635:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.186635:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.186636:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.186636:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.186637:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:6.0:1619677448.186639:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.186642:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677448.186650:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.186652:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.186653:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.186653:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.2:1619677448.186653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:6.0:1619677448.186653:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.186654:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677448.186656:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.186658:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.186660:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.186662:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.186663:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.186664:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.186666:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.186667:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.186668:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.186669:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.186673:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.186675:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.186678:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.186679:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.186680:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.186681:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.186682:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.186683:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.186683:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677448.186694:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.186697:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.186698:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.186700:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.186701:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.186703:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.186722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:21.2:1619677448.186724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:30.0:1619677448.186730:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677448.186730:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:21.2:1619677448.186733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:2.0:1619677448.186733:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:30.0:1619677448.186734:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:18.0:1619677448.186734:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:5.0:1619677448.186736:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:31.0:1619677448.186737:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:2.0:1619677448.186738:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.186739:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:30.0:1619677448.186740:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.186741:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.186742:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:2.0:1619677448.186742:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.186743:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.186743:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.186744:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.186745:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677448.186745:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.186746:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:30.0:1619677448.186747:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.186748:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.186748:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.186749:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.186749:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.186751:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.186751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:5.0:1619677448.186751:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.186751:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677448.186753:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:5.0:1619677448.186753:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.186755:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:18.0:1619677448.186757:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:2.0:1619677448.186757:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.186760:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.186760:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:30.0:1619677448.186761:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.186761:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.186762:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.186763:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.186763:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.186764:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677448.186764:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.186765:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:18.0:1619677448.186766:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.186766:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.186766:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.186767:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:30.0:1619677448.186768:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.186768:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677448.186768:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:30.0:1619677448.186769:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:18.0:1619677448.186769:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96718204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.186769:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:30.0:1619677448.186770:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677448.186770:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:30.0:1619677448.186771:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:18.0:1619677448.186771:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.186771:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:30.0:1619677448.186772:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:18.0:1619677448.186772:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677448.186772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.186772:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:30.0:1619677448.186773:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.186773:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:18.0:1619677448.186773:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.186774:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.2:1619677448.186776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:2.0:1619677448.186776:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:18.0:1619677448.186777:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.186777:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.186778:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.186780:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.186780:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.186781:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.186782:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:21.2:1619677448.186783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.0:1619677448.186783:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:18.0:1619677448.186783:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.186783:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.186784:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.186784:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.0:1619677448.186785:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:18.0:1619677448.186785:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:2.0:1619677448.186785:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:21.2:1619677448.186786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677448.186786:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.186787:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.186787:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.186787:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:2.0:1619677448.186787:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.186788:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.186788:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:13.0:1619677448.186788:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.186788:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677448.186789:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677448.186790:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:13.0:1619677448.186790:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.186790:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:5.0:1619677448.186791:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:12.0:1619677448.186792:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677448.186792:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.186792:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:13.0:1619677448.186793:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.186794:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.186794:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677448.186796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000800:00000200:31.0:1619677448.186797:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.186799:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.186801:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.186801:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677448.186801:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:31.0:1619677448.186803:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:20.0:1619677448.186803:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:13.0:1619677448.186803:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.186803:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:31.0:1619677448.186804:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.186804:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.186805:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.186806:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.186806:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:13.0:1619677448.186806:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:12.0:1619677448.186806:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.186807:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.186807:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:13.0:1619677448.186807:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.186807:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:31.0:1619677448.186808:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.186808:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.186809:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:13.0:1619677448.186810:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.186810:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:12.0:1619677448.186811:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677448.186812:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.186812:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.186812:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677448.186813:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.186816:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.186816:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.186817:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.186818:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:17.0:1619677448.186818:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.186819:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.186819:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.186820:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.186820:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677448.186821:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.186821:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677448.186822:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.186823:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:17.0:1619677448.186824:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.186824:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.186824:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:17.0:1619677448.186825:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.186826:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:17.0:1619677448.186827:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.186828:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.186829:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.186830:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:17.0:1619677448.186830:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.186831:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.186832:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.186833:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.186834:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.186834:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.186835:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.186836:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.186838:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:17.0:1619677448.186838:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.186838:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.186839:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.186840:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.186841:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.186842:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.186842:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.186847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.186853:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.186855:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.186856:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.186857:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.186858:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.186859:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.186861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.186868:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.186871:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.186872:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.186873:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.186876:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.186877:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.186879:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.186880:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.186881:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.186882:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.186886:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.186888:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.186890:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.186891:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.186892:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.186893:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.186894:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.186895:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.186895:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.186906:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.186908:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.186909:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.186910:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.186912:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.186913:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.187090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.187095:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.187098:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.187099:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.187101:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.187102:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.187104:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.187105:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.187107:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.187108:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.187109:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.187112:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.187114:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.187117:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.187117:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.187118:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.187120:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.187121:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.187122:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.187123:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.187123:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.187128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.187133:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:1.0:1619677448.187134:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677448.187136:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:1.0:1619677448.187136:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:31.0:1619677448.187137:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.187137:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.187137:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677448.187139:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.187140:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.187142:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.187150:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.187153:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.187153:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.187154:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.187155:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.187156:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.187195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.187202:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.187205:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.187206:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.187207:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.187210:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.187212:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.187213:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.187214:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.187215:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.187216:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.187220:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.187222:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.187225:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.187226:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.187226:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.187228:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.187228:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.187229:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.187229:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.187234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.187242:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.187244:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.187245:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.187246:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.187247:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.187249:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.187354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.187360:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.187363:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.187363:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.2:1619677448.187365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:17.0:1619677448.187366:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.187368:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.187370:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.187371:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.187373:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.187373:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.187373:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:17.0:1619677448.187374:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.187376:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.187377:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:17.0:1619677448.187378:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.187379:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:17.0:1619677448.187380:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.187382:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.187382:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.187383:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.187384:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:10.0:1619677448.187384:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:17.0:1619677448.187385:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:10.0:1619677448.187385:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.187386:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.187386:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.187386:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.187387:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:10.0:1619677448.187387:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.187388:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.187391:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.187393:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.187396:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.187396:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.187397:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.187398:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.187399:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.187400:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.187400:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:6.0:1619677448.187400:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.187403:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.187404:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.187404:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.187405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:6.0:1619677448.187406:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.187408:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677448.187413:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.187415:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.187416:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.187417:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.187419:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.187420:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.187422:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:22.0:1619677448.187423:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.187424:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.187424:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.187425:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.187425:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.187450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.187455:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.187457:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.187458:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.187459:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.187462:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.187464:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.187465:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.187466:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.187467:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.187468:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.187471:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.187473:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.187476:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.187477:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.187478:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.187480:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.187481:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.187482:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.187482:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677448.187494:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.187497:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.187498:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.187500:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.187502:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.187503:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.187534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.187541:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.187544:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.187544:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.187547:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.187549:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.187551:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677448.187552:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.187553:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677448.187554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:18.0:1619677448.187555:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.187556:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677448.187559:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.187561:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677448.187561:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.187564:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:18.0:1619677448.187564:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.187565:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.187565:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677448.187566:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:24.0:1619677448.187567:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.187568:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677448.187569:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.187569:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:24.0:1619677448.187570:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.187570:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:24.0:1619677448.187571:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:21.2:1619677448.187572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:24.0:1619677448.187573:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96718194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.187574:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.187575:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.187576:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.2:1619677448.187578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:24.0:1619677448.187579:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.187580:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677448.187580:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.187582:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:13.0:1619677448.187582:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:24.0:1619677448.187584:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.187584:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:13.0:1619677448.187584:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:31.0:1619677448.187585:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.187585:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:13.0:1619677448.187585:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.187585:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:31.0:1619677448.187587:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.187587:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677448.187587:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.187587:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:13.0:1619677448.187587:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.187587:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:24.0:1619677448.187588:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:31.0:1619677448.187589:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:24.0:1619677448.187589:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677448.187589:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:31.0:1619677448.187590:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.187590:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.187590:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677448.187590:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:31.0:1619677448.187593:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.187593:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677448.187593:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:8.2:1619677448.187593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677448.187593:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.187594:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.187595:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.187595:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.187596:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.187597:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.187597:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677448.187598:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:5.0:1619677448.187599:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96718192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.187600:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.187600:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.187600:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:5.0:1619677448.187601:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.187601:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.187602:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:5.0:1619677448.187602:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.187603:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:17.0:1619677448.187603:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96718191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.187603:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.187604:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.187605:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.187606:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.187607:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.187607:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:17.0:1619677448.187607:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.187608:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677448.187609:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.187609:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:5.0:1619677448.187609:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.187609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.187610:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.187611:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677448.187611:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.187612:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.187613:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:17.0:1619677448.187614:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.187614:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.187615:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.187615:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677448.187616:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.187616:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.187617:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.187617:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.187617:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.187618:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677448.187619:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677448.187619:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.187619:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677448.187620:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.187620:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677448.187621:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:17.0:1619677448.187621:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.187621:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.0:1619677448.187621:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:10.0:1619677448.187622:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:26.0:1619677448.187623:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.187623:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.187623:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:17.0:1619677448.187624:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.187624:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.187625:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:10.0:1619677448.187625:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.187626:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.187627:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.187629:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.187630:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96718190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.187631:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:26.0:1619677448.187632:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.187633:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.187633:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:26.0:1619677448.187634:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:12.0:1619677448.187634:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.187635:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:12.0:1619677448.187636:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.187636:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.187637:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:22.0:1619677448.187638:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:13.0:1619677448.187638:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.187638:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.187639:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.187639:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677448.187640:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:10.0:1619677448.187640:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.187641:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:22.0:1619677448.187641:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.187641:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.187641:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:6.0:1619677448.187642:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.187643:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.187643:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.187645:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.187646:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.187646:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.187646:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:26.0:1619677448.187648:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.187648:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.187650:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.187651:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.187652:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.187657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.187661:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.187664:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.187664:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.187665:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.187665:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.187666:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.187667:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.187667:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.187668:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.187670:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.187688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.187696:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.187699:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.187701:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.187702:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.187704:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.187706:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.187707:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.187708:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.187709:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.187710:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.187714:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.187715:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.187718:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.187719:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.187720:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.187722:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.187722:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.187723:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.187723:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.187728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.187735:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.187738:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.187739:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.187740:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.187741:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.187744:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.187801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.0:1619677448.187805:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.187808:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677448.187810:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.187810:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.187812:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.187814:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.187815:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.187816:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.187817:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.187818:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.187821:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.187823:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.187825:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.187826:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.187827:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.187828:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.187829:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.187830:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.187830:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.187834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.187841:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.187843:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:5.2:1619677448.187843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:20.0:1619677448.187844:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.187846:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.187848:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:1.0:1619677448.187848:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.187849:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.2:1619677448.187849:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:1.0:1619677448.187851:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.187852:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.187854:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.187855:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:1.0:1619677448.187856:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.187858:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:1.0:1619677448.187858:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:18.0:1619677448.187859:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:1.0:1619677448.187859:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.187861:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.187861:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.187862:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.187863:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.187864:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.187866:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.187868:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.187868:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.187869:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.187870:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.187870:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.187872:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.187874:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.187875:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.187875:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.187876:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.187877:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677448.187878:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.187878:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.187879:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.187880:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.187880:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677448.187881:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.187881:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.187883:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.187884:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.187884:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.187885:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.187889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677448.187890:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.187892:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.187893:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.187895:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.187896:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:12.0:1619677448.187896:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.187898:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677448.187899:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.187900:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.187900:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.187902:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.187905:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.188025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.188033:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.188036:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.188037:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.188039:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.188042:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.188043:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.188044:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.188046:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.188047:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.188048:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.188051:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.188053:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.188056:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.188057:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.188058:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.188059:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.188060:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.188060:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.188061:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.188073:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.188076:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.188077:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.188077:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.188080:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.188083:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.188174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.188180:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.188183:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.188183:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.188185:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.188188:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.188189:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.188191:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.188192:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.188193:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.188194:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.188197:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.188199:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.188201:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.188202:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.188203:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.188205:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.188205:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.188207:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.188208:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.188208:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.188213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.188218:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677448.188218:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677448.188220:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:12.0:1619677448.188220:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:17.0:1619677448.188221:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.188222:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677448.188222:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.188223:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.188225:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.188226:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.188415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.188417:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.188420:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.188422:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.188423:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.188424:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.188426:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.188427:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.188428:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.188430:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.188431:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.2:1619677448.188433:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:20.0:1619677448.188438:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.188440:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.188442:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.188443:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.188444:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.188445:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:20.0:1619677448.188445:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.188447:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.188447:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.188447:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.2:1619677448.188448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677448.188449:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:31.0:1619677448.188450:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.188450:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.188450:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677448.188451:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:30.0:1619677448.188451:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.188453:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.188453:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:31.0:1619677448.188454:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.188454:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.188455:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.188455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.188456:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.188458:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.188459:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.188461:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.188461:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.188461:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677448.188462:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.188462:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.188463:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96718181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.188464:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.188464:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.188464:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:30.0:1619677448.188465:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677448.188465:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.188465:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677448.188466:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.188466:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:13.0:1619677448.188467:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.188468:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.188470:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.188470:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.188471:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.188472:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.188472:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.188473:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:13.0:1619677448.188473:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.188474:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.188474:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.188475:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.188475:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.188475:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.188477:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.188477:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.188477:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.188478:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.188478:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.188479:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.188480:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677448.188483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.188489:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.188491:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:12.0:1619677448.188491:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.188492:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.188493:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:12.0:1619677448.188493:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.188494:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.188495:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677448.188495:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.188496:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677448.188508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.188516:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.188519:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.188519:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.188521:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.188521:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.188525:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.188579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.188585:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.188588:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.188588:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.188591:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.188593:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.188595:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.188596:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.188597:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.188598:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.188599:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.188603:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.188605:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.188607:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.188608:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.188609:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.188611:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.188611:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.188612:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.188612:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.188623:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.188626:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.188627:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.188628:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.188630:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.188631:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.188642:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:4.2:1619677448.188643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677448.188645:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.188646:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.188647:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.188647:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.188649:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.188650:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.188653:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.188654:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.188656:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.188658:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.188660:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.188661:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.188662:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.188663:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.188664:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.188667:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.188669:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.188672:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.188673:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.188673:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.188675:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.188675:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.188676:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.188677:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677448.188689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.188691:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.188693:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.188695:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:6.0:1619677448.188695:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.188696:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.188697:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.188699:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.188700:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.188701:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:22.0:1619677448.188702:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.188702:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.188704:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.188705:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.188707:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.188708:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.188708:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.188711:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.188713:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.188715:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.188716:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.188717:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.188718:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.188719:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.188720:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.188721:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.188722:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:21.2:1619677448.188727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.188732:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.188734:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.188734:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.188735:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.188755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.188762:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.188765:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.188765:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.188767:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.188768:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.188769:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.188781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.188788:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.188791:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.188792:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.188793:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.188793:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.188794:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.188837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.188842:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.188844:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.188846:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677448.188847:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.188849:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.188850:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:5.0:1619677448.188854:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677448.188855:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.188856:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.188857:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:5.0:1619677448.188858:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.188858:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:18.0:1619677448.188859:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.188860:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.188863:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.188863:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.188865:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:18.0:1619677448.188866:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.188867:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:5.0:1619677448.188868:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.188869:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.188869:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:18.0:1619677448.188870:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.188871:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.188871:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:18.0:1619677448.188872:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.188874:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:18.0:1619677448.188876:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.188876:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.188877:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.188877:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:18.0:1619677448.188878:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.188878:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:10.0:1619677448.188879:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677448.188880:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.188881:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.188881:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.188882:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677448.188882:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.188884:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.188884:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:10.0:1619677448.188884:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.188885:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.188886:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.188887:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:10.0:1619677448.188887:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.188888:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677448.188889:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:10.0:1619677448.188889:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96718175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.188890:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.188891:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.188892:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.188892:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.188892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000400:00000200:10.0:1619677448.188893:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.188894:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677448.188895:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:10.0:1619677448.188896:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.188897:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:30.0:1619677448.188898:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:10.0:1619677448.188898:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.188899:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.188900:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:10.0:1619677448.188901:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.188901:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:10.0:1619677448.188902:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.188902:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.188904:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.188904:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.188905:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.188905:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677448.188909:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.188912:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.188912:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.188913:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.188913:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.188914:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:13.0:1619677448.188916:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.188918:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.188919:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.188919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:13.0:1619677448.188921:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.188922:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.188923:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677448.188926:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.188929:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.188930:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.188931:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.188932:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.188934:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677448.188938:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.188941:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.188941:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:21.2:1619677448.188942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:12.0:1619677448.188942:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.188942:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.188943:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.188946:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.188949:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677448.188951:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.188951:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.188954:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.188955:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:20.2:1619677448.188957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:5.0:1619677448.188957:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.188958:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.188959:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.188960:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.188963:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.188964:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.188965:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:31.0:1619677448.188968:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:5.0:1619677448.188968:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.188969:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.188969:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:31.0:1619677448.188970:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.188971:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677448.188971:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.188971:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.188972:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000400:00000200:31.0:1619677448.188973:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.188973:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:31.0:1619677448.188974:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.188976:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.188976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000400:00000200:31.0:1619677448.188977:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.188978:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.188979:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.188982:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.188984:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.188984:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:31.0:1619677448.188986:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.188986:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:31.0:1619677448.188987:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:20.0:1619677448.188987:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.188988:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:20.0:1619677448.188988:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.188989:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677448.188989:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:31.0:1619677448.188990:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.188990:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677448.188990:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:31.0:1619677448.188991:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677448.189001:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.189004:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.189005:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.189006:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.189008:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.189009:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.189132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.189137:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.189139:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.189142:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.189142:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.189144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:5.0:1619677448.189145:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.189148:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677448.189149:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.189150:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.189151:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:5.0:1619677448.189151:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.189152:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:8.0:1619677448.189154:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.189155:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:5.0:1619677448.189156:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.189159:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.189159:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:8.0:1619677448.189161:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.189162:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677448.189163:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:5.0:1619677448.189163:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677448.189163:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:8.0:1619677448.189164:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96718171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.189165:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.189165:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677448.189166:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:8.0:1619677448.189167:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.189167:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:5.0:1619677448.189167:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.189167:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.189171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:8.0:1619677448.189171:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.189172:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.189175:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.189176:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.189176:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.189178:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.189178:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:18.0:1619677448.189179:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:8.0:1619677448.189179:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:18.0:1619677448.189181:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:8.0:1619677448.189181:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.189181:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:18.0:1619677448.189182:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.189182:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.189184:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.189187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:18.0:1619677448.189187:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:13.0:1619677448.189190:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.189193:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:24.0:1619677448.189194:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:13.0:1619677448.189194:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.189195:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.189196:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:12.0:1619677448.189196:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.189197:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677448.189197:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677448.189198:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:21.2:1619677448.189265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.189272:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.189275:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.189277:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.189279:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.189279:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.189280:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677448.189282:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.189283:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.189284:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.189285:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677448.189288:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.189290:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.189293:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.189294:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677448.189299:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.189301:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677448.189301:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.189303:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.189304:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.189305:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677448.189311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677448.189316:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.189319:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.189319:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677448.189320:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:22.0:1619677448.189320:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.189322:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:22.0:1619677448.189322:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.189323:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.189324:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.189325:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.189373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.189379:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.189382:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.189382:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.189385:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.189388:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.189389:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.189391:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.189392:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.189393:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.189394:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.189397:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.189399:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.189402:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.189403:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.189404:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.189406:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.189406:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.189407:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.189407:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.189420:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.189422:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.189423:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.189425:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.189427:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.189428:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.189433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.189440:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.189443:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.189445:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.189447:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.189448:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.189449:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.189451:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.189452:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.189453:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.189454:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.189457:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.189459:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.189463:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.189463:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.189464:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.189466:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.189467:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.189467:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.189468:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.189473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.189481:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.189483:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.189485:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.189486:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.189487:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.189489:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.189490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.189498:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.189501:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.189502:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.189502:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.189503:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.189504:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.189718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.189723:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.189725:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.189728:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.189728:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.189730:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.189731:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.189732:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.189734:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.189735:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.189736:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.189739:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.189741:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.189744:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.189744:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.189745:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.189747:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.189747:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.189748:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.189748:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677448.189761:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.189764:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.189764:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.189765:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.189767:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.189768:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.189831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.189838:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.189841:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.189843:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.189845:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.189846:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.189848:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.189849:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718166] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.189850:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.189851:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.189852:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.189856:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.189858:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.189860:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.189861:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.189861:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.189863:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.189863:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.189864:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.189865:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677448.189874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:13.0:1619677448.189875:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.189877:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.189878:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.189880:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.189880:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:12.0:1619677448.189881:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.189882:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.189883:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.189885:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.189886:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.189889:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.189890:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.189891:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.189893:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.189894:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.189895:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:21.2:1619677448.189896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000400:00000200:2.0:1619677448.189898:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.189899:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.189901:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:2.0:1619677448.189901:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.189902:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.189903:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:2.0:1619677448.189903:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.189904:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.189904:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:20.0:1619677448.189905:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.189905:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.189905:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.0:1619677448.189906:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.189908:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.189909:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.189910:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.189910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677448.189912:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.189913:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.189914:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:18.0:1619677448.189917:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:20.0:1619677448.189918:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.189919:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677448.189920:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.189920:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.189921:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.189921:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.189922:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000400:00000200:18.0:1619677448.189922:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.189923:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:18.0:1619677448.189924:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677448.189925:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.189925:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.189927:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677448.189928:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677448.189929:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:8.2:1619677448.189931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:21.2:1619677448.189934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.189940:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.189941:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677448.189941:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.189942:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000400:00000200:5.0:1619677448.189942:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.189943:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:26.0:1619677448.189944:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea3be00] (18)++
00000800:00000200:20.0:1619677448.189944:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.189944:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.189945:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677448.189946:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:26.0:1619677448.189947:0:19941:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a3aea3be00] (19)--
00000800:00000200:26.0:1619677448.189947:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea3be00] (18)--
00000800:00000200:26.0:1619677448.189948:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea3be00] (17)++
00000800:00000200:26.0:1619677448.189949:0:19941:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a3aea3be00] (18)--
00000800:00000200:26.0:1619677448.189950:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea3be00] (17)--
00000800:00000200:26.0:1619677448.189951:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea3be00] (16)++
00000800:00000200:26.0:1619677448.189952:0:19941:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a3aea3be00] (17)--
00000800:00000200:26.0:1619677448.189953:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea3be00] (16)--
00000800:00000200:26.0:1619677448.189954:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea3be00] (15)++
00000800:00000200:26.0:1619677448.189956:0:19941:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a3aea3be00] (16)--
00000800:00000200:26.0:1619677448.189956:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea3be00] (15)--
00000800:00000200:26.0:1619677448.189957:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea3be00] (14)++
00000800:00000200:26.0:1619677448.189958:0:19941:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a3aea3be00] (15)--
00000800:00000200:26.0:1619677448.189958:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea3be00] (14)--
00000800:00000200:26.0:1619677448.189959:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea3be00] (13)++
00000800:00000200:26.0:1619677448.189959:0:19941:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a3aea3be00] (14)--
00000800:00000200:26.0:1619677448.189960:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea3be00] (13)--
00000800:00000200:26.0:1619677448.189961:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea3be00] (12)++
00000800:00000200:26.0:1619677448.189961:0:19941:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a3aea3be00] (13)--
00000800:00000200:26.0:1619677448.189962:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea3be00] (12)--
00000800:00000200:26.0:1619677448.189962:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea3be00] (11)++
00000800:00000200:26.0:1619677448.189963:0:19941:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a3aea3be00] (12)--
00000800:00000200:26.0:1619677448.189964:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea3be00] (11)--
00000800:00000200:26.0:1619677448.189964:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea3be00] (10)++
00000800:00000200:26.0:1619677448.189965:0:19941:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a3aea3be00] (11)--
00000800:00000200:26.0:1619677448.189966:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea3be00] (10)--
00000800:00000200:26.0:1619677448.189967:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea3be00] (9)--
00000800:00000200:5.2:1619677448.190057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.190065:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.190068:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.190068:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.190070:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.190073:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.190074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:10.0:1619677448.190074:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.190075:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.190076:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.190077:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.190078:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.190081:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.190081:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:10.0:1619677448.190082:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677448.190084:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.190085:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:10.0:1619677448.190085:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.190085:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.190086:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:8.0:1619677448.190087:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.190088:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.190088:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.190089:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.190089:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:8.0:1619677448.190089:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.190090:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.190091:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96718162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.190093:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.190093:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.190094:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.190098:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.190099:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677448.190100:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.190102:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677448.190102:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677448.190103:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.190103:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.190103:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:12.0:1619677448.190104:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.190105:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.190105:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:12.0:1619677448.190106:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677448.190106:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677448.190107:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677448.190107:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.190111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.190119:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.190121:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.190123:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.190123:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.190125:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.190128:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.190153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.190160:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.190163:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.190165:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.190167:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.190168:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.190170:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.190171:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.190172:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.190173:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.190174:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.190177:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.190179:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.190181:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.190181:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.190182:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.190184:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.190184:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.190185:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.190186:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677448.190206:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.190208:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.190209:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.190211:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.190213:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.190213:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.190225:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.190227:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.190228:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.190229:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.190229:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.190230:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.190321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.190327:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.190330:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.190331:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.190333:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.190334:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.190335:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.190337:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.190338:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.190339:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.190340:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.190344:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.190346:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.190348:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.190349:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.190350:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.190352:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.190352:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.190353:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.190353:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:21.2:1619677448.190365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.190374:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.190377:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.190379:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.190381:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.190381:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.190383:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.190384:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.190385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.190386:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.190387:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.190388:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.190391:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.190393:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.190393:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677448.190396:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.190396:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.190396:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.190397:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.190397:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:20.0:1619677448.190398:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.190399:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.190399:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.190399:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.190400:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.190400:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677448.190401:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.190414:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:21.2:1619677448.190416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.190417:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.190419:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.190419:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.190420:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:22.0:1619677448.190421:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.190421:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.190424:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.190425:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677448.190426:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.190427:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.190429:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.190431:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.190432:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.190434:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.190435:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:6.0:1619677448.190435:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.190437:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.190438:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.190439:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.190439:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.190439:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.190440:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677448.190441:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.190444:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.190445:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.190446:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.190448:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.190448:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.190449:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.190449:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:26.0:1619677448.190461:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.190463:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.190464:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.190465:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.190467:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.190470:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.190496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.190503:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.190506:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.190506:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.190508:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.190510:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.190512:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.190513:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.190514:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.190515:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.190516:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.190520:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.190522:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.190524:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.190525:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.190526:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.190528:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.190528:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.190528:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.190529:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677448.190530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.190537:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.190539:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.190540:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.190541:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:22.0:1619677448.190541:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:26.0:1619677448.190543:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.190544:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:22.0:1619677448.190544:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.190545:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.190545:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:26.0:1619677448.190546:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718156] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.190547:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.190547:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.190548:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.190549:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.190551:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:26.0:1619677448.190553:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.190555:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.190557:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.190558:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.190558:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.190559:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.190560:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.190560:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.190561:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.190562:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:22.0:1619677448.190562:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.190562:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.190563:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.190564:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.190565:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677448.190573:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.190576:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.190576:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.190578:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677448.190580:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.190581:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.190770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.190778:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.190780:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.190782:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.190783:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.190784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:24.0:1619677448.190786:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.190788:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.190789:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.190791:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.190792:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:24.0:1619677448.190792:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.190792:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.190795:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:24.0:1619677448.190797:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.190798:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.190799:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677448.190799:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.190800:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.190802:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:24.0:1619677448.190802:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.190803:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:31.0:1619677448.190804:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.190804:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677448.190805:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.190805:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:31.0:1619677448.190806:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.190806:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:24.0:1619677448.190806:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.190806:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.190807:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:31.0:1619677448.190809:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.190811:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:8.2:1619677448.190811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.190814:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.190815:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677448.190815:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.190817:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677448.190818:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.190818:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.190819:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677448.190819:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.190822:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.190823:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.190825:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.190825:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.190828:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677448.190830:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.190833:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.190834:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.190834:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.190837:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.190839:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.190942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.190949:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.190952:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.190953:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.190955:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.190958:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.190959:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.190960:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.190961:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.190963:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.190963:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.190966:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.190968:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.190971:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.190972:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.190972:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.190974:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.190974:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.190975:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.190977:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.190977:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.190982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.190987:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:26.0:1619677448.190990:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677448.190990:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.190990:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.190991:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.190992:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.190993:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677448.190993:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.190994:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677448.190996:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.190998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.191003:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.191006:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.191008:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.191009:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.191010:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.191011:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.191013:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.191014:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.191015:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.191016:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.191019:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.191021:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.191023:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.191024:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.191024:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.191026:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.191026:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.191027:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.191028:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.191032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.191039:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.191042:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.191042:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.191044:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.191045:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.191046:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:8.2:1619677448.191050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.191058:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.191060:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.191061:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.191062:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.191062:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.191063:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.2:1619677448.191128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.191134:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.191137:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.191138:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.191140:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.191142:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.191143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:1.0:1619677448.191144:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.191145:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.191147:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677448.191148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677448.191148:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.191149:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677448.191149:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.191151:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:1.0:1619677448.191152:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.191154:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.191154:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.0:1619677448.191155:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677448.191155:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677448.191155:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.191157:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:1.0:1619677448.191157:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.191158:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.191158:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:5.0:1619677448.191158:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.191158:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.191159:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:5.0:1619677448.191160:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.191160:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:10.0:1619677448.191161:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.191161:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.191161:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.191161:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677448.191162:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:1.0:1619677448.191162:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:10.0:1619677448.191164:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.191166:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677448.191166:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.191167:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:10.0:1619677448.191168:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96718149] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.191169:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.191170:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.191170:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.191171:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.191171:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677448.191172:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677448.191173:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.191174:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:10.0:1619677448.191174:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.191174:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.191175:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.191175:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:10.0:1619677448.191176:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:22.0:1619677448.191177:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.191178:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.191178:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.191178:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.191179:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.191179:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:22.0:1619677448.191180:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.191180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:10.0:1619677448.191181:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.191181:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.191182:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:10.0:1619677448.191182:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.191182:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677448.191187:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.191190:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.191191:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.191191:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.191192:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:2.0:1619677448.191193:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.191194:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.191195:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.191195:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.191196:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:26.0:1619677448.191197:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.191199:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.191237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.191243:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.191246:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.191247:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.191248:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.191250:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.191251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:18.0:1619677448.191252:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.191253:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.191254:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.191255:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.191256:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.191258:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:18.0:1619677448.191260:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.191261:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.191261:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.191262:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:18.0:1619677448.191263:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.191264:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.191264:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677448.191264:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.191266:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.191266:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.191267:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.191267:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:8.0:1619677448.191267:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.191268:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.191269:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96718147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.191271:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.191271:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.191273:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677448.191276:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.191278:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.191281:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677448.191282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.191282:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.191283:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.191284:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.191285:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.191285:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.191286:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.191290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.191290:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.191292:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.191296:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:9.2:1619677448.191299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.191300:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:2.0:1619677448.191300:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.191302:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:20.0:1619677448.191303:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.191304:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.191304:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:20.0:1619677448.191305:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.191305:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.191308:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.191308:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677448.191309:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677448.191310:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677448.191310:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.191311:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.191312:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.191382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.191388:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.191391:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.191392:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.191393:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.191395:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.191397:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.191398:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.191399:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.191400:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.191401:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.191404:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.191406:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.191407:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.191408:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.191409:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.191410:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.191411:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.191411:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.191412:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.191417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.191424:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.191426:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.191426:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.191428:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.191428:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.191429:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.191527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.191533:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.191536:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.191537:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.191539:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.191542:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.191543:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.191544:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.191546:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.191547:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.191548:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.191550:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.191552:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.2:1619677448.191553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:17.0:1619677448.191554:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.191555:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.191556:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.191557:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.191558:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.191559:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.191560:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.191560:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.191561:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.191565:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:5.2:1619677448.191565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.191566:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.191567:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.191570:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.191571:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:1.0:1619677448.191571:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:10.0:1619677448.191572:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.191572:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:10.0:1619677448.191573:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.191573:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.191573:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.191574:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:10.0:1619677448.191575:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.191575:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:6.0:1619677448.191575:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.191576:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.191577:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.191579:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.191580:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.191581:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:10.0:1619677448.191582:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.191585:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.191586:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.191586:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.191588:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.191589:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.191589:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677448.191590:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.191590:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677448.191593:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.191594:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.191594:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.191595:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.191595:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677448.191602:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.191604:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.191605:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.191607:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.191609:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:4.2:1619677448.191609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677448.191610:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.191616:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.191619:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.191620:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.191622:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.191624:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.191625:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.191626:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.191628:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.191629:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.191629:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.191633:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.191634:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.191636:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.191637:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.191637:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.191639:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.191640:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.191641:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.191642:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.191642:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.191648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.2:1619677448.191649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.191654:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.191655:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:10.0:1619677448.191655:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:30.0:1619677448.191656:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.191657:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.191657:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.191658:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:30.0:1619677448.191658:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.191658:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.191660:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677448.191660:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.191661:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:31.0:1619677448.191662:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.191662:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677448.191662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:31.0:1619677448.191664:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.191666:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.191667:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.191668:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.191668:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:31.0:1619677448.191669:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.191670:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:22.0:1619677448.191671:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.191671:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.191674:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.191675:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.191677:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:6.0:1619677448.191677:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.191678:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:31.0:1619677448.191679:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.191680:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:6.0:1619677448.191680:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.191681:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:6.0:1619677448.191681:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.191682:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:6.0:1619677448.191682:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.191683:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677448.191683:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.191684:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.191686:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.191686:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:6.0:1619677448.191687:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.191689:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:21.2:1619677448.191692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677448.191693:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.191693:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.191694:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:6.0:1619677448.191694:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.191696:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:6.0:1619677448.191696:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:26.0:1619677448.191697:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.191697:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:6.0:1619677448.191697:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.191697:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.191698:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677448.191699:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:26.0:1619677448.191699:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.191702:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:26.0:1619677448.191702:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:30.0:1619677448.191703:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.191704:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677448.191706:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.191709:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.191710:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.191711:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.191713:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.191714:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.191787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.191795:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.191799:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.191800:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.191802:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.191805:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.191807:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.191808:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.191809:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.191810:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.191811:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.191815:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.191816:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.191819:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.191820:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.191821:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.191823:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.191823:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.191824:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.191825:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.191826:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677448.191829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.191832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:18.0:1619677448.191837:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.0:1619677448.191838:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:18.0:1619677448.191840:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.191841:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.191841:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.191841:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:2.0:1619677448.191841:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.191843:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.191843:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:18.0:1619677448.191846:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.191905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.191911:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.191913:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.191914:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.191916:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.191918:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.191920:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.191921:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.191922:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.191923:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.191924:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.191927:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.191929:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.191931:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.191932:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.191932:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.191934:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.191934:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.191935:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.191935:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.191940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.191947:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.191949:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.191949:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.191951:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.191952:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.191953:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.192030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.192037:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.192040:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.192041:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.192043:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.192045:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.192047:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.192048:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.192049:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.192050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:10.0:1619677448.192050:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.192051:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.192055:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.192056:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:10.0:1619677448.192057:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.192058:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:10.0:1619677448.192059:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.192060:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.192061:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:10.0:1619677448.192061:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:5.0:1619677448.192061:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677448.192062:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.192062:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.192063:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:5.0:1619677448.192063:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.192064:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:5.0:1619677448.192065:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677448.192066:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.192068:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.192068:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.192070:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.192073:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:31.0:1619677448.192075:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.192075:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.192077:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.192078:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.192078:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.192079:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677448.192079:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:31.0:1619677448.192080:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.192081:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.192081:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.192082:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.0:1619677448.192082:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.192083:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.192084:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.192085:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.2:1619677448.192090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677448.192092:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.192093:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:30.0:1619677448.192095:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.192095:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:20.0:1619677448.192095:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:30.0:1619677448.192096:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.192096:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.192096:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677448.192097:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.192097:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.192098:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:30.0:1619677448.192099:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:26.0:1619677448.192099:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.192099:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.192101:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:26.0:1619677448.192105:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.192117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.192122:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.192124:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.192126:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.192127:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.192129:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.192130:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.192132:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.192133:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.192134:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.192135:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.192138:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.192140:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.192142:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.192143:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.192143:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.192145:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.192145:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.192146:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.192147:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.192151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.192158:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.192160:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.192161:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.192163:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.192163:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.192165:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.2:1619677448.192173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.192179:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.192181:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.192182:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.192184:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.192186:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.192187:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.192188:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.192189:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.192191:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.192191:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.192194:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.192196:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.192198:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.192198:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.192199:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.192201:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.192201:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.192202:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.192202:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.192207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.192214:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.192216:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.192216:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.192217:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.192218:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.192219:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.192302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.192309:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.192312:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.192313:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.192315:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.192317:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.192318:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.192319:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.192321:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.192322:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.192323:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.192325:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.192327:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.192329:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.192330:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.192331:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.192332:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.192332:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.192333:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.192333:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.192344:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.192345:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.192346:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.192347:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.192348:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.192349:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.192349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.192356:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.192358:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.192360:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.192361:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.192361:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000400:00000200:24.0:1619677448.192364:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.192364:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.192365:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:24.0:1619677448.192365:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:10.0:1619677448.192365:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.192365:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:24.0:1619677448.192366:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.192366:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:24.0:1619677448.192368:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.192368:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.192369:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.192372:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.192374:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.192376:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.192377:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.192378:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.192379:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.192380:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.192381:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.192382:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.192382:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.192385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.192388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.192393:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.192395:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677448.192395:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.192396:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.192397:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.192397:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.192398:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.192399:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.192399:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677448.192402:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.192479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.192485:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.192488:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.192489:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.192490:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.192492:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.192494:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.192494:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.192496:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.192497:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.192498:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.192501:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.192502:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.192505:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.192506:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.192507:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.192509:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.192509:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.192510:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.192510:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677448.192521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:13.0:1619677448.192522:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.192523:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677448.192525:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677448.192526:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:13.0:1619677448.192526:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.192528:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.192529:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.192529:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.192529:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.192531:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.192531:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:20.0:1619677448.192533:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.192534:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.192537:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:20.0:1619677448.192540:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.192540:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.192541:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.192543:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:12.0:1619677448.192543:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.192544:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677448.192545:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.192545:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.192545:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677448.192546:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.192546:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.192548:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:20.0:1619677448.192548:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:31.0:1619677448.192550:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.192552:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.192552:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677448.192552:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.192553:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.192553:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:31.0:1619677448.192554:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:20.0:1619677448.192555:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:31.0:1619677448.192556:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.192556:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.192557:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.192557:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:31.0:1619677448.192558:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.192558:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000400:00000200:31.0:1619677448.192559:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:20.0:1619677448.192559:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677448.192562:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.192564:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.2:1619677448.192564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.192566:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.192567:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.192567:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.192568:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.192569:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.192569:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.192569:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:30.0:1619677448.192570:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.192570:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.192571:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:5.0:1619677448.192571:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:30.0:1619677448.192572:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.192572:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677448.192573:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.192573:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677448.192573:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.192576:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.192577:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.192578:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.192580:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:20.2:1619677448.192580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:30.0:1619677448.192581:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.192583:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.192584:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.192586:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677448.192587:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.192589:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.192590:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.192591:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:21.2:1619677448.192600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.192606:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.192609:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.192610:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.192612:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.192614:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.192616:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.192617:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.192618:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.192619:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.192621:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.192624:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.192626:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.192629:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.192630:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.192631:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.192633:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.192633:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.192634:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.192634:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677448.192645:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.192648:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.192649:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.192650:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.192651:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.192653:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.192755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.192760:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.192763:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.192764:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.192766:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.192769:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.192770:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.192772:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677448.192772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:17.0:1619677448.192773:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.192774:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.192775:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.192778:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.192780:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.192780:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:26.0:1619677448.192783:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:17.0:1619677448.192783:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.192783:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.192784:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.192784:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:26.0:1619677448.192786:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.192786:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.192787:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.192788:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.192788:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:26.0:1619677448.192789:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:17.0:1619677448.192789:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.192790:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.192791:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.192792:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677448.192792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:26.0:1619677448.192793:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.192794:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.2:1619677448.192795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.192798:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.192799:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677448.192799:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:1.0:1619677448.192800:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677448.192801:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:26.0:1619677448.192802:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677448.192802:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.192802:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.192802:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:26.0:1619677448.192803:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:1.0:1619677448.192803:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.192804:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:24.0:1619677448.192804:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.192804:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.192805:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.192806:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.192806:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.192807:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:24.0:1619677448.192808:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.192813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677448.192820:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:22.0:1619677448.192821:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677448.192822:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.192823:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.192823:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:22.0:1619677448.192824:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:8.0:1619677448.192824:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.192824:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677448.192824:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.192825:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.192827:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.192830:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.192898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.192906:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.192909:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.192911:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.192913:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.192913:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.192915:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.192916:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.192918:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.192919:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.192919:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.192922:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.192924:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.192927:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.192927:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.192928:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.192929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:31.0:1619677448.192930:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.192930:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.192931:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.192932:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.192935:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.192938:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.192938:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.192940:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.192942:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677448.192943:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.192944:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:6.0:1619677448.192944:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677448.192945:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.192946:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.192947:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.192947:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.192948:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:6.0:1619677448.192948:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.192949:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:6.0:1619677448.192950:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.192953:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.192955:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.192958:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.192958:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.192959:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.192961:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.192961:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.192962:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.192962:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.192967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.192975:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.192978:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.192979:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.192979:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.192981:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.192983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.192986:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.192988:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.192990:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.192991:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.192992:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.192992:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.192993:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.193006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.193014:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.193017:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.193017:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.193020:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.193023:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.193024:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.193025:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.193027:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.193028:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.193029:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.193032:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.193034:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.193036:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.193037:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.193038:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.193039:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.193040:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.193040:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.193041:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.193051:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.193054:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.193055:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.193056:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.193058:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.193059:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.193077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.193082:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.193085:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.193086:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.193088:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.193091:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.193092:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.193094:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.193095:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.193096:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.193097:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.193100:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.193101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:1.0:1619677448.193102:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.193105:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:1.0:1619677448.193105:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.193106:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.193107:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:1.0:1619677448.193107:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.193109:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.193109:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.193110:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.193110:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.193110:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.193111:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:5.0:1619677448.193113:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.193114:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.193116:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.193117:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.193118:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.193119:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:12.0:1619677448.193121:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:5.0:1619677448.193122:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.193124:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.193124:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:12.0:1619677448.193125:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.193126:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677448.193127:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.193127:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:13.0:1619677448.193128:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.193128:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:12.0:1619677448.193130:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.193130:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.193130:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.193131:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.193132:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:4.2:1619677448.193135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.2:1619677448.193136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:26.0:1619677448.193142:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.193144:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:20.0:1619677448.193144:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:10.0:1619677448.193145:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677448.193146:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:26.0:1619677448.193147:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.193147:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.193149:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.193150:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.193150:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:26.0:1619677448.193151:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:20.0:1619677448.193151:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:26.0:1619677448.193152:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.193154:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.193155:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.193156:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.193159:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.193161:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.193163:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.193164:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.193164:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.193166:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.193166:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.193167:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.193168:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.193169:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.2:1619677448.193174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:4.2:1619677448.193175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.193178:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.193179:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.193181:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.193181:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.193182:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:5.0:1619677448.193182:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:30.0:1619677448.193183:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.193184:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:31.0:1619677448.193185:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.193185:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677448.193185:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:5.0:1619677448.193185:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.193186:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677448.193187:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:5.0:1619677448.193187:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.193189:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.193190:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.193191:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.193193:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.193193:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.193197:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.193199:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.193202:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.193203:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.193204:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.193205:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.193206:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.193207:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.193207:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.193218:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.193220:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.193221:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.193223:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.193224:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.193226:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.193237:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.193240:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.193241:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.193241:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.193242:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.193243:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.193335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.193343:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.193345:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.193348:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.193350:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.193350:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.193351:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.193352:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.193354:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.193354:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.193356:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.193359:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.193360:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.193362:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.193363:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.193364:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.193365:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.193366:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:5.2:1619677448.193366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:31.0:1619677448.193367:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.193367:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:18.0:1619677448.193372:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.193375:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.193376:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:30.0:1619677448.193378:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:18.0:1619677448.193378:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.193381:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.193381:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.193381:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.193382:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:31.0:1619677448.193384:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677448.193384:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.193384:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.193386:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:18.0:1619677448.193386:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.193387:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.193388:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.193391:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.193392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:18.0:1619677448.193393:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.193395:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.193396:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.193397:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.193399:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.193399:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.193400:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.193400:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:26.0:1619677448.193401:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.193404:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.193404:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677448.193405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677448.193406:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.193409:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.193410:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.193412:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.193413:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.193414:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.193414:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:26.0:1619677448.193415:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.193416:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.193417:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.193417:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:26.0:1619677448.193419:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000400:00000200:17.0:1619677448.193420:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.193422:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.193422:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677448.193425:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.193425:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.193426:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.193428:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.193429:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.193430:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.193430:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.193442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.193450:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.193452:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.193453:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.193455:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.193455:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.193458:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.193526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.193533:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.193536:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.193537:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.193538:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.193541:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.193543:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.193544:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.193545:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.193546:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.193547:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.193550:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.193552:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.193554:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.193554:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.193555:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.193556:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.193557:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.193557:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.193558:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.193569:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.193571:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.193572:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.193573:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.193575:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.193576:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.193609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.193617:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.193620:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.193620:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.193622:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.193625:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.193626:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.193627:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.193629:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.193630:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.193630:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.193633:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.193635:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.193637:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.193638:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.193639:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.193640:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.193640:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.193641:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.193641:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.2:1619677448.193646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.193652:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677448.193653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:13.0:1619677448.193653:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677448.193655:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.193656:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.193656:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.193656:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.193658:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.193658:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.193659:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677448.193660:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:1.0:1619677448.193660:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.193661:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677448.193662:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.193663:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677448.193663:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.193664:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.193666:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.193666:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.193667:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.193668:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677448.193669:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.193671:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.193672:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.193672:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.193674:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.193675:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.193675:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:22.0:1619677448.193676:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:1.0:1619677448.193678:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.193679:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.193679:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677448.193680:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.193681:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677448.193682:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:1.0:1619677448.193682:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.193682:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.193683:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.193685:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.193686:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.193686:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.193688:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.193689:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.193690:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.193691:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.193692:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:13.0:1619677448.193694:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:9.2:1619677448.193695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:13.0:1619677448.193696:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:21.2:1619677448.193697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:13.0:1619677448.193697:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.193699:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.193700:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.193702:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.193702:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677448.193702:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:6.0:1619677448.193704:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.193705:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677448.193705:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.193706:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.193706:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:2.0:1619677448.193706:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.193708:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.193711:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.193714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.193722:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.193725:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.193726:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.193726:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.193727:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.193728:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.193914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.193922:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.193924:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.193925:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.193927:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.193930:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.193931:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.193933:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718112] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.193934:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.193935:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.193936:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:4.2:1619677448.193936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:26.0:1619677448.193940:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.193941:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.0:1619677448.193942:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677448.193944:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.193945:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.193945:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.0:1619677448.193945:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.193946:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677448.193947:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.193947:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.193948:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.193948:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:8.0:1619677448.193948:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.193950:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.193951:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.193953:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96718111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.193954:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.193955:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.193956:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.193959:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.193960:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:8.0:1619677448.193960:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.193962:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.193963:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.193963:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.193964:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.193964:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.193964:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.193966:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.193967:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.193967:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:8.0:1619677448.193967:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.193967:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.193968:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.193972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.193975:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.193978:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.193978:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.193979:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.193980:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.193980:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677448.193980:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.193983:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.193984:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.193984:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.193987:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.193990:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.193993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.2:1619677448.193998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:30.0:1619677448.194000:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.194003:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.194005:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.194005:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:31.0:1619677448.194007:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.194007:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.194007:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000400:00000200:30.0:1619677448.194008:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:8.0:1619677448.194008:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:30.0:1619677448.194009:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.194010:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.194011:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.194012:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.194013:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677448.194013:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.194015:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677448.194016:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.194016:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96718109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.194018:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:24.0:1619677448.194018:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.194018:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.194019:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:30.0:1619677448.194021:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.194022:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.194022:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677448.194023:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.194024:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.194024:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.194025:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677448.194025:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677448.194026:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677448.194026:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.194027:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677448.194027:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.194029:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.194029:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677448.194030:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677448.194030:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677448.194035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:12.0:1619677448.194036:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.194038:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.2:1619677448.194039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:12.0:1619677448.194039:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.194040:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.194041:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:12.0:1619677448.194042:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.194043:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677448.194043:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.194044:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.194045:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677448.194045:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:8.0:1619677448.194046:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.194047:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.0:1619677448.194048:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.194050:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.194050:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.194053:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.194055:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.194056:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.194057:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.194058:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.194059:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.194063:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.194065:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.194068:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.194069:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.194070:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.194072:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.194072:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.194073:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.194073:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677448.194084:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.194087:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.194088:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.194090:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.194091:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.194093:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.194127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.194132:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.194135:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.194137:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.194138:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.194140:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.194141:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.194142:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.194144:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.194145:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.194146:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.194149:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.194151:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.194153:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.194154:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.194154:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.194156:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.194157:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.194158:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677448.194160:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677448.194160:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:8.2:1619677448.194162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.2:1619677448.194165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:21.2:1619677448.194166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.194171:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.194172:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677448.194172:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:30.0:1619677448.194173:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.194173:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000400:00000200:5.0:1619677448.194173:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.194174:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.194175:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.194175:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.194176:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:20.0:1619677448.194176:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677448.194177:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:30.0:1619677448.194178:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.194179:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.194180:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.194182:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.194183:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.194184:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.194185:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.194186:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.194190:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.194192:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.194195:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.194195:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.194196:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.194198:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.194198:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.194199:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.194200:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.194204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.194212:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.194214:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.194215:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.194217:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.194218:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.194220:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.194536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.194542:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.194545:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.194546:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.194547:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.194550:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.194551:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.194553:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.194554:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.194555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:2.0:1619677448.194555:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.194556:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.194559:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.194560:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.194563:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.194563:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.194564:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.194564:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.194566:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:2.0:1619677448.194566:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.194566:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.194567:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:31.0:1619677448.194568:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.194568:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.194569:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:30.0:1619677448.194570:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.194571:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.194572:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.194573:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.194573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.194574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000400:00000200:31.0:1619677448.194575:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.194576:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.194577:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.194580:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.194580:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.194581:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:1.0:1619677448.194581:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.194582:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.194582:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.194583:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:31.0:1619677448.194584:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.194584:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:31.0:1619677448.194585:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.194585:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.194585:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.194586:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.194587:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:1.0:1619677448.194587:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.194588:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.194589:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677448.194589:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677448.194590:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.194601:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.194604:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.194605:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.194606:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.194609:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.194610:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.194670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.194675:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.194678:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.194680:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.194681:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.194682:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.194684:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.194685:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.194687:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.194688:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.194689:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.194692:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.194694:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.194696:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.194697:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.194698:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.194700:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.194700:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.194701:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.194701:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.2:1619677448.194702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.194710:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.194713:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:22.0:1619677448.194714:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:30.0:1619677448.194715:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.194716:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677448.194717:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:22.0:1619677448.194717:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.194718:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.194718:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:30.0:1619677448.194720:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:22.0:1619677448.194720:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.194721:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.194723:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.194723:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:30.0:1619677448.194724:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.194725:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.194728:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.194730:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.194732:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.194733:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.194734:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.194735:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.194736:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.194737:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.194737:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677448.194747:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.194750:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.194750:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.194752:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.194754:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.194754:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677448.194765:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.194767:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.194768:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.194769:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.194770:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.194771:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.194782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.194788:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.194791:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.194791:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.194794:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.194796:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.194797:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.194799:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.194800:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.194801:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.194802:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.194806:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.194808:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.194811:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.194811:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.194812:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.194814:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.194814:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.194815:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.194815:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.194820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:21.2:1619677448.194822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.194828:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.194830:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677448.194830:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:24.0:1619677448.194831:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.194832:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:24.0:1619677448.194833:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.194834:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.194836:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.194837:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.194838:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:8.0:1619677448.194838:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:24.0:1619677448.194839:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.194840:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.194841:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.194842:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.194843:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.194846:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.194848:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.194851:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.194851:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.194852:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.194854:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.194854:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.194854:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.194855:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677448.194866:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.194868:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.194869:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.194870:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.194872:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.194873:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.194927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.194933:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.194936:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.194937:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.194938:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.194941:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.194942:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.194943:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.194944:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.194945:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.194946:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.194949:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.194950:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.194952:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.194952:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.194953:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.194955:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.194955:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.194956:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.194956:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.194961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.194968:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.194970:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.194971:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.194972:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.194972:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.194974:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.195172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.195179:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.195182:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.195184:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.195186:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.195186:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.195188:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.195189:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.195190:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.195191:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.195192:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.195195:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.195197:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.195199:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.195200:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.195201:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.195202:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.195203:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.195204:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.195206:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.195206:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677448.195212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.195219:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.195222:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:12.0:1619677448.195222:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.195223:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.195224:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.195224:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.195225:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.195227:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.195229:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.195230:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.195239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.195243:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.195246:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:13.0:1619677448.195247:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.195248:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.195249:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:13.0:1619677448.195250:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.195251:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.195251:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.195251:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:20.0:1619677448.195252:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.195253:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.195253:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.195254:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:20.0:1619677448.195255:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.195256:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.195257:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:20.2:1619677448.195260:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:20.0:1619677448.195265:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.195266:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:30.0:1619677448.195268:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.195268:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.195269:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.195269:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:30.0:1619677448.195270:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:21.2:1619677448.195271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:20.0:1619677448.195271:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.195271:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000400:00000200:30.0:1619677448.195272:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.195272:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.195273:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:31.0:1619677448.195274:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.195274:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.195276:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:8.2:1619677448.195277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:6.0:1619677448.195277:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:30.0:1619677448.195278:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.195279:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.195280:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.195280:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:30.0:1619677448.195281:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.195282:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.195283:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.195285:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:30.0:1619677448.195286:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.195286:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.195287:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:30.0:1619677448.195288:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.195288:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:5.0:1619677448.195288:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.195289:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96718095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.195290:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.195290:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.195291:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.195292:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.195292:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.195292:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:30.0:1619677448.195293:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.195298:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:6.0:1619677448.195298:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:8.2:1619677448.195299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000800:00000200:30.0:1619677448.195300:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.195300:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.195302:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:6.0:1619677448.195302:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.195303:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.195304:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:6.0:1619677448.195304:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:8.2:1619677448.195306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.195307:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.195308:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.195309:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.195310:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:20.2:1619677448.195311:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:6.0:1619677448.195311:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.195312:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.195312:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677448.195315:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.195316:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.195317:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.195318:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.195318:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.195319:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.195319:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.195319:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.195320:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.195320:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.195321:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.195323:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:5.0:1619677448.195323:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677448.195324:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677448.195324:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:10.0:1619677448.195324:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:31.0:1619677448.195325:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:10.0:1619677448.195327:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.195328:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.195328:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.195330:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.195332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.195333:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.0:1619677448.195340:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.195342:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.195343:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.195344:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.195345:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.195346:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.195392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.195397:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.195399:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.195401:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.195402:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.195403:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.195404:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677448.195405:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.195406:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.195407:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.195408:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677448.195411:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.195413:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677448.195415:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.195416:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677448.195417:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677448.195419:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677448.195419:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.195420:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.195420:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.195425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.2:1619677448.195431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.195436:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.195438:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.195438:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.195439:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.195440:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.195441:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.195443:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.195444:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.195445:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.195445:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677448.195446:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.195447:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.195449:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.195450:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.195451:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.195453:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.195456:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.195458:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.195460:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.195461:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.195462:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.195464:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.195465:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.195465:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.195466:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.195470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.195477:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.195480:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.195480:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.195482:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.195483:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.195485:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.195643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.195651:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.195653:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.195654:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.195656:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.195658:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.195660:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.195661:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.195662:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.195663:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.195665:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.195668:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.195669:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.195672:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.195672:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.195673:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.195675:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.195675:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.195676:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.195676:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.195705:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.195707:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.195708:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.195710:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.195712:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.195713:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.195806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677448.195813:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.195816:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677448.195817:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677448.195818:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.195818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677448.195821:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.195822:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.195823:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.195824:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.195824:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.195825:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.195826:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:1.0:1619677448.195827:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.195828:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.195830:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.195831:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.195833:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:1.0:1619677448.195834:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.195835:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.195835:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:1.0:1619677448.195835:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:10.0:1619677448.195836:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:1.0:1619677448.195837:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.195838:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.195838:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000400:00000200:1.0:1619677448.195838:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.195839:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.195839:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:1.0:1619677448.195839:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.195840:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.195844:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.195845:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.195848:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.195849:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.195849:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:31.0:1619677448.195850:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:1.0:1619677448.195851:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:31.0:1619677448.195852:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:1.0:1619677448.195852:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.195853:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.195853:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.195854:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000400:00000200:31.0:1619677448.195855:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.195855:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.195857:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.195857:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:9.2:1619677448.195857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.2:1619677448.195859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.195864:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:17.0:1619677448.195865:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677448.195866:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:17.0:1619677448.195867:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:24.0:1619677448.195868:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.195868:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677448.195868:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677448.195869:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677448.195870:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.195873:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677448.195945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.195950:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.195953:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.195954:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677448.195955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:2.0:1619677448.195956:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.195959:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.195960:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:5.0:1619677448.195961:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:2.0:1619677448.195962:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718089] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.195963:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:2.0:1619677448.195963:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.195964:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.195966:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:2.0:1619677448.195966:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.195967:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.195969:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.195969:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.195971:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:2.0:1619677448.195971:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.195972:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.195973:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.195975:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.195975:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.195976:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.195976:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.195976:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.195978:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:5.0:1619677448.195979:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.195980:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677448.195981:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.195982:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.195983:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.195983:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677448.195984:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.195984:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677448.195985:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677448.195987:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677448.195988:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.2:1619677448.195989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677448.195989:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:26.0:1619677448.195991:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:5.0:1619677448.195991:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.195992:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677448.195993:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:9.2:1619677448.195993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:10.0:1619677448.195994:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.195995:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.195996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677448.195997:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.195997:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.195999:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:26.0:1619677448.196000:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.0:1619677448.196000:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:18.0:1619677448.196000:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.0:1619677448.196002:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.196002:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677448.196002:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.196002:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677448.196003:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677448.196004:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.196005:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.196006:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.196007:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.196010:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.196035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.196042:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.196045:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.196046:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.196047:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.196049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:10.0:1619677448.196049:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.196051:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.196052:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.196054:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.196054:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.196055:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:10.0:1619677448.196056:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.196058:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.196059:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:10.0:1619677448.196059:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.196061:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677448.196062:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.196063:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.196065:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.196065:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:10.0:1619677448.196066:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:2.0:1619677448.196066:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.196067:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:2.0:1619677448.196067:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.196068:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:2.0:1619677448.196068:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.196069:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677448.196069:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.196070:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.196070:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677448.196073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.2:1619677448.196073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:2.0:1619677448.196073:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.196074:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.2:1619677448.196075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.196076:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.196076:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.196077:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.196078:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.196079:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.196079:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.196080:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:30.0:1619677448.196081:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.196081:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.0:1619677448.196082:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677448.196083:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.196084:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:9.2:1619677448.196084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.196085:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677448.196086:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:20.0:1619677448.196086:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.196086:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:30.0:1619677448.196087:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.196087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677448.196088:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677448.196088:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.196089:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.196090:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.196090:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.196091:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:30.0:1619677448.196092:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677448.196093:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.196093:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.196093:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:30.0:1619677448.196094:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.196094:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.196094:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:26.0:1619677448.196095:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.196095:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.196095:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:30.0:1619677448.196096:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.196096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.196096:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:30.0:1619677448.196097:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677448.196097:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:26.0:1619677448.196098:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.196098:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.196098:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:26.0:1619677448.196099:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.196100:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:26.0:1619677448.196101:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.196102:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.196102:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.196103:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.196105:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.196106:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.196107:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.196107:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.196107:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:30.0:1619677448.196108:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.196108:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.196109:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677448.196109:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.196109:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.196109:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:9.2:1619677448.196109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:30.0:1619677448.196111:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677448.196112:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.196112:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.196112:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.196113:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.196113:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.196114:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.196116:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.196116:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.196117:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:2.0:1619677448.196117:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.196119:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.196119:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:2.0:1619677448.196120:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.196121:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:2.0:1619677448.196121:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (23)++
00000800:00000200:18.0:1619677448.196122:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (25)--
00000800:00000200:2.0:1619677448.196122:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (25)--
00000800:00000200:2.0:1619677448.196122:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (24)--
00000400:00000200:18.0:1619677448.196123:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.196124:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677448.196125:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:18.0:1619677448.196125:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.196125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677448.196128:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.196128:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.196129:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.196129:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:22.0:1619677448.196131:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.196132:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677448.196134:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:10.0:1619677448.196134:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.196135:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.196136:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677448.196165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.196170:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.196173:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.196173:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.196175:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.196177:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.196179:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.196180:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.196181:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.196182:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.196183:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.196187:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.196189:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.196192:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.196192:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.196193:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.196195:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.196195:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.196196:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.196197:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:21.2:1619677448.196205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:10.0:1619677448.196207:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.196209:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:5.0:1619677448.196209:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:26.0:1619677448.196210:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.196210:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.196211:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:10.0:1619677448.196213:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.196214:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.196214:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.196217:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:5.0:1619677448.196217:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.196218:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.196219:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.196221:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.196222:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.196223:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.196226:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.196228:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.196230:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.196231:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.196232:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.196233:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.196234:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.196235:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.196235:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.196240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.196247:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.196250:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.196250:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.196252:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.196253:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.196254:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.2:1619677448.196298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.196305:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.196308:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.196309:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.196311:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.196313:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.196315:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.196316:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.196317:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.196318:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.196319:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.196322:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.196324:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.196327:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.196328:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.196328:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.196330:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.196331:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.196332:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677448.196333:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.196333:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677448.196337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677448.196340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:17.0:1619677448.196345:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:10.0:1619677448.196347:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:17.0:1619677448.196348:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.196348:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.196349:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.196349:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.196350:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:17.0:1619677448.196351:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.196351:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:17.0:1619677448.196354:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.196356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.196363:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.196365:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.196366:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.196367:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.196368:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.196368:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.196411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.196419:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.196422:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.196424:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.196427:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.196427:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.196429:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.196431:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.196432:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.196433:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.196434:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.196437:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.196438:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.196441:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.196442:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.196443:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.196445:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.196445:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.196446:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.196447:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.196451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.196459:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.196461:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.196462:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.196464:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.196464:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.196467:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.196567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.196574:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.196577:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.196578:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.196580:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.196583:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.196584:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.196586:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.196587:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.196588:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.196589:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.196592:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.196593:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.196596:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.196597:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.196598:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.196599:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.196600:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.196600:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.196601:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677448.196615:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.196617:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.196618:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.196619:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.196621:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.196623:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.196646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.196655:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.196657:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.196660:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.196661:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.196663:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.196664:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.196665:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.196667:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.196668:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.196669:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.196672:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.196674:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.196676:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.196676:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.196677:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.196678:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.196679:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.196679:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.196680:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677448.196690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677448.196691:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.196694:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.196694:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.196696:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:12.0:1619677448.196696:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.196698:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.196699:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:13.0:1619677448.196699:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.196700:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.196701:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.196704:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.196706:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.196707:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.196708:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.196709:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.196710:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.196713:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.196715:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.196716:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.196717:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.196718:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.196719:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.196720:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.196720:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.196721:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.196725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:21.2:1619677448.196730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:1.0:1619677448.196732:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.196734:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.196734:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.196735:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:2.0:1619677448.196736:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:1.0:1619677448.196736:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.196737:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:2.0:1619677448.196739:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.196740:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.196741:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.196744:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.196746:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.196747:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.196748:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.196749:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.196750:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.196754:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.196755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.196755:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.196759:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.196760:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.196761:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.196762:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.196763:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.196763:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.196764:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:18.0:1619677448.196765:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.196765:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.196766:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677448.196767:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.196768:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.196769:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.196770:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.196772:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.196772:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.196773:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.196775:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:18.0:1619677448.196776:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.196778:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:18.0:1619677448.196778:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.196778:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.196779:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.196781:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.196782:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:10.0:1619677448.196782:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.196783:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.196784:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:18.0:1619677448.196785:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.196785:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.196786:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.196787:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677448.196797:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.196799:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.196801:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.196802:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.196804:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.196805:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.196974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.196981:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.196984:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.196984:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.196986:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.196989:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.196990:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.196992:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.196993:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.196994:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.196995:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.196998:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.197000:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.197002:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.197003:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.197004:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.197005:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.197006:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.197007:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.197008:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.197008:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.197014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.197021:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.197022:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.197023:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.197024:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.197028:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.197030:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.197031:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.197032:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.197034:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.197035:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.197049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.197057:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.197059:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.197061:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.197063:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.197063:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.197065:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.197066:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.197067:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.197068:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.197069:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.197071:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.197073:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.197076:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.197077:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.197077:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.197079:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.197079:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.197080:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.197081:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.197092:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.197095:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.197096:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.197096:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.197098:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.197102:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.197190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.197193:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.197196:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.197198:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.197199:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.197200:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.197202:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677448.197203:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.197205:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.197206:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.197207:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677448.197210:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.197212:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677448.197216:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.197217:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677448.197218:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.197219:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677448.197220:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.197221:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.197222:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677448.197232:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.197235:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.197236:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.197238:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.197238:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.197240:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.197246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.197249:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:17.0:1619677448.197251:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.197252:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.197253:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.197254:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.197254:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:17.0:1619677448.197254:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.197255:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.197256:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:17.0:1619677448.197257:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.197258:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.197260:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.197261:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.197263:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.197263:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.197265:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.197268:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.197269:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.197271:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677448.197271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677448.197272:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.197272:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.197273:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.197274:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.197275:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.197275:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.197277:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677448.197279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.197280:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.197281:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.197283:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.197283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000400:00000200:26.0:1619677448.197285:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.197287:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:5.0:1619677448.197287:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:1.0:1619677448.197287:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:26.0:1619677448.197288:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.197289:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.197289:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:5.0:1619677448.197290:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:1.0:1619677448.197290:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.197291:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.197291:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.197293:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.197293:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.197298:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.197299:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:26.0:1619677448.197300:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.2:1619677448.197301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000400:00000200:5.0:1619677448.197301:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.197302:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.197304:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.197305:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.197306:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.197307:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.197308:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.197308:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677448.197308:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:17.0:1619677448.197310:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.197311:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.197311:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:26.0:1619677448.197312:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.197312:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.197312:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000400:00000200:5.0:1619677448.197312:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.197313:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.197313:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677448.197314:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.197314:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677448.197315:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.197316:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.197316:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.197316:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.197317:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.197318:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.197319:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.197320:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.197321:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.197321:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.197326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:22.0:1619677448.197329:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.197332:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677448.197333:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:6.0:1619677448.197333:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.197334:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.197335:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:22.0:1619677448.197336:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.197336:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.197338:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.197339:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677448.197339:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.197340:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:6.0:1619677448.197349:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.197352:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.197352:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.197353:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.197354:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.197355:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.197358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.197365:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.197368:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.197369:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.197371:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.197374:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.197376:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.197377:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.197378:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.197379:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.197380:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.197384:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.197385:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.197389:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.197389:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.197390:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.197392:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.197392:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.197393:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.197394:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677448.197405:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.197407:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.197408:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.197409:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.197411:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.197414:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.197430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.197436:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.197438:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.197440:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.197441:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.197442:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.197444:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.197445:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.197447:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.197448:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.197449:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.197452:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.197454:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.197456:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.197457:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.197458:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.197459:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.197460:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.197461:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.197462:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.197466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.197473:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.197476:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.197477:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.197478:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.197479:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.197482:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.197517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.197524:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.197527:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.197527:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.197529:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.197530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677448.197533:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.197534:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.197535:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.197536:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.197536:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.197537:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.197539:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.197539:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.197539:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.197542:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.197545:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.197545:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.197547:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.197547:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.197549:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.197550:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.197550:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.197551:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.197551:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:1.0:1619677448.197551:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.197552:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.197553:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.197553:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.197554:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.197555:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:1.0:1619677448.197555:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.197557:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.197560:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.197560:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.197561:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.197563:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.197563:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.197564:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.197565:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.197570:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.197573:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.197574:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.197575:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.197576:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:30.0:1619677448.197577:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.197578:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:6.0:1619677448.197579:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.197580:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.197580:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.197583:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.197585:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.197597:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.197600:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.197601:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.197601:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.197602:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.197603:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.197682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:21.2:1619677448.197686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677448.197690:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.197693:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.197693:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:30.0:1619677448.197694:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:8.0:1619677448.197695:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.197696:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:8.0:1619677448.197697:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.197698:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.197699:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:31.0:1619677448.197700:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:8.0:1619677448.197700:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.197701:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.197701:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.197703:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677448.197703:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.197705:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.197705:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677448.197707:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.197708:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.197709:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677448.197709:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.197711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:8.0:1619677448.197712:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:30.0:1619677448.197714:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.197715:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.197715:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:30.0:1619677448.197716:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:17.0:1619677448.197716:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:8.0:1619677448.197716:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.197718:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677448.197719:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.197719:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:30.0:1619677448.197720:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.197720:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:17.0:1619677448.197720:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:8.0:1619677448.197720:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.197720:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:1.0:1619677448.197720:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:30.0:1619677448.197722:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.197723:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677448.197723:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.197724:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:9.2:1619677448.197724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.197725:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:17.0:1619677448.197725:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.197726:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677448.197726:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:9.2:1619677448.197727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:17.0:1619677448.197728:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96718062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.197729:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.197730:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.197731:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:21.2:1619677448.197732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.197733:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.197735:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:17.0:1619677448.197735:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.197736:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.197736:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:17.0:1619677448.197737:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677448.197737:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.197738:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.197738:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.197740:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:17.0:1619677448.197740:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.197741:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:17.0:1619677448.197741:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:2.0:1619677448.197741:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.197743:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:18.0:1619677448.197743:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677448.197743:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:31.0:1619677448.197744:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677448.197744:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677448.197744:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:31.0:1619677448.197745:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677448.197745:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:2.0:1619677448.197745:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.197746:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:17.0:1619677448.197746:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:4.2:1619677448.197755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.197758:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.197761:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.197762:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.197763:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.197764:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.197764:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.197768:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.197768:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.197770:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.197770:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.197771:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.197772:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000400:00000200:26.0:1619677448.197773:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:10.0:1619677448.197773:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.197773:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.197774:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.197775:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.197776:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.197776:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:26.0:1619677448.197777:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.197778:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.197781:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.197783:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.197786:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.197786:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.197787:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.197789:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.197790:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.197790:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.197791:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.197795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.197803:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.197806:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.197807:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.197807:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.197809:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.197812:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.197814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.197821:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.197823:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.197824:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.197825:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.197826:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.197826:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.197843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.197849:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.197851:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.197852:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.197855:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.197858:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.197859:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.197860:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.197861:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.197862:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.197863:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.197866:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.197868:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.197871:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.197872:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.197872:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.197874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677448.197874:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.197874:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.197876:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.197877:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.197877:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.197881:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.2:1619677448.197882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.197884:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.197886:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.197888:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677448.197888:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677448.197888:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:31.0:1619677448.197889:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.197890:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:17.0:1619677448.197890:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.197891:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.197891:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:31.0:1619677448.197892:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.197892:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677448.197892:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.197893:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.197893:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.197894:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.197895:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.197895:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:31.0:1619677448.197898:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.197900:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:21.2:1619677448.197900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:31.0:1619677448.197902:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.197902:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.197903:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:13.0:1619677448.197903:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.197904:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.197904:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.197905:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.197905:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677448.197906:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677448.197906:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.197906:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677448.197906:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677448.197907:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.197907:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677448.197907:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.197908:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.197909:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.197911:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.197914:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.197916:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:12.0:1619677448.197917:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:6.0:1619677448.197917:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.197919:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.197919:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.197920:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.197920:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.197921:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:12.0:1619677448.197922:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.197924:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:6.0:1619677448.197925:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.197927:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677448.197927:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.197930:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.197931:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.197931:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.197934:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.197934:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.197935:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.197936:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677448.197957:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.197960:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.197962:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.197963:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.197964:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.197967:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.197968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.197975:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.197978:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.197978:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.197979:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.197981:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.197981:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.197982:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.197982:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.197983:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.197984:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:10.0:1619677448.197984:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.197986:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.197987:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.197988:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.197989:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.197990:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.197994:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.197995:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.197999:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.197999:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.198000:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.198002:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.198002:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.198003:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.198004:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.198005:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.198010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.2:1619677448.198013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:22.0:1619677448.198015:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677448.198016:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.198017:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.198018:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.198019:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.198019:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:6.0:1619677448.198019:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.198020:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.198020:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.198021:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:20.0:1619677448.198021:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.198022:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.198022:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.198024:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.198025:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:22.0:1619677448.198026:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:20.0:1619677448.198027:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.198028:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.198029:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.198030:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.198034:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.198035:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.198038:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.198039:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.198040:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.198041:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.198042:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.198043:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.198044:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.198054:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.198056:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.198057:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.198059:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.198060:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.198062:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.198158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.198166:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.198168:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.198169:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.198171:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.198174:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.198176:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.198177:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.198178:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.198179:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.198180:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.198183:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.198185:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.198187:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.198187:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.198188:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.198189:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.198190:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.198191:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.198191:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.198212:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.198214:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.198215:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.198216:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.198218:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.198219:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.198242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.198249:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.198251:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.198252:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.198254:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.198256:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.198257:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.198259:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.198260:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.198261:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.198262:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.198264:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.198265:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.198267:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.198268:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.198268:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.198270:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.198270:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.198271:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.198271:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.198281:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.198283:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.198284:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.198285:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.198286:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.198287:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.198314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.198320:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.198322:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.198323:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.198325:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.198328:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.198330:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.198331:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.198332:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.198333:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.198334:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.198338:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.198339:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.198342:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.198343:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.198344:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.198345:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.198346:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.198346:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.198347:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.198360:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.198362:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.198363:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.198365:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.198367:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.198368:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.198451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.198459:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.198462:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.198462:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.198464:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.198466:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.198467:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:21.2:1619677448.198467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000400:00000200:26.0:1619677448.198468:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.198470:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.198471:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.198471:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:26.0:1619677448.198472:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.198474:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:26.0:1619677448.198476:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.198477:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.198477:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.198478:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.198480:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.198480:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.198481:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.198482:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:5.0:1619677448.198482:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:26.0:1619677448.198483:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.198483:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677448.198483:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.198484:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:5.0:1619677448.198484:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.198485:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:5.0:1619677448.198486:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.198486:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.198490:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.198492:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677448.198494:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:5.0:1619677448.198494:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.198495:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.198495:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:10.0:1619677448.198496:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.198497:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.198497:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.198497:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.198497:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000400:00000200:10.0:1619677448.198499:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.198499:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:10.0:1619677448.198500:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.0:1619677448.198500:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:5.0:1619677448.198501:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:8.2:1619677448.198504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:21.2:1619677448.198506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.198511:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.198512:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677448.198512:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.198513:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000400:00000200:5.0:1619677448.198513:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.198514:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.198515:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.198515:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.198516:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.198516:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677448.198517:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:26.0:1619677448.198518:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.198519:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.198520:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.198520:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.198521:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.198578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.198587:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.198589:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.198591:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.198594:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.198594:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.198596:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.198597:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.198598:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.198599:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.198600:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.198603:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.198605:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.198607:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.198608:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.198608:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.198610:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.198610:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.198611:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.198612:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677448.198618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:13.0:1619677448.198622:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.198624:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.198625:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.198625:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:13.0:1619677448.198626:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.198628:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.198628:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677448.198628:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.198629:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.198631:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.198634:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.198635:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.198637:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.198638:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.198639:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.198640:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.198644:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.198645:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.198647:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.198648:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.198649:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.198651:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.198651:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.198652:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.198653:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.198654:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.198657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.198659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.198664:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:17.0:1619677448.198665:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677448.198666:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.198667:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.198668:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677448.198668:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.198669:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.198669:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.198671:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.198673:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.198684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.198687:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.198689:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.198691:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.198693:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.198694:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.198695:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.198697:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.198698:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.198699:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.198700:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.198704:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.198706:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.198709:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.198710:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.198711:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.198712:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.198713:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.198714:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.198714:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.2:1619677448.198715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.198721:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.198724:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.198725:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.198727:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.198729:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.198731:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.198732:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.198733:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.198734:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.198735:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:1.0:1619677448.198736:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.198738:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.198739:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.198739:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.198741:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.198741:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.198742:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.198744:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:1.0:1619677448.198744:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.198745:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.198745:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.198747:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.198748:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.198748:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.198749:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.198759:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.198762:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.198763:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.198765:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.198766:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.198767:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677448.198777:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.198779:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.198780:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.198781:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.198782:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.198783:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.198837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:4.2:1619677448.198840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.198846:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.198848:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.198848:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.198849:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.198851:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.198852:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.198853:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.198854:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.198855:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.198856:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.198858:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.198859:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.198860:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.198862:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.198863:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.198864:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.198865:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.198866:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.198866:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.198866:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.198870:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.198870:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.198871:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.198872:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.198872:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.198874:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.198875:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.198875:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.198875:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.198876:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.198876:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (23)++
00000800:00000200:10.0:1619677448.198877:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (24)--
00000800:00000200:26.0:1619677448.198878:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.198878:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.198879:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.198879:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.198879:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.198880:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.198880:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677448.198883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.198887:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:26.0:1619677448.198890:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677448.198890:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:31.0:1619677448.198891:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677448.198891:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.198891:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.198893:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:26.0:1619677448.198893:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:31.0:1619677448.198894:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.198894:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677448.198894:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.198895:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:31.0:1619677448.198896:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.198898:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.198898:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677448.198900:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.198909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677448.198917:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677448.198920:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677448.198920:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677448.198922:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677448.198925:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.198926:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677448.198927:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.198928:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.198930:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.198930:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677448.198933:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677448.198934:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.198936:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.198936:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677448.198937:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.198938:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.198939:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677448.198940:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677448.198941:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677448.198942:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:5.2:1619677448.198947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677448.198954:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677448.198956:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677448.198957:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677448.198958:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:5.2:1619677448.198981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.198987:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.198990:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.198991:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.198993:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.198996:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.198997:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.198998:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.199000:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.199000:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.199001:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.199004:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.199006:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.199008:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.199009:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.199010:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.199011:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.199012:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.199013:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.199013:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.199017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.199025:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.199027:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.199028:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.199028:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.199031:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.199033:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:9.2:1619677448.199036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.199042:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.199044:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.199045:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.199045:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:21.2:1619677448.199046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:24.0:1619677448.199047:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677448.199050:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677448.199051:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.199053:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.199054:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.199055:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.199056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:24.0:1619677448.199057:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.199059:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.199060:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.199061:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.199062:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.199063:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.199064:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.199066:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:24.0:1619677448.199067:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.199068:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.199069:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.199070:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.199071:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.199072:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:24.0:1619677448.199072:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.199072:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.199073:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:30.0:1619677448.199074:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.199074:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:30.0:1619677448.199075:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.199075:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:30.0:1619677448.199076:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.199076:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.199076:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.199077:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.199077:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:30.0:1619677448.199080:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.199081:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.2:1619677448.199083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:30.0:1619677448.199084:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.199085:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.199086:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.199087:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:12.0:1619677448.199087:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677448.199088:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.199089:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:12.0:1619677448.199089:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:30.0:1619677448.199090:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:12.0:1619677448.199090:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.199090:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:12.0:1619677448.199091:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.199093:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (23)--
00000800:00000200:12.0:1619677448.199093:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.2:1619677448.199093:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:13.0:1619677448.199094:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677448.199097:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.199098:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677448.199103:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.199105:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.199107:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.199108:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.199108:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.199112:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.199188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.199193:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.199196:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.199197:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.199199:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.199202:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.199204:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.199205:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.199206:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.199208:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.199208:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.199212:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.199214:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.199217:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.199217:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.199218:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.199220:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.199220:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.199221:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.199222:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.199232:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.199235:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.199236:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.199237:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.199239:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.199240:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.199262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.199269:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.199272:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.199273:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.199275:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.199277:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.199278:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.199280:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.199281:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.199282:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.199283:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.199286:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.199288:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.199290:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.199291:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.199292:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.2:1619677448.199299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.199301:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677448.199301:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.199302:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.199303:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677448.199308:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.199311:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.199312:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.199312:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.199314:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.199317:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.199349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.199357:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.199360:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.199362:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.199364:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.199365:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.199366:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.199367:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.199369:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.199371:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.199371:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.199374:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.199376:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.199378:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.199379:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.199379:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.199380:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.199381:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.199382:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.199382:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.199394:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.199397:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.199397:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.199399:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.199401:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.199401:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677448.199410:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.199413:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.199413:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.199414:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.199414:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.199416:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.199427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.199433:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.199435:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.199437:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.199439:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.199440:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.199442:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.199443:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.199444:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.199445:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.199446:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.199449:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.199450:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.199453:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.199454:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.199454:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.199456:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.199456:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.199457:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.199458:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.199462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.199470:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.199472:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.199473:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.199475:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.199475:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.199477:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:8.2:1619677448.199484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.199492:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.199494:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.199495:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.199496:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.199497:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.199498:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.2:1619677448.199654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.199662:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.199665:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.199666:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.199668:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.199671:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.199672:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.199673:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.199674:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.199676:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.199677:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.199680:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.199681:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.199683:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.199684:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.199685:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.199686:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.199687:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.199687:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.199688:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677448.199698:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.199701:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.199702:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.199703:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.199705:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.199705:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.199740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.199747:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.199750:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.199751:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.199753:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.199755:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.199756:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.199758:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.199759:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.199760:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.199761:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.199764:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.199765:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.199768:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.199769:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.199769:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.199771:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.199771:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.199772:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.199772:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.2:1619677448.199777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.199783:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:22.0:1619677448.199785:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677448.199785:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.199786:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:22.0:1619677448.199787:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.2:1619677448.199787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:22.0:1619677448.199788:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.199788:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.199789:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:2.0:1619677448.199790:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.199791:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:2.0:1619677448.199792:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:1.0:1619677448.199792:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677448.199793:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.199794:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.199795:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.199795:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.199796:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:2.0:1619677448.199796:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.199798:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:1.0:1619677448.199799:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.199801:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.199801:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.199803:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:1.0:1619677448.199803:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.199804:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.199806:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.199806:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:1.0:1619677448.199806:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.199807:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:1.0:1619677448.199807:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.199808:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:2.0:1619677448.199809:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.199809:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.199810:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.199811:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.199812:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:1.0:1619677448.199812:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.199813:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.2:1619677448.199815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.199817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:1.0:1619677448.199817:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.199817:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.199818:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.199820:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.199820:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.199821:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.199822:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:18.0:1619677448.199823:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:1.0:1619677448.199823:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.199824:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677448.199825:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.199826:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.199827:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.2:1619677448.199828:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.199829:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.199830:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.199831:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.199833:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:5.0:1619677448.199833:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.199835:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:5.0:1619677448.199835:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677448.199836:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.199837:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677448.199849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.199857:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.199860:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.199862:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.199863:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.199864:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.199866:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.199867:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.199869:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.199870:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.199871:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.199874:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.199876:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.199879:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.199880:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.199881:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.199883:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.199883:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.199884:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.199886:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.199887:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.199889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:9.2:1619677448.199891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.2:1619677448.199892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.2:1619677448.199897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:8.0:1619677448.199898:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:5.0:1619677448.199899:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.199900:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:8.0:1619677448.199901:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:5.0:1619677448.199901:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.199902:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.199902:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.199903:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.199904:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.199905:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:31.0:1619677448.199906:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:8.0:1619677448.199906:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.199906:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.199907:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677448.199908:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:20.0:1619677448.199908:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.199908:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677448.199909:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.0:1619677448.199909:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:31.0:1619677448.199910:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.199910:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.199912:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677448.199913:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.199915:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.199916:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.199917:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677448.199920:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.199922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:20.0:1619677448.199922:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.199924:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.199925:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.199926:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.199928:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.199928:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:26.0:1619677448.199929:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:20.0:1619677448.199929:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.199930:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:26.0:1619677448.199932:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677448.199933:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677448.199935:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.199939:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.199941:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.199942:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.199942:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.199943:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.199944:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.199944:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.199945:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.199946:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.199946:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.199949:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.199949:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.199950:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.199951:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.199955:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.199955:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.199956:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.199957:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.199958:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.199958:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677448.199959:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677448.199964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.199972:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:21.2:1619677448.199973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:17.0:1619677448.199974:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.199975:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.199975:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.199978:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.199980:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:17.0:1619677448.199981:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:24.0:1619677448.199983:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.199984:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.199985:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.199988:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.199989:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.199990:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.199992:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.199993:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.199993:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.199996:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.199998:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.200000:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.200000:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.200001:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.200002:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.200003:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.200004:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.200004:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677448.200014:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.200017:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.200017:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.200019:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.200021:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.200022:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.200396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:5.2:1619677448.200397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:6.0:1619677448.200402:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677448.200403:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.200405:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.200406:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677448.200406:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.200407:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677448.200409:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.200411:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.200412:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.200413:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677448.200413:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.200415:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.200415:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:6.0:1619677448.200417:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.200417:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.200419:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.200419:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.200420:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677448.200420:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.200422:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:6.0:1619677448.200426:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.200427:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.200429:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677448.200430:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:6.0:1619677448.200432:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.200433:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:1.0:1619677448.200433:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.200434:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:1.0:1619677448.200434:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.200435:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:6.0:1619677448.200436:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.200437:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677448.200437:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:6.0:1619677448.200438:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.200438:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677448.200438:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.200438:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.200439:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.200450:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.200453:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.200453:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.200454:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.200456:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.200457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:21.2:1619677448.200458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:10.0:1619677448.200459:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.0:1619677448.200462:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:8.0:1619677448.200464:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:5.0:1619677448.200464:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:8.0:1619677448.200466:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.200467:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.200467:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.200467:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.200468:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.200469:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.200470:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.200471:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:8.0:1619677448.200472:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:5.0:1619677448.200472:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.200474:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.200475:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.200476:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.200479:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.200481:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.200483:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.200484:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.200484:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:9.2:1619677448.200485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677448.200486:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.200487:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.200487:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.200488:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:24.0:1619677448.200491:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.2:1619677448.200492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:24.0:1619677448.200494:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.200496:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.200496:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.200497:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.200498:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.0:1619677448.200501:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.200503:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.200504:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.200505:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.200506:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.200508:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.2:1619677448.200598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.200606:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.200609:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.200611:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.200614:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.200614:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.200616:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.200617:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.200619:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.200620:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.200621:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.200624:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.200626:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.200629:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.200630:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.200631:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.200632:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.200633:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.200634:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.200634:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:21.2:1619677448.200636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:20.2:1619677448.200636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677448.200639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.200643:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.200645:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:8.0:1619677448.200647:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:5.0:1619677448.200647:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.200648:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.200649:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.200649:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.200650:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:20.0:1619677448.200650:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.200650:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:20.0:1619677448.200652:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.200652:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677448.200653:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.200654:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.200655:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.200655:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.200656:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.200657:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.200657:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677448.200658:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96718023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.200660:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.200661:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:8.0:1619677448.200661:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.200662:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.200662:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.200662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:8.0:1619677448.200663:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.200664:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.200664:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.200666:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.200667:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.200667:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000400:00000200:8.0:1619677448.200667:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.200668:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.200668:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.200669:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.200669:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677448.200669:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.200669:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677448.200670:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.200671:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.200672:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:5.0:1619677448.200672:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.200673:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677448.200673:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.200673:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:26.0:1619677448.200675:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.200675:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.200675:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.200675:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.200675:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677448.200677:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.200678:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:8.0:1619677448.200678:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.200678:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.200679:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:26.0:1619677448.200680:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96718022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.200680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677448.200680:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:26.0:1619677448.200681:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.200682:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.200683:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677448.200683:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677448.200684:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:9.2:1619677448.200684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.200685:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:26.0:1619677448.200686:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.200686:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.200687:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.200687:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:26.0:1619677448.200688:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:12.0:1619677448.200688:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.200689:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.200691:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.200692:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.200692:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:12.0:1619677448.200692:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.200692:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.200693:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.200694:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.200695:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.200695:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.200696:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:2.0:1619677448.200696:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.200697:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:18.0:1619677448.200697:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.200698:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677448.200698:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.200702:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.200703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.200709:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677448.200710:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.200711:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.200712:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.200712:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677448.200712:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.200713:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.200714:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.200715:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.200718:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.200732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.200738:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.200740:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.200741:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.200743:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.200745:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.200747:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.200748:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.200749:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.200750:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.200751:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.200755:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.200756:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.200759:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.200759:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.200760:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.200762:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.200762:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.200763:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.200763:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.200768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.200774:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.200776:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.200777:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.200778:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.200779:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.200780:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.200858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.200866:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.200869:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.200871:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.200872:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.200874:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.200875:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677448.200877:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.200878:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.200878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677448.200879:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.200880:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677448.200883:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.200884:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.200884:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.200886:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:30.0:1619677448.200887:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.200887:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.200888:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677448.200889:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.200890:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:17.0:1619677448.200890:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.200891:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.200892:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677448.200892:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.200893:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:17.0:1619677448.200894:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.200895:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.200896:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.200897:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.200898:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.200902:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.200904:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:6.0:1619677448.200904:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.200907:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.200907:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.200907:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677448.200908:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.200908:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:6.0:1619677448.200908:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.200910:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:6.0:1619677448.200910:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.200911:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.200911:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.200912:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.200913:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677448.200923:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.200926:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.200927:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.200928:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.200929:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.200931:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.201068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.201075:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.201078:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.201079:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.201081:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.201084:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.201085:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.201086:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.201088:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.201089:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.201090:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.201092:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.201094:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.201096:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.201097:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.201097:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.201099:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.201099:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.201100:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.201100:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:4.2:1619677448.201106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677448.201110:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.201112:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.201113:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.201113:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677448.201114:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.201114:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.201116:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:24.0:1619677448.201116:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.201117:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.201119:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.201121:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.201122:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.201124:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96718017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.201125:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.201126:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.201127:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.201130:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.201132:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.201134:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.201135:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.201136:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.201137:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.201138:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.201138:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.201139:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.201143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.201151:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.201154:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.201155:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.201155:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.201158:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.201161:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.201163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.201170:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.201173:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.201174:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.201174:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.201175:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.201175:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.201205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.201211:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.201213:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.201214:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.201216:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.201218:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.201220:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677448.201221:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.201222:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.201223:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.201224:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677448.201228:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.201229:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677448.201232:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.201233:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677448.201234:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.201235:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677448.201236:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.201236:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677448.201237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677448.201237:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:31.0:1619677448.201245:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.201247:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.201250:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.201251:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.201252:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.201252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677448.201254:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.201255:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.201256:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.201257:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.201258:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.201260:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:31.0:1619677448.201262:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.201263:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:31.0:1619677448.201264:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:20.0:1619677448.201264:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.201265:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.201267:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.201267:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.201268:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.201268:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.201268:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677448.201270:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.201270:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.201271:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.201272:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.201276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.201284:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.201287:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.201287:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.201289:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.201290:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.201292:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.201293:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677448.201298:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.201300:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.201303:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.201303:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.201305:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.201306:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.201307:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718014] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.201309:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.201310:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.201311:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.201314:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.201316:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.201318:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.201319:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.201320:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.201321:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.201322:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.201323:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.201325:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.201325:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:9.2:1619677448.201328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677448.201330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.201333:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.201335:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.201336:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.201336:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:2.0:1619677448.201336:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.201339:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.201340:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.201341:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.201342:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.201344:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.201352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.201355:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.201357:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.201359:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.201360:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.201361:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.201363:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677448.201364:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.201366:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.201367:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.201368:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677448.201371:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.201373:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677448.201375:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.201376:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677448.201376:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677448.201378:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677448.201379:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.201380:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.201381:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.201385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.201393:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.201395:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.201396:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.201396:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.201399:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.201401:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.201414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.201421:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.201424:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.201425:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.201425:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.201426:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.201426:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.201544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.0:1619677448.201547:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.201550:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677448.201552:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.201553:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.201555:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.201556:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.201558:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.201559:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.201560:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.201561:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.201564:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.201565:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.201567:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.201568:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.201568:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.201570:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.201570:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.201571:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.201572:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.201576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.201584:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.201586:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.201587:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.201589:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.201590:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.201591:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.2:1619677448.201599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.201606:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.201609:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.201610:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.201612:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.201614:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.201616:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.201617:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.201618:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.201620:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.201620:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.201624:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.201625:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.201628:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.201628:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.201629:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.201630:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.201631:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.201632:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.201632:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.201637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.201643:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.201645:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.201645:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.201646:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.201647:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.201648:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.201675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.201682:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.201684:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.201685:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.201687:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.201689:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.201691:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.201692:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.201693:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.201694:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.201695:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.201698:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.201700:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.201703:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.201704:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.201704:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.201706:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.201707:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.201707:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.201708:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:21.2:1619677448.201712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.201719:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:12.0:1619677448.201720:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:22.0:1619677448.201721:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:12.0:1619677448.201722:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:6.0:1619677448.201722:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:12.0:1619677448.201723:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.201724:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677448.201725:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.201727:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.201727:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:22.0:1619677448.201728:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:12.0:1619677448.201728:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.201728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:22.0:1619677448.201730:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.201731:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.201732:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.201732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000400:00000200:22.0:1619677448.201733:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.201736:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.201736:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.201738:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:22.0:1619677448.201739:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:21.2:1619677448.201739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677448.201740:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:13.0:1619677448.201740:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:26.0:1619677448.201741:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000400:00000200:10.0:1619677448.201741:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.201742:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.201743:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:22.0:1619677448.201743:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:13.0:1619677448.201743:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:22.0:1619677448.201744:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:10.0:1619677448.201744:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.201746:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.201746:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000400:00000200:13.0:1619677448.201746:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.201746:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:22.0:1619677448.201747:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.201747:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677448.201747:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677448.201748:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677448.201748:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677448.201750:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.201750:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.201751:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.201751:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.201752:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:13.0:1619677448.201753:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96718007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.201754:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.201755:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.201756:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.201757:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:13.0:1619677448.201757:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.201757:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.201758:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96718006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.201759:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.201760:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.201761:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677448.201761:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.201762:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:13.0:1619677448.201763:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677448.201766:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.201766:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.201767:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.201767:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.201767:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:13.0:1619677448.201768:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.201768:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.201769:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677448.201770:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.201770:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:13.0:1619677448.201771:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:10.0:1619677448.201771:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677448.201772:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:10.0:1619677448.201772:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.201773:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.201773:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:10.0:1619677448.201773:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.201774:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.201774:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.201775:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677448.201776:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.201777:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.201777:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:9.2:1619677448.201777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.201778:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677448.201778:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.201779:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.201780:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677448.201780:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677448.201781:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.201781:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.201783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.201784:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:6.0:1619677448.201785:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677448.201785:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:21.2:1619677448.201786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:6.0:1619677448.201788:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677448.201788:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:22.0:1619677448.201789:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:18.0:1619677448.201789:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:6.0:1619677448.201789:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.201790:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.201790:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:6.0:1619677448.201792:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.201793:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:2.0:1619677448.201793:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.201793:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.201794:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.201794:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.201795:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:26.0:1619677448.201796:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:2.0:1619677448.201796:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:1.0:1619677448.201796:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.201797:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:9.2:1619677448.201798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.2:1619677448.201798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677448.201799:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:10.0:1619677448.201802:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:10.0:1619677448.201803:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677448.201805:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.201806:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.201806:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:18.0:1619677448.201808:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.201808:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.201809:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000400:00000200:10.0:1619677448.201809:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.201811:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.201811:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:10.0:1619677448.201813:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.201814:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.201815:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.201816:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.201817:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.201818:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.201821:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.201823:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.201824:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.201825:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.201825:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.201826:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.201827:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.201828:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.201829:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677448.201838:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.201840:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.201841:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.201841:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.201842:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.201844:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.201905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.201913:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.201916:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.201918:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.201920:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.201920:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.201922:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.201923:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.201924:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.201925:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.201926:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.201928:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.201930:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.201932:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.201933:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.201933:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.201935:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.201935:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.201937:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.201939:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.201939:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677448.201945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.201947:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.201949:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.201950:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.201951:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.201953:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.201953:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.201954:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.201955:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.201956:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.201957:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.201966:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.201969:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.201970:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.201970:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.201971:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.201972:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.201999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.202007:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.202009:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.202010:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.202012:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.202014:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.202015:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677448.202016:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96718003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.202018:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.202019:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.202019:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.202022:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.202024:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.202027:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.202028:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.202029:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.202031:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.202031:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.202032:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.202032:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.202037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.202045:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.202048:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.202049:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.202051:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.202051:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.202054:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.202088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.202091:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.202094:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.202096:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.202096:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.202098:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.202099:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.202101:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.202102:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.202103:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.202104:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.202107:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.202109:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.202112:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.202113:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.202113:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.202115:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.202116:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.202116:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.202117:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.202128:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.202131:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.202132:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.202133:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.202135:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.202136:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.202144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.202150:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677448.202153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.202153:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.202154:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.202156:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.202159:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677448.202159:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.202160:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:17.0:1619677448.202161:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:2.0:1619677448.202162:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96718001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.202162:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:2.0:1619677448.202164:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.202165:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.202165:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.202166:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.202167:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.202169:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.202170:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96718000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.202170:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.202172:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.202173:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.202173:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.202174:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.202176:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.202177:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:17.0:1619677448.202178:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.202178:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.202180:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.202180:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.202181:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.202181:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677448.202182:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.202182:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677448.202183:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.202184:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.202185:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.202185:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.202187:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.202188:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.202189:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.202191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677448.202193:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.2:1619677448.202194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.202196:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.202196:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.202198:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.202199:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:13.0:1619677448.202200:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.202201:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677448.202201:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.202201:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.202202:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:1.0:1619677448.202202:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.202203:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.202204:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.202204:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.202205:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.202205:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.202349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677448.202351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.202357:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.202358:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.202360:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677448.202361:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.202362:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677448.202363:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.202365:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677448.202365:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.202366:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.202367:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.202368:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.202369:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.202369:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.202371:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.202371:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.202373:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.202373:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.202374:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.202374:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.202376:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.202379:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.202381:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.202383:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.202384:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.202385:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.202385:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.202386:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.202388:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.202388:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.202389:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.202389:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.202389:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.202390:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.202390:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.202392:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.202392:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.202393:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.202393:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.202403:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:6.0:1619677448.202406:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (24)++
00000800:00000200:22.0:1619677448.202407:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:6.0:1619677448.202407:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000400:00000200:22.0:1619677448.202408:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.202408:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.202410:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.202410:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.202414:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.202414:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.202415:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.202417:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.202418:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.202418:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.202418:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.202419:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.202569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.0:1619677448.202574:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.202577:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677448.202580:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.202581:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.202582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:5.0:1619677448.202584:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.202586:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:18.0:1619677448.202589:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:5.0:1619677448.202591:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:18.0:1619677448.202592:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.202592:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:5.0:1619677448.202593:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.202594:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:18.0:1619677448.202595:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.202595:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.202596:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.202597:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.202597:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:18.0:1619677448.202598:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.202598:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677448.202600:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.202600:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.202602:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.202603:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.202603:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.202603:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.202605:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.202605:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677448.202606:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:18.0:1619677448.202607:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.202607:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.202608:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.202608:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.202609:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:10.0:1619677448.202610:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.202611:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.202611:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.202611:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.202611:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000400:00000200:10.0:1619677448.202612:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.202613:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:18.0:1619677448.202614:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.202614:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:5.0:1619677448.202615:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:10.0:1619677448.202616:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.202617:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677448.202617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:18.0:1619677448.202618:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.202618:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.202619:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.202620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.2:1619677448.202620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677448.202621:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.202621:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.202621:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.202622:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.202623:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:10.0:1619677448.202623:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.202624:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.202625:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:10.0:1619677448.202625:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:10.0:1619677448.202625:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.202626:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.202626:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.0:1619677448.202626:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.202627:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:10.0:1619677448.202627:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.0:1619677448.202627:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.202628:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000400:00000200:5.0:1619677448.202628:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.202629:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:17.0:1619677448.202629:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:21.2:1619677448.202630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:1.0:1619677448.202630:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.202631:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.202631:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.202632:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:17.0:1619677448.202633:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.202634:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.202634:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:5.0:1619677448.202634:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:17.0:1619677448.202636:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:2.0:1619677448.202636:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:17.0:1619677448.202637:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96717994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.202637:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677448.202637:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.202638:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:17.0:1619677448.202639:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.202639:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.202639:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:17.0:1619677448.202640:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.202640:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:17.0:1619677448.202641:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:12.0:1619677448.202641:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.202642:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.202642:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677448.202643:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.202643:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.202645:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.202646:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.202647:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677448.202648:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.202651:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.202651:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.202652:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.202654:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.202654:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.202655:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.202656:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.202660:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.202663:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.202664:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.202665:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.202665:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.202667:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.202668:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.202670:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.202672:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.202672:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.202674:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.202676:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.202684:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.202687:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.202688:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.202688:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.202689:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.202689:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.202711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.202717:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.202720:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.202721:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.202723:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.202726:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.202728:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.202729:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.202730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:18.0:1619677448.202730:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.202731:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.202732:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.202733:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.202735:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.2:1619677448.202736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:18.0:1619677448.202736:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:20.0:1619677448.202738:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.202738:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.202740:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.202741:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.202741:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.202741:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:20.0:1619677448.202742:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:18.0:1619677448.202742:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.202743:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:20.0:1619677448.202744:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.202744:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677448.202745:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677448.202745:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.202745:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.202745:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:20.0:1619677448.202746:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.202747:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:30.0:1619677448.202748:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677448.202752:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.202752:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.202752:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.202754:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:30.0:1619677448.202756:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.202756:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.202757:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677448.202757:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:6.0:1619677448.202757:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677448.202758:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:30.0:1619677448.202759:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96717991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.202759:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:30.0:1619677448.202760:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.202760:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.202760:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:6.0:1619677448.202760:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.202761:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.202761:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.202762:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:4.2:1619677448.202762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:30.0:1619677448.202763:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.202763:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:6.0:1619677448.202763:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:30.0:1619677448.202766:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.202766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677448.202768:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.202769:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.202770:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:26.0:1619677448.202771:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:30.0:1619677448.202772:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.202772:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.202772:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.202773:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677448.202773:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:26.0:1619677448.202774:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.202775:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.202775:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.202776:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677448.202776:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.202776:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:30.0:1619677448.202777:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:26.0:1619677448.202777:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:17.0:1619677448.202777:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.202777:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.202779:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96717990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.202779:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.202780:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.202781:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.202782:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.202783:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:26.0:1619677448.202785:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.202787:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.202788:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.2:1619677448.202788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.202789:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.202789:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.202790:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.202791:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.202792:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.202792:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677448.202792:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.202792:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.202793:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.202794:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.202794:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.202795:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677448.202796:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677448.202798:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.202799:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.202799:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.202800:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.202801:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:31.0:1619677448.202804:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.202807:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.202808:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.202809:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.202810:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.202812:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.203007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.203015:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.203017:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.203019:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.203021:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.203022:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.203023:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.203025:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.203026:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.203027:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.203028:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.203031:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.203032:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.203035:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.203036:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.203036:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.203036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.203038:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.203038:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.203039:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.203040:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.203041:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.203043:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677448.203044:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.203044:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.203044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.203046:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.203048:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.203049:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.203050:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.203050:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.203051:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.203052:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:30.0:1619677448.203054:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.203054:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677448.203055:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.203055:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.203057:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.203057:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.203058:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.203058:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.203059:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.203060:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.203060:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677448.203061:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.203062:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.203062:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677448.203062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.203070:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.203072:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.203073:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.203074:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.203074:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.203076:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.203077:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.203077:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.203079:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.203080:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.203082:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.203083:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.203084:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.203087:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.203088:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.203088:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.203089:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.203090:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.203090:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.203092:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.203092:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.203093:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.203093:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.203094:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.203094:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.203095:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.203097:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.203097:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.203099:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.203100:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.203100:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.203106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.203111:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.203113:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.203114:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.203114:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.203115:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677448.203115:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.203116:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.203117:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.203117:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.203120:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677448.203129:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.203131:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.203132:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.203132:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.203133:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.203134:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.203151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.203157:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.203160:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.203160:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.203162:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.203165:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.203166:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.203167:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.203169:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.203170:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.203170:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.203174:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.203175:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.203179:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.203179:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.203180:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.203182:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.203182:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.203183:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.203184:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.203195:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.203198:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.203198:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.203200:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.203201:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.203203:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.203208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677448.203211:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.203214:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.203215:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.203216:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.203216:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.203216:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.203218:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.203218:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.203219:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.203221:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.203224:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.203225:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.203226:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.203227:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.203228:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.203229:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.203232:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.203234:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.203236:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.203237:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.203238:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.203239:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.203240:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.203241:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.203242:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.203242:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.203248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.203251:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.203254:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.203254:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.203255:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:12.0:1619677448.203256:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.203257:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:12.0:1619677448.203258:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.203258:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.203259:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.203259:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.203260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.203265:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.203268:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.203269:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.203271:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.203273:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.203275:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.203276:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.203278:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.203279:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.203280:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.203283:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.203285:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.203288:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.203288:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.203289:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.203291:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.203291:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.203292:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.203292:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.203297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.203305:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.203308:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.203309:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.203309:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.203311:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.203314:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.203346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.203354:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.203357:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.203358:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.203360:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.203362:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.203364:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.203365:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.203366:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.203367:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.203368:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.203372:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.203373:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.203376:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.203377:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.203377:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.203379:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.203379:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.203380:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.203380:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.203392:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.203395:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.203395:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.203397:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.203399:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.203399:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.203425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.203433:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.203436:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.203437:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.203439:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.203443:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.203444:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.203445:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.203447:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.203448:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.203449:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.203453:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.203454:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.203457:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.203458:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.203458:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.203461:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.203461:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.203462:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.203462:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.203467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.203474:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.203478:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.203479:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.203479:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.203481:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.203484:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.203515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.203522:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.203525:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.203527:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.203529:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.203529:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.203531:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.203532:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.203533:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.203534:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.203535:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677448.203538:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.203540:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.203543:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.203543:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.203544:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.203546:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.203546:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.203548:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.203549:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.203550:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677448.203552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.203555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.203559:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.203561:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.203562:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.203563:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:18.0:1619677448.203563:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.203565:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.203566:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.203567:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.203568:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677448.203568:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.203569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.203575:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.203578:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.203580:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.203581:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.203582:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.203583:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.203584:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.203586:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.203587:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.203587:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.203590:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.203592:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.203594:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.203595:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.203596:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.203597:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.203598:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.203599:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.203600:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.203603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.2:1619677448.203611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.203612:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.203614:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.203615:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.203615:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.203615:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.203616:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.203618:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.203619:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:20.0:1619677448.203620:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.203620:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.0:1619677448.203621:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.203622:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.203623:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.203625:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.203626:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.203626:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.2:1619677448.203627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000400:00000200:20.0:1619677448.203630:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.203631:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.2:1619677448.203633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677448.203634:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.203635:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.203635:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.203637:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.203638:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.203638:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.203639:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.203639:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.203640:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:17.0:1619677448.203640:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.203640:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.203641:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.203641:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677448.203642:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.203643:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:17.0:1619677448.203643:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.2:1619677448.203645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.203645:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.203646:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.203649:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.203650:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:17.0:1619677448.203651:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677448.203651:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:17.0:1619677448.203652:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717978] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.203653:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:17.0:1619677448.203653:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.203653:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:30.0:1619677448.203654:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.203654:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.203654:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.203655:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:30.0:1619677448.203656:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.203656:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.203657:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:17.0:1619677448.203659:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.203660:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677448.203661:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.203664:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.203665:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.203666:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.203668:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.203669:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.203669:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.203670:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.203681:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:4.2:1619677448.203683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677448.203684:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.203685:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.203686:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.203688:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.203689:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.203690:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.203693:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.203694:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.203696:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.203698:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.203699:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.203701:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.203702:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.203703:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.203704:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.203707:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.203709:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.203712:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.203713:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.203714:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.203715:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.203716:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.203716:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.203717:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.203729:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.203733:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.203734:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.203734:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.203736:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.203739:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.203808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.203815:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.203819:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.203821:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.203822:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.203824:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.203825:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.203827:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.203828:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.203829:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.203830:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.203833:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.203834:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.203837:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.203838:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.203839:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.203841:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.203841:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.203843:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.203844:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.203845:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.203847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.203851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.203854:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.203857:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.203858:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.203859:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.203859:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.203859:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.203861:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.203862:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677448.203862:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677448.203863:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.203869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.203877:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.203879:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.203880:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.203881:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.203882:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.203883:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.203891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.2:1619677448.203897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:10.0:1619677448.203898:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.203901:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.203902:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677448.203902:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677448.203904:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.203905:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.203906:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677448.203906:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.203908:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:6.0:1619677448.203908:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.203910:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.203910:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.203911:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.203912:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.203913:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.203914:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:6.0:1619677448.203914:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.203915:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.203916:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.203917:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.203923:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.203924:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.203926:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.203927:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.203929:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.203930:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.203931:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.203931:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:6.0:1619677448.203931:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.203932:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.203933:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.203934:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.203934:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.203934:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.203935:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677448.203935:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:10.0:1619677448.203936:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677448.203936:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.203937:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:21.2:1619677448.203942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677448.203946:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.203948:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.203949:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.203949:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.203950:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:30.0:1619677448.203951:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.203951:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.203952:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677448.203953:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.203953:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:30.0:1619677448.203955:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:22.0:1619677448.203955:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.203956:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.203956:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.203958:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.203962:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.203971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.203978:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.203980:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.203981:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.203983:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.203986:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.203988:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.203989:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.203990:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.203991:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.203992:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.203995:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.203997:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.203999:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.203999:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.204000:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.204001:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.204002:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.204002:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.204003:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.204013:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.204015:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.204015:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.204016:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.204017:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.204019:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.204206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.204212:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.204214:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.204215:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.204217:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.204220:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.204222:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.204223:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.204224:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.204225:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.204226:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.204229:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.204231:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.204233:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.204234:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.204234:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.204236:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.204236:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.204238:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.204239:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.204239:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.204244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.204249:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677448.204250:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.204251:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.204252:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.204253:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.204253:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.204254:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.204254:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.204256:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.204259:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.204310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.204313:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.204315:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.204317:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.204319:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.204320:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.204322:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677448.204323:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.204324:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.204325:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.204326:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.204330:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.204332:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.204334:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.204335:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.204336:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.204337:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.204338:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.204338:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.204339:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677448.204344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.204352:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.204355:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.204356:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.204356:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.204358:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677448.204360:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.204388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.204396:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.2:1619677448.204397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:31.0:1619677448.204399:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.204401:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.204403:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.204404:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.204404:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:31.0:1619677448.204405:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.204407:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.204407:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:31.0:1619677448.204408:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.204408:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:31.0:1619677448.204409:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.204411:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.204411:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.204414:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.204414:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.204416:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.204416:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.204417:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.204418:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.204419:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.204420:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:18.0:1619677448.204420:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.204421:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677448.204421:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.204422:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.204423:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.204424:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677448.204424:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.204425:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677448.204426:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.204428:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677448.204429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677448.204429:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.204430:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.204432:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.204432:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.204433:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.204433:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677448.204436:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.204437:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:8.2:1619677448.204437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677448.204439:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:30.0:1619677448.204440:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:13.0:1619677448.204440:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.204442:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.204442:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.204444:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:12.0:1619677448.204444:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:30.0:1619677448.204445:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.204445:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:13.0:1619677448.204445:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:30.0:1619677448.204447:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:17.0:1619677448.204447:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:30.0:1619677448.204448:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.204448:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.204449:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:30.0:1619677448.204450:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.204451:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.204452:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.204453:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.204453:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.204455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000400:00000200:30.0:1619677448.204457:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.204459:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.204462:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.204462:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:1.0:1619677448.204462:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.204463:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.204465:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677448.204465:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.204465:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.204466:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:1.0:1619677448.204466:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:30.0:1619677448.204467:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:1.0:1619677448.204467:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.204468:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:1.0:1619677448.204468:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.204471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.204476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677448.204479:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.204482:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677448.204483:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677448.204483:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.204484:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.204485:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.204486:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.204487:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.204487:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:26.0:1619677448.204488:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.204490:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.204490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000400:00000200:26.0:1619677448.204492:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.204493:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.204494:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.204495:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.204496:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.204498:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000400:00000200:26.0:1619677448.204499:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.204500:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.204501:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.204501:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:21.2:1619677448.204502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:20.0:1619677448.204502:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.204502:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.204503:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:26.0:1619677448.204504:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.204505:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.204506:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.204508:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.204508:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.204509:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:24.0:1619677448.204509:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:26.0:1619677448.204510:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.204511:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677448.204512:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.204513:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677448.204514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:24.0:1619677448.204515:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.204516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:24.0:1619677448.204517:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.204519:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.204520:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96717966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.204521:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:24.0:1619677448.204522:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.204522:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.204523:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.204523:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:2.0:1619677448.204524:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.204525:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.204525:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:2.0:1619677448.204525:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.204526:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.204526:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.204527:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:2.0:1619677448.204528:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.204529:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.204531:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677448.204532:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.204532:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.204533:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.204535:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.204535:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.204536:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.204537:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.204538:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.2:1619677448.204541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.204543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677448.204548:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.204550:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.204551:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.204551:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:20.0:1619677448.204553:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.204553:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.204553:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.204555:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677448.204555:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.204556:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.204576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.204583:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.204586:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.204587:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.204589:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.204592:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.204594:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.204595:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.204596:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.204597:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.204598:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.204601:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.204602:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.204605:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.204606:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.204606:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.204608:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.204609:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.204609:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.204610:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.204614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.204622:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.204624:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.204625:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.204627:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.204627:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.204629:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.204690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.204695:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.204698:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.204700:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.204700:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.204702:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.204704:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.204705:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.204706:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.204707:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.204708:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.204712:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.204714:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.204716:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.204717:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.204717:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.204719:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.204719:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.204720:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.204721:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.204725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.204732:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.204734:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.204735:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.204737:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.204737:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.204739:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:21.2:1619677448.204784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.204790:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.204793:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.204794:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.204795:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.204798:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.204799:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.204801:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.204802:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.204803:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.204804:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.204806:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.204808:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.204810:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.204811:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.204812:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.204814:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.204814:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.204815:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.204815:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.204820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.204828:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.204830:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.204831:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.204831:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.204833:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.204836:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.205027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677448.205035:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677448.205038:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677448.205040:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.205041:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677448.205042:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.205044:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.205045:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.205047:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.205048:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.205049:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.205052:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.205054:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.205056:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.205057:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.205058:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.205059:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.205060:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677448.205061:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87b3e5800] (22)++
00000800:00000200:31.0:1619677448.205063:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:31.0:1619677448.205063:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:20.2:1619677448.205069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:12.0:1619677448.205072:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.205075:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.205076:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.205077:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)++
00000400:00000200:12.0:1619677448.205077:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.205079:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:30.0:1619677448.205079:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:13.0:1619677448.205079:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.205080:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:12.0:1619677448.205080:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677448.205093:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.205096:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.205096:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.205097:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.205098:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.205099:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.205161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.205167:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.205170:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.205171:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.205173:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.205175:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.205177:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.205178:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.205179:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.205180:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.205181:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.205185:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.205187:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.205190:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.205191:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.205192:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.205193:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.205194:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.205195:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.205195:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.205206:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.205208:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.205209:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.205211:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.205212:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.205214:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.205221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.205229:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.205231:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:21.2:1619677448.205231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:30.0:1619677448.205234:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.205235:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.205236:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.205237:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:18.0:1619677448.205237:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:30.0:1619677448.205239:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.205240:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.205240:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:30.0:1619677448.205241:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.205241:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.205243:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:30.0:1619677448.205244:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677448.205246:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.205247:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677448.205248:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.205249:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.205250:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:18.0:1619677448.205250:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.205251:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.205252:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.205254:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.205254:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.205255:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:18.0:1619677448.205256:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:30.0:1619677448.205257:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.205257:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.205258:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:18.0:1619677448.205258:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.205259:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:18.0:1619677448.205259:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.205260:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.205261:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.205262:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.205263:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.205263:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.205264:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677448.205268:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:9.2:1619677448.205268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677448.205271:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.205272:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.205274:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.205275:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677448.205275:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:13.0:1619677448.205276:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677448.205278:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.205279:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.205279:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.205281:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.205283:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.205419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.205427:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.205430:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.205430:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.205433:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.205436:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.205437:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.205439:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.205440:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.205441:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.205442:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.205445:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.205447:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.205450:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.205451:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.205452:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.205453:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.205454:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.205455:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.205455:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677448.205469:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.205471:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.205472:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.205473:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.205474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:6.0:1619677448.205475:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.205478:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677448.205481:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.205484:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.205484:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.205486:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.205488:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.205489:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.205491:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.205492:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.205493:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.205494:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.2:1619677448.205496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:18.0:1619677448.205497:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.205499:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677448.205499:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.205500:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.205501:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.205502:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:18.0:1619677448.205502:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.205503:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.205503:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:20.0:1619677448.205504:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.205505:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:5.0:1619677448.205505:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.205506:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.205506:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.205506:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:20.0:1619677448.205508:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677448.205509:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.205509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677448.205510:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.205511:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.205511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000400:00000200:20.0:1619677448.205512:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677448.205516:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.205517:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677448.205518:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.205518:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.205518:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.205519:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:2.0:1619677448.205519:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.205520:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.205520:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677448.205521:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677448.205521:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.205521:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.205521:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.205522:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677448.205523:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.205523:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677448.205524:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.205524:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.205525:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.205529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:9.2:1619677448.205532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.205538:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.205540:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.205540:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.205540:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.205541:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.205542:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.205542:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.205543:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.205544:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.205545:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677448.205545:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.205546:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.205590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.205596:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677448.205597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:17.0:1619677448.205599:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.205600:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.205602:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.205604:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000400:00000200:17.0:1619677448.205605:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.205607:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:12.0:1619677448.205607:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:17.0:1619677448.205608:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.205610:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.205610:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.205611:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.205612:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.205612:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677448.205613:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.205616:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.205618:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:12.0:1619677448.205618:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:12.0:1619677448.205620:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.205621:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.205622:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.205622:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:12.0:1619677448.205622:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.205622:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.205624:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677448.205624:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.205625:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.205625:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.205626:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677448.205627:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.205629:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.205632:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.205632:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.205633:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.205635:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.205636:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.205636:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677448.205636:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677448.205637:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:13.0:1619677448.205638:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.205638:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677448.205639:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.205640:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.205643:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677448.205644:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.205645:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.205646:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.205647:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.205647:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.205649:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.205664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.0:1619677448.205668:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:12.0:1619677448.205669:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.205670:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:12.0:1619677448.205672:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.205672:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.205673:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:12.0:1619677448.205673:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:5.0:1619677448.205673:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.205674:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.205675:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677448.205675:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.205677:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.205678:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.205679:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.205680:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.205681:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.205684:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.205686:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.205688:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.205689:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.205690:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.205691:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.205692:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.205693:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.205694:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.205698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.205706:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.205709:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.205709:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.205711:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.205712:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.205713:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:21.2:1619677448.205923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.205930:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.205933:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.205934:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.205936:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.205939:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.205940:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.205942:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.205943:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.205944:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.205945:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.205948:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.205949:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.205952:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.205952:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.205953:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.205955:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.205955:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.205956:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.205956:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677448.205967:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.205969:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.205970:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.205971:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.205973:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.205974:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.205985:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.205988:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.205989:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.205989:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.205990:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.205992:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.206010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.206016:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.206019:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.206020:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.206022:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.206024:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.206026:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.206027:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.206028:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.206029:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.206030:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.206034:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.206035:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.206038:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.206039:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.206039:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.206041:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.206042:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.206042:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.206043:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.206047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.206054:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.206056:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677448.206057:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.206058:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.206060:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.206060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:24.0:1619677448.206063:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:31.0:1619677448.206069:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:9.2:1619677448.206069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:31.0:1619677448.206072:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.206074:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.206074:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000400:00000200:31.0:1619677448.206076:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.206077:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.0:1619677448.206077:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677448.206078:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:24.0:1619677448.206078:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:31.0:1619677448.206079:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.206079:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (21)++
00000400:00000200:31.0:1619677448.206080:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.206080:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.206080:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:31.0:1619677448.206081:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.206082:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.206086:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.206087:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.206090:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.206091:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.206091:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.206093:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.206093:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.206094:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.206095:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677448.206106:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.206108:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.206109:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.206110:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.206112:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.206113:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.206114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.206120:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.206122:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.206125:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.206125:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.206127:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.206129:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.206130:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717949] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.206132:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.206133:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.206134:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.206137:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.206139:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.206141:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.206142:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.206143:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.206144:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.206145:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.206146:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.206146:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.206151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.206158:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.206161:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.206162:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.206162:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.206165:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.206167:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.206179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.206187:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.206190:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.206192:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.206194:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.206194:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.206196:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677448.206197:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.206199:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.206200:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.206201:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677448.206204:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.206206:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.206210:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.206210:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677448.206211:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.206213:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677448.206213:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.206214:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.206215:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.206226:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.206229:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.206230:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.206230:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.206232:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.206233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677448.206235:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:10.0:1619677448.206241:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.206243:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.206244:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677448.206244:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:10.0:1619677448.206246:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.206248:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.206248:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.206249:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.206249:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.206250:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.206250:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:10.0:1619677448.206251:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.206252:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.206253:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.206254:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.206255:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.206258:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.206260:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.206261:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.206262:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.206263:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.206264:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.206264:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.206265:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.206266:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677448.206275:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:20.2:1619677448.206276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:26.0:1619677448.206277:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.206278:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.206278:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:10.0:1619677448.206279:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.206280:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.206281:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.0:1619677448.206281:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.206283:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.206284:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.206285:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.206287:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.206288:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.206289:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.206290:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.206291:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.206299:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.206301:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.206304:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.206305:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.206305:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.206307:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.206308:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.206308:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.206309:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:4.2:1619677448.206316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.206321:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:10.0:1619677448.206323:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.206324:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.206325:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.206326:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.206327:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677448.206328:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.206329:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:10.0:1619677448.206329:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.206330:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:5.2:1619677448.206330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.206331:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)++
00000400:00000200:10.0:1619677448.206331:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.206332:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:30.0:1619677448.206332:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:30.0:1619677448.206332:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:21.2:1619677448.206332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:10.0:1619677448.206332:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:31.0:1619677448.206333:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:10.0:1619677448.206334:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717945] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.206335:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.206336:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.206337:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.206337:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:22.0:1619677448.206338:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.206340:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:22.0:1619677448.206341:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:1.0:1619677448.206341:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.206342:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.206342:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:17.0:1619677448.206344:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.206345:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.206345:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:17.0:1619677448.206347:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.206348:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.206348:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:10.0:1619677448.206348:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.206349:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.206349:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:22.0:1619677448.206350:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677448.206350:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.206351:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.206351:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.206351:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.206352:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677448.206353:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.206353:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.206354:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.206354:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.206355:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.206355:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:22.0:1619677448.206357:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:8.2:1619677448.206358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:17.0:1619677448.206361:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.206362:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.206364:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:22.0:1619677448.206365:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677448.206367:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.206367:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:22.0:1619677448.206368:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.206368:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.206368:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:22.0:1619677448.206369:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:5.0:1619677448.206369:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:22.0:1619677448.206370:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:5.0:1619677448.206370:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.206371:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.206371:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.206372:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:20.0:1619677448.206372:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677448.206372:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:5.0:1619677448.206372:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.206373:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.206374:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677448.206374:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.206374:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677448.206375:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677448.206376:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:9.2:1619677448.206377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.206379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.206384:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:1.0:1619677448.206384:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:26.0:1619677448.206385:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:24.0:1619677448.206387:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:1.0:1619677448.206387:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:26.0:1619677448.206388:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:24.0:1619677448.206388:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.206388:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:8.0:1619677448.206388:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:1.0:1619677448.206388:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.206389:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.206389:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677448.206391:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.206392:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.206393:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:26.0:1619677448.206395:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:4.2:1619677448.206454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.206461:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.206464:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.206465:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.206467:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.206469:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.206471:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.206472:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.206474:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.206475:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.206476:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.206479:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.206481:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.206484:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.206485:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.206485:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.206487:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.206487:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.206489:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.206490:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.206490:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.206493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.206496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.206501:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.206503:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.206504:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.206504:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677448.206505:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677448.206505:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.206506:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.206506:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.206507:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.206510:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.206621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.206626:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.206628:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.206629:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.206631:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.206634:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.206635:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.206636:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.206637:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.206638:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.206639:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.206642:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.206644:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.206647:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.206647:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.206648:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.206650:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.206650:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.206650:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.206651:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.206656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.206662:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.206665:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677448.206665:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.206666:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.206668:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.206671:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677448.206680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.206686:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.206688:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.206690:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.206691:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.206693:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.206694:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.206696:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.206697:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.206698:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.206699:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.206703:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.206704:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.206707:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.206708:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.2:1619677448.206709:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.206712:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.206713:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.206714:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.206714:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.206716:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677448.206716:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.206717:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677448.206717:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.206718:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.206719:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677448.206719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000400:00000200:5.0:1619677448.206721:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.206722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (21)++
00000400:00000200:5.0:1619677448.206723:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.206725:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.206726:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.206727:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:5.0:1619677448.206727:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.206728:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.206728:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.206728:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.206731:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.206732:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.206732:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.206733:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677448.206733:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677448.206734:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.206734:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.206735:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677448.206735:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677448.206737:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.206738:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.206739:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.206741:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.206741:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.206742:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.206742:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.206747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.206755:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.206757:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.206758:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.206759:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.206761:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.206763:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.206814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.206821:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.206824:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.206825:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.206827:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.206830:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.206831:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.206832:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.206834:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.206835:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.206836:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.206839:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.206841:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.206843:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.206844:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.206844:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.206846:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.206846:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.206848:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.206849:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.206849:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.206855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.206860:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.206861:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.206862:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.206863:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.206864:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.206865:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.206865:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.206866:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.206866:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.206867:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:4.2:1619677448.206881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.206889:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.206891:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.206892:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.206894:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.206896:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.206897:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.206898:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.206899:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.206900:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.206901:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.206904:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.206905:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.206908:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.206908:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.206909:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.206910:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.206911:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.206912:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.206912:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677448.206917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.206924:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:6.0:1619677448.206924:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677448.206927:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:6.0:1619677448.206927:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:24.0:1619677448.206928:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677448.206928:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:8.0:1619677448.206929:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.206929:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.206931:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.206932:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.206934:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:6.0:1619677448.206934:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:8.0:1619677448.206935:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.206936:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.206938:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.206938:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.206942:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.206943:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.206946:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.206947:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.206947:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.206949:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.206950:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.206950:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.206951:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.206955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.206963:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.206966:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.206967:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.206967:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.206970:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.206973:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.206986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.206991:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.206994:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.206995:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.206997:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.206998:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.207000:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.207001:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.207002:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.207003:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.207004:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.207008:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.207009:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.207012:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.207013:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.207014:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.207016:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.207016:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.207017:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.207017:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677448.207029:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.207031:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.207032:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.207034:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.207036:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.207037:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.207095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677448.207102:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677448.207105:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677448.207107:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.207108:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677448.207109:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.207110:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677448.207112:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677448.207113:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.207114:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677448.207115:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:13.0:1619677448.207118:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.207120:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677448.207123:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.207124:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:13.0:1619677448.207125:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:13.0:1619677448.207126:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:13.0:1619677448.207127:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677448.207128:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677448.207130:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677448.207130:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:9.2:1619677448.207133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.2:1619677448.207136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:18.0:1619677448.207141:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.207143:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:12.0:1619677448.207143:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000400:00000200:18.0:1619677448.207145:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.207145:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:2.0:1619677448.207145:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:12.0:1619677448.207146:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:18.0:1619677448.207147:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.207147:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:18.0:1619677448.207150:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.207152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.207159:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.207162:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.207163:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.207163:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.207164:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.207165:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.207187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.207194:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.207197:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.207199:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.207201:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.207202:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.207203:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.207205:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.207206:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.207207:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.207208:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.207211:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.207213:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.207215:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.207216:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.207216:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.207218:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.207218:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.207219:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.207220:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.207229:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.2:1619677448.207229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677448.207232:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.207233:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.207234:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.207234:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.207236:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.207237:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.207237:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.207238:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.207240:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.207243:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.207245:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.207246:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.207247:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.207248:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.207249:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.207253:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.207254:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.207257:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.207258:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.207258:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.207260:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.207260:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.207261:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.207261:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.207274:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:4.2:1619677448.207274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677448.207276:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.207277:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.207277:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.207279:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.207281:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.207282:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:10.0:1619677448.207284:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.207285:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.207287:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.207289:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.207291:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.207292:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.207298:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.207299:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.207301:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.207304:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.207306:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.207309:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.207310:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.207310:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.207312:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.207312:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.207313:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.207314:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.2:1619677448.207324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:6.0:1619677448.207326:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677448.207327:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.207329:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.207330:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.207330:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:6.0:1619677448.207331:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.207332:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.207333:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.207333:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.207334:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.207335:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:6.0:1619677448.207336:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:20.0:1619677448.207337:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.207338:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.207339:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.207340:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.207344:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.207346:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.207349:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.207350:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.207350:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.207352:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.207353:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.207354:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.207355:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.207356:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.207361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677448.207365:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.207366:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677448.207368:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.207368:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:13.0:1619677448.207369:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.207369:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.207370:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:13.0:1619677448.207371:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.207372:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.207374:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.207417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677448.207424:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.207427:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677448.207428:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677448.207430:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.207433:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.207434:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:26.0:1619677448.207435:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.207436:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.207437:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.207438:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:26.0:1619677448.207441:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.207443:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:26.0:1619677448.207446:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.207447:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:26.0:1619677448.207447:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:26.0:1619677448.207449:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:26.0:1619677448.207449:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.207450:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677448.207451:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677448.207455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.207464:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.207466:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.207467:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.207468:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.207470:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.207473:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.207522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.207530:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.207533:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.207534:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.207536:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.207539:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.207540:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.207542:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.207543:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.207544:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.207545:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.207548:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.207549:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.207552:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.207552:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.207553:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.207554:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.2:1619677448.207555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.207555:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.207556:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.207558:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.207558:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.207560:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.207561:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.207563:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.207563:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:4.2:1619677448.207564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:10.0:1619677448.207565:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.207567:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:22.0:1619677448.207568:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:10.0:1619677448.207568:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.207570:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.207570:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.207571:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:10.0:1619677448.207571:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677448.207571:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.207572:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:22.0:1619677448.207572:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.207573:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.207573:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.207574:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.207574:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.207577:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:22.0:1619677448.207578:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677448.207579:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.207582:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.207583:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.207583:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.207585:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.207586:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.207587:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677448.207588:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.207589:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:9.2:1619677448.207591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.207593:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.207594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:6.0:1619677448.207596:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.207597:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.207597:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.207598:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.2:1619677448.207598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:6.0:1619677448.207599:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677448.207599:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.207601:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.207603:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:18.0:1619677448.207603:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.207603:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.207605:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.207605:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:2.0:1619677448.207605:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:26.0:1619677448.207606:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.207607:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:26.0:1619677448.207608:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.207608:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:2.0:1619677448.207609:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:26.0:1619677448.207611:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.207611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:26.0:1619677448.207613:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.207614:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.207616:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.207616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:26.0:1619677448.207617:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.207618:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.207618:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677448.207619:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.207621:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.207621:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.207622:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677448.207623:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.207623:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.207624:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.207624:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:26.0:1619677448.207625:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.207626:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.207628:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.207628:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.207629:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.207629:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677448.207631:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.207631:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.207631:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:31.0:1619677448.207632:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:26.0:1619677448.207632:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677448.207633:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.207633:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.207633:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:31.0:1619677448.207635:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.207636:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.207636:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:9.2:1619677448.207637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677448.207639:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.207641:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.207643:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.207644:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.207644:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677448.207645:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.207647:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.207647:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.207647:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.207648:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.207648:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.207649:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.207650:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:2.0:1619677448.207650:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.207651:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677448.207653:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.207657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.207658:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.207661:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.207662:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.207662:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.207664:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:10.0:1619677448.207664:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.207667:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.207667:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677448.207667:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:30.0:1619677448.207668:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.207676:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.207679:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.207680:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.207680:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.207680:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.207681:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.207778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.0:1619677448.207782:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.207785:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677448.207787:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.207788:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.207790:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.207791:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.207793:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.207794:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.207795:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.207796:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.207799:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.207801:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.207803:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.207804:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.207805:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.207806:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.207807:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.207807:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.207808:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.207813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.207821:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.207823:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.207824:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.207826:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.207827:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.207828:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:8.2:1619677448.207832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.207839:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.207842:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.207843:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.207844:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677448.207844:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.207846:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:4.2:1619677448.207877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.207884:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.207887:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.207888:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.207889:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.207892:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.207894:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.207895:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717923] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.207897:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.207898:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.207899:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.207902:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.207903:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.207906:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.207907:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.207908:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.207910:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.207910:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.207911:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.207911:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.207923:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.207926:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.207927:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.207927:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.207930:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.207932:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.207995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.208003:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.208006:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.208008:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.208010:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.208010:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.208012:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:30.0:1619677448.208013:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.208014:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.208015:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.208016:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677448.208019:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.208021:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677448.208024:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.208024:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:30.0:1619677448.208025:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677448.208027:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677448.208027:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.208028:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.208029:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677448.208031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:8.2:1619677448.208033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:18.0:1619677448.208037:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.208040:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:17.0:1619677448.208041:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:2.0:1619677448.208041:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.208043:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.208044:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.208044:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.208045:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.208046:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.208047:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677448.208047:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.208049:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.208050:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.208050:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:18.0:1619677448.208051:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.208052:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.208055:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.208057:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.208059:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.208060:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.208061:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.208063:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.208064:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.208064:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.208065:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.208069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.208077:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.208080:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.208081:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.208082:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.208083:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.208085:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.208134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.208139:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.208142:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.208143:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.208145:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.208147:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.208149:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.208150:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.208151:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.208152:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.208153:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.208157:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.208158:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.208161:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.208162:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.208163:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.208165:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.208165:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.208166:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.208166:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.208178:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.208181:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.208182:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.208183:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.208185:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.208186:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.208189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.208192:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.208195:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.208197:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.208197:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.208198:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.208199:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.208200:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:13.0:1619677448.208200:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.208201:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:20.0:1619677448.208202:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.208202:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.208202:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677448.208203:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.208203:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:20.0:1619677448.208204:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.208205:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.208209:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.208210:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.208213:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.208214:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.208215:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.208217:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.208218:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.208219:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.208219:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.208229:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:21.2:1619677448.208231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:12.0:1619677448.208231:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.208232:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.208234:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.208235:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.208237:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.208237:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.208240:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.208241:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.208243:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.208245:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.208246:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677448.208248:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.208249:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.208250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:2.0:1619677448.208250:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.208251:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677448.208254:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.208256:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677448.208257:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677448.208258:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.208259:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677448.208260:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:2.0:1619677448.208260:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677448.208261:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677448.208262:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:31.0:1619677448.208263:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.208264:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:2.0:1619677448.208264:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.208264:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:31.0:1619677448.208265:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.208265:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:31.0:1619677448.208267:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.208268:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.208270:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.208270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:31.0:1619677448.208271:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.208272:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:6.0:1619677448.208275:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:31.0:1619677448.208276:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.208276:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:31.0:1619677448.208277:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677448.208278:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:6.0:1619677448.208278:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.208279:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.208279:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:6.0:1619677448.208279:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.208280:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677448.208281:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.208282:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:6.0:1619677448.208282:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:31.0:1619677448.208283:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.208285:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.208285:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:6.0:1619677448.208285:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677448.208287:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:4.2:1619677448.208288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.208289:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.208289:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.208291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.208296:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.2:1619677448.208297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.208300:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.208300:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677448.208303:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:5.0:1619677448.208303:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.208304:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.208305:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.208306:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.208306:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.208306:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.208308:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:26.0:1619677448.208308:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.208309:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677448.208309:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:30.0:1619677448.208310:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:26.0:1619677448.208310:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:8.2:1619677448.208310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000400:00000200:26.0:1619677448.208311:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717916] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.208313:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.208313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:26.0:1619677448.208314:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.208315:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.208318:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.208318:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.208320:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.208320:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.208320:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.208321:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.208322:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.208322:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.208323:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.0:1619677448.208323:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:26.0:1619677448.208324:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.208325:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.208325:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.208325:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.208327:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.208327:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.208327:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677448.208328:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.208328:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.208328:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:30.0:1619677448.208329:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.208330:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.208332:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.208333:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.208334:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.208337:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.208338:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.208340:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.208341:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.208342:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.208342:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:30.0:1619677448.208343:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.208344:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.208344:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.208345:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.208345:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.208346:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.208346:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.208348:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.208351:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677448.208356:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.208358:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.208359:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.208361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:13.0:1619677448.208361:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.208363:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.208364:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:13.0:1619677448.208364:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677448.208366:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.208368:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.208369:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.208371:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.208373:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677448.208374:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.208376:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.208377:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.208378:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677448.208382:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.208383:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.208386:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.208387:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.208388:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.208390:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.208390:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.208391:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.208391:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:10.0:1619677448.208402:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.208404:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.208405:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.208406:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.208408:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.208409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:10.0:1619677448.208410:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:18.0:1619677448.208416:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.208418:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.208419:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.208421:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.208424:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.208425:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.208427:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.208428:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.208429:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.208430:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.208434:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.208436:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.208439:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.208439:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.208440:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.208442:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.208443:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.208443:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.208444:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677448.208454:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.208457:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.208458:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.208460:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.208461:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.208463:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.208494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.208501:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.208504:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.208505:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.208507:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.208510:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.208511:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.208512:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.208514:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.208515:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.208516:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.208519:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.208521:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.208524:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.208525:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.208526:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.208527:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.208528:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.208528:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.2:1619677448.208528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:24.0:1619677448.208529:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.208533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.208533:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.208536:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.208537:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.208538:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.208541:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:6.0:1619677448.208541:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.208543:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:17.0:1619677448.208544:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:6.0:1619677448.208544:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.208544:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.208545:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.208546:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.208547:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.208547:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.208548:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.208550:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:6.0:1619677448.208552:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.208552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:6.0:1619677448.208554:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.208557:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.208557:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.208557:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.208558:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.208560:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.208560:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.208560:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:6.0:1619677448.208561:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.208562:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.208562:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:1.0:1619677448.208562:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.208564:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.208566:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.208567:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.208569:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.208569:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.208571:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.208574:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:1.0:1619677448.208574:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.208576:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:1.0:1619677448.208576:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.208577:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.208579:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.208580:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.208580:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.208580:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.208581:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.208582:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.208583:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:1.0:1619677448.208583:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.208584:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.208585:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.208586:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.208590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.208595:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.208596:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677448.208597:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677448.208598:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:13.0:1619677448.208598:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.208599:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.208600:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677448.208600:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.208601:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.208603:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.208612:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.208615:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.208616:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.208617:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.208617:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.208619:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.208738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.0:1619677448.208742:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.208745:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677448.208747:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.208748:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.208750:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.208752:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.208753:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.208754:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.208755:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.208756:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.208760:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.208762:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.208764:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.208765:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.208765:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.208767:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.208768:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.208768:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.208769:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.208773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.208780:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.208782:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.208783:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.208783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:20.0:1619677448.208785:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.208787:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.208788:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:26.0:1619677448.208791:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.208793:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.208794:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.208796:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.208799:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.208800:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.208802:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717908] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.208803:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.208804:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.208805:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.208808:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.208810:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.208813:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.208813:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.208814:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.208816:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.208816:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.208817:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.208817:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.208828:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.208830:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.208831:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.208832:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.208834:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.208835:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.208885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.208893:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.208896:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.208898:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.208900:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.208900:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.208901:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.208903:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.208904:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.208905:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.208906:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.208909:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.208911:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.208913:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.208914:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.208914:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.208916:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.208916:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.208917:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.208918:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.208928:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.208930:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.208930:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.208932:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.208932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677448.208934:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.208935:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.208937:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.208940:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.208941:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.208943:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.208946:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.208947:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.208949:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.208950:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.208951:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.208952:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.208955:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.208956:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.208959:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.208960:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.208961:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.208962:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.208963:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.208963:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.208964:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.208975:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.208977:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.208978:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.208980:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.208981:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.208982:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.209158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.209164:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.209167:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.209168:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.209170:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.209173:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.209174:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.209176:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.209177:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.209178:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.209179:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.209183:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.209184:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.209188:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.209188:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.209189:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.209191:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.209192:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.209193:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:20.2:1619677448.209193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:22.0:1619677448.209194:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.209194:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:21.2:1619677448.209199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677448.209201:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.209203:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.209205:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.209205:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:6.0:1619677448.209205:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:4.2:1619677448.209205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.209207:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:6.0:1619677448.209207:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:31.0:1619677448.209208:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.209208:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.209208:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.209209:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:31.0:1619677448.209210:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.209210:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.209211:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.209213:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.209213:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.209214:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.209215:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.209215:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.209216:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.209217:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:31.0:1619677448.209219:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.209219:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:10.0:1619677448.209219:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.209220:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.209221:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.209221:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.209222:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.209223:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.209224:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.209225:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:10.0:1619677448.209225:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.209226:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:10.0:1619677448.209226:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.209226:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.209228:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.209228:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.209229:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.209230:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:10.0:1619677448.209230:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.209231:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:21.2:1619677448.209234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:10.0:1619677448.209234:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677448.209234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.209235:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.209235:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.209237:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.209238:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.209238:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.209239:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677448.209240:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.0:1619677448.209241:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677448.209243:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:20.0:1619677448.209244:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:9.2:1619677448.209244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677448.209245:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.209245:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.209246:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.209247:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.209247:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677448.209247:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.209249:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:20.0:1619677448.209250:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:18.0:1619677448.209250:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.209250:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.209252:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.209252:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677448.209254:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.209255:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.209256:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.209257:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:18.0:1619677448.209261:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.209261:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.209262:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.209264:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.209266:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.209267:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.209268:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.209270:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.209270:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.209271:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.209271:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677448.209276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.209283:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.209285:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.209286:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.209288:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.209288:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.209290:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.209632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.209641:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.209643:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.209646:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.209648:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.209649:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.209650:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.209651:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.209652:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.209654:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.209654:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.209658:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.209659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.209660:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.209662:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.209663:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.209663:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.209665:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.209665:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.209666:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.209666:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.209667:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677448.209668:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677448.209668:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.209670:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.209672:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.209673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:31.0:1619677448.209674:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677448.209675:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.209676:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.209677:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.209678:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.209678:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.209679:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.209680:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.209681:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.209681:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.209681:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.209683:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.209683:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.209683:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:30.0:1619677448.209684:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:10.0:1619677448.209684:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.209686:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.209686:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.209687:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.209687:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.209689:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:26.0:1619677448.209689:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.209690:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677448.209690:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:31.0:1619677448.209691:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:26.0:1619677448.209691:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.209692:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.209693:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.209693:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677448.209693:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.209694:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:8.2:1619677448.209696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677448.209697:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.209698:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:21.2:1619677448.209699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.209701:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.209701:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.209702:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.209703:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.209704:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.209704:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.209704:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:26.0:1619677448.209705:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.209706:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:17.0:1619677448.209706:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.209707:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.209708:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.209709:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.209709:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.209710:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:17.0:1619677448.209710:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.209713:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.209715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:12.0:1619677448.209716:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.209719:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.209719:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.209721:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.209721:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677448.209723:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.209724:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677448.209724:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.209725:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.209725:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.209726:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.209727:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.209782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.209788:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.209790:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.209792:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.209793:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.209794:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.209795:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.209797:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.209798:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.209799:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.209800:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.209803:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.209805:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.209807:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.209808:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.209808:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.209810:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.209810:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.209811:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.209812:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.2:1619677448.209815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677448.209817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:4.2:1619677448.209818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677448.209822:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.2:1619677448.209824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:5.0:1619677448.209824:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.209826:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:10.0:1619677448.209826:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677448.209826:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.209827:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.209828:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.209829:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.209829:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:5.0:1619677448.209829:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.209830:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.209831:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:8.0:1619677448.209832:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:10.0:1619677448.209833:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.209833:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.209835:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.209835:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:5.0:1619677448.209835:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.209836:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.209836:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.209837:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.209838:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.209838:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.209839:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.209840:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.209841:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:10.0:1619677448.209841:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.209841:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.209842:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:10.0:1619677448.209842:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.209843:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.209844:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.209844:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.209846:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96717895] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.209846:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:8.0:1619677448.209847:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.209848:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.209848:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.209850:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.209850:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.209851:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.209851:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.209851:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.209853:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.209854:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.209854:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:8.0:1619677448.209854:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.209854:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.209854:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677448.209855:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.209855:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.0:1619677448.209856:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.209857:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.209857:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.209859:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:8.0:1619677448.209859:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.209860:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.209860:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677448.209860:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.209861:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.0:1619677448.209862:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (23)--
00000800:00000200:21.2:1619677448.209863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:9.2:1619677448.209863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.0:1619677448.209863:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.209863:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.209864:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:30.0:1619677448.209865:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:4.2:1619677448.209866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.209868:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:21.2:1619677448.209868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:30.0:1619677448.209869:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.209869:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:30.0:1619677448.209871:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.209872:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.209872:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:26.0:1619677448.209873:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:18.0:1619677448.209873:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:6.0:1619677448.209873:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677448.209874:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:22.0:1619677448.209874:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.209875:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000800:00000200:2.0:1619677448.209875:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:26.0:1619677448.209876:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.209877:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677448.209877:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.209877:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677448.209877:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:22.0:1619677448.209878:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.209879:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.209880:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.209881:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:22.0:1619677448.209882:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.209883:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.209883:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.209883:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:22.0:1619677448.209884:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.209884:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.209884:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.209885:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.209885:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.209886:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677448.209888:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.209888:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96717893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.209889:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.209890:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.209891:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.209892:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.209892:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.209895:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.209895:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677448.209896:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.209896:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.209896:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.209897:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677448.209897:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:22.0:1619677448.209898:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.209899:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.209899:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677448.209899:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677448.209900:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677448.209900:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.209900:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.209900:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677448.209902:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.209903:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.2:1619677448.209904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.209904:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.209905:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677448.209905:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677448.209907:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677448.209908:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.209908:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.209909:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677448.209912:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.209914:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.209915:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.209915:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.209917:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.209919:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:17.0:1619677448.209920:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677448.209922:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.209923:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.209923:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.209926:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:26.0:1619677448.209929:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:10.0:1619677448.209943:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.209945:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.209946:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.209946:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.209947:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.209948:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.209950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.209956:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.209959:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.209960:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.209962:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.209964:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.209966:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.209967:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.209968:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.209969:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.209970:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.209973:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.209974:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.209976:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.209977:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.209977:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.209979:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.209980:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.209981:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.209981:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.209985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.209993:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.209995:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.209996:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.209997:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.209998:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.210001:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.210011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677448.210019:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677448.210022:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677448.210024:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.210025:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677448.210026:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.210028:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677448.210029:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.210031:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.210032:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677448.210033:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677448.210037:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.210039:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677448.210042:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.210043:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:12.0:1619677448.210044:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677448.210046:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:12.0:1619677448.210047:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677448.210048:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677448.210048:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677448.210052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.210060:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.210062:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.210064:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.210064:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.210066:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.210069:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.210119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:17.0:1619677448.210125:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677448.210128:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677448.210129:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677448.210130:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.210133:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.210135:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677448.210136:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.210137:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.210138:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.210139:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677448.210142:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.210143:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.210146:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.210147:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:17.0:1619677448.210147:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677448.210149:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677448.210149:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677448.210150:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677448.210150:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677448.210155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.210162:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.2:1619677448.210166:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677448.210169:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.210170:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (22)--
00000400:00000200:20.0:1619677448.210171:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.210172:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.210174:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.210175:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677448.210176:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.210179:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.210180:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.210181:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.210182:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.210183:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.210184:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.210185:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.210186:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.210187:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.210188:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.210189:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.210191:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.210191:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:31.0:1619677448.210193:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.210193:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.210194:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.210195:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.210196:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.210198:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.210199:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:30.0:1619677448.210199:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.210200:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.210201:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.210201:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.210202:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (23)++
00000800:00000200:31.0:1619677448.210203:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (24)--
00000800:00000200:31.0:1619677448.210204:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.210204:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.210204:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.210205:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.210205:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.210207:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.210207:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.210209:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.210210:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.210211:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.210213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:13.0:1619677448.210214:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.2:1619677448.210216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677448.210216:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.210217:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.210218:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.210220:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.210221:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677448.210221:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677448.210224:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:5.0:1619677448.210224:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.210224:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.210226:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:5.0:1619677448.210226:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.210227:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677448.210227:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.210228:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.210229:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677448.210233:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.210236:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.210237:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.210237:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.210238:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.210239:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.210281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677448.210286:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.210290:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677448.210290:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677448.210293:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.210300:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.210301:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.210303:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.210304:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.210305:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.210307:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.210310:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.210312:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.210314:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.210314:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.210315:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.210317:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.210317:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677448.210318:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677448.210318:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.2:1619677448.210323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.210331:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.210333:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.210334:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.210334:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.210336:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.210337:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.210345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.210351:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.2:1619677448.210352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677448.210354:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.210355:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.210357:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.210358:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677448.210359:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.210360:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:1.0:1619677448.210361:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:18.0:1619677448.210362:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.210362:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677448.210363:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.210364:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.210365:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.210366:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677448.210366:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.210368:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.210369:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.210370:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.210371:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.210373:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:1.0:1619677448.210373:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.210374:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.210376:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.210377:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.210377:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:1.0:1619677448.210377:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.210379:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:1.0:1619677448.210379:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.210380:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.210380:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.210381:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:1.0:1619677448.210382:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.210383:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.210383:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.210385:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.210386:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.210386:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.210387:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.210393:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.210396:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.210396:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.210397:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.210398:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:26.0:1619677448.210399:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.210400:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.210401:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.210402:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:12.0:1619677448.210403:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.210405:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.210406:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.210578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.210585:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.210588:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.210589:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.210591:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.210594:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.210595:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.210596:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.210597:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.210599:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.210600:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.210603:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.210605:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.210606:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.210607:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.210608:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.210609:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.210610:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.210610:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.210611:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.210620:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.210622:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.210623:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.210623:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.210624:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.210626:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.210696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.210702:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.210705:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.210705:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.210707:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.210709:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.210711:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.210712:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.210713:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.210715:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.210715:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.210719:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.210721:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.210724:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.210725:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.210725:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.210727:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.210728:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.210728:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.210729:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.210739:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.210742:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.210742:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.210743:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.210745:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.210748:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.210821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.210828:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.210830:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.210831:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.210833:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.210835:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.210836:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.210837:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.210838:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.210840:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.210841:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.210844:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.210845:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.210847:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.210848:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.210848:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.210850:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.210850:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.210851:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.210852:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.210853:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.210858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.2:1619677448.210862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.210863:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.210864:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.210865:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.210866:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.210867:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.210867:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.210868:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.210868:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.210868:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.210869:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.210869:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:17.0:1619677448.210871:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.210872:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:21.2:1619677448.210874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000400:00000200:17.0:1619677448.210874:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.210877:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.210878:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:20.0:1619677448.210879:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:17.0:1619677448.210879:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.210879:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000400:00000200:17.0:1619677448.210881:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.210881:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.210882:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:26.0:1619677448.210883:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:17.0:1619677448.210883:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.210883:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.210884:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.210885:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:20.0:1619677448.210886:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.210886:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.0:1619677448.210886:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:17.0:1619677448.210887:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.210889:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.210889:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:20.0:1619677448.210890:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.210892:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.210892:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.210893:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.210893:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:20.0:1619677448.210894:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.210894:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:20.0:1619677448.210895:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:17.0:1619677448.210896:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.210896:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.210897:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.210897:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:20.0:1619677448.210899:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.210900:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.210903:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.210904:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.210904:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.210905:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.210906:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.210908:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:13.0:1619677448.210908:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.0:1619677448.210909:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677448.210909:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:13.0:1619677448.210911:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:8.2:1619677448.210911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000400:00000200:13.0:1619677448.210912:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.210914:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.210915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (21)++
00000800:00000200:12.0:1619677448.210915:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.210917:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677448.210920:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.210921:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677448.210921:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.210922:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000400:00000200:5.0:1619677448.210922:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.210923:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.210924:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.210925:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.210926:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677448.210926:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.2:1619677448.210961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.210969:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.210972:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.210974:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.210977:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.210977:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.210979:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.210980:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.210981:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.210982:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.210984:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.210987:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.210989:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.210992:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.210993:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.210994:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.210995:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.210996:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.210997:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.210998:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.211001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.211008:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.211011:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.211012:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.211014:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.211014:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.211016:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.211020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.211027:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.211030:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.211031:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.211032:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.211033:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.211034:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.211047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.211052:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.211055:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.211056:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.211058:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.211061:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.211062:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.211064:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.211065:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.211066:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.211067:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.211070:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.211071:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.211073:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.211074:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.211075:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.211077:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.211077:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.211078:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.211079:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.211080:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677448.211083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.211086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.211091:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.211092:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.211093:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.211093:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.211093:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.211094:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.211095:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.211095:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.211096:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677448.211097:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.211216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.211222:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.211224:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.211225:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.211226:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.211229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:2.0:1619677448.211229:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.211230:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:20.0:1619677448.211231:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:2.0:1619677448.211231:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.211232:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.211233:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.211233:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.211234:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:5.0:1619677448.211237:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:2.0:1619677448.211237:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.211238:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:20.0:1619677448.211240:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.211241:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.211242:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.211242:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.211242:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:20.0:1619677448.211244:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:2.0:1619677448.211244:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.211244:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:20.0:1619677448.211245:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.211245:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.211246:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000400:00000200:20.0:1619677448.211247:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.211247:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:20.0:1619677448.211248:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.211249:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.2:1619677448.211250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.211252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000400:00000200:20.0:1619677448.211252:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.211254:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.211257:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.211257:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.211258:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.211258:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:20.0:1619677448.211259:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.211259:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:18.0:1619677448.211260:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:5.0:1619677448.211260:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.211261:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.211261:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677448.211261:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.0:1619677448.211262:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677448.211262:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677448.211263:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.2:1619677448.211263:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677448.211264:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.211265:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.211268:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677448.211268:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.211271:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.211272:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677448.211273:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677448.211274:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.211276:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.211277:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.211277:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.211278:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:13.0:1619677448.211279:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.211279:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.211280:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.211282:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677448.211282:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.211283:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.211284:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.211287:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.211289:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.211292:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.211293:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.211297:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.211299:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.211300:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.211301:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.211302:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.211303:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:4.2:1619677448.211304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677448.211308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.211312:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677448.211313:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677448.211314:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:26.0:1619677448.211315:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:13.0:1619677448.211316:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.211316:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.211316:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.211317:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.211317:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.211318:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.211318:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677448.211318:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.211320:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.211320:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:26.0:1619677448.211321:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:13.0:1619677448.211321:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:26.0:1619677448.211322:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717874] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.211324:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.211325:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.211326:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.211329:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.211330:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.211333:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.211334:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.211335:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.211336:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.211337:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.211337:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.211338:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.211350:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.211353:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.211354:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.211354:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.211357:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.211360:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.211450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.211455:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.211458:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.211458:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.211460:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.211463:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.211464:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.211465:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.211466:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.211467:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.211468:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.211472:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.211473:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.211476:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.211477:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.211477:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.211479:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.211479:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:21.2:1619677448.211480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677448.211480:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.211480:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.211486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.211488:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.211491:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.211494:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.211494:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.211495:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.211496:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.211496:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:30.0:1619677448.211497:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:17.0:1619677448.211497:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.211497:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.211499:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.211500:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.211501:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.211502:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.211503:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.211503:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.211504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000400:00000200:30.0:1619677448.211506:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.211508:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.211510:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.211511:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.211511:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.211512:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.211513:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.211514:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.211514:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.211514:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.211515:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.211515:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677448.211515:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.211516:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.211517:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:12.0:1619677448.211525:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.211527:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.211528:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.211530:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.211531:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.211532:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.211541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.211547:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.211548:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.211549:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.211551:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.211551:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.211552:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.211552:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:1.0:1619677448.211552:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.211553:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.211555:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677448.211555:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.211556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:1.0:1619677448.211556:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.211558:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.211559:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.211560:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.211561:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:10.0:1619677448.211564:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:1.0:1619677448.211565:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.211566:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.211567:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677448.211567:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:10.0:1619677448.211569:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.211570:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.211571:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.211571:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:10.0:1619677448.211572:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.211573:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.211573:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.211574:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:1.0:1619677448.211574:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.211575:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.211575:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:10.0:1619677448.211576:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.211577:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.211578:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:9.2:1619677448.211579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:10.0:1619677448.211581:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.211583:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:24.0:1619677448.211585:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:10.0:1619677448.211585:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.211586:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.211586:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.211587:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:24.0:1619677448.211588:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:10.0:1619677448.211588:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:24.0:1619677448.211589:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.211589:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.211589:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677448.211589:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.211591:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.211595:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:26.0:1619677448.211599:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.211601:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.211602:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.211602:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.211604:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.211605:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.211660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.211663:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.211666:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.211668:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.211669:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.211671:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.211672:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.211674:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717869] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.211675:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.211676:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.211677:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.211680:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.211682:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.211685:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.211686:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.211687:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.211688:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.211689:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.211690:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.211691:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.211692:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.2:1619677448.211697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:31.0:1619677448.211701:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.211702:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:31.0:1619677448.211704:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:5.0:1619677448.211704:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:31.0:1619677448.211705:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.211705:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.211706:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:31.0:1619677448.211707:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:30.0:1619677448.211708:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.211710:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.211719:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.211721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.211722:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.211723:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.211724:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.211724:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.211726:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.211727:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.211730:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.211732:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.211734:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.211734:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.211736:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.211737:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.211738:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.211739:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.211740:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.211743:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.211745:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.211747:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.211748:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.211748:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.211748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:30.0:1619677448.211750:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.211750:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.211751:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.211752:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:10.0:1619677448.211756:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.211758:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.211759:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.211760:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.211761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:12.0:1619677448.211761:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:10.0:1619677448.211762:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.211764:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:10.0:1619677448.211764:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677448.211765:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.211765:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.211765:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:10.0:1619677448.211766:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.211767:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.211767:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.211768:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:10.0:1619677448.211768:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.211768:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:12.0:1619677448.211769:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677448.211771:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.211771:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.211773:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.211773:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.211775:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.211775:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.211777:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.211778:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.211778:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.211779:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.211779:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677448.211779:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.211780:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:5.0:1619677448.211780:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.211782:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.211782:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:13.0:1619677448.211783:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.211783:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.211783:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677448.211784:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.211784:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:5.0:1619677448.211784:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.211785:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.211786:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677448.211786:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:9.2:1619677448.211788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.211789:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.211790:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677448.211791:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000800:00000200:5.0:1619677448.211793:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (24)--
00000800:00000200:5.0:1619677448.211794:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.211795:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.211796:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:2.0:1619677448.211796:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:5.0:1619677448.211797:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677448.211798:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000800:00000200:18.0:1619677448.211800:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000400:00000200:2.0:1619677448.211800:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.211801:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000400:00000200:18.0:1619677448.211801:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.211802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:2.0:1619677448.211802:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.211803:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.211804:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:18.0:1619677448.211805:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:20.0:1619677448.211806:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:2.0:1619677448.211806:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.211807:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677448.211807:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677448.211808:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.211818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.211825:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.211828:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.211829:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.211829:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.211830:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.211830:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.211845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:21.2:1619677448.211849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:20.0:1619677448.211850:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.211853:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.211855:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.211856:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:31.0:1619677448.211857:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:20.0:1619677448.211857:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.211859:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:31.0:1619677448.211860:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:20.0:1619677448.211860:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.211862:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.211863:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.211863:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:20.0:1619677448.211863:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.211864:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:31.0:1619677448.211865:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.211867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:31.0:1619677448.211868:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677448.211869:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.211870:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.211871:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.211872:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.211873:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:20.0:1619677448.211873:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677448.211874:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.211875:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.211876:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:26.0:1619677448.211877:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:20.0:1619677448.211877:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:31.0:1619677448.211878:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.211878:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:10.0:1619677448.211878:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.211879:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.211879:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.211880:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.211880:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:31.0:1619677448.211881:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:26.0:1619677448.211882:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.211883:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.211884:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000400:00000200:26.0:1619677448.211884:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.211884:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96717863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.211885:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:8.2:1619677448.211885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:31.0:1619677448.211886:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000400:00000200:26.0:1619677448.211886:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.211887:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:26.0:1619677448.211887:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.211887:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.211888:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677448.211888:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:26.0:1619677448.211891:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.211892:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:9.2:1619677448.211892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.0:1619677448.211892:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.211894:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:26.0:1619677448.211895:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.211895:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.211896:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.211896:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.211897:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.211897:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.211898:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.211898:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677448.211898:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:26.0:1619677448.211899:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.211899:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:26.0:1619677448.211900:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:24.0:1619677448.211901:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:8.2:1619677448.211901:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000800:00000200:8.0:1619677448.211906:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.211907:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.211908:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.211910:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:8.0:1619677448.211910:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677448.211912:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677448.211913:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.211914:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.211914:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:22.0:1619677448.211915:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677448.211915:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:6.0:1619677448.211916:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.211917:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.211917:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:22.0:1619677448.211919:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.211922:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.211922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.211928:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.211931:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.211931:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.211934:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.211937:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.211938:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.211939:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.211941:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.211942:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.211943:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.211946:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.211948:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.211951:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.211951:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.211952:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.211954:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.211954:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.211955:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.211955:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.211959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.211966:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.211968:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.211970:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.211970:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677448.211972:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.211974:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677448.212218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.2:1619677448.212223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677448.212226:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.212228:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.212231:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.212231:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.212232:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.212233:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.212233:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.212234:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:30.0:1619677448.212235:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677448.212236:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.212236:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.212237:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.212238:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.212239:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.212240:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.212242:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677448.212243:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.212243:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.212245:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.212246:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.212246:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.212247:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.212249:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.212250:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:10.0:1619677448.212250:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.212251:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.212252:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.212252:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.212253:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.212254:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.212254:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.212255:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.212256:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.212257:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.212258:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.212259:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.212260:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.212260:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.212265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.212266:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.212269:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.212270:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.212270:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.212272:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:6.0:1619677448.212272:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.212275:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:6.0:1619677448.212275:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:18.0:1619677448.212277:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.212277:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.212279:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.212282:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677448.212293:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.212299:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.212300:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.212301:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.212302:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.212302:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.212309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.212317:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.212320:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.212321:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.212322:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.212326:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.212327:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.212329:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.212330:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.212331:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.212332:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.212336:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.212337:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.212340:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.212340:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.212341:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.212343:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.212344:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.212345:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.212346:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.212346:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.212350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.212352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.2:1619677448.212356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:10.0:1619677448.212359:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.212360:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.212361:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.212362:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677448.212362:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677448.212363:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:21.2:1619677448.212365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:2.0:1619677448.212365:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.212366:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677448.212367:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.212368:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.212368:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.212369:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:5.0:1619677448.212370:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.212371:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.212371:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.212372:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (23)++
00000800:00000200:8.0:1619677448.212372:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:2.0:1619677448.212372:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677448.212373:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (24)--
00000800:00000200:20.0:1619677448.212373:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.212373:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (24)--
00000400:00000200:2.0:1619677448.212374:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.212375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:8.0:1619677448.212375:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:5.0:1619677448.212375:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:2.0:1619677448.212375:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.212376:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:2.0:1619677448.212377:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.212378:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677448.212379:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.212381:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.212381:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.212383:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000400:00000200:8.0:1619677448.212383:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:2.0:1619677448.212383:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:8.0:1619677448.212384:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96717857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.212385:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:8.0:1619677448.212385:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.212386:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:2.0:1619677448.212387:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677448.212388:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.212388:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:26.0:1619677448.212389:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.212389:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.212389:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.212391:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.212391:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:26.0:1619677448.212392:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.212392:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.212392:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:26.0:1619677448.212393:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:8.0:1619677448.212393:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.212393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677448.212395:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96717856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.212395:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.212396:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.212397:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.212398:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:8.0:1619677448.212398:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.212399:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.212399:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.212399:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.2:1619677448.212400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:26.0:1619677448.212401:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.212401:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.212401:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:17.0:1619677448.212402:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677448.212402:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:1.0:1619677448.212402:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:26.0:1619677448.212403:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.212403:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677448.212404:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.212404:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:26.0:1619677448.212405:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.212405:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.212405:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:26.0:1619677448.212406:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:1.0:1619677448.212406:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.212407:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.212407:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.212408:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000400:00000200:20.0:1619677448.212408:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.212409:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:12.0:1619677448.212409:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.212409:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:1.0:1619677448.212409:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.212410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000400:00000200:1.0:1619677448.212410:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677448.212411:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.212411:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.212411:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.212412:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677448.212412:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96717855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.212413:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.212414:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96717855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.212414:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.212415:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:1.0:1619677448.212415:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.212416:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.212416:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.212416:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:1.0:1619677448.212416:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.212417:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:20.0:1619677448.212417:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.212417:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677448.212417:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:13.0:1619677448.212418:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.212418:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.212418:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:24.0:1619677448.212419:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.212419:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:20.0:1619677448.212419:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.212420:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:10.0:1619677448.212420:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:13.0:1619677448.212421:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:10.0:1619677448.212421:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.212422:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:1.0:1619677448.212423:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.212424:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.212425:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.212426:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:1.0:1619677448.212426:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677448.212428:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.212429:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.212429:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677448.212430:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.212430:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.212431:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.212432:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.212432:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.212432:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.212433:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.212434:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677448.212434:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677448.212435:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.212435:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.212436:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.212438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:9.2:1619677448.212440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.212446:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:18.0:1619677448.212447:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:17.0:1619677448.212447:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.212448:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.212450:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:17.0:1619677448.212450:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.212450:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:18.0:1619677448.212451:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.212451:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.212453:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.212453:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677448.212456:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.212518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.212525:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.212528:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.212528:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.212530:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.212532:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.212534:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.212535:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.212536:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.212537:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.212538:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.212542:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.212544:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.212547:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.212548:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.212548:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.212550:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.212550:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.212551:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.212552:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.212564:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.212567:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.212568:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.212568:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.212571:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.212574:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677448.212585:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.212588:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.212589:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.212589:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.212590:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.212590:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.212740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.212748:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.212751:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.212753:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.212755:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.212755:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.212757:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.212759:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.212760:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.212761:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.212762:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.212765:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.212767:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.212769:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.212769:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.212770:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.212772:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.212772:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.212773:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.212774:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.212794:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.212796:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.212797:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.212798:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.212800:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.212800:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.212837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.212844:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.212847:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.212848:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.212848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677448.212850:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.212852:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.212854:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:1.0:1619677448.212854:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.212855:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.212857:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.212857:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.212857:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:10.0:1619677448.212859:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.212860:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.212860:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.212862:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.212864:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.212864:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.212866:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.212867:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.212868:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.212869:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.212870:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.212870:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.212871:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.212871:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.212873:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:1.0:1619677448.212873:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.212874:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.212874:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.212875:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:1.0:1619677448.212875:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.212878:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.212879:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:9.2:1619677448.212880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.212880:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.212881:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.212882:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.212883:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.212884:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.212885:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.212887:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.2:1619677448.212889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.212890:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.212891:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.212891:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.212893:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.212895:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:2.0:1619677448.212896:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677448.212897:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.212898:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.212899:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.212904:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.212906:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.212907:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.212909:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.212910:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.212912:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.212953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.212959:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.212961:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.212962:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.212964:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.212967:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.212968:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.212969:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.212971:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.212972:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.212973:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:21.2:1619677448.212975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000400:00000200:18.0:1619677448.212976:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.212978:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.212979:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.212980:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.212980:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:18.0:1619677448.212981:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.212982:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.212983:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.212983:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:5.0:1619677448.212983:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:18.0:1619677448.212984:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.0:1619677448.212985:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.212985:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.212987:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.212988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677448.212989:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.212990:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.212991:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.212992:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.212993:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:2.0:1619677448.212996:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:5.0:1619677448.212997:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.212998:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677448.212998:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.212998:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.213000:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.213000:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.213001:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000400:00000200:2.0:1619677448.213001:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.213002:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.213004:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:2.0:1619677448.213004:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677448.213005:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:9.2:1619677448.213006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.213006:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.213006:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.213010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:18.0:1619677448.213013:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.213016:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.213017:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.213017:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.213018:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:18.0:1619677448.213018:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.213018:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677448.213021:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:4.2:1619677448.213022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677448.213023:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:20.0:1619677448.213025:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.213025:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.213026:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:20.0:1619677448.213027:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.213028:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:26.0:1619677448.213029:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.213029:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677448.213030:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.213031:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.213032:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.213033:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:5.0:1619677448.213033:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.213034:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.213035:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.213036:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.213037:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.213038:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.213038:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.213039:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.213040:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.213042:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.213043:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.213044:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.213045:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.213046:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.213046:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.213049:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.213049:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.213049:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.213050:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.213052:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.213052:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.213052:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.213053:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.213054:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:26.0:1619677448.213055:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.213056:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.213056:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.213058:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.213058:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.213059:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.213060:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.213065:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.213067:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.2:1619677448.213067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:30.0:1619677448.213068:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.213070:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.213071:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.213072:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.213072:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:30.0:1619677448.213073:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:6.0:1619677448.213074:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.213075:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:20.0:1619677448.213075:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:6.0:1619677448.213076:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.213077:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.213078:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.213078:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.213080:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.213081:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:20.0:1619677448.213082:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677448.213083:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.213084:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.213085:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.213086:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677448.213090:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.213092:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677448.213094:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.213095:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677448.213096:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.213097:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677448.213098:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.213098:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.213099:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.213103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.213111:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.213113:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.213114:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.213115:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.213117:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.213119:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.213322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.213329:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.213331:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.213332:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:21.2:1619677448.213333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:2.0:1619677448.213334:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.213336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:2.0:1619677448.213337:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.213338:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.213339:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.213341:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:2.0:1619677448.213341:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.213342:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.213342:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677448.213343:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:24.0:1619677448.213345:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.213345:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:17.0:1619677448.213346:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.213346:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:4.2:1619677448.213347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677448.213348:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.213350:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.213350:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:1.0:1619677448.213350:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.213353:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.213353:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.213353:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.213354:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.213354:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.213355:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:1.0:1619677448.213355:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:8.0:1619677448.213356:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:2.0:1619677448.213356:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.213357:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:2.0:1619677448.213357:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:1.0:1619677448.213357:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.213358:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:8.0:1619677448.213358:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96717843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.213358:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.213358:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:1.0:1619677448.213359:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.213360:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.213360:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.213361:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.213362:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.2:1619677448.213363:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:10.0:1619677448.213364:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.213366:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.213366:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.213367:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96717841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.213367:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.213368:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.213369:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.213369:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.213370:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.213371:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.213371:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:1.0:1619677448.213372:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.213373:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.213374:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:1.0:1619677448.213374:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:8.0:1619677448.213375:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.213375:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.213376:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.213376:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.213377:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:10.0:1619677448.213377:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.213377:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.213377:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677448.213378:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.213378:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.213380:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.213380:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.0:1619677448.213381:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.213382:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.213383:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.213383:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:8.0:1619677448.213383:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.213384:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:8.0:1619677448.213385:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.213385:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:10.0:1619677448.213386:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.213386:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677448.213386:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:21.2:1619677448.213387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:10.0:1619677448.213387:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677448.213387:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:10.0:1619677448.213388:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.213390:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:9.2:1619677448.213391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.213393:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.213394:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.213396:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.213397:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:18.0:1619677448.213397:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:31.0:1619677448.213399:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677448.213400:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:22.0:1619677448.213401:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677448.213402:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:22.0:1619677448.213403:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:18.0:1619677448.213403:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.213404:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677448.213404:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:22.0:1619677448.213405:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.213405:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.213405:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.213408:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.213408:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677448.213408:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.213410:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96717840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.213411:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:18.0:1619677448.213411:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.213413:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.213414:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677448.213417:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.213417:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.213418:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677448.213419:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677448.213420:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677448.213424:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.213424:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:18.0:1619677448.213425:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:18.0:1619677448.213427:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:18.0:1619677448.213427:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.213428:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677448.213429:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:30.0:1619677448.213440:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.213443:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.213443:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.213445:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.213446:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.213448:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.213628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.213636:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.213638:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.213640:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.213642:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.213643:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.213644:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.213645:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.213647:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.213648:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.213649:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.213652:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.213653:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.213656:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.213657:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.213658:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.213660:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.213661:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.213662:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.213662:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.213666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.213674:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.213676:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.213677:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.213679:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.213680:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.213682:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.213712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.2:1619677448.213713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.213719:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.213721:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.213721:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:30.0:1619677448.213724:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677448.213724:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.213724:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.213727:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.213727:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.213727:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.213729:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.213729:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.213730:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.213731:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677448.213732:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.213733:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.213733:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.213734:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.213734:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.213735:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.213736:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.213739:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.213741:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.213742:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.213744:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.213746:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.213747:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.213747:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.213748:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.213748:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.213748:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.213750:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.213750:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.213751:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.213751:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.213752:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.213752:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.213753:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.213754:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.213755:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677448.213761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.213763:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677448.213765:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:12.0:1619677448.213766:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.213766:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.213767:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:31.0:1619677448.213768:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:26.0:1619677448.213768:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.213768:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.213768:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.213770:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.213770:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.213771:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.213771:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677448.213771:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.213772:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.213773:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:13.0:1619677448.213783:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.213785:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.213786:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.213787:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.213787:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.213789:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.213813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:6.0:1619677448.213819:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677448.213822:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677448.213823:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:6.0:1619677448.213825:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.213827:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.213829:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.213830:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.213831:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.213832:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.213833:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.213837:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.213839:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.213841:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.213842:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.213843:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.213844:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.213845:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677448.213846:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:6.0:1619677448.213847:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:6.0:1619677448.213847:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:9.2:1619677448.213851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.2:1619677448.213853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677448.213858:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000800:00000200:2.0:1619677448.213858:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677448.213860:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:22.0:1619677448.213860:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677448.213861:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:2.0:1619677448.213861:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.213862:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.213862:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.213864:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.213867:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.213874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.213881:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.213883:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.213884:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.213886:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.213889:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.213890:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.213891:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.213893:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.213894:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.213895:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:4.2:1619677448.213897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:10.0:1619677448.213898:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.213900:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.213903:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.213904:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677448.213904:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.213905:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677448.213906:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:10.0:1619677448.213906:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.213907:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677448.213907:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:10.0:1619677448.213908:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:24.0:1619677448.213909:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.213910:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.213910:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:24.0:1619677448.213912:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.213913:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.213914:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96717834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.213915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000400:00000200:24.0:1619677448.213916:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.213916:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677448.213916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:24.0:1619677448.213917:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.213920:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:24.0:1619677448.213921:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.213921:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:24.0:1619677448.213922:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.213922:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.213923:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:20.0:1619677448.213923:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:6.0:1619677448.213923:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.213924:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.213925:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.213926:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:24.0:1619677448.213926:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:20.0:1619677448.213926:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.213927:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677448.213927:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:22.0:1619677448.213927:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.213927:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:26.0:1619677448.213928:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:24.0:1619677448.213928:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.213929:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.213929:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:20.0:1619677448.213929:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.213930:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677448.213930:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:20.0:1619677448.213931:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.213932:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.213934:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677448.213934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677448.213935:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.213936:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.213938:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.213940:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.213942:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.213942:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677448.213943:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.213943:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.213945:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.213945:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:17.0:1619677448.213945:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.213945:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.213946:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000400:00000200:17.0:1619677448.213946:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.213947:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:17.0:1619677448.213948:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.213951:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.213951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.213958:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.213961:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.213962:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.213963:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.213964:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.213965:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.2:1619677448.214090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.214096:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.214098:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.214099:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.214101:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.214103:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.214105:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.214106:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.214107:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.214108:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.214109:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.214112:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.214114:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.214117:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.214118:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.214118:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.214120:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.214121:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.214121:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.214122:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.214133:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.214136:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.214137:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.214138:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.214140:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.214141:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.214219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.214225:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.214228:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.214228:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.214230:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.214233:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.214234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:6.0:1619677448.214234:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.214236:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.214238:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.214239:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.214240:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.214242:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:6.0:1619677448.214243:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.214245:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.214245:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:6.0:1619677448.214245:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.214247:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:6.0:1619677448.214248:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.214249:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:10.0:1619677448.214250:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.214250:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:10.0:1619677448.214252:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:6.0:1619677448.214252:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:10.0:1619677448.214253:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717830] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.214253:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.214254:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:10.0:1619677448.214255:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.214255:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.214255:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.214256:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677448.214257:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:21.2:1619677448.214260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677448.214260:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.214262:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677448.214265:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:10.0:1619677448.214265:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.214266:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:10.0:1619677448.214266:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.214266:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677448.214267:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:30.0:1619677448.214268:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:22.0:1619677448.214268:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:10.0:1619677448.214268:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.214268:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:30.0:1619677448.214269:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.214269:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:6.0:1619677448.214269:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.214270:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:31.0:1619677448.214271:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.214271:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.214274:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.2:1619677448.214274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.214281:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:31.0:1619677448.214283:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.214284:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.214285:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677448.214286:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.214286:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:17.0:1619677448.214286:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.214287:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.214288:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:17.0:1619677448.214288:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:31.0:1619677448.214289:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677448.214291:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.214298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.214306:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.214308:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.214309:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.214309:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.214310:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.214311:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.214550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.214558:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.214561:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.214563:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.214565:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.214566:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.214567:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.214568:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.214570:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.214570:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.214571:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.214575:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.214576:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:5.2:1619677448.214577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677448.214578:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.214579:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.214579:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.214581:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.214581:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.214583:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.214584:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.214585:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677448.214586:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.214588:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.214589:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.214591:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.214591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677448.214594:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.214595:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:12.0:1619677448.214595:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:26.0:1619677448.214596:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.214597:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.214597:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:26.0:1619677448.214598:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677448.214598:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.214599:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:26.0:1619677448.214599:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:12.0:1619677448.214600:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.214601:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.214602:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677448.214602:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.214602:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.214603:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.214604:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.214605:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.214606:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.214607:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.214607:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.214609:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.214609:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.214611:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.214612:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.214612:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.214618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677448.214621:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.214623:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.214624:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.214625:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.214625:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:12.0:1619677448.214627:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.214627:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:13.0:1619677448.214628:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.214628:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.214629:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677448.214705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.214711:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.214714:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.214715:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.214717:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.214720:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.214722:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.214723:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.214724:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.214725:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.214726:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.214730:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.214732:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.214735:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.214736:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.214736:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.214738:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.214738:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.214739:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.214740:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:26.0:1619677448.214750:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.214753:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.214753:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.214754:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.214756:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.214759:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.214773:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.214775:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.214776:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.214776:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.214778:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.214778:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.214792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.214799:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.214801:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.214802:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.214804:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.214806:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.214807:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.214809:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.214810:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.214811:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.214812:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.214815:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.214816:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.214818:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.214819:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.214820:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.214821:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.214822:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.214823:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.214823:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.214834:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.214836:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.214837:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.214838:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.214840:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.214841:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.214855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.214861:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.214864:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.214865:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.214867:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.214870:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.214871:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.214873:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.214874:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.214875:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.214876:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.214880:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.214881:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.214884:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.214885:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.214886:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.214888:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.214888:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.214889:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.214889:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677448.214901:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.214904:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.214905:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.214906:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.214907:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.214909:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.214936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.214943:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.2:1619677448.214944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677448.214946:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677448.214947:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.214948:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.214949:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.214949:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:31.0:1619677448.214950:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.214952:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.214952:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.214953:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677448.214954:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.214955:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.214955:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.214957:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.214957:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.214959:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.214959:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.214961:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.214962:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.214964:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.214965:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.214968:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.214970:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.214971:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.214972:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.214972:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.214973:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.214975:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.214975:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.214975:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.214976:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.214976:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.214977:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677448.214977:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.214978:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677448.214979:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.214980:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.214982:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.214982:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677448.214986:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:20.2:1619677448.214987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677448.214989:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677448.214990:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.214992:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.214992:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677448.214993:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677448.214994:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677448.214995:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.214995:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.214995:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:13.0:1619677448.214996:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.214996:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:13.0:1619677448.214997:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.214997:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.215000:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.215053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:21.2:1619677448.215056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.0:1619677448.215059:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:17.0:1619677448.215060:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:5.0:1619677448.215062:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:17.0:1619677448.215063:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677448.215064:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:20.0:1619677448.215065:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.215065:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677448.215067:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.215069:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.215070:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.215070:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:17.0:1619677448.215072:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.215072:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.215073:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.215074:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.215075:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.215076:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.215077:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.215077:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:17.0:1619677448.215079:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.215083:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000400:00000200:17.0:1619677448.215084:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.215085:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677448.215087:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.215087:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.215088:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.215089:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:17.0:1619677448.215090:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.215090:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:17.0:1619677448.215091:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.215091:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:17.0:1619677448.215092:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.215092:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677448.215093:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:17.0:1619677448.215094:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.215094:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:5.0:1619677448.215094:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:17.0:1619677448.215095:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677448.215096:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677448.215097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:21.2:1619677448.215099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.215105:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.215106:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677448.215107:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:12.0:1619677448.215107:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.215107:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000400:00000200:20.0:1619677448.215108:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.215108:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.215109:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.215110:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.215110:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:12.0:1619677448.215111:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677448.215112:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:12.0:1619677448.215112:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.215113:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.215115:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.215116:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677448.215126:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.215129:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.215130:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.215130:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.215131:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.215132:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.215308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.215316:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.215319:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.215321:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.215321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:30.0:1619677448.215323:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.215324:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.215325:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.215326:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.215328:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.215328:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:30.0:1619677448.215329:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.215330:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.215331:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.215332:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:30.0:1619677448.215335:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.215335:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.215338:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.215338:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.215340:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677448.215341:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.215341:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.215342:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.215342:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:2.0:1619677448.215343:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.215344:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.215344:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:2.0:1619677448.215344:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.215345:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.215346:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.215346:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:2.0:1619677448.215348:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.215350:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.215353:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.215354:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.215355:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.215357:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.215357:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:12.0:1619677448.215358:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677448.215358:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.215358:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677448.215360:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.215361:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.215363:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.215365:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.215366:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.215367:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.215369:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.215370:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.215371:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.215372:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.215374:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.215384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.215392:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.215395:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.215396:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.215398:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.215400:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.215402:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.215403:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.215404:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.215405:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.215406:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.215410:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.215412:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.215415:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.215415:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.215416:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.215418:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.215418:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.215419:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.215419:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.215430:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.215432:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.215433:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.215435:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.215436:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.215437:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.215503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.215509:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.215512:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.215513:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.215515:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.215518:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.215519:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.215520:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.215522:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.215523:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.215524:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.215528:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.215529:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.215533:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.215533:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.215534:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.215536:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.215536:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.215537:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.215538:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677448.215549:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.215552:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.215553:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.215555:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.215555:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.215558:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.215685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.215691:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.215694:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.215695:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.215697:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.215700:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.215702:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.215703:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.215704:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.215705:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.215706:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.215710:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.215711:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.215713:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.215714:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.215714:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.215716:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.215717:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.215718:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.215718:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.215723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.215729:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.215731:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.215732:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.215733:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.215734:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.215735:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.215762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.215770:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.215773:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.215774:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.215776:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.215779:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.215780:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.215781:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.215783:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.215784:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.215785:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.215788:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.215790:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.215793:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.215794:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.215795:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677448.215795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.215797:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.215797:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.215798:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.215799:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677448.215800:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.2:1619677448.215802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.215802:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.215803:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.215804:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.215807:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.215808:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.215809:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96717814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.215810:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:8.0:1619677448.215810:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.215811:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.215812:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:8.0:1619677448.215812:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.215813:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.215813:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:8.0:1619677448.215815:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.215816:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.215817:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.215819:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.0:1619677448.215821:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.215822:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.215822:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.215824:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.215825:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.215825:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.215826:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.215830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.215838:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.215841:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.215842:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.215842:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.215845:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.215847:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.215879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.215886:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.215889:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.215890:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.215892:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.215894:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.215895:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.215897:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.215898:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.215899:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.215900:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.215902:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.215904:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.215905:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677448.215905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:26.0:1619677448.215906:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.215907:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.215908:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.215909:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.215909:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.215910:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.215912:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.215914:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.215916:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.215917:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.215919:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.215920:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.215921:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:5.0:1619677448.215921:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:10.0:1619677448.215922:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.215922:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.215923:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:5.0:1619677448.215924:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.215925:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.215926:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.215927:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:10.0:1619677448.215928:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:5.0:1619677448.215930:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.215932:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:5.0:1619677448.215935:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.215936:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677448.215937:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.0:1619677448.215939:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677448.215939:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.215940:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.215941:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677448.215945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.215948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.0:1619677448.215951:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.215953:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:17.0:1619677448.215954:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.215954:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.215955:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.215957:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.215957:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.215958:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.215959:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.215960:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:17.0:1619677448.215962:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.215964:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.215965:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.215967:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.215968:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.215969:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.215972:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.215973:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.2:1619677448.215975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677448.215975:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.215976:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.215976:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.215978:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.215978:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.215979:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.215979:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.215983:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:8.2:1619677448.215984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.215986:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.215988:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.215989:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.215990:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.215991:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:30.0:1619677448.215992:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.215993:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.215993:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:30.0:1619677448.215994:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.215994:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.215995:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.215995:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.215996:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.215997:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.215998:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:30.0:1619677448.216000:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.216002:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.216004:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.216005:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.216006:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.216008:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.216008:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.216009:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.216010:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677448.216019:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.216022:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.216023:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.216024:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.216026:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.216027:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677448.216038:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.216040:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.216041:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.216042:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.216042:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.216043:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.216069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.216074:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.216077:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.216079:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.216080:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.216081:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.216083:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.216084:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.216086:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.216087:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.216088:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.216091:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.216093:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 440 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.216096:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.216097:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.216098:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.216100:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.216100:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.216101:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.216102:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:31.0:1619677448.216113:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.216116:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.216117:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.216118:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.216120:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.216121:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.216133:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.216135:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.216136:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.216137:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.216138:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.216139:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.216161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.216168:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.216171:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.216172:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.216173:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.216175:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.216176:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.216177:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.216178:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.216179:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.216180:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:21.2:1619677448.216180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:26.0:1619677448.216183:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.216185:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.216186:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.216187:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.216187:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.216188:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.216189:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677448.216189:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.216189:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.216190:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.216190:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:31.0:1619677448.216191:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.216193:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.216193:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.216195:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677448.216196:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.216197:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.216198:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.216199:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.216200:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:31.0:1619677448.216202:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.216202:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.216203:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.216204:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.216204:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.216205:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.216207:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.216207:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:10.0:1619677448.216207:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677448.216208:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.216210:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.216210:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.216211:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.216212:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.216216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.216223:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.216226:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.216227:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.216227:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.216229:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.216232:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.216234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.216241:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.216244:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.216245:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.216245:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.216246:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.216247:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.216312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.216319:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.216322:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.216323:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.216324:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.216326:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.216327:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.216329:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.216330:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.216331:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.216332:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.216334:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.216336:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.216337:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.216338:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.216338:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.216340:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.216340:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.216341:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.216341:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677448.216342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.216349:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.216352:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:2.0:1619677448.216352:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.216353:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:10.0:1619677448.216353:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.216354:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.216355:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:2.0:1619677448.216355:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.216356:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.216356:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.216358:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:2.0:1619677448.216358:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677448.216359:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717805] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.216360:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.216361:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.216362:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.216366:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.216368:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.216371:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.216371:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.216372:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.216374:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.216375:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.216375:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.216376:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677448.216381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.216389:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.216392:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.216393:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.216394:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.216396:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.216397:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.216463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.216466:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.216469:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.216471:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.216472:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:20.0:1619677448.216473:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.216475:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:5.0:1619677448.216475:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.216476:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.216477:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:20.0:1619677448.216478:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.216479:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.216480:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:20.0:1619677448.216481:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.216481:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.216484:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.216485:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.216486:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.216487:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:20.0:1619677448.216488:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:17.0:1619677448.216489:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.216490:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.216491:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.216491:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:17.0:1619677448.216491:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.216492:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:17.0:1619677448.216492:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:20.0:1619677448.216494:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.216494:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.216495:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.216496:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677448.216496:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.216497:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.216500:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677448.216500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.216501:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.216502:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.216504:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.216504:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.216505:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.216506:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.216507:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.216508:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:9.2:1619677448.216509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:18.0:1619677448.216510:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.216511:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.216512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.216512:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.216513:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.216516:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.0:1619677448.216516:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:1.0:1619677448.216517:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677448.216519:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:1.0:1619677448.216519:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:24.0:1619677448.216520:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.216520:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.216520:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.216521:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.216522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:8.0:1619677448.216522:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.216525:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.216529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:2.0:1619677448.216529:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.216532:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.216534:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.216534:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677448.216535:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:2.0:1619677448.216535:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.216535:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677448.216538:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.216539:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.216539:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.216540:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.216541:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.216549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.216554:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.216557:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.216558:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.2:1619677448.216559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:22.0:1619677448.216561:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.216564:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.216565:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.216567:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.216568:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:22.0:1619677448.216568:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.216569:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.216570:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.216571:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.216574:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.216575:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:22.0:1619677448.216575:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.216577:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.216577:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.2:1619677448.216577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:30.0:1619677448.216580:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:22.0:1619677448.216580:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.216581:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.216581:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.216582:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.216582:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:30.0:1619677448.216583:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.216583:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:30.0:1619677448.216584:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.216584:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.216584:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:30.0:1619677448.216585:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.216585:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677448.216585:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.216587:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.216588:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.216589:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.216589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.216592:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.2:1619677448.216593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:5.0:1619677448.216594:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.216595:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.216596:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.216596:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:5.0:1619677448.216596:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:17.0:1619677448.216597:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:5.0:1619677448.216597:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96717800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.216598:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.216599:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677448.216599:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:5.0:1619677448.216599:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.216600:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.216600:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:17.0:1619677448.216600:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:5.0:1619677448.216600:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.216601:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:17.0:1619677448.216601:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.216601:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.216601:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.216602:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:17.0:1619677448.216604:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.216604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:18.0:1619677448.216605:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677448.216606:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.216607:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.216608:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:18.0:1619677448.216609:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:5.0:1619677448.216609:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.216610:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96717799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.216612:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.216612:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.216613:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:18.0:1619677448.216613:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.216613:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.216614:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:18.0:1619677448.216614:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:5.0:1619677448.216614:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.216615:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.216616:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677448.216617:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.216617:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:18.0:1619677448.216619:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:5.2:1619677448.216619:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:20.0:1619677448.216621:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.216621:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:18.0:1619677448.216622:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:5.0:1619677448.216622:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677448.216623:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677448.216625:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.216626:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.216627:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677448.216627:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.216628:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:2.0:1619677448.216629:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.216630:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:13.0:1619677448.216630:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.216631:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.216632:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.216632:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677448.216632:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:2.0:1619677448.216632:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.216633:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:13.0:1619677448.216633:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.216633:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:13.0:1619677448.216635:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.216635:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96717798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.216637:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677448.216637:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.216638:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677448.216638:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677448.216639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677448.216639:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.216642:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.216643:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.216645:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:2.0:1619677448.216645:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.216645:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.216646:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.216647:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.216647:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:26.0:1619677448.216648:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.216648:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.216648:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:10.0:1619677448.216649:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:2.0:1619677448.216649:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:26.0:1619677448.216650:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.216650:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.216651:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:10.0:1619677448.216652:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.216653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.216654:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.216654:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.216655:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:26.0:1619677448.216656:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.216656:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:5.2:1619677448.216656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000400:00000200:10.0:1619677448.216657:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.216659:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.216660:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.216661:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.216662:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.216664:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:10.0:1619677448.216664:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.216664:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:18.0:1619677448.216665:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.216665:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:10.0:1619677448.216666:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.216666:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.216667:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.216667:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.216668:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:10.0:1619677448.216669:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.216670:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.216670:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.216671:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.216674:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.216674:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.216675:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.216676:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.216677:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677448.216679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677448.216682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.216687:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.216689:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:2.0:1619677448.216690:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.216691:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:18.0:1619677448.216691:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.216691:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.216692:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.216693:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:2.0:1619677448.216693:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.216696:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.216736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.216742:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.216745:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.216746:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.216748:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.216750:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.216752:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.216753:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.216755:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.216756:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.216757:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.216760:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.216761:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.216764:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.216764:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.216765:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.216767:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.216767:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.216768:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.216768:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.216781:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.216783:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.216785:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.216785:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.216787:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.216790:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.216814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.216821:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.216824:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.216825:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.216827:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.216830:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.216831:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.216833:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.216834:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.216835:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.216836:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.216839:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.216841:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.216844:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.216844:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.216845:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.216847:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.216847:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.216848:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.216848:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677448.216859:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.216861:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.216862:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.216863:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.216865:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.216866:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.216877:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.216879:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.216880:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.216881:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.216881:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.216883:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.216957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.216964:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.216967:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.216968:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.216970:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.216972:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.216973:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.216975:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.216976:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.216977:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.216978:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.216981:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.216982:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.216985:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.216986:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.216986:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.216988:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.216989:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.216990:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.216990:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.217001:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.217004:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.217005:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.217006:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.217008:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.217011:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.217051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.217058:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.2:1619677448.217058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677448.217061:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.217063:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.217063:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.217065:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.217066:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.217066:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.217067:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677448.217069:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:17.0:1619677448.217069:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.217070:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.217071:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.217072:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.217072:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.217074:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:17.0:1619677448.217074:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.217076:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.217078:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.217079:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.217080:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.217081:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.217083:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:17.0:1619677448.217084:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.217086:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.217087:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:17.0:1619677448.217087:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.217088:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.217090:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.217090:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.217090:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.217090:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.217091:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677448.217091:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.217092:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677448.217093:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.217093:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.217094:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.217095:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.217096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.217104:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.217106:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:12.0:1619677448.217106:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:18.0:1619677448.217108:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.217108:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:12.0:1619677448.217109:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:18.0:1619677448.217110:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:12.0:1619677448.217110:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.217112:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.217113:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:13.0:1619677448.217113:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.217115:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.217155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.217162:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.217165:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.217167:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.217169:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.217170:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.217171:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.217172:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.217174:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.217175:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.217176:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:20.2:1619677448.217178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:30.0:1619677448.217179:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.217182:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.217184:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.217184:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677448.217184:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.217185:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.217186:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.217186:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:30.0:1619677448.217187:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.217188:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.217189:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.217189:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.217190:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.217190:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:5.0:1619677448.217191:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.217193:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.217194:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.217196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:5.0:1619677448.217196:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.217197:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.217198:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.217201:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:5.0:1619677448.217201:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.217202:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.217203:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.217203:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.217204:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:31.0:1619677448.217205:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.217205:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.217206:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677448.217206:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.217207:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.217207:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.217207:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.217208:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.217209:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.217209:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.217210:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.217211:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.217211:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.217215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.217223:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.217226:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.217227:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.217227:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.217229:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.217231:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.217242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.217248:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.217250:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.217252:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.217253:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.217254:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.217256:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.217257:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.217258:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.217259:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.217261:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.217264:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.217266:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.217267:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.217268:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.217269:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.217270:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.217271:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.217272:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.217272:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.217276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.217285:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.217287:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.217288:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.217289:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.217290:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.217292:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:8.2:1619677448.217296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.217305:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.217307:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.217308:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.217309:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.217309:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.217311:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.2:1619677448.217318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.217324:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.217327:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.217327:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.217330:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.217332:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.217333:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.217335:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.217336:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.217337:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.217338:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:4.2:1619677448.217339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677448.217341:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.217343:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.217346:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.217347:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.217347:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.217347:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.217349:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:10.0:1619677448.217350:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:1.0:1619677448.217350:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.217350:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.217351:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677448.217351:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:10.0:1619677448.217353:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.217356:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.217356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:10.0:1619677448.217357:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.217358:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.217359:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.217360:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.217361:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677448.217362:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:10.0:1619677448.217365:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.217365:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.217366:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.217366:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.217367:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:8.0:1619677448.217368:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.217370:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.217370:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.217371:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677448.217371:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:10.0:1619677448.217373:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.217373:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.217374:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.217374:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.217378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677448.217384:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.217387:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677448.217388:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.217388:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:6.0:1619677448.217388:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:24.0:1619677448.217389:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.217389:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677448.217391:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.217392:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.217392:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.217394:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.217397:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.217541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.217546:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.217548:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.217549:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.217551:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.217553:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.217554:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.217555:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.217557:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.217558:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.217559:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.217562:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.217563:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.217565:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.217566:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.217567:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.217568:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.217569:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.217570:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.217571:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.217571:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.217575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.217576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.217581:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:17.0:1619677448.217583:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677448.217583:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.217583:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.217584:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677448.217585:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.217586:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.217587:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.217588:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.217591:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.217650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.217658:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.217660:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.217661:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.217663:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.217663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:26.0:1619677448.217666:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.217666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677448.217667:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.217669:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.217669:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:26.0:1619677448.217670:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.217671:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.217671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677448.217672:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:1.0:1619677448.217672:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.217673:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.217673:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677448.217674:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.217675:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.217678:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.217678:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.217679:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.217680:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.217681:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.217682:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:18.0:1619677448.217682:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.217683:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:26.0:1619677448.217684:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.217684:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:18.0:1619677448.217684:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.217684:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.217685:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677448.217685:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.217686:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.217686:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677448.217686:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.217686:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.217687:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:1.0:1619677448.217687:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.217688:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.217688:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:18.0:1619677448.217688:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.217688:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:26.0:1619677448.217689:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:31.0:1619677448.217690:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:26.0:1619677448.217690:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000400:00000200:18.0:1619677448.217690:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.217691:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:18.0:1619677448.217691:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.217692:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96717782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.217693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677448.217693:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.217694:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.217695:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.217696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677448.217696:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:31.0:1619677448.217697:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.217697:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.217699:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.217700:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.217700:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000400:00000200:31.0:1619677448.217701:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.217701:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:2.0:1619677448.217702:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677448.217702:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:18.0:1619677448.217703:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.217703:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:1.0:1619677448.217703:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.217704:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.217704:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.217704:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:1.0:1619677448.217704:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:18.0:1619677448.217705:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.217705:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:2.0:1619677448.217705:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.217705:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:10.0:1619677448.217706:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677448.217706:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.217707:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:2.0:1619677448.217707:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.217708:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.217708:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:18.0:1619677448.217708:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:9.2:1619677448.217708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.217709:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.217709:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.217709:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.217710:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:31.0:1619677448.217711:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.217711:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.2:1619677448.217711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.217712:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.217712:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:31.0:1619677448.217713:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:18.0:1619677448.217713:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:8.2:1619677448.217713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.217713:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.217714:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677448.217715:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:18.0:1619677448.217715:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.217716:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677448.217716:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677448.217716:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.217718:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.217719:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677448.217719:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677448.217720:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:21.2:1619677448.217721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677448.217721:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.217722:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.217723:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.217723:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:22.0:1619677448.217725:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:8.0:1619677448.217725:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.217727:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.0:1619677448.217727:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.217727:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.217728:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677448.217729:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.217729:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:5.0:1619677448.217729:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.217730:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:22.0:1619677448.217730:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.217732:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:22.0:1619677448.217732:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.217732:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677448.217733:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.217734:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.217735:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677448.217743:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.217746:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.217747:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.217747:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.217748:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.217748:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.217748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.217755:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.217758:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.217758:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.217760:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.217763:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.217765:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.217766:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.217768:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.217769:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.217770:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.217773:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.217775:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.217778:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.217778:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.217779:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.217781:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.217781:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.217782:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.217782:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.217794:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.217797:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.217798:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.217798:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.217800:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.217803:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.217840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.217843:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.217846:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.217848:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.217849:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.217850:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.217852:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677448.217853:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.217854:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.217855:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.217856:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677448.217860:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.217861:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677448.217863:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.217864:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677448.217865:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.217866:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677448.217867:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.217868:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.217869:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.217873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.217881:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.217883:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.217884:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.217884:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.217886:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.217889:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.218042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.218048:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.218050:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.218051:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.218053:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.218055:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.218057:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.218058:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.218059:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.218060:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.218061:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.218064:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.218065:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.218068:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.218069:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.218069:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.218071:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.218071:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.218072:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.218072:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.218085:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.218087:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.218088:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.218088:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.218090:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.218093:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.218105:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.218107:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.218108:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.218109:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.218110:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.218110:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.218156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.218162:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.218165:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.218166:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.218168:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.218171:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.218172:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677448.218173:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.218174:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.218176:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.218176:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.218180:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.218182:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.218184:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.218185:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.218186:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.218187:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.218188:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.218189:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.218190:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.218190:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:8.2:1619677448.218194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.218196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.218201:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.218203:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:5.0:1619677448.218203:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:18.0:1619677448.218204:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.218205:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677448.218205:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.218206:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.218208:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.218208:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.218210:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.218239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.218247:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.218250:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.218252:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.218254:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.218255:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.218257:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.218258:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717777] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.218259:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.218261:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.218261:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.218265:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.218267:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.218270:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.218271:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.218271:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.218273:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.218274:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.218274:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.218275:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.218288:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.218291:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.218292:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.218293:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.218299:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.218303:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.218330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.218335:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.218338:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.218340:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.218341:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.218342:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.218344:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.218345:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.218346:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.218347:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.218348:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.218351:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.218353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:20.0:1619677448.218353:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.218355:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.218356:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.218357:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.218358:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.218359:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.218360:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.218360:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:31.0:1619677448.218361:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677448.218364:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:8.2:1619677448.218365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000400:00000200:31.0:1619677448.218366:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.218368:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677448.218369:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.218370:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677448.218371:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717775] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.218373:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.218373:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.218373:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:31.0:1619677448.218374:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.218375:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.218376:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.218378:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.218378:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.218379:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.218380:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:31.0:1619677448.218381:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677448.218384:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.218385:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677448.218386:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.218387:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677448.218388:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677448.218389:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677448.218390:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677448.218402:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.218405:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.218406:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.218406:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.218408:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.218412:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.218434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.218440:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.218443:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.218443:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.218446:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.218448:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.218449:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.218450:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.218452:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.218453:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.218454:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.218457:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.218459:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.218461:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.218461:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.218462:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.218463:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.218464:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.218464:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.218465:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.218469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.218477:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.218479:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.218479:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.218480:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.218481:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.218483:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.218561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.218569:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.218571:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.218572:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.218575:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.218578:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.218579:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.218581:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717773] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.218582:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.218583:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.218584:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.218587:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.218589:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.218592:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.218593:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.218594:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.218595:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.218596:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.218596:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.218597:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.218601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.218609:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.218611:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.218612:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.218612:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.218615:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:1.0:1619677448.218617:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.218876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.218884:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.218887:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.218889:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.218890:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.218892:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.218894:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.218895:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.218896:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.218897:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.218898:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.218902:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.218903:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.218906:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.218907:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.218908:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.218910:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.218910:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.218911:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.218911:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.218916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.218922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:20.0:1619677448.218928:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:31.0:1619677448.218930:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:20.0:1619677448.218931:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:31.0:1619677448.218932:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:20.0:1619677448.218932:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.218933:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.218934:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:31.0:1619677448.218935:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.218936:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:8.2:1619677448.218937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677448.218939:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.218939:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:31.0:1619677448.218941:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:20.0:1619677448.218941:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677448.218943:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.218943:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.218944:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000400:00000200:31.0:1619677448.218945:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.218945:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.218945:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000400:00000200:31.0:1619677448.218946:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.218946:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:31.0:1619677448.218947:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.218950:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.218952:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.218955:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.218955:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.218956:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.218957:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.218958:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677448.218959:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677448.218960:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:10.0:1619677448.218969:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.218972:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.218972:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.218973:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.218974:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.218977:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.218992:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.218995:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.218995:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.218996:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.218997:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.218997:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.219061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.219068:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.219071:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.219073:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.219075:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.219075:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.219076:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.219078:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.219078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:30.0:1619677448.219079:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.219080:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.219080:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.219082:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677448.219083:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.219085:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:30.0:1619677448.219086:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.219087:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.219088:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.219088:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677448.219089:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.219089:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677448.219089:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.219091:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.219091:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:5.0:1619677448.219091:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677448.219092:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:5.0:1619677448.219092:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.219093:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:5.0:1619677448.219094:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.219095:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.219096:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.219099:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.219101:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.219104:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.219104:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.219105:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:5.0:1619677448.219106:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (24)++
00000800:00000200:5.0:1619677448.219108:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (25)--
00000800:00000200:13.0:1619677448.219109:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:5.0:1619677448.219109:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:13.0:1619677448.219110:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.219110:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677448.219111:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.219111:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000400:00000200:13.0:1619677448.219112:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.219112:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677448.219116:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.2:1619677448.219117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677448.219117:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.219119:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677448.219119:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.219120:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.219120:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:13.0:1619677448.219121:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.219122:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.219122:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677448.219122:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.219123:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.219123:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.219123:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677448.219123:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.219124:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.219125:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.219235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:31.0:1619677448.219243:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677448.219245:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:31.0:1619677448.219247:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.219250:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.219250:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:31.0:1619677448.219252:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.219253:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.219254:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.219255:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.219256:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677448.219259:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.219261:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.219264:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.219264:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.219265:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.219267:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.219267:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677448.219268:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:31.0:1619677448.219269:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:9.2:1619677448.219273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.219281:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.219283:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.219285:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.219285:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.219287:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.219290:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.219299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:9.2:1619677448.219302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.219308:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.219310:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.219311:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.219312:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677448.219313:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.219313:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.219314:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.219315:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.219316:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.219316:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.219317:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:2.0:1619677448.219317:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:18.0:1619677448.219319:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.219320:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.219321:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.219322:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.219325:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.219327:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.219329:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.219330:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.219331:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.219333:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.219333:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.219334:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.219335:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.219336:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.2:1619677448.219341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.219346:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:6.0:1619677448.219347:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.2:1619677448.219347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.219348:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.219349:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677448.219350:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677448.219350:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677448.219351:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.219351:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.219353:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.219354:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.219355:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677448.219357:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.219358:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.219359:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.219361:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.219363:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.219364:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.219366:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.219367:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.219368:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.219371:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.219372:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.219375:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.219376:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.219377:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.219378:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.219379:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.219380:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.219380:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.219392:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.219395:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.219396:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.219398:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.219399:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.219402:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.219433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.219439:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.219441:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.219444:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.219444:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.219446:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.219448:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.219450:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.219451:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.219452:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.219453:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.219456:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.219458:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.219460:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.219460:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.219461:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.219462:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.219463:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.219464:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677448.219465:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677448.219466:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:8.2:1619677448.219469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:21.2:1619677448.219472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.219477:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.219479:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677448.219479:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.219480:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677448.219480:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.219480:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.219481:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.219482:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.219483:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677448.219484:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:21.2:1619677448.219623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.219631:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.219633:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.219636:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.219637:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.219638:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.219640:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.219641:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.219642:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.219644:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.219645:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.219647:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.219649:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.219651:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.219652:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.219653:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.219655:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.219655:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.219656:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.219657:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.219667:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.219670:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.219670:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.219672:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.219674:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.219675:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.219731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.219739:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.219741:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.219742:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.219743:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.219746:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.219747:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.219749:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717763] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.219750:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.219751:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.219751:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.219754:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.219756:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.219758:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.219759:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.219759:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.219761:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.219761:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.219762:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.219762:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.219773:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.219775:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.219776:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.219777:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.219779:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.219780:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.219801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.219806:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.219809:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.219809:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.219811:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.219814:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.219815:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.219817:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.219818:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.219819:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.219820:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.219823:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.219825:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.219827:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.219828:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.219829:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.219831:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.219831:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.219832:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.219832:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.219837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.219845:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.219847:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.219848:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.219848:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.219851:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.219853:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.219855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.219858:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.219861:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.219863:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.219864:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.219865:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.219867:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.219868:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.219870:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.219871:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.219872:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.219876:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.219877:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.219880:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.219881:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.219882:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.219884:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.219884:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.219885:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.219886:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.219896:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.219899:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.219900:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.219901:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.219903:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.219904:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.219918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.219926:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.219928:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:12.0:1619677448.219929:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677448.219929:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.219931:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.219932:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.219932:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.219933:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:24.0:1619677448.219934:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.219934:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.219934:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677448.219935:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.219936:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717760] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.219938:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.219939:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.219940:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.219943:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.219944:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.219947:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.219948:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.219949:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.219951:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.219952:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.219952:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.219953:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.219958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.219966:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.219969:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.219970:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.219970:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.219972:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.219975:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.220013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.220018:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.220021:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.220022:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.220024:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.220027:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.220028:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.220030:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.220031:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.220032:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.220033:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.220036:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.220037:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.220040:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.220041:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.220041:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.220043:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.220043:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.220045:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.220046:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.220047:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.220052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.220057:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677448.220057:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.220059:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.220060:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.220060:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.220061:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.220061:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.220061:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.220063:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.220066:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.220185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.220191:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.2:1619677448.220195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.0:1619677448.220195:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.220197:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.220198:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.220201:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.220203:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:5.0:1619677448.220203:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:5.0:1619677448.220204:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.220206:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:5.0:1619677448.220206:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.220207:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.220209:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:5.0:1619677448.220209:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677448.220210:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.220212:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.220213:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.220214:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.220215:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.220215:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:30.0:1619677448.220217:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.220218:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.220218:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.220219:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000400:00000200:30.0:1619677448.220220:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.220220:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.0:1619677448.220222:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677448.220222:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:30.0:1619677448.220223:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.220224:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:30.0:1619677448.220225:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.220225:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.220226:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:30.0:1619677448.220227:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.220228:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:9.2:1619677448.220228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:30.0:1619677448.220229:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.220231:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.220231:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.2:1619677448.220231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677448.220232:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677448.220233:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:30.0:1619677448.220234:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.220234:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677448.220234:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:20.0:1619677448.220235:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.220236:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.220237:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.0:1619677448.220237:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.220238:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.220238:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.220240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:8.0:1619677448.220241:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.220242:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677448.220243:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677448.220245:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.220245:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.220247:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.220248:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.220249:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.220249:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.220251:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.220251:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.220252:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677448.220318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.220326:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.220329:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.220330:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.220332:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.220334:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.220336:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.220337:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.220338:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.220339:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.220340:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.220344:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.220345:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.220348:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.220348:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.220349:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.220350:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.220351:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.220352:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.220352:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.220364:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.220367:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.220367:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.220369:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.220371:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.220372:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.220471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.220478:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.220481:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.220482:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.220483:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.220485:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.220487:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.220488:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.220490:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.220491:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.220492:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.220496:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.220498:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.220501:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.220501:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.220502:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.220504:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.220504:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.220505:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.220505:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:10.0:1619677448.220516:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.220519:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.220519:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.220520:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.220522:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.220525:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.220546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.220551:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.220554:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.220555:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.220557:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.220559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000400:00000200:2.0:1619677448.220560:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.220561:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.220562:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.220564:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.220564:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.220565:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:2.0:1619677448.220565:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.220567:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:2.0:1619677448.220569:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.220570:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:2.0:1619677448.220571:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:20.0:1619677448.220572:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.220573:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.220574:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.220574:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:20.0:1619677448.220575:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.220576:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:2.0:1619677448.220576:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.220576:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:20.0:1619677448.220577:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717753] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.220577:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.220578:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:20.0:1619677448.220579:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.220580:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.220581:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:8.2:1619677448.220583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:20.0:1619677448.220584:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.220586:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.220588:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.220589:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.220589:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.220591:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.220591:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.220592:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.220592:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.220592:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.220593:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:5.0:1619677448.220593:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.220595:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.220595:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.220597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.220600:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677448.220601:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.220602:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.2:1619677448.220603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:5.0:1619677448.220603:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.220604:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.220606:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.220607:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677448.220608:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.2:1619677448.220610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.220611:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.220613:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.220615:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.220616:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677448.220616:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.220617:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:20.0:1619677448.220618:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.220618:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677448.220619:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.220620:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.220620:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000400:00000200:20.0:1619677448.220621:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.220621:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.220621:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.220622:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:20.0:1619677448.220624:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.220625:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.220628:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.220629:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.220630:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.220632:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.220632:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.220633:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.220634:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.220644:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.220647:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.220648:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.220649:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.220650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:13.0:1619677448.220650:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.2:1619677448.220651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:12.0:1619677448.220652:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677448.220657:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.220659:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:31.0:1619677448.220660:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:6.0:1619677448.220660:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.220662:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.220663:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.220665:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.220665:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.220666:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677448.220667:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:22.0:1619677448.220667:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.220668:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.220669:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.220670:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.220670:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.220672:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.220672:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677448.220674:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.220675:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.220676:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.220682:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.220682:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.220685:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.220686:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.220688:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.220689:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.220689:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677448.220690:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.220690:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.220691:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.220692:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.220692:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677448.220693:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.220693:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:22.0:1619677448.220693:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:31.0:1619677448.220694:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:22.0:1619677448.220694:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677448.220695:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.220698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.220705:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.220706:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677448.220707:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.220709:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.220709:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.220710:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.220711:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.220711:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.220712:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.220712:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.220714:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.0:1619677448.220716:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.220819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:5.2:1619677448.220820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:24.0:1619677448.220826:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677448.220828:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677448.220829:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.220830:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677448.220831:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.220831:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:24.0:1619677448.220832:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.220833:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.220835:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.220835:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.220837:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:24.0:1619677448.220837:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677448.220838:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.220838:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.220840:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.220840:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.220841:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.220841:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.220842:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:24.0:1619677448.220842:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:4.2:1619677448.220846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:26.0:1619677448.220847:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.220847:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.220849:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.220850:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:24.0:1619677448.220852:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.220853:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:26.0:1619677448.220854:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.220854:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.220855:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:24.0:1619677448.220855:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:10.0:1619677448.220855:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.220856:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:24.0:1619677448.220856:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677448.220856:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:24.0:1619677448.220857:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:26.0:1619677448.220858:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:24.0:1619677448.220858:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000400:00000200:10.0:1619677448.220858:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.220859:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.220859:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677448.220859:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677448.220860:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677448.220861:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.220861:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.220862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:10.0:1619677448.220863:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.220864:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96717747] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.220865:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677448.220865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677448.220866:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.220868:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.220871:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.220871:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.2:1619677448.220871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.220872:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677448.220872:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.220873:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:10.0:1619677448.220874:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.220874:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:6.0:1619677448.220874:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.220875:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677448.220875:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:10.0:1619677448.220875:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.220875:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677448.220875:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.220876:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677448.220876:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.220877:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.220877:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677448.220877:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.220878:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677448.220878:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.220879:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.220879:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:1.0:1619677448.220879:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:10.0:1619677448.220880:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.220881:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677448.220881:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:1.0:1619677448.220882:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.220883:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.220884:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:1.0:1619677448.220884:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.220885:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:4.2:1619677448.220886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677448.220886:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.220888:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.220889:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.220890:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.220891:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:1.0:1619677448.220891:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.220893:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.220894:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:1.0:1619677448.220894:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.220895:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677448.220895:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.220895:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.220896:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677448.220896:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:22.0:1619677448.220897:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.220897:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.220900:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677448.220900:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.220900:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.220901:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.220903:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.220904:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.220904:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.220905:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.220909:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.220911:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.220912:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.220913:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.220914:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.220914:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677448.220915:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.220918:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.220919:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.220921:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.220922:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.220924:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.221212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677448.221219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.221224:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.221226:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.221228:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.221229:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677448.221230:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.221231:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.221232:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.221233:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.221235:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.221236:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.221238:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.221239:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.221240:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.221241:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.221243:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.221244:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.221245:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.221246:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.221246:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.221247:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.221250:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.221251:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.221254:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.221254:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.221255:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.221255:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.221256:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.221256:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.2:1619677448.221257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:26.0:1619677448.221258:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.221258:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.221259:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.221259:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.221260:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.221260:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.221260:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (23)++
00000800:00000200:26.0:1619677448.221262:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.221262:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (24)--
00000800:00000200:10.0:1619677448.221262:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.221263:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.221264:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:8.2:1619677448.221264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.221267:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:5.2:1619677448.221267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677448.221269:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.221271:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.221272:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.221272:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:10.0:1619677448.221272:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:31.0:1619677448.221273:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.221274:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96717743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.221274:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:31.0:1619677448.221275:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.221275:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:10.0:1619677448.221275:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.221276:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.221277:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.221277:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:31.0:1619677448.221278:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.221278:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:20.0:1619677448.221278:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.221278:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.221279:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.0:1619677448.221279:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.221280:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.221281:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.221281:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:31.0:1619677448.221282:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.221282:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677448.221284:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.221286:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.221287:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.221287:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.221289:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.221290:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.221291:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.221291:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677448.221303:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.221305:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.221306:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.221308:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.221310:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.221312:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.221320:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.221322:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.221323:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.221324:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.221324:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.221325:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.221371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.221377:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.221380:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.221381:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.221383:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.221386:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.221387:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677448.221389:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.221390:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.221391:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.221392:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677448.221396:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.221397:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677448.221401:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.221401:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677448.221402:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677448.221404:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677448.221404:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.221405:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.221405:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677448.221417:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.221420:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.221421:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.221421:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.221423:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:26.0:1619677448.221427:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677448.221450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.0:1619677448.221454:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.221457:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677448.221459:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.221460:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.221462:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.221463:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.221465:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.221466:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.221467:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.221468:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:21.2:1619677448.221472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:5.0:1619677448.221472:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.221474:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.221477:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.221478:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.221478:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:24.0:1619677448.221480:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677448.221480:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.221481:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.221481:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:24.0:1619677448.221482:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:5.0:1619677448.221482:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.0:1619677448.221483:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.221485:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.221486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:4.2:1619677448.221486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677448.221488:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.221490:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.221491:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96717740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.221493:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.221493:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:24.0:1619677448.221494:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.221495:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.221495:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:26.0:1619677448.221496:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.221496:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:20.0:1619677448.221497:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.221497:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.221499:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.221500:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.221500:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.221500:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:24.0:1619677448.221501:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.221502:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:10.0:1619677448.221502:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.221504:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.221504:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:24.0:1619677448.221505:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.221506:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:10.0:1619677448.221506:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.221507:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.221508:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.221508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000800:00000200:24.0:1619677448.221509:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.221509:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:10.0:1619677448.221509:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677448.221511:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000400:00000200:10.0:1619677448.221512:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.221513:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.221513:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:10.0:1619677448.221514:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.221516:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.221517:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.221517:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:10.0:1619677448.221518:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:21.2:1619677448.221519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:10.0:1619677448.221519:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.221519:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:12.0:1619677448.221520:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:10.0:1619677448.221520:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.221520:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:10.0:1619677448.221521:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.221522:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.221523:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (23)--
00000800:00000200:12.0:1619677448.221523:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.221523:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.221523:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677448.221524:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:12.0:1619677448.221525:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.221526:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:12.0:1619677448.221527:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.221528:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677448.221528:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.221529:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677448.221529:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:30.0:1619677448.221533:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.221535:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.221536:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.221537:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.221539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.221540:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.221541:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.221542:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.221544:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.221546:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.221547:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.221549:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.221550:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.221551:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.221553:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.221554:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.221554:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.221557:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.221559:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.221561:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.221561:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.221562:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.221563:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.221564:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.221565:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.221565:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.221575:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.221577:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.221578:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.221580:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.221581:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.221582:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.221608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.221610:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.221613:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.221615:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677448.221617:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.221617:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.2:1619677448.221618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:20.0:1619677448.221618:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.221620:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717737] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.221621:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.221622:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.221623:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.221624:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:20.0:1619677448.221626:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:22.0:1619677448.221628:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:20.0:1619677448.221628:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.221628:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.221631:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.221631:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.221632:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.221633:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.221633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:22.0:1619677448.221634:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.221635:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.221635:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:22.0:1619677448.221636:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:20.0:1619677448.221636:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.221636:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:22.0:1619677448.221637:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.221639:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.221639:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.221639:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:22.0:1619677448.221641:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.221642:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.221643:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:22.0:1619677448.221645:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.221646:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:2.0:1619677448.221646:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.221648:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.221649:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:2.0:1619677448.221649:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.221650:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677448.221650:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.221651:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.221651:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:2.0:1619677448.221651:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.221652:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96717735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.221653:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:13.0:1619677448.221653:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677448.221653:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677448.221654:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.221654:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.221655:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:2.0:1619677448.221655:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.221656:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677448.221656:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.221657:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:8.2:1619677448.221659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:2.0:1619677448.221660:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.221662:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.221665:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.221665:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677448.221666:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (23)++
00000800:00000200:12.0:1619677448.221667:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.221668:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (24)++
00000800:00000200:2.0:1619677448.221669:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:2.0:1619677448.221669:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677448.221670:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.221670:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:12.0:1619677448.221671:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.221671:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.221671:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (24)--
00000800:00000200:2.0:1619677448.221671:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677448.221672:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677448.221672:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.221673:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.221674:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:5.0:1619677448.221674:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.221676:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.221677:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.221678:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.221679:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.221679:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.221680:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.221682:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.2:1619677448.221683:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.221685:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:5.0:1619677448.221686:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:30.0:1619677448.221687:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.221689:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.221689:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.221689:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677448.221690:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:17.0:1619677448.221691:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:30.0:1619677448.221692:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.221692:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677448.221694:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.221695:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.221695:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.221696:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.221698:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.221700:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.221700:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.221701:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.221702:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.221703:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.221704:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.221705:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.221705:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.221706:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.221706:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.221708:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.221709:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.221709:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.221710:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:17.0:1619677448.221711:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:4.2:1619677448.221711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.221712:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.221713:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677448.221714:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.221715:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677448.221715:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:17.0:1619677448.221717:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:26.0:1619677448.221718:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.2:1619677448.221718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677448.221718:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.221718:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.221719:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.221720:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:12.0:1619677448.221720:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:10.0:1619677448.221721:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.221723:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.221723:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677448.221724:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.221725:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.221726:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:26.0:1619677448.221726:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:12.0:1619677448.221726:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.221727:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:31.0:1619677448.221728:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:26.0:1619677448.221728:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.221728:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677448.221728:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:31.0:1619677448.221729:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677448.221729:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.221729:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.221729:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.221730:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:26.0:1619677448.221730:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.221732:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:13.0:1619677448.221732:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.221734:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:26.0:1619677448.221735:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.221737:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.221740:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.221741:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.221742:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.221743:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.221744:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.221745:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.221745:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.221757:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.221760:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.221760:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.221761:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.221763:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.221766:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.221942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919065782400] (20)++
00000800:00000200:18.0:1619677448.221948:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (21)++
00000800:00000200:18.0:1619677448.221951:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:2.0:1619677448.221951:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000800:00000200:20.2:1619677448.221954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:18.0:1619677448.221954:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.221956:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:18.0:1619677448.221956:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.221958:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:20.0:1619677448.221959:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:18.0:1619677448.221959:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.221961:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.221962:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.221962:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.221962:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:18.0:1619677448.221963:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677448.221966:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.221968:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.221969:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.221971:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:18.0:1619677448.221971:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.221972:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000400:00000200:20.0:1619677448.221973:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.221973:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:20.0:1619677448.221974:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.221975:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.221975:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:18.0:1619677448.221975:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (21)++
00000400:00000200:20.0:1619677448.221976:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.221976:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (22)--
00000800:00000200:18.0:1619677448.221977:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (21)--
00000400:00000200:20.0:1619677448.221979:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.221981:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.2:1619677448.221981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.221984:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.221984:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.221985:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.221987:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.221987:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.221988:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.221988:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677448.221989:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.221991:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.221992:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.221994:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.221994:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.221996:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:22.0:1619677448.222000:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.222003:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.222004:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.222004:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.222006:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.222008:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.222034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.222042:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.222044:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.222045:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.222047:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.222050:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.222051:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.222053:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.222054:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.222055:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.222056:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.222059:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.222061:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.222063:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.222064:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.222064:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.222066:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.222066:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.222067:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.222069:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.222069:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.222075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.222080:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.222081:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.222082:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.222083:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.222083:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.222084:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.222084:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.222085:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.222085:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.222086:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677448.222171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.222179:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.222181:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.222182:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.222183:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.222185:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.222186:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.222187:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.222188:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.222189:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.222190:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.222192:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.222193:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.222194:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.222195:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.222195:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.222197:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.222197:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.222197:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.222198:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677448.222199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.222206:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.222208:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:31.0:1619677448.222209:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:10.0:1619677448.222209:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.222210:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.222211:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.222211:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.222212:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.222213:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.222213:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.222213:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:31.0:1619677448.222214:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.222215:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.222217:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.222218:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.222219:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.222222:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.222223:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.222225:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.222226:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.222226:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.222228:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.222228:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.222228:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.222229:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.222229:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.222230:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.222231:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.222231:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.222232:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.222232:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:30.0:1619677448.222240:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.222242:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.222243:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.222244:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.222246:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.222247:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.222304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.222310:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.222313:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.222313:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.222315:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.222318:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.222320:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.222321:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.222322:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.222323:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.222324:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.222327:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.222329:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.222332:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.222333:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.222333:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.222335:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.222336:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.222337:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.222337:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.222349:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.222352:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.222353:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.222354:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.222356:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.222357:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677448.222365:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.222368:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.222369:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.222370:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.222370:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.222371:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.222406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.222414:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.222416:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.222417:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.222419:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.222420:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.222422:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.222423:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.222424:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.222425:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.222426:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.222430:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.222432:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.222435:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.222436:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.222436:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.222438:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.222439:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.222439:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.222440:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.222452:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.222455:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.222456:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.222457:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.222459:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.222462:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.222485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.222494:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.222496:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.222499:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.222500:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.222501:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.222502:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.222503:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.222504:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.222505:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.222506:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.222510:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.222511:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.222514:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.222515:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.222515:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.222517:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.222518:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.222519:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.222519:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.222531:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.222533:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.222534:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.222535:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.222537:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.222540:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.222542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.222547:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:6.0:1619677448.222548:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.222550:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:22.0:1619677448.222551:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.222551:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.222552:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : ACK - routed
00000800:00000200:6.0:1619677448.222552:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.222553:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.222553:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:6.0:1619677448.222554:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:20.0:1619677448.222555:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.222556:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.222558:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.222559:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.222560:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.222561:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.222564:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.222566:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.222568:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.222569:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.222569:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.222571:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.222571:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.222572:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.222573:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.222577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.222585:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.222587:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.222588:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.222589:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:20.0:1619677448.222590:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.222591:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.2:1619677448.222623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.222629:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.222632:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.222633:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.222635:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.222638:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.222640:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.222641:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717722] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.222643:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.222644:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.222645:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.222647:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.222649:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.222652:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.222653:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.222654:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.222656:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.222656:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.222657:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.222658:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.222659:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.222664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.222668:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.222669:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677448.222671:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677448.222671:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:12.0:1619677448.222672:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.222672:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.222673:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677448.222674:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.222675:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.222676:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.222712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.222719:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.222721:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.222722:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.222724:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.222726:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.222728:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.222729:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.222730:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.222731:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.222732:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.222735:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.222737:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.222740:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.222741:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.222742:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.222743:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:21.2:1619677448.222744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:8.0:1619677448.222744:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.222745:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.222745:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.222749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.222750:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.222752:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.222753:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.222755:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.222757:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.222758:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:22.0:1619677448.222759:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.222760:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.222760:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:22.0:1619677448.222761:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.222761:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:22.0:1619677448.222762:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.222762:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.222764:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.222764:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.222766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:22.0:1619677448.222767:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.222770:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.222771:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.222772:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.222773:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.222773:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.222774:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (23)++
00000800:00000200:17.0:1619677448.222775:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.222775:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (24)--
00000800:00000200:22.0:1619677448.222777:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:22.0:1619677448.222777:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.222778:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:17.0:1619677448.222778:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677448.222779:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:17.0:1619677448.222779:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.222779:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.222780:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.222782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.222785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677448.222789:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677448.222790:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:1.0:1619677448.222791:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:6.0:1619677448.222792:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:1.0:1619677448.222792:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.222793:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.222793:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.222794:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:1.0:1619677448.222794:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.222797:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.222876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.222882:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.222885:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.222886:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.222888:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.222891:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.222892:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.222894:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.222895:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.222896:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.222897:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.222900:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.222902:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.222905:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.222905:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.222906:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.222908:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.222908:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.222910:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.222911:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.222911:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.222914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.222916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.222922:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:17.0:1619677448.222922:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677448.222924:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.222924:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677448.222924:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677448.222925:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:17.0:1619677448.222925:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.222926:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.222928:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.222930:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.223017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.223023:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.223025:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.223026:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:4.2:1619677448.223027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677448.223028:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.223030:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.223032:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.223033:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.223034:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677448.223034:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.223035:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.223037:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:1.0:1619677448.223037:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.223038:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.223041:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.223041:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.223044:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.223044:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.223045:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.223047:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.223047:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.223047:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:26.0:1619677448.223048:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.223048:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:26.0:1619677448.223049:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.223050:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.223050:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.223050:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.223051:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.223051:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677448.223054:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.223056:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.223058:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.223059:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.223059:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.223061:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.223061:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.223062:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:13.0:1619677448.223062:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:26.0:1619677448.223063:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:13.0:1619677448.223065:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.223065:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.223067:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.223068:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.223070:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.223073:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.223076:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.223076:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.223078:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.223080:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.223080:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:21.2:1619677448.223082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.223087:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.223090:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.223091:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:30.0:1619677448.223093:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:2.0:1619677448.223093:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.223095:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.223096:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:2.0:1619677448.223096:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.223097:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:2.0:1619677448.223097:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:30.0:1619677448.223098:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.223099:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:2.0:1619677448.223099:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.223100:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.223101:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.223102:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.223105:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.223107:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.223110:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.223110:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.223111:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.223113:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.223113:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.223114:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.223115:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677448.223119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.223127:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.223130:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.223130:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.223132:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.223133:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.223135:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.223317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.223325:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.223328:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.223330:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.223332:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.223333:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.223334:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.223336:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.223337:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.223338:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.223339:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.223342:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.223344:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.223346:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.223347:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.223348:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.223350:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.223350:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.223351:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.223352:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677448.223352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.223358:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.223360:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.223361:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.223363:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.223363:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:17.0:1619677448.223366:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.223366:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.223367:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.223368:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:12.0:1619677448.223369:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.223370:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.223370:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:17.0:1619677448.223371:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.223372:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.223372:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:17.0:1619677448.223373:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.223377:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.223379:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.223382:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.223383:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.223383:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.223385:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.223386:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.223386:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.223387:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.223391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.223398:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.223401:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.223402:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.223402:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.223404:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.223407:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677448.223449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.223457:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.223460:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.223461:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.223463:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.223465:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.223466:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.223468:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.223469:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.223470:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.223471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000400:00000200:10.0:1619677448.223471:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.223474:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.223475:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:10.0:1619677448.223476:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.223477:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:10.0:1619677448.223479:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.223480:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:10.0:1619677448.223480:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:5.0:1619677448.223480:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.223481:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.223482:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677448.223482:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.223483:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.223484:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:5.0:1619677448.223484:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:10.0:1619677448.223485:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.223485:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:5.0:1619677448.223486:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717712] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.223487:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.223488:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.223489:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.2:1619677448.223491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:5.0:1619677448.223493:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.223495:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.223495:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 440 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:13.0:1619677448.223497:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:26.0:1619677448.223498:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:13.0:1619677448.223498:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.223498:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.223499:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.223499:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.223500:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:13.0:1619677448.223500:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:26.0:1619677448.223501:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:12.0:1619677448.223501:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.223501:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.223502:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677448.223502:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.223502:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.223503:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677448.223505:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:5.0:1619677448.223505:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:21.2:1619677448.223510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (21)++
00000800:00000200:30.0:1619677448.223514:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.223516:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:30.0:1619677448.223517:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.223518:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.223518:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677448.223519:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.223519:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:30.0:1619677448.223520:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.223521:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.223523:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.223589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.223592:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.223595:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.223597:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.223598:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.223600:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.223602:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.223603:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.223604:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.223605:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.223606:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.2:1619677448.223609:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:20.0:1619677448.223614:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.223616:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.223616:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.223619:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:20.0:1619677448.223619:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.223620:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.223621:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:31.0:1619677448.223622:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.223623:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677448.223623:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.223623:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.223624:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.223625:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:31.0:1619677448.223626:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:20.0:1619677448.223626:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000400:00000200:31.0:1619677448.223627:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.223627:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677448.223628:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.223629:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.223630:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:20.2:1619677448.223631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.223634:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.223635:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.223636:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.223636:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:30.0:1619677448.223637:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.223638:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.223638:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.223639:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.223639:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:30.0:1619677448.223639:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.223639:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.223640:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.223641:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677448.223642:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.223642:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.223642:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.223643:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.223643:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.223644:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.2:1619677448.223646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.2:1619677448.223648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.223652:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.223655:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.223656:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.223656:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:1.0:1619677448.223657:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.223658:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.223659:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.223660:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.223660:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.223661:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:1.0:1619677448.223662:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.223663:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.223664:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:1.0:1619677448.223664:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.223665:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.223666:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.223670:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.223672:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.223675:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.223676:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.223676:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.223678:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.223678:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.223679:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.223680:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.223690:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.223693:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.223694:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.223695:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.223697:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.223698:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.223762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.223770:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.223773:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.223775:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.223776:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.223777:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.223779:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.223780:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.223782:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.223782:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.223784:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.223787:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.223789:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.223791:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.223792:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.223793:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.223794:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.223795:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.223796:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.223798:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.223798:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677448.223804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.223807:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.223809:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:21.2:1619677448.223810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:13.0:1619677448.223810:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.223811:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.223812:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.223813:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.223813:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.223814:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.223815:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.223816:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677448.223817:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.223820:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.223821:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.223823:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.223825:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677448.223826:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.223827:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:12.0:1619677448.223828:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.223829:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.223829:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:18.0:1619677448.223830:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.223830:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.223830:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:18.0:1619677448.223831:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.223831:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:18.0:1619677448.223832:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.223835:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.223837:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.223840:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.223841:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.223842:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.223843:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.223844:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.223845:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.223845:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677448.223856:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.223858:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.223859:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.223861:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.223862:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.223864:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.223946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.223952:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.223955:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.223955:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.223958:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.223960:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.223962:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.223963:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.223964:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.223965:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.223966:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.223969:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.223971:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.223973:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.223974:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.223974:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.223976:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.223976:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.223978:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.223979:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.223979:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.223982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.223985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.223991:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.223992:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.223992:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.223992:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.223992:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.223993:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.223994:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.223994:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.223995:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.223996:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.224001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.224008:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.224010:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.224011:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.224012:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.224012:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677448.224012:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:21.2:1619677448.224082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.224088:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.224091:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.224092:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.224094:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.224096:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.224098:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.224099:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.224100:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.224101:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.224103:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.224106:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.224107:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.224110:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.224111:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.224112:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.224113:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.224114:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.224115:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.224115:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.224125:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.224128:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.224129:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.224130:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.224132:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.224135:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:26.0:1619677448.224142:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.224145:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.224145:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.224146:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.224147:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:21.2:1619677448.224147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677448.224148:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:31.0:1619677448.224155:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.224157:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.224159:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.224161:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.224162:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.224163:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.224165:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.224166:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.224167:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.224168:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.224171:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.224173:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.224176:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.224177:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.224177:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.224179:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.224179:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.224180:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.224181:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.224190:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.224193:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.224194:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.224195:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.224197:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.224197:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.224233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677448.224235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:10.0:1619677448.224240:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.224241:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:10.0:1619677448.224243:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.224244:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677448.224244:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.224245:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:10.0:1619677448.224247:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.224248:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.224250:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.224251:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.224252:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.224253:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.224253:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.224255:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.224256:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.224257:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.224258:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.224258:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.224259:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677448.224259:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.224264:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000400:00000200:10.0:1619677448.224265:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.224267:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.224268:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677448.224269:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.224270:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.224271:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.224271:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.224272:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677448.224272:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.224273:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677448.224273:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.224274:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.224274:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677448.224275:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.224275:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.224276:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.224277:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.224278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.224286:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.224288:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.224288:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.224289:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677448.224289:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.224291:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.224292:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.224292:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.224298:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.224299:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:6.0:1619677448.224300:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.224303:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.224314:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.224317:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.224317:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.224318:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.224318:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.224319:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.224426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.224429:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.224431:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.224433:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.224435:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.224436:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.224437:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.224439:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.224440:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.224441:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.224442:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.224446:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.224446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677448.224448:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.224451:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.224452:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.224452:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.224454:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677448.224454:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.224455:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.224455:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.224456:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677448.224457:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.224458:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.224460:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.224463:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.224465:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.224466:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.224467:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.224468:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:26.0:1619677448.224468:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.224469:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.224471:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.224472:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.224473:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.224474:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.224475:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.224476:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.224476:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.224478:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.224478:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.224479:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.224481:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.224481:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.224482:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.224483:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.224487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.224494:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.224497:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.224498:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.224499:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.224500:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.224504:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.224547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.224552:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.224555:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.224557:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677448.224558:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.224559:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.224561:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677448.224562:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.224564:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.224565:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.224566:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.224569:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.224571:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.224574:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.224575:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677448.224575:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.224577:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.224578:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.224578:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.224579:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:10.0:1619677448.224589:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.224592:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.224593:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.224593:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.224595:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:10.0:1619677448.224598:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.224610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.224618:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.224621:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.224623:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.224625:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.224625:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.224627:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677448.224628:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.224629:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.224630:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.224631:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.224635:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.224636:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:21.2:1619677448.224637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:31.0:1619677448.224639:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.224640:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.224640:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.224642:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.224642:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:31.0:1619677448.224643:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.224644:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.224644:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677448.224644:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.224647:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.224647:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.224649:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.224649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677448.224651:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.224652:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.224653:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.224654:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.224655:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:1.0:1619677448.224657:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:20.0:1619677448.224658:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.224660:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:1.0:1619677448.224660:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.224661:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.224661:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.224662:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.224663:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.224664:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:1.0:1619677448.224664:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.224665:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.224666:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:1.0:1619677448.224666:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677448.224667:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.224668:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.224671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.224679:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.224681:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.224682:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.224684:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.224684:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.224686:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.2:1619677448.224768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.224774:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.224777:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.224778:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.224779:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.224781:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.224782:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.224784:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.224785:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.224786:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.224787:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.224790:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.224791:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.224794:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.224795:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.224795:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.224797:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.224797:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.224798:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.224799:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.2:1619677448.224806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:6.0:1619677448.224811:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:18.0:1619677448.224812:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677448.224814:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.224815:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.224815:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:6.0:1619677448.224815:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.224816:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:6.0:1619677448.224818:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.224819:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.224820:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:18.0:1619677448.224821:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.224822:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.224824:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.224825:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.224826:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.224827:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.224830:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.224832:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.224834:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.224835:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.224835:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.224837:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.224838:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.224838:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.224839:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.224843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.224851:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.224853:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.224853:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.224854:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.224855:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.224857:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.224864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.224871:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.224874:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.224874:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.224875:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.224875:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.224875:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:21.2:1619677448.224971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.224978:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.224981:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.224982:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.224984:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.224986:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.224988:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.224989:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.224990:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.224991:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.224992:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.224995:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.224997:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.225000:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.225000:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.225001:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.225003:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.225004:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.225004:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.225005:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.225009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.225017:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.225020:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.225021:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.225021:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.225023:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.225026:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.225052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.225057:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.225060:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.225061:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.225063:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.225066:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.225067:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.225069:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.225070:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.225071:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.225072:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.225076:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.225077:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.225081:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.225082:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.225083:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.225085:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.225085:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.225086:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.225086:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677448.225098:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.225100:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.225101:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.225103:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.225104:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.225106:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.225111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.225116:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.225118:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.225119:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.225120:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.225120:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.225121:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.225121:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677448.225122:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.225123:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.225123:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.225125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:30.0:1619677448.225126:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.225127:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.225128:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.225130:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.225131:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.225132:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.225132:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.225134:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:30.0:1619677448.225135:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.225135:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.225137:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.225138:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.225140:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.225140:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.225141:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.225141:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:26.0:1619677448.225142:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:30.0:1619677448.225143:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.225143:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677448.225143:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.225144:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677448.225144:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.225145:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:26.0:1619677448.225145:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.225146:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.225149:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.225151:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.225155:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.225155:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.225156:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.225156:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.225157:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:10.0:1619677448.225157:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.225159:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:10.0:1619677448.225159:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.225160:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.225161:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.225161:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.225162:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.225163:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.225163:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.225169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.225172:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.225175:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.225175:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.225175:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.225176:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.225177:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.225178:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.225178:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.225179:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.225181:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.225372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.225381:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.225383:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.225384:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.225386:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.225389:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.225390:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.225392:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.225393:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.225394:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.225394:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.225397:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.225399:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.225401:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.225401:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.225402:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.225404:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.225404:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.225405:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.225405:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677448.225418:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.225420:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.225421:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.225422:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.225425:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.225426:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.225456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.225462:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.225465:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.225465:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.225467:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.225470:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.225471:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.225473:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.225474:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.225475:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.225476:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.225479:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.225481:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.225484:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.225485:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.225485:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.225487:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.225488:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.225488:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.225489:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.225500:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.225503:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.225504:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.225505:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.225506:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.225508:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.225576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.225582:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.225586:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.225586:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.225588:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.225590:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.225592:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.225593:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717688] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.225594:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.225595:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.225596:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.225600:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.225602:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.225605:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.225606:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.225606:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.225608:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.225608:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.225610:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.225611:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.225611:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.2:1619677448.225617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.225620:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:2.0:1619677448.225622:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:26.0:1619677448.225623:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.225623:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.225624:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.225624:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.225625:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:10.0:1619677448.225626:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.225626:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:10.0:1619677448.225629:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.225719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.225728:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.225731:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.225733:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.225733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.225734:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.225735:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.225737:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.225738:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.225739:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.225740:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.225740:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677448.225741:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.225742:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.225743:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:30.0:1619677448.225745:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.225746:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.225748:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.225749:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.225750:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.225751:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:26.0:1619677448.225751:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677448.225752:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:26.0:1619677448.225752:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.225753:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.225754:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.225754:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677448.225754:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.225755:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677448.225755:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.225756:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:26.0:1619677448.225758:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.225760:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.2:1619677448.225760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.225762:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.225763:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.225764:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.225765:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.225765:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.225766:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.225767:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677448.225769:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.225771:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.225772:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.225774:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.225775:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.225777:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.225777:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677448.225780:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.225781:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.225781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000400:00000200:31.0:1619677448.225782:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.225783:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.225785:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.0:1619677448.225789:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.225792:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.225793:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.225794:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.225794:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.225795:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.225808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.0:1619677448.225812:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.225814:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677448.225817:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.225817:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.225818:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.225820:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.225821:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.225822:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.225823:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.225824:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.225827:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.225829:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.225831:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.225832:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.225833:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.225834:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.225835:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.225835:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.225836:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.225840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.225848:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.225850:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.225850:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.225852:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.225853:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.225854:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:8.2:1619677448.225865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000800:00000200:21.2:1619677448.225870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.0:1619677448.225875:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:30.0:1619677448.225877:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.225877:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.225878:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.225878:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677448.225879:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:30.0:1619677448.225880:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.0:1619677448.225880:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:30.0:1619677448.225882:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.225884:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.225885:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.225886:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.225888:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.225889:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.225890:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.225891:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.225894:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.225896:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:21.2:1619677448.225896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:30.0:1619677448.225898:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.225899:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.225900:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.225901:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.225902:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.225902:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677448.225903:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.225905:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.225905:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677448.225905:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.225906:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.225908:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.225910:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.225911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677448.225912:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:10.0:1619677448.225912:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:18.0:1619677448.225913:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.225914:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.225915:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.225915:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.225916:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:18.0:1619677448.225916:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.225916:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.225919:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:18.0:1619677448.225920:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000400:00000200:10.0:1619677448.225920:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.225921:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.225922:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677448.225922:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677448.225923:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677448.225923:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.225925:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.225926:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677448.225927:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677448.225928:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.225929:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.225929:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.225930:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677448.225931:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.225934:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.225934:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.225935:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.225936:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.225936:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:22.0:1619677448.225943:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.225945:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.225946:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.225946:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.225948:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.225951:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.225951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.225957:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.225960:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.225961:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.225963:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.225966:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.225967:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.225968:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717682] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.225970:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.225971:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.225972:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.225975:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.225976:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.225979:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.225979:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.225980:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.225982:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.225982:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.225983:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.225984:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.225987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.225995:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.225997:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.225998:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.226000:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.226001:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.226002:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.226011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.226019:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.226022:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.226023:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.226025:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.226027:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.226028:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.226030:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.226031:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.226032:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.226033:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.226036:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.226038:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.226041:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.226042:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.226043:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.226044:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.226045:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.226046:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.226047:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.226047:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.226053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677448.226057:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677448.226060:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:13.0:1619677448.226060:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.226061:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.226062:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:13.0:1619677448.226062:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.226063:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.226064:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.226065:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.226065:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:4.2:1619677448.226120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.226128:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.226130:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.226131:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.226133:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.226135:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.226136:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.226138:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.226139:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.226140:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.226140:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.226144:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.226145:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.226147:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.226148:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.226149:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.226150:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.226151:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.226151:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.226151:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.226162:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.226165:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.226165:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.226167:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.226169:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.226170:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.226182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.226190:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.2:1619677448.226192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.226193:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.226195:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.226197:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.226197:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.226198:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677448.226199:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.226201:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.226201:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.226201:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677448.226203:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.226205:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.226205:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.226206:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:17.0:1619677448.226208:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.226209:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.226209:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:30.0:1619677448.226211:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:17.0:1619677448.226211:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.226212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:30.0:1619677448.226213:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.226213:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.226214:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:17.0:1619677448.226214:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.226215:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:17.0:1619677448.226215:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.226216:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.226217:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.226218:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:17.0:1619677448.226218:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.226218:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677448.226220:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.226220:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677448.226220:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.226221:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.226221:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:17.0:1619677448.226223:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.226224:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:6.0:1619677448.226224:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.226225:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.226226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677448.226227:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.226227:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:6.0:1619677448.226227:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.226228:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.226228:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.226228:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.226229:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96717677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.226230:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:6.0:1619677448.226231:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.226232:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.226232:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.226232:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.226233:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677448.226234:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.226234:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.226235:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.226236:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:6.0:1619677448.226236:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.226237:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.226238:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:6.0:1619677448.226238:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.226239:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677448.226239:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.226241:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:6.0:1619677448.226241:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677448.226242:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.226242:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.226243:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:12.0:1619677448.226244:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.226244:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:13.0:1619677448.226245:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677448.226245:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.226246:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.226246:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677448.226247:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:12.0:1619677448.226247:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.226250:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.226251:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.226251:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:9.2:1619677448.226251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.226252:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.226253:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:18.0:1619677448.226258:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.226261:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.226262:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.226262:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.226264:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.226267:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.226377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.226382:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.226385:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.226386:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.226388:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.226390:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.226392:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.226393:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.226394:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.226395:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.226396:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.226400:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.226401:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.2:1619677448.226403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677448.226404:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.226405:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.226406:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.226408:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.226408:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677448.226409:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.226409:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.226409:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677448.226411:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.226413:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.226414:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.226416:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.226418:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.226419:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.226420:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.226421:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.226422:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:5.0:1619677448.226422:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.226424:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.226425:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.226426:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.226426:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.226428:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.226428:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.226431:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.0:1619677448.226431:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.226432:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.226433:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.226435:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.226435:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.226436:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.226437:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.226448:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.2:1619677448.226448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:13.0:1619677448.226450:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.226451:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.226453:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.226455:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.226455:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.226456:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.226458:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.226459:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.226461:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.226463:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.226464:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:26.0:1619677448.226465:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.226466:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.226467:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.226467:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.226468:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.226468:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.226468:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:26.0:1619677448.226469:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.226469:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.226470:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677448.226472:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.226474:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.226476:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.226478:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.226478:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.226480:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.226480:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.226481:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.226481:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.226492:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.226494:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.226495:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.226496:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.226498:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.226499:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.226510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677448.226511:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.226513:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.226514:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.226515:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.226515:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.226516:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.226517:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.226520:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.226521:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.226522:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.226525:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.226526:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.226527:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.226528:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.226529:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.226530:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.226533:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.226534:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.226537:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.226538:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.226538:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.226540:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.226541:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.226542:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.226542:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.2:1619677448.226544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.226553:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:22.0:1619677448.226554:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:30.0:1619677448.226555:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:22.0:1619677448.226557:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:30.0:1619677448.226558:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.226558:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.226558:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:30.0:1619677448.226559:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.226560:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.226561:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.226561:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.226562:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.226564:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.226564:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:30.0:1619677448.226565:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.226566:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.226569:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.226569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677448.226570:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.226573:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.226574:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.226575:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.226576:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.226577:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:24.0:1619677448.226577:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677448.226578:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.226578:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:24.0:1619677448.226580:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.226581:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.226582:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.226583:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:24.0:1619677448.226586:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.226588:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.226589:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96717671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.226591:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.226591:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:24.0:1619677448.226592:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.226593:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.226594:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.226595:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.226596:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.226597:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.226597:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:24.0:1619677448.226599:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.226600:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:24.0:1619677448.226602:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.226603:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.226603:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.226605:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.226606:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.226607:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.226607:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.226614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.226622:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.226625:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.226626:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.226626:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.226628:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.226631:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.226635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.226641:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.226644:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.226645:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.226645:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.226646:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.226646:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.226819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.226827:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.226830:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.226831:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.226833:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.226836:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.226837:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.226838:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.226839:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677448.226839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:26.0:1619677448.226840:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.226842:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.226845:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.226846:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.226847:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.226848:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.226850:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.226850:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.226851:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:21.2:1619677448.226851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677448.226852:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:10.0:1619677448.226852:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.226853:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:26.0:1619677448.226854:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:10.0:1619677448.226855:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.226856:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.226857:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.226858:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677448.226858:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.226859:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.226860:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.226860:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677448.226861:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677448.226861:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.2:1619677448.226861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.2:1619677448.226861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:10.0:1619677448.226862:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.226864:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.226865:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677448.226865:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.226865:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:13.0:1619677448.226866:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:31.0:1619677448.226867:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.226867:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:31.0:1619677448.226868:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677448.226868:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:13.0:1619677448.226868:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:10.0:1619677448.226868:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.226868:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:13.0:1619677448.226869:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.226869:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:31.0:1619677448.226870:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96717668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.226870:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:10.0:1619677448.226870:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.226871:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:31.0:1619677448.226872:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.226872:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.226872:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.226872:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677448.226872:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.226873:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.226873:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677448.226874:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.226874:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.226874:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:5.0:1619677448.226875:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.226877:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:30.0:1619677448.226878:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.226879:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.226879:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96717667] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.226880:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.226880:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677448.226880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677448.226881:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:30.0:1619677448.226882:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:5.0:1619677448.226882:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.226883:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.226883:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96717666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.226883:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:6.0:1619677448.226885:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:30.0:1619677448.226886:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.226887:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.226887:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:5.0:1619677448.226887:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.226888:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.226888:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.226888:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.226889:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677448.226889:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.226890:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.226890:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:31.0:1619677448.226891:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.226891:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.226892:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:6.0:1619677448.226892:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.226893:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677448.226893:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.226894:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.226895:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677448.226895:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677448.226896:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.226896:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677448.226896:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.226897:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.226898:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677448.226898:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.226898:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:30.0:1619677448.226899:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.226900:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.226900:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:30.0:1619677448.226901:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.226901:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.226902:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.226902:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:30.0:1619677448.226903:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.226903:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.226904:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.226905:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677448.226906:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.2:1619677448.226906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:12.0:1619677448.226908:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.226909:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.226910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:12.0:1619677448.226911:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.226912:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.226913:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677448.226915:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.226916:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.226917:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.226918:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:20.0:1619677448.226919:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.226920:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.226921:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.226921:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.226922:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:5.0:1619677448.226923:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.226924:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.226926:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.226939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.226945:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.226948:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.226948:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.226950:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.226953:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.226955:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.226956:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.226957:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.226958:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.226960:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.226963:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.226965:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.226968:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.226969:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.226970:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.226971:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.226972:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.226973:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.226974:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.226974:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.226980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.226984:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.226985:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677448.226986:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677448.226987:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:13.0:1619677448.226987:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.226988:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677448.226989:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.226989:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.226990:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.226992:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.227070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.227073:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.227075:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.227078:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.227078:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.227080:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.227081:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.227083:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.227084:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.227085:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.227086:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.227090:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.227091:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.227094:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.227095:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.227096:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.227097:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.227098:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.227099:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.227099:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.2:1619677448.227103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.227110:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.227112:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.227113:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.227115:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.227116:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.227116:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.227118:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.227119:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.227121:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.227122:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.227123:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.227124:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.227125:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.227126:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.227127:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.227128:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:22.0:1619677448.227129:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.227131:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.227133:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.227134:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.227134:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.227136:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.227137:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.227138:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.227139:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.227139:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.227142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.227145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677448.227150:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677448.227151:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:5.0:1619677448.227152:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:6.0:1619677448.227153:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:5.0:1619677448.227153:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.227154:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.227154:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677448.227155:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.227155:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.227157:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.227158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.227166:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.227169:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.227170:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.227170:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.227171:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.227172:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.227228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.227235:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.227237:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.227238:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.227240:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.227241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:10.0:1619677448.227243:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.227245:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.227246:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.227247:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:10.0:1619677448.227247:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.227248:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.227249:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.227250:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.227250:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:10.0:1619677448.227252:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.227254:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.227255:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.227256:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.227257:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.227257:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:18.0:1619677448.227258:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:10.0:1619677448.227258:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:18.0:1619677448.227259:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.227260:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.227260:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:18.0:1619677448.227261:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.227261:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.227261:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:18.0:1619677448.227262:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.227263:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.227266:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.227268:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.227270:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.227271:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:18.0:1619677448.227271:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.227272:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.227273:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.227274:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.227274:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.227274:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:10.0:1619677448.227274:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.227275:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.227275:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:26.0:1619677448.227276:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.227278:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:8.2:1619677448.227279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.227287:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.227289:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.227290:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.227291:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.227293:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.227298:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.227327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.227335:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.227338:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.227340:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.227343:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.227343:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.227344:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.227345:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.227347:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.227348:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.227349:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.227352:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.227354:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.227355:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.227356:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.227357:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.227358:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.227359:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.227360:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.227361:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.227372:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.227374:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.227375:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.227376:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.227378:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.227379:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.227483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677448.227491:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677448.227494:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677448.227496:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.227498:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677448.227498:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.227500:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677448.227501:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.227502:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.227503:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677448.227504:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677448.227507:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.227509:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677448.227512:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.227513:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:12.0:1619677448.227513:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677448.227516:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:12.0:1619677448.227516:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677448.227517:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677448.227518:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677448.227522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.227530:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.227533:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.227534:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.227534:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.227536:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.227539:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.227576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.227582:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.227584:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.227586:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.227587:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.227589:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.227590:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.227591:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.227593:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.227594:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.227595:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.227598:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.227600:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.227603:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.227604:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.227604:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.227606:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.227607:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.227608:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.227610:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.227610:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.227615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.227618:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677448.227619:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.227620:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.227621:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.227621:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.227622:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.227622:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.227624:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.227625:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.227627:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.227640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.227647:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.227649:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.227650:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.227652:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.227655:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.227656:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.227657:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.227659:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.227660:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.227661:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.227664:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.227665:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.227668:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.227669:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.227670:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.227672:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.227672:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.227673:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.227673:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.227678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.227685:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.227687:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.227688:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.227689:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.227691:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.227694:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.227697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677448.227703:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.227705:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.227706:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.227706:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.227707:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.227708:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.227768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:4.2:1619677448.227769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677448.227775:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:10.0:1619677448.227776:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677448.227777:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.227779:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:10.0:1619677448.227779:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.227780:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:8.0:1619677448.227781:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.227782:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.227784:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.227784:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.227785:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:8.0:1619677448.227786:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.227787:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.227787:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.227788:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.227789:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.227789:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.227790:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:8.0:1619677448.227790:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.227791:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.227795:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.227796:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.227798:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.0:1619677448.227798:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.227800:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.227801:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:8.0:1619677448.227801:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.227802:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.0:1619677448.227802:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.227803:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:8.0:1619677448.227803:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.227804:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677448.227804:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.227805:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.227805:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.0:1619677448.227805:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.227806:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.227806:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.227810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.227816:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.227818:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.227819:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.227819:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:31.0:1619677448.227820:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.227822:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.227822:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:31.0:1619677448.227823:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.0:1619677448.227824:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.227825:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.227825:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:20.0:1619677448.227827:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.227827:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.227830:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.227831:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000400:00000200:5.0:1619677448.227831:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677448.227833:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.227833:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000400:00000200:5.0:1619677448.227833:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.227834:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.227834:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.227835:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.227835:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:30.0:1619677448.227836:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.227836:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:5.0:1619677448.227837:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.227838:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.227839:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.227839:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.227840:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.227840:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:5.0:1619677448.227840:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.227841:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.227843:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:5.0:1619677448.227844:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.227846:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.227848:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.227849:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.227849:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.227851:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.227851:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.227852:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.227853:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677448.227857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.227865:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.227868:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.227868:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.227869:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.227870:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677448.227873:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.227964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.227972:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.227975:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.227977:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.227979:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.227980:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.227981:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.227983:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.227984:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.227985:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.227986:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.227990:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.227991:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.227994:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.227995:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.227996:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.227997:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.227998:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.227999:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.227999:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677448.228000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.228005:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.228007:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.228009:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.228009:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.228011:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.228011:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.228012:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.228012:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.228014:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.228015:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.228015:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.228015:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.228017:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.228017:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:20.0:1619677448.228018:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.228019:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.228022:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.228024:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.228026:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.228027:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.228027:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.228029:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.228030:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:13.0:1619677448.228030:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.228031:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677448.228032:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677448.228033:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:13.0:1619677448.228033:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.228034:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.228034:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.228035:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.2:1619677448.228035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:13.0:1619677448.228036:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.228038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.228044:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677448.228045:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:5.0:1619677448.228045:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.228046:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.228046:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.228047:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:5.0:1619677448.228047:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.228049:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.228049:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.228051:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.2:1619677448.228154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.228161:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.228164:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.228165:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.228167:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.228169:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.228171:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.228172:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.228173:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.228174:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.228175:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.228178:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.228180:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.228183:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.228184:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.228185:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.228187:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.228187:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.228188:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.228189:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677448.228193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.228201:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.228204:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.228205:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.228205:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.228207:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.228210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677448.228211:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677448.228216:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.228218:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.228219:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.228221:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.228223:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.228225:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.228226:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.228227:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.228228:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.228229:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.228232:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.228234:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.228237:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.228238:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.228238:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.228240:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.228241:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.228241:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.228242:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.228246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.228253:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.228255:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.228256:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.228256:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.228258:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.228261:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.228572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.228580:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.2:1619677448.228582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.228583:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.228585:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.228587:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.228588:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.228589:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:30.0:1619677448.228589:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.228590:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.228591:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.228591:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.228592:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.228593:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.2:1619677448.228594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:31.0:1619677448.228597:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.228598:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.228599:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.228600:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:30.0:1619677448.228600:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.228600:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.228602:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.228603:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.228603:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:31.0:1619677448.228604:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.228604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:17.0:1619677448.228604:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677448.228605:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677448.228606:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.228607:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677448.228607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677448.228608:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:31.0:1619677448.228609:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.228609:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677448.228609:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.228610:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677448.228610:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677448.228611:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.228611:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677448.228611:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:30.0:1619677448.228612:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:1.0:1619677448.228612:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96717647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.228613:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.228613:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:30.0:1619677448.228614:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:6.0:1619677448.228614:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677448.228614:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.228615:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.228615:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:1.0:1619677448.228615:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.228616:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:1.0:1619677448.228616:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:31.0:1619677448.228617:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:22.0:1619677448.228617:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.228618:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.228618:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:31.0:1619677448.228619:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:24.0:1619677448.228619:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677448.228619:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.228620:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:22.0:1619677448.228621:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677448.228621:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.228622:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.228622:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:22.0:1619677448.228622:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96717646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.228624:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.228624:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.228624:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:22.0:1619677448.228625:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.228625:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:22.0:1619677448.228626:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:1.0:1619677448.228626:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.228627:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.228627:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.228627:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.228628:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.2:1619677448.228629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:12.0:1619677448.228630:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.228630:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.228630:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677448.228631:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.228631:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.228631:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.228631:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.228632:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.228633:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677448.228634:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:8.0:1619677448.228634:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.228635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677448.228636:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:8.0:1619677448.228636:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:30.0:1619677448.228637:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:22.0:1619677448.228637:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.228638:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000400:00000200:8.0:1619677448.228638:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96717645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.228639:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:22.0:1619677448.228639:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:20.0:1619677448.228639:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.228639:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.228640:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:21.2:1619677448.228640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:8.0:1619677448.228640:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.228641:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:22.0:1619677448.228641:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.228641:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.228642:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677448.228642:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.228642:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.228643:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677448.228644:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:8.0:1619677448.228644:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.228644:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.228645:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:24.0:1619677448.228646:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.228647:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.228649:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.228649:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.228651:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.228652:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.228654:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:10.0:1619677448.228654:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:8.0:1619677448.228654:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.228655:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.228655:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.228656:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:8.0:1619677448.228656:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.228657:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:24.0:1619677448.228657:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.228657:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.228658:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:10.0:1619677448.228659:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.228660:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.228660:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.228660:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.228661:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:24.0:1619677448.228662:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:8.0:1619677448.228662:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:24.0:1619677448.228663:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96717644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.228663:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:8.2:1619677448.228664:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:24.0:1619677448.228665:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.228666:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.228667:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.228668:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.228671:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.228672:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.228672:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677448.228675:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.228675:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.228675:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.228676:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.228676:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:17.0:1619677448.228676:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.228678:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:17.0:1619677448.228678:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.228679:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.228679:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.228680:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:17.0:1619677448.228681:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:9.2:1619677448.228684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.228693:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.228695:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.228697:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.228697:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.228698:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.228702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.228705:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.228706:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.228709:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.228709:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.228710:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.228710:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.228711:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.228940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.0:1619677448.228944:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.228947:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677448.228949:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.228949:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.228951:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.228952:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.228953:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.228954:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.228955:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.228956:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.228959:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.228961:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.228964:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.228965:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.228965:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.228967:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.228968:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.228968:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.228969:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.228973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.228981:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.228983:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.228984:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.228985:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.228986:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.228987:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.2:1619677448.229042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.229047:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.229050:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.229052:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.229053:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.229054:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.229056:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677448.229057:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.229059:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.229060:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.229060:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677448.229064:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.229065:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.229067:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.229068:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.229068:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.229070:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.229070:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.229071:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.229072:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.229076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.229081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.229085:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:21.2:1619677448.229086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677448.229087:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.229088:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:1.0:1619677448.229088:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.229089:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.229090:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.229092:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.229093:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677448.229094:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677448.229094:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.229094:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677448.229095:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.229096:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:30.0:1619677448.229097:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:17.0:1619677448.229098:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.229099:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.229100:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.229101:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677448.229101:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.229102:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.229102:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677448.229105:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677448.229106:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.229107:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.229108:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.229109:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:30.0:1619677448.229110:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.229111:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.229112:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.229112:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.229113:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.229114:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:30.0:1619677448.229115:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.229115:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.229116:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:30.0:1619677448.229117:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.229117:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.229118:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.229120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677448.229121:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.229121:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.229122:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.2:1619677448.229123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.229124:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.229125:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.229126:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.229128:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.229128:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677448.229129:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.229130:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.229130:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.229131:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.229131:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.229131:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.229132:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.229133:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.229134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.229134:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.229135:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:6.0:1619677448.229138:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.229141:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677448.229142:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677448.229142:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.229142:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.229145:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:6.0:1619677448.229145:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.229146:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.229147:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.229148:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677448.229156:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.229158:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.229159:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.229160:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.229161:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.229161:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.229191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.229198:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.229201:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.229202:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.229204:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.229207:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.229208:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.229209:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677448.229210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:26.0:1619677448.229211:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.229212:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.229214:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.229217:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:26.0:1619677448.229218:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.229219:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.229220:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.229221:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.229223:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.229224:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.229225:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:10.0:1619677448.229225:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:26.0:1619677448.229226:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677448.229226:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.229227:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.229228:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:10.0:1619677448.229228:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.229229:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.229230:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.229231:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.229231:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677448.229233:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.229233:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.229234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.229235:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.229238:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.229239:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.229239:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.229241:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.229241:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.229241:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677448.229242:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.229242:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677448.229243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:18.0:1619677448.229244:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.229245:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.229246:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.229248:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.229251:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.0:1619677448.229251:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.229254:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:6.0:1619677448.229254:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:24.0:1619677448.229255:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.229257:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.229257:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.229258:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.229258:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.229260:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.229261:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:6.0:1619677448.229261:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.229263:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96717637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.229264:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.229264:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:8.0:1619677448.229265:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.229266:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677448.229269:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.229270:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.229273:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.229274:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.229274:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.229276:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.229277:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:22.0:1619677448.229278:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:8.0:1619677448.229278:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.229279:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.229280:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677448.229281:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.229282:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.229282:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.229283:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.229283:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.229285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.229293:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.2:1619677448.229294:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.229301:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.229302:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (22)--
00000800:00000200:20.2:1619677448.229302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:24.0:1619677448.229303:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677448.229308:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677448.229310:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677448.229311:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.229311:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.229313:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:20.0:1619677448.229313:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.229314:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:5.0:1619677448.229315:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:26.0:1619677448.229316:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677448.229316:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:26.0:1619677448.229318:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.229320:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.229321:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.229322:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.229323:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.229324:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.229327:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.229329:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.229331:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.229332:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.229333:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.229335:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.229335:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.229336:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.229337:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.229341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.229349:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.229352:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.229353:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.229355:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.229355:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.229358:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.229490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.229496:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.229499:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.229500:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.229502:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.229504:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.229506:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.229507:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.229508:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.229510:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.229511:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.229514:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.229516:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.229519:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.229520:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.229521:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.229523:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.229523:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.229525:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.229526:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.229526:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677448.229531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.229537:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:13.0:1619677448.229538:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.229539:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.229539:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.229540:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677448.229540:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.229541:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.229543:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.229545:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.229546:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.229584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.229586:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.229589:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.229591:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.229592:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.229593:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.229595:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677448.229596:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.229598:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.229599:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.229600:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.229603:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.229604:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.229607:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.229608:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.229608:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.229610:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.229611:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.229612:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.229613:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.229614:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677448.229616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677448.229619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.229624:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:17.0:1619677448.229625:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677448.229627:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:17.0:1619677448.229628:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:5.0:1619677448.229628:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.229628:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:17.0:1619677448.229629:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.229629:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.229631:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.229633:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.229709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.229715:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.229717:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.229718:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.229720:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.229722:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.229723:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.229724:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.229725:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.229726:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.229727:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.229730:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.229732:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.229733:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.229734:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.229735:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.229736:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.229737:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.229737:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.229738:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.229742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.229750:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.229752:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.229752:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.229753:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.229754:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.229755:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.229760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.229768:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.229770:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.229771:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.229771:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.229771:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.229772:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.229834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.229840:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.229842:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.229845:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.229845:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.229847:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.229848:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677448.229850:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.229851:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.229852:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.229853:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677448.229856:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.229857:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.229859:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.229861:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:21.2:1619677448.229862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:5.0:1619677448.229862:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.229863:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.229864:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.229865:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.229866:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677448.229868:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.2:1619677448.229869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677448.229871:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.229872:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.229873:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:17.0:1619677448.229876:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:2.0:1619677448.229876:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.229878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:2.0:1619677448.229878:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:17.0:1619677448.229879:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:2.0:1619677448.229879:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.229880:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.229880:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.229880:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:2.0:1619677448.229881:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.229882:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.229882:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:17.0:1619677448.229884:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:2.0:1619677448.229885:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:31.0:1619677448.229887:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.229887:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:31.0:1619677448.229890:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:2.0:1619677448.229890:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.229891:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677448.229891:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:31.0:1619677448.229892:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.229893:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677448.229894:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:31.0:1619677448.229895:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.229895:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677448.229895:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.229896:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.229896:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:31.0:1619677448.229897:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.229898:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.229899:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.229900:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.229901:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:21.2:1619677448.229901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:31.0:1619677448.229904:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.229905:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:31.0:1619677448.229906:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.229907:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:31.0:1619677448.229908:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.229908:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:31.0:1619677448.229909:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.229909:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677448.229909:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:10.0:1619677448.229909:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.229910:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.229910:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:31.0:1619677448.229911:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.229911:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:10.0:1619677448.229911:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:31.0:1619677448.229912:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.229913:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.229913:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.229914:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677448.229921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:12.0:1619677448.229923:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.229926:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.229926:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.229927:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:12.0:1619677448.229928:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.229930:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.229930:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677448.229930:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.229931:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.229933:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.229936:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.229937:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.229938:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717629] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.229940:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.229941:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.229942:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.229945:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.229947:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.229950:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.229950:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.229951:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.229953:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.229953:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.229954:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.229955:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.229959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.229967:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.229970:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.229971:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.229971:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.229973:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.229977:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.229978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.229984:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.229987:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.229988:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.229989:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.229992:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.229994:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.229995:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.229997:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.229998:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.229999:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.230002:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.230004:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.230007:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.230008:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.230009:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.230011:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.230012:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.230012:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.230013:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:26.0:1619677448.230023:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.230026:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.230026:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.230027:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.230029:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.230032:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:4.2:1619677448.230060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677448.230066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.230070:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.230073:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.230074:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.230076:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.230076:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.230078:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.230079:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.230080:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.230081:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.230082:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.230083:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.230084:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.230085:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.230087:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.230089:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.230090:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.230090:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.230091:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.230092:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.230092:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.230095:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.230096:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.230096:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.230097:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.230098:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.230099:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.230099:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.230100:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.230101:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.230101:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (23)++
00000800:00000200:26.0:1619677448.230102:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.230102:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (24)--
00000800:00000200:26.0:1619677448.230103:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.230103:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.230104:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.230104:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.230105:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (23)++
00000800:00000200:10.0:1619677448.230105:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.230106:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.230107:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677448.230108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.2:1619677448.230111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.230113:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.230115:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.230115:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.230115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.230117:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:30.0:1619677448.230117:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.230117:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.230118:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:10.0:1619677448.230118:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677448.230119:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.230119:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.230120:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677448.230120:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.230120:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677448.230120:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:31.0:1619677448.230121:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.230121:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.230122:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.230122:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.230123:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.230123:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:1.0:1619677448.230123:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.230124:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:26.0:1619677448.230124:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.230125:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:1.0:1619677448.230125:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.230126:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.230128:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.230129:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.230130:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.230131:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.230134:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.230136:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.230139:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.230140:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.230140:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.230142:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.230143:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.230144:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.230144:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.230155:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.230158:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.230159:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.230160:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.230161:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.230163:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.230252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.230258:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.230261:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.230262:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.230263:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.230266:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.230268:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.230269:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.230271:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.230271:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.230273:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.230276:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.230278:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.230281:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.230282:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.230283:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.230284:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.230285:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.230286:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.230286:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677448.230298:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.230301:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.230302:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.230304:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.230305:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.230307:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.230307:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.230308:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.230308:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.230309:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.230309:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.230447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.230455:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.230457:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.230458:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.230460:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.230462:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.230464:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.230465:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.230466:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.230468:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.230468:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.230472:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.230474:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.230477:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.230478:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.230479:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.230480:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.230481:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.230481:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.230482:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.230495:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.230498:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.230499:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.230499:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.230502:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.230505:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.230522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.230528:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.230531:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.230532:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.230534:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.230537:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.230538:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.230539:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717622] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.230541:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.230542:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.230542:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.230545:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.230547:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.230550:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.230551:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.230552:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.230553:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.230554:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.230555:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.230555:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.230566:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.230568:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.230569:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.230571:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.230572:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.230574:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.230727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.230732:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.230735:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.230736:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.230738:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.230740:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.230742:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.230744:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.230745:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.230746:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.230747:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.230751:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.230753:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.230756:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677448.230756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677448.230757:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.230757:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.230759:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.230759:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.230760:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.230761:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.230763:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.230766:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.230767:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.230769:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.230772:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.230773:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:10.0:1619677448.230773:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.230774:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.230775:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.230776:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.230776:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.230777:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.230778:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.230778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:10.0:1619677448.230779:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.230780:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:10.0:1619677448.230782:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.230784:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.230785:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:24.0:1619677448.230785:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677448.230787:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.230788:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:10.0:1619677448.230788:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.230789:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677448.230789:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.230790:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.230790:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.230791:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.230791:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.230792:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:24.0:1619677448.230793:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.230794:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.230796:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96717619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.230797:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.230798:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.230799:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.230802:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.230803:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.230806:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.230807:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.230807:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.230809:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.230810:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.230811:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.230812:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.230813:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677448.230814:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.2:1619677448.230815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677448.230817:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.230818:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.230818:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:4.2:1619677448.230818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:22.0:1619677448.230820:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.230823:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677448.230823:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:8.0:1619677448.230825:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:21.2:1619677448.230826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677448.230826:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.230827:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.230827:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677448.230827:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.230828:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.230829:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.230829:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.230832:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:31.0:1619677448.230833:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.230835:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.230838:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.230838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:30.0:1619677448.230839:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.230840:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.230841:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677448.230843:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.230845:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.230845:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:31.0:1619677448.230846:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.230847:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:24.0:1619677448.230847:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.230848:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:31.0:1619677448.230851:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.230851:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.230853:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:24.0:1619677448.230854:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.230855:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.230856:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96717617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.230857:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.230858:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677448.230858:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:24.0:1619677448.230858:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.230859:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.230859:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.230860:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677448.230861:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.230862:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.230862:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:24.0:1619677448.230863:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.230865:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.230867:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.230868:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.230869:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.230871:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.230871:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.230872:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.230872:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677448.230874:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.230877:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:9.2:1619677448.230877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.230877:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.230878:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.230880:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.230883:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677448.230885:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.230887:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.230889:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.230889:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.230891:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.230895:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.230933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.230939:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.230941:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.230942:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.230944:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.230946:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.230947:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.230948:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717616] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.230950:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.230951:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.230952:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.230955:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.230957:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.230959:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.230960:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.230960:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.230962:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.230962:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.230963:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.230963:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.230968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.230969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.230974:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.230974:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:17.0:1619677448.230977:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:2.0:1619677448.230977:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.230977:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.230978:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.230979:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:17.0:1619677448.230980:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.230980:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.230981:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:17.0:1619677448.230983:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.230985:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.230986:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.230987:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677448.230987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:17.0:1619677448.230989:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.230989:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.230993:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.230994:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.230995:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677448.230997:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.230998:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.230998:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:17.0:1619677448.230998:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.230998:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.230999:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677448.230999:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.231000:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.231002:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.231002:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.231004:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.231005:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.231005:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.231010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.231015:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:1.0:1619677448.231015:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.231017:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:30.0:1619677448.231018:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:1.0:1619677448.231018:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677448.231019:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.231019:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677448.231021:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.231022:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.231024:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.231083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.231091:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.231093:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.231094:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.231096:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.231099:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.231100:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.231101:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.231102:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.231103:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.231104:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.231107:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.231109:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.231110:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.231111:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.231112:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.231113:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.231113:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.231114:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.231115:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.231125:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.231126:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.231127:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.231128:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.231128:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.231130:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.231146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.231154:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.231156:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.231158:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.231160:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.231161:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.231162:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.231163:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.231165:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.231165:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.231166:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.231169:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.231171:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.231173:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.231173:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.231174:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.231175:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.231176:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.231177:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.231177:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.231187:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.231189:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.231190:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.231191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:30.0:1619677448.231192:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.231193:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.231194:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:26.0:1619677448.231198:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.231200:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.231201:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.231202:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.231204:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.231205:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.231206:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.231208:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.231209:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.231210:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.231212:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.231214:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.231217:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.231217:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.231218:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.231220:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.231220:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.231221:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.231221:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.231244:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.231247:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.231247:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.231248:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.231250:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.231253:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.231276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.231279:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.231281:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.231283:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.231285:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.231285:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.231287:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.231288:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.231290:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.231291:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.231292:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.231299:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.231301:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.231304:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.231305:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.231306:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.231307:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.231308:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.231308:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.231309:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.231322:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.231324:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.231325:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.231327:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.231329:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.231330:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.231445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.231453:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.231455:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.231456:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.231458:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.231461:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.231463:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.231464:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.231465:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.231466:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.231467:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.231470:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.231472:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.231475:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.231475:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.231476:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.231478:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.231478:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.231479:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.231481:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.231481:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.231487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.231492:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.231493:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.231495:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.231496:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677448.231496:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.231496:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.231497:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.231497:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.231498:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.231501:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.231561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.231569:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.231572:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.231574:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.231576:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.231576:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.231577:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.231579:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.231580:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.231581:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.231582:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.231585:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.231586:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.231589:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.231590:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.231591:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.231593:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.231593:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.231594:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.231595:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.231600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.231608:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.231611:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.231612:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.231613:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.231614:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.231617:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.231621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.231629:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.231632:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.231633:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.231634:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.231637:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.231639:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.231640:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.231641:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.231642:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.231643:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.231646:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.231647:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.231648:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.231649:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.231650:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.231651:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.231651:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.231652:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.231652:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.231663:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.231664:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.231665:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.231666:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.231667:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.231668:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.231679:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.231681:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.231682:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.231682:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.231683:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.231683:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.231720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.231726:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.231728:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.231729:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.231731:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.231734:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.231735:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.231736:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.231738:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.231739:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.231739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:22.0:1619677448.231740:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.231743:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.231745:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:17.0:1619677448.231745:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.231748:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.231748:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.231748:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677448.231749:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.231749:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.231751:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.231751:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:17.0:1619677448.231751:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.231753:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:17.0:1619677448.231753:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.231754:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.231754:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:17.0:1619677448.231755:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.231756:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.231757:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677448.231757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:17.0:1619677448.231758:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.231759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:17.0:1619677448.231759:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.231763:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.231764:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:17.0:1619677448.231765:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.231765:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:6.0:1619677448.231766:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.231766:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.231767:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677448.231767:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:17.0:1619677448.231768:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.231768:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.231769:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:17.0:1619677448.231769:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.231769:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:2.0:1619677448.231770:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.231771:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.231772:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.231773:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.231773:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677448.231773:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:12.0:1619677448.231784:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.231787:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.231788:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.231789:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.231791:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.231792:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677448.231802:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.231805:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.231806:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.231806:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.231807:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.231808:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.231823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:4.2:1619677448.231823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677448.231829:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.231830:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.231832:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.231832:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:26.0:1619677448.231833:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.231833:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:18.0:1619677448.231837:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.231838:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.231839:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.231840:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.231840:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.231842:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.231843:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.231843:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.231845:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.231845:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.231846:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.231847:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.231848:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.231849:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.231854:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.231855:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.231857:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.231858:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.231860:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.231861:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.231861:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.231862:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.231862:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.231863:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.231864:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.231864:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.231865:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.231865:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.231866:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.231867:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:10.0:1619677448.231867:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.231868:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.231876:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.231878:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.231879:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.231879:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.231881:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.231881:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.231884:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:22.0:1619677448.231884:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.231884:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.231885:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.231887:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.231892:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.231894:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.231896:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.231897:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.231897:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.231898:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.231899:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.231929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.231937:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.231940:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.231942:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.231944:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.231945:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.231946:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677448.231948:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.231949:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.231950:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.231951:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677448.231955:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.231957:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.231960:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.231961:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677448.231961:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.231963:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677448.231964:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.231965:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.231965:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.231977:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.231979:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.231980:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.231981:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.231983:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.231986:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.231997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.232005:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.232008:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.232010:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.232012:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.232013:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.232014:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.232016:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717602] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.232017:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.232018:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.232019:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.232021:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.232023:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.232026:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.232027:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.232027:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.232029:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.232030:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.232031:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.232031:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.232036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.232044:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.232046:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.232047:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.232049:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.232050:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.232051:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.232053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.232061:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.232064:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.232065:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.232065:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.232066:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.232067:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.232084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.232092:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.232095:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.232095:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.232097:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.232100:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.232101:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.232102:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.232104:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.232105:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.232106:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.232109:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.232110:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.232113:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.232114:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.232114:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.232116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:10.0:1619677448.232116:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.232116:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.232117:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.232117:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677448.232122:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.232124:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.232126:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:12.0:1619677448.232127:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:2.0:1619677448.232127:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.232130:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.232131:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.232131:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.232133:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.232133:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:13.0:1619677448.232134:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677448.232134:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.232136:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677448.232136:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.232137:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.232138:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.232141:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.232143:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.2:1619677448.232145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677448.232146:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.232146:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.232147:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.232149:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.232149:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.232150:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:5.0:1619677448.232151:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.232151:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.0:1619677448.232154:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.232156:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.232156:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.232159:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.232161:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:22.0:1619677448.232162:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:5.0:1619677448.232162:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.232163:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.232164:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.232165:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:5.0:1619677448.232165:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677448.232166:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.232166:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.232169:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.232169:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.232171:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677448.232172:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.232175:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.232176:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.232176:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.232178:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.232179:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.232180:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.232181:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.232182:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.232187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.232190:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677448.232191:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.232192:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.232193:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677448.232193:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677448.232194:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:13.0:1619677448.232194:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.232194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:13.0:1619677448.232196:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.232197:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.232199:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.232200:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.232203:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.232204:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.232206:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.232209:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.232210:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.232212:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.232213:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.232214:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.232215:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.232219:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.232221:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.232224:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.232225:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.232226:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.232227:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.232228:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.232229:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.232229:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.232240:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.232243:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.232244:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.232245:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.232246:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.232249:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.232260:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.232263:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.232264:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.232265:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.232265:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.232266:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.232440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.232447:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.232450:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.232451:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.232453:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.232456:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.232457:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.232458:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.232460:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.232460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:26.0:1619677448.232461:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.232462:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.232465:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.232467:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.232469:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.232470:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.232471:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:26.0:1619677448.232471:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.232472:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:30.0:1619677448.232473:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.232473:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.232474:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.232474:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.232475:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.232476:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.232476:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.232478:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.232479:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.232480:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.232481:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.232482:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.232485:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.232486:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.232488:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:30.0:1619677448.232489:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.232490:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.232491:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.232491:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:30.0:1619677448.232492:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.232492:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.232492:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.232493:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.232494:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:6.0:1619677448.232495:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.232496:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.232497:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:6.0:1619677448.232497:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.2:1619677448.232499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.232503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.232511:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.232511:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.232513:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677448.232514:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.232515:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.232515:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.232516:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.232516:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.232517:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.232519:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.232598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.232603:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.232606:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.232608:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.232608:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677448.232610:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.232612:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677448.232613:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.232615:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.232616:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.232617:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677448.232620:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.232622:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677448.232625:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.232626:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677448.232627:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677448.232629:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.232629:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.232630:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.232630:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677448.232641:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.232644:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.232645:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.232647:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677448.232648:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.232650:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.232659:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.232662:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.232663:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.232664:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.232664:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.232665:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.232677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.232684:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.232687:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.232689:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.232689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:31.0:1619677448.232691:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.232691:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.232693:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.232694:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.232695:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.232695:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:31.0:1619677448.232696:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.232697:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.232698:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.232699:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:31.0:1619677448.232701:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.232702:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.232704:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.232705:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.232706:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.232707:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:18.0:1619677448.232707:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:31.0:1619677448.232708:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677448.232708:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.232709:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:18.0:1619677448.232709:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.232710:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677448.232710:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.232711:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:18.0:1619677448.232711:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.232712:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.232713:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677448.232715:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.232717:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:21.2:1619677448.232718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.232719:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.232720:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.232720:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.232722:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.232722:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677448.232722:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.232723:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.232724:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677448.232724:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.232725:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.232726:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:12.0:1619677448.232727:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.232728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.232729:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.232729:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677448.232729:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.232730:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.232731:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677448.232736:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.232739:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.232740:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.232740:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.232740:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.232742:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.232743:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.232743:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.232744:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677448.232745:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:12.0:1619677448.232745:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.232746:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.232886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.232892:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.232894:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.232895:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.232897:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.232899:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.232901:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.232902:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.232903:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.232905:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.232906:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:4.2:1619677448.232908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:22.0:1619677448.232909:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.232910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:22.0:1619677448.232911:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.232914:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.232914:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.232914:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677448.232915:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.232916:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:8.0:1619677448.232916:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677448.232917:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.232918:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.232918:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.232919:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:20.0:1619677448.232919:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.232919:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.232920:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677448.232920:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.232922:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.232923:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:8.2:1619677448.232923:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:20.0:1619677448.232925:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.232926:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.232927:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.232928:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677448.232929:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.232931:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:20.0:1619677448.232932:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.232932:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.232932:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.232933:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.232934:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:8.0:1619677448.232935:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.232935:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.232936:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96717590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.232937:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.232938:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.232938:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:8.0:1619677448.232938:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.232938:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.232938:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:8.0:1619677448.232939:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.232940:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:8.0:1619677448.232940:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.232941:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.232941:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.232942:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:8.0:1619677448.232943:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.232944:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.0:1619677448.232945:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.232948:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.232949:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.232949:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.232951:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.232951:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677448.232952:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:8.0:1619677448.232952:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677448.232953:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677448.232953:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:30.0:1619677448.232954:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.232956:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.232956:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.232957:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.232957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.232958:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.232958:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.232958:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.232959:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:30.0:1619677448.232960:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.232961:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.232961:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:30.0:1619677448.232962:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.232963:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.232964:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:1.0:1619677448.232964:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:30.0:1619677448.232967:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.232967:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.232968:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.232968:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.232969:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:1.0:1619677448.232970:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.232972:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.232973:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.232973:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.232973:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677448.232975:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.2:1619677448.232975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.232976:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.232977:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.232977:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.232981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677448.232983:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.232985:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.232986:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.232987:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.232988:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.232988:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677448.232989:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.232991:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.232992:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.232994:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.232994:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.232997:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.233027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.233033:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.233036:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.233037:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.233039:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.233042:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.233043:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.233045:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.233046:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.233047:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.233048:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.233052:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.233053:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.233056:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.233057:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.233058:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.233060:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.233060:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.233061:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.233061:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.233072:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.233075:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.233075:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.233077:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.233078:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.233080:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.233123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.233131:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.233134:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.233134:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677448.233137:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.233139:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.233141:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.233142:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717587] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.233143:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.233144:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.233145:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.233149:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.233150:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.233153:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.233154:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.233155:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.233157:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.233157:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.233158:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.233158:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.233162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.233170:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.233172:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.233173:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.233174:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.233176:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.233177:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.233182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.233189:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.233192:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.233193:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.233194:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.233194:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.233196:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.233282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.233289:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.233293:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677448.233299:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.233300:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.233303:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.233305:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.233306:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.233307:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.233308:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.233309:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.233312:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.233314:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.233316:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.233317:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.233318:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.233319:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.233320:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.233320:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.233321:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.233333:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:21.2:1619677448.233334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:13.0:1619677448.233335:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.233336:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.233337:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.233339:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.233341:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.233341:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677448.233344:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677448.233344:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000400:00000200:6.0:1619677448.233346:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.233349:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.233351:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:6.0:1619677448.233352:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.233353:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.233354:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.233355:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:6.0:1619677448.233358:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.233360:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677448.233363:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.233364:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:6.0:1619677448.233365:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:6.0:1619677448.233367:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:6.0:1619677448.233367:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677448.233368:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677448.233368:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:26.0:1619677448.233380:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.233382:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.233383:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.233384:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.233386:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.233389:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.233730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.233738:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.233740:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.233743:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.233744:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.233745:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.233746:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.233748:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.233749:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.233750:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.233751:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.233754:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.233756:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.233759:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.233760:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.233761:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.233763:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.233763:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.233764:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.233765:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.2:1619677448.233767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677448.233769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.233773:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.233775:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.233777:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.233778:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.233779:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.233780:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.233780:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.233782:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.233782:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.233784:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.233785:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.233786:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.233787:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.233790:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:20.0:1619677448.233791:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.233791:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.233792:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677448.233793:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.233795:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.233796:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.233796:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.233798:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.233799:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.233799:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.233800:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677448.233812:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.233814:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.233815:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.233815:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.233817:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.233820:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.233848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.233856:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.233859:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.233861:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.233862:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.233864:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.233865:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.233867:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.233868:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.233869:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.233870:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.233873:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.233875:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.233877:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.233878:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.233879:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.233880:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.233881:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.233882:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.233882:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.233893:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.233895:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.233896:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.233897:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.233899:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.233900:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677448.233911:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.233914:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.233914:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.233915:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.233915:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.233916:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.233953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677448.233960:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.233963:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677448.233964:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677448.233965:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.233968:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.233970:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.233971:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.233972:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.233973:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.233974:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.233977:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.233978:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.233981:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.233981:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.233982:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.233983:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.233984:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.233985:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.233985:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:30.0:1619677448.233996:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.233998:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.233999:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.234000:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.234002:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.234003:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.234116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.234122:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.234125:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.234126:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.234128:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.234131:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.234133:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.234134:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.234135:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.234136:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.234137:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.234141:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.234143:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.234145:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.234146:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.234146:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.234148:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.234148:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.234150:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.234151:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.234151:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.234156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.234162:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.234162:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677448.234164:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677448.234164:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.234165:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677448.234166:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.234166:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.234166:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.234168:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.234170:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.234178:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.234181:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.234181:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.234182:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.234183:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.234183:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.234184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.234191:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.234194:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.234195:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.234197:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.234199:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.234201:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.234202:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.234203:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.234204:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.234205:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.234208:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.234210:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.234212:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.234213:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.234214:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.234215:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.234215:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.234216:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.234216:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.234226:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.234228:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.234229:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.234229:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.234231:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.234232:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.234358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.234365:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.234367:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.234368:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.234370:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.234373:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.234374:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.234375:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717578] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.234377:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.234378:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.234379:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677448.234382:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.234384:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.234387:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.234388:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.234388:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.234390:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.234390:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.234391:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.234391:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.234396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.234399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:5.0:1619677448.234405:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677448.234406:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.0:1619677448.234407:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.234408:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.234409:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.234410:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:5.0:1619677448.234410:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.234411:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:2.0:1619677448.234412:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.234413:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:2.0:1619677448.234414:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.234416:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.234417:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.234418:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.234420:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.234420:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.234424:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.234425:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.234428:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.234428:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.234429:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.234431:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.234431:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.234432:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.234432:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677448.234446:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.234448:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.234449:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.234449:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.234452:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677448.234454:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.234459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.234467:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.234470:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.234471:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.234473:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.234475:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.234476:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.234478:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.234479:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.234480:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.234481:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.234484:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.234486:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.234488:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.234489:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.234489:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.234491:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.234491:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.234493:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.234494:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.234494:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.234500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.234503:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.234506:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.234506:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.234507:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:31.0:1619677448.234508:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.234509:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:31.0:1619677448.234510:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.234510:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.234511:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:26.0:1619677448.234511:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677448.234528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.234535:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.234538:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.234539:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.234541:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.234542:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.234543:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.234544:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.234545:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.234546:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.234547:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.234550:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.234552:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.234556:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.234556:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.234557:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.234559:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.234559:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.234560:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.234561:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.234572:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.234575:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.234576:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.234576:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.234578:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.234581:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.234598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.234605:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.234608:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.234608:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.234611:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.234613:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.234614:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.234616:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.234617:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.234618:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.234619:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.234623:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.234625:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.234627:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.234628:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.234629:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.234631:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.234631:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.234632:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.234633:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.234637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.234643:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.234646:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.234647:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.234648:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.234650:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.234653:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677448.234695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.234698:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.234700:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.234702:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.234703:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.234705:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.234706:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677448.234708:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717573] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.234709:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.234710:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.234711:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677448.234714:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.234716:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677448.234719:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.234720:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677448.234720:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.234722:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677448.234722:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.234723:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.234724:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.234728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.234736:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.234739:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.234740:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.234740:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.234742:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.234745:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.234758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.234764:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.2:1619677448.234765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:5.0:1619677448.234768:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.234770:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.234770:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:6.0:1619677448.234771:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.234773:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.234773:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:5.0:1619677448.234773:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.234775:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:5.0:1619677448.234775:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.234777:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.234777:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717572] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.234778:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.234779:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.234780:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.234781:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.234781:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.234781:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.234783:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.234785:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.234785:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000400:00000200:6.0:1619677448.234786:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.234788:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.234788:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.234789:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.234792:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:5.0:1619677448.234792:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.234793:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96717570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.234793:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:6.0:1619677448.234794:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.234794:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:22.0:1619677448.234795:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.234796:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.234796:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:5.0:1619677448.234796:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.234796:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:22.0:1619677448.234797:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.234797:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.234798:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:6.0:1619677448.234801:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.234802:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.234802:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.234803:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.234804:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.234805:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.234806:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.234806:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)++
00000800:00000200:22.0:1619677448.234807:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.234807:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (24)--
00000800:00000200:22.0:1619677448.234808:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.234808:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.234809:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677448.234810:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677448.234810:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.234810:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.234811:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.234812:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677448.234813:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.234814:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.234816:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:8.2:1619677448.234816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.234817:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.234817:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:30.0:1619677448.234819:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677448.234819:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:5.2:1619677448.234819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.234820:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.234821:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.234823:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.234826:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:10.0:1619677448.234826:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:1.0:1619677448.234826:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.234827:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.234828:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.234829:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.234830:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.234831:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.234833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:17.0:1619677448.234834:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.234834:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.234835:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.234836:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:1.0:1619677448.234836:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:26.0:1619677448.234837:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000400:00000200:1.0:1619677448.234837:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.234839:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:1.0:1619677448.234839:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.234840:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.234840:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:1.0:1619677448.234840:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.234841:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.234842:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677448.234842:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:26.0:1619677448.234843:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.234843:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:2.0:1619677448.234843:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:1.0:1619677448.234845:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.234846:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.234848:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:18.0:1619677448.234849:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.234849:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.234850:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677448.234850:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:18.0:1619677448.234851:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.234851:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.234852:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:18.0:1619677448.234853:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.234853:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677448.234854:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.234854:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.234854:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:18.0:1619677448.234855:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.234858:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.234858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:18.0:1619677448.234860:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.234863:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.234864:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.234865:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.234865:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:18.0:1619677448.234867:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.234867:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677448.234867:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:18.0:1619677448.234868:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:17.0:1619677448.234868:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.234868:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:18.0:1619677448.234869:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.234870:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677448.234870:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.234872:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.234875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:8.2:1619677448.234877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:13.0:1619677448.234880:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.234880:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:13.0:1619677448.234883:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.234883:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:13.0:1619677448.234884:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.234884:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.234885:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677448.234885:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:13.0:1619677448.234886:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.234887:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.234887:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.234888:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:13.0:1619677448.234889:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.234889:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.234890:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.234891:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.235040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.235047:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.235050:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.235050:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.235053:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.235054:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.235056:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.235057:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.235058:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.235059:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.235060:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.235063:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.235065:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.235068:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.235069:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.235069:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.235071:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.235072:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.235073:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.235073:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.235074:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.235079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.235084:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:4.2:1619677448.235085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677448.235086:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:26.0:1619677448.235086:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677448.235087:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.235088:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:31.0:1619677448.235089:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.235089:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.235090:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.235090:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.235091:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:24.0:1619677448.235092:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.235094:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.235095:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.235097:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.235100:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.235101:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.235102:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.235104:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.235104:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.235105:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.235108:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.235110:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.235112:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.235113:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.235114:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.235116:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.235116:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.235116:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.2:1619677448.235116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.0:1619677448.235117:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.235121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.235126:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.235128:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677448.235131:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.235131:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:17.0:1619677448.235133:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.235133:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.235134:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.235134:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.235136:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.235136:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.235137:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.235138:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.235139:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.235142:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.235143:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.235143:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.235145:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.235145:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677448.235148:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.235149:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.235150:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.235152:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.235152:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.235153:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.235153:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.235165:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.235168:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.235169:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.235169:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.235171:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.235173:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.235211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.235219:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.235221:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.235222:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.235224:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.235226:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.235227:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.235229:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.235230:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.235231:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.235232:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.235235:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.235236:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.235238:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.235239:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.235240:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.235241:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.235242:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.235242:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.235243:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.235253:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.235256:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.235257:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.235258:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.235260:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.235261:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.235565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.235573:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.235575:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.235576:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.235578:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.235581:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.235583:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:4.2:1619677448.235583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:8.0:1619677448.235584:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.235585:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.235586:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.235587:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.235590:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:8.0:1619677448.235590:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.235591:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.235593:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.235594:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677448.235594:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.235595:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.235595:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.235595:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.235597:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.235597:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:26.0:1619677448.235598:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.235598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:8.0:1619677448.235598:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000400:00000200:26.0:1619677448.235599:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.235600:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.235600:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.235600:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:26.0:1619677448.235601:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.235602:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.235603:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.2:1619677448.235603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677448.235604:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:21.2:1619677448.235605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000400:00000200:26.0:1619677448.235606:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.235607:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.235607:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677448.235608:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:22.0:1619677448.235610:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.235611:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.235611:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677448.235612:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677448.235612:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:22.0:1619677448.235612:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.235613:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.235613:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677448.235614:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.235614:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.235614:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.235615:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.235615:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677448.235615:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.235615:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:22.0:1619677448.235615:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96717561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.235616:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:20.0:1619677448.235616:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.235617:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:22.0:1619677448.235617:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.235617:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:22.0:1619677448.235618:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.235619:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.235619:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:22.0:1619677448.235623:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.235625:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.2:1619677448.235629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677448.235630:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.235631:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.235631:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.235632:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.235633:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.235634:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:6.0:1619677448.235634:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.235635:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.235636:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:6.0:1619677448.235636:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.235636:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.235637:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.235638:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.235639:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677448.235639:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.235639:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677448.235639:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.235642:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.235643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:1.0:1619677448.235644:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.235645:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.235646:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.235647:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.235648:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:1.0:1619677448.235648:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.235649:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:22.0:1619677448.235650:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:30.0:1619677448.235651:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677448.235651:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.235652:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:1.0:1619677448.235653:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.235654:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:1.0:1619677448.235654:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:30.0:1619677448.235655:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.235657:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.235657:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.235657:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:31.0:1619677448.235658:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.235658:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.235659:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677448.235660:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:1.0:1619677448.235660:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.235662:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.235663:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.235663:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.235666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.235669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.235673:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.235674:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.235675:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.235675:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677448.235676:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.235676:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.235676:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:31.0:1619677448.235677:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.235677:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:1.0:1619677448.235677:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.235678:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:17.0:1619677448.235678:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.235679:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:1.0:1619677448.235679:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.235680:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677448.235680:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.2:1619677448.235754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.235760:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.235763:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.235764:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.235766:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.235769:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.235770:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.235771:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.235773:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.235774:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.235775:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.235778:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.235780:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.235783:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.235784:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.235784:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.235786:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.235786:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.235787:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.235788:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677448.235800:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.235803:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.235803:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.235805:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.235807:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.235808:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.235858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.235863:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.235866:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.235868:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.235869:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.235870:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.235871:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.235872:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.235873:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.235874:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.235875:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.235876:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.235877:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.235879:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.235881:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677448.235881:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.235883:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.235884:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.235884:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.235885:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.235886:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.235889:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.235890:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.235890:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.2:1619677448.235891:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677448.235892:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677448.235894:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.235894:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.235895:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677448.235895:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.235896:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.235897:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.235897:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.235897:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)++
00000800:00000200:20.0:1619677448.235898:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (24)++
00000800:00000200:5.0:1619677448.235899:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (25)++
00000800:00000200:17.0:1619677448.235900:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:5.0:1619677448.235900:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (26)--
00000800:00000200:1.0:1619677448.235900:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.235901:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (26)++
00000800:00000200:5.0:1619677448.235901:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (25)--
00000800:00000200:20.0:1619677448.235902:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (25)--
00000400:00000200:17.0:1619677448.235902:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.235902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.235902:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (25)--
00000800:00000200:8.2:1619677448.235903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.235904:0:19934:0:(o2iblnd_cb.c:929:kiblnd_post_tx_locked()) 10.51.2.71@o2ib3(1): redundant or enough NOOP
00000400:00000200:17.0:1619677448.235904:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.235906:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (23)++
00000400:00000200:17.0:1619677448.235906:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.235907:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96717556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.235908:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.235909:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (23)--
00000400:00000200:17.0:1619677448.235909:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.235910:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:17.0:1619677448.235910:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.235910:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677448.235911:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:17.0:1619677448.235911:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.235911:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:20.0:1619677448.235912:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677448.235912:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (23)--
00000800:00000200:2.0:1619677448.235913:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:1.0:1619677448.235913:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:18.0:1619677448.235914:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.235914:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.235914:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.235916:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.235918:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:2.0:1619677448.235918:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.235919:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.235921:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677448.235922:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.235922:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.235923:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.235923:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677448.235924:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.235926:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.235926:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.235927:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.235928:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.235939:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.235942:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.235943:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.235943:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.235945:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.235948:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.235957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.235965:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.235967:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.235968:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.235970:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.235973:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.235975:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.235976:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.235978:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.235978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:10.0:1619677448.235979:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.235980:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.235984:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.235986:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.235986:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.235987:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677448.235989:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.235990:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.235991:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.235991:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.235992:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677448.235993:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.235994:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.235995:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.235995:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:26.0:1619677448.235996:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.235997:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.236000:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.236000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.236001:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.236002:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.236003:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.236003:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.236004:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.236004:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.236005:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.236005:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.236007:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.236007:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.236008:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.236008:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:18.0:1619677448.236008:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.236011:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.236012:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.236012:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.236014:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.236017:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.236018:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.236020:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.236021:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.236021:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.236023:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.236025:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.236040:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.236042:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.236042:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.236043:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.236043:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.236044:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:22.0:1619677448.236162:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.236165:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.236165:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.236166:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.236167:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.236167:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.236177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677448.236183:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677448.236186:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677448.236187:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677448.236189:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.236191:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.236193:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677448.236194:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.236195:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.236196:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.236197:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677448.236200:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.236202:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.236204:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.236205:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:1.0:1619677448.236206:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677448.236207:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677448.236208:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677448.236208:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677448.236209:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:6.0:1619677448.236221:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.2:1619677448.236221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:22.0:1619677448.236224:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.236224:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.236225:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.236227:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.236227:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677448.236229:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677448.236230:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.236231:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.236232:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.236235:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.236237:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.236238:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.236239:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.236240:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.236241:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.236244:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.236246:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.236248:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.236248:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.236249:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.236251:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.236251:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.236252:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.236252:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.236258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.236264:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.236266:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.236267:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.236268:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.236269:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.236270:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.236324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677448.236325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677448.236332:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.236333:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.236335:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:30.0:1619677448.236336:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677448.236336:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.236338:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.236339:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.236340:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677448.236340:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.236341:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.236342:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.236343:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.236343:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.236345:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.236345:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.236347:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.236347:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.236348:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.236348:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.236349:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.236352:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.236354:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.236357:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.236357:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.236358:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.236358:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.236359:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.236360:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.236361:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.236361:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.236362:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.236362:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.236362:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.236363:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (22)++
00000800:00000200:30.0:1619677448.236364:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (23)--
00000800:00000200:30.0:1619677448.236365:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.236366:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.236367:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.236368:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677448.236374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677448.236376:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:22.0:1619677448.236379:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:6.0:1619677448.236379:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (24)++
00000400:00000200:22.0:1619677448.236380:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.236380:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:31.0:1619677448.236381:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:6.0:1619677448.236381:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.236382:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.236383:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.236384:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.236385:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.236385:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.236386:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.236387:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.236387:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.236389:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.236390:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.236391:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.236391:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.236392:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.236406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.236411:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.236413:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.236414:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.236416:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.236419:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.236420:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.236422:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.236423:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.236424:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.236425:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.236428:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.236431:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.236434:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.236434:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.236435:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.236437:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.236437:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.236438:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.236438:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.236443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.236450:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.236453:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.236454:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.236454:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.236456:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.236458:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.236473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.236480:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.236483:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.236484:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.236484:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.236485:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.236486:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.236498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.236506:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.236509:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.236512:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.236513:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.236515:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.236516:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.236518:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.236519:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.236520:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.236521:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.236525:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.236526:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.236529:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.236530:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.236530:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.236532:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.236532:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.236533:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.236534:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.2:1619677448.236539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.236544:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:17.0:1619677448.236545:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.236547:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.236547:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.236548:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.236548:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677448.236549:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.236551:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:17.0:1619677448.236551:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.236552:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:17.0:1619677448.236553:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.236555:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.236556:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717547] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.236557:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.236559:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.236560:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.236563:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.236565:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.236568:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.236569:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.236570:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.236572:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.236572:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.236573:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.236573:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.236584:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.236587:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.236588:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.236590:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.236591:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.236593:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.236617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.236624:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.236627:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.236628:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.236630:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.236633:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.236635:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.236636:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.236637:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.236638:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.236639:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.236642:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.236643:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.236646:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.236647:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.236648:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.236650:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.236650:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.236652:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.236653:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.236653:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677448.236656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677448.236659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.236663:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.236666:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:2.0:1619677448.236666:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.236667:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.236667:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.236668:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.236669:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.236670:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:2.0:1619677448.236670:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.236673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677448.236673:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.236674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.236678:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.236681:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:18.0:1619677448.236682:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677448.236683:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.236684:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677448.236684:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.236685:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.236686:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.236686:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.236687:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.236687:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:5.0:1619677448.236688:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.236689:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.236690:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.236691:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.236692:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.236696:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.236698:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.236701:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.236701:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.236702:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.236704:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.236704:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.236705:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.236706:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.236718:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.236720:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.236721:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.236723:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.236725:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.236726:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.236749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.236756:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.236759:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.236759:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.236761:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.236764:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.236765:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.236766:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.236768:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.236769:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.236770:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.236772:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.236774:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.236775:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.236776:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.236777:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.236778:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.236778:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.236779:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.236779:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.236789:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.236791:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.236792:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.236793:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.236793:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.236794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:10.0:1619677448.236795:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:2.0:1619677448.236800:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.236802:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.236804:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.236805:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.236807:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.236809:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.236810:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.236811:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.236812:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.236813:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.236817:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.236819:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.236822:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.236822:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.236823:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.236825:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.236825:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.236826:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.236826:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.236831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.236837:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.236840:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.236841:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.236843:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.236844:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.236846:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.236849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:4.2:1619677448.236850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677448.236855:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.236857:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677448.236858:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.236859:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:13.0:1619677448.236859:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.236859:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.236860:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.236860:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.236860:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.236862:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.236864:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.236865:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.236867:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.236868:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.236869:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.236870:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.236874:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.236875:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.236878:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.236879:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.236880:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.236882:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.236882:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.236883:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.236884:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.236889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.236896:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.236899:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.236900:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.236902:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.236902:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.236905:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.236906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.236913:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.236916:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.236917:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.236918:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.236918:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.236919:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.236931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.236939:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.236942:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.236944:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.236945:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.236945:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.236947:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.236948:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.236950:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.236951:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.236952:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:5.2:1619677448.236952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677448.236955:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.236956:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.236957:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.236959:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.236959:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.236960:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.236960:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:30.0:1619677448.236961:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.236961:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.236962:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.236963:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:1.0:1619677448.236963:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.236965:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.236966:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:1.0:1619677448.236966:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.236967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:1.0:1619677448.236967:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.236968:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.236969:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.236970:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.236971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:1.0:1619677448.236971:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:13.0:1619677448.236974:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:1.0:1619677448.236974:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.236976:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:13.0:1619677448.236977:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.236977:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.236979:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:1.0:1619677448.236979:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677448.236980:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.236980:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.236980:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677448.236981:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:1.0:1619677448.236981:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.236982:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.236983:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677448.236983:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677448.236983:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.236984:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.236984:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.236985:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677448.236989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.236994:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.236997:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.236999:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.236999:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:25.2:1619677448.237000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:5.0:1619677448.237001:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.237003:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677448.237004:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.237006:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:5.0:1619677448.237006:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.237007:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.237008:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.237009:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.237010:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.237012:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.237012:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:30.0:1619677448.237013:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.237014:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.237015:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.0:1619677448.237017:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.237018:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677448.237018:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677448.237020:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.237021:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.237022:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.237022:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.237026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.237033:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.237036:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.237037:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.237038:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677448.237039:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.237041:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.237047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.237054:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.237056:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.237057:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.237058:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.237058:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.237059:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.237143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.237150:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.237152:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.237154:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.237157:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.237157:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.237158:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.237159:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.237161:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.237162:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.237163:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.237165:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.237167:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.237169:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.237170:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.237171:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.237172:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.237173:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.237173:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.237174:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.237178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.237184:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.237186:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.237187:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.237188:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.237190:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.237192:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.237222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.237229:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.237231:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.237232:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.237234:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.237236:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.237238:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.237239:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.237240:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.237241:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.237242:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.2:1619677448.237243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:10.0:1619677448.237245:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.237247:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.237251:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.237251:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.237252:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.237252:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.237252:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:26.0:1619677448.237254:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.237254:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.237255:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.237256:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.237256:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:26.0:1619677448.237257:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.237257:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.237259:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.237260:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.237260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:26.0:1619677448.237261:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.237262:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.237263:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.237266:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.237267:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:22.0:1619677448.237268:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.237270:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.237270:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.237271:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677448.237271:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.237272:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.237272:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.237273:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.237273:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.237274:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:22.0:1619677448.237274:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.237276:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.237276:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:22.0:1619677448.237277:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.2:1619677448.237279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.237283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677448.237290:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:10.0:1619677448.237290:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677448.237292:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:20.0:1619677448.237293:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677448.237297:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:20.2:1619677448.237298:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677448.237299:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:10.0:1619677448.237302:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:20.0:1619677448.237303:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.237304:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.237306:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.237307:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:30.0:1619677448.237309:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.237311:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.237313:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.237314:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.237315:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.237316:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.237317:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.237318:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.237319:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.237322:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.237324:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.237326:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.237327:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.237327:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.237329:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.237329:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.237330:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.237331:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.237335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.237344:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.237346:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.237347:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.237348:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.237350:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.237351:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.237398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.237405:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.237408:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.237409:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.237412:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.237414:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.237416:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.237417:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.237419:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.237420:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.237421:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.237424:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.237426:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.237429:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.237430:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.237431:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.237432:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.237433:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.237434:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.237435:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.237436:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.237439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.237440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677448.237444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:13.0:1619677448.237446:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.237446:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.237448:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.237448:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.237449:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677448.237449:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677448.237450:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.237452:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:13.0:1619677448.237452:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.237453:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.237455:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:13.0:1619677448.237455:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.237456:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.237458:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.237458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:26.0:1619677448.237461:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.237462:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.237463:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.237464:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.237466:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.237466:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677448.237466:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.237469:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:26.0:1619677448.237470:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.237471:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677448.237472:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:8.0:1619677448.237472:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.237475:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677448.237475:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.237476:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.237476:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:8.0:1619677448.237476:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.237477:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96717532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.237478:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.237478:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:4.2:1619677448.237478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.237479:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677448.237479:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.237479:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.237480:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.237481:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.237482:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.237483:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:25.2:1619677448.237483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:8.0:1619677448.237483:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.237485:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.237485:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677448.237485:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:26.0:1619677448.237487:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.237487:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.237487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677448.237488:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:12.0:1619677448.237489:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677448.237489:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677448.237490:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.237490:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:26.0:1619677448.237491:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:8.0:1619677448.237491:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:12.0:1619677448.237492:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.237492:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.237492:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.237492:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:26.0:1619677448.237493:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96717531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.237494:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.237495:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.237496:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.237496:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.237496:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:26.0:1619677448.237497:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.237497:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.237497:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.2:1619677448.237497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677448.237499:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.237500:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677448.237501:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.237503:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.237504:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:26.0:1619677448.237506:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.237507:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.237507:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.237507:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.237508:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.237508:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:26.0:1619677448.237509:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.237510:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.237510:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677448.237510:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.237511:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677448.237514:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:25.2:1619677448.237515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.237523:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.237527:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.237528:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.237528:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.237531:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.237534:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.237699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.237707:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.237710:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.237712:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.237714:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.237714:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.237716:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.237717:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.237718:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.237719:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.237720:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.237723:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.237725:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.237727:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.237728:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.237728:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.237730:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.237730:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.2:1619677448.237730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.237731:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.237732:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677448.237734:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.237735:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:25.2:1619677448.237735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.237737:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.237738:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.237739:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.237740:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.237741:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.237741:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:31.0:1619677448.237742:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.237743:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.237743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:31.0:1619677448.237744:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.237744:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.237744:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.237746:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.237747:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.237748:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.237748:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.237749:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:24.0:1619677448.237750:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:31.0:1619677448.237752:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.237753:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:24.0:1619677448.237753:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.237753:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:31.0:1619677448.237754:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.237755:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.237756:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.237756:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.237756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:24.0:1619677448.237756:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.237757:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:24.0:1619677448.237758:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.237760:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:8.2:1619677448.237761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:24.0:1619677448.237762:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96717528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.237762:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677448.237763:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.237765:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.237765:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677448.237766:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:12.0:1619677448.237766:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.237766:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.237767:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.237768:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677448.237769:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.237770:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:24.0:1619677448.237771:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.237773:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:24.0:1619677448.237774:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.237774:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:5.0:1619677448.237774:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.237775:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677448.237775:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.237776:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:24.0:1619677448.237777:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.237777:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.237778:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677448.237778:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:24.0:1619677448.237779:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.237783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.237791:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.237795:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.237796:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.237796:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.237798:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.237801:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.238040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.238043:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.238046:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.238048:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.238049:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.2:1619677448.238050:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:20.0:1619677448.238051:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.238052:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.238053:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.238055:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.238056:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.238056:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:20.0:1619677448.238057:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.238059:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.238060:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677448.238061:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.238062:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.238063:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.238065:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.238066:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.238067:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:1.0:1619677448.238067:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:20.0:1619677448.238068:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677448.238068:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.238070:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.238070:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677448.238070:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.238070:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.238071:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.238072:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677448.238072:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.238075:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.238076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.238077:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.238080:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.238081:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.238081:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677448.238083:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677448.238083:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.238083:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.238084:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.238085:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:1.0:1619677448.238085:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:2.0:1619677448.238086:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.238087:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.238088:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.238089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677448.238091:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677448.238096:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.238099:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.238099:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.238101:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.238102:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.238104:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.238187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.238195:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.238198:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.238200:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.238202:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.238203:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.238205:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677448.238206:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.238207:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.238208:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.238210:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677448.238213:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.238215:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.238218:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.238219:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.238219:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.238221:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.238222:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.238223:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.238223:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.238227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.238235:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.238238:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.238239:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.238239:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.238241:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.238244:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.238267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.238273:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.238276:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.238277:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.238278:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.238281:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.238282:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.238284:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.238285:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.238286:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.238287:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.238290:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.238291:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.238294:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.238294:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.238299:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.238301:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.238301:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.238302:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.238302:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677448.238304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:8.2:1619677448.238307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.238313:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.238316:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.238316:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:5.0:1619677448.238316:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:10.0:1619677448.238318:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.238318:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.238319:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.238320:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.238321:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:10.0:1619677448.238321:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.238323:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:5.0:1619677448.238323:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:10.0:1619677448.238324:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.238326:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.238326:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.238326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000400:00000200:10.0:1619677448.238328:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.238331:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.238333:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.238334:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.238336:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.238336:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.238336:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.238337:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.238338:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.238338:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.238339:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.238339:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.238340:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:5.0:1619677448.238340:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:10.0:1619677448.238341:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.238341:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677448.238346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.238355:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.238357:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.238358:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.238359:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.238361:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.238364:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.238442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.238448:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.238451:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.238452:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.238454:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.238456:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.238458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677448.238458:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.238459:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.238460:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.238461:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.238462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:2.0:1619677448.238462:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.238465:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677448.238466:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.238467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:26.0:1619677448.238468:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:21.2:1619677448.238469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.238469:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:10.0:1619677448.238469:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677448.238469:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.238472:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.238472:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.238472:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.238473:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:18.0:1619677448.238474:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.238474:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677448.238475:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.238476:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677448.238476:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:26.0:1619677448.238477:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677448.238477:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.238477:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.238477:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.238478:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677448.238479:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:26.0:1619677448.238479:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.238479:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:2.0:1619677448.238480:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.238480:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:18.0:1619677448.238481:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.238481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677448.238482:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.238482:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.238482:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:30.0:1619677448.238483:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677448.238483:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.238483:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.238484:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677448.238485:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.238485:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.238486:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.238490:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.238491:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.238492:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.238493:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.238495:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:26.0:1619677448.238495:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:2.0:1619677448.238495:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:18.0:1619677448.238496:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.238497:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96717519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.238497:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96717519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.238498:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.238498:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:31.0:1619677448.238499:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.238499:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:2.0:1619677448.238499:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.238501:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.238501:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.238501:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677448.238501:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.238502:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677448.238503:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.238503:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.238504:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.238504:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.238506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:2.0:1619677448.238509:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.238510:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.238510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.238512:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.238513:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.238513:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.238514:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:18.0:1619677448.238514:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.238514:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.238515:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:18.0:1619677448.238515:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.238515:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677448.238516:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.238516:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.238516:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:2.0:1619677448.238516:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000800:00000200:31.0:1619677448.238517:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:30.0:1619677448.238517:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.238517:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.238518:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.238518:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:10.0:1619677448.238518:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:2.0:1619677448.238518:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (24)--
00000800:00000200:18.0:1619677448.238519:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:10.0:1619677448.238519:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677448.238519:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (22)++
00000800:00000200:31.0:1619677448.238520:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.238520:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:18.0:1619677448.238520:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:10.0:1619677448.238520:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.238521:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677448.238521:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (23)++
00000800:00000200:31.0:1619677448.238522:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.238522:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677448.238522:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (23)--
00000800:00000200:31.0:1619677448.238523:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:18.0:1619677448.238523:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:8.2:1619677448.238523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.238523:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.238524:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000800:00000200:2.0:1619677448.238524:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.238525:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.238525:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (24)++
00000800:00000200:2.0:1619677448.238526:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.238527:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.238527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.238527:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (24)++
00000800:00000200:21.2:1619677448.238528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.238528:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (25)--
00000800:00000200:2.0:1619677448.238529:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (24)--
00000800:00000200:18.0:1619677448.238530:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:1.0:1619677448.238530:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677448.238531:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.238531:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.238531:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.238532:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:2.0:1619677448.238532:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.238533:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:1.0:1619677448.238533:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677448.238534:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677448.238534:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:2.0:1619677448.238534:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.238534:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.238535:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677448.238535:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677448.238536:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:1.0:1619677448.238536:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.238537:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.238538:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.238538:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:20.0:1619677448.238540:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.238540:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.238543:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.238559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.238566:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.238569:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.238570:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.238570:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.238571:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.238572:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.238607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677448.238613:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.238616:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677448.238617:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677448.238619:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.238621:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.238623:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677448.238624:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.238625:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.238627:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.238628:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677448.238631:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.238633:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677448.238636:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.238637:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677448.238637:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677448.238639:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677448.238640:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.238641:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677448.238641:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:25.2:1619677448.238645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.238651:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.238654:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.238655:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.238655:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.238657:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.238660:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677448.238749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.238756:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.238759:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.238761:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.238762:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.238764:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.238765:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677448.238767:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.238768:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.238769:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.238770:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677448.238773:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.238775:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.238778:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.238779:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.238780:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.238781:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.238782:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.238784:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.238785:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.238786:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677448.238787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.238792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.238795:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.238797:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.238798:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.238799:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:1.0:1619677448.238799:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.238801:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.238802:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.238803:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.238804:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677448.238804:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.239105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.239111:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.239114:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.239115:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.239117:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.239120:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.239121:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.239122:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.239124:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.239125:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.239125:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.239128:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.239129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677448.239130:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.239133:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.239134:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.239135:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.239136:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.239137:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.239137:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.239138:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:10.0:1619677448.239139:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:2.0:1619677448.239139:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:26.0:1619677448.239140:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.239142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:10.0:1619677448.239142:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.239144:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.239146:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.239148:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.239149:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.239150:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:10.0:1619677448.239151:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.239152:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.239152:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.239153:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.239155:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.239156:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.239158:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.239158:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.239159:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.239160:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.239162:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.239162:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.239164:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.239164:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.239165:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.239165:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.239170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.239176:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.239179:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.239179:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.239180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:31.0:1619677448.239181:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.239183:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.239183:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.239186:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.239188:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.239191:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.239191:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.239193:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.239195:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.239196:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.239197:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.239199:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.239200:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.239203:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.239204:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.239207:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.239208:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.239209:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.239211:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.239212:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.239212:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.239213:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.239217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.239224:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.239227:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.239228:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.239230:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.239232:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.239233:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.239281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.239288:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.239291:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677448.239296:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.239299:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.239301:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.239303:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.239305:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.239306:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.239307:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.239308:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.239311:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.239313:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.239316:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.239316:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.239317:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.239319:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.239320:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:21.2:1619677448.239320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.239321:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.239323:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.239323:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.239324:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.239326:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.2:1619677448.239326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.239328:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.239329:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.239329:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.239329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:30.0:1619677448.239331:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.239332:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.239333:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.239333:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.239334:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.239337:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.239338:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.239338:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.239338:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:4.2:1619677448.239338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677448.239339:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.239339:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.239340:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.239340:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.239341:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.239341:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.239341:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.239342:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.239343:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.239343:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.239344:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677448.239344:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.239344:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.239345:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677448.239346:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677448.239348:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:25.2:1619677448.239349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677448.239349:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.239350:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.239352:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.239354:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.239356:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677448.239356:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.239357:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96717510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.239358:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.239359:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:8.0:1619677448.239359:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677448.239360:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.239360:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677448.239361:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.239363:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:8.0:1619677448.239363:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.239364:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677448.239365:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.239367:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.239368:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.239369:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.239370:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.239371:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.239372:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.239373:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.239374:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.239376:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.239379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.239385:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.239386:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.239387:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677448.239388:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:1.0:1619677448.239388:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.239389:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.239389:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.239390:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677448.239391:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.239394:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.239567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.239573:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.239576:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.239578:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.239578:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.239581:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.239583:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.239584:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.239585:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.239586:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.239587:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.239591:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.239593:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.239596:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.239597:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.239598:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.239599:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.239600:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.239602:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.239603:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.239603:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.239605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.239608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.239611:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:31.0:1619677448.239612:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.239613:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.239614:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677448.239615:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677448.239615:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:31.0:1619677448.239616:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.239617:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.239619:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.239620:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.239638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.239644:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.239647:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.239648:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.239649:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.239649:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.239651:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.239694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.239699:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.239702:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.239703:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.239705:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.239707:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.239709:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.239710:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.239711:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.239712:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.239713:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.239717:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.239719:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.239721:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.239722:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.239723:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:4.2:1619677448.239725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677448.239725:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.239725:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.239726:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.239726:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.239731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:26.0:1619677448.239733:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.239735:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.239736:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677448.239737:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:26.0:1619677448.239738:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.239739:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:26.0:1619677448.239740:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.239740:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.239741:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.239742:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:20.2:1619677448.239742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:26.0:1619677448.239743:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.239743:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.239745:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.239745:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677448.239746:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.239746:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:26.0:1619677448.239747:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.239748:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.239751:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.239751:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677448.239752:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.239752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:26.0:1619677448.239754:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:20.0:1619677448.239754:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.239756:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.239756:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:26.0:1619677448.239757:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.239757:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:20.0:1619677448.239757:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.239758:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:26.0:1619677448.239759:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.239759:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677448.239759:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.239760:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677448.239760:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677448.239760:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.239761:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:20.0:1619677448.239761:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.0:1619677448.239761:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.239762:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.239763:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.239763:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:25.2:1619677448.239765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:20.0:1619677448.239765:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.239767:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.239770:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.239771:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.239771:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:20.0:1619677448.239772:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.239773:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.239773:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.239774:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.239774:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.239774:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:10.0:1619677448.239775:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.239776:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.239777:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.239777:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677448.239778:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.239779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.239782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.239787:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.239788:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677448.239789:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.239790:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.239790:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.239791:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.239791:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:31.0:1619677448.239792:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.239793:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.239795:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.240186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.240192:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.240196:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.240196:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.240199:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.240202:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.240204:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.240205:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.240206:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.240207:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.240208:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.240212:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.240213:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.240216:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.240217:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.240218:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.240220:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.240220:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.240221:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.240222:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.240223:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:25.2:1619677448.240226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.240228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677448.240231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677448.240233:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.240233:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:12.0:1619677448.240235:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.240235:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.240236:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677448.240237:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:12.0:1619677448.240237:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.240237:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677448.240239:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.240239:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.240240:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.240241:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677448.240242:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:17.0:1619677448.240243:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.240246:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.240247:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.240248:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.240250:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.240251:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.240252:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.240255:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.240257:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.240260:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.240261:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.240261:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.240263:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.240263:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.240264:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.240265:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.240269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.240276:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.240278:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.240279:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.240279:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.240282:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.240284:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.2:1619677448.240450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.240457:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.2:1619677448.240458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:26.0:1619677448.240460:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.240461:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.240463:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.240464:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.240464:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:26.0:1619677448.240465:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.240466:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.240467:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.240468:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:26.0:1619677448.240469:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.240471:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.240471:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.240472:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.240474:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.240475:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:26.0:1619677448.240476:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.240477:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.240478:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.240478:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.240479:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.240480:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:18.0:1619677448.240480:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.240481:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:18.0:1619677448.240481:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.240482:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:18.0:1619677448.240484:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.240485:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.240486:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677448.240486:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.240487:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.240488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.240489:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.240490:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677448.240490:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677448.240492:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677448.240493:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.240494:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.240495:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.240495:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:10.0:1619677448.240495:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.240496:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:25.2:1619677448.240497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.240498:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.240498:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.240500:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:21.2:1619677448.240500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:10.0:1619677448.240500:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.240501:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.240502:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.240502:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.240503:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.240504:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.240506:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:26.0:1619677448.240507:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:2.0:1619677448.240508:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.240509:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677448.240510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.240510:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677448.240516:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.240518:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.240519:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.240519:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.240520:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.240520:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.240568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.240574:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.240577:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.240578:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.240580:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.240582:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.240584:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.240585:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.240587:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.240588:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.240589:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.240592:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.240594:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.240597:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.240598:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.240598:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.240600:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.240600:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.240601:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.240602:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.240607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.240615:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.240618:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.240619:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.240619:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.240621:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.240624:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.240673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.240680:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.240683:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.240684:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.240686:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.240687:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.240689:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.240690:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.240690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000400:00000200:26.0:1619677448.240691:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.240692:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.240693:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.240695:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:26.0:1619677448.240697:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.240698:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:26.0:1619677448.240699:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.240699:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677448.240700:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.240701:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.240702:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.240703:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:17.0:1619677448.240703:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.240704:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:17.0:1619677448.240704:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:26.0:1619677448.240705:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.240705:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677448.240705:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.240706:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:17.0:1619677448.240707:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.240708:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.240709:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:25.2:1619677448.240710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:17.0:1619677448.240712:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.240714:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.240717:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.240718:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677448.240718:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.240718:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.240720:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.240720:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:30.0:1619677448.240721:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.240722:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.240722:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000400:00000200:30.0:1619677448.240723:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.240723:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677448.240724:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:31.0:1619677448.240725:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.240726:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:9.2:1619677448.240726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.240731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:24.0:1619677448.240732:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.240735:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.240736:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.240736:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.240737:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000400:00000200:24.0:1619677448.240738:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.240739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677448.240739:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677448.240739:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677448.240740:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:24.0:1619677448.240741:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.0:1619677448.240745:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.240748:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.240750:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.240750:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.240753:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.240755:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.240756:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.240758:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.240759:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.240760:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.240763:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.240765:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.240768:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.240769:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.240770:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.240772:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.240772:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.240773:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.240774:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.240785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.240792:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.240795:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.240796:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.240797:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.240799:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:21.2:1619677448.240800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.240803:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.240804:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.240806:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.240807:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.240808:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.240810:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.240811:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.240812:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717497] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.240814:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.240815:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.240816:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.240819:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.240821:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.240824:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.240825:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.240826:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.240828:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.240828:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.240829:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.240829:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.240834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.240841:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.240844:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.240845:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.240845:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.240848:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.240850:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.240852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:25.2:1619677448.240853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:17.0:1619677448.240857:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.240860:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:6.0:1619677448.240860:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677448.240861:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.240863:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.240863:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.240864:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.240864:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.240865:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000400:00000200:17.0:1619677448.240866:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.240866:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:17.0:1619677448.240868:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.240869:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.240870:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.240871:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.240872:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.240875:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.240877:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.240879:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.240880:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.240880:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.240882:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.240882:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.240884:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.240884:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.240885:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.240888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.240891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.240897:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.240897:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.240898:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.240898:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.240899:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.240899:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.240900:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.240900:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.240901:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.240901:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.241155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.241161:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.241164:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.241165:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.241167:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.241170:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.241171:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.241172:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.241173:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.241174:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.241175:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.241178:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.241179:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:21.2:1619677448.241180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:18.0:1619677448.241181:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.241182:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.241182:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.241184:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.241184:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.241185:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.241185:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:22.0:1619677448.241186:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.241189:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:9.2:1619677448.241190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.241190:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.241191:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.241194:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.241195:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.241197:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.241197:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:22.0:1619677448.241198:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.241199:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.241199:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:22.0:1619677448.241200:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677448.241200:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.241201:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.241203:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.241205:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.241205:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.241206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677448.241207:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.241210:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.241211:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.241212:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.241213:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.241213:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677448.241214:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.241216:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677448.241216:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:10.0:1619677448.241216:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:22.0:1619677448.241217:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.241217:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:25.2:1619677448.241219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000400:00000200:10.0:1619677448.241219:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.241221:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.241222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677448.241223:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.241224:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.241226:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:10.0:1619677448.241226:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.241227:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.241227:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.241227:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:26.0:1619677448.241228:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.241228:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:6.0:1619677448.241229:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.241230:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:26.0:1619677448.241231:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.241231:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677448.241232:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.241234:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.241236:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.241237:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:10.0:1619677448.241237:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.241237:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.241238:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.241240:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.241240:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.241241:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.241241:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.241245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.241251:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.241253:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.241253:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.241254:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.241255:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.241256:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.241298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.241306:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.241308:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.241309:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.241310:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.241312:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.241314:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.241314:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.241316:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.241316:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.241317:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.241320:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.241321:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.241325:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.241325:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.241326:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.241328:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.241328:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.241329:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.241329:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.241334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.241343:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.241345:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.241346:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.241348:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.241350:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.241351:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.241393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.241398:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.241401:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.241403:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677448.241404:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.241406:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.241408:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677448.241410:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.241411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:5.0:1619677448.241411:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.241413:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.241414:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677448.241419:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:30.0:1619677448.241420:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.241421:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.241422:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.0:1619677448.241424:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.241424:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:30.0:1619677448.241425:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.241425:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.241427:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.0:1619677448.241427:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.241427:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:30.0:1619677448.241428:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.241428:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000400:00000200:30.0:1619677448.241429:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:5.0:1619677448.241429:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:30.0:1619677448.241431:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.241432:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.241433:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.241433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:30.0:1619677448.241434:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.241437:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.241439:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.241440:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.241442:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.241442:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.241442:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.241443:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:31.0:1619677448.241444:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677448.241444:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.241446:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.241446:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.241447:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.241447:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.241448:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.241448:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.241452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.241460:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.241463:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.241464:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.241464:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.241467:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.241470:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.241515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.241521:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.241524:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.241525:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.241527:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.241529:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.241530:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.241531:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.241533:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.241534:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.241535:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.241538:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.241540:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.241543:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.241543:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.241544:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.241546:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.241547:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.241548:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.241548:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.241552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.241559:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.241562:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.241563:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.241565:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.241567:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.241568:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.241568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.241575:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.241578:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.241579:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.241581:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.241584:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.241584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:26.0:1619677448.241585:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.241587:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.241588:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.241589:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.241590:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:12.0:1619677448.241591:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:26.0:1619677448.241593:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.241593:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.241594:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677448.241595:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:12.0:1619677448.241595:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.241595:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.241597:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.241598:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.241598:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.241599:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.241601:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.241601:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.241602:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.241604:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.241604:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.241607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.241610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.241614:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.241617:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.241617:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.241618:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.241618:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.241619:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.241619:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.241620:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.241621:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.241624:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.241672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.241677:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.241680:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.241680:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.241682:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.241685:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.241686:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.241687:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.241689:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.241690:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.241691:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.241694:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.241696:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.241699:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.241700:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.241700:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.241702:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.241702:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.2:1619677448.241702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:22.0:1619677448.241703:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.241703:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.241708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677448.241708:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.241711:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.241712:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:26.0:1619677448.241714:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:18.0:1619677448.241714:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.241716:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.241717:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:18.0:1619677448.241717:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:10.0:1619677448.241717:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.241718:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.241718:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.241720:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.241721:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.241722:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.241723:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.241724:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:25.2:1619677448.241726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000400:00000200:18.0:1619677448.241726:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.241728:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.241730:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.241730:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.241731:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.241732:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677448.241733:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.241733:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.241734:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:26.0:1619677448.241735:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677448.241735:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:10.0:1619677448.241735:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.241736:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.241737:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.241738:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:9.2:1619677448.241739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677448.241745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677448.241746:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677448.241748:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.241748:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.241749:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.241750:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.241751:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:2.0:1619677448.241752:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.241753:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.241754:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677448.241754:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:20.0:1619677448.241756:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.241758:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.241759:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.241760:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.241761:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.241762:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.241766:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.241768:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.241770:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.241771:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.241772:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.241774:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.241774:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.241775:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.241776:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.2:1619677448.241780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:9.2:1619677448.241780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.241787:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677448.241788:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677448.241790:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:18.0:1619677448.241790:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.241791:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.241791:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:31.0:1619677448.241794:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.241794:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:18.0:1619677448.241794:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.241796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677448.241797:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.241798:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:18.0:1619677448.241799:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:31.0:1619677448.241800:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.241801:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.241801:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677448.241802:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.241803:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.241803:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.241804:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.241805:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000400:00000200:31.0:1619677448.241806:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.241806:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.241806:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677448.241809:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.241811:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.241812:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.241812:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.241814:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.241814:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:21.2:1619677448.241814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:31.0:1619677448.241815:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.241816:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.241820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.241821:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.241824:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.241825:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677448.241826:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:8.0:1619677448.241827:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.241828:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:8.0:1619677448.241829:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.241830:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.241831:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:13.0:1619677448.241832:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.241832:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:8.0:1619677448.241833:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96717483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.241834:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:8.0:1619677448.241834:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.241835:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.241836:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677448.241839:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.241841:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:25.2:1619677448.241843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.241844:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.241845:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.241845:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.241847:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.241848:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.241848:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677448.241849:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.241849:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677448.241852:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.241853:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.241853:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:8.2:1619677448.241853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:12.0:1619677448.241854:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.241855:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677448.241862:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.241864:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.241865:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.241867:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.241867:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.241870:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.241908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.241913:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:4.2:1619677448.241914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677448.241916:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.241917:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.241919:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.241921:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677448.241922:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.241923:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:26.0:1619677448.241924:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.241925:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677448.241925:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.241926:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.241927:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.241928:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.241929:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.241931:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.241932:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.241933:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.241934:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.241935:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.241936:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.241937:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.241938:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.241938:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.241939:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.241940:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:26.0:1619677448.241942:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.241942:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.241942:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.241943:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.241943:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.241944:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.241947:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677448.241947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.241948:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.241948:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.241950:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.241951:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.241951:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.241952:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:12.0:1619677448.241954:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:25.2:1619677448.241956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:12.0:1619677448.241956:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.241957:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.241959:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.241961:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.241962:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.241964:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.241967:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.241967:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.241968:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.241970:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.241973:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.242001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.242008:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.242011:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.242014:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.242015:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.242016:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.242018:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.242019:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.242020:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.242022:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.242022:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.242026:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.242027:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.242030:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.242031:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.242031:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.242032:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.242033:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.242035:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.242036:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.242037:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.242038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.242042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.242044:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.242046:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.242047:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.242047:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.242049:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.242050:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.242052:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:10.0:1619677448.242052:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677448.242053:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.242054:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677448.242141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677448.242148:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677448.242151:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677448.242153:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.242155:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.242155:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677448.242156:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677448.242158:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.242159:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.242160:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.242161:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677448.242164:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.242165:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677448.242168:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.242168:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.242169:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.242170:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.242171:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677448.242172:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677448.242172:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677448.242176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.242183:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.242185:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.242186:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.242186:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.242188:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.242191:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.242215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.242221:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.242224:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.242225:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.242226:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.242229:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.242230:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.242232:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.242233:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.242233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677448.242234:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.242234:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.242237:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.242238:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.242239:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.2:1619677448.242240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677448.242241:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:26.0:1619677448.242242:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.242242:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.242243:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.242243:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:17.0:1619677448.242244:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.242245:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.242245:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.242246:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.242246:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.242247:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.242247:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.242248:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.242250:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.242251:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:25.2:1619677448.242251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:17.0:1619677448.242251:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.242253:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.242254:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:17.0:1619677448.242254:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.242255:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.242257:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.242258:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.242259:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:22.0:1619677448.242259:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:30.0:1619677448.242260:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96717476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.242260:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.242261:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.242261:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:30.0:1619677448.242262:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.242262:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:30.0:1619677448.242263:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:22.0:1619677448.242263:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.242263:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677448.242264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:17.0:1619677448.242264:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:22.0:1619677448.242265:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.242265:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.242267:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.242267:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677448.242268:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.242268:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.242269:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.242269:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677448.242269:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.242271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.242272:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.242273:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.242274:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677448.242275:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:25.2:1619677448.242275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:22.0:1619677448.242275:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677448.242276:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.242277:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:22.0:1619677448.242277:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:30.0:1619677448.242278:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:6.0:1619677448.242279:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:22.0:1619677448.242280:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.242280:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:13.0:1619677448.242282:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:6.0:1619677448.242282:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000400:00000200:22.0:1619677448.242283:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.242283:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.242283:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.242284:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.242285:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:13.0:1619677448.242285:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677448.242285:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.242286:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:22.0:1619677448.242287:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96717475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677448.242287:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.242288:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000400:00000200:22.0:1619677448.242289:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.242289:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:22.0:1619677448.242290:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.242290:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.242290:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.242291:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:13.0:1619677448.242291:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677448.242292:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.242301:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.242302:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:12.0:1619677448.242303:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677448.242305:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.242306:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.242307:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.242309:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.242309:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.242310:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.242311:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.242342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.242350:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.242353:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.242354:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.242355:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.242357:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.242358:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.242362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.242368:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.242371:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.242372:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.242373:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.242373:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.242374:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.242382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.242390:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.242392:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.242393:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.242395:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.242398:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.242399:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.242401:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.242402:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.242403:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.242404:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.242407:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.242409:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.242411:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.242412:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.242413:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.242415:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.242415:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.242416:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.242416:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.242421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.242430:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.242432:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.242434:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.242435:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.242437:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.242439:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.242439:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.242440:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.242441:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (23)++
00000800:00000200:5.0:1619677448.242441:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.242442:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.242443:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.242444:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.242475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.242482:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.242485:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.242487:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.242489:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.242490:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.242491:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.242492:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.242494:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.242495:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.242496:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.242500:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.242501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677448.242502:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.242505:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.242505:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.242506:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.242507:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.242508:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.242508:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.242509:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.242509:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.242511:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.242512:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.242513:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.242513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:26.0:1619677448.242517:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.242518:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.242519:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.242519:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:26.0:1619677448.242520:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.242521:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.242522:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.242522:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.242523:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.242525:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.242526:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.242526:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.242528:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.242529:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.242530:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.242531:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.242532:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.242533:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.242534:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.242535:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.242536:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.242537:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.242539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.242542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.242545:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.242547:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.242548:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.242549:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.242549:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:12.0:1619677448.242551:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.242551:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:13.0:1619677448.242552:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.242552:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.242553:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:4.2:1619677448.242554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.242561:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.242564:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.242564:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.242566:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.242569:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.242570:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.242571:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.242573:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.242573:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.242574:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.242577:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.242579:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.242581:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.242582:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.242582:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.242584:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.242585:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.242585:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.242586:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.242590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.242598:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.242600:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.242602:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.242602:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.242604:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.242607:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.242641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.242647:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.242650:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.242651:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.242653:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.242655:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.242657:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.242658:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.242660:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.242661:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.242662:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.242666:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.242667:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.242671:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.242671:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.242672:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.242674:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.242674:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.242675:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.242676:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.242677:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.242680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.2:1619677448.242682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.242687:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677448.242687:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.242689:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.242689:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.242690:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:17.0:1619677448.242690:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677448.242691:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.242693:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.242694:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.242696:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.242723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.242731:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.242734:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.242736:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.242737:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.242738:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.242740:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.242741:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.242742:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.242743:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.242744:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.242748:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.242750:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.242752:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.242752:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.242753:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.242754:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.242755:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.242756:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.242757:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.242762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.242768:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.242770:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.242771:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.242773:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.242775:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.242776:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.242779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:12.0:1619677448.242785:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.242787:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.242788:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.242789:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.242789:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.242790:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.242881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:4.2:1619677448.242886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677448.242888:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.242891:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:26.0:1619677448.242892:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677448.242892:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.242894:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.242895:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.242896:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:8.0:1619677448.242896:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.242898:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.242898:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.242899:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.242900:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.242901:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.242901:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.242902:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:8.0:1619677448.242902:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.242903:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.242905:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.242906:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.242906:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.242907:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.0:1619677448.242908:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.242911:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.242912:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.0:1619677448.242912:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.242912:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.242913:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.242914:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.242915:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:8.0:1619677448.242915:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.242915:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:26.0:1619677448.242916:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.0:1619677448.242916:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.242916:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:26.0:1619677448.242917:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.242918:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.242919:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.242920:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.242920:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.242920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:25.2:1619677448.242922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.242926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.242929:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.242929:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:30.0:1619677448.242931:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.242932:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.242932:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:10.0:1619677448.242933:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:5.0:1619677448.242933:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.242934:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.242935:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.242935:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:5.0:1619677448.242935:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.242936:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.242936:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.242938:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677448.242938:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.242938:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.243113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.243121:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.243123:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:21.2:1619677448.243123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:31.0:1619677448.243126:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.243128:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.243128:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.243129:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:30.0:1619677448.243129:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:31.0:1619677448.243130:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.243131:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.243132:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.243133:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.243134:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.243136:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.243137:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.243139:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.243139:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.243140:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.243141:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.243143:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.243143:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.243144:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.243147:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.243147:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.243148:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:30.0:1619677448.243148:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.243150:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.243150:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.243150:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.243152:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.243153:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.243153:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.243154:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.243154:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.243154:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.243155:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.243155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.243156:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.243157:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.243158:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.243158:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677448.243159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.243161:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:9.2:1619677448.243162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:12.0:1619677448.243164:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.243164:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.243166:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.243167:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:12.0:1619677448.243168:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.243169:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:13.0:1619677448.243169:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.243170:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:2.0:1619677448.243170:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:31.0:1619677448.243171:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:2.0:1619677448.243173:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.243174:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.243174:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.243176:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.243179:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.243251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.243258:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:20.2:1619677448.243260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:24.0:1619677448.243261:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.243262:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677448.243263:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.2:1619677448.243263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:24.0:1619677448.243264:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.243266:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:24.0:1619677448.243267:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.243268:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.243269:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.243270:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.243270:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.243271:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.243271:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677448.243272:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:24.0:1619677448.243272:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.243272:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677448.243274:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.243274:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.243275:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:20.0:1619677448.243275:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.243275:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.243277:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.243277:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.243278:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.243280:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.243281:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.243282:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.243283:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96717462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.243283:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.243284:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.243285:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.243285:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.243286:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.243287:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.243287:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.243288:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.243288:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.243288:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.243289:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:26.0:1619677448.243290:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.243290:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.243290:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677448.243291:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677448.243291:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.243292:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.243292:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:20.0:1619677448.243292:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.243293:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.243298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677448.243298:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.243299:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677448.243299:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677448.243301:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.243301:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.243302:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.243304:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.243304:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.243305:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:21.2:1619677448.243305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:26.0:1619677448.243306:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677448.243307:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:9.2:1619677448.243307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:25.2:1619677448.243309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677448.243309:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.243310:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.243310:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.243312:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.243312:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:22.0:1619677448.243314:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677448.243315:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.0:1619677448.243315:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.243315:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.243316:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677448.243317:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:18.0:1619677448.243318:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.243319:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.243319:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:12.0:1619677448.243320:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.243321:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.243322:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:12.0:1619677448.243323:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.243325:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677448.243325:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.243327:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.243345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.243352:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.243355:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.243356:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.243358:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.243360:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.243362:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.243363:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.243365:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.243366:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.243366:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.243370:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.243371:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.243375:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.243376:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.243376:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.243378:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.243378:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.243379:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.243379:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677448.243383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:25.2:1619677448.243384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677448.243388:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:6.0:1619677448.243391:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677448.243391:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.243392:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677448.243394:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:2.0:1619677448.243394:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:22.0:1619677448.243395:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.243395:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.243397:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.243398:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.243398:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677448.243400:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.243401:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:2.0:1619677448.243401:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.243403:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.243403:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677448.243407:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.243409:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677448.243411:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.243412:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677448.243413:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677448.243415:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677448.243415:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.243416:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.243417:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.243421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.243428:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.2:1619677448.243430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677448.243431:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.243432:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.243432:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.243434:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:1.0:1619677448.243436:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.243437:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677448.243439:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.243440:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.243442:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.243445:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.243446:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:25.2:1619677448.243447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000400:00000200:1.0:1619677448.243448:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.243449:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.243450:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.243451:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.243454:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:1.0:1619677448.243455:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.243457:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.243457:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.243457:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.243458:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.243459:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.243460:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677448.243460:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.243461:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.243461:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.243463:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.243464:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.243464:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.243465:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.243469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.243477:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.243480:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.243481:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.243482:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.243484:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.243485:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.243615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.243622:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.243625:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.243627:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.243629:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.243630:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.243631:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.243632:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.243634:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.243635:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.243636:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.243639:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.243641:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.243644:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.243644:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.243645:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.243647:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.243647:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.243648:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.243649:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.243652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.243658:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.243661:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.243662:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.243663:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.243665:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.243666:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.243736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.243744:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.243746:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.243747:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.243749:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.243752:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.243753:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.243754:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.243756:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.243757:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.243757:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.2:1619677448.243759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:26.0:1619677448.243760:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.243762:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.243765:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.243766:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.243766:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:8.0:1619677448.243766:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:26.0:1619677448.243768:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.243768:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677448.243768:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:26.0:1619677448.243769:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677448.243769:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:26.0:1619677448.243770:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:8.0:1619677448.243771:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.243773:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.243774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:8.0:1619677448.243775:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.243776:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96717456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.243777:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.243778:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.243779:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.243782:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:8.0:1619677448.243782:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.243783:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.243784:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.243785:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.243785:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:8.0:1619677448.243786:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.243786:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.243787:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:22.0:1619677448.243788:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.243789:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.243789:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:22.0:1619677448.243790:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677448.243790:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.243791:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.243794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.243801:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.243803:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.243804:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.243806:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.243807:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.243808:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.2:1619677448.243815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.243823:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.243826:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.243827:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.243827:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.243828:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.243828:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.243861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.243868:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.243871:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.243873:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.243875:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.243875:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.243877:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.243878:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.243879:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.243880:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.243881:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.243885:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.243886:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.243889:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.243890:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.243891:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.243892:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.243893:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.243894:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.243894:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.243898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.243908:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.243910:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.243911:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.243913:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.243913:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.243916:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.244049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.244051:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.244054:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.244056:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.244056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:20.0:1619677448.244058:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.244059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:20.0:1619677448.244059:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.244061:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.244062:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677448.244062:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.244063:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.244064:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.244065:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.244065:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:6.0:1619677448.244065:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677448.244066:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:22.0:1619677448.244068:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.244068:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:20.0:1619677448.244069:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.244069:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.244071:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.244072:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.244073:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.244073:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.244074:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.244074:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.244075:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.244076:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:18.0:1619677448.244076:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.244076:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.244077:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:18.0:1619677448.244077:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.244078:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:18.0:1619677448.244078:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.244078:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:20.0:1619677448.244079:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.244080:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000400:00000200:6.0:1619677448.244080:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96717452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.244081:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:18.0:1619677448.244082:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.244082:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.244083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:6.0:1619677448.244083:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.244084:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.244085:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.2:1619677448.244086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677448.244088:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.244088:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.244089:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:6.0:1619677448.244089:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.244091:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.244091:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677448.244091:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.244091:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:22.0:1619677448.244092:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677448.244092:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.244092:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:22.0:1619677448.244094:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677448.244094:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:22.0:1619677448.244095:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.244095:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.244095:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677448.244096:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.244096:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.244096:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:22.0:1619677448.244097:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.244097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.244098:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.244099:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.244099:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.244100:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677448.244100:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.244101:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677448.244104:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.2:1619677448.244106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:1.0:1619677448.244107:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.244108:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.244108:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.244110:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.244113:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:1.0:1619677448.244113:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.244116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.244116:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.244117:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.244118:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.244119:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.244121:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677448.244123:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.244126:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.244127:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.2:1619677448.244127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:1.0:1619677448.244127:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.244128:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.244129:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.244129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677448.244135:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.244137:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.244138:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.244139:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.244140:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.244140:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.244140:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.244141:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.244142:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:10.0:1619677448.244143:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.244145:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.244147:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.244148:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.244150:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.244151:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.244152:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.244155:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.244157:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.244160:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.244160:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.244161:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.244163:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.244164:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.244164:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.244165:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.244170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.244177:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.244180:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.244181:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.244181:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.244184:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.244187:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.244256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.244261:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.244264:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.244265:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.244267:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.244269:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.244270:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.244272:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.244273:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.244274:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.244275:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.244278:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.244280:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.244283:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.244283:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.244284:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.244286:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.244286:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.244287:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.244287:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.244292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.244300:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.244302:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.244303:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.244304:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.244306:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.244309:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.244474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.244480:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.244483:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.244484:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.244486:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.244489:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.244490:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.244492:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.244493:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.244494:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.244495:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.244498:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.244500:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.244503:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.244503:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.244504:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.244506:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.244506:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.244507:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.244507:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.244513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.244521:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.244524:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.244525:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.244527:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.244528:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.244530:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.244530:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.244531:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.244531:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.244531:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.244532:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.244602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.244605:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.244607:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.244610:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.244611:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.244612:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.244614:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.244615:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.244616:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.244617:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.244618:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.244622:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.244623:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.244626:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.244627:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.244628:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.244630:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.244630:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.244631:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.244631:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.244636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.244644:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.244646:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.244647:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.244649:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:31.0:1619677448.244651:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.244653:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.244658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677448.244664:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.244667:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.244668:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.244669:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.244669:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.244671:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.244692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.244699:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.244702:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.244704:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.244706:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.244706:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.244708:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.244709:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.244710:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.244711:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.244712:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.244715:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.244716:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.244720:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.244721:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.244721:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.244723:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.244724:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.244725:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.244727:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.244727:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677448.244729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.244733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.244737:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:31.0:1619677448.244740:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:2.0:1619677448.244740:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.244741:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.244741:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.244742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:31.0:1619677448.244743:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:2.0:1619677448.244743:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.244744:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.244745:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677448.244746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:2.0:1619677448.244747:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677448.244749:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.244751:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.244751:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:24.0:1619677448.244754:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:2.0:1619677448.244754:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.244756:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:2.0:1619677448.244756:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.244757:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677448.244758:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.244760:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.244761:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.244761:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.244763:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.244763:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.244764:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.244766:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.244767:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96717445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.244768:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.244769:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.244770:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.244770:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.244771:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.244772:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.244773:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.244774:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:24.0:1619677448.244775:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.244775:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.244776:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.244776:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.244777:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.244777:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:24.0:1619677448.244779:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.244780:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.244781:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:9.2:1619677448.244782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.244783:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.244783:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.244784:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.244784:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.244788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.244789:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.244791:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.244792:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.244793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677448.244793:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.244794:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.244796:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677448.244796:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677448.244798:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677448.244799:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.244800:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.244800:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.244801:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:17.0:1619677448.244802:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.244804:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.244804:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.244805:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:5.0:1619677448.244806:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.244808:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.244809:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.244811:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.244812:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.244813:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.244816:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.244818:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.244821:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.244822:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.244823:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.244825:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.244826:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.244826:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.244827:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.244831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.244838:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.244841:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.244841:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.244843:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.244844:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.244847:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.244849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.244855:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.244858:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.244859:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.244859:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.244861:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.244862:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.245082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.245088:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.245091:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.245091:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.245093:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.245096:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.245097:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.245099:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.245100:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.245101:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.245102:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.245105:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.245107:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.245110:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.245110:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.245111:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.245113:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.245113:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.245114:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.245115:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.245119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.245125:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.245128:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.245129:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.245131:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.245132:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.245133:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.245347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.245356:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.245358:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.245359:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.245361:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.245364:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.245365:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.245366:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.245368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:10.0:1619677448.245368:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.245369:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.245370:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.245373:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.245375:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.245376:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.245378:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.245379:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:10.0:1619677448.245379:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.245380:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.245381:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:31.0:1619677448.245382:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.245382:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.245383:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.245383:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:31.0:1619677448.245384:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.245384:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.245386:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.245387:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.245388:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677448.245388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677448.245389:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.245390:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.245393:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.245395:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.245397:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.245397:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677448.245398:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.245399:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.245400:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.245400:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:31.0:1619677448.245401:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677448.245401:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.245401:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.245402:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.245402:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:2.0:1619677448.245403:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.245406:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.245407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.2:1619677448.245408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.245414:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.245415:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677448.245417:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.245418:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:2.0:1619677448.245418:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:20.0:1619677448.245419:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.245419:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.245420:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.245420:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:20.0:1619677448.245421:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.245422:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.245425:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.245425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.245433:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.245436:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.245436:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.245439:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.245441:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.245443:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.245444:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.245446:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.245447:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.245448:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.245451:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.245453:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.245455:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.245456:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.245456:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.245458:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.245458:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.245459:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.245460:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.245464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.245472:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.245475:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.245476:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.245477:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.245479:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.245482:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.245483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.245490:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.245493:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.245493:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.245494:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.245495:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.245495:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.245504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.245510:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.245512:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.245513:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.245515:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.245517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:17.0:1619677448.245518:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.245520:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.245521:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.245522:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.245522:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:17.0:1619677448.245523:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.245524:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.245525:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.245526:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677448.245529:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.245529:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.245530:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.245531:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.245533:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:17.0:1619677448.245534:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.245534:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.245535:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.245536:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:2.0:1619677448.245536:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.245537:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:2.0:1619677448.245537:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.245538:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677448.245538:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.245539:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.245540:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.245541:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:2.0:1619677448.245541:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.245543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677448.245543:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.2:1619677448.245546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.245546:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.245547:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.245548:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.245550:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.245550:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677448.245551:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677448.245551:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:1.0:1619677448.245551:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:2.0:1619677448.245552:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.245553:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677448.245553:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:12.0:1619677448.245554:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677448.245554:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.245555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:12.0:1619677448.245555:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.245555:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677448.245557:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.245558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:13.0:1619677448.245559:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.245560:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677448.245561:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.245564:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:13.0:1619677448.245564:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.245565:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.245566:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:13.0:1619677448.245566:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.245567:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.245568:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677448.245568:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.245569:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.245583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.245589:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.245592:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.245594:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.245594:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.245597:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.245599:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.245600:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.245601:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.245602:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.245603:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.245607:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.245609:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.245612:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.245613:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.245614:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.245615:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.245616:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.245617:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.245617:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.245621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.245627:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:4.2:1619677448.245628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:12.0:1619677448.245630:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.245631:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.245632:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.245634:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.245635:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677448.245635:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.245638:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.245638:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677448.245640:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.245643:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.245645:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.245646:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.245647:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.245648:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.245649:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.245652:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.245654:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.245657:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.245657:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.245658:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.245660:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.245660:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.245661:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.245661:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.245666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.245672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:17.0:1619677448.245673:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.245676:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.245677:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.245677:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677448.245677:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.245679:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.245680:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.245681:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677448.245682:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:2.0:1619677448.245683:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.245686:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.245688:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.245689:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.245691:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.245692:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.245693:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.245696:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.245697:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.245700:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.245700:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.245701:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.245703:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.245703:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.245704:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.245705:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.245709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.245716:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.245719:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.245720:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.245720:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.245722:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.245724:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.245825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.2:1619677448.245831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:1.0:1619677448.245831:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.245833:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.0:1619677448.245834:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677448.245834:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.245836:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:1.0:1619677448.245837:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.245839:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.245839:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:1.0:1619677448.245840:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.245841:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:20.0:1619677448.245842:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.245843:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.245844:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677448.245844:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.245845:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.245846:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.245847:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.245847:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:20.0:1619677448.245848:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.245849:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677448.245852:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.245854:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677448.245855:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.0:1619677448.245857:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677448.245857:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.245858:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.245859:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.0:1619677448.245860:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.245860:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:20.0:1619677448.245861:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.245861:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:1.0:1619677448.245861:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.245862:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.245863:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:1.0:1619677448.245863:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677448.245864:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.245865:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.245866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.0:1619677448.245866:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.245870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:6.0:1619677448.245874:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.245876:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.245877:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.245877:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:6.0:1619677448.245878:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.245879:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.245880:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:6.0:1619677448.245880:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.245881:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.245882:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:10.0:1619677448.245883:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:10.0:1619677448.245887:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.245889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.245895:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.245898:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.245899:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.245899:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.245900:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.245900:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.245989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.245996:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.245998:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.245999:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.246001:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.246004:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.246005:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.246006:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.246008:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.246009:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.246010:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.246013:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.246015:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.246017:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.246018:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.246018:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.246019:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.246020:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.246021:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.246022:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:21.2:1619677448.246023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:10.0:1619677448.246023:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.246025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:4.2:1619677448.246028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.246029:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.246031:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.246032:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677448.246033:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.246034:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:22.0:1619677448.246034:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.246035:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.246037:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.246037:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.246037:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.246038:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.246039:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:22.0:1619677448.246039:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:10.0:1619677448.246039:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.246040:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.246040:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.246040:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:22.0:1619677448.246041:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.246042:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.246043:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.246046:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.246048:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.246051:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.246052:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.246052:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.246054:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.246055:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.246056:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.246057:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.246058:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.246060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.246063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.246067:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677448.246069:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:17.0:1619677448.246070:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:6.0:1619677448.246070:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:17.0:1619677448.246071:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.246071:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.246072:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.246073:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:17.0:1619677448.246074:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.246076:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.246141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.246147:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.246150:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.246152:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.246152:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.246155:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.246156:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.246158:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.246159:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.246160:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.246161:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.246164:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.246166:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.246169:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.246170:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.246171:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.246172:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.246173:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.246174:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.246174:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.246178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.246185:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.246188:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.246189:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.246190:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.246192:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.246193:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.246254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.246260:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.246263:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.246264:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.246266:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.246268:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.246269:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.246270:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.246271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:1.0:1619677448.246271:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.246272:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.246273:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.246276:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.246278:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.246278:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.246280:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.246281:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:1.0:1619677448.246281:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.246282:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.246283:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:30.0:1619677448.246284:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.246284:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.246284:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.246285:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677448.246285:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677448.246287:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.246288:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.246289:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.246290:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.246290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:30.0:1619677448.246291:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.246292:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.246300:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:30.0:1619677448.246301:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.246302:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.246302:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.246303:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.246303:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.246305:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.246306:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.246306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677448.246307:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.246307:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.246307:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:30.0:1619677448.246309:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.246310:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.246311:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.246311:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.246315:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:25.2:1619677448.246317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.246317:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.246318:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.246320:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.246322:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.246323:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.246324:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.246325:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:10.0:1619677448.246326:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.246327:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.246327:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:6.0:1619677448.246327:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.246328:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.246328:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.246331:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.246331:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.246334:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677448.246334:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677448.246335:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.246336:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.246337:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.246338:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.246338:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.246339:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.246340:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.246341:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.246343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.2:1619677448.246346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.246352:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.246353:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.246354:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.246355:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.246355:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.246356:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.246357:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.246357:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.246358:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.246358:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.246381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.246387:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.246390:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.246391:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.246394:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.246396:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.246397:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.246399:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717426] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.246400:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.246401:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.246402:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.246405:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.246407:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.246409:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.246410:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.246410:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.246412:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.246412:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.246413:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.246413:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.246418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.246425:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.246427:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.246428:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.246429:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.246430:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.246431:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.246522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.246530:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.246533:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.246534:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.246536:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.246538:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.246539:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.246540:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.246541:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.246543:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.246544:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.246547:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.246548:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.246551:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.246552:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.246553:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.246555:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.246555:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.246556:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.246556:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.246561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.246569:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.246572:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.246573:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.246573:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.246576:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.246578:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.246596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:5.2:1619677448.246597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:6.0:1619677448.246601:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.246603:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.246604:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.246605:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677448.246606:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:6.0:1619677448.246607:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.246607:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.246609:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.246609:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.246611:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677448.246612:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.246612:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.246613:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677448.246614:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.246615:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.246616:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.246616:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.246617:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.246617:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677448.246619:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677448.246621:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.246623:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.246624:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677448.246626:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677448.246627:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.246628:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.246628:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:6.0:1619677448.246628:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:17.0:1619677448.246629:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677448.246629:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:17.0:1619677448.246631:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.246631:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.246631:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:17.0:1619677448.246632:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.246632:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.246633:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.246633:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.246633:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.246636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:8.2:1619677448.246638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.246643:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.246645:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:1.0:1619677448.246645:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:26.0:1619677448.246646:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.246646:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:1.0:1619677448.246648:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:26.0:1619677448.246649:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.246649:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.246649:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.246651:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.246652:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:1.0:1619677448.246653:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.246805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.246812:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.246815:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.246816:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.246818:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.246821:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.246822:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.246823:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.246825:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.246826:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.246827:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.246830:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.246832:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.246835:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.246836:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.246837:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.246838:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.246839:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.246840:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.246840:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.246844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.246852:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.246855:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.246856:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.246856:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.246859:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.246861:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.246863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.246870:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.246873:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.246874:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.246874:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.246875:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.246875:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.246881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.246888:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.246891:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.246892:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.246893:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.246896:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.246897:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.246899:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.246900:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.246901:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.246902:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:20.2:1619677448.246904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:10.0:1619677448.246906:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.246907:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.246910:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.246911:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.246912:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.246912:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.246913:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.246914:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.246914:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:31.0:1619677448.246915:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:10.0:1619677448.246915:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:31.0:1619677448.246918:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.246919:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.246919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677448.246920:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.246922:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.246923:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.246924:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.246925:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.246926:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:20.0:1619677448.246927:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.246929:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:31.0:1619677448.246930:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.246930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:20.0:1619677448.246930:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.246932:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:20.0:1619677448.246932:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:31.0:1619677448.246934:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.246934:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.246935:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677448.246935:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677448.246936:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.246937:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:8.0:1619677448.246937:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:31.0:1619677448.246938:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.246939:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.246939:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:24.0:1619677448.246940:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.246940:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677448.246942:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.246943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:8.0:1619677448.246946:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.246947:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.246948:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96717419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.246949:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:8.0:1619677448.246950:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.246951:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.246952:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:8.0:1619677448.246952:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.246953:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.246955:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.246955:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.246956:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.246957:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:8.0:1619677448.246957:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.246960:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.246961:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.246962:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.246964:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.246964:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.246965:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.246965:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.246970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.246978:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.246980:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.246981:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.246983:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.246983:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.246986:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.247037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.247044:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.247047:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.247048:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.247049:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.247053:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.247054:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.247056:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.247057:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.247058:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.247059:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.247062:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.247064:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.247066:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.247067:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.247068:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.247070:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.247070:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.2:1619677448.247070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:26.0:1619677448.247071:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.247071:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.247075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677448.247076:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.247079:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.247079:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.247081:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.247083:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:22.0:1619677448.247084:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.247085:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:22.0:1619677448.247086:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:20.0:1619677448.247086:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.247087:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.247087:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.247088:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.247089:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.247089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.0:1619677448.247089:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:22.0:1619677448.247090:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:20.0:1619677448.247092:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.247093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000400:00000200:22.0:1619677448.247094:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.247096:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:31.0:1619677448.247097:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677448.247098:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.247099:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677448.247099:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.247100:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:31.0:1619677448.247101:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.247101:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:5.0:1619677448.247101:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677448.247102:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677448.247103:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677448.247103:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.247103:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677448.247103:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.247104:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.247104:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000400:00000200:31.0:1619677448.247105:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:5.0:1619677448.247105:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.247106:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:31.0:1619677448.247107:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717416] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.247107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:20.0:1619677448.247107:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:31.0:1619677448.247108:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.247109:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.247110:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.247113:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.247114:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:31.0:1619677448.247115:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.247116:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.247117:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.247117:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.247118:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.247119:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.247120:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:10.0:1619677448.247120:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.247122:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.247122:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.247123:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.247123:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.247123:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:25.2:1619677448.247128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.247135:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.247138:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.247139:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.247139:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.247142:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.247145:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.247146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.247153:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.247155:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.247156:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.247157:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.247158:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.247158:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.247231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.247236:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677448.247237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:17.0:1619677448.247239:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.247240:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.247242:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.247243:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:17.0:1619677448.247243:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.247245:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:18.0:1619677448.247246:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:17.0:1619677448.247246:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.247247:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677448.247248:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.247249:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.247250:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.247250:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:18.0:1619677448.247251:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.247253:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.247254:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.247255:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.247256:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.247257:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.247257:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:18.0:1619677448.247258:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.247261:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.247262:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.247262:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.247263:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:18.0:1619677448.247264:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.247264:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.247265:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.247266:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:18.0:1619677448.247267:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.247267:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:18.0:1619677448.247268:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.247268:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677448.247268:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.247270:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.247270:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:9.2:1619677448.247270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:18.0:1619677448.247271:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.247272:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.2:1619677448.247273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.2:1619677448.247276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:24.0:1619677448.247278:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:1.0:1619677448.247278:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.247280:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:24.0:1619677448.247281:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677448.247281:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.247282:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677448.247282:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.247283:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.247283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000400:00000200:24.0:1619677448.247285:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.247288:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.0:1619677448.247288:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:1.0:1619677448.247289:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:5.0:1619677448.247291:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.247291:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.247292:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677448.247292:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:20.0:1619677448.247294:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.247299:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.247300:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.247301:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:1.0:1619677448.247303:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.247305:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.247306:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.247308:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.247309:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.247310:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.247314:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677448.247316:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.247318:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.247319:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.247320:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.247322:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.247322:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677448.247323:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677448.247324:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:9.2:1619677448.247328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.247335:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.247338:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.247339:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.247339:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.247341:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677448.247344:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.247349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677448.247354:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.247357:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.247358:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.247358:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.247359:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.247360:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.247444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.247453:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.247455:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.247458:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.247460:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.247460:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.247462:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.247463:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.247464:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.247465:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.247466:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.247469:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.247471:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.247473:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.247474:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.247475:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.247476:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.247477:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.247478:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.247478:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.247482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.247490:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.247492:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.247493:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.247494:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.247497:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.247498:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.247504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.247510:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.247513:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.247513:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.247514:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.247515:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.247515:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.247591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.247599:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.247602:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.247602:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.247604:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.247607:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.247609:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.247610:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717411] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.247611:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.247612:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.247613:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.247616:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.247617:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.247620:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.247621:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.247622:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.247622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:26.0:1619677448.247623:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.247624:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.247625:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.247625:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.247628:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.247631:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.247631:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.247634:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.247636:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.247638:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.247639:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.247640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:6.0:1619677448.247640:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.247641:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.247642:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.247647:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.247649:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.247649:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.247650:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.247651:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.247651:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.247652:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:22.0:1619677448.247653:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.247653:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.247655:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.247656:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.247656:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.247657:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.247657:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677448.247658:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677448.247661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.247669:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.247671:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.247672:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.247673:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.247674:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.247677:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.247693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.2:1619677448.247697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677448.247699:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.247701:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.247702:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.247704:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.247705:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.247707:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.247708:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:17.0:1619677448.247709:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.247710:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.247711:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.247712:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:17.0:1619677448.247712:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.247713:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.247713:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.247714:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.247715:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.247716:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.247718:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.247718:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.247719:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.247720:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.247721:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.247723:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.247724:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.247725:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.247725:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.247726:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.247727:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.247728:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.247728:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.247728:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.247729:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:17.0:1619677448.247729:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.247730:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.247731:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.247732:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.247733:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:25.2:1619677448.247733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.247734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:31.0:1619677448.247735:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.247735:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677448.247737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:8.2:1619677448.247738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:12.0:1619677448.247741:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:10.0:1619677448.247741:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:20.2:1619677448.247744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.247744:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.247744:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.247744:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:6.0:1619677448.247744:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:26.0:1619677448.247745:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:6.0:1619677448.247746:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677448.247747:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:12.0:1619677448.247747:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.247748:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.247749:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:10.0:1619677448.247749:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.247750:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.247752:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.247752:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:10.0:1619677448.247752:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.247753:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.247754:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:30.0:1619677448.247755:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:20.0:1619677448.247755:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.247755:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677448.247755:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:30.0:1619677448.247756:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.247757:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:6.0:1619677448.247757:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.247757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000400:00000200:20.0:1619677448.247758:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.247758:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.247759:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.247760:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.247760:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.247761:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:10.0:1619677448.247761:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.247762:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:20.0:1619677448.247763:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:10.0:1619677448.247763:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:8.0:1619677448.247764:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677448.247767:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677448.247767:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000400:00000200:6.0:1619677448.247767:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.247769:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.247771:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.247771:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.247772:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:8.0:1619677448.247774:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.247774:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.247775:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.247776:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.247776:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000400:00000200:8.0:1619677448.247776:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:6.0:1619677448.247776:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.247777:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:8.0:1619677448.247777:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96717405] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.247777:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:8.0:1619677448.247778:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.247778:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:10.0:1619677448.247779:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000400:00000200:8.0:1619677448.247779:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.247779:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:10.0:1619677448.247780:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000400:00000200:8.0:1619677448.247780:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.247780:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:10.0:1619677448.247781:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677448.247781:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:8.2:1619677448.247783:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.2:1619677448.247787:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:1.0:1619677448.247791:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:8.0:1619677448.247793:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.247794:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.247795:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.247795:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.247795:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.247796:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:1.0:1619677448.247797:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.247798:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.247798:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:8.0:1619677448.247799:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677448.247799:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.247800:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677448.247800:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.0:1619677448.247801:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:8.0:1619677448.247801:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677448.247801:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677448.247803:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed9d3400] (22)++
00000400:00000200:5.0:1619677448.247804:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.247805:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.247806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.2:1619677448.247806:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:26.0:1619677448.247808:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677448.247808:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (23)--
00000800:00000200:8.0:1619677448.247809:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:26.0:1619677448.247810:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:5.2:1619677448.247810:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (21)++
00000800:00000200:10.0:1619677448.247811:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:13.0:1619677448.247812:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:26.0:1619677448.247813:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.247813:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.247814:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:13.0:1619677448.247815:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:26.0:1619677448.247816:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.247816:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.247817:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)++
00000800:00000200:20.0:1619677448.247817:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677448.247818:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:20.0:1619677448.247818:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.247818:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:26.0:1619677448.247819:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.247819:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed9d3400] (23)--
00000800:00000200:20.0:1619677448.247819:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000400:00000200:13.0:1619677448.247819:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.247819:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677448.247820:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:20.0:1619677448.247820:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:26.0:1619677448.247821:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.247821:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677448.247822:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.247822:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:24.0:1619677448.247822:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000400:00000200:26.0:1619677448.247826:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.247827:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.247830:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.247830:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.247831:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.247832:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.247833:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.247833:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.247834:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.247838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.247845:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.247847:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.247848:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.247849:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.247851:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.247852:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.247856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.247862:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.247865:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.247866:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.247867:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.247867:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.247868:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.247906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.247913:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.247915:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.247917:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.247919:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.247920:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.247921:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.247922:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.247924:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.247924:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.247926:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.247929:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.247930:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.247932:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.247933:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.247934:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.247935:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.247936:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.247937:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.247939:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.247939:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.247940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.247945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.247947:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.247949:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.247950:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.247952:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.247953:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.247953:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.247954:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.247955:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.247955:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.247956:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677448.248007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.248015:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.248017:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.248018:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.248020:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.248022:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.248024:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.248025:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.248026:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.248027:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.248028:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.248031:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.248033:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.248036:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.248036:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.248037:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.248039:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.248039:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.248040:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677448.248042:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.248042:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677448.248045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.248046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:20.2:1619677448.248048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:17.0:1619677448.248052:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:13.0:1619677448.248053:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:4.2:1619677448.248054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677448.248055:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:13.0:1619677448.248056:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:1.0:1619677448.248056:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.248057:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.248057:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.248058:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:17.0:1619677448.248059:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:12.0:1619677448.248059:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:26.0:1619677448.248060:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.248060:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:13.0:1619677448.248060:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.248061:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.248062:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:13.0:1619677448.248062:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:10.0:1619677448.248062:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:26.0:1619677448.248063:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:13.0:1619677448.248064:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.248064:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:13.0:1619677448.248065:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.248067:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.248067:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677448.248068:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.248070:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.248071:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:13.0:1619677448.248072:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.248073:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.248074:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.248074:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.248075:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.248076:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:13.0:1619677448.248077:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.248078:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677448.248078:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677448.248079:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.248080:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.248081:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:13.0:1619677448.248081:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677448.248081:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677448.248082:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:26.0:1619677448.248084:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.248085:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.248085:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:9.2:1619677448.248086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.248087:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.248087:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.248088:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.248090:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.248090:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.248093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.248094:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:4.2:1619677448.248096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.248097:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.248098:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.248098:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.248100:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.248100:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:10.0:1619677448.248102:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:18.0:1619677448.248103:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677448.248103:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.248104:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.248105:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:5.0:1619677448.248105:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.248106:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.248106:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.248107:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.248108:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.248172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.248180:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.248183:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.248185:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.248186:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.248187:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.248188:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.248190:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.248191:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.248192:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.248193:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.248196:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.248197:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.248199:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.248200:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.248201:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.248202:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.248203:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.248204:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.248204:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.248208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.248214:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.248216:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.248217:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.248218:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.248220:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.248222:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.248231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:13.0:1619677448.248237:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.248240:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.248240:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.248241:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.248242:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.248243:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.248291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.248300:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.2:1619677448.248301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:31.0:1619677448.248303:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.248305:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.248308:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.248308:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677448.248308:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:31.0:1619677448.248309:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.248310:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.248311:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.248311:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.248312:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:31.0:1619677448.248313:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.248315:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.248315:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.248317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:2.0:1619677448.248317:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.248319:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.248320:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.248320:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:2.0:1619677448.248321:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.248322:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.248323:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:2.0:1619677448.248323:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.248324:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.248324:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.248326:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.248326:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677448.248327:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.248327:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.248328:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.248329:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677448.248329:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.248329:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.248330:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677448.248330:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:31.0:1619677448.248331:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.248331:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677448.248331:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.248332:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96717396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.248333:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.248334:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.248334:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.248335:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.248335:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:25.2:1619677448.248336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.248336:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.248337:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:20.0:1619677448.248339:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.248340:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.0:1619677448.248341:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.248341:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.248342:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:12.0:1619677448.248343:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:2.0:1619677448.248343:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.0:1619677448.248344:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.248344:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.248345:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677448.248346:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:20.0:1619677448.248347:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:12.0:1619677448.248347:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.248348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.0:1619677448.248348:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.248349:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:12.0:1619677448.248349:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.248350:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.248351:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677448.248351:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.248352:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.248352:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677448.248353:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.248353:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:9.2:1619677448.248353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:13.0:1619677448.248355:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.248356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:25.2:1619677448.248357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.248359:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:13.0:1619677448.248360:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.248361:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:13.0:1619677448.248361:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.248362:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677448.248362:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:2.0:1619677448.248362:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677448.248363:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:13.0:1619677448.248364:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.248364:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.248364:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.248364:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:18.0:1619677448.248365:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:13.0:1619677448.248365:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.248365:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:18.0:1619677448.248366:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.248366:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.248366:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.248367:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677448.248368:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.248372:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.248377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.248384:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.248387:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.248387:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.248389:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.248392:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.248394:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.248395:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.248396:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.248397:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.248398:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.248402:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.248403:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.248405:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.248406:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.248407:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.248408:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.248408:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.248409:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.248410:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.248415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.248422:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.248424:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.248424:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.248425:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.248426:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.248428:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.248431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.248439:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.248441:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.248442:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.248444:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.248447:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.248448:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.248449:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.248451:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.248452:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.248453:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.248456:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.248458:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.248460:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.248461:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.248462:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.248463:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.248464:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.248464:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.248465:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.248469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.248475:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.248478:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.248478:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.248480:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.248482:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.248483:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.248698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.248706:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.248708:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.248710:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.248712:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.248713:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.248714:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.248715:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.248717:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.248718:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.248718:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.248722:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.248723:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.248726:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.248727:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.248728:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.248729:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.248730:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.248731:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.248731:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.248735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.248743:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.248746:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.248747:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.248749:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.248749:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.248752:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.248873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.248881:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.248884:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.248886:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.248888:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.248888:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.248890:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.248891:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.248893:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.248894:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.248895:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:5.2:1619677448.248897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:31.0:1619677448.248898:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.248899:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.248902:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.248903:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.248903:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:26.0:1619677448.248904:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.248905:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.248906:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.248906:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:31.0:1619677448.248907:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.248907:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.248907:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.248909:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.248911:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.248912:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:9.2:1619677448.248912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:26.0:1619677448.248913:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.248915:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.248915:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.248916:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.248919:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.248919:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:26.0:1619677448.248920:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.248922:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.248922:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:26.0:1619677448.248923:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:2.0:1619677448.248923:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.248924:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:18.0:1619677448.248924:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:26.0:1619677448.248925:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.248925:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:2.0:1619677448.248925:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.248926:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.248927:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677448.248928:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:25.2:1619677448.248931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:9.2:1619677448.248932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.248938:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.248940:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:18.0:1619677448.248940:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000400:00000200:30.0:1619677448.248942:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.248943:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.248944:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677448.248944:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.248944:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.248944:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.248945:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.248946:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:30.0:1619677448.248947:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.248991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.248996:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.248999:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.249000:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.249002:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.249005:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.249006:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.249008:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.249009:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.249010:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.249011:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.2:1619677448.249012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:2.0:1619677448.249014:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.249016:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677448.249019:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:2.0:1619677448.249019:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.249019:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.249020:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677448.249021:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.249022:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:2.0:1619677448.249022:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.249022:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.249023:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:8.0:1619677448.249024:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.249024:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:8.0:1619677448.249027:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.249028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:8.0:1619677448.249029:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.249029:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96717389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.249031:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.249032:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.249033:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.249035:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.249036:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677448.249037:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677448.249039:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.249040:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.249040:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.249040:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677448.249041:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:22.0:1619677448.249042:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.249042:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.249044:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.249044:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.249045:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.249045:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677448.249046:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677448.249050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.249058:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.249060:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.249061:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.249062:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.249064:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.249067:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.249076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.249083:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.249085:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.249087:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.249087:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.249088:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.249088:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.249096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.249101:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.249104:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.249105:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.249107:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.249108:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.249110:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.249111:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.249113:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.249114:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.249115:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.249118:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.249120:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.249123:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.249123:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.249124:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.249126:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.249127:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.249127:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.249128:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.249132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.249139:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.249142:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.249143:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.249144:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.249145:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.249148:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.249381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.249389:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.249392:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.249394:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.249396:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.249397:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.249399:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.249400:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.249402:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.249403:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.249404:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.249407:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.249409:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.249413:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.249414:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.249414:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:4.2:1619677448.249414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.249416:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.249417:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.249418:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.249418:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:10.0:1619677448.249421:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677448.249423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.249424:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.249425:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.249426:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.249428:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.249430:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:20.0:1619677448.249431:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:10.0:1619677448.249431:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.249432:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.249433:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.249434:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:10.0:1619677448.249434:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.249436:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.249437:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.249437:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.249438:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:10.0:1619677448.249438:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.249439:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.249440:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:20.0:1619677448.249441:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.249441:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:10.0:1619677448.249442:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.249443:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.249443:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:5.0:1619677448.249444:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.249445:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:10.0:1619677448.249445:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.249445:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.249446:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:10.0:1619677448.249446:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.249446:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677448.249447:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.249447:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.249448:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.249450:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.249451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:5.0:1619677448.249451:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.249452:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.249453:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.249457:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:20.0:1619677448.249457:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677448.249458:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.249459:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.249460:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:5.0:1619677448.249460:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.249461:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.249461:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677448.249463:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.249463:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96717384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.249464:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:20.0:1619677448.249464:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.249466:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:20.0:1619677448.249466:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.249466:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.249470:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.249470:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:20.0:1619677448.249471:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.249471:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.249473:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.249473:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.249474:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.249474:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.249475:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.249476:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.249476:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677448.249477:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.249477:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.249477:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.249479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.249479:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.249480:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.249481:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.249483:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.249483:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.249485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.249487:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.2:1619677448.249488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:6.0:1619677448.249490:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.249491:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.249491:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.249492:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:6.0:1619677448.249493:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.249494:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:30.0:1619677448.249495:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:6.0:1619677448.249495:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:30.0:1619677448.249496:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.249496:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:30.0:1619677448.249497:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.249497:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.249497:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677448.249498:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.249501:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.249529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.249535:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.249538:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.249539:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.249541:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.249543:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.249545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:18.0:1619677448.249545:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.249546:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.249548:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.249549:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.249550:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.249551:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.249553:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:18.0:1619677448.249555:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.249556:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.249557:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.249558:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.249560:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.249560:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:18.0:1619677448.249561:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:2.0:1619677448.249561:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.249562:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677448.249563:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.249563:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.249564:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:2.0:1619677448.249565:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.249566:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.249567:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.249567:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677448.249568:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.249569:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.249570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.249571:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.249574:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.249575:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.249575:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.249577:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.249578:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677448.249578:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.249578:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.249578:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677448.249580:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.249581:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.249581:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:25.2:1619677448.249583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:17.0:1619677448.249584:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.249586:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:13.0:1619677448.249591:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.249594:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.249594:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.249596:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.249598:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.249599:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.249827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.249834:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.249837:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.249837:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.249839:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.249842:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.249843:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.249844:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.249846:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.249847:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.249848:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.249851:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.249853:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.249855:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.249856:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.249857:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.249858:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.249858:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.249859:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.249860:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.249864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.249870:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.249873:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.249873:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.249875:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.249877:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.249878:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.249883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677448.249889:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.249892:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.249893:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.249893:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.249894:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.249895:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.250050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.250056:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.250059:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.250060:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.250062:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.250065:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.250067:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.250068:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.250069:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.250071:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.250071:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.250075:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.250076:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.250079:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.250080:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.250080:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.250082:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.250082:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.250083:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.250084:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.250085:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.2:1619677448.250087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:21.2:1619677448.250090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:12.0:1619677448.250094:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.250096:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.250097:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.250097:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:12.0:1619677448.250098:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.250099:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:13.0:1619677448.250100:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.250100:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677448.250101:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677448.250102:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.2:1619677448.250130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.250137:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.250140:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.250141:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.250143:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.250144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:1.0:1619677448.250145:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.250147:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.250148:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.250149:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.250150:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:1.0:1619677448.250150:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.250151:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.250154:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:1.0:1619677448.250154:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.250155:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677448.250157:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:22.0:1619677448.250158:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.250160:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.250160:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.250161:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:22.0:1619677448.250162:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:1.0:1619677448.250162:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:22.0:1619677448.250163:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.250163:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:22.0:1619677448.250164:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.250164:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677448.250165:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.250165:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:22.0:1619677448.250166:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.250166:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.250167:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.250169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:22.0:1619677448.250169:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.250171:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.2:1619677448.250172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.250174:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.250174:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.250175:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.250175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:13.0:1619677448.250176:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:22.0:1619677448.250177:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.250177:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.250177:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677448.250178:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.250179:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:17.0:1619677448.250179:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:13.0:1619677448.250179:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:22.0:1619677448.250180:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677448.250180:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677448.250180:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.250181:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677448.250181:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:13.0:1619677448.250182:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.250183:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.2:1619677448.250183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.250183:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.2:1619677448.250185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677448.250185:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677448.250187:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.250187:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677448.250189:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.250190:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.250190:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.250191:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.250191:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:5.0:1619677448.250192:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677448.250193:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.250193:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:5.0:1619677448.250193:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.250194:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677448.250194:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:5.0:1619677448.250194:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.250195:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.250196:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:20.0:1619677448.250199:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.250200:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:5.0:1619677448.250200:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:8.2:1619677448.250202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.250202:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677448.250205:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.250206:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677448.250207:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.250209:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.250209:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.250209:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.250210:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.250211:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.250211:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677448.250212:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.250212:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.250212:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.250213:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:25.2:1619677448.250225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.250231:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.250234:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.250235:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.250237:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677448.250238:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.250240:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.250243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.250250:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.250254:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.250255:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.250256:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.250259:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.250260:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.250261:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.250263:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.250263:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.250265:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.250268:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.250269:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.250272:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.250273:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.250273:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.250275:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.250275:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.250276:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.250276:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.250280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.250287:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.250290:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.250290:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.250292:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.250294:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.250299:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.250360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.250368:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.250371:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.250372:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.250373:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.250375:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.250377:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.250378:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.250379:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.250380:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.250381:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.250384:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.250385:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.250388:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.250389:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.250389:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.250391:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.250392:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.250393:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.250394:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.250394:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.250397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.250400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.250406:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677448.250407:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.250408:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.250409:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.250409:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.250410:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677448.250410:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.250411:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677448.250412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:1.0:1619677448.250412:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.250415:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677448.250417:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.250420:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.250422:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.250423:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.250425:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.250426:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.250428:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.250429:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.250430:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.250431:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.250434:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.250436:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.250439:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.250440:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.250441:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.250443:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.250443:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.250444:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.250444:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.250449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.250457:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.250459:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.250460:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.250462:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.250464:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.250465:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.250528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.250534:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.250537:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.250537:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.250539:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.250542:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.250543:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.250544:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.250546:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.250547:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.250548:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.250551:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.250553:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.250556:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.250556:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.250557:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.250559:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.250559:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.250560:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.250561:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.250565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.250572:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.250574:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.250575:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.250577:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.250578:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.250580:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.250643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.250650:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.250653:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.250654:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.250656:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.250658:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.250660:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.250661:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.250662:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.250664:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.250665:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.250668:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.250670:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.250673:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.250674:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.250674:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.250674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:26.0:1619677448.250676:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.250677:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.250678:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.250679:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.250680:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677448.250681:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:25.2:1619677448.250682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677448.250685:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:4.2:1619677448.250685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.250685:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.250687:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.250690:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.250690:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:18.0:1619677448.250692:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:10.0:1619677448.250692:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.250692:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.250693:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:18.0:1619677448.250693:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.250694:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677448.250694:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.250694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:18.0:1619677448.250695:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.250696:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.250696:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.250696:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.250697:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:18.0:1619677448.250697:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.250697:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.250699:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:26.0:1619677448.250701:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:18.0:1619677448.250701:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.250701:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:18.0:1619677448.250703:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:10.0:1619677448.250704:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.250706:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677448.250707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677448.250707:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.250707:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:10.0:1619677448.250707:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.250708:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:18.0:1619677448.250709:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:10.0:1619677448.250709:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.250710:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.250711:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:10.0:1619677448.250711:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.250711:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.250712:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.250713:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:10.0:1619677448.250713:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:22.0:1619677448.250715:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:8.2:1619677448.250715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:10.0:1619677448.250716:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.250717:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.250718:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677448.250718:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:5.2:1619677448.250718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:22.0:1619677448.250719:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.250719:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.250720:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:10.0:1619677448.250720:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.250721:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.250721:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.250723:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.250723:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.250723:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:1.0:1619677448.250723:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:10.0:1619677448.250724:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.250725:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.250726:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:2.0:1619677448.250726:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.250726:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677448.250726:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:2.0:1619677448.250727:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:1.0:1619677448.250727:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.250728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:17.0:1619677448.250728:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.250729:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.250731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677448.250732:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.250733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:26.0:1619677448.250736:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.250738:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.250739:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.250740:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:26.0:1619677448.250741:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.250741:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677448.250741:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.250742:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.250742:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.250743:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.250744:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677448.250744:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.250744:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.250745:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.250746:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.250746:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.250770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.250778:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.250781:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.250782:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.250783:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.250786:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.250788:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.250789:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.250790:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.250791:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.250792:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.250795:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.250797:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.250799:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.250800:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.250800:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.250802:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.250802:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.250802:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.250803:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.250807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.250814:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.250817:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.250818:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.250819:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.250821:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.250821:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.250835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:13.0:1619677448.250841:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.250843:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.250844:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.250845:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.250845:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.250847:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.250947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.250954:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.250957:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.250959:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.250961:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.250962:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.250963:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.250965:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.250966:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.250967:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.250968:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.250971:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.250972:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.250976:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.250977:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.250977:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.250979:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.250980:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.250981:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.250982:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.250983:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.250985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.250989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.250997:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.250997:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677448.251000:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:20.0:1619677448.251000:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677448.251001:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.251001:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.251002:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.251003:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.251004:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.2:1619677448.251005:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677448.251006:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677448.251011:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.251014:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.251015:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.251017:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.251020:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.251021:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.251023:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.251024:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.251025:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.251026:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.251029:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.251031:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.251033:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.251034:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.251035:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.251036:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.251037:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.251038:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.251038:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.251043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.251050:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.251053:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.251054:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.251054:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.251056:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.251057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:6.0:1619677448.251058:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677448.251062:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.251065:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.251067:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:5.0:1619677448.251067:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.251069:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.251070:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.251071:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.251072:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.251073:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.251075:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.251076:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.251077:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.251079:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.251081:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.251081:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.251082:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.251083:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.251084:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.251085:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.251086:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.251089:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.251089:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.251090:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.251090:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.251091:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.251092:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.251093:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)++
00000800:00000200:5.0:1619677448.251094:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (24)--
00000800:00000200:5.0:1619677448.251094:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.251095:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.251095:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.251095:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.251096:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:25.2:1619677448.251098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.251098:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.251099:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.2:1619677448.251100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677448.251100:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.251102:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.251102:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.251104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.251106:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.2:1619677448.251107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.251108:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.251108:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.251109:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.251110:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.251110:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.251110:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:31.0:1619677448.251112:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.251112:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677448.251113:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.251114:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.251114:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:31.0:1619677448.251115:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.251115:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.251115:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677448.251116:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:30.0:1619677448.251116:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:12.0:1619677448.251116:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.251117:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.251117:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.251117:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.251118:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677448.251118:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677448.251119:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.251120:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.251121:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:25.2:1619677448.251123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:31.0:1619677448.251124:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.251126:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.251129:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.251129:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.251130:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.251130:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.251132:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:13.0:1619677448.251132:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.251133:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.251133:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677448.251134:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.251134:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.251134:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.251134:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.251136:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677448.251138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.251146:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.251149:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.251150:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.251150:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.251153:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.251156:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.251218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.251225:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.251228:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.251229:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.251231:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.251233:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.251234:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.251236:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.251237:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.251238:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.251239:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.251242:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.251244:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.251247:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.251248:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.251248:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.251250:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.251251:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.251252:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.251252:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.251256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.251264:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.251267:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.251268:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.251268:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.251271:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.251274:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.251277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.251284:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.2:1619677448.251285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:6.0:1619677448.251286:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.251287:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.251287:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.251288:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.251289:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677448.251291:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.251297:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.251299:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.251301:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.251303:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.251305:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.251306:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.251307:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.251308:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.251309:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.251313:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.251315:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.251318:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.251318:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.251319:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.251321:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.251321:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.251322:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.251322:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.251327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.251335:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.251337:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.251338:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.251340:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.251342:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.251343:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.251430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.2:1619677448.251434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.251438:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.251440:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.251441:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.251441:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.251443:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:26.0:1619677448.251444:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.251444:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.251446:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.251447:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.251449:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677448.251449:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.251450:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.251451:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.251452:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.251453:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.251454:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.251455:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.251455:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.251456:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677448.251457:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.251461:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.251462:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.251464:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.251465:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.251467:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.251467:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.251468:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:17.0:1619677448.251468:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.251469:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:17.0:1619677448.251469:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.251471:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:17.0:1619677448.251471:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.251472:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.251472:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.251473:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.251473:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.251474:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677448.251474:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.251476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:25.2:1619677448.251480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.251482:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.251484:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.251485:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.251486:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.251487:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.251488:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677448.251489:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:22.0:1619677448.251490:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.251491:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.251491:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.251494:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.251495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:22.0:1619677448.251495:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:26.0:1619677448.251501:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.251504:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.251505:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.251505:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.251506:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.251507:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.251556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.251562:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.251565:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.251566:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.2:1619677448.251567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:18.0:1619677448.251568:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.251570:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.251572:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:6.0:1619677448.251572:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:18.0:1619677448.251573:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.251575:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.251575:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.251576:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:18.0:1619677448.251576:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.251577:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:6.0:1619677448.251579:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.251581:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.251583:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:6.0:1619677448.251584:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.251586:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.251586:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:18.0:1619677448.251587:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677448.251587:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:6.0:1619677448.251587:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.251589:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.251589:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:6.0:1619677448.251589:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.251589:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.251591:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:6.0:1619677448.251591:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.251592:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.251592:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:6.0:1619677448.251594:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.251595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.251596:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:21.2:1619677448.251597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:6.0:1619677448.251599:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.251600:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.251601:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.251603:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.251603:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:22.0:1619677448.251604:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.251604:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.251605:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.251605:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.251605:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677448.251605:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:22.0:1619677448.251606:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.251606:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677448.251607:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677448.251607:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:22.0:1619677448.251608:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:8.2:1619677448.251610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.251611:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.251615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677448.251618:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677448.251621:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.251621:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.251622:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.251622:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.251624:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:1.0:1619677448.251625:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.251626:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.251626:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.251627:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:5.0:1619677448.251629:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.251631:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.251632:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.251633:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.251634:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.251635:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.251639:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.251641:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.251644:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.251645:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.251645:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.251647:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.251648:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.251648:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.251649:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.251653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.251660:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.251662:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.251663:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.251665:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.251666:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.251668:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.251715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.251722:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.251725:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.251726:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.251728:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.251730:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.251731:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.251732:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.251734:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.251735:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.251736:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.251739:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.251741:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.251744:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.251745:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.251746:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.251747:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.251748:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.251748:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.251749:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.251753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.251761:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.251764:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.251764:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.251765:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.251767:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.251769:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.251771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:22.0:1619677448.251778:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.251780:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.251781:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.251782:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.251783:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.251783:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.251805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.251811:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.251814:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.251814:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.251817:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.251818:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.251820:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.251821:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.251823:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.251824:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.251825:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.251828:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.251830:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.251832:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.251833:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.251834:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.251836:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.251836:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.251837:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.251838:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.251839:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.251841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.251844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.251849:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.251850:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677448.251852:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677448.251852:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677448.251853:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.251853:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.251853:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.251854:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.251855:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.251857:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.252074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.252081:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.252084:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.252085:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.252087:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.252090:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.252092:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.252093:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.252094:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.252095:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.252096:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.252099:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.252101:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.252104:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.252105:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.252105:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.2:1619677448.252106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.252107:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.252107:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.252109:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.252110:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.252111:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.252113:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:25.2:1619677448.252113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.252114:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.252115:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.252117:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.252119:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.252119:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.252120:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:26.0:1619677448.252121:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.252121:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.252122:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.252123:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.252124:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.252124:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:26.0:1619677448.252125:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000400:00000200:22.0:1619677448.252126:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.252128:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.2:1619677448.252128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:22.0:1619677448.252129:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:26.0:1619677448.252130:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.252131:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.252131:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.252133:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.252133:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.252134:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.252135:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:30.0:1619677448.252136:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:25.2:1619677448.252138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.252139:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.252141:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.252143:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.252143:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.252144:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:30.0:1619677448.252145:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.252146:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.252147:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.252147:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:30.0:1619677448.252148:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677448.252148:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.252149:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:12.0:1619677448.252150:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677448.252151:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:30.0:1619677448.252152:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.252152:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.252154:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.252157:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.252158:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.252159:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.252161:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.252161:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.252162:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.252163:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.252167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.252174:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.252177:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.252178:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.252180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677448.252180:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.252182:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.252184:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.252186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.252188:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.252191:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.252193:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.252194:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.252195:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.252196:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.252197:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.252198:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677448.252198:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.252198:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000400:00000200:31.0:1619677448.252199:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.252199:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:31.0:1619677448.252200:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.252200:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:31.0:1619677448.252201:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.252202:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.252206:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.252207:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.252210:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.252210:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.252211:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.252212:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.252213:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.252214:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.252216:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.252217:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.252218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.252222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.252224:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.252226:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.252227:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.252229:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.252230:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677448.252230:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.252231:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.252232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:30.0:1619677448.252233:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.252233:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.252234:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.252235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:6.0:1619677448.252237:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.252240:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.252241:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677448.252241:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:6.0:1619677448.252243:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.252244:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.252245:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.252245:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.252246:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:6.0:1619677448.252246:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.252247:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:6.0:1619677448.252247:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.252249:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.252250:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.252251:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.252252:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.252255:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.252257:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.252260:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.252260:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.252261:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.252263:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.252264:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.252264:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.252265:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.252269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.252277:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.252280:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.252281:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.252281:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.252283:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.252285:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.252381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.252388:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.252391:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.252392:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.252394:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.252397:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.252398:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.252400:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.252401:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.252402:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.252403:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.252406:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.252408:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.252411:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.252411:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.252412:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.252414:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.252415:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.252415:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.252416:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.252420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.252420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677448.252428:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:10.0:1619677448.252428:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.252431:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:10.0:1619677448.252431:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.252432:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677448.252432:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.252432:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:10.0:1619677448.252434:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.252435:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.252437:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.252438:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:10.0:1619677448.252439:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.252441:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.252442:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.252443:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.252444:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.252447:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.252449:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.252452:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.252453:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.252453:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.252455:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.252455:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.252456:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.252458:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.252458:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.252461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:4.2:1619677448.252464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.252470:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:26.0:1619677448.252471:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.252472:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:26.0:1619677448.252473:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677448.252473:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:5.2:1619677448.252473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.252474:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.252474:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.252475:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.252476:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.252479:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677448.252479:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.252481:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.252482:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.252484:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.252487:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.252488:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.252490:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.252491:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.252492:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.252493:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.252497:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.252498:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.252501:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.252502:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.252503:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.252505:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.252505:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.252506:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.252507:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.252510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.252518:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.252521:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.252521:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.252523:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.252525:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.252526:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.252542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.2:1619677448.252546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:26.0:1619677448.252550:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.252552:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:26.0:1619677448.252553:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.252553:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.252556:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.252556:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.252556:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.252559:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.252560:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.252561:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677448.252562:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.252563:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.252564:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.252565:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.252566:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.252567:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.252568:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.252568:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.252569:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677448.252570:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.252575:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.252575:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.252578:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677448.252578:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.252580:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.252581:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.252581:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.252582:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.252582:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:2.0:1619677448.252582:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.252584:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:2.0:1619677448.252584:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.252584:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:26.0:1619677448.252585:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.252585:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:26.0:1619677448.252586:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:2.0:1619677448.252586:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:26.0:1619677448.252587:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.252590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:9.2:1619677448.252590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.252598:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:30.0:1619677448.252599:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:18.0:1619677448.252600:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.252600:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.252601:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:2.0:1619677448.252601:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:30.0:1619677448.252602:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.252603:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.252604:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.252606:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:18.0:1619677448.252606:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677448.252607:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.252708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.252714:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.252717:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.252717:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.252720:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.252722:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.252723:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.252724:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.252726:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.252727:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.252728:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.252731:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.252733:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.252736:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.252736:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.252737:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.252739:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.252739:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.252740:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.252740:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.252745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.252753:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.252755:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.252756:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.252757:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.252759:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.252761:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.252800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.252807:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.252810:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.252813:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.252814:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.252815:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.252816:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.252818:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.252819:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.252820:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.252821:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.252824:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.252826:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.252828:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.252829:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.252829:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.252831:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.252831:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.252832:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.252833:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.252836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.252843:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.252845:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.252846:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.252848:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.252850:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.252851:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.252963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.252970:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.252973:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.252975:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.252977:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.252977:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.252977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677448.252979:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.252980:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.252981:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.252982:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.252983:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.252983:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.252986:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.252986:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677448.252987:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.252990:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.252991:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:17.0:1619677448.252993:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.252994:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.252995:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:17.0:1619677448.252995:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:30.0:1619677448.252996:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:17.0:1619677448.252996:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.252998:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.252998:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.252998:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.252999:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:17.0:1619677448.252999:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.253000:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:17.0:1619677448.253000:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.253003:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.253004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677448.253005:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.253008:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.253009:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.253009:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.253011:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.253012:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.253012:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.253012:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677448.253013:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677448.253014:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.253016:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.253017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:5.0:1619677448.253017:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.253018:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.253020:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677448.253024:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.253026:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.253027:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.253029:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.253030:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.253032:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.253101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677448.253106:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677448.253109:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677448.253110:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677448.253112:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.253114:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.253116:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:22.0:1619677448.253117:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.253118:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.253119:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.253120:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:22.0:1619677448.253124:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.253126:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677448.253129:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.253130:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:22.0:1619677448.253130:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:22.0:1619677448.253132:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:22.0:1619677448.253133:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677448.253134:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677448.253135:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677448.253136:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:25.2:1619677448.253138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:21.2:1619677448.253141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (21)++
00000800:00000200:31.0:1619677448.253145:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:6.0:1619677448.253146:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:31.0:1619677448.253148:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:6.0:1619677448.253148:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000400:00000200:31.0:1619677448.253149:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.253149:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677448.253150:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000400:00000200:31.0:1619677448.253151:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.253152:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.253155:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.253166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.253173:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.253175:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.253176:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.253177:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.253178:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.253179:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.253187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.253194:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.253196:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.253199:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.253200:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.253201:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.253203:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.253204:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.253205:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.253206:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.253207:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.253210:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.253212:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.253214:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.253215:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.253216:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.253217:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.253217:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.253218:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.253219:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.253222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.253229:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.253232:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.253233:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.253234:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.253235:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.253236:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.253349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.253357:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.253359:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.253360:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.253362:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.253364:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.253366:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.253367:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.253368:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.253369:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.253370:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.253373:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.253375:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.253377:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.253377:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.253378:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.253379:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.253380:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.253380:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.253381:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.253386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.253393:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:4.2:1619677448.253393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.253396:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.253396:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.253398:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.253400:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.253400:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.253401:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:26.0:1619677448.253403:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.253404:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.253405:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.253408:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.253409:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.253410:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.253411:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.253412:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.253413:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.253416:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.253417:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.253420:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.253421:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.253421:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.253423:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.253423:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.253424:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.253424:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.253430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.253437:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.253440:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.253441:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.253442:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.253444:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.253446:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.253517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.253523:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.253526:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.253527:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.253529:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.253531:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.253533:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.253534:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.253535:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.253536:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.253537:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.253540:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.253541:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.253543:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.253544:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.253545:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.253546:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.2:1619677448.253547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677448.253547:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.253547:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.253548:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.253552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.253554:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.253557:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.253559:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.253559:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.253561:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677448.253561:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:30.0:1619677448.253562:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.253562:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.253562:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:30.0:1619677448.253564:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.253565:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.253565:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.253567:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.253567:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:30.0:1619677448.253568:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.253569:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.253572:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.253574:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.253576:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.253577:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.253578:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.253579:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.253580:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.253581:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.253582:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.253585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.253592:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.253595:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.253595:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.253597:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.253599:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.253600:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.253732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.253740:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.253743:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.253745:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.253747:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.253747:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.253748:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.253749:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.253751:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.253752:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.253753:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.253756:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.253758:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.253760:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.253760:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.253761:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.253762:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.253763:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.253764:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.253764:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.253768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.253774:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.253776:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.253777:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.253779:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.253780:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.253781:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.253797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.253803:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.253806:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.253806:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.253807:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.253807:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.253808:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.253827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.253833:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.253836:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.253837:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.253839:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.253842:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.253843:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.253844:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.253845:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.253847:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.253848:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.253851:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.253852:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.253855:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.253855:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.253856:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.253858:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.253858:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.253859:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.253860:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.253860:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:21.2:1619677448.253861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:9.2:1619677448.253864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.253866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677448.253869:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677448.253871:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677448.253871:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:22.0:1619677448.253873:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.253873:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.253874:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:18.0:1619677448.253874:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:2.0:1619677448.253874:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.253875:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.253876:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.253877:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.253878:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677448.253878:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.253880:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.253881:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.253882:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.253883:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677448.253887:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.253888:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.253889:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.253890:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.253890:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.253891:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.253892:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.253892:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (23)++
00000800:00000200:2.0:1619677448.253892:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.253894:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (24)--
00000800:00000200:18.0:1619677448.253895:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.253895:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (24)++
00000800:00000200:2.0:1619677448.253895:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (24)--
00000800:00000200:18.0:1619677448.253896:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.253896:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677448.253896:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.253897:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:9.2:1619677448.253900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.253907:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.253909:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.253909:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.253911:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.253911:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:21.2:1619677448.253912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:18.0:1619677448.253913:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:8.0:1619677448.253919:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.253922:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.253923:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.253924:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.253927:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.253929:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.253930:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.253931:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.253932:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.253934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:8.0:1619677448.253934:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.253937:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.253938:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.253940:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:8.0:1619677448.253941:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.253942:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.253943:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.253943:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.253945:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.0:1619677448.253945:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.253945:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:5.0:1619677448.253945:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.253946:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.253946:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:5.0:1619677448.253948:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.253949:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:9.2:1619677448.253951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677448.253951:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.253952:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.253953:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.253954:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.253957:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.253958:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677448.253959:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.253961:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.253962:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.253962:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.253962:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:2.0:1619677448.253962:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.253963:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:2.0:1619677448.253964:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.253965:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.253965:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.253966:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.253967:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:2.0:1619677448.253967:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:25.2:1619677448.253971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.253977:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.253980:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.253981:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.253981:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.253983:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.253986:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.254002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.254010:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.254012:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.254013:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.254015:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.254017:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.254019:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.254020:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.254021:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.254022:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.254023:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.254026:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.254028:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.254031:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.254031:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.254032:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.254033:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.254034:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.254035:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.254036:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.254036:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.254036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:25.2:1619677448.254039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.254042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677448.254042:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.254045:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.254046:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.254046:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:1.0:1619677448.254047:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.254048:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.254049:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.254049:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.254050:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:13.0:1619677448.254051:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.254051:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:1.0:1619677448.254051:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:12.0:1619677448.254052:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.254052:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:1.0:1619677448.254052:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.254054:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.254054:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:1.0:1619677448.254054:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.254055:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.254056:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.254060:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.254062:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.254065:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.254065:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.254066:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.254068:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.254069:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.254070:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.254071:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.254072:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.254074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.2:1619677448.254077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.254081:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677448.254082:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:30.0:1619677448.254084:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:17.0:1619677448.254084:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:30.0:1619677448.254085:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.254085:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.254085:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.254087:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.254087:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.254090:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.254099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.254102:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.254105:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.254107:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.254108:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.254110:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.254112:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.254113:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.254115:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.254115:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.254117:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.254120:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.254122:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.254124:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.254125:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.254126:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.254128:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.254129:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.254130:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.254131:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.254135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.254142:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.254145:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.254146:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.254146:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.254148:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.254150:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.254157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.254164:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.254167:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.254169:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.254171:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.254171:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.254173:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.254174:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717326] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.254176:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.254177:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.254178:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.254181:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.254183:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.254185:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.254186:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.254187:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.254189:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.254189:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.254191:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.254192:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.254193:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.254194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.254199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.254201:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.254204:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.254205:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.254206:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.254207:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:13.0:1619677448.254208:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.254209:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.254209:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:12.0:1619677448.254209:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.254210:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.254213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:13.0:1619677448.254219:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.254222:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.254222:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.254223:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.254224:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.254226:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.254360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.254367:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.254370:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.254372:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.254374:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.254374:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.254375:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.254377:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.254379:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.254380:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.254380:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.254383:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.254385:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.254387:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.254388:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.254388:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.254390:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.254390:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.254391:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.254393:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.254393:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.254396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.254399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.254403:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.254406:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.254406:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.254407:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:12.0:1619677448.254408:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.254409:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.254410:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.254410:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677448.254411:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.254412:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677448.254547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.254554:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.254556:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.254558:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.254559:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.254561:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.254563:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.254564:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.254565:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.254566:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.254567:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.254571:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.254572:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.254575:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.254576:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.254577:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.254579:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.254579:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.254580:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.254580:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.254584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.254591:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.254594:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.254594:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.254596:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.254597:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.254599:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.254639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.254645:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.254647:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.254648:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.254650:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.254653:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.254654:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.254656:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.254657:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.254658:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.254659:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.254663:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.254665:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.254668:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.254668:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.254669:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.254671:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.254671:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.254672:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.254672:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.254677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.254684:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.254687:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.254688:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.254689:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.254691:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.254692:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.254692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37c600] (20)++
00000800:00000200:25.2:1619677448.254699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.254700:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37c600] (21)++
00000800:00000200:26.0:1619677448.254703:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.1@o2ib3
00000800:00000200:10.0:1619677448.254704:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37c600] (22)--
00000800:00000200:30.0:1619677448.254706:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:26.0:1619677448.254706:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.1@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.254709:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.254709:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.254710:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.254710:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:26.0:1619677448.254710:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:30.0:1619677448.254711:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.254712:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:26.0:1619677448.254712:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.254713:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.254714:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.254715:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677448.254718:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.254719:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677448.254723:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.254723:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:26.0:1619677448.254724:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:26.0:1619677448.254726:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:26.0:1619677448.254726:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37c600] (21)++
00000800:00000200:26.0:1619677448.254727:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37c600] (22)--
00000800:00000200:26.0:1619677448.254728:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37c600] (21)--
00000800:00000200:8.2:1619677448.254733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.254741:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.254744:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.254745:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.254747:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.254747:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.254750:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:21.2:1619677448.255038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.255045:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.255048:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.255049:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.255051:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.255054:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.255055:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.255057:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.255058:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.255059:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.255060:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.255063:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.255064:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.255066:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.255067:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.255068:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.255069:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.255070:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.255071:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.255071:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:4.2:1619677448.255074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:25.2:1619677448.255075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.255081:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:10.0:1619677448.255081:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.255083:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.255084:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:12.0:1619677448.255084:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.255084:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.255086:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.255086:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.255088:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.255089:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:10.0:1619677448.255089:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.255090:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.255091:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.255093:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.255094:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.255095:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.255098:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.255098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:10.0:1619677448.255099:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.255101:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.255102:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.255102:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.255104:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.255104:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.255105:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:10.0:1619677448.255105:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.255106:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:18.0:1619677448.255107:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.255108:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:25.2:1619677448.255109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:18.0:1619677448.255110:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.255113:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.255114:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:30.0:1619677448.255116:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:18.0:1619677448.255116:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.255117:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.255118:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.255119:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.255119:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.255119:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.255122:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.255122:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.255124:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.255125:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677448.255126:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.255128:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.255129:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.255129:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.255131:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.255132:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.255132:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.255133:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.255137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.255145:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.255147:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.255149:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.255149:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.255151:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.255153:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.255156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.255164:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.255166:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.255167:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.255167:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.255168:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.255169:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.255172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.255179:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.255182:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.255182:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.255184:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.255187:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.255188:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.255190:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.255191:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.255192:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.255193:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.255196:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.255198:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.255200:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.255201:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.255202:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.255203:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.255204:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.255205:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.255205:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.255209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.255215:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.255217:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.255218:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.255220:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.255221:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.255222:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.255259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.255263:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.255265:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.255267:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.255268:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.255270:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.255271:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.255273:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.255274:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.255275:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.255276:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.255279:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.255281:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.255284:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.255284:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.255285:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.255287:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.255287:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.255288:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.255289:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.255292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.255300:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.255303:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.255304:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.255306:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.255307:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.255309:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.255315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.255322:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.255324:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.255325:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.255326:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.255326:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.255327:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.255354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.255359:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.255362:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.255363:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.255365:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.255367:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.255369:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:21.2:1619677448.255370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:6.0:1619677448.255370:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.255371:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.255372:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.255373:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.255375:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:6.0:1619677448.255376:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.255378:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.255378:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.255379:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:6.0:1619677448.255380:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.255381:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.255382:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:2.0:1619677448.255382:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.255384:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.255384:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677448.255384:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.255385:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.255385:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:2.0:1619677448.255386:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677448.255387:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.255388:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.255389:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.255390:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:8.2:1619677448.255391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:2.0:1619677448.255393:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.255395:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.255397:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.255398:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.255399:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677448.255399:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.255400:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.255401:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:2.0:1619677448.255401:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.255401:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:5.0:1619677448.255402:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.255402:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:5.0:1619677448.255404:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.255405:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.255406:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.255407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.255415:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.255417:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.255418:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.255419:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.255420:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.255422:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.255438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.255443:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.255446:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.255448:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.255449:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.255451:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.255453:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.255454:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.255455:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.255456:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.255457:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.255461:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.255463:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.255465:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.255466:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.255467:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.255469:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.255470:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.255470:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.255471:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.255475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.255476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.255483:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:10.0:1619677448.255483:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.2:1619677448.255485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677448.255486:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.255486:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.255486:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.255487:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677448.255489:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.255490:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.255491:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.255491:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.255492:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:10.0:1619677448.255492:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.255494:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:1.0:1619677448.255494:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.255495:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.255496:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.255497:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.255497:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.255498:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.255499:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.255500:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.255502:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.255504:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.255505:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.255506:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.255507:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.255508:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.255508:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.255511:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.255512:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:1.0:1619677448.255512:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.255513:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.255514:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.255514:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.255514:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.255515:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.255516:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677448.255517:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.255518:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.255519:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:25.2:1619677448.255520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:1.0:1619677448.255521:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:1.0:1619677448.255521:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.255522:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.255522:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.255527:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:30.0:1619677448.255529:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (24)--
00000400:00000200:30.0:1619677448.255530:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.255532:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.255534:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:30.0:1619677448.255534:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:31.0:1619677448.255535:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (24)--
00000400:00000200:31.0:1619677448.255536:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.255536:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:31.0:1619677448.255537:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.255539:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.255540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.255548:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.255550:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.255551:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.255552:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.255552:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.255553:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.255721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.255729:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.255731:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.255734:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.255736:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.255736:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.255737:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.255739:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.255740:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.255741:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.255742:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.255745:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.255747:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.2:1619677448.255749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:30.0:1619677448.255750:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.255751:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.255752:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.255753:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.255754:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.255755:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.255755:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.0:1619677448.255757:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.2:1619677448.255760:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:24.0:1619677448.255761:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.255765:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:20.2:1619677448.255767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:8.0:1619677448.255767:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.255769:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.255770:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:20.0:1619677448.255772:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:8.0:1619677448.255772:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96717310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.255773:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.255774:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:8.0:1619677448.255774:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.255774:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:20.0:1619677448.255775:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.255775:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.255775:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.255777:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.255779:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.255780:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.255781:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.255782:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677448.255783:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.0:1619677448.255783:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.255783:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.255785:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:8.0:1619677448.255786:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.255786:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717309] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.255787:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.255787:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677448.255787:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.255788:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.255789:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.0:1619677448.255790:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.255790:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.255791:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.255792:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.255793:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:5.0:1619677448.255793:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.255795:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:9.2:1619677448.255796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.255799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677448.255799:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.255800:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.255801:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:18.0:1619677448.255803:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677448.255803:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.255804:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.255805:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:24.0:1619677448.255806:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:18.0:1619677448.255806:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:5.0:1619677448.255806:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000400:00000200:18.0:1619677448.255807:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.255807:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677448.255807:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.255808:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:25.2:1619677448.255809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:24.0:1619677448.255809:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.255810:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:18.0:1619677448.255810:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.255812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677448.255813:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677448.255815:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.255815:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.255817:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:31.0:1619677448.255818:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677448.255818:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.255818:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.255818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:31.0:1619677448.255819:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.255821:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.255822:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.255824:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:2.0:1619677448.255825:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.255828:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.255829:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.255829:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.255830:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.255831:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.255861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.255868:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.255871:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.255872:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.255874:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.255876:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.255878:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.255879:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.255880:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.255881:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.255882:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.255885:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.255887:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.255890:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.255891:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.255891:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.255893:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.255894:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.255895:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.255897:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.255897:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677448.255899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677448.255903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.255907:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.255910:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:10.0:1619677448.255910:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:18.0:1619677448.255911:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.255911:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:2.0:1619677448.255911:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.255912:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677448.255913:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.255913:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:18.0:1619677448.255916:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.256086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.256091:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.256094:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.256094:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.256096:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.256098:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.256100:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.256101:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.256102:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.256103:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.256104:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.256108:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.256109:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.256112:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.256112:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.256113:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.256115:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.256115:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.256116:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.256117:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.256118:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.256121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.256123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.256129:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:17.0:1619677448.256129:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677448.256131:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.256131:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677448.256131:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677448.256132:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:17.0:1619677448.256132:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.256133:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.256134:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.256137:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.256142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.256149:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.256152:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.256153:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.256155:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.256158:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.256159:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.256160:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.256162:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.256163:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.256164:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.256167:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.256169:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.256172:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.256173:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.256174:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.256176:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.256176:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.256177:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.256177:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.256181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.256189:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.256191:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.256193:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.256193:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.256195:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.256198:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.256280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.256287:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.256290:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.256291:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.256292:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.256301:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.256302:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.256303:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.256304:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.256305:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.256306:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.256309:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.256311:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.256313:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.256314:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.256315:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.256317:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.256317:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.256318:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.256318:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.256323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.256331:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.256334:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.256335:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.256335:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.256337:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.256339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:18.0:1619677448.256340:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:26.0:1619677448.256346:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677448.256349:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:10.0:1619677448.256350:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:26.0:1619677448.256352:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.256355:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.256356:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.256357:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.256357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:26.0:1619677448.256359:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.256359:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.256361:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.256364:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.256366:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.256366:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.256368:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.256369:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.256370:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:10.0:1619677448.256370:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.256371:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:10.0:1619677448.256372:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.256373:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:10.0:1619677448.256374:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.256375:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.256376:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.256377:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.256378:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000400:00000200:10.0:1619677448.256378:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.2:1619677448.256379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.256380:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75ca00] (22)++
00000800:00000200:26.0:1619677448.256381:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (23)--
00000400:00000200:10.0:1619677448.256381:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.256382:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:26.0:1619677448.256383:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.256383:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.256386:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.256386:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.256387:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.256387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677448.256388:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.256388:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:10.0:1619677448.256389:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.256389:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.256390:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:17.0:1619677448.256391:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.256391:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:10.0:1619677448.256392:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)++
00000400:00000200:1.0:1619677448.256392:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.256393:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75ca00] (23)--
00000800:00000200:10.0:1619677448.256394:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:1.0:1619677448.256394:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.256395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.256395:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:1.0:1619677448.256397:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.256400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:12.0:1619677448.256403:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.256405:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677448.256407:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:12.0:1619677448.256407:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.256408:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.256409:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:13.0:1619677448.256410:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.256411:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:12.0:1619677448.256411:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.256411:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.256412:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.256413:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.256520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.256522:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.256525:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.256527:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.256528:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.256531:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.256532:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.256534:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.256535:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.256536:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.256537:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.256540:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.256542:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:21.2:1619677448.256544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677448.256545:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.256546:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.256547:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.256548:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.256549:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.256550:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.256550:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.256553:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.256555:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:25.2:1619677448.256555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:30.0:1619677448.256558:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.256560:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.256560:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.256562:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:30.0:1619677448.256562:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:31.0:1619677448.256563:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.256563:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.256564:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.256564:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.256566:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.256566:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.256567:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.256569:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.256570:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:30.0:1619677448.256570:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.256572:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.256575:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.256576:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.256577:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.256578:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.256579:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.256580:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.256580:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.256584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.256592:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.256594:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.256596:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.256596:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.256598:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.256601:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.256626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.256632:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.256634:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.256635:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.256637:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.256639:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.256641:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.256642:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.256643:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.256644:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.256645:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.256649:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.256651:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.256654:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.256655:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.256655:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.256657:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.256658:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.256658:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.256659:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.256663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.256670:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.256673:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.256674:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.256676:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.256676:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.256679:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.256684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677448.256685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:26.0:1619677448.256691:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677448.256693:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:26.0:1619677448.256694:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.256695:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677448.256695:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.256696:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:26.0:1619677448.256697:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.256699:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.256700:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.256701:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677448.256701:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.256703:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.256703:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.256704:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.256704:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.256706:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.256706:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.256707:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677448.256707:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.256708:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.256712:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.256712:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.256714:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677448.256714:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.256717:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.256717:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.256718:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.256718:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677448.256718:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.256719:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.256720:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677448.256720:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.256721:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677448.256721:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:26.0:1619677448.256722:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677448.256722:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:26.0:1619677448.256723:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.0:1619677448.256723:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.256726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:25.2:1619677448.256730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:12.0:1619677448.256732:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.256734:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.256735:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.256737:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.256738:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:13.0:1619677448.256739:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.256740:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677448.256741:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.256742:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.256742:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.256745:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.256748:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.256762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.256768:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.256770:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.256771:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.256773:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.256776:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.256777:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.256779:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.256780:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.256781:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.256782:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:4.2:1619677448.256786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:1.0:1619677448.256786:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.256787:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.2:1619677448.256790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677448.256790:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.256791:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.256792:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:24.0:1619677448.256794:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.256794:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.256794:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.256795:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677448.256796:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:1.0:1619677448.256796:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677448.256797:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677448.256798:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:24.0:1619677448.256799:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.256800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.256801:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:24.0:1619677448.256802:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.256803:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.256803:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:30.0:1619677448.256805:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:24.0:1619677448.256805:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96717296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.256806:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.256806:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.256807:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.256807:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:31.0:1619677448.256808:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:24.0:1619677448.256808:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.256809:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.256810:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:31.0:1619677448.256811:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.256811:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.256811:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.256813:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.256813:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.256813:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.256814:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:24.0:1619677448.256815:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.256817:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:31.0:1619677448.256818:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.256818:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677448.256819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:24.0:1619677448.256819:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.256820:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:24.0:1619677448.256820:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.256822:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.256822:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677448.256823:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.256823:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:24.0:1619677448.256823:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.256823:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:31.0:1619677448.256824:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.256826:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.256826:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.256827:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:22.0:1619677448.256827:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.256828:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.256828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.256830:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.256830:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.256831:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.256832:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.256832:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.2:1619677448.256832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677448.256835:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.256838:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.256839:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.256839:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.256840:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:17.0:1619677448.256841:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.256843:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:17.0:1619677448.256844:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:5.0:1619677448.256844:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.256846:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.256846:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.256849:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.256851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.256859:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.256861:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.256862:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.256863:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.256864:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.256865:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.256893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.256901:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.256903:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.256904:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.256906:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.256909:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.256911:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.256912:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.256913:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.256914:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.256915:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.256919:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.256920:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.256922:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.256923:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.256923:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.256925:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.256925:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.256926:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.256926:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.256931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.256937:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.256940:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.256941:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.256942:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.256943:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.256945:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.256982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.256988:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.256990:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.256991:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.256994:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.256996:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.256998:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.256999:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.257000:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.257001:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.257002:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.257005:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.257007:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.257009:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.257009:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.257010:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.257011:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.257012:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.257012:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.257013:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.257017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.257024:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.257026:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.257027:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.257028:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.257029:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.257030:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.257037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.257044:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.257046:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.257047:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.257047:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.257048:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.257048:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.257083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.257088:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.257091:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.257092:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.257094:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.257096:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.257097:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.257098:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.257100:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.257101:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.257102:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.257105:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.257107:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.257109:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.257110:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.257111:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.257112:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.257112:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.257113:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.257114:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.257118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.257121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.257127:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.257128:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.257128:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.257129:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.257130:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.257132:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.257132:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.257135:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.257136:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.257137:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677448.257137:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677448.257138:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.257139:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.257141:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.257141:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.257143:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.257146:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.257148:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.257151:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.257152:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.257153:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.257154:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.257155:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.257156:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.257157:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.257158:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.257160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.257163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.257167:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.257168:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677448.257170:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677448.257170:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.257170:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677448.257171:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.257171:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677448.257173:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.257174:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.257176:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.257280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.257287:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.257290:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.257290:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.257293:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.257299:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.257301:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:21.2:1619677448.257301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000400:00000200:26.0:1619677448.257302:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.257304:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.257305:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.257306:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.257309:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000400:00000200:26.0:1619677448.257309:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.257311:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000800:00000200:26.0:1619677448.257311:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.257313:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.257314:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.257315:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.257315:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677448.257316:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.257316:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:26.0:1619677448.257317:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.257317:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.257318:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:26.0:1619677448.257318:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677448.257319:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.257319:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:31.0:1619677448.257320:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.257321:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.257322:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:8.2:1619677448.257324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677448.257325:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.257327:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.257328:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.257329:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.257330:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.257331:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.257332:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677448.257332:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:20.0:1619677448.257332:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:31.0:1619677448.257333:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:20.0:1619677448.257334:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:21.2:1619677448.257335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677448.257335:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.257336:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.257338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677448.257339:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.257341:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:30.0:1619677448.257344:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.2:1619677448.257344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.257345:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.257346:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677448.257347:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.257347:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.257348:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.257349:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.257350:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.257351:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.257351:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000400:00000200:30.0:1619677448.257352:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:31.0:1619677448.257353:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.257353:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.0:1619677448.257353:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.257354:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.257354:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000400:00000200:30.0:1619677448.257355:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.257355:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000400:00000200:30.0:1619677448.257356:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.257356:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:30.0:1619677448.257357:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.257357:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:30.0:1619677448.257360:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.257362:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.257365:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.257366:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.257366:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.257368:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.257368:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.257369:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.257370:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.257374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.257382:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.257385:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.257385:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.257386:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.257388:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.257391:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.257402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.257408:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.257411:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.257412:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.257414:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.257417:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.257418:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.257420:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.257421:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.257422:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.257423:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.257426:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.257428:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.257431:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.257432:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.257432:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.257434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677448.257434:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.257435:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.257435:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.257436:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.257440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.257442:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.257445:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.257447:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.257447:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:31.0:1619677448.257449:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.257449:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:12.0:1619677448.257450:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:31.0:1619677448.257451:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:12.0:1619677448.257451:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.257452:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.257453:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.257454:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.257455:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.257455:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:31.0:1619677448.257456:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:12.0:1619677448.257456:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:31.0:1619677448.257459:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.257461:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.257463:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.257463:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.257464:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.257465:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.257466:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.257467:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.257467:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.257471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.257477:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.257480:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.257481:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.257482:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.257484:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.257485:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.257486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.257492:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.257495:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.257496:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.257497:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.257497:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.257499:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.257622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.257628:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.257631:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.257631:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.257633:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.257636:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.257638:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.257639:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.257640:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.257641:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.257642:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.257646:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.257648:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.257650:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.257651:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.257652:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.257654:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.257654:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.257655:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.257655:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.257660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.257667:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.257669:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.257670:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.257672:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.257673:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.257675:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.257677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:12.0:1619677448.257683:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.257686:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.257687:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.257688:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.257688:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.257690:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.257761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.257768:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.257771:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.257773:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.257775:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.257775:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.257777:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.257778:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717284] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.257779:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.257780:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.257781:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.257785:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.257786:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.257790:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.257791:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.257791:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.257793:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.257793:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.257794:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.257795:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.257799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.257807:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.257809:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.257810:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.257811:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.257813:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.257816:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.257858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.257863:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.257866:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.257867:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.257869:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.257872:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.257873:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.257875:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.257876:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.257877:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.257878:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.257881:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.257883:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.257885:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.257886:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.257887:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.257888:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.257889:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.257890:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.257890:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.257894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:21.2:1619677448.257900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:1.0:1619677448.257901:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.257903:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.257904:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.257905:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.257905:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.257905:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.257907:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:22.0:1619677448.257908:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.257909:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.257910:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.257913:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.257914:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.257915:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.257917:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.257918:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.257919:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.257922:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.257924:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.257927:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.257928:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.257928:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.257930:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.257931:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.257931:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.257932:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.257936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.257942:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.257945:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.257946:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.257946:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.257948:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.257951:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677448.257957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677448.257960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.257963:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.257965:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.257965:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.257967:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.257968:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.257969:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.257971:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.257973:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.257974:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.257976:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.257977:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.257978:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.257979:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.257981:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677448.257982:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.257983:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.257984:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.257985:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.257985:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.257986:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677448.257989:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.257989:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.257990:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.257991:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:5.0:1619677448.257991:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.257993:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.257994:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.257994:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.257995:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.257995:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.257996:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677448.257996:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.0:1619677448.257996:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677448.257997:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.257998:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:25.2:1619677448.257999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.257999:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.258000:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.258001:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.258002:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:9.2:1619677448.258004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.258005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:13.0:1619677448.258006:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.2:1619677448.258007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:13.0:1619677448.258008:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.258009:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.258010:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:18.0:1619677448.258011:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:13.0:1619677448.258011:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.258012:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.258012:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:13.0:1619677448.258013:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677448.258013:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.258014:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:18.0:1619677448.258014:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:8.0:1619677448.258014:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.258015:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.258015:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:2.0:1619677448.258015:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677448.258016:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:8.0:1619677448.258017:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.258018:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.258020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:18.0:1619677448.258020:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.258021:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.258022:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.258022:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:18.0:1619677448.258024:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.258025:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.258026:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:12.0:1619677448.258027:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677448.258029:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.258030:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.258031:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677448.258031:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.258032:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.258032:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.258033:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.258033:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677448.258034:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.258034:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.258034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:18.0:1619677448.258036:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.258036:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.258037:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.258038:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:26.0:1619677448.258042:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:9.2:1619677448.258042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.258045:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.258046:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.258047:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.258049:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:26.0:1619677448.258050:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.258051:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:26.0:1619677448.258052:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677448.258052:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.258053:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.258053:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.258054:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.258055:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.258056:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.258056:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:26.0:1619677448.258057:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.258061:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.258063:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.258065:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.258066:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.258066:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.258068:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.258068:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.258070:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.258071:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.258072:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.258074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.2:1619677448.258077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.258081:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.258083:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.258084:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.258084:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:31.0:1619677448.258086:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.258086:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:30.0:1619677448.258087:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.258087:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.258088:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:10.0:1619677448.258088:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677448.258164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.258172:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.258175:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.258177:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.258178:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.258179:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.258180:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.258182:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.258183:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.258184:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.258185:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.258188:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.258190:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.258193:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.258194:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.258194:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.258196:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.258196:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.258198:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.258199:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.258200:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.258202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.258206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.2:1619677448.258206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:6.0:1619677448.258210:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677448.258212:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.258214:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.258214:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677448.258215:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:17.0:1619677448.258215:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:6.0:1619677448.258215:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.258216:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.258217:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:6.0:1619677448.258217:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.258218:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677448.258218:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.258219:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.258219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:17.0:1619677448.258220:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.258222:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:6.0:1619677448.258222:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:17.0:1619677448.258223:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717276] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.258224:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.258224:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:17.0:1619677448.258225:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.258226:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:6.0:1619677448.258226:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.258227:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.258227:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.258228:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.258229:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:17.0:1619677448.258230:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.258232:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.258235:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.258236:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.258237:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.258238:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.258239:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.258240:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.258241:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.258242:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.258245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.258246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.2:1619677448.258251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:13.0:1619677448.258251:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677448.258252:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677448.258254:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677448.258254:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:13.0:1619677448.258255:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.258255:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.258256:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677448.258257:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.258257:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.258259:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677448.258259:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.258261:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.258264:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.258266:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.258266:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.258267:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:4.2:1619677448.258268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:31.0:1619677448.258269:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.258270:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.258271:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.258272:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.258274:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.258275:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.258277:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.258277:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.258278:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.258280:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.258280:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:26.0:1619677448.258280:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.258281:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:26.0:1619677448.258282:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.258283:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.258283:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.258284:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:26.0:1619677448.258284:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.258285:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.258286:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000400:00000200:26.0:1619677448.258286:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.258287:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677448.258287:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.258288:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:25.2:1619677448.258288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:26.0:1619677448.258290:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.258292:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.2:1619677448.258292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.258295:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:26.0:1619677448.258299:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.258299:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.258300:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.258301:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:26.0:1619677448.258301:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:12.0:1619677448.258301:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:26.0:1619677448.258302:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677448.258302:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677448.258302:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.258303:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:12.0:1619677448.258303:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.258304:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:26.0:1619677448.258304:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:30.0:1619677448.258305:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677448.258305:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:12.0:1619677448.258305:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.258306:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.258308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:4.2:1619677448.258311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.258317:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.258318:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.258318:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.258319:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.258320:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.258320:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.258320:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.258321:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.258322:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.258322:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.258425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.258433:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.258435:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.258437:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.258439:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.258439:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.258441:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.258442:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.258443:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.258444:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.258445:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.258448:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.258449:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.258451:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.258452:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.258453:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.258454:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.258455:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.258455:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.258456:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.258459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.258466:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.258468:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.258469:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.258469:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.258470:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.258473:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.258682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.258688:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.258690:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.2:1619677448.258691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677448.258691:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.258693:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.258696:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.258697:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:31.0:1619677448.258698:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.258699:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.258700:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.258701:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.258702:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:17.0:1619677448.258703:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.258705:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.258706:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:17.0:1619677448.258706:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.258708:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.258709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:17.0:1619677448.258709:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.258710:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:17.0:1619677448.258710:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.258711:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.258711:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:31.0:1619677448.258712:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.258712:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677448.258713:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.258714:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.258714:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.258714:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.258715:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.258715:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.258715:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.258718:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.258718:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:31.0:1619677448.258719:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.258720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:22.0:1619677448.258722:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.258723:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.258725:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.258725:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.258726:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.258726:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:22.0:1619677448.258726:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.258727:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96717270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.258728:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:22.0:1619677448.258728:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.258728:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.258729:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.258730:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.258730:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:22.0:1619677448.258730:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.258730:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:22.0:1619677448.258731:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.258731:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.258732:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:25.2:1619677448.258734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:1.0:1619677448.258734:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.258735:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.258736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.258737:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.258739:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.258740:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.258740:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677448.258740:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.258741:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.258741:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (23)++
00000800:00000200:22.0:1619677448.258743:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (24)--
00000800:00000200:13.0:1619677448.258743:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:22.0:1619677448.258744:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:13.0:1619677448.258744:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.258744:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.258745:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:17.0:1619677448.258745:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.258745:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (23)++
00000800:00000200:22.0:1619677448.258746:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000400:00000200:13.0:1619677448.258746:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.258746:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (24)--
00000800:00000200:1.0:1619677448.258746:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.258747:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677448.258747:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.258749:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.2:1619677448.258749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.258752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.258757:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677448.258757:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:17.0:1619677448.258759:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:6.0:1619677448.258760:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.258761:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:17.0:1619677448.258761:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.258761:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677448.258761:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.258762:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.258765:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.258771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.258777:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.258780:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.258781:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.2:1619677448.258783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000400:00000200:18.0:1619677448.258783:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.258784:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.258787:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677448.258788:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.258789:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:18.0:1619677448.258790:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.258791:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000400:00000200:18.0:1619677448.258792:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.258793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:18.0:1619677448.258793:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677448.258796:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.258797:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000400:00000200:2.0:1619677448.258798:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.258799:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:2.0:1619677448.258799:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677448.258800:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.258801:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:2.0:1619677448.258801:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.258802:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.258803:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:30.0:1619677448.258804:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:18.0:1619677448.258806:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.258807:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677448.258807:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000400:00000200:30.0:1619677448.258808:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.258808:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:2.0:1619677448.258809:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.258810:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000400:00000200:30.0:1619677448.258811:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.258811:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:30.0:1619677448.258812:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:18.0:1619677448.258812:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.258812:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677448.258813:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.2:1619677448.258813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000400:00000200:30.0:1619677448.258814:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96717267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.258814:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677448.258815:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.258815:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (21)++
00000400:00000200:30.0:1619677448.258816:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.258816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677448.258816:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (23)++
00000400:00000200:30.0:1619677448.258817:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.258818:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.258818:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (24)--
00000800:00000200:2.0:1619677448.258818:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677448.258819:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677448.258819:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:1.0:1619677448.258819:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:30.0:1619677448.258822:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.258822:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)++
00000800:00000200:17.0:1619677448.258822:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677448.258822:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:30.0:1619677448.258824:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.258825:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (24)--
00000800:00000200:10.0:1619677448.258825:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (24)++
00000400:00000200:26.0:1619677448.258826:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.258826:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (24)--
00000400:00000200:1.0:1619677448.258826:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.258827:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.258827:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.258828:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:26.0:1619677448.258828:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000400:00000200:1.0:1619677448.258828:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.258829:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:1.0:1619677448.258829:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677448.258830:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:10.0:1619677448.258830:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.258831:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677448.258831:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96717266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.258832:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:1.0:1619677448.258832:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.258833:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.258833:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:1.0:1619677448.258833:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.258834:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.258834:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:1.0:1619677448.258834:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.258835:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.258835:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.258836:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (21)++
00000800:00000200:8.2:1619677448.258836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.258837:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.258838:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:1.0:1619677448.258838:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.258841:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.258844:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.258844:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.258845:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:17.0:1619677448.258846:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.258846:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:17.0:1619677448.258847:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.258848:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:17.0:1619677448.258849:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.258849:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677448.258850:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677448.258851:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:1.0:1619677448.258852:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.258853:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:25.2:1619677448.258854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.258862:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.258865:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.258865:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.258866:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.258868:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.258870:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.259244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.259251:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.259254:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.259255:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.259257:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.259259:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.259260:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.259262:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.259263:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.259264:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.259265:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.259268:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.259270:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.259272:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.259273:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.259274:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.259275:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.259276:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.259276:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.259277:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.259281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.259288:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.259290:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.259291:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.259292:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.259295:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.259300:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.259333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.259341:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.259343:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.259344:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.259346:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.259348:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.259350:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.259351:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.259352:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.259353:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.259354:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.259357:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.259359:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.259361:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.259362:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.259363:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.259364:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.259365:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.259365:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.259366:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.259370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.259379:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.259382:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.259383:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.259383:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.259386:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.259388:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.259394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.259401:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.259404:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.259406:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.259408:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.259409:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.259411:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.259412:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.259413:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.259414:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.259414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:30.0:1619677448.259415:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.259418:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.259420:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.259422:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.259422:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.259423:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.259424:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.259424:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:30.0:1619677448.259425:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.259425:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.259426:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.259426:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.259427:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:26.0:1619677448.259427:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.259429:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.259430:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.259431:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.259432:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.259432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:26.0:1619677448.259434:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.259434:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.259437:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.259439:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:12.0:1619677448.259440:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:26.0:1619677448.259442:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.259442:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:26.0:1619677448.259443:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.259443:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:12.0:1619677448.259443:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.259444:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.259445:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.259445:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.259446:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:12.0:1619677448.259446:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.259447:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.259447:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.259450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:25.2:1619677448.259454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:12.0:1619677448.259456:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.259458:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.259459:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.259460:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.259460:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:21.2:1619677448.259461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:12.0:1619677448.259461:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677448.259462:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.259465:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.259466:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.259466:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677448.259468:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:22.0:1619677448.259468:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.259471:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677448.259471:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:31.0:1619677448.259473:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.259475:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.259475:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.259477:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.259478:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.259479:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.259480:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.259481:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.259484:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.259484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:31.0:1619677448.259485:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.259487:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.259488:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.259489:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.259489:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:31.0:1619677448.259490:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.259491:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.259491:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.259492:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.259492:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677448.259493:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:6.0:1619677448.259495:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.259496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:6.0:1619677448.259497:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.259498:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:6.0:1619677448.259500:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.259501:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.259502:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677448.259502:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.259503:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:12.0:1619677448.259504:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.259505:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.259506:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677448.259507:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.259508:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677448.259509:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.259509:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.259511:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.259511:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677448.259512:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.259514:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677448.259514:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677448.259515:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677448.259516:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677448.259521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.259528:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.259531:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.259532:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.259532:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.259534:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.259537:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.259557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.259563:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.259565:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.259566:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.259568:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.259570:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.259572:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.259573:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.259574:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.259575:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.259577:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.259580:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.259582:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.259584:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.259585:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.259586:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.259588:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.259588:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.259589:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.259589:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.259594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.259600:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.259603:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.259603:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677448.259604:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.259606:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.259609:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677448.259663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.259668:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.259671:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.259671:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.259674:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.259676:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.259678:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.259679:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.259680:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.259681:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.259682:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.259686:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.259688:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.259690:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.259691:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.259691:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.259693:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.259693:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.259694:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.259695:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.259699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.259706:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.259709:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.259710:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.259710:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.259712:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.259715:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.259833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.259840:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.259843:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.259844:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.259846:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.259849:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.259850:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.259851:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.259853:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.259854:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.259855:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.259858:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.259860:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.259862:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677448.259863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:10.0:1619677448.259863:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.259864:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.259865:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.259866:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.259866:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.259867:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.259869:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:25.2:1619677448.259871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.259872:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:5.0:1619677448.259874:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.259875:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:5.0:1619677448.259877:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.259878:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:5.0:1619677448.259878:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:30.0:1619677448.259880:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:5.0:1619677448.259880:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.259881:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.259881:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.259882:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.259883:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.259883:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.259884:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.259886:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:5.0:1619677448.259887:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.259888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.259889:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.259891:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.259892:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.259893:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.259895:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.259895:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.259895:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:5.0:1619677448.259896:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:5.0:1619677448.259897:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:31.0:1619677448.259898:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.259899:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.259900:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.259900:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:8.2:1619677448.259901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.259902:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.0:1619677448.259909:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.259911:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.259912:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.259914:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.259914:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.259916:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.260043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.260050:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.260054:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.260054:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.260056:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.260059:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.260060:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.260062:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.260063:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.260064:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.260065:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.260068:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.260070:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.260073:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.260074:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.260075:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.260077:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.260078:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.260078:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.260079:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.260083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.260091:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.260094:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.260095:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.260095:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.260097:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.260100:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.260231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.260239:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.260241:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.260242:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.260244:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.260247:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.260248:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.260249:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717254] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.260251:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.260252:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.260253:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.260256:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.260258:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.260261:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.260261:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.260262:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.260263:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.260264:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.260265:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.260265:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.260270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.260278:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.260280:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.260281:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.260282:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.260284:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.260287:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.260316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.260322:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.260324:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.260326:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.260327:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.260328:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.260330:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.260331:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.260332:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.260333:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.260334:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.260337:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.260339:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.260341:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.260342:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.260342:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.260344:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.260344:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.260345:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.260346:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.260351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.260359:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.260361:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.260362:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.260362:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.260364:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.260366:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.260369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.260375:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.260377:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.260378:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.260381:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.260384:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.260385:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.260386:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.260387:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.260389:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.260389:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.260393:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.260395:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.260397:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.260398:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.260399:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.260401:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.260401:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.260402:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.260402:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.260406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.260413:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.260416:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.260417:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677448.260417:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.260420:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.260423:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.2:1619677448.260567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.260575:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.260577:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.260578:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.260580:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.260583:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.260585:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.260586:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.260587:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.260588:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.260589:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.260593:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.260593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.260594:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.260598:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.260599:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.260600:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.260600:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.260602:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.260602:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.260603:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:26.0:1619677448.260604:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.260604:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.260605:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:26.0:1619677448.260606:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.260606:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.260608:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:8.2:1619677448.260608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:26.0:1619677448.260609:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.260610:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.260611:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.260612:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.260615:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.260617:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.260617:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.260619:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677448.260620:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.260620:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:20.0:1619677448.260620:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.260621:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.260622:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:20.0:1619677448.260622:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.260623:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677448.260623:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.260624:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.260624:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677448.260625:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:25.2:1619677448.260628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.260636:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.260639:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.260640:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.260640:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.260643:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.260645:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.260654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.260660:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.260663:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.260664:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.260666:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.260669:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.260670:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.260672:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.260673:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.260674:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.260675:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.260678:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.260680:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.260683:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.260683:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.260684:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.260686:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.260686:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.260687:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.260687:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.260692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.260700:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.260702:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.260703:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.260703:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.260706:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.260708:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.260709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.260716:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.260719:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.260720:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.260720:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.260721:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.260722:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.260793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:20.2:1619677448.260797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:24.0:1619677448.260800:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.260802:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:5.0:1619677448.260802:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:8.0:1619677448.260803:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.260805:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.260805:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.260807:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.260808:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.260809:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.260810:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.260810:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.260812:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.260812:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.260813:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.260814:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.260815:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.260815:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.260816:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.260817:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.260818:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.260822:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.260823:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.260826:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.260826:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.260829:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.260829:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.260830:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.260830:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.260831:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.260831:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.260833:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.260833:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.260834:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677448.260834:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.260835:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:24.0:1619677448.260836:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.260836:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677448.260836:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677448.260839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:9.2:1619677448.260840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.260847:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:17.0:1619677448.260847:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677448.260850:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:17.0:1619677448.260850:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.260851:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.260851:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677448.260851:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:18.0:1619677448.260852:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.260853:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.260854:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.260856:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677448.260857:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.260858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.260866:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.260868:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.260869:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.260870:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.260871:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.260871:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.260876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.260879:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.260881:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.260884:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.260884:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.260886:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.260888:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.260889:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.260891:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.260892:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.260893:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.260896:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.260897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677448.260898:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.260901:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.260902:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.260903:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.260905:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.260905:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677448.260906:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.260907:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:10.0:1619677448.260907:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.260908:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677448.260908:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.260908:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:10.0:1619677448.260910:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.260911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.260913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:10.0:1619677448.260913:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.260914:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.260916:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717245] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.260917:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.260918:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.260919:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677448.260919:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.260919:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.260921:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:13.0:1619677448.260922:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:10.0:1619677448.260922:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.260922:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:13.0:1619677448.260923:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.260923:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:13.0:1619677448.260924:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.260924:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.260926:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.260927:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.260927:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.260927:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.260928:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.260930:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.260930:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.260931:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.260931:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.260935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.260942:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.260944:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.260945:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.260946:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.260948:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.260949:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.261077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.261084:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.261087:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.261088:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.261089:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.261092:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.261093:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.261095:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.261096:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.261097:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.261097:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.261100:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.261101:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.261103:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.261104:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.261105:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.261106:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.261106:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.261108:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.261109:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.261109:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.261112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.261115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.261120:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.261122:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:10.0:1619677448.261122:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:30.0:1619677448.261123:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.261124:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.261124:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.261124:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.261125:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.261127:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.261128:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.261273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.261279:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.261282:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.261283:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.261285:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.261287:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.261288:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.261290:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.261291:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.261291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:18.0:1619677448.261292:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.261298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:26.0:1619677448.261299:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:18.0:1619677448.261299:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.261302:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:18.0:1619677448.261303:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.261303:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677448.261304:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:26.0:1619677448.261306:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.261306:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:18.0:1619677448.261307:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.261308:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.261308:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.261308:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.261309:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.261309:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:26.0:1619677448.261310:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:18.0:1619677448.261310:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:26.0:1619677448.261311:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.261311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677448.261311:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.261312:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:8.0:1619677448.261312:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.261313:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.261313:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000400:00000200:26.0:1619677448.261314:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.261314:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:26.0:1619677448.261315:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.2:1619677448.261316:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677448.261318:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.261319:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.261319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:30.0:1619677448.261320:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.261320:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.261321:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:30.0:1619677448.261322:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:8.0:1619677448.261322:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96717241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.261323:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.261324:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.261324:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:8.0:1619677448.261324:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.261325:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677448.261325:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.261325:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.261325:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:1.0:1619677448.261325:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:30.0:1619677448.261326:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.261327:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677448.261327:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.261327:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.261327:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:30.0:1619677448.261328:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:26.0:1619677448.261328:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:2.0:1619677448.261328:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677448.261328:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:26.0:1619677448.261329:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:17.0:1619677448.261329:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.261329:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:1.0:1619677448.261329:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.261330:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96717240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.261330:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:8.0:1619677448.261330:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.261331:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.261332:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.261332:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:30.0:1619677448.261333:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677448.261333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.261333:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:30.0:1619677448.261334:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:8.0:1619677448.261335:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.261335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677448.261336:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.0:1619677448.261336:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000400:00000200:30.0:1619677448.261337:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.261338:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (24)--
00000800:00000200:30.0:1619677448.261339:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.0:1619677448.261339:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:30.0:1619677448.261341:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.261341:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:30.0:1619677448.261342:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.261342:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:10.0:1619677448.261342:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677448.261342:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.261342:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:30.0:1619677448.261343:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.261344:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000800:00000200:30.0:1619677448.261345:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.261345:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.261345:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:2.0:1619677448.261345:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (24)++
00000800:00000200:30.0:1619677448.261346:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.261346:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.261346:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:18.0:1619677448.261346:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.261346:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:2.0:1619677448.261346:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000400:00000200:2.0:1619677448.261347:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.261348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000400:00000200:18.0:1619677448.261348:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.261349:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.261351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.261353:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.261353:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.261354:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677448.261356:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.261358:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.261359:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.261360:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677448.261360:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.261363:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.261364:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.261366:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.261366:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.261369:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.261430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.261437:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.261440:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.261440:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677448.261443:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:20.2:1619677448.261445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:10.0:1619677448.261445:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.261446:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:20.0:1619677448.261447:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:10.0:1619677448.261448:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.261449:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.261450:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:10.0:1619677448.261451:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.261452:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.261452:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.261453:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.261455:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.261455:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000400:00000200:20.0:1619677448.261457:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677448.261457:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:20.0:1619677448.261458:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.261460:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.261460:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.261461:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.261461:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:20.0:1619677448.261462:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.261462:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.261464:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.261464:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.261465:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:20.0:1619677448.261466:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.261467:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.261467:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:20.0:1619677448.261468:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.2:1619677448.261470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.261471:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.261472:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.2:1619677448.261473:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677448.261477:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.261478:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:20.0:1619677448.261479:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.261479:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.261480:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677448.261481:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677448.261481:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677448.261481:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.261481:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.261482:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.261483:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.261484:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677448.261484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:17.0:1619677448.261484:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:26.0:1619677448.261485:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:17.0:1619677448.261487:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:13.0:1619677448.261492:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.261494:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.261495:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.261497:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.261499:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.261500:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.261501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:25.2:1619677448.261504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.261506:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.261509:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.261510:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677448.261511:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677448.261512:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.261513:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.261514:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.261515:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.261515:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:1.0:1619677448.261515:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.261517:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677448.261517:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.261518:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.261519:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.261520:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.261521:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.261524:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.261526:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:21.2:1619677448.261527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:1.0:1619677448.261529:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.261529:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.261530:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.261532:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:22.0:1619677448.261533:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677448.261533:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.261534:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.261535:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677448.261536:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:1.0:1619677448.261536:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.261537:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677448.261538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:22.0:1619677448.261539:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.261541:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.261541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677448.261543:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:24.0:1619677448.261544:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:22.0:1619677448.261544:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.261545:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.261546:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.261546:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677448.261547:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:22.0:1619677448.261547:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:24.0:1619677448.261548:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.261548:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.0:1619677448.261548:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.261549:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.261550:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677448.261551:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.261551:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.261554:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.2:1619677448.261554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:24.0:1619677448.261555:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677448.261556:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.261557:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.261558:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:9.2:1619677448.261559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:22.0:1619677448.261560:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.261560:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677448.261560:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:22.0:1619677448.261561:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.261562:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677448.261563:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677448.261564:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677448.261565:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.261566:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:2.0:1619677448.261568:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.261570:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000400:00000200:2.0:1619677448.261570:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:2.0:1619677448.261571:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.261572:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.261573:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677448.261574:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.261574:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (21)++
00000800:00000200:5.0:1619677448.261574:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:2.0:1619677448.261574:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.261575:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.261576:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.261576:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.0:1619677448.261577:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.261577:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.261578:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.261579:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.261580:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.261581:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.261582:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677448.261584:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.261585:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:2.0:1619677448.261585:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677448.261587:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:2.0:1619677448.261588:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677448.261589:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677448.261589:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:8.2:1619677448.261593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677448.261599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677448.261601:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.261603:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.261604:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.261604:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.261606:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677448.261607:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.261608:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.261609:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677448.261609:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:10.0:1619677448.261611:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.261614:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.261615:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.261617:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.261618:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.261619:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.261620:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.2:1619677448.261622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:10.0:1619677448.261623:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.261625:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.261628:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.261628:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.261629:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.261629:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.261631:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:10.0:1619677448.261631:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.261631:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:10.0:1619677448.261632:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.261632:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677448.261633:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.261633:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:18.0:1619677448.261636:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.261637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:18.0:1619677448.261638:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.261639:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.261640:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.261641:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.261642:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:6.0:1619677448.261645:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:18.0:1619677448.261646:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.261647:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:22.0:1619677448.261648:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.261648:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.261649:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.261651:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.261652:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.261652:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:6.0:1619677448.261652:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.261654:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.261655:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:6.0:1619677448.261655:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677448.261656:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:25.2:1619677448.261657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.261657:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.261658:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.261661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.261662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:22.0:1619677448.261664:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.261667:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:22.0:1619677448.261667:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.261668:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.261668:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.261668:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:22.0:1619677448.261669:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.261669:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:26.0:1619677448.261670:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:2.0:1619677448.261670:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:10.0:1619677448.261671:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:2.0:1619677448.261671:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:26.0:1619677448.261672:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.261673:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:26.0:1619677448.261674:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.261677:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.261681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.261686:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.261689:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.261690:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.261690:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.261691:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.261692:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.261793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.261799:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.261802:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.261803:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.261805:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.261808:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.261809:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.261810:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.261812:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.261813:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.261813:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.261816:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.261818:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.261820:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.261821:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.261821:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.261823:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.261824:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.261825:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.261825:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.261829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.261837:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.261839:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.261840:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.261842:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.261843:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.261845:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.261849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.261857:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.261859:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.261860:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.261861:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.261861:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.261863:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.261956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.261964:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.261967:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.261968:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.261969:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.261972:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.261974:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.261975:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.261976:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.261977:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.261978:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.261982:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.261984:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.261986:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.261987:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.261987:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.261989:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.261989:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.261990:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.261990:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.261994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.262001:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:4.2:1619677448.262001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.262004:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.262005:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.262006:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.262008:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.262008:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.262009:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.262011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677448.262011:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.262012:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.262013:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.262016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:10.0:1619677448.262016:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.262017:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.262018:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.262019:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677448.262019:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.262020:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.262021:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677448.262021:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.262021:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.262024:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:31.0:1619677448.262025:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.262025:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.262025:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:30.0:1619677448.262028:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.262028:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.262029:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:30.0:1619677448.262030:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677448.262031:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.262032:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.262032:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.262033:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677448.262034:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.262034:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.262035:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.262035:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:2.0:1619677448.262035:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.262036:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.262037:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.262037:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.262037:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677448.262037:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.262038:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677448.262038:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.262039:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:2.0:1619677448.262039:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677448.262043:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.262043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:2.0:1619677448.262044:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:30.0:1619677448.262045:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.262046:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.262048:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.262048:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.262049:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.262049:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.262049:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677448.262050:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677448.262050:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.262051:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.262052:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.262052:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.262052:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677448.262053:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:6.0:1619677448.262053:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.262054:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677448.262054:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:30.0:1619677448.262055:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:6.0:1619677448.262055:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.262055:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:30.0:1619677448.262056:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677448.262056:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677448.262057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:6.0:1619677448.262057:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.262058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.262059:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.262061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:13.0:1619677448.262064:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:21.2:1619677448.262065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.262067:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:13.0:1619677448.262067:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677448.262068:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:13.0:1619677448.262068:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.262068:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:18.0:1619677448.262069:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.262069:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:13.0:1619677448.262070:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.262070:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:12.0:1619677448.262071:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.262071:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.262072:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.262072:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677448.262073:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.262073:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677448.262074:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677448.262075:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.262076:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.262077:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677448.262139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.262144:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.262147:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.262148:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.262149:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.262152:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.262154:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.262155:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.262156:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.262157:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.262158:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.262161:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.262163:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.262165:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.262166:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.262167:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.262169:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.262169:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.262170:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.262171:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.262174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.262181:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.262184:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.262184:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.262185:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.262187:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.262188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:26.0:1619677448.262190:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677448.262196:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.262198:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.262200:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.262202:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.262203:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.262205:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.262206:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.262207:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.262208:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.262209:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.262213:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.262214:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.262217:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.262218:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.262219:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.262221:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.262221:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.262222:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.262224:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.262224:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.262227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.262230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.262235:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.262237:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:30.0:1619677448.262238:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:5.0:1619677448.262238:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.262240:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.262240:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.262241:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.262242:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.262242:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.262243:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.262376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.262382:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.262385:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.262385:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.262387:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.262390:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.262392:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.262393:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.262394:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.262395:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.262397:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.262400:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.262402:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.262405:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.262406:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.262407:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.262409:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.262409:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.262410:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.262410:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.262415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.262423:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.262425:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.262426:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.262428:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.262430:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.262431:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.262500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.262506:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:21.2:1619677448.262506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:22.0:1619677448.262509:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.262510:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.262511:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.262514:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.262514:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000400:00000200:22.0:1619677448.262515:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:4.2:1619677448.262515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.262516:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:22.0:1619677448.262517:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.262519:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:22.0:1619677448.262519:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.262520:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677448.262520:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.262521:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:10.0:1619677448.262521:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.262522:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:22.0:1619677448.262525:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.262527:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:10.0:1619677448.262528:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.262530:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.262530:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.262530:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:22.0:1619677448.262531:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.262532:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:10.0:1619677448.262532:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.262533:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.262534:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.262534:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677448.262534:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.262535:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677448.262535:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.262536:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.262539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000400:00000200:10.0:1619677448.262539:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.262541:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.262541:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:26.0:1619677448.262543:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.262544:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96717222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.262545:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.262546:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.262547:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.262550:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.262551:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)++
00000400:00000200:26.0:1619677448.262552:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.262552:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (23)++
00000800:00000200:26.0:1619677448.262553:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.262554:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (24)--
00000800:00000200:10.0:1619677448.262555:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.262556:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677448.262557:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.262558:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.262558:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677448.262559:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.262559:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.262560:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.262561:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.262561:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.262561:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)++
00000800:00000200:26.0:1619677448.262562:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.262563:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.262563:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (24)--
00000400:00000200:10.0:1619677448.262563:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.262564:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)++
00000800:00000200:26.0:1619677448.262564:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (24)--
00000400:00000200:26.0:1619677448.262565:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.262565:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.262566:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.262566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:9.2:1619677448.262567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.262568:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.262569:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.262570:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:10.0:1619677448.262570:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677448.262573:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677448.262573:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.262574:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:18.0:1619677448.262575:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.262577:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.262578:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.262579:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.262581:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.262584:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.262586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.2:1619677448.262590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.262593:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:2.0:1619677448.262593:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.262596:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:18.0:1619677448.262597:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.262597:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.262598:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.262598:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000400:00000200:20.0:1619677448.262599:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.262599:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:2.0:1619677448.262599:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:20.0:1619677448.262601:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.262602:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677448.262604:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.262605:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.262606:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.262607:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.262610:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.262612:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.262614:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.262615:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.262615:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.262617:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.262618:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.262618:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.262619:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677448.262624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.262632:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.262635:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.262636:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.262636:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.262638:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677448.262640:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.262656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.262661:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.262663:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.262664:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.262666:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.262668:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.262669:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.262670:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.262672:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.262672:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.262673:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.262676:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.262678:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.262680:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.262681:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.262681:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.262683:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.262684:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.262684:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.262685:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.262689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.262695:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.262697:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.262698:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.262699:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.262699:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.262701:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.262706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.262713:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.262715:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.262716:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.262716:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.262717:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.262717:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.262763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.262772:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.262775:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.262777:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.262779:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.262780:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.262782:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.262783:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.262784:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.262785:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.262786:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.262790:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.262791:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.262794:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.262795:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.262796:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.262797:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.262798:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.262799:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.262799:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.262803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.262811:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.262814:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.262815:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.262816:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.262817:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.262819:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.262932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.2:1619677448.262938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:1.0:1619677448.262938:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.262940:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.262941:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.262943:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.262944:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:1.0:1619677448.262945:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.262947:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:5.2:1619677448.262947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:1.0:1619677448.262947:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.262948:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.262949:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.262950:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:1.0:1619677448.262950:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.262951:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.262951:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.262952:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:1.0:1619677448.262953:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677448.262954:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.262956:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.262957:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.262957:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.262958:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.262959:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.262960:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.262960:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:18.0:1619677448.262961:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:4.2:1619677448.262962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:1.0:1619677448.262963:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.262964:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.262965:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000400:00000200:2.0:1619677448.262966:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:1.0:1619677448.262966:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:18.0:1619677448.262967:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.262967:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96717216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.262967:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.262968:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.262968:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.262969:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.262969:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.262969:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677448.262970:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:2.0:1619677448.262970:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.262971:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.262972:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.262973:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677448.262974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:2.0:1619677448.262975:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.262978:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.262978:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.262980:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.262980:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.262981:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677448.262981:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:2.0:1619677448.262981:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:24.0:1619677448.262982:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.262982:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677448.262982:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:2.0:1619677448.262982:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.262984:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.262984:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:8.0:1619677448.262984:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.262984:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.262984:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)++
00000400:00000200:24.0:1619677448.262985:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.262986:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96717215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.262986:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.262986:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:8.0:1619677448.262986:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.262986:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000400:00000200:24.0:1619677448.262987:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.262987:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677448.262987:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:24.0:1619677448.262988:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.262990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677448.262990:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:9.2:1619677448.262990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:24.0:1619677448.262994:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.262995:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.262996:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:13.0:1619677448.262997:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677448.262997:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:2.0:1619677448.262998:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677448.262999:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.263000:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:13.0:1619677448.263000:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.263000:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:24.0:1619677448.263001:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677448.263001:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.263002:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:13.0:1619677448.263002:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.263003:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.263003:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:12.0:1619677448.263004:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677448.263005:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:2.0:1619677448.263005:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.263006:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.263006:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:13.0:1619677448.263006:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.263008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.263008:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677448.263010:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.263012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.263016:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.263019:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677448.263019:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.263020:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.263021:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.263021:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.263022:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.263023:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.263023:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677448.263026:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.263086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.263094:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.263097:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.263099:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.263100:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.263101:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.263103:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.263104:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.263106:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.263106:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.263107:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.263110:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.263112:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.263114:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.263115:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.263115:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.263117:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.263117:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.263118:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.263119:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.263122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.263129:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.263131:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.263132:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.263133:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.263135:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.263136:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.263140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.263147:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.263149:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.263150:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.263151:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.263151:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.263153:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.263245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.263251:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.263254:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.263255:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.263257:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.263259:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.263261:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.263262:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.263264:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.263265:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.263266:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.263269:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.263270:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.263273:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.263274:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.263275:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.263276:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.263277:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.263278:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.263279:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.263280:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.263282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.263285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.263290:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677448.263290:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.263292:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677448.263293:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677448.263294:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677448.263294:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.263299:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.263300:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.263302:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.263305:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.263465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.2:1619677448.263465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.263471:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.263473:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:30.0:1619677448.263474:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.263474:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.263476:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.263477:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:1.0:1619677448.263478:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.263480:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.263480:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:31.0:1619677448.263482:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.263482:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.263482:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.263483:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.263484:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.263484:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.263485:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677448.263486:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.263488:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.263489:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.263490:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.263491:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.263492:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:30.0:1619677448.263495:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.263497:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.263498:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.263499:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.263499:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.263500:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.263501:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.263501:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.263501:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.263501:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.263502:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.263502:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.263503:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.263504:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.263505:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.263505:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.263507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.2:1619677448.263509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.2:1619677448.263510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.263515:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:31.0:1619677448.263517:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:22.0:1619677448.263518:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:31.0:1619677448.263519:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:22.0:1619677448.263519:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.263519:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677448.263519:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.263521:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.263521:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:31.0:1619677448.263522:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.263523:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:17.0:1619677448.263523:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.263523:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.263524:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.263525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000400:00000200:17.0:1619677448.263525:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.263526:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.263527:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.263529:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.263530:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.263531:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.263532:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:6.0:1619677448.263532:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.263534:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:31.0:1619677448.263536:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.263536:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.263536:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.263537:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.263538:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.263538:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677448.263539:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.263541:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.263542:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.263542:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.263544:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.263544:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.263545:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.263546:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.263550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.263557:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.263559:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.263560:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.263562:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.263564:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.263565:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.263565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.263573:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.263576:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.263576:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.263578:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.263581:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.263582:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.263583:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.263585:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.263586:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.263587:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.263590:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.263591:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.263594:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.263595:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.263596:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.263598:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.263598:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.263599:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.263600:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.263601:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.263604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.263607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.263613:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:10.0:1619677448.263614:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:20.0:1619677448.263615:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.263616:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.263616:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.263616:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.263617:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:20.0:1619677448.263618:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.263620:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.263621:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.263621:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.263622:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.263623:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (23)++
00000800:00000200:5.0:1619677448.263623:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.263624:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (24)--
00000800:00000200:20.0:1619677448.263625:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677448.263625:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:21.2:1619677448.263672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.263677:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.263680:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.263680:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.263682:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.263685:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.263687:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.263688:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.263689:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.263690:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.263691:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.263695:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.263697:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.263699:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.263700:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.263701:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.263703:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.263703:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.263705:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.263706:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.263706:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.263709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.263711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.263717:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:1.0:1619677448.263718:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677448.263719:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.263719:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.263720:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677448.263720:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.263722:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.263722:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.263724:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.263727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.263729:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677448.263730:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.263732:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.263733:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.263735:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.263737:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.263738:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.263739:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.263741:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.263742:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.263743:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.263746:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.263748:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.263750:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.263751:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.263752:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.263753:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.263754:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.263754:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.263755:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.263759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.263767:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.263769:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.263769:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.263771:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.263771:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.263772:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.263778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.263785:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.263787:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.263788:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.263788:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.263789:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.263789:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.263846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.263853:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.263856:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.263858:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.263859:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.263860:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.263862:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.263863:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.263864:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.263865:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.263866:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.263870:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.263871:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.263874:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.263875:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.263875:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.263877:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.263877:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.263878:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.263879:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.263882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.263889:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.263892:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.263892:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.263894:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.263896:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.263896:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.2:1619677448.263971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.263977:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.263979:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.263981:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.263982:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.263984:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.263986:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.263987:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.263988:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.263989:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.263990:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.263994:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.263996:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.263999:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.264000:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.264000:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.264002:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.264003:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.264003:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.264004:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.264008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.264015:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.264017:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.264018:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.264020:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.264021:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.264022:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.264208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.264216:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.264219:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:21.2:1619677448.264220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:31.0:1619677448.264221:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.264223:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.264223:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.264224:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:30.0:1619677448.264224:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:31.0:1619677448.264225:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.264225:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.264227:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.264228:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.264229:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.264231:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.264233:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.264234:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.264235:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.264236:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.264237:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.264238:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.264239:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.264240:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.264242:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.264243:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:30.0:1619677448.264243:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.264244:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.264245:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.264245:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.264246:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.264247:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.264247:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.264247:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.264248:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.264249:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.264249:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.264250:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.264251:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677448.264251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.264252:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.264253:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.264254:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.264255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:13.0:1619677448.264257:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.264259:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:21.2:1619677448.264260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:13.0:1619677448.264260:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.264262:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.264262:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:12.0:1619677448.264263:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.264264:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.264265:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:13.0:1619677448.264265:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:10.0:1619677448.264265:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.264267:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:10.0:1619677448.264267:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.264269:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.264270:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677448.264270:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677448.264271:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677448.264341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.264348:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.264351:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.264351:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.264352:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677448.264354:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.264355:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.264357:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.264358:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.264359:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.264360:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.264361:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.264362:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.264363:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.264364:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.264366:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.264367:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.264367:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.264368:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.264369:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.264369:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.264372:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.264374:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.264376:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.264376:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.264376:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.264377:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.264377:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.264378:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.264379:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.264379:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)++
00000800:00000200:26.0:1619677448.264380:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.264380:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (24)++
00000800:00000200:10.0:1619677448.264380:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (24)--
00000800:00000200:26.0:1619677448.264381:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (24)--
00000800:00000200:10.0:1619677448.264381:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.264382:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.264382:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.264384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.2:1619677448.264387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:25.2:1619677448.264388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.264392:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.264394:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.264394:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.264396:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.264396:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.264396:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.264397:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.264398:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.264399:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:22.0:1619677448.264399:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:10.0:1619677448.264399:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.264400:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677448.264400:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.264400:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:10.0:1619677448.264401:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.264403:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.264404:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.264405:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.264407:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.264408:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.264408:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.264411:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:10.0:1619677448.264412:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.264414:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.264417:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.264418:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.264418:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.264419:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.264420:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.264421:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.264421:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.264425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.264432:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.264435:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.264436:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.264438:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.264440:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.264440:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.264526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:5.2:1619677448.264529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:6.0:1619677448.264531:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.264534:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.264535:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677448.264536:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:6.0:1619677448.264536:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.264538:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:6.0:1619677448.264539:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.264540:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677448.264541:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.264542:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.264543:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.264544:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.264545:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.264545:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.264546:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677448.264547:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.264548:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.264550:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.264551:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.264552:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677448.264552:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.264554:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.264556:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.264558:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.264558:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.264559:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677448.264559:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.264560:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.264561:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:6.0:1619677448.264561:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.264562:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677448.264562:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.264563:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:6.0:1619677448.264563:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677448.264564:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.264564:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677448.264565:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.264565:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.264567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.2:1619677448.264570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677448.264575:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:1.0:1619677448.264577:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:31.0:1619677448.264578:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.264579:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.264580:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.264580:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.264580:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.264581:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677448.264582:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:1.0:1619677448.264582:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.264583:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:1.0:1619677448.264584:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.264717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.264725:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.264728:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.264730:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.264731:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.264732:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.264734:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.264735:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.264736:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.264737:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.264738:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.264742:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.264744:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.264747:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.264748:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.264749:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.264750:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.264751:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.264752:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.264752:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.264756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.264765:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.264768:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.264769:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.264769:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.264771:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.264775:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.264780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.264787:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.264790:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.264791:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.264791:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.264792:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.264792:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.264806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.264811:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.264813:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.264814:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.264816:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.264818:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.264819:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677448.264820:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.264821:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.264822:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.264823:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677448.264827:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.264829:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.264831:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.264832:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677448.264833:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677448.264834:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.264835:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.264835:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.264836:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.264841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.264848:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.264850:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.264851:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.264852:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.264854:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.264857:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.264884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677448.264887:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677448.264890:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677448.264892:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.264893:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677448.264895:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.264897:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:20.0:1619677448.264898:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.264899:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.264900:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.264901:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:20.0:1619677448.264904:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.264906:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:20.0:1619677448.264909:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.264909:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:20.0:1619677448.264910:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:20.0:1619677448.264912:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:20.0:1619677448.264913:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677448.264913:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677448.264914:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:25.2:1619677448.264918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.264924:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.264927:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.264928:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.264928:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.264930:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.264933:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:25.2:1619677448.264938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000800:00000200:4.2:1619677448.264944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.264948:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.264950:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.264951:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.264952:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.264952:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.264953:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:10.0:1619677448.264954:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.264955:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.264956:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.264958:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.264959:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.264961:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.264962:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.264963:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.264964:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.264965:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.264967:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.264969:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.264972:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.264973:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.264974:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.264975:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.264976:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.264977:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.264977:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.264981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.264989:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.264992:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.264993:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.264993:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.264995:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.264998:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.265021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.265029:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.265032:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.265034:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.265035:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.265037:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.265038:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.265039:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.265040:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.265041:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.265041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:31.0:1619677448.265042:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.265045:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.265046:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:31.0:1619677448.265047:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.265049:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.265049:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:31.0:1619677448.265050:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.265050:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.265051:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.265051:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.265052:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.265052:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.265054:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:5.0:1619677448.265054:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.265055:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.265056:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:5.0:1619677448.265056:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.265057:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.265058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:5.0:1619677448.265058:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.265059:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.265060:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.2:1619677448.265061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:5.0:1619677448.265063:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.265065:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677448.265065:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.265067:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.265068:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.265068:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.265068:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.265069:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.265070:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:13.0:1619677448.265070:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.265071:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677448.265071:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.265072:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:13.0:1619677448.265072:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677448.265072:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677448.265073:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677448.265073:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.265073:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677448.265074:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.265077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:25.2:1619677448.265079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:12.0:1619677448.265084:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.265085:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.265085:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.265086:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.265087:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.265088:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.265088:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.265088:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.265089:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.265090:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.265091:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677448.265092:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.265223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.265229:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.265232:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.265233:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.265235:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.265237:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.265238:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.265240:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.265241:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.265242:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.265243:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.265246:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.265248:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.265251:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.265252:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.265253:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.265254:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.265255:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.265256:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.265257:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.265258:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.265260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.265263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.265266:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677448.265268:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677448.265269:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677448.265270:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.265271:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.265272:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677448.265272:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.265273:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677448.265274:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.265276:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.265307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.265315:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.265317:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.265319:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.265321:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.265322:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.265323:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677448.265324:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.265325:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.265326:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.265327:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677448.265330:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.265332:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.265335:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.265335:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.265336:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.265338:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.265338:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.265339:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.265340:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.265345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677448.265349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.0:1619677448.265352:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:1.0:1619677448.265353:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677448.265355:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:1.0:1619677448.265355:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.265356:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:20.0:1619677448.265357:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.265357:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.265358:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:1.0:1619677448.265359:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.265360:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.265361:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:8.2:1619677448.265361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677448.265363:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.265364:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.265364:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:20.0:1619677448.265365:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.265365:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677448.265366:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.265368:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.265368:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.265369:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000400:00000200:20.0:1619677448.265370:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.265370:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.265370:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.265371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677448.265373:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.265375:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.265376:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.265377:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.265378:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.265378:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.265379:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677448.265380:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.0:1619677448.265380:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:26.0:1619677448.265381:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.265382:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.265384:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.265384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:26.0:1619677448.265387:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.265388:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.265389:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.265391:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.265391:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:26.0:1619677448.265392:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.265393:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.265394:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.265395:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.265396:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.265397:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.265397:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.265398:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.265399:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:26.0:1619677448.265400:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.265401:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.265402:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.265403:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.265404:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.265405:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.265406:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.265407:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.265409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:4.2:1619677448.265412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.2:1619677448.265418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.265420:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.265421:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.265422:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:10.0:1619677448.265422:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.265423:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.265424:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.265424:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.265425:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.265426:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.265426:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.265426:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.265428:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:26.0:1619677448.265429:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.265429:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.265431:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.265431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000400:00000200:26.0:1619677448.265432:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.265434:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.265435:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.265437:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.265438:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.265438:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.265439:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.265440:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000400:00000200:26.0:1619677448.265441:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.265441:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.265442:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.265443:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.265443:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.265446:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677448.265446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:26.0:1619677448.265447:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.265447:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.265449:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.265449:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.265450:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.265450:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677448.265452:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:25.2:1619677448.265454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677448.265455:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.265456:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.265458:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.265461:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.265461:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:22.0:1619677448.265462:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.265464:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.265464:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:22.0:1619677448.265465:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.265465:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.265466:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.265468:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:13.0:1619677448.265468:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.265468:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.265470:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:22.0:1619677448.265471:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.265473:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.265476:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.265477:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.265477:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.265479:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.265479:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.265480:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.265481:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.265485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.265493:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.265496:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.265497:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.265497:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.265499:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.265502:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.265509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.265516:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.265519:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.265520:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.265522:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.265525:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.265526:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.265528:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.265529:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.265530:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.265531:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.265535:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.265536:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.265539:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.265540:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.2:1619677448.265540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:24.0:1619677448.265541:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.265543:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.265543:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:24.0:1619677448.265544:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.265545:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.265545:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677448.265545:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.265547:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.265548:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:9.2:1619677448.265549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677448.265550:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.265551:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.265553:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.265554:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.265555:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.265556:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.265557:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:20.0:1619677448.265559:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.265559:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.265560:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.265561:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.265561:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.265562:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.265564:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.265565:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.265566:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.265566:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677448.265568:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.265568:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.265569:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.265570:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.265574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.265580:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.265583:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.265584:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.265585:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.265586:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.265588:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.265595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.265602:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.265604:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.265605:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.265606:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.265606:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.265608:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.265724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.265730:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.265733:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.265734:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.265736:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.265739:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.265740:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.265742:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.265743:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677448.265743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:17.0:1619677448.265744:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.265745:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.265748:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.265749:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.265750:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.265752:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.265753:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.265754:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.265754:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.265754:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.265756:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:10.0:1619677448.265756:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.265757:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.265757:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.265758:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:10.0:1619677448.265758:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.265760:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.265761:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.265762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:10.0:1619677448.265762:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.265763:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.265764:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.265768:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.265769:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.2:1619677448.265769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:10.0:1619677448.265770:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.265772:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.265772:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.265772:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.265773:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:31.0:1619677448.265774:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.265774:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.265775:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.265775:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.265775:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:10.0:1619677448.265776:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.265776:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.265777:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677448.265777:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677448.265778:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.265779:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.265780:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.265781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:2.0:1619677448.265783:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.265784:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.265785:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.265787:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.265787:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.265789:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:2.0:1619677448.265789:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.265792:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.265792:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:2.0:1619677448.265792:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.265793:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.265794:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.265795:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.265797:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.265797:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.265798:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.265799:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.265800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.265800:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.265800:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.265801:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.265802:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.265806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.265807:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.265810:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.265810:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.265811:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.265812:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.265812:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677448.265814:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.265816:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.265817:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.265819:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.265819:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.265822:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.265846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.265854:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.265857:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.265857:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.265860:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.265863:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.265864:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.265865:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.265866:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.265868:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.265868:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.265872:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.265873:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.265875:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.265876:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.265876:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.265878:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.265878:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.265879:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.265880:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.265880:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.265883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.2:1619677448.265886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.265891:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.265892:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.265893:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.265894:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.265895:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:10.0:1619677448.265895:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.265896:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.265896:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.265897:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.265897:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:4.2:1619677448.265997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.266005:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.266007:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.266008:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.266010:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.266013:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.266014:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.266016:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.266017:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.266018:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.266019:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.266022:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.266023:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.266026:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.266027:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.266027:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.266029:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.266029:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.266030:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.266030:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.266035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677448.266037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.266040:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.266042:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677448.266043:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:20.0:1619677448.266045:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.266045:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.266046:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.266046:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:1.0:1619677448.266046:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.266047:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.266049:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677448.266049:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.266051:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.266052:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.266052:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:20.0:1619677448.266053:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.266054:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.2:1619677448.266056:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:20.0:1619677448.266062:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.266064:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677448.266064:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.266066:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.266067:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:20.0:1619677448.266067:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.266067:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.266068:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:20.0:1619677448.266069:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.266069:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677448.266070:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.266071:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:26.0:1619677448.266072:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.266072:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000400:00000200:26.0:1619677448.266073:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:20.0:1619677448.266073:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677448.266075:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.266075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:26.0:1619677448.266076:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.266077:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.266078:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:20.2:1619677448.266078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677448.266081:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.266082:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677448.266082:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.266083:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:26.0:1619677448.266085:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.266085:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677448.266085:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:26.0:1619677448.266086:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:6.0:1619677448.266086:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.266086:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677448.266087:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677448.266087:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.266088:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:6.0:1619677448.266088:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.266088:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677448.266089:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.266090:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.266090:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:6.0:1619677448.266091:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.2:1619677448.266094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.266102:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.266105:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.266106:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.266106:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.266108:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.266110:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.266243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.266250:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.266253:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.266254:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.266256:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.266259:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.266260:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.266261:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.266262:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.266263:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.266264:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.266267:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.266268:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.266270:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.266271:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.266271:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.266273:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.266273:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.266274:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.266275:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.266279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.266286:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.266288:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.266289:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.266291:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.266292:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.266298:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.266302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.2:1619677448.266305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677448.266308:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.266311:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.266312:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.266312:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.266313:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.266313:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.266314:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.266316:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.266317:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.266318:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.266321:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.266322:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.266323:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.266324:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.266325:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.266326:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.266329:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.266330:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.266332:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.266333:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.266333:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.266335:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.266335:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.266336:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.266336:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.266340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.266347:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.266349:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.266350:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.266351:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.266353:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.266354:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.266403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.266411:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.266413:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.266414:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.266416:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.266418:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.266420:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.266421:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.266422:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.266423:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.266424:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677448.266427:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.266429:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.266432:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.266432:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.266433:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.266435:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.266435:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.266436:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.266436:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.266440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.266449:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.266451:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.266452:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.266454:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.266455:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.266457:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.266533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.266539:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.266542:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.266543:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.266545:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.266548:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.266549:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.266550:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.266552:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.266553:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.266554:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.266557:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.266559:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.266561:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.266562:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.266563:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.266564:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.266565:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.266566:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.266567:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.266568:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.266570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.266573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677448.266578:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677448.266578:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:20.0:1619677448.266580:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:2.0:1619677448.266580:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.266581:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:20.0:1619677448.266582:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.266582:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:20.0:1619677448.266583:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.266584:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.266586:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.266679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.266687:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.266689:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.266690:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.266692:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.266694:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.266696:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.266697:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.266699:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.266699:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.266701:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.266704:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.266706:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.266708:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.266709:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.266710:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.266711:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.266711:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.266712:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.266713:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.266716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.266723:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.266726:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.266726:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.266728:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.266729:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.266731:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.266736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.266742:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.266744:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.266745:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.266746:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.266746:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.266748:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.266762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.266767:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.266770:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.266772:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.266773:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.266775:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.266777:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.266778:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.266779:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.266781:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.266782:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.266785:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.266787:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.266790:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.266791:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.266791:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.266793:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.266794:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.266795:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.266795:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.266799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.266806:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.266808:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.266809:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.266811:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.266812:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.266814:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.266877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.266883:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.266886:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.266887:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.266889:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.266891:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.266893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:18.0:1619677448.266893:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.266894:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.266895:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.266896:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.266897:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.266900:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677448.266900:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.266902:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.266903:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:18.0:1619677448.266904:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.266905:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:31.0:1619677448.266906:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.266906:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.266907:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677448.266907:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:31.0:1619677448.266908:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.266908:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.266908:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.266909:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:31.0:1619677448.266910:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.266911:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.266912:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.266913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:31.0:1619677448.266914:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.266915:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.2:1619677448.266916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:31.0:1619677448.266918:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.266920:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.266921:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:31.0:1619677448.266922:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.266922:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.266923:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.266924:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.266924:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677448.266925:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:22.0:1619677448.266925:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.266925:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677448.266925:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677448.266926:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.266926:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.266927:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:22.0:1619677448.266927:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:17.0:1619677448.266927:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.266929:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.266930:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.266931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677448.266932:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.266933:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.266935:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.266936:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.266937:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.266938:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677448.266938:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.266941:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.266941:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.266941:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677448.266944:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.266944:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.266945:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.266947:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.266947:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.266948:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.266949:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.266951:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.266951:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.266952:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.266952:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.266956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.266963:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.266965:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.266966:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.266968:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.266969:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.266971:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.267019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.267026:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.267029:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.267030:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.267032:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.267034:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.267036:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.267037:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.267038:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.267040:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.267040:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.2:1619677448.267043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:8.0:1619677448.267044:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.267045:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.267048:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.267049:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.267049:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.267051:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:8.0:1619677448.267051:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.267051:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.267052:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.267052:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:30.0:1619677448.267053:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.267055:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.267056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.267057:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.267057:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.267059:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.267060:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717166] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.267061:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.267062:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.267062:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:30.0:1619677448.267063:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:12.0:1619677448.267065:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.267065:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.267066:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677448.267067:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.267068:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.267069:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.267069:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.267070:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:12.0:1619677448.267070:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.267071:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.2:1619677448.267071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:30.0:1619677448.267072:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.267073:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.267074:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.267074:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:2.0:1619677448.267077:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:25.2:1619677448.267078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:2.0:1619677448.267080:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.267081:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.267083:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.267084:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.267086:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:2.0:1619677448.267086:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.267087:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.267087:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.267089:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.267089:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.267090:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:2.0:1619677448.267091:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.267092:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:2.0:1619677448.267092:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.267093:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.267096:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.267098:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.267101:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.267102:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.267103:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.267105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:2.0:1619677448.267105:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.267106:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.267106:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.267107:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:6.0:1619677448.267110:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:25.2:1619677448.267111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.267113:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.267114:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.267116:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.267117:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677448.267118:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.267120:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:6.0:1619677448.267120:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:13.0:1619677448.267121:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.267121:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677448.267123:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.267123:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:6.0:1619677448.267123:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.267124:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.267125:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.267126:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:6.0:1619677448.267129:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.267131:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.267133:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.267134:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.267135:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.267137:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.267137:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.267138:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.267139:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.267143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.267151:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.267153:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.267154:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.267156:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.267157:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.267158:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.267167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.267174:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.2:1619677448.267175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:26.0:1619677448.267177:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.267178:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.267180:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.267180:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:26.0:1619677448.267182:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.267182:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:26.0:1619677448.267184:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.267185:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.267185:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.267185:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.267186:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.267188:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.267188:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677448.267188:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.267190:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677448.267191:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.267192:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.267193:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.267194:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677448.267199:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.267201:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.267202:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.267204:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.267205:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.267206:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.267207:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.267207:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.267208:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.267209:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.267209:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.267210:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.267210:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:26.0:1619677448.267211:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.267211:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.267211:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:26.0:1619677448.267212:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.267212:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.267213:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.267215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:8.2:1619677448.267216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.267219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677448.267222:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.267224:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.267225:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.267225:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.267226:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000400:00000200:30.0:1619677448.267227:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.267227:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.267227:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.267228:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.267228:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.267229:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677448.267230:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.267230:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.267231:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.0:1619677448.267231:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:5.0:1619677448.267233:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.267412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.267420:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.267423:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.267423:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.267425:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.267428:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.267430:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.267431:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.267432:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.267434:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.267434:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.267438:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.267439:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.267443:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.267443:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.267444:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.267446:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.267446:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.267447:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.267448:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.267453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.267461:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.267464:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.267465:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.267465:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.267467:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.267470:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.267495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.267503:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.267506:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.267508:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.267510:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.267510:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.267512:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.267513:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.267514:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.267515:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.267516:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.267520:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.267521:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.267524:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.267525:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.267526:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.267527:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.267528:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.267529:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.267529:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.267534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.267542:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.267544:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.267545:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.267547:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.267548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:5.0:1619677448.267549:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.267550:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.267553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.267558:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.267560:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.267561:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677448.267561:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000400:00000200:24.0:1619677448.267564:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.267564:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.267565:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.267566:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.267566:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:24.0:1619677448.267567:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.267568:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:20.0:1619677448.267568:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:24.0:1619677448.267569:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.267571:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.267571:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.267572:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.267576:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.267578:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.267580:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.267581:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.267581:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.267583:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.267583:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.267585:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.267586:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.267586:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.2:1619677448.267589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:21.2:1619677448.267592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:12.0:1619677448.267596:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.267598:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.267599:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.267599:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:12.0:1619677448.267600:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.267601:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.267601:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677448.267602:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.2:1619677448.267602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.0:1619677448.267603:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677448.267604:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.267607:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.267608:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.267608:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.267608:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.267609:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.267610:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677448.267611:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.267611:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.267614:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.267617:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.267618:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.267619:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.267620:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.267622:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.267623:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.267626:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.267627:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.267629:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.267630:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.267630:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.267632:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.267632:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.267633:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.267633:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.267638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.267645:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.267647:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.267648:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.267648:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.267649:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.267651:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.267941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.267949:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.267952:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.267954:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.267956:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.267957:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.267958:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.267960:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.267961:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.267962:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.267963:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.267966:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.267967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.267968:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.267971:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.267971:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:21.2:1619677448.267971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:30.0:1619677448.267972:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677448.267973:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.267974:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.267975:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.267976:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.267976:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:17.0:1619677448.267976:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:6.0:1619677448.267976:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677448.267977:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.267979:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.267979:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.267980:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:17.0:1619677448.267980:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.267980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:17.0:1619677448.267982:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.267982:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.267983:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717156] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.267984:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.267985:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.267985:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.267986:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677448.267987:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.267988:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.267989:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.267989:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:17.0:1619677448.267991:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.267991:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.267991:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:6.0:1619677448.267992:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:5.0:1619677448.267992:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.267994:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:17.0:1619677448.267994:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.267994:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.267996:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.267997:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.267997:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:6.0:1619677448.267997:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.267998:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:17.0:1619677448.267998:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.267999:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:20.0:1619677448.267999:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:17.0:1619677448.267999:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.267999:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.268000:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.268000:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.268001:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677448.268001:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677448.268001:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.268002:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677448.268002:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:17.0:1619677448.268003:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.268003:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.268004:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:8.2:1619677448.268005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.268005:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.268006:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.268007:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.268008:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.2:1619677448.268008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.2:1619677448.268010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.268013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.268014:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.268014:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.268015:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.268015:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.268016:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.268016:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.268017:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.268018:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:1.0:1619677448.268018:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.268019:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:5.0:1619677448.268019:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:1.0:1619677448.268019:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677448.268021:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:5.0:1619677448.268021:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677448.268022:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:5.0:1619677448.268022:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.268023:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677448.268024:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.268024:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.268025:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.268159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.268166:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.268169:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.268170:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.268172:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.268175:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.268176:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.268178:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.268179:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.268180:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.268181:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.268185:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.268186:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.268189:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.268190:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.268191:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.268193:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.268193:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.268195:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.268196:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.268196:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.268199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.268202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.268207:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:10.0:1619677448.268209:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:20.0:1619677448.268210:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.268210:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.268211:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:20.0:1619677448.268212:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.268212:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.268213:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.268213:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.268215:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.268270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.268275:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677448.268276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:17.0:1619677448.268278:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.268279:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.268280:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.268282:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.268283:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:17.0:1619677448.268284:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.268285:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.268286:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:17.0:1619677448.268287:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.268287:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:17.0:1619677448.268288:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.268290:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677448.268291:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.268299:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.268301:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:24.0:1619677448.268302:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.268303:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:17.0:1619677448.268304:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677448.268305:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96717152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.268305:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.268305:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:24.0:1619677448.268306:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.268307:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.268307:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.268308:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.268308:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.268308:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.268309:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677448.268311:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.268313:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:9.2:1619677448.268314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.268316:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.268317:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.268318:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.268320:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.268320:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.268321:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.268322:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677448.268322:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.268323:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.268323:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.268324:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.268325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:8.0:1619677448.268326:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.268329:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677448.268333:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.268335:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.268336:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.268338:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.268339:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.268340:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.268584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.268591:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.268594:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.268595:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.268597:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.268599:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.268601:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.268602:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.268603:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.268604:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.268605:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.268608:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.268610:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.268612:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.268613:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.268613:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.268615:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.268615:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.268616:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.268616:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.268620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.268628:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.268630:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.268631:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.268632:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.268635:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.268636:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.268646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.268653:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.268656:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.268658:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.268660:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.268660:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.268662:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.268662:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.268664:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.268665:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.268665:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.268668:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.268670:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.268672:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.268673:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.268673:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.268675:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.268675:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.2:1619677448.268676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677448.268677:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.268678:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.268679:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.268680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.268681:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677448.268684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.268684:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.268685:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.268687:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677448.268687:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.268689:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.268690:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.268690:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.268692:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.268692:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:30.0:1619677448.268693:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677448.268693:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:1.0:1619677448.268693:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717149] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.268695:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:13.0:1619677448.268695:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677448.268695:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.268696:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:1.0:1619677448.268696:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.268697:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:1.0:1619677448.268697:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:25.2:1619677448.268699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:1.0:1619677448.268700:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.268702:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.268705:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.268706:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.268706:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.268706:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.268708:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:12.0:1619677448.268709:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.268709:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.268709:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677448.268710:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.268710:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.268711:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.268712:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.268713:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.268714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.268722:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.268725:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:5.2:1619677448.268725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:22.0:1619677448.268726:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.268726:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.268728:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.268731:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677448.268732:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.268735:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.268735:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.268738:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.268741:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.268742:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.268744:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.268745:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.268746:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.268747:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.268750:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.268752:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.268753:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.268754:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.268755:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.268756:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.268756:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.268757:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.268758:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.268762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.268769:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.268771:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.268771:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.268773:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.268773:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.268775:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.268894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.268902:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.268905:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.268907:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.268909:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.268909:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.268911:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.268912:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.268914:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.268915:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.268916:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.268919:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.268920:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.268923:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.268924:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.268925:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.268927:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.268927:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.268929:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.268930:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.268931:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.268933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.268937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.268942:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.268944:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677448.268945:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:5.0:1619677448.268945:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.268947:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:20.0:1619677448.268947:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.268947:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.268948:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.268949:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.268949:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.269026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.269033:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.269036:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.269037:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.269038:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.269041:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.269042:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.269044:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.269045:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.269046:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.269046:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.269050:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.269051:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.269054:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.269055:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.269055:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.269057:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.269057:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.269058:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.269058:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.269063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.269071:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.269074:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.269074:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.269075:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.269077:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.269080:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.269376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.269381:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.269384:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.269385:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.269387:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.269389:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.269391:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.269392:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.269393:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.269394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:17.0:1619677448.269395:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.269396:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.269399:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.269400:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.269401:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.269403:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:17.0:1619677448.269404:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.269404:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677448.269405:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.269405:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:22.0:1619677448.269406:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.269407:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:22.0:1619677448.269408:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.269408:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.269408:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677448.269409:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:17.0:1619677448.269409:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677448.269411:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.269412:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.269413:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.269414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:22.0:1619677448.269414:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.269417:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.269419:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.269421:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.269422:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:13.0:1619677448.269422:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:22.0:1619677448.269423:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677448.269424:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:4.2:1619677448.269424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677448.269425:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.269425:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:13.0:1619677448.269425:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.269426:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.269427:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:13.0:1619677448.269427:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.269429:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.269430:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.2:1619677448.269431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.269434:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.269437:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.269438:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677448.269439:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:24.0:1619677448.269440:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.269441:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:24.0:1619677448.269442:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.269442:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.269443:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.269445:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.269445:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.269446:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96717143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.269448:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:24.0:1619677448.269449:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677448.269449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000400:00000200:24.0:1619677448.269450:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.269451:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.269455:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.269456:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.269457:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.269459:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.269460:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.269460:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.269461:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.269461:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.269462:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.269462:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.269463:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677448.269464:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677448.269465:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.269466:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.269466:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.2:1619677448.269467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.2:1619677448.269469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.269475:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.269477:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.269478:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.269480:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.269480:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.269480:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.269481:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.269482:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.269482:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:1.0:1619677448.269483:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.269484:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.269485:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.269486:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:31.0:1619677448.269487:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.269488:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.269489:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.269492:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.269494:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.269496:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.269496:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.269497:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.269498:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.269499:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.269500:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.269501:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.269504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.269510:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.269513:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.269514:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.269515:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.269517:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.269517:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.269614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.269621:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.269624:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.269625:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.269626:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.269630:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.269631:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.269632:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.269634:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.269635:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.269635:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.269639:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.269640:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.269643:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.269644:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.269644:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.269646:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.269646:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.269647:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.269647:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.269652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.269656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:6.0:1619677448.269659:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.269662:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.269663:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.269664:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:6.0:1619677448.269664:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.269666:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.269667:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:6.0:1619677448.269668:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:30.0:1619677448.269669:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.269670:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.269670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000400:00000200:30.0:1619677448.269671:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.269673:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.269674:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.269675:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.269676:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.269677:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.269677:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:30.0:1619677448.269680:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.269681:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.269681:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.269682:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.269682:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.269683:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.269683:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677448.269684:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.269685:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.269686:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.269687:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.269688:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.269689:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.269690:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.269693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.269699:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.269701:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.269702:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.269704:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.269705:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.269706:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.269750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.269757:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.269759:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.269760:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.269762:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.269764:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.269766:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.269767:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.269768:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.269770:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.269771:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.269774:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.269775:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.269777:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.269778:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.269779:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.269781:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.269781:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.269782:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.269782:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.269786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.269794:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.269797:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.269797:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.269798:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.269800:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677448.269801:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.269856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.269862:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.269864:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.269865:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.269867:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.269869:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.269871:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.269872:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.269874:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.269875:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.269875:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.269879:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.269880:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.269883:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.269883:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.269884:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.269885:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.269886:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.269886:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.269887:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.269891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.269898:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.269900:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.269901:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.269902:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.269904:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.269907:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.269965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.269968:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.269970:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.269973:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.269974:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.269976:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.269978:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.269979:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.269980:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.269981:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.269982:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.269986:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.269988:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.269991:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.269992:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.269992:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.269995:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.269995:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.269996:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.269996:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.270000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.270007:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.270009:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.270010:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.270012:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.270013:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.270015:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.270077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.270085:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.270087:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.270090:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.270091:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.270092:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.270093:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.270095:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.270096:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.270097:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.270098:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.270101:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.270103:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.270106:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.270106:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.270107:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.270108:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.270109:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.270110:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.270111:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.270114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.270120:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.270122:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.270123:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.270123:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.270125:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.270127:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.270197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.270205:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.270207:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.270208:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.270209:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.270212:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.270213:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.270214:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.270215:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.270216:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.270217:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.270220:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.270221:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.270224:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.270224:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.270225:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.270227:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.270227:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.270228:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.270230:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.270230:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.270232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.2:1619677448.270236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.270239:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.270241:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:21.2:1619677448.270241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:30.0:1619677448.270242:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.270243:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.270243:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:31.0:1619677448.270245:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.270245:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:30.0:1619677448.270246:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:26.0:1619677448.270246:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677448.270246:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.270247:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677448.270249:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.270249:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.270251:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.270254:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.270255:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.270257:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.270258:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.270259:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.270260:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.270263:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.270264:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.270267:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.270267:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.270268:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.270270:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.270271:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.270271:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.270272:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.270276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.270284:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.270286:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.270287:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.270287:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.270289:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.270292:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.270358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677448.270362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:31.0:1619677448.270366:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.270369:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:10.0:1619677448.270370:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:31.0:1619677448.270371:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.270373:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.270373:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:31.0:1619677448.270374:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.270374:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:5.2:1619677448.270374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:31.0:1619677448.270376:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.270376:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.270377:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.270378:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.270379:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.270380:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.270380:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.270380:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:31.0:1619677448.270381:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.270381:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.270383:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:10.0:1619677448.270383:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.270384:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.270384:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677448.270385:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.270386:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.270387:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.270388:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.270390:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.270391:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:10.0:1619677448.270391:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.270392:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.270393:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.270394:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.270394:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.270394:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.270395:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.270396:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:17.0:1619677448.270396:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.270397:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96717131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.270397:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.270397:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.270398:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:17.0:1619677448.270399:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.270399:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:25.2:1619677448.270400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:17.0:1619677448.270400:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.270400:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:17.0:1619677448.270401:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.270401:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.270401:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:4.2:1619677448.270402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:17.0:1619677448.270404:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.270406:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.2:1619677448.270406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:13.0:1619677448.270408:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:26.0:1619677448.270409:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.270410:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.270410:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:17.0:1619677448.270411:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:26.0:1619677448.270412:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:17.0:1619677448.270412:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (23)++
00000400:00000200:13.0:1619677448.270412:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.270413:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.270414:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677448.270414:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.270415:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.270415:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.270415:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:12.0:1619677448.270416:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:17.0:1619677448.270417:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677448.270417:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:1.0:1619677448.270417:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:26.0:1619677448.270418:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.270418:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:12.0:1619677448.270418:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.270418:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.270419:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.270419:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677448.270419:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000400:00000200:26.0:1619677448.270420:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:17.0:1619677448.270420:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:12.0:1619677448.270420:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677448.270421:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677448.270421:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:1.0:1619677448.270421:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.270422:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96717130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677448.270422:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.270422:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:26.0:1619677448.270423:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.270423:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:26.0:1619677448.270424:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677448.270424:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.270424:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.2:1619677448.270424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.270426:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:13.0:1619677448.270426:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677448.270428:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677448.270429:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:26.0:1619677448.270430:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.270430:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.270430:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.270431:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.270432:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.270434:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.270434:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.270435:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.270437:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.270437:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.270438:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.270440:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.270440:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.270442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.270446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.270449:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.270451:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.270452:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.270453:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:30.0:1619677448.270454:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.270455:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.270455:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.270455:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.270456:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677448.270457:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.2:1619677448.270468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.270473:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.270476:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.270478:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.270479:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.270481:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.270483:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.270484:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.270486:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.270486:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.270488:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.270491:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.270493:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.270496:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.270497:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.270497:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.270499:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.270500:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.270501:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.270501:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.270505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.270512:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.270514:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.270515:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.270517:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.270518:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.270519:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.270565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.270567:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.270570:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.270572:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.270573:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.270575:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.270577:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.270578:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.270580:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.270581:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.270582:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.270585:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.270587:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.270589:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.270590:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.270591:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.270593:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.270594:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.270594:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.270595:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.270599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.2:1619677448.270599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677448.270606:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:26.0:1619677448.270607:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.270609:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.270610:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.270610:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.270610:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:31.0:1619677448.270612:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.270612:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.270613:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.270615:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:26.0:1619677448.270615:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.270616:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.270617:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.270618:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.270618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:26.0:1619677448.270619:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.270620:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.270623:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.270625:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.270625:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.270626:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.270627:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:21.2:1619677448.270627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.270628:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.270628:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.270629:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.270629:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:26.0:1619677448.270629:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.270629:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.270630:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.270630:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.270631:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.270631:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.270632:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677448.270634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.270635:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.270637:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.270639:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.270640:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.270640:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:10.0:1619677448.270640:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:30.0:1619677448.270641:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.270642:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:30.0:1619677448.270643:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.270643:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.270643:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.270644:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.270645:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.270645:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.270648:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.270650:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.270650:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:30.0:1619677448.270652:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.270653:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.270653:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:25.2:1619677448.270654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (22)++
00000800:00000200:30.0:1619677448.270655:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:30.0:1619677448.270656:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.270656:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.270657:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.270660:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.270663:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.270663:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)++
00000800:00000200:10.0:1619677448.270663:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)++
00000800:00000200:26.0:1619677448.270664:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (24)--
00000800:00000200:10.0:1619677448.270664:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (24)--
00000800:00000200:26.0:1619677448.270665:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.270665:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.270666:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.270666:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.270668:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.270705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.270711:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.270714:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.270715:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.270717:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.270720:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.270721:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.270723:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.270724:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.270725:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.270726:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.270730:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.270731:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.270734:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.270734:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.270735:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.270737:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.270737:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.270738:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.270738:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.270743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.270751:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.270753:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.270754:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.270756:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.270757:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.270759:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.270801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.270803:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.270806:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.270808:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.270809:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.270811:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.270812:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.270814:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.270815:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.270816:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.270818:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:4.2:1619677448.270819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:20.0:1619677448.270823:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.270825:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.270826:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.270828:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.270829:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:20.0:1619677448.270829:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.270829:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.270829:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.270831:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.270831:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.270832:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677448.270833:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.270833:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.270834:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000400:00000200:26.0:1619677448.270835:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:20.0:1619677448.270835:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.2:1619677448.270835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677448.270836:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.270837:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.270837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:26.0:1619677448.270838:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.270839:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.2:1619677448.270839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677448.270841:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.270843:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:26.0:1619677448.270843:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.270844:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:26.0:1619677448.270845:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.270845:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677448.270845:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.270846:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:17.0:1619677448.270846:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.270847:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.270847:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:31.0:1619677448.270848:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.270848:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.270848:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677448.270849:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:26.0:1619677448.270849:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (23)++
00000400:00000200:17.0:1619677448.270849:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.270849:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.270850:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:17.0:1619677448.270850:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:31.0:1619677448.270851:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:26.0:1619677448.270851:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:17.0:1619677448.270851:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.270852:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.270852:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.270853:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:17.0:1619677448.270853:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.270854:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.270855:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:25.2:1619677448.270857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:17.0:1619677448.270859:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.270860:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.270863:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.270864:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677448.270864:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.270865:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.270866:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:17.0:1619677448.270866:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:30.0:1619677448.270867:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.270867:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.270867:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.270868:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677448.270869:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.270870:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.270871:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.270872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.270879:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.270881:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.270882:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.270884:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.270885:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.270886:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.270986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.270994:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.270996:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.270997:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.270999:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.271001:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.271003:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.271004:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.271005:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.271006:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.271007:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.271010:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.271012:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.271013:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.271014:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.271015:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.271016:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.271016:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.271017:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.271017:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.271021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.271027:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.271029:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.271029:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.271030:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.271031:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.271032:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.271037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.271045:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.271047:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.271049:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.271051:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.271051:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.271053:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.271054:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.271055:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.271056:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.271057:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.271059:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.271061:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.271063:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.271064:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.271064:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.271066:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.271066:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.271068:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.271069:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.271070:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.271071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.271075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.271077:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.271079:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.271080:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.271082:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.271083:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.271083:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.271084:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.271085:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.271086:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.271087:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.271090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.271096:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.271098:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.271099:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.271100:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.271100:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.271102:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.271137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.271144:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.271146:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.271147:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.271149:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.271151:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.271152:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.271153:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.271154:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.271155:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.271156:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.271159:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.271160:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:21.2:1619677448.271164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:10.0:1619677448.271164:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.271165:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.271165:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.271167:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.271167:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.271168:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.271168:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.0:1619677448.271171:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.2:1619677448.271173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:8.0:1619677448.271173:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.271174:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.271176:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.271179:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.271180:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.271181:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96717118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.271181:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:8.0:1619677448.271182:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.271183:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.271183:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.271184:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:8.0:1619677448.271184:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:6.0:1619677448.271185:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.271187:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.271188:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.271191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:8.0:1619677448.271191:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677448.271191:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677448.271194:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.271194:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.271195:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.271197:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677448.271198:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:8.0:1619677448.271198:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.271198:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.271199:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677448.271201:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.271202:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.271202:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.271203:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:9.2:1619677448.271203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677448.271204:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677448.271211:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:21.2:1619677448.271214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:18.0:1619677448.271214:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.271215:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.271215:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.271218:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.271219:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:9.2:1619677448.271220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.271222:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.271223:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677448.271224:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.271225:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677448.271225:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.271227:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:6.0:1619677448.271227:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.271228:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000400:00000200:6.0:1619677448.271228:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:2.0:1619677448.271228:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.271229:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000400:00000200:6.0:1619677448.271229:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.271230:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:6.0:1619677448.271231:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.271232:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.271233:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.271236:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.271238:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.271241:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.271242:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.271242:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.271244:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.271244:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.271245:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.271246:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.271250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.271257:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.271260:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.271261:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.271262:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.271263:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.271266:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.271332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.271338:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.271340:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.271343:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.271343:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.271346:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.271347:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.271348:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.271350:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.271351:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.271352:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.271355:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.271357:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.271360:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.271360:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.271361:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.271362:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.271363:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.271364:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.271365:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.2:1619677448.271369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:25.2:1619677448.271370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.271377:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677448.271377:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.271380:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:22.0:1619677448.271380:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.271381:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.271381:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677448.271381:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:26.0:1619677448.271383:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.271383:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.271386:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.271386:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.271386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677448.271387:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.271389:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.271390:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.271391:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.271392:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:1.0:1619677448.271392:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.271394:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.271395:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.271396:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.271399:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:1.0:1619677448.271399:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.271401:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.271402:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.271402:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:1.0:1619677448.271402:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.271403:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.271404:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:1.0:1619677448.271404:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:26.0:1619677448.271405:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.271405:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:1.0:1619677448.271405:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.271406:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.271407:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.271408:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:25.2:1619677448.271409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:1.0:1619677448.271412:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.271414:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.271415:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.271417:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.271417:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.271417:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.271418:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:1.0:1619677448.271418:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.271418:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:10.0:1619677448.271420:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.271420:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.271420:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.271421:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:1.0:1619677448.271421:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.271422:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.271426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.271434:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.271436:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.271437:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.271439:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.271441:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.271442:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.271464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.271471:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.271474:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.271476:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.271478:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.271478:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.271480:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.271481:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.271482:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.271483:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.271484:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.271487:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.271489:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.271491:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.271492:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.271493:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.271494:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.271495:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.271495:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.271496:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.271500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.271507:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.271510:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.271510:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.271512:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.271513:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.271515:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.271533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.271540:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.271543:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.271544:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.271545:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.271547:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.271548:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.271549:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717112] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.271550:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.271551:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.271552:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.271556:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.271557:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.271560:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.271561:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.271562:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.271563:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.271564:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.271564:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.271565:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.271570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.271578:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.271580:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.271581:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.271582:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.271584:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.271587:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.271672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.271679:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.271682:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.271683:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.271685:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.271687:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.271688:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.271690:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.271691:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.271692:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.271693:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.271696:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.271698:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.271701:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.271702:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.271703:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.271704:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.271705:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.271707:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.271708:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.271708:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.271711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.271713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677448.271719:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677448.271719:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.271721:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:20.0:1619677448.271722:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:2.0:1619677448.271722:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:20.0:1619677448.271723:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.271723:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:20.0:1619677448.271724:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.271725:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.271727:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.271730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.271737:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.271740:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.271741:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.271742:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.271742:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.271744:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.271949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.271954:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.271957:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.271958:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.271960:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.271963:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.271964:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.271966:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.271967:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.271968:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.271969:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.271972:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.271974:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.271977:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.271978:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.271978:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.271980:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.271981:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.271981:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.271982:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.271986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.271993:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.271996:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.271997:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.271998:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.271999:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.272001:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.272137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.272143:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.272146:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.272147:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.272149:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.272152:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.272153:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.272155:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.272156:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.272157:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.272158:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.272162:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.272163:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.272166:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.272166:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.272167:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.272169:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.272169:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.272170:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.272170:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.272175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.272182:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.272185:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.272186:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.272186:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.272188:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.272190:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.272192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.272198:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.272200:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.272201:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.272203:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.272205:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.272207:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.272208:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.272210:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.272210:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.272212:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.272215:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.272217:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.272219:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.272220:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.272221:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.272223:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.272223:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.272224:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.272225:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.272226:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:25.2:1619677448.272228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:21.2:1619677448.272231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.272235:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:6.0:1619677448.272236:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:10.0:1619677448.272237:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.272238:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.272238:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.272238:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.272239:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677448.272240:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.272240:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.272243:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.272337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.272343:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677448.272344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:17.0:1619677448.272346:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.272347:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.272349:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.272351:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:17.0:1619677448.272351:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.272352:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:24.0:1619677448.272354:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:17.0:1619677448.272354:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.272355:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.272355:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:17.0:1619677448.272356:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.272357:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677448.272358:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.272361:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.272361:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.272362:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.272363:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96717106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.272363:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:4.2:1619677448.272363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677448.272365:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.272366:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.272366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:17.0:1619677448.272366:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677448.272367:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.272367:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.272368:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.272369:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.272370:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.272371:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:24.0:1619677448.272371:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.272371:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677448.272372:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.272372:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.272372:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.272373:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:2.0:1619677448.272373:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:10.0:1619677448.272374:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677448.272375:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:25.2:1619677448.272376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677448.272376:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.272376:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:26.0:1619677448.272377:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.272377:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.2:1619677448.272377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.272378:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677448.272379:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.272380:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.272380:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.272380:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.272381:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:20.2:1619677448.272381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:26.0:1619677448.272382:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:24.0:1619677448.272382:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:2.0:1619677448.272382:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.272382:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:2.0:1619677448.272383:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:26.0:1619677448.272384:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96717105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.272384:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:2.0:1619677448.272385:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96717105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.272385:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.272385:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677448.272386:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:9.2:1619677448.272386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.272386:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677448.272387:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.272387:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.272387:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.272389:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677448.272389:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677448.272389:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.272389:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.272390:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:13.0:1619677448.272391:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677448.272391:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.272392:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:12.0:1619677448.272392:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677448.272395:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677448.272396:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.272396:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.272396:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:26.0:1619677448.272397:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.272397:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.272398:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000400:00000200:30.0:1619677448.272399:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.272399:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.272400:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.272401:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:2.0:1619677448.272401:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.272402:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96717103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.272402:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.272403:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.272404:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.272405:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.272405:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.272406:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.272406:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.272406:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:30.0:1619677448.272407:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.272407:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.272407:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.272408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677448.272408:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:26.0:1619677448.272409:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.272409:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:30.0:1619677448.272410:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.272410:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.272411:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:2.0:1619677448.272411:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:30.0:1619677448.272412:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.272412:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:26.0:1619677448.272413:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:2.0:1619677448.272413:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:26.0:1619677448.272414:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.272415:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.272415:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677448.272415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.2:1619677448.272415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.272416:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677448.272416:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)++
00000800:00000200:30.0:1619677448.272417:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (23)++
00000400:00000200:31.0:1619677448.272418:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.272419:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (24)--
00000800:00000200:30.0:1619677448.272419:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:21.2:1619677448.272419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:4.2:1619677448.272419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.272420:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.272420:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.272421:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:31.0:1619677448.272422:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:30.0:1619677448.272422:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.272423:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96717102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.272424:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:31.0:1619677448.272425:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.272425:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:5.0:1619677448.272425:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (23)++
00000400:00000200:31.0:1619677448.272426:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.272426:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.272426:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:31.0:1619677448.272427:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.272427:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:5.0:1619677448.272427:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (24)--
00000800:00000200:2.0:1619677448.272427:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:18.0:1619677448.272428:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.272428:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:10.0:1619677448.272429:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:5.0:1619677448.272429:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.272429:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677448.272430:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (23)++
00000800:00000200:10.0:1619677448.272430:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677448.272430:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.0:1619677448.272431:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (24)--
00000800:00000200:18.0:1619677448.272431:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:10.0:1619677448.272431:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:5.0:1619677448.272431:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.272432:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.272432:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.272433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.272434:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.272434:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.272434:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.272436:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.272437:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677448.272438:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.272438:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.272439:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.272441:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.272441:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.272441:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.272442:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.272443:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677448.272443:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.272445:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.272445:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.272445:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.272446:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:8.2:1619677448.272447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.272454:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.272457:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.272458:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.272458:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.272461:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.272464:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.272610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677448.272618:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677448.272620:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677448.272621:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677448.272623:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.272626:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.272628:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677448.272629:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.272631:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.272632:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.272633:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677448.272636:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.272638:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677448.272641:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.272642:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:8.0:1619677448.272642:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677448.272644:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:8.0:1619677448.272644:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677448.272645:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677448.272646:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.2:1619677448.272650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.272658:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.272661:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.272662:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.272662:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.272664:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.272667:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.272735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.272742:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.272745:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.272746:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.272748:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.272751:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.272752:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.272754:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.272755:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.272756:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.272757:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.272760:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.272762:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.272765:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.272766:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.272766:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.272768:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.272769:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.272770:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.272771:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.272772:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677448.272774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.272777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.272782:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.272784:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.272785:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.272785:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:2.0:1619677448.272786:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.272787:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.272787:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:2.0:1619677448.272788:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.272789:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677448.272791:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.272879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.272887:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.272891:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.272893:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.272894:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.272896:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.272897:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.272899:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.272900:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.272901:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.272902:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.272905:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.272907:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.272909:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.272909:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.272910:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.272911:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.272912:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.272913:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.272915:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.272915:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.272917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.2:1619677448.272920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.2:1619677448.272921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.272925:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:1.0:1619677448.272926:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.272927:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.272927:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.272928:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:31.0:1619677448.272929:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.272929:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:17.0:1619677448.272929:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.272930:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.272931:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:1.0:1619677448.272931:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.272932:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.272933:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677448.272934:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:1.0:1619677448.272934:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.272935:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:25.2:1619677448.272936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000400:00000200:1.0:1619677448.272936:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.272938:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.272939:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.272940:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.272943:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000400:00000200:1.0:1619677448.272943:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.272945:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.272946:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.272946:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.272947:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.272947:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.272948:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:1.0:1619677448.272948:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.272949:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.272950:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.272951:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.272952:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.272953:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.272953:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.272957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.272964:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.272967:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.272967:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.272969:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.272970:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.272972:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.272976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.272983:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.272985:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.272986:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.272987:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.272987:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.272989:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.273094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.273101:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.273103:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.273104:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.273106:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.273109:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.273111:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.273112:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.273114:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.273115:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.273116:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.273119:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.273120:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.273123:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.273124:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.273125:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.273127:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.273127:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.273128:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.273130:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.273130:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677448.273133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.273136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:18.0:1619677448.273140:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.273143:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:8.0:1619677448.273143:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:18.0:1619677448.273144:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.273144:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:2.0:1619677448.273144:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.273145:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.273146:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:18.0:1619677448.273147:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.273150:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.273154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.273160:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.273163:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.273164:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.273166:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.273169:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.273170:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.273171:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.273172:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.273173:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.273174:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.273177:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.273180:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.273182:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.273183:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.273183:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.273185:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.273185:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.273186:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.273186:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.273192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.273200:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.273202:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.273203:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.273205:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.273205:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.273207:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.273280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.273283:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.273285:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.273287:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.273288:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.273290:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.273292:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677448.273297:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.273299:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.273300:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.273301:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.273304:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.273305:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.273308:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.273308:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.273309:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.273311:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.273312:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.273312:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.273313:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677448.273318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.273326:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.273328:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.273329:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.273330:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.273332:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677448.273334:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.273341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.273348:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.273350:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.273351:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.273351:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.273352:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.273353:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.273405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.273413:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.273416:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.273417:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.273419:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.273422:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.273423:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.273425:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.273426:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.273427:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.273428:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.273431:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.273433:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.273435:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.273436:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.273437:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.273439:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.273439:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.273440:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.273440:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.273444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.273453:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.273455:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.273456:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.273458:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.273458:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.273461:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.273462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.273469:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.273472:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.273473:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.273473:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.273474:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.273475:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.273500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.273506:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.273509:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.273510:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.273512:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.273515:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.273516:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.273517:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.273518:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.273520:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.273520:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.273524:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.273526:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.273529:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.273530:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.273530:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.273532:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.273532:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.273533:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.273533:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.273539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.273545:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.273548:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.273549:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.273549:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.273552:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.273555:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.273559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.2:1619677448.273560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.0:1619677448.273566:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:30.0:1619677448.273568:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:24.0:1619677448.273569:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.273569:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677448.273570:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:30.0:1619677448.273571:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:8.0:1619677448.273571:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.273572:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:30.0:1619677448.273573:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.273575:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.273576:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.273577:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.273578:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.273580:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.273581:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.273582:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.273585:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.273587:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.273590:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.273591:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.273591:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.273593:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.273594:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.273595:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.273595:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.273599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.273607:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.273610:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.273610:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.273612:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.273613:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.273615:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.273833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.273840:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.273843:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.273844:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.273846:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.273848:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.273850:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.273851:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.273852:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.273853:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.273854:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.273857:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.273858:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.273861:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.273861:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.273862:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.273863:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.273864:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.273864:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.273865:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:21.2:1619677448.273867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:25.2:1619677448.273869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.273874:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.273877:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:13.0:1619677448.273877:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:31.0:1619677448.273879:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.273879:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.273880:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.273881:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:13.0:1619677448.273881:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.273882:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.273884:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:13.0:1619677448.273884:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.273884:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:31.0:1619677448.273885:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.273886:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.273887:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.273888:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.273891:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.273893:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.273895:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.273896:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.273897:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.273898:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.273899:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.273900:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.273900:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.273905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.273913:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:4.2:1619677448.273915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677448.273916:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.273916:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.273917:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.273919:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.273922:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.273923:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677448.273924:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.273925:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.273927:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.273930:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.273931:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.273933:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717089] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.273934:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.273935:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.273935:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.273938:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.273940:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.273942:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.273943:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.273943:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.273945:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.273945:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.273946:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.273946:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.273951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.273957:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.273960:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.273960:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.273962:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.273963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677448.273964:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.273964:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:1.0:1619677448.273969:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:25.2:1619677448.273971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.273971:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.273972:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.273974:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.273977:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:1.0:1619677448.273977:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.273979:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:31.0:1619677448.273980:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.273980:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.273981:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.273981:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.273982:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:1.0:1619677448.273982:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.273983:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.273984:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:1.0:1619677448.273984:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.273987:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.273989:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.273991:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.273992:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.273993:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.273994:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.273995:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.273995:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.273996:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.274000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.274008:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.274011:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.274012:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.274012:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.274014:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.274017:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.274034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.274041:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.274044:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.274045:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.274047:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.274049:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.274051:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.274052:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.274053:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.274055:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.274056:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.274059:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.274061:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.274064:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.274065:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.274065:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.274067:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.274067:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.274068:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.274068:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.274073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.274080:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.274083:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.274084:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.274084:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.274087:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.274089:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.274249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.274255:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.274257:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.274260:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.274260:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.274262:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.274264:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.274265:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.274267:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.274268:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.274269:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.274272:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.274274:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.274277:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.274277:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.274278:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.274280:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.274281:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.274281:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.274282:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.274286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.274293:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.274300:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.274301:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.274301:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677448.274303:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.274305:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.274310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.274316:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.274319:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.274320:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.274322:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.274324:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.274325:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.274326:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.274328:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.274329:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.274329:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.274332:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.274334:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.274336:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.274336:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.274337:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.274339:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.274339:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.274340:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:21.2:1619677448.274341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.274341:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.274346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.274350:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.274352:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:18.0:1619677448.274354:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.274355:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:18.0:1619677448.274356:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.274356:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.274357:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.274358:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.274358:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.274359:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.274360:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.274361:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.274362:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.274363:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.274364:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.274365:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677448.274366:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.274366:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.274367:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.274367:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.274368:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.274368:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.274369:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677448.274371:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.274372:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.274373:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.274375:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.274375:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.274376:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.274377:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.274377:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.274380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.274383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:12.0:1619677448.274387:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.274388:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:12.0:1619677448.274390:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.274390:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.274390:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.274391:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:12.0:1619677448.274392:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.274392:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677448.274394:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.274395:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.274433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.274439:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.274442:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.274443:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.274444:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.274447:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.274448:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.274450:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.274451:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.274452:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.274453:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.274456:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.274457:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.274460:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.274461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:1.0:1619677448.274461:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.274462:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.274463:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.274464:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.274465:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:20.2:1619677448.274466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:1.0:1619677448.274466:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.274466:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.274469:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:9.2:1619677448.274469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.274471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.274472:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:26.0:1619677448.274474:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:13.0:1619677448.274474:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.274475:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:24.0:1619677448.274476:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:13.0:1619677448.274476:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.274477:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:17.0:1619677448.274477:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:13.0:1619677448.274478:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:10.0:1619677448.274478:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:24.0:1619677448.274479:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:17.0:1619677448.274479:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.274480:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677448.274480:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.274480:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:26.0:1619677448.274481:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677448.274481:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.274481:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677448.274482:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.274483:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.274483:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677448.274483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677448.274484:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.274485:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:13.0:1619677448.274485:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.274487:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.274487:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:26.0:1619677448.274489:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.274489:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.274490:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.274491:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:13.0:1619677448.274491:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.274491:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.274492:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:13.0:1619677448.274494:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.274495:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:13.0:1619677448.274495:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:10.0:1619677448.274495:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.274496:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.274497:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:10.0:1619677448.274497:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.274498:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:13.0:1619677448.274498:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677448.274499:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000400:00000200:10.0:1619677448.274499:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.274500:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96717080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.274501:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000400:00000200:10.0:1619677448.274501:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.274502:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:10.0:1619677448.274502:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.274503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:10.0:1619677448.274503:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.274506:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.274507:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:10.0:1619677448.274507:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.274507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:26.0:1619677448.274508:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.274509:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.274510:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.274511:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:12.0:1619677448.274511:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:26.0:1619677448.274512:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.274512:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.274513:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677448.274513:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.274513:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.274513:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.274514:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:13.0:1619677448.274514:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:10.0:1619677448.274514:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:13.0:1619677448.274515:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000400:00000200:12.0:1619677448.274515:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.274516:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:10.0:1619677448.274516:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.2:1619677448.274516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:12.0:1619677448.274517:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.274517:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.274517:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:13.0:1619677448.274518:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.274518:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:12.0:1619677448.274519:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:25.2:1619677448.274522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.274524:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:25.2:1619677448.274526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:20.0:1619677448.274527:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.274528:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.274529:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:6.0:1619677448.274530:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.274530:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:13.0:1619677448.274532:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.274533:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:6.0:1619677448.274533:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.274534:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.274534:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.274535:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.274536:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.274537:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:6.0:1619677448.274537:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.274538:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.274539:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677448.274540:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.274542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.274549:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.274552:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.274552:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.274552:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.274554:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.274554:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.274585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.274593:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.274596:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.274598:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.274601:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.274601:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.274603:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.274604:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.274606:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.274606:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.274607:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.274611:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.274612:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.274614:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.274615:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.274616:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.274617:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.274618:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.274619:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.274620:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.274623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.274630:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.274632:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.274636:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677448.274636:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.274637:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.274639:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.274823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.274830:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.274832:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.274834:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.274836:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.274836:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.274838:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.274839:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.274840:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.274841:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.274842:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.274845:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.274846:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.274849:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.274850:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.274850:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.274852:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.274853:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.274854:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.274856:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.274856:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.274858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.274862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677448.274866:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.274869:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.274869:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:30.0:1619677448.274870:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:6.0:1619677448.274870:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.274872:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:6.0:1619677448.274872:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.274873:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.274874:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.274875:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.274938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.274946:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.274948:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.274949:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.274951:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.274954:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.274956:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.274957:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.274958:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.274959:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.274960:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.274963:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.274964:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.274967:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.274968:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.274969:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.274971:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.274971:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.274973:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.274974:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.274974:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677448.274977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677448.274980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.274985:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.274987:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:10.0:1619677448.274987:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:2.0:1619677448.274988:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.274989:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.274989:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.274990:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.2:1619677448.274990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:18.0:1619677448.274991:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.274991:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:18.0:1619677448.274994:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677448.274996:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.274999:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.275000:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.275002:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.275005:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.275006:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.275008:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.275009:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.275010:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.275011:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.275015:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.275016:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.275019:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.275020:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.275020:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.275022:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.275022:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.275023:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.275023:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.275028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.275036:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.275038:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.275039:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.275040:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.275041:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.275044:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.275090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.275098:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.275101:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.275103:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.275105:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.275106:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.275108:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.275109:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.275110:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.275111:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.275112:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.275116:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.275117:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.2:1619677448.275119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:30.0:1619677448.275121:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.275121:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.275122:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.275124:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.275124:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.275125:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677448.275125:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677448.275126:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.275127:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.275129:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:8.2:1619677448.275130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.275131:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.275132:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:5.0:1619677448.275132:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.275134:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.275134:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677448.275135:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.275135:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.275136:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.275137:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.275137:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.275139:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.275141:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.275142:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:5.0:1619677448.275142:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.275144:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.275147:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.275148:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677448.275148:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.275150:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.275151:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.275152:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.275152:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.275156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.275162:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.275165:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.275166:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.275166:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.275168:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:26.0:1619677448.275171:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.275176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.275181:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.275184:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.275185:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.275185:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.275186:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.275187:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.275277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.275283:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.275285:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.275287:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.275288:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.275291:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.275292:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.275299:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.275300:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.275301:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.275302:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.275306:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.275307:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.275311:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.275312:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.275313:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.275314:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.275315:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.275316:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.275316:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.275321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.275328:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:5.2:1619677448.275329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.275331:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.275332:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.275334:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.275334:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.275335:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.275337:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.275338:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.275339:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.275339:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.275342:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.275344:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.275345:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.275346:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.275347:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.275348:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.275351:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.275353:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.275355:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.275356:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.275356:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.275358:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.275359:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.275359:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.275359:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.275364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.275371:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.275374:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.275375:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.275376:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.275379:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.275380:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.275383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.275389:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.275392:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.275392:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.275395:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.275397:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.275399:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.275400:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.275401:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.275402:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.275403:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.275406:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.275408:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.275411:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.275412:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.275412:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.275415:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.275415:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.275416:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.275416:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677448.275418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.275421:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:25.2:1619677448.275422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.275423:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.275426:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.275426:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677448.275429:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:20.0:1619677448.275429:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.275430:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677448.275431:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.275432:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.275432:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.275433:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.275434:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.275435:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.275435:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.275435:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.275438:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:20.0:1619677448.275439:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.275441:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.275444:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.275445:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.275446:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.275448:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.275449:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.275449:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.275450:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.275454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.275461:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.275464:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.275465:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.275466:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.275468:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.275469:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.275577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.275584:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.275587:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.275588:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.275590:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.275590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:26.0:1619677448.275593:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.275594:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.275595:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.275596:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.275596:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:26.0:1619677448.275598:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.275598:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.275599:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.275600:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.275603:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.275604:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.275605:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.275606:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.275607:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.275608:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.275609:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.275610:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:18.0:1619677448.275610:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.275611:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:18.0:1619677448.275611:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.275612:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:18.0:1619677448.275612:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.275613:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.275614:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.275614:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:18.0:1619677448.275615:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.275617:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.2:1619677448.275618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.275618:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.275619:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.275620:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.275621:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.275621:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.275622:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.275623:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677448.275626:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:9.2:1619677448.275627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677448.275629:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.275629:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.275630:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.275631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:22.0:1619677448.275632:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.275636:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677448.275636:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:21.2:1619677448.275637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677448.275637:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.275637:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.275638:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.275639:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:18.0:1619677448.275641:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.275641:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.275643:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.275644:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677448.275644:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.275644:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.275645:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.275646:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.275647:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.275648:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:18.0:1619677448.275650:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.275651:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677448.275651:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.275652:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.275652:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.275655:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.275656:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.275657:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.275658:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.275658:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.275660:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.275661:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.275661:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.275661:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:31.0:1619677448.275662:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.275662:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.275663:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.275664:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.275664:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:31.0:1619677448.275665:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.275665:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:31.0:1619677448.275667:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.275670:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677448.275670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.275671:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.275671:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.275673:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.275674:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.275674:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.275675:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.275679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.275679:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.275681:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.275682:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.275684:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.275684:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.275685:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.275687:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:5.0:1619677448.275687:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:26.0:1619677448.275688:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.275688:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.275689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000400:00000200:10.0:1619677448.275690:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.275693:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.0:1619677448.275697:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.275699:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.275700:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.275701:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.275702:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.275703:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.275744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.275750:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.275753:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.275754:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.275756:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.275758:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.275760:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.275761:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.275763:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.275764:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.275765:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.275768:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.275770:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.275773:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.275774:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.275775:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.275777:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.275777:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.275778:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.275778:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.275782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.275786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:12.0:1619677448.275789:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.275791:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.275792:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.275793:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:12.0:1619677448.275794:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.275795:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.275796:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:12.0:1619677448.275797:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:30.0:1619677448.275798:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.275800:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.275801:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.275802:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.275803:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.275805:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.275806:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.275807:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.275810:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.275811:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.275814:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.275815:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.275815:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.275817:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.275818:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.275819:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.275820:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.275821:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.275822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.275827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677448.275829:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.275831:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.275832:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.275833:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.275835:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:12.0:1619677448.275835:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.275836:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.275837:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.275838:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.275839:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.2:1619677448.275900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.275907:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.275910:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.275911:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.275912:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.275915:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.275916:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.275917:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.275919:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.275920:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.275920:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.275924:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.275924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:26.0:1619677448.275925:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.275927:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.275927:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.275928:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.275929:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.275929:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:26.0:1619677448.275930:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.275930:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.275931:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:20.0:1619677448.275931:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.275932:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.0:1619677448.275932:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.275934:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.275935:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:25.2:1619677448.275936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:20.0:1619677448.275936:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.275938:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.275939:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.275940:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.275942:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:20.0:1619677448.275943:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.275944:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.275945:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.275945:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.275947:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.275948:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.275949:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.275949:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677448.275949:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.275949:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.275951:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.275952:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.275953:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.275955:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.275955:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.275957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.275960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677448.275963:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.275965:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677448.275966:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.275967:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.275967:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:12.0:1619677448.275968:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.275968:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677448.275969:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.275969:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.275971:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.275978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.275984:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.275987:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.275988:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.275989:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.275991:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.275991:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.275995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.276002:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.276004:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.276005:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.276007:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.276010:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.276011:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.276012:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.276013:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.276014:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.276015:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.276018:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.276019:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.276022:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.276023:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.276024:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.276025:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.276026:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.276027:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.276028:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.276028:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.276031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.276034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.276039:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.276041:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.276042:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677448.276043:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677448.276043:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.276043:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.276044:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.276045:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.276045:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.276048:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.276103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.276109:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.276111:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.276112:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.276114:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.276117:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.276118:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.276120:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.276121:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.276122:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.276123:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.276126:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.276128:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.276130:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.276131:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.276132:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.276133:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.276134:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.276134:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.276134:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.276139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.276147:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.276150:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.276151:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.276151:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.276153:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.276155:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.276251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.276259:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.276262:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.276264:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.276265:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.276266:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.276268:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.276269:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.276270:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.276271:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.276272:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.276276:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.276278:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.276281:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.276282:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.276283:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.276284:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.276285:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.276286:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.276286:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677448.276286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677448.276291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:10.0:1619677448.276293:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.276298:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677448.276300:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:10.0:1619677448.276301:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.276303:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.276303:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.276304:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.276304:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.276306:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.276307:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677448.276307:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.276309:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.276310:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.276310:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:10.0:1619677448.276311:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.276312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:10.0:1619677448.276312:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.276315:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.276317:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.276317:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.276320:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.276320:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:10.0:1619677448.276321:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.276322:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.0:1619677448.276323:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.276323:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.276324:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.276324:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.276325:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.276325:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:5.0:1619677448.276325:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.276327:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.276329:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.276330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:5.0:1619677448.276330:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.276331:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.276332:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.276335:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.276337:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.276339:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.2:1619677448.276340:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677448.276342:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.276343:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.276343:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:5.0:1619677448.276344:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.276345:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.276345:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.276346:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.276346:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.276348:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.276348:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.276348:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:22.0:1619677448.276349:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:17.0:1619677448.276349:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.276349:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.276349:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.276350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000400:00000200:1.0:1619677448.276351:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.276354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677448.276354:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.276355:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.276356:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.276357:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:1.0:1619677448.276357:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.276359:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.276360:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.276360:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.276361:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:13.0:1619677448.276361:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677448.276361:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.276361:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.276362:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677448.276363:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:1.0:1619677448.276363:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677448.276364:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.276365:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:13.0:1619677448.276366:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.276368:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.276368:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.276368:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:13.0:1619677448.276369:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.276369:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.276371:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.276372:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.276372:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.276373:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.276377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.276383:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.276386:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.276387:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.276389:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.276390:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.276391:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.276400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:13.0:1619677448.276406:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.276409:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.276410:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.276410:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.276411:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.276413:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.276433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.276441:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.276443:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.276444:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.276446:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.276449:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.276450:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.276452:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.276453:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.276454:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.276455:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.276458:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.276460:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.276462:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.276462:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.276463:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.276464:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.276465:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.276465:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.276466:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.276470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.276476:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:21.2:1619677448.276477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677448.276478:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.276479:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.276481:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.276482:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.276483:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.276483:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.276485:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677448.276486:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677448.276488:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.276490:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.276491:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.276492:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.276494:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.276494:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.276495:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.276499:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.276500:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.276503:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.276504:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.276505:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.276506:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.276507:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.276508:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677448.276508:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677448.276513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.276521:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.276524:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.276525:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.276526:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.276527:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.276530:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.276743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.276748:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.276751:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.276753:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.276753:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.276756:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.276757:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.276758:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.276760:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.276761:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.276762:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.276765:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.276767:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.276769:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.276770:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.276771:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.276772:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.276773:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.276773:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.276774:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.276778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.276786:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.276788:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.2:1619677448.276788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:22.0:1619677448.276789:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.276789:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.276791:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.276794:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677448.276794:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.276797:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.276797:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.276799:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.276802:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.276803:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.276804:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.276805:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.276806:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.276807:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.276810:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.276812:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.276815:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.276816:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.276816:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.276818:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.276818:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.276820:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.276821:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.276821:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.276824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.276826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677448.276830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677448.276831:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:17.0:1619677448.276832:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.276833:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.276834:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.276835:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:2.0:1619677448.276835:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:20.0:1619677448.276836:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.276836:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:1.0:1619677448.276836:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.276837:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:20.0:1619677448.276838:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:17.0:1619677448.276838:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.276838:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.276839:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:17.0:1619677448.276840:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.276842:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.276843:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.276845:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.276846:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.276847:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.276850:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.276852:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.276855:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.276856:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.276857:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.276859:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.276860:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.276861:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.276862:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.276863:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.276865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.2:1619677448.276868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.276873:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:1.0:1619677448.276873:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.276875:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:31.0:1619677448.276876:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:1.0:1619677448.276876:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677448.276877:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.276877:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677448.276879:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.276880:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.276882:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.276937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.276944:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.276947:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.276948:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.276950:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.276953:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.276955:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.276956:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.276957:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.276958:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.276959:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.276962:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.276964:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.276967:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.276968:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.276968:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.276970:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.276970:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.276971:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.276972:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.276976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.276984:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.276987:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.276988:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.276988:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.276990:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.276993:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.277178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.277185:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.277188:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.277189:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.277190:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.277193:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.277195:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.277196:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.277197:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.277198:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.277199:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.277202:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.277203:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.277206:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.277207:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.277207:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.277209:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.277210:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.277211:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.277212:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.277212:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677448.277215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.277218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677448.277223:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.277225:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:18.0:1619677448.277226:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.277226:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.277227:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.277228:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.277228:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.277229:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:18.0:1619677448.277229:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.277231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677448.277232:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677448.277237:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.277240:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.277240:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.277243:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.277246:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.277247:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.277248:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.277249:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.277250:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.277251:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.277255:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.277256:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.277258:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.277259:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.2:1619677448.277260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677448.277260:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.277261:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.277261:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.277262:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.277263:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.277267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.277269:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.277272:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.277274:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.277275:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:1.0:1619677448.277275:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:30.0:1619677448.277277:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.277277:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.277277:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.277278:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:17.0:1619677448.277278:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:30.0:1619677448.277280:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.277280:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.277281:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.277282:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.277282:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:30.0:1619677448.277283:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.277286:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.277287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.277288:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.277291:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.277292:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.277292:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.277293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:17.0:1619677448.277296:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.277301:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.277301:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:8.0:1619677448.277301:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.277301:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.277303:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.277303:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:17.0:1619677448.277303:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.277304:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.277305:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677448.277305:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.277305:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.0:1619677448.277305:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.2:1619677448.277307:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:8.0:1619677448.277313:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.277315:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.277316:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:8.0:1619677448.277317:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.277318:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96717046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.277319:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.277319:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:8.0:1619677448.277320:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.277320:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.277321:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.277322:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.277322:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.277325:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.277325:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.0:1619677448.277327:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.277329:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.277330:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.277331:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.277332:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.277333:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.277334:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.277335:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.277338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.277346:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.277349:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.277350:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.277350:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.277352:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.277355:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.277439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.277442:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.277444:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:4.2:1619677448.277446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677448.277447:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.277448:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.277450:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.277452:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677448.277453:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.277455:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.277455:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677448.277456:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.277457:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.277458:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.277458:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:20.0:1619677448.277461:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.277462:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.277464:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677448.277466:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.277466:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.277467:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677448.277468:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:10.0:1619677448.277468:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.277469:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.277470:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677448.277470:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:10.0:1619677448.277470:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.277471:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.277471:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:10.0:1619677448.277471:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.277472:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.277476:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.277476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.277477:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.277480:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.277480:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.277481:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.277482:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.277483:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:10.0:1619677448.277483:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.277484:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:17.0:1619677448.277485:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:10.0:1619677448.277485:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.277485:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677448.277487:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.277487:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.277488:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.277489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.277491:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.277491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.277496:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:8.2:1619677448.277496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.277498:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:31.0:1619677448.277499:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.277500:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.277500:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:31.0:1619677448.277501:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.277501:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.277502:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.277503:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.277503:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677448.277504:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677448.277506:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.277506:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.277507:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.277508:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.277508:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.277572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.277578:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.277581:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.277582:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.277583:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.277586:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.277587:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.277589:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.277590:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.277591:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.277592:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.277596:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.277598:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.277601:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.277602:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.277602:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.277604:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.277605:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.277605:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.277606:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.277610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.277617:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.277620:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.277621:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.277623:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.277625:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.277626:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.277855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.277862:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.277865:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.277865:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.277868:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.277870:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.277871:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:5.2:1619677448.277871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677448.277872:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.277874:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.277875:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.277876:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.277877:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.277879:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:10.0:1619677448.277880:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.277880:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.277882:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.277882:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.277884:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.277884:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.277885:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:17.0:1619677448.277886:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677448.277886:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:17.0:1619677448.277887:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.277887:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.277888:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.277888:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677448.277889:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.277889:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:17.0:1619677448.277890:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.277891:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:25.2:1619677448.277893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:17.0:1619677448.277894:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.277896:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.277899:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.277900:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677448.277900:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.277900:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.277902:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.277903:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:17.0:1619677448.277903:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.277903:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677448.277904:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.277904:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677448.277906:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.277907:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.277908:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.277908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:25.2:1619677448.277911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:13.0:1619677448.277915:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.277917:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677448.277918:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:13.0:1619677448.277918:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.277920:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.277921:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.277921:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:12.0:1619677448.277921:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.277922:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.277922:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:13.0:1619677448.277922:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.277924:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.277977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.277985:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.277987:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.277988:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.277990:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.277993:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.277994:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.277995:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.277997:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.277998:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.277999:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.278002:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.278003:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.278006:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.278007:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.278008:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.278010:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.278010:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.278011:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.278011:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.278016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677448.278023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677448.278023:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.278026:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.278027:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.278028:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.278030:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.278031:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677448.278033:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677448.278034:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.278036:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.278036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:31.0:1619677448.278038:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.278039:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.278040:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.278042:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.278043:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.278043:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677448.278044:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.278045:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.278046:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.278047:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.278047:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000400:00000200:30.0:1619677448.278048:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.278048:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.278049:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677448.278050:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.278053:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.278054:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.278054:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.278056:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.278057:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.278058:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.278058:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.278062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.278070:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.278073:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.278074:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.278076:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.278076:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.278079:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.278127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.278134:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.278137:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.278138:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.278140:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.278142:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.278144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:10.0:1619677448.278144:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.278145:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.278146:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.278148:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.278148:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.278150:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677448.278152:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.278153:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:10.0:1619677448.278154:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.278154:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.278156:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.278157:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.278157:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:22.0:1619677448.278158:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.278158:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:22.0:1619677448.278160:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:10.0:1619677448.278160:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.278160:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677448.278161:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.278161:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.278161:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.278162:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.278163:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.278164:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:25.2:1619677448.278166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:22.0:1619677448.278168:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.278170:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.278174:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.278174:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.278174:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.278175:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.278175:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.278176:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.278177:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.278178:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:6.0:1619677448.278179:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.278180:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.278181:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.278181:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.278182:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677448.278182:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.2:1619677448.278183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:25.2:1619677448.278186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:21.2:1619677448.278188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677448.278190:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.278192:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.278193:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.278193:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.278194:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.278195:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.278195:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.278196:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677448.278196:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.278196:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.278197:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.278197:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.278198:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677448.278198:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:6.0:1619677448.278198:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677448.278199:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:21.2:1619677448.278318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.278325:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.278328:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.278329:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.278330:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.278333:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.278334:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677448.278335:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.278337:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.278338:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.278339:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677448.278342:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.278344:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.278346:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.278347:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.278347:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.278349:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.278349:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.278350:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.278350:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.278356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.278363:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.278365:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.278366:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.278367:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.278368:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.278369:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.278514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.278522:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.278525:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.278527:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.278529:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.278530:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.278531:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.278533:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.278534:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.278535:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.278536:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.278539:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.278541:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.278543:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.278544:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.278544:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.278546:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.278546:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.278548:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.278548:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677448.278551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:25.2:1619677448.278552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.278556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:26.0:1619677448.278560:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:12.0:1619677448.278560:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.278562:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.278562:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:26.0:1619677448.278563:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:12.0:1619677448.278563:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.278564:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:12.0:1619677448.278565:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.278565:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:26.0:1619677448.278567:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.278567:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:12.0:1619677448.278567:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.278568:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677448.278568:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.278570:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.278571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:5.0:1619677448.278571:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.278572:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677448.278573:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677448.278574:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.278575:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.278576:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.278577:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677448.278577:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.278578:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.278578:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.278580:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.278580:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677448.278581:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:12.0:1619677448.278581:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.278582:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.278583:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.278584:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677448.278585:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.278586:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.278588:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.278589:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.278591:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.278592:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.278592:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:26.0:1619677448.278593:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.278593:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.278594:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.278594:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.278595:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:26.0:1619677448.278596:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.278596:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.278597:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:5.0:1619677448.278597:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.278598:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:26.0:1619677448.278599:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:5.0:1619677448.278599:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:26.0:1619677448.278600:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677448.278601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677448.278604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:8.2:1619677448.278605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.278606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:1.0:1619677448.278609:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.278612:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.278613:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.278613:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000400:00000200:1.0:1619677448.278613:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.278614:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:10.0:1619677448.278614:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:5.0:1619677448.278614:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.278615:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:1.0:1619677448.278615:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.278616:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:10.0:1619677448.278616:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:5.0:1619677448.278616:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:20.0:1619677448.278617:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.278617:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:5.0:1619677448.278618:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:1.0:1619677448.278618:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677448.278619:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:20.0:1619677448.278619:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.278620:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.278621:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (23)++
00000800:00000200:20.0:1619677448.278622:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (24)--
00000800:00000200:5.0:1619677448.278622:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (24)--
00000800:00000200:5.0:1619677448.278623:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.278624:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.278634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:5.0:1619677448.278639:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:5.0:1619677448.278642:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:5.0:1619677448.278644:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.278645:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:5.0:1619677448.278646:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.278647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:5.0:1619677448.278648:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:5.0:1619677448.278649:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.278651:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.278652:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.278653:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.278654:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677448.278656:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.278657:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.278658:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.278659:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.278660:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.278662:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.278663:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.278663:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:10.0:1619677448.278664:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:5.0:1619677448.278664:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677448.278665:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:10.0:1619677448.278666:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.278666:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000400:00000200:10.0:1619677448.278667:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.278667:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000400:00000200:10.0:1619677448.278668:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.278668:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000400:00000200:10.0:1619677448.278669:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.278669:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:25.2:1619677448.278671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:10.0:1619677448.278672:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.278674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:10.0:1619677448.278674:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.278677:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000800:00000200:10.0:1619677448.278677:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.278678:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:10.0:1619677448.278678:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.278679:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000800:00000200:10.0:1619677448.278679:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.278680:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:12.0:1619677448.278680:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:10.0:1619677448.278680:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.278681:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000400:00000200:12.0:1619677448.278681:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.278681:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.278681:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:12.0:1619677448.278682:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.278682:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:13.0:1619677448.278684:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.278685:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.2:1619677448.278686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.278693:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.278696:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.278697:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.278698:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.278699:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.278701:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.278790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.278797:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.278800:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.278801:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.278803:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.278806:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.278808:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.278809:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.278811:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.278812:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.278813:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.278816:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.278818:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.278820:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.278821:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.278821:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.278823:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.278824:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.278824:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.278825:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.278828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.278836:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.278838:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.278839:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.278841:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.278842:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.278843:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.278998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.279004:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.279007:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.279008:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.279010:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.279013:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.279014:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.279015:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.279017:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.279018:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.279019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:17.0:1619677448.279019:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.279022:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.279024:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.279026:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.279026:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.279027:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.279028:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.279029:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.279029:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.279030:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677448.279030:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.279030:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.279031:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.279032:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.279035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:6.0:1619677448.279035:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.279036:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.279037:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.279039:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.279040:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.279042:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.279044:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.279045:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.279046:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.279046:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.279048:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.279048:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.279051:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.279051:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.279052:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677448.279052:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.279053:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.279055:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.279055:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.279056:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.279056:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.279061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.279069:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.279071:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.279072:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.279074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677448.279074:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.279074:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.279077:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.279079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.279083:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.279085:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677448.279087:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.279088:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.279089:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.279089:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.279090:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.279090:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000400:00000200:30.0:1619677448.279091:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677448.279091:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.279092:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:30.0:1619677448.279093:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.279093:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:30.0:1619677448.279094:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.279095:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.279096:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.279099:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.279102:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.279105:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.279106:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.279106:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.279108:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.279108:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.279110:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.279111:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.279112:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.279113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.279118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.279119:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.279122:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.279123:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.279124:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.279125:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.279126:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.279127:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:12.0:1619677448.279127:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.279128:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.279129:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677448.279274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677448.279275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:26.0:1619677448.279282:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.279284:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.279284:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.279285:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.279287:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:5.2:1619677448.279287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:26.0:1619677448.279288:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.279289:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.279290:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.279291:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.279291:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677448.279291:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:5.2:1619677448.279293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:18.0:1619677448.279298:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:26.0:1619677448.279300:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.279302:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.279302:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.279303:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.279303:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.279304:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.279304:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000400:00000200:26.0:1619677448.279304:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.279304:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:31.0:1619677448.279305:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.279305:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:30.0:1619677448.279306:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:31.0:1619677448.279307:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.279309:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.279310:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.279311:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.279311:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.279312:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.279313:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.279314:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.279314:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:26.0:1619677448.279314:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.279314:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.279315:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.279315:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677448.279316:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96717024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.279316:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:26.0:1619677448.279317:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:30.0:1619677448.279318:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.279318:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:18.0:1619677448.279318:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96717024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.279319:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.279319:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:18.0:1619677448.279319:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.279320:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000400:00000200:18.0:1619677448.279320:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.279321:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.279321:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677448.279322:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.279325:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677448.279325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.279326:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.279327:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.279327:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.279327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.279328:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.279329:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:18.0:1619677448.279329:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.279330:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.279330:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.279330:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.279332:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:18.0:1619677448.279332:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.279333:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.279334:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.279334:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:2.0:1619677448.279334:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:30.0:1619677448.279335:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:18.0:1619677448.279335:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677448.279335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.279336:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (23)++
00000800:00000200:2.0:1619677448.279336:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:30.0:1619677448.279337:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677448.279337:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:18.0:1619677448.279337:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000800:00000200:10.0:1619677448.279337:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:2.0:1619677448.279337:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.279338:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677448.279338:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.279339:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871380600] (22)++
00000800:00000200:18.0:1619677448.279339:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (24)--
00000800:00000200:10.0:1619677448.279339:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:18.0:1619677448.279340:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:30.0:1619677448.279341:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (23)--
00000800:00000200:30.0:1619677448.279341:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:18.0:1619677448.279341:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.279342:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677448.279343:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677448.279343:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.2:1619677448.279343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677448.279343:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.279345:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:20.0:1619677448.279346:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:18.0:1619677448.279346:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.279347:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.279347:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.279348:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.279348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (21)++
00000400:00000200:20.0:1619677448.279349:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.279350:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677448.279350:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:20.0:1619677448.279352:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:1.0:1619677448.279353:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:20.0:1619677448.279354:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677448.279354:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.279355:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.279357:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (22)++
00000400:00000200:1.0:1619677448.279357:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.279359:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871380600] (23)--
00000800:00000200:31.0:1619677448.279360:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:31.0:1619677448.279361:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:1.0:1619677448.279361:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.279389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677448.279394:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.279397:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677448.279398:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677448.279399:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.279402:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.279403:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.279404:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.279406:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.279407:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.279408:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.279411:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.279413:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 504 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:5.2:1619677448.279415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.279416:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.279417:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.279418:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.279420:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.279421:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677448.279421:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.279422:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677448.279422:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.279423:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:17.0:1619677448.279424:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.279425:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.279426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:1.0:1619677448.279426:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.279427:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.279429:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.279430:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.279431:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.279431:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:8.0:1619677448.279433:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:1.0:1619677448.279434:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.279435:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677448.279436:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:24.0:1619677448.279437:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.279437:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.279438:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.279439:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:8.0:1619677448.279440:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.279440:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.279441:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:8.0:1619677448.279443:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:1.0:1619677448.279443:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.279444:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.279445:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.279446:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.279447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.279451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.279455:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.279456:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677448.279458:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677448.279458:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:22.0:1619677448.279459:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.279459:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.279459:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.279461:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.279461:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.279464:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.279641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677448.279646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.0:1619677448.279651:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.279653:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677448.279654:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:26.0:1619677448.279656:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:20.0:1619677448.279657:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677448.279657:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:5.0:1619677448.279657:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.279659:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.279660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:5.0:1619677448.279661:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.279662:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.279663:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.279664:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.279665:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.279665:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.279667:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.279667:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:5.0:1619677448.279667:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.279668:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.279669:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.279670:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:24.0:1619677448.279670:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:5.0:1619677448.279670:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677448.279671:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.279675:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.279675:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.279676:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.279677:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.279679:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:24.0:1619677448.279679:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:5.0:1619677448.279679:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:24.0:1619677448.279680:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96717018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.279682:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.279682:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677448.279682:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.279683:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:24.0:1619677448.279683:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.279683:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:24.0:1619677448.279684:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.279684:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.279685:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:24.0:1619677448.279685:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.279686:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.279686:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.0:1619677448.279686:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.279687:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.0:1619677448.279687:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:24.0:1619677448.279688:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.279688:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.279689:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:25.2:1619677448.279690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:24.0:1619677448.279690:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.279690:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:24.0:1619677448.279693:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.279693:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:25.2:1619677448.279694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677448.279694:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:24.0:1619677448.279695:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.2:1619677448.279695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:24.0:1619677448.279696:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.279696:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:31.0:1619677448.279697:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:24.0:1619677448.279697:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677448.279698:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677448.279700:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:20.0:1619677448.279700:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:31.0:1619677448.279701:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.279701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.279701:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.279702:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.279702:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:31.0:1619677448.279703:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.279704:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.279705:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:13.0:1619677448.279705:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.279706:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.279707:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.279707:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:12.0:1619677448.279709:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.279709:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.279710:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:13.0:1619677448.279710:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677448.279710:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.279712:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.279715:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.279721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:25.2:1619677448.279723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.279728:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.279728:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.279731:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677448.279731:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.279732:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.279732:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:30.0:1619677448.279733:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.279733:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.279734:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677448.279735:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.279736:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.279737:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.279739:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.279740:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.279741:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.279742:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.279745:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.279747:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.279749:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.279750:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.279750:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.279752:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.279752:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.279753:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.279754:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.279757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.279764:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.279766:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.279767:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.279768:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.279770:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.279771:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.279776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.279783:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.279786:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.279786:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.279788:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.279790:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.279792:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.279793:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.279794:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.279795:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.279796:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.279799:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.279800:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.279803:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.279804:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.279805:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.279806:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.279807:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.279807:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.279808:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.279812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.279820:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.279823:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.279824:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.279825:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.279827:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.279830:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.279852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.279858:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.279861:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.279861:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.279863:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.279866:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.279867:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.279868:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.279870:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.279870:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.279872:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.279875:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.279877:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.279879:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.279880:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.279880:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.279882:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.279882:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.279883:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.279883:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.279888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.279896:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.279898:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.279899:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.279899:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.279901:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.279903:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.279904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:6.0:1619677448.279912:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.279914:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.279915:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.279916:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.279917:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.279917:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.280155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.280162:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.280165:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.280166:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.280168:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.280170:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.280172:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.280173:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717014] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.280174:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.280175:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.280176:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.280180:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.280181:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:21.2:1619677448.280183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677448.280184:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.280185:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.280186:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.280188:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.280188:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.280189:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.280189:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:31.0:1619677448.280191:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.2:1619677448.280192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:31.0:1619677448.280194:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.280196:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.280196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:31.0:1619677448.280198:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.280198:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:31.0:1619677448.280199:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:30.0:1619677448.280199:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.280201:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.280201:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677448.280201:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:30.0:1619677448.280202:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:18.0:1619677448.280202:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:31.0:1619677448.280203:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.280204:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.280205:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.280205:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:9.2:1619677448.280206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677448.280208:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.280208:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.280210:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.280210:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.280211:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.280212:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.280212:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96717012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.280213:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.280214:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96717012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.280214:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.280215:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.280216:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.280216:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:30.0:1619677448.280217:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.280217:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.280217:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.280218:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.280218:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.280221:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.280221:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:18.0:1619677448.280221:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:31.0:1619677448.280222:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:2.0:1619677448.280222:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:31.0:1619677448.280223:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:30.0:1619677448.280223:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.280224:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.280225:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.280225:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677448.280226:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.280227:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.280227:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.280227:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.280228:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677448.280229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.280229:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.280229:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.280230:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.280230:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.280231:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.280232:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.280232:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.280233:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677448.280234:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.280234:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677448.280234:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.280234:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677448.280235:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:30.0:1619677448.280236:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:10.0:1619677448.280237:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:30.0:1619677448.280238:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.280238:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677448.280238:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.280238:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.280238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:10.0:1619677448.280240:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.280240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.280243:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:18.0:1619677448.280245:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:20.2:1619677448.280246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677448.280247:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.280248:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.280249:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.280250:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:20.0:1619677448.280252:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677448.280252:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:31.0:1619677448.280254:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.280255:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677448.280256:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:20.0:1619677448.280256:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.280258:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.280258:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.280258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677448.280258:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.280259:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677448.280261:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677448.280266:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.280269:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.280270:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.280270:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.280271:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.280271:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.280281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.280286:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.280289:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.280290:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.2:1619677448.280290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:22.0:1619677448.280292:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.280292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:22.0:1619677448.280299:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.280299:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677448.280300:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:5.0:1619677448.280300:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677448.280301:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:22.0:1619677448.280302:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.280303:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:1.0:1619677448.280303:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677448.280304:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.280305:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.280305:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:17.0:1619677448.280306:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.280306:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.280307:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:17.0:1619677448.280309:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.280310:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.280310:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.280311:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:22.0:1619677448.280312:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:17.0:1619677448.280313:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.280313:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.280315:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.280315:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96717009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.280316:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.280316:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:17.0:1619677448.280316:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.280316:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.280317:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:17.0:1619677448.280317:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:5.0:1619677448.280318:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.280319:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000400:00000200:5.0:1619677448.280319:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.280320:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.280320:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.280321:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:17.0:1619677448.280322:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.280324:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.280325:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:9.2:1619677448.280326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677448.280327:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.280328:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.280329:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.280329:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.0:1619677448.280330:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.280331:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677448.280331:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.280332:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.280332:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.280333:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.280333:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:18.0:1619677448.280334:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:17.0:1619677448.280334:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677448.280334:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.280335:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.280336:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677448.280337:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:9.2:1619677448.280337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:18.0:1619677448.280338:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.280338:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:25.2:1619677448.280340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:18.0:1619677448.280340:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.280342:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:8.0:1619677448.280345:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.280347:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:31.0:1619677448.280348:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:24.0:1619677448.280348:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.280349:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.280350:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.280351:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.280351:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.280353:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.280354:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:30.0:1619677448.280355:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.280356:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.280375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.280383:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.280386:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.280386:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.280388:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.280391:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.280393:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.280394:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.280396:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.280396:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.280398:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.280401:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.280403:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.280406:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.280407:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.280408:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.280409:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.280410:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.280411:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.280411:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.280427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.280434:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.280437:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:2.0:1619677448.280437:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.280439:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.280439:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.280440:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (23)++
00000800:00000200:18.0:1619677448.280441:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (24)--
00000400:00000200:2.0:1619677448.280441:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.280442:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.280443:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.280444:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.280467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.280472:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.280475:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.280476:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.280477:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.280479:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.280480:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.280482:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.280483:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.280484:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.280485:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.280488:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.280490:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.280493:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.280494:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.280495:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.280497:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.280497:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.280498:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.280498:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.280502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.280509:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.280511:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.280512:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.280513:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.280515:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.280517:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.280577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.280583:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.280586:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.280587:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.280589:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.280591:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.280592:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.280594:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96717005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.280595:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.280596:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.280597:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.280600:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.280601:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.280604:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.280605:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.280606:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.280608:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.280608:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.280609:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.280609:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.280614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.280621:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.280624:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.280625:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.280625:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.280628:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.280631:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.280646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.280652:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.280655:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.280656:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.280658:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.280661:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.280662:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.280664:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.280665:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.280666:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.280667:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.280669:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.280671:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.280673:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.280674:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.280674:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.280675:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.280676:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.280677:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.280677:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.280682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.280688:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.280691:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.280691:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.280692:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.280693:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.280694:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.280746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.280752:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677448.280753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677448.280754:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.280755:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.280757:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.280760:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.280761:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677448.280761:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.280762:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96717003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.280764:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:17.0:1619677448.280764:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.280765:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.280766:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.280766:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.280767:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.280769:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.280769:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.280770:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.280771:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.280772:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.280774:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.280774:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.280775:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.280775:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:30.0:1619677448.280776:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:17.0:1619677448.280776:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.280777:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.280778:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.280778:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677448.280779:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.280779:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.280781:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:25.2:1619677448.280783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.280784:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.280785:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.280785:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.280787:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.280787:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.280788:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.280789:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.280791:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:25.2:1619677448.280792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:13.0:1619677448.280794:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.280795:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.280797:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.280798:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:12.0:1619677448.280798:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.280799:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.280801:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.280802:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.280802:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.280803:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.280806:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.280819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.280826:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.280829:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.280830:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.280831:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.280834:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.280835:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.280837:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96717001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.280838:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.280839:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.280840:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.280843:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.280844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677448.280844:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.280847:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.280847:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.280848:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.280849:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.280850:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.280850:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.280851:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677448.280852:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.2:1619677448.280853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677448.280854:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:25.2:1619677448.280855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:31.0:1619677448.280856:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.280856:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.280858:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.280859:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.280859:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:31.0:1619677448.280861:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.280862:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96717000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.280862:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.280862:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677448.280862:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677448.280864:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.280864:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:31.0:1619677448.280865:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.280865:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:20.0:1619677448.280865:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.280865:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.280867:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:13.0:1619677448.280868:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677448.280868:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.280869:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96716999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.280870:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.280871:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.280871:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.280872:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:20.0:1619677448.280873:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:12.0:1619677448.280873:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.280875:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.280876:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:25.2:1619677448.280876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.280877:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:20.0:1619677448.280877:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.280878:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:31.0:1619677448.280879:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.280879:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.280880:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.280880:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677448.280881:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.280882:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:13.0:1619677448.280882:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.0:1619677448.280883:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.280885:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.280885:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677448.280885:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.280886:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:20.0:1619677448.280887:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677448.280887:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:20.0:1619677448.280888:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.280888:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.280889:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:25.2:1619677448.280891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:12.0:1619677448.280892:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.280893:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.280894:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.280896:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.280897:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677448.280899:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.280901:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.280902:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.280903:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.280904:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.280907:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.280983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.280990:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.280992:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.280993:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.280995:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.280997:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.280999:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.281000:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716998] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.281002:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.281003:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.281004:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.281007:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.281008:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.281011:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.281012:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.281013:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.281014:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.281015:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.281016:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.281016:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.281020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.281028:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.281031:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.281031:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.281032:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.281034:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.281037:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.281136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.281141:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.281144:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.281144:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.281146:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.281148:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.281150:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.281151:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.281153:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.281154:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.281155:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.2:1619677448.281156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:6.0:1619677448.281159:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.281160:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.281162:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.281163:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.281164:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.281164:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.281165:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.281165:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.281166:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.281167:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.281167:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677448.281167:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.281168:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:1.0:1619677448.281170:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.281171:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:25.2:1619677448.281172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:1.0:1619677448.281173:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.281174:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.281175:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.281176:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.281179:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:1.0:1619677448.281180:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.281182:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:1.0:1619677448.281182:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.281183:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.281185:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.281185:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.281186:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.281186:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.281187:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.281188:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.281189:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:1.0:1619677448.281189:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.281190:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.281191:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.281192:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.281194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.281197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.281201:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677448.281202:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.281204:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:13.0:1619677448.281204:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.281204:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.281205:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.281206:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677448.281206:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.281208:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.281209:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.281232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.281237:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.281240:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.281241:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.281243:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.281245:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.281246:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.281247:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.281249:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.281250:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.281251:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.281255:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.281257:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.281260:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.281260:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.281261:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.281263:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.281263:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.281264:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.281264:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.281269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.281275:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.281278:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.281278:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.281279:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.281281:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.281284:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.281289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.281294:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.281300:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.281301:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.281302:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.281303:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.281303:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.281423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.281432:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.281434:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.281437:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.281438:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.281439:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.281440:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.281441:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.281442:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.281444:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.281444:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.281448:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.281450:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.281452:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.281453:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.281453:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.281455:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.281456:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.281456:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.281457:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.281461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.281469:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.281472:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.281472:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.281474:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.281476:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.281477:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.281486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.281492:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.281495:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.281496:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.281498:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.281501:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.281502:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.281504:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.281505:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.281506:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.281507:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.281510:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.281512:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.281515:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.281515:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.281516:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.281518:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.281518:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.281519:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.281519:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.281524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.281531:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.281533:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.281535:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.281536:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.281537:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.281540:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.281551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.281559:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.281562:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.281562:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.281564:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.281567:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.281568:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.281569:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.281570:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.281571:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.281573:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.281575:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.281577:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.281579:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.281580:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.281580:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.281582:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.281582:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.281583:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.281584:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.281585:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.281587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.2:1619677448.281590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.281595:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.281596:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.281596:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.281598:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.281599:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.281600:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.281600:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.281601:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.281601:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.281602:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677448.281613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677448.281617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.0:1619677448.281622:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677448.281623:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677448.281624:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:18.0:1619677448.281626:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:20.0:1619677448.281627:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.281627:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.281627:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.281630:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.281630:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.281631:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.281633:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.281633:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.281634:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.281635:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.281635:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.281636:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.281637:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.281637:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.281638:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.281639:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.281645:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.281645:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.281647:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.281649:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.281650:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.281651:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.281652:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.281652:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.281652:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.281653:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.281654:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.281655:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677448.281655:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.281656:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:5.0:1619677448.281656:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677448.281657:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:5.0:1619677448.281657:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.281657:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677448.281658:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677448.281661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.2:1619677448.281661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.281664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677448.281669:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677448.281670:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677448.281671:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:18.0:1619677448.281671:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.281671:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.281671:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.281672:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.281672:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.281673:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.281673:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.281674:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:20.0:1619677448.281675:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.281675:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:2.0:1619677448.281676:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.281678:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677448.281680:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.281736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.2:1619677448.281740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:31.0:1619677448.281743:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.281746:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:18.0:1619677448.281746:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:31.0:1619677448.281748:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.281749:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.281749:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:30.0:1619677448.281750:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.281751:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.281752:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.281753:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.281754:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.281754:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.281756:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.281756:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.281757:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.281758:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.281758:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.281759:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.281761:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.281762:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.281764:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.281766:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.281768:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.281769:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.281770:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.281771:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.281771:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.281772:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.281773:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.281774:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:18.0:1619677448.281774:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.281775:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.281775:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677448.281776:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.281776:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677448.281777:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:25.2:1619677448.281778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.281778:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.281778:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.281779:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.281780:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.2:1619677448.281782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.281785:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:21.2:1619677448.281786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:12.0:1619677448.281787:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677448.281788:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.281789:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:12.0:1619677448.281790:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.281791:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:2.0:1619677448.281791:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:30.0:1619677448.281792:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:13.0:1619677448.281792:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.281792:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:30.0:1619677448.281793:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:13.0:1619677448.281793:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.281793:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:30.0:1619677448.281794:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.281794:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.281794:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:13.0:1619677448.281795:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.281796:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.281797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.281799:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.281800:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.281802:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:21.2:1619677448.281803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:13.0:1619677448.281803:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.281803:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.281804:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.281804:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677448.281809:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.2:1619677448.281811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677448.281812:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.281813:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.281815:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.281817:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677448.281818:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.281820:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:17.0:1619677448.281820:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.281820:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677448.281821:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.281823:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.281824:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.281824:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.281826:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.281827:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.281828:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.281830:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.281830:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.281831:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.281833:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.281833:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.281834:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.281836:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.281837:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.281838:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:17.0:1619677448.281838:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.281840:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.281840:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.281840:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.281841:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.281842:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:17.0:1619677448.281842:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.281843:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677448.281843:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.281843:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.281845:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.281845:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:25.2:1619677448.281846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.281846:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.281847:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:21.2:1619677448.281848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.2:1619677448.281851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.281852:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:6.0:1619677448.281853:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:30.0:1619677448.281855:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:6.0:1619677448.281855:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:30.0:1619677448.281856:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.281856:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.281857:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:30.0:1619677448.281858:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.281858:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.281859:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.281861:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:1.0:1619677448.281861:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.281862:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.281863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.281863:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.281864:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.281866:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677448.281869:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.281872:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.281873:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.281874:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.281875:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.281876:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.281971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677448.281978:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677448.281980:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677448.281982:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677448.281984:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.281987:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.281989:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677448.281990:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.281991:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.281992:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.281993:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677448.281997:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.281998:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677448.282001:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.282002:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677448.282003:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677448.282004:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677448.282005:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677448.282005:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677448.282006:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:25.2:1619677448.282010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.282018:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.282021:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.282021:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.282022:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.282024:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.282028:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.282030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.282037:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.282040:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.282040:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.282041:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.282042:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.282042:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.282154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.282162:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.282165:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.282167:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.282169:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.282170:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.282171:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.282173:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.282174:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.282175:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.282176:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.282179:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.282181:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.282184:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.282185:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.282186:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.282187:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.282188:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.282189:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.282190:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.282194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.282203:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.282205:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.282207:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.282208:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.282208:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.282211:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.282289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.282292:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.282299:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:5.0:1619677448.282299:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.282301:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.282303:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.282305:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.282306:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.282307:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.282308:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.282310:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.282313:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.282315:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.282318:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.282319:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.282319:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.282321:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.282322:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.282322:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.282323:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.282328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.282335:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.282338:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.282339:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.282340:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.282342:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.282343:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.282431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.282438:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.282441:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.282442:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.282443:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.282446:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.282447:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.282449:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.282450:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.282451:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.282452:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677448.282455:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.282457:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.282460:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.282460:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.282461:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.282463:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.282463:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:5.2:1619677448.282464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.0:1619677448.282465:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.282466:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.282466:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.2:1619677448.282469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677448.282470:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677448.282473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:17.0:1619677448.282473:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.282474:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.282476:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.282477:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.282478:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:17.0:1619677448.282479:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.282480:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.282480:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:17.0:1619677448.282481:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.282481:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.282481:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.282482:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.282482:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.282483:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:17.0:1619677448.282484:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.0:1619677448.282484:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:5.0:1619677448.282484:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.282485:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:17.0:1619677448.282488:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.282488:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677448.282490:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.282493:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.282493:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.282494:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.282496:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.282497:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.282497:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.282498:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.282503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.282510:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.282513:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.282514:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.282516:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.282517:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.282519:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.282590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.282598:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.282601:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.282601:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.282603:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.282605:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.282607:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.282608:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.282609:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.282610:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.282611:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.282615:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.282617:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.282619:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.282620:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.282620:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.282622:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.282622:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.282623:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.282624:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.282625:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.282627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.282630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.282634:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.282637:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:10.0:1619677448.282637:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:30.0:1619677448.282638:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.282639:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.282639:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.282640:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.282641:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.282641:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.282642:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.282654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.282662:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.282665:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.282665:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.282667:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.282669:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.282671:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.282672:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.282674:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.282675:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.282675:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.282678:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.282680:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.282682:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.282682:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.282683:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.282684:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.282685:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.282685:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.282686:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.282690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.282697:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.282699:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.282700:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.282701:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.282703:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.282704:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.282716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.282723:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.282727:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.282727:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.282729:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.282732:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.282734:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.282735:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716978] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.282736:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.282737:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.282738:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.282742:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.282744:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.282746:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.282747:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.282747:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.282749:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.282749:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.282750:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.282750:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.282755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.282762:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.282765:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.282766:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.282766:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.282768:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.282770:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.282773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.282780:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.282783:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.282783:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.282783:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.282784:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.282785:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.282858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.282865:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.282867:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.282868:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.282870:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.282874:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.282875:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.282876:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.282878:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.282879:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.282880:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.282883:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.282885:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.282888:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.282888:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.282889:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.282891:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.282891:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.282892:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.282892:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.282896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.282904:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.282907:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.282908:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.282908:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.282910:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.282913:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.283008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.283015:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.283018:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.283018:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.283021:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.283023:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.283024:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.283026:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.283027:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.283028:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.283029:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.283032:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.283033:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.283035:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.283036:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.283037:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.283038:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.283039:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.283040:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.283040:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.283044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.283051:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.283053:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.283054:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.283055:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.283057:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.283058:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.283074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.283078:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.283080:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.283082:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.283083:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.283085:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.283087:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677448.283088:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.283089:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.283090:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.283091:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.283095:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.283097:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.283099:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.283100:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.283100:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.283102:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.283103:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.283103:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.283104:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677448.283108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.283109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677448.283115:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.283116:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.283118:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:31.0:1619677448.283119:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:17.0:1619677448.283119:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.283119:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.283122:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000400:00000200:31.0:1619677448.283123:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.283123:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677448.283125:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:31.0:1619677448.283126:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.283128:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.283129:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.283130:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.283132:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.283133:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677448.283136:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.283137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.283138:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.283141:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.283142:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.283143:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.283144:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.283145:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.283145:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.283146:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.283146:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677448.283146:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677448.283148:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.283148:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.283150:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.283151:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:9.2:1619677448.283151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:30.0:1619677448.283153:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.283154:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.283155:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.283156:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:2.0:1619677448.283158:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:30.0:1619677448.283159:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.283161:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677448.283161:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.283162:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.283162:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.283163:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.283164:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.283164:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:2.0:1619677448.283165:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.283166:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.283166:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.283168:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:2.0:1619677448.283168:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:30.0:1619677448.283170:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.283171:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.283172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.2:1619677448.283176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.283181:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.283182:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.283183:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.283184:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:31.0:1619677448.283185:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.283185:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.283186:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.283187:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.283188:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.283189:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.283190:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.283190:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.283190:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.283191:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.283192:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.283306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.283312:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.283315:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.283316:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.283317:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.283320:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.283321:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.283323:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.283324:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.283325:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.283326:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.283329:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.283331:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.283333:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677448.283334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677448.283334:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.283334:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.283336:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.283336:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.283338:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.283339:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.283339:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.283342:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677448.283343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.283344:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.283345:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.2:1619677448.283345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.283347:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.283349:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.283351:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:17.0:1619677448.283351:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:26.0:1619677448.283352:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.283352:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.283352:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:26.0:1619677448.283353:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.283353:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.283353:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.283354:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.283355:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.283355:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.283355:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:26.0:1619677448.283356:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.283356:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677448.283357:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:26.0:1619677448.283359:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.283361:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.283364:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.283365:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.283366:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.283368:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.283368:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.283369:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.283369:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.283374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.283383:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.283386:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.283387:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.283387:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.283389:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.283392:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.283418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.283426:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.283428:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.283429:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.283431:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.283434:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.283435:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.283436:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.283438:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.283439:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.283439:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.283442:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.283444:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.283446:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.283446:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.283447:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.283448:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.283449:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.283450:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.283450:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677448.283451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:25.2:1619677448.283454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.283460:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:13.0:1619677448.283461:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.283463:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000800:00000200:13.0:1619677448.283464:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:30.0:1619677448.283465:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.283465:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.283467:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.283467:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.283468:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677448.283468:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:13.0:1619677448.283468:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.283469:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:30.0:1619677448.283470:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.283471:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.283472:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.283473:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:25.2:1619677448.283474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:30.0:1619677448.283477:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.283478:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677448.283480:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.283482:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.283482:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:30.0:1619677448.283483:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677448.283483:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.283483:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.283484:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:30.0:1619677448.283485:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677448.283485:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:13.0:1619677448.283485:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.283486:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:21.2:1619677448.283486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:13.0:1619677448.283486:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.283487:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:9.2:1619677448.283491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.283496:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.283498:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.283500:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:18.0:1619677448.283501:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:2.0:1619677448.283501:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.283503:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.283504:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.283505:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677448.283505:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.283507:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.283508:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.283509:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.283510:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:2.0:1619677448.283514:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.283515:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.283515:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.283516:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677448.283517:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677448.283520:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.283521:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677448.283522:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677448.283523:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677448.283524:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.283525:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.283526:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.283526:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677448.283529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:21.2:1619677448.283531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:26.0:1619677448.283535:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:2.0:1619677448.283537:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:26.0:1619677448.283538:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.283539:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.283539:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:2.0:1619677448.283540:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.283540:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:26.0:1619677448.283541:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.283541:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677448.283545:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:4.2:1619677448.283640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.283646:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.283649:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.283649:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.283651:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.283653:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.283654:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.283655:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.283657:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.283658:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.283659:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.283662:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.283663:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.283666:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.283666:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.283667:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.283668:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.283669:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.283669:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.283670:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.283674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.283681:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.283683:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.283684:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.283685:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.283687:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.283688:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.283693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.283700:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.283703:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.283704:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.283704:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.283705:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.283706:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.283789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.283797:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.283799:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.283800:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.283802:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.283805:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.283807:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.283808:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.283809:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.283810:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.283811:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.283814:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.283816:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.283818:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.283819:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.283820:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.283821:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.283822:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.283823:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.283824:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.283825:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.283828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.283830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.2:1619677448.283834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677448.283836:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.283837:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.283838:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677448.283839:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677448.283839:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.283840:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677448.283840:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677448.283840:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.283841:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.0:1619677448.283842:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:1.0:1619677448.283842:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.283845:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.283845:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.283845:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:5.0:1619677448.283847:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.283849:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.283850:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.283851:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.283852:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.283853:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.283857:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.283859:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.283861:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.283862:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.283863:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.283865:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.283865:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.283867:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.283868:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.283869:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:9.2:1619677448.283871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677448.283874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.283877:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:18.0:1619677448.283878:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677448.283879:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.283879:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.283880:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677448.283881:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.283882:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.283883:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.283884:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.283887:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.283900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.283907:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.283910:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.283911:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.283911:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.283912:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.283913:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.283918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.283925:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.283928:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.283930:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.283931:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.283933:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.283934:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.283935:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.283937:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.283938:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.283939:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.283942:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.283944:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.283946:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.283947:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.283948:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.283949:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.283950:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.283951:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.283951:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.283955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.283962:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.283964:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.283965:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.283967:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.283968:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.283969:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.283981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.283986:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.283989:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.283990:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.283992:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.283995:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.283996:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.283998:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.283999:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.284000:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.284001:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.284004:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.284006:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.284009:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.284009:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.284010:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.284012:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.284012:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.284013:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.284013:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677448.284017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.2:1619677448.284018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.284025:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.284026:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:26.0:1619677448.284028:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:20.0:1619677448.284028:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:10.0:1619677448.284028:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677448.284029:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.284031:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.284032:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.284032:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:26.0:1619677448.284034:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.284034:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:26.0:1619677448.284035:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:8.2:1619677448.284036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000400:00000200:26.0:1619677448.284037:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.284038:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.284039:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.284040:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.284043:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.284044:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.284045:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.284047:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.284048:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.284048:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677448.284048:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:26.0:1619677448.284049:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.284049:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.284049:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:26.0:1619677448.284050:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.284050:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:26.0:1619677448.284051:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.284052:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.284053:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.284054:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.284056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.2:1619677448.284058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:4.2:1619677448.284062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.284065:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.284067:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.284067:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.284068:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.284069:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.284069:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.284070:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.284072:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.284072:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.284072:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.284073:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.284074:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.284074:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:26.0:1619677448.284075:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:10.0:1619677448.284075:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.284076:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:26.0:1619677448.284077:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.284078:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.284079:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.284080:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.284083:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.284085:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.284087:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.284088:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.284089:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.284091:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.284091:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.284092:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.284092:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.284097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.284105:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.284107:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.284108:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.284109:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.284111:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.284114:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.284386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.284392:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.284395:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.284395:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.284397:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.284399:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.284400:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.284401:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.284403:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.284404:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.284405:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.284408:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.284410:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.284411:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.284412:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.284413:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.284414:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.284415:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.284415:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.284416:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.284421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.284428:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.284430:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.284431:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.284432:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.284433:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.284434:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.284438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.284445:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.284447:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.284448:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.284448:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.284449:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.284449:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.284480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.284483:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.284485:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.284488:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.284489:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.284490:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.284491:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.284493:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.284494:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.284495:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.284496:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.284500:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.284501:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.284505:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.284505:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.284506:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.284508:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.284509:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.284510:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.284510:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.284515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.284523:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.284525:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.284526:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.284526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:13.0:1619677448.284528:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.284530:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.284531:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.284534:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.284537:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.284538:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.284540:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.284543:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.284544:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.284545:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.284547:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.284548:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.284549:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.284552:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.284554:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.284557:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.284558:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.284559:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.284560:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.284561:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.284561:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.284562:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677448.284566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.2:1619677448.284567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.284574:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.0:1619677448.284575:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.284577:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:5.0:1619677448.284577:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:24.0:1619677448.284578:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:5.0:1619677448.284578:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.284580:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:8.0:1619677448.284580:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.284580:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.284583:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.284584:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:8.0:1619677448.284585:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:8.2:1619677448.284587:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000400:00000200:8.0:1619677448.284591:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.284592:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.284593:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.284594:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.284595:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.284597:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.284598:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000400:00000200:8.0:1619677448.284598:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.284599:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.284600:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.284600:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.284601:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.0:1619677448.284603:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.284604:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.284605:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.284607:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.284607:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.284608:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.284609:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.284613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.284621:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.284623:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.284625:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.284625:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.284626:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.284630:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.284750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.284757:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.284760:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.284761:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.284763:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.284765:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.284767:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.284768:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.284769:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.284770:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.284771:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.284774:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.284775:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.284778:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.284779:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.284779:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.284781:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.284782:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.284783:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.284783:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677448.284787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.284795:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.284798:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.284799:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.284799:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.284801:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.284804:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.284814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.284822:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.284825:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.284827:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.284830:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.284830:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.284831:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.284833:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.284834:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.284835:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.284836:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.284839:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.284841:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.284844:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.284845:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.284845:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.284847:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.284847:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.284848:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.284849:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.284853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.284861:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.2:1619677448.284864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:6.0:1619677448.284864:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.284865:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.284866:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.284867:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:6.0:1619677448.284867:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.284869:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.284871:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.284871:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677448.284872:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.284874:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.284876:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677448.284877:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.284879:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.284880:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.284881:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677448.284884:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.284885:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677448.284888:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.284889:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677448.284890:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.284892:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677448.284892:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.284893:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.284894:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.284898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.284904:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.284907:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.284908:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.284910:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.284911:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.284913:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.285068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.285073:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.285075:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.285077:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.285078:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.285080:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.285082:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677448.285083:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.285084:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.285085:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.285086:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677448.285090:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.285091:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.285094:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.285094:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.285095:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.285097:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.285097:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.285098:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.285099:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.285103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.285110:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.285113:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.285114:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.285114:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.285116:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.285118:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.285120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.285125:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.2:1619677448.285126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:22.0:1619677448.285128:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.285129:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.285131:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.285132:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:22.0:1619677448.285133:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.285134:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:18.0:1619677448.285135:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.285135:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:22.0:1619677448.285136:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.285138:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.285138:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.285139:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.285140:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.285141:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.285142:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.285144:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.285145:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.285146:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.285147:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.285147:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.285148:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677448.285150:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.285151:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:2.0:1619677448.285151:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.285152:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.285153:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.285154:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.285154:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677448.285154:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677448.285155:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677448.285156:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.285157:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.285158:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.2:1619677448.285159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.285159:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.285160:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.285161:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.285161:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.285165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677448.285166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.0:1619677448.285171:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677448.285174:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.285174:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.285174:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.285174:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:20.0:1619677448.285176:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.285176:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:30.0:1619677448.285177:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:18.0:1619677448.285177:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.285177:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.285178:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.285179:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:30.0:1619677448.285180:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.285180:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.285181:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:18.0:1619677448.285182:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:30.0:1619677448.285183:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.285184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:30.0:1619677448.285185:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.285187:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.285188:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.285189:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.285190:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.285191:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677448.285193:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.285195:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.285195:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.285196:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.285196:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.285196:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.285197:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677448.285200:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.285200:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.285201:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.285203:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.285203:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.285204:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.285205:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.285209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.285216:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.285219:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.285220:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.285222:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.285222:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.285225:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.285246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.285253:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.285256:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.285257:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.285259:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.285262:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.285263:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.285265:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716949] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.285266:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.285267:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.285268:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.285271:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.285273:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.285276:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.285277:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.285277:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.285279:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.285279:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.285280:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.285281:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677448.285284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:25.2:1619677448.285285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677448.285290:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:6.0:1619677448.285293:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677448.285294:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:22.0:1619677448.285299:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:18.0:1619677448.285299:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:6.0:1619677448.285301:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (22)--
00000400:00000200:18.0:1619677448.285302:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.285302:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.285304:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.285304:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.285306:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.285307:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.285307:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:18.0:1619677448.285308:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.285310:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.285311:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.285314:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.285316:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.285319:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.285320:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.285320:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.285322:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.285322:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.285323:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.285324:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.285328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.285336:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.285338:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.285339:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.285340:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.285342:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.285345:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.285486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677448.285494:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677448.285497:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677448.285499:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.285501:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677448.285502:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.285503:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677448.285504:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.285506:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.285507:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.285508:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677448.285511:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.285513:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.285516:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.285517:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:31.0:1619677448.285518:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.285520:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.285521:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677448.285522:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677448.285523:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677448.285524:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677448.285526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.285533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:2.0:1619677448.285534:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.285537:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.285538:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.285538:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.285540:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.285541:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677448.285543:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677448.285544:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:2.0:1619677448.285544:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677448.285545:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:5.2:1619677448.285593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.285599:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.285602:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.285603:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.285605:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.285606:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.285608:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.285609:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.285611:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.285612:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.285613:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.285616:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.285618:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.285621:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.285622:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.285622:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.285624:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.285624:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.285625:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.285625:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.285630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.285638:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.285640:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.285641:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.285641:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:21.2:1619677448.285642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:22.0:1619677448.285643:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.285645:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:30.0:1619677448.285649:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.285652:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.285655:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.285656:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.285658:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.285659:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.285661:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716945] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.285662:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.285663:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.285664:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.285667:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.285669:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.285671:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.285672:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.285672:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.285674:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.285675:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.285676:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.285677:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.285678:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.285679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.285683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.285687:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.285689:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.285690:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.285691:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:12.0:1619677448.285691:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.285693:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:12.0:1619677448.285693:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.285694:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677448.285694:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.285695:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677448.285735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.285743:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.285745:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.285747:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.285749:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.285750:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.285751:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.285752:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.285753:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.285754:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.285755:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.285758:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.285759:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.285761:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.285762:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.285762:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.285764:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.285764:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.285765:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:5.2:1619677448.285765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677448.285766:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.285770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:21.2:1619677448.285770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:10.0:1619677448.285772:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.285775:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.285776:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:12.0:1619677448.285776:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:24.0:1619677448.285777:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:10.0:1619677448.285777:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.285778:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.285779:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.285780:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:10.0:1619677448.285780:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.285781:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.285781:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:8.0:1619677448.285781:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677448.285782:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.285783:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:10.0:1619677448.285783:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.285784:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.285784:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.285785:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.285786:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.285786:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:24.0:1619677448.285788:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.285789:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96716942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.285789:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.285791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:24.0:1619677448.285791:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.285791:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.285791:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:24.0:1619677448.285792:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.285793:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.285794:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.285794:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:24.0:1619677448.285795:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.285795:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.285796:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677448.285797:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:13.0:1619677448.285797:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.285797:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.285798:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.285799:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677448.285800:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.285800:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:25.2:1619677448.285801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:24.0:1619677448.285801:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:13.0:1619677448.285801:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:24.0:1619677448.285802:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:13.0:1619677448.285802:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.285802:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.285803:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.285803:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:13.0:1619677448.285803:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677448.285804:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:5.2:1619677448.285804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677448.285805:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:26.0:1619677448.285809:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:8.2:1619677448.285809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.285810:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.285811:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.285813:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.285814:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.285815:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.285815:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.285816:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.0:1619677448.285816:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:10.0:1619677448.285816:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.285817:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.0:1619677448.285819:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.285820:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.285821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000400:00000200:20.0:1619677448.285822:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.285823:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.285825:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:26.0:1619677448.285827:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.285830:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.285830:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.285831:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.285832:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.285832:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.285867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.285873:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.285876:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.285876:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.285879:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.285881:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.285882:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.285884:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.285885:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.285886:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.285887:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.285890:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.285892:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.285895:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.285896:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.285897:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.285898:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.285899:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.285899:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.285900:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.285904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.285911:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.285913:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.285914:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.285915:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.285917:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.285920:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677448.286105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677448.286107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.286111:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.286113:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.286114:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.286115:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.286116:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.286117:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.286118:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.286119:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.286120:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.286121:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677448.286122:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.286123:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.286124:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.286124:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.286125:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.286126:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.286127:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.286127:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677448.286128:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.286130:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.286132:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.286134:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.286134:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.286136:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.286137:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.286138:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.286138:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.286140:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.286140:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.286140:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677448.286141:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.286141:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.286142:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.286142:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677448.286143:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.286144:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.286145:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:25.2:1619677448.286146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.286146:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.286147:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.286150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:21.2:1619677448.286150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:5.2:1619677448.286152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.286155:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.286156:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:12.0:1619677448.286157:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:6.0:1619677448.286157:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:1.0:1619677448.286158:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677448.286159:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.286159:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.286160:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677448.286160:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.286161:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677448.286161:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.286161:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.286161:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677448.286162:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.286163:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677448.286163:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.286165:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.286166:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.286166:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677448.286167:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:22.0:1619677448.286168:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:6.0:1619677448.286168:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.286170:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.286171:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.286172:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.286173:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.286177:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.286179:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.286182:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.286183:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.286184:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.286186:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.286187:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.286187:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.286188:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.286192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.286199:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.286202:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.286203:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.286204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:30.0:1619677448.286205:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.286206:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.286208:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.286210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677448.286210:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.286213:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.286215:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.286217:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.286218:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:18.0:1619677448.286219:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.286221:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:18.0:1619677448.286221:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677448.286223:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.286224:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.286224:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:18.0:1619677448.286224:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.286226:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.286226:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.286226:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677448.286229:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.286230:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.286231:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.286232:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.286234:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.286234:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:31.0:1619677448.286235:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:18.0:1619677448.286236:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.286237:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677448.286238:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:31.0:1619677448.286239:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.286240:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.286240:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677448.286241:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677448.286241:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.286241:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677448.286244:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.286245:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.286246:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:25.2:1619677448.286246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.286247:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.286248:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.286249:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.286250:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:6.0:1619677448.286253:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.2:1619677448.286254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:6.0:1619677448.286256:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.286257:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.286257:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.286259:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.286261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:6.0:1619677448.286261:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677448.286264:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.286267:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:24.0:1619677448.286268:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:5.0:1619677448.286268:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.286270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:20.0:1619677448.286270:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.286270:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.286271:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.286271:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.286273:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.286274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.0:1619677448.286274:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:24.0:1619677448.286276:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.286277:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.286278:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.286278:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:24.0:1619677448.286280:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.286281:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:24.0:1619677448.286281:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.286281:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.286281:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:24.0:1619677448.286282:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:22.0:1619677448.286282:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.286283:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677448.286284:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677448.286284:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:30.0:1619677448.286287:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.286287:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.286287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:31.0:1619677448.286288:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677448.286289:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.286290:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.286291:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:24.0:1619677448.286292:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.286292:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:24.0:1619677448.286293:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:30.0:1619677448.286300:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.286300:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:24.0:1619677448.286301:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.286301:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:30.0:1619677448.286302:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.286303:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.286303:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.286303:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:30.0:1619677448.286304:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677448.286304:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:2.0:1619677448.286304:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.286305:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:2.0:1619677448.286306:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.286307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:2.0:1619677448.286308:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.286309:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.286309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:2.0:1619677448.286310:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96716933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.286311:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.286312:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.286313:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.286313:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:2.0:1619677448.286313:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.286314:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.286314:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:2.0:1619677448.286314:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.286315:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.286316:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.286317:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.286317:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:30.0:1619677448.286318:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.286318:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:2.0:1619677448.286318:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.286319:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.286320:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.286320:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:25.2:1619677448.286322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.286322:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.286322:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.286323:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.286324:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:20.0:1619677448.286325:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.286325:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.286325:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.286326:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.286327:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:5.0:1619677448.286327:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:2.0:1619677448.286327:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.286327:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:20.0:1619677448.286328:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96716932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.286328:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:20.0:1619677448.286329:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.286329:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:20.0:1619677448.286330:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.286330:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:20.0:1619677448.286332:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:13.0:1619677448.286333:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.2:1619677448.286333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:13.0:1619677448.286334:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.286335:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677448.286336:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.286337:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.286338:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:13.0:1619677448.286338:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677448.286340:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.286340:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:25.2:1619677448.286341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.286341:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.286342:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.286342:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.286343:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:5.0:1619677448.286343:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.286344:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.286344:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.286344:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.286345:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677448.286346:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:12.0:1619677448.286347:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.286347:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:25.2:1619677448.286349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:12.0:1619677448.286351:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.286351:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.286352:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.286353:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.286354:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.286357:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.286360:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.286360:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.286361:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.286363:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.286365:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.286415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.286421:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.286424:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.286424:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.286427:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.286429:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.286431:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.286432:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.286434:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.286435:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.286436:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.286439:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.286440:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.286443:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.286444:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.286444:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.286446:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.286447:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.286447:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.286448:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.286452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.286460:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.286462:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.286463:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.286463:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.286465:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.286468:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.286590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.286597:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.286600:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.286600:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:20.2:1619677448.286603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:8.0:1619677448.286603:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.286605:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.286606:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.286608:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.286609:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.286610:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.286611:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:8.0:1619677448.286611:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.286614:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.286616:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.286616:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.286617:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.286618:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.286619:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.286621:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:8.0:1619677448.286621:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.286622:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.286622:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.286622:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677448.286623:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.286624:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.286624:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.286624:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:31.0:1619677448.286625:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.286625:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.286626:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:31.0:1619677448.286628:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.286630:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.2:1619677448.286630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.286633:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.286633:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.286634:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.286636:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.286636:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.286638:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:17.0:1619677448.286638:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.286639:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.286640:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677448.286640:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.2:1619677448.286641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.286641:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.286642:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.286644:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.286647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677448.286647:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.286648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.286653:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:1.0:1619677448.286655:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.286656:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.286656:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.286657:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.286658:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.286659:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.286659:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.286659:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.286659:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.286660:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.2:1619677448.286660:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677448.286660:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.286661:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.286662:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677448.286662:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677448.286663:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.2:1619677448.286667:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.286668:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677448.286669:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.286669:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.286670:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.286671:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:5.0:1619677448.286671:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677448.286672:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.286672:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.0:1619677448.286672:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:26.0:1619677448.286674:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.286675:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.286676:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.286677:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.286678:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677448.286678:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.286680:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.286680:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.286681:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.286682:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.286683:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.286683:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.286684:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677448.286685:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.286686:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.286688:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.286690:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.286692:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.286693:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.286695:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.286695:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.286696:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.286696:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.286697:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:17.0:1619677448.286697:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.286698:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.286698:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.286699:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.286699:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.286700:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677448.286700:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.286701:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.286702:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.286702:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.286703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:25.2:1619677448.286707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.286709:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.286711:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.286712:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.286712:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.286714:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.286715:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677448.286716:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:6.0:1619677448.286717:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.286718:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.286718:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.286720:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.286722:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.286783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.286790:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.286793:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.286794:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.286796:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.286799:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.286800:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.286801:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.286803:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.286804:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.286805:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677448.286808:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.286810:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.2:1619677448.286810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:24.0:1619677448.286813:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.286814:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.286814:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677448.286815:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677448.286816:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.286816:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.286818:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:22.0:1619677448.286818:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:24.0:1619677448.286819:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:6.0:1619677448.286819:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677448.286820:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:22.0:1619677448.286821:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.286822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:22.0:1619677448.286823:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.286824:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:5.2:1619677448.286824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000400:00000200:22.0:1619677448.286826:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.286827:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.286827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000400:00000200:22.0:1619677448.286828:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.286829:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.286830:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:22.0:1619677448.286833:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.286833:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:8.0:1619677448.286833:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000400:00000200:20.0:1619677448.286834:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.286835:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.286836:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:22.0:1619677448.286836:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.286836:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.286836:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:5.0:1619677448.286836:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.286837:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.286838:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:22.0:1619677448.286839:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.286839:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:8.0:1619677448.286839:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.286840:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677448.286840:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.286840:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.286842:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.286843:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:8.0:1619677448.286843:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.286844:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.286845:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000400:00000200:8.0:1619677448.286845:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:22.0:1619677448.286846:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:8.0:1619677448.286846:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96716924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.286847:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.286848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:8.0:1619677448.286848:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.286849:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:21.2:1619677448.286851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:8.0:1619677448.286852:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.286854:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677448.286855:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:8.0:1619677448.286856:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.286856:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:8.0:1619677448.286857:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:30.0:1619677448.286858:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:8.0:1619677448.286858:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:6.0:1619677448.286858:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:30.0:1619677448.286859:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.286859:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.286859:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:6.0:1619677448.286859:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.286860:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677448.286860:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000400:00000200:30.0:1619677448.286861:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.286861:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:4.2:1619677448.286861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.286862:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.286864:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:9.2:1619677448.286864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:26.0:1619677448.286868:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.286870:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:24.0:1619677448.286871:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:10.0:1619677448.286871:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.286873:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.286873:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.286873:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.286874:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:26.0:1619677448.286875:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.286875:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.286877:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:24.0:1619677448.286877:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:26.0:1619677448.286878:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716923] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.286879:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.286880:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.286881:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.286884:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.286886:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.286889:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.286890:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.286890:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.286892:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.286892:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.286893:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.286894:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.286895:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.286898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.286900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.286905:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.286907:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.286908:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.286909:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.286909:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.286910:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.286910:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.286911:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.286912:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.286915:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.286918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.286925:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.286928:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.286928:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.286929:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.286930:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.286931:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.286977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.286982:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.286985:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.286986:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.286988:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.286991:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.286993:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.286994:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.286995:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.286996:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.286997:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.287000:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.287001:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.287004:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.287005:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.287006:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.287008:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.287008:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.287010:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.287011:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.287011:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:25.2:1619677448.287014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.287016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:13.0:1619677448.287021:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.287022:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:13.0:1619677448.287024:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.287024:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:13.0:1619677448.287025:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.287025:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:13.0:1619677448.287026:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.287026:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677448.287027:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.287029:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.287085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.287090:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.287093:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.287094:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.287096:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.287098:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.287100:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.287101:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.287102:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.287103:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.287104:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.287108:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.287110:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.287112:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.287113:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.287113:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.287115:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.287115:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.287116:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.287116:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.287121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.287129:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.287131:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.287132:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.287132:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.287134:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.287136:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.287137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.287145:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.287148:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.287150:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.287152:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.287153:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.287155:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.287156:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.287157:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.287158:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.287159:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.287162:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.287164:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.287166:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.287167:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.287168:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.287169:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.287170:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.287171:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.287171:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.287175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.287181:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.287184:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.287184:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.287186:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.287188:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.287190:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.287276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.287283:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.287286:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.287288:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.287290:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.287290:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.287291:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.287292:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.287298:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.287299:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.287300:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.287303:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.287305:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.287307:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.287308:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.287308:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.287310:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.287310:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.287311:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.287311:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.287316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.287323:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.287325:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.287326:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.287327:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.287329:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.287331:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.287481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.287490:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.287493:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.287494:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.287496:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.287499:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.287500:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.287501:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.287503:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.287504:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.287504:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.287507:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.287509:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.287512:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.287512:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.287513:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.287515:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.287515:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.287516:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.287516:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677448.287519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.2:1619677448.287520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:17.0:1619677448.287526:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.287528:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:17.0:1619677448.287529:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.287530:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.287531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:17.0:1619677448.287532:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.287532:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.287533:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:17.0:1619677448.287535:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.287535:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.287536:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:8.0:1619677448.287537:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.287538:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.287538:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:22.0:1619677448.287539:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:17.0:1619677448.287539:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.287540:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96716917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.287541:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.287542:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:17.0:1619677448.287542:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:8.0:1619677448.287542:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.287543:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.287543:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:8.0:1619677448.287545:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.287546:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.287546:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677448.287548:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.287549:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677448.287551:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:8.0:1619677448.287551:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.287553:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.287554:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.287554:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.287555:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.0:1619677448.287556:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.287557:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:8.0:1619677448.287557:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:17.0:1619677448.287558:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.287558:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677448.287559:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.287560:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.0:1619677448.287560:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.287560:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:17.0:1619677448.287561:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677448.287561:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.287562:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.287563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.2:1619677448.287565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.2:1619677448.287566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.287570:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:1.0:1619677448.287571:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677448.287573:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:1.0:1619677448.287573:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:5.0:1619677448.287574:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:1.0:1619677448.287574:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677448.287575:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.287575:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677448.287576:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.287576:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:30.0:1619677448.287578:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:5.0:1619677448.287578:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.287579:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.287580:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.0:1619677448.287580:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.287582:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.287625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.287632:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.287635:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.287636:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.287638:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.287640:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.287642:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:20.2:1619677448.287643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:10.0:1619677448.287643:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.287645:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.287645:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.287646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:20.0:1619677448.287646:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:10.0:1619677448.287646:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.287648:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:5.0:1619677448.287651:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.287652:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.287652:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677448.287653:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.287655:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.287655:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:6.0:1619677448.287655:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.287656:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677448.287657:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677448.287658:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.287658:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.287659:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:6.0:1619677448.287659:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.287660:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.287660:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:20.0:1619677448.287661:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.287661:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:20.0:1619677448.287662:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.287662:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.287662:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677448.287662:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.287663:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677448.287664:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.287665:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96716913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.287666:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.287667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:6.0:1619677448.287667:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.287668:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.287669:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:6.0:1619677448.287669:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:20.0:1619677448.287672:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.287673:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.287673:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:12.0:1619677448.287673:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:6.0:1619677448.287673:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.287675:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:6.0:1619677448.287675:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677448.287676:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:12.0:1619677448.287676:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:20.0:1619677448.287677:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000400:00000200:12.0:1619677448.287677:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.287678:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:6.0:1619677448.287678:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.287679:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:12.0:1619677448.287679:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.287679:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.287679:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:13.0:1619677448.287680:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:25.2:1619677448.287681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:6.0:1619677448.287681:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:12.0:1619677448.287682:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677448.287682:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.287683:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.287683:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677448.287684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:26.0:1619677448.287687:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:9.2:1619677448.287688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:26.0:1619677448.287690:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.287690:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.287690:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000400:00000200:26.0:1619677448.287691:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.287692:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:26.0:1619677448.287693:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.287693:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.287693:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:26.0:1619677448.287696:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:18.0:1619677448.287697:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.287700:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.287701:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.287701:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.2:1619677448.287703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:18.0:1619677448.287703:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.287706:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677448.287708:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.287711:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.287713:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.287713:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.287716:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.287718:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.287719:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.287721:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.287722:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.287723:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.287726:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.287728:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.287730:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.287731:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.287731:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.287733:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.287734:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.287735:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.287736:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.287737:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.287739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.287741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.287744:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:22.0:1619677448.287746:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677448.287746:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.287747:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.287748:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677448.287749:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.287750:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.287750:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.287751:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.287753:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.287930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907b780e00] (20)++
00000800:00000200:2.0:1619677448.287936:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907b780e00] (21)++
00000800:00000200:2.0:1619677448.287939:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.52@o2ib3
00000800:00000200:18.0:1619677448.287940:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907b780e00] (22)--
00000400:00000200:2.0:1619677448.287941:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.52@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.287944:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.287946:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677448.287947:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.287948:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.287949:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.287951:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677448.287953:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.52@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.287955:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677448.287958:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.287958:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:2.0:1619677448.287959:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:2.0:1619677448.287961:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:2.0:1619677448.287961:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907b780e00] (21)++
00000800:00000200:2.0:1619677448.287962:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907b780e00] (22)--
00000800:00000200:2.0:1619677448.287963:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907b780e00] (21)--
00000800:00000200:8.2:1619677448.287967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.2:1619677448.287976:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.0:1619677448.287979:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.287982:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.287982:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.287984:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:20.0:1619677448.287984:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.287985:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:30.0:1619677448.287987:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:20.0:1619677448.287987:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:30.0:1619677448.287989:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.287990:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.287991:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.287993:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.287994:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.287995:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.287996:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.287997:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.288001:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.288003:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.288005:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.288006:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.288007:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.288008:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.288009:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.288010:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.288010:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.288014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.288020:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.288022:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.288023:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.288025:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.288026:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.288027:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.288036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.288042:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.288045:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.288045:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.288047:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.288050:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.288052:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.288053:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.288054:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.288055:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.288056:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.288060:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.288062:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.288065:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.288065:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.288066:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.288068:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.288068:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.288069:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.288069:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.288074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.288081:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.288083:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.288084:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.288086:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.288087:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.288089:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.288119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.288126:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.288129:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.288129:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.288131:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.288133:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.288135:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.288136:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716908] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.288137:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.288138:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.288139:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.288143:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.288145:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.288148:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.288148:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.288149:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.288151:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.288151:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.288152:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.288152:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.288157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.288165:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.288168:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.288168:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.288169:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.288171:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.288174:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.288202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.288208:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.288211:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.288212:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.288214:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.288217:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.288218:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.288219:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.288220:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.288221:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.288222:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.288225:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.288227:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.288230:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.288230:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.288231:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.288233:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.288233:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.288234:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.288234:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.288239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.288246:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.288249:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.288250:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.288250:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.288252:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.288254:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.288273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.288280:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.288283:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.288284:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.288286:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.288288:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.288290:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.288291:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.288293:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.288298:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.288300:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.288303:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.288305:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.288308:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.288309:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.288310:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.288312:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.288312:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.288313:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.288313:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.288318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.288327:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.288329:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.288331:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.288331:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.288333:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.288336:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.288396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.288404:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.288407:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.288408:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.288411:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.288411:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.288412:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.288413:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.288414:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.288415:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.288416:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.288419:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.288420:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.288423:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.288423:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.288424:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.288425:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.288426:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.288427:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.288428:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.288431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.288438:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.288441:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.288442:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.288443:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.288445:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.288446:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.288449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.288455:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.288458:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.288458:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.288459:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.288460:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.288460:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.288532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.288538:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.288541:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.288542:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.288545:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.288547:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.288549:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.288550:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.288551:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.288552:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.288553:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.288557:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.288558:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.288561:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.288562:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.288562:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.288564:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.288565:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.288565:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.288566:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.288570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.288577:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.288580:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.288581:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.288583:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.288584:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.288586:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.288591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677448.288597:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.288600:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.288601:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.288602:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.288602:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.288604:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.288621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.288627:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.288630:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.288631:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.288632:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.288635:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.288637:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:20.2:1619677448.288638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:18.0:1619677448.288638:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.288639:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.288640:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.288641:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677448.288644:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.288644:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677448.288646:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:5.0:1619677448.288646:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.288649:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677448.288649:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.288649:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.288650:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677448.288651:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:5.0:1619677448.288651:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.288653:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677448.288653:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:5.0:1619677448.288653:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:18.0:1619677448.288654:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.288654:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:5.0:1619677448.288654:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.288656:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.288656:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.288658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000400:00000200:5.0:1619677448.288658:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.288661:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.288663:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.288665:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:5.0:1619677448.288666:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.288667:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.288667:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.288668:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.288668:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.288669:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.288669:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.288670:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:10.0:1619677448.288671:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.288671:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.288671:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:10.0:1619677448.288674:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:25.2:1619677448.288675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.288681:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.288684:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.288685:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.288687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000400:00000200:12.0:1619677448.288687:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.288688:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.288690:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.288695:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.288698:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677448.288698:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677448.288700:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.288703:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.288704:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.288705:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.288707:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.288708:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.288709:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.288712:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.288714:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.288716:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.288717:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.288718:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.288719:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.288720:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.288720:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677448.288721:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677448.288725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.288731:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.288734:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.288734:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.288736:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.288737:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.288738:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.288745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:12.0:1619677448.288752:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.288754:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.288755:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.288756:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.288756:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.288758:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.288772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.288779:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.288782:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.288783:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.288784:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.288787:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.288789:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:5.2:1619677448.288789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:10.0:1619677448.288790:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.288791:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.288793:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.288794:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.288798:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:10.0:1619677448.288798:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.288799:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.288800:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.288801:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.288802:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.288803:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.288803:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:26.0:1619677448.288804:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.288805:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:10.0:1619677448.288805:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:26.0:1619677448.288806:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.288808:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.288808:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.288809:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.288810:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:25.2:1619677448.288811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.288811:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.288812:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:26.0:1619677448.288813:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.288813:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.288814:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.288816:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.288817:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.288817:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.288818:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:26.0:1619677448.288819:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.288819:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.288820:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:26.0:1619677448.288820:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:30.0:1619677448.288821:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.288822:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.288822:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.288822:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.288824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.288825:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.288826:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.288827:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:5.2:1619677448.288828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.288829:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.288830:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.288831:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.288833:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.288834:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:10.0:1619677448.288835:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.288837:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.288838:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.288839:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:4.2:1619677448.288973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.288980:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.288983:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.288984:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.288985:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.288988:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.288990:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.288991:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.288992:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.288993:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.288994:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.288997:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.288998:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.289001:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.289002:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.289002:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.289004:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.289005:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.289005:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.289006:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.289010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.289018:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.289020:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.289021:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.289021:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.289023:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.289026:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.289127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.289134:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.289137:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.289139:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.289141:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.289141:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.289142:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.289144:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.289145:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.289146:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.289147:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.289150:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.289152:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.289154:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.289155:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.289155:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.289157:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.289157:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.289159:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.289160:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.289161:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.289162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.289166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.289169:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.289171:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.289172:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.289173:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.289174:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677448.289175:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.289176:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:13.0:1619677448.289176:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.289177:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.289178:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677448.289261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677448.289268:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677448.289270:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677448.289272:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.289275:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677448.289275:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.289276:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:21.2:1619677448.289276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:30.0:1619677448.289277:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.289279:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.289280:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.289280:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.289283:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.289283:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677448.289285:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.289285:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.289286:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677448.289288:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.289288:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.289289:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.289290:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:22.0:1619677448.289290:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.289291:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.289292:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:22.0:1619677448.289292:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:30.0:1619677448.289298:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677448.289298:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:22.0:1619677448.289299:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716895] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.289299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:22.0:1619677448.289300:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.289302:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.289303:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.289306:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.289306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677448.289308:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:20.0:1619677448.289308:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:22.0:1619677448.289311:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.289311:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677448.289312:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000400:00000200:20.0:1619677448.289312:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.289313:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:20.0:1619677448.289313:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.289314:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.289314:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.289315:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677448.289315:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.289316:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677448.289317:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:22.0:1619677448.289317:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677448.289317:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:10.0:1619677448.289317:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.289319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000400:00000200:26.0:1619677448.289320:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.289321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000400:00000200:26.0:1619677448.289322:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.289324:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.289325:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.289326:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:26.0:1619677448.289327:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.289328:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.289328:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.289329:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:5.0:1619677448.289329:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.289330:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.289330:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000400:00000200:10.0:1619677448.289331:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.289331:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.289332:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.289333:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:10.0:1619677448.289333:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.289334:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.289336:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.289338:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.289338:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.289339:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.289339:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.289340:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.289341:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.289342:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.289342:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.289342:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.289343:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.289343:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.289343:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.289344:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:26.0:1619677448.289345:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.289345:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.289347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:4.2:1619677448.289351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.289356:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.289357:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.289358:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.289359:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.289360:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.289360:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.289360:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.289361:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.289362:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.289362:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.289362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cffdc4000] (20)++
00000800:00000200:17.0:1619677448.289370:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cffdc4000] (21)++
00000800:00000200:17.0:1619677448.289373:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.11@o2ib3
00000800:00000200:1.0:1619677448.289373:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc4000] (22)--
00000400:00000200:17.0:1619677448.289376:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.11@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.289378:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.289380:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:17.0:1619677448.289381:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.289382:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.289383:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.289384:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:17.0:1619677448.289387:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.289389:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:17.0:1619677448.289393:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f90d380] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.289393:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e49d37c00] (20)++
00000800:00000200:17.0:1619677448.289394:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e49d37c00] (21)++
00000800:00000200:17.0:1619677448.289396:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e49d37c00] (22)--
00000800:00000200:17.0:1619677448.289396:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cffdc4000] (21)++
00000800:00000200:17.0:1619677448.289398:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cffdc4000] (22)++
00000800:00000200:17.0:1619677448.289400:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cffdc4000] (23)--
00000800:00000200:17.0:1619677448.289400:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc4000] (22)--
00000800:00000200:24.2:1619677448.289403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (21)++
00000800:00000200:5.2:1619677448.289405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cffdc4000] (21)++
00000800:00000200:24.0:1619677448.289406:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)++
00000800:00000200:24.0:1619677448.289409:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e49d37c00] (23)--
00000400:00000200:24.0:1619677448.289411:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.289411:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)--
00000800:00000200:1.0:1619677448.289411:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cffdc4000] (22)++
00000400:00000200:24.0:1619677448.289413:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.289413:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cffdc4000] (23)--
00000800:00000200:1.0:1619677448.289414:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc4000] (22)--
00000800:00000200:1.0:1619677448.289414:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc4000] (21)--
00000800:00000200:24.0:1619677448.289416:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)--
00000800:00000200:5.2:1619677448.289445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.289453:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.289455:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.289456:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.289458:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.289460:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.289461:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.289462:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.289463:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.289464:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.289465:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.289468:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.289470:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.289473:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.289474:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.289474:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.289476:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.289477:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.289477:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.289478:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677448.289489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:25.2:1619677448.289492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.0:1619677448.289495:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.289497:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.289499:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.289500:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677448.289500:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:5.0:1619677448.289502:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.289503:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.289503:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.289504:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.289504:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.289506:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.289507:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.289508:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.289509:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.289510:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:5.0:1619677448.289510:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.289514:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.289515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:5.0:1619677448.289515:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.289518:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.289518:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.289519:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.289521:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.289522:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677448.289523:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.0:1619677448.289523:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.289523:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677448.289525:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.289527:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:9.2:1619677448.289527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.289527:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.289528:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.289528:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677448.289535:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.289538:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.289539:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.289539:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.289541:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.289544:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.289765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.289772:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.289775:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.289776:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.289778:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.289781:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.289782:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.289783:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.289785:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.289786:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.289787:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.289791:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.289792:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.289794:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.289795:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.289796:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.289797:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.289798:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.289799:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.289799:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.289803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.289811:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.289814:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.289814:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.289816:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.289818:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.289818:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:21.2:1619677448.289872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.289879:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:21.2:1619677448.289880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677448.289882:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.289883:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.289884:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.289887:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:8.0:1619677448.289887:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.289889:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:8.0:1619677448.289889:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.289891:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.289892:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.289892:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.289893:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:8.0:1619677448.289893:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.289894:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.289894:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677448.289896:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.289898:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.289899:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.289900:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.289901:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.289901:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677448.289901:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.289904:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.289905:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.289905:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.289905:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.289907:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677448.289907:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.289907:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.289908:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.289909:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:30.0:1619677448.289910:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.289911:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.289912:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.289913:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:9.2:1619677448.289913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.289914:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.289915:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.289915:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.289919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.289921:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.289924:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.289924:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.289925:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.289927:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.289927:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.289930:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.289931:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.289931:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:6.0:1619677448.289931:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.289933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:6.0:1619677448.289933:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.289936:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.289938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.289940:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.289943:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.289944:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.289944:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.289945:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.289945:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677448.289946:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.289949:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.289949:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.289950:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.289951:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.289952:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.290060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.290063:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.290065:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.290067:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.290068:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.290070:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.290072:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.290073:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.290074:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.290075:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.290076:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.290080:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.290082:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.290084:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.290085:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.290086:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.290088:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.290089:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.290090:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.290090:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.290094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.290102:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.290104:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.290105:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.290107:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.290109:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.290110:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.290158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.290163:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.290166:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.290167:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.290169:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.290171:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.290173:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.290174:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.290175:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.290176:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.290177:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.290181:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.290182:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.290185:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.290186:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.290187:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.290188:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.290189:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.290190:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.290190:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.290194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.290200:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.290203:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.290204:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.290204:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.290206:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.290209:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:4.2:1619677448.290261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.290268:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.290270:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.290271:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.290273:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.290275:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.290276:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.290277:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.290278:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.290279:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.290280:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.290284:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.290285:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.290287:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.290288:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.290288:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.290290:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.290290:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.290291:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.290291:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.290297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.290303:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.290305:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.290306:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.290307:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.290307:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.290309:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.290317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.290323:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.290325:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.290326:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.290326:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.290327:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.290327:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.290526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.290533:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.290536:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.290537:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.290538:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.290541:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.290541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:24.0:1619677448.290542:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.290544:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.290545:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.290546:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.290547:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.290547:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.290550:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:24.0:1619677448.290551:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.290551:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:24.0:1619677448.290553:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.290553:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.290556:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.290556:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.290557:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.290557:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:18.0:1619677448.290557:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:24.0:1619677448.290559:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.290559:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:18.0:1619677448.290559:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.290560:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:18.0:1619677448.290560:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.290561:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677448.290561:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.290562:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.290565:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.290565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.290567:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.290569:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.290570:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.290570:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.290572:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.290572:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.290573:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677448.290573:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677448.290574:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677448.290575:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.290576:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.290577:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.290578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:17.0:1619677448.290579:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.290582:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677448.290585:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.290588:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.290589:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.290589:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.290591:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.290593:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.290604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.290611:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.290614:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.290615:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.290617:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.290620:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.290621:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.290622:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.290624:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.290625:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.290626:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.290629:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.290631:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.290634:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.290635:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.290635:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.290637:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.290638:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.290639:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.290639:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.290643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.290651:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.290653:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.290654:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.290655:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.290657:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.290660:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.290675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.290683:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.290686:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.290688:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.290691:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.290691:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.290692:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.290693:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.290695:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.290696:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.290697:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.290699:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.290701:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.290703:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.290704:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.290704:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.290706:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.290707:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.290708:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.290708:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677448.290711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:25.2:1619677448.290714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.290721:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.290722:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.290722:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.290723:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.290724:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.290725:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.290726:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.290727:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.290729:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.290730:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.290732:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.290733:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.290734:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.290735:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.290736:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.290737:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:10.0:1619677448.290738:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.290739:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.290742:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.290743:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.290743:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.290745:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.290746:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.290747:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.290747:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.290751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.290759:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.290762:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.290762:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.290763:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.290765:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:1.0:1619677448.290768:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.290785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.2:1619677448.290791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.290793:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.290795:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:17.0:1619677448.290796:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.290796:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.290798:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.290799:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.290800:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.290801:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.290802:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677448.290802:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.290804:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.290805:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.290805:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.290807:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.290807:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677448.290809:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677448.290809:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.290811:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.290812:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.290813:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677448.290816:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.290817:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.290818:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.290820:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.290820:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.290821:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.290821:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.290823:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.290823:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.290823:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.290824:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.290824:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677448.290824:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.290824:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.290825:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.290826:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.290827:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.290828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:17.0:1619677448.290828:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.290831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677448.290834:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.290836:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.290836:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.290837:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.290838:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.290838:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:10.0:1619677448.290840:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:1.0:1619677448.290841:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.290841:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.290842:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.290843:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.290845:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.290848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.290855:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.290857:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.290858:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.290859:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.290859:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677448.290859:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.2:1619677448.290910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.290918:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.290921:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.290923:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.290924:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.290925:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.290927:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.290928:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.290928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:30.0:1619677448.290930:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.290931:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.290932:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.290934:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677448.290935:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.290937:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.290937:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.290939:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.290940:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.290940:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677448.290941:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.290942:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:5.0:1619677448.290942:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.290943:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:30.0:1619677448.290944:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.290944:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.290945:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:5.0:1619677448.290945:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.290946:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:5.0:1619677448.290946:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.290947:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.290948:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.2:1619677448.290950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:5.0:1619677448.290952:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.290953:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.290956:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.290957:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.290957:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.290957:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.290958:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.290959:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.290959:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.290960:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:20.0:1619677448.290961:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.290962:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.290963:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.290963:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.290964:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:9.2:1619677448.290965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.290965:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.290968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.290971:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.290973:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:2.0:1619677448.290973:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677448.290974:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.290974:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677448.290976:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.290977:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.290977:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.290979:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.290981:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.290999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677448.291007:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677448.291009:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677448.291010:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677448.291012:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677448.291015:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.291016:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677448.291017:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.291019:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.291020:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.291021:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677448.291024:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677448.291026:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.291027:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.291028:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677448.291028:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.291030:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.291030:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677448.291031:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677448.291031:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677448.291035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.291041:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.291043:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.291044:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.291044:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.291046:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677448.291047:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.291054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677448.291059:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.291062:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.291062:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.291063:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.291063:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.291064:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677448.291158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.291161:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.291163:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.291166:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.291166:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.291169:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.291170:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.291171:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716874] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.291172:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.291173:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.291174:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.291177:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.291180:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:4.2:1619677448.291182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677448.291183:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.291185:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.291185:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.291187:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.291188:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.291189:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.291189:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:26.0:1619677448.291190:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.291192:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.291193:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.2:1619677448.291193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:9.2:1619677448.291194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677448.291195:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.291197:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.291198:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.291199:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.291199:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.291201:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.291202:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.291202:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677448.291202:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.291203:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.291204:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.291204:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.291205:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.291205:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:1.0:1619677448.291205:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.291207:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.291208:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.291209:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:18.0:1619677448.291210:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:1.0:1619677448.291210:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.291211:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.291211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:1.0:1619677448.291212:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.291213:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:1.0:1619677448.291213:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.291214:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.291216:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.291217:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:1.0:1619677448.291217:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.291218:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.291219:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.291219:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.291219:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.291220:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.291221:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.291221:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677448.291221:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.291221:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.291222:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.291222:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.291223:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.291223:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677448.291223:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.291224:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.291224:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677448.291224:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:25.2:1619677448.291225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677448.291225:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.291226:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.291227:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.291230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.291232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.291234:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.291237:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.291238:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.291238:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677448.291238:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677448.291239:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.291240:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.291240:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.291240:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:22.0:1619677448.291241:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.291241:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.291242:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.291243:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.291243:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.291244:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677448.291245:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.291373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.291379:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.291382:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.291383:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.291384:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.291387:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.291388:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.291389:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.291391:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.291392:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.291393:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.291396:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.291398:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.291401:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.291401:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.291402:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.291404:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.291404:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.291405:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.291405:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.291410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.291419:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.291421:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.291422:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.2:1619677448.291422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:6.0:1619677448.291422:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.291424:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677448.291427:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677448.291430:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.291433:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.291435:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.291437:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.291438:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.291440:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:5.2:1619677448.291440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:31.0:1619677448.291441:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.291442:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.291443:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.291444:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.291446:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:31.0:1619677448.291448:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.291449:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:31.0:1619677448.291450:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.291450:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677448.291452:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.291452:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.291454:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.291454:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:2.0:1619677448.291455:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.291456:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.291457:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:2.0:1619677448.291457:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:31.0:1619677448.291458:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:2.0:1619677448.291458:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716869] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.291459:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.291460:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.291460:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:2.0:1619677448.291460:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.291461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:2.0:1619677448.291461:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:25.2:1619677448.291462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:2.0:1619677448.291465:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.291466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677448.291466:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.291467:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.291469:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677448.291469:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.291469:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.291470:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:1.0:1619677448.291470:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.291471:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.291471:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:13.0:1619677448.291472:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:2.0:1619677448.291472:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:1.0:1619677448.291472:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.291473:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.291473:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.291473:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:30.0:1619677448.291474:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:13.0:1619677448.291474:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.291475:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.291476:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:1.0:1619677448.291476:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:30.0:1619677448.291477:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.291477:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:13.0:1619677448.291477:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.291478:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.291478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:1.0:1619677448.291478:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.291479:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.291480:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.291481:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.291484:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.291486:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.291487:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:1.0:1619677448.291488:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.291489:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:5.0:1619677448.291490:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:1.0:1619677448.291490:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:5.0:1619677448.291491:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.291491:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.291492:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.291493:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.291493:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.291493:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.291493:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677448.291495:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:25.2:1619677448.291498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.291505:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.291507:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.291508:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.291508:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.291510:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.291513:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.291601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.291607:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.291610:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.291611:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.291612:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.291615:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.291617:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.291618:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.291620:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.291620:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.291622:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.291625:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.291626:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.291629:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.291630:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.291631:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.291632:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.291633:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.291633:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.291634:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.291638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.291645:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.291648:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.291649:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.291649:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.291651:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.291654:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.291675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.2:1619677448.291679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.291684:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.291686:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677448.291687:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:30.0:1619677448.291688:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.291689:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.291690:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.291691:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.291692:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.291693:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.291694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677448.291695:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.291696:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.291697:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:17.0:1619677448.291699:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.291700:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.291700:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.291701:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:17.0:1619677448.291702:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:31.0:1619677448.291703:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.291703:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677448.291703:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677448.291705:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.291705:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.291706:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.291707:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677448.291708:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.291710:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.291710:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.291711:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.291711:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.291711:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:17.0:1619677448.291711:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96716864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.291713:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.291713:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.291713:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:17.0:1619677448.291713:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.291714:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:17.0:1619677448.291714:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.291715:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677448.291715:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.291716:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677448.291717:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.291718:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.291718:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.291718:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.291718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:17.0:1619677448.291719:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.291720:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.291720:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.291721:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677448.291721:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.291722:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.291724:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:17.0:1619677448.291724:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.291724:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.291725:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.291726:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:8.2:1619677448.291726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:26.0:1619677448.291727:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.291727:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.291727:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.291727:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:17.0:1619677448.291728:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.291729:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.291729:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.291732:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.291733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.291734:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:25.2:1619677448.291736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.291736:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.291737:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.291739:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.291740:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.2:1619677448.291740:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677448.291742:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:12.0:1619677448.291742:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:20.0:1619677448.291743:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:12.0:1619677448.291743:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.291745:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.291745:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:13.0:1619677448.291746:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.291747:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:12.0:1619677448.291748:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.291749:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.291749:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.291750:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.291750:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.291750:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.291751:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.291752:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.291753:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.291756:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.291758:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.291759:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.291760:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.291761:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.291762:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.291765:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.291767:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.291770:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.291771:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.291771:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.291773:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.291774:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.291774:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.291775:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.291779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.291785:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.291788:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.291788:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.291790:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.291792:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.291792:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:4.2:1619677448.291861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.291868:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.291871:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.291872:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.291873:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.291875:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.291877:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.291878:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.291879:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.291880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:10.0:1619677448.291880:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.291881:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.291884:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.291885:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.291885:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.291888:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.291888:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:10.0:1619677448.291889:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.291889:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.291890:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.291891:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.291891:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.291892:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.291893:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:6.0:1619677448.291893:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.291894:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.291895:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.291895:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.291897:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.291898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:6.0:1619677448.291898:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.291899:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.291900:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:4.2:1619677448.291900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677448.291904:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.291906:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.291907:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.291907:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.291908:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.291909:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677448.291909:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.291909:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.291910:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677448.291910:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.291911:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.291911:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.291911:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.291913:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.291914:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.291914:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.291915:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.291916:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677448.291916:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.291916:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:25.2:1619677448.291919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:21.2:1619677448.291921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677448.291926:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677448.291927:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:30.0:1619677448.291928:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:22.0:1619677448.291928:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.291929:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:30.0:1619677448.291930:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.291930:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677448.291932:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.291932:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.291935:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.291975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.291981:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.291984:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.291985:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.291987:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.291989:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.291990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:1.0:1619677448.291991:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.291992:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.291993:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.291994:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.291995:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677448.291996:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.291998:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:1.0:1619677448.291998:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.292000:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677448.292000:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:2.0:1619677448.292002:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.292003:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.292004:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.292004:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.292004:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:2.0:1619677448.292006:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677448.292006:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.292006:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.292007:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.292007:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:2.0:1619677448.292008:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.292009:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.292010:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.292011:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:25.2:1619677448.292012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:2.0:1619677448.292014:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.292016:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.292019:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677448.292019:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.292020:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.292021:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.292022:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.292023:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.292023:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.292023:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.292023:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.292024:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:6.0:1619677448.292025:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.292025:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.292026:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:25.2:1619677448.292028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.292028:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.292029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677448.292031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677448.292035:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.292036:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:31.0:1619677448.292037:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.292038:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677448.292039:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:12.0:1619677448.292039:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.292040:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:18.0:1619677448.292040:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.292041:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:12.0:1619677448.292041:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.292042:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.292042:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.292044:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:12.0:1619677448.292044:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677448.292045:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.292047:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.292048:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.292048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:31.0:1619677448.292049:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.292050:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.292051:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.292054:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.292055:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.292056:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.292057:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.292057:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.292058:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.292058:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.292060:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.292060:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.292061:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.292061:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.292063:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.292063:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.292064:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.292065:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.292069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.292076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:20.0:1619677448.292077:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.292079:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.292080:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.292082:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.292083:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:5.0:1619677448.292084:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.292085:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:10.0:1619677448.292086:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677448.292087:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.2:1619677448.292087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000400:00000200:10.0:1619677448.292089:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.292092:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.292094:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:5.0:1619677448.292094:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:10.0:1619677448.292095:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.292096:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.292097:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.292097:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.292098:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.292098:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.292099:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.292099:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.292101:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:10.0:1619677448.292102:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.292104:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.292107:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.292107:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.292108:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.292110:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.292110:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.292111:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.292111:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677448.292115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.292122:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.292125:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.292126:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.292128:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.292128:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.292131:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.292265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.292270:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.292273:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.292274:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.292276:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.292279:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.292280:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.292282:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.292283:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.292284:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.292285:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.292288:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.292290:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.292293:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.292294:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.292299:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.292301:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.292301:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.292302:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.292303:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.292304:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.292307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.292309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.292314:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.292316:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677448.292317:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.292318:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.292318:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.292318:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.292319:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677448.292320:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.292322:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.292322:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.292443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.292447:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.292449:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.292451:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.292452:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.292454:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.292455:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.292457:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.292458:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.292459:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.292460:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.292463:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.292464:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.292467:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.292468:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.292468:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.292470:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.292470:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.292471:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.292472:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.292476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.292483:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:4.2:1619677448.292484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677448.292486:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.292487:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.292488:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.292489:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.292491:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:26.0:1619677448.292492:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.292495:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:25.2:1619677448.292495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.292495:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.292498:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.292501:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.292502:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:6.0:1619677448.292502:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:26.0:1619677448.292503:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.292505:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.292505:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.292506:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.292506:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:26.0:1619677448.292507:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.292507:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.292508:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.292508:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:26.0:1619677448.292511:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.292513:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.292515:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.292516:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.292516:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:21.2:1619677448.292517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:26.0:1619677448.292518:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.292518:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.292519:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.292520:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.292524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.0:1619677448.292524:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.292527:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.292528:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.292529:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.292531:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:8.0:1619677448.292531:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.292533:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:8.0:1619677448.292533:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.292534:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.292534:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96716853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.292536:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.292536:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.292537:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.292538:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:8.0:1619677448.292538:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.292539:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.292541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:8.0:1619677448.292541:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.292542:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.292545:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.292546:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.292546:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.292548:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.0:1619677448.292548:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.292548:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.292550:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:31.0:1619677448.292551:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.292551:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:8.0:1619677448.292551:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:31.0:1619677448.292552:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:8.0:1619677448.292552:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:31.0:1619677448.292553:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.292553:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.2:1619677448.292554:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.292558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:20.2:1619677448.292561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677448.292562:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:24.0:1619677448.292565:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:5.0:1619677448.292565:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.292566:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.292567:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.292568:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677448.292568:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.292568:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.292569:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:24.0:1619677448.292569:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677448.292571:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677448.292572:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.292574:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.292576:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.292577:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.292578:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.292580:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.292581:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.292582:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.292582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:31.0:1619677448.292583:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.292584:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:31.0:1619677448.292586:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.292587:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:31.0:1619677448.292588:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:20.0:1619677448.292590:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.292590:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677448.292591:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.292591:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.292592:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:20.0:1619677448.292592:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.292593:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.292594:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:20.0:1619677448.292594:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:31.0:1619677448.292595:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.292595:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.292595:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.292596:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.292597:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.292598:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:25.2:1619677448.292599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:20.0:1619677448.292602:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.292604:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:13.0:1619677448.292605:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.0:1619677448.292607:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677448.292608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677448.292608:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:13.0:1619677448.292608:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.292608:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.292609:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.292610:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:13.0:1619677448.292610:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.292611:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.292611:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.292612:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:12.0:1619677448.292612:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.292613:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.292616:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677448.292618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.292619:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.292621:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.292621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.292623:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.292623:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.292624:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:30.0:1619677448.292624:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677448.292626:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.292626:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.292627:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.292627:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.292628:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.292628:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:31.0:1619677448.292629:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000400:00000200:30.0:1619677448.292629:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.292630:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.292632:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.292632:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.292632:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.292632:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:31.0:1619677448.292633:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:12.0:1619677448.292633:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.292634:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.292634:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.292637:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.292638:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.292638:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.292640:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.292640:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.292642:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.292643:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.292644:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.292647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.292649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.292653:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.292655:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.292656:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.292657:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:12.0:1619677448.292657:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.292659:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:12.0:1619677448.292659:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.292660:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.292661:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.292661:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.292683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.292689:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.292692:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.292692:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.292694:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.292697:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.292699:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.292700:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.292701:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.292702:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.292703:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.292707:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.292708:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.292711:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.292713:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.292713:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.292715:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.292715:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.292716:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.292716:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.2:1619677448.292718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:25.2:1619677448.292721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.292725:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.292727:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:10.0:1619677448.292728:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.292729:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.292730:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.292731:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.292731:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.292733:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.292733:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.292734:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:10.0:1619677448.292735:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:31.0:1619677448.292736:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:10.0:1619677448.292736:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.292737:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.292738:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.292739:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.292742:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.292744:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.292747:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.292747:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.292748:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.292750:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.292750:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.292751:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.292752:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.292753:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.292755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.292758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.292762:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.292764:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.292765:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.292766:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:12.0:1619677448.292766:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.292768:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:12.0:1619677448.292768:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.292769:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:13.0:1619677448.292769:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.292770:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:4.2:1619677448.293080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.293087:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.293089:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.293090:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.293091:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.293094:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.293095:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.293096:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.293097:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.293098:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.293099:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.293101:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.293103:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.293105:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.293105:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.293106:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.293107:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.293108:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.293108:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.293109:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.293113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.293120:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.293123:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.293124:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.293125:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.293127:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.293128:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.293168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.293175:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.293178:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.293179:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.293180:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.293184:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.293185:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.293186:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.293187:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.293188:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.293189:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.293192:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.293194:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.293197:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.293197:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.293198:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.293199:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.293200:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.293201:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.293202:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.293203:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.293205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.293208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.293213:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.293215:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.293215:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.293216:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.293216:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.293217:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.293218:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.293218:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.293218:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.293221:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.293268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.293274:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.293276:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.293277:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.293279:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.293281:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.293283:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.293284:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.293285:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.293286:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.293287:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.293290:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.293292:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.293299:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.293300:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.293300:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.293301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:17.0:1619677448.293302:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.293303:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.293303:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.293304:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.293307:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:25.2:1619677448.293309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.293310:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.293311:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.293313:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.293316:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.293316:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:22.0:1619677448.293317:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.293319:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.293319:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:22.0:1619677448.293320:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.293320:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.293321:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.293322:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:12.0:1619677448.293322:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.293323:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.293324:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:22.0:1619677448.293326:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.293328:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.293331:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.293332:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.293333:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.293335:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.293336:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.293337:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.293337:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.293341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.293349:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.293352:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.293353:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.293354:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.293356:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.293357:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.293361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.293368:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.293371:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.293372:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.293372:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.293373:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.293374:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.293463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.293471:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.293473:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.293474:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.293476:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.293479:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.293481:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.293482:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.293483:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.293484:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.293485:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677448.293489:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.293491:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.293493:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.293494:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.293494:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.293496:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.293497:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.293497:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.293498:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.293503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.293512:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.293514:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.293515:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.293517:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.293518:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.293520:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.293522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.293529:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.293532:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.293533:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.293533:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.293534:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.293535:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.293541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.293549:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.293552:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.293554:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.293555:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.293556:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.293557:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.293559:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716842] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.293560:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.293561:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.293562:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.293566:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.293567:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.293569:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.293570:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.293570:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.293572:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.293573:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.293574:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.293576:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.293576:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.293578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.2:1619677448.293579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.2:1619677448.293583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:4.2:1619677448.293584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.293589:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.293590:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:26.0:1619677448.293590:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:30.0:1619677448.293591:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.293592:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.293593:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:30.0:1619677448.293593:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.293593:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:31.0:1619677448.293594:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.293594:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.293595:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:10.0:1619677448.293595:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:30.0:1619677448.293596:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (23)--
00000400:00000200:26.0:1619677448.293596:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.293597:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.293598:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.293598:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:10.0:1619677448.293598:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.293599:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.293599:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.293599:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (23)--
00000400:00000200:26.0:1619677448.293599:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.293600:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.293601:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.293602:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.293604:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.293605:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.293606:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.293607:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.293609:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.293610:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.293610:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.293611:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.293612:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.293612:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.293615:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.293616:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:10.0:1619677448.293616:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.293617:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.293617:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.293619:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.293619:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.293620:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.293620:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.293621:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.293621:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.293621:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.293622:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677448.293623:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.293623:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:25.2:1619677448.293624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.293624:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.293625:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.293625:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.293629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:13.0:1619677448.293630:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.293633:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:21.2:1619677448.293634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:13.0:1619677448.293634:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.293635:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.293637:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.293638:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:12.0:1619677448.293638:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.293640:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.293640:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.293642:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:30.0:1619677448.293642:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.293642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.293643:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677448.293644:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:31.0:1619677448.293645:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.293645:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.293646:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.293647:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.293648:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.293649:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.293650:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.293651:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.293651:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.293652:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.293652:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.293652:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:31.0:1619677448.293653:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.293653:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.293654:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677448.293656:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.293657:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.293659:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.293660:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.293661:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.293662:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.293663:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.293664:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.293664:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.293667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.293674:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.293676:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.293677:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.293678:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.293680:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.293681:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.293712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:30.0:1619677448.293720:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677448.293722:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:30.0:1619677448.293724:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.293726:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677448.293726:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.293728:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.293729:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.293731:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.293731:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.293732:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.293736:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.293737:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.293740:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.293741:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.293742:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.293743:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.293744:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677448.293745:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677448.293745:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:5.2:1619677448.293748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.2:1619677448.293750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:1.0:1619677448.293754:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.293756:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677448.293757:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.293758:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.293759:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.293760:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:1.0:1619677448.293760:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.293761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:2.0:1619677448.293762:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677448.293763:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.293765:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677448.293766:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.293766:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716837] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.293767:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677448.293768:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.293769:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.293770:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.293770:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:18.0:1619677448.293771:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.293772:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.293772:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.293773:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.293773:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.293774:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.293775:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.293775:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.293777:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:2.0:1619677448.293779:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677448.293780:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.293781:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:2.0:1619677448.293782:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.293782:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:18.0:1619677448.293783:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.293783:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.293784:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:1.0:1619677448.293784:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.293784:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.293785:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.293785:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.293785:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.293785:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.293786:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:1.0:1619677448.293786:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677448.293787:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.293788:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.293788:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.293788:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.293789:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:25.2:1619677448.293790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:18.0:1619677448.293790:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.293791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677448.293791:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.293791:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:9.2:1619677448.293794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.293796:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:21.2:1619677448.293797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.0:1619677448.293797:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.293799:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.293800:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.293800:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:31.0:1619677448.293802:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.293803:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.293803:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677448.293803:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:5.0:1619677448.293803:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.293804:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.293805:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.293805:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:31.0:1619677448.293806:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:5.0:1619677448.293806:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.293807:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.293807:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:5.0:1619677448.293808:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:2.0:1619677448.293808:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:5.0:1619677448.293809:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.293809:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.293810:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.293810:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:5.0:1619677448.293811:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.293812:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.293813:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.293816:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.293818:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.293821:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.293822:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.293823:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.293825:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.293825:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.293826:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.293827:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.293830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.293837:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.293840:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.293841:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.293842:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.293843:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.293845:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.293953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.293958:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.293961:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.293961:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.293964:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.293966:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.293968:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.293969:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.293970:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.293971:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.293972:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.293975:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.293977:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.293980:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.293981:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.293981:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.293983:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.293984:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.293985:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.293985:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.294000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.294008:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.294010:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.294011:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.294011:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.294013:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.294016:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.294260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.294265:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.294268:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.294268:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.294270:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.294272:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.294274:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.294275:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.294276:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.294277:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.294278:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.294281:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.294283:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.294285:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.294286:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.294286:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.294288:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.294288:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.294289:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.294290:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.294296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.294304:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.294306:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.294307:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.294307:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.294309:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.294312:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.294381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.2:1619677448.294384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:4.2:1619677448.294385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677448.294391:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:26.0:1619677448.294392:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.294393:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.294394:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:8.0:1619677448.294394:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.294395:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:10.0:1619677448.294395:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677448.294397:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.294397:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.294398:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.294400:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.294400:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.294401:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.294402:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.294402:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.294404:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.294404:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.294405:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.294406:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.294407:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.294407:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.294408:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.294408:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.294410:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.294411:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716830] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.294413:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.294413:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.294414:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.294415:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.294415:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.294417:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.294419:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.294419:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.294420:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.294421:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677448.294422:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.294422:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.294423:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.294423:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.294423:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.294424:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677448.294424:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000800:00000200:26.0:1619677448.294425:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.294426:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677448.294426:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (23)--
00000800:00000200:8.0:1619677448.294426:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:26.0:1619677448.294427:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.294427:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.294427:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.294427:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:10.0:1619677448.294428:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.294428:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000800:00000200:26.0:1619677448.294429:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.294430:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (24)--
00000800:00000200:9.2:1619677448.294430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (24)++
00000800:00000200:10.0:1619677448.294431:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.294432:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.294433:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.294434:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677448.294439:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (24)++
00000800:00000200:4.2:1619677448.294439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.294441:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (25)--
00000800:00000200:2.0:1619677448.294442:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (25)++
00000400:00000200:18.0:1619677448.294443:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.294443:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (25)--
00000400:00000200:2.0:1619677448.294444:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.294445:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.294446:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.294447:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.294449:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:18.0:1619677448.294449:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (24)--
00000800:00000200:2.0:1619677448.294449:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.294450:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677448.294450:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.294451:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:18.0:1619677448.294452:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.294452:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.294454:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.294454:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.294455:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.294456:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.294456:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (23)++
00000800:00000200:2.0:1619677448.294457:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (24)--
00000800:00000200:2.0:1619677448.294457:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.294458:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.294460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.294466:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.294468:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.294469:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.294471:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.294473:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.294475:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.294476:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.294478:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.294479:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.294480:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.294483:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.294485:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.294488:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.294489:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.294489:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.294492:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.294492:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.294493:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.294494:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.294495:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.294497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.294498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:5.2:1619677448.294500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.294505:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:26.0:1619677448.294506:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:17.0:1619677448.294506:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.294508:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.294508:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677448.294508:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:26.0:1619677448.294509:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:17.0:1619677448.294509:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677448.294509:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.294510:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:12.0:1619677448.294511:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.294512:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.294513:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.294514:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:26.0:1619677448.294515:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.294517:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.294518:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.294519:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.294520:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.294521:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.294525:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.294526:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.294529:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.294530:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.294531:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.294532:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.294533:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.294534:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677448.294535:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.294536:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677448.294538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677448.294541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:1.0:1619677448.294546:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.294549:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.294549:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:1.0:1619677448.294549:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.294550:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.294551:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.294552:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:1.0:1619677448.294552:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.294553:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:1.0:1619677448.294555:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.294570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.294573:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.294575:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.294578:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.294578:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.294580:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.294582:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.294583:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.294585:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.294585:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.294586:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.294590:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.294591:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:21.2:1619677448.294593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677448.294594:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.294595:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.294595:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.294597:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.294598:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.294599:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.294599:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.294602:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677448.294603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.294604:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.294607:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.294609:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.294610:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.294610:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:30.0:1619677448.294611:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.294612:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.294612:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.294613:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.294614:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.294615:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677448.294615:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.294616:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.294616:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:30.0:1619677448.294621:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.294622:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.294622:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.294625:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.294625:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.294626:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.294628:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.294628:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.294629:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.294630:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.294633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.294639:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.294642:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.294642:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.294644:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.294646:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.294646:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.294652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.294659:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.294661:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.294662:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.294662:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.294663:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.294664:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.294720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.294726:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.294729:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.294730:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.294732:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.294734:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.294735:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.294737:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.294738:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.294739:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.294740:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.294744:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.294746:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.294749:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.294749:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.294750:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.294752:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.294752:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.294753:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.294753:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.294758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.294764:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.294767:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.294767:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.294768:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.294770:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.294773:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677448.295047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.295053:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.295055:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.2:1619677448.295057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:5.0:1619677448.295057:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.295058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:5.0:1619677448.295060:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.295062:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.295062:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:20.0:1619677448.295063:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:5.0:1619677448.295063:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.295064:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:10.0:1619677448.295065:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677448.295065:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.295066:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.295067:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.295068:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.295068:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:20.0:1619677448.295071:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.295072:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.295072:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.295074:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.295075:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.295075:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.295077:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:10.0:1619677448.295077:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.295078:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.295079:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.295081:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.295081:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.295082:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.295082:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.295084:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:10.0:1619677448.295084:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.295087:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.295088:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.295088:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:20.0:1619677448.295089:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.295090:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.295090:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.295091:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.295092:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:5.0:1619677448.295092:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.295093:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:10.0:1619677448.295094:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.295094:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.295095:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.295096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.295096:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.295097:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:10.0:1619677448.295097:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.295098:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.2:1619677448.295099:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.295099:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.295100:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.295101:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:10.0:1619677448.295101:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.295102:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.295103:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.295103:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:10.0:1619677448.295103:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677448.295104:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.295104:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.295104:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677448.295105:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677448.295105:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:5.0:1619677448.295105:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.295106:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677448.295107:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.295107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.295107:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:31.0:1619677448.295109:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.295110:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:9.2:1619677448.295110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.295112:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:6.0:1619677448.295115:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:24.0:1619677448.295117:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:6.0:1619677448.295117:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.295118:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:24.0:1619677448.295119:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:6.0:1619677448.295119:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.295120:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.295121:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.295121:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.295122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:24.0:1619677448.295123:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.295124:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677448.295127:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.295127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677448.295130:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.295132:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.295133:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677448.295134:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677448.295135:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.295137:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:26.0:1619677448.295138:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.295138:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:26.0:1619677448.295140:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:8.0:1619677448.295140:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.295141:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.295143:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.295143:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.295144:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.295145:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677448.295146:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:8.0:1619677448.295146:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.295148:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.295149:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.295150:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.295150:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.295153:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:8.0:1619677448.295154:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.295156:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.295156:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.295157:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.295157:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.295159:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.295159:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677448.295159:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.295160:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677448.295160:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.295160:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.295161:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677448.295162:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.295162:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.295163:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.295164:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677448.295165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:8.2:1619677448.295168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.295173:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.295176:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677448.295176:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:22.0:1619677448.295177:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.295177:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.295178:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:22.0:1619677448.295179:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.295179:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.295179:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.295182:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:17.0:1619677448.295182:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.295185:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.295188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.295195:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.295198:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.295199:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.295199:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.295200:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.295201:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.295378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.295385:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.295388:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.295389:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.295391:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.295393:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.295395:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.295396:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.295398:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.295399:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.295399:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.295403:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.295404:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.295407:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.295408:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.295408:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.295411:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.295411:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.295412:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.295412:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.295417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.295425:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:21.2:1619677448.295428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677448.295428:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.295429:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.295430:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.295431:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.295435:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677448.295436:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.295439:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.295441:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.295443:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.295443:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.295445:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.295446:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.295447:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.295448:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.295449:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.295452:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.295454:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.295457:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.295458:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.295458:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.295460:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.295460:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.295461:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.295462:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.295466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.295474:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.295477:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.295478:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.295478:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.295480:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.295484:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.295494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.295500:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.295503:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.295505:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.295505:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.295507:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.295509:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.295510:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.295512:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.295513:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.295514:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.295518:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.295519:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.295522:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.295523:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.295524:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.295525:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.295526:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.295527:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.295527:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.295531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.295539:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.295542:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.295543:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.295543:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.295545:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.295548:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.295595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.295602:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.295605:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.295606:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.295608:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.295611:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.295612:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.295613:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.295615:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.295616:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.295616:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.295619:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.295621:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.295622:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.295623:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.295624:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.295625:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.295626:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.295626:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.295627:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.295631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.295637:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.295638:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.295639:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.295640:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.295641:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.295642:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.295657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.295663:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.295666:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.295666:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.295668:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.295670:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.295672:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.295673:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.295674:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.295675:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.295676:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.295680:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.295681:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.295683:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.295684:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.295684:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.295686:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.295687:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.295687:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.295688:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:21.2:1619677448.295690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:9.2:1619677448.295692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677448.295695:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.295698:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:18.0:1619677448.295699:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677448.295699:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.295700:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.295701:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.295702:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.295703:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:22.0:1619677448.295704:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.295704:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.295706:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:18.0:1619677448.295706:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:22.0:1619677448.295707:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.295709:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.295710:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.295711:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.295715:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.295717:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.295719:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.295720:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.295720:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.295722:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.295723:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.295724:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.295725:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.295726:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.295729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.295730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.295735:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:20.0:1619677448.295736:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677448.295737:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.295737:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.295738:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677448.295739:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.295739:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.295741:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.295742:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.295744:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.295958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677448.295965:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.295968:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677448.295969:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677448.295970:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.295972:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.295974:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.295975:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.295976:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.295977:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.295978:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.295982:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.295984:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.295987:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.295987:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.295988:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.295990:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.295990:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.295991:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677448.295992:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677448.295993:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:21.2:1619677448.295996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.2:1619677448.295996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:21.2:1619677448.296001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:31.0:1619677448.296004:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.296005:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:31.0:1619677448.296006:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677448.296007:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:26.0:1619677448.296008:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:1.0:1619677448.296008:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:31.0:1619677448.296009:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.296009:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.296010:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677448.296010:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:31.0:1619677448.296011:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.296011:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677448.296011:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.296011:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:31.0:1619677448.296013:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677448.296014:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.296014:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:31.0:1619677448.296016:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.296016:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.296016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000400:00000200:31.0:1619677448.296017:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677448.296020:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.296022:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.296023:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677448.296024:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.296025:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.296025:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.296026:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.296026:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.296027:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.296027:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:31.0:1619677448.296028:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.296028:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.296028:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:31.0:1619677448.296029:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.296031:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.296032:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677448.296033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:21.2:1619677448.296037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677448.296039:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.296042:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.296043:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.296043:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.296045:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677448.296045:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.296047:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.296048:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677448.296048:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:30.0:1619677448.296049:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677448.296054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.296060:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.296063:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.296064:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.296066:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.296069:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.296070:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.296072:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.296073:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.296074:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.296075:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.296078:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.296079:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.296082:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.296083:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.296083:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.296085:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.296086:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.296086:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.296087:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.296091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.296099:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.296102:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.296102:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.296103:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.296105:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.296108:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.296166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.296171:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.296174:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.296175:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.296177:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.296180:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.296181:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.296182:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.296183:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.296184:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.296185:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.296189:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.296190:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.296192:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.296193:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.296193:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.296195:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.296196:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.296196:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.296197:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.296201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.296208:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.296211:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.296211:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.296212:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.296213:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.296214:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.296339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.296348:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.296350:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.296352:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.296355:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.296355:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.296356:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.296358:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.296359:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.296360:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.296361:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.296364:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.296366:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.296369:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.296370:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.296370:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.296372:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.296373:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.296374:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.296375:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.296379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.296383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.0:1619677448.296389:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677448.296391:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.296391:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.296392:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.296394:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:5.0:1619677448.296394:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.296395:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:31.0:1619677448.296397:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.296397:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677448.296398:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.296399:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.296401:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.296402:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.296404:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.296405:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.296406:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.296409:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.296409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:31.0:1619677448.296410:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.296412:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.296413:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.296414:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.296415:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.296415:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677448.296416:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.296416:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.296417:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677448.296418:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.296419:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.296420:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.296421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:2.0:1619677448.296422:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.296424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:2.0:1619677448.296424:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.296425:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.296426:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.296427:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.296428:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.296431:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:2.0:1619677448.296431:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.296432:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.296433:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.296433:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.296434:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.296435:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:2.0:1619677448.296435:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.296436:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.296436:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.296437:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677448.296438:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.296439:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.296439:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.296439:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:10.0:1619677448.296440:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:2.0:1619677448.296440:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.296440:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:26.0:1619677448.296441:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.296441:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.296443:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.296444:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.296445:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677448.296445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677448.296446:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.296447:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.296450:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.296451:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.296452:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.296453:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.296453:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.296454:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:18.0:1619677448.296454:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.296455:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.296456:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.296456:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:18.0:1619677448.296456:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.296456:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.296457:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:18.0:1619677448.296458:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:26.0:1619677448.296459:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.296459:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.296462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:9.2:1619677448.296462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.2:1619677448.296465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.296469:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:2.0:1619677448.296470:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.296471:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.296472:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.296473:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:2.0:1619677448.296473:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.296473:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.296474:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:18.0:1619677448.296474:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:10.0:1619677448.296474:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.296474:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.296475:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:2.0:1619677448.296475:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:26.0:1619677448.296476:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.296476:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.296477:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677448.296593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677448.296599:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.296602:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677448.296602:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677448.296604:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.296607:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.296608:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.296609:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716805] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.296611:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.296612:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.296613:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.296616:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.296618:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.296621:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.296622:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.296622:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.296624:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.296624:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.296626:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677448.296627:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677448.296627:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:25.2:1619677448.296630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.296632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:26.0:1619677448.296636:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:2.0:1619677448.296638:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:26.0:1619677448.296639:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.296640:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.296640:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.296640:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677448.296640:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:26.0:1619677448.296642:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.296642:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:26.0:1619677448.296645:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.296655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.296661:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.296664:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.296664:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.296665:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.296666:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.296666:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.296667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.296674:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.296677:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.296678:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.296680:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.296682:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.296683:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.296684:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.296685:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.296686:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.296687:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.296691:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.296692:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.296695:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.296696:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.296697:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.296698:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.296699:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.296700:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.296701:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.296701:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.296704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.296707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.296713:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.296714:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.296716:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677448.296716:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.296717:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.296717:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.296717:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.296718:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.296719:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.296722:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.296723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:25.2:1619677448.296726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.296731:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677448.296734:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:22.0:1619677448.296734:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.296734:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:22.0:1619677448.296736:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.296737:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.296737:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.296738:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.296739:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.296739:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:10.0:1619677448.296740:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.296742:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677448.296743:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.296745:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.296745:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.296746:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.2:1619677448.296747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677448.296750:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.296751:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.296753:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.296754:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.296755:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.296755:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677448.296756:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:10.0:1619677448.296757:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:1.0:1619677448.296757:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.296758:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000400:00000200:17.0:1619677448.296759:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.296759:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677448.296759:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000400:00000200:17.0:1619677448.296760:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.296762:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.296763:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.296763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:17.0:1619677448.296764:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.296765:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.296766:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.296769:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.296771:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:17.0:1619677448.296771:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677448.296773:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.296774:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.296774:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.296775:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.296775:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:20.0:1619677448.296776:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.296777:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.296777:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.296777:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:17.0:1619677448.296778:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.296779:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677448.296779:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.296783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.2:1619677448.296784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:24.0:1619677448.296790:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.296793:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.296794:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.296794:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:5.0:1619677448.296794:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:24.0:1619677448.296796:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.296796:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.296797:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.296798:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.296798:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:24.0:1619677448.296799:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.0:1619677448.296800:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.296881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.296889:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.296892:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.296894:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.296895:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.296897:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.296898:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.296899:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.296901:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.296902:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.296902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:30.0:1619677448.296903:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.296906:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.296908:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.296909:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.296910:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677448.296910:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.296911:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.296911:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:31.0:1619677448.296912:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.296913:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.296913:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:31.0:1619677448.296914:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.296914:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.296915:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:31.0:1619677448.296916:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:30.0:1619677448.296916:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.296917:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.296918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:31.0:1619677448.296919:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.296919:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.296920:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.296923:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.296925:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677448.296926:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:31.0:1619677448.296928:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.296928:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:13.0:1619677448.296928:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:31.0:1619677448.296929:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:13.0:1619677448.296929:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.296930:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.296931:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.296931:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.296932:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.296933:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.296933:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.296934:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.2:1619677448.296937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.296945:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.296947:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.296948:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.296948:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.296951:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.296954:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.296972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.296978:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.2:1619677448.296979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.296981:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.296983:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.296984:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.296984:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.296985:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:18.0:1619677448.296987:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.296989:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677448.296989:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.296990:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.296992:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.296993:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.296994:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677448.296996:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.296998:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:2.0:1619677448.296999:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.297000:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.297000:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:18.0:1619677448.297001:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.2:1619677448.297001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:2.0:1619677448.297001:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.297002:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:2.0:1619677448.297003:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.297004:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.297004:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.297004:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.297007:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.297008:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677448.297008:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.297009:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.297010:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.297010:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677448.297011:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:9.2:1619677448.297011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.297011:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.297012:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:10.0:1619677448.297012:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:2.0:1619677448.297012:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.297013:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677448.297014:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.297014:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.297014:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.297015:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.297015:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.297015:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.297016:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:26.0:1619677448.297017:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.297017:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.297017:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000400:00000200:26.0:1619677448.297018:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:2.0:1619677448.297018:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.297019:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.297019:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:26.0:1619677448.297020:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.297020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677448.297021:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.297022:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.297023:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:5.2:1619677448.297023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000400:00000200:26.0:1619677448.297026:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.297028:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.297028:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.297029:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.297029:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:26.0:1619677448.297030:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.297030:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:18.0:1619677448.297030:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.297030:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:26.0:1619677448.297031:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.297031:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:26.0:1619677448.297032:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:18.0:1619677448.297032:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.297032:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.297033:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.297033:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:2.0:1619677448.297033:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:26.0:1619677448.297034:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:18.0:1619677448.297034:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:25.2:1619677448.297038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.297044:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.297046:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.297047:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.297047:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.297049:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.297050:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.297059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.297066:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.297069:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.297070:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.297071:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.297074:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.297075:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.297076:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.297078:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.297079:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.297080:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.297083:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.297084:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.297087:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.297088:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.297088:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.297090:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.297090:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.297091:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.297091:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.297096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.297104:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.297106:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.297107:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.297108:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.297110:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.297112:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.297113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.297120:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.297123:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.297124:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.297125:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.297128:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.297130:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.297131:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.297132:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.297134:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.297134:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.297137:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.297139:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.297141:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.297142:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.297142:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.297144:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.297144:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.297146:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.297147:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.297147:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.2:1619677448.297149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:21.2:1619677448.297153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:13.0:1619677448.297155:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.297158:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:5.2:1619677448.297158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:13.0:1619677448.297159:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.297160:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.297160:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:12.0:1619677448.297162:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.297162:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.297162:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677448.297163:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677448.297163:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.297164:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:1.0:1619677448.297166:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.297167:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.297168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:1.0:1619677448.297169:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.297172:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.297173:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:12.0:1619677448.297174:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:1.0:1619677448.297174:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.297175:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.297177:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.297177:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.297178:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:1.0:1619677448.297178:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:12.0:1619677448.297179:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.297179:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.297181:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:1.0:1619677448.297181:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.297183:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.297185:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.297186:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.297187:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.297188:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.297189:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.297190:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.297191:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.297194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.297201:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.297203:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.297204:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.297204:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.297206:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.297209:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.297212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677448.297218:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.297220:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.297221:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.297221:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.297222:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.297223:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677448.297314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.297322:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.297325:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.297327:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.297329:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.297330:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.297331:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.297332:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.297334:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.297335:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.297336:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.297339:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.297341:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.297343:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.297344:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.297344:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.297346:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.297346:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.297347:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:4.2:1619677448.297347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.297348:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.297352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.297355:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.297358:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.297359:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.297360:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.297360:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:26.0:1619677448.297362:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.297362:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:26.0:1619677448.297363:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:13.0:1619677448.297363:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.297365:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677448.297365:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.297366:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.297366:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:26.0:1619677448.297367:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.297367:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:26.0:1619677448.297368:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.297371:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.297373:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.297375:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.297376:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.297376:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.297378:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.297378:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.297379:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.297380:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.297383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.297390:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.297392:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.2:1619677448.297392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:31.0:1619677448.297393:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.297395:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.297397:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.297398:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.0:1619677448.297398:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.297401:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:25.2:1619677448.297403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:5.0:1619677448.297403:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.297404:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.297406:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.297407:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677448.297409:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:5.0:1619677448.297409:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.297410:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.297411:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.297412:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.297412:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.297413:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.297414:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.297415:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:5.0:1619677448.297415:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.297416:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.0:1619677448.297417:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.297419:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.297420:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.297421:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.297422:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.297423:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.297424:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.297424:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.297429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.297436:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.297439:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.297440:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.297440:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.297442:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.297444:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.297588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.297595:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.297598:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.297600:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.297602:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.297603:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.297605:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.297606:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.297606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677448.297608:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.297609:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.297610:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.297612:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677448.297613:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.297615:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.297615:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.297615:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.297618:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.297618:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.297619:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.297620:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:17.0:1619677448.297621:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.297622:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.297622:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677448.297622:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:30.0:1619677448.297624:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.297624:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:17.0:1619677448.297624:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.297625:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.297626:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.297627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:17.0:1619677448.297627:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.297631:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.297632:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:13.0:1619677448.297634:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.297635:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.297636:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.297636:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:13.0:1619677448.297636:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.297637:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.297638:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.297639:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:13.0:1619677448.297639:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.297640:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677448.297640:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.297641:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:13.0:1619677448.297641:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677448.297642:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.297647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.297652:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.297654:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:9.2:1619677448.297655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.297655:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.297655:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677448.297662:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.297665:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677448.297665:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677448.297666:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.297668:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.297670:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.2:1619677448.297726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.297733:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.297736:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.297737:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.297739:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.297741:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.297743:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677448.297744:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.297745:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.297746:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.297747:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.297750:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.297752:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.297755:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.297755:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.297756:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.297758:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.297758:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.297759:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.297760:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.297764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.297772:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.297774:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.297775:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.297777:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.297778:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.297780:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.297799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.297806:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.297809:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.297810:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.297811:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.297814:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.297816:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.297817:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.297819:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.297819:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.297820:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.297823:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.297824:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.297826:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.297827:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.297827:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.297829:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.297829:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.297830:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.297831:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.297835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.297841:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.297844:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.297844:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.297846:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.297848:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.297849:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.297901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677448.297906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.297908:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.297911:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.297911:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.297912:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.297913:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.297914:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:10.0:1619677448.297915:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.297915:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.297917:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.297918:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.297918:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.297920:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.297920:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.297921:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.297921:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.297922:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:20.2:1619677448.297923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:17.0:1619677448.297924:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.297926:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.297926:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.297927:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.297928:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.297929:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.297930:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.297931:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.297932:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.297933:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.297933:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.297934:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:17.0:1619677448.297934:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.297935:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.297935:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677448.297936:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.297936:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.297937:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677448.297937:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.297937:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677448.297938:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.297938:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:30.0:1619677448.297939:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.297940:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:17.0:1619677448.297940:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:30.0:1619677448.297941:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96716784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.297941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.297941:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.297941:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.297942:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677448.297943:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.297944:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.297945:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:25.2:1619677448.297946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:30.0:1619677448.297948:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.297949:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:30.0:1619677448.297950:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.297951:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.297952:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.297953:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.297953:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677448.297954:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.297954:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.297955:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:22.0:1619677448.297955:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.297955:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.297956:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.297957:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.297957:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.297958:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:22.0:1619677448.297958:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:12.0:1619677448.297958:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.297959:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.297960:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.297960:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.297961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:12.0:1619677448.297961:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.2:1619677448.297962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.297966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:6.0:1619677448.297969:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.0:1619677448.297970:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:22.0:1619677448.297972:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.297972:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.297972:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.297973:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:5.0:1619677448.297973:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677448.297974:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:6.0:1619677448.297974:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:5.0:1619677448.297974:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.297976:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:20.0:1619677448.297976:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.297976:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.297977:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.297978:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.297979:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.298004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:26.0:1619677448.298011:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677448.298014:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:10.0:1619677448.298014:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:26.0:1619677448.298017:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.298019:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.298021:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:26.0:1619677448.298022:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716783] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.298023:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.298025:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.298025:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677448.298029:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.298030:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677448.298033:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.298034:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:26.0:1619677448.298035:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:26.0:1619677448.298037:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:26.0:1619677448.298037:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677448.298038:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:26.0:1619677448.298038:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:8.2:1619677448.298043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.298050:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.298053:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.298054:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.298055:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.298056:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.298058:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:21.2:1619677448.298084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.298090:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.298093:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.298093:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.298095:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.298098:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.298099:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.298100:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.298101:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.298103:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.298104:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.298107:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.298109:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.298112:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.298113:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.298114:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.298116:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.298116:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.298117:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.298117:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.298122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.298128:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.298131:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.298132:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.298134:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.298135:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.298137:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.298260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.298265:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.298268:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.298269:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.298271:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.298274:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.298276:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.298277:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.298278:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.298279:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.298280:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.298283:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.298285:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.298287:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.298288:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.298289:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.298291:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.298291:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.298292:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.298292:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.298297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.298305:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.298308:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.298309:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.298311:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.298313:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.298314:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.298327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.298335:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.298338:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.298340:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.298342:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.298343:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.298344:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.298346:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.298347:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.298348:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.298349:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.298352:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.298354:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.298356:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.298357:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.298358:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.298359:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.298360:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.298361:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.298363:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.298363:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.298364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.298369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.2:1619677448.298371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:12.0:1619677448.298371:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.298373:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.298373:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.298375:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.298376:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.298377:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.298378:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.298378:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.298379:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.298379:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.298380:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677448.298381:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.298383:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.298384:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.298385:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.298387:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.298388:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.298389:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.298391:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.298392:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.298395:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.298397:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.298399:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.298400:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.298401:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.298402:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.298403:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.298404:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.298404:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.298408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.298416:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.298418:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.298419:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.298419:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.298422:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.298424:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.298471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.298479:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.298482:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.298482:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.298485:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.298487:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.298489:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.298491:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.298492:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.298493:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.298494:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.298497:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.298499:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.298502:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.298503:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.298503:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.298505:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.298505:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.298506:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.298506:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.298511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.298518:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.2:1619677448.298519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677448.298521:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.298522:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.298522:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.298524:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.298525:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.298527:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:17.0:1619677448.298527:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.298528:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.298530:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.298533:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.298535:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.298536:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716777] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.298537:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.298538:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.298539:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.298543:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.298545:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.2:1619677448.298546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.298548:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677448.298548:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.298548:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.298549:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.298551:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:17.0:1619677448.298551:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.298551:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.298552:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.298552:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:20.0:1619677448.298553:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.298554:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.298556:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.298557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:20.0:1619677448.298557:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.298559:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.298560:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.298561:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.298562:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:13.0:1619677448.298563:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:20.0:1619677448.298566:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.298566:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.298567:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.298569:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:13.0:1619677448.298569:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.298569:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677448.298572:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.298572:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677448.298573:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.298573:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.298575:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.298576:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.298578:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.298579:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.298580:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.298581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.298585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:31.0:1619677448.298588:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.298590:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:5.0:1619677448.298590:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000400:00000200:31.0:1619677448.298591:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.298592:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:31.0:1619677448.298593:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.298593:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.298594:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677448.298595:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.298596:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.298652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.298659:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.298662:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.298663:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.298665:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.298666:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.298668:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.298669:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716775] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.298670:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.298672:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.298673:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.298676:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.298679:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.298682:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.298682:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.298683:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.298685:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.298685:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.298686:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.298686:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.298691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.298698:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.298701:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.298702:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.298702:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.298704:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.298707:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.298710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:21.2:1619677448.298712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:17.0:1619677448.298717:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.298719:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677448.298720:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.298720:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.298720:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.298721:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.298722:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677448.298722:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.298723:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.298724:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.298727:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.298728:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.298730:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.298731:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.298732:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.298733:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.298736:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.298738:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.298741:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.298742:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.298743:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.298745:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.298745:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.298746:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.298747:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.298751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.298757:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.298760:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.298760:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.298761:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.298763:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.298766:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.298786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.298792:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.298795:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.298795:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.298797:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.298800:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.298802:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.298803:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716773] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.298804:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.298805:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.298806:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.298810:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.298812:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.298814:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.298815:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.298816:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.298817:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.298818:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.298819:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.298820:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.298821:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.298823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.298825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.298831:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677448.298831:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.298833:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.298833:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.298834:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.298834:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.298835:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.298835:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.298837:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.298839:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.298882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.298890:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.298893:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.298895:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.298896:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.298898:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.298899:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.298900:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.298902:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.298903:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.298904:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.298908:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.298909:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.298912:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.298912:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.298913:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.298914:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.298915:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.298916:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.298917:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.298920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.298926:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.298929:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.298930:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.298931:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.298933:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.298934:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.299037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.299043:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.299046:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.299047:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.299049:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.299051:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.299053:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677448.299054:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.299056:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.299057:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.299058:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677448.299061:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.299063:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.299065:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.299066:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677448.299067:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677448.299068:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.299069:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.299069:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.299070:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.299074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.299082:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.299084:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.299085:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.299085:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.299088:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.299090:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.299093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:21.2:1619677448.299098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677448.299098:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.299101:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.299101:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.299103:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.299105:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677448.299106:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.299108:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:6.0:1619677448.299108:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.299109:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.299111:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:6.0:1619677448.299111:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.299112:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.299112:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.299113:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.299114:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.299116:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.299118:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.299118:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.299119:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.299121:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.299121:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.299122:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.299124:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.299125:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.299125:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677448.299126:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.299127:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.299128:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.299128:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.299128:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.299129:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677448.299131:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.299132:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.299132:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.2:1619677448.299133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.299134:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.299135:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.299136:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.299136:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.299140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.299140:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.299143:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.299144:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.299144:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.299146:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677448.299146:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.299148:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.299149:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.299149:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:12.0:1619677448.299151:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.299152:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.299153:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.299159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.299166:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.299168:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.299169:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.299170:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.299170:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.299171:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.299190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.299196:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.299198:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.299199:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.299201:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.299203:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.299205:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.299206:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.299207:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.299208:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.299209:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.299213:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.299214:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.299216:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.299217:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.299218:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.299220:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.299220:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.299221:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.299221:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.299226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.299233:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.299235:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.299235:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.299236:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.299237:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.299239:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.299251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.299256:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.299259:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.299261:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.299262:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.299264:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.299266:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.299267:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.299268:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.299269:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.299270:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.299274:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.299276:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.299279:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.299279:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.299280:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.299282:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.299283:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.299283:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.299284:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.299285:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.299288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.299290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677448.299292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:13.0:1619677448.299294:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.299298:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.299300:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.299300:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.299301:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677448.299301:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677448.299302:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:13.0:1619677448.299302:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.299302:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.299302:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.299304:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:13.0:1619677448.299304:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.299304:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.299307:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677448.299308:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.299310:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:5.0:1619677448.299312:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.299313:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.299314:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.299315:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:5.0:1619677448.299318:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.299320:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:5.0:1619677448.299323:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.299323:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677448.299324:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.0:1619677448.299326:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677448.299326:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.299327:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.299328:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677448.299333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:4.2:1619677448.299337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677448.299340:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.299343:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.299343:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.299344:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.299344:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:24.0:1619677448.299346:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.299347:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.299347:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:24.0:1619677448.299349:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:10.0:1619677448.299349:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.299352:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.299354:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.299355:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.299356:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.299357:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.299358:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.299361:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.299362:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.299366:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.299366:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.299367:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.299369:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.299370:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.299371:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.299372:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.299373:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677448.299375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677448.299378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.299383:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.299385:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:2.0:1619677448.299386:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.299387:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:18.0:1619677448.299387:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.299387:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.299388:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677448.299389:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.299390:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677448.299393:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.299522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.299530:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.299532:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.299533:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.299535:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.299538:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.299539:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.299541:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.299542:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.299543:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.299544:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.299547:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.299548:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.299550:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.299551:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.299551:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.299553:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.299554:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.299554:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.299555:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.299559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.299565:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.299568:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.299568:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.299570:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.299572:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.299573:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.299770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.299776:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.299779:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.299779:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.299782:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.299784:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.299785:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.299786:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716763] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.299786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:22.0:1619677448.299788:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.299789:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.299790:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.299792:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677448.299793:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.299795:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.299795:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.299796:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677448.299798:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.299798:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.299799:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.299799:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:17.0:1619677448.299800:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.299801:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.299801:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:17.0:1619677448.299801:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:22.0:1619677448.299802:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.299803:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:17.0:1619677448.299803:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.299804:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.299805:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.299806:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.2:1619677448.299807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:17.0:1619677448.299810:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.299812:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.299815:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.299815:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.299816:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.299817:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.299817:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.299817:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.299819:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.299819:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:1.0:1619677448.299819:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.299820:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.299821:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677448.299822:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.299823:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.299825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.299832:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.299833:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.299834:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.299835:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.299835:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.299836:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.299846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.299852:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.299855:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.299856:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.299858:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.299860:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.299861:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.299863:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.299864:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.299865:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.299866:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.299869:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.299871:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.299873:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.299874:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.299875:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.299876:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.299877:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.299877:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.299878:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.299882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.299889:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.299892:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.299893:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.299893:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.299895:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.299897:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.300212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.300219:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.300222:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.300223:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.300225:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.300227:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.300229:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.300230:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716760] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.300231:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.300232:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.300233:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:21.2:1619677448.300235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:26.0:1619677448.300236:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.300238:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.300241:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.300242:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.300242:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.300242:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:26.0:1619677448.300244:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.300245:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.300245:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677448.300245:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:26.0:1619677448.300246:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:24.0:1619677448.300246:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.300247:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.300250:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:8.0:1619677448.300254:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.300256:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.300257:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96716759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.300258:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:8.0:1619677448.300258:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.300259:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.300260:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:8.0:1619677448.300260:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.300261:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.300263:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.300263:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.300264:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.300266:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.300267:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.0:1619677448.300269:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.300269:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.300270:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.300273:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.300273:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.300274:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.300274:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.300279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.300287:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.300289:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.300291:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.300291:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.300297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677448.300297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:2.0:1619677448.300298:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.300301:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677448.300304:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.300305:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677448.300307:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:10.0:1619677448.300308:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.300309:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677448.300310:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.300310:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.300312:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.300313:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.300314:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.300314:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.300316:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.300317:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.300319:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.300319:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.300320:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.300320:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.300321:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.300322:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.300323:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.300325:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.300327:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.300328:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.300330:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.300331:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.300332:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.300332:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.300333:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.300333:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.300334:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.300334:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.300335:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.300335:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.300336:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677448.300336:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677448.300337:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:5.0:1619677448.300337:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:10.0:1619677448.300338:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.300339:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.300341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.2:1619677448.300344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:25.2:1619677448.300345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.300349:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.300352:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:26.0:1619677448.300352:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:13.0:1619677448.300352:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:30.0:1619677448.300353:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.300354:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.300354:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:30.0:1619677448.300355:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.300355:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.300355:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.300355:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.300355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cffdc4000] (20)++
00000800:00000200:31.0:1619677448.300356:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:13.0:1619677448.300357:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.300359:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:13.0:1619677448.300359:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.300360:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677448.300361:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cffdc4000] (21)++
00000800:00000200:17.0:1619677448.300364:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.11@o2ib3
00000800:00000200:1.0:1619677448.300365:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc4000] (22)--
00000800:00000200:21.2:1619677448.300366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000400:00000200:17.0:1619677448.300367:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.11@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.300369:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.300370:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:17.0:1619677448.300372:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.300373:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.300373:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000400:00000200:17.0:1619677448.300374:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.300375:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000800:00000200:10.0:1619677448.300375:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677448.300376:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:17.0:1619677448.300380:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.300380:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.300382:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000400:00000200:10.0:1619677448.300383:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.300384:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:17.0:1619677448.300385:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f90d380] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.300386:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e49d37c00] (20)++
00000400:00000200:10.0:1619677448.300386:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.300387:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e49d37c00] (21)++
00000400:00000200:10.0:1619677448.300387:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.300388:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e49d37c00] (22)--
00000400:00000200:10.0:1619677448.300388:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.300389:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cffdc4000] (21)++
00000400:00000200:10.0:1619677448.300389:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.300390:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cffdc4000] (22)--
00000800:00000200:17.0:1619677448.300390:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc4000] (21)--
00000400:00000200:10.0:1619677448.300393:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.300394:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:24.2:1619677448.300395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (21)++
00000800:00000200:10.0:1619677448.300397:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.300398:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.300398:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:24.0:1619677448.300399:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)++
00000800:00000200:10.0:1619677448.300400:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.300400:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:24.0:1619677448.300401:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e49d37c00] (23)--
00000800:00000200:10.0:1619677448.300401:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000400:00000200:24.0:1619677448.300402:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.300402:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.0:1619677448.300402:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)--
00000400:00000200:24.0:1619677448.300404:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.300406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:24.0:1619677448.300408:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)--
00000800:00000200:31.0:1619677448.300413:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.300415:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.300416:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.300417:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.300419:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.300420:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.300425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.300431:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.300434:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.300435:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.300435:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.300436:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.300437:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.300576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677448.300584:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677448.300587:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677448.300588:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.300591:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.300591:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677448.300592:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677448.300593:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.300594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:31.0:1619677448.300595:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.300596:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.300596:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677448.300599:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.300600:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677448.300601:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677448.300603:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:31.0:1619677448.300604:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.300604:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:31.0:1619677448.300605:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:31.0:1619677448.300606:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677448.300606:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.300608:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.300609:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.300609:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677448.300610:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:6.0:1619677448.300610:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:31.0:1619677448.300611:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000400:00000200:6.0:1619677448.300611:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716753] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.300612:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.300613:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677448.300614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677448.300614:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.300618:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.300620:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.300622:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677448.300622:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.300623:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.300624:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.300625:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:6.0:1619677448.300625:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:18.0:1619677448.300626:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.300626:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677448.300626:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677448.300627:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:18.0:1619677448.300628:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.300628:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.300628:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.300631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.300632:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.300633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:9.2:1619677448.300635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:22.0:1619677448.300639:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:1.0:1619677448.300640:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677448.300641:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.300642:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677448.300642:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:22.0:1619677448.300643:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677448.300643:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677448.300643:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.300644:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.300645:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.300646:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000400:00000200:1.0:1619677448.300646:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.300647:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.300647:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.300648:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677448.300648:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.300649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.2:1619677448.300650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677448.300657:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.300658:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677448.300659:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.300660:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.300660:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.300661:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:17.0:1619677448.300661:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.300662:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:30.0:1619677448.300663:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.300665:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.300666:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.300667:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.300669:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.300670:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.300671:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.300672:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.300675:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.300676:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.300679:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.300680:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.300680:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.300682:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.300683:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.300684:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.300684:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.300688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.300694:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.2:1619677448.300695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:13.0:1619677448.300697:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.300697:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.300699:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.300701:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.300702:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.300703:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.300705:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:25.2:1619677448.300706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.300706:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.300708:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.300710:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.300712:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:13.0:1619677448.300712:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:26.0:1619677448.300713:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.300714:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.300715:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.300715:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.300716:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:13.0:1619677448.300716:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.300716:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.300717:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.300718:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:26.0:1619677448.300719:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.300720:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.300723:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.300724:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.300725:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.300727:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.300727:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.300728:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.300728:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.300733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.300741:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.300743:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.300744:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.300746:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.300746:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.300749:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.300826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.300833:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.300835:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.300836:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.300838:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.300841:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.300842:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.300844:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.300845:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.300846:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.300847:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.300850:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.300851:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.300854:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.300855:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.300855:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.300857:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.300857:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.300858:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.300859:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.300863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.300870:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.2:1619677448.300873:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677448.300875:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.300878:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.300879:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.300881:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:5.0:1619677448.300881:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.300883:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:5.0:1619677448.300883:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:10.0:1619677448.300884:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.300886:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.300889:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.300890:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.300892:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.300893:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.300894:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.300895:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.300898:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.300899:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.300901:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.300902:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.300903:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.300904:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.300905:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.300905:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.300905:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.300910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.300916:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.300919:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.300919:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.300921:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.300922:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.300923:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.300925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:25.2:1619677448.300931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.300931:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.300934:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.300935:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.300936:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.300938:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677448.300939:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.300941:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:13.0:1619677448.300941:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.300941:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:18.0:1619677448.300942:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.300942:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:18.0:1619677448.300943:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.300943:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:18.0:1619677448.300944:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.300944:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:18.0:1619677448.300945:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.300948:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.300950:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.300953:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.300954:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.300954:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.300956:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.300956:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.300958:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.300959:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.300960:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.300961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.300965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:12.0:1619677448.300968:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.300970:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.300970:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:12.0:1619677448.300971:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.300972:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:12.0:1619677448.300973:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.300973:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:13.0:1619677448.300974:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.300974:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677448.300976:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.300994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.301000:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.301002:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.301003:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.301005:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.301008:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.301009:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.301011:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716747] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.301012:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.301013:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.301014:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.301016:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.301018:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.301020:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.301020:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.301021:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.301023:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.301023:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.301024:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.301024:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.301029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.301037:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.301039:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.301040:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.301040:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.301042:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.301044:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.301065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.301068:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.301070:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.301073:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.301074:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.301076:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.301077:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.301079:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.301080:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.301081:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.301082:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.301086:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.301087:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.301091:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.301091:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.301092:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.301094:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.301095:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.301096:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.301096:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.301100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.301107:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.301110:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.301111:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.301112:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.301113:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.301115:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.301154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.301160:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.301162:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.301163:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.301165:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.301168:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.301169:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.301170:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.301172:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.301172:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.301174:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.301177:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.301179:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.301180:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.301181:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.301182:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.301183:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.301184:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.301184:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.301185:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.301189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.301196:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.301198:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.301199:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.301200:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.301201:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.301202:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.301233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.301240:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.301243:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.301244:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.301246:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.301248:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.301249:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.301251:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.301252:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.301253:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.301254:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.301258:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.301259:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.301262:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.301263:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.301264:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.301266:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.301266:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.301267:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.301267:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.301272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.301277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:5.0:1619677448.301280:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.301282:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.301283:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.301284:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677448.301285:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.301285:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.301287:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.301288:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677448.301288:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.301291:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000400:00000200:8.0:1619677448.301299:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.301301:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000400:00000200:8.0:1619677448.301302:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.301303:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.301304:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.301304:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000400:00000200:8.0:1619677448.301304:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.0:1619677448.301305:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:8.0:1619677448.301305:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.301306:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:8.0:1619677448.301306:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.301307:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.301308:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.301311:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.301313:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.301315:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.301316:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.301317:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.301318:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.301319:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.301320:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.301321:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.301322:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.301324:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.301328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.301332:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.301335:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.301335:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.301336:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.301336:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.301337:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.301338:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.301338:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.301339:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.2:1619677448.301339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677448.301341:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677448.301348:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.301351:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.2:1619677448.301351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:30.0:1619677448.301353:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.301355:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.301355:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.301357:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.301358:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.301359:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.301359:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:30.0:1619677448.301360:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.301362:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.301362:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.301363:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:30.0:1619677448.301365:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.301365:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.301367:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.301367:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.301369:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.301370:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.301371:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.301372:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.301372:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:26.0:1619677448.301372:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.301372:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.301373:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.301374:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:26.0:1619677448.301376:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.301377:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.301378:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.301378:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.301379:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.301380:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.301381:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.301381:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:25.2:1619677448.301381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.301383:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.301383:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.301384:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.301385:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.301388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:6.0:1619677448.301389:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.301392:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.301393:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.301394:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.301395:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:6.0:1619677448.301396:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.301397:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.301399:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.301399:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:31.0:1619677448.301400:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.301401:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.301403:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.301473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.301480:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.301483:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.301483:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677448.301485:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.301488:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.301489:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.301491:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.301492:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.301493:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.301494:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.301497:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.301498:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.301501:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.301502:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.301503:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.301505:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.301505:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.301506:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.301506:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.301511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.301518:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.301521:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.301522:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.301522:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.301524:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.301527:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.301533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907b780e00] (20)++
00000800:00000200:18.0:1619677448.301538:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907b780e00] (21)++
00000800:00000200:18.0:1619677448.301541:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.52@o2ib3
00000800:00000200:2.0:1619677448.301542:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907b780e00] (22)--
00000400:00000200:18.0:1619677448.301543:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.52@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.301545:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.301546:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.301547:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.301549:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.301550:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.301551:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.301554:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.52@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.301556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:18.0:1619677448.301556:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.301558:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.301558:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.301559:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.301561:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.301561:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907b780e00] (21)++
00000800:00000200:18.0:1619677448.301562:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907b780e00] (22)--
00000800:00000200:18.0:1619677448.301562:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907b780e00] (21)--
00000800:00000200:10.0:1619677448.301563:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.301565:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677448.301566:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:9.2:1619677448.301566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677448.301568:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.301571:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.301572:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.301573:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.301573:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:10.0:1619677448.301575:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.301575:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:10.0:1619677448.301576:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.301576:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.301577:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:10.0:1619677448.301577:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.301579:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.301581:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.301583:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.301584:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.301586:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.301587:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.301588:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.301590:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.301590:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.301591:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.301592:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677448.301596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.301603:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.301606:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.301607:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.301607:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.301609:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.301612:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.301623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.301631:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.301633:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.301636:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.301637:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.301639:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.301640:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.301642:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716737] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.301643:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677448.301643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:30.0:1619677448.301644:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.301645:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.301649:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.301650:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.301650:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.301652:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.301653:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.301654:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.301655:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.301655:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:26.0:1619677448.301655:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.301657:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.301657:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.301658:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677448.301658:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.301659:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:26.0:1619677448.301659:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.301660:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.301662:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.301663:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.301663:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.2:1619677448.301663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:26.0:1619677448.301666:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.301668:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.301671:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.301671:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677448.301672:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.301672:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.301673:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677448.301674:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.301674:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677448.301674:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.301676:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:5.0:1619677448.301676:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.301677:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.301677:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677448.301677:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.301679:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:25.2:1619677448.301680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.301683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.301688:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.301690:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.301691:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.301691:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.301692:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.301692:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.301693:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.301694:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.301694:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.301697:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.301698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.301705:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.301707:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.301708:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.301709:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.301710:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.301710:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.301893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.301900:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.301903:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.301905:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.301906:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.301908:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.301909:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.301910:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.301912:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.301913:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.301914:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677448.301917:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.301919:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.301922:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.301923:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.301923:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.301925:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.301925:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.301926:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.301927:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.301931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.301938:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.301941:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.301942:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.301942:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.301944:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.301948:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.302028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.302035:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.302038:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.302039:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.302041:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.302044:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.302045:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.302047:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.302048:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.302049:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.302050:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.302053:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.302054:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.302056:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.302057:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.302058:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.302059:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.302060:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.302060:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.302061:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.302065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:4.2:1619677448.302071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677448.302072:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.302075:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.302075:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.302077:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:13.0:1619677448.302077:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.302079:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.302080:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:12.0:1619677448.302080:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.302081:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.302082:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.302086:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.302087:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.302088:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.302089:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.302090:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.302091:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.302094:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.302095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.302096:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.302099:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.302100:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.302100:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.302103:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.302103:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.302104:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.302104:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.302105:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.302106:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.302106:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:25.2:1619677448.302108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:10.0:1619677448.302108:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.302110:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.302111:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.302112:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.302114:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.302114:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.302115:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:22.0:1619677448.302116:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.302119:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:10.0:1619677448.302119:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.302120:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.302120:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677448.302120:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.302123:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.302123:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.302124:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.302124:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:22.0:1619677448.302125:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677448.302126:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.302126:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.302127:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.302128:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.302132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.302138:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.302141:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.302142:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.302144:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.302145:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.302146:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.302152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.302158:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.302161:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.302162:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.302162:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.302163:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.302164:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.302254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.302260:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.302263:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.302263:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.302265:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.302266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:1.0:1619677448.302267:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.302269:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.302270:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.302271:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.302272:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.302273:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.302274:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.302277:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:1.0:1619677448.302277:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.302280:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.302280:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.302281:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.302283:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.302283:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.302283:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.302284:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.302285:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.302286:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.302286:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.302286:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.302287:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.302288:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.302288:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:31.0:1619677448.302289:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.302289:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.302290:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677448.302292:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.302292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.302298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.302300:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.302300:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.302302:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677448.302303:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677448.302303:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.302304:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.302304:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.302304:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:13.0:1619677448.302305:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.302306:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.302306:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677448.302306:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.302306:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677448.302307:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.302308:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:17.0:1619677448.302308:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.302309:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:13.0:1619677448.302309:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.302310:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.302312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.302316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.302318:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.302321:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.302322:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.302323:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.302324:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:13.0:1619677448.302325:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.302326:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.302327:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.302328:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.302328:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.2:1619677448.302350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.302356:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.302359:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.302360:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.302362:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.302364:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.302365:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.302366:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.302367:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.302368:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.302369:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.302372:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.302373:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.302375:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.302376:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.302377:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.302378:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.302379:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.302379:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.302380:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.302384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.302392:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.302393:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.302394:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.302395:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.302396:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.302397:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.302403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.302410:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.302412:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.302413:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.302414:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.302414:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.302415:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.302445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.302453:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.302456:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.302458:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.302460:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.302460:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.302462:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.302463:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.302464:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.302465:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.302466:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.302470:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.302471:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.302474:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.302475:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.302475:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.302477:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.302477:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.302479:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.302480:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.302481:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.302482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.302486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.302489:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.302491:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.302492:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.302493:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.302494:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677448.302495:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.302496:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:13.0:1619677448.302496:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.302497:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.302498:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677448.302547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.302554:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.302557:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.302558:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.302560:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.302562:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.302563:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.302565:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.302566:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.302567:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.302568:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.302571:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.302573:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.302575:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.302575:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.302576:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.302577:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.302578:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.302579:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.302579:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.302583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.302587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:31.0:1619677448.302590:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.302590:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:31.0:1619677448.302593:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.302593:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.302593:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.302595:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:31.0:1619677448.302596:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.302596:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:30.0:1619677448.302597:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.302598:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:20.0:1619677448.302598:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.302600:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677448.302601:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.302602:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.302603:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.302604:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677448.302608:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.302610:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677448.302613:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.302613:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677448.302614:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.302616:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677448.302616:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.302617:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.302618:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.302622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.302628:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.302631:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.302632:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.302634:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677448.302635:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.302637:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.302689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.302695:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.302698:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.302699:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.302700:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.302703:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.302705:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677448.302706:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.302707:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.302708:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.302709:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677448.302713:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:21.2:1619677448.302714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:2.0:1619677448.302715:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677448.302718:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.302719:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677448.302719:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677448.302721:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:8.0:1619677448.302722:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:2.0:1619677448.302722:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.302723:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.302723:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677448.302724:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.302725:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.2:1619677448.302727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000400:00000200:8.0:1619677448.302727:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.302730:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.302732:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:10.0:1619677448.302733:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:8.0:1619677448.302733:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96716724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.302734:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.302735:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.302736:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.302736:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:8.0:1619677448.302736:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.302737:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.302740:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000400:00000200:8.0:1619677448.302740:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.302742:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.302743:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:8.0:1619677448.302745:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.302746:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.302746:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.302748:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.302748:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.302749:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.302750:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.302751:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.2:1619677448.302753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:21.2:1619677448.302756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:12.0:1619677448.302759:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.302762:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.302762:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.302763:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:12.0:1619677448.302764:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.302765:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.302766:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677448.302766:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.302767:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677448.302767:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.302828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.302834:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.302837:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.302837:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.302839:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.302842:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.302844:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.302845:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.302846:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.302847:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.302848:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.302852:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.302853:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.302856:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.302857:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.302858:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.302860:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.302861:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.302861:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.302862:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.302866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.302872:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.302875:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.302876:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.302877:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.302879:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.302880:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.302889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.302895:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.302898:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.302899:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.302899:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.302900:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.302901:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.302918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.302925:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.302927:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.302929:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.302930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:31.0:1619677448.302932:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.302932:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.302933:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.302935:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716722] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.302936:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.302936:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:31.0:1619677448.302937:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.302938:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:6.0:1619677448.302939:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.302940:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.302943:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.302944:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.302945:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.302946:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:6.0:1619677448.302947:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.302948:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.302949:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.302950:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.302950:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:6.0:1619677448.302950:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.302951:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.302952:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.302952:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677448.302952:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.302953:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.302954:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:6.0:1619677448.302956:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.302957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:6.0:1619677448.302957:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.302961:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.302962:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.302962:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.302963:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:6.0:1619677448.302964:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.302965:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.302965:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677448.302966:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:6.0:1619677448.302966:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:26.0:1619677448.302967:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.302967:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.302969:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.302969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:25.2:1619677448.302970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.302972:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.0:1619677448.302975:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.302977:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.302978:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:30.0:1619677448.302979:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.302980:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.302980:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.302980:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.302982:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.302983:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:5.0:1619677448.302983:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.302984:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677448.302985:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:5.0:1619677448.302985:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.302987:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.302988:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.302989:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.2:1619677448.302990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:5.0:1619677448.302992:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.302993:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.302996:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.302997:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.302997:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.302997:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.302999:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.302999:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.303000:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:5.0:1619677448.303000:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.303001:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:31.0:1619677448.303002:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.303004:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.303005:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.303005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:31.0:1619677448.303006:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.303008:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.303009:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.303010:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.303011:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:22.0:1619677448.303012:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:31.0:1619677448.303014:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.303014:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677448.303015:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:22.0:1619677448.303015:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.303015:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.303018:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.303018:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:22.0:1619677448.303018:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.303019:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.303021:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:22.0:1619677448.303021:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677448.303022:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.303022:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.303023:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.303027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.303033:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.303035:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.303036:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.303038:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.303039:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.303040:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.303055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.303062:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.303065:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.303066:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.303067:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.303070:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.303071:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.303072:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.303074:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.303075:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.303075:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.303079:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.303081:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.303083:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.303084:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.303084:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.303086:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.303086:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.303087:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.303088:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.303092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.303100:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.303102:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.303103:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.303104:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.303106:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.303108:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.303110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.303117:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.303120:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.303120:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.303120:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.303121:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.303122:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.303135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.303141:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.303144:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.303145:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.303147:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.303150:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.303151:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.303153:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.303154:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.303155:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.303156:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.303159:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.303161:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.303162:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.303163:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.303164:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.303165:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.303166:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.303167:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.303168:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.303169:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677448.303172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.303174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.303180:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.303181:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.303182:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:2.0:1619677448.303182:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.303183:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.303183:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.303183:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.303184:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.303185:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.303185:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:4.2:1619677448.303190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.303197:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.2:1619677448.303199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.303200:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.303200:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.303202:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.303204:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.303205:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.303206:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:17.0:1619677448.303207:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:10.0:1619677448.303208:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.303208:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.303210:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.303211:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.303211:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.303213:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.303213:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.303216:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.303217:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.303218:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.303219:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.303220:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.303221:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:10.0:1619677448.303221:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.303224:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.303225:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.303225:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.303226:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.303227:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:10.0:1619677448.303227:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.303228:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.303229:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.303229:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677448.303230:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.303230:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.303231:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:25.2:1619677448.303233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.303233:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.303233:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.303234:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.303235:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.303239:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:9.2:1619677448.303239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.303242:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.303243:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.303245:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.303245:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:30.0:1619677448.303246:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.303247:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:24.0:1619677448.303248:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.303249:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.303249:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:25.2:1619677448.303251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:24.0:1619677448.303251:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.303254:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:30.0:1619677448.303257:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.303260:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.303261:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.303262:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.303263:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.303264:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.303266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.303273:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.303276:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.303278:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.303280:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.303280:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.303281:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.303282:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.303284:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.303285:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.303286:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.303289:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.303291:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.303293:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.303294:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.303300:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.303301:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.303302:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.303302:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.303303:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.303307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.303315:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.303317:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.303318:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.303320:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.303321:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.303322:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.303435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.303441:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.303444:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.303445:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.303446:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.303449:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.303450:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.303452:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.303453:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.303454:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.303455:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.303457:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.303459:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.303462:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.303463:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.303464:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.303465:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.303466:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.303466:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.303467:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.303471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.303478:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.303481:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.303482:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.303482:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.303484:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.303487:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.303606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.303614:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.303617:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.303619:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.303621:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.303622:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.303623:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.303624:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716712] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.303625:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.303626:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.303627:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.303631:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.303632:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.303634:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.303635:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.303636:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.303637:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.303638:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.303639:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.303639:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.303643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.303650:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.303652:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.303653:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.303654:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.303655:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.303657:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.303660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.303668:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.303671:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.303673:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.303675:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.303675:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.303676:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.303677:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.303678:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.303679:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.303680:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.303683:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.303685:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.2:1619677448.303685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:30.0:1619677448.303688:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.303688:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.303689:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.303690:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.303691:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.303691:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.303692:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.303693:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677448.303693:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.303695:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.303696:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.303697:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.303697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:20.0:1619677448.303699:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.303700:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.303702:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.303703:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.303704:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.303705:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.303706:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.303707:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.303707:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.303709:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.303709:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.303712:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.303713:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.303713:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.303713:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677448.303714:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.303716:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.303717:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.303718:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.303719:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.303720:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.303722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.303724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.2:1619677448.303725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677448.303728:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.303730:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.303731:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.303731:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:17.0:1619677448.303733:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:12.0:1619677448.303733:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.303733:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:12.0:1619677448.303734:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.303734:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677448.303734:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.303735:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677448.303736:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.303736:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.303737:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:17.0:1619677448.303739:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.303741:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.303742:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.303743:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.303744:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.303745:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.303749:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.303751:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.303755:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.303755:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.303756:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.303758:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.303758:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.303760:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.303761:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.303762:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.303764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.2:1619677448.303767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.303771:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:1.0:1619677448.303772:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677448.303773:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:1.0:1619677448.303774:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:31.0:1619677448.303775:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.303775:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677448.303776:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.303776:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.303777:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.303779:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.304012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.304018:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.304020:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.304021:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.304023:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.304026:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.304027:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.304029:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.304030:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.304031:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.304032:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.304035:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.304037:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.304039:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.304040:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.304041:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.304042:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.304043:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.304043:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.304044:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.304048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.304055:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.304057:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.304058:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.304060:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.304061:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.2:1619677448.304062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:13.0:1619677448.304063:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.304068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.304070:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.304073:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:12.0:1619677448.304074:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.304074:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.304076:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.304077:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.304078:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.304078:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.304078:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.304079:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:26.0:1619677448.304080:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:12.0:1619677448.304080:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:26.0:1619677448.304081:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.304082:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.304083:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.304085:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.304087:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.304089:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.304092:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.304093:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.304094:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.304095:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.304096:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.304097:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.304098:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.304099:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.304101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.304104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.304110:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677448.304111:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677448.304113:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.304113:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:10.0:1619677448.304114:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.304114:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677448.304114:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.304115:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677448.304116:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.304119:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.304121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:22.0:1619677448.304128:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.304131:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.304131:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.304132:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.304133:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.304133:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.304191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:21.2:1619677448.304195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677448.304197:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.304200:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.304201:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:30.0:1619677448.304202:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677448.304203:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.304205:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:18.0:1619677448.304205:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.304206:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.304207:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.304208:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.304209:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677448.304209:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.304210:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.304211:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.304212:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.304212:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.304214:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.304216:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.304216:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.304217:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.304218:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.304219:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.304222:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.304223:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.304223:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.304224:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.304225:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.304225:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.304226:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.304227:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:30.0:1619677448.304228:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.304228:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.304228:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:30.0:1619677448.304229:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.304229:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:4.2:1619677448.304230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.304231:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.304231:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677448.304232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.304233:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.2:1619677448.304233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677448.304234:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.304235:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.304236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.304237:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.304239:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677448.304239:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:10.0:1619677448.304240:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.304241:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.2:1619677448.304241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.304241:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:6.0:1619677448.304242:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677448.304242:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677448.304243:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:12.0:1619677448.304243:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677448.304243:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.304243:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.304244:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677448.304245:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:10.0:1619677448.304245:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.304245:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:12.0:1619677448.304246:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.304247:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:6.0:1619677448.304247:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677448.304248:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:12.0:1619677448.304248:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.304249:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.304249:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.304250:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:12.0:1619677448.304250:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677448.304250:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.304251:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677448.304251:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.304252:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:10.0:1619677448.304252:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.304256:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.304257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.304258:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.304260:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.304260:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.304261:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.304262:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:13.0:1619677448.304263:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.304263:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.304264:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.304265:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:13.0:1619677448.304266:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.304266:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:13.0:1619677448.304267:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.304267:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:25.2:1619677448.304268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:13.0:1619677448.304268:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.304269:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.304271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.304275:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.304277:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.304278:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.304278:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677448.304279:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.304280:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:31.0:1619677448.304281:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.304281:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.304282:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.304282:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.2:1619677448.304415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.304421:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.304424:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.304426:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.304427:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.304429:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.304431:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.304432:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.304433:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.304434:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.304435:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.304439:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.304440:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.304444:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.304444:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.304445:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.304447:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.304447:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.304448:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.304449:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.304453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.304461:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.2:1619677448.304461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677448.304463:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.304464:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.304466:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.304467:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.304468:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.304469:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.304470:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.304471:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.304473:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.304475:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.304477:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.304478:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716702] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.304480:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.304481:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.304482:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.304485:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.304487:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.304489:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.304490:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.304491:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.304492:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.304493:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.304493:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.304494:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.304498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.304507:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.304509:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.304510:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.304510:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.304512:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.304515:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.304550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.304555:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.304558:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.304558:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.304560:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.304563:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.304564:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.304565:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.304567:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.304568:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.304569:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.304572:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.304574:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.304577:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.304578:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.304579:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.304581:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.304581:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.304582:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.304582:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.304586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.304593:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.304596:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.304597:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.304597:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.304600:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.304603:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:25.2:1619677448.304611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000800:00000200:26.0:1619677448.304616:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.304619:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.304619:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.304620:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.304621:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.304622:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677448.304669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.304676:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.304678:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.304679:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.304681:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.304683:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.304685:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.304686:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.304687:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.304688:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.304689:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.304692:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.304694:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.304697:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.304698:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.304698:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.304700:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.304701:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.304701:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.304702:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677448.304706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.304712:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.304714:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.304715:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.304717:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.304719:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.304719:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.304824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.304827:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.304830:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.304832:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.304833:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.304835:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.304837:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.304838:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.304840:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.304841:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.304842:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.304845:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.304847:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.304850:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.304850:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.304851:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.304853:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.304853:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.304854:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.304855:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.304859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.304867:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.304870:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.304870:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.304872:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.304874:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.304875:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.304880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.304887:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.304890:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.304891:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.304893:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.304895:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.304897:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.304898:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.304899:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.304900:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.304901:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677448.304904:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.304906:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.304909:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.304910:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.304911:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.304912:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.304913:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.304913:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.304914:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.304918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.2:1619677448.304919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.304924:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.304927:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:5.0:1619677448.304927:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:1.0:1619677448.304928:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.304929:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:17.0:1619677448.304930:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.304930:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.304932:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.304932:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.304933:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:17.0:1619677448.304934:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.304935:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.304936:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.304936:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:17.0:1619677448.304937:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.304938:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.304941:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.304943:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.304945:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.304946:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.304947:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.304948:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.304948:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.304949:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.304950:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.304954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.304962:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.304964:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.304964:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.304965:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.304966:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.304968:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.304977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.304985:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.304987:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.304988:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.304991:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.304993:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.304995:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.304996:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.304998:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.304999:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.305000:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.305003:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.305005:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.305008:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.305009:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.305010:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.305011:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.305012:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.305012:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.305013:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.305018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.305026:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.305029:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.305030:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.305030:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.305032:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.305036:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.305072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.305080:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.305083:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.305085:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.305087:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.305088:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.305089:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.305091:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.305092:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.305093:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.305094:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.305098:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.305099:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.305103:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.305103:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.305104:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.305106:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.305106:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.305107:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.305108:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.305112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.305120:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.305122:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.305123:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.305125:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.305126:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.305128:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.305132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.305139:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.305142:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.305143:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.305143:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.305144:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.305145:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.305253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.305258:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.305261:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.305262:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.305264:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.305267:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.305269:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.305270:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.305271:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.305272:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.305273:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.305277:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.305279:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.305281:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.305282:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.305283:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.305284:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.305285:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.305286:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.305286:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.305290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.305299:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.305302:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.305302:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.305303:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.305305:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.305307:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.305327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.305335:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.305338:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.305339:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.305341:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.305344:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.305345:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.305347:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.305348:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.305349:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.305350:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.305353:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.305355:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.305358:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.305359:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.305359:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.305361:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.305361:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.305362:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.305362:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.2:1619677448.305367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:25.2:1619677448.305368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.305374:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.305376:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.305376:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.305378:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.305379:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677448.305379:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.305380:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.305380:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.305381:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.305382:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.305383:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.305384:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.305386:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:5.0:1619677448.305386:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.305387:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.305388:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:25.2:1619677448.305390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000400:00000200:5.0:1619677448.305391:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.305393:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.305396:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.305396:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.305397:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.305398:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.305399:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.305400:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.305400:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677448.305401:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.305401:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.305401:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677448.305402:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.305403:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.305404:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.305405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.305413:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.305416:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.305417:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.305418:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.305420:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.305422:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.305423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.305430:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.305433:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.305434:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.305434:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.305435:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.305436:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.305494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.305500:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.305503:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.305504:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.305506:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.305508:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.305509:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.305511:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.305512:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.305513:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.305514:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.305518:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.305520:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.305522:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.305523:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.305524:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.305525:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.305526:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.305526:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.305527:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.305532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.305540:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.305543:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.305544:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.305544:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.305546:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.305548:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.305563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.305570:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.305573:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.305574:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.305576:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.305579:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.305580:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.305581:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.305583:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.305584:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.305585:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.305588:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.305590:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.305592:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.305593:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.305594:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.305595:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.305596:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.305597:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.305597:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.305601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.305607:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.305610:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.305610:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.305612:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.305614:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.305615:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.305665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.305672:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.305675:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.305676:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.305677:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.305679:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.305681:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.305682:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.305683:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.305684:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.305685:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.305688:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.305689:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.305693:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.305693:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.305694:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.305695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677448.305695:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.305696:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.305696:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.305697:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.305701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.305703:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.305705:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.305707:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.305708:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:31.0:1619677448.305709:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.305710:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.305711:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:20.0:1619677448.305711:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.2:1619677448.305711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677448.305712:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716688] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.305712:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.305714:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.305714:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.305715:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.305716:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.305716:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.305717:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677448.305719:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:30.0:1619677448.305720:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.305720:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.305722:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.305722:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:17.0:1619677448.305722:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.305724:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.305725:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:30.0:1619677448.305726:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.305727:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.305727:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:17.0:1619677448.305727:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.305728:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.305729:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:17.0:1619677448.305729:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.305730:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.305730:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.305731:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.305733:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.305733:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:17.0:1619677448.305733:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.305735:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.2:1619677448.305735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677448.305737:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.305737:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.305738:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677448.305739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677448.305740:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.305740:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.305741:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.305742:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:5.0:1619677448.305742:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677448.305743:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.305744:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.305745:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.305746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.305747:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.305747:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.2:1619677448.305748:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677448.305749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:31.0:1619677448.305750:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.305750:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.305751:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.305753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:5.0:1619677448.305753:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.305754:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677448.305754:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.305756:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:5.0:1619677448.305756:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677448.305756:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.305757:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:5.0:1619677448.305757:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000400:00000200:1.0:1619677448.305757:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.305758:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677448.305758:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.305758:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:17.0:1619677448.305759:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.305760:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677448.305760:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677448.305760:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.305761:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:6.0:1619677448.305761:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.305761:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.305761:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:1.0:1619677448.305761:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677448.305762:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:6.0:1619677448.305764:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.305765:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.305767:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.305768:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.305770:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.305771:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.305775:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.305777:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.305779:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.305780:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.305781:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.305782:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.305783:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.305784:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.305785:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.305786:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.305788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.305791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.305796:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677448.305797:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:17.0:1619677448.305798:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:22.0:1619677448.305799:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:17.0:1619677448.305799:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.305800:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677448.305800:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.305801:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:17.0:1619677448.305801:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.305804:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.305807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.305814:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.305817:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.305817:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.305818:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.305819:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.305819:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.305871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.305878:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.305881:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.305882:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.305884:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.305887:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.305889:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.305890:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.305892:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.305893:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.305894:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677448.305897:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.305899:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.305902:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.305903:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.305903:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.305905:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.305905:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.305906:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.305907:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.305911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.305918:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.305920:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.305921:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.305923:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.305924:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.305926:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.305958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.305965:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.305968:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.305969:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.305971:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.305973:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.305975:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.305976:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.305977:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.305978:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.305979:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.305983:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.305984:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.305987:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.305988:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.305988:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.305990:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.305990:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.305991:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.305991:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.305996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.306004:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.306007:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.306008:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.306009:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.306011:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.306014:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.306168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.306175:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.306178:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.306179:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.306181:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.306183:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.306185:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.306186:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.306187:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.306188:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.306189:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.306192:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.306194:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.306196:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.306197:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.306197:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.306198:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.306199:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.306200:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.306200:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.306204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.306212:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.306214:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.306215:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.306216:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.306218:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.306219:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.306286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.306292:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.306298:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.306299:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677448.306302:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.306304:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.306305:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.306306:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716682] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.306307:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.306308:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.306309:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.306312:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.306314:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.306317:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.306317:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.306318:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.306319:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.306320:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.306321:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.306322:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.306323:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.306326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.306329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.306334:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.306337:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677448.306337:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:22.0:1619677448.306338:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.306338:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.306339:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677448.306340:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.306340:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.306341:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677448.306342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677448.306343:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677448.306350:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.306352:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.306355:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.306357:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.306357:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.306359:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.306360:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.306362:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.306363:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.306364:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.306367:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.306368:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.306371:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.306372:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.306373:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.306374:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.306375:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.306376:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.306376:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.306380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.306388:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.306391:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.306392:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.306392:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.306394:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.306397:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.306422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.306426:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.306429:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.306430:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.306431:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.306434:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.306435:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.306436:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.306437:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.306438:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.306439:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.2:1619677448.306440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:22.0:1619677448.306442:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.306444:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.306446:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.306447:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.306448:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:22.0:1619677448.306448:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.306449:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.306450:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.306450:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677448.306451:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:22.0:1619677448.306451:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:30.0:1619677448.306453:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.306455:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.306456:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.306456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677448.306458:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.306459:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.306460:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.306461:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.306462:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.306464:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:30.0:1619677448.306466:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.306467:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:30.0:1619677448.306468:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:17.0:1619677448.306468:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.306469:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.306470:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.306472:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.306472:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.306473:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677448.306473:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677448.306475:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.306476:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.306477:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.306478:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.306481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.306489:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.306492:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.306493:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.306494:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.306495:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.306497:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.306646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.306653:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.306656:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.306657:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.306659:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.306662:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.306663:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.306664:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.306665:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.306666:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.306667:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.306670:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.306672:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.306675:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.306675:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.306676:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.306678:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.306678:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.306679:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.306679:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.306683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.306688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:20.2:1619677448.306688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.306691:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.306692:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677448.306693:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:18.0:1619677448.306694:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:6.0:1619677448.306694:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.306695:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:20.0:1619677448.306695:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.306696:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.306696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:20.0:1619677448.306698:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.306698:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:6.0:1619677448.306699:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.306699:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:20.0:1619677448.306700:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.306701:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.306701:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:18.0:1619677448.306702:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.306702:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.306703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000400:00000200:20.0:1619677448.306703:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.306703:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:20.0:1619677448.306704:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.306704:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.306705:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.306706:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677448.306706:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.306707:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:18.0:1619677448.306708:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.306710:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.306710:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.306711:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:18.0:1619677448.306711:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.306711:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.306712:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.306713:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.306714:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.306714:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.306714:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.306715:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000400:00000200:10.0:1619677448.306715:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:22.0:1619677448.306716:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.306716:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:10.0:1619677448.306716:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96716675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.306717:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.306717:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.306718:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:10.0:1619677448.306718:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.306719:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.306720:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.306720:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.306721:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.306721:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:10.0:1619677448.306721:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:20.0:1619677448.306722:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.306723:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677448.306723:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.306723:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.306724:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677448.306725:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.306726:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677448.306726:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:25.2:1619677448.306727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.306727:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:10.0:1619677448.306727:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677448.306728:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:10.0:1619677448.306729:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.306730:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.306731:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.306732:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.306733:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677448.306734:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:10.0:1619677448.306734:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.306735:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.306736:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:13.0:1619677448.306737:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:25.2:1619677448.306738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:13.0:1619677448.306738:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.306740:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.306741:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:5.2:1619677448.306741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677448.306742:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.306742:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677448.306743:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.306743:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.306744:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:12.0:1619677448.306744:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.306746:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.306747:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.306747:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.306747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.306748:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:31.0:1619677448.306749:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.306750:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:30.0:1619677448.306751:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.306751:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.306752:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:26.0:1619677448.306752:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.306753:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.306756:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:25.2:1619677448.306757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:13.0:1619677448.306757:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.306757:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:4.2:1619677448.306757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:13.0:1619677448.306758:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.306759:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.306764:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:24.0:1619677448.306765:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677448.306767:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.306768:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:24.0:1619677448.306768:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.306768:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677448.306769:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.306769:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.306771:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:24.0:1619677448.306771:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.306773:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.306775:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.306776:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.306778:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.306779:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.306780:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.306783:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.306785:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.306787:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.306788:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.306789:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.306790:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.306791:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.306791:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.306792:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.306796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.306804:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.306807:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.306808:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.306808:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.306810:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.306813:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.306937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677448.306944:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.306947:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677448.306948:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677448.306950:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.306953:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.306954:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.306956:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.306957:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.306958:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.306959:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.306961:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.306963:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.306965:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.306966:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.306967:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.306968:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.306969:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.306969:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.306970:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677448.306974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.306980:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.306982:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.306983:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.306984:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.306986:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.306987:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.306997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.307003:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.307005:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.307008:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.307008:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677448.307010:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.307012:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677448.307014:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.307015:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.307016:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.307017:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677448.307021:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.307023:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677448.307026:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.307027:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677448.307027:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677448.307029:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.307030:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.307031:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.307032:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.307033:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.307035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.2:1619677448.307038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.307041:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:31.0:1619677448.307042:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:20.0:1619677448.307043:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:31.0:1619677448.307044:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:20.0:1619677448.307044:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:31.0:1619677448.307045:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.307045:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:31.0:1619677448.307047:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677448.307048:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.307049:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.307098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.307105:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.307107:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.307108:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.307110:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.307113:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.307114:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.307116:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.307117:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.307118:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.307119:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.307123:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.307124:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.307127:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.307127:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.307128:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.307130:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.307130:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.307131:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.307131:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.307135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.2:1619677448.307139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:30.0:1619677448.307142:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.307145:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:2.0:1619677448.307145:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:30.0:1619677448.307146:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.307147:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.307148:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.307148:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:31.0:1619677448.307149:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.307150:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:2.0:1619677448.307151:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.307154:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.307155:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.307156:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.307158:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.307159:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.307160:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.307163:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.307164:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.307166:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.307167:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.307168:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.307169:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.307170:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.307171:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.307171:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.307176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.307183:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.307185:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.307185:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.307187:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.307187:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.307188:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.307262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.307269:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.307272:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.307274:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.307276:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.307277:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.307279:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.307280:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.307281:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.307283:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.307284:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.307287:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.307288:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.307291:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.307292:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.307292:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.307299:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.307300:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677448.307300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.307301:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.307301:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.307308:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.307310:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.307311:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.307312:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.307314:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.307315:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.307352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.307358:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.307361:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.307361:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:21.2:1619677448.307363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:17.0:1619677448.307363:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.307365:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.307366:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.307368:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.307369:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.307369:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:17.0:1619677448.307370:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.307372:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677448.307372:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.307373:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.307375:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677448.307376:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.307376:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.307378:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:2.0:1619677448.307378:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.307379:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716667] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.307380:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.307381:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.307382:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.307382:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.307383:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.307383:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.307385:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.307385:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.307386:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:2.0:1619677448.307386:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677448.307387:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677448.307388:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.307390:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.307390:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:25.2:1619677448.307391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677448.307391:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.307392:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.307393:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.307393:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.307394:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.307399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.307399:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.307402:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.307403:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.307403:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.307405:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.307407:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677448.307407:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677448.307409:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.307410:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.307410:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.307412:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.307414:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.307430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.307437:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.307440:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.307440:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.307441:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.307441:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.307441:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.307688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.307694:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.307696:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.307697:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.307699:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.307700:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.307701:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.307702:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.307704:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.307705:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.307706:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.307708:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.307710:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.307711:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.307712:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.307712:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.307713:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.307713:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.307714:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.307715:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.307719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.307726:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.307728:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.307728:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.307730:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.307730:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.307731:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.307770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.307779:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.2:1619677448.307779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.307782:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.307784:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.307785:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.307786:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.307786:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.307787:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.307789:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.307789:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.307789:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:30.0:1619677448.307791:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.307793:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.307793:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.307794:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.307795:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.307796:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.307798:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.307798:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.307800:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.307801:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.307801:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.307802:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.307804:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.307805:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.307806:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677448.307806:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.307807:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.307808:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.307808:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.307809:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.307809:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677448.307811:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.307812:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:25.2:1619677448.307813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.307813:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.307814:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.307815:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.307815:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.307816:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.307819:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:25.2:1619677448.307822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.307822:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.307823:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.307824:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.307824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677448.307826:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.307826:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.307827:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.307830:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.307831:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.307832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.307832:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:12.0:1619677448.307833:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.307834:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.307835:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:13.0:1619677448.307835:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.307836:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677448.307837:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.307837:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.307840:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.307840:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.307841:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.307841:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:10.0:1619677448.307841:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:13.0:1619677448.307842:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.307842:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.307843:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677448.307843:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.307844:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.307845:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.307849:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.307851:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.307853:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.307854:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.307854:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.307856:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.307856:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.307857:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.307857:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.307862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.307868:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.307871:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.307872:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.307873:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.307875:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.307875:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.2:1619677448.307922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.307928:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.307931:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.307932:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.307934:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.307935:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.307937:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.307938:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.307939:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.307941:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.307942:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.307945:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.307947:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.307950:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.307951:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.307951:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.307953:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.307953:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.307954:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.307955:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.307959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.307965:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.307968:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.307969:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.307971:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.307972:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.307974:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.308085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.308091:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.308094:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.308095:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.308097:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.308099:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.308101:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.308102:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.308103:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.308104:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.308105:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.308109:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.308110:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.308114:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.308114:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.2:1619677448.308115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677448.308115:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.308117:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.308117:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.308118:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.308118:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.308123:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:9.2:1619677448.308123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:30.0:1619677448.308125:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.308127:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.308128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:31.0:1619677448.308129:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.308130:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.308130:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:30.0:1619677448.308131:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.308132:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.308132:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:30.0:1619677448.308133:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.308134:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.308134:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.308134:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.308136:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.308136:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:8.0:1619677448.308137:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.308138:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.308139:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:30.0:1619677448.308140:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.308140:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:26.0:1619677448.308141:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:9.2:1619677448.308141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:30.0:1619677448.308142:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.308143:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.308145:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.308145:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:30.0:1619677448.308146:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.308146:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:26.0:1619677448.308147:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.308147:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:30.0:1619677448.308148:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.308148:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677448.308148:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.308149:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677448.308149:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.308149:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:30.0:1619677448.308150:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:26.0:1619677448.308150:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.308150:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.308150:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.308151:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.308152:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:26.0:1619677448.308153:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:25.2:1619677448.308153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.308155:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.308158:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.308158:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.308159:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:12.0:1619677448.308160:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:26.0:1619677448.308161:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.308161:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.308162:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:12.0:1619677448.308162:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:26.0:1619677448.308163:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:12.0:1619677448.308163:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.308164:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:12.0:1619677448.308165:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.308166:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.2:1619677448.308167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:12.0:1619677448.308168:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.308169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:1.0:1619677448.308174:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:10.0:1619677448.308176:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:21.2:1619677448.308177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677448.308177:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.308178:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.308178:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.308179:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.308180:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:10.0:1619677448.308180:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:1.0:1619677448.308180:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:1.0:1619677448.308183:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:31.0:1619677448.308184:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.308187:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.2:1619677448.308188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000400:00000200:31.0:1619677448.308189:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.308191:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.308191:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.308192:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677448.308194:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.308195:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.308195:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677448.308196:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.308197:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.308198:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.308199:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.308199:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:31.0:1619677448.308200:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.308200:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.308200:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:31.0:1619677448.308202:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677448.308204:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.308205:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.308205:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.308207:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.308208:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.308209:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.308210:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.308211:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677448.308212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:21.2:1619677448.308217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677448.308218:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.308221:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.308222:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.308222:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.308224:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677448.308224:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.308226:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677448.308226:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:30.0:1619677448.308227:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.308228:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677448.308296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.308302:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:4.2:1619677448.308304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677448.308304:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.308306:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.308308:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.308310:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.308312:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677448.308312:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677448.308313:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.308314:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.308315:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677448.308315:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.308316:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.308317:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.308318:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.308321:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.308321:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.308322:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.308323:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.308323:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.2:1619677448.308325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:10.0:1619677448.308325:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.308326:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.308326:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.308327:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:2.0:1619677448.308327:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:20.0:1619677448.308328:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.308328:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.308330:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677448.308330:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677448.308330:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:10.0:1619677448.308331:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.308331:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.308332:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:20.0:1619677448.308333:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.308334:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.308334:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.308335:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.308336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:20.0:1619677448.308337:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677448.308337:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.308338:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.308338:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:20.0:1619677448.308339:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96716655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.308340:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.308340:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.308340:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677448.308341:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.308341:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:20.0:1619677448.308342:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.308342:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:5.2:1619677448.308342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.308343:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.308345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:20.0:1619677448.308345:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.308345:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.0:1619677448.308348:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.308348:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677448.308348:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:4.2:1619677448.308349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.308351:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677448.308351:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.308351:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:30.0:1619677448.308353:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.308353:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.308353:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677448.308354:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.308354:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:13.0:1619677448.308354:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.308354:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.308356:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:26.0:1619677448.308356:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:1.0:1619677448.308356:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.308357:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.308357:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.308357:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677448.308357:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:30.0:1619677448.308358:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.308358:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.308358:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677448.308358:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:1.0:1619677448.308358:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:26.0:1619677448.308359:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:1.0:1619677448.308359:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96716654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.308360:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.308360:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:1.0:1619677448.308360:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.308361:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:1.0:1619677448.308361:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.308362:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.308362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677448.308362:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.2:1619677448.308365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677448.308366:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.308368:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.308369:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.308370:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677448.308371:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677448.308371:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.308372:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)++
00000400:00000200:12.0:1619677448.308373:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.308373:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:1.0:1619677448.308373:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000400:00000200:12.0:1619677448.308374:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.308374:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.308375:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677448.308375:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:1.0:1619677448.308375:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.308376:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.308377:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.308377:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.308378:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.308379:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.308380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.308384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.308387:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.308389:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677448.308389:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.308390:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.308391:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.308392:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677448.308392:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.308393:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.308393:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.308394:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.308471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.308479:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.308481:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.308482:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.308484:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.308486:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.308488:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.308489:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.308490:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.308491:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.308492:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.308496:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.308497:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.308500:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.308501:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.308502:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.308503:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.308504:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.308505:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.308505:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.308509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.308515:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.308517:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.308518:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.308520:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.308521:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.308522:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.308526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:13.0:1619677448.308533:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.308535:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.308536:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.308537:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.308537:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.308538:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.308614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.308622:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.308624:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.308626:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.308629:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.308629:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.308630:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.308631:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.308633:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.308634:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.308634:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.308637:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.308639:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.308641:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.308642:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.308642:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.308644:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.308644:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.308645:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.308646:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.308649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.308656:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.308658:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.308659:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.308661:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.308662:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.308663:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.308669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.308675:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.308677:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.308678:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.308679:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.308679:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.308680:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.308710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.308716:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.308719:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.308719:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.308722:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.308724:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.308726:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.308727:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.308728:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.308730:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.308731:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.308734:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.308736:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.308739:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.308739:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.308740:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.308742:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.308742:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.308743:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.308743:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.308748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.308755:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.308757:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.308758:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.308760:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.308761:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.308763:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.308903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.308910:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.308913:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.308915:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.308915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:31.0:1619677448.308917:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.308918:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.308919:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.308920:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.308921:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.308922:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.308922:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:30.0:1619677448.308923:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:22.0:1619677448.308924:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.308925:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.308928:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.308929:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.308931:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:22.0:1619677448.308931:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.308932:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677448.308933:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.308934:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:22.0:1619677448.308934:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.308935:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:22.0:1619677448.308935:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.308936:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:22.0:1619677448.308936:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.308937:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:22.0:1619677448.308937:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.308938:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.308938:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:22.0:1619677448.308940:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.308942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:22.0:1619677448.308942:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.308945:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.308945:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.308946:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.308948:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:22.0:1619677448.308948:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.308949:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.308949:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.308950:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677448.308951:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.308952:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.308954:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.308954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.308955:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.308956:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.308957:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.308960:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.308960:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.308962:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.308964:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.308965:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.308971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677448.308978:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.308980:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.308981:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.308982:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.308983:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.308985:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.309052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.309057:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.309060:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.309062:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.309063:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.309065:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.309067:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.309068:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.309070:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.309070:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.309071:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.309074:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.309076:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.309079:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.309080:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.309081:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.309082:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.309083:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.309084:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.309084:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.309088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.309094:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.309097:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.309098:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.309100:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.309101:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.309102:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.309162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.309169:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.309172:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.309173:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.309174:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.309177:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.309179:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:24.0:1619677448.309180:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.309181:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.309182:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.309183:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:24.0:1619677448.309186:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.309187:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:24.0:1619677448.309190:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.309190:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:24.0:1619677448.309191:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:24.0:1619677448.309193:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:24.0:1619677448.309193:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.309194:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.309195:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.309199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.2:1619677448.309204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.309205:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.309207:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.309208:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.309208:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.309210:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.309210:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.309212:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:26.0:1619677448.309213:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:1.0:1619677448.309213:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.309215:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.309218:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.309219:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:25.2:1619677448.309220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000400:00000200:17.0:1619677448.309221:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.309222:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.309223:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.309224:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:10.0:1619677448.309226:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000400:00000200:17.0:1619677448.309227:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.309228:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.309229:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:17.0:1619677448.309229:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:10.0:1619677448.309229:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.309230:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.309231:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:17.0:1619677448.309232:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.309233:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.309234:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.309235:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.309236:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.309237:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.309237:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.309241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.309248:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.309250:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.309251:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.309252:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.309254:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.309257:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677448.309291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.309293:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.309300:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.309301:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.309303:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677448.309306:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.309308:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677448.309309:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.309310:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.309312:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.309313:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677448.309316:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.309318:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.309321:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.309322:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.309322:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.309324:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.309325:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.309326:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.309326:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.309330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.309338:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.309340:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.309341:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.309341:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.309344:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:5.2:1619677448.309344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:26.0:1619677448.309346:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:18.0:1619677448.309351:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.309353:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.309354:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.309357:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.309359:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.309361:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.309362:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.309363:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.309364:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.309365:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.309369:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.309370:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.309373:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.309374:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.309374:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.309376:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.309377:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.309378:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.309379:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.309379:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.309383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.309384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.309390:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:5.0:1619677448.309392:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677448.309392:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.309392:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.309393:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677448.309395:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.309395:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.309397:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.309398:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.309400:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.309413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.309421:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.309424:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.309426:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.309428:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.309429:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.309430:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.309431:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.309432:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.309433:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.309434:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.309438:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.309439:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.309442:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.309443:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.309444:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.309445:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.309446:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.309447:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.309447:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.309452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.309459:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.309462:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.309463:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.309463:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.309466:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.309468:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.309473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.309480:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.309482:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.309483:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.309483:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.309484:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.309485:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.309489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.309494:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.309497:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.309497:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.309499:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.309501:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.309502:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677448.309503:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.309504:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.309505:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.309506:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677448.309509:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.309511:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.309512:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.309513:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.309514:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.309515:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.309516:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.309516:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.309517:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.309521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.309528:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.309530:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.309530:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.309531:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.309532:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.309533:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.309534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.309541:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.309544:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.309545:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.309547:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.309549:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.309551:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.309552:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.309554:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.309555:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.309556:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.309559:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.309560:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.309564:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.309564:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.309565:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.309567:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.309568:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.309568:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.309569:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.309574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677448.309574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.309581:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677448.309581:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677448.309584:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:18.0:1619677448.309584:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:8.0:1619677448.309585:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677448.309585:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:24.0:1619677448.309586:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.309586:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.309588:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.309589:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.309590:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.309591:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.309591:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:24.0:1619677448.309593:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.309594:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.309594:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.309597:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.309599:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.309602:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.309602:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.309603:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.309605:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.309605:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.309606:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.309606:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.309611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.309619:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.309622:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.309623:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.309623:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.309625:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.309628:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.309674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.309677:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.309679:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.309681:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.309682:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.309684:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.309686:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677448.309687:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.309689:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.309690:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.309691:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677448.309694:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.309695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677448.309696:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677448.309698:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.309699:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677448.309700:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.309701:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.309701:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.309702:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.309703:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.309703:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677448.309704:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.309704:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.309706:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.309707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:17.0:1619677448.309709:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.309711:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.309712:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.309714:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.309715:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.309716:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.309716:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.309717:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.309718:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.309720:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.309720:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.309722:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.309723:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.309724:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.309724:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677448.309725:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.309726:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.309728:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.309728:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.309729:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.309729:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.309733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.309740:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.309742:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.309742:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.309744:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.309744:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.309745:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.309793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.309799:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.309802:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.309802:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.309805:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.309807:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.309809:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.309810:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.309812:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.309813:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.309814:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.309817:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.309818:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.309821:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.309821:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.309822:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.309824:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.309824:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.309825:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.309826:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.309830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.309838:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.309841:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.309841:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.309843:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.309843:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.309845:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.309867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.309874:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.309877:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.309877:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.309879:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.309882:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.309884:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.309885:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.309887:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.309888:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.309889:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.309892:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.309894:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.309897:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.309897:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.309898:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.309900:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.309900:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.309901:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.309901:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.309906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.309914:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.309916:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.309917:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.309918:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.309919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:22.0:1619677448.309920:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.309922:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677448.309926:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.309929:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:21.2:1619677448.309930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fdae00] (20)++
00000400:00000200:31.0:1619677448.309931:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.309933:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.309934:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.309935:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fdae00] (21)++
00000400:00000200:31.0:1619677448.309936:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.309937:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.309938:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.309938:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.38@o2ib3
00000400:00000200:31.0:1619677448.309940:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.309940:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.38@o2ib3 : ACK - routed
00000400:00000200:31.0:1619677448.309941:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.309941:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (22)--
00000400:00000200:31.0:1619677448.309944:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.309946:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:20.0:1619677448.309947:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.309949:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.309949:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:31.0:1619677448.309950:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:20.0:1619677448.309950:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.309951:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:20.0:1619677448.309952:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.309953:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.309953:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:20.0:1619677448.309953:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.309954:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:31.0:1619677448.309955:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.309956:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.309957:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.309957:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.38@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.309959:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.2:1619677448.309959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.309963:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.309966:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.309966:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.309967:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677448.309967:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.309968:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:5.0:1619677448.309968:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.309970:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.309970:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000400:00000200:5.0:1619677448.309970:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.309972:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fdae00] (21)++
00000800:00000200:30.0:1619677448.309973:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:20.0:1619677448.309973:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fdae00] (22)--
00000800:00000200:20.0:1619677448.309973:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (21)--
00000800:00000200:5.0:1619677448.309973:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.309974:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677448.309974:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677448.309975:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:9.2:1619677448.309976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.309982:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.309985:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.309985:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.309986:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.309988:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677448.309991:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.2:1619677448.310123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.2:1619677448.310127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.310131:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.310133:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.310133:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.310134:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.310135:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.310136:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.310137:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.310139:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.310140:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.310140:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.310142:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.310143:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:17.0:1619677448.310144:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.310145:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.310147:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.310147:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.310148:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.310149:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.310150:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.310151:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:17.0:1619677448.310153:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.310155:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.310158:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.310158:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.310159:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.310160:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.310160:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:17.0:1619677448.310162:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.310163:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.310163:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.310163:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:17.0:1619677448.310164:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:10.0:1619677448.310164:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:17.0:1619677448.310166:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.310166:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.310166:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.310166:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.310167:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.310168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.310168:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.310171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.2:1619677448.310171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.310176:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:1.0:1619677448.310177:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:30.0:1619677448.310178:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.310178:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.310179:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:1.0:1619677448.310179:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:31.0:1619677448.310180:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.310180:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.310180:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.310180:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.310181:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677448.310182:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.310184:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.310184:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.310185:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.310186:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.310203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.310209:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.310212:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.310213:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.310215:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.310217:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.310219:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.310220:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.310222:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.310223:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.310224:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.310227:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.310229:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.310232:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.310233:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.310234:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.310235:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.310236:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.310237:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.310238:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.310238:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.310241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.310244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677448.310248:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:18.0:1619677448.310249:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:26.0:1619677448.310250:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:18.0:1619677448.310251:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:10.0:1619677448.310251:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.310252:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.310252:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.310253:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:26.0:1619677448.310254:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.310257:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.310497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.310504:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.310507:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.310509:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.310512:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.310512:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.310513:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.310514:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.310516:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.310517:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.310518:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.310521:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.310523:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.310526:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.310527:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.310527:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.310529:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.310530:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.310531:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.310531:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.310536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.310544:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.310547:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.310548:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.310549:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.310551:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.310553:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.310584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.310590:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.310593:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.310594:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.310596:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.310599:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.310601:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.310602:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716629] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.310603:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.310604:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.310605:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.310608:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.310610:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.310612:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.310613:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.310614:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.310615:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.310616:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.310616:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.310617:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677448.310621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:25.2:1619677448.310622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.310629:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.310631:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:6.0:1619677448.310631:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:26.0:1619677448.310632:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677448.310633:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.310634:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:10.0:1619677448.310634:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.310634:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.310636:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.310636:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.310637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:10.0:1619677448.310638:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.310639:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.310640:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:10.0:1619677448.310641:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.310641:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.310642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000400:00000200:10.0:1619677448.310643:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.310643:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.310644:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.310645:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.310646:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.310648:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.310648:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000400:00000200:10.0:1619677448.310648:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.310649:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.310650:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:10.0:1619677448.310650:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.310651:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:6.0:1619677448.310652:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.310653:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.310654:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:6.0:1619677448.310654:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.310655:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:6.0:1619677448.310655:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:10.0:1619677448.310656:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.310657:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677448.310657:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.310658:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.310658:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.310658:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.310659:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.310660:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:25.2:1619677448.310662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:6.0:1619677448.310663:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.310666:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.310669:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.310671:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.2:1619677448.310671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:6.0:1619677448.310671:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.310671:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.310673:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.310673:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000400:00000200:22.0:1619677448.310674:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.310674:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:22.0:1619677448.310676:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.310676:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.310677:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:5.0:1619677448.310677:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677448.310678:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.310678:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677448.310679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.310679:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677448.310679:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.2:1619677448.310682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677448.310682:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.310682:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.310686:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.310687:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677448.310688:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:5.0:1619677448.310688:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.310689:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.310689:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:22.0:1619677448.310690:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:5.0:1619677448.310690:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.310691:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677448.310691:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:5.0:1619677448.310691:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.310691:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.310692:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:5.0:1619677448.310692:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677448.310693:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.310696:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.310696:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677448.310698:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.310702:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.310703:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.310703:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.310706:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.310706:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.310707:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.310708:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.310712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.310718:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.310721:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.310722:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.310724:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.310725:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.310727:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.310734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.310741:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.310744:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.310745:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.310747:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.310750:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.310751:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.310752:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.310753:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.310754:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.310755:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.310759:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.310760:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.310763:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.310764:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.310764:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.310766:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.310766:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.310767:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.310768:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.310769:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.310771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.2:1619677448.310774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.310778:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.310780:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.310781:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.310782:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.310782:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:31.0:1619677448.310784:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.310784:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:30.0:1619677448.310785:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:26.0:1619677448.310785:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.310785:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.310790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.2:1619677448.310796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:31.0:1619677448.310797:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.310800:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.310800:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.310801:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.310802:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.310802:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:1.0:1619677448.310802:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677448.310804:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677448.310805:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677448.310807:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.310810:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.310811:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677448.310812:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.310814:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.310815:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.310816:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677448.310819:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.310821:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.310824:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.310825:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:1.0:1619677448.310825:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677448.310827:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677448.310827:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677448.310828:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677448.310828:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:25.2:1619677448.310833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.310840:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.310842:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.310843:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.310843:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.310845:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.310848:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.311002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.311010:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.311013:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.311015:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.311017:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.311018:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.311020:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.311021:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.311022:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.311023:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.311024:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.311028:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.311029:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.311033:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.311033:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.311034:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.311036:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.311036:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.311037:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.311038:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.311042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.311049:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.311052:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.311052:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.311054:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.311055:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.311058:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.311059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.2:1619677448.311064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.311066:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.311069:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.311070:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.311071:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:10.0:1619677448.311072:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.311074:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.311074:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.311075:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:5.0:1619677448.311075:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.311075:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000400:00000200:10.0:1619677448.311076:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716622] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.311076:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.311077:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:10.0:1619677448.311078:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.311079:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.311079:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.311082:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.311084:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.311087:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.311087:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.311088:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.311089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:10.0:1619677448.311090:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.311091:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.311091:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.311092:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:18.0:1619677448.311095:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:9.2:1619677448.311096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.311098:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:18.0:1619677448.311100:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.311100:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.311102:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.311103:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.311104:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.311105:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.311105:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.311107:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.311107:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.311108:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.311110:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.311111:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.311114:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.311114:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.311115:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677448.311116:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.311119:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.311119:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.311120:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.311122:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.311122:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.311123:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.311123:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.311128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.311136:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.311138:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.311139:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.311139:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.311141:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.311143:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.311148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.311155:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.311157:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.311158:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.311159:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.311160:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.311160:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.311189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.311194:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.311196:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.311197:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.311199:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.311200:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.311201:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.311202:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.311204:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.311205:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.311206:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.311208:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.311210:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.311211:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.311212:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.311213:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.311214:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.311215:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.311215:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.311215:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.311220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.311227:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.311228:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.311229:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.311230:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.311230:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.311232:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.311329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.311337:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.311340:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.311341:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.311342:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.311345:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.311347:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.311347:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677448.311348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677448.311349:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.311350:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.311351:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:5.2:1619677448.311352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:24.0:1619677448.311354:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.311356:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.2:1619677448.311357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:26.0:1619677448.311358:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677448.311359:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.311360:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:24.0:1619677448.311360:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.311360:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.311361:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677448.311362:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.311362:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:10.0:1619677448.311362:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677448.311363:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.311364:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000400:00000200:10.0:1619677448.311364:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.311365:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:24.0:1619677448.311365:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000400:00000200:26.0:1619677448.311366:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.311366:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:30.0:1619677448.311368:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:10.0:1619677448.311368:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.311368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:26.0:1619677448.311369:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.311370:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.311371:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.311372:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:26.0:1619677448.311372:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.311372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000400:00000200:26.0:1619677448.311373:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.311374:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.311374:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677448.311375:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:10.0:1619677448.311376:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677448.311377:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.311377:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.311378:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.311378:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:30.0:1619677448.311379:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.311379:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:10.0:1619677448.311380:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.311380:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:30.0:1619677448.311381:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:10.0:1619677448.311381:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.311381:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:5.0:1619677448.311381:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.311382:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:10.0:1619677448.311383:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:8.0:1619677448.311383:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677448.311384:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.311384:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.311386:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.311387:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.311387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.311388:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.311389:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.311390:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:26.0:1619677448.311390:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.311390:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.311391:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.311391:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.311391:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.311392:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.311393:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.311393:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.311394:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.311394:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:26.0:1619677448.311394:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.311394:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677448.311395:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.311395:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.311395:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:30.0:1619677448.311396:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:26.0:1619677448.311396:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (23)++
00000800:00000200:20.0:1619677448.311396:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:10.0:1619677448.311396:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677448.311397:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677448.311397:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:10.0:1619677448.311397:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.0:1619677448.311397:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:30.0:1619677448.311398:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:26.0:1619677448.311398:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (24)--
00000800:00000200:26.0:1619677448.311398:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.311399:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.311400:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.311400:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.311400:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.311401:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.311401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.311401:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677448.311402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:4.2:1619677448.311403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.2:1619677448.311405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:25.2:1619677448.311408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.311409:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:10.0:1619677448.311410:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:31.0:1619677448.311411:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:2.0:1619677448.311411:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:31.0:1619677448.311412:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.311413:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.311413:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.311413:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.311414:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.311414:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677448.311414:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:30.0:1619677448.311415:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:18.0:1619677448.311415:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.311415:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.311416:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:31.0:1619677448.311417:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:22.0:1619677448.311417:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:2.0:1619677448.311417:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.311419:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.311419:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.311420:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.311421:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.311421:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:22.0:1619677448.311422:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.311423:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.311425:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.311426:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.311427:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.311428:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.311432:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.311433:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.311433:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.311434:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677448.311435:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.311438:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.311439:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.311440:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.311441:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.311442:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.311442:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.311443:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.311447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.311456:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.311459:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.311460:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.311460:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.311462:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.311464:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.311465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.311472:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.311474:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.311475:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.311476:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.311476:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.311477:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.311535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.311542:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.311545:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.311547:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.311549:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.311550:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.311552:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.311553:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.311554:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.311555:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.311556:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.311559:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.311561:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.311564:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.311565:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.311565:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.311567:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.311567:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.311568:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.311569:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.311572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.311580:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.311582:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.311583:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.311584:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.311587:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.311588:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.311693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.311700:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.311702:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.311703:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.311705:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.311708:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.311710:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.311711:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.311712:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.311713:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.311714:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.311717:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.311718:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.311721:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.311722:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.311722:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.311724:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.311725:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.311725:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.311726:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.311730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.311738:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.311741:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.311742:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.311742:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.311744:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.311747:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.311772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.311775:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.311777:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.311779:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.311781:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.311782:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.311783:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677448.311785:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.311786:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.311787:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.311788:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.311792:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.311794:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.311796:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.311797:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.311797:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.311799:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.311799:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.311800:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.311801:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677448.311805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.311814:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.311817:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.311817:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.311818:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.311820:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677448.311822:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.311957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.311964:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.311967:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.311969:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.311971:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.311972:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.311973:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.311975:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.311976:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.311977:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.311978:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.311982:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.311984:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.311986:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.311987:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.311987:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.311989:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.311989:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.311991:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.311991:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.311994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.312001:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.312003:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.312004:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.312004:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.312006:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.312009:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.312013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:4.2:1619677448.312016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.312022:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.312024:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.312025:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.312026:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677448.312027:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.312028:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.312030:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.312031:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.312032:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.312033:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.312034:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.312035:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.312038:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.312040:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.312040:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.312041:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.312042:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.312044:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.312044:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.312045:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.312047:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.312048:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.312049:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:10.0:1619677448.312049:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.312050:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.312051:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.312051:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.312052:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.312053:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.312054:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.312054:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.312055:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.312055:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.312055:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:25.2:1619677448.312056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.312058:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.312059:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.312060:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.312060:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.312061:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.312062:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.312063:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.312063:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.312064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:10.0:1619677448.312064:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.312066:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:6.0:1619677448.312071:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:25.2:1619677448.312073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:6.0:1619677448.312074:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.312075:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.312075:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.312077:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.312080:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:6.0:1619677448.312081:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:26.0:1619677448.312082:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.312083:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.312083:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.312084:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.312084:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.312115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:5.0:1619677448.312121:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:5.0:1619677448.312124:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:20.0:1619677448.312126:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:5.0:1619677448.312126:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.312128:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.312130:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677448.312132:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.312133:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.312134:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.312135:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.312138:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.312140:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.312143:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.312144:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677448.312144:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.312146:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.312147:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:5.0:1619677448.312147:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:5.0:1619677448.312148:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:25.2:1619677448.312152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.312158:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.312160:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.312161:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.312161:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.312164:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.312167:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.312170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.312176:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.312179:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.312180:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.312182:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.312184:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.312185:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.312186:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.312188:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.312189:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.312190:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.2:1619677448.312192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:18.0:1619677448.312193:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.312194:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.312196:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.312197:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.312197:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.312199:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.312200:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:18.0:1619677448.312200:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.312200:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.312201:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677448.312202:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.312203:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.312205:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.312205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677448.312208:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.312209:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.312211:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.312212:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.312212:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:26.0:1619677448.312213:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.312214:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.312214:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.312215:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.312216:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.312217:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000400:00000200:26.0:1619677448.312218:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.312220:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.312222:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:26.0:1619677448.312224:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677448.312224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.312225:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.312226:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.312228:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.312228:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.312229:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.312229:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:18.0:1619677448.312231:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.312235:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.312235:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:18.0:1619677448.312236:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (23)++
00000800:00000200:2.0:1619677448.312236:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (25)--
00000800:00000200:18.0:1619677448.312237:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (25)--
00000800:00000200:18.0:1619677448.312237:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.312238:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.312238:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.312239:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.312243:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.312313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.312319:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.312322:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.312322:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.312324:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.312326:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.312328:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.312329:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.312330:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.312331:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.312332:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.312336:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.312338:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.312340:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.312341:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.312342:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.312343:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.312344:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.312345:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.312345:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.312350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.312359:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.312361:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.312362:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.312362:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.312364:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.312367:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.312374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.312382:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.2:1619677448.312384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:26.0:1619677448.312385:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.312385:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.312387:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.312390:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.312390:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:10.0:1619677448.312392:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:18.0:1619677448.312393:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.312393:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:10.0:1619677448.312394:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.312396:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.312396:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.312397:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.312398:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677448.312399:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.312401:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.312403:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.312404:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.312405:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.312406:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:2.0:1619677448.312406:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.312407:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.312409:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.312410:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.312410:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:2.0:1619677448.312411:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.312412:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.312413:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.312413:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677448.312413:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.312414:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677448.312416:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.312417:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.312417:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:25.2:1619677448.312418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:2.0:1619677448.312419:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.312420:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.312420:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.312421:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:30.0:1619677448.312425:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:25.2:1619677448.312425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.312428:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.312429:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.312430:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.312432:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:13.0:1619677448.312432:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.312433:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.312435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:13.0:1619677448.312435:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.312436:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.312438:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.312440:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.312441:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.312442:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.312445:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.312445:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.312446:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:21.2:1619677448.312446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.312448:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.312449:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.312450:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.312453:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.312455:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.312457:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.312458:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.312459:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.312460:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716602] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.312461:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.312463:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.312463:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.312467:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.312469:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.312471:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.312472:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.312473:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.312474:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.312475:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.312476:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.312478:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.312478:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.312480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.312484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.312486:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.312488:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.312489:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.312490:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.312491:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.312492:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.312493:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.312494:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.312495:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.312496:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677448.312517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.312525:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.312527:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.312528:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.312530:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.312533:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.312534:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.312536:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.312537:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.312538:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.312539:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.312542:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.312543:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.312546:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.312547:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.312548:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.312549:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.312550:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.312551:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.312552:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.312553:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.312555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.312558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.312564:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677448.312565:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.312566:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.312567:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.312567:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.312568:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677448.312568:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677448.312568:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.312570:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.312573:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.312576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.312579:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.312581:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.312584:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.312585:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.312587:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.312588:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677448.312589:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.312591:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.312592:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.312592:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677448.312596:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.312598:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.312600:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.312601:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.312601:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.312603:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.312604:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.312605:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.312606:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.312607:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677448.312609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.312612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.312616:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.312618:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.312618:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.312618:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.312619:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:20.0:1619677448.312620:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.312620:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.312621:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.312622:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677448.312623:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.2:1619677448.312662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677448.312670:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677448.312673:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677448.312675:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.312677:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677448.312678:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.312679:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.312680:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.312682:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.312683:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.312684:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.312687:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.312688:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.312691:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.312692:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.312693:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.312695:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.312695:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677448.312697:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87b3e5800] (22)++
00000800:00000200:31.0:1619677448.312698:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:31.0:1619677448.312699:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:8.2:1619677448.312701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.312704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:5.0:1619677448.312708:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.312710:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.312711:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.312713:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)++
00000800:00000200:20.0:1619677448.312713:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.312713:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.312716:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:5.0:1619677448.312716:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677448.312717:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677448.312718:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:5.2:1619677448.312753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.312759:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.312762:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.312762:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.312765:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.312766:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.312768:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.312769:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.312770:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.312772:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.312772:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.312776:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.312778:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.312781:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.312781:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.312782:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.312782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677448.312785:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.312786:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.312786:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.312787:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677448.312788:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.312790:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677448.312790:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:25.2:1619677448.312791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:17.0:1619677448.312792:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.312794:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.312795:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677448.312796:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.312797:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:17.0:1619677448.312798:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.312798:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.312799:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:13.0:1619677448.312800:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.312801:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.312803:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677448.312803:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.312803:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.312806:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677448.312807:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.312809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.312809:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.312810:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:17.0:1619677448.312810:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:17.0:1619677448.312812:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:17.0:1619677448.312813:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.312814:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677448.312815:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:12.0:1619677448.312815:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.312816:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:25.2:1619677448.312818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:12.0:1619677448.312818:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.312819:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.312819:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.312820:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.312822:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.312822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:26.0:1619677448.312824:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.312827:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:1.0:1619677448.312827:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:10.0:1619677448.312828:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.312829:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.312830:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000400:00000200:26.0:1619677448.312831:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.312831:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677448.312832:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:26.0:1619677448.312834:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677448.312860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.312867:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.312870:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.312872:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.312874:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.312875:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.312876:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.312878:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.312879:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.312880:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.312881:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.312884:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.312886:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.312889:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.312889:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.312890:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.312891:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.312892:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.312893:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.312893:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.312897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.312903:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.312906:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.312906:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.312908:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.312910:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.312911:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.312948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.312955:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.312958:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.312959:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.312960:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.312963:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.312964:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.312965:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.312966:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.312967:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.312968:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.312971:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.312972:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.312974:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.312975:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.312975:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.312977:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.312977:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.312978:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.312979:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.312980:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.312983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.312985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.312989:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.312992:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.312992:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.312992:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677448.312994:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.312994:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.312995:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.312996:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.312996:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.312997:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.313140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.313146:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.313148:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.313149:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.313151:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.313154:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.313156:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.313157:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.313158:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.313159:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.313160:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.313164:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.313165:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.313169:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.313169:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.313170:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.313172:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.313172:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.313173:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.313174:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.313175:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.313177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.313179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677448.313182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.313184:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.313185:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:13.0:1619677448.313187:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.313187:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.313188:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:1.0:1619677448.313188:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:2.0:1619677448.313189:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:13.0:1619677448.313190:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.313190:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.313190:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.313190:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677448.313192:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.313192:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.313193:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:18.0:1619677448.313194:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.313196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:18.0:1619677448.313196:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.313197:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.313199:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.313200:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.313201:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.313203:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677448.313204:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.313206:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.313207:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.313209:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.313209:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.313210:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677448.313210:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.313211:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677448.313212:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.313213:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:18.0:1619677448.313213:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.313213:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:30.0:1619677448.313214:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.313214:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.313215:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:30.0:1619677448.313216:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.313217:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.313218:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:25.2:1619677448.313218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677448.313222:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.313223:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.313225:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.313226:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.313227:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:30.0:1619677448.313228:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (24)++
00000800:00000200:12.0:1619677448.313228:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677448.313229:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.313230:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677448.313230:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.313231:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.313232:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.313232:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.313232:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.313233:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:25.2:1619677448.313235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.313242:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.313244:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.313245:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.313246:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.313248:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.313249:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.313250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.313256:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.313259:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.313260:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.313260:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.313261:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.313261:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.313339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.313342:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.313344:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.313347:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.313348:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.313348:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.313350:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.313351:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.313352:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.313354:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.313354:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.313358:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.313360:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.313362:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.313363:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.313364:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.313366:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.2:1619677448.313367:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.0:1619677448.313371:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:25.2:1619677448.313372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.313372:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.313372:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677448.313376:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.313379:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:12.0:1619677448.313380:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:31.0:1619677448.313381:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.313382:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:31.0:1619677448.313383:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.313383:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.313384:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.313385:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:12.0:1619677448.313385:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.313387:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.313387:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:31.0:1619677448.313388:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.313388:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677448.313389:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.313390:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.313394:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.313395:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.313397:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.313398:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.313399:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.313401:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.313401:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.313403:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.313404:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.313405:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.313406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.2:1619677448.313410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.313415:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.313416:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.313417:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.313419:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.313419:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:30.0:1619677448.313419:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.313420:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.313421:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.313422:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.313423:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:21.2:1619677448.313505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.313513:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.313515:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.313516:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.313518:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.313521:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.313522:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:5.2:1619677448.313522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:8.0:1619677448.313523:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.313524:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.313525:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.313526:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.313529:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677448.313529:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.313531:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:8.0:1619677448.313531:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.313532:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.313534:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.313534:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.313535:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.313535:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.313537:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.313537:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:17.0:1619677448.313538:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.313538:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:17.0:1619677448.313539:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:8.0:1619677448.313539:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:17.0:1619677448.313540:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.313542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:17.0:1619677448.313542:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.313543:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.313544:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.313547:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.313549:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.313549:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.313551:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:17.0:1619677448.313552:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.313552:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:13.0:1619677448.313552:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.313553:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:13.0:1619677448.313554:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.313555:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.313555:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.313556:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.313556:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.313557:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.313557:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.313562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.313569:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.313571:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.313572:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.313574:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.313575:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.313577:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.313592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.313598:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.313600:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.313602:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677448.313603:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.313605:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.313606:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.313608:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716587] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.313609:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.313610:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.313611:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.313614:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.313616:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.313618:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.313619:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.313620:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.313621:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.313622:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.313622:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.313623:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677448.313628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.313636:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:21.2:1619677448.313638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677448.313638:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.313639:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.313640:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.313641:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.313644:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677448.313646:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.313648:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.313651:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.313652:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.313653:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.313655:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.313656:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.313658:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.313659:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.313660:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.313663:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.313665:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.313667:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.313668:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.313669:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.313670:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.313671:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.313672:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.313672:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.313676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.313682:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.313685:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.313686:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.313687:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.313689:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.313690:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.313696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677448.313703:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677448.313706:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677448.313708:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.313710:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677448.313710:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.313711:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.313713:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.313714:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.313715:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.313716:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.313719:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.313721:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.313724:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.313725:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.313725:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.313727:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.313728:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677448.313729:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:30.0:1619677448.313731:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:30.0:1619677448.313732:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:9.2:1619677448.313733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.313737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:18.0:1619677448.313741:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.313744:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:31.0:1619677448.313745:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)++
00000400:00000200:18.0:1619677448.313745:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.313745:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.313747:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc2e00] (23)--
00000400:00000200:18.0:1619677448.313747:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.313748:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677448.313749:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:18.0:1619677448.313750:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.313786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.313793:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.313796:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.313797:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.313799:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.313801:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.313802:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.313803:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.313805:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.313806:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.313806:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.313809:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.313811:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.313812:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.313813:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.313813:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.313815:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.313815:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.313816:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.313817:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.313821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.313827:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.313828:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.313829:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.313830:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.313831:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.313832:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.313856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.313859:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.313861:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.313863:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.313864:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.313866:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.313867:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.313868:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.313870:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.313871:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.313872:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.313875:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.313876:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.2:1619677448.313877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677448.313879:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.313880:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.313880:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.313882:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.313882:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.313883:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677448.313883:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.313884:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677448.313886:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.313887:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.313888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.2:1619677448.313890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:1.0:1619677448.313890:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.313893:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.313894:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.313895:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.313896:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:1.0:1619677448.313897:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.313898:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.313898:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:1.0:1619677448.313898:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.313899:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.313899:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.313900:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.313901:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.313901:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:6.0:1619677448.313902:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.313904:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.313905:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.313905:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:10.0:1619677448.313907:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.313907:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:25.2:1619677448.313908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000400:00000200:10.0:1619677448.313909:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.313910:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.313910:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.313911:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.313911:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.313911:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:10.0:1619677448.313913:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.313913:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.313913:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.313914:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.313915:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677448.313915:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.313915:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:10.0:1619677448.313917:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.313918:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:25.2:1619677448.313919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677448.313919:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.313919:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.313920:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.313920:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.313921:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677448.313922:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.313923:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:10.0:1619677448.313924:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677448.313926:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)++
00000800:00000200:10.0:1619677448.313926:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:10.0:1619677448.313927:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.313928:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.313928:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.313929:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677448.313930:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.313931:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.313933:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677448.313934:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677448.313934:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677448.313935:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.313935:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677448.313936:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.313938:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.313942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:12.0:1619677448.313948:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.313951:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.313952:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.313953:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.313953:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.313955:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.314009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.314015:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.314018:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.314019:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.314021:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.314023:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.314024:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.314025:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.314027:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.314028:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.314028:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.314031:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.314033:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.314035:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.314036:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.314036:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.314037:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.314038:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.314039:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.314039:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.314043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.314049:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.314052:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.314052:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.314054:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.314055:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.314056:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.314124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.314132:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.314134:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.314136:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.314138:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.314139:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.314140:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.314141:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.314143:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.314144:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.314144:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.314147:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.314149:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.314151:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.314152:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.314152:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.314154:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.314154:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.314155:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.314156:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.314159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.314166:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.314169:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.314170:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.314171:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.314172:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.314173:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.314194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.2:1619677448.314199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:31.0:1619677448.314201:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.314203:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:2.0:1619677448.314204:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:31.0:1619677448.314205:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.314207:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.314207:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677448.314207:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.314208:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:31.0:1619677448.314209:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.314210:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.314211:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716578] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.314212:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.314213:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.314213:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.314214:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.314215:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.314216:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.314218:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.314219:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.314220:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.314221:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.314223:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.314225:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.314226:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.314226:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.314227:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.314227:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.314228:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.314229:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.314230:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:2.0:1619677448.314230:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.314231:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.314231:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.314232:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.314232:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677448.314233:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.314233:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:25.2:1619677448.314234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.314235:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.314236:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.314236:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:21.2:1619677448.314238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.2:1619677448.314239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677448.314240:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.2:1619677448.314242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677448.314243:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.314243:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.314245:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.314246:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.314246:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.314247:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.314248:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:18.0:1619677448.314248:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:30.0:1619677448.314249:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.314250:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677448.314250:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:5.0:1619677448.314250:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:25.2:1619677448.314251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.314251:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.314252:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677448.314252:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.314253:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.314255:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.314255:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.314258:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.314258:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:13.0:1619677448.314261:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.314262:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.314262:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.314263:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.314264:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.314285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.314290:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.314294:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.314299:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677448.314302:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.314304:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.314306:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.314307:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.314309:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.314310:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.314311:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.314315:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.314317:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.2:1619677448.314319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677448.314320:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.314320:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.314321:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.314322:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677448.314323:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.314324:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:17.0:1619677448.314324:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.314324:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.314325:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:20.0:1619677448.314326:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.314327:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.314329:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.314330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:20.0:1619677448.314330:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.314331:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.314333:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.314334:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.314335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:20.0:1619677448.314335:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.314337:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:20.0:1619677448.314338:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.314340:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677448.314340:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.314341:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.314341:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:30.0:1619677448.314343:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.314343:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.314344:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.314344:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:31.0:1619677448.314345:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.314345:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.314345:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:30.0:1619677448.314346:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.0:1619677448.314347:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.314347:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:2.0:1619677448.314347:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.314348:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.314349:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:2.0:1619677448.314350:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.314352:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:25.2:1619677448.314353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:2.0:1619677448.314353:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.314355:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.314356:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.314357:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.314360:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:2.0:1619677448.314360:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.314362:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.314363:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.314363:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:12.0:1619677448.314365:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.314366:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.314367:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.314367:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.314368:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.314368:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:2.0:1619677448.314370:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.314371:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.314371:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.314372:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.314376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.314382:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.314385:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.314386:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.314388:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.314389:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.314390:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.314514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.314521:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.314524:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.314525:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.314527:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.314529:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.314531:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.314532:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716573] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.314533:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.314534:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.314535:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.314539:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.314540:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.314544:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.314544:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.314545:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.314547:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.314547:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.314548:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.314548:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.314553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.314561:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.314564:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.314565:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.314566:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.314568:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.314571:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.314575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.314580:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.314583:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.314584:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.314586:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.314589:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.314590:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.314592:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716572] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.314593:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.314594:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.314595:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.314598:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.314599:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.314601:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.314602:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:21.2:1619677448.314603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:1.0:1619677448.314603:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.314604:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.314605:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.314606:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.314607:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.314607:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677448.314608:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.314611:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:8.2:1619677448.314611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677448.314612:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.314613:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.314614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677448.314616:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.314618:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.314619:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.314620:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.314620:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:22.0:1619677448.314621:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.314621:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.314621:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:22.0:1619677448.314622:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.314622:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.314622:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.314623:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.314624:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.314624:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.314625:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677448.314626:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:22.0:1619677448.314627:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.314629:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.314631:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.314632:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.314633:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.314635:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.314636:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.314636:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.314637:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.314641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.314648:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:21.2:1619677448.314649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:31.0:1619677448.314651:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.314652:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.314654:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.314654:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:30.0:1619677448.314655:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.314657:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677448.314657:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677448.314658:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677448.314660:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.314663:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.314665:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.314666:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.314668:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.314669:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.314670:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.314673:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.314675:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.314678:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.314679:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.314680:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.314681:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.314682:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.314683:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677448.314683:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:25.2:1619677448.314687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.314693:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.314696:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.314697:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.314697:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.314699:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.314702:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.314704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.314711:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.314714:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.314715:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.314717:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.314719:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.314721:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.314722:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.314724:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.314724:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.314726:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.314729:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.314731:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.314733:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.314734:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.314734:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.314736:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.314736:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.314737:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.314739:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.314739:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.314741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.314745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.314748:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.314750:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.314751:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.314752:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.314752:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:13.0:1619677448.314754:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.314754:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.314754:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:12.0:1619677448.314755:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.314756:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.2:1619677448.314859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.314865:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.314867:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.314868:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.314870:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.314871:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.314873:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.314874:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.314876:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.314877:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.314878:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.314881:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.314882:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.314885:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.314886:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.314887:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.314889:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.314889:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.314890:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.314890:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.314895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.314901:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.314904:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.314904:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.314905:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.314907:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.314910:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.314915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677448.314920:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.314923:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.314924:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.314924:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.314925:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.314925:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.314932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.314938:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.314940:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.314942:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.314944:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.314947:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.314948:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.314949:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.314951:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.314952:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.314953:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.314956:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.314958:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.314961:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.314962:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.314963:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.314964:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.314965:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.314966:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.314966:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.314970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.314977:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.314980:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.314981:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.314983:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.314984:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.314985:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.315038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.315046:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.315049:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.315051:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.315053:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.315054:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.315055:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.315057:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.315058:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.315059:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.315060:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.315063:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.315064:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.315068:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.315069:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.315069:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.315071:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.315072:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.315073:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.315074:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.315077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.315086:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.315089:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.315090:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.315092:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.315092:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.315095:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.315142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.315150:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.315152:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.315153:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.315155:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.315158:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.315159:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.315160:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.315161:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.315162:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.315163:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.315166:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.315167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.315168:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.315170:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.315171:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.315171:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.315173:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.315173:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.315173:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.315174:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.315174:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677448.315175:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.315176:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.315178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:17.0:1619677448.315179:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.315182:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.315183:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.315184:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.315184:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:17.0:1619677448.315185:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.315186:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.315187:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:12.0:1619677448.315187:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.315188:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.315191:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.315191:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677448.315191:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.315193:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677448.315194:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.2:1619677448.315196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.315196:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.315196:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.315197:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.315199:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.315199:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.315200:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.315201:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.315202:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.2:1619677448.315205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:13.0:1619677448.315206:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.315207:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.315207:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.315208:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.315209:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677448.315213:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.315215:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.315215:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.315216:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.315218:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.315219:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.315223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.315230:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:21.2:1619677448.315232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:17.0:1619677448.315233:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.315234:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.315234:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.315235:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.315236:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:6.0:1619677448.315238:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.315240:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.315241:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.315244:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.315246:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.315248:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.315249:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.315251:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.315252:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.315253:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.315257:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.315259:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.315262:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.315263:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.315264:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.315266:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.315266:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.315268:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.315269:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.315269:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:25.2:1619677448.315272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:21.2:1619677448.315275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677448.315278:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677448.315280:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:31.0:1619677448.315281:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.315282:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.315282:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.315282:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.315283:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:31.0:1619677448.315284:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.315285:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.315287:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.315288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.2:1619677448.315289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.315291:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.315298:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.315300:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:5.0:1619677448.315300:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677448.315302:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.315302:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.315303:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.315303:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.315304:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.315305:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:20.0:1619677448.315305:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.315307:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677448.315308:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.315309:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.315310:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.315311:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677448.315314:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.315316:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677448.315318:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.315319:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677448.315320:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.315321:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677448.315322:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.315323:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.315325:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.315325:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677448.315328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677448.315330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.315336:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677448.315336:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:5.0:1619677448.315339:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:1.0:1619677448.315339:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.315340:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.315340:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.315340:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677448.315340:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.315343:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.315345:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.315348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.315356:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.315359:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.315360:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.315361:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.315364:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.315366:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.315367:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.315368:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.315369:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.315370:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.315373:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.315375:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.315378:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.315378:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.315379:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.315381:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.315381:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.315382:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.315383:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.315387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.315394:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.315397:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.315398:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.315399:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.315400:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.315403:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.315445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.315451:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.315454:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.315455:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.315457:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.315460:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.315461:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.315462:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.315464:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.315465:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.315466:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.315469:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.315471:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.315474:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.315475:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.315476:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.315477:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.315478:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.315479:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.315479:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.315484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.315491:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.315494:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.315495:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.315497:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.315499:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.315500:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.315500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:25.2:1619677448.315503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.315505:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.315508:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.315509:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.315510:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:17.0:1619677448.315511:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.315512:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.315513:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.315513:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:17.0:1619677448.315514:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:13.0:1619677448.315514:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:17.0:1619677448.315515:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.315515:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.315516:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:17.0:1619677448.315517:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.315518:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.315519:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.315522:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.315524:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.315526:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.315526:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.315527:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.315529:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.315529:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.315530:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.315530:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.315534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.315541:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.315543:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.315544:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.315545:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.315546:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.315547:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.315554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.315561:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.315563:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.315564:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.315564:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.315565:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.315566:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.315566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.315574:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.315576:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.315577:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.315579:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.315582:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.315584:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.315585:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.315586:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.315587:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.315588:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.315592:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.315594:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.315596:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.315597:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.315597:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.315599:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.315599:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.315600:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.315600:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.315605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.315611:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.315613:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.315614:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.315616:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.315618:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.315618:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.315640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.315648:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.315651:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.315653:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.315655:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.315656:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.315657:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.315658:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.315659:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.315660:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.315660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:31.0:1619677448.315661:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.315664:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.315665:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677448.315666:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.315668:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.315668:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:31.0:1619677448.315669:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.315669:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677448.315669:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:31.0:1619677448.315671:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.315671:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:2.0:1619677448.315671:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677448.315672:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.315673:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:2.0:1619677448.315673:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.315674:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677448.315675:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.315676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:2.0:1619677448.315677:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.315678:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.315678:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677448.315681:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:12.0:1619677448.315683:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677448.315683:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.315684:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.315685:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:2.0:1619677448.315685:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:12.0:1619677448.315686:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.315686:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:12.0:1619677448.315687:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.315687:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.315688:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:13.0:1619677448.315689:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677448.315689:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.315690:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.315690:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:12.0:1619677448.315691:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677448.315693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.315696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.315702:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.315703:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.315703:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:18.0:1619677448.315704:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.315704:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.315705:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.315706:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.315706:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.315707:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677448.315708:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.315790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.315798:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.315801:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.315803:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.315805:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.315805:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.315807:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.315808:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.315809:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.315810:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.315811:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.315814:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.315816:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.315819:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.315820:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.315821:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.315822:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.315823:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.315824:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.315825:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.315828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.315836:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.315839:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.315840:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.315840:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.315842:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.315845:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.315846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.315853:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.315856:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.315857:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.315858:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.315858:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.315859:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.316021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.316028:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.316031:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.316032:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.316034:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.316036:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.316037:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.316039:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.316040:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.316041:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.316042:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.2:1619677448.316043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:10.0:1619677448.316045:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.316046:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.316049:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.316049:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.316050:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.316050:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.316051:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.316052:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:10.0:1619677448.316052:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.316053:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.316054:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:6.0:1619677448.316054:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.316055:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.316056:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.316057:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.316058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.316059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:6.0:1619677448.316059:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.316060:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.316062:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677448.316062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677448.316063:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.316064:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.316067:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.316068:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:22.0:1619677448.316068:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.316068:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.316069:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.316070:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.316071:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.316071:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.316072:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.316072:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.316072:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.316073:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.316074:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.316074:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.316075:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.316075:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.316076:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.316076:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.316077:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:26.0:1619677448.316078:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.316078:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.316079:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:26.0:1619677448.316080:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:6.0:1619677448.316080:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:26.0:1619677448.316081:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.316081:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:26.0:1619677448.316083:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.316084:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.316084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000400:00000200:26.0:1619677448.316085:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.316089:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.316091:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.316091:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.316092:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.316094:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.316094:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.316095:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.316096:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:10.0:1619677448.316096:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.316098:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.316099:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.316099:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.316101:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.316101:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:10.0:1619677448.316101:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:8.2:1619677448.316104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.316112:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.316114:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.316115:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.316117:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.316118:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.316120:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.316143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.316148:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.316151:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.316153:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.316154:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.316156:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.316157:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.316159:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.316160:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.316161:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.316162:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.316166:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.316167:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.316170:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.316171:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.316171:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.316173:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.316174:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.316175:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.316176:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.316180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.316186:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.316189:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.316190:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.316192:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.316193:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.316195:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.316209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.316217:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.316220:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.316222:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.316224:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.316224:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.316226:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.316227:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.316228:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.316229:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.316230:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.316233:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.316235:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.316237:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.316238:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.316239:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.316240:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.316241:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.316242:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.316244:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.316244:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.316246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.316250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677448.316253:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.316255:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.316256:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.316257:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.316258:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:12.0:1619677448.316259:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.316260:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:13.0:1619677448.316260:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.316261:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.316262:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.2:1619677448.316477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.316485:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:4.2:1619677448.316485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:10.0:1619677448.316488:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.316489:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.316490:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.316493:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.316493:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677448.316494:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.316496:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.316496:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.316497:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:10.0:1619677448.316497:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.316498:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.316499:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:8.0:1619677448.316499:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.316501:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.316502:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.316503:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:10.0:1619677448.316504:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:8.0:1619677448.316504:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96716548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.316506:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.316506:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.316507:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.316508:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.316508:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.316509:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.316511:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.316511:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:8.0:1619677448.316511:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.316512:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.316512:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677448.316512:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.316515:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.316516:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.316516:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:25.2:1619677448.316517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:8.0:1619677448.316518:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.316518:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.316519:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.316519:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.316524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.316526:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.316528:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.316529:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.316530:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.316532:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.316532:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677448.316534:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677448.316534:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.316535:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.316536:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.316538:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.316540:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.316646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.316653:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.316656:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.316657:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.316658:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.316661:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.316662:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.316664:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716547] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.316665:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.316666:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.316667:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.316670:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.316671:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.316674:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.316674:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.316675:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.316677:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.316677:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.316678:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.316678:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.316683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.316690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:6.0:1619677448.316691:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.316693:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.316694:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.316695:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.316697:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:6.0:1619677448.316697:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.316700:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:6.0:1619677448.316700:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677448.316701:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.316703:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.316706:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.316707:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.316709:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.316710:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.316711:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.316712:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677448.316715:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.316716:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.316719:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.316720:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.316721:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.316722:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.316723:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.316724:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.316724:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.316728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.316736:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.316739:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.316740:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.316741:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.316742:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.316744:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.316746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.316754:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.316756:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.316757:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.316758:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.316758:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.316759:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.316897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.316902:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.316905:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.316907:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677448.316908:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.316909:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.316911:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.316912:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.316913:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.316914:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.316915:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.316919:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.316920:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.316923:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.316923:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.316924:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.316925:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.316926:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.316927:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.316928:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.316929:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677448.316931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677448.316934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.316937:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.316939:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.316939:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:17.0:1619677448.316939:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677448.316940:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677448.316941:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.316942:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.316943:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.316944:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677448.316947:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.317103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.317109:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.317112:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.317113:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.317115:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.317117:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.317119:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.317120:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.317121:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.317122:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.317123:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.317127:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.317128:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.317131:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.317131:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.317132:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.317134:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.317134:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.317135:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.317135:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.317140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.317147:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.317149:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.317150:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.317150:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.317152:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.317154:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.317220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.317227:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.317230:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.317230:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.317232:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.317236:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.317237:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.317239:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.317240:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.317241:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.317242:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.317245:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.317247:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.317250:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.317250:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.317251:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.317253:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.317254:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.317254:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.317255:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677448.317255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:9.2:1619677448.317259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.317260:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.317263:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.317264:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.317265:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.317267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677448.317267:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:17.0:1619677448.317267:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.317269:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:18.0:1619677448.317270:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:17.0:1619677448.317270:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.317271:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.317271:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.317271:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:17.0:1619677448.317272:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.317273:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.317273:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677448.317275:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.317277:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:17.0:1619677448.317277:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.317278:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677448.317279:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:31.0:1619677448.317280:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.317282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677448.317282:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.317282:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:31.0:1619677448.317283:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.317283:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677448.317283:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:31.0:1619677448.317285:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:30.0:1619677448.317285:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677448.317285:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.317285:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.317286:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.317286:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.317287:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.317287:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:31.0:1619677448.317288:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.317288:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.317288:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677448.317289:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:25.2:1619677448.317291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:30.0:1619677448.317292:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.317293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.317300:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.317300:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.317300:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:30.0:1619677448.317301:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.317302:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677448.317302:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:30.0:1619677448.317303:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:22.0:1619677448.317303:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677448.317303:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677448.317304:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96716540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.317304:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.317304:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.317304:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677448.317306:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.317307:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.317307:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.317308:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.317309:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.317310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677448.317311:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.317312:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.317312:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:30.0:1619677448.317312:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.317313:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.317314:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.317315:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.317315:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.317316:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677448.317316:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.317316:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.317317:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.317318:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.317318:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677448.317318:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.317319:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.317320:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.317320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.317321:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:6.0:1619677448.317321:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.317322:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.317322:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.317323:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:25.2:1619677448.317324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677448.317324:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:26.0:1619677448.317327:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.317330:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.317331:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.317331:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:10.0:1619677448.317331:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.317333:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.317334:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.317334:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.317336:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:12.0:1619677448.317336:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.317338:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.317339:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.317340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.317346:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.317349:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.317349:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.317350:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.317351:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.317351:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.317414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.317420:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.317423:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.317423:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.317425:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.317428:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.317430:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.317431:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.317432:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.317433:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.317434:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.317438:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.317440:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.317442:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.317443:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.317444:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.317445:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.317446:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.317447:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.317447:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.317452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.317460:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.317462:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.317463:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.317464:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.317465:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.317468:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.317479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.317484:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.317487:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.317488:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.317490:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.317491:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.317493:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.317494:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.317495:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.317496:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.317497:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.317500:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.317502:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.317504:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.317505:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.317505:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.317507:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.317508:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.317508:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.317509:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.317513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.2:1619677448.317518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:6.0:1619677448.317521:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.317523:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.317524:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.317524:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.317526:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:6.0:1619677448.317526:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:26.0:1619677448.317528:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.317529:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677448.317529:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:26.0:1619677448.317531:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.317531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000400:00000200:26.0:1619677448.317534:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.317535:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.317536:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.317538:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.317539:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.317539:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:26.0:1619677448.317540:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:22.0:1619677448.317542:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.317542:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:26.0:1619677448.317543:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.317543:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.317544:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.317545:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:26.0:1619677448.317546:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.317548:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.317549:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.317549:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.317550:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.317551:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.317552:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.317553:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.317554:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.317556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.2:1619677448.317559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.317564:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.317565:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.317566:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.317568:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:10.0:1619677448.317568:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.317569:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.317569:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.317570:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.317570:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.317571:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:4.2:1619677448.317637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677448.317641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:26.0:1619677448.317644:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.317647:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:17.0:1619677448.317647:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:10.0:1619677448.317647:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.317650:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.317650:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677448.317650:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:26.0:1619677448.317652:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.317653:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.317654:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.317655:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.317655:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.317656:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.317657:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.317658:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.317659:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.317660:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677448.317661:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.317662:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.317664:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.317666:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.317668:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.317669:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.317671:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.317672:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.317673:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.317673:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.317674:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.317674:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.317675:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677448.317675:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.317676:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.317677:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.317677:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677448.317677:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:17.0:1619677448.317678:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.317679:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677448.317679:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677448.317682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:25.2:1619677448.317683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.317689:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.317690:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:2.0:1619677448.317692:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:10.0:1619677448.317693:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.317694:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:18.0:1619677448.317694:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:10.0:1619677448.317694:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.317694:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.317697:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.317698:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.317699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.317702:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:2.0:1619677448.317703:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.317704:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.317707:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.317707:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.317708:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.317709:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.317710:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.317843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.317848:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.317851:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.317852:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.317854:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.317856:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.317858:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.317859:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.317860:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.317861:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.317862:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.317865:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.317866:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.317869:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.317869:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.317870:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.317872:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.317872:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.317873:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.317874:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.317878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.317885:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.317888:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.317889:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.317889:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.317891:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.317893:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.318036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.318038:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.318041:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.318043:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.318044:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.318045:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.318047:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.318048:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.318050:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.318051:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677448.318052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:20.0:1619677448.318053:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.318057:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.318059:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.318059:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.318062:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:20.0:1619677448.318062:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.318063:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.318063:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.318064:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.318064:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.318066:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.318066:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677448.318067:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.318067:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677448.318068:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:20.0:1619677448.318068:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:26.0:1619677448.318070:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.318071:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.318072:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.318072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:26.0:1619677448.318073:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.318075:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.318075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:26.0:1619677448.318077:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677448.318078:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677448.318080:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.318081:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.318081:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.318081:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677448.318081:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.318081:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.318083:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:12.0:1619677448.318083:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.318084:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.318084:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:13.0:1619677448.318084:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.318085:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:2.0:1619677448.318085:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:26.0:1619677448.318086:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.318086:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:12.0:1619677448.318086:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:18.0:1619677448.318087:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.318089:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.318089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677448.318091:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:4.2:1619677448.318092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:18.0:1619677448.318093:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.318094:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.318095:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.318097:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.318098:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.318099:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:2.0:1619677448.318100:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.318101:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.318101:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:2.0:1619677448.318101:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.318102:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.318102:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677448.318103:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.318104:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.318106:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.318106:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.318107:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.318107:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677448.318108:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.318109:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.318110:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.318111:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.318111:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.318116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.318124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677448.318124:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.318126:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.318127:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.318127:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.318129:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.318131:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677448.318132:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.318135:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.318137:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.318139:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.318139:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677448.318139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000400:00000200:31.0:1619677448.318141:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.318142:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.318143:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.318144:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.318146:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.318146:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.318149:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.318150:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.318150:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.318150:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.318151:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.318152:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.318152:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677448.318154:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.318155:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.318156:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.318157:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.318158:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.318159:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.318159:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.318163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.318169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:13.0:1619677448.318169:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.318172:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.318173:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.318174:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:13.0:1619677448.318174:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.318175:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.318176:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677448.318177:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.318178:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.318180:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.318182:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.318183:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.318185:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.318186:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.318187:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.318188:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.318191:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.318193:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.318195:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.318196:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.318197:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.318198:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.318199:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.318199:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.318200:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.318204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.318212:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.318214:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.318215:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.318215:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.318218:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.318220:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.318300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.318307:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.318310:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.318311:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.318312:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.318316:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.318317:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.318318:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.318320:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.318321:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.318322:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.318325:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.318327:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.318329:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.318330:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.318331:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.318332:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.318333:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.318334:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.318334:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.318338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.318347:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.318349:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.318350:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.2:1619677448.318351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:1.0:1619677448.318351:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.318353:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.318356:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677448.318357:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.318359:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.318360:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.318362:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.318364:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.318365:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.318366:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.318367:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.318369:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.318370:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.318373:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.318375:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.318377:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.318378:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.318379:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.318381:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.318381:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.318382:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.318382:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.318387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.318394:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.318397:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677448.318397:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677448.318398:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.318400:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.318403:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677448.318405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:8.0:1619677448.318411:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.318413:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677448.318414:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.318414:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.318415:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.318415:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677448.318501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.318509:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.318511:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.318512:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.318514:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.318517:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.318519:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.318520:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.318521:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.318522:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.318523:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.318526:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.318528:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.318530:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.318531:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.318532:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.318533:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.318534:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.318534:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.318535:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.318539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.318547:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.318550:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.318550:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.318552:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.318554:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.318555:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.318636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.318642:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.318646:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.318646:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.318648:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.318650:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.318652:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.318653:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.318655:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.318656:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.318657:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.318660:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.318662:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.318665:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.318666:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.318667:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.318668:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.318669:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.318670:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.318670:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.318674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.318682:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.318685:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.318686:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.318687:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.318689:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:21.2:1619677448.318690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:12.0:1619677448.318690:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.318698:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.318701:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.318703:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.318705:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.318705:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.318706:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.318708:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.318710:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.318711:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.318712:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.318715:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.318717:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.318719:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.318720:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.318721:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.318722:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.318723:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.318724:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.318726:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.318726:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.318728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.318732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.318734:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.318736:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.318737:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.318738:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.318739:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.318740:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.318741:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:12.0:1619677448.318741:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.318742:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.318743:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.318747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.318753:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.318755:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.318756:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.318757:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.318757:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.318759:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.318785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.318791:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.318793:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.318796:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.318796:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.318799:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.318800:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677448.318802:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.318803:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.318804:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.318805:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677448.318809:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.318811:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.318814:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.318814:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.318815:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.318817:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.318817:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.318818:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.318819:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.318823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.318830:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.318832:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.318833:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.318835:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.318836:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.318837:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.318842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.318850:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.318853:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.318855:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.318857:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.318858:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.318859:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.318860:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.318861:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.318863:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.318863:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.318867:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.318868:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.318871:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.318872:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.318872:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.318874:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.318874:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.318875:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.318876:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.318879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.318886:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.318888:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.318889:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.318890:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.318892:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.318894:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.318898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:4.2:1619677448.318899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677448.318904:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.318907:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:12.0:1619677448.318907:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.318907:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.2:1619677448.318907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677448.318908:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.318908:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.318909:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:12.0:1619677448.318909:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.318910:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.318912:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.318912:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.318914:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.318915:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:1.0:1619677448.318915:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.318916:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.318918:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.318919:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.318919:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.318920:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.318921:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.318922:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.318923:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.318925:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.318926:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.318927:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.318928:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.318929:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.318929:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:26.0:1619677448.318931:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.318932:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.318932:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:1.0:1619677448.318933:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.318934:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.318934:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.318935:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.318936:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677448.318936:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.318938:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677448.318939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.318939:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.318940:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.318941:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.318942:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.318943:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.318943:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.318946:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:9.2:1619677448.318947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:31.0:1619677448.318949:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.318950:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.318952:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.318953:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:24.0:1619677448.318953:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:31.0:1619677448.318954:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:24.0:1619677448.318956:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.318957:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.318957:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677448.318959:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.318961:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.2:1619677448.318980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.318988:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.318990:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.318991:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.318993:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.318996:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.318997:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.318998:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.319000:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.319000:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.319002:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.319005:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.319007:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.319009:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.319010:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.319011:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.319012:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.319013:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.319013:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.319014:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677448.319018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.319024:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.319027:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.319027:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.319029:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.319031:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.2:1619677448.319031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:13.0:1619677448.319032:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677448.319037:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677448.319039:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677448.319040:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677448.319042:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.319045:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.319047:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:17.0:1619677448.319048:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.319050:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.319051:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.319052:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:17.0:1619677448.319055:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.319057:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:17.0:1619677448.319059:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.319060:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:17.0:1619677448.319061:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:17.0:1619677448.319062:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:17.0:1619677448.319063:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677448.319064:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677448.319064:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:9.2:1619677448.319068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.319076:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.319078:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.319080:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.319080:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.319082:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.319084:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.319188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.319194:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.319196:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.319199:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.319199:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.319201:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.319203:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.319204:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.319205:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.319206:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.319207:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.319211:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.319213:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.319216:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.319216:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.319217:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.319219:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.319220:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.319221:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.319222:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.319223:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.319225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.319228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.319231:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677448.319231:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.319233:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.319234:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.319234:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.319234:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.319235:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.319236:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.319238:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.319239:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.319281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.319289:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.319291:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.319292:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.319300:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.319302:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.319303:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.319304:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.319306:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.319307:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.319308:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.319311:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.319312:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.319315:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.319315:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.319316:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.319318:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.319318:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.319318:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.319319:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.319324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.319332:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.319334:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.319335:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.319336:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.319338:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.319339:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.319342:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.319343:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.319344:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.319344:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.319346:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.319400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.319407:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.319410:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.319412:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.319414:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.319414:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.319415:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.319416:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.319417:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.319418:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.319419:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.319422:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.319424:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.319427:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.319428:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.319428:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.319430:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.319431:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.319431:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.319432:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.319436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.319444:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.319447:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.319447:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.319448:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.319450:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.319454:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.319456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.319464:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.2:1619677448.319466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:31.0:1619677448.319467:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.319469:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.319471:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.319472:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.319472:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:31.0:1619677448.319473:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.319474:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.319476:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.319476:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.319476:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:31.0:1619677448.319479:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.319479:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.319480:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677448.319482:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.319484:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.319484:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.319486:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.319487:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:2.0:1619677448.319487:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.319488:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.319489:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.319490:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:2.0:1619677448.319490:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.319491:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.319492:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.319493:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.319493:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:2.0:1619677448.319493:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.319494:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:2.0:1619677448.319495:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:25.2:1619677448.319498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:2.0:1619677448.319498:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.319499:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.319499:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.319500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.319501:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.319501:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.319503:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:30.0:1619677448.319504:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:2.0:1619677448.319504:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.319504:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677448.319506:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.319507:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.319508:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.319508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:1.0:1619677448.319508:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:30.0:1619677448.319509:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.319509:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.319509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677448.319510:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:1.0:1619677448.319511:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.319512:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.319514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:1.0:1619677448.319514:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.319515:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.0:1619677448.319516:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677448.319516:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:1.0:1619677448.319516:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.319517:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.319518:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:1.0:1619677448.319518:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.319519:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.319519:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.319519:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:1.0:1619677448.319519:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.319520:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.0:1619677448.319521:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:20.0:1619677448.319522:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.319522:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:1.0:1619677448.319523:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.319524:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.319525:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677448.319525:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677448.319526:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.2:1619677448.319527:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:1.0:1619677448.319528:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.319528:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.319529:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.319531:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.319531:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677448.319532:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.319533:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:8.0:1619677448.319534:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.319534:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.319534:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.319535:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.2:1619677448.319537:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.319537:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.2:1619677448.319539:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.319540:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:8.0:1619677448.319540:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.319542:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96716511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.319543:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.319543:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.319543:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:8.0:1619677448.319544:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.319544:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677448.319545:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:8.0:1619677448.319545:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.319546:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.319546:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.319546:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:17.0:1619677448.319547:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.319547:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.319548:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:8.0:1619677448.319549:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.319549:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:17.0:1619677448.319550:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.319551:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.319552:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.0:1619677448.319554:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.319555:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.319555:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.319557:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.319558:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.319559:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.319560:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.319561:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.2:1619677448.319562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:21.2:1619677448.319567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:13.0:1619677448.319570:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.319572:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:24.0:1619677448.319573:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:13.0:1619677448.319573:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.319574:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.319575:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.319576:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.319577:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677448.319577:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.319578:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.319657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.319665:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.319668:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.319669:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.319670:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.319673:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.319674:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.319675:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.319677:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.319678:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.319679:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.319682:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.319684:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.319687:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.319688:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.319688:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.319690:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.319690:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.319691:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.319691:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677448.319694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:25.2:1619677448.319696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.319701:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.319703:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.319705:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:22.0:1619677448.319706:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.319707:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.319708:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.319709:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.319710:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.319711:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.319712:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.319714:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.319715:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.319716:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.2:1619677448.319719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:6.0:1619677448.319720:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.319721:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.319721:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.319721:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.319722:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677448.319723:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.319724:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:22.0:1619677448.319726:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.319726:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.319727:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.319727:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677448.319728:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:20.0:1619677448.319729:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.319730:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.319730:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:20.0:1619677448.319730:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:22.0:1619677448.319731:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:20.0:1619677448.319731:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.319732:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.319733:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677448.319733:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.319733:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.319735:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:25.2:1619677448.319736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:21.2:1619677448.319738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:20.0:1619677448.319738:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.319739:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.319741:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.319742:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.319742:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.319742:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:6.0:1619677448.319743:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:20.0:1619677448.319744:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.319745:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677448.319745:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.319745:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677448.319745:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:31.0:1619677448.319746:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.319746:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677448.319746:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:31.0:1619677448.319747:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.319747:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677448.319749:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.319750:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.2:1619677448.319750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:25.2:1619677448.319751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.319758:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.319758:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.319760:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:30.0:1619677448.319761:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.319761:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.319762:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.319763:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.319763:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:5.0:1619677448.319763:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.319764:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.319765:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.0:1619677448.319765:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.319837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677448.319843:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677448.319846:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677448.319846:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:1.0:1619677448.319848:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.319850:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.319852:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677448.319853:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.319854:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.319855:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.319856:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.319860:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.319861:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.319864:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.319864:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:1.0:1619677448.319865:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.319867:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677448.319868:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677448.319868:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677448.319869:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677448.319873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.319881:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.319883:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.319884:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.319885:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.319887:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.319889:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.319990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.319995:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.319998:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.319999:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.320000:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.320003:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.320004:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.320005:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.320006:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.320007:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.320008:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.320011:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.320012:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.320015:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.320015:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.320016:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.320018:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.320018:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.320019:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.320019:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677448.320021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.320024:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:9.2:1619677448.320024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677448.320026:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.320028:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.320029:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.320030:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.320030:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:20.0:1619677448.320032:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:24.0:1619677448.320033:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:20.0:1619677448.320033:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.320033:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:20.0:1619677448.320034:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.320034:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.320036:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.320037:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.320038:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.320040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677448.320041:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.320041:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.0:1619677448.320043:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.2:1619677448.320044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677448.320045:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.320046:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.320047:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.320048:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.320049:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.320049:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.320050:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677448.320051:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677448.320051:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:26.0:1619677448.320052:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677448.320053:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.320054:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:30.0:1619677448.320055:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677448.320055:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677448.320055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677448.320056:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.320058:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677448.320058:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.320059:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.320061:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.320061:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.320062:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.320062:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.320063:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677448.320063:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.320063:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:26.0:1619677448.320065:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.320065:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:26.0:1619677448.320066:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.320066:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.320067:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.320067:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.320067:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.320069:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.320070:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.320071:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:26.0:1619677448.320072:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.320073:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.320074:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:26.0:1619677448.320074:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.320075:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.320076:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:26.0:1619677448.320076:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.320077:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.320077:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.320078:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.320078:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.320079:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.320079:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.320080:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.320080:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.320081:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.2:1619677448.320083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:25.2:1619677448.320085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.320091:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:13.0:1619677448.320092:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677448.320093:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.320094:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:13.0:1619677448.320094:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.320095:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.320095:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.320098:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:1.0:1619677448.320098:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:13.0:1619677448.320099:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.320100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677448.320102:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677448.320103:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.320105:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.320108:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.320108:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.320108:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.320109:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.320110:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.320266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.320273:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.320276:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.320277:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.320279:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.320281:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.320283:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.320284:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.320285:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.320286:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.320287:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.320290:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.320292:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.320299:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.320300:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.320300:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.320302:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.320302:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.320304:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.320305:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.320305:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.320308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.320311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677448.320317:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677448.320319:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:5.0:1619677448.320319:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.320320:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.320321:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.320322:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:5.0:1619677448.320322:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.320323:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.320324:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.320325:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.320428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.320431:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.320433:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.320435:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.320436:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.320438:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:20.0:1619677448.320442:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.320444:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.320445:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.320446:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.320447:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:20.0:1619677448.320447:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.320448:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.320450:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:20.0:1619677448.320453:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.320454:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.320454:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.320455:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:30.0:1619677448.320457:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.320458:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.320459:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:20.0:1619677448.320459:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.320459:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677448.320460:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.320461:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.320461:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:30.0:1619677448.320462:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.320462:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.320462:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677448.320463:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.320463:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:30.0:1619677448.320466:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.320468:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:9.2:1619677448.320468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.320471:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.320472:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.320473:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.320475:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.320475:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677448.320476:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:30.0:1619677448.320477:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.320478:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.320479:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:2.0:1619677448.320479:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.320480:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.320480:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.320481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:2.0:1619677448.320482:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.320484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677448.320485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677448.320487:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.320488:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677448.320489:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.320491:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:2.0:1619677448.320491:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.320492:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:5.0:1619677448.320492:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.320492:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677448.320493:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:2.0:1619677448.320493:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.320494:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.320494:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.320494:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677448.320495:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.320496:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.320497:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.320497:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.320499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.2:1619677448.320499:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.2:1619677448.320502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:17.0:1619677448.320505:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.320507:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677448.320508:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:24.0:1619677448.320509:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:1.0:1619677448.320509:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.320510:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.320511:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000400:00000200:17.0:1619677448.320511:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.320512:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:20.0:1619677448.320512:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.320512:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.320513:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:17.0:1619677448.320514:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.320514:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:24.0:1619677448.320515:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.320516:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:24.0:1619677448.320517:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.320517:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.320519:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677448.320519:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.320520:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.320521:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.320522:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.320522:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:24.0:1619677448.320524:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.320525:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677448.320526:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.320529:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:24.0:1619677448.320530:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.320532:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.320533:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.320533:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.320534:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:24.0:1619677448.320535:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.320536:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:17.0:1619677448.320536:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.320536:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.320537:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.320537:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.320538:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677448.320539:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.320539:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.320540:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.320541:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.320542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:9.2:1619677448.320545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.320549:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.320552:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.320553:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.320553:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:31.0:1619677448.320554:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.320556:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:18.0:1619677448.320556:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:31.0:1619677448.320557:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:2.0:1619677448.320557:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.320558:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.320558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:18.0:1619677448.320560:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.320563:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.320565:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.320568:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.320569:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.320571:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.320574:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.320575:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.320577:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716497] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.320578:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.320579:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.320580:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.320583:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.320585:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.320587:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.320588:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.320588:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.320590:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.320590:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.320591:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.320591:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.320595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:21.2:1619677448.320598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.320603:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.320605:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.320606:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.320607:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.320608:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.320608:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.320610:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.320610:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.320611:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.320612:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.320613:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.320615:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.320616:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.320617:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.320618:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.320619:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677448.320622:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.320623:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.320626:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.320627:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.320627:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.320629:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.320630:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.320631:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.320631:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.320635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.320643:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.320646:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.320646:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.320647:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.320649:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.320652:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.320691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.320696:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.320699:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.320700:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.320702:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.320704:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.320705:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.320706:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.320708:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.320709:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.320710:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.320712:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.320714:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.320716:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.320717:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.320717:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.320719:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.320719:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.320720:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.320720:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.320725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677448.320725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.320732:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.320732:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.320734:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.320735:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:2.0:1619677448.320735:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.320736:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.320736:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.2:1619677448.320737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:2.0:1619677448.320737:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.320739:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.320739:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:26.0:1619677448.320742:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.320743:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:30.0:1619677448.320744:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:26.0:1619677448.320744:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.320746:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.320747:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:26.0:1619677448.320747:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.320748:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.320749:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.320750:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677448.320752:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.320752:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.320753:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.320754:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:30.0:1619677448.320755:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.320755:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.320756:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.320756:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:30.0:1619677448.320757:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.320757:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:30.0:1619677448.320758:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.320758:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:30.0:1619677448.320761:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.320762:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.320763:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.320763:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.320764:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.320764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.320765:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.320766:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:30.0:1619677448.320766:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:30.0:1619677448.320768:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:30.0:1619677448.320768:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677448.320769:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677448.320770:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:10.0:1619677448.320770:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.320772:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:25.2:1619677448.320773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000400:00000200:10.0:1619677448.320773:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.320775:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.320775:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.320776:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.320777:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.320777:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:26.0:1619677448.320778:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.320778:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.320780:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.320782:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.320926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.320932:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.320934:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.320935:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.320937:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.320940:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.320941:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.320942:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.320944:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.320944:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.320946:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.320949:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.320951:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.320954:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.320955:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.320955:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.320957:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.320958:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.320958:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.320959:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.320963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.320970:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.320973:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.320974:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.320975:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.320977:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.320978:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.321039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.321045:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.321047:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.321048:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.321050:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.321052:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.321054:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.321055:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.321056:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.321057:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.321059:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.321062:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.321064:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.321067:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.321068:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.321068:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.321070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:18.0:1619677448.321070:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.321071:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.321072:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.321073:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.321074:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.321076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:22.0:1619677448.321076:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.321078:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:21.2:1619677448.321079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:6.0:1619677448.321079:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.321081:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.321082:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:2.0:1619677448.321084:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:26.0:1619677448.321085:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:22.0:1619677448.321085:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.321085:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:22.0:1619677448.321086:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677448.321086:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.321086:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.321087:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:22.0:1619677448.321088:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.321088:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.321088:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:22.0:1619677448.321089:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.321090:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.321091:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.321092:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.321095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000400:00000200:22.0:1619677448.321095:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.321097:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.321099:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.321100:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.321101:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:22.0:1619677448.321101:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.321102:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.321103:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.321103:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677448.321104:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:22.0:1619677448.321104:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.321104:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.321105:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.321106:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.321107:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:8.2:1619677448.321108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.321116:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.321118:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.321119:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.321121:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.321121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677448.321123:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.321124:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:10.0:1619677448.321128:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.321131:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.321132:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.321134:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.321136:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.321138:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.321139:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.321140:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.321141:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.321142:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.321145:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.321147:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.321150:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.321150:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.321151:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.321153:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.321153:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.321154:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.321156:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.321156:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.321158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.321160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677448.321162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.321165:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677448.321167:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.321168:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.321168:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.321169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.321170:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:12.0:1619677448.321170:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.321172:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:30.0:1619677448.321173:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.321173:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.321174:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:12.0:1619677448.321174:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:30.0:1619677448.321175:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.321175:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.321175:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:30.0:1619677448.321176:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.321177:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.321177:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.321177:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677448.321178:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.321178:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:30.0:1619677448.321179:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.321180:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.321181:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.321182:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.321184:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.321184:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.321185:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677448.321186:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.321186:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.321188:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.321189:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.321190:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.321190:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.321191:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.321191:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.321193:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:26.0:1619677448.321193:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.321194:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.321194:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.321195:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.321195:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.321198:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.321198:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.321199:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.2:1619677448.321199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.321201:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.321201:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.321202:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.321202:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677448.321206:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:25.2:1619677448.321207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.321209:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.321210:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.321211:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.321212:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.321214:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:6.0:1619677448.321214:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.321217:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.321218:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.321219:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.321221:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.321224:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.321394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.321400:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.321403:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.321403:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.321406:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.321409:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.321410:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.321412:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.321413:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.321414:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.321415:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.321418:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.321420:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.321424:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.321424:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.321425:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.321427:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.321427:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.321429:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.321430:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.321430:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.321433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.321436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.321441:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677448.321442:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677448.321444:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677448.321444:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.321444:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677448.321445:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.321445:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677448.321447:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.321448:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.321450:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.321455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:12.0:1619677448.321462:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.321464:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.321465:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.321466:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.321466:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.321468:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.321488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.321494:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.321497:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.321499:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.321499:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677448.321502:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.321503:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677448.321505:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.321506:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.321507:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.321508:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.321512:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.321514:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.321517:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.321517:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677448.321518:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.321520:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.321521:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.321522:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.321522:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.321526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.321533:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.321535:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.321536:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.321537:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.321539:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:10.0:1619677448.321542:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.321569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:20.2:1619677448.321571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.321573:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.321576:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:8.0:1619677448.321577:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:20.0:1619677448.321578:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.321578:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.0:1619677448.321579:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.321580:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:20.0:1619677448.321580:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.321582:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:8.0:1619677448.321583:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.321584:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.321585:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.321586:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.321586:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.321587:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:8.0:1619677448.321588:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.321590:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96716483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.321592:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.321592:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.321593:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.321595:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:8.0:1619677448.321595:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.321598:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.321599:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:8.0:1619677448.321599:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.321600:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.0:1619677448.321601:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.321602:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.321602:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.321603:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.321603:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.0:1619677448.321604:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.321605:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.321606:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.321607:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:25.2:1619677448.321608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.0:1619677448.321608:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.321609:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.321609:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.321613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.321615:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.321618:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.321619:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.321620:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.321622:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:18.0:1619677448.321622:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677448.321623:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677448.321625:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.321626:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.321626:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.321628:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.321631:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.321730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.321737:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.321740:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.321742:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.321744:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.321745:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.321746:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.321747:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.321749:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.321750:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.321751:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.321754:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.321756:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.321759:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.321759:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.321760:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.321762:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.321762:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.321763:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.321763:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.321768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.321776:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.321778:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.321779:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.321781:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.321781:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.321784:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.321926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.321932:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.321934:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.321935:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.321937:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.321940:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.321942:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:5.2:1619677448.321942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:17.0:1619677448.321943:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.321944:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.321945:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.321946:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.321949:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677448.321949:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.321951:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.321952:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.321953:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.321954:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.321955:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.321955:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:26.0:1619677448.321956:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.321957:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.321957:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.321958:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.321958:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.321959:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677448.321960:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.321961:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.321962:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.321963:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.321963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:26.0:1619677448.321964:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.321967:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.321969:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.321971:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.321972:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:22.0:1619677448.321972:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:26.0:1619677448.321973:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.321974:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.321974:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677448.321974:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:26.0:1619677448.321975:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677448.321975:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.321976:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677448.321976:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.321977:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.321979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:22.0:1619677448.321980:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:10.0:1619677448.321985:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.321987:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.321988:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.321989:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.321990:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.321992:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.322007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.322012:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:4.2:1619677448.322012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677448.322015:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.322015:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:21.2:1619677448.322017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:6.0:1619677448.322017:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.322019:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677448.322020:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.322022:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:26.0:1619677448.322023:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.322023:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.322023:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.322024:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677448.322025:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.322027:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677448.322027:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.322027:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.322028:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:26.0:1619677448.322029:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.322030:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.322031:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:6.0:1619677448.322032:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.322033:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.322035:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.322035:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:6.0:1619677448.322035:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:31.0:1619677448.322037:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.322037:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.322038:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.322038:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000400:00000200:31.0:1619677448.322039:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.322039:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.322039:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:31.0:1619677448.322040:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.322041:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.322041:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000400:00000200:31.0:1619677448.322042:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.322042:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.322042:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.322042:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.322043:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.322046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000400:00000200:31.0:1619677448.322047:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.322048:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.322050:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.322051:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.322052:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.322053:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.322054:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.322054:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.322054:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.322055:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.322055:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.322055:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:31.0:1619677448.322056:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.322056:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677448.322056:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.322057:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.322058:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:26.0:1619677448.322058:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:10.0:1619677448.322058:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.322059:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:26.0:1619677448.322059:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.322060:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.322060:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.322061:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:25.2:1619677448.322061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.322062:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:25.2:1619677448.322065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:21.2:1619677448.322068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.322068:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:25.2:1619677448.322069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000800:00000200:13.0:1619677448.322070:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.322071:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.322072:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.322073:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:12.0:1619677448.322074:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.322075:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.322075:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.322075:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:13.0:1619677448.322075:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.322076:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.322077:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.322077:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.322077:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.322077:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.322078:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:21.2:1619677448.322078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:10.0:1619677448.322078:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:31.0:1619677448.322079:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.322079:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:26.0:1619677448.322079:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:31.0:1619677448.322080:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.322080:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:31.0:1619677448.322081:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.322081:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677448.322082:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.322083:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.322084:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:24.0:1619677448.322086:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.322088:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.322089:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.322091:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.322093:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.322095:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.322096:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.322098:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.322099:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.322100:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677448.322103:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.322104:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.322107:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.322108:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.322109:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.322110:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.322111:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.322112:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.322112:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.322116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.322124:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.322126:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.322127:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.322129:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.322130:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.322132:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.322191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.322197:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.322200:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.322201:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.322203:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.322206:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.322207:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.322209:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.322210:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.322211:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.322212:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.322215:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.322216:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:21.2:1619677448.322219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:8.0:1619677448.322219:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.322219:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.322220:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.322221:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.322222:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.322223:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.322223:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677448.322225:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.322227:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.322228:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677448.322228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677448.322230:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.322233:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.322234:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:1.0:1619677448.322235:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:2.0:1619677448.322236:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.322237:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.322238:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.322238:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.322239:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:2.0:1619677448.322239:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677448.322239:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.322242:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.322243:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677448.322244:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677448.322245:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.322248:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.322248:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.322249:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.322251:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.322251:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.322252:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.322253:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.322257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.322264:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.322267:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.322268:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.322268:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.322270:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.322273:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.322278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.322285:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.322287:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.322288:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.322289:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.322290:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.322290:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.322465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.322473:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.322476:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.322478:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.322480:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.322481:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.322482:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.322483:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.322485:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.322486:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.322487:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.322490:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.322492:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.322494:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.322495:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.322496:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.322497:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.322498:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.322499:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.322499:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.322504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.322511:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.322513:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.322514:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.322516:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.322518:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.322519:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.322532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.322539:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.322541:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.322542:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.322544:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.322546:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.322548:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.322549:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.322550:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.322551:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.322553:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.322556:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.322558:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.322560:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.322561:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.322562:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.322563:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.322564:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.322564:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.322565:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.322569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.322577:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.322579:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.322580:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.322581:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.322583:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.322585:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.322760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.322767:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.322770:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.322772:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.322774:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.322775:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.322776:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.322777:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.322778:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.322780:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.322781:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677448.322784:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.322786:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.322789:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.322790:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.322790:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.322792:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.322793:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.322793:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.322794:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.322799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.2:1619677448.322803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677448.322807:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.322809:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:5.0:1619677448.322809:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.322810:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.322811:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.322813:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.322813:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:20.0:1619677448.322814:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.322815:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:18.0:1619677448.322817:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:18.0:1619677448.322818:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:5.0:1619677448.322818:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.322819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.322820:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.322820:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:5.0:1619677448.322821:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.322821:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000400:00000200:5.0:1619677448.322822:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:2.0:1619677448.322822:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.322823:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000400:00000200:5.0:1619677448.322824:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.322824:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.322825:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.322825:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:5.0:1619677448.322826:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.322826:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:5.0:1619677448.322827:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.322828:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.322828:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.322830:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.322832:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.322832:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.322833:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.322835:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.322835:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.322836:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.322838:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.322839:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:18.0:1619677448.322840:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.322840:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.322842:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.322842:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.322842:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.322844:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:18.0:1619677448.322845:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.322845:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:18.0:1619677448.322846:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.322846:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.322846:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.322847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:18.0:1619677448.322848:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677448.322849:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.322849:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.2:1619677448.322850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677448.322850:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677448.322853:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677448.322854:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:20.0:1619677448.322855:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.322856:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.322857:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.322857:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677448.322858:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.322859:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.322861:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.322862:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.322862:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677448.322863:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.322863:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677448.322864:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.322867:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.322869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.322875:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.322878:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.322878:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.322879:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.322880:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.322881:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.323016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.323022:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.323025:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.323026:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.323028:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.323031:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.323032:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.323034:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.323035:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.323036:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.323037:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.323040:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.323041:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.323043:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.323043:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.323044:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.323045:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.323046:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.323047:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.323047:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.323051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.323052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.323058:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.323059:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.323061:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:1.0:1619677448.323061:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:18.0:1619677448.323062:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.323062:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.323063:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.323064:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.323064:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.323066:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:1.0:1619677448.323067:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.323069:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.323070:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.323071:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677448.323072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:1.0:1619677448.323072:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.323073:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.323077:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.323078:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:2.0:1619677448.323079:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.323082:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.323082:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.323082:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.323083:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.323083:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677448.323083:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:2.0:1619677448.323084:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677448.323084:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.323086:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.323086:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.323087:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:21.2:1619677448.323088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:1.0:1619677448.323088:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.323092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:6.0:1619677448.323094:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.323096:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.323097:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677448.323099:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:6.0:1619677448.323099:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.323101:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677448.323102:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:6.0:1619677448.323102:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.323103:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.323104:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.323105:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.323106:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.323107:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.323108:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.323109:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:6.0:1619677448.323109:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.323113:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.323115:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.323117:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.323118:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.323119:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.323120:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.323121:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.323121:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.323122:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.323127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.323134:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.323137:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.323138:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.323138:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.323140:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.323142:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.323181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.323189:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.323192:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.323193:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.323195:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.323197:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.323199:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.323200:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.323202:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.323202:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.323204:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.323207:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.323209:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.323212:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.323213:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.323214:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.323215:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.323216:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.323217:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677448.323218:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.323218:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677448.323221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.323225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.323231:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677448.323232:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677448.323233:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.323234:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:20.0:1619677448.323234:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677448.323235:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:20.0:1619677448.323235:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.323237:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.323238:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.323240:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.323442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.323449:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.323452:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.323453:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.323455:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.323458:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.323459:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.323461:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.323462:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.323463:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.323463:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.323466:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.323468:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.323470:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.323471:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.323472:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.323474:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.323475:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.323476:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.323477:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.323477:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677448.323481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677448.323483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.323489:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.323490:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.323492:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:18.0:1619677448.323492:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.323493:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.323493:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.323494:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.323494:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:18.0:1619677448.323495:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.323498:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.323552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.323560:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.323562:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.323563:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.323565:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.323568:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.323569:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.323570:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.323571:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.323572:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.323573:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.323576:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.323578:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.323579:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.323580:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.323580:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.323582:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.323582:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.323583:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.323584:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.323588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.323595:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.323598:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.323599:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.323600:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.323602:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.323603:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.323710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.323716:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.323719:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.323719:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.323721:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.323724:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.323725:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.323726:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.323728:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.323729:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.323730:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.323733:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.323735:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.323736:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.323737:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.323738:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.323739:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.323740:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.323741:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.323742:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.323742:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.323746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.323749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677448.323752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:17.0:1619677448.323754:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.323755:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.323756:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.323756:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.323756:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.323757:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.323758:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.323758:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:1.0:1619677448.323758:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.323759:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677448.323759:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677448.323761:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.323762:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.323763:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.323765:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.323766:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.323768:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.323769:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.323770:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.323771:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.323774:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.323776:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.323778:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.323779:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.323779:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.323781:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.323781:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.323782:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.323783:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.323787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.323794:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.323797:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.323798:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.323798:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.323800:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.323802:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.323809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.323815:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.323818:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.323819:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.323819:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.323820:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.323820:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.323872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.323880:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.323882:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.323885:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.323886:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.323887:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.323889:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.323890:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.323891:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.323893:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.323894:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.323897:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.323898:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.323901:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.323902:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.323902:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.323904:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.323904:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.323906:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.323906:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.323909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.323916:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.323918:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.323919:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.323919:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.323920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:10.0:1619677448.323921:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.323923:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.0:1619677448.323926:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.323929:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.323931:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.323931:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.323934:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.323936:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677448.323937:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.323939:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.323940:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.323941:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677448.323944:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.323946:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677448.323949:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.323950:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677448.323951:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677448.323952:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.323953:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.323954:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.323954:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.323958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:21.2:1619677448.323959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:31.0:1619677448.323965:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:22.0:1619677448.323965:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677448.323968:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:22.0:1619677448.323968:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.323969:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:31.0:1619677448.323970:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.323971:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.323972:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.323972:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:22.0:1619677448.323974:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.323975:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:22.0:1619677448.323975:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:25.2:1619677448.323976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000400:00000200:22.0:1619677448.323977:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.323978:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.323979:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.323980:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.323983:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000400:00000200:22.0:1619677448.323983:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.323985:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.323986:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.323987:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.323987:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:22.0:1619677448.323988:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.323989:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.323989:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:22.0:1619677448.323989:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.323989:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.323991:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.323991:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.323993:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.323994:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.323994:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.323997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.323999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677448.324004:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677448.324005:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.324007:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:1.0:1619677448.324007:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.324008:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.324008:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.324009:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:1.0:1619677448.324009:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.324011:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.324013:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.324085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.324092:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.324095:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.324096:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.324097:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.324100:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.324102:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.324103:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.324105:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.324105:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.324107:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.324110:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.324112:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.324115:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.324115:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.324116:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.324118:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.324118:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.324119:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.324121:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.324121:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.324124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.324127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.324131:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.324134:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.324134:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.324135:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.324135:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.324136:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.324137:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.324137:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.324137:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677448.324140:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.324289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.324298:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.324300:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.324301:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.324303:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.324305:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.324306:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.324308:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.324309:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.324310:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.324311:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.324314:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.324315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:26.0:1619677448.324316:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.324319:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.324320:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.324320:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.324322:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.324322:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.324322:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:26.0:1619677448.324323:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.324323:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:18.0:1619677448.324325:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.324326:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:25.2:1619677448.324328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:18.0:1619677448.324328:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.324330:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.324331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:18.0:1619677448.324331:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.324332:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.324334:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.324335:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.324336:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.324337:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.324339:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:18.0:1619677448.324339:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.324340:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:6.0:1619677448.324340:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.324341:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.324341:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.324343:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:22.0:1619677448.324343:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.324344:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.324345:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.324345:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.324346:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.324347:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:22.0:1619677448.324347:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677448.324347:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:30.0:1619677448.324348:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.324348:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.324348:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:25.2:1619677448.324349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677448.324349:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:30.0:1619677448.324350:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.324351:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.324352:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.324353:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.324354:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:8.2:1619677448.324354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.324356:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:30.0:1619677448.324358:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.324359:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.324360:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:22.0:1619677448.324360:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.324360:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.324361:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:20.0:1619677448.324362:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677448.324362:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:30.0:1619677448.324363:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.324364:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.324365:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.324365:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.324366:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.324367:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.324367:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:20.0:1619677448.324367:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.324368:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.324369:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.324370:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677448.324371:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.324371:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.324373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:8.2:1619677448.324375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.324377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.324380:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.324383:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.324384:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.324384:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677448.324385:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:5.0:1619677448.324385:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.324386:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.324387:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677448.324388:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:20.0:1619677448.324388:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.324388:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:31.0:1619677448.324389:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.324389:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.324390:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.324391:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.324393:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.324643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.324649:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.324652:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.324653:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.324655:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.324656:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.324658:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.324659:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.324661:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.324661:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.324663:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.324666:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.324668:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.324669:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.324670:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.324671:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.324672:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.324672:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.324673:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.324674:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.324678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.324685:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.324687:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.324688:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.324689:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.324690:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.324691:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.324698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.324705:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.324708:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.324709:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.324711:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.324714:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.324715:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.324716:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.324718:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.324719:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.324720:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.324723:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.324725:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.324728:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.324729:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.324730:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.324731:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.324732:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.324732:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.324733:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.324738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.324745:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.324748:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.324749:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.324750:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.324752:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.324754:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.324806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.324813:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.324816:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.324817:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.324819:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.324822:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.324823:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.324824:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.324825:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.324826:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.324827:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.324830:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.324831:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.324833:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.324834:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.2:1619677448.324834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:26.0:1619677448.324835:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.324836:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.324837:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.324837:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.324837:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677448.324840:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:25.2:1619677448.324842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:2.0:1619677448.324843:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.324844:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.324846:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.324849:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:2.0:1619677448.324849:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.324850:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:31.0:1619677448.324851:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:2.0:1619677448.324851:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.324852:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.324853:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.324854:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.324854:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.324855:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.324856:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.324857:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:2.0:1619677448.324858:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.324860:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.324863:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.324864:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.324864:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.324866:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.324866:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.324868:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.324869:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.324869:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.324873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.324874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.324880:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.324882:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.324882:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677448.324882:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677448.324883:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677448.324885:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.324886:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.324887:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.324888:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.324890:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.324899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.324907:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.324910:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.324912:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.324914:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.324916:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.324917:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.324918:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677448.324919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:30.0:1619677448.324920:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.324921:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.324922:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.324925:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.324926:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677448.324927:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.324928:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.324929:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677448.324930:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.324931:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:24.0:1619677448.324931:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.324932:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.324933:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.324934:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:24.0:1619677448.324934:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.324935:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.324935:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:24.0:1619677448.324935:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.324937:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96716448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.324938:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.324939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:24.0:1619677448.324939:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.324940:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.324943:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.324945:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677448.324946:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:24.0:1619677448.324948:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.324949:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.324949:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677448.324949:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.324950:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.324951:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.324951:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.324952:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.324952:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:13.0:1619677448.324952:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.324954:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.324954:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.2:1619677448.324957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.324965:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.324968:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.324968:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.324969:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.324971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:1.0:1619677448.324971:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.324974:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.324974:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677448.324977:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.324979:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.324979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.324980:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.324982:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.324984:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.324985:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.324986:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677448.324987:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:17.0:1619677448.324989:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.324990:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.324990:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:20.0:1619677448.324991:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.324991:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.324991:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:20.0:1619677448.324992:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.324993:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.324993:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677448.324995:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677448.324996:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:20.0:1619677448.324997:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.324998:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.325000:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:6.0:1619677448.325001:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.325002:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677448.325003:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.325003:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.325004:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.325005:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:6.0:1619677448.325005:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.325006:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.325007:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.325007:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:6.0:1619677448.325007:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.325008:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.325009:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:6.0:1619677448.325010:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.325012:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:25.2:1619677448.325013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.325014:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.325014:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.325015:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.325017:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.325017:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677448.325018:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677448.325018:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:12.0:1619677448.325019:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:25.2:1619677448.325022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:12.0:1619677448.325022:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.325023:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.325025:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.325026:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.325027:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677448.325029:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.325032:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.325032:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.325033:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.325034:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.325036:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.325038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.325045:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.325048:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.325050:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.325052:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.325053:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.325054:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.325056:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.325057:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.325058:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.325059:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.325063:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.325064:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.325067:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.325068:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.325069:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.325070:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.325071:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.325072:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.325072:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.325076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.325084:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.325086:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.325088:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.325088:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.325090:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.325093:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.325211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.325218:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.325221:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.325222:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.325224:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.325226:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.325227:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.325229:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.325230:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.325231:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.325232:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.325235:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.325237:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.325240:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.325240:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.325241:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.325243:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.325243:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.325243:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.325244:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.325249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.325256:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.325259:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.325259:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.325260:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.325262:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.325264:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.325312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.325317:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.325320:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.325320:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.325322:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.325324:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.325325:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.325326:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.325327:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.325329:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.325330:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.325332:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.325334:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.325336:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.325337:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.325338:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.325340:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.325340:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.325341:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.325341:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.325347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.325355:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.325357:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.325358:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.325360:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.325360:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.325362:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.325524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.325530:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.325533:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.325533:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:4.2:1619677448.325534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677448.325535:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.325537:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.325538:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.325540:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.325541:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.325542:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677448.325542:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.325543:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:26.0:1619677448.325545:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.325545:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677448.325548:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.325549:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.325551:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:26.0:1619677448.325552:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.325553:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:1.0:1619677448.325554:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.325555:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.325555:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.325555:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:26.0:1619677448.325556:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.325557:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.325557:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.325557:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.325558:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.325558:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.325559:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677448.325562:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.325563:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:9.2:1619677448.325564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:26.0:1619677448.325567:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.325567:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.325568:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.325570:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.325570:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677448.325570:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:26.0:1619677448.325571:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.325572:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.325573:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677448.325573:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.325574:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.325574:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.325575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:8.0:1619677448.325576:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.325578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677448.325579:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677448.325583:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677448.325585:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.325586:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.325586:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.325587:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.325587:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.325588:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.325589:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.325589:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.325592:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.325664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:21.2:1619677448.325665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.325671:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677448.325673:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677448.325674:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.325675:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:30.0:1619677448.325677:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:2.0:1619677448.325678:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:30.0:1619677448.325680:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.325681:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:2.0:1619677448.325681:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.325682:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677448.325683:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.325684:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.325685:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677448.325686:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.325687:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.325687:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.325688:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677448.325689:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.325690:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.325692:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.325693:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.325695:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:30.0:1619677448.325696:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.325697:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.325698:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.325698:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.325698:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.325700:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.325700:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677448.325701:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.325701:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:30.0:1619677448.325702:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.325702:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677448.325702:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.325703:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:30.0:1619677448.325704:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.325705:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.325706:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.325706:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:9.2:1619677448.325706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.325709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:25.2:1619677448.325710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677448.325715:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.325716:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.325716:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:18.0:1619677448.325717:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.325717:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677448.325717:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.325718:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.325719:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.325719:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677448.325720:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:2.0:1619677448.325720:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677448.325721:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:13.0:1619677448.325721:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.325722:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.325725:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.325726:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.325861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.325863:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.325866:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.325868:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.325869:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.325870:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.325872:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677448.325874:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.325875:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.325876:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.325877:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677448.325880:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.325882:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.325885:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.325886:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.325887:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.325889:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.325889:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.325891:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.325892:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.325892:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.325894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.2:1619677448.325897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:26.0:1619677448.325901:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.325903:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:5.0:1619677448.325903:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:10.0:1619677448.325904:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.325905:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.325905:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.325905:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.325906:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:26.0:1619677448.325907:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:20.2:1619677448.325907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:26.0:1619677448.325910:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.0:1619677448.325911:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.325912:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:5.0:1619677448.325912:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.325914:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.325915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000400:00000200:20.0:1619677448.325917:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.325919:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:4.2:1619677448.325920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:20.0:1619677448.325921:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.325922:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.325925:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.325925:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.325926:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.325928:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.325928:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.325929:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677448.325929:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.325929:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.325930:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.325931:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:20.0:1619677448.325931:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.325931:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.325932:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.325934:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.325934:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.325935:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.325935:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:26.0:1619677448.325937:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.325937:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.325938:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677448.325939:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.325939:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.325939:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.325939:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:26.0:1619677448.325941:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.325942:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.325943:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:25.2:1619677448.325943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:26.0:1619677448.325946:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.325948:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.2:1619677448.325949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:31.0:1619677448.325950:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:26.0:1619677448.325951:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.325953:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.325953:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.325953:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.325953:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:31.0:1619677448.325955:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.325955:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.325956:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.325957:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.325957:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.325957:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:31.0:1619677448.325958:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.325958:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.325958:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.325959:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.325960:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:25.2:1619677448.325961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:10.0:1619677448.325963:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.325964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:10.0:1619677448.325964:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677448.325965:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.325967:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.325967:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.325968:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.325970:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.325971:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677448.325971:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:6.0:1619677448.325972:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.325973:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:10.0:1619677448.325973:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.325973:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.325975:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.325976:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.325976:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.325977:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.325978:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.325978:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.325979:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.325979:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.325980:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.325980:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:22.0:1619677448.325981:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:10.0:1619677448.325981:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:6.0:1619677448.325981:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.325982:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:6.0:1619677448.325982:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000400:00000200:22.0:1619677448.325983:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.325983:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.325983:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.325984:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.325984:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.325985:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:22.0:1619677448.325986:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.325986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:22.0:1619677448.325987:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.325991:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.325993:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:5.0:1619677448.325994:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:22.0:1619677448.325996:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.325996:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677448.325997:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.325997:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:5.0:1619677448.325997:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.325999:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000400:00000200:5.0:1619677448.325999:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.326000:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.326000:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677448.326000:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.326001:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677448.326002:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:25.2:1619677448.326005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.326011:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.326014:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.326014:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.326015:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.326017:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.326020:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677448.326082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.326089:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.326093:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.326093:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.326095:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.326098:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.326099:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.326100:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.326102:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.326103:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.326104:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.326107:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.326109:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.326111:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.326112:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.326113:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.326114:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.326114:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.326116:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.326117:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.326117:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.2:1619677448.326117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:25.2:1619677448.326120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.326122:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677448.326123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:17.0:1619677448.326125:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:12.0:1619677448.326126:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677448.326126:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.326128:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.326129:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.326129:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.326130:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.326130:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:12.0:1619677448.326131:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.326132:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:8.0:1619677448.326132:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.326132:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:17.0:1619677448.326133:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.326133:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.326134:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677448.326134:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:17.0:1619677448.326135:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.326136:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.326137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:17.0:1619677448.326137:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.326140:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.326141:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.326143:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.326144:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.326145:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677448.326146:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677448.326147:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.326149:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:17.0:1619677448.326149:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.326150:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.326150:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677448.326150:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:17.0:1619677448.326151:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.326151:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.326152:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:25.2:1619677448.326154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.326160:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.326163:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.326164:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.326165:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.326166:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.326168:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.326180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.326187:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.326190:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.326191:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.326192:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.326195:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.326197:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.326198:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.326199:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.326200:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.326201:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.326205:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.326206:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.326209:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.326210:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.326211:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.326212:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.326213:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.326214:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.326215:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.326216:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.326218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.326221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.326226:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:26.0:1619677448.326229:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677448.326229:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.326229:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.326230:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.326231:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.326231:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.326232:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677448.326232:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.326235:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.326353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.326360:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.326363:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.326364:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.326365:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.326368:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.326369:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.326371:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.326372:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.326373:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.326373:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.326376:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.326378:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.326380:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.326381:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.326381:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.326383:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.326383:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.326384:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.326385:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.326391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.326399:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.326402:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.326402:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.326403:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.326405:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.326407:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.326542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.326545:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.326547:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.326549:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.326551:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.326552:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.326553:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677448.326555:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.326556:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.326557:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.326558:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677448.326561:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.326563:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.326566:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.326567:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.326567:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.326569:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.326569:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.326570:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.326571:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.326576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.326584:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.326586:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.326587:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.326588:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.326589:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.326590:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.326595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.326603:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.326605:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.326606:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.326607:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.326607:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.326608:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.326739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.326745:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.326748:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.326749:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.326751:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.326754:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.326755:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.326757:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.326758:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.326759:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.326760:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.326764:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.326765:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.326769:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.326769:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.326770:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.326772:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.326772:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.326774:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.326775:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.326775:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.326779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.326780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.326785:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.326787:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:13.0:1619677448.326787:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677448.326788:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.326789:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.326790:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.326790:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.326792:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.326794:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.326795:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.326797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.326803:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.326805:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.326806:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.326808:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.326811:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.326812:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.326814:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.326815:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.326816:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.326817:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.326820:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.326822:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.326824:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.326825:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.326826:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.326828:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.326828:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.326829:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.326829:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.326834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.326841:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.326844:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.326844:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.326846:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.326847:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.326848:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.326850:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.2:1619677448.326853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.326858:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.326860:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:20.0:1619677448.326861:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.326861:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677448.326863:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.326864:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.326865:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.326865:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.326866:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:26.0:1619677448.326867:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.326868:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:5.0:1619677448.326868:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:26.0:1619677448.326870:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716426] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.326871:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.326872:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.326873:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.326877:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.326878:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.326881:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.326881:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.326882:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.326884:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.326884:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.326885:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.326886:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.326890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.326896:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.326899:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.326900:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.326901:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.326903:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.2:1619677448.326904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:12.0:1619677448.326904:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.326912:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.326915:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.326917:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.326919:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.326920:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.326921:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.326922:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.326923:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.326924:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.326925:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.326928:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.326929:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.326932:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.326933:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.326934:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.326935:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.326936:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.326937:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.326937:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.326941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.326949:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.326951:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.326952:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.326954:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.326954:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.326957:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.327066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.327073:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.327075:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.327076:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.327078:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.327080:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.327082:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.327083:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.327084:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.327086:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.327087:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.327089:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.327091:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.327093:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.327093:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.327094:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.327095:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.327095:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.327096:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.327097:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.327101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.327106:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.327108:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.327109:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.327109:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.327110:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.327112:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.327144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.327151:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.327154:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.327155:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.327156:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.327159:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.327160:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.327161:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716423] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.327163:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.327163:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.327164:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.327167:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.327169:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.327172:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.327173:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.327174:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.327176:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.327176:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.327177:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.327177:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.327182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.327190:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.327193:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.327194:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.327195:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.327196:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.327200:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.327226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.327232:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.327235:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.327237:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.327237:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.327239:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.327240:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.327242:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.327243:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.327244:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.327245:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.327249:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.327251:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.327254:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.327255:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.327255:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.327257:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.327258:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.327259:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.327260:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.327261:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.327263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.327265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.327269:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677448.327270:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.327271:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.327271:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677448.327272:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677448.327272:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:31.0:1619677448.327273:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.327275:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.327276:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.327278:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.327429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.327437:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.327440:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.327442:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.327444:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.327445:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.327446:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.327448:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.327449:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.327450:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.327451:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.327454:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.327456:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.327458:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.327458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677448.327459:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.327460:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.327461:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.327462:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.327463:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.327465:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.327465:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.327466:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:25.2:1619677448.327467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.327469:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.327470:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.2:1619677448.327471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677448.327471:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.327473:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.327474:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:13.0:1619677448.327476:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:10.0:1619677448.327476:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.327477:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.327478:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.327478:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.327478:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.327478:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.327479:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:13.0:1619677448.327480:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.327481:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.327482:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:12.0:1619677448.327482:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:10.0:1619677448.327482:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.327483:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677448.327483:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.327484:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.327484:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:13.0:1619677448.327485:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.327485:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.327486:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.327486:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.327486:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.327486:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.327487:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.327488:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.327489:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.327489:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.327490:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.327494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.327499:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.327501:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.327502:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.327502:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.327504:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.327506:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.327510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.327516:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.327519:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.327520:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.327522:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.327525:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.327526:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.327528:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.327529:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.327530:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.327531:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:21.2:1619677448.327532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:1.0:1619677448.327534:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.327535:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.327538:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.327539:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.327540:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.327540:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.327542:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.327542:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.327543:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:1.0:1619677448.327543:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.327544:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677448.327545:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.327547:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.327547:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.327548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.327549:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.327550:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.327551:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.327552:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.327553:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:13.0:1619677448.327554:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:31.0:1619677448.327557:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.327557:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.327558:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.327559:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:13.0:1619677448.327559:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.327560:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.327562:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.327562:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.327563:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.327563:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.327565:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.327566:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.327567:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.327568:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.327569:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677448.327571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.327574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.327579:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:30.0:1619677448.327582:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:18.0:1619677448.327582:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.327583:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.327583:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.327584:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.327585:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.327586:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:18.0:1619677448.327586:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.327589:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.327771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.327779:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.327781:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.327782:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.327784:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.327787:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.327788:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.327790:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.327791:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.327792:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.327793:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.327796:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.327798:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.327800:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.327801:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.327801:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.327803:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.327803:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.327804:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.327805:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.327808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.327815:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.327817:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.327818:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.327819:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.327821:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.327822:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.327835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.327842:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.327844:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.327846:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.327849:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.327849:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.327851:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.327852:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716416] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.327853:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.327854:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.327855:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.327858:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.327859:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.327862:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.327862:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.327863:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.327864:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.327865:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.327866:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.327866:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.327870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.327876:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.327878:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.327879:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.327881:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.327882:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.327883:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.327897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.327903:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.327905:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.327906:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.327907:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.327907:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.327908:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.327923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.327930:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.327933:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.327934:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.327935:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.327937:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.327938:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.327939:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.327941:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.327942:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.327943:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.327946:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.327947:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.327950:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.327950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.327951:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.327951:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.327953:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.327953:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.327954:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.327954:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677448.327956:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.327958:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:25.2:1619677448.327959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.327959:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.327961:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.327964:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.327965:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.327966:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.327967:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:17.0:1619677448.327967:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.327968:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.327969:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.327970:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.327971:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.327971:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:17.0:1619677448.327973:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.327974:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.327976:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.327977:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:17.0:1619677448.327978:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.327979:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.327979:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:25.2:1619677448.327981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.327981:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (23)--
00000800:00000200:20.2:1619677448.327982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677448.327982:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.327983:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.327983:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677448.327985:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.327987:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:6.0:1619677448.327988:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:20.0:1619677448.327989:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.327991:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (23)++
00000800:00000200:21.2:1619677448.327991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:6.0:1619677448.327991:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.327991:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677448.327992:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000400:00000200:20.0:1619677448.327992:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.327992:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (23)++
00000400:00000200:22.0:1619677448.327993:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.327993:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (25)--
00000800:00000200:6.0:1619677448.327993:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:20.0:1619677448.327994:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.327995:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.327995:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.327996:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.327997:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.327997:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:20.0:1619677448.327998:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.327999:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:20.0:1619677448.327999:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.328000:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.328001:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:20.0:1619677448.328003:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.328005:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.328006:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677448.328008:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.328008:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.328009:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.328009:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677448.328009:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:20.0:1619677448.328011:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.328011:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.328012:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:2.0:1619677448.328012:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.328013:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.328013:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:2.0:1619677448.328013:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.328014:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:9.2:1619677448.328017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677448.328018:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.328020:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.328023:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.328024:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.328025:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.328026:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.328027:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.328027:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.328028:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.328028:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.328029:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.328029:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:18.0:1619677448.328030:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.328031:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:9.2:1619677448.328033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.328036:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.328038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.328040:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.328041:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.328042:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.328042:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677448.328043:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.328043:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.328044:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.328045:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.328045:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.328045:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.328046:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677448.328046:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.328093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.328100:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.2:1619677448.328101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:26.0:1619677448.328103:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.328104:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.328106:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.328108:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677448.328109:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.328110:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:31.0:1619677448.328111:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:26.0:1619677448.328112:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716411] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.328114:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.328114:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.328115:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.328116:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.328116:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677448.328116:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.328119:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.328120:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:31.0:1619677448.328121:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.328122:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.328123:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.328123:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:31.0:1619677448.328124:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.328124:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.328124:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:31.0:1619677448.328125:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.328126:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.328126:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.328127:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677448.328128:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.328129:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.328129:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.328130:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:25.2:1619677448.328131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.328133:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.328134:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.328134:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:4.2:1619677448.328135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.328136:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.328136:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.328137:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.328138:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.328138:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.328141:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.328142:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.328142:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:8.2:1619677448.328142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.328143:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.328144:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.328144:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.328145:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.328146:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677448.328146:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.328150:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.328153:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.328154:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.328155:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.328156:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.328158:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.328186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.328194:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.328196:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.328198:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.328200:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.328201:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.328203:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.328204:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.328205:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.328206:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.328207:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.328210:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.328211:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.328213:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.328214:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.328214:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.328216:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.328216:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.328217:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.328217:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.328221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.328228:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.328230:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.328231:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.328232:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.328234:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.328235:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.328260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.328266:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.328268:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.328269:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.328271:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.328273:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.328275:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.328276:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.328277:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.328278:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.328279:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.328283:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.328284:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.328287:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.328287:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.328288:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.328290:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.328290:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.328291:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.328291:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.328297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.328306:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.328308:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.328309:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.328309:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.328312:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.328314:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.328348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.328354:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.328357:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.328358:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.328360:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.328362:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.328364:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.328365:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.328367:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.328368:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.328369:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.328372:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.328374:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.328376:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.328377:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.328377:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.328379:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.328380:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.328380:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.328381:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.328386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.328393:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.328396:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.328396:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.328397:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.328399:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.328402:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.328473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.328479:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.328482:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.328482:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.328484:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.328487:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.328488:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.328489:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.328491:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.328492:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.328493:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.328497:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.328498:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.328501:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.328501:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.328502:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.328504:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.328504:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.328505:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:5.2:1619677448.328505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:17.0:1619677448.328506:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.328507:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.328509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.328512:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.2:1619677448.328512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.328515:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.328516:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677448.328517:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:10.0:1619677448.328517:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.328517:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.328519:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677448.328520:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:10.0:1619677448.328520:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.328520:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677448.328521:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.328521:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:6.0:1619677448.328521:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.328522:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677448.328523:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.328523:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716405] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.328524:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.328525:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.328526:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:10.0:1619677448.328526:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.328530:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.328532:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.328534:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.328535:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.328536:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.328538:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.328538:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.328539:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.328540:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.328541:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.328543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.328546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.328550:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677448.328553:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:12.0:1619677448.328553:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.328554:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.328555:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.328555:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:12.0:1619677448.328555:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.328556:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.328557:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.328559:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.328698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.328707:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.328709:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.328711:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.328714:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.328714:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.328715:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.328716:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.328718:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.328719:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.328720:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.328723:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.328724:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.328726:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.328727:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.328728:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.328729:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.328730:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.328731:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.328731:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.328734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.328740:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.328743:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.328744:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.328744:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.328746:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.328748:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.328757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.328763:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.328765:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.328766:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.328766:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.328767:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.328767:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.328768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677448.328772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.328774:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677448.328777:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.328777:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.328778:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677448.328780:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:2.0:1619677448.328780:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.328781:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.328783:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.328783:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.328784:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.328786:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.328786:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.328787:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.328788:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.328788:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.328789:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.328790:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.328790:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.328791:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.328792:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.328797:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.328798:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.328800:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:2.0:1619677448.328801:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.328803:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.328804:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:2.0:1619677448.328804:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.328805:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:2.0:1619677448.328805:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.328806:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.328807:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.328808:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:2.0:1619677448.328808:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677448.328809:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.328810:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.328810:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677448.328810:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.328811:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.328813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.2:1619677448.328815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.328821:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:1.0:1619677448.328823:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.328824:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.328825:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.328825:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.328826:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.328827:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.328827:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.328828:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.328830:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.328831:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:1.0:1619677448.328834:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.328917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.328923:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.328926:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.328926:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.328928:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.328931:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.328932:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.328933:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.328935:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.328936:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.328937:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.328940:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.328942:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.328945:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.328946:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.328947:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.328948:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.328949:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.328950:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.328951:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.328952:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677448.328954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.328957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.328962:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677448.328962:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677448.328964:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.328964:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677448.328964:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:22.0:1619677448.328965:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:18.0:1619677448.328965:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.328966:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.328967:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.328970:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.329005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.329010:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.329013:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.329014:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.329016:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.329018:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.329020:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.329021:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.329023:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.329024:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.329025:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.329027:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.329027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677448.329029:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.329032:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.329033:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.329034:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.329034:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.329035:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.329036:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.329036:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.329037:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.329037:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.329038:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.329040:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.329041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:10.0:1619677448.329042:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.329043:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.329045:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.329046:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.329047:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.329048:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:10.0:1619677448.329048:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:12.0:1619677448.329050:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.329051:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.329051:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677448.329053:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.329053:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:13.0:1619677448.329054:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.329055:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.329056:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.329056:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.329056:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.329058:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.329058:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.329059:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.329060:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.329064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.329070:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.329073:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.329073:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.329075:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.329077:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.329077:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:25.2:1619677448.329080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.329087:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.329089:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.329090:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.329091:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.329091:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.329093:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.329105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.329107:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.329110:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.329112:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.329113:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.329115:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.329117:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677448.329118:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.329119:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.329120:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.329121:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677448.329125:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.329126:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677448.329130:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.329130:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677448.329131:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677448.329133:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677448.329134:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.329134:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.329135:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677448.329139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.329145:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.329148:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.329149:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.329149:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.329151:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.329154:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677448.329260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.329265:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.329268:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.329270:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.329270:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.329273:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.329274:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.329275:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.329277:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.329278:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.329279:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.329282:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.329283:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.329286:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.329287:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.329287:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.329289:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.329290:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.329291:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.329291:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.329297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.329305:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.329307:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.329308:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.329308:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.329310:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.329313:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.329384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.329392:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.329395:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.329397:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.329400:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.329400:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.329401:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.329403:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.329404:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.329405:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.329406:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.329409:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.329411:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.329413:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.329413:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.329414:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.329415:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.329416:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.329417:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.329418:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.329422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.329428:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.329431:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.329431:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.329433:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.329435:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.329436:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.329508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:5.2:1619677448.329510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.329516:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.329517:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.329518:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.329518:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000400:00000200:18.0:1619677448.329520:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.329520:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.329523:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.329524:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.329525:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.329526:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.329528:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.329529:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.329530:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.329531:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.329533:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.329533:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.329534:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.329535:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.329536:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.329536:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.329539:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.329540:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.329540:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.329541:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.329542:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.329543:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.329544:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.329544:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677448.329545:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.329545:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677448.329546:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.329546:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677448.329547:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:18.0:1619677448.329548:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.329548:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.329548:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:25.2:1619677448.329549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.329549:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.329550:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.329555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.329557:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.329559:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.329560:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.329560:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.329562:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.329563:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:22.0:1619677448.329565:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677448.329565:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.329566:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.329567:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.329569:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.329570:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.329575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.329583:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.329585:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.329586:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.329588:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.329590:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.329592:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.329593:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.329594:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.329595:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.329596:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.329599:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.329601:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.329604:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.329604:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.329605:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.329607:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.329607:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.329608:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.329608:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.329613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.329620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:1.0:1619677448.329621:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.329623:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.329624:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.329625:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.329626:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:1.0:1619677448.329626:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.329629:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.329629:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:1.0:1619677448.329629:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:6.0:1619677448.329631:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.329634:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.329635:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.329636:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.329638:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.329639:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.329640:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.329643:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.329645:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.329647:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.329648:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.329649:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.329651:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.329651:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.329652:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.329652:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.329656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.329665:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.329667:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.329668:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.329668:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.329670:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.329672:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.329695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.329702:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.329705:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.329706:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.329707:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.329710:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.329712:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.329713:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.329716:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.329716:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.329718:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677448.329721:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.329723:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.329726:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.329727:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.329728:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.329729:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.329730:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.329731:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.329731:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.329735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.329743:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.329745:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.329747:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.329748:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.329748:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.329751:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.329784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.329792:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.329795:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.329795:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.329797:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.329800:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.329801:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.329803:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.329804:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.329805:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.329806:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.329810:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.329810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.329811:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.329814:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.329815:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.329815:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.329816:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.329817:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.329817:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.329818:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677448.329818:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:26.0:1619677448.329819:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677448.329819:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.329821:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.329823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:1.0:1619677448.329824:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.329825:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.329826:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.329828:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.329828:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.329830:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:22.0:1619677448.329831:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.329833:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:1.0:1619677448.329833:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.329834:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.329835:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.329835:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:22.0:1619677448.329837:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.329837:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.329838:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.329839:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.329840:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:22.0:1619677448.329841:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677448.329841:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.329841:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.329842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.329842:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.329846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677448.329849:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.329852:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.329853:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:6.0:1619677448.329853:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.329854:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.329854:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.329854:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:17.0:1619677448.329856:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.329857:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.329857:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.329859:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.329860:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.329884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.329891:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.329894:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.329896:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.329897:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.329898:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.329900:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.329901:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.329902:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.329903:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.329904:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.329908:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.329910:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.329913:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.329914:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.329914:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.329916:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.329916:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.329918:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.329919:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.329920:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.329922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.329926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.2:1619677448.329926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677448.329929:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.329932:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677448.329933:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.329933:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.329934:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:22.0:1619677448.329934:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.329936:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.329937:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.329937:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.329937:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.329937:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:30.0:1619677448.329938:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.329940:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:10.0:1619677448.329940:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.329941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000400:00000200:10.0:1619677448.329943:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.329945:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.329946:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.329947:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.329948:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.329948:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:10.0:1619677448.329949:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677448.329951:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.329951:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.329953:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000400:00000200:10.0:1619677448.329954:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.329954:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.329954:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677448.329956:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.329959:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.329960:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.329960:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.329962:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.329963:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.329963:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.329964:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.329969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.329976:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.329979:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.329980:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.329981:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.329982:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.329986:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.330133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.330140:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.330143:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.330145:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.330147:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.330148:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.330150:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.330151:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.330152:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.330153:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.330154:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.330157:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.330159:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.330161:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.330161:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.330162:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.330163:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.330164:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.330165:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.330165:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.330169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.330175:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.330178:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.330179:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.330180:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.330182:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.330183:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.330195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.330202:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.330204:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.330205:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.330207:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.330210:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.330212:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.330213:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.330214:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.330215:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.330216:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.330219:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.330221:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.330223:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.330224:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.330225:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:4.2:1619677448.330225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.330227:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.330227:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.330228:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.330229:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:26.0:1619677448.330230:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.330232:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.330232:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.330233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:26.0:1619677448.330234:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.330236:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.330238:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.330239:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.330240:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.330241:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.330241:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.330241:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.330243:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.330244:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.330245:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.330246:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.330246:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.330248:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.330249:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:26.0:1619677448.330251:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677448.330251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.330252:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.330252:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.330254:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.330255:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.330255:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.330256:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.330258:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:9.2:1619677448.330260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.330261:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.330262:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.330263:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.330263:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.330264:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677448.330268:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.330271:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.330272:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.330272:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.330275:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.330278:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.330323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.330328:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.330331:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.330332:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.330334:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.330336:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.330338:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.330339:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.330340:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.330342:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.330342:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.330346:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.330348:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.330351:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.330352:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.330353:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.330355:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.330356:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.330357:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.330357:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.330362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.330369:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.330372:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.330373:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.330375:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.330377:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.330378:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.330522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.330530:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.330532:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.330533:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.330536:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.330536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:10.0:1619677448.330538:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.330540:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.330541:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.330542:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.330544:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677448.330544:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.330545:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:24.0:1619677448.330546:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.330547:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:10.0:1619677448.330548:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.330549:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.330550:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:24.0:1619677448.330552:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.330553:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677448.330554:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:10.0:1619677448.330554:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:24.0:1619677448.330555:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96716381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.330555:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:24.0:1619677448.330556:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.330556:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:24.0:1619677448.330557:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.330557:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:24.0:1619677448.330558:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.330558:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.330559:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.330559:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:24.0:1619677448.330561:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.330562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677448.330562:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.330565:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677448.330565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677448.330566:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.330566:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.330567:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.330568:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.330568:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.330569:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677448.330569:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.330572:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.330572:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.330573:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:8.2:1619677448.330573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.330574:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.330574:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.330574:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.330575:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.330576:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.330579:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677448.330582:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.330584:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.330585:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.330586:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.330588:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.330591:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.330731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.330737:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.330740:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.330741:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.330743:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.330745:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.330747:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.330748:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.330749:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.330750:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.330751:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.330754:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.330756:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.330759:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.330759:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.330760:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.330762:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.330762:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.330763:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.330763:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.330767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.330775:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.330778:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.330779:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.330779:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.330781:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.330784:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.330797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:30.0:1619677448.330805:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677448.330808:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:30.0:1619677448.330810:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.330811:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677448.330813:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.330814:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.330816:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.330817:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.330818:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.330819:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.330822:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.330823:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.330826:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.330827:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.330827:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.330829:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.330829:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677448.330830:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677448.330831:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:25.2:1619677448.330835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.330843:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.330845:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.330846:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.330848:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.330850:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.330851:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.330865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:4.2:1619677448.330869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677448.330871:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.330873:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.330876:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677448.330876:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677448.330876:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.330878:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.330879:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.330880:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:5.0:1619677448.330880:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.330882:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.330883:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.330884:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.330885:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.330886:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.330887:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.330888:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.330890:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.330891:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.330892:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.330894:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.330894:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.330895:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.330897:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.330898:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:10.0:1619677448.330899:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.330899:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.330901:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.330901:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.330902:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.330902:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.330903:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:10.0:1619677448.330904:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.330905:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.330906:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:25.2:1619677448.330907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.330908:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.330908:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.330909:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.330909:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.330913:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:21.2:1619677448.330914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.2:1619677448.330914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.330916:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.330917:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.330919:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.330920:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.330921:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:31.0:1619677448.330922:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.0:1619677448.330922:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677448.330924:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:8.0:1619677448.330924:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.330925:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:20.0:1619677448.330925:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.330927:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.330927:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:8.0:1619677448.330928:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.330930:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:8.0:1619677448.330931:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.330932:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.330934:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.330935:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.330936:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.330937:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.330940:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.330942:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.330945:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.330945:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.330946:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.330948:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.330949:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.330950:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.330951:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.330952:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677448.330954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.330958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:5.2:1619677448.330959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:18.0:1619677448.330962:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677448.330965:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:18.0:1619677448.330965:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.330965:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.330966:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.330967:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:24.0:1619677448.330967:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.330968:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:18.0:1619677448.330968:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.330970:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:24.0:1619677448.330970:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:18.0:1619677448.330971:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.330971:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:26.0:1619677448.330973:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.330976:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.330977:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.330978:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.330979:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.330980:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.330981:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.330984:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.330986:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.330988:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.330989:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.330990:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.330991:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.330992:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677448.330993:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:26.0:1619677448.330993:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:8.2:1619677448.331000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.331008:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.331010:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.331011:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.331013:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.331013:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.331016:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.331174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.331181:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.331184:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.331185:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.331186:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.331189:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.331191:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.331192:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.331193:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.331194:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.331195:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.331198:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.331199:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.331202:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.331203:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.331203:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.331205:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.331205:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.331206:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.331206:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.331211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.331219:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.331222:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.331223:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.331223:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.331225:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.331229:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.331249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.331254:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.331256:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.331257:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.331259:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.331261:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.331263:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.331264:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.331265:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.331266:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.331267:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.331270:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.331272:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.331275:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.331275:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.331276:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.331278:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.331278:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.331279:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.331279:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.331285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.331291:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.331297:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.331300:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.331301:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.331302:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.331305:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.331370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677448.331378:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677448.331381:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677448.331383:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.331385:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.331385:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677448.331386:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:13.0:1619677448.331388:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677448.331389:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.331390:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677448.331391:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:13.0:1619677448.331395:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.331396:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:13.0:1619677448.331399:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.331400:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:13.0:1619677448.331401:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:13.0:1619677448.331402:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:13.0:1619677448.331403:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677448.331403:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677448.331404:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677448.331408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.331415:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.331418:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.331418:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677448.331419:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.331421:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.331422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.0:1619677448.331423:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677448.331427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.331428:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.331431:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.331432:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677448.331433:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000400:00000200:17.0:1619677448.331434:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.331436:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.331436:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.331437:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000400:00000200:17.0:1619677448.331437:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:8.0:1619677448.331437:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.331438:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.331438:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:17.0:1619677448.331439:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.331440:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.331441:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.331442:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.331446:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.331448:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.331451:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.331452:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.331453:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.331455:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.331455:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.331456:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.331457:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.331458:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.331461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.2:1619677448.331463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.331468:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:30.0:1619677448.331469:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:1.0:1619677448.331470:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.331471:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.331472:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:1.0:1619677448.331472:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677448.331473:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.331475:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.331476:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.331478:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.331542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.331550:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.331552:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.331553:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.331555:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.331557:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.331559:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.331560:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677448.331561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:8.0:1619677448.331562:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.331562:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.331564:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.331567:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.331568:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677448.331569:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.331570:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.331572:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677448.331572:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.331573:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.331573:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.331574:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.331575:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:10.0:1619677448.331576:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.331576:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677448.331577:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:8.0:1619677448.331577:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.331578:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.331578:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:10.0:1619677448.331579:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.331580:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.331581:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.331581:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:8.2:1619677448.331581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:10.0:1619677448.331584:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.331584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677448.331586:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.331588:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.331589:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:10.0:1619677448.331589:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.331590:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:24.0:1619677448.331591:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.331591:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:10.0:1619677448.331591:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.331592:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.331592:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.331593:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.331593:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.331593:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.331593:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.331593:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:24.0:1619677448.331595:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677448.331595:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.331597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.331598:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.331602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.331606:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.331608:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.331609:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.331610:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.331611:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:6.0:1619677448.331612:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.331614:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.331615:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.331615:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677448.331615:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.331616:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.331617:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.331691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.331699:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.331701:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.331702:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.331704:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.331707:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.331709:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.331710:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.331711:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.331712:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.331713:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.331715:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.331717:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.331718:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.331719:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.331719:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.331721:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.331721:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.331722:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.331723:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.331726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.331732:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.331734:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.331735:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.331735:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.331736:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.331738:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.331749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.331755:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.331758:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.331759:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.331761:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.331763:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.331764:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.331766:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.331767:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.331768:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.331769:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.331772:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.331774:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.331776:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.331777:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.331777:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.331779:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.331780:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.331781:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.331781:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.331786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.331792:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.331794:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.331795:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.331796:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.331796:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.331798:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.331810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.2:1619677448.331811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677448.331817:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.331819:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677448.331819:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.331820:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.331821:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.331821:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:31.0:1619677448.331822:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:2.0:1619677448.331822:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:31.0:1619677448.331824:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.331826:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.331826:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.331827:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.331828:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.331830:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.331831:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.331832:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.331834:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.331836:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.331838:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.331839:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.331840:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.331841:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.331842:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.331844:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.331845:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.331845:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.331847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.331851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.331854:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.331856:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.331857:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.331858:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.331859:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:13.0:1619677448.331860:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.331861:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.331862:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.331862:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.331863:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.331867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:13.0:1619677448.331873:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.331876:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.331877:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.331877:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.331878:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.331879:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.331897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.331900:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.331902:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.331904:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.331905:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.331907:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.331909:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.331910:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.331912:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:20.0:1619677448.331914:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.331916:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.331917:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.331917:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.331919:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.331921:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.331921:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.331923:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.331924:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.331926:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.331926:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677448.331927:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.331928:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677448.331928:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.331929:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.331930:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:5.0:1619677448.331930:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.331931:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.331932:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.331933:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:5.0:1619677448.331934:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.331935:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.331935:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.331936:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:9.2:1619677448.331936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.331936:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.331939:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677448.331940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.331940:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.2:1619677448.331941:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677448.331942:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.331943:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.331943:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677448.331943:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677448.331944:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677448.331944:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.331946:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:5.0:1619677448.331946:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.331946:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:18.0:1619677448.331947:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.331947:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.331947:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:2.0:1619677448.331947:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.331948:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:18.0:1619677448.331949:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.331950:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:21.2:1619677448.331951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677448.331951:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.331952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.331952:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:17.0:1619677448.331952:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.331955:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.331957:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:30.0:1619677448.331958:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677448.331958:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.331958:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:17.0:1619677448.331959:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.331960:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.331961:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:17.0:1619677448.331961:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:12.0:1619677448.331961:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.331962:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.331964:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.331965:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.331965:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677448.331965:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.331966:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.331968:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.331968:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677448.331969:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:30.0:1619677448.331970:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.331971:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.331972:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.331972:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.331973:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000400:00000200:30.0:1619677448.331974:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.331974:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:30.0:1619677448.331975:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.331976:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.331976:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.331977:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677448.331978:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.331978:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.331980:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.331982:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677448.331982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:30.0:1619677448.331983:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.331984:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.331985:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.331986:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.331987:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.331987:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677448.331988:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:25.2:1619677448.331990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677448.331991:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.331991:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.331992:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.331994:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.331997:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677448.331997:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677448.331999:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.332000:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.332001:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.332003:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.332003:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:9.2:1619677448.332003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677448.332009:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:25.2:1619677448.332010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677448.332012:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.332013:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.332013:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.332014:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.332015:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677448.332016:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.332019:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.332020:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.332020:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.332021:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.332022:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.332118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.332123:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.332126:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.332127:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.332129:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.332132:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.332133:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.332135:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.332136:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.332137:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.332138:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.332141:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.332143:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.332146:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.332147:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.332147:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.332149:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.332149:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.332151:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.332152:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.332152:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:25.2:1619677448.332155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:21.2:1619677448.332157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.332161:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:6.0:1619677448.332162:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:26.0:1619677448.332164:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.332164:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.332165:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.332165:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.332165:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.332166:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:26.0:1619677448.332167:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.332170:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:4.2:1619677448.332317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.332325:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.332328:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.332328:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.332330:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.332332:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.332333:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.332334:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.332335:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.332336:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.332337:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.332340:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.332342:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.332343:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.332344:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.332345:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.332346:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.332346:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.332347:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.332349:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.332349:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.332352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:4.2:1619677448.332355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.332361:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.332362:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.332363:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.332364:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.332365:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.332365:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.332366:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.332366:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.332367:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.332367:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677448.332443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.332450:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677448.332451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677448.332452:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.332453:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.332456:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.332458:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.332459:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:1.0:1619677448.332460:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.332461:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.332462:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.332462:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:1.0:1619677448.332463:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.332464:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.332466:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.332466:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.332468:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.332469:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677448.332470:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.332471:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.332472:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.332472:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.332473:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.332474:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:20.2:1619677448.332475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677448.332475:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.332476:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.332477:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:26.0:1619677448.332478:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.332478:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.332479:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.332479:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.332480:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677448.332480:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.332481:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.332482:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.332483:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.332483:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.332484:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.332484:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.332485:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.332485:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.332485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.332486:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:20.2:1619677448.332486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:31.0:1619677448.332488:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.332489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.332489:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.332490:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.332491:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677448.332491:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:31.0:1619677448.332492:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:12.0:1619677448.332492:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:31.0:1619677448.332493:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96716357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.332494:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.332494:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:5.0:1619677448.332494:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677448.332495:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.332495:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.332495:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:31.0:1619677448.332496:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.332497:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677448.332497:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.332497:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.332497:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.332497:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.332498:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:13.0:1619677448.332499:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.332501:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:10.0:1619677448.332501:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.332502:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.332502:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677448.332504:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:20.0:1619677448.332504:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677448.332506:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.332508:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.332508:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.332509:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.332509:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:20.0:1619677448.332509:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.332510:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.332511:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.332512:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.332512:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.332513:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.332513:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.332515:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.332518:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677448.332518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.332519:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.332519:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.332521:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.332521:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.332522:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.332523:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677448.332524:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.2:1619677448.332525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:21.2:1619677448.332526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677448.332526:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.332527:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.332527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:20.0:1619677448.332528:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.332531:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.332531:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677448.332532:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677448.332533:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:30.0:1619677448.332534:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.332534:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.332535:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677448.332536:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.332536:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.332537:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.332538:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:5.0:1619677448.332538:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.332539:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.332540:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:2.0:1619677448.332540:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.332541:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.332541:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.332541:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.332543:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.332544:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.332544:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.332545:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.332546:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.332547:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.332547:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677448.332550:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.332551:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.332552:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.332553:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.332553:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.332556:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.332557:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.332557:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.332557:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.332559:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.332559:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.332559:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.332561:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:30.0:1619677448.332562:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.332562:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:30.0:1619677448.332563:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.332563:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.332563:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:30.0:1619677448.332565:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.332565:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:9.2:1619677448.332565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.332566:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.332567:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677448.332568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:25.2:1619677448.332570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.332573:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.332574:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.332574:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:18.0:1619677448.332575:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.332575:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.332576:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.332577:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.332577:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.332577:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.332578:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677448.332579:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677448.332580:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.332581:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.332583:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.332584:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.332585:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.332622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.332629:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.332631:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.332632:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.332634:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.332637:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.332638:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.332639:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.332641:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.332642:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.332642:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.332646:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.332647:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.332650:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.332651:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.332652:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.332653:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.332654:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.332654:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.332655:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.332660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.332668:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.332670:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.332671:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.332672:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.332674:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.332677:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.332902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.332907:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.2:1619677448.332910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:17.0:1619677448.332910:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.332910:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:21.2:1619677448.332911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677448.332912:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:1.0:1619677448.332912:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.332915:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:1.0:1619677448.332915:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.332917:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:31.0:1619677448.332918:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677448.332918:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.332918:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:1.0:1619677448.332918:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.332919:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.332920:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.332921:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:1.0:1619677448.332921:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.332922:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:1.0:1619677448.332922:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:20.0:1619677448.332924:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.332925:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.332926:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.332926:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.332927:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.332927:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.332928:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.332928:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:1.0:1619677448.332930:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:20.0:1619677448.332932:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677448.332933:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.332934:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:1.0:1619677448.332934:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.332934:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:31.0:1619677448.332935:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.332936:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96716350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.332936:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.332936:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.332937:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.332937:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.332937:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677448.332938:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.332938:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:31.0:1619677448.332939:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.332939:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:31.0:1619677448.332940:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.332941:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677448.332941:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:25.2:1619677448.332942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.0:1619677448.332942:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.332942:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:31.0:1619677448.332943:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.332945:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:25.2:1619677448.332946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.332947:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.332948:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.332949:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.332950:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677448.332950:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:31.0:1619677448.332951:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.332952:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.332952:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.332952:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:30.0:1619677448.332953:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:22.0:1619677448.332953:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.332953:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677448.332956:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:25.2:1619677448.332956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677448.332956:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.332957:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.332958:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:30.0:1619677448.332959:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677448.332960:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.332961:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:12.0:1619677448.332962:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:25.2:1619677448.332963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:12.0:1619677448.332964:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.332965:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.332966:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.332968:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.2:1619677448.332969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:12.0:1619677448.332969:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.332972:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.332974:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.332975:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.332976:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.332976:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.332978:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.332978:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677448.332979:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:30.0:1619677448.332980:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.332981:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.332983:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.332985:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.332986:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.332987:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.332988:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.332989:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.332992:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.332993:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.332996:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.332997:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.332997:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.332999:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.332999:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.333000:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.333000:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.333005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.333013:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.333015:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.333016:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.333018:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.333019:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.333021:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.333051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677448.333057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.333062:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.333064:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677448.333066:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.333066:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.333067:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677448.333068:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.333069:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.333070:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.333071:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.333072:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.333073:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.333074:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.333077:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.333079:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.333079:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.333080:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.333081:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:10.0:1619677448.333081:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:26.0:1619677448.333082:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:10.0:1619677448.333082:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.333083:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.333084:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.333085:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.333085:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.333086:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.333087:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.333088:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.333089:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.333089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:10.0:1619677448.333089:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.333091:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.333091:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.333093:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.333093:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.333094:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.333094:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.333095:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:26.0:1619677448.333096:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.333096:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.333097:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.333098:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.333098:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.333098:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.333099:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:21.2:1619677448.333101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:10.0:1619677448.333101:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.333101:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.333103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.333106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.333106:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:25.2:1619677448.333109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.333109:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.333110:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:6.0:1619677448.333111:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:2.0:1619677448.333111:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.333114:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:2.0:1619677448.333114:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.333115:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677448.333115:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.333115:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.333116:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.333116:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000400:00000200:2.0:1619677448.333116:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:26.0:1619677448.333117:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:2.0:1619677448.333117:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.333118:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.333118:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000400:00000200:6.0:1619677448.333118:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.333119:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:2.0:1619677448.333119:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.333120:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.333120:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:2.0:1619677448.333120:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.333121:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:6.0:1619677448.333121:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:2.0:1619677448.333121:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.333125:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.333126:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.333129:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.333130:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.333131:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.333133:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.333133:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.333134:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.333134:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677448.333137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:25.2:1619677448.333138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.333140:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.333143:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:26.0:1619677448.333145:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:20.0:1619677448.333145:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.333146:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677448.333147:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:20.0:1619677448.333147:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.333148:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.333148:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:20.0:1619677448.333149:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677448.333150:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.333151:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.333152:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.333153:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:20.0:1619677448.333153:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.333154:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677448.333157:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.333159:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.333161:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.333162:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.333163:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.333164:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.333165:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.333165:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.333166:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.333170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.333177:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.333180:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.333181:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.333182:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.333183:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.333184:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.333191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.333200:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.333202:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.333203:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.333204:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.333204:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.333205:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.333288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.333294:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.333300:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.333301:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677448.333304:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.333306:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.333307:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.333308:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.333310:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.333311:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.333312:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.333315:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.333317:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.333320:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.333320:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.333321:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.333322:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.333323:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.333324:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.333325:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.333326:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.333329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.333331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.333337:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.333338:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.333339:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677448.333340:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677448.333340:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677448.333341:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.333341:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.333342:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.333343:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.333345:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.333347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:6.0:1619677448.333354:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.333357:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.333357:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.333358:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.333359:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.333359:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.333428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.333436:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.333439:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.333441:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.333443:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.333444:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.333446:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.333447:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.333448:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.333449:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.333450:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.333454:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.333455:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.333458:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.333459:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.333460:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.333461:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.333462:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.333463:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.333465:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.333465:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.333467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.333471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.333474:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.333477:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.333477:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.333479:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:13.0:1619677448.333479:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.333481:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:13.0:1619677448.333481:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.333482:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677448.333482:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.333483:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677448.333558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.333566:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.333568:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.333570:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.333573:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.333573:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.333574:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.333575:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.333577:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.333578:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.333578:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.333581:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.333583:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.333586:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.333587:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.333587:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.333589:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.2:1619677448.333589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677448.333590:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.333591:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.333591:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:10.0:1619677448.333596:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.2:1619677448.333596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.333599:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.333600:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.333601:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.333604:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.333604:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:10.0:1619677448.333605:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:5.0:1619677448.333606:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:10.0:1619677448.333607:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.333607:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.333608:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.333609:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:10.0:1619677448.333609:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.333609:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.333610:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.333613:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.333613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.333615:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677448.333616:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.333617:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.333617:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.333618:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.333619:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677448.333619:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.333620:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677448.333621:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.333621:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677448.333621:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:10.0:1619677448.333622:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.0:1619677448.333622:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.333622:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:10.0:1619677448.333623:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.333626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.333633:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.333636:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.333637:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.333638:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.333640:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.333641:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.333704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.333710:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.333713:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.333714:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.333716:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.333718:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.333720:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.333721:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.333722:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.333723:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.333724:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.333727:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.333729:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.333730:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.333731:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.333732:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.333733:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.333734:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.333734:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.333735:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.333739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.333746:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.333748:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.333749:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.333749:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.333750:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.333752:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.333828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.333831:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.333834:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.333836:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.333837:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.333838:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.333839:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677448.333841:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.333842:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.333843:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.333844:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677448.333847:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.333848:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.333851:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.333851:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.333852:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.333854:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.333854:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.333855:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.333855:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.333859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.333866:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.333869:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.333869:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.333871:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.333872:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.333873:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.333914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.333922:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.333924:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.333926:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.333927:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.333930:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.333932:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.333933:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.333935:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.333936:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.333936:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.333939:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.333941:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.333943:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.333944:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.333945:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.333946:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.333947:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.333947:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.333948:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.333952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.333959:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.333961:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.333962:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.333963:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.333965:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.333966:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.334072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.334078:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.334080:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.334081:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.334083:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.334085:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.334086:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.334087:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.334089:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.334090:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.334091:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.334094:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.334096:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.334099:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.334100:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.334100:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.334102:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.334103:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.334104:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.334104:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.334108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.334115:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.334118:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677448.334118:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677448.334119:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.334121:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.334124:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677448.334157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.334164:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.334167:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.334168:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.334170:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.334172:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.334174:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677448.334175:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.334176:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.334176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:18.0:1619677448.334177:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.334179:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:21.2:1619677448.334182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000400:00000200:18.0:1619677448.334183:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.334183:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.334184:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.334185:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.334187:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.334187:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.334188:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.334189:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:2.0:1619677448.334189:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.334190:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:18.0:1619677448.334191:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.334191:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:26.0:1619677448.334192:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677448.334193:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:2.0:1619677448.334193:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.334195:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.334196:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.334197:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.334197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:2.0:1619677448.334198:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.334199:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.334199:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.334200:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:26.0:1619677448.334201:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:18.0:1619677448.334201:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:26.0:1619677448.334202:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96716334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.334203:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.334203:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.334204:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.334205:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.334205:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.334205:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.334206:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.334207:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.334207:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.334208:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.334210:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.334210:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.334211:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.334211:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677448.334211:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.334211:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.334213:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.334213:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.334214:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.334215:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.334215:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:26.0:1619677448.334216:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.334216:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (21)++
00000800:00000200:9.2:1619677448.334216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:26.0:1619677448.334217:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (22)++
00000800:00000200:26.0:1619677448.334219:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:26.0:1619677448.334220:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:8.2:1619677448.334220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (23)++
00000800:00000200:26.0:1619677448.334221:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677448.334222:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677448.334223:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:18.0:1619677448.334223:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.334226:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.334227:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.334227:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677448.334228:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)++
00000800:00000200:18.0:1619677448.334228:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:21.2:1619677448.334229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:18.0:1619677448.334229:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.0:1619677448.334230:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (24)--
00000400:00000200:20.0:1619677448.334231:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.334233:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.334235:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)++
00000800:00000200:20.0:1619677448.334236:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (24)--
00000800:00000200:10.0:1619677448.334236:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:5.0:1619677448.334236:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (24)--
00000400:00000200:5.0:1619677448.334237:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.334238:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.334239:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677448.334239:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:5.0:1619677448.334239:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.334240:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:5.0:1619677448.334241:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.334287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.334295:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.334301:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.334302:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.334305:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.334308:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.334309:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.334311:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.334312:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.334313:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.334314:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.334317:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.334318:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.334320:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.334321:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.334321:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.334323:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.334324:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.334324:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.334325:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.334330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.2:1619677448.334331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677448.334337:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:26.0:1619677448.334339:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.334340:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.334340:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.334342:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.334342:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.334343:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.334344:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.334345:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:26.0:1619677448.334345:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.334347:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.334347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:26.0:1619677448.334349:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.334350:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.334351:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.334352:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.334353:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677448.334354:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.334356:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.334356:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.334357:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.334357:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:26.0:1619677448.334357:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677448.334358:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.334359:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.334360:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.334361:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.334361:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.334363:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.334364:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.334365:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.334365:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.334369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.334377:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.334380:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.334381:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.334381:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.334383:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.334386:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.334438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.334444:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.334447:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.334448:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.334450:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.334452:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.334454:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.334455:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.334456:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.334457:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.334458:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.334462:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.334463:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.334466:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.334467:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.334467:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.334469:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.334470:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.334471:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.334472:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.334473:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.334475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.2:1619677448.334477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.2:1619677448.334478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.334481:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:18.0:1619677448.334484:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:10.0:1619677448.334484:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:1.0:1619677448.334484:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.334485:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.334485:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.334486:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.334487:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:1.0:1619677448.334487:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:18.0:1619677448.334488:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677448.334488:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.334488:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.334491:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.334492:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:1.0:1619677448.334494:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.334495:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.334496:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.334498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:1.0:1619677448.334498:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.334498:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.334499:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.334503:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.334504:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677448.334505:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.334507:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.334508:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:1.0:1619677448.334508:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.334509:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:31.0:1619677448.334510:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.334510:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.334511:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.334511:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.334512:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677448.334513:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.334513:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.334514:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.334516:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.334516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:31.0:1619677448.334517:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.334518:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.334519:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677448.334523:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.334523:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:31.0:1619677448.334524:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677448.334526:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:31.0:1619677448.334527:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677448.334527:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.334527:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:31.0:1619677448.334528:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.334528:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:24.0:1619677448.334529:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.334530:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.334531:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677448.334531:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:31.0:1619677448.334532:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.334532:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.334536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.334544:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.334547:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.334548:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.334548:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.334550:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.334554:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.334609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:20.2:1619677448.334615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:26.0:1619677448.334617:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:21.2:1619677448.334618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:26.0:1619677448.334620:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.334621:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.334622:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:30.0:1619677448.334623:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:6.0:1619677448.334623:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:26.0:1619677448.334625:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.334626:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:26.0:1619677448.334626:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:22.0:1619677448.334626:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.334626:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:26.0:1619677448.334628:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.334629:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.334629:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.334630:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.334630:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.334631:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.334632:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.334632:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.334632:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.334633:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.334635:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.334635:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.334636:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.334637:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000400:00000200:30.0:1619677448.334638:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.334639:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.334640:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.334640:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.334640:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.334641:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:6.0:1619677448.334642:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:26.0:1619677448.334643:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.334644:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.334644:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.334646:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.334646:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.334647:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:6.0:1619677448.334647:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.334648:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.334648:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:6.0:1619677448.334650:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.334652:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.2:1619677448.334652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.334653:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.334654:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.334654:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.334655:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:6.0:1619677448.334655:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.334656:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:6.0:1619677448.334656:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.334657:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:6.0:1619677448.334657:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677448.334658:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:6.0:1619677448.334658:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.334658:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677448.334660:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.334660:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677448.334661:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.334662:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677448.334662:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:25.2:1619677448.334663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:20.0:1619677448.334663:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.334663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:20.0:1619677448.334664:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.334666:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.2:1619677448.334667:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677448.334671:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:31.0:1619677448.334672:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:20.0:1619677448.334673:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677448.334673:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:31.0:1619677448.334674:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:18.0:1619677448.334674:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.334674:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:31.0:1619677448.334675:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.334675:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.334676:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:31.0:1619677448.334677:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.334677:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:18.0:1619677448.334677:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.334678:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:18.0:1619677448.334679:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:30.0:1619677448.334680:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677448.334681:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.334952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.334957:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.334960:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.334962:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.334962:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.334964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:5.0:1619677448.334965:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.334969:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.334970:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.334972:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:5.0:1619677448.334972:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.334973:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.334974:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677448.334975:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.334975:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677448.334978:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.334979:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.334980:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.334981:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:8.0:1619677448.334982:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.334983:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96716324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.334984:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677448.334985:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.334985:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:8.0:1619677448.334986:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.334986:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:8.0:1619677448.334987:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.334988:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:8.0:1619677448.334990:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.334992:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.334992:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.334993:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:25.2:1619677448.334994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.334994:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:8.0:1619677448.334995:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.334995:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.0:1619677448.334996:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.334997:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.334998:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.334999:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.334999:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:30.0:1619677448.335000:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.2:1619677448.335000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:8.0:1619677448.335000:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:30.0:1619677448.335003:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677448.335003:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000400:00000200:30.0:1619677448.335004:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.335005:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:8.2:1619677448.335005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:30.0:1619677448.335006:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.335006:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677448.335007:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.335007:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677448.335008:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.0:1619677448.335009:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.335011:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.335012:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.335014:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.335015:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.335017:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.335084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.335090:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.335093:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.335094:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.335096:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.335099:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.335100:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.335102:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.335103:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.335104:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.335105:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.335108:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.335110:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.335113:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.335114:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.335114:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.335116:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.335116:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.335118:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.335119:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.335119:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.335122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.2:1619677448.335124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.335128:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:1.0:1619677448.335129:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677448.335131:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677448.335131:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.0:1619677448.335132:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677448.335132:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677448.335133:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.335134:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677448.335135:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.335138:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677448.335196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.335203:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.335206:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.335206:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.335209:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.335212:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.335213:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.335214:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.335216:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.335217:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.335218:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.335221:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.335223:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.335226:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.335227:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.335227:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.335229:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.335229:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.335230:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.335230:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.335235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.335243:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.335245:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.335246:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.335247:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.335249:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.335252:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.335263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.335269:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.335272:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.335273:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.335275:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.335277:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.335279:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.335280:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.335282:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.335283:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.335284:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.335287:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.335289:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.335291:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.335292:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.335293:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.335299:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.335299:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.335300:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.335300:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.335306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.335314:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.335317:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.335318:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.335318:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.335320:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.335322:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.335368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.335374:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.335377:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.335378:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.335380:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.335382:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.335383:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.335385:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.335386:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.335387:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.335388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:6.0:1619677448.335388:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.335392:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.335393:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.335396:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:6.0:1619677448.335396:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.335396:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.335397:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.335399:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:6.0:1619677448.335399:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.335399:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.335400:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.335400:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:31.0:1619677448.335401:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.335403:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.335403:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.335404:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.335406:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.335406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:31.0:1619677448.335407:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.335408:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.335409:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.335412:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.335414:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677448.335414:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.335416:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:31.0:1619677448.335417:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.335417:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:18.0:1619677448.335417:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.335417:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.335418:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.335419:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:18.0:1619677448.335419:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.335420:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.335421:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.335422:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:18.0:1619677448.335422:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:25.2:1619677448.335425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.335432:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.335434:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.335435:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.335437:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.335438:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.335439:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.335491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.335499:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.335501:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.335502:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.335504:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.335507:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.335508:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.335510:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.335511:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.335512:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.335512:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.335515:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.335517:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.335520:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.335521:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.335521:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.335523:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.335524:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.335524:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.335525:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.335529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.335537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:2.0:1619677448.335537:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.335540:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.335541:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.335541:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.335543:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.335543:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677448.335546:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677448.335546:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:2.0:1619677448.335547:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:1.0:1619677448.335548:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.335551:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.335553:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.335554:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.335555:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.335556:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.335557:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.335560:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.335562:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.335565:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.335566:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.335566:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.335568:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.335569:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677448.335570:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677448.335571:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677448.335572:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:25.2:1619677448.335574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.335577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677448.335582:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:12.0:1619677448.335583:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677448.335584:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677448.335585:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677448.335586:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:12.0:1619677448.335586:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.335587:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.335588:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.335590:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.335592:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.335603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.335608:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.335611:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.335613:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.335613:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.335616:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.335618:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.335619:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.335620:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.335621:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.335622:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.335627:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.335628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.0:1619677448.335629:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.335632:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.335633:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.335634:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.335635:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.335636:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.335637:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.335637:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.335639:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.335639:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677448.335640:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:25.2:1619677448.335641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:31.0:1619677448.335642:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.335644:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.2:1619677448.335644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.335645:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.335646:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:20.0:1619677448.335647:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:31.0:1619677448.335648:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.335649:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.335649:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677448.335649:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:31.0:1619677448.335650:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.335650:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677448.335651:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.335651:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677448.335651:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:22.0:1619677448.335652:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.335653:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:31.0:1619677448.335655:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.335655:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.335657:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677448.335658:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.2:1619677448.335660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.335661:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.335662:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.335662:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.335664:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.335665:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.335666:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.335666:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.335667:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:9.2:1619677448.335670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.335670:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.335671:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.335671:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.335672:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.335673:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.335673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677448.335678:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.335680:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:10.0:1619677448.335680:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.335681:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.335682:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.335683:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.335684:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677448.335684:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.335686:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.335687:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.335688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000400:00000200:10.0:1619677448.335689:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.335690:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.335692:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.335693:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.335694:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.335695:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.335696:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.335698:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.335699:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:10.0:1619677448.335699:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.335700:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:10.0:1619677448.335701:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.335701:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.335702:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:10.0:1619677448.335704:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.335704:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.335705:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.2:1619677448.335705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.335707:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.335707:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.335708:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.335710:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.335710:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677448.335711:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.2:1619677448.335713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.335713:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.335714:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:4.2:1619677448.335716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677448.335716:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.335719:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.335720:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:20.0:1619677448.335722:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:1.0:1619677448.335722:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.335723:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:1.0:1619677448.335723:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.335724:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:1.0:1619677448.335724:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.335725:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:20.0:1619677448.335725:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.335725:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.335726:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677448.335727:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.335728:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.335728:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:1.0:1619677448.335729:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.335730:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677448.335731:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.335734:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.335734:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.335735:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.335737:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.335738:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677448.335739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:1.0:1619677448.335739:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.335739:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.335743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677448.335745:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.335748:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.335749:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:13.0:1619677448.335750:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:18.0:1619677448.335751:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.335753:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.335753:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:18.0:1619677448.335754:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:13.0:1619677448.335754:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.335756:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677448.335756:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.335757:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.335757:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:18.0:1619677448.335758:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.335759:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:18.0:1619677448.335760:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:25.2:1619677448.335763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:18.0:1619677448.335763:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.335765:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.335766:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.335767:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.335767:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.335769:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:12.0:1619677448.335769:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677448.335770:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.335771:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.335772:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:12.0:1619677448.335772:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.335773:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:12.0:1619677448.335773:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.335773:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.335774:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.335775:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677448.335775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.335778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.335783:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.335785:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.335786:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.335786:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:2.0:1619677448.335786:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.335787:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.335788:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.335788:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.335789:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677448.335790:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.335794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.335800:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.335803:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.335804:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.335804:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.335805:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.335805:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.335949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.335956:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.335959:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.335960:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.335961:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.335965:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.335966:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.335968:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.335969:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.335970:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.335971:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.335975:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.335976:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.335979:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.335980:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:4.2:1619677448.335980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677448.335981:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.335983:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.335983:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.335984:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.335985:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:10.0:1619677448.335987:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.335989:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:9.2:1619677448.335989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.335990:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.335992:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.335995:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.335996:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:2.0:1619677448.335996:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:10.0:1619677448.335997:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.335998:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.335999:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.335999:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.336000:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:10.0:1619677448.336000:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.336000:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.336003:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.336004:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.336006:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.336008:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.336009:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.336010:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.336011:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.336013:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.336014:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.336014:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.336015:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.336019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.336027:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.336029:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.336030:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.336030:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.336032:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.336035:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.336140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.336145:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.336148:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.336149:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.336151:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.336153:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.336154:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.336156:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716309] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.336157:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.336158:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.336159:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.336163:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.336164:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.336167:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.336168:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.336168:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.336170:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.336171:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.336171:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.336172:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.336176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.336183:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.336186:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.336187:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.336187:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.336189:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.336191:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.336217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.336224:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.336227:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.336228:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.336230:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.336232:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.336234:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.336235:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.336236:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.336237:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.336239:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.336242:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.336244:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.336247:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.336247:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.336248:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.336250:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.336250:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.336251:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.336251:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.336256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.336263:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.336266:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.336267:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.336267:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.336269:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.336272:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.336276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:4.2:1619677448.336277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677448.336283:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.336284:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.336286:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.336287:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:17.0:1619677448.336287:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.336287:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:10.0:1619677448.336288:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677448.336288:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.336289:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:26.0:1619677448.336290:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.336293:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.336299:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.336300:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.336301:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.336302:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.336303:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.336306:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.336308:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.336310:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.336311:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.336312:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.336313:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.336314:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.336315:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.336315:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.336320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.336329:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.336332:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.336333:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.336335:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.336335:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.336338:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.336340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.336347:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.336350:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.336351:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.336351:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.336352:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.336353:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.336569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.336577:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.336580:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.336581:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.336582:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.336585:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.336586:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.336588:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.336589:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.336590:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.336591:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.336594:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.336596:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.336598:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.336599:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.336600:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.336602:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.336602:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.336603:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.336604:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.336609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.336617:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.336620:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.336621:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.336621:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:21.2:1619677448.336622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:18.0:1619677448.336623:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.336626:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677448.336627:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.336630:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.336631:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.336633:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.336635:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.336637:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.336639:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.336640:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.336641:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.336642:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.336645:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.336647:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.336649:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.336650:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.336651:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.336652:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.336653:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.336654:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.336655:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.336656:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.336659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.336661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.336665:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:20.0:1619677448.336666:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:22.0:1619677448.336667:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.336668:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677448.336668:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677448.336669:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:20.0:1619677448.336669:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.336671:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.336672:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.336673:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.336676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.336683:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.336686:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.336687:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.336688:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.336688:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.336690:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.336742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.336750:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.336753:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.336755:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.336757:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.336758:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.336760:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.336761:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.336762:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.336763:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.336764:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.336767:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.336769:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.336771:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.336772:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.336773:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.336774:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.336775:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.336776:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.336777:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.336780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.336782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:13.0:1619677448.336788:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677448.336789:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:13.0:1619677448.336790:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.336791:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.336792:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:13.0:1619677448.336792:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.336794:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.336795:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:31.0:1619677448.336796:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.336796:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:13.0:1619677448.336796:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677448.336798:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677448.336799:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.336800:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.336800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:31.0:1619677448.336801:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.336802:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677448.336805:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.336805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677448.336806:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.336807:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677448.336809:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.336809:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.336810:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:12.0:1619677448.336810:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.336810:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677448.336811:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:12.0:1619677448.336811:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.2:1619677448.336811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.336811:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677448.336812:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:12.0:1619677448.336812:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.336813:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677448.336813:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677448.336814:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:2.0:1619677448.336814:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.336815:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.336817:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.336818:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:9.2:1619677448.336818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:2.0:1619677448.336819:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.336820:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:2.0:1619677448.336821:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:1.0:1619677448.336821:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:2.0:1619677448.336822:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.336824:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.336824:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:2.0:1619677448.336824:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.336825:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.336826:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677448.336827:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.336827:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.336828:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:17.0:1619677448.336828:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.336828:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.336830:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96716301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.336830:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.336831:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.336831:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.336833:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.336833:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:17.0:1619677448.336834:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.336834:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:2.0:1619677448.336835:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.336836:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.336837:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:17.0:1619677448.336838:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.336839:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.336839:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677448.336840:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.336840:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.336841:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677448.336843:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.336844:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.336845:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:9.2:1619677448.336845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.336846:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.336847:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.336847:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.336848:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.336852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677448.336852:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.336855:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.336855:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.336856:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.336857:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.336859:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677448.336859:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.336861:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.336862:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.336864:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.336865:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.336866:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677448.336866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.2:1619677448.336870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677448.336874:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.336875:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.336877:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.336878:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.336878:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677448.336878:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.336879:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.336880:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677448.336881:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.336881:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.336883:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.336884:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.336886:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.336887:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.336889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000400:00000200:5.0:1619677448.336889:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.336890:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.336894:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000400:00000200:5.0:1619677448.336894:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.336896:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.336897:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677448.336898:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:5.0:1619677448.336899:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.336900:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.336901:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.336901:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:6.0:1619677448.336903:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.336903:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:6.0:1619677448.336904:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:5.0:1619677448.336904:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.336904:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.336905:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:6.0:1619677448.336906:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.336907:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.336908:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.336909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:6.0:1619677448.336909:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.336913:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.336914:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.336916:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:6.0:1619677448.336917:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.336918:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:30.0:1619677448.336920:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:6.0:1619677448.336920:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (24)++
00000400:00000200:30.0:1619677448.336921:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.336923:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (24)--
00000400:00000200:30.0:1619677448.336923:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.336923:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:6.0:1619677448.336923:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677448.336924:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677448.336925:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:30.0:1619677448.336926:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:25.2:1619677448.336928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.336935:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.336938:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.336939:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.336940:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.336942:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.336943:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.337129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.337136:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.337139:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.337140:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.337142:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.337145:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.337147:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.337148:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.337149:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.337150:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.337151:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.337154:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.337156:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.337159:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.337159:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.337160:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.337162:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.337162:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.337164:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.337165:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.337165:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.337168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:4.2:1619677448.337171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.337175:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.337178:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:10.0:1619677448.337178:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:22.0:1619677448.337179:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.337179:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.337180:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.337180:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.337181:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.337182:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.337185:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.337194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.337202:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.337205:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.337206:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.337208:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.337210:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.337212:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.337213:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.337214:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.337215:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.337216:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.337219:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.337220:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.337222:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.337223:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.337223:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.337225:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.337225:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.337226:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.337227:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.337228:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.337230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.2:1619677448.337233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.337238:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.337240:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.337240:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.337242:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:10.0:1619677448.337242:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.337243:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.337243:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.337244:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.337244:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.337245:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:4.2:1619677448.337267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.337274:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.337277:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.337278:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.337279:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.337282:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.337283:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.337284:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.337285:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.337286:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.337287:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.337290:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.337292:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.337300:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.337301:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.337301:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.337303:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.337303:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.337304:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.337305:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.337309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:21.2:1619677448.337311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.337317:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677448.337317:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:6.0:1619677448.337319:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.337320:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677448.337320:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.337321:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.337321:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.337322:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.337323:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.337324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:6.0:1619677448.337325:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.337326:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:1.0:1619677448.337327:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:6.0:1619677448.337328:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.337329:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.337330:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.337331:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.337333:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:6.0:1619677448.337334:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.337335:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:6.0:1619677448.337336:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.337338:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.337339:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.337340:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.337340:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.337340:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.337340:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.337342:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:6.0:1619677448.337342:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.337342:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:30.0:1619677448.337343:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.337343:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:30.0:1619677448.337344:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.337344:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:30.0:1619677448.337345:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.337346:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:8.2:1619677448.337348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.337350:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.337352:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.337354:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.337355:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.337356:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.337356:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677448.337357:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.337358:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.337358:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677448.337359:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.337359:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.337359:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.337361:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.337362:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.337363:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:25.2:1619677448.337364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.337371:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.337374:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.337375:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.337376:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.337379:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.337380:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.337390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.337397:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.337400:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.337402:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.337404:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.337404:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.337405:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.337407:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.337408:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.337409:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.337410:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.337413:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.337414:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.2:1619677448.337415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677448.337416:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.337417:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.337417:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.337419:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.337419:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.337420:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.337421:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.337422:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.337423:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677448.337423:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:25.2:1619677448.337424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.337424:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.337426:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.337428:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.337429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677448.337429:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.337431:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.337431:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:17.0:1619677448.337432:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.337433:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.337433:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:17.0:1619677448.337434:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677448.337434:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.337436:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:13.0:1619677448.337436:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.337437:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:17.0:1619677448.337438:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.337439:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:13.0:1619677448.337439:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.337440:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.337440:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677448.337440:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.337442:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.337443:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.337444:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.337445:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.337445:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.337446:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.337447:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.337451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.337458:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.337460:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.337461:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.337461:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.337463:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.337465:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.337470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.337477:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.337479:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.337480:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.337481:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.337481:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.337482:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.337611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.337617:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.337619:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.337621:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.337622:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.337624:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.337626:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.337627:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.337629:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.337629:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.337631:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.337634:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.337636:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.337639:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.337640:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.337641:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.337642:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.337643:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.337643:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.337644:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.337648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.337655:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.337658:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.337659:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.337661:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.337662:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.337663:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.337666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677448.337672:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.337676:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.337677:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.337678:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.337679:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.337680:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.337734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.337740:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.337743:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.337744:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.337746:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.337748:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.337749:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.337750:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.337752:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.337753:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.337754:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.337757:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.337758:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.337760:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.337761:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.337761:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.337763:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.337763:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.337764:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.337765:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.337766:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677448.337769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.337772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.337778:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.337779:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.337779:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.337779:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.337780:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.337780:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.337781:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.337781:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.337782:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677448.337783:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.337809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.337814:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.337817:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.337818:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.337820:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.337821:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.337822:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.337824:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.337825:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.337826:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.337827:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.337830:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.337832:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.337833:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.337834:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.337834:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.337836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677448.337836:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.337836:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.337837:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.337839:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.337839:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.337842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.337844:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.2:1619677448.337844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.337847:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.337848:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.337850:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.337850:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:26.0:1619677448.337851:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.337851:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.337851:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:17.0:1619677448.337852:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.337852:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.337853:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.337854:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:1.0:1619677448.337854:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:26.0:1619677448.337855:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.337855:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.337855:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677448.337857:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.337857:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:26.0:1619677448.337858:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.337859:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.337862:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.337864:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.337867:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.337868:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.337868:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.337870:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.337870:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.337871:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.337872:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.337876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.337882:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.337884:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.337885:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.337887:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.337888:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.337889:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.337981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.337989:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.337991:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.337993:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.337995:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.337995:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.337997:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.337998:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.337999:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.338000:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.338001:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.338004:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.338005:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.2:1619677448.338005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:31.0:1619677448.338008:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.338008:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.338008:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:31.0:1619677448.338009:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.338010:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.338010:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:31.0:1619677448.338011:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.338011:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.338012:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:20.0:1619677448.338012:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.338013:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.338015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.338016:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.338017:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677448.338019:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.338020:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.338021:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.338022:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:13.0:1619677448.338022:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.338024:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:20.0:1619677448.338025:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677448.338025:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.338026:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.338027:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:12.0:1619677448.338028:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.338029:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677448.338031:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.338032:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677448.338032:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677448.338034:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677448.338035:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.338036:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.338037:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677448.338040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.338047:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.338049:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.338050:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.338050:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.338053:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.338056:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.2:1619677448.338075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.338080:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.338083:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.338084:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.338086:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.338087:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.338089:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.338090:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.338092:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.338093:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.338094:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.338097:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.338099:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.338102:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.338103:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.338103:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.338105:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.338106:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.338106:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.338107:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.338111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.338117:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.338120:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.338121:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.338123:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.338124:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.338126:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.338172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.338179:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.338182:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.338184:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.338186:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.338186:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.338188:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.338189:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716284] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.338190:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.338192:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.338193:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.338196:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.338198:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.338201:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.338201:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.338202:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:4.2:1619677448.338202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.338204:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.338205:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.338205:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.338206:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:10.0:1619677448.338209:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677448.338210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.338212:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.338213:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.338214:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.338217:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.338218:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:10.0:1619677448.338219:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.338220:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.338221:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.338221:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:10.0:1619677448.338222:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.338222:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.338223:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.338224:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.338224:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.338227:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.338229:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.338229:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:10.0:1619677448.338231:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.338232:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.338232:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.338234:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.338234:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.338235:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.338236:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.338239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.2:1619677448.338239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677448.338246:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:26.0:1619677448.338246:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.338249:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:26.0:1619677448.338249:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:31.0:1619677448.338250:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.338250:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:31.0:1619677448.338251:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.338252:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.338253:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.338254:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:26.0:1619677448.338255:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.338257:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.338258:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.338259:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.338260:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.338261:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.338264:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.338266:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.338268:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.338269:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.338270:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.338271:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.338272:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.338272:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.338273:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.338277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.338285:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.338288:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.338288:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.338289:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.338291:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.338297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.338302:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.338303:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.338305:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.338305:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.338307:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.338307:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.338308:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.338467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.338473:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.338476:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.338476:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.338478:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.338481:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.338483:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.338484:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.338485:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.338486:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.338487:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.338490:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.338492:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.338495:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.338496:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.338496:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.338498:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.338498:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.338499:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.338500:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.338504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.338511:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.338514:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.338515:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.338517:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.338518:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.338519:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.338521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.338527:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.338531:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.338532:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.338532:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.338533:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.338534:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.338561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677448.338568:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677448.338570:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677448.338572:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.338574:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.338574:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677448.338576:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:13.0:1619677448.338577:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677448.338579:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.338580:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677448.338581:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:13.0:1619677448.338584:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.338586:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.338589:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.338589:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:13.0:1619677448.338590:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:13.0:1619677448.338592:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:13.0:1619677448.338593:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677448.338593:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677448.338594:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677448.338599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.338605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677448.338607:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.338610:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.338610:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.338612:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677448.338612:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.338613:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.338615:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677448.338615:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:30.0:1619677448.338617:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.338620:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.338620:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.338621:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.338623:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.338624:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.338625:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.338626:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.338629:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.338630:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.338633:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.338633:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.338634:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.338635:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.338636:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.338637:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.338637:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.338641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.338648:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.338650:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.338651:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.338651:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.338653:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.338656:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.338659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.338665:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.338667:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.338668:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.338668:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.338669:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.338669:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.338671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.338678:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.338681:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.338682:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.338683:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.338686:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.338687:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.338688:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.338689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677448.338690:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.338691:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.338691:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.338695:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.338695:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.338696:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:1.0:1619677448.338697:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.338698:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.338698:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.338699:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.338700:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:1.0:1619677448.338701:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.338702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677448.338702:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.338702:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.338703:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.338703:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:1.0:1619677448.338703:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.338705:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.338706:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.338707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:1.0:1619677448.338707:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.338708:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.338709:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.338712:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.338713:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:1.0:1619677448.338713:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.338714:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.338715:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:31.0:1619677448.338716:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.338716:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:1.0:1619677448.338716:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.338717:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.338717:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:31.0:1619677448.338718:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.338718:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677448.338719:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.338720:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:1.0:1619677448.338720:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.338720:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.338721:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.338721:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677448.338722:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96716276] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.338723:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.338724:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.338725:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:8.2:1619677448.338725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.338728:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:31.0:1619677448.338729:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.338729:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.338730:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.338731:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.338731:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.338732:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.338733:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.338733:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:17.0:1619677448.338733:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.338734:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.338734:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677448.338735:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.338735:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677448.338735:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.338736:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.338737:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677448.338737:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.338738:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.338738:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.338739:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:17.0:1619677448.338740:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677448.338741:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.338741:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.338743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.338747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:21.2:1619677448.338749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:5.0:1619677448.338752:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:4.2:1619677448.338753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677448.338755:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:5.0:1619677448.338755:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:18.0:1619677448.338756:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:5.0:1619677448.338756:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.338757:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.338758:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:5.0:1619677448.338758:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.338759:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:18.0:1619677448.338759:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:4.2:1619677448.338759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677448.338760:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.338760:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:24.0:1619677448.338761:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:18.0:1619677448.338762:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.338762:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:24.0:1619677448.338764:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:18.0:1619677448.338764:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.338765:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677448.338766:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:18.0:1619677448.338766:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.338768:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.338769:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:24.0:1619677448.338769:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.338769:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677448.338770:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.338771:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.338772:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.338772:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.338773:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.338774:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.338775:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96716274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.338776:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.338777:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.338778:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.338779:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.338779:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:18.0:1619677448.338779:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.338781:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:18.0:1619677448.338782:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.338783:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96716273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.338783:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.338783:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:26.0:1619677448.338784:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.338784:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.338785:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.338785:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.338786:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.338786:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.338786:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:24.0:1619677448.338787:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:18.0:1619677448.338787:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:26.0:1619677448.338789:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.338790:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.338791:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:25.2:1619677448.338791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:24.0:1619677448.338791:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.338791:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.338793:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.338794:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677448.338795:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.338795:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:26.0:1619677448.338796:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.338796:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (23)++
00000800:00000200:24.0:1619677448.338796:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.338797:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677448.338798:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (24)--
00000800:00000200:10.0:1619677448.338798:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (24)++
00000800:00000200:26.0:1619677448.338799:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.338799:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (24)--
00000800:00000200:8.2:1619677448.338799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.338800:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.338800:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:10.0:1619677448.338800:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.338801:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.338802:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.338802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.2:1619677448.338803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.338806:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.338807:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:1.0:1619677448.338807:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:10.0:1619677448.338808:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.338809:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.338809:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.338810:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.338810:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.338811:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:10.0:1619677448.338811:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.338811:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.338812:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:10.0:1619677448.338813:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:8.0:1619677448.338813:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677448.338813:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.338814:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677448.338817:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.339072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.339077:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.339080:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.339080:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.339082:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.339084:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.339085:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.339086:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.339087:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.339089:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.339090:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.339093:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.339095:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.339096:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.339097:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.339098:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.339099:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.339099:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.339101:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.339102:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.339102:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.339105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.339108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.339113:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.339114:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.339114:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.339114:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.339115:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.339115:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.339116:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.339117:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.339117:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677448.339118:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:21.2:1619677448.339212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.339218:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.2:1619677448.339221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677448.339221:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.339222:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.339224:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.339226:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.339227:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.339229:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.339230:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:2.0:1619677448.339230:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.339231:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.339232:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.339233:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677448.339236:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.339237:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.339237:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:30.0:1619677448.339240:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.339240:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.339242:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.339243:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.339244:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.339244:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.339244:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:30.0:1619677448.339246:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.339246:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:30.0:1619677448.339247:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.339247:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.339247:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:30.0:1619677448.339248:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.339248:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:30.0:1619677448.339251:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.339252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.339253:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.339256:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.339257:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.339258:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.339259:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:30.0:1619677448.339260:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.339260:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.339261:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.339262:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:6.0:1619677448.339262:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.339263:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.339263:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.339265:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:8.2:1619677448.339266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.339268:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.339272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677448.339272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:20.0:1619677448.339277:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677448.339278:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677448.339279:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.339279:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:20.0:1619677448.339280:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.339281:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:5.0:1619677448.339281:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.339282:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.339282:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:5.0:1619677448.339284:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.339285:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:18.0:1619677448.339285:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.339287:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:18.0:1619677448.339287:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.339287:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.339288:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.339289:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000400:00000200:5.0:1619677448.339289:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:20.0:1619677448.339290:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.339290:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:18.0:1619677448.339290:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:20.0:1619677448.339291:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.339291:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.339292:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:18.0:1619677448.339292:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.339298:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.339298:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.339299:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.339300:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.339301:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677448.339302:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:5.0:1619677448.339305:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.339307:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:18.0:1619677448.339309:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.339311:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.339312:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.339313:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.339313:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677448.339314:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.339314:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.339315:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.339316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:5.0:1619677448.339316:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.339316:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677448.339317:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.339317:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677448.339318:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.339318:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677448.339318:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677448.339319:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.339322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.339324:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:8.2:1619677448.339325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.339327:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.339328:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.339330:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677448.339331:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:26.0:1619677448.339332:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.339332:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.339333:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.339333:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:26.0:1619677448.339334:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:5.0:1619677448.339334:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.339335:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.339335:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.339336:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96716267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.339337:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.339337:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.339338:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.339338:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.339338:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677448.339339:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:26.0:1619677448.339340:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:20.0:1619677448.339340:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:26.0:1619677448.339343:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.339345:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.339348:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.339349:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.339350:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.339352:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.339352:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.339353:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677448.339354:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.339355:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677448.339358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677448.339361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:1.0:1619677448.339366:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:10.0:1619677448.339368:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:1.0:1619677448.339368:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.339369:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.339370:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:1.0:1619677448.339370:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.339371:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.339372:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:1.0:1619677448.339372:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:1.0:1619677448.339375:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.339377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.339384:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.339387:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.339388:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.339388:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.339389:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.339390:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.339430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.339438:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.339441:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.339444:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.339446:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.339446:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.339448:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.339449:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.339450:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.339451:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.339452:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.339455:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.339457:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.339459:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.339460:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.339460:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.339462:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.339463:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.339464:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.339464:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.339468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.339476:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.339478:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.339479:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.339481:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.339483:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.339484:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.339502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.339509:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.339512:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.339512:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.339514:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.339517:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.339518:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.339519:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.339520:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.339521:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.339522:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.339525:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.339526:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.339528:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.339528:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.339529:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.339530:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.339530:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.339531:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.339531:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.339535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.339541:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.339543:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.339544:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.339545:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.339545:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.339547:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.339593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.0:1619677448.339598:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.339600:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677448.339602:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677448.339603:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.339605:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.339606:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.339607:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.339609:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.339610:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.339610:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.339613:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.339615:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.339616:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.339617:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.339618:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.339619:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.339620:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.339620:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.339621:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.339625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.339632:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.339634:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.339635:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.339637:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.339638:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.339639:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:8.2:1619677448.339652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.339659:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.339662:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.339663:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.339663:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677448.339664:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.339665:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:21.2:1619677448.339760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.339767:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.339770:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.339771:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.339773:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.339775:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.339777:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.339778:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.339780:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.339781:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.339782:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677448.339786:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.339787:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.339791:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.339791:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.339792:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.339794:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.339794:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.339795:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.339796:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.339796:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.2:1619677448.339799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.339802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:21.2:1619677448.339803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677448.339808:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:30.0:1619677448.339810:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:8.0:1619677448.339810:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:5.0:1619677448.339810:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.339811:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.339812:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:30.0:1619677448.339813:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:8.0:1619677448.339813:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:5.0:1619677448.339813:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.339814:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.339814:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:30.0:1619677448.339815:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.339817:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677448.339817:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:30.0:1619677448.339818:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.339820:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.339821:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.339822:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.339823:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.339824:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.339827:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.339829:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.339832:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.339833:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.339834:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.339835:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.339836:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.339837:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.339838:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.339842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.339850:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.339852:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.339853:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.339855:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.339855:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.339858:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.339870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.339877:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.339880:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.339881:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.339882:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.339885:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.339887:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.339888:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.339889:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.339890:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.339891:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.339894:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.339896:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.339899:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.339899:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.339900:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.339902:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.339902:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.339903:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.339903:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.339908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.339916:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.339919:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.339920:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.339920:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.339922:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.339924:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.340068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677448.340074:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677448.340077:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677448.340078:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677448.340080:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.340082:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.340084:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677448.340085:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.340087:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.340088:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.340088:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677448.340091:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.340093:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677448.340096:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.340097:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:2.0:1619677448.340097:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:2.0:1619677448.340099:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:2.0:1619677448.340100:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677448.340101:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677448.340102:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677448.340103:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:9.2:1619677448.340106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.340108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (21)++
00000800:00000200:8.0:1619677448.340112:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:18.0:1619677448.340114:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:18.0:1619677448.340115:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:8.0:1619677448.340115:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.340116:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:18.0:1619677448.340116:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:8.0:1619677448.340116:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.340117:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000400:00000200:8.0:1619677448.340118:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.340122:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.2:1619677448.340160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.340168:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.340170:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.340171:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.340173:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.340175:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.340176:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.340178:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.340179:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.340180:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.340181:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.340184:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.340186:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.340189:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.340189:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.340190:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.340192:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.340192:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.340193:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.340194:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.340195:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.340199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.340200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.2:1619677448.340204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:12.0:1619677448.340205:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.340207:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.340208:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.340208:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:12.0:1619677448.340209:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.340210:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.340210:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.0:1619677448.340210:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677448.340211:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.340212:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.0:1619677448.340212:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:12.0:1619677448.340213:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677448.340214:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.340215:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.340217:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.340219:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.340220:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.340221:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.340222:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.340223:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.340227:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.340229:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.340232:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.340233:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.340233:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.340235:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.340236:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.340238:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.340239:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.340239:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.340241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.340244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.340247:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677448.340248:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.340250:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.340250:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677448.340250:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677448.340251:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:13.0:1619677448.340251:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.340253:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.340254:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.340256:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.340396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:17.0:1619677448.340402:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677448.340405:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677448.340405:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677448.340407:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.340409:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.340411:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.340412:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.340413:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.340414:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.340415:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.340419:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.340421:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.340424:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.340425:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.340425:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.340427:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.340428:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677448.340428:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677448.340429:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:25.2:1619677448.340433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.340441:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.340444:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.340445:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.340446:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.340448:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.340449:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.340456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.340462:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.340465:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.340466:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.340466:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.340468:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.340469:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.340549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.340555:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.340558:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.340558:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.340561:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.340563:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.340564:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.340565:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.340567:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.340568:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.340568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:22.0:1619677448.340569:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:21.2:1619677448.340571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000400:00000200:22.0:1619677448.340572:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.340574:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.340575:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677448.340577:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.340577:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677448.340577:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:31.0:1619677448.340578:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677448.340578:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677448.340579:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:22.0:1619677448.340580:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.340580:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:31.0:1619677448.340581:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.340581:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.340581:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677448.340582:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:30.0:1619677448.340583:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677448.340583:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.340585:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.340586:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677448.340586:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.340586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:20.0:1619677448.340588:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.340589:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.340590:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.340591:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.340592:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.340593:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.340593:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.340594:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.340594:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:20.0:1619677448.340595:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.2:1619677448.340597:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:2.0:1619677448.340597:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.340598:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.340599:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.340599:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.340600:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.340601:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:20.0:1619677448.340602:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.340602:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:2.0:1619677448.340603:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:31.0:1619677448.340604:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.340604:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:31.0:1619677448.340605:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677448.340605:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 440 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.340606:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:5.0:1619677448.340606:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677448.340608:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.340608:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.340608:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.340609:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:5.0:1619677448.340609:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.340610:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677448.340610:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.340611:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:5.0:1619677448.340611:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:31.0:1619677448.340612:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.340612:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677448.340612:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96716253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.340613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:5.0:1619677448.340614:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.340616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:5.2:1619677448.340617:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:25.2:1619677448.340618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677448.340620:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.340621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:5.0:1619677448.340621:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.340622:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:18.0:1619677448.340622:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:1.0:1619677448.340622:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.340623:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:2.0:1619677448.340623:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.0:1619677448.340624:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:18.0:1619677448.340624:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:13.0:1619677448.340624:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.340625:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:18.0:1619677448.340625:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.340626:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:4.2:1619677448.340626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677448.340626:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:1.0:1619677448.340626:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:13.0:1619677448.340627:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:5.0:1619677448.340627:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000400:00000200:18.0:1619677448.340628:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:13.0:1619677448.340628:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.340630:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677448.340630:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.340630:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.340630:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.340631:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.340631:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.340632:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:18.0:1619677448.340632:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.340633:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677448.340633:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677448.340633:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.340633:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.340634:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677448.340634:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:2.0:1619677448.340634:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.340634:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:30.0:1619677448.340635:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677448.340635:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:13.0:1619677448.340635:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.340635:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:2.0:1619677448.340636:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.340636:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.340637:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:5.0:1619677448.340637:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.340637:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:10.0:1619677448.340638:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.340638:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.340638:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:2.0:1619677448.340639:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.340639:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.340641:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.340641:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.340641:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.340642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:2.0:1619677448.340642:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.340642:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.340643:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.340644:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.340646:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677448.340647:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.340648:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96716250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.340649:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:26.0:1619677448.340650:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.340650:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.340651:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.340652:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.340652:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.340652:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.340653:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.340654:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.340655:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.340655:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.340655:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.340655:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.340656:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.340657:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.340657:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.340658:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.340658:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.340658:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.340659:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:2.0:1619677448.340659:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.340659:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.340660:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.340660:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.340662:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.340662:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677448.340663:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677448.340663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677448.340663:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:26.0:1619677448.340664:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.340664:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.340664:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:26.0:1619677448.340666:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.340666:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.340667:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:25.2:1619677448.340667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.340669:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.340670:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.340672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:12.0:1619677448.340672:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.340673:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.340674:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.340675:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:4.2:1619677448.340675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677448.340676:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.340676:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:13.0:1619677448.340677:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.340679:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.340679:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.340680:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677448.340680:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:12.0:1619677448.340681:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.340682:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.340682:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:10.0:1619677448.340682:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:13.0:1619677448.340683:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.340683:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.340683:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677448.340683:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.340684:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.340684:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:13.0:1619677448.340685:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.340685:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.340685:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.340686:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.340687:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.340690:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.340759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.340767:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.340770:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.340772:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.340774:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.340776:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.340777:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.340778:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.340780:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.340781:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.340782:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.340785:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.340786:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.340789:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.340789:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.2:1619677448.340789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:31.0:1619677448.340790:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.340791:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.340792:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.340793:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.340793:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:18.0:1619677448.340795:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:25.2:1619677448.340797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:18.0:1619677448.340798:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.340799:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.340801:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.340803:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:18.0:1619677448.340804:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.340805:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.340806:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.340806:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.340807:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.340808:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.340808:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.340809:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.340810:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.340810:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:18.0:1619677448.340810:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.340813:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.340815:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.340818:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.340818:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.340819:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.340821:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.340821:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.340822:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.340822:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.340827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677448.340836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677448.340836:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677448.340838:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677448.340838:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.340839:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.340839:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.340841:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:17.0:1619677448.340842:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.340844:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.340844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.340844:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.340847:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.340847:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:20.0:1619677448.340848:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:17.0:1619677448.340848:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677448.340849:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716247] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.340850:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.340851:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.340852:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.340852:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000400:00000200:20.0:1619677448.340853:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.340853:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.340853:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.340854:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:20.0:1619677448.340857:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.340859:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.340861:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.340861:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.340862:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.340863:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.340864:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.340865:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.340865:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.340869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.340877:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.340880:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.340881:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.340882:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.340883:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.340885:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.340895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.340903:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.340905:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.340906:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.340907:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.340907:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.340908:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.340908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.340916:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.340918:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.340919:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.340921:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.340924:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.340925:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.340927:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.340928:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.340929:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.340930:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.340933:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.340935:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.340937:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.340938:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.340939:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.340940:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.340941:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.340941:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.340942:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.340946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.340952:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.340955:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.340956:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.340957:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.340959:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.340961:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.341029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.341035:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.341038:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.341038:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.341040:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.341043:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.341044:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.341045:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716245] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.341047:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.341048:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.341049:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.341052:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.341054:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.341057:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.341057:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.341058:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.341060:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.341060:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.341061:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.341061:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.341066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.341073:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.341075:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.341076:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.341078:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.341079:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.341080:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.341096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.341101:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.341104:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.341105:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.341107:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.341110:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.341111:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.341112:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.341114:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.341115:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.341116:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.341119:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.341121:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.341123:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.341124:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.341125:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.341126:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.341127:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.341127:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.341128:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.341132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.341134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677448.341140:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.341141:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.341143:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.341144:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.341144:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:6.0:1619677448.341144:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.341145:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.341146:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:10.0:1619677448.341147:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.341149:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:10.0:1619677448.341150:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.341152:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.341153:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.341154:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.341155:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.341156:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.341160:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.341161:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.341165:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.341165:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.341166:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.341168:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.341168:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.341169:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.341170:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.341174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:21.2:1619677448.341177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.341182:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.341183:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.341183:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.341184:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.341185:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.341186:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.341188:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.341190:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.341191:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.341191:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677448.341192:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.341193:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.341194:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.341196:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.341196:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.341197:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.341201:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.341203:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.341205:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.341206:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.341207:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.341209:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.341209:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.341210:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.341210:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.341215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.341221:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.341224:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.341224:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.341225:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.341227:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.341230:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:25.2:1619677448.341235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000800:00000200:10.0:1619677448.341241:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.341244:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.341244:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.341245:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.341246:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.341246:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677448.341327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.341334:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.341337:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.341339:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.341341:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.341342:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.341343:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.341344:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.341345:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.341346:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.341347:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.341350:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.341352:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.341354:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.341355:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.341356:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.341357:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.341358:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.341359:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.341359:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.341364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.341371:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.341373:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.341374:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.341376:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.341378:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.341379:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.341383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.341389:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.341392:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.341392:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.341393:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.341394:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.341394:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.341414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.341417:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.341419:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.341422:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.341423:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.341424:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.341425:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.341427:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.341428:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.341429:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.341430:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.341433:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.341435:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.341438:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.341439:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.341439:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.341441:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.341442:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.341442:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.341443:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.341447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:21.2:1619677448.341450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:30.0:1619677448.341454:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.341456:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.341457:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.341457:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:30.0:1619677448.341459:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.341460:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:24.0:1619677448.341460:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.341461:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:30.0:1619677448.341462:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:24.0:1619677448.341462:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.341464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:24.0:1619677448.341465:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.341466:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.341468:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.341469:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.341470:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.341470:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:24.0:1619677448.341471:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.341473:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:24.0:1619677448.341474:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.341474:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677448.341476:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:17.0:1619677448.341476:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.341479:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.341479:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.341480:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.341481:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:17.0:1619677448.341481:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:24.0:1619677448.341482:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:17.0:1619677448.341482:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.341483:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.341483:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:17.0:1619677448.341483:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.341484:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:17.0:1619677448.341484:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.341485:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.2:1619677448.341488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:17.0:1619677448.341489:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.341491:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.341493:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.341494:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.341495:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.341496:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677448.341497:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.341497:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.341498:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.341498:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677448.341499:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.341499:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.341501:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.341502:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:25.2:1619677448.341503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.341504:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677448.341510:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.341513:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.341514:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.341515:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.341516:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.341518:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.341520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.341526:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.341529:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.341530:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.341531:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.341531:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.341533:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.341558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.341566:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.341568:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.341569:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.341571:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.341574:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.341575:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.341576:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.341578:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.341579:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.341580:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.341583:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.341584:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.341588:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.341588:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.341589:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.341591:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.341591:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.341592:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.341592:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.341597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.341605:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.341607:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.341608:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.341609:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.341611:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.341614:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.341619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.341626:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.341628:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.341629:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.341629:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.341630:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.341631:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.341662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.341666:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.341669:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.341669:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.341671:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.341674:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.341675:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.341676:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.341677:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.341678:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.341679:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.341682:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.341684:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.341687:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.341688:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.341688:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.341690:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.341691:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.341692:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.341693:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.341693:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:25.2:1619677448.341696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:21.2:1619677448.341698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.341703:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:10.0:1619677448.341703:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:22.0:1619677448.341705:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.341706:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:10.0:1619677448.341706:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.341707:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.341707:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677448.341707:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.341709:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.341712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:10.0:1619677448.341713:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.0:1619677448.341715:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.341717:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.341719:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.341720:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.341722:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.341724:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677448.341725:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.341727:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.341727:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.341729:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677448.341732:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.341734:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677448.341737:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.341738:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677448.341739:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.341740:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677448.341741:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.341742:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.341742:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.341746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.341753:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.341756:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.341756:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.341758:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677448.341759:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.341761:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.341888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.341896:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.341899:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.341901:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.341902:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.341903:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.341904:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.341905:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.341907:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.341908:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.341909:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.341912:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.341913:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.341916:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.341916:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.341917:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.341918:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.341919:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.341920:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.341920:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.341924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.341931:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.341933:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.341934:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.341935:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.341936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:12.0:1619677448.341937:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.341938:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.341940:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.341943:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677448.341945:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.341945:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.341948:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.341949:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.341951:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.341952:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.341953:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.341954:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:20.2:1619677448.341956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:5.0:1619677448.341957:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.341959:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.341959:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.341960:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:5.0:1619677448.341961:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.341961:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000400:00000200:20.0:1619677448.341962:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.341962:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.341964:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.341964:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000400:00000200:20.0:1619677448.341965:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.341966:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677448.341966:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677448.341967:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000400:00000200:20.0:1619677448.341968:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.341968:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.341969:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.341969:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.341970:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.341970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000400:00000200:20.0:1619677448.341971:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:21.2:1619677448.341973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (21)++
00000400:00000200:20.0:1619677448.341974:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.341976:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.341977:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.341978:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.341978:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.341979:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.341980:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.341980:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677448.341980:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.341982:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.341982:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.341983:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.341983:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677448.341983:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677448.341984:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.341984:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677448.341984:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.341985:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677448.341986:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:25.2:1619677448.341988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.341994:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.341997:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.341998:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.342000:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.342001:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.342003:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.342016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.342023:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.342026:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.342027:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.342028:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.342031:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.342032:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.342033:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.342035:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.342035:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.342036:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.342039:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.342041:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.342043:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.342043:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.342044:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.342045:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.342046:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.342047:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.342047:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.342051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.342057:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.342059:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.342059:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.342060:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.342061:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.342062:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.342073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.342079:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.342081:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.342082:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.342082:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.342083:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.342083:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.342099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.342105:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.342107:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.342108:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.342110:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.342113:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.342115:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.342116:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.342117:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.342118:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677448.342118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:6.0:1619677448.342119:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.342123:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.342125:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.342125:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.342129:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.342129:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:6.0:1619677448.342129:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.342130:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.342131:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:10.0:1619677448.342132:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.342132:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.342133:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.342133:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677448.342134:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.342134:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677448.342135:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.342136:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.342137:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.342138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:10.0:1619677448.342139:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.342140:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.342143:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.342145:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:10.0:1619677448.342145:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.342148:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.342148:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.342148:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.342148:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.342149:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677448.342150:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.342151:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.342151:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.342152:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.342152:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.342153:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677448.342153:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.342157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.342166:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.342168:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.342170:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.342170:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.342172:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.342176:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.342323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.342331:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.342334:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.342335:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.342337:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.342340:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.342341:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.342342:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.342344:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.342345:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.342346:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.342349:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.342351:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.342354:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.342355:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.342356:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.342358:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.342358:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.342359:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.342359:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.342364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.342372:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.342375:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.342376:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.342376:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.342378:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.342381:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.342456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.2:1619677448.342460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677448.342462:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.342465:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.342465:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:30.0:1619677448.342468:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677448.342468:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.342471:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:1.0:1619677448.342471:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.342473:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677448.342474:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.342475:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:1.0:1619677448.342475:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.342476:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.342477:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.342477:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.342478:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677448.342479:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.342480:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.342482:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.342483:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.342483:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.342485:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.342485:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.342488:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.342489:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.342489:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.342490:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.342491:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.342492:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.342492:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.342493:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:30.0:1619677448.342494:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.342494:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:30.0:1619677448.342495:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.342495:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.342495:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.342497:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.342497:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.342498:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.342498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.342499:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677448.342499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:25.2:1619677448.342502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.342505:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677448.342506:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.342507:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677448.342508:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677448.342508:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677448.342509:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.342509:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.342509:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677448.342510:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.342512:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.342512:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.342512:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.342514:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:13.0:1619677448.342514:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.342516:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.342517:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.342537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.342545:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.342548:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.342548:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.342551:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.342554:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.342555:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.342557:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.342558:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.342559:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.342560:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.342563:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.342565:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.342567:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.342568:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.342568:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.342570:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.342570:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.342572:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.342573:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.342573:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.342576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.342579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.342582:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.342585:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.342585:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.342586:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677448.342587:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.342588:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.342588:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.342589:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.342590:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677448.342590:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677448.342691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.342697:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.342699:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.342700:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.342702:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.342704:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.342706:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.342707:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.342709:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.342710:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.342711:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.342714:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.342716:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.342719:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.342720:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.342721:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.342722:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.342723:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.342724:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.342724:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.342728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.342735:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.342737:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.342738:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.342740:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.342741:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:21.2:1619677448.342743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:13.0:1619677448.342743:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.342747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:6.0:1619677448.342749:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.342752:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.342752:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:12.0:1619677448.342753:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:6.0:1619677448.342755:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.342756:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.342757:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.342757:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.342758:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:6.0:1619677448.342758:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.342759:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:6.0:1619677448.342759:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.342760:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.342762:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.342763:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.342764:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.2:1619677448.342765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677448.342766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:6.0:1619677448.342767:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.342768:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.342771:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.342772:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.342772:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.342773:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.342774:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.342774:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.342774:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.342775:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677448.342776:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:6.0:1619677448.342776:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:26.0:1619677448.342777:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.342777:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.342779:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.342779:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.342780:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.342780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.342782:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.342782:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.342784:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.342784:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.342786:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.342786:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.342788:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.342788:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.342788:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:30.0:1619677448.342789:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.342789:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.342791:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.342791:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.342791:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.342792:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.342794:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.342794:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.342796:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.342797:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.342797:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677448.342799:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.342799:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.342801:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.342802:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.342802:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.342802:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.342803:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.342803:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.342804:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.342805:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.342805:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.342806:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.342806:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.342807:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:26.0:1619677448.342807:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.342809:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.342809:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.342810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:25.2:1619677448.342813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.342815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.342816:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.342818:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.342819:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.342819:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.342820:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:10.0:1619677448.342821:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.342822:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:10.0:1619677448.342822:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677448.342823:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:13.0:1619677448.342823:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.342824:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.342826:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.342826:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:12.0:1619677448.342826:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.342827:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677448.342827:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.342883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.342889:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.342892:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.342892:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.342894:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.342897:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.342898:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.342900:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.342901:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.342902:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.342903:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.342907:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.342909:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.342911:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.342912:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.342912:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.342914:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.342915:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.342915:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.342916:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.342920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.342928:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.342930:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.342931:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.342931:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.342933:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.342935:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.343087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.343092:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.343094:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.343096:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.343098:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.343099:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.343101:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.343102:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.343104:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.343105:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.343106:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.343109:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.343111:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.343114:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.343114:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.343115:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.343117:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.343118:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:4.2:1619677448.343118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677448.343119:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.343121:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:21.2:1619677448.343122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.2:1619677448.343123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:26.0:1619677448.343126:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.343128:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.343129:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677448.343130:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:5.2:1619677448.343130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677448.343131:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.343132:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.343133:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:26.0:1619677448.343134:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:8.0:1619677448.343134:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677448.343134:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:26.0:1619677448.343135:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.343135:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:24.0:1619677448.343136:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.343137:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.343137:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:26.0:1619677448.343138:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.343138:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:5.0:1619677448.343138:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.343139:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.343139:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.343140:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677448.343140:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.343140:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.343141:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.343142:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96716217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.343142:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.343142:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:26.0:1619677448.343144:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.343144:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.343145:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.343145:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.343146:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.343147:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.343147:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.343149:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96716216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.343151:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.343151:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:17.0:1619677448.343151:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.343152:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:24.0:1619677448.343152:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.343152:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.343153:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.343154:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.343154:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.343155:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677448.343155:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.343156:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:17.0:1619677448.343157:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.343158:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.343159:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:17.0:1619677448.343159:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:25.2:1619677448.343160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:24.0:1619677448.343160:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.343161:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.343162:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.343162:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:17.0:1619677448.343162:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.343163:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:17.0:1619677448.343163:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.343164:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (23)++
00000800:00000200:17.0:1619677448.343166:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (24)--
00000800:00000200:17.0:1619677448.343166:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.343167:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.343167:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.343167:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (23)++
00000800:00000200:9.2:1619677448.343168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.343170:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000800:00000200:22.0:1619677448.343171:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (23)++
00000400:00000200:6.0:1619677448.343171:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.343172:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000400:00000200:22.0:1619677448.343173:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.343173:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.343174:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.343175:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:22.0:1619677448.343177:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.343177:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.343178:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677448.343178:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.343179:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.343180:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.343181:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.343181:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.343182:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000400:00000200:2.0:1619677448.343182:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.343183:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.343183:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677448.343185:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.343187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.343195:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.343198:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.343199:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.343199:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.343200:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.343201:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.343313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919065782400] (20)++
00000800:00000200:2.0:1619677448.343319:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (21)++
00000800:00000200:2.0:1619677448.343321:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:18.0:1619677448.343322:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000400:00000200:2.0:1619677448.343324:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.343326:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.343328:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677448.343329:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.343330:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.343331:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.343332:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677448.343336:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.343338:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677448.343340:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.343341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.343341:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:2.0:1619677448.343342:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:2.0:1619677448.343343:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:2.0:1619677448.343344:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (21)++
00000800:00000200:2.0:1619677448.343344:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (22)--
00000800:00000200:2.0:1619677448.343345:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (21)--
00000800:00000200:1.0:1619677448.343347:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.2:1619677448.343350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:1.0:1619677448.343350:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.343351:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.343353:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.343355:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.343356:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.343357:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.343358:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:1.0:1619677448.343359:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.343360:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:1.0:1619677448.343360:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.343361:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.343361:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:20.0:1619677448.343364:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.343364:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:1.0:1619677448.343365:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.343366:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:1.0:1619677448.343367:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.343370:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.343370:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.343371:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.343373:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.343373:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.343374:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.343374:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.343379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.343388:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.343390:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.343391:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.343392:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.343393:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.343396:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.343450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.343458:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.343460:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.343461:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.343463:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.343466:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.343467:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.343469:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.343470:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.343471:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.343472:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.343476:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.343478:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.343480:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.343481:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.343482:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.343483:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.343484:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.343485:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.343485:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.343490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.343497:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:5.2:1619677448.343498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:17.0:1619677448.343500:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.343501:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.343501:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.343503:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.343504:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677448.343506:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677448.343507:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.343508:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.343510:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.343512:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.343514:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.343515:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.343516:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.343517:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.343518:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.343521:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.343523:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.343525:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.343526:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.343526:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.343528:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.343528:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.343530:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.343531:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.343531:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677448.343534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.343536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.343542:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.343543:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.343543:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.343544:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:2.0:1619677448.343544:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.343545:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.343546:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.343546:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.343547:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677448.343547:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.343660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677448.343664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677448.343666:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.343668:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.343669:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.343671:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:1.0:1619677448.343671:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.343673:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.343674:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.343675:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:1.0:1619677448.343675:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.343676:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.343677:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.343678:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.343679:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.343680:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.343680:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:26.0:1619677448.343682:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.343684:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.343684:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.343686:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.343687:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.343687:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:26.0:1619677448.343689:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.2:1619677448.343689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677448.343690:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.343691:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.343692:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:26.0:1619677448.343694:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.343694:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.343694:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.343696:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.343696:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.343697:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.343697:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.343698:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.343699:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.343699:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.343700:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:9.2:1619677448.343700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.343701:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:26.0:1619677448.343701:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.343702:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.2:1619677448.343702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.343703:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.343703:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:31.0:1619677448.343705:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.343705:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:30.0:1619677448.343706:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677448.343706:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:31.0:1619677448.343707:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:25.2:1619677448.343707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.343707:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677448.343707:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:31.0:1619677448.343708:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96716209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.343709:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:31.0:1619677448.343710:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.343710:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677448.343710:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:31.0:1619677448.343711:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.343711:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.343711:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.343711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677448.343712:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.343712:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.343714:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.343715:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:31.0:1619677448.343717:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.343717:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.343718:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.343718:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677448.343718:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:31.0:1619677448.343719:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:12.0:1619677448.343719:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.343720:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:9.2:1619677448.343720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:10.0:1619677448.343721:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.343722:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.343722:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.343722:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.343723:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:12.0:1619677448.343723:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.343724:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.343725:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.343726:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.343727:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.343727:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:24.0:1619677448.343727:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.343729:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.343730:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.343731:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.343731:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.2:1619677448.343731:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.343736:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.0:1619677448.343741:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.343744:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.343745:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.343747:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.343747:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.343749:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.343830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.343837:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.343840:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.343843:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.343844:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.343846:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.343847:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.343848:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.343850:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.343851:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.343852:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.343855:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.343856:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.343859:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.343859:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.343860:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.343861:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.343862:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.343864:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.343865:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:21.2:1619677448.343865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:30.0:1619677448.343866:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.343867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.343871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.343871:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:13.0:1619677448.343873:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.343874:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.343875:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:13.0:1619677448.343876:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.343877:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.343877:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677448.343879:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:13.0:1619677448.343879:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.343879:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677448.343879:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.343880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677448.343880:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:13.0:1619677448.343881:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.343882:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.343882:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:2.0:1619677448.343882:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.343883:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:2.0:1619677448.343883:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.343884:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.343885:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.343887:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677448.343888:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.343890:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:2.0:1619677448.343890:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.343891:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677448.343891:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.343892:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:10.0:1619677448.343893:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.343893:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.343894:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:10.0:1619677448.343895:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.343895:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.343896:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.343896:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:10.0:1619677448.343897:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.343898:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.343899:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.343901:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.343901:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.343905:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.343907:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:9.2:1619677448.343907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.343910:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.343911:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.343911:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.343913:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.343913:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.343914:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.343914:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.343915:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:18.0:1619677448.343916:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.343917:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.343918:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:25.2:1619677448.343919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:18.0:1619677448.343919:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.343921:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677448.343927:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.343930:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.343930:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.343931:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.343932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:22.0:1619677448.343933:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.343936:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677448.343938:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.343940:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.343941:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.343943:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.343946:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.343947:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.343949:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.343950:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.343951:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.343952:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.343956:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.343957:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.343961:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.343962:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.343962:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.343964:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.343965:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.343965:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.343966:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.343970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.343976:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.343979:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.343979:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.343982:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.343983:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.343985:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.343991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:13.0:1619677448.343997:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.344000:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.344001:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.344001:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.344002:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.344004:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.344188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.344193:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.344196:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.344198:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.344199:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.344200:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.344201:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.344203:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.344204:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.344205:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.344206:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.344209:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.344211:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.344213:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.344214:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.344215:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.344216:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.344217:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.344217:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.344218:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.344227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.344234:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.344237:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.344238:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.344239:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.344240:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.344241:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.344241:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.344242:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.344243:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677448.344243:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.344244:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.2:1619677448.344296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.344302:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.344304:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.344306:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.344307:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.344309:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.344310:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.344312:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.344313:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.344314:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.344315:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.344318:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.344320:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.344322:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.344323:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.344323:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.344325:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.344325:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.344326:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.344326:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.344331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.344339:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.344341:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.344341:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.344342:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.344343:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.344345:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.344435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.344443:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.344446:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.344448:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.344449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:31.0:1619677448.344451:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.344452:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677448.344452:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.344453:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.344453:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.344454:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.344455:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.344456:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.344457:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.344459:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.344460:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.344461:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.344463:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.344463:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.344464:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.344465:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.344466:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.344467:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.344469:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.344470:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:30.0:1619677448.344470:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.344471:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.344472:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.344472:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.344473:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.344474:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.344474:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677448.344474:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.344475:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.344476:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.344476:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.344477:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.344478:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:25.2:1619677448.344478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.344479:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.344479:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.344483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.344486:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.344488:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.344489:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.344490:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.344491:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.344492:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.344492:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.344492:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.344493:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:13.0:1619677448.344494:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.344495:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.344496:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.344619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.344626:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.344629:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.344630:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.344631:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.344634:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.344635:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.344637:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.344638:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.344639:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.344640:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.344643:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.344644:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.344647:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.344648:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.344649:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.344651:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.344651:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.344653:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.344654:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.344654:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677448.344656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.344660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:2.0:1619677448.344664:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.0:1619677448.344667:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:2.0:1619677448.344667:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.344668:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.344668:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.344669:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.344670:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.344671:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:2.0:1619677448.344671:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.344674:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.344675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.2:1619677448.344677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:18.0:1619677448.344683:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.344685:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677448.344685:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.344686:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.344686:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.344687:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.344688:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:18.0:1619677448.344688:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.344689:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.344691:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.344694:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.344696:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.344697:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.344698:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.344699:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.344700:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.344704:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.344705:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.344708:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.344709:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.344709:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.344711:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.344711:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.344712:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.344712:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.344717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.2:1619677448.344723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:6.0:1619677448.344725:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.344728:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.344728:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.344729:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.344729:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:6.0:1619677448.344731:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.344732:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.344734:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677448.344734:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:5.0:1619677448.344734:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.344736:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.344738:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:25.2:1619677448.344739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000400:00000200:5.0:1619677448.344739:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.344740:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.344741:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.344742:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:22.0:1619677448.344746:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:5.0:1619677448.344746:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.344748:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677448.344749:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.344750:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.344750:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.344751:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:5.0:1619677448.344751:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.344751:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:22.0:1619677448.344752:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677448.344752:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.344754:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.344755:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.344756:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.344757:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.344758:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677448.344760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677448.344763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:4.2:1619677448.344763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.0:1619677448.344767:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:17.0:1619677448.344768:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:20.0:1619677448.344769:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.344770:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:24.0:1619677448.344771:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677448.344771:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677448.344771:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.344772:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.344772:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.344773:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:17.0:1619677448.344774:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.344774:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.344777:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.344777:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:24.0:1619677448.344779:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.344781:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.344782:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.344783:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.344784:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.344785:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.344788:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.344790:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.344792:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.344793:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.344794:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.344795:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.344796:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.344797:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.344797:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.344801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.344809:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.344811:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.344812:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.344813:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.344815:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.344817:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.344848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.344856:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.344858:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.344861:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.344862:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.344863:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.344865:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.344866:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.344867:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.344868:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.344869:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.344873:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.344874:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.344877:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.344878:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.344879:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:4.2:1619677448.344879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.344881:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.344881:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.344882:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.344883:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:10.0:1619677448.344886:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677448.344887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.344889:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.344890:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.344891:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.344894:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.344895:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:10.0:1619677448.344895:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.344896:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.344897:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.344898:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.344898:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.344898:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.344900:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.344900:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.344900:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.344903:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:10.0:1619677448.344903:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.344905:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.344907:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.344907:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.344908:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.344909:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.344909:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677448.344909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.344910:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.344911:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.344914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.344916:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.344919:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.344920:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:5.0:1619677448.344920:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.344921:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.344921:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.344922:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:20.0:1619677448.344922:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:26.0:1619677448.344923:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.344924:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.344925:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.344927:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.345181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.0:1619677448.345185:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.345187:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677448.345189:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677448.345190:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.345192:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.345193:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.345194:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.345196:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.345197:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.345198:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.345201:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.345203:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.345205:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.345206:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.345207:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.345208:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.345209:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.345209:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.345210:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.345214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.345221:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.345224:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.345224:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.345226:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.345227:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.345228:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.2:1619677448.345237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.345244:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.345247:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.345249:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.345251:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.345252:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.345254:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.345255:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.345257:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.345258:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.345259:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.345263:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.345264:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.345268:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.345268:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.345269:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.345269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:30.0:1619677448.345271:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.345272:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.345273:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:5.0:1619677448.345274:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677448.345275:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.345275:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.345277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.345277:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.345279:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.2:1619677448.345281:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:5.0:1619677448.345281:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.345283:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.345284:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.345285:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.345285:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.345286:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.345287:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:5.0:1619677448.345287:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.345288:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677448.345289:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:20.0:1619677448.345289:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.345291:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:20.0:1619677448.345291:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.345291:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:31.0:1619677448.345292:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.345295:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677448.345299:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.345300:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677448.345300:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.345302:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.345303:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.345304:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.345305:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.345306:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.345307:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.345308:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.345309:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677448.345312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677448.345314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.345317:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.345319:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.345320:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:17.0:1619677448.345320:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677448.345321:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677448.345323:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.345324:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.345324:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.345326:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677448.345328:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.345353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.345360:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.345364:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.345364:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677448.345366:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.345369:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.345371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:8.0:1619677448.345371:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.345372:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.345374:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.345375:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.345376:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.345379:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:8.0:1619677448.345379:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.345381:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.345382:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.345384:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.345384:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.345385:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.345386:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677448.345386:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677448.345387:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.345387:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.345388:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:31.0:1619677448.345389:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:8.0:1619677448.345389:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.345389:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:31.0:1619677448.345390:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.345391:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.345392:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.345393:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.2:1619677448.345393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677448.345396:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.345398:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.345401:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.345401:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:31.0:1619677448.345402:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.345402:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.345404:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.345404:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:31.0:1619677448.345405:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.345405:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677448.345405:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.345406:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:20.0:1619677448.345406:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.345407:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.345410:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.345410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.345418:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.345420:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.345421:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.345422:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.345424:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.345426:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.345541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.345546:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.345549:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.345551:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.345551:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.345553:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.345555:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.345556:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.345557:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.345558:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.345559:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.345563:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.345564:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.345567:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.345568:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.345569:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.345570:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.345571:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.345572:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.345572:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677448.345573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:25.2:1619677448.345577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.345582:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.345584:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.345585:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:12.0:1619677448.345585:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677448.345587:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.345588:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.345588:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.345589:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.345591:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.345591:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:13.0:1619677448.345592:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.345592:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.345593:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.345594:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677448.345594:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.345595:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.345598:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.345600:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.345602:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.345602:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.345603:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.345604:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.345604:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.345605:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.345607:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.345607:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.345609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:4.2:1619677448.345613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.345618:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.345619:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.345620:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.345621:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.345622:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.345622:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.345623:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.345623:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.345624:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.345624:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677448.345677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.345683:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.345686:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.345687:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.345688:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.345691:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.345692:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.345694:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.345695:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.345696:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.345697:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.345700:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.345702:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.345704:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.345704:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.345705:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.345707:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.345707:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.345708:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.345708:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.345713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.345720:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.345722:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.345722:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.345723:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.345724:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.345725:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.345989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677448.345997:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677448.346000:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677448.346002:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.346003:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677448.346005:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.346007:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677448.346008:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677448.346010:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.346011:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677448.346012:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677448.346015:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.346017:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677448.346020:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.346021:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:13.0:1619677448.346021:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677448.346023:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:13.0:1619677448.346024:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677448.346025:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677448.346026:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677448.346028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.2:1619677448.346030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.346035:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.346037:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:26.0:1619677448.346038:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.346039:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.346040:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.346040:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.346041:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.346041:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:26.0:1619677448.346042:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.346043:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.346043:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.346045:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.346046:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.346046:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:26.0:1619677448.346047:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.346048:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.346051:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.346052:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:8.2:1619677448.346052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:26.0:1619677448.346055:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.346056:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.346056:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.346058:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.346058:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.346059:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677448.346059:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.346060:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677448.346062:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.346062:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.346063:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:25.2:1619677448.346064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.346064:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.346064:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677448.346072:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.346075:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.346075:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.346076:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.346078:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.346079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677448.346081:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:30.0:1619677448.346088:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.346091:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.346093:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.346094:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.346096:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.346097:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.346099:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.346100:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.346101:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.346102:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.346105:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.346107:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.346109:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.346110:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.346110:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.346110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.2:1619677448.346111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:30.0:1619677448.346112:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.346112:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.346112:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.346113:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.346114:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677448.346115:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.2:1619677448.346117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000400:00000200:20.0:1619677448.346117:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.346118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677448.346118:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:20.0:1619677448.346120:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.346120:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.346120:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:24.0:1619677448.346121:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.346121:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:20.0:1619677448.346122:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677448.346122:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.346124:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.346124:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:24.0:1619677448.346125:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.346126:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.346126:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:20.0:1619677448.346127:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677448.346127:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.346128:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.346128:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:12.0:1619677448.346129:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.346130:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:13.0:1619677448.346130:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677448.346132:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96716182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.346132:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.346134:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.346134:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.346135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677448.346135:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.346135:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.346136:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.346137:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.346137:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:12.0:1619677448.346138:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.346139:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677448.346139:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.346140:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.346141:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.346141:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.346142:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.346142:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.346143:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.346143:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677448.346143:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677448.346147:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.346148:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.346150:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 440 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.346151:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.346153:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.346153:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.346154:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.346154:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.346154:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.346155:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.346155:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.346156:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.346156:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.346157:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.346157:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)++
00000800:00000200:24.0:1619677448.346158:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.346158:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677448.346159:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677448.346159:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.346159:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (23)++
00000800:00000200:24.0:1619677448.346160:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677448.346160:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.346161:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (24)--
00000800:00000200:5.0:1619677448.346161:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (23)--
00000800:00000200:24.0:1619677448.346162:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.346163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.346163:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.346163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.346164:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:21.2:1619677448.346166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (21)++
00000800:00000200:8.2:1619677448.346168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:13.0:1619677448.346171:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.346172:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:17.0:1619677448.346172:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:13.0:1619677448.346173:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677448.346174:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:17.0:1619677448.346174:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:13.0:1619677448.346174:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.346175:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:17.0:1619677448.346175:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.346175:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:13.0:1619677448.346176:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.346177:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:12.0:1619677448.346177:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:17.0:1619677448.346178:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.346178:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677448.346179:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:13.0:1619677448.346179:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.346179:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:20.0:1619677448.346180:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.346181:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.346181:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.346184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.346186:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:4.2:1619677448.346186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677448.346188:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.346190:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.346191:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.346192:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.346192:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.346193:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:10.0:1619677448.346194:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.346196:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.346197:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.346199:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.346202:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.346203:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.346205:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.346206:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.346207:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.346208:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.346211:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.346213:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.346216:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.346217:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.346218:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.346220:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.346220:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.346221:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.346221:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.346226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.346233:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.346236:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.346236:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.346237:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.346239:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.346242:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.346305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.346310:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.346312:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.346313:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.346315:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.346317:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.346319:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.346320:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.346321:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.346322:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.346323:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.346326:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.346328:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.346330:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.346331:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.346332:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.346334:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.346334:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.346335:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.346335:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.346341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.346349:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.346352:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.346353:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.346353:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.346355:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.346357:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.346517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.346524:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.346526:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.346527:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.346527:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.346528:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.346529:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.346551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.346556:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.346559:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.346561:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.346562:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.346564:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.346566:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.346567:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.346568:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.346569:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.346570:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.346574:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.346575:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.346578:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.346579:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.346580:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.346581:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.346582:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.346583:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.346583:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.346588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.346595:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.2:1619677448.346596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:12.0:1619677448.346598:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.346599:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.346601:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.346602:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.346603:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.346604:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.346606:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.346607:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.346609:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.346612:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.346614:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.346615:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.346617:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.346618:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.346619:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:5.2:1619677448.346619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677448.346622:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.346624:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.346625:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.346627:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.346628:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.346628:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.346628:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.346628:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:26.0:1619677448.346630:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:1.0:1619677448.346630:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.346631:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.346631:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.346632:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:1.0:1619677448.346632:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.346633:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.346635:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.346636:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677448.346637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677448.346637:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.346638:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.346642:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.346644:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.346646:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677448.346647:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.346647:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.346648:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677448.346649:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.346650:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.346650:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.346650:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.346650:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.346651:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:2.0:1619677448.346652:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.346652:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677448.346655:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:25.2:1619677448.346656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.346662:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.346665:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.346666:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.346668:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.346669:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.346671:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.346678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.346685:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.346688:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.346690:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.346693:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.346693:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.346694:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.346695:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.346697:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.346698:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677448.346698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:31.0:1619677448.346699:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.346702:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.346703:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.346706:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.346706:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677448.346707:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.346707:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677448.346708:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:31.0:1619677448.346709:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.346709:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677448.346709:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677448.346711:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677448.346711:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.346713:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.346713:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.346714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:8.0:1619677448.346714:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.346715:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.346716:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96716174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.346718:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.346718:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.346719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:8.0:1619677448.346719:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:12.0:1619677448.346720:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.346722:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:8.0:1619677448.346722:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677448.346723:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.346724:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:12.0:1619677448.346725:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.346725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.346726:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.346726:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.346727:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677448.346727:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.346727:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.346728:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677448.346728:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.346729:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.346730:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677448.346730:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.346730:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.346731:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.346731:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677448.346732:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.346735:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.346735:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:8.2:1619677448.346736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:10.0:1619677448.346737:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.346740:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.346741:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.346742:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.346743:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.346744:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:10.0:1619677448.346744:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.346745:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.346746:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.346747:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.346747:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:10.0:1619677448.346748:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.346750:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.346751:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.346753:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677448.346753:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.346754:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.346755:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.346756:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.346757:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.346757:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.346758:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.346762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.346769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.346773:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.346774:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.346774:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:22.0:1619677448.346776:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.346776:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:22.0:1619677448.346778:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.346778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000400:00000200:6.0:1619677448.346779:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.346781:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.346782:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.346783:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:6.0:1619677448.346783:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:22.0:1619677448.346784:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.346785:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.346786:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000400:00000200:6.0:1619677448.346786:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.346787:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.346789:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677448.346789:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:6.0:1619677448.346789:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677448.346793:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.346793:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.346795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:24.0:1619677448.346796:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.346796:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:24.0:1619677448.346797:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:20.0:1619677448.346798:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:24.0:1619677448.346799:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96716171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.346799:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.346800:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:6.0:1619677448.346800:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.346800:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:24.0:1619677448.346801:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.346802:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.346802:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.346802:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:24.0:1619677448.346803:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:6.0:1619677448.346803:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:20.0:1619677448.346804:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.346804:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:6.0:1619677448.346805:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.346805:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677448.346806:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.346807:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.346808:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:9.2:1619677448.346808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:24.0:1619677448.346809:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:21.2:1619677448.346810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:20.0:1619677448.346810:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96716170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.346811:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.346811:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.346812:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677448.346812:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:20.0:1619677448.346812:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.346813:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:24.0:1619677448.346814:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.346814:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677448.346815:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677448.346816:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:22.0:1619677448.346816:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677448.346816:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:20.0:1619677448.346817:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:22.0:1619677448.346818:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.346819:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677448.346819:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.346819:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:9.2:1619677448.346819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:6.0:1619677448.346819:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:18.0:1619677448.346820:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.346820:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677448.346822:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.346823:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:18.0:1619677448.346823:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.346824:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.346825:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.0:1619677448.346826:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.346826:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:8.0:1619677448.346826:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:20.0:1619677448.346827:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.346828:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677448.346828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:8.0:1619677448.346828:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.346829:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.346829:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.346831:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.346832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:8.0:1619677448.346832:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:2.0:1619677448.346835:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:26.0:1619677448.346838:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:2.0:1619677448.346838:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.346839:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.346839:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:9.2:1619677448.346840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:2.0:1619677448.346840:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.346840:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:26.0:1619677448.346841:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.346842:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.346842:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.346845:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:24.0:1619677448.346847:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:26.0:1619677448.346848:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:24.0:1619677448.346850:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677448.346850:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:25.2:1619677448.346851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:24.0:1619677448.346851:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.346852:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.346852:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:10.0:1619677448.346857:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.346859:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.346860:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.346860:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.346861:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.346862:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.347003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.347009:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.347012:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.347012:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.347015:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.347016:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.347018:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677448.347019:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.347020:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.347021:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.347022:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677448.347026:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.347028:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.347030:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.347031:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677448.347032:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677448.347034:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.347034:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.347035:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.347036:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.347037:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677448.347040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:21.2:1619677448.347042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677448.347047:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677448.347048:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:22.0:1619677448.347050:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677448.347050:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:22.0:1619677448.347051:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.347051:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677448.347051:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.347052:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:22.0:1619677448.347053:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.347056:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.347066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.2:1619677448.347070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:26.0:1619677448.347074:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.2:1619677448.347074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.347076:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:18.0:1619677448.347076:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:10.0:1619677448.347077:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677448.347079:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:1.0:1619677448.347079:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.347080:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.347080:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:21.2:1619677448.347081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:26.0:1619677448.347082:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.347082:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:18.0:1619677448.347083:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.347083:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.347084:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.347085:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.347085:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.347085:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.347087:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.347087:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.347087:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.347089:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677448.347089:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.347089:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.347089:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:30.0:1619677448.347091:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677448.347091:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.347091:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.347091:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.347092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:18.0:1619677448.347092:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.347093:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.347093:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.347094:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.347095:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:1.0:1619677448.347095:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677448.347096:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.347096:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.347098:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.347098:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.347099:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.347100:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677448.347101:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:30.0:1619677448.347102:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96716165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.347102:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.347103:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.347103:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:18.0:1619677448.347103:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.347103:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.347104:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.347104:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.347105:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677448.347106:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.347106:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.347106:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.347106:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.347106:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:30.0:1619677448.347107:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.347107:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.347108:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:5.0:1619677448.347108:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:26.0:1619677448.347109:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:1.0:1619677448.347109:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.347110:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677448.347110:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.347110:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:508, d:32, s:-96716164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.347110:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.347110:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:18.0:1619677448.347111:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:5.0:1619677448.347111:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.347112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.347112:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.347112:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000400:00000200:30.0:1619677448.347113:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.347113:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:5.0:1619677448.347113:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.347113:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.347114:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:1.0:1619677448.347114:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:18.0:1619677448.347115:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.347115:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.2:1619677448.347115:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677448.347115:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:30.0:1619677448.347116:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.347116:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.347118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:30.0:1619677448.347119:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.347119:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.347119:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.2:1619677448.347119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.347120:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (24)++
00000800:00000200:30.0:1619677448.347122:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (25)--
00000800:00000200:30.0:1619677448.347122:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.347122:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (25)--
00000800:00000200:10.0:1619677448.347122:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:5.2:1619677448.347122:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:13.0:1619677448.347123:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.347124:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.347124:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.347124:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:13.0:1619677448.347125:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.347125:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:10.0:1619677448.347125:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677448.347126:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:5.0:1619677448.347126:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:25.2:1619677448.347127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.347127:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677448.347128:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677448.347129:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677448.347129:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677448.347130:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.347130:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:8.0:1619677448.347130:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.347130:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.347130:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:17.0:1619677448.347131:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.347131:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.347132:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:8.0:1619677448.347132:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.347132:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.347133:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.347133:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.347133:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.347134:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.347136:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677448.347136:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.0:1619677448.347136:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.347137:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:17.0:1619677448.347138:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:13.0:1619677448.347138:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.347138:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000400:00000200:13.0:1619677448.347139:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.347140:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)++
00000800:00000200:5.0:1619677448.347140:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:5.0:1619677448.347141:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677448.347142:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.347142:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.347143:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.347143:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:5.0:1619677448.347143:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677448.347144:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.347144:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.347144:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:25.2:1619677448.347145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.347145:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.347149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.347152:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677448.347152:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.347154:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:13.0:1619677448.347154:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677448.347155:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:13.0:1619677448.347155:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.347156:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:13.0:1619677448.347157:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.347158:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.347159:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.347223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.347228:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.347231:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.347232:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.347234:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.347237:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.347239:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.347240:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.347241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:2.0:1619677448.347241:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.347242:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.347243:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.347246:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677448.347247:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.347249:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:2.0:1619677448.347249:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.347250:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.347252:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.347253:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.347253:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:6.0:1619677448.347254:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.347254:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:6.0:1619677448.347256:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:2.0:1619677448.347256:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.347256:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:6.0:1619677448.347257:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.347257:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.347258:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:6.0:1619677448.347259:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.347260:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.347261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:6.0:1619677448.347261:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.347264:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.347266:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:12.0:1619677448.347268:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677448.347268:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.347269:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.347269:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677448.347270:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.347271:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.347271:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.347271:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.347272:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:12.0:1619677448.347273:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.347273:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677448.347274:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.347276:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677448.347277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.347285:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.347287:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.347288:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.347290:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.347291:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.347292:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.347361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.347368:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.347371:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.347372:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.347374:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.347376:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.347378:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.347379:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.347380:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.347381:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.347382:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.347386:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.347387:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.347390:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.347391:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.347392:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.347393:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.347394:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.347394:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.347395:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.347400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.347408:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.347411:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.347412:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.347412:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.347414:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.347417:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.347522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.347530:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.347533:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.347534:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.347535:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.347538:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.347540:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.347541:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.347543:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.347543:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.347544:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.347547:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.347549:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.347551:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.347552:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.347552:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.347554:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.347554:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.347555:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.347555:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677448.347559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.347567:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.347569:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.347570:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.347572:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.347573:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.347575:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.347577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.347583:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.347584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:12.0:1619677448.347586:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.347587:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.347587:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.347588:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.347588:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.347591:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.347594:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.347595:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.347596:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.347599:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.347600:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.347601:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.347602:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.347603:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.347604:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.347607:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.347608:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.347610:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.347611:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.347611:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.347613:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.347613:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.347614:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.347614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.347615:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.347618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:1.0:1619677448.347621:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.347623:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.347624:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.347624:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.347626:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:1.0:1619677448.347626:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.347627:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.347628:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:1.0:1619677448.347628:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.347629:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.347630:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:10.0:1619677448.347632:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:1.0:1619677448.347632:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.347633:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.347634:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.347635:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.347638:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.347640:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.347642:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.347643:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.347643:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.347645:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.347645:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.347646:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.347646:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.347651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.347659:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.347661:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.347662:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.347662:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.347664:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.347665:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.347668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.347676:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.347679:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.347679:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.347680:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.347680:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.347681:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.347697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:21.2:1619677448.347701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:5.0:1619677448.347702:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.347705:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677448.347707:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:6.0:1619677448.347707:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:5.0:1619677448.347707:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.347709:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:5.0:1619677448.347709:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.347710:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:5.0:1619677448.347711:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677448.347712:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.347713:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.347714:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.347715:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.347715:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.347716:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.347716:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677448.347718:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716156] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.347719:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.347720:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.347721:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.347722:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.347722:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.347725:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.347726:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.347726:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.347726:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:6.0:1619677448.347728:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.347728:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.347729:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.347730:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.347730:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:6.0:1619677448.347731:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.347732:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.347733:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.2:1619677448.347734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:6.0:1619677448.347735:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.347735:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.347736:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.347737:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.347741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.347742:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.347744:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.347745:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.347746:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.347747:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:31.0:1619677448.347748:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.347748:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:31.0:1619677448.347752:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.347752:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.347754:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.347756:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.347757:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.347888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.347896:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.347899:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.347901:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.347903:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.347903:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.347905:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.347906:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.347907:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.347908:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.347910:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.347913:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.347915:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.347919:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.347919:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.347920:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.347922:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.347922:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.347923:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.347924:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.347928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.347936:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.347939:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.347940:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.347940:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.347942:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.347945:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.347966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.347973:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.2:1619677448.347974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677448.347975:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.347977:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.347978:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.347979:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000400:00000200:26.0:1619677448.347980:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.347980:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:26.0:1619677448.347981:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.347983:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.347984:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.347985:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.347986:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.347988:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.347990:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.347990:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.347992:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.347992:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.347993:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.347994:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.347995:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.347996:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.347999:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.348000:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.348002:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.348002:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.348003:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.348003:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.348003:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.348003:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.348005:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.348005:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.348006:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.348006:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677448.348007:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677448.348008:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.348008:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677448.348009:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.348009:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:26.0:1619677448.348010:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.348011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:25.2:1619677448.348014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.348017:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:4.2:1619677448.348017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.348019:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.348020:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.348022:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.348022:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:30.0:1619677448.348023:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.348024:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.348024:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.348025:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.348026:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:10.0:1619677448.348026:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.348026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.348027:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.348027:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.348028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.348028:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.348029:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.348029:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677448.348032:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.348034:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.348035:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.348035:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.348038:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.348038:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.348039:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.348039:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.348040:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:17.0:1619677448.348040:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.348041:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:17.0:1619677448.348041:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.348043:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.348044:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.348045:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.348046:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.348049:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.348050:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.348052:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.348052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:17.0:1619677448.348053:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.348053:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.348055:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.348055:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.348055:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.348056:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677448.348058:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:21.2:1619677448.348060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.2:1619677448.348061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.348061:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.348062:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.348064:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.348066:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.348068:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677448.348068:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.348069:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:2.0:1619677448.348069:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.348070:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.348070:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.348071:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.348072:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:17.0:1619677448.348072:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:2.0:1619677448.348072:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.348073:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:2.0:1619677448.348073:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.348074:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.348076:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.348076:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:2.0:1619677448.348077:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.348079:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.348080:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:24.0:1619677448.348081:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.348082:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96716150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.348083:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677448.348084:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.348084:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.348084:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677448.348085:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.348086:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.348086:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677448.348089:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.348090:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:24.0:1619677448.348091:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.348091:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.348091:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.348092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677448.348094:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.348095:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.348096:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.348097:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.348098:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.348099:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.348099:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677448.348099:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.348102:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.348103:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.348103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:12.0:1619677448.348105:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.348106:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.348107:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677448.348111:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.348114:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.348115:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.348115:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.348117:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.348120:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.348323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.348329:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.348332:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.348333:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.348334:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.348336:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.348337:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.348339:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716149] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.348340:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.348341:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.348342:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.348345:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.348347:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.348350:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.348351:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.348352:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.348354:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.348354:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.348355:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.348355:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.348360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.348368:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.348370:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.348371:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.348373:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.348375:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.348376:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.348390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.348399:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.348402:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.348403:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.348406:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.348406:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.348408:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.348409:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.348410:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.348411:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.348412:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.348415:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.348417:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.348420:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.348421:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.348422:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.348423:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.348424:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.348425:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.348425:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.348430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.348438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677448.348439:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.348441:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.348442:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.348444:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.348445:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.348447:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677448.348447:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:26.0:1619677448.348450:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.348450:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.348452:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.348454:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.348455:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.348456:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.348458:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.348459:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.348459:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.348463:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.348464:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.348466:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.348467:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.348468:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.348469:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.348470:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.348470:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.348471:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.348475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.348482:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.348484:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.348485:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.348487:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.348488:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.348490:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.348592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.348597:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.348600:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.348601:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.348603:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.348606:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.348608:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.348609:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.348610:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.348611:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.348612:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.348615:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.348617:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.348619:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.348620:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.348621:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.348623:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.348623:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.348625:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.348626:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.348626:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.348629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.348631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.348635:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.348636:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677448.348637:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677448.348638:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:12.0:1619677448.348638:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.348639:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.348640:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677448.348640:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.348641:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.348643:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.348827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.348832:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.348835:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.348837:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.348838:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.348839:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.348840:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.348842:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.348843:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.348844:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.348845:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.348848:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.348850:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.348853:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.348854:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.348854:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.348856:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.348856:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.348857:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.348858:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.348862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.2:1619677448.348870:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:4.2:1619677448.348872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677448.348874:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.348876:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:2.0:1619677448.348876:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:5.0:1619677448.348877:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.348879:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677448.348879:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.348880:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:18.0:1619677448.348880:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.348880:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:5.0:1619677448.348881:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:26.0:1619677448.348882:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.348882:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:2.0:1619677448.348883:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.348885:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.348885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:2.0:1619677448.348886:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.348887:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.348887:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.348889:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.348890:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.348891:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.348891:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.348891:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677448.348892:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.348893:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.348894:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:2.0:1619677448.348894:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.348894:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:10.0:1619677448.348895:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.348897:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.348898:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.348899:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.348901:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.348902:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.348902:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.348904:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.348904:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:10.0:1619677448.348905:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677448.348905:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.348905:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677448.348905:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96716142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.348907:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.348907:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:1.0:1619677448.348907:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.348908:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.348908:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677448.348908:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:1.0:1619677448.348908:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.348909:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677448.348909:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000400:00000200:1.0:1619677448.348909:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677448.348910:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:10.0:1619677448.348911:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.348911:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.348912:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677448.348912:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.348913:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.348913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.348914:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.2:1619677448.348916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:25.2:1619677448.348917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.348917:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.348918:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.348919:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (23)++
00000800:00000200:1.0:1619677448.348921:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (24)--
00000800:00000200:1.0:1619677448.348921:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.348922:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677448.348922:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.348922:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677448.348923:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.348923:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:18.0:1619677448.348924:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.348924:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.348924:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:22.0:1619677448.348925:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (23)++
00000800:00000200:2.0:1619677448.348925:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.348926:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.348926:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:22.0:1619677448.348928:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000800:00000200:18.0:1619677448.348928:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677448.348928:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (24)++
00000400:00000200:22.0:1619677448.348929:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.348929:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000400:00000200:6.0:1619677448.348930:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.348931:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.348932:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.348935:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.348936:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.348937:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.348944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:6.0:1619677448.348951:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.348954:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.348955:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.348955:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.348956:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.348956:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.348987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.348994:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.348997:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.348998:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.349000:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.349003:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.349004:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.349006:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.349007:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.349008:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.349009:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.349013:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.349015:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.349017:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.349018:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.349018:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.349020:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:21.2:1619677448.349021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677448.349021:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.349022:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.349024:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.349024:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.2:1619677448.349026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.349028:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.2:1619677448.349030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:30.0:1619677448.349031:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:13.0:1619677448.349032:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:30.0:1619677448.349033:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.349034:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:31.0:1619677448.349035:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.349035:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.349035:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.349036:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:30.0:1619677448.349037:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:13.0:1619677448.349037:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.349038:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.349038:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:12.0:1619677448.349038:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.349039:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677448.349039:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:30.0:1619677448.349040:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.349040:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.349041:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:24.0:1619677448.349041:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:30.0:1619677448.349044:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.349045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.349046:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.349048:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.349049:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.349049:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.349051:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.349051:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.349052:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677448.349052:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.349053:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.349054:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.349055:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:25.2:1619677448.349056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:12.0:1619677448.349056:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.349056:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.349058:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.349062:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.349065:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.349065:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.349066:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.349067:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.349070:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.349163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677448.349166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:17.0:1619677448.349168:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.349171:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.349171:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.349173:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677448.349174:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.349176:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.349176:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.349177:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:17.0:1619677448.349178:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.349179:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.349180:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.349181:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.349182:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.349183:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.349183:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.349185:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.349187:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.349187:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.349188:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.349190:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.349190:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.349192:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.349193:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.349194:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.349195:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.349195:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.349196:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.349197:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.349197:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.349197:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.349198:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.349200:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.349201:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:25.2:1619677448.349202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.349202:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.349203:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.349204:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.349205:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.349205:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.349209:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:25.2:1619677448.349211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.349211:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.349212:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.349214:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.349215:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.349216:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.349217:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.349220:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.349220:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.349222:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.349224:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.349225:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.349274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.349282:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.349285:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.349286:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.349288:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.349289:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.349290:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.349291:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.349293:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.349298:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.349299:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:4.2:1619677448.349299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:31.0:1619677448.349303:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.349304:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.349306:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.349306:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.349307:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.349307:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.349308:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:31.0:1619677448.349309:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.349309:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.349310:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.349310:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.349311:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:26.0:1619677448.349311:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.349313:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.349315:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:25.2:1619677448.349315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:26.0:1619677448.349316:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.349317:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.349318:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.349319:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.349322:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.349322:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:26.0:1619677448.349323:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677448.349325:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.349325:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.349326:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.349327:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.349327:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:13.0:1619677448.349327:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.349329:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677448.349329:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.349330:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.349330:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:12.0:1619677448.349330:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.349331:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.349335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.349343:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.349346:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.349348:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.349348:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.349350:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.349352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.349355:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.349356:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.349358:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.349358:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.349359:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.349359:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.349360:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.349447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.349453:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.349456:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.349456:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.349458:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.349460:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.349462:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.349463:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.349464:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.349465:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.349466:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.349469:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.349471:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.349474:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.349474:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.349475:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.349477:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.349477:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.349478:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.349479:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.349484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.349492:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.349494:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.349495:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.349496:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.349498:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677448.349498:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:21.2:1619677448.349517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:5.2:1619677448.349517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677448.349523:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677448.349524:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.349526:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:17.0:1619677448.349527:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.349527:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677448.349527:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:18.0:1619677448.349530:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.349531:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.349531:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.349533:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677448.349533:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.349534:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.349535:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677448.349536:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.349536:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.349537:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.349538:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.349539:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.349539:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.349541:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677448.349542:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.349545:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.349545:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.349547:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.349547:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.349548:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.349549:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.349550:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677448.349550:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.349551:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677448.349551:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.349551:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677448.349552:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.349552:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677448.349553:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.349554:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.349555:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.349555:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.349556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:25.2:1619677448.349560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677448.349563:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:13.0:1619677448.349566:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677448.349566:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.349566:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.349567:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.349568:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.349569:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.349570:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.349570:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:13.0:1619677448.349571:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.349573:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.349574:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.349703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.349712:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.349714:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.349717:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.349718:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.349719:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.349720:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.349722:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.349723:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.349724:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.349725:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.349729:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.349730:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.349734:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.349735:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.349736:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.349737:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.349738:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.349739:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.349739:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.349743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.349751:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.349754:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.349755:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.349756:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.349757:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.349759:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.349817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.349825:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.349828:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.349830:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.349832:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.349833:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.349834:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.349836:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.349837:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.349838:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.349839:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.349842:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.349844:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.349846:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.349847:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.349847:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.349849:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.349850:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.349851:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.349853:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.349853:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.349855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.349859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.349861:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.349863:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.349864:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.349866:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:12.0:1619677448.349866:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.349867:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.349868:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:12.0:1619677448.349868:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.349869:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.349870:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.349874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.349880:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.349883:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.349883:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.349884:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.349885:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.349886:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.349938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.349944:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.349946:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.349947:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.349949:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.349951:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.349952:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.349954:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.349955:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.349956:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.349957:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.349960:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.349962:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.349964:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.349965:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.349966:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.349968:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.349968:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.349969:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.349969:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.349974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.349981:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.349983:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.349984:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.349986:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.349987:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.349988:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.349995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.350002:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.350005:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.350006:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.350007:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.350007:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.350009:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.350055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.0:1619677448.350058:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.350061:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677448.350063:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.350064:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.350065:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.350066:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.350067:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.350069:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.350070:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.350071:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.350074:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.350076:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.350079:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.350079:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.350080:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.350082:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.350082:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.350083:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.350083:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.350087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.350094:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.350097:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.350098:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.350099:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.350100:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.350101:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:8.2:1619677448.350112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.350119:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.350122:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.350123:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:4.2:1619677448.350123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677448.350124:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677448.350124:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.350125:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:10.0:1619677448.350131:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.350134:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.350135:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.2:1619677448.350136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:10.0:1619677448.350137:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.350140:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.350141:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:5.0:1619677448.350141:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:10.0:1619677448.350142:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.350144:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.350144:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:10.0:1619677448.350145:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.350146:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.350147:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.350147:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.350149:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.350149:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.350151:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.350151:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677448.350152:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.350153:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.350154:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.350155:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.350155:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:5.0:1619677448.350155:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.350156:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.350158:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.350158:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.350159:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:5.0:1619677448.350159:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.350160:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.350161:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.350161:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.350163:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677448.350164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.350164:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.350165:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.350166:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.350167:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:4.2:1619677448.350167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677448.350168:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.350168:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677448.350171:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.2:1619677448.350172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.350174:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.350174:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.350175:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.350176:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.350176:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.350176:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.350177:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.350178:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.350180:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:6.0:1619677448.350181:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677448.350182:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:25.2:1619677448.350184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000400:00000200:17.0:1619677448.350184:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.350184:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.350186:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.350189:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:22.0:1619677448.350192:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.350195:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.350195:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.350196:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.350197:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.350197:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.350243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.350250:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.350253:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.350255:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.350257:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.350257:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.350259:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.350260:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.350262:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.350263:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.350264:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.350267:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.350269:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.350271:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.350272:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.350273:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.350274:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.350275:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.350276:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.350278:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.350278:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.350280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.350284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.350286:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.350289:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.350289:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.350291:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.350292:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:13.0:1619677448.350292:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.350294:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.350299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.350300:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.350301:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.350302:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677448.350303:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.350306:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.350307:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.350308:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.350308:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.350309:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.350332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.350341:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.350343:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.350344:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.350346:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.350349:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.350350:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.350351:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.350352:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.350353:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.350354:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.350357:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.350358:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.350361:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.350361:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.350362:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.350363:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.350364:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.350364:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.350365:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.350369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.350375:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.350377:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.350378:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.350380:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.350382:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.350382:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:21.2:1619677448.350466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.350474:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.350477:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.350478:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.350480:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.350482:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.350483:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.350485:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.350486:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.350487:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.350487:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677448.350490:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.350492:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.350495:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.350496:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.350496:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.350497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:8.0:1619677448.350498:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.350498:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.350499:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.350499:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677448.350500:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.350503:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:8.2:1619677448.350504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:20.0:1619677448.350505:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.350507:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.350507:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.350509:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:5.0:1619677448.350509:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:20.0:1619677448.350510:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.350510:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.350511:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.350511:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.350513:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.350513:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.350514:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.350517:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:20.0:1619677448.350518:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.350518:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677448.350520:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.350523:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.350524:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.350524:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.350526:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.350527:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.350528:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.350528:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.350532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.350540:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.350542:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.350543:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.350545:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.350546:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.350548:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.350702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.350709:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.350712:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.350713:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.350715:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.350717:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.350719:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.350720:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.350721:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.350722:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.350723:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.350727:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.350729:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.2:1619677448.350730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677448.350732:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.350732:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.350733:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.350735:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.350735:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.350736:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.350736:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.350737:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.350740:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:25.2:1619677448.350741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:31.0:1619677448.350742:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.350744:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.350744:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.350746:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.350747:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.350748:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.350749:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.350749:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:31.0:1619677448.350750:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.350751:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.350752:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:31.0:1619677448.350753:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.350753:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.350755:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:6.0:1619677448.350755:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.350757:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.350758:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:6.0:1619677448.350758:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677448.350759:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.350760:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.350761:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.350762:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.350762:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.350766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.350768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:12.0:1619677448.350773:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.350775:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:30.0:1619677448.350776:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:12.0:1619677448.350776:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.350777:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.350778:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:12.0:1619677448.350779:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:30.0:1619677448.350780:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.350780:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.350782:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.350782:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.350784:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.350785:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.350786:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.350787:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.350788:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:20.2:1619677448.350789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:30.0:1619677448.350791:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.350792:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.350794:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.350795:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.350796:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.350796:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:30.0:1619677448.350797:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.350797:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.350798:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.350798:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:30.0:1619677448.350799:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.350800:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.350801:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:25.2:1619677448.350802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:10.0:1619677448.350804:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.350805:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677448.350806:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.350808:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.350808:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.350809:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:10.0:1619677448.350809:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.350811:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.350812:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.350812:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000400:00000200:31.0:1619677448.350813:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.350814:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.350815:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.350816:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:10.0:1619677448.350817:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.350817:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.350818:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.350820:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.350820:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.350821:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.350821:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.350825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.350833:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.350835:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.350836:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.350838:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.350838:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.350841:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.350889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.350895:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.350898:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.350898:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.350900:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.350903:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.350904:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.350906:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.350907:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.350908:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.350909:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.350912:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.350914:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.350916:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.350916:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.350917:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.350919:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.350919:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.350920:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.350921:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.350922:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.350925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.350928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.350933:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.350934:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.350935:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.350935:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.350936:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.350936:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.350937:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.350937:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.350938:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677448.350939:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:21.2:1619677448.351017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.351022:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.351024:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.351026:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.351028:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.351030:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.351031:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.351032:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.351033:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.351034:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.351035:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.351039:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.351040:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.351043:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.351044:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.351044:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.351046:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.351046:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.351048:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677448.351049:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677448.351050:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:8.2:1619677448.351051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:21.2:1619677448.351055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.351059:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.351061:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.351062:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000400:00000200:5.0:1619677448.351062:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.351063:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677448.351063:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.351063:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.351065:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.351066:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677448.351067:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:4.2:1619677448.351165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.351172:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.351175:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.351175:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.351178:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.351180:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.351182:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.351183:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.351184:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.351185:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.351186:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.351189:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.351191:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.351193:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.351194:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.351194:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.351196:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.351196:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.2:1619677448.351196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:26.0:1619677448.351197:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.351197:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.351201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.351201:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.351204:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.351206:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.351206:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.351207:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:5.0:1619677448.351208:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.351209:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.351210:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.351210:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:26.0:1619677448.351211:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:5.0:1619677448.351211:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.351212:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.351213:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.351214:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.351215:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:5.0:1619677448.351215:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:25.2:1619677448.351219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000400:00000200:5.0:1619677448.351219:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.351221:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.351224:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.351225:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.351225:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.351225:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.351227:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.351227:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.351228:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.351228:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.351229:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.351229:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.351229:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.351229:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677448.351230:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.351233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.351236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.351239:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:12.0:1619677448.351240:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.351242:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:12.0:1619677448.351242:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.351243:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.351244:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.351244:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:12.0:1619677448.351245:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.351246:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.351247:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677448.351248:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677448.351249:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.351250:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.351251:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.351252:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:12.0:1619677448.351254:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:20.0:1619677448.351256:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.351257:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677448.351260:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.351261:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677448.351262:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677448.351264:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677448.351264:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.351265:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.351265:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677448.351273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.351280:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.351283:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.351284:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.351284:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.351286:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677448.351289:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.351293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.351301:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.351303:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.351304:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.351306:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.351308:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.351310:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.351311:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.351312:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.351313:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.351314:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.351317:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.351318:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.351321:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.351322:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.351323:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.351324:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.351324:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.351326:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.351327:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.351327:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.351330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.351334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.351339:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.351341:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.351341:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.351342:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.351343:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677448.351343:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.351344:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.351345:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677448.351345:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.351347:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.351358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.351366:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.351369:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.351371:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.351373:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.351373:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.351374:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.351375:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716112] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.351377:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.351378:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.351378:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.351381:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.351383:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.351385:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.351386:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.351386:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.351388:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.351388:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.351389:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.351390:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.351394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.351401:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.351404:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.351405:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.351406:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.351408:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.351409:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.351461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.351469:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.351472:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.351473:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.351474:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.351475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:10.0:1619677448.351477:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.351478:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.351479:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.351481:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.351482:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.351482:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:10.0:1619677448.351483:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:8.0:1619677448.351485:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.351486:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:10.0:1619677448.351486:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.351487:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:8.0:1619677448.351488:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.351489:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.351490:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:8.0:1619677448.351490:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.351491:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.351492:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:8.0:1619677448.351492:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:10.0:1619677448.351493:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:8.0:1619677448.351493:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96716110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.351494:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.351495:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000400:00000200:8.0:1619677448.351495:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.351495:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.351496:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:8.0:1619677448.351496:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:25.2:1619677448.351498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:8.0:1619677448.351500:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.351501:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.2:1619677448.351501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677448.351504:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.351505:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:8.0:1619677448.351505:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.351505:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.351507:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:8.0:1619677448.351507:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.351507:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:30.0:1619677448.351508:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.351508:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677448.351508:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.351508:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:30.0:1619677448.351509:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.351511:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.351511:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:30.0:1619677448.351512:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:26.0:1619677448.351512:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.351513:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.2:1619677448.351513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:25.2:1619677448.351515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.351521:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677448.351521:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.351523:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:31.0:1619677448.351524:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677448.351524:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.351525:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.351525:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.351526:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677448.351526:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.351526:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.351528:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.0:1619677448.351529:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.351533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.351540:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.351543:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.351544:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.351544:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.351545:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.351547:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.351548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677448.351550:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677448.351555:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.351557:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:30.0:1619677448.351558:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.351558:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.351560:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.351561:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:10.0:1619677448.351562:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.351563:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.351563:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.351564:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.351565:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.351565:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.351566:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.351567:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.351567:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.351568:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.351568:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.351570:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.351571:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.351571:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.351572:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.351573:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.351575:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.351575:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.351575:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.351576:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.351577:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.351577:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.351577:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.2:1619677448.351578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677448.351578:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.351578:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.351579:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.351580:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.351580:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.351581:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.351582:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:25.2:1619677448.351582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.351583:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.351583:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.351583:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.351586:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:25.2:1619677448.351587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.351588:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:20.0:1619677448.351588:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.351588:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.351590:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.351591:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.351591:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.351592:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:5.0:1619677448.351592:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677448.351593:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:5.0:1619677448.351594:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96716107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.351595:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:26.0:1619677448.351595:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.351596:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.351596:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.351597:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:5.0:1619677448.351597:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.351598:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.351598:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.351599:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.351601:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:5.0:1619677448.351602:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.351604:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.351607:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.351607:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.351608:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.351610:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.351611:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.351611:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.351612:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.351616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.351623:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.351625:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.351626:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.351628:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.351629:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.351630:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.351637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.351643:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.351646:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.351647:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.351647:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.351648:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.351649:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.351844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.0:1619677448.351847:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677448.351850:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:20.0:1619677448.351852:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.351853:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677448.351855:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.351856:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:20.0:1619677448.351858:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.351859:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.351860:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.351861:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677448.351864:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.351866:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.351869:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.351869:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.351870:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.351872:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.351872:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677448.351873:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.0:1619677448.351874:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677448.351878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.351885:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.351888:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.351888:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.351890:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.351891:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.351892:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.351901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.351908:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.351911:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.2:1619677448.351912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:30.0:1619677448.351913:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.351914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677448.351915:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.351915:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.351917:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677448.351917:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677448.351918:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.351919:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:10.0:1619677448.351919:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:30.0:1619677448.351920:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.351922:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.351922:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:20.0:1619677448.351922:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.351922:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:5.0:1619677448.351922:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677448.351923:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.351925:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.351926:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.351927:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:30.0:1619677448.351928:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.351928:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.351928:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.351930:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.351930:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:30.0:1619677448.351931:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.351931:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.351932:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.351933:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.351933:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.351934:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.351934:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.351935:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.351935:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.351937:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.351937:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677448.351937:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.351938:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677448.351938:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.351939:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677448.351940:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.2:1619677448.351942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.351942:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.351943:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.351944:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677448.351944:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.351944:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.351946:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677448.351946:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.351947:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.351947:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.351948:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677448.351948:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.351948:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.351949:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.351950:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677448.351950:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677448.351951:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.351951:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.351951:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.351952:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.351952:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.2:1619677448.351952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:13.0:1619677448.351953:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.351954:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.2:1619677448.351955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677448.351955:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677448.351957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.351960:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677448.351960:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:5.0:1619677448.351962:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:1.0:1619677448.351962:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.351963:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.351963:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:1.0:1619677448.351963:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.351964:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677448.351965:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:1.0:1619677448.351966:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.351968:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:1.0:1619677448.351968:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677448.351969:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.351969:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.351972:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.351975:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.352033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.0:1619677448.352037:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.352040:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677448.352042:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.352042:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677448.352044:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.352046:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.352047:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.352048:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.352050:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.352050:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.352054:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.352056:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.352057:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.352058:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.352059:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.352060:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.352060:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.352061:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.352062:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.352066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.352073:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.352076:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.352076:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.352078:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.352079:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.352080:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:21.2:1619677448.352185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.352191:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.352194:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.352194:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.352196:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.352198:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.352200:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.352202:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.352203:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.352204:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.352205:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.352208:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.352210:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.352212:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.352213:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.352213:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.352215:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.352215:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.352216:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.352217:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.352218:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.352221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.352223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.352228:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:17.0:1619677448.352229:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.352231:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:6.0:1619677448.352231:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.352231:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:17.0:1619677448.352232:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.352232:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677448.352233:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.352235:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.352237:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.352249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.352255:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.2:1619677448.352257:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677448.352260:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.352261:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.352263:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.352263:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677448.352265:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.352266:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.352267:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677448.352267:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.352269:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.352269:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.352271:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.352271:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.352272:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.352273:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.352273:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.352274:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.352276:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.352277:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.352278:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.352279:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.352280:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.352282:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.352283:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.352284:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.352284:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.352285:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.352286:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.352287:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.352287:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.352288:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.352288:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.352289:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.352289:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677448.352289:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.352290:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.352291:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.352292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.352292:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.352299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677448.352301:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.352304:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.352305:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.352306:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:12.0:1619677448.352307:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.352308:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:13.0:1619677448.352308:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:17.0:1619677448.352309:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.352309:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.352310:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.352311:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.352313:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.352314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.352321:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.352323:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:21.2:1619677448.352325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677448.352325:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.352325:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.352325:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.352326:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677448.352333:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.352336:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.352338:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.352340:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.352341:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.352342:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.352343:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.352345:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.352346:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.352347:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.352350:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.352350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.352352:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.352354:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.352355:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.352356:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.352357:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.352358:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.352358:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.352359:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:26.0:1619677448.352360:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:31.0:1619677448.352361:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.352361:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677448.352361:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.352363:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.352363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.352363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:26.0:1619677448.352366:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.352366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677448.352368:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.352369:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.352370:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.352370:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.352371:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:26.0:1619677448.352371:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.352372:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:26.0:1619677448.352372:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:13.0:1619677448.352372:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.352373:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.352373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677448.352374:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.352375:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:13.0:1619677448.352375:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.352376:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.352376:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.352376:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.352377:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:13.0:1619677448.352377:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.352378:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:30.0:1619677448.352378:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.352378:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.352380:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:26.0:1619677448.352381:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.352382:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.352382:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:30.0:1619677448.352383:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.352383:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:30.0:1619677448.352384:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.352385:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.352385:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.352386:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:10.0:1619677448.352386:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.352387:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:30.0:1619677448.352388:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.352389:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:10.0:1619677448.352389:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.352390:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.352391:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.352392:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96716095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.352393:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.352393:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.352394:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:30.0:1619677448.352394:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:10.0:1619677448.352394:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.352395:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:8.2:1619677448.352395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.352396:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.352396:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.352397:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.352399:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000400:00000200:10.0:1619677448.352399:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.352400:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:26.0:1619677448.352400:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.352401:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677448.352401:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:26.0:1619677448.352401:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.352401:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677448.352402:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.2:1619677448.352402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.352403:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.352404:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677448.352405:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:10.0:1619677448.352405:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:4.2:1619677448.352405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.352406:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:20.0:1619677448.352407:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.352407:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.352407:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:20.0:1619677448.352409:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.352409:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:10.0:1619677448.352409:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.0:1619677448.352409:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.352410:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:25.2:1619677448.352412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.352412:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677448.352412:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:10.0:1619677448.352412:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:17.0:1619677448.352413:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.352413:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677448.352413:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.352415:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:17.0:1619677448.352415:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.352418:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:17.0:1619677448.352418:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.352420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:31.0:1619677448.352421:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.352422:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.352424:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.352425:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.352427:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:1.0:1619677448.352427:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.352430:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.352431:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.352431:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.352432:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.352433:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.352486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.352494:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.352496:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.352497:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.2:1619677448.352498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:24.0:1619677448.352499:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.352502:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.352504:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:18.0:1619677448.352504:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:24.0:1619677448.352505:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.352506:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.352507:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.352507:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.352508:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:24.0:1619677448.352509:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.352510:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.352513:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.352513:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.352515:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.352515:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.352516:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.352518:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.352518:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.352519:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:18.0:1619677448.352519:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.352520:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:18.0:1619677448.352520:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:24.0:1619677448.352521:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.352522:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.352522:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.352523:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677448.352523:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.352525:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.2:1619677448.352527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.352528:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.352529:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.352530:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.352532:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.352532:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.352533:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.352534:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:17.0:1619677448.352534:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677448.352535:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.352537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:17.0:1619677448.352537:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.352538:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.352538:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:21.2:1619677448.352540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000400:00000200:17.0:1619677448.352541:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.352543:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677448.352544:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:2.0:1619677448.352545:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:10.0:1619677448.352546:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.352547:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:2.0:1619677448.352547:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:10.0:1619677448.352548:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.352548:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.352549:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:10.0:1619677448.352550:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.352553:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.352620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.352626:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.352629:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.352630:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.352632:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.352634:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.352636:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.352637:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.352638:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.352639:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.352640:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:5.2:1619677448.352640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:22.0:1619677448.352644:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.352646:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677448.352646:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.352649:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.352649:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.352649:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:22.0:1619677448.352650:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.352651:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.352652:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000400:00000200:1.0:1619677448.352652:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.352653:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:1.0:1619677448.352653:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.352654:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.352654:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:1.0:1619677448.352654:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.352656:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.352657:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.352658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000400:00000200:1.0:1619677448.352658:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.352659:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.352662:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.352664:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:1.0:1619677448.352664:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.352666:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.352667:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:1.0:1619677448.352667:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.352667:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677448.352668:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.352669:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.352669:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.352669:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.352670:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.352671:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.352671:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:21.2:1619677448.352673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:26.0:1619677448.352674:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:8.2:1619677448.352675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.352678:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.352681:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:17.0:1619677448.352682:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677448.352682:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.352684:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:17.0:1619677448.352684:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.352685:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.352685:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:18.0:1619677448.352687:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.352687:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.352688:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:17.0:1619677448.352688:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:18.0:1619677448.352690:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.352691:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.352692:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.352693:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677448.352696:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.352698:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.352699:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.352700:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.352701:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.352702:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.352703:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.352703:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.352704:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.352708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.352715:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.352717:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.352718:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.352719:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.352720:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.352722:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.352847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.352854:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.352857:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.352858:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.352860:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.352862:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.352863:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677448.352865:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716089] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.352866:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.352867:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.352868:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.352872:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.352873:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.352877:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.352878:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.352878:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.352880:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.352881:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.352882:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677448.352883:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.352884:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677448.352888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.352889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.352896:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:5.0:1619677448.352896:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:26.0:1619677448.352898:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.352899:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:5.0:1619677448.352899:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:26.0:1619677448.352900:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:5.0:1619677448.352900:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.352902:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.352902:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.352905:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.352914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.352921:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.352924:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.352926:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.352928:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.352929:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.352931:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677448.352932:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.352933:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.352934:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.352935:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677448.352938:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.352940:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.352943:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.352944:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.352944:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.352945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:30.0:1619677448.352946:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.352946:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.352947:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.352948:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677448.352950:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.352952:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:8.2:1619677448.352952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677448.352954:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.352955:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.352957:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.352958:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677448.352959:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.352959:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:20.0:1619677448.352960:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.352961:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.352962:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.352962:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.352963:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.352963:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.352964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:20.0:1619677448.352966:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.352967:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.352968:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 440 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.352970:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.352971:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677448.352972:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.352972:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.352972:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.352973:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677448.352973:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677448.352975:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000400:00000200:17.0:1619677448.352975:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.352976:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000400:00000200:17.0:1619677448.352976:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.352977:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.352977:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:17.0:1619677448.352977:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.352979:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.352980:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.352981:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677448.352981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:17.0:1619677448.352982:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.352986:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.352987:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.0:1619677448.352987:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.352990:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.352991:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.352991:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677448.352991:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.352992:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.352992:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.352994:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:8.0:1619677448.352994:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.352995:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.352996:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.352996:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677448.352997:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.2:1619677448.352998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:25.2:1619677448.353000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:18.0:1619677448.353004:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677448.353007:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:18.0:1619677448.353007:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.353007:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677448.353009:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.353010:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.353010:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.353012:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.353012:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.353013:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:18.0:1619677448.353013:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677448.353015:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:18.0:1619677448.353015:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.353016:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.353017:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.353018:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.353021:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.353023:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:4.2:1619677448.353024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677448.353026:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.353027:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.353028:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.353030:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.353030:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.353031:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:10.0:1619677448.353031:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.353032:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:10.0:1619677448.353034:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.353035:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.353035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.353037:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.353040:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.353041:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:12.0:1619677448.353042:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677448.353043:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.353044:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:10.0:1619677448.353044:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.353045:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.353045:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677448.353047:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.353047:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:13.0:1619677448.353049:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.353050:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677448.353050:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.353052:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:25.2:1619677448.353054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.353054:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.353055:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.353056:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.353057:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.353057:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.353058:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.353059:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:13.0:1619677448.353060:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:25.2:1619677448.353063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:13.0:1619677448.353063:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.353064:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.353064:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.353065:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.353066:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.353069:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.353072:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.353073:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.353074:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.353076:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.353076:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.353081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:21.2:1619677448.353084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:31.0:1619677448.353088:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.353090:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677448.353091:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.353091:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.353092:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.353093:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.353093:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.353093:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:31.0:1619677448.353094:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:6.0:1619677448.353096:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.353099:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.353100:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.353101:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.353103:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.353104:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.353105:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.353108:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.353110:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.353112:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.353113:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.353114:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.353115:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.353116:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.353117:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.353117:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.353121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.353129:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.353132:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.353133:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.353133:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.353135:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.353138:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.353144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.353151:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.353154:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.353155:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.353155:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.353156:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.353157:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.353167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:24.0:1619677448.353174:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677448.353177:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677448.353178:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677448.353179:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.353182:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.353183:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:24.0:1619677448.353184:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.353186:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.353187:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.353188:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:24.0:1619677448.353191:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.353192:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:24.0:1619677448.353194:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.353195:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677448.353196:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.353197:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.353198:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677448.353198:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677448.353199:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.2:1619677448.353213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.2:1619677448.353214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.0:1619677448.353220:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:1.0:1619677448.353220:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.353223:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677448.353223:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677448.353223:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.353223:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:24.0:1619677448.353224:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.353226:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.353226:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.353228:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:1.0:1619677448.353229:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.353231:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.353232:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.353233:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.353234:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.353235:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.353239:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.353241:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.353242:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.353243:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.353244:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.353245:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.353245:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.353247:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.353248:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.353248:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.353251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.353254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.353260:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.353260:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.353260:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:17.0:1619677448.353261:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.353261:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.353262:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.353263:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.353263:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.353264:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677448.353264:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.2:1619677448.353452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.353457:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.353459:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.353460:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.353461:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.353463:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.353465:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.353466:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.353467:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.353468:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.353469:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.353472:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.353473:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.353476:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.353477:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.353477:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.353479:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.353479:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.353480:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.353480:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.353486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.353494:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.353496:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.353497:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.353498:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.353500:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.353502:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.353517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.353525:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.353528:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.353529:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.353531:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.353533:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.353534:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.353536:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.353537:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.353538:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.353539:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.353543:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.353544:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.353547:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.353548:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.353548:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.353550:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.353550:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.353551:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.353551:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.353555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.353563:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.353565:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.353566:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.353567:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.353569:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.353570:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.353610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.353618:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.353621:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.353623:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.353626:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.353626:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.353627:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.353629:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.353630:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.353631:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.353632:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677448.353635:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.353637:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.353640:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.353640:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.353641:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.353643:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.353643:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.353644:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.353644:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.353649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.353657:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.353660:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.353661:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.353661:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.353664:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.353666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.2:1619677448.353667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:2.0:1619677448.353667:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:6.0:1619677448.353672:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677448.353674:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677448.353675:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.353675:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:24.0:1619677448.353678:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.353678:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:6.0:1619677448.353678:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.353681:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.353681:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.353682:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:8.0:1619677448.353683:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.353683:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.353685:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.353686:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.353687:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.353688:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.353688:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:8.0:1619677448.353689:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.353690:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.353692:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.353693:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.353696:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:8.0:1619677448.353697:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.353699:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.353699:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.353699:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.353700:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.0:1619677448.353702:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.353702:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.353702:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677448.353703:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.353703:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.353703:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.353704:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677448.353705:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.353705:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.2:1619677448.353708:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.2:1619677448.353712:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.353715:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.353716:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.0:1619677448.353716:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.353719:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.353719:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.353720:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:5.0:1619677448.353721:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.353722:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:17.0:1619677448.353723:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.353723:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677448.353724:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.353724:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000400:00000200:1.0:1619677448.353724:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.353725:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.353726:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000400:00000200:1.0:1619677448.353726:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.353727:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)++
00000800:00000200:5.0:1619677448.353727:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.353729:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:1.0:1619677448.353729:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677448.353730:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.353757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.353763:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.353766:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.353766:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.353769:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.353771:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.353773:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.353774:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.353775:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.353776:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.353777:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.353781:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.353783:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.353786:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.353787:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.353788:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.353790:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.353790:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.353791:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.353791:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.353795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.353800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:13.0:1619677448.353802:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.353805:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.353806:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.353807:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.353808:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.353808:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.353810:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.353811:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.353812:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.353814:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.353816:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.353818:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.353819:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.353820:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.353821:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.353822:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.353825:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.353827:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.353828:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.353829:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.353830:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.353831:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.353832:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.353832:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.353833:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.2:1619677448.353835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:25.2:1619677448.353836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:17.0:1619677448.353840:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.353843:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:10.0:1619677448.353843:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.353844:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:1.0:1619677448.353844:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.353845:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.353846:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:17.0:1619677448.353846:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.353847:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.353848:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.353849:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:17.0:1619677448.353850:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.353852:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.353854:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.353855:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.353856:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.353859:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.353861:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.353864:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.353865:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.353865:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.353867:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.353868:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.353869:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.353869:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.353873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.353880:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.353883:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.353884:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.353886:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.353887:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.353889:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.353973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.353980:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.353983:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.353985:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.353987:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.353988:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.353990:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.353991:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.353993:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.353994:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.353995:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.353998:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.354000:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.354003:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.354004:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.354005:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.354007:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.354007:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.354008:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.354009:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.354013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.354021:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.354023:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.354024:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.354026:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.354027:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.354029:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.354118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:4.2:1619677448.354121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677448.354124:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.354127:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:26.0:1619677448.354128:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.354129:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.354129:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.354131:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.354132:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:5.0:1619677448.354132:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.354134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000400:00000200:5.0:1619677448.354134:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.354135:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.354136:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.354136:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.354138:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.354138:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.354139:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.354139:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.354140:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:5.0:1619677448.354140:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.354141:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.354141:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:26.0:1619677448.354142:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.354143:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.354144:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.354145:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.354147:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.354148:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:5.0:1619677448.354149:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.354150:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96716069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.354152:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.354152:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.354153:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.354154:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:26.0:1619677448.354155:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.354158:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.354158:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.354159:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.354160:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.354160:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.354161:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.354162:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.354162:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.354162:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.354163:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.354163:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.354163:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000800:00000200:20.0:1619677448.354164:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:26.0:1619677448.354165:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.354165:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677448.354166:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.354166:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677448.354166:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (23)--
00000800:00000200:20.0:1619677448.354167:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:5.0:1619677448.354167:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.354168:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:9.2:1619677448.354168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677448.354168:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.354170:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677448.354170:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.354172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:2.0:1619677448.354176:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:2.0:1619677448.354178:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000800:00000200:18.0:1619677448.354179:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (24)++
00000800:00000200:5.0:1619677448.354179:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:18.0:1619677448.354180:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000400:00000200:2.0:1619677448.354180:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.354181:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.354181:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.354182:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.354183:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.354183:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.354185:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.354185:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.354187:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:18.0:1619677448.354188:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.354188:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.354189:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.354191:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.354192:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.354192:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.354193:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.354194:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.354258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677448.354265:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.354268:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677448.354269:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677448.354272:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.354274:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.354275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:10.0:1619677448.354276:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.354277:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.354279:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.354280:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.354281:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:10.0:1619677448.354281:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.354283:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.354284:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:10.0:1619677448.354285:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.354286:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677448.354287:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.354289:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.354290:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:10.0:1619677448.354290:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.354291:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.354291:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.354292:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677448.354299:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.354300:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.354300:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.354300:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:25.2:1619677448.354301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677448.354301:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.354301:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.354302:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:18.0:1619677448.354304:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.354306:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.354308:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.354308:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.354309:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.354309:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.354310:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.354311:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:13.0:1619677448.354311:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.354311:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.354312:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.354312:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:13.0:1619677448.354313:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.354315:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:9.2:1619677448.354316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677448.354317:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.354324:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.354326:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.354327:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.354327:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.354329:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.354331:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.354430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.354436:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.354439:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.354440:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.354442:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.354445:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.354446:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.354447:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.354449:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.354450:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.354451:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.354454:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.354456:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.354458:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.354459:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.354460:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.354461:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.354462:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.354462:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.354463:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.354468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.354475:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.354478:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.354479:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.354479:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.354481:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.354483:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.354556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.354565:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.354567:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.354569:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.354572:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.354572:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.354574:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.354575:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.354577:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.354578:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.354579:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.354582:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.354584:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.354587:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.354588:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.354588:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.354590:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.354591:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.354592:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.354593:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.354594:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.354597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.354600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.354607:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:30.0:1619677448.354608:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.354610:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:20.0:1619677448.354610:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:30.0:1619677448.354611:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.354611:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.354612:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.354613:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.354614:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.2:1619677448.354615:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:20.0:1619677448.354616:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677448.354622:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.354624:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.354625:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.354627:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.354629:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.354631:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.354632:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.354633:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.354634:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.354635:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.354639:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.354640:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.354643:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.354644:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.354645:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.354646:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.354646:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.354648:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.354649:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.354649:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.354652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.354655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.354660:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:5.0:1619677448.354662:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677448.354662:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.354663:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.354664:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677448.354665:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.354666:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.354667:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.354668:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.354670:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.354707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.354709:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.354712:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.354714:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.354715:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.354717:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.354718:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677448.354720:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.354721:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.354722:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.354723:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677448.354726:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.354728:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677448.354731:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.354731:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677448.354732:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.354734:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677448.354735:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.2:1619677448.354736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:20.0:1619677448.354736:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.354737:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.354738:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.354740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.2:1619677448.354743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:24.0:1619677448.354744:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.354746:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:30.0:1619677448.354747:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:8.0:1619677448.354747:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677448.354748:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:30.0:1619677448.354749:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:24.0:1619677448.354749:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.354750:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.354750:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.354751:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:30.0:1619677448.354752:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.354752:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.354752:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677448.354753:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:24.0:1619677448.354753:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.354754:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96716062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.354755:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:24.0:1619677448.354756:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.354757:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.354758:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.354761:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.354763:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.354766:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.354767:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.354767:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.354769:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.354769:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.354771:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.354772:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.354772:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.2:1619677448.354774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:21.2:1619677448.354778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:12.0:1619677448.354781:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.354783:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.354784:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.354785:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.354785:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:13.0:1619677448.354787:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.354787:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.354788:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677448.354789:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677448.354789:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.354792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:13.0:1619677448.354798:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.354801:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.354802:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.354802:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.354803:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.354803:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.354803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.354810:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.354813:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.354813:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.354815:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.354818:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.354819:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.354821:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.354822:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.354823:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.354824:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.354827:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.354829:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.354832:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.354833:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.354833:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.354835:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.354835:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.354836:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.354837:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.354841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.354849:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.354852:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.354852:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:21.2:1619677448.354853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:6.0:1619677448.354853:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.354855:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.354858:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677448.354860:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.354863:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.354865:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.354867:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.354868:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.354869:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.354870:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.354872:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.354872:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.354873:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.354876:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.354878:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.354880:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.354880:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.354881:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.354882:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.354883:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.354884:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.354884:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.354888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.354894:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.354897:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.354897:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.354899:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.354900:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.354901:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.354910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.354916:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677448.354916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:17.0:1619677448.354919:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.354920:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:21.2:1619677448.354921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000400:00000200:17.0:1619677448.354922:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.354923:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677448.354925:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.354926:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:10.0:1619677448.354926:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.354927:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677448.354927:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:17.0:1619677448.354928:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.354929:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.354930:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000400:00000200:17.0:1619677448.354930:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.354931:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.354931:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:17.0:1619677448.354932:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677448.354933:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.354933:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.354935:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677448.354936:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.354936:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.354937:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:17.0:1619677448.354937:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.354938:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.354939:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.354940:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.354940:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:10.0:1619677448.354940:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.354941:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677448.354942:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.354943:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:17.0:1619677448.354943:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.354944:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.354945:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.354947:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.354947:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.354947:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.354948:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.354948:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.354949:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.354950:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.354950:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.354951:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.354953:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.354953:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677448.354953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:18.0:1619677448.354954:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:10.0:1619677448.354954:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:18.0:1619677448.354955:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:10.0:1619677448.354955:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.354955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.354957:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:10.0:1619677448.354957:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677448.354958:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:10.0:1619677448.354958:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677448.354959:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:18.0:1619677448.354959:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677448.354959:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:10.0:1619677448.354960:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:1.0:1619677448.354960:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677448.354961:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:10.0:1619677448.354961:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.354961:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677448.354962:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.354962:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:25.2:1619677448.354963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:24.0:1619677448.354963:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.354963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:9.2:1619677448.354963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:1.0:1619677448.354963:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.354964:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677448.354965:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.354967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677448.354968:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677448.354969:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:12.0:1619677448.354969:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:22.0:1619677448.354971:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:2.0:1619677448.354971:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:12.0:1619677448.354972:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:6.0:1619677448.354972:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:12.0:1619677448.354973:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.354974:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:26.0:1619677448.354975:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:22.0:1619677448.354975:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.354975:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.354976:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:13.0:1619677448.354976:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677448.354976:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.354977:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:22.0:1619677448.354977:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.354977:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.354978:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:12.0:1619677448.354978:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.354978:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677448.354979:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.354980:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.354980:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:22.0:1619677448.354981:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.354982:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.354983:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:9.2:1619677448.354983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000400:00000200:22.0:1619677448.354987:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.354989:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.354991:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.354991:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.354992:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.354993:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.354994:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.354995:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.354995:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677448.354995:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.354995:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.354996:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.354997:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677448.354997:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:22.0:1619677448.354998:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.354999:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.355001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.355004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.355009:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:5.0:1619677448.355010:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677448.355012:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.355012:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.0:1619677448.355013:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.355013:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.355014:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:5.0:1619677448.355015:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.355016:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.355018:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.355171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677448.355178:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677448.355181:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677448.355183:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.355184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.355185:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677448.355186:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.355188:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677448.355189:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.355190:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.355191:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.355191:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677448.355192:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:26.0:1619677448.355195:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.355195:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:30.0:1619677448.355196:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.355197:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.355198:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677448.355200:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.355200:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.355201:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:30.0:1619677448.355202:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:26.0:1619677448.355202:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:30.0:1619677448.355203:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000400:00000200:26.0:1619677448.355203:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.355204:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000400:00000200:26.0:1619677448.355204:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.355205:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:26.0:1619677448.355205:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.355206:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000400:00000200:26.0:1619677448.355206:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.355209:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.355209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:26.0:1619677448.355211:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.355213:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.355214:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.355215:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.355215:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:26.0:1619677448.355217:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:21.2:1619677448.355217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:8.0:1619677448.355217:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:26.0:1619677448.355218:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.355218:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:24.0:1619677448.355218:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.355218:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.355220:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.355220:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:8.0:1619677448.355220:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.355222:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:9.2:1619677448.355223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.355223:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.0:1619677448.355226:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:4.2:1619677448.355226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677448.355228:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.355228:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:9.2:1619677448.355230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:2.0:1619677448.355230:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:20.0:1619677448.355231:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.355232:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:10.0:1619677448.355233:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:2.0:1619677448.355233:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:20.0:1619677448.355234:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.355234:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.355234:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.355235:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.355236:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000400:00000200:20.0:1619677448.355236:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.355236:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.355236:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.355237:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:20.0:1619677448.355237:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677448.355237:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.355238:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.355239:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.355239:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.355240:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:2.0:1619677448.355240:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677448.355241:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:20.0:1619677448.355241:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.355243:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.355245:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.355246:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.355247:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.355248:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.355249:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.355250:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677448.355251:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677448.355252:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:8.2:1619677448.355254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:21.2:1619677448.355258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.355263:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.355265:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677448.355265:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.355266:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677448.355266:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.355266:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.355268:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.355268:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.355269:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677448.355270:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:8.2:1619677448.355274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000800:00000200:20.2:1619677448.355280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.0:1619677448.355286:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:31.0:1619677448.355288:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.355289:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.355289:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.355290:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:31.0:1619677448.355291:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:20.0:1619677448.355291:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.355292:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:31.0:1619677448.355293:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.355296:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.355299:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.355301:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.355302:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.355304:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.355305:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.355306:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.355309:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.355311:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.355314:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.355314:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.355315:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.355317:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.355317:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.355318:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.355318:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.355323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.355330:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:5.2:1619677448.355331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:30.0:1619677448.355333:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.355334:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.355336:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.355337:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:18.0:1619677448.355337:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:30.0:1619677448.355338:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:18.0:1619677448.355340:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.355341:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.355343:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.355344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000400:00000200:18.0:1619677448.355345:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.355346:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.355348:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.355349:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.355350:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000400:00000200:18.0:1619677448.355350:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.355351:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.355354:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.355354:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.355355:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000400:00000200:18.0:1619677448.355355:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.355356:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.355356:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:18.0:1619677448.355357:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.355360:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.355361:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.355362:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.355364:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.355364:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.355365:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.355365:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.355371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.355379:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.355381:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.355382:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.355384:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.355385:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.355386:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.355493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.2:1619677448.355496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677448.355502:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.355503:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.355505:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677448.355506:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.355506:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:4.2:1619677448.355507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:31.0:1619677448.355508:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.355508:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677448.355509:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.355511:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.355513:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.355513:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.355514:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677448.355515:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.355515:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677448.355516:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.355517:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.355518:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.355518:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.355518:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.355519:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677448.355520:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.355520:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.355521:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:10.0:1619677448.355522:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.355525:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.355526:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.355527:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.355527:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:17.0:1619677448.355529:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:10.0:1619677448.355529:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96716048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.355530:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.355530:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.355532:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.355532:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.355533:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.355533:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000400:00000200:10.0:1619677448.355533:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.355534:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.355534:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.355535:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.355536:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.355536:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.355537:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.355537:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.355538:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.355538:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.355539:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.355539:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.355540:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.355541:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.355542:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.355542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.355542:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.355543:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.355543:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:9.2:1619677448.355543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:10.0:1619677448.355545:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.355545:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.355546:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.355546:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677448.355548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677448.355549:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:25.2:1619677448.355550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:13.0:1619677448.355551:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677448.355552:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.355553:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.355553:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.355555:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.355556:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.355556:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.355557:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.355557:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:31.0:1619677448.355558:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.355558:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:13.0:1619677448.355558:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.355559:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.355560:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.355560:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.355560:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.355560:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.355561:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.355562:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.355562:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.355564:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:12.0:1619677448.355564:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.355564:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.355565:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.355566:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.355566:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.355702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.355710:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.355712:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.355714:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.355716:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.355716:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.355718:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.355719:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.355720:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.355721:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.355722:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.355726:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.355727:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.355730:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.355730:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.355731:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.355732:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.355733:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.355734:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.355734:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.355738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.355744:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.355746:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.355747:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.355749:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.355751:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.355752:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.355774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.355781:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.355784:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.355785:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.355786:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.355788:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.355790:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.355791:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.355792:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.355793:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.355794:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.355797:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.355797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677448.355799:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.355802:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.355802:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.355803:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.355804:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677448.355805:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.355806:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:24.0:1619677448.355806:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.355806:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.355807:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:10.0:1619677448.355807:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.2:1619677448.355808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:26.0:1619677448.355809:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.355811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677448.355812:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.355813:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.355814:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.355816:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.355817:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.355817:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:26.0:1619677448.355818:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.355818:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:2.0:1619677448.355819:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:10.0:1619677448.355821:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.355822:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.355822:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.355823:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:10.0:1619677448.355823:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.355823:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.355824:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.355826:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677448.355826:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.355828:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96716044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.355829:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.355829:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:10.0:1619677448.355831:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.355831:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.355835:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.355835:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.355836:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.355836:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.355837:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.355838:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.355839:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.355840:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.355840:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.355841:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.355841:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.355841:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.355842:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.355843:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:25.2:1619677448.355844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.355844:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.355845:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.355845:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.355849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.355856:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.355858:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.355858:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:22.0:1619677448.355859:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.355859:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.355860:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.355862:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.355862:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.355866:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.355866:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.355867:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677448.355868:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.355871:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.355872:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.355872:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.355873:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.355873:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.355923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.355926:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.355928:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.355931:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.355931:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.355933:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.355935:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.355936:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.355938:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.355939:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.355940:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.355943:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.355945:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.355949:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.355949:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.355950:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.355952:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.355952:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.355953:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.355954:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.2:1619677448.355957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:25.2:1619677448.355958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.355964:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:31.0:1619677448.355965:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.355967:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:26.0:1619677448.355967:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.355967:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:31.0:1619677448.355968:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.355970:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.355970:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:30.0:1619677448.355971:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.355973:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:26.0:1619677448.355973:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.355974:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.355975:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.355977:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.355978:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.355979:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.355982:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.355983:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.355986:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.355987:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.355988:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.355990:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.355991:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.355991:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.355992:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677448.355996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.356004:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.356007:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.356008:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.356008:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.356010:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.356013:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.356014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.356022:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.356024:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.356025:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.356025:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.356026:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.356027:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.356225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:4.2:1619677448.356229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.356233:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.2:1619677448.356233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:30.0:1619677448.356236:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:10.0:1619677448.356236:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677448.356238:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.356238:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:5.0:1619677448.356238:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.356239:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.356240:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.356240:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.356241:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.356241:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:30.0:1619677448.356242:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.356244:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.356244:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:10.0:1619677448.356244:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.356244:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.356245:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.356246:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.356246:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.356247:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.356247:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.356248:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.356249:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.356250:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.356250:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.356252:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.356252:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.356253:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.356254:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.356255:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:5.0:1619677448.356255:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.356256:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.356257:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.356258:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.356259:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.356260:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:10.0:1619677448.356260:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.356261:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:5.0:1619677448.356261:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.356262:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.356262:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:10.0:1619677448.356262:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.356263:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:25.2:1619677448.356265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.356265:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.356265:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.356266:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.356266:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.356267:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.356268:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.356268:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677448.356268:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.356269:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.356269:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.356270:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.356271:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.356272:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:5.0:1619677448.356272:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.356272:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.356274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.356275:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.356275:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.356277:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.356278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.356279:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.356279:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:22.0:1619677448.356281:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:21.2:1619677448.356282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677448.356284:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:12.0:1619677448.356284:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677448.356285:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.356286:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.356287:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:22.0:1619677448.356288:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:12.0:1619677448.356288:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.356290:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:12.0:1619677448.356290:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.356291:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677448.356291:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.356292:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.356298:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.356299:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.356300:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.356300:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.356302:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.356304:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.356305:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.356306:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.356307:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.356310:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.356312:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.356315:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.356315:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.356316:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.356318:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.356318:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.356319:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.356320:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.356333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.356341:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.356343:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.356344:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.356345:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.356347:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.356348:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.356359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.356364:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.356366:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.356368:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.356369:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.356371:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.356373:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.356374:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.356375:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.356376:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.356377:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.356380:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.356381:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.356384:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.356385:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.356385:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.356387:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.356387:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.356388:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.356389:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.356394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.356401:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.356404:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.356405:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.356406:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.356407:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.356408:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.2:1619677448.356625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.356633:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.356636:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.356637:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.356639:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.356642:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.356644:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.356645:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.356646:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.356647:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.356648:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.356652:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.356653:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.356656:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.356656:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.356657:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.356659:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.356659:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.356660:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.356660:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.356664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.356671:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.356673:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.356674:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.356675:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.356677:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.356678:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.356726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.356734:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.356737:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.356738:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.356740:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.356740:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.356742:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.356743:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.356744:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.356745:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.356746:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.356749:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.356751:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.356754:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.356755:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.356756:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.356757:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.356758:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.356759:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.356761:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.356761:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.356763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.356768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.356775:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677448.356776:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.356778:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:20.0:1619677448.356778:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677448.356779:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.356779:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.356780:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.356781:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.356781:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:8.2:1619677448.356784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.356788:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.356791:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.356792:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.356793:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.356794:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.356794:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.356795:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.356796:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.356796:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.356797:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:5.0:1619677448.356797:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.356798:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.356799:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.356800:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.356802:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.356803:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.356804:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.356805:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.356808:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.356810:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.356813:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.356813:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.356814:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.356816:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.356816:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.356818:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.356819:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.356819:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.356822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.356824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.356829:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:6.0:1619677448.356829:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677448.356831:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.356832:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677448.356832:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.356833:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.356833:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:6.0:1619677448.356833:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.356835:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.356838:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.356841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.356848:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.356851:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.356852:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.356852:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.356853:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.356853:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.356911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.356919:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.356922:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.356923:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.356925:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.356927:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.356929:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.356930:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.356932:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.356933:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.356934:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.356937:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.356938:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.356941:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.356942:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.356943:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.356945:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.356946:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.356946:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.356947:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.356951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.356960:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.356963:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.356964:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.356964:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.356966:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.356969:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.357054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.357060:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.357062:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.357063:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.357065:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.357068:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.357069:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.357071:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.357072:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.357073:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.357074:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.357077:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.357078:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.357080:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.357081:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.357081:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.357083:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.357083:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.357084:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.357084:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.357089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.357096:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.357098:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.357099:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.357100:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.357100:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.357102:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.357165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.357172:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.2:1619677448.357173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:24.0:1619677448.357175:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.357175:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677448.357176:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:24.0:1619677448.357178:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.357178:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.357180:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.357181:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.357181:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:24.0:1619677448.357182:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677448.357182:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.357184:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.357184:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.357185:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.357186:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.357187:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.357187:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.357189:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.357189:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.357190:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.2:1619677448.357194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:20.0:1619677448.357195:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.357196:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.357198:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677448.357199:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.357201:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.357202:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677448.357203:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.357203:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.357203:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677448.357204:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.357204:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.357205:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677448.357205:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.357205:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:24.0:1619677448.357206:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.357206:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:24.0:1619677448.357207:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677448.357207:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:10.0:1619677448.357207:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.357208:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.357208:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677448.357208:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:10.0:1619677448.357209:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.357211:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:25.2:1619677448.357212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:10.0:1619677448.357212:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96716029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.357212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:10.0:1619677448.357213:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.357214:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.357215:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.357219:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.357220:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.357221:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:1.0:1619677448.357221:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677448.357222:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677448.357223:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.357223:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:6.0:1619677448.357223:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.357223:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:22.0:1619677448.357224:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.357224:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:17.0:1619677448.357225:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:10.0:1619677448.357225:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:1.0:1619677448.357225:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.357226:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:6.0:1619677448.357226:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.357227:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.357227:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677448.357228:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:1.0:1619677448.357228:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.357230:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:25.2:1619677448.357231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:8.2:1619677448.357232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.357238:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:17.0:1619677448.357239:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.357240:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.357241:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.357242:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.357243:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.357243:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.357243:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:21.2:1619677448.357244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677448.357244:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.357245:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677448.357246:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.357247:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677448.357248:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.357251:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.357253:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.357255:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.357255:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.357257:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.357258:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.357259:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.357260:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.357261:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.357264:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.357266:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.357269:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.357270:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.357270:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.357272:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.357273:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.357274:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.357274:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.357278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.357279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.357286:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.357286:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.357289:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:22.0:1619677448.357289:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.357289:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.357290:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.357290:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.357292:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.357293:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.357300:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:26.0:1619677448.357301:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.357303:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.357305:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.357306:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.357307:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.357308:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.357311:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.357313:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.357315:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.357316:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.357316:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.357318:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.357319:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.357320:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.357321:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.357322:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.357325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.357328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.357332:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.357335:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.357335:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.357335:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:30.0:1619677448.357337:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.357337:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.357337:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.357338:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.357339:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.357341:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.357453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.0:1619677448.357457:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.357460:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677448.357462:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.357463:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.357465:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.357466:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.357467:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.357469:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.357470:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.357471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:5.0:1619677448.357471:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.357475:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.357477:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677448.357477:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.357479:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.357480:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:5.0:1619677448.357480:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.357480:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:22.0:1619677448.357481:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.0:1619677448.357482:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.357482:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000400:00000200:6.0:1619677448.357483:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.357483:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.357484:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:6.0:1619677448.357485:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.357486:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.357488:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.357489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000400:00000200:6.0:1619677448.357489:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.357490:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.357491:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.357494:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.357495:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.357496:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.357498:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:6.0:1619677448.357498:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.357499:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.357499:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.357500:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:20.0:1619677448.357501:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.357502:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.357502:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677448.357502:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.357503:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:6.0:1619677448.357503:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.357504:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.357508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.357515:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.357518:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.357519:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.357520:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.357521:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.357523:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.357526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.357532:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.357535:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.357536:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.357536:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.357537:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.357538:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.357737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1200] (20)++
00000800:00000200:20.2:1619677448.357742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677448.357744:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677448.357747:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:1.0:1619677448.357747:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:5.0:1619677448.357748:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677448.357749:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.357750:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:1.0:1619677448.357751:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.357753:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.357753:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.357753:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:1.0:1619677448.357754:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.357755:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.357756:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.357757:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677448.357757:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.357758:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.357758:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.357760:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.357761:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.357762:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.357767:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.357767:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.357770:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 584 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.357771:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.357773:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.357774:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.357774:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.357775:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.357776:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677448.357776:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.357777:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677448.357777:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:1.0:1619677448.357778:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:5.0:1619677448.357779:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.357780:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.357780:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677448.357780:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdbb1200] (22)++
00000800:00000200:1.0:1619677448.357781:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (23)--
00000800:00000200:1.0:1619677448.357782:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:8.2:1619677448.357783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:25.2:1619677448.357784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.2:1619677448.357787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:22.0:1619677448.357791:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677448.357791:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.357793:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:17.0:1619677448.357793:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)++
00000800:00000200:22.0:1619677448.357794:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:21.2:1619677448.357794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:20.0:1619677448.357794:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.357795:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.357795:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdbb1200] (23)--
00000800:00000200:6.0:1619677448.357795:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:20.0:1619677448.357796:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.357796:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:22.0:1619677448.357797:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.357797:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:5.0:1619677448.357798:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.357799:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:22.0:1619677448.357800:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677448.357800:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:4.2:1619677448.357802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677448.357803:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.357804:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.357805:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.357808:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.357809:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677448.357810:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.357811:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.357813:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.357813:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:2.0:1619677448.357813:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.357814:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.357815:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.357817:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.357819:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.357819:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.357821:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:2.0:1619677448.357821:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.357822:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.357823:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.357824:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.357824:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:10.0:1619677448.357825:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.357825:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677448.357826:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.357826:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.357827:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.2:1619677448.357828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.357828:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:20.2:1619677448.357829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000400:00000200:10.0:1619677448.357829:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.357829:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.357830:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:10.0:1619677448.357831:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.357833:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.357833:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:9.2:1619677448.357833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.357834:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:1.0:1619677448.357834:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677448.357835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:10.0:1619677448.357835:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:6.0:1619677448.357835:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:10.0:1619677448.357836:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.357836:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.357837:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677448.357837:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:22.0:1619677448.357838:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:17.0:1619677448.357838:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.357838:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000400:00000200:1.0:1619677448.357840:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.357841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.357841:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:6.0:1619677448.357841:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.357842:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.357843:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.357843:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.357843:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:1.0:1619677448.357843:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.357844:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.357844:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:1.0:1619677448.357844:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:2.0:1619677448.357845:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:6.0:1619677448.357846:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:2.0:1619677448.357846:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:1.0:1619677448.357846:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96716020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.357847:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:18.0:1619677448.357847:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.357848:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96716020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.357848:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.357849:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:18.0:1619677448.357849:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:1.0:1619677448.357849:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.357850:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.357850:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.357850:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.357851:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:6.0:1619677448.357852:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.357853:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.357853:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.357854:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.357855:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:6.0:1619677448.357858:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.357858:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.357860:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.357861:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.357861:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.357861:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:25.2:1619677448.357862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:6.0:1619677448.357863:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.357863:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.357863:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.357864:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:1.0:1619677448.357864:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.357865:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.357865:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.357866:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677448.357867:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:10.0:1619677448.357868:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:6.0:1619677448.357868:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:8.2:1619677448.357869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.357869:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:10.0:1619677448.357870:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:6.0:1619677448.357870:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:26.0:1619677448.357871:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.357871:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.357872:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:9.2:1619677448.357872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.357873:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.357875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (21)++
00000800:00000200:17.0:1619677448.357876:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.357878:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.357879:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.357879:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:22.0:1619677448.357880:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000800:00000200:1.0:1619677448.357880:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.357881:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.357882:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:22.0:1619677448.357883:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:22.0:1619677448.357883:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:17.0:1619677448.357883:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:2.0:1619677448.357883:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.357884:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677448.357885:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000400:00000200:2.0:1619677448.357885:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.357888:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.357948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.357955:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.357958:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.357960:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.357961:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.357963:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.357964:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.357966:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.357967:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.357968:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.357969:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.357973:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.357974:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.357977:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.357978:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.357978:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.357980:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.357980:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.357981:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.357982:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677448.357984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:25.2:1619677448.357985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:6.0:1619677448.357989:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.357991:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:6.0:1619677448.357992:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:26.0:1619677448.357993:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:22.0:1619677448.357993:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:26.0:1619677448.357995:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.357995:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:6.0:1619677448.357995:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.357997:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.357997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:6.0:1619677448.357998:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.357999:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.358000:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.358001:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.358002:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:6.0:1619677448.358002:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.358003:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.358004:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:6.0:1619677448.358004:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.358006:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.358008:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.358009:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.358010:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.358011:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:6.0:1619677448.358011:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:26.0:1619677448.358013:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.358013:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.358014:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.358014:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:26.0:1619677448.358015:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.358015:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677448.358016:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677448.358017:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.358017:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.358018:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.358018:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.2:1619677448.358018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:26.0:1619677448.358020:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.358022:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.2:1619677448.358022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.358024:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:26.0:1619677448.358025:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.358025:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.358026:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.358027:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:26.0:1619677448.358028:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.358028:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:26.0:1619677448.358029:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.358029:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:18.0:1619677448.358030:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.358030:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:26.0:1619677448.358031:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.358032:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:18.0:1619677448.358032:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.358032:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:18.0:1619677448.358033:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:17.0:1619677448.358033:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.358033:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.358034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677448.358035:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96716015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.358036:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.358037:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.358037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677448.358038:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:18.0:1619677448.358039:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.358040:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.358043:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:18.0:1619677448.358044:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.358044:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:2.0:1619677448.358044:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.358046:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.358046:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:2.0:1619677448.358046:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.358047:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.358048:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:2.0:1619677448.358048:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.358050:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.358051:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.358051:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.358052:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.358053:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.358054:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.358054:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.358055:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.358055:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.358056:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.358056:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.358057:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677448.358057:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.358058:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:25.2:1619677448.358060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.358068:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.358070:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.358071:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.358073:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.358075:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.358076:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.358144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.358150:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.358153:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.358153:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.358155:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.358157:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.358159:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.358160:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716014] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.358161:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.358162:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.358163:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.358166:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.358168:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.358171:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.358172:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.358173:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.358175:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.358175:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.358177:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.358178:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.358178:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.358181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.358183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.358187:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677448.358188:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677448.358190:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.358191:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.358191:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.358191:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677448.358192:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.358192:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.358193:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.358195:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.358259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.358265:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.358267:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.358268:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.358270:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.358274:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.358275:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.358276:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.358278:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.358279:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.358280:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.358284:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.358285:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.358288:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.358289:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.358289:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.358291:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.358292:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.358293:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.358297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.358298:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:10.0:1619677448.358305:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.358308:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.358308:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.358309:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.358311:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.358314:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.358391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.358398:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.358401:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.358401:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.358403:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.358405:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.358407:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.358408:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.358409:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.358410:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.358411:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.358415:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.358416:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.358420:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.358420:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.358421:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.358423:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.358423:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.358424:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.358424:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.358429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.358438:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.358440:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.358442:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.358443:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.358444:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.2:1619677448.358446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.358449:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.358450:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.358452:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.358453:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.358454:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.358454:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.358455:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.358503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.358511:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.358514:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.2:1619677448.358515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677448.358516:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.358518:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.358518:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.358520:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.358521:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.358521:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677448.358523:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.358524:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.358524:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.358524:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677448.358525:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677448.358528:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.358529:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.358530:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.358532:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:17.0:1619677448.358532:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.358533:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.358534:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.358535:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.358535:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:17.0:1619677448.358535:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.358536:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.358537:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:17.0:1619677448.358537:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.358538:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677448.358541:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.358542:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:17.0:1619677448.358542:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:25.2:1619677448.358543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.358544:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.358545:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677448.358546:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.358546:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.358547:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.358549:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.358549:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:21.2:1619677448.358550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.358550:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.358550:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.358551:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.358552:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.358553:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.358553:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.358555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:10.0:1619677448.358555:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.358558:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677448.358560:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.358561:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.358562:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.358564:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.358564:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677448.358565:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.358566:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.358566:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:31.0:1619677448.358568:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.358568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:31.0:1619677448.358571:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.358571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:20.0:1619677448.358573:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.358575:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:22.0:1619677448.358577:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:20.0:1619677448.358577:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.358578:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:22.0:1619677448.358579:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:20.0:1619677448.358579:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.358581:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:6.0:1619677448.358581:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677448.358582:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.358583:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.358584:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.358585:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.358586:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.358586:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677448.358588:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.358589:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.358590:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000400:00000200:22.0:1619677448.358591:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.358592:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.358593:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:20.0:1619677448.358593:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.358595:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.358596:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.358596:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:22.0:1619677448.358597:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.358598:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:22.0:1619677448.358599:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677448.358599:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.358600:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.358601:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:4.2:1619677448.358601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677448.358602:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.358602:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.358603:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.358605:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.358605:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.2:1619677448.358605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:22.0:1619677448.358606:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.358607:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.358611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677448.358612:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.0:1619677448.358612:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.358614:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:8.0:1619677448.358615:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:5.0:1619677448.358615:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.358616:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:5.0:1619677448.358617:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:20.0:1619677448.358618:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:8.0:1619677448.358619:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.358619:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:5.0:1619677448.358620:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:8.0:1619677448.358621:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.358621:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.358622:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.358623:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:8.2:1619677448.358624:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000400:00000200:2.0:1619677448.358624:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.358627:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:8.0:1619677448.358628:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.358630:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96716007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.358631:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.358632:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000400:00000200:8.0:1619677448.358632:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.358633:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.358635:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.358636:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.358636:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.358637:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:8.0:1619677448.358637:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.358638:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:8.0:1619677448.358639:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.358642:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.358643:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.358644:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.358646:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.358646:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.358647:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.358649:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.358649:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.358651:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.358655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.358659:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.358661:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677448.358662:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.358662:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.358663:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.358664:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.358665:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677448.358665:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.358666:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677448.358668:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.358696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.358699:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.358702:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.358704:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.358705:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.358706:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.358708:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677448.358709:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.358710:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.358711:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.358712:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677448.358716:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.358717:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.358720:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.358720:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.358721:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.358723:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.358723:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.358724:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.358725:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.358729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.358736:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.358739:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.358739:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.358741:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.358742:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.358743:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.358769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.358777:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.358780:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.358781:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.358783:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.358785:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.358787:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.358788:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96716005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.358790:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.358791:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.358792:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.358795:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.358797:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.358799:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.358800:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.358801:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.358803:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.358803:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.358804:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.358804:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.358808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.358816:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.358819:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.358820:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.358820:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.358822:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:17.0:1619677448.358825:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.358831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.358838:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.358841:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.358841:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.358843:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.358846:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.358848:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.358849:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.358851:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.358851:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.358853:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.358856:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.358858:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.358860:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.358861:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.2:1619677448.358862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:18.0:1619677448.358862:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.358864:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.358864:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.358865:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.358865:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677448.358866:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.358868:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.358870:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.358870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.358871:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.358872:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.358874:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677448.358875:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96716003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.358877:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.358877:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:20.0:1619677448.358878:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.358879:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:1.0:1619677448.358879:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.358880:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.358881:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:20.0:1619677448.358883:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.358883:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.358885:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677448.358886:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.2:1619677448.358887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.358888:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.358889:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.358890:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.358892:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.358892:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.358893:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.358894:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677448.358894:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.358897:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.358897:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:25.2:1619677448.358898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:17.0:1619677448.358898:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.358899:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.358899:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677448.358904:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.358907:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.358907:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.358908:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.358910:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.358914:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.358915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.358921:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.358923:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.358924:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.358924:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.358926:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.358926:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.358954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.358959:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.358962:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.358963:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.358965:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.358967:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.358969:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.358970:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.358971:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.358972:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.358973:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.358977:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.358979:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:5.2:1619677448.358981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677448.358981:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.358982:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.358983:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.358985:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.358985:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.358986:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.358986:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.358988:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.358991:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.358992:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.358993:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.358996:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.358998:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.358999:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96716001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.359000:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.359001:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677448.359001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:10.0:1619677448.359002:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.359005:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.359007:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.0:1619677448.359007:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:10.0:1619677448.359009:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.359010:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.359010:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:8.0:1619677448.359010:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.359011:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.359011:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.359012:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.359012:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.359013:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:8.0:1619677448.359013:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.359014:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677448.359016:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:25.2:1619677448.359017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.359023:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.359025:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.359025:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.359026:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.359027:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.359029:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.359195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.359200:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.359203:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.359204:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.359206:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.359208:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.359210:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.359211:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96716000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.359213:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.359213:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.359215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:17.0:1619677448.359215:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.359218:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.359220:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.359223:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677448.359223:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.359224:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.359224:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.359226:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:17.0:1619677448.359226:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.359227:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.359227:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677448.359228:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.359228:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.359230:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.359230:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.359232:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.359233:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.359235:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.359235:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.359237:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.359240:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.359242:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.359245:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677448.359245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.359246:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.359247:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.359248:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.359249:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.359250:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.359250:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677448.359253:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.2:1619677448.359254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:12.0:1619677448.359255:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.359256:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.359258:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.359260:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.359260:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.359262:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.359265:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.359266:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.359268:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.359268:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.359270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.359271:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.359275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:13.0:1619677448.359277:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.359280:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.359281:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.359281:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.359282:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.359283:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.359283:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.359286:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.359288:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.359290:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.359291:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.359292:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.359298:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715998] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.359299:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.359301:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.359301:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.359305:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.359307:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.359309:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.359309:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.359310:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.359311:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.359312:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.359313:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.359313:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.359317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.359325:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.359327:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.359328:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.359328:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.359330:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.359332:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.359440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.359448:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.359451:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.359451:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.359453:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.359455:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.359457:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.359458:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.359459:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.359460:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.359461:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.359464:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.359466:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.359468:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.359469:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.359469:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.359471:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.359471:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.359472:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.359472:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.359476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:21.2:1619677448.359483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:12.0:1619677448.359483:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.359486:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.359487:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.359488:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.359489:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:12.0:1619677448.359490:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.359491:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677448.359492:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.359494:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.359495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:5.0:1619677448.359495:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.359496:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.359498:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:20.0:1619677448.359499:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.359501:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.359501:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:20.0:1619677448.359502:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.359503:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.359505:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.359506:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.359506:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:20.0:1619677448.359507:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.359507:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:21.2:1619677448.359508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:12.0:1619677448.359508:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677448.359509:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 440 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.359512:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.359512:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:20.0:1619677448.359513:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.359515:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677448.359515:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:6.0:1619677448.359515:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677448.359517:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:6.0:1619677448.359517:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.359518:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677448.359518:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677448.359519:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:25.2:1619677448.359520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:6.0:1619677448.359520:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.359522:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.359523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (21)++
00000400:00000200:6.0:1619677448.359524:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.359525:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.359526:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.359527:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:6.0:1619677448.359527:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.359527:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000400:00000200:6.0:1619677448.359528:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.359529:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:13.0:1619677448.359530:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:5.0:1619677448.359530:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:13.0:1619677448.359531:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.359531:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:6.0:1619677448.359532:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677448.359533:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.359534:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677448.359534:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.359536:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677448.359536:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.359537:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.359537:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.359539:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.359539:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.359540:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.359541:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.359542:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.2:1619677448.359544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677448.359546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.359548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677448.359552:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677448.359554:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:20.0:1619677448.359554:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.359554:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:22.0:1619677448.359555:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:20.0:1619677448.359555:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677448.359556:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677448.359556:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.359556:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.359557:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:20.0:1619677448.359558:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.359559:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.359561:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.359562:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.359563:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.359564:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.359566:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.359566:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.359568:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.359568:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:5.0:1619677448.359569:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.359571:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.359573:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.359574:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.359574:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.359577:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.359577:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.359579:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.359580:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.359581:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:9.2:1619677448.359583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677448.359586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.359589:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.359591:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:2.0:1619677448.359591:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677448.359592:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.359592:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677448.359593:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.359594:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.359595:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.359596:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.359599:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.359625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.359632:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.359635:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.359635:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.359637:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.359640:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.359642:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.359643:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.359644:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.359645:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.359646:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.359649:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.359651:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.359654:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.359655:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.359656:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.359657:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.359658:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.359659:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.359661:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.359661:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.359663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.359666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.359670:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677448.359671:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.359673:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:12.0:1619677448.359673:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:17.0:1619677448.359674:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677448.359674:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.359675:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677448.359675:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.359677:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.359678:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.359723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.359730:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.359733:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.359734:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.359735:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.359738:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.359739:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.359740:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.359742:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.359743:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.359744:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.359747:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.359749:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.359752:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.359753:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.359753:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.359755:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.359755:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.359756:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.359756:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.359761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.359769:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.359772:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.359772:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.359773:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.359775:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.359777:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.359880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:6.0:1619677448.359886:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677448.359888:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677448.359889:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:6.0:1619677448.359891:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.359893:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.359895:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677448.359896:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.359897:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.359898:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.359900:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677448.359903:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.359905:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677448.359907:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.359908:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:6.0:1619677448.359909:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677448.359911:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:6.0:1619677448.359911:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677448.359912:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677448.359912:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:8.2:1619677448.359917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.359925:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.359927:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.359928:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.359928:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.359930:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.359932:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.359956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.359964:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.359967:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.359969:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.359971:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.359971:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.359973:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.359974:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.359976:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.359977:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.359978:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.359981:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.359983:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.359986:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677448.359986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:30.0:1619677448.359987:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.359988:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677448.359988:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677448.359989:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.359990:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.359991:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.359991:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:5.2:1619677448.359991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:30.0:1619677448.359992:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:20.0:1619677448.359993:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.359995:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.359996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:20.0:1619677448.359997:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677448.359997:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.359998:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.360000:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.360000:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:20.0:1619677448.360001:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677448.360001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:20.0:1619677448.360002:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.360003:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.360005:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:18.0:1619677448.360007:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.360007:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:26.0:1619677448.360008:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677448.360008:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.360008:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.360010:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.360011:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:20.0:1619677448.360011:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.360011:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677448.360011:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.360012:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:20.0:1619677448.360014:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.360014:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96715988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.360015:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.360015:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.360016:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677448.360016:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.360017:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.360017:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.360018:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:18.0:1619677448.360018:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.360019:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677448.360019:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.360020:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.360020:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:26.0:1619677448.360021:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96715987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.360023:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.360023:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:26.0:1619677448.360024:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.360024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677448.360024:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.360024:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.360025:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:26.0:1619677448.360026:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.360027:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.360029:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.360030:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.360030:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.360031:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:13.0:1619677448.360031:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677448.360032:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.360033:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.360033:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:13.0:1619677448.360033:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677448.360034:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:13.0:1619677448.360034:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.360035:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.360035:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.360035:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:26.0:1619677448.360036:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.360036:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:13.0:1619677448.360036:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.360037:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.360038:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.360038:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.360039:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.360039:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:13.0:1619677448.360039:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677448.360039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.360042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:25.2:1619677448.360044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.360047:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:5.0:1619677448.360049:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677448.360050:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.360050:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:22.0:1619677448.360051:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.360051:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:2.0:1619677448.360051:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:5.0:1619677448.360052:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.360054:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677448.360054:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.360054:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.360055:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.360055:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.360057:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.360057:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677448.360061:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.360064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.360072:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.360074:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.360075:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.360076:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.360076:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.360077:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.360160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.360168:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.360170:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.360171:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.360173:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.360176:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.360177:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.360178:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.360179:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.360180:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.360181:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.360184:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.360185:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.360187:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.360188:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.360188:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.360190:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.360191:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.360192:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.360192:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.360196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.360202:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.360205:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.360205:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.360207:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.360209:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.360210:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.360272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.360275:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.360277:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.360279:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.360280:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.360282:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.360283:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.360285:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.360286:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.360287:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.360288:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.360291:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.360297:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.360300:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.360301:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.360302:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.360303:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.360304:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.360305:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.360305:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.360310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.360317:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:21.2:1619677448.360320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:12.0:1619677448.360320:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.360320:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.360322:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.360325:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.360326:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677448.360328:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.360331:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.360331:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.360333:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.360336:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.360337:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.360339:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.360340:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.360341:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.360342:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677448.360345:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.360347:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.360350:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.360351:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.360352:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.360353:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.360354:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.360355:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.360356:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.360357:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.2:1619677448.360359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.360363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677448.360369:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.0:1619677448.360370:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:5.0:1619677448.360371:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.360372:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.360373:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.360373:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.360374:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:5.0:1619677448.360374:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.360376:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.360377:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.360409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677448.360410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677448.360417:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:1.0:1619677448.360417:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.360420:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:17.0:1619677448.360420:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.360420:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.360423:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.360424:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.360425:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:1.0:1619677448.360426:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.360427:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.360428:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677448.360430:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.360430:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.360432:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.360432:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.360433:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.360434:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.360434:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677448.360435:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.360436:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.360439:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.360441:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.360442:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677448.360443:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.360445:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.360446:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.360446:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.360447:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.360447:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.360448:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.360449:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.360450:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.360450:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:1.0:1619677448.360450:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.360451:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:1.0:1619677448.360451:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.360451:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.360452:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.360455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:9.2:1619677448.360455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.360462:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:13.0:1619677448.360463:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:24.0:1619677448.360465:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:13.0:1619677448.360465:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:24.0:1619677448.360466:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.360466:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.360466:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:13.0:1619677448.360468:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.360469:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.360469:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.360470:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.360472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.360472:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:12.0:1619677448.360479:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.360481:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.360482:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.360483:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.360483:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.360485:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.360533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.360540:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.360543:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.360545:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.360547:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.360548:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.360549:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.360551:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.360552:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.360553:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.360554:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.360557:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.360559:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.360561:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.360562:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.360562:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.360564:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.360564:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.360565:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.360566:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.360569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.360576:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.360578:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.360579:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.360580:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.360582:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.360583:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.360623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.360631:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.360634:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.360635:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.360637:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.360639:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.360640:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:21.2:1619677448.360640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:26.0:1619677448.360641:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.360643:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.360643:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.360645:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.360646:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:26.0:1619677448.360648:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.360649:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.360649:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.360650:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677448.360653:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.360653:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.360654:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.360654:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.360656:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.360656:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677448.360656:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.360657:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:22.0:1619677448.360657:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.360658:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.360659:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.360659:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.360660:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.360661:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.360662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:22.0:1619677448.360662:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:4.2:1619677448.360665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677448.360666:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.360668:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.360671:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.360671:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.360672:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.360672:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.360672:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.360673:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:6.0:1619677448.360673:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.360674:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.360674:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677448.360675:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.360675:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.360675:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.360676:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.360677:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:21.2:1619677448.360677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:22.0:1619677448.360678:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677448.360678:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.360679:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.2:1619677448.360681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.360685:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.360687:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.360689:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.360690:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.360691:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.360692:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.360692:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.360693:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.360694:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.360695:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715978] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.360696:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.360697:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.360698:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.360701:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.360703:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.360704:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.360705:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.360705:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.360706:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.360706:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677448.360707:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.360709:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.360709:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.360710:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.360711:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.360714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.360722:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.2:1619677448.360726:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:20.0:1619677448.360728:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677448.360729:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.360730:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.360731:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.360732:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677448.360734:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:18.0:1619677448.360735:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.360736:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.360738:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.360740:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.360742:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.360743:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.360745:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.360746:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.360747:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.360750:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.360752:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.360755:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.360755:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.360756:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.360757:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.360758:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.360759:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.360759:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.360763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.360771:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.360773:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.360774:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.360776:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.360777:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.360778:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.360790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.360797:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.360800:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.360801:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.360802:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.360802:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.360804:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.360818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.360824:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.360827:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.360828:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.360830:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.360832:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.360834:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.360835:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.360836:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.360837:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.360838:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.360840:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.360842:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.360844:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.360845:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.360845:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.360847:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.360847:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.360848:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.360849:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.360852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.360860:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.360863:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.360864:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.360864:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.360866:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.360869:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.360873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.360881:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.360884:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.360886:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.360888:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.360889:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.360890:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.360891:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.360893:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.360894:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.360895:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.360898:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.360899:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.360903:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.360904:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.360905:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.360906:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.360907:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.360907:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.360908:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.360912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.360920:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.360923:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.360923:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.360925:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.360925:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.360928:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.360990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.360997:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.361000:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.361001:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.361003:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.361006:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.361007:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.361008:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.361010:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.361011:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.361012:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.361015:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.361017:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.361020:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.361021:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.361021:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.361023:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.361023:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.361024:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.361024:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677448.361027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:25.2:1619677448.361029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:8.0:1619677448.361034:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:22.0:1619677448.361036:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.0:1619677448.361036:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.361037:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.361038:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.361039:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.361040:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.361040:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:8.0:1619677448.361041:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.361043:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.361043:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.361045:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96715973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.361046:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:8.0:1619677448.361046:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.361047:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.361048:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.361052:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.361053:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.361056:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.361057:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.361058:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.361060:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.361061:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.361061:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.361062:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.361066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.361074:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.361077:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.361078:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.361078:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.361080:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.361084:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.361110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.361118:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.361120:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.361121:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.361123:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.361126:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.361128:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.361129:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.361130:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.361131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:10.0:1619677448.361132:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.361132:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.361136:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.361137:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.361138:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.361139:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.361140:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.361140:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.361141:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:10.0:1619677448.361142:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.361142:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677448.361143:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.361143:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.361144:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.361145:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.361146:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.361147:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.361148:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.361148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:30.0:1619677448.361149:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.361150:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.361151:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.361154:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:30.0:1619677448.361154:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.361155:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.361156:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.361156:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.361157:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.361158:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.361159:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.361159:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.361159:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.361160:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.361161:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.361161:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.361163:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.361165:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.361165:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.2:1619677448.361166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:25.2:1619677448.361167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.361169:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.2:1619677448.361171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.361171:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.361173:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.361173:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.361175:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:20.0:1619677448.361176:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.361176:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.361176:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.361178:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677448.361179:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:12.0:1619677448.361179:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.361180:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.361180:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.361181:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:20.0:1619677448.361181:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.361182:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677448.361182:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.361182:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.361183:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:20.0:1619677448.361183:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.361187:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.361188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.361189:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.361192:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.361192:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.361193:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.361194:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:20.0:1619677448.361196:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:20.0:1619677448.361196:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.361197:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.361198:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.361198:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.361199:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.361200:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677448.361200:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.361202:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.361204:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.361205:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.361207:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.361208:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.361210:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.361293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.361303:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.361306:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.361308:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.361311:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.361311:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.361312:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.361314:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.361315:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.361316:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.361317:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.361321:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.361322:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.361324:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.361325:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.361326:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.361327:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.361328:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.361329:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.361330:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.361331:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.361334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.361337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.361341:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.361343:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.361344:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.361345:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.361346:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.361348:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:13.0:1619677448.361348:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.361348:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.361349:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.361350:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.2:1619677448.361541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.361547:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677448.361549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:17.0:1619677448.361550:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.361551:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.361553:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.361556:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.361557:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677448.361558:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.361559:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.361560:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:17.0:1619677448.361561:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.361561:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:17.0:1619677448.361562:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.361564:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:26.0:1619677448.361565:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.361567:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.361568:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.361569:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.361570:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.361570:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:26.0:1619677448.361571:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.361572:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.361573:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:17.0:1619677448.361573:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.361574:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.361574:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.361576:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000400:00000200:26.0:1619677448.361577:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.361577:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.361578:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.361578:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.361579:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.361581:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677448.361581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.361582:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:9.2:1619677448.361582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:26.0:1619677448.361583:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.361585:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.361585:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.361587:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.361588:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.361589:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.361590:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677448.361590:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:25.2:1619677448.361591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.361592:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:24.0:1619677448.361592:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.361593:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.361593:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677448.361593:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:26.0:1619677448.361594:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.361594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:24.0:1619677448.361595:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.361597:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.361598:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:24.0:1619677448.361598:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:26.0:1619677448.361599:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:30.0:1619677448.361600:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:26.0:1619677448.361600:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.361601:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.361602:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:30.0:1619677448.361603:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.361603:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.361603:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:26.0:1619677448.361604:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.361604:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.361605:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:26.0:1619677448.361605:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.361605:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.361607:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:26.0:1619677448.361609:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.361609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.361611:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.361613:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.361614:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.361615:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.361616:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.361617:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.361617:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.361618:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.361618:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.361618:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.361619:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.361620:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.361620:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.361621:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:9.2:1619677448.361623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677448.361627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:2.0:1619677448.361631:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.0:1619677448.361634:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677448.361634:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.361635:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.361635:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.361636:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.361637:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677448.361637:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.361639:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.361641:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.361641:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:8.0:1619677448.361643:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.361644:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.361645:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.361646:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.361647:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.361651:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.361652:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.361655:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.361656:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.361656:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.361658:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.361658:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.361659:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.361660:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.361664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.361672:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.361674:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.361675:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.361675:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.361678:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.361681:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.361720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.0:1619677448.361724:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.361727:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677448.361729:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.361729:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.361732:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.361733:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.361734:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.361735:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.361736:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.361737:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.361740:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.361742:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.361744:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.361744:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.361745:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.361747:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.361747:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.361748:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.361749:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.361753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.361760:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.361762:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.361763:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.361765:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.361766:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.361767:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:21.2:1619677448.361834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.361842:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.361844:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.361847:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.361848:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.361849:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.361850:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.361852:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.361853:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.361854:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.361855:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677448.361859:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.361860:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.361863:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.361864:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.361865:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.361867:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.361867:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.361868:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.361869:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.361873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.361880:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.361883:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.361884:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.361884:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.361887:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.361890:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.361946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.361954:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.361957:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.361959:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.361961:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.361962:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.361963:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.361964:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.361966:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.361967:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.361967:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.361970:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.361972:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.2:1619677448.361972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:30.0:1619677448.361974:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.361974:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.361975:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.361976:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.361977:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.361978:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.361978:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:2.0:1619677448.361978:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.361981:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:25.2:1619677448.361982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:18.0:1619677448.361982:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.361984:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.361986:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.361987:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.361988:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.361989:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:2.0:1619677448.361990:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.361991:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.361992:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:2.0:1619677448.361992:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677448.361993:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.361996:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.361996:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:2.0:1619677448.361996:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.361998:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:2.0:1619677448.361999:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.362001:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.362002:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.362003:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.362005:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.362005:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.362006:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.362006:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.362011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.362018:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.362020:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.362021:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.362022:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.362023:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.362024:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.362027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.362034:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.362037:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.362038:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.362038:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.362038:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.362039:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.362287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.362294:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.362301:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.362302:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.362304:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.362307:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.362308:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.362310:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.362311:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.362312:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.362313:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.362317:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.362319:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.362321:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.362322:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.362323:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.362324:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.362325:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.362325:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.362326:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.362331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.362339:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.362341:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.362342:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.362343:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.362346:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.362347:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.362452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.362458:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.362461:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.362463:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.362464:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.362465:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.362466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:5.0:1619677448.362467:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.362468:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.362470:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.362471:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.362472:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.362474:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:5.0:1619677448.362475:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.362477:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.362477:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:5.0:1619677448.362477:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.362479:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.362480:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:26.0:1619677448.362481:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.362481:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:26.0:1619677448.362483:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.362483:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.362483:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.362484:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677448.362485:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:5.0:1619677448.362485:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:26.0:1619677448.362486:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.362488:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.362489:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.362489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:26.0:1619677448.362490:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.362493:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.362495:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677448.362497:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.362498:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.362499:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.362499:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677448.362500:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.362501:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.362501:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:22.0:1619677448.362501:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.362501:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.362502:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.362503:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:6.0:1619677448.362503:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.362506:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.2:1619677448.362508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.362516:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.362519:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.362520:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.362520:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.362522:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.362525:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.362546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.362554:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.362557:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.362559:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.362560:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.362561:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.362563:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.362564:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.362565:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.362566:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.362567:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.362571:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.362572:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.362574:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.362575:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.362576:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.362577:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.362578:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.362578:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.362579:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.362583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.362590:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.362593:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.362593:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.362595:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.362597:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.362598:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.362604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.362611:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.362614:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.362615:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.362616:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.362618:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.362619:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.362621:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.362622:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.362623:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.362624:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.362627:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.362629:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.362630:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.362631:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.362631:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.362633:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.362633:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.362634:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.362635:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.362639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.362646:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.362648:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.362649:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.362650:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.362651:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.362652:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.362742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.2:1619677448.362746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.362749:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.362752:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:1.0:1619677448.362752:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.362753:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:21.2:1619677448.362753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:1.0:1619677448.362754:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.362755:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.362756:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.362758:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:1.0:1619677448.362758:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.362759:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.362760:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:1.0:1619677448.362760:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.362761:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677448.362762:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:5.0:1619677448.362763:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:1.0:1619677448.362763:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.362764:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.362764:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.362766:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.362766:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.362767:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.362767:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.362768:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.362768:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:20.0:1619677448.362769:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677448.362769:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.362771:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.362773:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.362774:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.362774:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.362775:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677448.362777:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.362777:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:20.0:1619677448.362779:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.362779:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.362780:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.362780:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.362781:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.0:1619677448.362782:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:1.0:1619677448.362782:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.362783:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.362783:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.362784:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.362784:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:1.0:1619677448.362784:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.362784:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677448.362785:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:10.0:1619677448.362785:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.362786:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.362787:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:10.0:1619677448.362787:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.362788:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:10.0:1619677448.362788:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:25.2:1619677448.362789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.0:1619677448.362789:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.362789:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:10.0:1619677448.362789:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.362790:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.362793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.2:1619677448.362793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:22.0:1619677448.362797:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.362799:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.362800:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.362801:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677448.362801:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:5.0:1619677448.362801:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:22.0:1619677448.362802:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.362804:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:5.0:1619677448.362804:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.362804:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.362805:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:18.0:1619677448.362806:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.362806:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.362806:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.362807:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:18.0:1619677448.362808:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.362808:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:18.0:1619677448.362812:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.362829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.362837:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.362840:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.362841:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.362843:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.362846:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.362847:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.362849:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.362850:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.362851:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.362852:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.362856:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.362857:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.362860:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.362861:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.362862:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.362863:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.362864:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.362865:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.362867:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.362867:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.362869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:4.2:1619677448.362872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.362877:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677448.362879:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.362880:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.362881:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.362881:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677448.362881:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.362882:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.362883:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.362883:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.362886:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.362935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.362940:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.362943:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.362943:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.362945:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.362947:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.362949:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.362950:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.362951:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.362952:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.362953:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.362957:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.362958:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.362961:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.362962:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.362962:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.362964:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.362965:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.362965:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.362966:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.362970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.362977:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.362980:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.362981:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.362981:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.362983:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.362985:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.363037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.363040:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.363043:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.363045:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.363046:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.363047:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.363048:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.363050:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.363051:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.363052:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.363053:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.363057:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.363058:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.363061:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.363061:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.363062:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.363063:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.363064:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.363065:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.363066:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.363067:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.363069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.363072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.363077:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.363078:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:6.0:1619677448.363079:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.363080:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.363080:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:22.0:1619677448.363081:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.363081:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:6.0:1619677448.363082:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.363082:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677448.363085:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.363089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.363096:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.363099:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.363099:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.363100:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.2:1619677448.363100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:22.0:1619677448.363101:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.363101:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677448.363108:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.363111:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.363113:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.363115:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.363116:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.363117:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.363119:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715949] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.363120:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.363121:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.363122:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.363125:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.363127:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.363130:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.363131:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.363132:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.363133:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.363134:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.363135:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.363135:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.363139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.363147:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.363149:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.363150:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.363152:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.363152:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.363155:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.363171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.363177:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.363180:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.363180:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.363182:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.363184:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.363185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:1.0:1619677448.363186:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.363187:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.363188:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:1.0:1619677448.363188:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.363189:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.363190:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:1.0:1619677448.363190:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.363193:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.363194:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677448.363195:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.363196:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.363197:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.363198:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677448.363199:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.363200:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.363201:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.363201:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:20.0:1619677448.363202:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.363202:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:20.0:1619677448.363203:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:1.0:1619677448.363204:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.363204:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.363205:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.363206:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000400:00000200:20.0:1619677448.363207:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677448.363207:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.363208:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:25.2:1619677448.363210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.363211:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.363212:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.2:1619677448.363212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.363213:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.363215:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677448.363215:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:31.0:1619677448.363216:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.363216:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:17.0:1619677448.363217:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:20.0:1619677448.363218:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.363218:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677448.363219:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:17.0:1619677448.363219:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:31.0:1619677448.363220:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.363220:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.363221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:1.0:1619677448.363221:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677448.363222:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.363223:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.2:1619677448.363223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:31.0:1619677448.363225:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.363225:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.363227:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.363227:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.363227:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.363229:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:25.2:1619677448.363229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.363229:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.363231:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:30.0:1619677448.363232:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.0:1619677448.363232:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.363233:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677448.363234:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.363235:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.363236:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.363236:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.363237:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.363238:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.363240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.363247:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.363249:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.363252:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.363254:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.363254:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.363256:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.363257:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.363258:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.363259:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.363260:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.363263:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.363264:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.363267:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.363267:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.363268:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.363269:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.363270:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.363271:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.363271:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.363275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:4.2:1619677448.363279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.363284:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.363286:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.363287:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.363288:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:26.0:1619677448.363289:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.363289:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.363291:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.363292:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:21.2:1619677448.363299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:10.0:1619677448.363299:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.363301:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.363302:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.363302:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.363303:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715945] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.363304:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.363305:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.363305:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:10.0:1619677448.363306:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.363309:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.363310:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:10.0:1619677448.363310:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.363311:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.363312:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.363313:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.363315:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.363315:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:10.0:1619677448.363316:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.363316:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:2.0:1619677448.363316:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.363318:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:2.0:1619677448.363318:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.363319:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.363319:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677448.363319:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.363320:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:2.0:1619677448.363320:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.363323:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.363325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.363325:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.363328:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.363329:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.363330:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.363332:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.363332:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677448.363333:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677448.363333:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.363334:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677448.363335:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.363336:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.363338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:5.0:1619677448.363338:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.363339:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.2:1619677448.363341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.363344:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.363345:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:5.0:1619677448.363346:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.363349:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:20.0:1619677448.363349:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.2:1619677448.363349:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:26.0:1619677448.363350:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.363350:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.363352:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.363353:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.363354:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.363354:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (22)--
00000800:00000200:25.2:1619677448.363355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.363355:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677448.363355:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:26.0:1619677448.363358:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.363359:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.363359:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.363359:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:26.0:1619677448.363361:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.363362:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000400:00000200:18.0:1619677448.363362:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.363362:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.363363:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.363364:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:18.0:1619677448.363365:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.363367:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.363368:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.363370:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.363371:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.363371:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.363374:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.363376:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.363378:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.363379:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.363380:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.363382:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.363382:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.363383:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.363384:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.363388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.363395:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.363398:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.363399:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.363401:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.363403:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.363404:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.363434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.363440:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.363443:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.363444:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.363446:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.363449:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.363451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:17.0:1619677448.363451:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.363452:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.363454:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.363454:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.363456:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.363457:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:17.0:1619677448.363459:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.363460:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.363461:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677448.363461:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:6.0:1619677448.363463:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.363464:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.363464:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.363465:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:6.0:1619677448.363465:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.363466:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:17.0:1619677448.363467:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.363467:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.363468:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677448.363468:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.363469:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.363469:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.363470:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.363471:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:25.2:1619677448.363472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:6.0:1619677448.363475:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.363477:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677448.363479:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677448.363480:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.363481:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.363481:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:13.0:1619677448.363482:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.363483:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.363483:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000400:00000200:13.0:1619677448.363484:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.363484:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.363484:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677448.363485:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677448.363485:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677448.363487:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.363489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:25.2:1619677448.363492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.363495:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.363497:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.363498:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:12.0:1619677448.363498:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.363498:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.363501:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.363501:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.363502:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.363503:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.363503:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.363503:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:12.0:1619677448.363505:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.363529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:24.0:1619677448.363536:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677448.363540:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000400:00000200:24.0:1619677448.363542:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.363542:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677448.363545:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.363547:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677448.363548:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.363549:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.363550:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.363551:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677448.363554:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.363556:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677448.363559:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.363559:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:24.0:1619677448.363560:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:24.0:1619677448.363562:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:24.0:1619677448.363562:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677448.363563:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677448.363563:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:25.2:1619677448.363567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.363575:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.363578:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.363578:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.363580:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.363582:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.363583:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.363645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.363652:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.363655:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.363656:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.363657:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.363660:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.363661:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.363663:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.363664:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.363665:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.363666:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.363669:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.363670:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.363673:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.363674:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.363674:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.363676:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.363676:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.363677:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.363678:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.363682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.363690:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.2:1619677448.363692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677448.363693:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.363693:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.363694:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.363696:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.363698:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.363699:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.363700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.363701:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.363702:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.363704:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.363706:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.363707:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:1.0:1619677448.363708:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.363709:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.363710:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.363711:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.363711:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000400:00000200:1.0:1619677448.363711:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.363712:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000400:00000200:1.0:1619677448.363712:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.363713:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:1.0:1619677448.363713:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.363716:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.363718:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.363720:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.363721:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.363722:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.363724:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.363724:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.363725:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.363725:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.363729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.363737:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.363739:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.363740:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.363740:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.363742:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.363744:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.363881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:4.2:1619677448.363882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.0:1619677448.363887:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:8.0:1619677448.363889:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677448.363890:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:8.0:1619677448.363892:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.363893:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:20.0:1619677448.363893:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.363894:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:8.0:1619677448.363895:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.363896:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.363897:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.363898:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.363898:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.363899:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677448.363900:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.363901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:20.0:1619677448.363902:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.363902:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.363903:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:8.0:1619677448.363904:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.363905:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.363906:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.363907:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.363908:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677448.363910:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:8.0:1619677448.363910:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.363911:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.363912:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677448.363912:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.363913:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:8.0:1619677448.363913:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.363914:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:10.0:1619677448.363914:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.363915:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.363916:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:8.0:1619677448.363916:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.363916:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.363917:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000400:00000200:10.0:1619677448.363917:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.363917:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.363918:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000400:00000200:10.0:1619677448.363918:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677448.363919:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:10.0:1619677448.363919:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96715935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.363919:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:10.0:1619677448.363921:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677448.363921:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000400:00000200:10.0:1619677448.363922:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.363923:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:21.2:1619677448.363925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (21)++
00000800:00000200:8.2:1619677448.363925:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (22)++
00000400:00000200:10.0:1619677448.363926:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.363927:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677448.363928:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.363929:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:10.0:1619677448.363930:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.363930:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677448.363930:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.0:1619677448.363930:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:10.0:1619677448.363931:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.363931:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.363932:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677448.363932:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000400:00000200:5.0:1619677448.363932:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.363933:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:17.0:1619677448.363933:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:10.0:1619677448.363933:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.363933:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.363934:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:5.0:1619677448.363934:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.363935:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677448.363935:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677448.363936:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:5.0:1619677448.363936:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:1.0:1619677448.363936:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677448.363937:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000400:00000200:17.0:1619677448.363937:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.363938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.363938:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677448.363939:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000400:00000200:17.0:1619677448.363939:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.363939:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.363940:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.363940:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.363941:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.363941:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.363942:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:8.2:1619677448.363942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677448.363943:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.363944:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677448.363945:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:13.0:1619677448.363945:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:5.0:1619677448.363945:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.363946:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:17.0:1619677448.363947:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677448.363948:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:5.0:1619677448.363948:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.363949:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677448.363949:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.363949:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.363950:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000400:00000200:5.0:1619677448.363950:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.363950:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.363951:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000400:00000200:13.0:1619677448.363951:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.363951:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.363952:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:5.0:1619677448.363952:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.363957:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.363958:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.363958:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.363962:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.363963:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.363963:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.363965:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.363966:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.363967:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.363967:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.363971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.363978:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.363980:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.2:1619677448.363980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:12.0:1619677448.363981:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.363983:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.363984:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.363985:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.363986:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.363988:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.363990:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.363991:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.363993:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.363995:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.363996:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.363998:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.363999:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.364000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:1.0:1619677448.364000:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.364003:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.364005:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.364008:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.364008:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.364008:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.364009:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.364010:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:1.0:1619677448.364011:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.364011:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.364012:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.364012:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677448.364013:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.364014:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.364015:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.364017:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:25.2:1619677448.364017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:30.0:1619677448.364018:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.364019:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.364020:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.364021:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:13.0:1619677448.364023:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:30.0:1619677448.364025:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.364026:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:30.0:1619677448.364027:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:13.0:1619677448.364027:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.364028:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.364029:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.364029:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.364030:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.364031:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:13.0:1619677448.364031:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.364032:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.364033:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.364034:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.364036:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.364036:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.364038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.364042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.364044:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.364046:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.364047:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.364047:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.2:1619677448.364048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:26.0:1619677448.364049:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.364051:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677448.364053:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.364055:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.364055:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.364056:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.364058:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:30.0:1619677448.364058:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:31.0:1619677448.364060:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.364060:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.364062:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.364064:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.364065:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.364066:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.364067:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.364068:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.364071:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.364073:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.364076:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.364077:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.364077:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.364079:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.364080:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.364081:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.364081:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.364085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.364093:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.364095:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.364096:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.364098:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.364099:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.364101:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:25.2:1619677448.364103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:12.0:1619677448.364110:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.364112:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.0.10.103@o2ib7
00000400:00000200:12.0:1619677448.364115:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.103@o2ib7 : GET - for me
00000800:00000200:13.0:1619677448.364116:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677448.364117:0:19943:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.103@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:12.0:1619677448.364120:0:19943:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.103@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:12.0:1619677448.364121:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.364122:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677448.364124:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:12.0:1619677448.364128:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : REPLY try# 0
00000800:00000200:12.0:1619677448.364129:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.364131:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.364132:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.364133:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.364134:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.364135:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:25.2:1619677448.364140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.364146:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.364148:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.364149:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.364151:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: REPLY: OK
00000800:00000200:12.0:1619677448.364152:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.364153:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.364237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.364244:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.364246:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.364247:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.364248:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.364251:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.364252:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.364253:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.364255:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.364256:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.364256:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.364260:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.364261:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.364264:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.364265:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.364265:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.364267:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.364267:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.364268:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.364268:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.364275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:21.2:1619677448.364280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:6.0:1619677448.364282:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.364285:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.364286:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.364286:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.364287:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:6.0:1619677448.364288:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.364290:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.364291:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:6.0:1619677448.364291:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:24.0:1619677448.364293:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.364299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:24.0:1619677448.364301:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.364303:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.364304:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.364304:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:24.0:1619677448.364305:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.364306:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.364307:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.364307:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:24.0:1619677448.364308:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677448.364311:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.364313:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.364314:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.364315:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:22.0:1619677448.364315:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.364317:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.364318:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.364318:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.364319:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.364319:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:22.0:1619677448.364320:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.364320:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.364321:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.364322:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.364322:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.364323:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:22.0:1619677448.364324:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.364326:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.2:1619677448.364328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677448.364329:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.364329:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.364330:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.364332:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.364332:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.364334:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.364335:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.364336:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677448.364337:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.2:1619677448.364338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.364340:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:21.2:1619677448.364341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677448.364341:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.364341:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.364343:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.364346:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677448.364346:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677448.364347:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:17.0:1619677448.364349:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:6.0:1619677448.364349:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:17.0:1619677448.364350:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.364350:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.364351:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677448.364351:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.364352:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.364356:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.364406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.364412:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.364414:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.364415:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.364417:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.364419:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.364421:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.364422:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.364423:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.364424:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.364425:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.364428:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.364430:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:5.2:1619677448.364433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677448.364433:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.364434:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.364434:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.364436:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.364436:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.364438:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.364439:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.364439:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.364441:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:9.2:1619677448.364442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:26.0:1619677448.364444:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:5.2:1619677448.364444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.364445:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.364447:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.364448:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:17.0:1619677448.364449:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:26.0:1619677448.364450:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.364451:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:17.0:1619677448.364451:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.0:1619677448.364451:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.364452:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:17.0:1619677448.364452:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677448.364452:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.364453:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.364454:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.364454:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.364455:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.364456:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.364456:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677448.364458:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:26.0:1619677448.364460:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.364462:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.364464:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.364465:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:9.2:1619677448.364465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:26.0:1619677448.364466:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.364467:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.364468:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.364468:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.364469:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677448.364471:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:25.2:1619677448.364473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677448.364473:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677448.364474:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.364474:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.364475:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.364475:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677448.364480:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.364483:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.364483:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.364485:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.364487:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.364488:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.364659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.364666:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.364669:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.364670:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.364671:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.364674:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.364675:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.364676:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.364678:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.364678:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.364679:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.364682:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.364684:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.364686:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.364687:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.364688:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.364690:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.364690:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.364692:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677448.364693:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.364693:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:9.2:1619677448.364696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677448.364699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:18.0:1619677448.364704:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.364706:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:18.0:1619677448.364707:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.364708:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:18.0:1619677448.364708:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.364708:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.364709:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.364710:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:18.0:1619677448.364710:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.364713:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.364714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.2:1619677448.364716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.364722:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.364723:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.364724:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.364724:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.364725:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.364726:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.364726:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.364727:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.364728:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.364729:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.364730:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.364732:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.364733:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.364734:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.364736:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.364737:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.364739:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.364741:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.364743:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.364744:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.364745:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.364747:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.364748:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.364748:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.364749:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.364753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.364762:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.364764:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.364765:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.364767:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.364767:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.364770:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.364792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.364795:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.364797:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.364799:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.364800:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.364802:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.364804:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.364805:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715923] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.364806:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.364807:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.364808:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.364812:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.364813:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:21.2:1619677448.364816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.364816:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.364817:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.364818:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.364819:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.364820:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.364820:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.364821:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.364821:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677448.364822:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677448.364823:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.364823:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.364825:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.364826:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:9.2:1619677448.364827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677448.364828:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.364829:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.364830:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.364831:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.364833:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.364834:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677448.364835:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.364836:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.364837:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:2.0:1619677448.364837:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.364838:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.364838:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:2.0:1619677448.364838:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.364839:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.364840:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.364840:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000400:00000200:2.0:1619677448.364840:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.364841:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:2.0:1619677448.364843:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.2:1619677448.364845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.364853:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.364855:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.364856:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.364858:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.364859:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.364860:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.2:1619677448.364912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.364918:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.364920:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.364922:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677448.364923:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.364925:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.364926:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.364928:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.364929:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.364930:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.364931:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.364934:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.364936:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.364938:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.364939:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.364939:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.364941:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.364942:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.364942:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.364943:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.364948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.364955:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.364957:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.364958:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.364960:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677448.364961:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.364962:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.364967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.364973:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.364975:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.364976:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.364977:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.364977:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.364978:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.365021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.365030:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.365033:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.365035:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.365037:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.365037:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.365039:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.365040:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.365041:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.365042:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.365043:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.365046:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.365047:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.365050:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.365051:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.365051:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.365053:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.365053:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.365054:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.365055:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.365058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.365062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:13.0:1619677448.365065:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.365067:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.365068:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.365069:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.365069:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:12.0:1619677448.365071:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.365072:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677448.365072:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.365073:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.365074:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.365077:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.365079:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.365080:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.365081:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.365082:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.365083:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677448.365086:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.365088:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.365091:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.365092:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.365092:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.365094:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.365095:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.365095:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.365096:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.365100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.365108:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.365110:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.365111:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.365112:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.365113:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.365116:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.365165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.365170:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.365173:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.365174:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.365176:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.365178:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.365180:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.365181:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.365183:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.365183:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.365185:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.365188:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.365190:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.365192:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.365193:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.365194:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.365195:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.365196:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.365197:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.365197:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.365201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.365210:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.365212:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.365213:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.365214:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.365215:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.365218:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.365219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.365226:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.365229:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.365230:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.365230:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.365231:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.365232:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.365250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.365258:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.365261:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.365262:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.365264:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.365266:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.365268:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.365269:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.365270:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.365272:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.365272:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:5.2:1619677448.365275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:10.0:1619677448.365276:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.365278:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.365280:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.365281:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:10.0:1619677448.365281:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.365282:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:18.0:1619677448.365283:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:10.0:1619677448.365283:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.365284:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.365284:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:2.0:1619677448.365284:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.365286:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.365286:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.365286:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:18.0:1619677448.365288:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.365289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:18.0:1619677448.365290:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.365291:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715916] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.365291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:18.0:1619677448.365292:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.365297:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.365298:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.365300:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:26.0:1619677448.365301:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:18.0:1619677448.365301:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.365302:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:18.0:1619677448.365302:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:31.0:1619677448.365303:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.365303:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.365304:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:31.0:1619677448.365305:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.365305:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:18.0:1619677448.365305:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.365306:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.365306:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.365307:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:20.2:1619677448.365307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677448.365308:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (23)--
00000800:00000200:18.0:1619677448.365308:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.365309:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.365309:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:18.0:1619677448.365309:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:30.0:1619677448.365310:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.365310:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:18.0:1619677448.365310:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:30.0:1619677448.365311:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:18.0:1619677448.365311:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.365312:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677448.365312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:9.2:1619677448.365315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.365318:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:1.0:1619677448.365319:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.365321:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.365322:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.365322:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.365324:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677448.365324:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.365325:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.365325:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:1.0:1619677448.365325:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.365326:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:2.0:1619677448.365326:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.365327:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677448.365327:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:5.0:1619677448.365327:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.365328:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.365328:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.365329:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.365329:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.365330:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677448.365330:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:1.0:1619677448.365330:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677448.365331:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677448.365332:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.365333:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.365334:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.365335:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.365335:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.365336:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.365337:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677448.365338:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.365342:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.365343:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.365345:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.365346:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.365346:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.365347:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.365348:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.365348:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.365349:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.365350:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.365350:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.365350:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.365351:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.365352:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.365352:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.365352:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677448.365353:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.365354:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.365357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.2:1619677448.365357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677448.365360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677448.365365:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:2.0:1619677448.365365:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:17.0:1619677448.365367:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:10.0:1619677448.365367:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677448.365368:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.365368:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.365369:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.365369:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.365369:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.365370:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.365371:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.365371:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:2.0:1619677448.365371:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.365372:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:10.0:1619677448.365374:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.365374:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.2:1619677448.365375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000400:00000200:10.0:1619677448.365376:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.365377:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.365379:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.365380:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.365381:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.365382:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.365382:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.365386:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:10.0:1619677448.365386:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.365386:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.365387:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.365388:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.365388:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:10.0:1619677448.365389:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.365392:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.365393:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.365394:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.365395:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.365396:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.365397:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.365398:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.365399:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.365402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.365404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.365410:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.365411:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.365413:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.365413:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677448.365413:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.365414:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.365414:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.365415:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.365417:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.365419:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.365523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.365530:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.365533:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.365534:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.365536:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.365539:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.365540:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.365542:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.365543:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.365544:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.365545:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.365548:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.365550:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.365552:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.365553:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.365554:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.365556:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.365556:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.365558:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.365559:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.365559:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.365562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.365565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.365570:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.365572:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677448.365572:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.365573:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.365574:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.365574:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.365575:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.365576:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.365576:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677448.365579:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.365727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.365733:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.365736:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.365737:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.365739:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.365741:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.365742:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.365744:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.365745:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.365746:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.365747:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.365750:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.365752:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.365754:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.365755:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.365755:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.365757:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.365758:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.365759:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.365759:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.365763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.365771:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.365774:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.365775:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.365775:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.365777:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.365779:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.365792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.365800:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.365803:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.365805:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.365806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:31.0:1619677448.365807:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.365807:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.365809:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.365810:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.365811:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.365812:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.365812:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:31.0:1619677448.365813:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.365815:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.365816:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:31.0:1619677448.365819:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.365820:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.365822:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.365823:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.365825:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.365825:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677448.365826:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:2.0:1619677448.365826:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.365827:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:2.0:1619677448.365828:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.365829:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.365829:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:2.0:1619677448.365829:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.365830:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.365831:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.365832:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.365833:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:2.0:1619677448.365833:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.365834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677448.365835:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.2:1619677448.365838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677448.365838:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.365839:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.365839:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.365841:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677448.365842:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.365842:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.365843:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:12.0:1619677448.365844:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:2.0:1619677448.365844:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000400:00000200:12.0:1619677448.365845:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.365845:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:30.0:1619677448.365847:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:12.0:1619677448.365847:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.365848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.365849:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:12.0:1619677448.365849:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.365850:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:21.2:1619677448.365850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:30.0:1619677448.365851:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677448.365851:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.365851:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.365854:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.365855:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.365856:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:12.0:1619677448.365856:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.365858:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.365859:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:13.0:1619677448.365859:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.365860:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.365860:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:21.2:1619677448.365872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.365877:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.365880:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.365881:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.365883:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.365885:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.365887:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677448.365888:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715908] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.365890:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.365891:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.365891:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677448.365895:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.365896:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677448.365899:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.365900:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677448.365900:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677448.365902:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677448.365902:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.365903:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.365903:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:21.2:1619677448.365906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:25.2:1619677448.365907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:20.0:1619677448.365911:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.365914:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:10.0:1619677448.365914:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:20.0:1619677448.365916:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677448.365916:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.365917:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:5.0:1619677448.365917:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.2:1619677448.365918:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:10.0:1619677448.365918:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.365920:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000400:00000200:20.0:1619677448.365922:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.365923:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:20.0:1619677448.365924:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:5.0:1619677448.365924:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:20.0:1619677448.365925:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.365925:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.365927:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.365928:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.365929:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:25.2:1619677448.365932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000400:00000200:20.0:1619677448.365932:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.365934:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.365935:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.365935:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.365936:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.365937:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:5.0:1619677448.365937:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.365938:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:20.0:1619677448.365938:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.365939:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.365940:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.365941:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.365941:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.365942:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (21)++
00000400:00000200:5.0:1619677448.365942:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.365943:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000400:00000200:5.0:1619677448.365943:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.365944:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:8.2:1619677448.365944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000400:00000200:5.0:1619677448.365944:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.365945:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.365946:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.365947:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:5.0:1619677448.365947:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.365948:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.365949:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.365950:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.365950:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.365951:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.365952:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.365953:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.365953:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.365954:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.365954:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.365955:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.0:1619677448.365955:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.365956:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.365958:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.365959:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.365959:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.365961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.365964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.365968:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677448.365968:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.365970:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.365971:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.365971:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.365971:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.365972:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.365973:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.365974:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.365976:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.366084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677448.366086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677448.366091:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.366093:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.366094:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.366095:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.366096:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677448.366097:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.366099:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.366100:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677448.366100:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.366100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677448.366101:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.366102:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.366102:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677448.366104:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.366104:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.366106:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.366106:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.366106:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.366108:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.366108:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.366109:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:10.0:1619677448.366109:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.366109:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677448.366110:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.366113:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.366115:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.366115:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.366115:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.366116:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.366117:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96715903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.366117:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.366118:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.366119:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.366120:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.366120:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.366121:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.366121:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:10.0:1619677448.366121:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.366121:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.366122:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.366122:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.366123:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.366123:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.366124:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.366124:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.366124:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.366125:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677448.366125:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.366125:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.366126:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677448.366127:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:25.2:1619677448.366129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.366130:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.366130:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.366131:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.366133:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.366133:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:25.2:1619677448.366134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.366134:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.366134:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.366135:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.366138:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.366139:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.366139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:31.0:1619677448.366141:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.366141:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.366142:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.366143:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:12.0:1619677448.366143:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.366144:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.366145:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:12.0:1619677448.366145:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.366147:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677448.366148:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:13.0:1619677448.366148:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677448.366148:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.366149:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.366151:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.366151:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.366152:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.366153:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:24.0:1619677448.366154:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677448.366154:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.366155:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677448.366157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:21.2:1619677448.366162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:8.0:1619677448.366163:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.366165:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.366166:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.366166:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.366167:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.366168:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:6.0:1619677448.366168:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.366171:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.366171:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.366173:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.366175:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.366177:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.366178:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.366180:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.366181:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.366182:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.366186:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.366188:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.366190:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.366191:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.366191:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.366193:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.366193:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.366194:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.366194:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.366199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.366206:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.366209:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.366210:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.366210:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.366212:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.366214:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.366372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.366380:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.366383:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.366385:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.366387:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.366387:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.366389:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.366390:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.366391:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.366391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:30.0:1619677448.366392:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.366393:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.366397:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.366398:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.366399:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.366401:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.366402:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.366402:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.366402:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:30.0:1619677448.366403:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.366403:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.366404:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677448.366404:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.366405:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.366406:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:26.0:1619677448.366407:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.366408:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.366409:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.366410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:26.0:1619677448.366411:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.366411:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.366412:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.366415:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.366416:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:26.0:1619677448.366416:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.366418:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.366419:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:26.0:1619677448.366419:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (23)++
00000400:00000200:31.0:1619677448.366420:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.366420:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (23)++
00000400:00000200:31.0:1619677448.366421:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.366421:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (24)--
00000800:00000200:26.0:1619677448.366422:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.366422:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.366423:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (23)--
00000800:00000200:26.0:1619677448.366423:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.366424:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:25.2:1619677448.366427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.366434:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.366436:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.366437:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.366438:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.366439:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.366441:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.366577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.366583:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.366586:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.366587:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.366589:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.366592:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.366593:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.366594:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.366596:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.366597:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.366598:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.366601:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.366603:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.366605:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.366605:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.366606:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.366607:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.366608:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.366608:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.366609:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.366614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.366622:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.366623:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.366624:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.366625:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.366626:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.366627:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.366630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.366637:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.366640:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.366640:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.366641:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.366641:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.366642:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.366725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.366732:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.366736:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.366738:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.366740:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.366740:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.366741:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.2:1619677448.366742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:30.0:1619677448.366743:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.366744:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.366745:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.366745:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677448.366746:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.366747:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:5.0:1619677448.366750:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677448.366751:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.366751:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.366754:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.366754:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.366755:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:30.0:1619677448.366756:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.366757:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:20.0:1619677448.366757:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.366758:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.366758:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.366759:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677448.366759:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.366760:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677448.366760:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.366761:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.366761:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:20.0:1619677448.366763:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.366765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.366765:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.366767:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.366767:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.366768:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.366770:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.366770:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.366771:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.366772:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.366772:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.366775:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.366776:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.366776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:12.0:1619677448.366777:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.366779:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.366780:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.366783:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.366786:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.366787:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.366789:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.366789:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.366791:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.366794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.366801:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.366804:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.366805:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.366807:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.366809:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.366810:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.366812:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.366813:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.366814:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.366815:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.366818:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.366820:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.366823:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.366824:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.366825:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.366827:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.366827:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.366828:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.366828:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.366833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.366841:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:21.2:1619677448.366844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:18.0:1619677448.366844:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.366845:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.366845:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.366847:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.366850:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:18.0:1619677448.366851:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677448.366852:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.366854:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.366855:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.366856:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.366856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:20.0:1619677448.366858:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:20.0:1619677448.366859:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715895] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.366860:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.366861:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.366862:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.366864:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677448.366865:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.366867:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.366868:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 440 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.366868:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.366868:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.366869:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.366869:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677448.366870:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.366870:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.366871:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.366873:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.366873:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.366874:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.366875:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.366878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.366885:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.366888:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.366889:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.366890:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.366891:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.366892:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.366899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.366907:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.366910:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.366911:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.366911:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.366912:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.366913:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.366935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677448.366935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:17.0:1619677448.366941:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:31.0:1619677448.366943:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677448.366944:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677448.366944:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:31.0:1619677448.366946:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:17.0:1619677448.366947:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.366949:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.366949:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:17.0:1619677448.366949:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.366951:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.366951:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.366953:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:17.0:1619677448.366953:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.366955:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.366955:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.366956:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.366957:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.366957:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.366958:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.366959:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:17.0:1619677448.366962:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.366964:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.366965:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.366967:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:17.0:1619677448.366968:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.366968:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.366969:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.366969:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.366970:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.366971:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:17.0:1619677448.366971:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.366972:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:17.0:1619677448.366972:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677448.366972:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:31.0:1619677448.366973:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677448.366973:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:31.0:1619677448.366974:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.366975:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.366977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:25.2:1619677448.366981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:12.0:1619677448.366983:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.366986:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:30.0:1619677448.366987:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:12.0:1619677448.366987:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.366989:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:12.0:1619677448.366989:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.366990:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.366990:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.366991:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:30.0:1619677448.366992:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.366993:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.366994:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.367114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.2:1619677448.367119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:26.0:1619677448.367121:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.367123:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.367124:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677448.367125:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:26.0:1619677448.367126:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.367127:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.367128:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:26.0:1619677448.367129:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.367130:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.367131:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.367132:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.367132:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.367134:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.367134:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.367135:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.367135:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.367136:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677448.367137:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.367139:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.367140:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.367142:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.367145:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:18.0:1619677448.367147:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.367148:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.367148:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.367149:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677448.367149:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.367150:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.367151:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.367151:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.367152:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:18.0:1619677448.367152:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.367153:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.367153:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.367155:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:25.2:1619677448.367156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677448.367156:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.367157:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.367157:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.367161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.367165:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.367167:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.367167:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.367168:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:13.0:1619677448.367169:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.367169:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.367170:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.367172:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677448.367172:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.367173:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677448.367174:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.367174:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.367245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.367252:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.367255:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.367255:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.367258:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.367260:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.367261:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.367263:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.367264:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.367265:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.367266:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.367269:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.367271:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.367274:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.367275:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.367276:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.367278:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.367278:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.367279:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.367279:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.367284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.367291:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.367295:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.367299:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.367300:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.367302:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.367305:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.367325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.367328:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.367331:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.367333:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.367334:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.367335:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.367337:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677448.367339:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.367340:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.367341:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.367342:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677448.367346:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.367347:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677448.367350:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.367351:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677448.367352:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677448.367353:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677448.367354:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.367354:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.367355:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677448.367360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.367367:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.367370:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.367370:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.367371:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.367373:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.367376:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.367386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.367393:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.367396:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.367396:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677448.367398:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.367401:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.367403:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.367404:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.367405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000400:00000200:8.0:1619677448.367405:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.367406:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.367407:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.2:1619677448.367408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000400:00000200:8.0:1619677448.367410:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.367412:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:8.0:1619677448.367412:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.367414:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.367414:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:26.0:1619677448.367415:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:8.0:1619677448.367415:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:10.0:1619677448.367416:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.0:1619677448.367416:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.367417:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:26.0:1619677448.367418:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.367418:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:8.0:1619677448.367418:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.367418:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:8.0:1619677448.367419:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.367419:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.367421:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.367422:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.367423:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677448.367423:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.367424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677448.367425:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.367425:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.367427:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.367427:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.367428:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.367429:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677448.367429:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.367431:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.367432:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.367433:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.367433:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.367434:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.367435:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.367436:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:17.0:1619677448.367437:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.367438:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677448.367438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:1.0:1619677448.367438:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.367439:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.367439:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:17.0:1619677448.367440:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.367440:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.367441:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.367441:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:17.0:1619677448.367441:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677448.367442:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677448.367442:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:5.2:1619677448.367442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.367445:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677448.367446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:1.0:1619677448.367446:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.367447:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.367447:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.367448:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.367449:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.367450:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677448.367450:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.367450:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677448.367451:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:10.0:1619677448.367452:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:1.0:1619677448.367452:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:31.0:1619677448.367453:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.367453:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.367454:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677448.367454:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.367454:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:31.0:1619677448.367455:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.367455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:10.0:1619677448.367455:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.367456:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:17.0:1619677448.367456:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.367458:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:17.0:1619677448.367458:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.367458:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.367460:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96715885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.367460:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96715885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.367460:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:31.0:1619677448.367461:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.367462:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.367462:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:31.0:1619677448.367463:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.367464:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.367465:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:17.0:1619677448.367465:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:13.0:1619677448.367465:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.367465:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.367467:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.367469:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:17.0:1619677448.367471:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.367471:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:31.0:1619677448.367472:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.367473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.367473:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677448.367475:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:17.0:1619677448.367476:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.367477:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677448.367478:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.367478:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.367479:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:17.0:1619677448.367479:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:12.0:1619677448.367479:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.367480:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:17.0:1619677448.367480:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.367481:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677448.367482:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.367482:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.367482:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:12.0:1619677448.367482:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.367483:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677448.367483:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.367483:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:31.0:1619677448.367484:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677448.367484:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.367485:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.367486:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.367486:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.367487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:9.2:1619677448.367488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.2:1619677448.367488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677448.367492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.367493:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677448.367495:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.367496:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.367496:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.367497:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:1.0:1619677448.367497:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.367498:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.367499:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.367499:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677448.367500:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:18.0:1619677448.367500:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:22.0:1619677448.367501:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.367501:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.367501:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.367503:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:18.0:1619677448.367503:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.367504:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677448.367504:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:30.0:1619677448.367505:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677448.367506:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.367510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.367518:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.367521:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.367521:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.367523:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.367525:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.367527:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.367528:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.367530:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.367531:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.367532:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.367536:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.367537:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.367540:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.367541:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.367542:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.367543:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.367544:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.367545:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.367545:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.367549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.367558:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.367560:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.367561:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.367563:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.367563:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.367566:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.367569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.367576:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.367579:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.367580:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.367580:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.367581:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.367582:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.367696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.367703:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.367706:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.367707:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.367709:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.367711:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.367712:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.367714:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.367715:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.367716:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.367717:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.367720:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.367722:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.367724:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.367724:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.367725:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.367726:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.367727:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.367728:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.367729:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.367730:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.367732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.367735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.367739:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.367742:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.367742:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.367742:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:30.0:1619677448.367744:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.367744:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.367744:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.367746:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.367746:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.2:1619677448.367747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.367750:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.367751:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.367753:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677448.367755:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.367755:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.367757:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.367758:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.367760:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.367761:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.367762:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.367763:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.367766:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.367767:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.367770:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.367771:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.367772:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.367774:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.367774:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.367775:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.367776:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.367779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.367787:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.367789:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.367790:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.367792:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.367793:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.367795:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.367810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.367815:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.367817:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.367820:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.367820:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.367822:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.367823:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.367825:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.367826:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.367827:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.367828:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.367831:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.367832:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.367835:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.367836:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.367836:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.367838:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.367839:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.367839:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.367840:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.367844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.367851:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.367854:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.367855:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.367855:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.367856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:6.0:1619677448.367857:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.367859:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677448.367862:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.367865:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.367865:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.367868:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.367870:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.367871:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.367872:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.367874:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.367875:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.367876:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.367879:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.367881:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.367883:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.367884:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.367885:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.367886:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.367887:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.367887:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.367888:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:21.2:1619677448.367891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:25.2:1619677448.367892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677448.367897:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677448.367900:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677448.367900:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.367901:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677448.367903:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:2.0:1619677448.367903:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:22.0:1619677448.367904:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.367904:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:2.0:1619677448.367906:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.367907:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.367907:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:22.0:1619677448.367908:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.367909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000400:00000200:2.0:1619677448.367909:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.367910:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.367911:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.367912:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.367915:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:6.0:1619677448.367916:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677448.367917:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.367918:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.367919:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677448.367919:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:22.0:1619677448.367920:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.367920:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677448.367920:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.367921:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677448.367921:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677448.367922:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677448.367922:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.367923:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.367924:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.367925:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:9.2:1619677448.367928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.367930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.367936:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.367938:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.367938:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.367938:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.367938:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.367939:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.2:1619677448.367940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:18.0:1619677448.367940:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.367940:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.367941:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677448.367943:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677448.367948:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.367950:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.367953:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.367954:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.367955:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.367957:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.367958:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.367960:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.367961:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.367962:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.367965:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.367967:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.367969:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.367970:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.367971:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.367972:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.367973:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.367973:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.367974:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.367978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.367984:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.367986:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.367987:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.367987:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.367989:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.367991:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.368025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.368030:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.368033:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.368035:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.368036:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.368037:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.368038:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.368039:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.368040:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.368041:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.368042:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.368045:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.368047:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.368048:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.368049:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.368050:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.368051:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.368052:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.368054:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677448.368055:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:4.2:1619677448.368055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677448.368056:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:8.2:1619677448.368058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:21.2:1619677448.368061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (21)++
00000800:00000200:26.0:1619677448.368062:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.368065:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.368066:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.368066:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:26.0:1619677448.368067:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.368068:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677448.368068:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.368069:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000400:00000200:5.0:1619677448.368069:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.368070:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.368070:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.368070:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.368071:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:26.0:1619677448.368072:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:20.0:1619677448.368072:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:26.0:1619677448.368073:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.368073:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:26.0:1619677448.368074:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.368075:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.368076:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.368079:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.368080:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.368084:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.368084:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.368085:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.368087:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.368087:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.368088:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.368089:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677448.368090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:25.2:1619677448.368093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.368097:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.368099:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.368102:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:22.0:1619677448.368103:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.368103:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.368104:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.368105:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.368106:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.368107:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.368108:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715874] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.368110:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.368111:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.368112:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.2:1619677448.368115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:6.0:1619677448.368116:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.368117:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.368117:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.368118:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.368119:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677448.368121:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:22.0:1619677448.368122:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.368123:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.368123:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:2.0:1619677448.368124:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:22.0:1619677448.368125:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:18.0:1619677448.368125:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:22.0:1619677448.368126:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.368127:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.368127:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:2.0:1619677448.368127:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.368129:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.368131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:2.0:1619677448.368131:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.368132:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.368133:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.368134:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.368135:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.368138:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:2.0:1619677448.368138:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.368139:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.368141:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.368142:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.368143:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.368144:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.368144:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.368144:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.368145:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.368147:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:2.0:1619677448.368147:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.368147:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.368148:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.368148:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.368153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.368159:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.368162:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.368163:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.368165:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.368166:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.368168:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.368187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.368193:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.368196:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.368197:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.368197:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.368198:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.368199:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.368219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.368226:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.368229:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.368231:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.368233:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.368234:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.368235:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.368236:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.368238:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.368239:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.368240:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.368243:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.368245:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.368247:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.368248:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.368248:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.368250:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.368251:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.368252:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.368254:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.368254:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.368256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.368260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.368262:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.368264:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.368265:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.368266:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.368268:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:13.0:1619677448.368268:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.368269:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.368270:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.368271:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.368272:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:4.2:1619677448.368338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.368345:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.368348:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.368348:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.368350:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.368353:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.368354:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.368355:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.368356:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.368357:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.368358:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.368361:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.368363:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.368366:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.368366:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.368367:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.368369:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.368369:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.368370:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.368370:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.368375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.368384:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.368387:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.368387:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.368388:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.368390:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.368393:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.368412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.368418:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.368421:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.368422:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.368424:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.368426:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.368427:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.368429:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.368430:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.368431:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.368432:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.368436:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.368437:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.368440:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.368441:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.368442:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.368444:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.368444:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.368445:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.368445:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.368450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.368457:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.368460:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.368460:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.368462:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.368464:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.368465:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.368566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.368574:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.368576:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.368578:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.368580:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.368581:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.368582:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.368583:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715869] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.368584:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.368585:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.368586:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.368589:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.368591:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.368594:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.368595:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.368596:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.368597:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.368598:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.368599:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.368599:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.368603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.368612:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.368615:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.368616:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.368616:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.368618:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.368622:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.368805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.368810:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.368813:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.368814:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.368816:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.368819:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.368820:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.368822:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.368823:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.368824:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.368825:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.368828:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.368830:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.368833:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.368834:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.368834:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.368836:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.368836:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.368837:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.368837:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.368842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.368848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:20.2:1619677448.368848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:12.0:1619677448.368848:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.368851:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.368852:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.368854:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.368855:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.368856:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:24.0:1619677448.368856:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:12.0:1619677448.368856:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.368859:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:24.0:1619677448.368859:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.368859:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:31.0:1619677448.368862:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.368863:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:24.0:1619677448.368863:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.368865:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.368866:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.368867:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.368868:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677448.368869:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.368869:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.368870:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.368871:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.368872:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.368872:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.368873:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677448.368873:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677448.368878:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.368879:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.368880:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.368881:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.368883:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.368883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:31.0:1619677448.368884:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.368884:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.368885:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.368885:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.368886:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.368887:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.368887:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.368888:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.368888:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:24.0:1619677448.368888:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.368888:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:1.0:1619677448.368889:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:24.0:1619677448.368890:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:1.0:1619677448.368891:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:24.0:1619677448.368892:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.368892:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:17.0:1619677448.368892:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:8.2:1619677448.368893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:9.2:1619677448.368894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677448.368894:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.368897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000400:00000200:1.0:1619677448.368897:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.368898:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.368900:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96715865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.368901:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:1.0:1619677448.368901:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.368902:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:1.0:1619677448.368902:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.368903:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.368904:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:8.0:1619677448.368904:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:2.0:1619677448.368904:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:20.0:1619677448.368905:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.368905:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.368905:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.368907:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.368907:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:1.0:1619677448.368907:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.368908:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.368908:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:2.0:1619677448.368908:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.368909:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:1.0:1619677448.368910:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:21.2:1619677448.368911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:20.0:1619677448.368912:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:2.0:1619677448.368913:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677448.368914:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.368914:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.368915:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.368917:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677448.368917:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.368917:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677448.368918:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677448.368919:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:22.0:1619677448.368920:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.368920:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.368922:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.368923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677448.368925:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.368927:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.368928:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.368930:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.368930:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:22.0:1619677448.368931:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.368932:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677448.368932:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.368933:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.368936:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:22.0:1619677448.368937:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677448.368937:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.368939:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677448.368941:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677448.368942:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.368943:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.368943:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.368945:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.368946:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.368947:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.368948:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.368949:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.368952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.368954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.368959:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:17.0:1619677448.368960:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677448.368960:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.368961:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677448.368962:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:6.0:1619677448.368962:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:17.0:1619677448.368963:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.368963:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.368965:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.368968:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.369034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677448.369040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677448.369041:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.369043:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.369044:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.369046:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.369047:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.369049:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677448.369049:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.369051:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.369052:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.369053:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677448.369053:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.369054:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.369055:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.369055:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.369056:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.369057:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.369058:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.369060:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.369060:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.369061:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.369062:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.369062:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.369065:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.369065:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.369066:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.369066:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.369067:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.369069:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.369069:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.369069:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.369069:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.369069:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.369070:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.369070:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.369071:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.369072:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.369073:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.369073:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.369074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:25.2:1619677448.369077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.369082:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:12.0:1619677448.369084:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.369085:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:12.0:1619677448.369086:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:5.0:1619677448.369086:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.369087:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.369088:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.369088:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:12.0:1619677448.369089:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.369090:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.369091:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:12.0:1619677448.369092:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.369094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.369101:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.369103:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.369104:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.369104:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.369105:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.369106:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.369146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:4.2:1619677448.369149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:10.0:1619677448.369154:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677448.369156:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677448.369156:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.369157:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677448.369159:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:10.0:1619677448.369159:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.369160:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.369162:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.369162:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.369164:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.369164:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.369165:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.369166:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:10.0:1619677448.369166:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.369168:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.369168:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.369169:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.369169:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:24.0:1619677448.369170:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.369170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000400:00000200:24.0:1619677448.369171:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:10.0:1619677448.369172:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.369174:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:24.0:1619677448.369175:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.369175:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:10.0:1619677448.369176:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.369176:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:24.0:1619677448.369177:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.369177:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:10.0:1619677448.369177:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.369178:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.369178:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.369179:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677448.369180:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.369180:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.369180:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677448.369181:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.369181:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.369181:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.369182:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.369183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:24.0:1619677448.369183:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.369183:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.369184:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.369184:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:20.0:1619677448.369184:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.369185:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96715859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.369186:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.369187:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.369188:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:8.2:1619677448.369188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.369189:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.369191:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.369192:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.369192:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.369193:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.369194:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:26.0:1619677448.369195:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.369196:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.0:1619677448.369196:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.369197:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:1.0:1619677448.369197:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677448.369198:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.369199:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.369200:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:1.0:1619677448.369200:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677448.369201:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.369201:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:17.0:1619677448.369201:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.369201:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.369203:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.369204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:8.2:1619677448.369206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:1.0:1619677448.369206:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677448.369210:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.2:1619677448.369212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.369214:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.369214:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.369215:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.369215:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.369216:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:17.0:1619677448.369217:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.369217:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:17.0:1619677448.369220:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:5.0:1619677448.369220:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.369220:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.369221:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.369223:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.369223:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.369224:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.369225:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:8.2:1619677448.369226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000400:00000200:17.0:1619677448.369227:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.369228:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.369230:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.369231:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.369232:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.369233:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000400:00000200:17.0:1619677448.369233:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.369236:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.369237:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000400:00000200:17.0:1619677448.369237:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.369238:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.369238:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.369239:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:17.0:1619677448.369240:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.369243:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.369243:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.369244:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.369246:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.369247:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.369248:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.369249:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.369250:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.369252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.369254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.369258:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.369260:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677448.369261:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.369262:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.369262:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.369263:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677448.369264:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.369264:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.369265:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.369266:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.369491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677448.369498:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.369501:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677448.369501:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677448.369503:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.369506:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.369508:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.369509:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.369510:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.369511:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.369512:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.369516:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.369517:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.369520:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.369521:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.369521:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.369523:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.369523:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.369525:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677448.369526:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677448.369526:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:8.2:1619677448.369530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.369532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:17.0:1619677448.369537:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677448.369538:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677448.369539:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:17.0:1619677448.369540:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:2.0:1619677448.369540:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:17.0:1619677448.369541:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.369541:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:1.0:1619677448.369541:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.369543:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.369546:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.369550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.369557:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.369560:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.369560:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.369561:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.369562:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.369562:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.369565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.369572:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.369575:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.369575:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677448.369578:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.369580:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.369582:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.369583:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.369584:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.369585:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.369586:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.369590:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.369592:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.369594:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.369595:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.369596:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.369598:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.369598:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.369599:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.369599:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.369605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.369613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677448.369613:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.369615:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.369616:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.369617:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.369619:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.369621:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.369622:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.369623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:31.0:1619677448.369624:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.369626:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.369628:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.369628:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.369629:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.369631:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.369631:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677448.369632:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.369633:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.369634:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677448.369635:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.369635:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.369635:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.369636:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.369637:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:31.0:1619677448.369638:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.369639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677448.369640:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.369642:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.369642:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.369643:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.369643:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.369644:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:31.0:1619677448.369645:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.369645:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677448.369646:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.369647:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:5.0:1619677448.369647:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677448.369648:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.369649:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677448.369649:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.369650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.369650:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.2:1619677448.369652:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:20.0:1619677448.369655:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.369656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677448.369656:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.369657:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:20.0:1619677448.369658:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.369659:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.369659:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:12.0:1619677448.369660:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.369661:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.369661:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:12.0:1619677448.369663:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.369664:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:20.0:1619677448.369664:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.369664:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677448.369664:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:30.0:1619677448.369666:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:20.0:1619677448.369666:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.369666:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.369667:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.369668:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.369668:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677448.369668:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.369670:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.369671:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.369672:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.369673:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.369676:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.369677:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:5.0:1619677448.369677:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.369678:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.369679:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.369680:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.369680:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.369682:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.369682:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677448.369682:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.369682:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.369683:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.369683:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:25.2:1619677448.369685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:5.0:1619677448.369685:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.369686:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.369686:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.369687:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:13.0:1619677448.369692:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.369695:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677448.369695:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.369697:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.369698:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677448.369699:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677448.369700:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.369700:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.369701:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.369701:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677448.369704:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.369711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.369718:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.369720:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.369721:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.369722:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.369722:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.369724:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.369733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.369741:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.369744:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.369744:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.369746:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.369748:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.369749:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.369750:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.369752:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.369753:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.369754:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.369757:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.369759:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.369760:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.369761:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.369762:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.369763:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.369763:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.369764:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.369766:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.369766:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.369768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.2:1619677448.369772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.2:1619677448.369776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.369777:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.369778:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.369778:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.369781:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:10.0:1619677448.369781:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.369782:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.369782:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.369782:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.369782:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.369783:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.369783:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:17.0:1619677448.369784:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.369785:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.369787:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.369790:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.369791:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.369793:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.369794:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.369795:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.369796:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.369799:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.369801:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.369804:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.369805:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.369806:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.369808:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.369809:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.369809:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.369810:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.369814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.369820:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.369823:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.369824:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.369826:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.369826:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.369828:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.369833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.369839:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.369842:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.369843:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.369844:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.369847:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.369848:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.369849:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.369851:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.369852:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.369853:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.369856:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.369858:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.369860:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.369860:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.369861:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.369863:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.369863:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.369864:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.369864:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.369869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.369876:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.369878:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.369879:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.369879:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.369880:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.369882:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.369916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.369923:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.369925:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.369926:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.369928:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.369930:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.369931:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.369932:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.369934:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.369935:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.369936:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.369939:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.369940:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.369942:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.369943:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.369944:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.369945:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.369945:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.369946:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.369947:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.369948:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.369950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:4.2:1619677448.369953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.369958:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.369960:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.369960:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.369962:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.369962:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.369963:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.369963:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.369964:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.369964:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.369965:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677448.370097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.370103:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.370106:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.370107:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.370109:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.370112:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.370113:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.370114:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.370115:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.370117:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.370118:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.370121:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.370123:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.370125:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.370126:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.370127:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.370129:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.370129:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.370130:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.370130:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.370135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.370142:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.370145:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.370145:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.370147:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.370148:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.370150:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.370210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.370215:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.370217:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.370219:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.370220:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.370222:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.370224:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.370226:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.370227:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.370228:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.370229:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.370232:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.370234:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.370237:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.370237:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.370238:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.370240:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.370240:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.370242:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.370243:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.370244:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.370245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.370249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.370252:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677448.370252:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:21.2:1619677448.370254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.370254:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:13.0:1619677448.370255:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677448.370256:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:13.0:1619677448.370256:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.370257:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677448.370258:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:13.0:1619677448.370258:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.370259:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.370260:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:13.0:1619677448.370260:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:20.0:1619677448.370262:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.370263:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.370265:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.370267:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.370268:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.370269:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.370270:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.370271:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.370274:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.370276:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.370278:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.370278:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.370279:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.370280:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.370281:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.370282:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.370283:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.370287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.2:1619677448.370299:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.0:1619677448.370304:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.370306:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:10.0:1619677448.370307:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:5.0:1619677448.370307:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.370309:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.370310:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:10.0:1619677448.370310:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.370311:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.0:1619677448.370311:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:10.0:1619677448.370312:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.370315:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.370316:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.370317:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.370318:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.370319:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.370320:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.370323:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.370325:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.370327:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.370327:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.370328:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.370329:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.370329:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.370330:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.370331:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.370335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.370341:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.370343:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.370343:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.370344:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.370345:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.370346:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.370479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.370485:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.370488:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.370489:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.370491:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.370492:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.370494:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.370495:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.370496:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.370497:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.370498:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.370501:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.370503:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.370506:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.370507:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.370507:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.370509:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.370509:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.370511:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.370512:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.370512:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.370516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.370517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:4.2:1619677448.370521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677448.370522:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.370524:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:20.0:1619677448.370525:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677448.370525:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.370526:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677448.370527:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.2:1619677448.370527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:20.0:1619677448.370528:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.370528:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677448.370530:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.370531:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.370532:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677448.370533:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:10.0:1619677448.370533:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.370533:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677448.370533:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.370535:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.370536:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:1.0:1619677448.370536:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.370537:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.370538:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.370540:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.370540:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.370541:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.370542:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677448.370542:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.370544:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:21.2:1619677448.370545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:10.0:1619677448.370546:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.370546:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715842] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.370547:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.370548:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.370549:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:1.0:1619677448.370549:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.370552:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:10.0:1619677448.370552:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.370553:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:1.0:1619677448.370553:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.370554:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:10.0:1619677448.370554:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677448.370555:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:1.0:1619677448.370555:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.370556:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.370556:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:24.0:1619677448.370557:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.370557:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.370557:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677448.370559:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.370559:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:24.0:1619677448.370560:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.370560:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:25.2:1619677448.370562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:24.0:1619677448.370562:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:1.0:1619677448.370562:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.370562:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:24.0:1619677448.370563:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96715841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.370563:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677448.370564:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.370564:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677448.370565:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.370566:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:25.2:1619677448.370568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677448.370569:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:24.0:1619677448.370570:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.370571:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:6.0:1619677448.370572:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.370573:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.370574:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.370575:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:24.0:1619677448.370576:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.370576:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.370577:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.370577:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:12.0:1619677448.370577:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.370578:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.370579:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:6.0:1619677448.370579:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677448.370580:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.370580:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:12.0:1619677448.370580:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.370581:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677448.370582:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.370583:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.2:1619677448.370585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.370593:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.370595:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.370596:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.370598:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.370599:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.370601:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.370602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.370610:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.370612:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.370613:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.370614:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.370614:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.370615:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.370707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:20.2:1619677448.370712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677448.370713:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.370716:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.370717:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:5.0:1619677448.370717:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:2.0:1619677448.370718:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.370720:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:2.0:1619677448.370721:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.370722:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.370722:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.370722:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.370724:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.370724:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.370725:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.370725:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.370727:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.370727:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.370728:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.370729:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.370730:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.370732:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.370732:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:5.0:1619677448.370736:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.370736:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.370738:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.370739:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.370740:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.370741:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.370741:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.370742:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.370743:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.370743:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.370744:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.370744:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.0:1619677448.370745:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.370745:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.370745:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.0:1619677448.370746:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.370747:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.370749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:9.2:1619677448.370751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677448.370757:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677448.370758:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:22.0:1619677448.370760:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.370761:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.370761:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:6.0:1619677448.370761:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:18.0:1619677448.370762:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.370762:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:22.0:1619677448.370763:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:18.0:1619677448.370764:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.370765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.370767:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677448.370768:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.370770:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.370772:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.370773:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.370773:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.370774:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.370775:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.370899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.370907:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.370910:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.370911:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.370913:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.370916:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.370917:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.370918:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.370919:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.370921:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.370921:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.370925:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.370927:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.370929:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.370930:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.370931:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.370932:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.370933:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.370933:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.370934:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.370938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.370946:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.370948:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.370949:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.370951:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.370953:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.370954:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.370956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.370962:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.370965:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.370966:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.370967:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.370967:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.370968:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.371145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.371152:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.371155:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.371156:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.371157:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.371159:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.371161:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.371162:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715837] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.371164:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.371164:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.371165:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.371168:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.371170:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.371173:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.371173:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.371174:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.371175:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.371176:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.371177:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.371177:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.371181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.371189:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.371192:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.371192:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.371193:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.371195:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.371198:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.371221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.371226:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.371229:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.371230:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.371232:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.371234:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.371235:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.371236:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.371237:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.371238:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.371238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:22.0:1619677448.371239:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.371242:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.371244:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.371244:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.371246:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:22.0:1619677448.371247:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.371248:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.371248:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.371248:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.371249:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.371250:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.371250:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.371251:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.371252:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:2.0:1619677448.371252:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.371253:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.371254:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.371256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:2.0:1619677448.371256:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.371257:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.371258:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.371261:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.371263:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677448.371263:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.371265:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.371265:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.371266:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.371266:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.371266:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.371268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:20.0:1619677448.371268:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.371268:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.371268:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.371269:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.371269:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677448.371271:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.371272:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:9.2:1619677448.371274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.371275:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.371278:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.371279:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.371281:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.371282:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.371284:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.371284:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.371284:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.371285:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:2.0:1619677448.371285:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.2:1619677448.371286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:18.0:1619677448.371287:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.371287:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96715834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.371290:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:8.0:1619677448.371290:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.371290:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.371292:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.371293:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.371298:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:8.0:1619677448.371300:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.371301:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:8.0:1619677448.371302:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.371303:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.371305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677448.371306:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.371306:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.371306:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.371307:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:26.0:1619677448.371308:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.371309:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.371309:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.371309:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:26.0:1619677448.371310:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.371310:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.371310:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:26.0:1619677448.371311:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.371311:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.371312:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.371314:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:8.2:1619677448.371314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.371314:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.371316:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.371318:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.371319:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:17.0:1619677448.371321:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.371322:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:5.0:1619677448.371322:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677448.371323:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.371323:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.371324:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:17.0:1619677448.371324:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96715832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.371325:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:17.0:1619677448.371325:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.371325:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677448.371326:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:17.0:1619677448.371326:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.371326:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.371327:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.371327:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:17.0:1619677448.371327:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.371328:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.371328:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.371331:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.371331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677448.371333:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:5.0:1619677448.371334:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677448.371336:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.371336:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:17.0:1619677448.371337:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:5.0:1619677448.371337:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:17.0:1619677448.371338:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.371339:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:17.0:1619677448.371339:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:5.0:1619677448.371339:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.371340:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.371340:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677448.371340:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.371341:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677448.371343:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:25.2:1619677448.371346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.2:1619677448.371350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:13.0:1619677448.371354:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.371357:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:20.0:1619677448.371358:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:13.0:1619677448.371358:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.371360:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:13.0:1619677448.371360:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.371361:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.371362:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:12.0:1619677448.371362:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677448.371363:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:13.0:1619677448.371363:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677448.371364:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.371413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.371418:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.371420:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.371422:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.371423:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.371425:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.371426:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.371427:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.371429:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.371430:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.371431:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.371434:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.371436:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.371438:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.371439:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.371440:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.371442:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.371442:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.371443:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.371443:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.371447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.2:1619677448.371453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677448.371455:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.371457:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.371458:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.371459:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.371460:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:31.0:1619677448.371461:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.371461:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:31.0:1619677448.371464:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.371466:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.371469:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.371469:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.371471:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.371472:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715830] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.371473:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.371474:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.371475:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.371478:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.371478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:31.0:1619677448.371480:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.371481:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:31.0:1619677448.371482:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.371483:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:20.0:1619677448.371483:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:31.0:1619677448.371484:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.371485:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.371486:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:20.0:1619677448.371486:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.371486:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677448.371487:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.371488:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.371488:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.371489:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:25.2:1619677448.371491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:20.0:1619677448.371491:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.371492:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.371493:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.371494:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677448.371497:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.371497:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.0:1619677448.371499:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:12.0:1619677448.371500:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.371501:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.371502:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.371503:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:12.0:1619677448.371503:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.371504:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.2:1619677448.371505:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:12.0:1619677448.371505:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677448.371507:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.371509:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:25.2:1619677448.371510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:20.0:1619677448.371510:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.371510:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.371511:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.371512:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.371512:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.371513:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.371514:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.371514:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.371515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:13.0:1619677448.371516:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:5.0:1619677448.371516:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.371518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000400:00000200:5.0:1619677448.371518:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.371519:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.371520:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.371521:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:13.0:1619677448.371521:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.371521:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:10.0:1619677448.371521:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:5.0:1619677448.371521:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.371522:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.371522:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.371522:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.371523:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:5.0:1619677448.371523:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.371524:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:12.0:1619677448.371524:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.371524:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.371525:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.371525:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.371526:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.371528:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.371529:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.371531:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.0:1619677448.371532:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.371533:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.371534:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.371536:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:25.2:1619677448.371537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677448.371537:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.371537:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.371538:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.371542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.371543:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.371546:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.371547:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.371547:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.371548:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.371548:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:2.0:1619677448.371550:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.371553:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.371554:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.371554:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.371556:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.371559:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.371561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.371568:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.371570:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.371571:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.371571:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.371572:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.371572:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.371669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.371677:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.371679:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.371682:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.371683:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.371684:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.371685:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.371687:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.371688:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.371689:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.371690:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.371694:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.371695:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.371698:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.371698:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.371699:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.371700:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.371701:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.371702:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.371703:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.371706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.371712:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.371715:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.371715:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.371717:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.371718:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.371719:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.371742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677448.371744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.371749:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.371750:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.371751:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.371752:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.371753:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.371753:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:10.0:1619677448.371756:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.371757:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.371758:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.371759:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677448.371759:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.371761:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.371761:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:10.0:1619677448.371762:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.371763:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.371764:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.371764:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.371765:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677448.371766:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.371767:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:10.0:1619677448.371771:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.371771:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.371774:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.371774:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.371776:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.371777:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.371777:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:10.0:1619677448.371778:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:1.0:1619677448.371778:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677448.371779:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677448.371780:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:10.0:1619677448.371781:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.371781:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.371781:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.371782:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677448.371782:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.371783:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:10.0:1619677448.371784:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677448.371784:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.371786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:25.2:1619677448.371787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.2:1619677448.371789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.371794:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677448.371794:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677448.371796:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.371796:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.371796:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.371797:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:17.0:1619677448.371798:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.371798:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:1.0:1619677448.371798:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677448.371799:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.371799:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.371800:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.371801:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677448.371801:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.371802:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.371804:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.371895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.371903:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.371906:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.371906:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.371908:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.371912:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.371913:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.371915:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.371916:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.371917:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.371918:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.371922:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.371924:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.371926:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.371927:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.371927:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.371929:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.371929:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.371930:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.371930:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.371934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.371938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:12.0:1619677448.371941:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.371944:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.371944:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.371946:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:12.0:1619677448.371946:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.371948:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.371949:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:12.0:1619677448.371949:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:31.0:1619677448.371951:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.371952:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.371953:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.371954:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.371955:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.371957:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.371958:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.371959:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.371962:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.371963:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.371965:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.371966:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.371966:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.371968:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.371968:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.371969:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.371970:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.371973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.371980:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.371982:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.371983:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.371985:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.371986:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.371987:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.372158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.372163:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.372166:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.372167:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.372169:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.372170:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.372172:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.372173:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.372175:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.372176:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.372177:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.372180:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.372181:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.372184:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.372185:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:4.2:1619677448.372185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:17.0:1619677448.372186:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.372187:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.372188:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.372188:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.372189:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677448.372192:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.2:1619677448.372193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.372195:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:21.2:1619677448.372195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:24.0:1619677448.372198:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.372198:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.372199:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:24.0:1619677448.372201:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.372201:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:2.0:1619677448.372201:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:8.0:1619677448.372202:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.372203:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:2.0:1619677448.372204:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:24.0:1619677448.372205:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96715821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.372205:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:8.0:1619677448.372205:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.372207:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.372208:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.372209:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.372210:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.372210:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.372212:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.372213:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.372214:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.372215:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.372216:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.372218:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:2.0:1619677448.372218:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:24.0:1619677448.372219:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.372219:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:2.0:1619677448.372221:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.372223:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.372223:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.372223:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:24.0:1619677448.372224:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.372224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:24.0:1619677448.372226:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.372226:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677448.372226:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.372227:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.372227:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677448.372227:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.372228:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.372230:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.372230:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:31.0:1619677448.372231:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.372231:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:8.2:1619677448.372232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.372232:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:31.0:1619677448.372234:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:25.2:1619677448.372236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:31.0:1619677448.372237:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.372238:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.372239:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.372239:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:31.0:1619677448.372241:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677448.372242:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96715819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.372242:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:1.0:1619677448.372242:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:31.0:1619677448.372243:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.372243:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.372243:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.372245:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.372245:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:1.0:1619677448.372245:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.372246:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:26.0:1619677448.372246:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.372246:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.372249:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.372250:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.372252:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:10.0:1619677448.372252:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:1.0:1619677448.372253:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677448.372255:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.372255:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.372256:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.372258:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.372258:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.372259:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.372260:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.372263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.372269:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.372272:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.372273:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.372273:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.372275:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.372277:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.372355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.372361:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.372364:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.372365:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.372367:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.372369:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.372371:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.372372:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.372373:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.372374:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.372375:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.372379:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.372381:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.372384:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.372384:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.372385:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.372387:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.372387:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.372388:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.372388:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.372393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.372400:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.372403:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.372403:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.372404:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.372406:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.372409:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677448.372474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.372477:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.372480:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.372482:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.372483:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.372484:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.372486:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:20.2:1619677448.372488:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:20.0:1619677448.372491:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.372493:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.372494:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.372495:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.372497:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:20.0:1619677448.372498:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.372499:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:20.0:1619677448.372500:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.372502:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.372502:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.372503:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.372504:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.372504:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.372504:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:30.0:1619677448.372505:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:20.0:1619677448.372505:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.372506:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677448.372507:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.372507:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677448.372508:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.372508:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:30.0:1619677448.372509:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.372510:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:8.2:1619677448.372512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:30.0:1619677448.372514:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.372516:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.372518:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.372518:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.372519:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.372520:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.372520:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:30.0:1619677448.372521:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.372522:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:5.0:1619677448.372522:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.372523:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.372524:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.372524:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:5.0:1619677448.372525:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.372526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.372526:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.2:1619677448.372528:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.2:1619677448.372530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.372532:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677448.372535:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:10.0:1619677448.372535:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.372536:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.372537:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.372538:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:31.0:1619677448.372539:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.372539:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:26.0:1619677448.372539:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.372540:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.372541:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.372541:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:10.0:1619677448.372541:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.372542:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677448.372543:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:10.0:1619677448.372543:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.372545:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.372546:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.372547:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.372548:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.372549:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.2:1619677448.372552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000400:00000200:10.0:1619677448.372552:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.372554:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.0:1619677448.372557:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:10.0:1619677448.372557:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.372557:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.372558:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.0:1619677448.372559:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:10.0:1619677448.372560:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.372560:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:20.0:1619677448.372561:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677448.372561:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:5.0:1619677448.372562:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.372563:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.372563:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.372563:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:20.0:1619677448.372565:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:25.2:1619677448.372566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:20.0:1619677448.372566:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.372567:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.372568:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.372569:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:5.2:1619677448.372569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:20.0:1619677448.372572:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.372573:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677448.372573:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:20.0:1619677448.372575:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.372576:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:20.0:1619677448.372576:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:6.0:1619677448.372576:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.372577:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:20.0:1619677448.372577:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:6.0:1619677448.372577:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.372578:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:20.0:1619677448.372578:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:26.0:1619677448.372579:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677448.372579:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000400:00000200:6.0:1619677448.372579:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.372580:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.0:1619677448.372580:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677448.372581:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677448.372581:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:6.0:1619677448.372582:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.2:1619677448.372584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:21.2:1619677448.372587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.372592:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.372594:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677448.372594:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.372595:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000400:00000200:5.0:1619677448.372595:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.372596:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.372596:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:20.0:1619677448.372597:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.372598:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677448.372599:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:21.2:1619677448.372724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.2:1619677448.372730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.372732:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.372734:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.372736:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.372737:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.372738:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.372739:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.372739:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.372740:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677448.372741:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.372742:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.372743:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.372744:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.372745:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.372745:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.372745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:30.0:1619677448.372746:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677448.372747:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.372749:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.372751:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.372751:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.372752:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.372752:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677448.372753:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:30.0:1619677448.372754:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.372755:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.372756:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.372757:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.372758:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:17.0:1619677448.372758:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.372759:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:10.0:1619677448.372759:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.372760:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.372761:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.372761:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677448.372762:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.372762:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:10.0:1619677448.372762:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.372763:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:17.0:1619677448.372764:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.372765:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000400:00000200:10.0:1619677448.372765:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96715811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.372766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:17.0:1619677448.372766:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:10.0:1619677448.372766:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.372767:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000400:00000200:10.0:1619677448.372767:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.372768:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.372768:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.372769:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:10.0:1619677448.372769:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.372773:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:10.0:1619677448.372773:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.372773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:26.0:1619677448.372774:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.372775:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.372775:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.372777:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.372778:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.372779:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.372780:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.372780:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:24.0:1619677448.372780:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:10.0:1619677448.372780:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:24.0:1619677448.372782:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:10.0:1619677448.372782:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:24.0:1619677448.372783:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.372783:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677448.372783:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:10.0:1619677448.372784:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:24.0:1619677448.372785:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.372785:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.372786:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677448.372787:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:25.2:1619677448.372788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.372791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:9.2:1619677448.372792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.372794:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.372796:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.372797:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.372798:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:8.0:1619677448.372798:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000400:00000200:31.0:1619677448.372799:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.372800:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.372800:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.372801:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:8.0:1619677448.372801:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:31.0:1619677448.372802:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.372802:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:24.0:1619677448.372802:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.372802:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.372803:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.372804:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:25.2:1619677448.372806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.372812:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.372815:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.372816:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.372817:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.372817:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.372818:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.373054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.373060:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.373063:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.373064:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.373065:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.373068:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.373069:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.373070:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.373072:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.373072:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.373073:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.373077:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.373078:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.373080:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.373081:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.373081:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.373083:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.373083:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.373084:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.373084:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.373088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.373095:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.373098:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.373098:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.373100:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.373102:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.373103:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.373136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.373143:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.373146:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.373148:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.373150:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.373150:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.373152:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.373153:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.373154:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.373155:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.373156:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.373159:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.373161:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.373163:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.373164:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.373164:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.373166:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.373166:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.373167:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.373168:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.373171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.373179:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.373181:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.373182:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.373184:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.373186:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.373187:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.373190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.373198:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.373200:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.373202:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.373204:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.373205:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.373206:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.373207:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.373209:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.373209:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.373210:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.373213:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.373215:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.373217:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.373217:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.373218:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.373219:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.2:1619677448.373219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:30.0:1619677448.373220:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.373221:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.373223:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.373223:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.373225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.373225:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.373228:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.2:1619677448.373229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.373230:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.373230:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.373232:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:5.0:1619677448.373233:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.373234:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.373235:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.373235:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:13.0:1619677448.373237:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.373237:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.373238:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677448.373238:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.373239:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677448.373239:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.373240:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:5.0:1619677448.373240:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.373241:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:5.0:1619677448.373241:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.373242:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:5.0:1619677448.373244:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.373246:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.373248:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.373249:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.373250:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.373252:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.373252:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:25.2:1619677448.373254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.373254:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.373255:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.373255:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.373258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.373260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677448.373260:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.373263:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.373264:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:22.0:1619677448.373265:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677448.373265:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677448.373265:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.373265:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.373266:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677448.373267:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:22.0:1619677448.373268:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677448.373268:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:22.0:1619677448.373269:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.373269:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677448.373270:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.373271:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.373274:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.373374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.373381:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.373384:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.373385:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.373387:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.373389:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.373390:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.373392:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.373393:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.373394:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.373395:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.373399:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.373400:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.373403:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.373403:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.373404:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.373405:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.373406:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.373407:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.373407:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.373412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.373419:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.373421:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.373422:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.373424:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.373426:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.373427:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.373455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.373460:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.373463:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:21.2:1619677448.373465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:20.0:1619677448.373465:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.373467:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.373467:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.373469:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.373470:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715805] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.373471:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.373471:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:20.0:1619677448.373472:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.373473:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.373473:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.373474:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.373476:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.373478:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.373479:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.373480:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 440 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:6.0:1619677448.373481:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.373482:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.373484:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.373484:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.373485:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.373485:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:6.0:1619677448.373485:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.373486:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:20.0:1619677448.373487:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.373488:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.373489:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.373489:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:6.0:1619677448.373489:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.373491:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:25.2:1619677448.373493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.373493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:6.0:1619677448.373494:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.373495:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.373495:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.373497:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.373498:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.373499:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:10.0:1619677448.373500:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:6.0:1619677448.373500:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:13.0:1619677448.373501:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677448.373501:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:25.2:1619677448.373503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.373503:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:13.0:1619677448.373504:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:26.0:1619677448.373505:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:13.0:1619677448.373505:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.373506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677448.373506:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.373507:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:13.0:1619677448.373507:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.373508:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:12.0:1619677448.373508:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:26.0:1619677448.373509:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.373509:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.373510:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677448.373511:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.373511:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:10.0:1619677448.373511:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:22.0:1619677448.373513:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:10.0:1619677448.373513:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.373514:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677448.373514:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.373515:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677448.373515:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.373516:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.373520:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.373520:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.373521:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:10.0:1619677448.373522:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.373525:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.373526:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.373527:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.373529:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.373529:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.373530:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.373531:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.373536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.373544:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.373547:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.373548:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.373550:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.373550:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.373553:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.373599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.373606:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.373608:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.373609:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.373611:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.373614:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.373616:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.373617:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.373618:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.373619:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.373620:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.373624:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.373625:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.373628:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.373628:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.373629:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.373630:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.373631:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.373632:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.373632:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.373636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.373644:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.373647:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.373648:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.373648:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.373650:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.373652:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.373657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:22.0:1619677448.373664:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.373667:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.373667:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.373668:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.373669:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.373670:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.373678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.2:1619677448.373681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677448.373681:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.373684:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.373686:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.373688:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677448.373689:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677448.373689:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.373691:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:31.0:1619677448.373692:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:20.0:1619677448.373693:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.373695:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.373695:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.373696:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677448.373696:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.373697:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677448.373698:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.373700:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.373701:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.373702:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:31.0:1619677448.373703:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.373704:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.373705:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677448.373708:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.373709:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:31.0:1619677448.373710:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.373710:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:20.0:1619677448.373711:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.373712:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.373713:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.373713:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.373714:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:31.0:1619677448.373715:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.373715:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.373715:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.373716:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.373717:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:30.0:1619677448.373717:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:20.0:1619677448.373717:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.373718:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:30.0:1619677448.373718:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96715799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.373718:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.373718:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000400:00000200:30.0:1619677448.373719:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.373719:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:30.0:1619677448.373720:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.373721:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.373722:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.373723:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.373724:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.373724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:9.2:1619677448.373724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.373725:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.373725:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.373726:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.373729:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.373730:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.373730:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:30.0:1619677448.373731:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.373732:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.373732:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:30.0:1619677448.373733:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677448.373733:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.373733:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.373733:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.373734:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.373734:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:26.0:1619677448.373735:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.373736:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.373737:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.373737:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.373738:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:8.2:1619677448.373738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:18.0:1619677448.373739:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.373742:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677448.373746:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.373748:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.373749:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.373751:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.373751:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.373754:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.373769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.373776:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.373779:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.373780:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.373781:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.373784:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.373785:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.373786:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.373788:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.373789:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.373790:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.373793:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.373794:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.373797:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.373798:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.373799:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.373800:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.373801:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.373802:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.373802:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.373806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.373814:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.373817:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.373818:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.373818:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.373820:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.373824:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.373839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.373847:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.373850:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.373850:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.373852:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.373855:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.373856:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.373858:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.373858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:24.0:1619677448.373859:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.373860:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.373861:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.373863:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:24.0:1619677448.373864:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.373865:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.373866:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.373867:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:24.0:1619677448.373868:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.373869:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:18.0:1619677448.373869:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:24.0:1619677448.373870:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.373872:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.373872:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:18.0:1619677448.373872:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.373873:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000400:00000200:18.0:1619677448.373873:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:24.0:1619677448.373874:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000400:00000200:18.0:1619677448.373874:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.373875:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:18.0:1619677448.373875:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.373876:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.373878:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:9.2:1619677448.373879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.373880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000400:00000200:18.0:1619677448.373881:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.373883:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.373886:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.373887:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677448.373887:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:18.0:1619677448.373888:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.373888:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.0:1619677448.373889:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.373890:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:18.0:1619677448.373890:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677448.373890:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:2.0:1619677448.373890:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.373891:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:2.0:1619677448.373891:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.373892:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.373893:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:2.0:1619677448.373893:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.373894:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:25.2:1619677448.373896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.373896:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677448.373903:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.373906:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.373907:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.373907:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.373909:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677448.373911:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.373914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.2:1619677448.373915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:22.0:1619677448.373921:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.373921:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:22.0:1619677448.373924:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.373924:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:22.0:1619677448.373925:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.373925:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.373925:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:22.0:1619677448.373926:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.373927:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:18.0:1619677448.373927:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.373930:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.373931:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677448.373932:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.373933:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.373934:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.373935:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677448.373938:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.373940:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677448.373943:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.373943:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677448.373944:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677448.373946:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677448.373947:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677448.373948:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677448.373948:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:25.2:1619677448.373952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.373958:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.373961:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.373961:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.373962:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.373964:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.373967:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.374032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.374038:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.374041:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.374042:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.374044:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.374046:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.374047:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.374049:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.374050:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.374051:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.374052:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.374055:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.374057:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.374060:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.374060:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.374061:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.374063:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.374063:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.374065:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.374066:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.374066:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.374069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.2:1619677448.374071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.374075:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:1.0:1619677448.374076:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677448.374078:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677448.374078:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:24.0:1619677448.374079:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.374079:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.374079:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677448.374081:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.374081:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677448.374084:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677448.374291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677448.374293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.374300:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.374301:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.374303:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.374303:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:10.0:1619677448.374304:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.374305:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.374307:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.374309:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.374310:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.374311:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:10.0:1619677448.374312:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.374313:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.374314:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:6.0:1619677448.374315:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.374316:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.374316:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.374317:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.374318:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.374318:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:10.0:1619677448.374319:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.2:1619677448.374320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:6.0:1619677448.374322:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.374324:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.374325:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.374327:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.374328:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677448.374328:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.374329:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.374330:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.374331:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:10.0:1619677448.374331:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.374331:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:10.0:1619677448.374332:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.374332:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.374332:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:31.0:1619677448.374333:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.374333:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.374333:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.374334:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.374334:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.374335:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.374336:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.374336:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677448.374337:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:10.0:1619677448.374337:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.374338:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.374338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677448.374339:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96715791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.374340:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.374341:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.374342:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:4.2:1619677448.374343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.374345:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.374347:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.374347:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.374349:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:31.0:1619677448.374350:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.374350:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:25.2:1619677448.374350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:17.0:1619677448.374350:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.374350:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.374351:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)++
00000800:00000200:31.0:1619677448.374351:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (23)++
00000800:00000200:31.0:1619677448.374353:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (24)--
00000800:00000200:26.0:1619677448.374353:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.374353:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677448.374353:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.374354:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.374354:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677448.374354:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.374355:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677448.374356:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677448.374358:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:22.0:1619677448.374361:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:6.0:1619677448.374361:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.374362:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.374364:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.374367:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:25.2:1619677448.374369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.374376:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.374378:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.374379:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.374379:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.374380:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.374383:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.374402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.374410:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.374412:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.374413:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.374415:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.374418:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.374419:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.374421:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.374422:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.374423:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.374424:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.374427:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.374429:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.374431:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.374432:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.374433:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.374434:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.374435:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.374436:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.374436:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.374440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.374447:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.374449:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.374450:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.374451:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.374453:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.374454:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.374469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.374477:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.374479:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.374481:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.374484:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.374484:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.374486:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.374487:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.374488:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.374490:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.374490:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.374494:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.374495:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.374498:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.374499:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.374499:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.374501:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.374502:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.374502:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.374503:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.374507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.374516:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.374518:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.374519:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.374521:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.374522:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.374524:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.374575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.374580:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.374583:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.374584:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.374586:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.374588:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.374589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000400:00000200:1.0:1619677448.374589:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.374591:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.374592:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.374593:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:1.0:1619677448.374593:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.374594:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.374596:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677448.374598:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:1.0:1619677448.374599:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.374600:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.374602:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:5.0:1619677448.374603:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.374604:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:1.0:1619677448.374605:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.374606:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.374606:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.374606:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:5.0:1619677448.374607:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.374608:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.374608:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:5.0:1619677448.374609:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:1.0:1619677448.374609:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.374609:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.374610:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:5.0:1619677448.374613:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.374614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.0:1619677448.374614:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.374617:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.374618:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.374618:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.374620:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.374621:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.374621:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:22.0:1619677448.374622:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.0:1619677448.374622:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:22.0:1619677448.374624:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.374625:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.374625:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:8.2:1619677448.374626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000400:00000200:22.0:1619677448.374627:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.374630:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.374632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:20.0:1619677448.374634:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.374636:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.374636:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.374638:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.374639:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:24.0:1619677448.374640:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:20.0:1619677448.374640:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:24.0:1619677448.374643:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677448.374643:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677448.374645:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000800:00000200:8.2:1619677448.374645:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000400:00000200:24.0:1619677448.374648:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.374650:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677448.374652:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.374653:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.374654:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000400:00000200:24.0:1619677448.374655:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.374655:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.2:1619677448.374657:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:24.0:1619677448.374659:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.374659:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:24.0:1619677448.374661:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.374661:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.374661:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.374662:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.374662:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:2.0:1619677448.374663:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:24.0:1619677448.374664:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.374665:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677448.374665:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.374666:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:24.0:1619677448.374667:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.374667:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:24.0:1619677448.374668:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677448.374669:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000400:00000200:2.0:1619677448.374669:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.374670:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677448.374671:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:2.0:1619677448.374672:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.374673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:2.0:1619677448.374674:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.374675:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677448.374676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:2.0:1619677448.374676:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.374677:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.374678:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.2:1619677448.374680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (21)++
00000800:00000200:6.0:1619677448.374681:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:2.0:1619677448.374681:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.374683:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.374684:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.374684:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.374685:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.374686:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677448.374686:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.374687:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.374687:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.374687:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:24.0:1619677448.374688:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677448.374688:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.374689:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:6.0:1619677448.374689:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677448.374689:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.374689:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:24.0:1619677448.374690:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677448.374690:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.374690:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.374691:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:24.0:1619677448.374692:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.374693:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:8.2:1619677448.374695:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:8.0:1619677448.374699:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.374701:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.374702:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96715784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.374703:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:8.0:1619677448.374703:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.374704:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.374705:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.374706:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.374706:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.374708:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.374709:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:8.0:1619677448.374710:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.374711:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.374713:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:8.0:1619677448.374717:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.374720:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.374720:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.0:1619677448.374721:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.374721:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.374723:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.374723:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:8.0:1619677448.374724:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.0:1619677448.374724:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:8.0:1619677448.374725:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:5.0:1619677448.374725:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.374725:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:8.0:1619677448.374726:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:20.0:1619677448.374727:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.0:1619677448.374727:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.374729:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.374732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.374736:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.374739:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.374739:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.374739:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.374740:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.374742:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.374742:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.374742:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.374743:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.374744:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.374748:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.374750:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.374752:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.374753:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.374753:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.374754:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.374755:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.374859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.374866:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.374869:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.374870:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.374871:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.374874:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.374876:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.374877:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715783] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.374878:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.374879:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.374879:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.374882:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.374884:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.374886:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.374887:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.374888:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.374890:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.374890:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.374891:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.374892:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.374896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.374904:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.374907:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.374908:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.374908:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.374910:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.374913:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.374916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.374923:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.374926:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.374927:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.374927:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.374928:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.374929:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.375011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.375016:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.375019:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.375020:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.375022:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.375024:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.375025:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.375027:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.375028:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.375029:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.375030:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.375033:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.375035:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.375038:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.375039:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.375039:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.375041:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.375041:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.375042:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.375042:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.375047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.375055:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.375057:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.375058:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.375059:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.375060:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.375062:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.375123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.375129:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.375131:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.375134:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.375134:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.375136:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.375137:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.375139:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.375140:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.375141:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.375142:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.375145:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.375147:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.375150:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.375151:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.375152:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.375154:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.375154:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.375155:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.375156:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.375160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.375168:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.375170:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.375171:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.375173:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.375175:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.375176:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.375178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.375186:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.375188:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.375189:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.375191:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.375193:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.375195:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.375196:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.375197:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.375198:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.375199:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.375203:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.375205:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.375207:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.375208:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.375208:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.375210:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.375210:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.375211:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.375211:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.375215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.375223:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.375225:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.375226:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.375227:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.375229:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.375230:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.375234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.375241:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.375243:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.375244:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.375245:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.375245:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.375246:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.375362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.375370:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.375372:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.375373:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.375375:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.375378:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.375379:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.375380:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.375381:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.375382:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.375383:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.375386:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.375387:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.375390:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.375391:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.375391:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.375393:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.375394:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.375395:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.375395:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.375400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.375409:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.375411:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.375412:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.375413:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.375415:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.375418:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.375458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.375464:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.375467:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.375467:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.375469:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.375471:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.375473:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.375474:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.375475:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.375476:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.375477:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.375481:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.375482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677448.375483:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.375486:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.375487:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.375487:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.375489:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.375490:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.375490:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.375490:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.375491:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:31.0:1619677448.375493:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.375495:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.375495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.375497:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.375498:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.375499:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.375501:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715777] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.375502:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.375503:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.375503:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:31.0:1619677448.375504:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.375505:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.375506:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.375507:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677448.375508:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.375509:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.375510:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.375511:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.375512:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.375512:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.375513:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.375515:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.375515:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.375516:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.375517:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.375520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.375527:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.375529:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.375530:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.375531:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.375533:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.375534:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.375542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.375548:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.375551:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.375552:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.375552:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.375553:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.375554:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.375634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.375640:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.375643:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.375644:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.375646:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.375648:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.375650:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.375651:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.375653:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.375654:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.375655:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.375658:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.375660:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.375662:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.375663:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.375664:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.375666:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.375666:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.375667:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.375668:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.375672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.375680:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.375682:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.375683:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.375685:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.375686:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.375687:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.375708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.375713:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.375715:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.2:1619677448.375718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.375718:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.375718:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.375719:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.375721:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677448.375722:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.375723:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.375724:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:5.2:1619677448.375726:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:5.0:1619677448.375730:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715775] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.375731:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.375732:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.375733:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:26.0:1619677448.375734:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:20.0:1619677448.375735:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.375737:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:20.0:1619677448.375737:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:10.0:1619677448.375737:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:20.0:1619677448.375738:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.375738:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:20.0:1619677448.375740:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.375741:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.375742:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.375742:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.375743:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.375743:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.375744:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.375745:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.375747:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677448.375747:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.375747:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.375747:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.375750:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96715773] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.375750:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:17.0:1619677448.375750:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.375751:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.375752:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.375753:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.375753:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.375753:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.375753:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.375753:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677448.375754:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.375754:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.375754:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:17.0:1619677448.375755:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.0:1619677448.375756:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000400:00000200:17.0:1619677448.375756:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96715772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.375756:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677448.375757:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677448.375757:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:26.0:1619677448.375758:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.375758:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000400:00000200:17.0:1619677448.375758:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.375758:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.375759:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:17.0:1619677448.375759:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.375760:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.375760:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:26.0:1619677448.375761:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.375761:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.375762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:9.2:1619677448.375763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:26.0:1619677448.375764:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.375764:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.375765:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.375766:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.375767:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:17.0:1619677448.375767:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.375768:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.375768:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.375769:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.375769:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677448.375769:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.375769:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677448.375770:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.375770:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:17.0:1619677448.375772:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.375772:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.375772:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:5.0:1619677448.375772:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:25.2:1619677448.375773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:24.0:1619677448.375773:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:17.0:1619677448.375773:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.375773:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:5.0:1619677448.375773:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.375774:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.375775:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.375775:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.375777:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:8.2:1619677448.375778:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677448.375779:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.375781:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:8.0:1619677448.375781:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.375782:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.375782:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.375784:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.375785:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:8.0:1619677448.375785:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:1.0:1619677448.375786:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.375788:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.375789:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.375789:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.375791:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.375792:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:25.2:1619677448.375793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:4.2:1619677448.375799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.375802:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.375805:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.375805:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.375806:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.375807:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.375807:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.375808:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.375808:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.375809:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.375811:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.375813:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.375814:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.375815:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.375817:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.375818:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.375819:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.375822:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.375824:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.375825:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.375826:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.375827:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.375828:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.375828:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.375829:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.375829:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.375833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.375839:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.375840:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.375841:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.375842:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.375843:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.375844:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.375924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.375927:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.375930:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.375932:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.375933:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.375935:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.375936:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.375938:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.375939:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.375940:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.375941:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.375944:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.375946:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.375950:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.375950:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.375951:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.375953:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.375953:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.375954:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.375955:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.375959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.375965:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.375968:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.375969:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.375970:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.375972:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:4.2:1619677448.375972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:13.0:1619677448.375973:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677448.375979:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.375981:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.375982:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.375984:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.375986:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.375988:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.375989:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.375991:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.375991:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.375993:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.375996:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.375998:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.376001:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.376001:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.376002:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.376004:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.376004:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.376005:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.376005:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.376010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.376018:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.376020:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.376021:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.376022:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.376024:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.376026:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.376081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.376089:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.376092:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.376093:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.376094:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.376096:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.376098:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.376099:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.376100:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.376101:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.376102:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.376105:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.376106:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.2:1619677448.376107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:10.0:1619677448.376109:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.376110:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.376110:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.376112:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.376112:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:18.0:1619677448.376113:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:10.0:1619677448.376113:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677448.376115:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.376115:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:18.0:1619677448.376116:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.376117:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.376118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:18.0:1619677448.376119:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.376121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:18.0:1619677448.376122:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.376123:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.376125:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.376126:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.376126:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:18.0:1619677448.376127:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.376128:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:18.0:1619677448.376128:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.376129:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.376129:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:26.0:1619677448.376130:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:1.0:1619677448.376130:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.376131:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.376132:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:18.0:1619677448.376132:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.376132:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.376134:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.376135:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677448.376137:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.376138:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.376138:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.376140:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.376140:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.376142:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.376143:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.376143:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.376146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.376148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677448.376153:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677448.376154:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:21.2:1619677448.376155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:17.0:1619677448.376155:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.376156:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.376156:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.376157:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677448.376157:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.376158:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.376159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.376159:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677448.376160:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677448.376161:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677448.376161:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.376162:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.376162:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:18.0:1619677448.376165:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.376165:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.376168:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.376168:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.376170:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.376171:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.376173:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.376174:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.376175:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.376176:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.376178:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.376180:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.376180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677448.376180:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.376181:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.376182:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.376182:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.376185:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:5.2:1619677448.376186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.376187:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.376187:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.376189:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.376189:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.376189:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.376190:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.376190:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.376190:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:2.0:1619677448.376190:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.376191:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677448.376192:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.376192:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677448.376192:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.376193:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (22)++
00000400:00000200:10.0:1619677448.376193:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.376193:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.376194:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.376194:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.376194:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.376195:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:10.0:1619677448.376195:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.376195:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:1.0:1619677448.376195:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:18.0:1619677448.376196:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677448.376196:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.376198:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.376198:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.376199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.376199:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96715764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.376199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677448.376200:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.376201:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.376202:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.376202:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.376203:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.376203:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96715763] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.376204:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.376206:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.376207:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.376207:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:1.0:1619677448.376207:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.376208:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.376209:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:12.0:1619677448.376210:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:2.0:1619677448.376210:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.376211:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:12.0:1619677448.376211:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.376211:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:2.0:1619677448.376212:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000400:00000200:1.0:1619677448.376212:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677448.376213:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.376214:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.376214:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.376215:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.376215:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.376216:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.376216:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:12.0:1619677448.376217:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.376217:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.376218:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.376218:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.376219:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.376219:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677448.376219:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.376220:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677448.376222:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.376222:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:25.2:1619677448.376224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.376224:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.376225:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.376225:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.376228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.376230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.376232:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.376234:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:13.0:1619677448.376235:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:22.0:1619677448.376236:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.376236:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677448.376236:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.376238:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.376238:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:13.0:1619677448.376238:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:17.0:1619677448.376239:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677448.376239:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.376240:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677448.376240:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.376242:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677448.376242:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.376244:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.376473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.376479:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.376481:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.376483:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.376484:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.376486:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.376488:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:21.2:1619677448.376490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:5.0:1619677448.376490:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.376492:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.376493:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.376494:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677448.376496:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:5.0:1619677448.376498:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.376499:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.376499:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:5.0:1619677448.376500:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:6.0:1619677448.376502:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.376503:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.376504:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.376504:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:6.0:1619677448.376505:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:5.0:1619677448.376505:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677448.376506:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.376506:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.376507:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:6.0:1619677448.376508:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.376508:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.376508:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:6.0:1619677448.376509:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.376510:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:9.2:1619677448.376513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677448.376513:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.376515:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.376517:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.376518:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.376519:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.376521:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677448.376521:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.376521:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.376521:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.376522:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677448.376524:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.376525:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.376525:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.376527:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.376527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.376530:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677448.376536:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:4.2:1619677448.376537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677448.376539:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.376540:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.376542:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.376542:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.376544:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:26.0:1619677448.376545:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.376547:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.376548:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.376550:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.376553:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.376555:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.376556:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715760] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.376558:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.376558:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.376560:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.376563:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.376565:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.376568:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.376569:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.376569:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.376571:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.376571:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.376572:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.376572:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.376577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.376584:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.376587:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.376588:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.376588:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.376590:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.376593:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.376595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:5.2:1619677448.376600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:6.0:1619677448.376601:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.376604:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.376605:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.376605:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.376606:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.376607:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.376607:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:10.0:1619677448.376610:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.376611:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.376612:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.376616:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.376617:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.376618:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.376620:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.376621:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.376622:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.376625:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.376627:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.376629:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.376630:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.376631:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.376633:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.376633:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.376634:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.376636:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.376636:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677448.376639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.376641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.376646:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:21.2:1619677448.376648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677448.376649:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:2.0:1619677448.376649:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.376650:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.376650:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.376651:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.376652:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.376653:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:2.0:1619677448.376653:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.376655:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677448.376657:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:9.2:1619677448.376657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:30.0:1619677448.376660:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.376662:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.376664:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677448.376664:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677448.376665:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.376666:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:18.0:1619677448.376667:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.376667:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:30.0:1619677448.376668:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.376668:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000400:00000200:30.0:1619677448.376669:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.376670:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.376670:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.376670:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:30.0:1619677448.376671:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.376674:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.376676:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.376678:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.376679:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.376679:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.376681:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.376681:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.376683:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.376684:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.376685:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.376686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.376691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.376693:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.376696:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.376696:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.376698:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:12.0:1619677448.376698:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.376700:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:13.0:1619677448.376700:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.376700:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.376701:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.376702:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677448.376845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.0:1619677448.376849:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.376851:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677448.376853:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.376854:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.376856:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.376857:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.376859:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.376860:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.376861:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.376862:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.376865:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.376867:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.376869:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.376870:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.376871:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.376872:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.376873:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.376874:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677448.376876:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:5.0:1619677448.376876:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:8.2:1619677448.376878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:21.2:1619677448.376881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.376885:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.376886:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677448.376886:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.376887:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000400:00000200:5.0:1619677448.376887:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.376888:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.376888:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.376889:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.376890:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.0:1619677448.376891:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.2:1619677448.376900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.376906:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.376909:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.376910:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.376911:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.376914:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.376915:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.376916:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.376918:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.376919:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.376920:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.376923:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.376925:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.376927:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.376928:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.376929:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.376931:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.376931:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.376932:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.376932:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:21.2:1619677448.376935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:8.2:1619677448.376937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.376940:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.376943:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.376943:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.376945:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.376945:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:18.0:1619677448.376947:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.376947:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:18.0:1619677448.376948:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677448.376948:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.376949:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.376949:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.376950:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.376951:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.376952:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.376953:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:1.0:1619677448.376954:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:18.0:1619677448.376956:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.376957:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.376960:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.376961:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.376961:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.376963:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.376963:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.376965:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.376966:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.376967:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.376969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.376972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.376975:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:2.0:1619677448.376977:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:10.0:1619677448.376978:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.376979:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.376979:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.376979:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.376980:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677448.376981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:10.0:1619677448.376981:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.376981:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:21.2:1619677448.376982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677448.376986:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.376987:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:31.0:1619677448.376988:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.376989:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:31.0:1619677448.376991:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:26.0:1619677448.376991:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677448.376993:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.376994:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.376995:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.376996:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.376996:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.376997:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:31.0:1619677448.376998:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.376998:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.376999:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.377000:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.377001:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.377001:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.377002:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.377002:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677448.377003:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.377008:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.377009:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.377009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677448.377011:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.377012:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.377014:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.377015:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.377015:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.377015:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.377016:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.377016:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.377016:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.377017:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677448.377018:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.377018:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.377018:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.377018:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.377019:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:31.0:1619677448.377020:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:17.0:1619677448.377020:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.377020:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677448.377021:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:31.0:1619677448.377022:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.377022:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.377022:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:25.2:1619677448.377023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:17.0:1619677448.377023:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.377024:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:8.2:1619677448.377024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:17.0:1619677448.377026:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96715752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.377027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:20.2:1619677448.377028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.377028:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.377029:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.377030:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.377030:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.377032:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.377033:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.377033:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:26.0:1619677448.377034:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:17.0:1619677448.377034:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.377034:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:12.0:1619677448.377035:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.377035:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.377036:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:13.0:1619677448.377036:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.377037:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677448.377037:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:17.0:1619677448.377037:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.377038:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.377038:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:1.0:1619677448.377038:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.377039:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.377040:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677448.377040:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.377041:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:17.0:1619677448.377041:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.377041:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:25.2:1619677448.377042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.377042:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.377042:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677448.377045:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:17.0:1619677448.377046:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.377047:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.377047:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.377048:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.377052:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.377053:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677448.377053:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677448.377054:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.377055:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.377058:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.377059:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.377061:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.377062:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.377063:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.377221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.377228:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.377231:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.377232:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.377234:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.377237:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.377239:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.377240:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.377241:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.377242:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.377243:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:4.2:1619677448.377246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:26.0:1619677448.377247:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.377249:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.377253:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.377254:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.377254:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.377255:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.377255:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.377257:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:10.0:1619677448.377257:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.377258:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.377259:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.377259:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.377260:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677448.377261:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.377261:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.377262:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.377263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677448.377264:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.377265:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.377265:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.377269:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.377270:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.377271:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.377273:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.377274:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:10.0:1619677448.377274:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.377274:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:18.0:1619677448.377275:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.377275:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:10.0:1619677448.377276:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.377276:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.377277:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:18.0:1619677448.377278:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.377278:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.377278:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677448.377281:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:25.2:1619677448.377282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:4.2:1619677448.377284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.377289:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:26.0:1619677448.377291:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.377292:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.377293:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.377297:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677448.377299:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.377301:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.377301:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.377302:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.377304:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.377338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f772e00] (20)++
00000800:00000200:10.0:1619677448.377346:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f772e00] (21)++
00000800:00000200:10.0:1619677448.377350:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.18@o2ib3
00000800:00000200:26.0:1619677448.377351:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f772e00] (22)--
00000400:00000200:10.0:1619677448.377353:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.12.18@o2ib3 : GET - for me
00000400:00000200:10.0:1619677448.377356:0:19940:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.12.18@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:10.0:1619677448.377358:0:19940:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.12.18@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:10.0:1619677448.377360:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f772e00] (21)++
00000800:00000200:10.0:1619677448.377361:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f772e00] (22)--
00000400:00000200:10.0:1619677448.377362:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.12.18@o2ib3 local destination
00000400:00000200:10.0:1619677448.377366:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.12.18@o2ib3(10.51.12.18@o2ib3:10.51.12.18@o2ib3) : REPLY try# 0
00000800:00000200:10.0:1619677448.377368:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.12.18@o2ib3
00000800:00000200:10.0:1619677448.377369:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3c4599280] -> 10.51.12.18@o2ib3 (2) version: 12
00000800:00000200:10.0:1619677448.377370:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a07f772e00] (21)++
00000800:00000200:10.0:1619677448.377371:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f772e00] (22)++
00000800:00000200:10.0:1619677448.377372:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a07f772e00] (23)--
00000800:00000200:10.0:1619677448.377372:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f772e00] (22)--
00000800:00000200:5.2:1619677448.377378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f772e00] (21)++
00000800:00000200:26.0:1619677448.377385:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f772e00] (22)++
00000800:00000200:26.0:1619677448.377387:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f772e00] (23)--
00000400:00000200:26.0:1619677448.377388:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.377389:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.12.18@o2ib3: REPLY: OK
00000800:00000200:10.0:1619677448.377389:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f772e00] (22)--
00000800:00000200:26.0:1619677448.377391:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f772e00] (21)--
00000800:00000200:20.2:1619677448.377443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.377451:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.377454:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.377456:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.377458:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.377459:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.377459:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.377460:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.377461:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.377462:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.377463:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.377466:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.377468:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.377469:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.377470:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.377471:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.377472:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.377473:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.377474:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.377474:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.377478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.377485:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.377488:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.377488:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.377490:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.377491:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.377492:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.377503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.377509:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.377512:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.377514:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.377514:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.377516:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.377517:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.377519:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.377520:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.377521:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.377522:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.377525:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.377526:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.377529:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.377530:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.377530:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.377532:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.377532:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.377533:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.377534:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677448.377539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.377547:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.2:1619677448.377549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677448.377550:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677448.377551:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677448.377551:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.377551:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.377553:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.377554:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:8.2:1619677448.377555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677448.377557:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.377557:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.377559:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.377559:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:20.0:1619677448.377561:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:1.0:1619677448.377561:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677448.377562:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715747] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.377563:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.377564:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.377564:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.377564:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000400:00000200:20.0:1619677448.377565:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.377565:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.377565:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:20.0:1619677448.377566:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.377569:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.377571:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.377575:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677448.377575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.0:1619677448.377577:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.377578:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.377580:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.377581:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.377582:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:8.0:1619677448.377582:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677448.377583:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.377584:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.377586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:24.0:1619677448.377586:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.377586:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:20.2:1619677448.377588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:8.0:1619677448.377588:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.377591:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.377592:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:8.0:1619677448.377593:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.377594:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96715746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.377594:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677448.377595:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:8.0:1619677448.377595:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.377596:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.377596:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.377596:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.377597:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677448.377598:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.377598:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.377598:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.377599:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.377601:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:8.0:1619677448.377601:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.377603:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.377606:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.377607:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.377607:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.377609:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.377609:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.377610:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.377611:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.377615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.377623:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.377626:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.377627:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.377627:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.377630:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.377632:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.377637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.377645:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.377648:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.377650:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.377652:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.377652:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.377654:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.377655:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.377656:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.377657:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.377658:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.377661:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.377662:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.377665:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.377665:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.377666:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.377667:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.377668:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.377669:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.377669:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.377673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.377679:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.377682:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.377683:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.377683:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.377685:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.377687:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.377701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.377707:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.377710:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.377711:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.377713:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.377715:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.377716:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.377717:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.377719:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.377720:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.377721:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:4.2:1619677448.377721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:6.0:1619677448.377724:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.377726:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.377728:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.377729:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.377730:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:26.0:1619677448.377731:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.377731:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677448.377731:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.377733:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.377733:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:26.0:1619677448.377734:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.377734:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.377735:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000400:00000200:26.0:1619677448.377736:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.377736:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:26.0:1619677448.377738:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:25.2:1619677448.377738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000400:00000200:26.0:1619677448.377739:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.377741:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.377741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.2:1619677448.377741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000400:00000200:26.0:1619677448.377742:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.377743:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.377745:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:22.0:1619677448.377747:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:10.0:1619677448.377747:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:1.0:1619677448.377747:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000400:00000200:26.0:1619677448.377748:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.377748:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:10.0:1619677448.377748:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.377749:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677448.377750:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:22.0:1619677448.377750:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677448.377750:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.377750:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677448.377751:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677448.377753:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.377754:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.377755:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.377755:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:1.0:1619677448.377755:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.377756:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.377756:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:1.0:1619677448.377757:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:26.0:1619677448.377758:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:25.2:1619677448.377758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000400:00000200:1.0:1619677448.377758:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.377759:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677448.377759:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.377760:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677448.377760:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.377761:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:1.0:1619677448.377761:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.377762:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.377764:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:8.2:1619677448.377764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.377765:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677448.377765:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.377766:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.377766:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.377767:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:1.0:1619677448.377767:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677448.377770:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.377771:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)++
00000800:00000200:1.0:1619677448.377772:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (23)++
00000800:00000200:5.0:1619677448.377773:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (24)++
00000800:00000200:1.0:1619677448.377774:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (25)--
00000800:00000200:5.0:1619677448.377775:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (24)--
00000800:00000200:1.0:1619677448.377775:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000400:00000200:5.0:1619677448.377776:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.377776:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677448.377777:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000400:00000200:5.0:1619677448.377778:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.377778:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:20.0:1619677448.377779:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (23)++
00000800:00000200:20.0:1619677448.377780:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (24)--
00000400:00000200:20.0:1619677448.377781:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.377781:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.377783:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.377783:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:20.0:1619677448.377786:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.377787:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677448.377789:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677448.377791:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677448.377792:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677448.377792:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677448.377799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.377806:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.377809:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.377810:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.377810:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.377811:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.377812:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.377896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.377904:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.377906:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.377907:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.377910:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.377912:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.377914:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.377915:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.377916:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.377917:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.377918:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.377921:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.377923:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.377926:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.377927:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.377927:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.377929:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.377929:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.377930:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.377931:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.377935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.377942:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:21.2:1619677448.377942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:22.0:1619677448.377945:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.377946:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.377946:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:5.0:1619677448.377946:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.377948:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:22.0:1619677448.377949:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.377951:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.377951:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.377952:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:5.0:1619677448.377953:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.377955:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.377956:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.377957:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.377958:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.377959:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.377963:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.377965:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.377967:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.377968:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.377968:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.377970:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.377970:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.377971:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.377972:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.2:1619677448.377974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.2:1619677448.377975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.2:1619677448.377976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:17.0:1619677448.377980:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.377983:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.377983:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:17.0:1619677448.377983:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.377984:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.377986:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.377986:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.377986:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.377987:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:20.0:1619677448.377989:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.377989:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.377989:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:31.0:1619677448.377990:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.377990:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:17.0:1619677448.377990:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:20.0:1619677448.377991:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:31.0:1619677448.377992:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.377992:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.377993:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.377994:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:17.0:1619677448.377995:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.377996:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.377996:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677448.377998:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.377999:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.378000:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.378001:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:17.0:1619677448.378003:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:31.0:1619677448.378005:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.378006:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.378006:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677448.378007:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:17.0:1619677448.378007:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.378009:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.378009:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.378009:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.378010:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:17.0:1619677448.378010:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.378011:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:17.0:1619677448.378011:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.378012:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.378013:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.378014:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.378015:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.378015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:25.2:1619677448.378019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:6.0:1619677448.378022:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:30.0:1619677448.378025:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:22.0:1619677448.378025:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.378025:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.378026:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.378028:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.378028:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.378028:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.378030:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.378030:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677448.378031:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.378033:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.378054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.378062:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.378064:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.378065:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.378067:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.378070:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.378071:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.378073:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715737] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.378074:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.378075:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.378076:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.378079:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.378081:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.378083:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.378084:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.378084:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.378086:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.378086:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.378087:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.378087:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.378091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.378100:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.378102:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.378103:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.378104:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.378107:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.378107:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:4.2:1619677448.378144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.378151:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.378153:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.378154:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.378156:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.378159:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.378160:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.378161:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.378162:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.378163:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.378164:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.378167:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.378169:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.378172:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.378173:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.378173:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.378175:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.378175:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.378176:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.378176:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.378192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.378200:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.378202:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.378203:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.378204:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.378206:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.378209:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.378212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.378219:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.378221:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.378222:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.378223:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.378224:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.378224:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.378422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.378427:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.378430:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.378430:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.378432:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.378435:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.378436:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.378438:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.378439:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.378440:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.378441:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.378445:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.378446:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.378449:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.378450:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.378450:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.378452:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.378452:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.378453:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.378453:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.378459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.378467:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.378470:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.378471:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.378472:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.378473:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.378476:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.378481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.378487:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.378490:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.378491:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.378491:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.378492:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.378492:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.378553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.378561:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.378564:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.378566:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.378568:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.378569:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.378570:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.378572:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.378573:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.378574:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.378575:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.378578:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.378580:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:4.2:1619677448.378581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.378584:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.378584:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.378585:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.378587:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.378587:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.378588:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.378589:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:10.0:1619677448.378589:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.378592:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.378593:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.378594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:10.0:1619677448.378595:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.378597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000400:00000200:10.0:1619677448.378598:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.378599:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.378600:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.378601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:10.0:1619677448.378601:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.378602:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.378603:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677448.378605:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000400:00000200:10.0:1619677448.378606:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.378607:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:10.0:1619677448.378608:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677448.378608:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:30.0:1619677448.378609:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677448.378609:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:10.0:1619677448.378610:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.378610:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:30.0:1619677448.378611:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:10.0:1619677448.378611:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:8.0:1619677448.378611:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677448.378611:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.378612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677448.378612:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:5.0:1619677448.378612:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.378613:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:8.0:1619677448.378613:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.378614:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.378614:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.378615:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:8.0:1619677448.378615:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:5.0:1619677448.378615:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.378616:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677448.378616:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:8.0:1619677448.378616:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96715732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.378617:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.378617:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:8.0:1619677448.378617:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.378618:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.378618:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677448.378619:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:25.2:1619677448.378619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:8.0:1619677448.378619:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.378620:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.378621:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:30.0:1619677448.378622:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.378623:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677448.378624:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.378624:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.378625:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677448.378625:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:5.0:1619677448.378626:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.378627:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677448.378628:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.378628:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.378629:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.378630:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:5.0:1619677448.378630:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96715730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.378631:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.378631:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677448.378631:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.378631:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)++
00000400:00000200:6.0:1619677448.378631:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.378631:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.378632:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (22)++
00000400:00000200:5.0:1619677448.378632:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.378633:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.378633:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.378634:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.378634:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (23)--
00000800:00000200:8.0:1619677448.378634:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:30.0:1619677448.378635:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:8.0:1619677448.378635:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:30.0:1619677448.378636:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677448.378636:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000400:00000200:5.0:1619677448.378637:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.378638:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.378638:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.378639:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.0:1619677448.378639:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.378640:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.378640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.378642:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.378642:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (23)++
00000800:00000200:6.0:1619677448.378643:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (24)--
00000800:00000200:5.0:1619677448.378643:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (24)++
00000800:00000200:9.2:1619677448.378644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.378645:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (24)++
00000800:00000200:5.0:1619677448.378646:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (24)--
00000800:00000200:5.0:1619677448.378646:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677448.378647:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (24)--
00000800:00000200:5.0:1619677448.378647:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677448.378648:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:5.0:1619677448.378648:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:6.0:1619677448.378649:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.378651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (22)++
00000400:00000200:6.0:1619677448.378651:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.378652:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:6.0:1619677448.378655:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677448.378655:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.378656:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.378656:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:2.0:1619677448.378656:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.378658:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.378659:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.378659:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.378660:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.378661:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.378662:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:6.0:1619677448.378663:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.378906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.2:1619677448.378912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:1.0:1619677448.378912:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.378915:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.378916:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.378917:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:1.0:1619677448.378918:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.378919:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:1.0:1619677448.378921:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.378922:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.378922:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:5.0:1619677448.378923:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.378924:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.378924:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.378925:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.378926:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:1.0:1619677448.378926:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.378928:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.378928:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677448.378930:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.378931:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.378932:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:1.0:1619677448.378932:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.378934:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:20.0:1619677448.378936:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.378936:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.378938:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.378938:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.378939:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:1.0:1619677448.378940:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.378940:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.378941:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.378941:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.378942:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:1.0:1619677448.378942:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677448.378943:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.378944:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.378945:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.378946:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.378946:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.378946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.2:1619677448.378950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:17.0:1619677448.378953:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.378955:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.378956:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.378957:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.378958:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.378958:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:17.0:1619677448.378960:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.0:1619677448.378960:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.378961:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.378963:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.378963:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.378965:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:8.2:1619677448.378966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.2:1619677448.378970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000800:00000200:1.0:1619677448.378974:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.378977:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.378977:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:20.0:1619677448.378978:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:1.0:1619677448.378978:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.378979:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.378979:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.0:1619677448.378981:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.378981:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.378982:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.378983:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.378984:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:4.2:1619677448.379052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.379059:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.379062:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.379062:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.379065:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.379067:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.379069:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.379070:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.379072:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.379073:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.379074:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.379077:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.379079:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.379082:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.379083:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.379084:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.379085:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.379086:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.379087:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.379088:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.379089:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.379091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.379095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.379100:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.379102:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:10.0:1619677448.379102:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:20.0:1619677448.379103:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.379104:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.379104:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677448.379105:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.379105:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.379106:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677448.379108:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.379164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.379170:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.379173:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.379174:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.379176:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.379178:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.379180:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677448.379181:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.379182:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.379183:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.379184:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677448.379187:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.379189:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.379192:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.379193:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677448.379193:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677448.379195:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.379196:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.379197:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.379198:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.379198:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677448.379201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:21.2:1619677448.379204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677448.379209:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677448.379210:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.379211:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:22.0:1619677448.379212:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677448.379212:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:22.0:1619677448.379213:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.379213:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677448.379213:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:22.0:1619677448.379215:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.379217:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.379227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:6.0:1619677448.379234:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.379237:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.379238:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.379238:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.379239:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.379239:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.379273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.379280:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.379283:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.379284:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.379286:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.379289:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.379290:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.379292:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.379293:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.379298:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.379299:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:4.2:1619677448.379301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:26.0:1619677448.379303:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.379305:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.379307:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.379308:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.379309:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:24.0:1619677448.379309:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677448.379310:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.379311:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.379312:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.379312:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677448.379312:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.379313:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.379315:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.379317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:24.0:1619677448.379318:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.379319:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.379320:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96715724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.379322:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.379323:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.379324:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:24.0:1619677448.379324:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.379327:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:24.0:1619677448.379327:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.379328:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.379328:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:31.0:1619677448.379329:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.379331:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.379332:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:24.0:1619677448.379332:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:31.0:1619677448.379333:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:24.0:1619677448.379333:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.379334:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.379335:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.379336:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.379336:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.379340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.379348:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.379351:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.379352:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.379352:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.379355:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.379357:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.379414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:20.2:1619677448.379418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:22.0:1619677448.379420:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.379423:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.379424:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677448.379425:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:22.0:1619677448.379426:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.379428:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:22.0:1619677448.379428:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.379430:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:31.0:1619677448.379432:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.379432:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.379432:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.379434:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.379434:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.379435:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.379436:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677448.379437:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.379438:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715722] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.379440:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.379441:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.379442:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.379447:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.379448:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.379449:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.379451:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.379453:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.379454:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.379454:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.379455:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.379455:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.379456:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.379456:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.379457:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.379457:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677448.379458:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.379458:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:22.0:1619677448.379458:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677448.379460:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.379461:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.379462:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.379463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.2:1619677448.379464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.379467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.379472:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.379472:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677448.379474:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:5.0:1619677448.379474:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:31.0:1619677448.379475:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.379475:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.379476:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:31.0:1619677448.379477:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.379477:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.379477:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677448.379477:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.2:1619677448.379477:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.379480:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.379481:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:5.0:1619677448.379481:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.379482:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.379482:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.379482:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.379483:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:1.0:1619677448.379483:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.379484:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.379484:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677448.379485:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:1.0:1619677448.379486:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.379487:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.379489:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.379490:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.379492:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.379493:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.379494:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.379496:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.379497:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.379500:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.379502:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.379504:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.379504:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.379505:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.379506:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.379507:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.379508:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.379509:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.379509:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.379512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.379515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.379521:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.379521:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.379521:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:17.0:1619677448.379522:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.379522:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.379523:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.379524:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.379524:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.379525:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677448.379525:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:21.2:1619677448.379608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.379615:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.379618:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.379620:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.379622:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.379623:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.379624:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.379625:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.379627:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.379628:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.379629:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.379631:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.379633:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.379636:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.379636:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.379637:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.379639:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.379639:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.379640:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.379641:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.379644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.379651:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.379654:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.379654:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.379656:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.379658:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.379659:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.379664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.379670:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.379673:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.379674:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.379674:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.379675:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.379676:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.379694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.379696:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.379699:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.379701:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.379702:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.379703:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.379705:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.379706:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.379707:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.379708:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.379709:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.379712:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.379714:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.379717:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.379718:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.379718:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.379720:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.379721:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.379722:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.379722:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.379726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.379733:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.379735:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.379736:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.379738:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.379739:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.379740:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.379795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.379800:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.379803:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.379805:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.379806:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.379808:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.379810:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677448.379811:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.379813:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.379813:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.379815:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.379818:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.379820:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.379823:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.379824:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677448.379824:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.379827:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.379827:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.379829:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.379830:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.379831:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.379832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.2:1619677448.379836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.379838:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:10.0:1619677448.379839:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:20.0:1619677448.379841:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.379841:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:26.0:1619677448.379842:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.379842:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:20.0:1619677448.379843:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:10.0:1619677448.379843:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.379845:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.379848:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.379852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.379858:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.379861:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.379862:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.379862:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.379863:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.379863:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.379877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:5.2:1619677448.379877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.379883:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.379884:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:1.0:1619677448.379886:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.379887:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.379887:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.379888:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:1.0:1619677448.379889:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.379891:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.379891:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.379893:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.379894:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.379894:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.379895:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677448.379896:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.379897:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.379897:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.379898:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.379899:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.379899:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.379900:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:5.2:1619677448.379900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:1.0:1619677448.379902:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.379904:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.379905:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.379906:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.379907:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:18.0:1619677448.379908:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:10.0:1619677448.379908:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.379908:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.379909:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.379910:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.379910:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.379911:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:10.0:1619677448.379911:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.379911:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.379912:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.379912:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:10.0:1619677448.379913:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677448.379915:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.379915:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.379915:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:8.2:1619677448.379915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.379916:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.379917:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:18.0:1619677448.379918:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.379919:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.379921:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96715715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.379921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677448.379922:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.379922:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:18.0:1619677448.379923:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.379924:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:18.0:1619677448.379925:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677448.379925:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.379926:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.379927:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.379929:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:18.0:1619677448.379931:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.379931:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.379932:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:21.2:1619677448.379934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.379934:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.379934:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.379936:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.379937:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.379938:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.379938:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.379939:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.379940:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.379941:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.379941:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677448.379942:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.379942:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.379943:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677448.379943:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.379944:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.379945:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.379946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:2.0:1619677448.379948:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.379949:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.379950:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.379951:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.379953:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:2.0:1619677448.379953:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.379953:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.379956:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.379956:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.379956:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.379957:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.379958:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.379959:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.379960:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.379960:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.379961:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677448.379961:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.379962:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.2:1619677448.379963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:2.0:1619677448.379963:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.379964:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.379964:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677448.379967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:9.2:1619677448.379968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.379971:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677448.379974:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000800:00000200:5.0:1619677448.379974:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:31.0:1619677448.379976:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.379976:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677448.379977:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.379978:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.379978:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:18.0:1619677448.379978:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.379979:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.379979:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.379979:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:2.0:1619677448.379979:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.379980:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:31.0:1619677448.379981:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:20.0:1619677448.379981:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:31.0:1619677448.379982:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.379982:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.379984:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.379984:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:31.0:1619677448.379985:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.379986:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.379990:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.379991:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.379995:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.379996:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.379996:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.379998:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.379999:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677448.380000:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:31.0:1619677448.380000:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:8.2:1619677448.380004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:4.2:1619677448.380009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677448.380011:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.380014:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.380015:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.380016:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.380017:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.380017:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.380019:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.380019:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.380020:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:26.0:1619677448.380021:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.380024:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.380026:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.380027:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715712] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.380029:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.380030:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.380031:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.380034:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.380035:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.380038:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.380039:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:21.2:1619677448.380039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:26.0:1619677448.380040:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.380042:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.380042:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.380043:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.380044:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677448.380044:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.380046:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.380048:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.380048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.380049:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.380051:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.380053:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677448.380054:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.380055:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.380055:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:20.0:1619677448.380056:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.380057:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:2.0:1619677448.380058:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.380059:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.380059:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.380061:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.380061:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.380064:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 440 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677448.380065:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677448.380067:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677448.380067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.380068:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677448.380068:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.380070:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677448.380070:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.380071:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.380072:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:18.0:1619677448.380075:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:25.2:1619677448.380076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:18.0:1619677448.380078:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.380078:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.380078:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.380080:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.380080:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677448.380083:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.380086:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.380087:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.380089:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677448.380089:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.380092:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.380099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.380106:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.380109:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.380110:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.380110:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.380110:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.380112:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.380152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.380158:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.380160:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.380161:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.380163:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.380165:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.380167:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.380168:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.380169:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.380170:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.380171:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.380174:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.380176:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.380179:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.380180:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.380181:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.380182:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.380183:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.380184:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.380184:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.380188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.380195:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.380198:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.380199:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.380200:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.380201:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.380203:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.380217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.380224:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.380227:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.380229:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.380230:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.380231:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.380233:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.380234:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.380235:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.380236:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.380238:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.380241:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.380243:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.2:1619677448.380246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677448.380247:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.380247:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.380248:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.380250:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.380250:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.380252:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.380253:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.380254:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677448.380254:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:25.2:1619677448.380256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.380256:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.380257:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:21.2:1619677448.380259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677448.380259:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.380262:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.380264:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:6.0:1619677448.380264:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:10.0:1619677448.380265:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.380266:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.380266:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677448.380267:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677448.380267:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:10.0:1619677448.380267:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.380268:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:6.0:1619677448.380268:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.380269:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.380270:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.380271:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:6.0:1619677448.380271:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.380272:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.380275:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677448.380275:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.380277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.380277:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.380277:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.380278:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.380279:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.380280:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.380281:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.380282:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.380283:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677448.380284:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:25.2:1619677448.380285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:22.0:1619677448.380287:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.380288:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.380288:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.2:1619677448.380288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677448.380289:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.380290:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677448.380292:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:26.0:1619677448.380299:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:31.0:1619677448.380300:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.380301:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.380301:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.380302:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.380302:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:31.0:1619677448.380303:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.380303:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677448.380305:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.380397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.380405:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.380408:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.380411:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.380412:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.380414:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.380415:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.380416:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.380418:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.380419:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.380420:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.380423:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.380424:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.380427:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.380428:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.380429:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.380430:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.380431:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.380432:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.380432:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.380437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.380446:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.380448:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.380449:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.380451:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.380451:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.380454:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.380490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.380496:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.380498:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.380499:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.380501:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.380504:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.380505:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.380507:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.380508:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.380509:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.380510:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.380514:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.380515:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.380519:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.380519:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.380520:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.380522:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.380523:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.380523:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.380524:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.380529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.380536:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.380539:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.380540:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.380541:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.380542:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.380544:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.380692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.380699:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.380702:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.380702:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677448.380705:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.380708:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.380709:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.380710:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.380711:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.380713:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.380714:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.380717:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.380718:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.380721:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.380722:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.380723:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.380725:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.380725:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.380726:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.380728:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.380728:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677448.380731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.380734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:2.0:1619677448.380739:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677448.380740:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:2.0:1619677448.380742:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:24.0:1619677448.380743:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:18.0:1619677448.380743:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.380743:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.380744:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.380745:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:2.0:1619677448.380746:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.380749:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.380755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677448.380758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677448.380760:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.380763:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.380764:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677448.380764:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677448.380766:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.380766:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:18.0:1619677448.380767:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.380767:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677448.380769:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.380770:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.380770:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.380772:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.380772:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.380773:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.380774:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.380775:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.380776:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.380778:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.380779:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.380779:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.380780:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.380781:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:21.2:1619677448.380782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:1.0:1619677448.380784:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.380785:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.380786:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.380786:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.380786:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.380788:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.380789:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.380789:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.380789:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677448.380789:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.380790:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677448.380790:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.380790:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.380792:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:1.0:1619677448.380792:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677448.380793:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.380793:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:30.0:1619677448.380794:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.380794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.380794:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.380795:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.380796:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.380797:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.380798:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.380800:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96715702] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.380800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677448.380801:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.380801:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000400:00000200:30.0:1619677448.380802:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.380803:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.380804:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:17.0:1619677448.380805:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:12.0:1619677448.380805:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.380807:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.380807:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:12.0:1619677448.380807:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.380808:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.380808:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:30.0:1619677448.380809:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.380809:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.380810:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677448.380811:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.380811:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:30.0:1619677448.380812:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.380812:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.380813:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.380813:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:13.0:1619677448.380813:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.380815:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.380815:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.380816:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.380816:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.380817:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.380820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.380826:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.380829:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.380829:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.380831:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.380833:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.380834:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.380841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.380847:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.380850:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.380851:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.380852:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.380852:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.380854:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.380976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fdae00] (20)++
00000800:00000200:20.0:1619677448.380981:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fdae00] (21)++
00000800:00000200:20.0:1619677448.380983:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.38@o2ib3
00000400:00000200:20.0:1619677448.380985:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.38@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.380986:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (22)--
00000400:00000200:20.0:1619677448.380988:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.380988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:20.0:1619677448.380990:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.380991:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.380993:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.380994:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.380995:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677448.380995:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.380998:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.380998:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:20.0:1619677448.380999:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.38@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.381001:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:8.0:1619677448.381002:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.381004:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.381005:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.381005:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:8.0:1619677448.381006:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.381007:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:8.0:1619677448.381007:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.0:1619677448.381008:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fdae00] (21)++
00000800:00000200:20.0:1619677448.381009:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fdae00] (22)++
00000400:00000200:8.0:1619677448.381009:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96715700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.381010:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fdae00] (23)--
00000400:00000200:8.0:1619677448.381010:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.381011:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (22)--
00000400:00000200:8.0:1619677448.381011:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.381012:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:25.2:1619677448.381013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.381015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fdae00] (21)++
00000400:00000200:8.0:1619677448.381015:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.381017:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.381019:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fdae00] (22)++
00000800:00000200:8.0:1619677448.381020:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.381021:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.381021:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677448.381022:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.381022:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fdae00] (23)--
00000800:00000200:8.0:1619677448.381023:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.381023:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677448.381023:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (22)--
00000800:00000200:5.0:1619677448.381023:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (21)--
00000800:00000200:22.0:1619677448.381024:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677448.381024:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:6.0:1619677448.381024:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.381025:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.381026:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.381026:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677448.381026:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:8.2:1619677448.381029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.381029:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.381032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.381037:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.381039:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.381039:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.381040:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.381040:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.381041:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.381041:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.381042:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.381042:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.381045:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.381069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:5.2:1619677448.381069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.381074:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.381076:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:17.0:1619677448.381077:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.381078:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.381079:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.381080:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:17.0:1619677448.381081:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.381082:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677448.381083:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.381085:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.381085:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.381086:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.381087:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677448.381088:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.381088:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.381089:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.381090:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.381090:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.381092:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.381094:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.381094:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.381097:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.381098:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677448.381100:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.381101:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.381101:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.381102:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.381104:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.381104:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.381104:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.381104:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.381105:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.381105:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.381106:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.381108:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.381108:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.381109:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.381109:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677448.381110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.2:1619677448.381114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677448.381116:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.381119:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.381120:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.381121:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.381122:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:13.0:1619677448.381123:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.381124:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677448.381124:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.381125:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.381127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:20.0:1619677448.381127:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.381127:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.381130:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:13.0:1619677448.381134:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.381137:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.381137:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.381138:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.381139:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.381140:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.381177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.381182:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.381185:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.381187:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.381188:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.381190:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.381191:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.381192:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.381194:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.381195:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.381196:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.381199:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.381201:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.381204:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.381205:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.381205:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.381207:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.381207:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.381208:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677448.381210:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677448.381210:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:8.2:1619677448.381213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:21.2:1619677448.381216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.381221:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.381223:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677448.381223:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.381224:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677448.381224:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.381224:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.381225:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.381226:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.381227:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:5.0:1619677448.381228:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.2:1619677448.381250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.381257:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.381260:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.381261:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.381263:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.381266:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.381267:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.381269:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.381270:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.381271:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.381272:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.381275:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.381277:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.381280:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.381280:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.381281:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.381283:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.381283:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.381284:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.381284:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.381290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.381292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677448.381299:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.381299:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.381302:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677448.381302:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.381303:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677448.381303:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.381305:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.381305:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.381307:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.381308:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677448.381308:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.381310:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.381311:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.381312:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.381313:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.381314:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.381318:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.381319:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.381322:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.381323:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.381324:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.381326:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.381326:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.381327:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.381328:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.381332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.381339:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.381342:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.381343:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.381345:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.381346:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.381347:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.381400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677448.381408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:10.0:1619677448.381408:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.381410:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.381411:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.381413:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.381414:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:10.0:1619677448.381415:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.381417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:10.0:1619677448.381417:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:2.0:1619677448.381417:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.381418:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:10.0:1619677448.381418:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.381421:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.381421:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.381422:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.381423:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677448.381423:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.381424:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.381426:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:2.0:1619677448.381426:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:6.0:1619677448.381427:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677448.381428:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.381429:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.381430:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.381431:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.381432:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.381432:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:22.0:1619677448.381434:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.381436:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.381437:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.381438:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:22.0:1619677448.381439:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.381441:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96715692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.381441:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.381442:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.381442:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.381442:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:22.0:1619677448.381443:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.381443:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:22.0:1619677448.381444:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.381444:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:22.0:1619677448.381445:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.381445:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.381445:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.381446:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.381446:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.381447:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:22.0:1619677448.381448:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.381448:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677448.381448:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:10.0:1619677448.381449:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.381450:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.2:1619677448.381451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.381453:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.381454:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.381455:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.381457:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:25.2:1619677448.381458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:22.0:1619677448.381458:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.381459:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.381460:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677448.381460:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.381461:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.381461:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.381463:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.381464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.381465:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:6.0:1619677448.381465:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.381467:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:6.0:1619677448.381467:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677448.381468:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.381469:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.381471:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.381471:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:26.0:1619677448.381474:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:17.0:1619677448.381474:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.381475:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.381475:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.381477:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.381480:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.381493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.381496:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.381499:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.381501:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.381502:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.381504:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.381505:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.381507:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.381508:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.381509:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.381510:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.2:1619677448.381512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:20.0:1619677448.381513:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.381515:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.381518:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.381518:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677448.381519:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.381520:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.381521:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:20.0:1619677448.381522:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.381522:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.381522:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677448.381523:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.381524:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677448.381524:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.381527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677448.381527:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.381528:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.381530:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.381531:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.381532:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.381533:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.381534:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.381536:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:18.0:1619677448.381537:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677448.381537:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.381539:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.381540:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.381540:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.381542:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.381542:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677448.381543:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.381544:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.381546:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.381546:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.381547:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.381549:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.381549:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:21.2:1619677448.381550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:25.2:1619677448.381551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.381555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:13.0:1619677448.381558:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677448.381559:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.381560:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.381560:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:13.0:1619677448.381561:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.381562:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:2.0:1619677448.381562:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:13.0:1619677448.381563:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.381563:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:31.0:1619677448.381564:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.381564:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.381564:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:30.0:1619677448.381566:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677448.381566:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677448.381567:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.381568:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.381569:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.381571:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.381572:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.381573:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.381576:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.381578:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.381580:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.381581:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.381582:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.381583:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.381584:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.381585:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.381585:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.381588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.381595:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.381597:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.381598:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.381599:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.381601:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.381602:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.381607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.381613:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.381616:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.381617:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.381617:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.381618:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.381619:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.381748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677448.381755:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677448.381758:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677448.381759:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677448.381760:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677448.381763:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.381764:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677448.381765:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715688] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.381767:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.381767:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.381769:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677448.381771:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677448.381773:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677448.381774:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.381775:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677448.381775:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.381777:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.381777:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677448.381778:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677448.381779:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677448.381783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.381789:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.381791:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.381791:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.381792:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.381793:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677448.381795:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.381953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.381960:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.381962:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.381964:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.381965:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.381968:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.381969:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.381970:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.381972:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.381972:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.381973:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.381976:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.381978:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.381981:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.381981:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:5.2:1619677448.381981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.381982:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.381984:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.381985:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.381986:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.381987:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.381988:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.381989:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.381989:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:25.2:1619677448.381990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:10.0:1619677448.381991:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.381993:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.381994:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.381995:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.381997:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.381997:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.381998:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:10.0:1619677448.381998:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.382001:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:10.0:1619677448.382001:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.382002:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.382002:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.382002:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.382004:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.382005:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.382005:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.382006:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.382007:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:10.0:1619677448.382007:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.382008:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.382009:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.382009:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.382013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.382019:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.382022:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.382022:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.382024:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.382026:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.382026:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:21.2:1619677448.382074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.382079:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.382082:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.382083:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.382085:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:20.2:1619677448.382086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:18.0:1619677448.382088:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.382089:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.382090:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.382092:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.382093:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:18.0:1619677448.382093:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.382094:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.382096:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:18.0:1619677448.382097:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000400:00000200:30.0:1619677448.382099:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.382099:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.382100:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.382101:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.382101:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.382102:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.382102:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677448.382103:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.382104:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.382104:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.382105:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.382105:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:30.0:1619677448.382106:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.382106:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.382106:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:30.0:1619677448.382108:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:9.2:1619677448.382110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677448.382111:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.382113:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.2:1619677448.382113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:30.0:1619677448.382115:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.382116:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.382117:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.382118:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:2.0:1619677448.382118:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:30.0:1619677448.382119:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.382119:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.382119:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:30.0:1619677448.382120:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.382120:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:2.0:1619677448.382120:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.382122:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.382122:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:2.0:1619677448.382123:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:25.2:1619677448.382124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.382124:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.382125:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.382126:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:5.0:1619677448.382128:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.382130:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:5.0:1619677448.382130:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.382131:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.382133:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:5.0:1619677448.382133:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.382134:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.382134:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.382135:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.382136:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.382137:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.382139:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:5.0:1619677448.382139:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.382141:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.382144:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.382145:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.382146:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.382148:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.382148:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.382150:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.382151:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.382152:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.382153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.382157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.382160:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677448.382160:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.382162:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.382163:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677448.382163:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677448.382164:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:12.0:1619677448.382164:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.382166:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.382167:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.382168:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.382260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.382267:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.382270:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.382271:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.382273:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.382275:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.382277:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.382278:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715682] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.382279:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.382280:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.382281:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.382284:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.382286:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.382289:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.382289:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.382290:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.382292:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.382292:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.382299:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:25.2:1619677448.382299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.382300:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.382300:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.382306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.382307:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.382310:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.382310:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.382311:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.382313:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.382314:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:5.2:1619677448.382315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.382316:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677448.382316:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677448.382317:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.382318:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677448.382320:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.382323:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.382324:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.382326:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.382329:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.382330:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.382331:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.382333:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.382333:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.382335:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.382338:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.382340:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:21.2:1619677448.382342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:1.0:1619677448.382343:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.382343:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.382344:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.382346:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.382346:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.382347:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677448.382347:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.382347:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.382350:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.382351:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:25.2:1619677448.382352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:22.0:1619677448.382352:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.382355:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.382356:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.382357:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.382359:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.382359:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:22.0:1619677448.382360:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.382361:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677448.382361:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:21.2:1619677448.382362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000400:00000200:13.0:1619677448.382362:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.382365:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677448.382365:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.382366:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677448.382367:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:22.0:1619677448.382368:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677448.382368:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677448.382369:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:22.0:1619677448.382370:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.382371:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.382372:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677448.382372:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.382373:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:22.0:1619677448.382374:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.382374:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:20.0:1619677448.382374:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.382376:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:20.0:1619677448.382376:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:22.0:1619677448.382377:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.382377:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677448.382377:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.382378:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.382379:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.382380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677448.382381:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:21.2:1619677448.382383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:20.0:1619677448.382384:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.382386:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.382388:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.382388:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677448.382389:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.382389:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:6.0:1619677448.382389:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:20.0:1619677448.382391:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.382391:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:17.0:1619677448.382391:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:6.0:1619677448.382391:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.382391:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677448.382392:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:6.0:1619677448.382392:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677448.382393:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:17.0:1619677448.382393:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.382393:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.382395:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.382397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:17.0:1619677448.382398:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.382401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.382404:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.382407:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.382407:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.382408:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677448.382409:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.382410:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:17.0:1619677448.382411:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.382411:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:1.0:1619677448.382411:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.382412:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.382413:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.382414:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.382447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.382455:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.382458:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.382460:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.382462:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.382463:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.382464:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.382466:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.382467:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.382468:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.382469:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.382472:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.382474:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.382477:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.382478:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.382479:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.382480:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.382481:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.382482:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.382482:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.382486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.382492:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.382494:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.382495:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.382496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:12.0:1619677448.382496:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.382498:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.382499:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677448.382502:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.382505:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.382506:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.382508:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.382510:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.382512:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.382513:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.382514:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.382515:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.382516:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.382519:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.382521:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.382524:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.382525:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.382525:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.382527:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.382527:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.382528:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.382529:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.382533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.382539:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.382542:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.382543:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.382544:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.382546:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.382549:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.382612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.382619:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.382622:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.382623:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.382625:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.382627:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.382628:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.382630:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.382631:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.382632:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.382633:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.382636:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.382638:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:4.2:1619677448.382639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677448.382641:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.382641:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.382642:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.382644:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.382644:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.382645:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.382645:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677448.382646:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.2:1619677448.382647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677448.382648:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.382649:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.382650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:10.0:1619677448.382651:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.382653:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.382654:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677448.382654:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.382655:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.382657:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:10.0:1619677448.382657:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.382658:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:10.0:1619677448.382658:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.382659:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.382659:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.382661:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677448.382661:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.382661:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:31.0:1619677448.382662:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.382662:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.382662:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.382663:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.382664:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.382666:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96715674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.382666:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.382666:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.382667:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.382667:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.382667:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:31.0:1619677448.382669:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.382669:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677448.382669:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.382669:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.382670:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.382670:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.382671:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:31.0:1619677448.382673:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.382674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.382675:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.382678:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.382679:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.382679:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.382680:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:31.0:1619677448.382681:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.382682:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.382682:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677448.382682:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:31.0:1619677448.382683:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:26.0:1619677448.382683:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.382683:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.382685:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.382687:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:8.2:1619677448.382687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.382695:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.382697:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.382698:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.382700:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.382701:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.382703:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.382722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.382727:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.382730:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.382732:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.382732:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.382734:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.382735:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.382737:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.382738:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.382739:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.382740:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.382744:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.382745:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.382749:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.382749:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.382750:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.382752:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.382752:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.382753:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.382754:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.382758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.382765:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.382768:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.382769:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.382771:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.382771:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.382775:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.382814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.382822:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.382824:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.382825:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.382827:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.382829:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.382830:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.382832:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.382833:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.382834:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.382835:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.382838:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.382840:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.382844:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.382844:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.382845:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.382847:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.382848:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.382848:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.382849:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.382853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.382859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.382864:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.382866:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.382867:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:2.0:1619677448.382867:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.382869:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.382869:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.382872:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.382873:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.382874:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:18.0:1619677448.382875:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.382875:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.382877:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.382878:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.382879:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.382881:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.382882:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.382885:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.382886:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.382889:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.382889:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.382890:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.382892:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.382892:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.382893:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.382893:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.382898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.382905:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.382908:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.382909:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.382909:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.382911:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.382913:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.382971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.382974:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.382977:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.382979:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.382980:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.382981:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.382983:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.382984:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.382985:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.382986:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.382987:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.382991:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.382992:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.382995:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.382996:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.382997:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.382998:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.382999:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.383000:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.383001:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.383004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.383012:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.383015:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.383016:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.383017:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:30.0:1619677448.383019:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.383021:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.383063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.383070:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.383073:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.383073:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.383076:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.383078:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.383080:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.383081:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.383082:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.383083:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.383084:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.383087:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.383088:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.383091:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.383091:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.383092:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.383094:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.383094:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.383095:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.383095:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.383099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.383106:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.383108:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.383109:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.383110:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.383112:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.383113:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.383117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677448.383124:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.383127:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.383128:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.383128:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.383129:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.383130:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.383260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.383268:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.383271:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.383272:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.383275:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.383275:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.383277:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.383278:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.383279:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.383280:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.383280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:31.0:1619677448.383281:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.383284:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.383286:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.383288:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.383289:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.383289:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.383289:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677448.383290:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.383291:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.383292:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:30.0:1619677448.383298:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.383299:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:8.2:1619677448.383299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.383300:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.383301:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715667] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.383303:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.383303:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.383304:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.383305:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.383307:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.383307:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.383307:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.383307:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677448.383309:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.383309:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.383309:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.383310:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.383311:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.383312:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:5.0:1619677448.383312:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.383313:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.2:1619677448.383313:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.383314:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.383315:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.383316:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.383317:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:5.0:1619677448.383317:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.383318:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.383319:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.0:1619677448.383319:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.383320:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:25.2:1619677448.383320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.383320:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.383320:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.383321:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.383322:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677448.383323:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.383324:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:21.2:1619677448.383324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.383327:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:10.0:1619677448.383327:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:30.0:1619677448.383329:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:10.0:1619677448.383330:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:30.0:1619677448.383331:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:26.0:1619677448.383331:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.383331:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.383332:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.383332:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:10.0:1619677448.383333:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.383336:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.383337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.383343:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.383345:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.383346:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.383346:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.383347:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.383347:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.383445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.383450:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.383452:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.383454:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.383455:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.383457:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.383459:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.383460:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.383461:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.383462:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.383463:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.383467:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.383469:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.383471:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.383472:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.383473:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.383474:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.383475:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.383476:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.383477:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.383481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.383488:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.383490:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.383491:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.383493:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.383493:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.383495:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:21.2:1619677448.383496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677448.383504:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677448.383507:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677448.383509:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.383510:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677448.383511:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.383513:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.383514:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.383515:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.383516:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.383517:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.383521:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.383522:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.383524:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.383525:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.383526:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.383527:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.383528:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677448.383529:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677448.383529:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:25.2:1619677448.383533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.383539:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.383542:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.383542:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.383544:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.383546:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.383547:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.383591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.383599:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.383601:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.383602:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.383604:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.383606:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.383608:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.383609:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.383610:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.383611:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.383612:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.383615:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.383617:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.383620:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.383621:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.383621:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.383623:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.383623:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.383624:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.383624:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.383629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677448.383633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677448.383637:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.383640:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:10.0:1619677448.383640:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.383640:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.383641:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.383643:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.383643:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.383644:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.383645:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.383646:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:10.0:1619677448.383648:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.383649:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:8.2:1619677448.383649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000400:00000200:10.0:1619677448.383650:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.383652:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.383653:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.383654:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.383655:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:10.0:1619677448.383657:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.383658:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.383659:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.383659:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.383659:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.383660:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.383660:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:10.0:1619677448.383661:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.383662:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.383662:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.383663:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.383664:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.383665:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.383665:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.383669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.383676:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.383679:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.383679:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.383681:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.383683:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.383684:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.383809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.383815:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.383818:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.383818:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.383820:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.383823:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.383824:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.383825:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.383827:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.383828:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.383829:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.383832:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.383833:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.383835:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.383836:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.383836:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.383838:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.383838:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.383840:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.383841:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.383841:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.383844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.383846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.383852:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.383853:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.383853:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.383854:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.383854:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.383854:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.383855:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.383856:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:21.2:1619677448.383857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:17.0:1619677448.383857:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.383857:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:22.0:1619677448.383863:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.383865:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.383866:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.383868:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.383871:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.383872:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.383873:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.383875:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.383876:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.383877:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.383880:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.383882:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.383885:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.383885:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.383886:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:5.2:1619677448.383886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677448.383888:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.383888:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.383889:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.383889:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.383894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:21.2:1619677448.383895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:26.0:1619677448.383896:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.383899:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.383900:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:26.0:1619677448.383902:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.383902:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:2.0:1619677448.383902:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:10.0:1619677448.383903:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.383904:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.383904:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.383905:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:10.0:1619677448.383905:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.383907:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677448.383907:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.383909:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.383910:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.383910:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.383911:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.383911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:2.0:1619677448.383912:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.383913:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.383914:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.383916:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:2.0:1619677448.383916:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.383917:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.383917:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.383918:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:2.0:1619677448.383918:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.383919:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.383919:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677448.383920:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.383922:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.383923:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.383924:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.383925:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.383926:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.383926:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.383927:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:18.0:1619677448.383927:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:2.0:1619677448.383927:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.383928:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96715658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.383929:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.383929:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.383930:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:18.0:1619677448.383930:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.383931:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.383932:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.383932:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:18.0:1619677448.383932:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:25.2:1619677448.383935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677448.383935:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.383936:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.383936:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.383936:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.383938:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.2:1619677448.383938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.383939:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.383939:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.383941:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.383941:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677448.383942:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.383942:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677448.383942:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.383943:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.383944:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:25.2:1619677448.383945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:13.0:1619677448.383945:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:10.0:1619677448.383945:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:18.0:1619677448.383946:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677448.383946:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.383947:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:13.0:1619677448.383947:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.383947:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.383948:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:18.0:1619677448.383948:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:10.0:1619677448.383948:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677448.383949:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:12.0:1619677448.383949:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677448.383950:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:13.0:1619677448.383950:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.383952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.383952:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.383954:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:21.2:1619677448.383955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:22.0:1619677448.383956:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.383956:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.383958:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.383959:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677448.383960:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677448.383960:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:12.0:1619677448.383961:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:25.2:1619677448.383962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000400:00000200:12.0:1619677448.383962:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.383962:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.383963:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:12.0:1619677448.383964:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.383964:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677448.383965:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.383967:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677448.383970:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:25.2:1619677448.383971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:22.0:1619677448.383973:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.383973:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.383973:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.383975:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.383975:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677448.383977:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.383980:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.383981:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.383982:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.383982:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.383984:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.384044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.384051:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.384054:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.384056:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.384058:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.384059:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.384060:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.384061:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.384063:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.384064:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.384065:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.384068:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.384070:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.384074:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.384074:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.384075:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.384077:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.384078:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.384078:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.384079:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.384083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.384091:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.384093:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.384094:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.384096:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.384097:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.384099:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.384155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.384160:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.384163:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.384165:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.384166:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.384167:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.384169:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677448.384170:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.384172:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.384173:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.384174:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.384177:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.384179:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.384182:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.384183:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677448.384184:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.384186:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.384186:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.384187:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.384188:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.384191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.384198:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.384200:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.384201:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.384202:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.384204:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.384206:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.384248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.384255:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.384257:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.384258:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.384260:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.384262:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.384264:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.384265:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.384266:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.384267:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.384268:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.384271:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.384273:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.384276:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.384276:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.384277:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.384279:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.384280:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.384281:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.384282:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.384282:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677448.384286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677448.384288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.384295:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.384300:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (23)--
00000800:00000200:18.0:1619677448.384300:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.384302:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.384302:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:18.0:1619677448.384303:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.384304:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.384304:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.384306:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.384308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.384311:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.384312:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.384314:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.384315:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.384316:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.384316:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.384317:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.384400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.2:1619677448.384408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677448.384409:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.384411:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.384412:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.384414:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.384414:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:30.0:1619677448.384416:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677448.384416:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.384417:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.384417:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677448.384419:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677448.384419:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.384420:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.384421:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.384422:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.384422:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.384423:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.384424:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.384425:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677448.384425:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.384427:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.2:1619677448.384428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:31.0:1619677448.384430:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.384431:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.384432:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677448.384433:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.384435:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677448.384436:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.384436:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.384437:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.384437:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677448.384437:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.384437:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.384438:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677448.384438:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.384439:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.384440:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677448.384440:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:6.0:1619677448.384440:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.384441:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.384441:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677448.384442:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.384442:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677448.384443:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:6.0:1619677448.384443:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.384445:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:25.2:1619677448.384446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:9.2:1619677448.384446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:6.0:1619677448.384446:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96715652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.384447:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.384448:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.384450:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.384453:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.384455:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:6.0:1619677448.384455:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.384456:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.384457:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.384458:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.384458:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.384459:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.384459:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.384459:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.384460:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.384461:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (22)++
00000400:00000200:22.0:1619677448.384462:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.384462:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.384463:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (23)--
00000800:00000200:6.0:1619677448.384464:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.384465:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.384465:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.384466:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677448.384466:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.384466:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.384469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.384477:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.384479:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.384480:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.384481:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.384482:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.384484:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.384555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.0:1619677448.384559:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.384561:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677448.384563:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.384564:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.384566:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.384567:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.384568:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.384570:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.384571:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.384572:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:5.2:1619677448.384574:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:5.0:1619677448.384579:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.384580:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.384581:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:1.0:1619677448.384582:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.384583:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.384583:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.384584:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.384585:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:1.0:1619677448.384585:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.384586:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.384587:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.384587:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000400:00000200:1.0:1619677448.384587:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.384588:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:1.0:1619677448.384588:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.384589:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.384590:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.384591:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.384592:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.2:1619677448.384593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000400:00000200:1.0:1619677448.384596:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.384597:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.384599:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.384600:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:1.0:1619677448.384600:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.384600:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.384602:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.384602:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.384603:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:1.0:1619677448.384603:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677448.384604:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.384604:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:20.0:1619677448.384605:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.384606:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.384607:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:8.2:1619677448.384608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.2:1619677448.384612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000800:00000200:17.0:1619677448.384615:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.384617:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.384618:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.384619:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:1.0:1619677448.384619:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.384620:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.384622:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.0:1619677448.384622:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.384623:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.384623:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677448.384624:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.384625:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:4.2:1619677448.384842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.384849:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.384852:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.384852:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.384855:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.384858:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.384859:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.384860:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.384861:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.384862:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.384863:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.384867:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.384869:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.384872:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.384873:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.384874:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.384875:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.384876:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.384877:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.384877:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.384881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.384889:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.2:1619677448.384889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:18.0:1619677448.384892:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.384893:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.384893:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.384895:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.384896:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677448.384898:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.384899:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.384900:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.384902:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.384905:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.384906:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.384908:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.384909:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.384910:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.384911:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.384914:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.384916:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.384918:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.384919:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.384919:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.384921:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.384921:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.384922:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.384922:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.384926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.384933:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.384936:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.384936:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.384938:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.384940:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.384941:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.384944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.384950:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.384953:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.384953:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.384954:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.384955:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.384955:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.385077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.385083:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.385085:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.385086:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.385088:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.385089:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.385091:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.385092:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.385094:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.385094:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.385095:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.2:1619677448.385099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:1.0:1619677448.385099:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.385100:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.385103:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.385104:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:2.0:1619677448.385105:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:1.0:1619677448.385105:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:21.2:1619677448.385106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:1.0:1619677448.385106:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:2.0:1619677448.385107:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:1.0:1619677448.385107:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.385107:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.385108:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677448.385109:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.385110:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.385112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.385112:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677448.385113:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.385114:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.385115:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677448.385115:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.385116:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.385117:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.385118:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.385119:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:2.0:1619677448.385119:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.385120:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.385120:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.385121:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:6.0:1619677448.385121:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.385122:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.385123:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96715645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.385125:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.385125:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:6.0:1619677448.385125:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.385126:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.385126:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.385127:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.385128:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.385129:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.385130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:6.0:1619677448.385131:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.385133:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.385134:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.385135:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.385136:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.385136:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.385136:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.385137:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.385138:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677448.385138:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.385139:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.385139:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677448.385140:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.385140:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677448.385140:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.385140:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:30.0:1619677448.385141:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:6.0:1619677448.385141:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:30.0:1619677448.385143:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.385144:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.385144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.385145:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.385145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:30.0:1619677448.385147:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:20.2:1619677448.385147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:30.0:1619677448.385148:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96715644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.385150:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.385150:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.385150:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:30.0:1619677448.385152:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.385152:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:18.0:1619677448.385152:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:5.0:1619677448.385152:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:20.0:1619677448.385153:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.385154:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.385155:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.385155:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.385156:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.385156:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:30.0:1619677448.385157:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.385157:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.385158:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.385159:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:18.0:1619677448.385159:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.385160:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.385160:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.385160:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.385161:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:5.0:1619677448.385161:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677448.385163:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.385163:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.385164:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.385165:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:5.0:1619677448.385165:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.385166:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.385167:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.385167:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:5.0:1619677448.385167:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.385168:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.385169:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:9.2:1619677448.385169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000400:00000200:5.0:1619677448.385170:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.385172:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:25.2:1619677448.385173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677448.385174:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.385175:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.385175:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.385176:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677448.385177:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.385178:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.385178:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677448.385179:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677448.385179:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677448.385179:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:12.0:1619677448.385180:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677448.385180:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.385181:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.385181:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:25.2:1619677448.385183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:12.0:1619677448.385183:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.385184:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.385186:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.385188:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.385189:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.385191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:22.0:1619677448.385191:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.385193:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.385194:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.385194:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.385196:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.385198:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677448.385198:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.385200:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.385201:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.385202:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.385203:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.385205:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.385227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:21.2:1619677448.385228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677448.385234:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:31.0:1619677448.385236:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.385236:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.385237:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:31.0:1619677448.385238:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677448.385240:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.385241:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.385242:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.385243:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.385243:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.385245:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.385245:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.385247:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.385247:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.385249:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.385250:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.385251:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.385251:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.385253:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.385253:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.385259:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.385259:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.385262:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.385262:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.385265:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.385265:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.385266:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.385266:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.385266:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.385267:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.385268:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.385269:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.385269:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.385269:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:31.0:1619677448.385270:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.385270:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:31.0:1619677448.385271:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.385271:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.385274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:25.2:1619677448.385275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.385281:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:6.0:1619677448.385283:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677448.385284:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.385285:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.385286:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.385286:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.385287:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.385287:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.385288:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.385289:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:6.0:1619677448.385289:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.385292:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.2:1619677448.385293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.385300:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.385303:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.385304:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.385305:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.385305:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.385307:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.385363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677448.385369:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.385372:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677448.385373:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677448.385375:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.385378:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.385379:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677448.385381:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.385382:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.385383:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.385384:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677448.385388:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.385390:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.385392:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.385393:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:17.0:1619677448.385394:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.385396:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677448.385396:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.385397:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677448.385397:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:25.2:1619677448.385402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.385410:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.385413:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.385414:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.385414:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.385416:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.385418:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.385431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.385439:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.385441:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.385442:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.385442:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.385443:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.385444:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.385447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.385452:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.385455:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.385456:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.385458:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.385460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:1.0:1619677448.385460:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.385462:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.385463:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.385464:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.385465:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.385466:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.0:1619677448.385468:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:21.2:1619677448.385470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:8.0:1619677448.385470:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:1.0:1619677448.385470:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.385471:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:1.0:1619677448.385471:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:8.0:1619677448.385473:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.385474:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:1.0:1619677448.385474:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.385475:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:8.0:1619677448.385476:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.385476:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:1.0:1619677448.385476:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.385478:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.385478:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.385479:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:8.0:1619677448.385479:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.385479:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.385479:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677448.385480:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96715638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.385480:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.385481:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.385481:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.385482:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.385483:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.385483:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:25.2:1619677448.385484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:5.0:1619677448.385485:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.385487:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.385488:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.385489:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:8.0:1619677448.385490:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.385491:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:8.0:1619677448.385493:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.385494:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:5.0:1619677448.385494:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.385495:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.385497:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.385497:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.385497:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:31.0:1619677448.385498:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:8.0:1619677448.385498:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.385498:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.385499:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.385500:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.385500:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:30.0:1619677448.385501:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.0:1619677448.385501:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.385501:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.385502:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.385502:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677448.385502:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.385503:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.385504:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:5.0:1619677448.385505:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:9.2:1619677448.385506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.385506:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:8.2:1619677448.385508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:21.2:1619677448.385511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (21)++
00000800:00000200:18.0:1619677448.385514:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677448.385516:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.385516:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:18.0:1619677448.385517:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:5.0:1619677448.385517:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:5.0:1619677448.385517:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.385518:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:2.0:1619677448.385518:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:20.0:1619677448.385519:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.385519:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.385519:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:20.0:1619677448.385521:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.385521:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:18.0:1619677448.385522:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.385524:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:18.0:1619677448.385525:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.385743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.385751:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.385754:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.385756:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.385758:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.385759:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.385761:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677448.385762:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.385764:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.385765:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.385766:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.385769:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.385771:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.385774:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.385775:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.385775:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.385777:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.385778:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.2:1619677448.385778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:31.0:1619677448.385779:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.385779:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.385783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.385783:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.385786:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.385787:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.385789:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.385791:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:2.0:1619677448.385791:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.385793:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:17.0:1619677448.385794:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:2.0:1619677448.385794:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.385795:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.385795:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:2.0:1619677448.385796:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.385796:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.385798:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.385798:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.385801:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:2.0:1619677448.385801:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.385803:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.385806:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.385807:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.385808:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.385810:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.385810:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.385811:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.385811:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.385816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.385823:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.385826:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.385827:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.385828:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.385830:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.385831:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.385886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.385892:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.385895:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.385895:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.385897:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.385900:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.385901:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.385903:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.385904:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.385905:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.385906:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.385910:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.385911:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.385914:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.385915:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.385916:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.385917:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.385918:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.385919:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.385920:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.385920:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.385923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.385926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.385930:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.385931:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677448.385932:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677448.385933:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:13.0:1619677448.385933:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.385934:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.385935:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677448.385935:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.385936:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.385938:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.386062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.386069:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.386072:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.386072:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.386075:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.386077:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.386079:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.386080:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.386081:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.386082:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.386083:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.386087:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.386089:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.386091:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.386092:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.386092:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.386094:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.386094:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.386095:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.386095:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.386099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.386105:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.386108:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.386108:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.386110:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.386112:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.386112:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.386191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677448.386198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:10.0:1619677448.386198:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.386201:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.386202:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.386203:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.386204:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:10.0:1619677448.386206:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.386207:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000400:00000200:10.0:1619677448.386208:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:2.0:1619677448.386208:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:10.0:1619677448.386209:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.386210:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.386211:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.386212:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.386213:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.386213:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.386215:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677448.386217:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.386219:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.386220:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.386220:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.386221:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.386222:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.386226:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.386226:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.386226:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.386227:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.386228:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.386229:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.386229:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.386230:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.386230:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:18.0:1619677448.386231:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.386232:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:18.0:1619677448.386232:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:18.0:1619677448.386234:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:25.2:1619677448.386235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.386235:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.386235:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677448.386236:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:25.2:1619677448.386240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:22.0:1619677448.386243:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.2:1619677448.386243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677448.386246:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.386246:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.386247:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:22.0:1619677448.386247:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.386249:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.386250:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.386251:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.386251:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:30.0:1619677448.386252:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.386252:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677448.386253:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.386254:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.386254:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.386256:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:26.0:1619677448.386256:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.386259:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.386260:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.386262:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.386263:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.386264:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.386265:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:21.2:1619677448.386265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:26.0:1619677448.386268:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.386270:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.386272:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.386273:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.386273:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.386273:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.386275:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677448.386275:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.386275:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.386276:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.386276:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:31.0:1619677448.386277:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.386279:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.386280:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.386280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:31.0:1619677448.386282:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.386283:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715629] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.386284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:31.0:1619677448.386285:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.386286:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.386286:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:31.0:1619677448.386287:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.386288:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.386288:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.386289:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:1.0:1619677448.386289:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.386290:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.386291:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.386292:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:1.0:1619677448.386292:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677448.386293:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.386293:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.386301:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.386302:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.386303:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.386303:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677448.386304:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.386305:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.386305:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677448.386305:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.386306:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.386307:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.386308:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:1.0:1619677448.386308:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.386309:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:1.0:1619677448.386309:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.386310:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:25.2:1619677448.386311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.386312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:1.0:1619677448.386313:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.386315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.386315:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.386318:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.386319:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.386319:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.386320:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:30.0:1619677448.386321:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:1.0:1619677448.386322:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (25)--
00000800:00000200:1.0:1619677448.386322:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.386323:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:13.0:1619677448.386323:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:1.0:1619677448.386323:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.386324:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:13.0:1619677448.386324:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.386324:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.386325:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.386326:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.386326:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.386326:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.386326:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:31.0:1619677448.386327:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.386328:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.386328:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:30.0:1619677448.386329:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.386329:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.386330:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.386331:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.386331:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.386332:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.386333:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.386333:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:30.0:1619677448.386334:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.386335:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.386336:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.386340:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.386341:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.386345:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.386345:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.386346:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:25.2:1619677448.386346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.386348:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.386348:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.386349:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.386350:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677448.386353:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.386355:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:8.2:1619677448.386355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:13.0:1619677448.386356:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.386357:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.386357:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.386358:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.386363:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.386365:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.386366:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.386368:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.386369:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:4.2:1619677448.386369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:5.0:1619677448.386371:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.0:1619677448.386376:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.386379:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.386380:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.386382:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.386384:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.386386:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.386387:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.386388:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.386389:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.386390:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.386393:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.386395:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.2:1619677448.386397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.0:1619677448.386398:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.386398:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.386399:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.386400:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677448.386401:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.386401:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.386401:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.386402:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677448.386403:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.386405:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.386406:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.386406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.386406:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.386408:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677448.386409:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.386410:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.386411:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.386411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:20.0:1619677448.386412:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.386414:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:20.0:1619677448.386416:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.386417:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.386417:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:2.0:1619677448.386417:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:17.0:1619677448.386418:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.386418:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.386420:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.386421:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:18.0:1619677448.386421:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677448.386421:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.386421:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:20.0:1619677448.386422:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.386424:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677448.386424:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677448.386424:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:2.0:1619677448.386424:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.386425:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.386427:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.386427:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:2.0:1619677448.386427:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.386428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000400:00000200:2.0:1619677448.386428:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.386429:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.386430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:2.0:1619677448.386431:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.386432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:2.0:1619677448.386432:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.386433:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.386436:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.386436:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.386437:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:2.0:1619677448.386438:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.386438:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.386440:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:5.0:1619677448.386440:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:2.0:1619677448.386440:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.386440:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.386441:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:5.0:1619677448.386441:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.386441:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677448.386441:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.386441:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.386441:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.386442:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.386442:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.386443:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.386443:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:1.0:1619677448.386443:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:17.0:1619677448.386444:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.386444:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:2.0:1619677448.386445:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.386446:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.386446:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677448.386447:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:9.2:1619677448.386449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.386452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.386458:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.386458:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.386459:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.386459:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.386460:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.386460:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.386462:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.386462:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.386463:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.386463:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:21.2:1619677448.386512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.0:1619677448.386516:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.386519:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677448.386521:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677448.386522:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.386524:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.386525:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.386527:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.386528:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.386529:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.386530:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.386533:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.386535:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.386536:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.386537:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.386538:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.386539:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.386540:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.386540:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.386541:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.386546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.386553:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.386556:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.386556:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.386558:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.386559:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.386560:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.2:1619677448.386581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.386589:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.386592:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.386592:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.386594:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.386596:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.386598:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.386599:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715622] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.386600:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.386601:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.386603:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.386606:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.386608:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.386611:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.386611:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.386612:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.386614:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.386614:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.386616:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677448.386617:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.386617:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:9.2:1619677448.386620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677448.386623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:2.0:1619677448.386628:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.386631:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:2.0:1619677448.386631:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.386632:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.386632:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.386633:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.386634:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:10.0:1619677448.386634:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:2.0:1619677448.386634:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.386638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.386641:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.386643:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.386645:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.386645:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.386646:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.386647:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.386648:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.386672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.386678:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.386681:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.386681:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.386683:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.386686:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.386687:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.386689:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.386690:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.386691:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.386692:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.386695:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.386697:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.386700:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.386700:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.386701:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.386703:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.386703:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.386704:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.386704:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.386709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.386716:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:21.2:1619677448.386717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:13.0:1619677448.386719:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.386719:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.386721:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.386722:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.386723:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:13.0:1619677448.386724:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677448.386726:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.386727:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677448.386729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:18.0:1619677448.386729:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.386731:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.386732:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.386733:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.386734:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:18.0:1619677448.386735:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.386736:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.386737:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.386737:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.386740:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.386740:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.386741:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:5.0:1619677448.386743:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.386744:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.386745:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:18.0:1619677448.386746:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.386746:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.386747:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.386748:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:5.0:1619677448.386748:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.386749:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:5.0:1619677448.386749:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.386750:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:5.0:1619677448.386750:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.386751:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.386751:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:5.0:1619677448.386753:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.386755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.386755:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.386758:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.386758:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.386759:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.386760:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.386761:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.386762:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.386762:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:6.0:1619677448.386763:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.2:1619677448.386766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.386766:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.386767:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:21.2:1619677448.386767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:6.0:1619677448.386767:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.386769:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677448.386771:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677448.386774:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.386775:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:17.0:1619677448.386777:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677448.386778:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:17.0:1619677448.386778:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.386778:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:1.0:1619677448.386778:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.386781:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.386782:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.386784:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:24.0:1619677448.386785:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.386786:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.386788:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.386789:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.386790:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677448.386790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677448.386791:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.386794:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.386796:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.386797:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677448.386799:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.386799:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.386799:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.386800:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677448.386800:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.386802:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.386802:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:10.0:1619677448.386802:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.386803:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.386803:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:10.0:1619677448.386805:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.386806:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:25.2:1619677448.386807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:10.0:1619677448.386807:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.386809:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.386809:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.386810:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:12.0:1619677448.386813:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:10.0:1619677448.386813:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.386815:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:12.0:1619677448.386816:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.386816:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.386818:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.386818:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.386819:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.386819:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:13.0:1619677448.386820:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.386821:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.386821:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.386821:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.386822:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.386823:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.386827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.386835:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:21.2:1619677448.386835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677448.386837:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.386838:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.386839:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.386841:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.386843:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677448.386844:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677448.386846:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.386848:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.386850:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.386851:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.386852:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677448.386854:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715616] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.386855:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.386856:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.386857:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.386860:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.386862:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.386864:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.386865:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.386866:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.386867:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.386868:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.386869:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.386869:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.386873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.386881:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.386884:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.386884:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.386885:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.386887:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.386890:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.386949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.386956:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.386959:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.386960:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.386962:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.386965:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.386966:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.386968:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.386969:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.386970:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.386971:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.386974:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.386974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:26.0:1619677448.386976:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.386979:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.386979:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677448.386979:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.386980:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.386982:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677448.386982:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:26.0:1619677448.386983:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.386983:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677448.386983:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677448.386984:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677448.386985:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.386987:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.386988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677448.386988:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.386990:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.386991:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.386992:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.386993:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.386996:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.386996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.386996:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677448.386997:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.386999:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.387000:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677448.387000:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.387000:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.387001:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.387001:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677448.387002:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677448.387002:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.387003:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.387004:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.387005:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:6.0:1619677448.387005:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677448.387005:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677448.387006:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:1.0:1619677448.387006:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.387007:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.387007:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.387009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:17.0:1619677448.387010:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.387011:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:21.2:1619677448.387012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:17.0:1619677448.387013:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.387015:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.387016:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:17.0:1619677448.387016:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.387017:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:17.0:1619677448.387017:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.387019:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.387019:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677448.387019:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677448.387020:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:20.0:1619677448.387020:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.387020:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.387022:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.387022:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.387023:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.387024:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677448.387025:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.387026:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.387027:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.387028:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.387029:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.387029:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.387029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:17.0:1619677448.387030:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.387034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.0:1619677448.387036:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.387039:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.387040:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.387041:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.387041:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.387042:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:6.0:1619677448.387042:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.387045:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.387046:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.387046:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.387048:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.387050:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.387215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.387222:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.387225:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.387226:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.387228:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.387231:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.387232:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.387234:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.387235:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.387236:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.387237:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.387240:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.387242:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.387245:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.387246:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.387246:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.387248:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.387248:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.387249:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.387251:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.387251:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.387254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.387257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.387262:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.387264:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.387265:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.387265:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.387266:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677448.387266:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.387267:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.2:1619677448.387267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677448.387267:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.387268:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.387270:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.387273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.387275:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.387277:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.387279:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.387280:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.387281:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.387282:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.387283:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.387283:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.387284:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:6.0:1619677448.387284:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000400:00000200:30.0:1619677448.387285:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.387285:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.387285:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:30.0:1619677448.387286:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.387287:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.387288:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.387291:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.387298:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.387301:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.387302:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.387302:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.387304:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.387305:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.387306:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.387308:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.387309:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.387311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.387315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.387319:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.387321:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:31.0:1619677448.387323:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:5.0:1619677448.387323:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.387324:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.387324:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.387325:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.387326:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.387327:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.387327:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.387346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.387349:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.387351:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.387353:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.387354:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.387356:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.387357:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.387358:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.387360:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.387361:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.387362:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.387365:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.387367:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.387370:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.387371:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.387371:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.387373:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.387374:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.387375:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.387375:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.387380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.387387:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.387390:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.387391:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.387392:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.387394:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.387396:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.387396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.387403:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.387406:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.387407:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.387408:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.387408:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.387409:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.387410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.387417:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.387420:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.387422:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.387424:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.387425:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.387426:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.387427:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.387429:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.387430:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.387431:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.387434:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.387436:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.387438:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.387439:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.387440:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.387442:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.387442:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.387443:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.387444:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.387447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.387454:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.387457:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.387457:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.387459:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.387461:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:21.2:1619677448.387462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:12.0:1619677448.387462:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.387466:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.387468:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:5.0:1619677448.387470:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.387471:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.387473:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.387475:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.387476:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.387478:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.387479:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.387480:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.387483:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.387485:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 440 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.387487:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.387488:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.387489:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.387490:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.387491:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.387491:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.387492:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.387496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.387504:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.387507:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.387508:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.387508:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.387510:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.387512:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.387610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.387617:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.387620:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.387621:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.387622:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.387625:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.387626:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.387628:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.387629:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.387630:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.387631:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.387635:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.387636:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.387639:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.387640:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.387641:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.387642:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.387643:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.387644:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.387645:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.387645:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.387648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.387651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.387655:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.387658:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.387658:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.387659:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.387659:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.387660:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.387661:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.387661:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.387662:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677448.387664:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.387666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.2:1619677448.387672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:4.2:1619677448.387674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677448.387675:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.387677:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.387678:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.387679:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.387679:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.387680:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.387680:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.387681:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677448.387682:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677448.387682:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.387682:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.387685:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:17.0:1619677448.387685:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.387686:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677448.387687:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.387689:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.387689:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.387690:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.387692:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.387692:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.387693:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.387694:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.387696:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.387697:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.387698:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.387699:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.387699:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.387700:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:17.0:1619677448.387703:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.387704:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.387704:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.387704:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.387706:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677448.387706:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.387707:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.387707:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.387708:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:17.0:1619677448.387708:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:26.0:1619677448.387709:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:17.0:1619677448.387709:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.387709:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.387710:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.387710:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.387711:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.387712:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.387712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.387714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:25.2:1619677448.387716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:13.0:1619677448.387719:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677448.387720:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:10.0:1619677448.387721:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:13.0:1619677448.387722:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677448.387722:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:13.0:1619677448.387723:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.387723:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.387724:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.387724:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.387724:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.387725:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:13.0:1619677448.387725:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.387726:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:10.0:1619677448.387727:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.387728:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.387728:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.387739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.2:1619677448.387743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:26.0:1619677448.387747:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.387749:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:1.0:1619677448.387749:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:10.0:1619677448.387750:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.387752:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.387752:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677448.387753:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:26.0:1619677448.387755:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.387755:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.387756:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.387757:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.387758:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.387759:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.387759:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:26.0:1619677448.387761:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.387761:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.387762:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.387763:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.387765:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.387766:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:26.0:1619677448.387769:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.387770:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.387771:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.387773:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:26.0:1619677448.387774:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.387774:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.387775:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.387776:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.387777:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677448.387777:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.387777:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:26.0:1619677448.387778:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.387779:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:1.0:1619677448.387779:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:26.0:1619677448.387780:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:1.0:1619677448.387780:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:26.0:1619677448.387781:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677448.387781:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677448.387782:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:9.2:1619677448.387783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.387783:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:5.2:1619677448.387786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:9.2:1619677448.387787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.387790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:20.2:1619677448.387791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677448.387791:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.387793:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:18.0:1619677448.387794:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:8.0:1619677448.387794:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677448.387795:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:10.0:1619677448.387795:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:2.0:1619677448.387795:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.387796:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:18.0:1619677448.387796:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.387796:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677448.387796:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.387797:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.387797:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:30.0:1619677448.387798:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:18.0:1619677448.387798:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.387798:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:8.0:1619677448.387798:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.387799:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:8.0:1619677448.387800:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.387801:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:18.0:1619677448.387801:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:30.0:1619677448.387803:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.387804:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:31.0:1619677448.387805:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.387805:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.387807:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.387808:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715602] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.387809:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.387810:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.387811:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.387815:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.387816:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.387819:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.387820:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.387821:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.387822:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.387823:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.387824:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.387824:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.387828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.387834:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.387837:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.387838:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.387839:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.387841:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.387842:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.387867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.387873:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.387876:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.387876:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.387878:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.387881:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.387882:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.387883:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.387885:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.387886:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.387887:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.387890:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.387892:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.387895:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.387895:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.387897:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.387898:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.387899:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.387899:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.387900:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.387904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.387913:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.387915:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.387916:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.387917:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.387918:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.387919:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.388041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.388049:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.388052:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.388055:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.388056:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.388057:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.388058:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.388060:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.388061:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.388062:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.388063:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.388066:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.388068:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.388070:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.388071:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.388072:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.388073:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.388074:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.388075:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.388075:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.388079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.388085:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.388088:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.388088:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.388090:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.388091:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.388092:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.388257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.388265:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.388267:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.388268:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.388270:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.388273:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.388274:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.388275:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.388276:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.388277:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.388278:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.388281:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.388282:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.388284:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.388284:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.388285:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.388286:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.388287:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.388288:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.388288:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.388292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.388299:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.388301:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.388301:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.388302:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.388303:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.388304:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.388313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.388320:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.388322:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.388323:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.388323:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.388323:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.388324:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.388357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.388363:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.388366:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.388368:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.388369:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.388370:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.388372:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.388373:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.388375:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.388376:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.388377:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.388379:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.388381:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.388384:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.388385:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.388386:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.388388:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.388388:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.388389:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.388390:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.388393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.388396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:13.0:1619677448.388401:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.388403:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677448.388404:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:13.0:1619677448.388404:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.388406:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.388407:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:12.0:1619677448.388408:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:31.0:1619677448.388409:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.388409:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677448.388411:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.388411:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.388413:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.388414:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.388415:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.388415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:31.0:1619677448.388416:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.388417:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.388420:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.388421:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.388422:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:12.0:1619677448.388424:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.388425:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.388425:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.388425:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677448.388426:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:12.0:1619677448.388426:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.388427:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.388428:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:12.0:1619677448.388428:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.388429:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.388430:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.388430:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.388435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.388443:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.388445:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.388446:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.388448:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.388449:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.388451:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.388494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:26.0:1619677448.388502:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677448.388505:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:10.0:1619677448.388505:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:26.0:1619677448.388507:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.388509:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.388511:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:26.0:1619677448.388512:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.388513:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.388514:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.388515:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677448.388519:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.388520:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 584 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677448.388523:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.388524:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:26.0:1619677448.388525:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:26.0:1619677448.388526:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:26.0:1619677448.388527:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677448.388527:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:26.0:1619677448.388528:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:8.2:1619677448.388532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.388540:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.388543:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.388543:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.388545:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.388546:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.388548:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:21.2:1619677448.388563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:20.0:1619677448.388568:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.388570:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677448.388572:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.388573:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.388575:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.388576:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.388578:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.388579:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.388580:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.388581:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.388584:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.388586:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.388588:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.388589:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.388590:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.388591:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.388592:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.388593:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.388593:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.388597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.388604:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.388606:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.388607:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.388609:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.388610:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.388611:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:8.2:1619677448.388618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.388625:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.388628:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.388628:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.388629:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.388630:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.388631:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.2:1619677448.388674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.388679:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.388682:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.388684:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.388684:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677448.388685:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.388687:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.388688:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.388689:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.388690:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.388691:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.388694:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.388696:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.388699:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.388700:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.388701:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.388702:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.388703:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.388704:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.388705:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.388706:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:4.2:1619677448.388707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:25.2:1619677448.388708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.388711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.388714:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:10.0:1619677448.388714:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677448.388715:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.388717:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.388717:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:10.0:1619677448.388717:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.388718:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677448.388718:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:13.0:1619677448.388718:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.388719:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.388719:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.388720:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677448.388721:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.388722:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.388723:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677448.388724:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.388725:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.388726:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.388727:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.388728:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.388732:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.388733:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.388736:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.388737:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.388738:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.388739:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.388740:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.388741:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.388741:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.388745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.388753:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.388755:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.388756:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.388758:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.388758:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.388761:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.388826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.388834:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.2:1619677448.388836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:24.0:1619677448.388837:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.388838:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.388839:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.388840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:24.0:1619677448.388842:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.388843:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.388844:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.388845:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:24.0:1619677448.388846:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.388847:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:24.0:1619677448.388847:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.388848:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:24.0:1619677448.388848:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677448.388850:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.388851:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.388851:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.388851:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:24.0:1619677448.388852:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.388853:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.388853:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.388854:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.388855:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:26.0:1619677448.388856:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.388857:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.388857:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.388858:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:24.0:1619677448.388858:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:30.0:1619677448.388859:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.388859:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.388859:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.388859:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.388860:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:30.0:1619677448.388861:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:26.0:1619677448.388861:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.388861:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.388861:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:26.0:1619677448.388862:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.388862:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:26.0:1619677448.388863:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.388864:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.388866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677448.388867:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:26.0:1619677448.388867:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.388869:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.388869:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.388870:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:30.0:1619677448.388871:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:26.0:1619677448.388871:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.388872:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:26.0:1619677448.388872:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.388873:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.388873:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.388874:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:26.0:1619677448.388874:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.388875:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.388875:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677448.388875:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:30.0:1619677448.388876:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:26.0:1619677448.388876:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.388877:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677448.388877:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:9.2:1619677448.388878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.388879:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.388879:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.388880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:1.0:1619677448.388881:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.388883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.388884:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.0:1619677448.388885:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:12.0:1619677448.388887:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:24.0:1619677448.388888:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.2:1619677448.388888:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:31.0:1619677448.388890:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677448.388890:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.388891:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.388892:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.388892:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.388893:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:8.0:1619677448.388893:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.388894:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677448.388894:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:8.0:1619677448.388894:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.388895:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.388895:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677448.388896:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.388897:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:25.2:1619677448.388899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.388899:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.388899:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.388900:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:9.2:1619677448.388900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.388901:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.388901:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.388905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.0:1619677448.388906:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:13.0:1619677448.388907:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.388909:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.388910:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:13.0:1619677448.388910:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.0:1619677448.388910:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.388910:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.388911:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:13.0:1619677448.388911:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:24.0:1619677448.388912:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:13.0:1619677448.388912:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.388913:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677448.388913:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677448.388913:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.388914:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.388916:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.388918:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.388920:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.388921:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.388922:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.388923:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.388924:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.388927:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.388929:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.388931:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.388932:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.388932:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.388933:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.388934:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.388935:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.388935:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.388939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.388947:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.388949:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.388950:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.388950:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.388951:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.388953:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.389004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.389010:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.389013:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.389014:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.389016:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.389018:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.389019:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.389021:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.389022:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.389023:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.389024:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.389027:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.389029:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.389031:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.389032:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.389032:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.389034:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.389035:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.389035:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.389036:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.389040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.389048:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.389051:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.389052:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.389052:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.389054:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.389056:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.389108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.389115:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.389118:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.389119:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.389121:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.389123:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.389125:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.389126:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715587] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.389127:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.389129:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.389129:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.389133:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.389135:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.389138:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.389139:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.389139:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.389141:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.389142:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.389143:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.389144:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.389145:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677448.389148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677448.389150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.389155:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.389157:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:18.0:1619677448.389158:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.389159:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:21.2:1619677448.389159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:2.0:1619677448.389159:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.389160:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677448.389160:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.389161:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:18.0:1619677448.389162:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.389164:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677448.389165:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677448.389167:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.389168:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.389170:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.389172:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.389174:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.389175:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.389176:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.389177:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.389178:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.389182:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.389184:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.389187:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.389187:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.389188:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.389190:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.389190:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.389191:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.389191:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.389195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.389202:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.389204:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.389205:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.389205:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.389208:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.389209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:10.0:1619677448.389211:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:2.0:1619677448.389215:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.389218:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.389219:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.389221:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.389223:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.389224:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.389225:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.389227:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.389228:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.389229:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.389232:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.389233:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.389235:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.389236:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.389236:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.389238:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.389238:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.389240:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.389241:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.389241:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:9.2:1619677448.389244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.389246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.389252:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.389253:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.389253:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:18.0:1619677448.389254:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.389254:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:18.0:1619677448.389255:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.389255:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.389256:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.389257:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.389257:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.389261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.389268:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.389270:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.389271:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.389272:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.389272:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.389273:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.389404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.389410:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.389413:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.389415:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.2:1619677448.389415:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:5.0:1619677448.389419:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.389422:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.389423:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:5.0:1619677448.389423:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.389424:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.389426:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.389426:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:5.0:1619677448.389426:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.389428:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.389429:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.389431:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.389434:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.389434:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.389435:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:5.0:1619677448.389435:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.389436:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.389438:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.389439:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.389440:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.389440:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:26.0:1619677448.389441:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:5.0:1619677448.389441:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.389443:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.389444:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.389444:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677448.389445:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.389445:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677448.389446:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.389449:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677448.389449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.389450:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.389451:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.389452:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.389452:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.389453:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.389454:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.389457:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:25.2:1619677448.389459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.389459:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.389460:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.389462:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.389464:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.389465:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677448.389466:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.389468:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.389469:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.389471:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.389472:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.389473:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.389538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.389545:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.389548:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.389549:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.389550:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.389553:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.389554:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.389555:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.389557:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.389558:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.389559:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.389562:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.389564:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.389567:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.389568:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.389568:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.389570:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.389571:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.389571:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.389572:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.389576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.389584:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.389587:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.389588:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.389588:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.389590:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.389593:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.389692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.389697:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.389700:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.389701:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.389702:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.389704:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.389706:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.389707:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.389708:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.389709:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.389710:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.389713:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.389715:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.389718:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.389719:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.389719:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.389721:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.389721:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.389722:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.389722:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.389727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.389733:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:21.2:1619677448.389733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:26.0:1619677448.389736:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.389737:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.389737:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.389739:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.389739:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.389742:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.389742:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677448.389743:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:22.0:1619677448.389744:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.389747:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.389749:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.389750:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.389752:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.389753:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.389754:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:21.2:1619677448.389756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.2:1619677448.389757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:22.0:1619677448.389758:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.389760:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.389761:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:22.0:1619677448.389762:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.389763:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.389763:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:17.0:1619677448.389763:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.389764:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.389766:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:20.0:1619677448.389766:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.389766:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.389766:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677448.389767:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.389767:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.0:1619677448.389767:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:22.0:1619677448.389768:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:17.0:1619677448.389769:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.389770:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.389771:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:17.0:1619677448.389771:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.389772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:20.0:1619677448.389773:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.389773:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.389774:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.389775:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.389776:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.389776:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.389777:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.389778:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:31.0:1619677448.389779:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:17.0:1619677448.389779:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.389782:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.389783:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.389783:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.389784:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.389785:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.389786:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:30.0:1619677448.389787:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.389787:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.389788:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.0:1619677448.389788:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.389789:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.389790:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:17.0:1619677448.389790:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.389791:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:17.0:1619677448.389791:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.389792:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:17.0:1619677448.389792:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.389793:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.389794:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:17.0:1619677448.389794:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.389795:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:17.0:1619677448.389795:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.389796:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:17.0:1619677448.389796:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.389798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:25.2:1619677448.389799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:21.2:1619677448.389802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (21)++
00000800:00000200:30.0:1619677448.389806:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.389807:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.389808:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:30.0:1619677448.389809:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677448.389809:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677448.389809:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000400:00000200:5.0:1619677448.389809:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.389810:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.389810:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.389811:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.389812:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.389813:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.389813:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.389814:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:31.0:1619677448.389815:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.389815:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:31.0:1619677448.389816:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:30.0:1619677448.389816:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.0:1619677448.389816:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000400:00000200:31.0:1619677448.389818:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.389818:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.389821:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.389823:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.389824:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.389826:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.389826:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.389828:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.389831:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.389832:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.389836:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.389836:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.389837:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.389839:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.389840:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.389841:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.389841:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.389845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.389852:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.389855:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.389856:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.389857:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.389858:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.389860:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.389931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.389939:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.389941:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.389942:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.389944:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.389946:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.389947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:10.0:1619677448.389947:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677448.389949:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.389950:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.389951:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.389952:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.389954:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677448.389955:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.389956:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.389957:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.389959:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.389959:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.389960:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.389960:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.389961:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.389961:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.389962:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:30.0:1619677448.389963:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677448.389963:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.389963:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:30.0:1619677448.389964:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.389964:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:30.0:1619677448.389965:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.389966:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.389967:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.2:1619677448.389968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.389970:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.389971:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.389973:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.389974:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.389974:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.389975:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677448.389976:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.389976:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.389977:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.389978:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677448.389978:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.389979:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.389980:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:25.2:1619677448.389981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.389981:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.389983:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677448.389988:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.389991:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.389991:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.389993:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.389994:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.389995:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.390095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.390102:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.390105:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.390105:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.390107:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.390110:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.390112:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.390113:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.390114:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.390115:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.390116:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.390119:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.390121:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.390123:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.390124:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.390125:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.390127:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.390127:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.390128:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.390128:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.390133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.390140:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.2:1619677448.390141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:6.0:1619677448.390143:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.390144:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.390144:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.390146:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.390147:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677448.390149:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677448.390150:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.390150:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.390152:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.390153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:18.0:1619677448.390155:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.390156:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:18.0:1619677448.390157:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:20.0:1619677448.390158:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:18.0:1619677448.390158:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715573] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.390160:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.390161:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.390161:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:18.0:1619677448.390162:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.390163:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677448.390164:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.390167:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.390168:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:20.0:1619677448.390169:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.390170:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715572] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.390171:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.390172:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.390172:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:20.0:1619677448.390173:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.390173:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:20.0:1619677448.390174:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.390175:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.390176:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.390176:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.390177:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:20.0:1619677448.390178:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.390179:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.2:1619677448.390181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.390182:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.390183:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.390184:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.390186:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.390186:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.390187:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.390188:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.390188:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.390189:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.390189:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.390190:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.390190:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.390191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677448.390192:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.390194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.390195:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677448.390196:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.390197:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.390198:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677448.390198:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.390199:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.390201:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.2:1619677448.390201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000400:00000200:13.0:1619677448.390202:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.390203:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.390204:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.390206:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.390208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.390208:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.390210:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.390211:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.390212:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.390213:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:12.0:1619677448.390214:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.390214:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677448.390217:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.390218:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.390218:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.390219:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.390220:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.390307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.390315:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.390317:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.390318:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.390320:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.390323:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.390324:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.390325:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.390327:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.390328:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.390329:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.390332:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.390334:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.390336:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.390337:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.390338:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.390340:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.390341:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.390342:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.390343:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:5.2:1619677448.390343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.0:1619677448.390344:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677448.390347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.390349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:1.0:1619677448.390350:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.390353:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.390353:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:2.0:1619677448.390355:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677448.390356:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:1.0:1619677448.390356:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.390358:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:1.0:1619677448.390358:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.390359:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.390359:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:18.0:1619677448.390359:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.390359:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.390359:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:8.0:1619677448.390360:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:1.0:1619677448.390361:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.390362:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.390363:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.390364:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.390365:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:1.0:1619677448.390365:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.390369:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.390371:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.390373:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.390374:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.390375:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.390377:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.390377:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.390378:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.390378:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.390383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.390391:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.390394:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.390394:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.390396:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.390398:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.390399:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.390458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.390467:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.390469:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.390470:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.390472:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.390475:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.390476:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.390477:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.390479:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.390480:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.390481:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.390484:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.390486:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.390489:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.390489:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.390490:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.390492:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.390492:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.390493:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.390493:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.390498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.390506:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.390508:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.390509:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.390510:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.390512:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.390515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:22.0:1619677448.390515:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677448.390522:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.390525:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.390526:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.390526:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.390527:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.390528:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.390546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.390551:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.390553:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.390554:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.390556:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.390559:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.390560:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.390561:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.390562:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.390564:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.390565:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.390568:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.390570:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.390573:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.390573:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.390574:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.390575:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.390576:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.390577:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.390578:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.390578:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677448.390582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.390583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.390588:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677448.390589:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:6.0:1619677448.390590:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.390591:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677448.390592:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:6.0:1619677448.390592:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:18.0:1619677448.390593:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.390593:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.390595:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.390597:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.390710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:26.0:1619677448.390717:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677448.390720:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:10.0:1619677448.390720:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:26.0:1619677448.390723:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.390726:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.390727:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:26.0:1619677448.390728:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.390730:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.390731:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.390732:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:26.0:1619677448.390735:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.390737:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.390739:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.390739:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:26.0:1619677448.390740:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.390741:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.390742:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677448.390743:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f787000] (22)++
00000800:00000200:26.0:1619677448.390745:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (23)--
00000800:00000200:26.0:1619677448.390745:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:25.2:1619677448.390746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:5.2:1619677448.390752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677448.390756:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.390757:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.390758:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.390760:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.390761:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (22)++
00000800:00000200:26.0:1619677448.390761:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f787000] (23)--
00000800:00000200:10.0:1619677448.390761:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.390762:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677448.390762:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:26.0:1619677448.390763:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:20.2:1619677448.390771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677448.390777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.390779:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.390782:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:17.0:1619677448.390782:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677448.390784:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.390785:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677448.390785:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:30.0:1619677448.390786:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.390786:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677448.390788:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.390789:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.390789:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.390791:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.390791:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.390792:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.390793:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.390793:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.390795:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.390797:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.390798:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.390798:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.390799:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.390801:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.390803:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.390804:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.390805:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.390805:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.390805:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.390807:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.390807:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.390809:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:17.0:1619677448.390809:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.390809:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.390810:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:17.0:1619677448.390810:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.390811:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.390812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:21.2:1619677448.390812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:17.0:1619677448.390812:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.390812:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.390814:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.390815:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.390815:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.2:1619677448.390817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:25.2:1619677448.390818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677448.390820:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:5.2:1619677448.390820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.390822:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:13.0:1619677448.390822:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.390823:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:8.0:1619677448.390823:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:13.0:1619677448.390824:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.390825:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:24.0:1619677448.390826:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.390826:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.390826:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.390826:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677448.390827:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:12.0:1619677448.390827:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677448.390828:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:24.0:1619677448.390828:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.390828:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677448.390828:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.390828:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:31.0:1619677448.390829:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:1.0:1619677448.390829:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677448.390830:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:13.0:1619677448.390830:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:12.0:1619677448.390830:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.390831:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.390831:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677448.390832:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96715564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.390832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:24.0:1619677448.390833:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.390834:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.390834:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677448.390835:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.390838:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:24.0:1619677448.390839:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.390841:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.390841:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.390842:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:24.0:1619677448.390844:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.390844:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:2.0:1619677448.390844:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.390845:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677448.390846:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.390847:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.390848:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:2.0:1619677448.390848:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:24.0:1619677448.390849:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.390849:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:2.0:1619677448.390849:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.390850:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.390851:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.390852:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:9.2:1619677448.390853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677448.390857:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.390859:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.390862:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.390862:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.390863:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.390864:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.390864:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677448.390865:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.390866:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:18.0:1619677448.390867:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.390867:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.390868:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.390869:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677448.390870:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.390871:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:25.2:1619677448.390872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.390877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:13.0:1619677448.390879:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.390881:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677448.390882:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:13.0:1619677448.390882:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.390885:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:13.0:1619677448.390885:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.390885:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677448.390887:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.390888:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.390888:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.390890:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.390891:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.390893:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.390894:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.390895:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.390896:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.390899:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.390901:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.390904:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.390905:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.390905:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.390907:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.390907:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.390909:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.390909:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.390913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.390921:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.390923:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.390924:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.390926:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.390926:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.390928:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:4.2:1619677448.391032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.391039:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.391042:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.391043:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.391044:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.391046:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.391048:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.391049:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.391050:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.391051:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.391052:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.2:1619677448.391056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:10.0:1619677448.391056:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.391057:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.391060:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.391061:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.391062:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.391064:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.391064:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.391064:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.391065:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.391065:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.391066:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.391068:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.391070:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.391070:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.391070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:31.0:1619677448.391072:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.391073:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.391074:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.391075:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.391076:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.391077:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:31.0:1619677448.391078:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.391080:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.391080:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.391081:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677448.391082:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.391082:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.391083:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.391084:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677448.391084:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.391085:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.391086:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.391087:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.391087:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.391087:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.391091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.391097:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.391099:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.391100:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.391102:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.391104:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.391105:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.391302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.391308:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.391311:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:21.2:1619677448.391312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:1.0:1619677448.391312:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.391314:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.391316:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.391318:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:2.0:1619677448.391318:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:17.0:1619677448.391319:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.391320:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:17.0:1619677448.391321:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.391322:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:17.0:1619677448.391322:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.391323:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677448.391324:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.391327:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.391327:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.391328:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:17.0:1619677448.391329:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:2.0:1619677448.391329:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.391331:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.391332:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.391332:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:2.0:1619677448.391332:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.391333:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:2.0:1619677448.391333:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.391335:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.391335:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.391336:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.391336:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:2.0:1619677448.391336:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.391338:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.391340:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.391341:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.391341:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:25.2:1619677448.391342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677448.391343:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.391343:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.391344:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.391345:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.391349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.391350:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.391353:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.391354:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.391354:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.391356:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:6.0:1619677448.391356:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.391358:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677448.391359:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.391359:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.391360:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.391361:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.391363:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.391367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.391374:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.391377:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.391378:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.391378:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.391378:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.391379:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.391509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.391517:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.391520:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.391521:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.391523:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.391525:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.391527:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.391528:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.391530:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.391531:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.391532:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.391535:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.391537:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.391539:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.391540:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.391541:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.391542:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.391543:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.391544:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.391545:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.391546:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.391548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.391551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.391556:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:10.0:1619677448.391558:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:30.0:1619677448.391559:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.391559:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.391560:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:30.0:1619677448.391561:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.391561:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.391562:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.391563:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.391564:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.391567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.391574:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.391576:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.391577:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.2:1619677448.391577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:30.0:1619677448.391578:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.391578:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.391579:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677448.391583:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.391586:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.391587:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.391588:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.391588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:18.0:1619677448.391590:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.391592:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.391593:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.391594:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.391595:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.391596:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:18.0:1619677448.391596:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.391599:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.391599:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:18.0:1619677448.391600:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.391602:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.391602:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.391603:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.391604:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:26.0:1619677448.391605:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.391605:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.391606:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:26.0:1619677448.391607:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:18.0:1619677448.391607:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:26.0:1619677448.391608:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.391608:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:26.0:1619677448.391609:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.391609:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000400:00000200:26.0:1619677448.391610:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.391610:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:26.0:1619677448.391611:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:9.2:1619677448.391612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677448.391615:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.391615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:26.0:1619677448.391616:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.391620:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.391620:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.391620:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.391621:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:26.0:1619677448.391622:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (24)++
00000800:00000200:18.0:1619677448.391622:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.391623:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:26.0:1619677448.391624:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (25)--
00000800:00000200:26.0:1619677448.391625:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677448.391625:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (24)--
00000800:00000200:26.0:1619677448.391626:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:18.0:1619677448.391626:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677448.391626:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000800:00000200:26.0:1619677448.391627:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000400:00000200:2.0:1619677448.391627:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.391628:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:2.0:1619677448.391629:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.391630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.2:1619677448.391633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.391633:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.391635:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.391637:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.391638:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.391638:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.391640:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:2.0:1619677448.391640:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.391642:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:2.0:1619677448.391642:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.391643:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.391644:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677448.391649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.391657:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.391659:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.391662:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.391663:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.391664:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.391665:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.391667:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.391668:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.391669:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.391670:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.391673:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.391675:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.391677:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.391677:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.391678:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.391680:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.391680:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.391681:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.391682:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677448.391683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:25.2:1619677448.391685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677448.391691:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.2:1619677448.391691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677448.391692:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677448.391693:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.391694:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677448.391695:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.391695:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.391696:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:13.0:1619677448.391697:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.391697:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.391699:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:8.0:1619677448.391699:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.391699:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.391700:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677448.391700:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677448.391702:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:25.2:1619677448.391703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:8.0:1619677448.391703:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.391703:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.391704:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.391705:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.391705:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.391706:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.391707:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.391708:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.391709:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677448.391709:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.391710:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.391711:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:13.0:1619677448.391712:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.391713:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.391713:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.391714:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.391715:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677448.391716:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.391717:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.391719:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677448.391720:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.391722:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.391723:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.391723:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:8.0:1619677448.391724:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.391724:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.391725:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.391726:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.391726:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.391726:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.391727:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.391727:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677448.391728:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677448.391728:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677448.391729:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677448.391732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.2:1619677448.391732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.391739:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:17.0:1619677448.391740:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.391741:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:17.0:1619677448.391742:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677448.391742:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677448.391743:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.391744:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.391744:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:24.0:1619677448.391745:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.391746:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.391748:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:17.0:1619677448.391751:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.391770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.391775:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.391778:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.391780:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.391781:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.391782:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.391784:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677448.391785:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.391786:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.391787:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.391788:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677448.391792:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.391794:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.391796:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.391797:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.391797:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.391799:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.391800:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.391800:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.391801:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.391805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.391813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:5.0:1619677448.391814:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.391817:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.391818:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.391819:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.0:1619677448.391820:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.391820:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.391822:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:5.0:1619677448.391822:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:2.0:1619677448.391823:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.391825:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.391826:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.391828:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.391829:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.391830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000400:00000200:18.0:1619677448.391831:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.391832:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.391833:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.391836:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.391837:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.391838:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.391840:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.391841:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.391841:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:18.0:1619677448.391841:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.391842:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677448.391842:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.391842:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.391843:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677448.391844:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.391844:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.391845:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.391846:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.391850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.391857:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.391860:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.391861:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.391863:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.391864:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.391865:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.391912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.391919:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.391921:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.391922:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.391925:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.391928:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.391929:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.391930:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.391931:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.391932:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.391933:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:21.2:1619677448.391934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000400:00000200:10.0:1619677448.391937:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.391939:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:10.0:1619677448.391939:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.391941:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.391942:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:10.0:1619677448.391942:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.391942:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:20.0:1619677448.391944:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.391944:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.391944:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.391945:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.391945:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.391945:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677448.391946:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.391947:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677448.391948:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.391949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:20.0:1619677448.391949:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.391950:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.391951:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677448.391954:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.391956:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.391956:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.391958:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.391959:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677448.391959:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.391959:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:31.0:1619677448.391960:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.391961:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.391962:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:30.0:1619677448.391963:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.391964:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.0:1619677448.391964:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.391965:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677448.391966:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677448.391967:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.391968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.391970:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.391971:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.391972:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:5.0:1619677448.391972:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.391973:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.391974:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.391975:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.391976:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.391977:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:20.0:1619677448.391978:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:5.0:1619677448.391979:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.391979:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:20.0:1619677448.391980:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715547] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.391981:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.391982:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.391983:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677448.391986:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.391988:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.391991:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.391992:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.391992:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.391994:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.391994:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.391995:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.391996:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.392000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.392006:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.392009:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.392009:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.392010:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.392012:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.392015:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.392068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.392074:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.392076:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.392078:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.392079:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.392082:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.392083:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677448.392085:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.392086:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.392087:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.392088:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677448.392091:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.392093:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677448.392095:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.392096:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677448.392096:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677448.392098:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677448.392099:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.392099:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.392100:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.392104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.392112:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.392114:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.392115:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.392115:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:20.2:1619677448.392117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:22.0:1619677448.392118:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.392120:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:30.0:1619677448.392125:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.392127:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.392130:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.392131:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.392132:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.392133:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.392135:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.392136:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.392137:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.392138:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.392141:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.392143:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.392146:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.392147:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.392148:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.392149:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.392150:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.392151:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.392151:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.392155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.392161:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.392163:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.392164:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.392166:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.392167:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.392168:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.392183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.392190:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.392192:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.392193:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.392195:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.392197:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.392199:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.392200:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.392202:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.392203:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.392204:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.392207:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.392209:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.392212:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.392213:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.392214:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.392216:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.392216:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.392217:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.392217:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.392221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:21.2:1619677448.392228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:12.0:1619677448.392228:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.392231:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.392232:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.392234:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.392235:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.392236:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.392236:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.392238:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677448.392239:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:25.2:1619677448.392239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:10.0:1619677448.392241:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.392243:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.392245:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:13.0:1619677448.392246:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:10.0:1619677448.392246:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.392247:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.392248:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.392249:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:10.0:1619677448.392249:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.392250:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:10.0:1619677448.392250:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:13.0:1619677448.392251:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.392251:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:10.0:1619677448.392254:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.392255:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.392257:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.392258:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.392259:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.392261:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.392261:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.392262:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.392262:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677448.392266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.392273:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.392275:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.392276:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.392277:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.392279:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.392280:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.392336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.392343:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.392346:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.392347:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.392348:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.392351:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.392351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:26.0:1619677448.392352:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.392353:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.392355:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.392355:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.392356:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.392359:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677448.392359:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.392360:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.392361:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677448.392363:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.392363:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:30.0:1619677448.392364:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.392364:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.392365:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.392366:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.392366:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.392366:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.392366:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:30.0:1619677448.392367:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:26.0:1619677448.392367:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:30.0:1619677448.392368:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.392370:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.392371:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.392371:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:25.2:1619677448.392372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:30.0:1619677448.392374:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.392376:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.392378:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.392379:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.392379:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.392379:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.392380:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.392380:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.392381:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.392381:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:31.0:1619677448.392382:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.392383:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.392384:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.392384:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.392384:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.392385:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.392386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.392390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.392392:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.392394:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.392395:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.392395:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.392397:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.392398:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:10.0:1619677448.392399:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677448.392400:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.392401:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.392402:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677448.392433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.392439:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.392442:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.392443:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.392445:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.392447:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.392449:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.392450:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.392451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:6.0:1619677448.392451:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.392452:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.392453:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.392456:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.392456:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.392458:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.392459:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:6.0:1619677448.392461:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.392461:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.392462:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677448.392462:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.392462:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.392464:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.392464:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.392465:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.392466:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.392466:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:5.0:1619677448.392466:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.392467:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.392469:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.392470:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.392471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:5.0:1619677448.392471:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.392474:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.392476:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.392479:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.392479:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.392480:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.392480:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:20.0:1619677448.392481:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.392481:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:20.0:1619677448.392482:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.392483:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.392484:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.392485:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.392485:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.392486:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677448.392487:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:25.2:1619677448.392489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.392495:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.392498:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.392499:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.392501:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.392502:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.392503:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.392568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.392575:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.392578:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.392578:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.392580:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.392582:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.392584:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.392585:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.392587:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.392587:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.392589:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.392592:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.392594:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.392596:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.392597:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.392598:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.392599:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.392599:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.392600:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.392600:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.392605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.392612:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.392614:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.392615:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.392616:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.392616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677448.392618:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.392619:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677448.392622:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.392625:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.392626:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.392628:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.392630:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.392632:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.392633:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.392635:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.392636:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.392637:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.392640:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.392642:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.392644:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.392645:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.392646:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.392648:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.392648:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.392649:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.392650:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.392651:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.392654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.392655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.392661:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.392661:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.392663:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677448.392664:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677448.392664:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677448.392665:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.392665:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.392665:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677448.392667:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.392670:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.392694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.392701:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.392704:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.392707:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.392708:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.392709:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.392711:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.392712:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.392713:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.392714:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.392715:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.392719:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.392721:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.392723:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.392724:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.392724:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.392726:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.392726:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.392727:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.392728:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.392731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.392738:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.392740:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.392741:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.392742:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.392744:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.392745:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.392748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.392755:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.392757:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.392758:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.392759:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.392759:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.392760:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.392806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70b000] (20)++
00000800:00000200:5.0:1619677448.392810:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.392812:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677448.392815:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:5.0:1619677448.392815:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.392818:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.392819:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.392820:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.392822:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.392823:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.392824:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.392827:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.392829:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.392831:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.392832:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.392833:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.392834:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.392835:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:5.0:1619677448.392835:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:5.0:1619677448.392836:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677448.392840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.392847:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.392849:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.392850:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.392852:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.392853:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.392854:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:8.2:1619677448.392858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.392865:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.392868:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.392868:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.392869:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677448.392869:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.392871:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:21.2:1619677448.392906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.392913:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.392916:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.392918:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.392920:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.392920:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.392922:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.392923:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.392925:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.392925:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.392927:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.392930:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.392932:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.392934:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.392935:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.392936:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.392937:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.392938:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.392939:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.392939:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.392943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.392949:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.392951:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.392952:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.392952:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.392954:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.392957:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.392960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.392966:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.392969:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.392969:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.392970:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.392971:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.392971:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.393059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.393065:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.393068:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.393069:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.393070:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.393073:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.393074:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677448.393075:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.393076:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.393077:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.393078:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.393082:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.393084:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.393086:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.393087:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.393088:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.393089:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.393090:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.393091:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.393091:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.393095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.393103:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.393105:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.393106:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.393108:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.393109:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.393111:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.393215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.393222:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.393225:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.393226:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.393228:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.393232:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.393233:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.393234:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.393235:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.393237:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.393237:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.393240:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.393242:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.393245:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.393245:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.393246:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.393247:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.2:1619677448.393248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.0:1619677448.393248:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.393249:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.393249:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.393254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.393255:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.393258:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.393260:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.393261:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.393262:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.393262:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.393263:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:17.0:1619677448.393264:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:30.0:1619677448.393265:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.393265:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:30.0:1619677448.393266:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.393266:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.393267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677448.393268:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.393268:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.393269:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.393272:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.393273:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677448.393273:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.393274:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.393274:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.393274:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.393277:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.393278:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.393279:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.393279:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:1.0:1619677448.393279:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.393280:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:30.0:1619677448.393281:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.393282:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:1.0:1619677448.393282:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.393283:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.393283:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:1.0:1619677448.393283:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.393284:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.393285:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:8.2:1619677448.393287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:1.0:1619677448.393288:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.393290:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.393293:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.393294:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:5.0:1619677448.393300:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:1.0:1619677448.393300:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677448.393302:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:1.0:1619677448.393302:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.393302:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677448.393303:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.393303:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.393304:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:5.0:1619677448.393305:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.393306:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.393308:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:25.2:1619677448.393309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.393309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677448.393317:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:12.0:1619677448.393317:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:21.2:1619677448.393319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:31.0:1619677448.393320:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000800:00000200:12.0:1619677448.393320:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.393321:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.393322:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.393323:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.393324:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.393325:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:12.0:1619677448.393325:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677448.393325:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:31.0:1619677448.393326:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.393327:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:6.0:1619677448.393328:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:31.0:1619677448.393329:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.393329:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:31.0:1619677448.393331:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.393331:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.393332:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.393333:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:6.0:1619677448.393333:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.393335:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.393336:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.393337:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.393338:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.393339:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.393340:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:6.0:1619677448.393340:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.393343:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.393344:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:6.0:1619677448.393344:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.393345:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.393346:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677448.393346:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.393347:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677448.393348:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677448.393349:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:6.0:1619677448.393349:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.393349:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.393350:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.393352:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.393352:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:9.2:1619677448.393353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.393353:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.393353:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.393358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.393360:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.393363:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.393364:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.393365:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.393366:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:18.0:1619677448.393367:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.393368:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.393369:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.393370:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:20.0:1619677448.393371:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.393372:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.393373:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.393384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.393391:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.2:1619677448.393393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:30.0:1619677448.393394:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.393396:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.393397:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.0:1619677448.393398:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:30.0:1619677448.393399:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.393400:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:30.0:1619677448.393401:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.393402:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.393403:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.393403:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:30.0:1619677448.393404:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.393405:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.393406:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.393406:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.393408:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.393409:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.393410:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.393411:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.393412:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.393413:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.393413:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.393416:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.393416:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.393417:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677448.393417:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:30.0:1619677448.393419:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.393419:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.393419:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.393420:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.393421:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677448.393422:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.393423:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.393423:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:25.2:1619677448.393424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:5.0:1619677448.393426:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:5.0:1619677448.393426:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.393427:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.393428:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:13.0:1619677448.393431:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.393434:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677448.393434:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.393436:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.393437:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677448.393438:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677448.393439:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.393439:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.393440:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.393440:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677448.393443:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.393447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.393453:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.393456:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.393457:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.393457:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.393458:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.393459:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.393460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.393468:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.393471:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.393472:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.393474:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.393476:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.393478:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.393479:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.393480:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.393481:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.393482:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.393486:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.393487:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.393489:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.393490:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.393491:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.393492:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.393493:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.393493:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.393494:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.393498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.393505:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.393507:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.393508:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.393509:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.393511:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.393512:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.393512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.393519:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.393522:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.393523:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.393524:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.393527:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.393528:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.393529:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.393530:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.393531:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.393532:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.393535:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.393536:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.393537:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.393538:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.393538:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.393539:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.393540:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.393540:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.393541:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.393545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.393551:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.393553:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.393553:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.393554:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.393555:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.393556:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.393650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677448.393656:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677448.393658:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677448.393660:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677448.393661:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.393663:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.393665:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:2.0:1619677448.393666:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.393667:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.393668:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.393669:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:2.0:1619677448.393673:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.393675:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:2.0:1619677448.393677:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677448.393678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677448.393678:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:2.0:1619677448.393678:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677448.393680:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:2.0:1619677448.393680:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:20.0:1619677448.393681:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.393681:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677448.393682:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:20.0:1619677448.393684:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.393686:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.393686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.393686:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.393688:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.393690:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.393691:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.393692:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.393693:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.393694:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:20.0:1619677448.393695:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.393696:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:21.2:1619677448.393697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:17.0:1619677448.393697:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.393698:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.393698:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.393699:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.393700:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.393702:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.0:1619677448.393703:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.393704:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.393705:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.393705:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.393707:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677448.393707:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.393707:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.393708:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677448.393709:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.393709:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677448.393711:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.393712:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.393712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677448.393713:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.393714:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.393716:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.393717:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.393718:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.393719:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:30.0:1619677448.393721:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.393721:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.393722:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.393724:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:13.0:1619677448.393724:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.393725:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.393726:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.393727:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.393727:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.393728:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.393729:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.393730:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.393731:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.393731:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.393735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.393741:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.393743:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.393744:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.393745:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.393747:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.393748:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.393838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:5.2:1619677448.393838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:24.0:1619677448.393845:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:13.0:1619677448.393846:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:24.0:1619677448.393848:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:13.0:1619677448.393849:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:8.0:1619677448.393849:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.393851:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.393852:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.393853:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:24.0:1619677448.393854:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.393855:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.393857:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:13.0:1619677448.393857:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.393858:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677448.393859:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.393860:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.393860:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.393862:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677448.393862:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.393863:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:13.0:1619677448.393863:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:13.0:1619677448.393868:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.393869:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.393871:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.393871:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.393874:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.393874:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:13.0:1619677448.393874:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.393875:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.393875:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677448.393876:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.393877:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.393878:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.393878:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677448.393878:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.393879:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677448.393879:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677448.393880:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677448.393881:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677448.393882:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:25.2:1619677448.393883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:9.2:1619677448.393885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.393887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:18.0:1619677448.393892:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:12.0:1619677448.393892:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.393893:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.393894:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.393895:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:12.0:1619677448.393895:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.393896:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:2.0:1619677448.393896:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.393897:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.393897:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677448.393897:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677448.393898:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:18.0:1619677448.393899:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.393899:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.393900:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:18.0:1619677448.393902:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.393904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:4.2:1619677448.393909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677448.393911:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.393914:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.393914:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.393915:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.393916:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.393916:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.393916:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.393919:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.393920:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.393922:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.393924:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.393925:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.393926:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.393927:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.393928:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.393929:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.393932:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.393934:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.393935:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.393936:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.393936:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.393937:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.393938:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.393939:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.393940:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.393941:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.393943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:4.2:1619677448.393946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.393952:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.393953:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.393954:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.393955:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:26.0:1619677448.393956:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.393956:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.393957:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.393957:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.393958:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.393958:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.394010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.394018:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.394021:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.394023:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.394025:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.394026:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.394027:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.394028:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.394030:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.394030:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.394031:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.394034:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.394036:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.394039:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.394040:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.394041:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.394042:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.394043:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.394044:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.394046:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.394046:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.394048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.394052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.394056:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.394059:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:30.0:1619677448.394060:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:5.0:1619677448.394060:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.394061:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.394062:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.394063:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.394064:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677448.394064:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:30.0:1619677448.394065:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:21.2:1619677448.394138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.394146:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.394148:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.394151:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.394152:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.394153:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.394155:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677448.394156:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.394157:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.394158:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.394159:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:5.2:1619677448.394160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:30.0:1619677448.394162:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.394164:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677448.394166:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.394167:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:30.0:1619677448.394167:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:10.0:1619677448.394168:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.394169:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:30.0:1619677448.394169:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.394170:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:30.0:1619677448.394171:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:26.0:1619677448.394171:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.394172:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.394173:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677448.394173:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.394175:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.394175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:10.0:1619677448.394176:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.394177:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.394178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677448.394179:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.394179:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.394180:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.394181:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:10.0:1619677448.394183:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.394183:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.394184:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.394184:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.394185:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.394186:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677448.394186:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.394187:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.394188:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:10.0:1619677448.394188:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.394188:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.394188:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:31.0:1619677448.394189:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.394190:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.394190:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.394190:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.394191:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.394191:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.394195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.394202:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.394204:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.394205:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.394207:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.394208:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.394209:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.394246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.394253:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.394256:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.394257:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.394258:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.394261:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.394262:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.394263:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.394265:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.394265:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.394266:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.394269:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.394271:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.394273:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.394274:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.394275:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.394277:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.394277:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.394278:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.394278:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.394283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.394291:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.394297:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.394297:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.394299:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.394301:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.394303:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.394408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.2:1619677448.394410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.394413:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.394414:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.394415:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677448.394417:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:20.0:1619677448.394418:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.394418:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.394419:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:1.0:1619677448.394421:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.394422:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.394423:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.394423:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.394425:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.394426:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:20.0:1619677448.394427:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.394428:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.394428:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.394429:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.394430:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.394430:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.394432:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:20.0:1619677448.394434:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.394436:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.394438:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.394439:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.394440:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.394440:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.394441:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.394443:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.394444:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.394444:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.394444:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.394445:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.394446:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.394447:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:1.0:1619677448.394447:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677448.394448:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677448.394448:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.394449:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.394449:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.394450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:25.2:1619677448.394453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.394453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677448.394458:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.394459:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.394459:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:12.0:1619677448.394460:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.394461:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677448.394461:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:12.0:1619677448.394462:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.394462:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.394462:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.394463:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.394464:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.394465:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.394465:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.394466:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.394467:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.394468:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.394472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:12.0:1619677448.394479:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.394481:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.394482:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.394483:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.394483:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.394485:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.394501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.394509:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.394512:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.394512:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.394515:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.394517:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.394519:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.394520:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.394521:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.394522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:10.0:1619677448.394522:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.394523:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.394526:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.394528:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.394528:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.394530:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.394530:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:22.0:1619677448.394531:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:10.0:1619677448.394531:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:6.0:1619677448.394531:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:10.0:1619677448.394532:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.394532:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677448.394533:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.394533:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.394534:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.394536:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.394537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:22.0:1619677448.394537:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.394539:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.394540:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.394541:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.394542:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.394543:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.394545:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.394546:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.394546:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.394547:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:22.0:1619677448.394548:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:26.0:1619677448.394549:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.394550:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:22.0:1619677448.394551:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.394552:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.394552:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.394554:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.394554:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.394555:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.394556:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.394560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.394566:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.394569:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.394570:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.394570:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.394572:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.394575:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:25.2:1619677448.394579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000800:00000200:26.0:1619677448.394585:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.394588:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.394588:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.394589:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.394590:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.394590:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.394734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.394740:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.394743:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.394744:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.394746:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.394748:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.394750:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.394751:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.394752:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.394753:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.394754:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.394758:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.394760:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.394762:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.394763:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.394764:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.394765:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.394766:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.394766:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.394767:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.394772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.394780:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.394783:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.394784:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.394784:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.394786:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.394786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677448.394788:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.394791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:10.0:1619677448.394793:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:4.2:1619677448.394794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.2:1619677448.394795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677448.394796:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.394796:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.394798:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.394798:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:10.0:1619677448.394800:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.394801:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.394801:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.394802:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:10.0:1619677448.394802:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:1.0:1619677448.394802:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.394803:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677448.394803:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:10.0:1619677448.394803:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.394803:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.394803:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.0:1619677448.394804:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:10.0:1619677448.394805:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.394806:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:24.0:1619677448.394806:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.394807:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.394808:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677448.394809:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.394809:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.394811:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.394811:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:24.0:1619677448.394811:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.394812:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96715507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.394812:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.394813:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.394814:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.394814:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.394815:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.394815:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:24.0:1619677448.394816:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.394817:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.394818:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.394818:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.394819:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.394819:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:24.0:1619677448.394820:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.394820:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:24.0:1619677448.394822:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.394822:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.394822:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.394823:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.394823:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.394823:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.394825:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677448.394825:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.394826:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.394826:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.394827:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.394828:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.394828:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:25.2:1619677448.394828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:24.0:1619677448.394828:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.394829:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677448.394830:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677448.394830:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:31.0:1619677448.394831:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677448.394831:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:31.0:1619677448.394832:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.394832:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677448.394832:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.394834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.394835:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:25.2:1619677448.394836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:4.2:1619677448.394837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677448.394838:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.394839:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.394839:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.394841:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.394843:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:13.0:1619677448.394843:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677448.394844:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677448.394845:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:17.0:1619677448.394846:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:13.0:1619677448.394846:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677448.394846:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677448.394847:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.394847:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.394847:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:25.2:1619677448.394848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:8.0:1619677448.394848:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.394849:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:13.0:1619677448.394849:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.394849:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:17.0:1619677448.394850:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.394851:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.394853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.394853:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677448.394855:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.394858:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.394859:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:12.0:1619677448.394859:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.394859:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.394860:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.394861:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677448.394862:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.394863:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.394864:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.394864:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.394866:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.394971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:20.2:1619677448.394975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:26.0:1619677448.394979:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:5.0:1619677448.394980:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.394981:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.394982:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:5.0:1619677448.394983:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:26.0:1619677448.394985:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677448.394986:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.394986:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.394987:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.394989:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:5.0:1619677448.394989:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.394991:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.394991:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.394992:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.394993:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.394994:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.394995:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.394995:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.394996:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:5.0:1619677448.394997:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.395002:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.395003:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.395005:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.395006:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.395009:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.395010:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.395010:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.395011:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.395011:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.395012:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.395014:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.395014:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.395015:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:5.0:1619677448.395015:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.395016:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:5.0:1619677448.395016:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677448.395017:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:5.0:1619677448.395017:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.395019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.2:1619677448.395020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:13.0:1619677448.395026:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.395028:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:13.0:1619677448.395028:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.395029:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.395030:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.395031:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.395031:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.395032:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677448.395033:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.395033:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:13.0:1619677448.395034:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.395035:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.395036:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.395036:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:5.0:1619677448.395037:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:5.0:1619677448.395037:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.395038:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.395137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.395145:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.395147:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.395148:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.395150:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.395153:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.395153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:8.0:1619677448.395155:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.395156:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.395158:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.395159:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.395160:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.395161:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.395163:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.395164:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:8.0:1619677448.395164:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.395166:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.395166:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.395169:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.395169:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.395170:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:10.0:1619677448.395171:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:8.0:1619677448.395171:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:10.0:1619677448.395172:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.395172:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:10.0:1619677448.395173:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.395173:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:4.2:1619677448.395173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:10.0:1619677448.395174:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.395174:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.395174:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:10.0:1619677448.395175:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:8.2:1619677448.395178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:10.0:1619677448.395179:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.395181:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.395181:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.395182:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677448.395184:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.395184:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.395185:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.395186:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677448.395186:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:26.0:1619677448.395187:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.395188:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:10.0:1619677448.395188:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677448.395188:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.395189:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.395190:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96715501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.395191:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.395191:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.395192:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.395193:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.395193:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:25.2:1619677448.395193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.395194:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677448.395194:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:10.0:1619677448.395196:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.395197:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000400:00000200:26.0:1619677448.395198:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.395198:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.395200:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:12.0:1619677448.395200:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:10.0:1619677448.395200:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:12.0:1619677448.395202:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:26.0:1619677448.395203:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677448.395203:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.395203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.395204:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.395204:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:12.0:1619677448.395204:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.395206:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:13.0:1619677448.395206:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.395206:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.395207:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.395208:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.395209:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.395210:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.395212:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:25.2:1619677448.395212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.395213:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.395214:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.395214:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:4.2:1619677448.395215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.395220:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:21.2:1619677448.395221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:6.0:1619677448.395222:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.395223:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.395223:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.395224:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.395225:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.395226:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.395226:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.395226:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.395227:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:6.0:1619677448.395229:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677448.395230:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:5.2:1619677448.395231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.395231:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.395233:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.395235:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.395236:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:1.0:1619677448.395237:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677448.395238:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.395239:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.395240:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.395240:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.395240:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:18.0:1619677448.395241:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677448.395243:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.395245:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.395247:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:1.0:1619677448.395247:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.395248:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:18.0:1619677448.395250:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.395250:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.395251:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.395251:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:1.0:1619677448.395251:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.395252:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.395253:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:1.0:1619677448.395253:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:18.0:1619677448.395254:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.395254:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.395255:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:1.0:1619677448.395257:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.395258:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:25.2:1619677448.395259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677448.395262:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.395262:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.395263:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.395265:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.395265:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.395266:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:1.0:1619677448.395266:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.395267:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.395269:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.395270:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.395270:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:25.2:1619677448.395271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:22.0:1619677448.395273:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.395275:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677448.395278:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.395281:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.395282:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.395284:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.395285:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.395287:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.395288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677448.395294:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.395302:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.395303:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.395303:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.395303:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.395304:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.395315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677448.395318:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:5.2:1619677448.395320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:20.0:1619677448.395321:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677448.395323:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.395326:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.395326:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:2.0:1619677448.395326:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:20.0:1619677448.395328:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677448.395329:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.395329:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.395330:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:20.0:1619677448.395331:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.395332:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.395333:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.395333:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.395335:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.395337:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.395337:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.395339:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715497] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.395340:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.395341:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.395342:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.395343:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.395343:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.395344:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.395344:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.395346:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:2.0:1619677448.395346:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.395347:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677448.395348:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677448.395348:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:2.0:1619677448.395348:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.395351:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.395351:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.395352:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:25.2:1619677448.395353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.395354:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.395354:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.395355:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.395355:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:26.0:1619677448.395360:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:8.2:1619677448.395360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.395363:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.395364:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.395364:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.395366:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.395369:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:17.0:1619677448.395369:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.395371:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.395372:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.395372:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:25.2:1619677448.395374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000400:00000200:17.0:1619677448.395374:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.395377:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677448.395380:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.395383:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.395383:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.395384:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.395385:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.395386:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.395406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.395412:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.395415:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:4.2:1619677448.395415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:22.0:1619677448.395416:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.395418:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.395420:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.395422:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000400:00000200:6.0:1619677448.395422:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.395423:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.395424:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.395425:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677448.395426:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:6.0:1619677448.395426:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.395427:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:24.0:1619677448.395430:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.395431:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.395433:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.395433:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:24.0:1619677448.395434:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.395435:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96715495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.395436:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677448.395437:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.395438:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.395438:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:24.0:1619677448.395439:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.395439:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:6.0:1619677448.395440:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.395441:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.395442:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:24.0:1619677448.395443:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.395443:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:24.0:1619677448.395444:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677448.395444:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:25.2:1619677448.395446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:24.0:1619677448.395447:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.395448:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.395449:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:21.2:1619677448.395449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677448.395451:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.395451:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677448.395452:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:30.0:1619677448.395453:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:24.0:1619677448.395453:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:24.0:1619677448.395454:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:22.0:1619677448.395454:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:30.0:1619677448.395456:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:22.0:1619677448.395456:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:30.0:1619677448.395457:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.395457:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.395457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677448.395458:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:30.0:1619677448.395459:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.395459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (21)++
00000800:00000200:31.0:1619677448.395460:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.395461:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:1.0:1619677448.395465:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:8.0:1619677448.395466:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:17.0:1619677448.395468:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.395468:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:8.0:1619677448.395469:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:8.0:1619677448.395469:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:1.0:1619677448.395469:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.395471:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000400:00000200:1.0:1619677448.395471:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.395474:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.395654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.395659:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.395662:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.395662:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.395664:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.395667:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.395668:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.395669:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.395670:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.395671:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.395672:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.395675:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.395677:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.395679:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.395680:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.395681:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.395682:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.395683:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.395684:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.395685:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.395685:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.395688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.395691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.395697:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.395697:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.395698:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.395698:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.395698:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.395699:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.395700:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.395700:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.395701:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.395701:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.395756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.395762:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.2:1619677448.395765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677448.395765:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.395766:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.395768:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.395770:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.395772:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677448.395773:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:18.0:1619677448.395773:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.395774:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.395775:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.395776:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:18.0:1619677448.395777:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.395779:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.395780:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.395781:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.395782:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.395783:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.395784:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.395785:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.395787:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.395788:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.395788:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:31.0:1619677448.395789:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.395789:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.395790:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.395791:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.395791:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.395792:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:31.0:1619677448.395793:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.395793:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:31.0:1619677448.395795:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:25.2:1619677448.395796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.395798:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.395799:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.395800:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.395801:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.395802:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.395803:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.395803:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677448.395804:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.395806:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:25.2:1619677448.395807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:13.0:1619677448.395807:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.395809:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.395811:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.395812:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.395813:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.395815:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.395816:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.395818:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.395819:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.395820:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.395938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.395943:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.395946:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.395947:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.395949:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.395952:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.395953:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.395955:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.395956:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.395957:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.395958:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.395962:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.395963:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.395966:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.395967:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.395968:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.395970:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.395970:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.395971:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.395972:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.395973:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.395975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.395978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:13.0:1619677448.395982:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.395984:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:13.0:1619677448.395985:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677448.395986:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.395986:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:13.0:1619677448.395986:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.395987:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:13.0:1619677448.395987:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.395988:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.395990:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.396032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.396038:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677448.396041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.396041:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.396043:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.396045:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.396046:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.396047:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:5.2:1619677448.396047:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:5.0:1619677448.396050:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.396052:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.396052:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677448.396052:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.396054:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.396055:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:5.0:1619677448.396056:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.396056:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677448.396057:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.396058:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.396058:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677448.396060:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.396061:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.396063:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.396063:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.396063:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.396064:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.396065:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.396066:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.396067:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.396067:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.396068:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.396069:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.396070:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.396071:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.396073:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.396075:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.396075:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:20.0:1619677448.396076:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.396076:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.396077:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.396078:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.396078:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.396079:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.396079:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.396079:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677448.396080:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.396081:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.396081:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.396082:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.396082:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.396082:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.396083:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.396084:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:25.2:1619677448.396085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.396085:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.396085:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.396086:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677448.396086:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.396086:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677448.396087:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.396087:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.396090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.2:1619677448.396091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.396093:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.396095:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.396096:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:12.0:1619677448.396096:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677448.396096:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.396098:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.396098:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:12.0:1619677448.396099:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.396100:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.396101:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677448.396101:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.396102:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:12.0:1619677448.396102:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.396102:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.396103:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:1.0:1619677448.396104:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.396105:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.396106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:1.0:1619677448.396107:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:25.2:1619677448.396110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.2:1619677448.396111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.396113:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.396116:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:13.0:1619677448.396116:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.396117:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.396117:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.396118:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677448.396118:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.396119:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677448.396119:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.396120:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.396120:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677448.396121:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677448.396121:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.396122:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.396122:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.396123:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:13.0:1619677448.396123:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677448.396125:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.396264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.396271:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.396274:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.396275:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.396276:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.396279:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.396281:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.396282:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.396284:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.396285:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.396286:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.396289:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.396291:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.396293:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.396294:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.396300:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.396302:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.396303:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.396303:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.396304:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.396308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.396316:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.396319:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.396320:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.396321:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.396323:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.396324:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.396392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.396400:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.396403:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.396404:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.396405:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.396407:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.396409:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.396410:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.396411:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.396412:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.396413:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.396415:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.396417:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.396418:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.396419:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.396419:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.396420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:10.0:1619677448.396421:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.396421:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.396422:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.396422:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.396426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:6.0:1619677448.396426:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.396429:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.396429:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.396431:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.396433:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:6.0:1619677448.396434:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.396435:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.396435:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.396436:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:6.0:1619677448.396436:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.396437:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.396438:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.396439:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.396439:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.396440:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:6.0:1619677448.396441:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.396444:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.396446:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.396448:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.396449:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.2:1619677448.396449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:6.0:1619677448.396450:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.396451:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.396452:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.396452:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.396453:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677448.396456:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:9.2:1619677448.396458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:1.0:1619677448.396458:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.396459:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.396461:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.396463:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.396465:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:1.0:1619677448.396465:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.396466:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.396467:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.396468:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:1.0:1619677448.396468:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.396469:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.396469:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.396469:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.396471:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.396473:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.396474:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:1.0:1619677448.396475:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.396478:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.396479:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.396480:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.396482:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.396482:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.396483:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.396484:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.396488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.396496:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.396498:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.396499:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.396501:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.396503:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.396503:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.396508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.396514:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.396517:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.396517:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.396519:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.396522:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.396524:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.396525:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.396525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000400:00000200:18.0:1619677448.396526:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.396527:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.396528:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.396532:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.396532:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:18.0:1619677448.396533:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.396535:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.396535:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:18.0:1619677448.396536:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.396537:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:13.0:1619677448.396537:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.396538:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:18.0:1619677448.396539:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.396539:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.396540:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:13.0:1619677448.396540:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.396541:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.396542:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:13.0:1619677448.396542:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:13.0:1619677448.396543:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677448.396545:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677448.396545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:13.0:1619677448.396546:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677448.396547:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:5.2:1619677448.396548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000400:00000200:13.0:1619677448.396550:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.396552:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:2.0:1619677448.396553:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.396554:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.396555:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.396555:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:13.0:1619677448.396555:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.396555:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.396556:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:13.0:1619677448.396556:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:18.0:1619677448.396557:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677448.396557:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:2.0:1619677448.396557:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.396558:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677448.396559:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:13.0:1619677448.396559:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:2.0:1619677448.396559:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677448.396560:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677448.396561:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677448.396565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.396573:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.396576:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.396577:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.396577:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.396579:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.396582:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.396583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.396590:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.396593:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.396594:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.396594:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.396595:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.396596:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.396756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.396764:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.396767:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.396769:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.396771:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.396771:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.396773:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.396774:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.396775:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.396776:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.396777:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.396780:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.396782:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.396784:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.396785:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.396785:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.396787:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.396788:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.396789:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.396789:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.396792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.396798:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.396801:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.396802:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.396803:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.396805:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.396806:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.396881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:5.2:1619677448.396883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677448.396888:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677448.396889:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.396891:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.396891:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677448.396892:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.396892:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677448.396895:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677448.396895:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.396897:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.396898:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677448.396898:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.396899:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.396900:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677448.396902:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.396902:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.396903:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.396904:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677448.396904:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.396905:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.396906:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677448.396909:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.396911:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:17.0:1619677448.396911:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.396913:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677448.396914:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.396914:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.396915:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.396916:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.396917:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.396917:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677448.396917:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:18.0:1619677448.396918:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677448.396918:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:18.0:1619677448.396919:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677448.396920:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.396920:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.396921:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.396922:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.396923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:9.2:1619677448.396926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:22.0:1619677448.396930:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:24.0:1619677448.396933:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:22.0:1619677448.396933:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.396934:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.396934:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677448.396936:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:22.0:1619677448.396936:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:24.0:1619677448.396937:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.396937:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.396939:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.396939:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677448.396942:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677448.396973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.396981:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.396984:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.396987:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.396988:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.396989:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.396990:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.396992:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.396993:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.396994:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.396995:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.396998:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.397000:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.397004:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.397005:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.397006:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.397007:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.397008:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.397009:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.397009:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.397014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.397022:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.397025:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.397026:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.397028:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.397028:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.397031:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.397048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.397055:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.2:1619677448.397057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:10.0:1619677448.397057:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.397058:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.397060:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.397062:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.397063:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:2.0:1619677448.397063:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:10.0:1619677448.397064:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.397065:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:10.0:1619677448.397066:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.397067:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:10.0:1619677448.397067:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.397068:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677448.397069:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.397072:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.397072:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.397073:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:2.0:1619677448.397074:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.397075:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.397076:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.397076:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:2.0:1619677448.397076:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.397077:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:2.0:1619677448.397077:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.397078:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.397079:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.397079:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.397080:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.397080:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:2.0:1619677448.397082:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.397084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:2.0:1619677448.397084:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.397087:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.397087:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.397088:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.397090:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.397091:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:2.0:1619677448.397091:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.397091:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.397092:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:30.0:1619677448.397094:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.397095:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.397096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.397097:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.397097:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.397099:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:13.0:1619677448.397103:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.397105:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.397106:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.397108:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.397109:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.397111:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.397113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.397116:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.397119:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.397121:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.397122:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.397123:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.397125:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677448.397126:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.397127:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.397128:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.397129:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677448.397133:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.397135:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677448.397138:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.397139:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677448.397139:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.397141:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677448.397142:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.397143:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.397143:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.397147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.397150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677448.397154:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:26.0:1619677448.397157:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677448.397157:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.397158:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.397158:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.397160:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.397161:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.397161:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677448.397163:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:26.0:1619677448.397164:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.397166:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.397167:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.397168:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.397169:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.397171:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.397171:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.397174:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.397176:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.397180:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.397180:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.397181:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.397183:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.397183:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.397184:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.397185:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.397189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.397197:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.397199:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.397200:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.397201:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.397203:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.397206:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.397322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.397328:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.397331:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.397332:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.397334:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.397337:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.397338:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.397339:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.397340:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.397341:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.397342:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.397345:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.397347:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.397349:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.397350:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.397351:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.397352:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.397353:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.397354:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.397355:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.397356:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.397359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.397361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.397366:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:1.0:1619677448.397367:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677448.397369:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.397369:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:1.0:1619677448.397369:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.397370:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.397370:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.397370:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677448.397372:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.397375:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.397425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.397433:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.397436:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.397438:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.397440:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.397441:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.397442:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.397443:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.397444:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.397446:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.397447:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.397450:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.397451:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.397454:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.397454:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.397455:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.397456:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.397457:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.397458:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.397459:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.397462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.397469:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.397471:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.397472:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.397472:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.397474:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.397477:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.397693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.397698:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.397701:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.397702:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.397704:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.397707:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.397708:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.397709:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.397710:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.397712:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.397713:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.397716:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.397718:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.397721:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.397721:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.397722:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.397724:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.397724:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.397725:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.397725:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677448.397730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:9.2:1619677448.397730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.397736:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:30.0:1619677448.397739:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:24.0:1619677448.397739:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.397740:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.397740:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:30.0:1619677448.397742:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:24.0:1619677448.397742:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.397744:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.397745:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:31.0:1619677448.397746:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.397746:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.397747:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:9.2:1619677448.397748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000400:00000200:30.0:1619677448.397749:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.397750:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.397751:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.397752:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:8.0:1619677448.397754:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000400:00000200:30.0:1619677448.397755:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.397756:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:30.0:1619677448.397757:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677448.397757:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.397757:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.397758:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:30.0:1619677448.397759:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.397759:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:30.0:1619677448.397760:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.397761:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.397762:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.397763:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.397764:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.397764:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.397768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.397775:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.397777:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.397778:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.397779:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.397781:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.397782:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.397872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.397878:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.397880:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.397882:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.397883:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.397885:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.397887:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677448.397888:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.397889:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.397890:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.397891:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677448.397895:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.397897:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.397899:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.397900:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.397900:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.397902:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.397903:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.397904:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.397905:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.397906:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677448.397908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677448.397911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.2:1619677448.397913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677448.397913:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.397915:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.397916:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677448.397916:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:20.0:1619677448.397917:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677448.397919:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:30.0:1619677448.397920:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677448.397920:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.397920:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.397922:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.397923:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677448.397924:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:30.0:1619677448.397925:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.397927:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.397927:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.397929:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.397930:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.397931:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.397932:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.397933:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.397936:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.397938:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.397940:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.397941:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.397942:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.397943:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.397944:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.397945:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.397947:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.397947:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.397949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.397953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.397956:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.397958:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.397959:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.397960:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.397961:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677448.397962:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.397963:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:13.0:1619677448.397963:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.397964:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.397965:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.397969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.397976:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.397978:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.397979:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.397980:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.397980:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.397981:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.397990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.397997:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.2:1619677448.397999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:26.0:1619677448.398000:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.398001:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.398002:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.398002:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.398004:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:26.0:1619677448.398005:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.398006:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.398007:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.398008:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.398008:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:26.0:1619677448.398009:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.398009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:26.0:1619677448.398010:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.398010:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.398011:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.398011:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.398013:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.398014:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.398015:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:26.0:1619677448.398016:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.398016:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.398017:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.398018:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.398018:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.398019:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677448.398021:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.398022:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:22.0:1619677448.398022:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.398022:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.398023:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.398024:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.398025:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.398025:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.398025:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.398025:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.398026:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.398026:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.398027:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96715465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.398028:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.398029:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.398029:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:25.2:1619677448.398030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:22.0:1619677448.398030:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.398030:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677448.398031:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.398032:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.398032:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.398033:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000400:00000200:22.0:1619677448.398034:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.398034:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.398035:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:22.0:1619677448.398036:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.398037:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:25.2:1619677448.398037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.398039:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:22.0:1619677448.398039:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.398039:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:31.0:1619677448.398040:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.398040:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677448.398040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000400:00000200:31.0:1619677448.398042:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.398042:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.398042:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677448.398043:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.398043:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:31.0:1619677448.398044:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:22.0:1619677448.398044:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677448.398044:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.2:1619677448.398045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:12.0:1619677448.398047:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:25.2:1619677448.398048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:12.0:1619677448.398048:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.398048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:12.0:1619677448.398049:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.398051:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.398051:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:12.0:1619677448.398052:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.398052:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677448.398053:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.398054:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:30.0:1619677448.398055:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.398055:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:5.0:1619677448.398055:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:26.0:1619677448.398056:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677448.398056:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:30.0:1619677448.398058:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.398058:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.398058:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:30.0:1619677448.398059:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:1.0:1619677448.398059:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.398060:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:17.0:1619677448.398060:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:10.0:1619677448.398060:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.398061:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:1.0:1619677448.398061:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.398062:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:10.0:1619677448.398062:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.398063:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.398064:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.398064:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:10.0:1619677448.398065:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.398067:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.398070:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.398072:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.398074:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.398075:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.398075:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.398077:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.398078:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.398078:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.398079:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677448.398081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:25.2:1619677448.398084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.398089:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.398091:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.398091:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677448.398092:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.398093:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.398093:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.398095:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.398095:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.398097:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.398098:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.398099:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.398100:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.398101:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.398103:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677448.398104:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.398105:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.398105:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.398106:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.398109:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.398110:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.398111:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.398113:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.398113:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.398115:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.398116:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.398117:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.398119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.398123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.398127:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.398129:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.398130:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.398131:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.398132:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.398132:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.398133:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.398134:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.398135:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.398135:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.398137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.398145:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.398147:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.398148:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.398149:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.398150:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.398150:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.398204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.398210:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.398212:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.398213:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.398215:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.398218:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.398219:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.398221:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.398222:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.398223:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.398224:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.398227:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.398229:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.398232:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.398232:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.398233:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.398235:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.398235:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.398236:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.398237:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.398241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.398248:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.398250:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.398251:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.398253:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.398254:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.398256:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.398423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.398431:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.398433:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.398435:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.398438:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.398438:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.398440:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677448.398441:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.398442:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.398443:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.398444:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677448.398447:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.398448:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677448.398451:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.398451:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.398452:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.398454:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.398454:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.398455:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.398456:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.398460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.398467:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.398470:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.398471:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.398471:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.398473:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.398474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:24.0:1619677448.398475:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:30.0:1619677448.398482:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.398485:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.398487:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.398489:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.398489:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.398491:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.398492:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.398493:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.398494:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.398495:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.398499:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.398500:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.398503:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.398503:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.398504:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.398505:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.398506:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.398506:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.398507:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.398511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.398518:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.398520:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.398521:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.398522:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.398524:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.398525:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.398529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.398535:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.398538:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.398538:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.398539:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.398540:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.398540:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.398631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.398639:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.398642:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.398642:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.398644:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.398646:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.398648:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.398649:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.398650:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.398651:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.398652:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.398655:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.398657:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.398659:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.398659:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.398660:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.398661:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.398662:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.398663:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.398664:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.398664:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.2:1619677448.398666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:25.2:1619677448.398667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.398668:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.2:1619677448.398670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677448.398671:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.398673:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.398674:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.398676:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:20.0:1619677448.398676:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.398677:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.398677:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677448.398678:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:30.0:1619677448.398679:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.398679:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.398679:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.398680:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.398680:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.398680:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.398681:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.398681:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:20.0:1619677448.398681:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.398682:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.398682:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:20.0:1619677448.398686:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.398688:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.398690:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.398691:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:21.2:1619677448.398692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:20.0:1619677448.398692:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.398694:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.398694:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.398695:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.398696:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677448.398699:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:25.2:1619677448.398700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.398701:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.398702:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.398704:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.398706:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677448.398707:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.398708:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:12.0:1619677448.398709:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.398710:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.398710:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.398712:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.398712:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.398713:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.398714:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:6.0:1619677448.398714:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:12.0:1619677448.398715:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:6.0:1619677448.398718:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.398720:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.398723:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.398723:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.398724:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.398726:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.398726:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.398727:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.398727:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.398732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.398740:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.398743:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.398743:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.398745:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.398746:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.398748:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.398786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.398793:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.398795:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.398796:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.398798:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.398801:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.398802:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.398804:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.398805:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.398806:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.398807:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.398811:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.398812:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.398815:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.398816:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.398817:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.398818:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.398819:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.398819:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.398820:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.398824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.398833:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.398836:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.398836:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.398837:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.398839:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.398842:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.398887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.398894:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.398897:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.398898:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.398900:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.398902:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.398904:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.398905:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.398906:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.398907:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.398907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:26.0:1619677448.398908:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.398911:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.398913:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.398914:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:26.0:1619677448.398916:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.398916:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:26.0:1619677448.398917:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.398917:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:26.0:1619677448.398918:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.398919:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:24.0:1619677448.398919:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.398920:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.398921:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:24.0:1619677448.398921:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.398922:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.398922:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:24.0:1619677448.398923:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.398924:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96715454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.398925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:24.0:1619677448.398926:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.398927:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.398928:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:4.2:1619677448.398928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:24.0:1619677448.398932:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.398933:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.398935:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:10.0:1619677448.398935:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:24.0:1619677448.398936:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.398936:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.398937:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.398938:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:10.0:1619677448.398938:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.398939:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:24.0:1619677448.398939:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.398939:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:20.0:1619677448.398939:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.398939:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677448.398940:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.398941:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000400:00000200:20.0:1619677448.398941:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.398942:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677448.398942:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:25.2:1619677448.398944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.398944:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.398948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:13.0:1619677448.398951:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.398953:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.398954:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.398955:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:13.0:1619677448.398956:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.398957:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.398957:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:13.0:1619677448.398958:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677448.398958:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.398959:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.398968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:12.0:1619677448.398974:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.398977:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.398978:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.398978:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.398979:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.398981:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.399058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.399066:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.399068:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.399071:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.399072:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.399073:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.399074:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.399075:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.399076:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.399077:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.399077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:31.0:1619677448.399078:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.399081:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.399083:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.399084:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.399085:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.399086:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.399086:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.399086:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.399087:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.399088:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.399088:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.399089:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.399090:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:30.0:1619677448.399090:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.399091:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.399091:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.399092:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.399093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:30.0:1619677448.399094:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.399094:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.399095:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.399098:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.399099:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.399099:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.399100:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.399101:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.399102:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.399102:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.399103:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.399103:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.399104:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.399105:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.399105:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.399105:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.399106:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.399107:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.399110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.399116:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.399119:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.399119:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.399121:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.399122:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.399123:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.399150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.399156:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.399159:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.399159:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.399161:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.399164:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.399165:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.399167:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.399168:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.399169:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.399170:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.399173:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.399174:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.399177:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.399178:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.399179:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.399180:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.399181:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.399182:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.399182:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.399186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.399192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:12.0:1619677448.399192:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.399195:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.399196:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.399198:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.399199:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.399199:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:12.0:1619677448.399200:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.399202:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.399203:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.399205:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.399207:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.399208:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.399209:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.399211:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.399212:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.399213:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.399216:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.399217:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.399220:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.399221:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.399222:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.399223:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.399224:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.399225:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677448.399226:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.399227:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677448.399229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677448.399232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:1.0:1619677448.399238:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:26.0:1619677448.399239:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:1.0:1619677448.399240:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.399241:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.399242:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.399242:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:1.0:1619677448.399242:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.399243:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:1.0:1619677448.399244:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:1.0:1619677448.399247:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.399314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.399322:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.399324:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.399325:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.399327:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.399330:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.399331:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.399332:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.399334:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.399335:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.399335:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.399339:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.399340:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.399342:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.399343:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.399343:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.399344:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.399345:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.399345:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.399346:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.399351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.399358:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.399361:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.399361:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.399363:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.399365:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.399366:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.399394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677448.399402:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677448.399404:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677448.399406:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.399408:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.399408:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677448.399409:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.399410:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.399411:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.399412:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.399413:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.399416:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.399418:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.399421:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.399422:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.399422:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.399424:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.399425:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677448.399426:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:31.0:1619677448.399427:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:31.0:1619677448.399428:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:25.2:1619677448.399430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.399434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:22.0:1619677448.399438:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:30.0:1619677448.399441:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:22.0:1619677448.399441:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.399441:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.399442:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.399444:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc2e00] (23)--
00000400:00000200:22.0:1619677448.399444:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.399445:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677448.399446:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:22.0:1619677448.399448:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.399571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.399579:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.399582:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.399584:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.399586:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.399587:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.399588:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.399590:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.399591:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.399592:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.399593:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.399596:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.399598:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.399601:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.399602:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.399602:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.399604:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.399604:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.399605:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.399606:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.399610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.399618:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.399621:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.399622:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.399622:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.399624:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.399628:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.399631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.2:1619677448.399634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:5.0:1619677448.399637:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.399640:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677448.399640:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.399642:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:20.0:1619677448.399643:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.399643:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.399644:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:5.0:1619677448.399646:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.399646:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.399648:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.399649:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.399649:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.399650:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.399651:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.399652:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.399653:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.399653:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.399654:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.399654:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.399656:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.399658:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.399660:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.399661:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.399663:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.399664:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.399665:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.399665:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.399666:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.399666:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.399667:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.399668:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.399669:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.399669:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677448.399670:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.399670:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677448.399671:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.399671:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677448.399671:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.399675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:9.2:1619677448.399675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677448.399681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677448.399682:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:13.0:1619677448.399683:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.399684:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.399685:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.399686:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.399686:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.399687:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:13.0:1619677448.399687:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.399688:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.399689:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:13.0:1619677448.399689:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.399690:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:12.0:1619677448.399690:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.399691:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:13.0:1619677448.399692:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677448.399693:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.399694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:26.0:1619677448.399696:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.399697:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.399698:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.399699:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.399700:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.399701:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:12.0:1619677448.399701:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.399703:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.399704:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:26.0:1619677448.399705:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.399705:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.399705:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.399707:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.399708:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.399710:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.399711:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.399712:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.399713:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:4.2:1619677448.399713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677448.399714:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.399714:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.399715:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.399720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.399725:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677448.399728:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.0:1619677448.399728:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.399729:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677448.399730:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:8.0:1619677448.399730:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.399731:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.399733:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.399733:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.399733:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:8.0:1619677448.399735:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:20.0:1619677448.399736:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:8.0:1619677448.399736:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.399737:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.399738:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.399739:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.399742:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.399744:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.399747:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.399748:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.399749:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.399750:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.399751:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.399751:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.399752:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.399756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.399764:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.399767:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.399768:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.399768:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.399770:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.399773:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.399774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.399781:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.399784:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.399785:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.399785:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.399786:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.399786:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.399816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.399821:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.399824:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.399825:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.399827:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.399829:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.399830:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.399832:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.399833:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.399834:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.399835:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.399839:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.399841:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.399843:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.399844:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.399845:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.399846:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.399846:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.399847:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.399848:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.399849:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.399852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.399854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.399859:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.399860:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:17.0:1619677448.399860:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677448.399861:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.399862:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677448.399862:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.399863:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.399864:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.399865:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.399868:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.399869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677448.399877:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677448.399880:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:21.2:1619677448.399881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:13.0:1619677448.399882:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.399882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677448.399884:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677448.399885:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.399886:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:2.0:1619677448.399886:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:13.0:1619677448.399888:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.399889:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.399889:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.399890:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:13.0:1619677448.399890:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.399892:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:13.0:1619677448.399892:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.399892:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:13.0:1619677448.399893:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677448.399893:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.399896:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.399897:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.399899:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:10.0:1619677448.399899:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.399900:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.399900:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.399901:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:13.0:1619677448.399902:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.399902:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.399903:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:2.0:1619677448.399903:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.399904:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:10.0:1619677448.399904:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.399905:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.399906:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:13.0:1619677448.399906:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000400:00000200:10.0:1619677448.399906:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.399906:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.399907:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677448.399907:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000400:00000200:10.0:1619677448.399907:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.399908:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:9.2:1619677448.399912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:10.0:1619677448.399913:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.399913:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.399916:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.399917:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.399918:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.399918:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.399919:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.399920:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.399920:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.399921:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:10.0:1619677448.399921:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.399921:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677448.399921:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.399922:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:10.0:1619677448.399922:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677448.399922:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677448.399923:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.399924:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.399925:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.399925:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:25.2:1619677448.399926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.399926:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.399927:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.399928:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.399928:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.399929:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.399930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.399932:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:21.2:1619677448.399933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:26.0:1619677448.399934:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.399935:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.399935:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.399938:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.399939:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:18.0:1619677448.399940:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.399941:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:9.2:1619677448.399941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000400:00000200:18.0:1619677448.399942:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.399942:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.399943:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:18.0:1619677448.399944:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.399944:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.399946:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.399946:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.399947:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.399948:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.399948:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.399949:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.399949:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.399949:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.399950:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.399950:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.399951:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.399951:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677448.399952:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.399952:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.399997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.400002:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.400005:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.400006:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.400008:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.400009:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.400010:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.400012:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.400013:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.400014:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.400015:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.400019:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.400021:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.400023:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.400024:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.400024:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.400026:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.400026:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.400027:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.400027:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.400032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.400040:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.400043:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.400044:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.400045:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.400046:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.400048:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.400151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677448.400157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677448.400157:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.400159:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.400160:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.400162:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.400164:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.400165:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:1.0:1619677448.400166:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.400167:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.400168:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.400168:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:1.0:1619677448.400169:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.400171:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.400172:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677448.400173:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.400174:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.400176:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.400176:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.400177:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.400178:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:10.0:1619677448.400179:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.400180:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.400181:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.400181:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.400182:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.400183:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:10.0:1619677448.400185:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.400185:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.400185:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.400187:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.400187:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.400188:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:10.0:1619677448.400189:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.400189:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.400190:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.400191:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:9.2:1619677448.400191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:10.0:1619677448.400192:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.400193:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.400194:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.400194:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.2:1619677448.400194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.400197:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:25.2:1619677448.400198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.400199:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677448.400200:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.400201:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.400201:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.0:1619677448.400201:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.400202:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677448.400203:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.400203:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.400205:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:24.0:1619677448.400206:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677448.400206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:12.0:1619677448.400207:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.400208:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.400210:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.400211:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:13.0:1619677448.400212:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.400213:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677448.400214:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.400215:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.400217:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.400220:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.400221:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.400223:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.400224:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.400225:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.400226:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.400230:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.400231:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.400235:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.400235:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.400236:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.400238:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.400238:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.400239:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.400239:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.400243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.400250:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.400252:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.400253:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.400253:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.400256:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.400259:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.400262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.400268:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.400270:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.400271:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.400272:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.400272:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.400273:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.400277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.400283:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.400286:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.400286:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.400288:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.400290:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.400292:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.400298:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.400299:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.400300:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.400301:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.400304:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.400306:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.2:1619677448.400307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677448.400310:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.400310:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.400311:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.400313:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.400313:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.400315:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:31.0:1619677448.400316:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.400316:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.400316:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.400319:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.2:1619677448.400320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677448.400321:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.400323:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:4.2:1619677448.400323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.400324:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.400325:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.400326:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.400327:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.400328:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.400329:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:20.0:1619677448.400329:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:10.0:1619677448.400330:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:20.0:1619677448.400331:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:31.0:1619677448.400332:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.400332:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.400332:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.400333:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677448.400334:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.400334:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.400335:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677448.400336:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.400337:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677448.400338:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.400338:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.400339:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.400341:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.400341:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.400343:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.400344:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.400345:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677448.400347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.400351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.400356:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:30.0:1619677448.400358:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:18.0:1619677448.400359:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:30.0:1619677448.400360:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:18.0:1619677448.400360:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.400360:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.400361:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.400362:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:18.0:1619677448.400363:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.400366:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.400471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.400477:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.400479:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.400481:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.400482:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.400484:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.400486:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.400487:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.400489:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.400489:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.400491:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.400494:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.400496:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.400499:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.400500:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.400500:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.400502:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.400503:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.400503:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.400504:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.400509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.400517:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.400519:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.400520:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.400522:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.400523:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.400524:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.400564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.0:1619677448.400567:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677448.400570:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:20.0:1619677448.400572:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.400573:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677448.400575:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.400576:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677448.400578:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.400579:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.400580:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.400581:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677448.400584:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.400586:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.400588:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.400589:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.400589:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.400591:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.400591:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677448.400593:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708600] (22)++
00000800:00000200:20.0:1619677448.400594:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (23)--
00000800:00000200:20.0:1619677448.400595:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:8.2:1619677448.400596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.400600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677448.400603:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.400604:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.400605:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.400605:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (22)++
00000400:00000200:20.0:1619677448.400606:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.400607:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.2:1619677448.400607:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677448.400608:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677448.400609:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (23)--
00000800:00000200:5.0:1619677448.400610:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708600] (23)--
00000800:00000200:5.0:1619677448.400611:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677448.400613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:1.0:1619677448.400613:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.400616:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.400617:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.400618:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.400620:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000400:00000200:1.0:1619677448.400621:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.400623:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677448.400623:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:20.0:1619677448.400624:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000400:00000200:1.0:1619677448.400624:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.400625:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.400626:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:1.0:1619677448.400626:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.400627:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:1.0:1619677448.400627:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.400628:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.400631:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.400633:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.400636:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.400637:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.400638:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.400639:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.400640:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.400640:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.400641:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677448.400643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677448.400645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:26.0:1619677448.400650:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677448.400652:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:26.0:1619677448.400653:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.400653:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677448.400654:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.400655:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.400655:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.400656:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.400657:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.400658:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.400659:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.400660:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.400660:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:26.0:1619677448.400662:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.400663:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.400664:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:9.2:1619677448.400664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000400:00000200:26.0:1619677448.400667:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.400669:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.400670:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.400670:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:26.0:1619677448.400671:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.400672:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:24.0:1619677448.400672:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:26.0:1619677448.400673:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:24.0:1619677448.400673:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.400673:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:26.0:1619677448.400674:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677448.400674:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:26.0:1619677448.400675:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.400675:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:24.0:1619677448.400675:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:25.2:1619677448.400679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.400685:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.400687:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.400687:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.400688:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.400689:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.400691:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.400773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.400776:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.400778:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.400781:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.400781:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.400782:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.400784:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.400785:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.400787:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.400787:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.400789:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.400792:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.400794:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.400797:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.400797:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.400798:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.400800:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.400800:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.400801:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.2:1619677448.400801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:20.0:1619677448.400802:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.400806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:2.0:1619677448.400807:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.400809:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:21.2:1619677448.400810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:18.0:1619677448.400810:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.400812:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.400813:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.400815:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:2.0:1619677448.400815:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.400816:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.400816:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677448.400816:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.400817:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.400818:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:31.0:1619677448.400819:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.400819:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:2.0:1619677448.400819:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.400820:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677448.400820:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.400822:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:2.0:1619677448.400822:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.400823:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.400826:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.400826:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.400828:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:2.0:1619677448.400828:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.400829:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715426] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.400830:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.400831:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.400831:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.400832:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:2.0:1619677448.400832:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.400832:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.400834:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.400834:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.400835:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.400835:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:6.0:1619677448.400836:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.400838:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.2:1619677448.400840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.400841:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.400842:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.400843:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.400845:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.400845:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.400846:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.400846:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677448.400848:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:25.2:1619677448.400850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.400850:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.400851:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.400851:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.400853:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.400856:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677448.400857:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.400860:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:8.2:1619677448.400861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000400:00000200:31.0:1619677448.400862:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.400863:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.400864:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.400866:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:1.0:1619677448.400868:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.400871:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.400872:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.400872:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.400873:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.400874:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.400958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.400964:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.400967:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.400968:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.400970:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.400972:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.400974:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.400975:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.400976:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.400978:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.400979:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.400982:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.400984:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.400987:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.400987:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.400988:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.400990:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.400990:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.400992:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.400993:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.400993:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.400996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.400998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677448.401002:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.401004:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:2.0:1619677448.401004:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:10.0:1619677448.401005:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.401006:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.401006:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.401006:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.401007:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:26.0:1619677448.401008:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.401011:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.401014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.401020:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.401022:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.401023:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.401025:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.401027:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.401029:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.401030:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.401032:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.401033:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.401033:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.401036:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.401038:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.401041:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.401042:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.401042:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.401044:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.401045:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.401045:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.401046:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.401050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.401057:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.401059:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.401060:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.401062:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.401063:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.401065:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.401150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.401156:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.401159:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.401161:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.401161:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.401164:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.401165:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.401167:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715423] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.401168:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.401169:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.401170:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.401173:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.401174:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.401177:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.401178:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.401178:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.401180:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.401180:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.401181:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.401183:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.401183:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.401186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.401188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.401191:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:22.0:1619677448.401193:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677448.401193:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.401194:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.401194:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677448.401196:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.401197:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.401198:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.401199:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.401202:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.401283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.401291:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:4.2:1619677448.401292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.401299:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.401300:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.401301:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.401302:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.401303:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.401304:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.401306:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.401306:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.401308:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.401310:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.401311:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.401312:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.401313:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.401314:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.401316:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.401317:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.401317:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.401318:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.401319:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.401319:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.401323:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.401323:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.401324:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.401325:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.401325:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.401327:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.401327:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.401328:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.401328:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.401329:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677448.401329:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.401330:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (23)--
00000800:00000200:26.0:1619677448.401330:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.401330:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677448.401331:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.401331:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.401331:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (23)--
00000800:00000200:25.2:1619677448.401334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.401343:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (23)++
00000800:00000200:22.0:1619677448.401346:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (24)++
00000800:00000200:6.0:1619677448.401346:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000800:00000200:22.0:1619677448.401347:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000400:00000200:6.0:1619677448.401347:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.401348:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.401349:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.401350:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.401353:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.401353:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.401354:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.401395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.401401:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.401404:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:5.2:1619677448.401404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.401405:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.401407:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.401410:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.401410:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677448.401411:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.2:1619677448.401413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:18.0:1619677448.401413:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.401413:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.401414:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677448.401415:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.401416:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.401418:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.401418:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.401420:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.401421:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:18.0:1619677448.401422:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.401422:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.401423:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.401424:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:18.0:1619677448.401425:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.401425:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.401426:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.401428:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:18.0:1619677448.401428:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.401428:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677448.401429:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.401429:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.401430:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677448.401431:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.401432:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.401432:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.401432:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.401433:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677448.401433:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:30.0:1619677448.401434:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:18.0:1619677448.401434:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:30.0:1619677448.401435:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96715418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.401437:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.401437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.401437:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.401438:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.401438:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.401438:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:30.0:1619677448.401439:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.401440:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.401440:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.401441:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.401442:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677448.401443:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.401445:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.401445:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:9.2:1619677448.401446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:30.0:1619677448.401448:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.401448:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.401448:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.401449:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.401450:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:13.0:1619677448.401450:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.401451:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.401452:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.401452:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677448.401452:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:30.0:1619677448.401453:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.401453:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677448.401453:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677448.401455:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.401456:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.2:1619677448.401458:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:8.0:1619677448.401461:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.401463:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.401465:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:8.0:1619677448.401466:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.0:1619677448.401468:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.401469:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.401471:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.401472:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.401474:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.401475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.401483:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.401486:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.401487:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.401488:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.401488:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.401489:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.401744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.401750:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.401753:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.401754:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.401756:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.401758:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.401760:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.401761:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.401763:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.401764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:17.0:1619677448.401764:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.401765:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.401768:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.401770:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.401771:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.401773:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.401774:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677448.401774:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.401774:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.401776:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.401776:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.401777:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.401778:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677448.401778:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:31.0:1619677448.401779:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.401779:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.401780:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677448.401781:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.401782:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715416] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.401782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.401783:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.401784:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.401784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.401785:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.401789:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.401789:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677448.401790:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.401790:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677448.401791:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677448.401792:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.401792:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.401793:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677448.401793:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.401793:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.401794:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:12.0:1619677448.401794:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.401795:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.401795:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677448.401797:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677448.401797:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.401797:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677448.401798:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.401799:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.401800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.401806:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.401809:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.401810:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.401811:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677448.401811:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.401813:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.401815:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.401816:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.401818:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.401819:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.401820:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.401887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.401892:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.401895:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.401897:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.401897:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.401899:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.401901:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.401902:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.401903:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.401904:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.401906:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.401909:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.401910:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.401913:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.401914:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.401914:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.401916:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.401917:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.401917:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.401918:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.401922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.401929:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.401932:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.401933:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.401933:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.401935:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.401937:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.401940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.401947:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.401949:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.401950:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.401950:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:21.2:1619677448.401951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:6.0:1619677448.401951:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.401952:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677448.401958:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.401961:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.401962:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.401964:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.401967:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.401968:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.401969:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.401971:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.401972:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.401973:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677448.401976:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.401978:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.401981:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.401982:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.401983:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.401985:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.401985:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.401986:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.401986:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.401992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.402000:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.402003:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.402004:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.402005:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.402006:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.402008:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.402042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.402048:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.402050:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.402051:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.402053:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.402056:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.402057:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.402058:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.402060:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.402061:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.402062:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.402065:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.402067:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.402069:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.402070:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.402070:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.402072:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.402073:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.402073:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.402074:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.402079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.402086:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.402088:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.402090:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.402090:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.402092:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.402094:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.402109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.2:1619677448.402113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.402117:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.402118:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.402120:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.402121:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.402121:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.402122:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.402124:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.402125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:17.0:1619677448.402125:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.402127:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.402128:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.402128:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677448.402130:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.402130:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.402130:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:10.0:1619677448.402131:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.402132:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.402133:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.402133:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.402133:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:10.0:1619677448.402134:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.402135:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:17.0:1619677448.402135:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.402136:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.402136:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677448.402138:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.402140:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.402140:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.402141:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.402141:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96715410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.402143:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.402143:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.402143:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.402144:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.402145:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.402146:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.402147:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.402147:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:17.0:1619677448.402148:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.402148:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:17.0:1619677448.402149:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.402149:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:17.0:1619677448.402150:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.402150:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:5.0:1619677448.402150:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677448.402151:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.402151:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.402152:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.402152:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.402152:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.0:1619677448.402152:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.402154:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677448.402155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.402155:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.402155:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.402157:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:25.2:1619677448.402158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.402158:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.402158:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.402159:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:26.0:1619677448.402161:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.402163:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:8.2:1619677448.402163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677448.402164:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.402164:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:12.0:1619677448.402165:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:26.0:1619677448.402166:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.402167:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:26.0:1619677448.402168:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:12.0:1619677448.402168:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.402170:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.402171:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.402171:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:12.0:1619677448.402172:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.402174:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.402175:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.402175:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.402177:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677448.402180:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.402253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.402258:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.402261:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.402262:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.402264:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.402267:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.402268:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.402269:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.402271:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.402272:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.402273:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.402276:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.402278:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.402279:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.402280:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.402281:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.402282:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.402283:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.402284:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.402285:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.402285:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677448.402288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.402291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.402300:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.402300:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.402301:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.402301:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.402301:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.402301:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.402302:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.402303:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:2.0:1619677448.402303:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.402305:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.402329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.402337:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.402340:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.402340:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.402342:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.402344:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.402346:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.402347:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.402348:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.402349:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.402350:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.2:1619677448.402354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:10.0:1619677448.402354:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.402356:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.402359:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.402359:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.402360:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.402362:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.402362:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.402362:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.402363:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.402363:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.402365:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.402367:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.402369:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.402369:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.402369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:30.0:1619677448.402371:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.402372:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.402373:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.402374:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.402375:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:6.0:1619677448.402377:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:30.0:1619677448.402378:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.402380:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.402380:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:30.0:1619677448.402381:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:6.0:1619677448.402381:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.402383:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.402384:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.402385:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.402385:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:6.0:1619677448.402386:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:30.0:1619677448.402388:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.402388:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.402389:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:25.2:1619677448.402389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677448.402391:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.402392:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.402394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677448.402396:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.2:1619677448.402398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:22.0:1619677448.402399:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.402400:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.402400:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.402401:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.402402:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677448.402405:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:31.0:1619677448.402406:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.402408:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:20.0:1619677448.402408:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:31.0:1619677448.402409:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.402409:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.402410:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.402411:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.402411:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.402414:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.402467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.402473:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.402475:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.402476:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.402478:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.402481:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.402482:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.402484:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.402485:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.402486:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.402487:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.402491:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.402493:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.402495:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.402496:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.402497:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.402499:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.402500:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.402500:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.402501:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.402505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.402505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.402513:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677448.402513:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:26.0:1619677448.402515:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:13.0:1619677448.402516:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:10.0:1619677448.402516:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:13.0:1619677448.402517:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.402518:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.402519:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.402520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:12.0:1619677448.402520:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:26.0:1619677448.402521:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.402522:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:26.0:1619677448.402523:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.402524:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715405] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.402525:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.402526:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.402527:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677448.402527:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.402530:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677448.402531:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.402533:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.402533:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.402534:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.402536:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.402536:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.402537:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:26.0:1619677448.402537:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:31.0:1619677448.402538:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.402538:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677448.402540:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.402540:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.402540:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.402540:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677448.402541:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.402542:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.402543:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.402543:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:31.0:1619677448.402544:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.402545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.402546:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.402549:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.402549:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.2:1619677448.402549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.402550:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.402552:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.402552:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.402553:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677448.402553:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677448.402554:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.402556:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:18.0:1619677448.402557:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.402557:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:25.2:1619677448.402558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:18.0:1619677448.402558:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.402558:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.402559:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.402560:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:18.0:1619677448.402560:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.402564:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677448.402565:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.402568:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.402569:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.402570:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.402572:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.402575:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.402602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.402608:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.402611:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.402612:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.402614:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.402616:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.402617:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.402618:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.402619:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.402621:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.402622:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.402624:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.402626:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.402628:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.402629:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.402629:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.402631:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.402632:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.402632:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.402633:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.402637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.402644:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.402646:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.402646:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.402648:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.402648:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.402650:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.402655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.402661:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.402663:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.402666:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.402666:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.402668:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.402670:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.402671:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.402672:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.402673:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.402674:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.402678:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.402679:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.402682:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.402683:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.402684:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.402686:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.402686:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.402687:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.402688:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.402692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.402698:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.402701:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.402702:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.402704:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.402705:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.402707:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.402710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677448.402717:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.402720:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.402720:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.402721:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.402722:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.402723:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.402932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.402937:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.402940:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.402941:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.402943:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.402945:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.402947:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.402948:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.402950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:1.0:1619677448.402950:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.402951:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.402952:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.402955:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.402956:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.402958:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677448.402959:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.402960:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.402960:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.402961:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.402961:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:1.0:1619677448.402962:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.402962:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.402963:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.402963:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.402964:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:10.0:1619677448.402966:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.402968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:10.0:1619677448.402968:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.402969:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.402970:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.402971:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.402972:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.402976:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:10.0:1619677448.402976:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.402978:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:10.0:1619677448.402978:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.402979:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.402980:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.402981:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.402981:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (21)++
00000400:00000200:22.0:1619677448.402982:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.402982:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677448.402984:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (23)--
00000800:00000200:10.0:1619677448.402984:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677448.402985:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.402985:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.402986:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.402990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.402997:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.402999:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.403000:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.403001:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.403002:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.403004:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.403016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.403023:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.403025:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.403026:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.403028:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.403030:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.403032:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.403033:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.403034:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.403035:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.403036:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.403039:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.403040:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.403042:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.403043:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.403043:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.403045:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.403045:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.403046:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.403047:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.403048:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.403050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.403053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:4.2:1619677448.403053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.2:1619677448.403058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677448.403060:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:30.0:1619677448.403061:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:10.0:1619677448.403062:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:31.0:1619677448.403063:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.403063:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:26.0:1619677448.403063:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.403063:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.403064:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.403064:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.403066:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.403067:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.403068:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:31.0:1619677448.403069:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.403069:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.403070:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.403071:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.403072:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.403073:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.403074:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.403075:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.403078:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.403079:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.403080:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.403081:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.403082:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.403082:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:30.0:1619677448.403083:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.403084:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677448.403084:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.403085:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.403087:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.403087:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.403088:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.403089:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (22)++
00000800:00000200:25.2:1619677448.403089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.403090:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.403090:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677448.403090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:30.0:1619677448.403091:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.403092:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.403093:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.403093:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.403094:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.403095:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.403095:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.403096:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.403096:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.403096:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:31.0:1619677448.403097:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.403097:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.403097:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:31.0:1619677448.403098:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.403098:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.403099:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.403099:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.403099:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.403100:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:31.0:1619677448.403101:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.403101:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:10.0:1619677448.403102:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:25.2:1619677448.403103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:10.0:1619677448.403104:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.403106:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.403107:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.403109:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.403110:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:10.0:1619677448.403110:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.403111:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.403111:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.403112:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.403114:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.403116:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.403116:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.403117:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.403118:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.403121:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.403122:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.403123:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:25.2:1619677448.403124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.403124:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.403125:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.403125:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.403126:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:10.0:1619677448.403128:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.403129:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.403130:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.403130:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:8.2:1619677448.403130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.403131:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.403132:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.0:1619677448.403137:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.403140:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.403141:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.403142:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.403143:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.403146:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.403265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:5.0:1619677448.403270:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:5.0:1619677448.403272:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:20.0:1619677448.403275:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:5.0:1619677448.403275:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.403278:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.403279:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:5.0:1619677448.403281:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.403282:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.403283:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.403284:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:5.0:1619677448.403287:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.403289:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:5.0:1619677448.403292:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.403293:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:5.2:1619677448.403298:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.0:1619677448.403302:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:17.0:1619677448.403304:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.403304:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:5.0:1619677448.403305:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:5.0:1619677448.403306:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:5.0:1619677448.403306:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:17.0:1619677448.403307:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.403308:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.403310:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.403311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000400:00000200:17.0:1619677448.403313:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.403314:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.403316:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.403317:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:17.0:1619677448.403317:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.403318:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.403319:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:26.0:1619677448.403320:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.403321:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.403322:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.403322:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.403324:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:26.0:1619677448.403325:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.403327:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:17.0:1619677448.403327:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.403328:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.403329:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.403331:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.403331:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.403332:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.403333:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.403334:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.403337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.403339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.403343:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:1.0:1619677448.403344:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677448.403346:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:1.0:1619677448.403346:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:24.0:1619677448.403347:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:1.0:1619677448.403347:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.403348:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.403348:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.403350:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.403353:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.2:1619677448.403386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.403394:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.403396:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.403397:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.403399:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.403401:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.403403:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.403404:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.403405:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.403406:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.403407:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.403411:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.403412:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.403415:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.403416:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.403416:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.403418:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.403419:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.403419:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.403420:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677448.403435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:25.2:1619677448.403436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.403444:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:22.0:1619677448.403444:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:31.0:1619677448.403447:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:22.0:1619677448.403447:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.403448:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.403448:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:31.0:1619677448.403449:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.403450:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.403451:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.403452:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:30.0:1619677448.403452:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.403453:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.403454:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:31.0:1619677448.403455:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.403456:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.403457:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.403460:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.403461:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.403463:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.403464:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.403464:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.403466:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.403466:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.403467:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.403468:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.403471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.403478:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.403480:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.403481:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.403483:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.403484:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.403485:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.403491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.403497:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.403500:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.403501:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.403501:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.403502:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.403503:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.403506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.403513:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.403515:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.403516:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.403518:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.403521:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.403523:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.403524:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.403525:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.403526:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.403527:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.403530:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.403531:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.403534:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.403535:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.403535:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.403537:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.403537:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.403538:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.403538:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.403543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.403550:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.403553:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.403553:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.403554:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.403556:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.403559:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.403622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.403630:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.403632:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.403634:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.403636:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.403636:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.403638:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677448.403639:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.403640:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.403641:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.403642:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677448.403645:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.403647:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677448.403649:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.403650:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.403651:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.403652:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.403653:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.403654:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.403656:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.403656:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677448.403658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:21.2:1619677448.403662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677448.403664:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.403666:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.403667:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.403667:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.403669:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.403670:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:24.0:1619677448.403671:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:30.0:1619677448.403672:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.403673:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.403674:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677448.403690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.403696:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.403699:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.403699:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677448.403702:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.403704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:8.0:1619677448.403704:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.403706:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677448.403707:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:8.0:1619677448.403707:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.403708:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.403709:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.403710:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:8.0:1619677448.403710:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.403713:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.403713:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:8.0:1619677448.403715:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.403716:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.403717:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.403718:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.403720:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.403720:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.403720:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:20.0:1619677448.403721:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.403721:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.403722:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.403723:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677448.403723:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677448.403723:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.403724:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.403724:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:20.0:1619677448.403727:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.403728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.403728:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.403731:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.403732:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.403733:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:20.0:1619677448.403735:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:20.0:1619677448.403736:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677448.403736:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:20.0:1619677448.403737:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.403737:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.403739:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677448.403740:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.403741:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.403743:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.403744:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.403744:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677448.403745:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.403745:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677448.403746:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.403749:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.403756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.403762:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.403765:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.403765:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.403766:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.403766:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.403767:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.403934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.2:1619677448.403938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:1.0:1619677448.403940:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.403942:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.403943:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677448.403944:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.403946:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:1.0:1619677448.403946:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.403947:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677448.403948:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.403949:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.403950:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.403951:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.403952:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.403952:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.403954:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677448.403954:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.403955:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.403956:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.403958:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.403959:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.403959:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.403960:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.2:1619677448.403960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677448.403962:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.403964:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.403965:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.403966:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.403966:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.403967:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.403968:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:1.0:1619677448.403968:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.403968:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.403969:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.403969:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.403970:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.403971:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:22.0:1619677448.403971:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.403972:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.403973:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.403973:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.403974:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:22.0:1619677448.403974:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.2:1619677448.403974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.403975:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.403975:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:30.0:1619677448.403976:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.403977:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.403978:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96715385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.403980:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677448.403980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.403981:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.403982:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.403982:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.403984:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:30.0:1619677448.403985:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.403985:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.403986:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.403987:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:17.0:1619677448.403987:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.403987:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:17.0:1619677448.403989:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677448.403990:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677448.403991:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.403991:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)++
00000400:00000200:5.0:1619677448.403991:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.403991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.403992:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (23)++
00000400:00000200:5.0:1619677448.403993:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.403994:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:30.0:1619677448.403995:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.403996:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677448.403996:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677448.403997:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.403997:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.403999:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677448.404000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.404002:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.404002:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.404005:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.404007:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:10.0:1619677448.404007:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.404009:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:20.0:1619677448.404010:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:10.0:1619677448.404010:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.404011:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.404011:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.404013:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.404013:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.404013:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:10.0:1619677448.404014:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.404019:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.404020:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:10.0:1619677448.404020:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.404023:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.404024:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.404025:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.404027:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.404027:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.404028:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.404028:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.404032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.404040:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.404042:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.404043:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.404045:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.404046:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.404048:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.404051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.404059:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.404061:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.404062:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.404063:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.404063:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.404065:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.404265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.404271:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.404274:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.404275:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.404277:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.404279:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.404281:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.404282:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.404284:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.404284:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.404285:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.404289:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.404290:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.404293:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.404294:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.404299:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.404300:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.404301:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.404302:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.404302:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.404306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.404314:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.404317:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.404318:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.404318:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.404321:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.404323:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.404459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.404465:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.404467:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.404469:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.404470:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.404472:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.404473:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.404475:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.404476:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.404477:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.404478:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.404482:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.404483:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.404486:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.404487:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.404488:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.404490:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.404490:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.404492:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.404493:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.404494:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.404496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.404499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.404502:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.404504:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.404504:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677448.404504:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.404505:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.404506:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.404507:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.404509:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.404511:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.404512:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.404556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.404564:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.404567:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.404567:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.404569:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.404572:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.404574:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.404575:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.404575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:26.0:1619677448.404576:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.404577:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.404578:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677448.404581:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:26.0:1619677448.404582:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.404583:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677448.404584:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.404585:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677448.404586:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.404587:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:6.0:1619677448.404587:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.404588:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.404590:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:6.0:1619677448.404590:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.404591:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.404591:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.404591:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:26.0:1619677448.404592:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.404592:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.404594:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.404595:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677448.404596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677448.404596:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.404599:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.404601:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.404604:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.404604:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.404605:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.404605:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677448.404607:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.404607:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677448.404607:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.404609:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677448.404609:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:2.0:1619677448.404609:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.404610:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.404610:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677448.404611:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.404612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.404614:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.404615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.404620:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677448.404621:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:17.0:1619677448.404622:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:9.2:1619677448.404622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:22.0:1619677448.404623:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.404624:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:17.0:1619677448.404624:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.404624:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.404625:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:17.0:1619677448.404626:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.404629:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.404629:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677448.404632:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:8.2:1619677448.404632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.404633:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.404633:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.404634:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.404635:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677448.404639:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.404642:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.404642:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.404643:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.404644:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.404644:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.404690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.404696:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.404698:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.404699:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.404701:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.404702:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.404703:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.404704:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.404706:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.404706:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.404707:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.404710:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.404712:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.404713:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.404714:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.404715:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.404716:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.404717:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.404717:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.404718:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.404722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.404729:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.404731:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.404732:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.404732:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.404733:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.404735:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.404835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.404840:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.404843:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.404844:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.404845:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.404847:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.404849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:2.0:1619677448.404849:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.404850:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.404852:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.404853:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.404854:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.404856:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:2.0:1619677448.404857:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.404859:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.404860:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.404860:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:2.0:1619677448.404862:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.404863:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:2.0:1619677448.404863:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.404864:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:10.0:1619677448.404865:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.404865:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.404866:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.404866:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:10.0:1619677448.404867:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:2.0:1619677448.404867:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:10.0:1619677448.404868:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.404869:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.404870:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.404871:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.2:1619677448.404871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:10.0:1619677448.404874:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.404876:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.404878:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:10.0:1619677448.404879:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.404880:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.404880:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.404881:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.404882:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.404882:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:20.0:1619677448.404883:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.404883:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.404883:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.404884:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:5.0:1619677448.404884:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.404886:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:9.2:1619677448.404888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.404896:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.404898:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.404899:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.404900:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.404902:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.404905:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.404979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.404986:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.404989:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.404989:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677448.404991:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.404994:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.404996:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.404997:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.404998:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.404999:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.405000:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.405003:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.405004:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.405007:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.405008:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.405008:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.405009:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.405010:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.405011:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.405011:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.405015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.405022:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.405024:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.405025:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.405026:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.405028:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.405029:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.405033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:21.2:1619677448.405036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:12.0:1619677448.405039:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.405042:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.405042:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677448.405042:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:12.0:1619677448.405043:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.405044:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.405045:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677448.405045:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.405046:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.405047:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.405050:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.405051:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677448.405053:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.405054:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.405055:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.405056:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677448.405059:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.405061:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677448.405063:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.405064:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677448.405065:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677448.405066:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677448.405067:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.405067:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.405068:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.405072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.405080:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.405083:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.405084:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.405084:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.405086:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:6.0:1619677448.405089:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.405093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:22.0:1619677448.405100:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.405103:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.405103:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.405104:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.405105:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.405105:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.405139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.2:1619677448.405141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:31.0:1619677448.405147:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.405147:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.405149:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:31.0:1619677448.405151:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677448.405152:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.405153:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.405154:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.405155:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:5.0:1619677448.405155:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.405157:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.405157:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.405159:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.405159:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.405160:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.405161:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.405162:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.405162:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.405163:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.405164:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.405165:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.405169:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.405170:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.405172:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.405173:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.405175:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.405176:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.405176:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.405177:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.405177:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.405178:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.405179:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.405179:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.405180:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.405181:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:31.0:1619677448.405182:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.405182:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677448.405182:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.405183:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.405185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:25.2:1619677448.405188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:13.0:1619677448.405191:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.405194:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:26.0:1619677448.405195:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:13.0:1619677448.405195:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.405196:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.405197:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:12.0:1619677448.405197:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:26.0:1619677448.405198:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.405198:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.405198:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.405200:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.405204:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.405365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.405373:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.405376:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.405377:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.405378:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.405381:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.405383:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.405384:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.405385:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.405386:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.405388:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.405391:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.405393:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.405396:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.405397:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.405397:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.405399:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.405400:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.405401:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.405402:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.405403:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.2:1619677448.405406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:8.2:1619677448.405406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.405408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.405412:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.405415:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677448.405415:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:6.0:1619677448.405415:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:26.0:1619677448.405416:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:20.0:1619677448.405417:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:6.0:1619677448.405417:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.405418:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:20.0:1619677448.405418:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.405419:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.405420:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:20.0:1619677448.405420:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.405420:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.405421:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:6.0:1619677448.405422:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:20.0:1619677448.405423:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:6.0:1619677448.405423:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.405424:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.405425:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.405426:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.405430:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.405431:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.405434:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.405435:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.405435:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.405437:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.405438:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.405439:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.405439:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.405444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.405452:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.405454:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.405455:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.405457:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.405459:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.405460:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.405477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.405483:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.405486:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.405487:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.405489:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.405492:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.405493:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677448.405495:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.405496:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.405497:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.405498:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677448.405501:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.405503:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.405506:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.405507:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:18.0:1619677448.405508:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677448.405509:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677448.405510:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.405511:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.405511:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.405515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.405523:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.405526:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.405526:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.405528:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.405530:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.405531:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.405618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.405625:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.405628:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.405629:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.405631:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.405634:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.405636:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.405637:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.405638:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.405639:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.405640:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.405644:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.405645:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.405648:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.405648:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.405649:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.405651:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.405651:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.405652:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.405652:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.405656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.405663:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:21.2:1619677448.405664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:13.0:1619677448.405665:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.405666:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.405667:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.405669:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.405669:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.405671:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.405673:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.405675:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.405677:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.405677:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.405678:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.405679:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.405681:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.405681:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.405682:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.405685:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.405687:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.405689:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.405689:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.405690:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.405691:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.405692:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.405693:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.405693:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.405697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.405703:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.405706:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.405706:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.405708:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.405710:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.405711:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.405727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.405734:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.405737:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.405738:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.405740:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.405742:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.405744:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.405745:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.405746:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.405747:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.405747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:24.0:1619677448.405748:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.405751:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.405752:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.405752:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:24.0:1619677448.405755:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.405755:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:24.0:1619677448.405756:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.405757:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.405757:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.405757:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.405758:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.405759:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.405760:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:5.0:1619677448.405760:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.405761:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000400:00000200:5.0:1619677448.405761:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:24.0:1619677448.405762:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:5.0:1619677448.405762:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.405764:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677448.405765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677448.405765:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.405766:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:4.2:1619677448.405767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:5.0:1619677448.405769:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.405771:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.405773:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677448.405773:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.405774:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.405774:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.405775:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.405776:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:5.0:1619677448.405776:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.405776:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:17.0:1619677448.405777:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.405777:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:5.0:1619677448.405777:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677448.405777:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.405778:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677448.405778:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:24.0:1619677448.405779:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677448.405779:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.405782:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.405782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.405790:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.405792:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.405793:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.405795:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.405796:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.405797:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.405904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.405910:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.405912:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.405913:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.405915:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.405916:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.405918:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.405919:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.405921:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.405922:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.405923:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.405926:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.405927:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.405930:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.405931:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.405932:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.405933:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.405934:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.405935:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.405935:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.405939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.405946:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.405949:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677448.405949:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677448.405950:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.405952:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.405955:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677448.405975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.405981:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.405984:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.405985:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.405986:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.405989:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.405991:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.405992:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.405993:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.405994:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.405995:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.405998:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.405999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677448.406000:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.406002:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.406003:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.406004:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.406005:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.406006:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677448.406006:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.406007:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.406008:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:26.0:1619677448.406009:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:24.0:1619677448.406009:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:10.0:1619677448.406010:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.406011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:26.0:1619677448.406012:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.406014:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.406014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000400:00000200:26.0:1619677448.406015:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.406016:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.406017:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.406017:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:26.0:1619677448.406018:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.406019:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:13.0:1619677448.406020:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.406020:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.406021:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:13.0:1619677448.406022:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.406023:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.406023:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:12.0:1619677448.406024:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.406024:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:26.0:1619677448.406025:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:13.0:1619677448.406025:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677448.406025:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:26.0:1619677448.406026:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.406027:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.406028:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.406029:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.406030:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.406030:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.406031:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.406035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.406041:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.406043:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.406043:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.406044:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.406045:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.406047:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.406267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:20.2:1619677448.406268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677448.406272:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677448.406273:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677448.406276:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677448.406276:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677448.406276:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:20.0:1619677448.406278:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.406280:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.406280:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.406283:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.406284:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.406285:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.406286:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.406286:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.406287:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.406288:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.406289:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.406289:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.406290:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.406291:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.406292:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.406301:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.406302:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.406304:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.406305:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.406306:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.406307:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.406308:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.406308:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.406309:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.406310:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.406310:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.406310:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.406311:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.406312:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677448.406312:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.406312:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.406313:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.406314:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.406317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:25.2:1619677448.406321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.406325:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.406328:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:12.0:1619677448.406328:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:22.0:1619677448.406329:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.406329:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.406331:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.406331:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.406331:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.406333:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:12.0:1619677448.406333:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.406335:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.406336:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.406338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:21.2:1619677448.406343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.2:1619677448.406343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677448.406344:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.406347:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.406348:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.406348:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.406349:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.406350:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.406350:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.406352:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.406352:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.406354:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.406355:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:17.0:1619677448.406356:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.406358:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.406359:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.406360:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.406360:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.406361:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.406362:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.406363:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.406364:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.406365:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.406365:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.406366:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.406367:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.406367:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.406368:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.406371:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.406373:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.406376:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.406376:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.406377:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.406378:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.406378:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.406379:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.406380:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.406380:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.406381:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.406381:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:30.0:1619677448.406382:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.406382:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:30.0:1619677448.406383:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677448.406383:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.406384:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.406385:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.406385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.406386:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677448.406388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.406391:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677448.406393:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:1.0:1619677448.406393:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:12.0:1619677448.406394:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.406395:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:12.0:1619677448.406396:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.406396:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.406397:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:1.0:1619677448.406397:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.406398:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.406398:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677448.406399:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.406399:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677448.406401:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.406403:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.406428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.406434:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.406436:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.406437:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.2:1619677448.406439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:6.0:1619677448.406439:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.406442:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.406443:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.406444:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.406445:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.406446:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:6.0:1619677448.406447:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.406448:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.406450:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:6.0:1619677448.406452:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.406453:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:6.0:1619677448.406454:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:31.0:1619677448.406456:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.406457:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.406458:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.406458:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.406459:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:31.0:1619677448.406460:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.406461:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.406461:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.406461:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:31.0:1619677448.406462:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.406462:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.406463:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:31.0:1619677448.406464:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.406464:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:25.2:1619677448.406467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:31.0:1619677448.406468:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.406470:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.406473:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.406473:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.406474:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.406475:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.406475:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.406476:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.406477:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.406477:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:30.0:1619677448.406478:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.406479:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.406480:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.406480:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.406481:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.406482:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.2:1619677448.406483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.406487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:21.2:1619677448.406492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:20.0:1619677448.406494:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677448.406495:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.406497:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:20.0:1619677448.406497:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677448.406498:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:18.0:1619677448.406498:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:31.0:1619677448.406499:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.406499:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.406500:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:4.2:1619677448.406500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:20.0:1619677448.406501:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.406501:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:5.0:1619677448.406502:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.406502:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677448.406503:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:5.0:1619677448.406504:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (24)--
00000400:00000200:18.0:1619677448.406505:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.406505:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.406506:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.406507:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677448.406507:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:18.0:1619677448.406508:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.406510:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:26.0:1619677448.406511:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:18.0:1619677448.406511:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.406511:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677448.406513:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.406515:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.406516:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.406516:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.406517:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.406519:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.406521:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.406521:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.406522:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.406523:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.406523:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.406524:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.406527:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.406527:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:18.0:1619677448.406528:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:26.0:1619677448.406529:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.406530:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677448.406530:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677448.406531:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.406531:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.406532:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:26.0:1619677448.406534:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.406534:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.406535:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:25.2:1619677448.406536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.406537:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.406537:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.406538:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.406539:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:12.0:1619677448.406542:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:25.2:1619677448.406543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:12.0:1619677448.406545:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.406546:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.406547:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.406549:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677448.406550:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:12.0:1619677448.406550:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.406553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:22.0:1619677448.406553:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.406554:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.406555:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.406557:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.406560:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677448.406560:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:25.2:1619677448.406562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:13.0:1619677448.406563:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.406563:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.406564:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.406564:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.406566:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677448.406569:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.406572:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.406573:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.406573:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.406574:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.406575:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.406769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677448.406771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.406777:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.406778:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.406779:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.406780:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.406782:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.406783:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.406786:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.406787:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.406788:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.406789:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.406790:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.406791:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.406792:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.406795:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.406797:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.406797:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.406798:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.406799:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.406799:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.406800:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.406803:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.406804:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.406804:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677448.406805:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.406806:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.406807:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.406807:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.2:1619677448.406808:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677448.406810:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.406810:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.406811:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:9.2:1619677448.406812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.406812:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.406813:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.406813:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.406814:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.406814:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677448.406815:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.406816:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:10.0:1619677448.406816:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677448.406817:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.406817:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.406819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.406819:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.406820:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:2.0:1619677448.406820:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.2:1619677448.406822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000400:00000200:10.0:1619677448.406823:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.406823:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.406824:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.406824:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.406826:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:10.0:1619677448.406826:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.406826:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.406828:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:10.0:1619677448.406828:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:5.0:1619677448.406828:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000400:00000200:31.0:1619677448.406829:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.406829:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.406829:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677448.406830:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:31.0:1619677448.406831:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.406831:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.406831:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:10.0:1619677448.406832:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.406832:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677448.406833:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:10.0:1619677448.406833:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.406835:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:10.0:1619677448.406836:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.406837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.406838:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.406840:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.406840:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.406841:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.406842:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.406843:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.406843:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.406844:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.406845:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:30.0:1619677448.406846:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.406846:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.406847:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.406848:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.406848:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:25.2:1619677448.406848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.406850:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.406850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677448.406853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677448.406855:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:26.0:1619677448.406858:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:18.0:1619677448.406858:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:26.0:1619677448.406859:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:2.0:1619677448.406859:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:26.0:1619677448.406860:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.406862:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.406863:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.406863:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:18.0:1619677448.406864:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.406864:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.406864:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.406865:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.406865:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.406866:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:18.0:1619677448.406866:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.406867:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.406869:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.406870:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.406871:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.406874:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.406876:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.406878:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.406879:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.406880:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.406882:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.406883:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.406883:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.406884:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.406888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.406894:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.406897:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.406898:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.406900:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.406901:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.406903:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.406922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.406928:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:4.2:1619677448.406928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677448.406931:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.406931:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.406934:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.406934:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677448.406937:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.406937:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.406938:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677448.406938:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.406940:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.406941:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.406942:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.406943:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.406943:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.406944:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.406945:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.406946:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.406949:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.406949:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.406950:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.406951:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.406952:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.406955:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.406956:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:10.0:1619677448.406956:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.406957:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.406958:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.406958:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.406959:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.406959:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.406960:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.406961:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.406962:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.406962:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:25.2:1619677448.406964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.406964:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.406964:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.406965:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.406966:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.406970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:13.0:1619677448.406971:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.406973:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.406974:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.406976:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.406977:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677448.406978:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:13.0:1619677448.406979:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.406980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:22.0:1619677448.406981:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.406982:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.406982:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.406985:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.406987:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677448.406987:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.406990:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.406990:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.406991:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.406992:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.406993:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.407037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.407044:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.407047:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.407048:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.407050:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.407052:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.407053:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.407055:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.407056:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.407057:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.407058:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.407062:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.407063:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.407066:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.407067:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.407067:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.407069:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.407069:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.407070:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.407070:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.407075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.407082:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.407085:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.407086:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.407086:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.407089:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.407092:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.407233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:21.2:1619677448.407238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.407242:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.407244:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.407246:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.407246:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.407247:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.2:1619677448.407248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:31.0:1619677448.407249:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.407250:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.407250:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.407251:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.407253:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.407254:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.407255:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.407255:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:30.0:1619677448.407257:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.407258:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.407258:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:26.0:1619677448.407258:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:30.0:1619677448.407259:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.407259:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.407261:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.407261:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.407262:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.407262:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.407263:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.407266:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.407266:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:30.0:1619677448.407266:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.407267:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.407267:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.407268:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:26.0:1619677448.407268:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.407269:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.407269:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.407270:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.407270:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:26.0:1619677448.407270:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:31.0:1619677448.407271:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.407271:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:26.0:1619677448.407271:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96715345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.407272:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.407272:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.407272:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:26.0:1619677448.407272:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.407273:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:26.0:1619677448.407273:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.407274:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:26.0:1619677448.407274:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:8.2:1619677448.407275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.407276:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.407276:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677448.407277:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.407278:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:25.2:1619677448.407278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.407281:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.407282:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.407283:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:21.2:1619677448.407283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.407284:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:26.0:1619677448.407285:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.407285:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.407285:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.407286:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.407286:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.407287:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:10.0:1619677448.407287:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.407287:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:26.0:1619677448.407288:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:21.2:1619677448.407288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:5.0:1619677448.407288:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.407289:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.407290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.407290:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.2:1619677448.407292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677448.407292:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:5.0:1619677448.407292:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.407293:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:25.2:1619677448.407298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.407301:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.407302:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.407303:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.407303:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.407303:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.407303:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677448.407303:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:31.0:1619677448.407304:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677448.407305:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:5.0:1619677448.407305:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677448.407306:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677448.407306:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.407306:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:24.0:1619677448.407306:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.407306:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:5.0:1619677448.407307:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677448.407309:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:26.0:1619677448.407309:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677448.407309:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.407310:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.407310:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:8.0:1619677448.407310:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.407311:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.407311:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:20.0:1619677448.407312:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.407313:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.407313:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.407315:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.407316:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677448.407317:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.407317:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.407318:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.407319:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.407319:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.407320:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.407322:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677448.407324:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.407326:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.407327:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.407329:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.407330:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.407331:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.407332:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.407332:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.407333:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.407333:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.407334:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677448.407334:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.407335:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.407336:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:8.0:1619677448.407336:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677448.407337:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:8.0:1619677448.407337:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677448.407338:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.0:1619677448.407338:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.407340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.2:1619677448.407341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.407344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.407348:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677448.407348:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677448.407349:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.407350:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:20.0:1619677448.407351:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.407351:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:20.0:1619677448.407352:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.407352:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677448.407352:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.407353:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.407353:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:2.0:1619677448.407353:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.407354:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677448.407355:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:2.0:1619677448.407355:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.407358:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.407450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.407456:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.407460:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.407460:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.407462:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.407465:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.407466:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.407468:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.407469:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.407470:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.407471:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.407475:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.407476:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.407479:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.407480:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.407481:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.407482:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.407483:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.407484:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.407484:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.407489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.407497:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.407500:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.407501:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.407501:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.407503:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.407505:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.407533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.407539:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.407542:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.407543:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.407545:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.407548:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.407549:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.407551:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.407552:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.407553:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.407554:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.407557:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.407558:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.407560:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.407561:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.407561:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.407563:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.407564:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.407565:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.407566:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.407566:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677448.407569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.407572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.407578:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.407579:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.407579:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.407580:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:2.0:1619677448.407580:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.407581:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.407582:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.407582:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.407583:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677448.407583:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.407590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.407597:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.407600:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.407600:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.407601:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.407601:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.407601:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:4.2:1619677448.407663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677448.407665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.407672:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.407674:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.407675:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.407675:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677448.407678:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.407678:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.407682:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.407682:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.407683:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.407685:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.407686:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.407687:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.407688:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.407690:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.407692:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.407693:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.407693:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.407694:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.407695:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.407695:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.2:1619677448.407697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:26.0:1619677448.407699:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.407699:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.407700:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.407701:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.407701:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.407702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:22.0:1619677448.407703:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.407703:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.407704:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.407705:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.407705:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (23)++
00000800:00000200:26.0:1619677448.407706:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.407706:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677448.407706:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:10.0:1619677448.407706:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (24)--
00000800:00000200:6.0:1619677448.407706:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:10.0:1619677448.407707:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.407708:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.407708:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.407709:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.407709:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.407709:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:25.2:1619677448.407709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:22.0:1619677448.407709:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.407710:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.407710:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (23)--
00000800:00000200:31.0:1619677448.407712:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:22.0:1619677448.407712:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.407712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.407714:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.407714:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:9.2:1619677448.407714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.407716:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:22.0:1619677448.407716:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96715338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.407718:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.407718:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.407718:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:31.0:1619677448.407719:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:22.0:1619677448.407720:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.407720:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.407720:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:31.0:1619677448.407721:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96715337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.407721:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:10.0:1619677448.407721:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677448.407721:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.407722:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677448.407722:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.407723:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.407723:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.407724:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.407725:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.407725:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.407726:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677448.407726:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:2.0:1619677448.407726:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.407727:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:22.0:1619677448.407727:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.407728:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:2.0:1619677448.407728:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.407731:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.407731:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.407732:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677448.407733:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:22.0:1619677448.407734:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.407735:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.407735:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.407737:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.407737:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.407737:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677448.407738:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:22.0:1619677448.407738:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:31.0:1619677448.407739:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677448.407739:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677448.407741:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.407741:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.407742:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.407743:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:9.2:1619677448.407743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.2:1619677448.407747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.407750:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.407753:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.407754:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.407754:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677448.407755:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:18.0:1619677448.407756:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.407758:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677448.407758:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.407759:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.407761:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.407761:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.407764:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.407765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.407772:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.407775:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.407776:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.407776:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.407777:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.407778:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.407792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.407800:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.407802:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.407805:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.407806:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.407808:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.407809:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.407810:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.407811:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.407813:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.407813:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.407817:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.407819:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.407821:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.407821:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.407822:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.407823:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.407824:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.407825:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.407826:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.407829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.407835:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.407837:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.407838:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.407838:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.407840:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.407843:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.407847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.407852:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.407855:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.407856:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.407858:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.407860:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.407862:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.407863:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.407864:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.407865:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.407866:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.407870:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.407872:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.407875:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.407875:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.407876:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.407878:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.407879:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.407879:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.407880:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.407884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.407884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc1200] (20)++
00000800:00000200:20.2:1619677448.407886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677448.407892:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc1200] (21)++
00000800:00000200:13.0:1619677448.407892:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677448.407892:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677448.407895:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:5.0:1619677448.407895:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:31.0:1619677448.407896:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.15@o2ib3
00000400:00000200:13.0:1619677448.407896:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.407898:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:13.0:1619677448.407898:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.407899:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc1200] (22)--
00000400:00000200:5.0:1619677448.407899:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.407900:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.3.15@o2ib3 : GET - for me
00000800:00000200:12.0:1619677448.407900:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.407901:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:5.0:1619677448.407902:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.407903:0:19945:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.3.15@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:5.0:1619677448.407904:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677448.407906:0:19945:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.3.15@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000400:00000200:5.0:1619677448.407906:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.407907:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.407908:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc1200] (21)++
00000400:00000200:5.0:1619677448.407908:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.407909:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc1200] (22)++
00000400:00000200:5.0:1619677448.407909:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.407911:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc1200] (23)--
00000400:00000200:5.0:1619677448.407913:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.407916:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.3.15@o2ib3 local destination
00000800:00000200:31.0:1619677448.407917:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a867b82280] -> 10.51.3.15@o2ib3 (2) version: 12
00000800:00000200:5.2:1619677448.407917:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc1200] (22)++
00000800:00000200:31.0:1619677448.407919:0:19945:0:(o2iblnd.c:1251:kiblnd_query()) peer_ni 10.51.3.15@o2ib3 ffff91a867b82280, alive 0 secs ago
00000800:00000200:21.2:1619677448.407919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:5.0:1619677448.407921:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:31.0:1619677448.407922:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.3.15@o2ib3(10.51.3.15@o2ib3:10.51.3.15@o2ib3) : REPLY try# 0
00000800:00000200:5.0:1619677448.407924:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.407925:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.3.15@o2ib3
00000800:00000200:5.0:1619677448.407925:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:2.0:1619677448.407925:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677448.407926:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a867b82280] -> 10.51.3.15@o2ib3 (2) version: 12
00000800:00000200:30.0:1619677448.407926:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc1200] (23)++
00000800:00000200:5.0:1619677448.407926:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.407927:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a87afc1200] (24)++
00000800:00000200:30.0:1619677448.407927:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc1200] (24)--
00000800:00000200:2.0:1619677448.407927:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:31.0:1619677448.407928:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc1200] (24)++
00000800:00000200:30.0:1619677448.407928:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc1200] (24)--
00000800:00000200:18.0:1619677448.407928:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:5.0:1619677448.407928:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.407929:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc1200] (24)--
00000800:00000200:5.0:1619677448.407929:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.407930:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a87afc1200] (24)--
00000800:00000200:31.0:1619677448.407930:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc1200] (22)--
00000800:00000200:5.0:1619677448.407930:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.407930:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:2.0:1619677448.407930:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.407933:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.407934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:2.0:1619677448.407934:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.407935:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.407936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc1200] (21)++
00000400:00000200:2.0:1619677448.407937:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.407938:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.407939:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.407942:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:2.0:1619677448.407942:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:30.0:1619677448.407943:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.407944:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.407944:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.407945:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc1200] (22)++
00000800:00000200:31.0:1619677448.407946:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc1200] (23)--
00000400:00000200:31.0:1619677448.407946:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.407946:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.407947:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.407948:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.3.15@o2ib3: REPLY: OK
00000800:00000200:2.0:1619677448.407948:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.407948:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.407950:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.407951:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc1200] (22)--
00000800:00000200:2.0:1619677448.407951:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.407951:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:30.0:1619677448.407952:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:2.0:1619677448.407952:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677448.407953:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.407954:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc1200] (21)--
00000800:00000200:25.2:1619677448.407956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.407964:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.407967:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.407968:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.407968:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.407970:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677448.407973:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.408144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.408149:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:4.2:1619677448.408151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677448.408152:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.408153:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.408155:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.408158:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.408158:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677448.408159:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:5.2:1619677448.408160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:17.0:1619677448.408161:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.408161:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:17.0:1619677448.408162:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.408163:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677448.408163:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.408164:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677448.408164:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.408165:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.408167:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:17.0:1619677448.408168:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.408169:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.408170:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.408171:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.408172:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.408173:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.408174:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.408175:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.408175:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:10.0:1619677448.408175:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.408176:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.408177:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.408177:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.408177:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.408179:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.408179:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:26.0:1619677448.408180:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:17.0:1619677448.408180:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.408180:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.408181:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.408182:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96715330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.408183:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.408183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.408183:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.408184:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.408185:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:5.2:1619677448.408185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.408188:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.408189:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.408189:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.408190:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.408190:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:26.0:1619677448.408191:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.408191:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (23)++
00000800:00000200:1.0:1619677448.408192:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:10.0:1619677448.408193:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (25)--
00000800:00000200:1.0:1619677448.408193:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.408194:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.408194:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (25)--
00000800:00000200:10.0:1619677448.408194:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.408194:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.408195:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:22.0:1619677448.408195:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.408195:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677448.408195:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:26.0:1619677448.408196:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.408196:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.408197:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:22.0:1619677448.408197:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.408197:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.408198:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.408199:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.408199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (22)++
00000800:00000200:26.0:1619677448.408200:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677448.408202:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:25.2:1619677448.408203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:22.0:1619677448.408204:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.408206:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.408206:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.408207:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.408208:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.408209:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:22.0:1619677448.408210:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677448.408212:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.408213:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.408215:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.408215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.408218:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.408219:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.408219:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677448.408222:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.408224:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.408225:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.408226:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.408227:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.408228:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.408229:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.408230:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.408231:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.408234:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.408235:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.408238:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.408239:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.408239:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.408241:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.408242:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.408242:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.408243:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.408247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.408253:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.408256:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.408256:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.408258:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.408260:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.408261:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.408321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.408329:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.408332:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.408334:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.408336:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.408336:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.408338:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.408339:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.408340:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.408341:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.408342:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.408345:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.408346:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.408348:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.408349:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.408349:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.408351:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.408351:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.408352:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.408353:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.408357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.408364:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.408366:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.408367:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.408368:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.408370:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.408371:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.408379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.408386:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.408388:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.408389:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.408391:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.408394:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.408395:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.408397:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.408398:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.408399:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.408400:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:21.2:1619677448.408401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000400:00000200:17.0:1619677448.408403:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.408405:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677448.408407:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:17.0:1619677448.408408:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.408409:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.408409:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:18.0:1619677448.408411:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:17.0:1619677448.408411:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.408411:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.408411:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:17.0:1619677448.408412:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677448.408413:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.408413:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:18.0:1619677448.408416:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.408417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:18.0:1619677448.408418:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.408419:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715326] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.408420:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.408421:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.408422:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677448.408424:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:18.0:1619677448.408426:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.408427:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.408428:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.408428:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677448.408428:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.408430:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.408431:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.408432:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.408432:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677448.408434:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:18.0:1619677448.408434:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.408435:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.408435:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677448.408436:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:9.2:1619677448.408437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:25.2:1619677448.408440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677448.408444:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.408447:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.408447:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677448.408448:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (21)++
00000800:00000200:13.0:1619677448.408449:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677448.408449:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.408450:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677448.408451:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.408452:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.408454:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.408456:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.408457:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.408577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.408580:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.408582:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.408585:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.408586:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.408588:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.408590:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.408591:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.408592:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.408593:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.408595:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:4.2:1619677448.408596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:20.0:1619677448.408600:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.408602:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.408604:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.408605:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.408606:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.408607:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.408607:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.408608:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677448.408609:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.408609:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.408610:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:10.0:1619677448.408610:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.408611:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:10.0:1619677448.408612:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.408614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.408614:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.408615:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.408616:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.408617:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.408618:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:12.0:1619677448.408621:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677448.408622:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.408623:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:10.0:1619677448.408623:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:12.0:1619677448.408624:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.408626:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.408627:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.408627:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677448.408628:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.408628:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.408628:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.408630:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.408631:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.408631:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.408632:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677448.408635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:9.2:1619677448.408636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.408641:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.408643:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:17.0:1619677448.408644:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.408644:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:2.0:1619677448.408646:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.408647:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.408647:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.408647:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.408649:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.408650:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.408651:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:2.0:1619677448.408653:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:1.0:1619677448.408653:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.408654:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.408655:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.408656:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.408659:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.408661:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.408664:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.408665:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.408665:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.408667:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.408667:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.408668:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.408669:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.408673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.408682:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.408684:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.408686:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.408686:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.408688:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.408690:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.408693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:22.0:1619677448.408700:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.408703:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.408703:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.408704:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.408705:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.408705:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.408750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.408758:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.408761:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.408761:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.408763:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.408766:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.408768:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.408769:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.408770:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.408771:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.408772:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.408776:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.408778:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.408780:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.408781:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.408782:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.408784:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.408784:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.408785:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.408785:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.408790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.408798:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.408801:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.408802:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.408803:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677448.408804:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.408806:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.408877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.408885:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.408887:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.408888:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.408890:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.408893:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.408894:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.408896:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.408897:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.408898:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.408899:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.408902:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.408903:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.408906:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.408907:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.408907:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.408909:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.408909:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.408911:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.408912:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.408912:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.408914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.408916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677448.408920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677448.408921:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.408922:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.408923:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.408924:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.408925:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.408925:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677448.408926:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.408928:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:17.0:1619677448.408928:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.408928:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.408929:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.408930:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.408930:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.408931:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:17.0:1619677448.408931:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.408932:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.408933:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.408935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:17.0:1619677448.408935:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.408936:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.408937:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.408940:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.408941:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.408942:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.408944:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.408944:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.408945:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.408945:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677448.408946:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:12.0:1619677448.408946:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.408947:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.408947:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677448.408948:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.408949:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.408949:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.408950:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.408954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.408961:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.408963:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.408964:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.408966:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.408967:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.408969:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.408975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.408982:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.408985:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.408987:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.408989:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.408989:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.408990:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.408992:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.408993:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.408994:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.408995:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.408998:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.408999:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.409002:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.409003:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.409003:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.409005:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.409005:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.409007:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.409008:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.409009:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.409010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.409015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.409016:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:4.2:1619677448.409016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677448.409019:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.409019:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.409021:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.409023:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.409024:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:13.0:1619677448.409024:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.409024:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.409026:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:10.0:1619677448.409026:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:31.0:1619677448.409027:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677448.409027:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.409028:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:10.0:1619677448.409029:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.409031:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.409033:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.409034:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.409035:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.409036:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.409037:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.409040:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.409041:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.409043:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.409044:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.409045:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.409046:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.409046:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.409047:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.409048:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.409052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.409059:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.409061:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.409062:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.409063:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.409065:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.409066:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.409390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.409396:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.409399:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.409400:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.409401:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.409404:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.409405:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.409406:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.409408:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.409409:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.409410:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.409413:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.409415:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.409418:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.409419:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.409419:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.409421:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.409422:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.409423:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.409424:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.409425:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:25.2:1619677448.409428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:21.2:1619677448.409430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.409435:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:22.0:1619677448.409435:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:26.0:1619677448.409438:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:22.0:1619677448.409438:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.409439:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:10.0:1619677448.409439:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:6.0:1619677448.409439:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:26.0:1619677448.409440:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.409442:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.409445:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.409523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.409529:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.2:1619677448.409532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677448.409532:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.409533:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.409535:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.409538:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.409539:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:30.0:1619677448.409540:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:1.0:1619677448.409541:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.409542:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.409543:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:1.0:1619677448.409543:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.409545:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.409547:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.409547:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.409549:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.409549:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.409551:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:1.0:1619677448.409551:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.409552:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.409554:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.409554:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.409555:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.409555:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:30.0:1619677448.409556:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.409556:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.409557:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.409558:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677448.409559:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.409559:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.409560:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:30.0:1619677448.409561:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.409561:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.409563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.409564:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.409565:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.2:1619677448.409565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.409566:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.409567:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.409568:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.409569:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.409570:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:17.0:1619677448.409570:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677448.409571:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677448.409573:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.409573:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.409574:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.409574:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.409575:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.409577:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.409579:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.409580:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.409581:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.409582:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.409582:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.409582:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677448.409583:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677448.409583:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.409584:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.2:1619677448.409585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.409586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.409592:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.0:1619677448.409593:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677448.409595:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:20.0:1619677448.409596:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:2.0:1619677448.409596:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:20.0:1619677448.409597:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.409598:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677448.409599:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.409599:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:18.0:1619677448.409600:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.409602:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:18.0:1619677448.409602:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.409603:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.409605:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.409605:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.409607:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.409610:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.409612:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.409613:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.409614:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.409615:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.409616:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.409616:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.409618:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.409619:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.409619:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:9.2:1619677448.409622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.409625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.409630:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.409632:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.409632:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.409632:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.409633:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.409634:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.409635:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.409635:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.409636:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677448.409636:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.409682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.409688:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.409691:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.409693:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.409693:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.409696:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.409697:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677448.409698:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.409700:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.409701:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.409702:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.409706:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.409707:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.409711:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.409711:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677448.409712:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.409714:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.409714:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.409715:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.409716:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.409720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.409726:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.409729:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:4.2:1619677448.409729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.409730:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.409730:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.409731:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:10.0:1619677448.409732:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.409733:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677448.409736:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.409736:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.409737:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.409739:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.409741:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.409742:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.409743:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.409744:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.409745:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.409749:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.409751:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.409754:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.409754:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.409755:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.409757:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.409757:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.409758:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.409759:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.409760:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.409763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.409766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.409771:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:10.0:1619677448.409773:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:20.0:1619677448.409774:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.409775:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.409775:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:20.0:1619677448.409776:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.409776:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.409777:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.409777:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.409779:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.409779:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.409785:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.409788:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.409789:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.409791:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.409793:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.409795:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.409796:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.409797:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.409798:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.409799:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.409803:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.409805:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.409807:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.409808:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.409809:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.409810:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.409811:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.409811:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.409812:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.409826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.409834:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.409836:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.409837:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.409839:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.409839:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.409841:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.409845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.409852:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.409855:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.409856:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.409857:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.409857:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.409858:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.410054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.410061:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.410064:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.410066:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.410068:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.410069:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.410071:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.410072:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.410073:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.410074:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.410075:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.410079:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.410081:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.410083:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.410084:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.410085:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.410086:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.410087:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.410088:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.410088:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.410092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.410098:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.410101:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.410102:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.410103:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.410105:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.410105:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:21.2:1619677448.410135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.410141:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.410144:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.410144:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.410147:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.410150:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.410151:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.410153:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715309] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.410154:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.410155:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.410156:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:21.2:1619677448.410158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:22.0:1619677448.410159:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.410161:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.410163:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.410164:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.410164:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:22.0:1619677448.410165:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.410166:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.410166:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:22.0:1619677448.410167:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.410168:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.410168:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677448.410168:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.410169:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.410172:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.410173:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.410175:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.410176:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.410177:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.410178:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.410181:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.410182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.410182:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.410185:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.410186:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.410187:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.410188:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.410189:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677448.410189:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.410190:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.410190:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677448.410192:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.410193:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.410194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.410194:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.410195:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.410197:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677448.410201:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.410203:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.410204:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.410206:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.410207:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.410208:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.410272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:4.2:1619677448.410274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:5.0:1619677448.410277:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.410280:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:24.0:1619677448.410282:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:5.0:1619677448.410282:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.410283:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:24.0:1619677448.410285:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:5.0:1619677448.410285:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.410286:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:5.0:1619677448.410287:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.410288:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.410288:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.410290:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.410290:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.410291:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.410292:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.410292:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.410299:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.410301:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.410302:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.410304:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.410304:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.410307:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.410310:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.410310:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.410311:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.410311:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.410312:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.410313:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.410313:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.410314:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:24.0:1619677448.410315:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.410315:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:24.0:1619677448.410316:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.410317:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.410319:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.2:1619677448.410319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.410320:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.410321:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.410322:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.410325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.410330:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.410332:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.410333:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.410334:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.410335:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.410335:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.410336:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.410338:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.410339:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.410340:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.410340:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677448.410341:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.410343:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.410344:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.410344:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.410345:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.410345:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.410407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.410415:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.410418:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.410419:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.410421:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.410424:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.410426:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.410427:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.410428:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.410429:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.410430:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.410434:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.410435:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.410438:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.410439:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.410440:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.410441:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.410442:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.410443:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.410443:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.410448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.410456:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.410459:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.410460:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.410460:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.410463:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.410465:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.410678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.410683:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.410685:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.410686:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.410688:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.410690:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.410691:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.410692:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.410694:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.410695:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.410696:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.410699:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.410701:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.410704:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.410705:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.410705:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.410707:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.410708:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.410708:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.410709:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.410713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.410719:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.410722:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.410723:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.410723:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.410725:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.410728:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677448.410813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.410819:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.410822:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.410823:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.410825:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.410828:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.410830:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677448.410831:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.410832:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.410833:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.410834:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677448.410838:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.410840:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.410843:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.410844:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:18.0:1619677448.410844:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677448.410846:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677448.410846:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.410848:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.410849:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.410849:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.410852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:21.2:1619677448.410854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.410859:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:13.0:1619677448.410860:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677448.410862:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.410862:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:13.0:1619677448.410863:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:2.0:1619677448.410863:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:13.0:1619677448.410864:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.410865:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.410867:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.410868:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.410931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.410939:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.410942:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.410944:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.410946:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.410946:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.410948:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.410949:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.410950:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.410951:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.410953:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.410956:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.410957:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.410960:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.410961:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.410961:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.410963:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.410963:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.410965:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.410966:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.410967:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.410968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.410973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.410974:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.410977:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.410977:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.410979:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.410980:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.410980:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.410982:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.410983:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.410983:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.410984:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677448.411040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.411043:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.411046:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.411048:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.411049:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.411051:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.411053:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677448.411054:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.411056:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.411057:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.411058:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.411061:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.411063:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.411065:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.411066:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.411067:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.411068:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.411069:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.411070:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.411070:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:5.2:1619677448.411074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.2:1619677448.411075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.411081:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.411082:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:26.0:1619677448.411084:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:17.0:1619677448.411085:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:10.0:1619677448.411085:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.411086:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.411086:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.411086:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.411088:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.411089:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.411090:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:26.0:1619677448.411091:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.411092:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.411093:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.411094:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.411095:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.411098:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.411099:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.411102:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.411102:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.411103:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.411105:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.411105:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.411106:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.411106:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.2:1619677448.411109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:25.2:1619677448.411110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677448.411115:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:13.0:1619677448.411116:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:18.0:1619677448.411118:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.411118:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:13.0:1619677448.411119:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.411119:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.411121:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.411122:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.411123:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:18.0:1619677448.411124:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.411124:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:18.0:1619677448.411125:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.411127:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.411128:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.411129:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.411130:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.411133:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.411135:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.411137:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.411138:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.411138:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.411140:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.411140:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.411141:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.411141:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.411146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.411153:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.411155:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.411156:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.411157:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.2:1619677448.411157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:2.0:1619677448.411158:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.411159:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.411163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:1.0:1619677448.411163:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677448.411165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:17.0:1619677448.411166:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.411166:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.411168:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.411171:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677448.411171:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.411172:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:18.0:1619677448.411173:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.411173:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.411174:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000400:00000200:1.0:1619677448.411174:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.411175:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.411175:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.411176:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:1.0:1619677448.411176:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.411177:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677448.411178:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.411178:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.411178:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.411179:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.411181:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.411183:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677448.411183:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.411184:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.411185:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.411187:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.411188:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.411188:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.411189:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.411189:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.411190:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:2.0:1619677448.411192:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.411192:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.411192:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.411193:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.411193:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677448.411194:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.411196:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677448.411197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.411197:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.411197:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.411199:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.411199:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.411200:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.411201:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677448.411204:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:9.2:1619677448.411205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677448.411207:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.411208:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.411209:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.411211:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.411212:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677448.411213:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.411215:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:25.2:1619677448.411216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:18.0:1619677448.411216:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.411216:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.411218:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.411220:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677448.411223:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.411226:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.411227:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.411227:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.411228:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.411229:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.411394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.411401:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.411404:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.411405:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.411407:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.411410:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.411411:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.411412:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.411414:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.411415:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.411416:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.411419:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.411421:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.411422:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.411423:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.411424:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.411425:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.411426:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.411427:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.411427:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.411432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.411438:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.411440:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.411440:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.411441:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.411442:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.411444:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.411460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.411468:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.411471:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.411473:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.411475:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.411476:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.411477:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.411478:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.411479:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.411480:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.411481:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.411484:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.411486:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.411488:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.411488:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.411489:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.411490:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.411491:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.411492:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.411492:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.411497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.411502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:12.0:1619677448.411504:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.411507:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.411507:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.411508:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:12.0:1619677448.411509:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.411510:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677448.411511:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.411512:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677448.411512:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:6.0:1619677448.411514:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.411517:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.411518:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.411520:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.411521:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.411522:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.411523:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.411527:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.411529:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.411531:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.411532:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.411532:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.411534:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.411534:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.411535:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.411536:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.411541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.411549:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.411552:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.411553:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.411553:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.411555:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.411558:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.411657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.411662:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.411665:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.411665:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.411667:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.411670:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.411672:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.411673:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.411674:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.411675:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.411676:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.411680:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.411681:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.411684:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.411685:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.411685:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.411687:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.411688:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.411688:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.411689:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.411693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.411700:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.411703:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.411704:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.411706:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.411707:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.411709:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.411738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.411744:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.411747:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.411749:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.411749:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.411752:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.411753:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:4.2:1619677448.411754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:5.0:1619677448.411755:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.411756:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.411757:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.411758:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677448.411761:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:5.0:1619677448.411762:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.411764:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:5.0:1619677448.411764:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677448.411765:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677448.411767:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677448.411768:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.411768:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.411769:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:8.0:1619677448.411771:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.411771:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.411771:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:8.0:1619677448.411772:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:5.0:1619677448.411772:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:8.0:1619677448.411774:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96715291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.411774:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.411774:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:8.0:1619677448.411775:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.411776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:8.0:1619677448.411776:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.411777:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.2:1619677448.411779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:8.0:1619677448.411780:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.411782:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:8.0:1619677448.411782:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677448.411783:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.411784:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.411785:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.411785:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.411785:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677448.411785:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677448.411786:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.411786:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.411786:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.411787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:13.0:1619677448.411787:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.411788:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677448.411789:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677448.411789:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677448.411790:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677448.411790:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.411791:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.411794:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.411797:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.411800:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.411801:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:17.0:1619677448.411802:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:8.0:1619677448.411803:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.411804:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:8.0:1619677448.411805:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.411805:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.411806:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.411807:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.411808:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.411808:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96715290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.411810:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.411811:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.411812:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.411814:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:8.0:1619677448.411815:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.411818:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.411820:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.411821:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.411821:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.411823:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.411823:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.411824:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.411824:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.411828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.411835:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.411837:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.411838:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.411838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:12.0:1619677448.411839:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.411841:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.411842:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.411844:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.2:1619677448.411846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:10.0:1619677448.411847:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.411848:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.411849:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.411851:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.411853:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000400:00000200:10.0:1619677448.411853:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.411854:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.411855:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.411856:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:10.0:1619677448.411856:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.411857:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677448.411858:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.411860:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677448.411861:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.411861:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.411862:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:31.0:1619677448.411863:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.411864:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.411865:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.411865:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.411866:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.411866:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:31.0:1619677448.411867:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.411867:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.411868:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:31.0:1619677448.411870:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.411871:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.411872:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.411872:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.411872:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.411874:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.411874:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.411875:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:25.2:1619677448.411875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.411876:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.411877:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677448.411878:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677448.411878:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:25.2:1619677448.411882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:22.0:1619677448.411883:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.411886:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.411887:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.411887:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.411888:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:22.0:1619677448.411889:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.411890:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.411891:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.411891:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.411893:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.411893:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:26.0:1619677448.411896:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.411900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.411906:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.411909:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.411909:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.411910:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.411910:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.411911:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.411998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677448.411999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677448.412005:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.412006:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.412008:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:30.0:1619677448.412009:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:10.0:1619677448.412009:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:30.0:1619677448.412011:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.412012:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677448.412012:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.412014:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.412015:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.412016:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.412016:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.412017:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.412018:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.412019:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.412019:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.412020:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.412021:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.412021:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.412022:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.412026:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.412027:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.412028:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.412029:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.412031:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.412032:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.412032:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.412033:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.412033:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.412033:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.412034:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.412035:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.412035:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.412035:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.412036:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.412036:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677448.412036:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.412037:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.412040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:8.2:1619677448.412041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.412046:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.412049:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.412049:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.412049:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:31.0:1619677448.412051:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.412051:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.412052:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.412053:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.412054:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:20.0:1619677448.412054:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.412055:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.412057:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:25.2:1619677448.412059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.2:1619677448.412065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:30.0:1619677448.412066:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.412069:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.412069:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.412070:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.412071:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:2.0:1619677448.412071:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:30.0:1619677448.412072:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:2.0:1619677448.412074:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.412075:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.412077:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.412078:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.412080:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.412081:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.412083:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.412084:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.412085:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.412088:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.412090:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.412092:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.412092:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.412093:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.412094:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.412095:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.412095:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.412096:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.412101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677448.412104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:18.0:1619677448.412108:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.412110:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.412110:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.412111:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:2.0:1619677448.412111:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.412112:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.412113:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.412114:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.412115:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.412116:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.412119:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.412120:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.412122:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715284] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.412123:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.412124:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.412125:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:21.2:1619677448.412128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:10.0:1619677448.412129:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.412130:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.412134:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.412134:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.412135:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.412135:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.412137:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.412137:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:31.0:1619677448.412138:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:10.0:1619677448.412139:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:31.0:1619677448.412140:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.412140:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.412140:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:30.0:1619677448.412142:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.412143:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.412143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677448.412144:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.412145:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.412146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:31.0:1619677448.412147:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.412147:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.412148:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.412151:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:31.0:1619677448.412152:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.412153:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.412153:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:5.0:1619677448.412154:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:31.0:1619677448.412155:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.412155:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.412156:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.412156:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.412156:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:5.0:1619677448.412156:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.412157:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.412157:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.412158:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.412158:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:31.0:1619677448.412159:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.412159:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.0:1619677448.412159:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677448.412160:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.412164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.412170:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.412173:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.412174:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.412175:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.412176:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.412177:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.412232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.412238:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.412241:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.412241:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.412243:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.412246:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.412247:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.412249:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.412250:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.412251:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.412252:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.412256:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.412258:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.412261:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.412262:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.412262:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.412264:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.412265:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.412265:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.412265:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.412270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.412271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.412274:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.412276:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:30.0:1619677448.412277:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:20.0:1619677448.412278:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.412279:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:5.0:1619677448.412279:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677448.412280:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.412281:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.412282:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.412283:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:20.0:1619677448.412283:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.412284:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.412285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:20.0:1619677448.412286:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.412287:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.412288:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:20.0:1619677448.412288:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.412289:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:20.0:1619677448.412291:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.412292:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.412292:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.412299:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.412299:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.412300:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.0:1619677448.412301:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.412303:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.412304:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.412305:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.412306:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.412307:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.412308:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.412308:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.412313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.412317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677448.412321:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.412324:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.412324:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.412325:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.412325:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.412327:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.412327:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:6.0:1619677448.412328:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.412330:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.412330:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.412333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000400:00000200:10.0:1619677448.412333:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.412334:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.412336:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.412337:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.412338:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.412339:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.412340:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.412340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:22.0:1619677448.412343:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.412343:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.412344:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.412344:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.412345:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.412345:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.412346:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677448.412346:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.412348:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.412348:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.412349:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.412349:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.412350:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:10.0:1619677448.412351:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.412351:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677448.412351:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.412352:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:2.0:1619677448.412353:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.412354:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.412354:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677448.412355:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:25.2:1619677448.412356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:2.0:1619677448.412356:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.412358:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677448.412359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677448.412359:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.412360:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.412363:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.412364:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677448.412365:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.412366:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.412367:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.412367:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.412368:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677448.412368:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.412368:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.412368:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.412369:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677448.412369:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:6.0:1619677448.412370:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.412371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677448.412371:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677448.412371:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.412372:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:6.0:1619677448.412373:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677448.412373:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:26.0:1619677448.412375:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.412376:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:25.2:1619677448.412377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.412378:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.412379:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.412381:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.412384:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.412384:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:26.0:1619677448.412386:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:12.0:1619677448.412386:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:26.0:1619677448.412387:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.412387:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.412388:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.412389:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.412390:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.412391:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:13.0:1619677448.412391:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.412392:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:26.0:1619677448.412394:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.412394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.412396:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.412398:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.412398:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.412399:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.412400:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:13.0:1619677448.412400:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.412401:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.412401:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.412402:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.412403:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.412404:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.412404:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.412405:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:25.2:1619677448.412406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:13.0:1619677448.412406:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.412412:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.412414:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.412414:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.412415:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.412416:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.412418:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.412591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.412598:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.412601:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.412602:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.412603:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.412605:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.412606:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.412607:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.412608:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.412609:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.412610:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.412613:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.412614:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.412616:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.412617:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.412618:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.412619:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.412620:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.412621:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.412621:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.412625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.412632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677448.412633:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.412636:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.412637:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.412637:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.412639:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:22.0:1619677448.412639:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.412642:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677448.412642:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:31.0:1619677448.412644:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.412646:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.412647:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.412648:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.412649:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715276] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.412651:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.412652:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.412653:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.412656:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.412656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677448.412657:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.412660:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.412660:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.2:1619677448.412660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677448.412661:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.412662:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.412663:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.412664:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.412664:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677448.412664:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677448.412666:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.412667:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:25.2:1619677448.412668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:17.0:1619677448.412669:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.0:1619677448.412670:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.412670:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.412670:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.412672:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.412673:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.412674:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:5.0:1619677448.412674:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.412675:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.412676:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:5.0:1619677448.412676:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.412677:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:17.0:1619677448.412677:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.412677:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.412678:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.412678:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.412679:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.412680:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.412680:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.412680:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.412683:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.412683:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:17.0:1619677448.412684:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:5.0:1619677448.412685:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.412688:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.412689:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.412691:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.412691:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.412691:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.412692:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.412694:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.412694:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:5.0:1619677448.412694:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.412695:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.0:1619677448.412695:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.412696:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.412696:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677448.412697:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.412698:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.412699:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:25.2:1619677448.412700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.412700:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.412701:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.412704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:12.0:1619677448.412706:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.2:1619677448.412706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.412709:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.412710:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.412711:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.412711:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.412711:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677448.412713:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.412714:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.412714:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677448.412714:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:6.0:1619677448.412715:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.412715:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677448.412716:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.412716:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.412717:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.412719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:6.0:1619677448.412720:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677448.412725:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.412728:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.412728:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.412729:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.412730:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.412731:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.412874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.412877:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.412879:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.412882:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.412882:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.412884:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.412886:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.412887:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.412889:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.412889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:20.0:1619677448.412890:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.412891:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.412894:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.412896:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.412897:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.412899:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:20.0:1619677448.412899:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.412900:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.412900:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677448.412901:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:26.0:1619677448.412902:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.412903:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.412903:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677448.412904:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.412904:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.412905:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:26.0:1619677448.412906:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.412907:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.412908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:26.0:1619677448.412909:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.412909:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.412911:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.412914:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.412915:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:4.2:1619677448.412915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.412916:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677448.412918:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.412918:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.412919:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.412920:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:12.0:1619677448.412920:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.412921:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677448.412921:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.412923:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.412923:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.412924:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.412924:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.412925:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.412926:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:20.2:1619677448.412926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:26.0:1619677448.412927:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.412928:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.412928:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.412929:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.412930:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.412930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677448.412932:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.412933:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.412933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.412934:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677448.412934:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.412935:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.412936:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:18.0:1619677448.412937:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:10.0:1619677448.412937:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.412939:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.412940:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:31.0:1619677448.412941:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.412941:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.412941:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:2.0:1619677448.412941:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.412942:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.412942:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.412943:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.412943:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:30.0:1619677448.412944:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:26.0:1619677448.412944:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:18.0:1619677448.412944:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.412945:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96715270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.412945:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.412946:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.412947:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.412948:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.412948:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.412949:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.412949:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.412950:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:30.0:1619677448.412951:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.412952:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.412952:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.412953:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.412953:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.412954:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.412955:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.412956:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.412956:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.412958:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.412958:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:25.2:1619677448.412958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.412959:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.412960:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.412963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677448.412965:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.412968:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.412969:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.412970:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.412970:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:22.0:1619677448.412972:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.412972:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.412973:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.412975:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:13.0:1619677448.412975:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.412976:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.412978:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.413006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:24.0:1619677448.413014:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677448.413017:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677448.413018:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677448.413020:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.413022:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.413024:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:24.0:1619677448.413025:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.413027:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.413027:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.413028:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:24.0:1619677448.413032:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.413034:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:24.0:1619677448.413036:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.413037:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:24.0:1619677448.413037:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:24.0:1619677448.413039:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:24.0:1619677448.413039:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677448.413040:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677448.413040:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:25.2:1619677448.413044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.413050:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.413053:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.413053:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.413054:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.413055:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.413058:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:25.2:1619677448.413066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000800:00000200:10.0:1619677448.413072:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.413074:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.413075:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.413075:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.413076:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.413077:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.413087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.413093:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.413096:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.413096:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.413098:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.413101:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.413102:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.413104:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.413105:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.413106:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.413107:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.413110:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.413112:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.413114:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.413115:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.413116:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.413117:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.413118:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.413118:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.413119:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.413123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.413131:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.413133:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.413134:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.413135:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.413137:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.413138:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.413143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:22.0:1619677448.413151:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.413153:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.413154:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.413154:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.413155:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.413156:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.413256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.413261:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.413264:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.413266:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.413267:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.413269:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.413271:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.413272:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.413274:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.413275:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.413276:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.413279:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.413281:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.413283:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.413284:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.413285:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.413287:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.413287:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.413289:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.413290:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.413291:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.413293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.413299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.413302:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677448.413302:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.413304:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.413305:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677448.413305:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677448.413306:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:12.0:1619677448.413306:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.413308:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.413310:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.413311:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.413383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.413391:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.413394:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.413396:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.413397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:31.0:1619677448.413398:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.413398:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.413399:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.413400:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.413402:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.413403:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.413404:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:1.0:1619677448.413404:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.413407:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.413407:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.413407:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677448.413410:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:1.0:1619677448.413410:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.413412:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.413412:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.413413:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:1.0:1619677448.413413:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:31.0:1619677448.413414:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.413415:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:1.0:1619677448.413415:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.413416:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677448.413416:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.413417:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:1.0:1619677448.413417:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.413418:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.413419:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.413420:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.413421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:1.0:1619677448.413421:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.413423:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.2:1619677448.413425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.413425:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.413426:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.413426:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.413428:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:1.0:1619677448.413428:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.413428:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.413429:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.413430:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:1.0:1619677448.413430:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677448.413431:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.413431:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.413433:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:26.0:1619677448.413433:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.413435:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.2:1619677448.413435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677448.413436:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.413436:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:30.0:1619677448.413437:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677448.413443:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.413445:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.413445:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.413446:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.413448:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.413449:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.413453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.413460:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.413463:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.413463:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.413465:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.413468:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.413470:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.413471:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.413472:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.413473:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.413474:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.2:1619677448.413478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:10.0:1619677448.413478:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.413479:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.413483:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.413484:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.413485:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.413487:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.413487:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.413487:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.413488:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.413488:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.413489:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:26.0:1619677448.413490:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.413490:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:21.2:1619677448.413491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:26.0:1619677448.413493:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.413493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:26.0:1619677448.413494:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.413495:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.413496:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.413497:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.413498:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.413499:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677448.413500:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:31.0:1619677448.413501:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:26.0:1619677448.413501:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.413502:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.413502:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.413503:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.413503:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.413504:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.413505:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.413505:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.413506:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:26.0:1619677448.413506:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.413506:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677448.413508:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.413508:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.413508:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:30.0:1619677448.413509:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.413509:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.413509:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:30.0:1619677448.413510:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.413510:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:31.0:1619677448.413512:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.413513:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677448.413513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.413514:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:30.0:1619677448.413514:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.413515:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677448.413518:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.413519:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:30.0:1619677448.413519:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:13.0:1619677448.413520:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.413521:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:30.0:1619677448.413521:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:13.0:1619677448.413522:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:30.0:1619677448.413523:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000400:00000200:13.0:1619677448.413523:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.413524:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677448.413524:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:13.0:1619677448.413524:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.413526:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:9.2:1619677448.413526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:13.0:1619677448.413527:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677448.413533:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.413535:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.413536:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.413536:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.413538:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.413540:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677448.413544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.413550:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.413553:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.413554:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.413555:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.413558:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.413560:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677448.413561:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.413563:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.413564:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.413565:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.2:1619677448.413568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000400:00000200:2.0:1619677448.413568:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.413569:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677448.413571:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.413573:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677448.413573:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677448.413575:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677448.413575:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677448.413576:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:2.0:1619677448.413576:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.413576:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677448.413578:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677448.413579:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677448.413580:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677448.413581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.413589:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.413592:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.413593:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.413593:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.413595:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.413597:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.413629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.413632:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.413634:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.413636:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.413637:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.413639:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.413641:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677448.413642:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.413644:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.413645:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.413646:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677448.413649:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.413651:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.413654:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.413655:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.413656:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.2:1619677448.413656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677448.413657:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.413658:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.413658:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.413659:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:1.0:1619677448.413662:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:25.2:1619677448.413663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:1.0:1619677448.413665:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.413666:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.413668:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.413669:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:1.0:1619677448.413669:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.413671:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:10.0:1619677448.413672:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:1.0:1619677448.413672:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.413673:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.413673:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.413673:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.413675:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.413676:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.413676:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.413679:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:1.0:1619677448.413679:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.413681:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:25.2:1619677448.413682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:1.0:1619677448.413684:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.413685:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.413685:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.413687:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:1.0:1619677448.413687:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.413687:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.413688:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.413689:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.413690:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.413691:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.413691:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.413692:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.413693:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.413693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.413700:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.413703:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.413704:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.413704:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.413706:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.413708:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.413714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.413722:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.413724:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.413727:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.413728:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.413729:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.413731:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.413732:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.413734:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.413735:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.413736:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.413739:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.413741:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.413743:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.413744:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.413745:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.413746:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.413747:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.413748:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.413750:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.413750:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.413752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.413756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.413758:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.413761:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.413761:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.413763:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:12.0:1619677448.413763:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.413764:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.413765:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.413766:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.413767:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.413767:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.2:1619677448.413767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.413775:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.413778:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.413778:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.413780:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.413782:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.413784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:10.0:1619677448.413784:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.413785:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.413786:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.413787:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.413788:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:24.0:1619677448.413791:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:10.0:1619677448.413791:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.413792:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:24.0:1619677448.413793:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.413794:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:10.0:1619677448.413795:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677448.413796:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.413796:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.413797:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:24.0:1619677448.413798:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.413798:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.413799:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:24.0:1619677448.413800:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:10.0:1619677448.413800:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.413800:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:24.0:1619677448.413801:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96715256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.413802:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.413803:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.413804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:24.0:1619677448.413804:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677448.413807:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.413809:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.413812:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.413812:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:6.0:1619677448.413812:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:24.0:1619677448.413813:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.413814:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.413815:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:21.2:1619677448.413815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.413815:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:24.0:1619677448.413816:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.413816:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677448.413816:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.413816:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.413817:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.413818:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.413818:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.413820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:22.0:1619677448.413821:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.413821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:6.0:1619677448.413822:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.2:1619677448.413823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:6.0:1619677448.413823:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.413824:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.413826:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.413827:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.413827:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:22.0:1619677448.413828:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.413829:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.413829:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:22.0:1619677448.413831:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.413831:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:18.0:1619677448.413831:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:5.0:1619677448.413831:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677448.413831:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677448.413832:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.413833:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.413835:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.413836:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.413836:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.413838:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.413840:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.413840:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.0:1619677448.413841:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:18.0:1619677448.413841:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715254] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.413842:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.413842:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.413843:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.413843:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000400:00000200:18.0:1619677448.413843:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.413844:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:18.0:1619677448.413844:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.413845:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.413845:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.413845:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000400:00000200:18.0:1619677448.413845:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.413845:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.413846:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.413846:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:22.0:1619677448.413847:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.413847:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677448.413847:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:5.0:1619677448.413847:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:22.0:1619677448.413849:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:5.0:1619677448.413849:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96715253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.413850:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.413850:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:18.0:1619677448.413850:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.413851:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.413852:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.413852:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.413853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:5.0:1619677448.413853:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:18.0:1619677448.413855:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.413855:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:21.2:1619677448.413856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677448.413856:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.413858:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.413858:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:5.0:1619677448.413858:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.413859:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:20.0:1619677448.413860:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677448.413860:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:20.0:1619677448.413861:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:18.0:1619677448.413861:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:6.0:1619677448.413861:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:5.0:1619677448.413861:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.413862:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.413863:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:20.0:1619677448.413864:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.413864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.413864:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.0:1619677448.413864:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.413865:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677448.413865:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.413866:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.413867:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.413867:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677448.413869:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.413871:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.413872:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677448.413872:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.413872:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:5.0:1619677448.413873:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.413874:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677448.413874:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:17.0:1619677448.413875:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:2.0:1619677448.413875:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677448.413876:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.413876:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677448.413876:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:25.2:1619677448.413878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:17.0:1619677448.413878:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.413880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.413883:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.413884:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677448.413885:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.413888:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:17.0:1619677448.413888:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.413888:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:30.0:1619677448.413889:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.413889:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.413889:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000400:00000200:30.0:1619677448.413890:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.413890:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677448.413892:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.413893:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.413969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.413975:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.413977:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.413978:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.413980:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.413983:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.413984:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.413985:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.413986:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.413988:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.413989:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.413992:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.413994:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.413997:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.413998:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.413998:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.414000:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.414001:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.414002:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.414003:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.414003:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.414006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.414008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.414012:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677448.414013:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.414015:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.0:1619677448.414015:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.414016:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:17.0:1619677448.414016:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677448.414017:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.414018:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.414019:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.414022:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.414122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.414129:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.414132:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.414133:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.414135:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.414137:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.414139:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.414140:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.414141:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.414142:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.414143:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.414146:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.414148:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.414150:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.414151:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.414152:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.414153:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.414154:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.414155:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.414156:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.414156:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.414162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.414169:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.414171:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.414171:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.414172:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.414186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.414193:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.414196:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.414197:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.414198:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.414200:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.414201:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.414259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.414264:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.414267:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.414268:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.414270:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.414273:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.414274:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.414275:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.414276:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.414277:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.414278:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.414281:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.414283:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.414285:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.414285:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.414286:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.414288:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.414288:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.414289:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.414289:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.414294:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.414302:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.414304:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.414305:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.414306:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.414307:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.414308:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.414490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.414498:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.414501:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.414503:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.414506:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.414506:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.414507:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.414509:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.414510:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.414511:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.414512:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.414515:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.414517:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.414520:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.414521:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.414521:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.414523:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:21.2:1619677448.414523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:30.0:1619677448.414524:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.414525:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.414525:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.414529:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:25.2:1619677448.414531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.414532:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:22.0:1619677448.414535:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.414536:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.414537:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:22.0:1619677448.414538:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.414539:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.414540:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.414540:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.414541:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.414542:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.414543:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.414544:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.414545:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.414548:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.414549:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.414549:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.414551:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.414551:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.414552:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.414552:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.414553:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.414554:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.414554:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677448.414555:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.414556:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.414558:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.414558:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.414559:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.414559:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.414565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.414573:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.414575:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.414576:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.414578:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677448.414579:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:20.0:1619677448.414582:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.414584:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677448.414586:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.414588:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.414591:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.414592:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.414593:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.414595:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.414596:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715247] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.414597:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.414598:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.414600:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.414603:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.414604:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.414607:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.414607:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.414608:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.414610:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.414610:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.414611:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.414611:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.414615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.414623:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.414625:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.414626:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.414627:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.414630:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.414630:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.414678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.414687:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.414689:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.414690:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.414691:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.414694:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.414695:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.414696:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.414698:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.414698:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.414699:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.414702:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.414703:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.414705:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.414706:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.414707:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.414708:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.414708:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.414709:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.414709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:26.0:1619677448.414710:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.414714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.414715:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.414718:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.414719:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:13.0:1619677448.414720:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:18.0:1619677448.414721:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.414722:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:18.0:1619677448.414723:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.414723:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.414724:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:13.0:1619677448.414725:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.414726:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715245] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.414726:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:18.0:1619677448.414728:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.414728:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:18.0:1619677448.414729:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.414729:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.414733:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.414734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.414735:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.414737:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.414738:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.414739:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.414740:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.414740:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677448.414740:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.414741:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.414742:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677448.414743:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.414744:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.414744:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.414745:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.414746:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677448.414746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.414754:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.414757:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.414758:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.414760:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.414760:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.414762:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.414764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.414772:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.414774:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.414775:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.414776:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.414776:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.414777:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.414943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.414949:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.414952:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.414952:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.414954:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.414956:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.414957:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.414958:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.414960:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.414961:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.414962:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.414966:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.414967:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.414970:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.414971:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.414972:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.414974:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.414974:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.414975:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.414975:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.414979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:4.2:1619677448.414985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677448.414986:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.414988:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.414989:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.414991:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.414992:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.414992:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677448.414994:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.414994:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.414995:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.414995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:10.0:1619677448.414997:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.415000:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.415001:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:10.0:1619677448.415002:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.415003:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.415004:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.415004:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.415005:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.415005:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:10.0:1619677448.415005:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.415006:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:10.0:1619677448.415006:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:12.0:1619677448.415007:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:10.0:1619677448.415010:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.415012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677448.415012:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.415015:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.415016:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.415016:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.415018:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.415018:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.415019:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.415020:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.415020:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.415023:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:8.2:1619677448.415024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:30.0:1619677448.415025:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.415027:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.415027:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.415029:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.415030:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.415031:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.415031:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:30.0:1619677448.415032:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.415033:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.415034:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.415035:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.415037:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.415037:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.415038:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.415040:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.415041:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:30.0:1619677448.415043:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.415044:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.415045:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.415046:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.415047:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.415048:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.415048:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.415052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.415060:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.415062:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.415063:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.415065:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.415065:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.415067:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.415343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.415349:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.415352:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.415353:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.415355:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.415357:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.415358:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.415360:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.415361:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.415362:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.415363:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.415367:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.415369:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.415372:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.415373:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.415373:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.415375:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.415376:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.415376:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.415377:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.415381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.415389:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.415391:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.415392:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.415393:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.415395:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.415398:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677448.415512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.415521:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.415524:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.415526:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.415528:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.415529:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.415530:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.415531:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.415533:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.415534:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.415535:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.415538:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.415540:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.415542:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.415543:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.415543:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.415545:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.415545:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.415546:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.415547:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.415550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.415558:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.415560:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.415561:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.415562:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.415564:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.415565:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.415695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.415701:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.415704:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.415705:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.415707:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.415710:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.415711:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.415713:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.415714:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.415715:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.415716:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.415719:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.415720:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.415722:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.415723:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.415723:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.415725:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.415725:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.415726:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.415726:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:21.2:1619677448.415730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:9.2:1619677448.415731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.415738:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.415739:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.415741:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:30.0:1619677448.415742:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:18.0:1619677448.415742:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.415742:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.415745:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.415745:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.415745:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.415746:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:30.0:1619677448.415748:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.415750:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.415751:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.415752:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677448.415752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:30.0:1619677448.415753:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.415754:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.415757:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.415759:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.415759:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.415762:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.415762:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.415763:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.415763:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677448.415764:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.415764:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.415764:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677448.415765:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.415765:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677448.415766:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.415767:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.415768:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.415771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.415779:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.415782:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.415783:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.415783:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:21.2:1619677448.415784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:22.0:1619677448.415785:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.415788:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677448.415790:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.415793:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.415793:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.415795:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.415797:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.415799:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.415800:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.415801:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.415802:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.415803:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.415807:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.415809:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.415812:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.415813:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.415813:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.415815:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.415815:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.415816:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.415817:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.415821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.415828:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.415830:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.415831:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.415833:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.415834:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.415835:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.415870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.415877:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.415880:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.415881:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.415882:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.415886:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.415887:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.415888:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.415890:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.415891:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.415892:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.415895:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.415897:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.415900:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.415901:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.415901:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.415903:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.415903:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.415905:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.415906:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.415907:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.415909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.415912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.415918:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677448.415919:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.415920:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677448.415921:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:1.0:1619677448.415921:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.415922:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.415922:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.415923:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677448.415924:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.415927:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.415946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.415953:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.415955:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.415956:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.415958:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.415960:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.415962:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.415963:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.415964:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.415965:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.415966:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.415969:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.415971:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.415974:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.415975:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.415975:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.415977:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.415978:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.415978:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.415979:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.415983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.415992:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.415994:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.415995:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.415997:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.415997:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.415999:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.416125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:21.2:1619677448.416129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:13.0:1619677448.416134:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:6.0:1619677448.416135:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677448.416137:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:6.0:1619677448.416137:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.416138:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:13.0:1619677448.416140:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.416140:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:6.0:1619677448.416141:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.416143:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.416144:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.416147:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:6.0:1619677448.416147:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:13.0:1619677448.416149:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.416149:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.416151:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.416152:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.416153:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677448.416154:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677448.416155:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:6.0:1619677448.416155:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:13.0:1619677448.416159:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.416161:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:13.0:1619677448.416164:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.416164:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.416165:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:13.0:1619677448.416165:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:6.0:1619677448.416166:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:13.0:1619677448.416167:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:13.0:1619677448.416167:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677448.416168:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677448.416169:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:6.0:1619677448.416169:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.416170:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.416171:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:25.2:1619677448.416173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:6.0:1619677448.416173:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.416173:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.416174:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.416175:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.416179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.416183:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.416184:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.416185:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.416186:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.2:1619677448.416186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000400:00000200:31.0:1619677448.416187:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.416188:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.416189:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677448.416189:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.416190:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:30.0:1619677448.416190:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.416192:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.416192:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:30.0:1619677448.416193:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:1.0:1619677448.416195:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677448.416196:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:25.2:1619677448.416198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:1.0:1619677448.416198:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.416200:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.416202:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677448.416203:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.416205:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:1.0:1619677448.416205:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.416206:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.416207:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.416208:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.416209:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.416209:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:1.0:1619677448.416210:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.416211:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.416211:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:1.0:1619677448.416212:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677448.416215:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.416216:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:1.0:1619677448.416216:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.416218:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677448.416218:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677448.416219:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677448.416220:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677448.416224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.416232:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.416234:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.416235:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.416237:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.416238:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.416239:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.416303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.416308:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.416310:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.416313:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.416313:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.416315:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.416317:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677448.416318:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.416320:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.416321:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.416322:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677448.416325:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.416327:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.416330:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.416330:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.416331:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.416333:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.416333:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.416334:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.416335:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.416339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.416348:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.416351:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.416352:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.416352:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.416354:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.416356:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.416367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.416373:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.416376:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.416377:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.416379:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.416382:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.416383:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.416384:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.416385:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.416387:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.416387:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.416391:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.416393:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.416396:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.416397:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.416397:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.416399:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.416399:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.416400:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.416400:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.416406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.416413:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.416416:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.416417:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.416419:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.416420:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.416421:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.416586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.416589:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.416592:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.416594:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.416595:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.416597:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.416598:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677448.416600:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.416601:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.416602:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.416603:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677448.416606:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.416608:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677448.416611:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.416612:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677448.416613:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677448.416615:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677448.416615:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.416616:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.416618:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.416618:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:9.2:1619677448.416620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.2:1619677448.416623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:24.0:1619677448.416627:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:5.2:1619677448.416628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.0:1619677448.416630:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.416631:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.416631:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677448.416632:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000400:00000200:24.0:1619677448.416633:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.416633:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677448.416635:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:24.0:1619677448.416636:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:1.0:1619677448.416636:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.0:1619677448.416637:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (23)--
00000800:00000200:17.0:1619677448.416637:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.416639:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (22)--
00000400:00000200:1.0:1619677448.416639:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.416640:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:1.0:1619677448.416642:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.416643:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:1.0:1619677448.416643:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.416644:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.416646:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.416646:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:1.0:1619677448.416646:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.416647:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.416649:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.416649:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.416652:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.416652:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.416654:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:1.0:1619677448.416654:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.416655:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.416657:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.416658:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.416658:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:26.0:1619677448.416659:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.416659:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:26.0:1619677448.416660:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.416661:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.416661:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.416661:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.416662:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677448.416663:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.416665:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:25.2:1619677448.416666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.416668:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.416669:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.416669:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.416671:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.416671:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.416672:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.416673:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.416673:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:25.2:1619677448.416676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.416676:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.416676:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.416678:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.416680:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.416681:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.416682:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.416684:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.416685:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.416687:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.416688:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.416689:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.416854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.416862:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.2:1619677448.416863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:31.0:1619677448.416865:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.416867:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.416870:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.416870:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.416870:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:31.0:1619677448.416871:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.416873:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.416873:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:31.0:1619677448.416874:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.416874:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:31.0:1619677448.416875:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.416877:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.416877:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.416879:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.416880:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.416881:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.416881:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.416882:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.416883:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.416884:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.416884:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.416885:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:10.0:1619677448.416885:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.416886:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.416888:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.416888:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677448.416888:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:31.0:1619677448.416890:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:10.0:1619677448.416890:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.416891:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.416892:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.416893:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.416894:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.2:1619677448.416894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.416895:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (22)++
00000800:00000200:10.0:1619677448.416897:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (24)--
00000800:00000200:10.0:1619677448.416897:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.2:1619677448.416898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.416898:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.416898:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:30.0:1619677448.416906:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.416906:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (23)++
00000800:00000200:30.0:1619677448.416909:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:20.0:1619677448.416909:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (24)--
00000800:00000200:30.0:1619677448.416910:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.416910:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.416911:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.416912:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.416912:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (23)++
00000800:00000200:5.0:1619677448.416913:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (24)--
00000400:00000200:5.0:1619677448.416914:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.416915:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.416916:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:5.0:1619677448.416916:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677448.416918:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.416918:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.416919:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (23)++
00000800:00000200:20.0:1619677448.416919:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.416920:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.416921:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.416939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.416946:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.416949:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.416950:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.416951:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.416954:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.416956:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.416957:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.416958:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.416959:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.416960:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:21.2:1619677448.416962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:26.0:1619677448.416963:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.416965:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.416967:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.416967:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.416968:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.416969:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.416970:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.416970:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.416971:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.416971:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.416972:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.416974:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.416975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:30.0:1619677448.416977:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.416978:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.416979:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:30.0:1619677448.416979:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677448.416980:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.416980:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.416981:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.416981:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.416982:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.416983:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.416983:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.416986:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.416986:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.416987:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.416988:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.416990:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.416991:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.416992:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.416993:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.416994:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.416995:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.416995:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.416999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.417005:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.417007:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.417008:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.417009:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.417010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:12.0:1619677448.417011:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.417012:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.417015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:8.0:1619677448.417017:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.2:1619677448.417018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677448.417020:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.417021:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677448.417021:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:8.0:1619677448.417022:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.417024:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:12.0:1619677448.417024:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.417024:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:8.0:1619677448.417025:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.417026:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:22.0:1619677448.417027:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.417028:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.417028:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677448.417028:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.417029:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.417029:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:8.0:1619677448.417029:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.417030:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.417030:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.417031:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677448.417031:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:22.0:1619677448.417032:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.417034:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.417035:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.417036:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.417037:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.417038:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.0:1619677448.417038:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.417041:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.417041:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:22.0:1619677448.417042:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.417042:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.417043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677448.417044:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.0:1619677448.417044:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.417044:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.417045:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.417046:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:22.0:1619677448.417047:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.417047:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.2:1619677448.417047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677448.417048:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:25.2:1619677448.417049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677448.417049:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.417050:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677448.417051:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.417051:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.417052:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:21.2:1619677448.417052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:17.0:1619677448.417052:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.417053:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:10.0:1619677448.417053:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.417054:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677448.417054:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677448.417055:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:13.0:1619677448.417055:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677448.417056:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.417057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:10.0:1619677448.417057:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.417058:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:24.0:1619677448.417059:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:21.2:1619677448.417059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:17.0:1619677448.417059:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.417059:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.417060:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.417061:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:17.0:1619677448.417061:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.417061:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.417061:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:24.0:1619677448.417062:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677448.417062:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.417063:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:17.0:1619677448.417063:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:13.0:1619677448.417063:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:10.0:1619677448.417063:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.417064:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:6.0:1619677448.417064:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:17.0:1619677448.417065:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.417066:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.417067:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:17.0:1619677448.417067:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.417067:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:31.0:1619677448.417068:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.417068:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677448.417068:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.417068:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:17.0:1619677448.417069:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.417070:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.417070:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:17.0:1619677448.417070:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677448.417070:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.417073:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:17.0:1619677448.417076:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.417076:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.417078:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677448.417079:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.417082:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.417082:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:10.0:1619677448.417082:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.417083:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:10.0:1619677448.417083:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.417084:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.417085:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.417086:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.417086:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.417087:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:10.0:1619677448.417087:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.2:1619677448.417087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:10.0:1619677448.417088:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:17.0:1619677448.417089:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.417090:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.417090:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:25.2:1619677448.417091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.417091:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.417092:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:9.2:1619677448.417092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.417094:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677448.417094:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:5.2:1619677448.417095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:17.0:1619677448.417097:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.417098:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:5.2:1619677448.417098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.417099:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:17.0:1619677448.417100:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.417101:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677448.417101:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677448.417102:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:17.0:1619677448.417102:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:12.0:1619677448.417102:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:8.0:1619677448.417102:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.417102:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:12.0:1619677448.417103:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.417103:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.417104:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96715218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.417104:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677448.417105:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.417106:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:17.0:1619677448.417106:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.417106:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:8.0:1619677448.417106:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.417107:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.417108:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.417108:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:17.0:1619677448.417108:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.417109:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.417109:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.417110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:17.0:1619677448.417112:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.417113:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.2:1619677448.417115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.417115:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.417116:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.417116:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.417116:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.417117:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.417118:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.417118:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.417119:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.417119:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:8.0:1619677448.417119:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.417120:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677448.417121:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.0:1619677448.417121:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:13.0:1619677448.417122:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.417122:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677448.417125:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:8.2:1619677448.417125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:13.0:1619677448.417126:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.417127:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.417127:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.417129:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677448.417133:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.417135:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.417136:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.417137:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.417138:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.417140:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.417146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.417153:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.417156:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.417157:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.417157:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.417158:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.417158:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.417343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.417350:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.417353:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.417354:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.417356:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.417358:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.417360:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.417361:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.417362:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.417363:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.417364:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.417368:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.417369:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.417372:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.417373:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.417373:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.417375:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.417375:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.417376:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.417378:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.417378:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.417381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.417384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.417389:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:10.0:1619677448.417391:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:30.0:1619677448.417392:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.417392:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.417393:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:30.0:1619677448.417394:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.417394:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.417395:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.417396:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.417397:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.417407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.417414:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.417417:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.417418:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.417419:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.417422:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.417423:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.417425:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.417426:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.417427:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.417428:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.417431:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.417433:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.417436:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.417437:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.417437:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.417439:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.417439:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.417440:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.417440:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.417445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.417453:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.417455:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.417456:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.417457:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.417459:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.417461:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.417474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.417480:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.417482:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.417483:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.417485:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.417487:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.417489:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.417490:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.417491:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.417492:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.417493:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.417497:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.417499:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.417501:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.417502:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.417503:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.417505:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.417505:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.417506:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.417506:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.417511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.417518:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.417521:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.417522:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.417524:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.417525:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.417527:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.417530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:21.2:1619677448.417532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.417537:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.417539:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.417540:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.417540:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.417541:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.417541:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.417542:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.417542:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677448.417544:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.417544:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.417546:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.417547:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.417548:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.417550:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.417551:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.417552:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.417555:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.417557:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.417559:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.417560:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.417561:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.417562:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.417563:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.417564:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.417565:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.417566:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.2:1619677448.417567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:25.2:1619677448.417568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.417572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.417574:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.417575:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677448.417577:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:12.0:1619677448.417577:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.417578:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.417578:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:12.0:1619677448.417579:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.417580:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:26.0:1619677448.417581:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.417581:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.417582:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.417582:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.417583:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:26.0:1619677448.417583:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.417583:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677448.417584:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.417586:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.417587:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.417588:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.417588:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.417591:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.417593:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.417594:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.417595:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.417596:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.417597:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.417598:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.417599:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.417599:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.417603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.417609:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.417611:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.417611:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.417612:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.417613:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.417615:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.417702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.417708:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.417711:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:4.2:1619677448.417712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:5.0:1619677448.417713:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.417715:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.417716:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.417717:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:26.0:1619677448.417719:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677448.417719:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.417720:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.417721:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.417722:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:5.0:1619677448.417722:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.417723:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.417726:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.417726:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.417728:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.417728:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.417729:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.417730:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.417732:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.417733:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.417733:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.417733:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:26.0:1619677448.417734:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.417735:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.417735:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.417736:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.417737:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.417737:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:26.0:1619677448.417738:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.417740:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:25.2:1619677448.417741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.417743:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.417744:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.417744:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.417746:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.417747:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.417748:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.417748:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:12.0:1619677448.417748:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.417750:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.417751:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.417753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:12.0:1619677448.417753:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.417755:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.417756:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.417761:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.417764:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.417765:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.417765:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.417767:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.417770:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.417891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.417897:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677448.417899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:1.0:1619677448.417900:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.417901:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.417903:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.417905:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000400:00000200:1.0:1619677448.417906:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.417907:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:22.0:1619677448.417908:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677448.417908:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:1.0:1619677448.417909:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.417910:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.417911:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.417912:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.417913:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.417914:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.417916:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:1.0:1619677448.417917:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.417918:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.417919:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.417921:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.417921:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.417922:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:1.0:1619677448.417924:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.417924:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.417925:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:22.0:1619677448.417926:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.417927:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.417927:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.417928:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:1.0:1619677448.417928:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.417929:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.417931:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.417932:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:25.2:1619677448.417933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.417933:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.2:1619677448.417934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:22.0:1619677448.417935:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:22.0:1619677448.417935:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677448.417936:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677448.417936:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:20.0:1619677448.417936:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.417939:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.417939:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.417941:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.417942:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:25.2:1619677448.417942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677448.417942:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677448.417943:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.417944:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.417945:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.417946:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:20.0:1619677448.417946:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677448.417948:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:20.0:1619677448.417948:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.417949:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.417949:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:20.0:1619677448.417949:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.417950:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677448.417950:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.417950:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.417951:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677448.417952:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.417954:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.417956:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.417958:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.0:1619677448.417960:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.417960:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.417961:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.417963:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.417964:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.417965:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.417966:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.417967:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.417969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.417972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.417976:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.417977:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677448.417979:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.417979:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.417980:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:31.0:1619677448.417981:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.417981:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.417981:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.417983:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.417984:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.418036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.418044:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.418047:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.418049:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.418050:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.418051:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.418052:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:21.2:1619677448.418052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:30.0:1619677448.418054:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.418055:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.418056:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.418057:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.418059:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.418060:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.418060:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.418062:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.418062:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.418064:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.418065:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.418065:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:31.0:1619677448.418066:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:30.0:1619677448.418066:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:31.0:1619677448.418067:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.418067:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:31.0:1619677448.418068:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.418069:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.418070:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.418071:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.418072:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.418072:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:31.0:1619677448.418073:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.418073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.418074:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.418075:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.418077:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.418078:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.418078:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.418079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:13.0:1619677448.418079:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:31.0:1619677448.418080:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.418080:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.418081:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677448.418081:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:31.0:1619677448.418082:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:13.0:1619677448.418082:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.418083:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.418085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.418085:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:12.0:1619677448.418085:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.418086:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677448.418088:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.418089:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.418091:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677448.418091:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:18.0:1619677448.418094:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.418094:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.418095:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:18.0:1619677448.418095:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.418095:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.418097:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.418098:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.418099:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.418100:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.418100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:18.0:1619677448.418101:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.418103:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:18.0:1619677448.418104:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.418104:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677448.418106:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.418107:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.418108:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677448.418109:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.418110:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.418110:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:10.0:1619677448.418110:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.418112:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.418113:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.418113:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:10.0:1619677448.418113:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.418114:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:10.0:1619677448.418114:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.418115:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677448.418116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:10.0:1619677448.418117:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.418118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:10.0:1619677448.418118:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.418119:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.418123:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.418124:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.418125:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:6.0:1619677448.418125:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.418126:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.418128:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.418128:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.418128:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.418129:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.418129:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:6.0:1619677448.418129:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.418130:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:26.0:1619677448.418131:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.418131:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:6.0:1619677448.418131:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:10.0:1619677448.418132:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:26.0:1619677448.418133:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:10.0:1619677448.418133:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.418134:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677448.418134:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:26.0:1619677448.418135:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.418135:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.418136:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.418136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:26.0:1619677448.418138:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.418139:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.418143:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.418143:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.418143:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.418144:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.418145:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.418146:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.418147:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.418147:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.418148:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.418148:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.418148:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.418150:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.418151:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.418151:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.418152:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.418156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.418162:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.418165:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.418166:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.418167:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.418169:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.418169:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.418172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1200] (20)++
00000800:00000200:17.0:1619677448.418178:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677448.418181:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:1.0:1619677448.418181:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:17.0:1619677448.418184:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677448.418187:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.418188:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:17.0:1619677448.418190:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.418191:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.418192:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.418193:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:17.0:1619677448.418196:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677448.418198:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677448.418200:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.418201:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:17.0:1619677448.418202:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:17.0:1619677448.418204:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:17.0:1619677448.418204:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677448.418205:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:17.0:1619677448.418205:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:8.2:1619677448.418210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.418218:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.418220:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.418221:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.418223:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.418223:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.418225:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.2:1619677448.418263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.418266:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.418269:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.418271:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.418271:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.418274:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.418275:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677448.418277:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.418278:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.418279:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.418280:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677448.418283:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.418285:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.418288:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.418289:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.418290:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.418292:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.418292:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.418297:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.418298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.418298:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:10.0:1619677448.418305:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.418307:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.418308:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.418309:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.418310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:10.0:1619677448.418311:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:10.0:1619677448.418314:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.0:1619677448.418316:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:25.2:1619677448.418318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677448.418318:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.418320:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.418321:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.418323:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.418324:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000400:00000200:5.0:1619677448.418324:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.418326:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.418327:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.418327:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:5.0:1619677448.418327:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.418328:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000400:00000200:5.0:1619677448.418328:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.418329:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.418329:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:5.0:1619677448.418329:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.418333:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.418335:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.418338:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.418339:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.418339:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.418341:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.418341:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.418342:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.418343:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.418347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.418354:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.418356:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.418357:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.418359:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.418361:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.418362:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.418503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.418510:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.418512:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.418513:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.418515:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.418517:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.418518:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.418520:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.418521:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.418522:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.418523:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.418526:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.418527:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.418530:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.418531:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.418532:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.418533:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.418534:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.418535:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.418535:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.418539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.418548:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.418551:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.418551:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.418552:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.418554:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.418556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.418559:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.418561:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.418563:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.418564:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.418564:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.418565:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.418566:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.418574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.418580:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.418583:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.418584:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.418586:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.418589:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.418590:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.418591:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.418592:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.418594:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.418594:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.418597:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.418599:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.418601:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.418601:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.418602:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.418603:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.418604:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.418604:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.418605:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.418610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.418618:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.418620:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.418620:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.418621:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.418622:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.418623:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.418626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.418633:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.418635:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.418636:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.418636:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.418637:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.418637:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.418643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.418646:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.418648:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.418651:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.418651:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.418654:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.418656:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.418657:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.418659:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.418659:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.418661:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.418664:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.418666:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.418669:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.418669:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.418670:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.418672:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.418673:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.418673:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.418674:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.418678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.418684:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.418687:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.418688:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.418690:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.418691:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.418693:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.418694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.418700:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.418703:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.418704:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.418705:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.418705:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.418706:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.418745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.418752:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:21.2:1619677448.418753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677448.418755:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.418756:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.418757:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.418760:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.418761:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677448.418762:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:31.0:1619677448.418763:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677448.418763:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.418765:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.418766:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.418766:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.418767:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677448.418768:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.418768:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.418770:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:31.0:1619677448.418771:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.418771:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.418772:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.418773:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.418773:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:31.0:1619677448.418774:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.418776:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.418777:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:31.0:1619677448.418778:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.418778:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.418780:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.418780:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.418780:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.418781:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.418782:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:31.0:1619677448.418783:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.418783:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:31.0:1619677448.418784:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677448.418786:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677448.418786:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:9.2:1619677448.418786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.418787:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.418788:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.418792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677448.418794:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.418797:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.418798:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.418798:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.418799:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:2.0:1619677448.418800:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.418802:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.418803:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.418803:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:2.0:1619677448.418804:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:17.0:1619677448.418805:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.418808:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.418848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.418856:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.418859:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.418861:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.418863:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.418864:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.418865:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.418867:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.418868:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.418869:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.418870:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.418873:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.418875:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.418877:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.418878:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.418878:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.418880:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.418880:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.418881:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.418882:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.418885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.418892:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.418894:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.418895:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.418896:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.418898:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.418899:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.419042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.419049:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.419052:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.419053:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.419055:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.419058:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.419059:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.419060:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.419061:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.419062:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.419063:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.419066:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.419068:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.419071:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.419072:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.419072:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.419074:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.419075:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.419076:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.419077:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.419078:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677448.419080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677448.419083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.419088:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.419090:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:18.0:1619677448.419091:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.419092:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:18.0:1619677448.419092:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.419092:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.419093:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.419094:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:18.0:1619677448.419094:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.419097:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.419177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677448.419181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.419183:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.419185:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.419186:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677448.419186:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677448.419188:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.419189:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.419190:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:2.0:1619677448.419191:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.419192:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.419192:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.419193:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.419194:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.419195:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.419195:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.419196:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.419197:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.419198:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.419198:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.419199:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.419200:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677448.419205:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.419205:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.419207:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677448.419209:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.419210:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.419211:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677448.419212:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.419212:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.419212:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677448.419213:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.419214:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677448.419215:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.419215:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.419216:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:1.0:1619677448.419216:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:2.0:1619677448.419217:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.419217:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677448.419217:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.419218:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.419221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:25.2:1619677448.419222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.419224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.419229:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.419230:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.419231:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.419231:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.419231:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677448.419232:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677448.419232:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677448.419233:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:17.0:1619677448.419233:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.419233:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.419233:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.419234:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.419236:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.419236:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.419239:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677448.419239:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.419241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:22.0:1619677448.419248:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.419250:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.419252:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.419252:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.419253:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.419253:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.419295:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.419303:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.419306:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.419307:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.419309:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.419312:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.419314:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.419315:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.419316:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.419317:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.419318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:8.0:1619677448.419318:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677448.419322:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.419323:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677448.419324:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:6.0:1619677448.419326:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.419327:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677448.419328:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.419329:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:6.0:1619677448.419329:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.419330:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.419332:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.419332:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:6.0:1619677448.419332:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.419333:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.419333:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:6.0:1619677448.419334:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.419335:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.419336:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677448.419337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:6.0:1619677448.419337:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.419338:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.419342:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.419344:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:5.0:1619677448.419345:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:6.0:1619677448.419347:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.419348:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.419348:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:5.0:1619677448.419348:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.419349:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.419350:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000400:00000200:5.0:1619677448.419350:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.419351:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:6.0:1619677448.419351:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.419351:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.419352:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677448.419354:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:25.2:1619677448.419357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.419364:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.419367:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.419367:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.419368:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.419370:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.419373:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677448.419402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.419405:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.419408:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.419410:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.419410:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.419412:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.419414:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677448.419415:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.419417:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.419418:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.419419:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677448.419422:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.419424:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.419426:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.419427:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.419428:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.419429:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.419430:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.419430:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.419431:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.419435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.419442:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.419444:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.419445:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.419447:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.419447:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.419449:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.419461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.419469:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.419471:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.419474:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.419476:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.419476:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.419477:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.419478:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.419480:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.419481:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.419482:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677448.419486:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.419487:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.419491:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.419491:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.419492:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.419494:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.419494:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.419495:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.419496:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.419500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.419509:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.419512:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.419513:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.419513:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.419515:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.419518:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.419522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.419529:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.419531:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.419532:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.419532:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.419533:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.419534:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.419606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.419612:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.419615:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.419616:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.419618:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.419620:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.419621:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.419622:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.419624:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.419625:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.419626:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.419629:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.419631:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.419633:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.419634:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.419634:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.419636:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.419636:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.419637:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.419638:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.419642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.419650:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.419652:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.419653:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.419653:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.419656:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.419658:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.419713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.419720:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.419723:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.419723:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.419725:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.419728:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.419729:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:5.2:1619677448.419729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000400:00000200:26.0:1619677448.419731:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.419732:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.419733:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.419733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:26.0:1619677448.419734:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.419737:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.419737:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:26.0:1619677448.419739:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.419739:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:13.0:1619677448.419740:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:18.0:1619677448.419742:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:13.0:1619677448.419742:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.419743:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.419743:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.419743:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:26.0:1619677448.419744:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677448.419744:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:18.0:1619677448.419745:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.419745:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.419746:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.419747:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:18.0:1619677448.419747:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.419747:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:26.0:1619677448.419748:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.419748:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:13.0:1619677448.419748:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.419749:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:13.0:1619677448.419750:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.419751:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677448.419751:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.419752:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677448.419752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677448.419753:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677448.419753:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.419755:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:13.0:1619677448.419757:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.419759:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.419761:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:13.0:1619677448.419762:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.419762:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.419763:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.419763:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:2.0:1619677448.419763:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.419764:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:2.0:1619677448.419765:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.419766:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.419766:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:13.0:1619677448.419767:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677448.419768:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677448.419769:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:2.0:1619677448.419769:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.419770:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.419770:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:18.0:1619677448.419771:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.419771:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677448.419772:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (22)++
00000800:00000200:8.2:1619677448.419772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.419775:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:18.0:1619677448.419775:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677448.419775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:18.0:1619677448.419777:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.419778:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.419778:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677448.419779:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)++
00000800:00000200:20.0:1619677448.419782:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (24)--
00000400:00000200:20.0:1619677448.419783:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.419783:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000400:00000200:20.0:1619677448.419784:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.419784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677448.419786:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677448.419786:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:20.0:1619677448.419787:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:12.0:1619677448.419787:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:20.0:1619677448.419789:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677448.419789:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:20.0:1619677448.419791:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.419792:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.419792:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:20.0:1619677448.419793:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.419793:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.419794:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677448.419795:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.419796:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.419802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.419809:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.419812:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.419815:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.419816:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.419818:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.419819:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.419820:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.419821:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.419821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:30.0:1619677448.419822:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.419823:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.419827:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.419827:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:30.0:1619677448.419829:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677448.419830:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:30.0:1619677448.419831:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.419831:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:30.0:1619677448.419832:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.419832:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:18.0:1619677448.419833:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.419834:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.419834:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.419835:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:18.0:1619677448.419835:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.419836:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:18.0:1619677448.419837:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677448.419838:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.419839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:18.0:1619677448.419839:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.419840:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.419842:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677448.419845:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.419846:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:18.0:1619677448.419847:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.419849:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:18.0:1619677448.419849:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.419850:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.419850:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677448.419851:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677448.419852:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.419853:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.419853:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:18.0:1619677448.419853:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677448.419853:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.419854:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.419855:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.419856:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.419858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:8.2:1619677448.419858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.419861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:30.0:1619677448.419864:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.419866:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677448.419866:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:30.0:1619677448.419867:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.419868:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.419868:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.419869:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677448.419869:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:2.0:1619677448.419869:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:20.0:1619677448.419870:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.419870:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:30.0:1619677448.419871:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:20.0:1619677448.419871:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.419871:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677448.419872:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.419874:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.419884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.419891:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.419894:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.419895:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.419896:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.419896:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.419897:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.419929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.2:1619677448.419933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.419936:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.419939:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.419939:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.419940:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.419942:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:10.0:1619677448.419943:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.419943:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.419945:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.419946:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.419948:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.419948:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.419949:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.419950:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.419951:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.419952:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.419953:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.419954:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.419954:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.419955:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.419956:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.419961:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.419962:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.419964:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.419965:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.419967:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.419967:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.419968:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.419968:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.419969:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.419969:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.419970:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.419971:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.419971:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.419972:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.419972:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.419973:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.419973:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.419973:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.2:1619677448.419976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:25.2:1619677448.419977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.419983:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.419984:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:12.0:1619677448.419986:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677448.419986:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.419987:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.419988:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:12.0:1619677448.419988:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.419990:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.419991:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.419991:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.419992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:1.0:1619677448.419992:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:12.0:1619677448.419994:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.419997:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.420000:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.420001:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.420003:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.420006:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.420007:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.420008:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.420010:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.420011:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.420012:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.420015:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.420017:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.420020:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.420021:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.420021:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.420023:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.420023:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.420024:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.420024:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.420046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.420053:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.420056:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.420056:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.420058:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.420059:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.420061:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.420069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.420076:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.420079:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.420080:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.420081:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.420082:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.420083:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.420201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.420206:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.420209:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.420210:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.420212:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.420214:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.420215:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.420216:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.420218:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.420219:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.420220:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.420223:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.420225:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.420228:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.420229:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.420229:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.420231:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.420232:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.420232:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.420233:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.420237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.420244:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.420247:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.420248:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.420249:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.420251:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.420252:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.420285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.420288:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.420290:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.420293:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.420293:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.420299:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.420300:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677448.420301:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.420303:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.420304:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.420305:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677448.420308:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.420310:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677448.420314:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.420314:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677448.420315:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677448.420317:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677448.420318:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.420318:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.420319:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:5.2:1619677448.420320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:9.2:1619677448.420324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:2.0:1619677448.420326:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.420329:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.420330:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:24.0:1619677448.420331:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:2.0:1619677448.420332:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.420333:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.420334:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:2.0:1619677448.420334:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.420335:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.420336:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:21.2:1619677448.420337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:24.0:1619677448.420338:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.420338:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.420340:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.420341:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:2.0:1619677448.420341:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.420343:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:2.0:1619677448.420343:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.420344:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:18.0:1619677448.420347:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.420348:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.420349:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.420349:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.420351:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677448.420352:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.420352:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.420353:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.420353:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:18.0:1619677448.420354:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.420354:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:18.0:1619677448.420355:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.420356:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.420357:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.420358:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.420359:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.420359:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677448.420360:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.420361:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.2:1619677448.420362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.420363:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.420364:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677448.420364:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.420366:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677448.420367:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.420367:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.420368:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:1.0:1619677448.420370:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.2:1619677448.420372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.420373:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.420374:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.420374:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.420376:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.420378:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677448.420380:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.420382:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.420383:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.420385:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.420385:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.420387:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.420428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.420436:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.420438:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.420439:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.420441:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.420443:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.420445:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.420446:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.420448:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.420449:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.420450:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.420453:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.420455:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.420458:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.420459:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.420459:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.420461:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.420461:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.420462:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.420462:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.420466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.420474:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.420476:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.420477:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.420479:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.420479:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.420482:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.420615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.420623:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.420626:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.420628:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.420630:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.420631:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.420632:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.420633:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.420634:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.420635:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.420636:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.420639:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.420641:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.420644:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.420645:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.420646:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.420648:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.420648:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.420649:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.420651:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.420651:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.420654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.420657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677448.420662:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:30.0:1619677448.420665:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677448.420665:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.420666:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:30.0:1619677448.420667:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:22.0:1619677448.420667:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.420668:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.420669:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:22.0:1619677448.420669:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.420672:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.420674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.420682:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.420685:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.420687:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.420688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677448.420689:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.420689:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.420691:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.420692:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.420693:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.420693:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677448.420694:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.420695:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.420696:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.420697:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677448.420700:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.420700:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.420702:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.420702:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.420703:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:30.0:1619677448.420705:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.420705:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.420706:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:17.0:1619677448.420706:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.420707:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:17.0:1619677448.420707:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.420708:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.420708:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.420709:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.420710:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.420711:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.420712:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:17.0:1619677448.420712:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.420714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.420714:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.2:1619677448.420717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677448.420717:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.420718:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677448.420718:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:17.0:1619677448.420720:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:17.0:1619677448.420720:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.420721:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:17.0:1619677448.420722:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.420723:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.420723:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677448.420724:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:31.0:1619677448.420725:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:12.0:1619677448.420725:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.420727:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.420728:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:13.0:1619677448.420728:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:5.2:1619677448.420728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.420729:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:12.0:1619677448.420729:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:31.0:1619677448.420730:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677448.420730:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677448.420730:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.420731:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677448.420732:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.420734:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677448.420734:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:25.2:1619677448.420736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.420736:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.420737:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.420738:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.420742:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.420745:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.420746:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.420747:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.420747:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.420748:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.420759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.420765:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.420767:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.420770:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.420770:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.420773:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.420774:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677448.420775:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.420777:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.420778:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.420779:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677448.420782:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.420784:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.420787:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.420787:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.420788:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.420790:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.420791:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.420792:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.420793:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.420794:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677448.420796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677448.420799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.420802:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677448.420803:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:20.0:1619677448.420804:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.420805:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.420805:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677448.420806:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.420807:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.420807:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.420809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:1.0:1619677448.420809:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.420812:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:10.0:1619677448.420816:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.420819:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.420819:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.420822:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.420822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1200] (20)++
00000400:00000200:10.0:1619677448.420824:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.420826:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.420827:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.420828:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000400:00000200:10.0:1619677448.420828:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.420829:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.420830:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.420831:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:1.0:1619677448.420831:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:17.0:1619677448.420835:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.420835:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.420837:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.420837:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:17.0:1619677448.420838:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:17.0:1619677448.420839:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.420840:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.420841:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.420841:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:17.0:1619677448.420842:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.420842:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:17.0:1619677448.420843:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:10.0:1619677448.420844:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.420844:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.420845:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:17.0:1619677448.420846:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.420847:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.420847:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.420848:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:9.2:1619677448.420850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.420851:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.420852:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:17.0:1619677448.420852:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:5.2:1619677448.420853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.420854:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:17.0:1619677448.420854:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677448.420855:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:17.0:1619677448.420856:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:2.0:1619677448.420857:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.420860:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.2:1619677448.420860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:2.0:1619677448.420860:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.420861:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.420861:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.420862:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.420863:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.420864:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:2.0:1619677448.420864:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.420867:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:2.0:1619677448.420867:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677448.420869:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.420870:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.420870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:20.0:1619677448.420872:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.420873:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.420874:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:18.0:1619677448.420877:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.420879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.420880:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.420882:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.420882:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.420883:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.420883:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.420883:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.420884:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.420884:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677448.420887:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.420890:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.420891:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.420892:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715166] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.420894:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.420895:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.420897:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.420900:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.420902:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.420905:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.420905:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.420906:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.420908:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.420909:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.420909:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.420910:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.420914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.420920:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.420923:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.420924:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.420924:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.420926:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.420929:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.421042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.421047:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.421050:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.421051:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.421053:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.421055:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.421056:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.421058:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.421059:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.421060:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.421061:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.421064:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.421066:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.421069:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.421070:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.421070:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.421072:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.421073:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.421073:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.421073:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.421078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.421085:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.421088:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.421089:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.421090:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.421092:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.421093:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.421096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.421102:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.421105:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.421106:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.421107:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.421107:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.421109:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.421209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.421217:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.421219:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.421221:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.421223:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.421224:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.421225:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.421226:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.421228:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.421229:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.421230:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.421233:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.421234:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.421238:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.421239:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.421239:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.421241:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.421242:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.421243:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.421243:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.421247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.421255:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.421258:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.421259:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.421261:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.421261:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.421263:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.421358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.421366:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.421368:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.421369:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.421371:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.421374:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.421375:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.421376:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.421378:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.421379:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.421380:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.421383:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.421384:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.421387:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.421388:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.421389:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.421391:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.421391:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.421392:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.421392:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.421397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.421405:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.421408:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.421408:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.421410:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.421411:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.421413:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.421586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.421592:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.421595:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.421596:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.421598:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.421601:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.421602:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.421604:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.421605:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.421606:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.421607:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.421611:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.421612:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.421616:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.421616:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.421617:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.421619:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.421619:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.421620:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.421621:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:21.2:1619677448.421625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:25.2:1619677448.421626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.421631:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:12.0:1619677448.421633:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:22.0:1619677448.421634:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.421635:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677448.421636:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.421636:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.421637:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677448.421638:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.421640:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.421640:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:22.0:1619677448.421641:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:12.0:1619677448.421641:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:22.0:1619677448.421643:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.421644:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.421645:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.421646:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.421649:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.421651:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.421654:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.421655:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.421655:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.421655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:22.0:1619677448.421657:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.421658:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.421659:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.421660:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.421661:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:10.0:1619677448.421662:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:8.2:1619677448.421663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677448.421665:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677448.421666:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:21.2:1619677448.421666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677448.421668:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.421671:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.421671:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:1.0:1619677448.421671:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:10.0:1619677448.421673:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:6.0:1619677448.421673:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:10.0:1619677448.421674:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.421674:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677448.421674:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:10.0:1619677448.421675:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.421675:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:1.0:1619677448.421675:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.421676:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:10.0:1619677448.421677:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.421678:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.421678:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.421681:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:10.0:1619677448.421682:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.421684:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.421686:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.421687:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.421688:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.421689:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.421690:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.421690:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.421691:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677448.421695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.421703:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.421705:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.421706:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.421707:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.421710:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.421711:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.421741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.2:1619677448.421748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.421749:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.421751:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.421752:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.421753:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.421754:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.421756:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.421756:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.421757:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.421759:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:17.0:1619677448.421759:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.421761:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.421762:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.421762:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.421764:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.421764:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.421765:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:17.0:1619677448.421766:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.421767:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.421768:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.421770:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.421772:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.421774:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.421775:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.421777:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.421777:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.421778:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.421778:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.421780:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.421780:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.421780:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.421781:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.421781:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677448.421781:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.421782:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.421784:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.421784:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:25.2:1619677448.421785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:17.0:1619677448.421785:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.421786:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.421789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.421792:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.421794:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.421795:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.421796:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:31.0:1619677448.421797:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.421798:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:13.0:1619677448.421798:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.421799:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.421800:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:13.0:1619677448.421801:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.421802:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.421804:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.421848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.2:1619677448.421852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:30.0:1619677448.421855:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.421857:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677448.421858:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.421860:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.421860:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.421861:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:31.0:1619677448.421862:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.421863:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.421863:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.421865:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.421866:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.421867:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.421868:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.421869:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.421869:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.421870:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.421871:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.421871:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.421872:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.421873:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.421876:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.421878:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.421880:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.421881:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.421883:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.421884:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.421884:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.421885:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.421885:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.421885:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.421886:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.421887:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.421887:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.421888:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.421888:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677448.421888:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.421889:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.421890:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:25.2:1619677448.421891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.421891:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677448.421895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.421896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:12.0:1619677448.421898:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.421900:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677448.421901:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:12.0:1619677448.421901:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.421903:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677448.421903:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:12.0:1619677448.421903:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.421904:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:13.0:1619677448.421904:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677448.421905:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:18.0:1619677448.421905:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:22.0:1619677448.421906:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.421906:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:12.0:1619677448.421907:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.421908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:22.0:1619677448.421909:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.421911:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677448.421914:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.421917:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.421918:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.421919:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.421919:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.421921:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.421941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.421949:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.421951:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.421954:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.421955:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.421956:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.421958:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.421959:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.421960:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.421961:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.421962:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.421966:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.421968:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.421971:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.421972:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.421973:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.421974:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.421975:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.421976:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.421976:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.421980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.421988:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.421991:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.421992:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.421994:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.421994:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.421996:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.422001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.422008:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.422011:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.422012:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.422014:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.422016:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.422018:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.422019:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.422021:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.422022:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.422023:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.422026:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.422028:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.422030:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.422031:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.422032:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.422034:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.422034:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.422035:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.422035:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.422040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.422047:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.422050:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.422051:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.422051:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.422053:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.422056:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.422059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.422067:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.422069:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.422070:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.422070:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.422071:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.422072:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.422098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677448.422106:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677448.422108:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677448.422110:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.422112:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677448.422113:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.422115:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677448.422116:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.422117:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.422118:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677448.422119:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677448.422123:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.422124:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:4.2:1619677448.422125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677448.422127:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.422128:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:12.0:1619677448.422128:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677448.422130:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677448.422131:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677448.422132:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677448.422132:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:10.0:1619677448.422132:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.422135:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.422136:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.422136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:10.0:1619677448.422137:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.422140:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.422141:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.422142:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.422143:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.422144:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.422144:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:10.0:1619677448.422145:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.422146:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.422147:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.422147:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.422148:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.422150:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.422151:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.422153:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.422153:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677448.422154:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.422154:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.422156:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.422156:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.422158:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.422159:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.422159:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.422162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.422165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.422168:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.422170:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.422172:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.422172:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:30.0:1619677448.422173:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.422174:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.422174:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.422174:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.422176:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.422176:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.2:1619677448.422194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.422201:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.422204:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.422205:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.422207:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.422209:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.422210:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677448.422212:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.422213:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.422214:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.422214:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.422217:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.422219:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.422222:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.422222:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.422223:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.422225:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.422225:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.422226:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677448.422227:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.422228:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677448.422230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.422234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.422241:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677448.422241:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:26.0:1619677448.422242:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.422243:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.422244:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:20.0:1619677448.422244:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.422244:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.422246:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.422247:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.422249:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.422318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:5.2:1619677448.422318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.422324:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677448.422325:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.422327:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:2.0:1619677448.422327:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.422328:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:1.0:1619677448.422328:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.422331:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.422331:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.422333:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.422334:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.422335:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.422336:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.422337:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.422337:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.422339:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.422339:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.422340:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.422341:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.422341:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.422342:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.422345:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.422347:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.422350:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.422351:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.422351:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.422352:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.422352:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.422354:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.422355:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.422355:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677448.422356:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.422356:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.422356:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.422356:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677448.422358:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677448.422359:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.422360:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.422360:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.422361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.422369:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.422371:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677448.422372:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.422374:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.422375:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677448.422376:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677448.422377:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.422377:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.422379:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.422379:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.422380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:12.0:1619677448.422381:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.422388:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.422391:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.422393:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.422395:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.422396:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.422397:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.422398:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.422400:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.422401:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.422402:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.422405:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.422407:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.422410:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.422411:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.422411:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.422413:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.422413:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.422415:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.422415:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.422418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.422425:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.422427:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.422428:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.422430:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.422431:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.422432:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.422502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.422509:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.422511:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.422512:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.422515:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.422517:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.422519:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.422520:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.422521:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.422522:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.422523:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.422527:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.422529:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.422532:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.422532:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.422533:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.422535:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.422535:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.422536:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.422537:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677448.422542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677448.422542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.422543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677448.422548:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.422549:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:10.0:1619677448.422550:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677448.422551:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677448.422552:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677448.422553:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677448.422553:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:5.0:1619677448.422553:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.422553:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.422554:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.422554:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:1.0:1619677448.422555:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.422556:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.422557:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.422558:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677448.422558:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:5.0:1619677448.422559:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.422560:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.422560:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.422561:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.422562:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.422563:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.422563:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.422565:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.422567:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.422567:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.422568:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.422570:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.422572:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.422573:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.422574:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.422574:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.422575:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.422576:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.422577:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.422577:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.422577:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.422578:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.422579:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.422579:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.422579:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:10.0:1619677448.422580:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.422581:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.422581:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.422582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:25.2:1619677448.422586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:12.0:1619677448.422589:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.422592:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:30.0:1619677448.422593:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:12.0:1619677448.422593:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.422594:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.422595:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.422596:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.422596:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.422597:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:30.0:1619677448.422598:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.422598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677448.422599:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.422600:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.422600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.422606:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677448.422606:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.422608:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:13.0:1619677448.422609:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.422609:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:13.0:1619677448.422610:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:26.0:1619677448.422611:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.422611:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.422611:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.422612:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677448.422614:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.422615:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.422616:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.422618:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.422619:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.422619:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.422623:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.422624:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.422627:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.422627:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.422628:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.422629:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.422630:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.422631:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.422631:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.422635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.422642:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.422644:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.422645:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.422646:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.422648:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.422648:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.2:1619677448.422761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.422767:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.422769:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.422770:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.422772:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.422775:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.422776:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.422777:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.422779:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.422780:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.422781:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.422784:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.422786:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.422788:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.422789:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.422790:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.422792:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.422792:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.422793:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.422794:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.422798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.422804:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.422807:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.422808:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.422808:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677448.422810:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.422813:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677448.422888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.422895:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.422898:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.422900:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.422902:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.422902:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.422904:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.422905:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.422906:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.422907:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.422908:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.422912:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.422913:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.422916:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.422916:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.422917:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.422919:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.422919:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.422920:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.422921:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.422924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.422930:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.422933:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.422934:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.422935:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.422937:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.422938:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.422997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.423004:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.423006:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.423007:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.423009:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.423011:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.423013:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.423014:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.423015:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.423016:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.423017:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.423020:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.423022:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.423025:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.423026:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.423026:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.423028:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.423028:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.423029:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.423030:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.423034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.423042:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.423045:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.423045:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.423046:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.423048:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.423051:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.423095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.423098:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.423100:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.423103:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.423104:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.423105:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.423107:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.423108:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.423110:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.423111:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.423112:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.423115:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.423117:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.423120:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.423120:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.423121:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.423123:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.423124:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.423125:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.423125:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.423129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.423135:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.2:1619677448.423137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677448.423138:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.423139:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.423141:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.423142:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.423142:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.423144:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.423145:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.423146:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.423148:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.423151:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.423153:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.423154:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.423155:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.423156:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.423157:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.423160:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.423162:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.423164:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.423164:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.423165:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.423167:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.423168:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.423169:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.423170:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.423170:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.423173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.423176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.423181:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.423182:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.423182:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:17.0:1619677448.423183:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.423183:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.423184:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.423185:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.423185:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.423186:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677448.423186:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.2:1619677448.423198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.423205:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.423208:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.423208:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.423210:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.423213:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.423215:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.423216:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.423217:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.423218:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.423219:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.423223:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.423224:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.423227:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.423228:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.423229:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.423230:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.423230:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.423231:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.423232:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.423236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.423242:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.423244:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.423245:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.423246:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.423248:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.423249:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.423436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7000] (20)++
00000800:00000200:30.0:1619677448.423445:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7000] (21)++
00000800:00000200:30.0:1619677448.423447:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.15@o2ib3
00000400:00000200:30.0:1619677448.423449:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.15@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.423452:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7000] (22)--
00000400:00000200:30.0:1619677448.423452:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.423453:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:30.0:1619677448.423454:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.423456:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.423457:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.423458:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:30.0:1619677448.423461:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.423462:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 2496 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:30.0:1619677448.423465:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f90d380] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.423466:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e49d37c00] (20)++
00000800:00000200:30.0:1619677448.423467:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e49d37c00] (21)++
00000800:00000200:30.0:1619677448.423468:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e49d37c00] (22)--
00000800:00000200:30.0:1619677448.423469:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7000] (21)++
00000800:00000200:30.0:1619677448.423470:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7000] (22)--
00000800:00000200:30.0:1619677448.423470:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7000] (21)--
00000800:00000200:24.2:1619677448.423475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (21)++
00000800:00000200:24.0:1619677448.423479:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)++
00000800:00000200:24.0:1619677448.423482:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e49d37c00] (23)--
00000400:00000200:24.0:1619677448.423483:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.423483:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)--
00000400:00000200:24.0:1619677448.423485:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.423487:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)--
00000800:00000200:4.2:1619677448.423521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.423528:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.423531:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.423532:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.423533:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.423536:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.423537:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.423538:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.423540:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.423541:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.423541:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.423544:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.423546:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.423548:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.423548:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.423549:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.423550:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.423551:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.423552:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.423552:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.423557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.423563:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.423564:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.423565:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.423566:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.423567:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.423568:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.423583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677448.423590:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.423592:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677448.423593:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677448.423594:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.423596:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.423598:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:10.0:1619677448.423598:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.423600:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.423601:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.423601:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:4.2:1619677448.423603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:10.0:1619677448.423605:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.423606:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:10.0:1619677448.423609:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.423609:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:10.0:1619677448.423610:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677448.423610:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677448.423612:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:10.0:1619677448.423612:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.423612:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.423613:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.0:1619677448.423613:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.423614:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.423615:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.423618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:8.0:1619677448.423618:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.423619:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.423620:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96715134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.423622:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.423622:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.423623:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.423626:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:8.0:1619677448.423626:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.423628:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:8.0:1619677448.423628:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.423629:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.423630:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.423631:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.423632:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.423632:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.423632:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.423634:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.423635:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.423635:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677448.423636:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:8.0:1619677448.423636:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.423640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.423648:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.423651:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.423652:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.423652:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.423654:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.423655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.423659:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.423660:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.423662:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.423663:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.423664:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.423664:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.423665:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.423756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.423761:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.423764:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.423764:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.423766:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.423768:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.423770:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.423771:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.423772:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.423773:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.423774:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.423777:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.423778:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.423781:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.423782:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.423782:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.423784:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.423784:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.423785:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.423785:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.423790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.423798:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.423800:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.423801:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.423802:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.423804:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.423806:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.423850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.423856:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.423859:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.423860:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.423862:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.423865:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.423866:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677448.423867:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.423869:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.423870:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.423871:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677448.423874:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.423876:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677448.423879:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.423880:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677448.423880:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677448.423882:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677448.423883:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.423883:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.423884:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.423888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.423894:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.423897:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.423897:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.423898:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.423900:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.423903:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677448.423915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.423921:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.423924:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.423926:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.423926:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.423929:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.423931:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.423932:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.423934:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.423935:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.423936:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.423939:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.423941:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.423944:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.423945:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.423946:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.423947:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.423948:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.423948:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.423949:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.423953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.423953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.423960:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.423960:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677448.423963:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:12.0:1619677448.423964:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.423964:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.423966:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.423967:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:12.0:1619677448.423967:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.423968:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:31.0:1619677448.423969:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.423970:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:12.0:1619677448.423970:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677448.423971:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.423972:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.423973:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.423974:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.423977:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.423979:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.423981:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.423982:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.423983:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.423984:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.423985:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.423986:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.423986:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.423990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.423996:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.423998:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.423999:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.424001:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.424003:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.424003:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.424227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.424229:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.424232:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.424235:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.424235:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.424237:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.424239:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677448.424241:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.424242:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.424243:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.424244:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.424248:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.424249:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.424252:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.424253:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.424254:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.424255:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.424256:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.424256:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.424257:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677448.424261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.424269:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.424272:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.424273:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.424273:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.424275:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.424277:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.424350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.424356:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.424359:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.424360:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.424362:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.424365:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.424367:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.424368:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.424369:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.424370:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.424371:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.424374:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.424376:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.424379:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.424380:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.424380:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.424382:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.424383:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.424383:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.424384:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.424389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.424397:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.424400:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.424401:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.424403:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.424404:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.424405:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.424407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.424415:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.424417:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.424418:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.424419:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.424419:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.424420:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.424457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:4.2:1619677448.424459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.424467:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.424468:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.424469:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.424469:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677448.424472:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.424472:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.424475:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.424475:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.424476:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.424478:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.424479:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.424480:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.424481:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.424484:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.424486:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.424486:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.424487:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.424488:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.424489:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.424490:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.424492:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.424493:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.424494:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:10.0:1619677448.424494:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.424495:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.424495:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.424496:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.424498:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (23)++
00000800:00000200:10.0:1619677448.424498:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.424499:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (24)--
00000800:00000200:10.0:1619677448.424499:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.424500:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.424500:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.424501:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.424501:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:25.2:1619677448.424502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.424502:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.424503:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.424503:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.424504:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.424504:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.424505:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.424505:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.424506:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.424506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:26.0:1619677448.424507:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.424507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.424507:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.424508:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.424508:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.424509:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.424509:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.424510:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677448.424511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677448.424514:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.424516:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.424517:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.424518:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.424518:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:13.0:1619677448.424521:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.424521:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.424521:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.424522:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.424522:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.424527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.424534:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.424537:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.424537:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.424538:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.424539:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.424540:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.424668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.424674:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.424677:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.424678:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.424680:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.424682:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.424683:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.424685:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.424686:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.424687:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.424688:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.424691:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.424693:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.424696:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.424696:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.424697:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.424699:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.424699:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.424700:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.424700:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.424705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.424711:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.424714:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.424715:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.424717:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.424718:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.424720:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.424808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.424816:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.424819:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.424821:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.424823:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.424824:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:30.0:1619677448.424824:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.424826:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.424827:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.424828:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.424829:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677448.424832:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.424833:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.424836:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.424837:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:31.0:1619677448.424838:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677448.424839:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677448.424840:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.424841:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.424843:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.424843:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677448.424845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:21.2:1619677448.424849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.424853:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:21.2:1619677448.424854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:30.0:1619677448.424856:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:17.0:1619677448.424856:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.424856:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.424857:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.424859:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:1.0:1619677448.424859:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.424860:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.424861:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677448.424861:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:1.0:1619677448.424862:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:24.0:1619677448.424864:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.2:1619677448.424865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000400:00000200:24.0:1619677448.424867:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.424870:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.424871:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.424872:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:24.0:1619677448.424873:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.424874:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.424875:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.424876:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.424876:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.424876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677448.424876:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.424876:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:24.0:1619677448.424877:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.424877:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.424878:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:24.0:1619677448.424880:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.424882:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.424883:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:24.0:1619677448.424884:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.424885:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.424885:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.424886:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:24.0:1619677448.424887:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.424887:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:30.0:1619677448.424888:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.424888:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.424889:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:31.0:1619677448.424890:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.424891:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.424892:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:25.2:1619677448.424892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:30.0:1619677448.424893:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.424895:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.424895:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.424896:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.424899:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.424899:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677448.424901:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.424901:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.424902:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.424903:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.424904:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677448.424904:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (23)++
00000400:00000200:13.0:1619677448.424904:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.424905:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677448.424906:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:30.0:1619677448.424907:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:30.0:1619677448.424907:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.424909:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.424910:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.424911:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.424911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.424916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.424918:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.424920:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.424921:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677448.424922:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677448.424922:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.424923:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.424924:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:31.0:1619677448.424925:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:13.0:1619677448.424925:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677448.424925:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.424926:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.424927:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.424928:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:12.0:1619677448.424928:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.424929:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.2:1619677448.424961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919063fd6000] (20)++
00000800:00000200:2.0:1619677448.424967:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919063fd6000] (21)++
00000800:00000200:2.0:1619677448.424971:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.65@o2ib3
00000800:00000200:18.0:1619677448.424972:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919063fd6000] (22)--
00000400:00000200:2.0:1619677448.424974:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.65@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.424977:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.424978:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:2.0:1619677448.424980:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.424981:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.424982:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.424983:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:2.0:1619677448.424986:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.65@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.424988:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:2.0:1619677448.424991:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f90d380] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.424992:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e49d37c00] (20)++
00000800:00000200:2.0:1619677448.424992:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e49d37c00] (21)++
00000800:00000200:2.0:1619677448.424994:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e49d37c00] (22)--
00000800:00000200:2.0:1619677448.424995:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919063fd6000] (21)++
00000800:00000200:2.0:1619677448.424996:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919063fd6000] (22)--
00000800:00000200:2.0:1619677448.424996:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919063fd6000] (21)--
00000800:00000200:24.2:1619677448.425000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (21)++
00000800:00000200:24.0:1619677448.425003:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)++
00000800:00000200:24.0:1619677448.425006:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e49d37c00] (23)--
00000400:00000200:24.0:1619677448.425007:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.425007:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (23)--
00000400:00000200:24.0:1619677448.425009:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.425012:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)--
00000800:00000200:24.2:1619677448.425019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (20)++
00000800:00000200:24.0:1619677448.425021:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)++
00000800:00000200:24.0:1619677448.425024:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677448.425025:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e49d37c00] (22)++
00000800:00000200:8.0:1619677448.425025:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)--
00000800:00000200:24.0:1619677448.425026:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e49d37c00] (22)--
00000800:00000200:24.0:1619677448.425026:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)--
00000800:00000200:20.2:1619677448.425063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.2:1619677448.425066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:20.0:1619677448.425066:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.425069:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.425071:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.425072:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677448.425073:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:20.0:1619677448.425073:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.425075:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:22.0:1619677448.425076:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:20.0:1619677448.425076:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.425077:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677448.425078:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.425079:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.425080:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.425081:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677448.425082:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.425084:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.425085:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.425085:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.425087:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.425087:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:22.0:1619677448.425088:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.425089:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:20.0:1619677448.425090:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.425091:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.425092:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677448.425093:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.425094:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.425094:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677448.425095:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677448.425096:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:22.0:1619677448.425097:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.425097:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.425097:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677448.425098:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.425099:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:25.2:1619677448.425100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.425101:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.425101:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.425101:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.425102:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677448.425102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677448.425107:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:9.2:1619677448.425107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677448.425108:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677448.425110:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.425110:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:13.0:1619677448.425111:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.425111:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:13.0:1619677448.425112:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.425112:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.425113:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.425115:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:13.0:1619677448.425115:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677448.425117:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.425118:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.425118:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.425121:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.425123:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.425137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.425142:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.425145:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.425146:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.425148:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.425150:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.425152:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.425153:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.425155:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.425156:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.425157:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.425160:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.425162:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.425164:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.425165:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.425165:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.425167:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.425167:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.425168:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.425169:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.425170:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677448.425173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677448.425175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.2:1619677448.425175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.425181:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.425182:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:5.0:1619677448.425182:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:18.0:1619677448.425183:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.425183:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.425184:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.425184:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.425185:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.425185:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:2.0:1619677448.425185:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.425186:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:5.0:1619677448.425187:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:2.0:1619677448.425187:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677448.425188:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:4.2:1619677448.425189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:5.0:1619677448.425190:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.425192:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677448.425193:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.425194:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.425195:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.425197:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677448.425197:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:5.2:1619677448.425199:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.425200:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.425201:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.425204:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.425205:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677448.425205:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.425206:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.425207:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:17.0:1619677448.425208:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.425208:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:1.0:1619677448.425208:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:10.0:1619677448.425210:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.425210:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.425211:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:1.0:1619677448.425211:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.425212:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.425212:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.425212:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:10.0:1619677448.425214:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.425214:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.425214:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:1.0:1619677448.425214:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.425216:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000400:00000200:1.0:1619677448.425216:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:5.0:1619677448.425217:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000400:00000200:1.0:1619677448.425217:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96715115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.425218:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:10.0:1619677448.425219:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.425219:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.425220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:1.0:1619677448.425220:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.425222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:10.0:1619677448.425222:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.425222:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:20.0:1619677448.425225:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:10.0:1619677448.425225:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.425225:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:31.0:1619677448.425226:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:10.0:1619677448.425226:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677448.425226:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.425227:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.425228:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:10.0:1619677448.425228:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.425228:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.425228:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677448.425229:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:20.0:1619677448.425229:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:10.0:1619677448.425229:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677448.425230:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.425230:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:31.0:1619677448.425232:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:1.0:1619677448.425232:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.425233:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:1.0:1619677448.425233:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.425233:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:9.2:1619677448.425234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.425235:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.425236:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:1.0:1619677448.425236:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.425237:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.425237:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.425241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.425241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:18.0:1619677448.425242:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.425244:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.425245:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.425246:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.425248:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:18.0:1619677448.425248:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.425248:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.425250:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:26.0:1619677448.425251:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:24.0:1619677448.425251:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:18.0:1619677448.425251:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.425252:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:8.0:1619677448.425252:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.425254:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.425255:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.425257:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.425257:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:26.0:1619677448.425259:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.425260:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.425260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000400:00000200:26.0:1619677448.425261:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.425262:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.425263:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677448.425266:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000400:00000200:26.0:1619677448.425267:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.425269:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677448.425269:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677448.425269:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.425270:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.425271:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.425271:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:26.0:1619677448.425272:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.425272:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.425273:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.425275:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.425275:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.425275:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.425276:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.425280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.425287:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.425290:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.425290:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.425292:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.425294:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.425301:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.425321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.425329:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.425332:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.425332:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677448.425334:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.425337:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.425338:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677448.425340:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.425341:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.425342:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.425343:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.425346:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.425348:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.425351:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.425351:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.425352:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.425354:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.425354:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.425355:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.425356:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.425361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.425369:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.425371:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.425372:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.425374:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.425375:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.425377:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.425423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.425430:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.425433:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.425435:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.425438:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.425438:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.425439:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.425440:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715112] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.425441:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.425442:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.425443:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.425446:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.425448:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.425450:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.425451:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.425451:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.425453:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.425453:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.425454:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.425455:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.425459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.425466:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.425469:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.425469:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.425471:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.425473:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.425474:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.425503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.425509:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.425512:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.425513:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.425515:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.425517:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.425518:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.425519:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.425521:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.425522:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.425523:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.425527:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.425528:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.425532:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.425532:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.425533:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.425535:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.425535:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.425536:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.425537:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.425538:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.425541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.425543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677448.425547:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:18.0:1619677448.425548:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:26.0:1619677448.425550:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:18.0:1619677448.425550:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:10.0:1619677448.425550:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.425551:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.425551:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.425552:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:26.0:1619677448.425553:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.425557:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.425560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.425566:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.425568:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.425569:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.425569:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.425570:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.425570:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.425777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.425782:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.2:1619677448.425783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:6.0:1619677448.425785:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.425786:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.425788:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.425790:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.425791:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:6.0:1619677448.425792:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:26.0:1619677448.425793:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.425794:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677448.425794:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.425796:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.425797:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.425797:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.425798:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:26.0:1619677448.425800:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.425803:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.425805:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:26.0:1619677448.425806:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.425807:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.425808:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.425809:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.425809:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.425809:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.425809:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:26.0:1619677448.425811:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:6.0:1619677448.425811:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.425811:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.425812:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.425813:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:26.0:1619677448.425814:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.425816:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.2:1619677448.425817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.425819:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.425820:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.425821:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.425823:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.425824:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.425824:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.425825:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.425825:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.425826:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:10.0:1619677448.425826:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.425827:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.425828:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.425829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.425831:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:25.2:1619677448.425837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000800:00000200:17.0:1619677448.425837:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.425840:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.425841:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.425841:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.425843:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000400:00000200:17.0:1619677448.425843:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.425845:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.425846:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.425846:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.425847:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (23)--
00000800:00000200:17.0:1619677448.425847:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:26.0:1619677448.425848:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.425861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:21.2:1619677448.425862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677448.425867:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:31.0:1619677448.425870:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.425870:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677448.425870:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677448.425872:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.425873:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.425875:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.425875:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.425877:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:1.0:1619677448.425877:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:31.0:1619677448.425878:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.425878:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.425880:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.425880:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.425881:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.425881:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.425882:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:31.0:1619677448.425883:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.425885:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.425887:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.425887:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.425890:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:31.0:1619677448.425891:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.425893:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.425893:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.425894:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:1.0:1619677448.425894:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:31.0:1619677448.425896:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.425896:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:31.0:1619677448.425897:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.425897:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.425897:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677448.425897:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677448.425898:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:31.0:1619677448.425899:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.425900:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.425901:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.425901:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.425902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:25.2:1619677448.425905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.425908:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.425910:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.425911:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:20.2:1619677448.425911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a8677f3800] (20)++
00000800:00000200:13.0:1619677448.425911:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677448.425911:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.425914:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:10.0:1619677448.425914:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:13.0:1619677448.425915:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.425916:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.425916:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:12.0:1619677448.425918:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.425919:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f3800] (21)++
00000800:00000200:13.0:1619677448.425919:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.425922:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.15.14@o2ib3
00000400:00000200:30.0:1619677448.425924:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.14@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.425925:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f3800] (22)--
00000400:00000200:30.0:1619677448.425926:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.425927:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.425929:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.425930:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.425931:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.425932:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.425935:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.425937:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.425939:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.425940:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.425941:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.425942:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.425943:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a8677f3800] (21)++
00000800:00000200:30.0:1619677448.425943:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a8677f3800] (22)--
00000800:00000200:30.0:1619677448.425944:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f3800] (21)--
00000800:00000200:25.2:1619677448.425948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.425954:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.425956:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:20.2:1619677448.425957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a8677f3800] (20)++
00000400:00000200:12.0:1619677448.425957:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.425958:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.425960:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.425961:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.425965:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f3800] (21)++
00000800:00000200:31.0:1619677448.425968:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.14@o2ib3
00000400:00000200:31.0:1619677448.425970:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.14@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.425972:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.425972:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f3800] (22)--
00000400:00000200:31.0:1619677448.425974:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677448.425975:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677448.425976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:31.0:1619677448.425977:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.425978:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.425978:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677448.425981:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.425983:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.425983:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.425985:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.425985:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.425985:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:31.0:1619677448.425986:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.425986:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.425988:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.425988:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a8677f3800] (21)++
00000400:00000200:26.0:1619677448.425988:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.425989:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a8677f3800] (22)--
00000800:00000200:31.0:1619677448.425990:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f3800] (21)--
00000400:00000200:26.0:1619677448.425991:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.425993:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:25.2:1619677448.425993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:26.0:1619677448.425994:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.425995:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.425996:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.425997:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.426000:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:26.0:1619677448.426000:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.426001:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.426002:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.426003:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.426005:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.426005:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.426005:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677448.426006:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.426006:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.426007:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.426008:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.426008:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.426009:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.426010:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.426014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.2:1619677448.426018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a8677f3800] (20)++
00000800:00000200:31.0:1619677448.426023:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.426025:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:25.2:1619677448.426025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.426026:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.426026:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.426027:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f3800] (21)++
00000800:00000200:31.0:1619677448.426028:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.426028:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.14@o2ib3
00000800:00000200:31.0:1619677448.426029:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.426030:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f3800] (22)--
00000400:00000200:30.0:1619677448.426030:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.14@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.426032:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.426032:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:30.0:1619677448.426033:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.426034:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.426035:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.426035:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:30.0:1619677448.426036:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.426036:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.426036:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:30.0:1619677448.426037:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:22.0:1619677448.426039:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.426041:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.426042:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:30.0:1619677448.426043:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.426045:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.426046:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.426046:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.426048:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.426048:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a8677f3800] (21)++
00000800:00000200:30.0:1619677448.426049:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a8677f3800] (22)--
00000800:00000200:30.0:1619677448.426050:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f3800] (21)--
00000800:00000200:25.2:1619677448.426053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.426060:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.426062:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.426063:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.426064:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.426066:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.426067:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.426072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.426078:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.426081:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.426082:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.426083:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.426083:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.426084:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.426151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:5.2:1619677448.426152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.426157:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677448.426158:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.426159:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.426161:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:17.0:1619677448.426161:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.426162:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:2.0:1619677448.426163:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.426164:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.426166:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.426167:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.426168:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677448.426169:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677448.426170:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.426171:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.426172:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.426172:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.426173:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.426174:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.426175:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677448.426175:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.426180:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.426181:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.426183:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677448.426184:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677448.426186:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.426187:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.426187:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:17.0:1619677448.426188:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.426188:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:2.0:1619677448.426188:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:17.0:1619677448.426190:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:2.0:1619677448.426190:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:17.0:1619677448.426191:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.426191:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.426192:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:17.0:1619677448.426193:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:2.0:1619677448.426193:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677448.426194:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.426195:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.426196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:9.2:1619677448.426197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.426199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.426204:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:8.0:1619677448.426204:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:1.0:1619677448.426205:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:10.0:1619677448.426206:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.426207:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:8.0:1619677448.426207:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:1.0:1619677448.426207:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:24.0:1619677448.426208:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:10.0:1619677448.426208:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.426208:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.426208:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.426209:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:10.0:1619677448.426211:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.426211:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.426214:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:8.0:1619677448.426214:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677448.426254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a8677f3800] (20)++
00000800:00000200:5.2:1619677448.426255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:31.0:1619677448.426261:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f3800] (21)++
00000800:00000200:31.0:1619677448.426263:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.14@o2ib3
00000800:00000200:13.0:1619677448.426263:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677448.426265:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:31.0:1619677448.426266:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.14@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.426267:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f3800] (22)++
00000400:00000200:31.0:1619677448.426268:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.426268:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.14@o2ib3
00000400:00000200:13.0:1619677448.426268:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.426269:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:31.0:1619677448.426270:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677448.426271:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677448.426271:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.426273:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.426273:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.426274:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677448.426274:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.426276:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:13.0:1619677448.426276:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.426278:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.426279:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.14@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.426279:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.426281:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.426281:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.426283:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:31.0:1619677448.426284:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:30.0:1619677448.426284:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96715098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677448.426284:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.426285:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.426286:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.426286:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.426287:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.426290:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.426290:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.426291:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:30.0:1619677448.426291:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.426291:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.426292:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677448.426292:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.426300:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677448.426300:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677448.426300:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.426301:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a8677f3800] (23)++
00000800:00000200:13.0:1619677448.426301:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:8.2:1619677448.426301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (22)++
00000800:00000200:13.0:1619677448.426302:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:31.0:1619677448.426303:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a8677f3800] (24)++
00000800:00000200:30.0:1619677448.426303:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.426303:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:30.0:1619677448.426304:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.426304:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677448.426305:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a8677f3800] (25)--
00000800:00000200:31.0:1619677448.426305:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f3800] (24)--
00000800:00000200:30.0:1619677448.426306:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (23)--
00000800:00000200:31.0:1619677448.426307:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f3800] (23)--
00000800:00000200:30.0:1619677448.426307:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a8677f3800] (22)++
00000800:00000200:9.2:1619677448.426307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (23)++
00000800:00000200:30.0:1619677448.426308:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a8677f3800] (23)--
00000800:00000200:30.0:1619677448.426309:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f3800] (22)--
00000800:00000200:5.0:1619677448.426310:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.2:1619677448.426311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a8677f3800] (21)++
00000800:00000200:5.0:1619677448.426313:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.426314:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.426315:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000400:00000200:5.0:1619677448.426315:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.426318:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000800:00000200:5.0:1619677448.426318:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.426319:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:18.0:1619677448.426319:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.426319:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (24)++
00000800:00000200:31.0:1619677448.426320:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f3800] (22)++
00000800:00000200:2.0:1619677448.426320:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000400:00000200:2.0:1619677448.426320:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.426322:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.426322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:31.0:1619677448.426323:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.14@o2ib3
00000400:00000200:2.0:1619677448.426323:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.426326:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f3800] (23)++
00000800:00000200:18.0:1619677448.426326:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:31.0:1619677448.426327:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.14@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.426327:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a8677f3800] (24)--
00000800:00000200:2.0:1619677448.426327:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.426328:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.426328:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:30.0:1619677448.426329:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f3800] (23)--
00000800:00000200:30.0:1619677448.426329:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f3800] (22)--
00000400:00000200:31.0:1619677448.426330:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.426330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.426331:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:31.0:1619677448.426332:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:18.0:1619677448.426332:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.426333:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677448.426334:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.426335:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.426335:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.426336:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.426336:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.426336:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.426336:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000400:00000200:31.0:1619677448.426337:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.426337:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.426338:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:2.0:1619677448.426338:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.426341:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.426341:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.426342:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.426343:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.426344:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.426345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a8677f3800] (21)++
00000800:00000200:31.0:1619677448.426346:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.426346:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.426349:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.426350:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.426350:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:2.0:1619677448.426350:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.426352:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.426352:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f3800] (22)++
00000800:00000200:2.0:1619677448.426352:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.426353:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a8677f3800] (23)++
00000800:00000200:31.0:1619677448.426354:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a8677f3800] (24)--
00000800:00000200:31.0:1619677448.426354:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f3800] (23)--
00000800:00000200:30.0:1619677448.426354:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.14@o2ib3
00000800:00000200:2.0:1619677448.426355:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.426356:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f3800] (22)--
00000800:00000200:2.0:1619677448.426356:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:30.0:1619677448.426357:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.14@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.426357:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677448.426359:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.426359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.426359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677448.426359:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.426360:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.426360:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:30.0:1619677448.426361:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:2.0:1619677448.426361:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:30.0:1619677448.426362:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96715095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.426364:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.426365:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.426365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677448.426366:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.426367:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677448.426367:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:31.0:1619677448.426368:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.2:1619677448.426368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a8677f3800] (21)++
00000400:00000200:30.0:1619677448.426370:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.426370:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:31.0:1619677448.426371:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.426371:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.426371:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.426372:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677448.426372:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:31.0:1619677448.426373:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.426374:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.426375:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677448.426375:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.426375:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:31.0:1619677448.426376:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96715094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.426376:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:12.0:1619677448.426376:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.426377:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:12.0:1619677448.426377:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.426378:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.426378:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:31.0:1619677448.426379:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.426379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:13.0:1619677448.426379:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:31.0:1619677448.426380:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:12.0:1619677448.426380:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677448.426383:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.426383:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a8677f3800] (22)++
00000800:00000200:30.0:1619677448.426384:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a8677f3800] (23)--
00000800:00000200:31.0:1619677448.426385:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.426385:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f3800] (22)--
00000800:00000200:25.2:1619677448.426385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.2:1619677448.426385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677448.426385:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.426386:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.426387:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.426387:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.426388:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.426388:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.426389:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (23)++
00000800:00000200:30.0:1619677448.426389:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f3800] (21)++
00000800:00000200:6.0:1619677448.426389:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.426390:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.14@o2ib3
00000800:00000200:31.0:1619677448.426391:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677448.426391:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.426392:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.426392:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.14@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.426392:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.426392:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:31.0:1619677448.426393:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.426393:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.426393:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677448.426393:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:31.0:1619677448.426394:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f3800] (22)--
00000400:00000200:30.0:1619677448.426394:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.426394:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.426395:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.426395:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:30.0:1619677448.426396:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:13.0:1619677448.426396:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.426396:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:30.0:1619677448.426397:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96715093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.426397:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:13.0:1619677448.426397:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677448.426397:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.426398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:30.0:1619677448.426399:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.426399:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.426400:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.426401:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:1.0:1619677448.426403:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:30.0:1619677448.426404:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.426405:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.2:1619677448.426405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a8677f3800] (21)++
00000800:00000200:30.0:1619677448.426406:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:22.0:1619677448.426408:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:30.0:1619677448.426409:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.426409:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:22.0:1619677448.426409:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.426409:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.426410:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:22.0:1619677448.426411:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.426412:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f3800] (22)++
00000800:00000200:30.0:1619677448.426412:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.426413:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a8677f3800] (22)++
00000800:00000200:31.0:1619677448.426414:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.14@o2ib3
00000800:00000200:30.0:1619677448.426414:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a8677f3800] (24)++
00000800:00000200:22.0:1619677448.426414:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:31.0:1619677448.426416:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.14@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.426416:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a8677f3800] (25)--
00000800:00000200:30.0:1619677448.426416:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f3800] (24)--
00000400:00000200:31.0:1619677448.426418:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.426418:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f3800] (23)++
00000800:00000200:8.2:1619677448.426418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.426419:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.14@o2ib3
00000400:00000200:31.0:1619677448.426420:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.426421:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.426423:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.426424:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.426424:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:5.0:1619677448.426426:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:30.0:1619677448.426427:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.14@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.426429:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.426430:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.426430:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677448.426431:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.426431:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.426432:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96715091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.426432:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.426433:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (22)--
00000400:00000200:30.0:1619677448.426434:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.426434:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.426434:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.426436:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677448.426437:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.426438:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.426439:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.426440:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.426440:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:30.0:1619677448.426440:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.426440:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:5.0:1619677448.426440:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:10.0:1619677448.426441:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.426442:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:8.2:1619677448.426442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.426443:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a8677f3800] (24)++
00000800:00000200:30.0:1619677448.426443:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.2:1619677448.426443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.426444:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a8677f3800] (25)--
00000400:00000200:26.0:1619677448.426444:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.426445:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f3800] (24)--
00000800:00000200:31.0:1619677448.426446:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f3800] (23)++
00000800:00000200:30.0:1619677448.426446:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.426446:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.426447:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a8677f3800] (24)--
00000800:00000200:30.0:1619677448.426447:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (22)++
00000400:00000200:26.0:1619677448.426447:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.426448:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f3800] (23)--
00000400:00000200:26.0:1619677448.426448:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:9.2:1619677448.426448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (23)++
00000800:00000200:31.0:1619677448.426449:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.426449:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (23)--
00000800:00000200:30.0:1619677448.426449:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a8677f3800] (22)++
00000400:00000200:26.0:1619677448.426449:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96715090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.426450:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:30.0:1619677448.426450:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a8677f3800] (23)--
00000800:00000200:20.0:1619677448.426450:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.426451:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f3800] (22)--
00000400:00000200:26.0:1619677448.426451:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.426452:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f3800] (21)++
00000400:00000200:26.0:1619677448.426452:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.426452:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.426453:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677448.426454:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.426454:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.14@o2ib3
00000800:00000200:20.0:1619677448.426454:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.426455:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.426455:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.426456:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677448.426456:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (23)++
00000400:00000200:31.0:1619677448.426457:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.426457:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.14@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.426458:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.426458:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (24)++
00000400:00000200:31.0:1619677448.426459:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:18.0:1619677448.426459:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (24)--
00000800:00000200:2.0:1619677448.426459:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (24)--
00000400:00000200:31.0:1619677448.426460:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96715089] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.426460:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:18.0:1619677448.426460:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.426460:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.426462:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.426463:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.426463:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.426463:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.426464:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.426464:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.426464:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:21.2:1619677448.426464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:31.0:1619677448.426465:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.426466:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.426467:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.426467:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:31.0:1619677448.426468:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.426468:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:18.0:1619677448.426468:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.426469:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.426470:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.426470:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:30.0:1619677448.426471:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.426471:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.2:1619677448.426472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677448.426473:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.426473:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:9.2:1619677448.426473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.426473:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:31.0:1619677448.426474:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.426474:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:30.0:1619677448.426474:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96715088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.426475:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:31.0:1619677448.426476:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:30.0:1619677448.426476:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.426477:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000400:00000200:2.0:1619677448.426477:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:30.0:1619677448.426478:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.426479:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.426479:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.426480:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (22)++
00000800:00000200:18.0:1619677448.426480:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:17.0:1619677448.426480:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:2.0:1619677448.426480:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.426481:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:25.2:1619677448.426481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677448.426481:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.426481:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.426482:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:2.0:1619677448.426482:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:31.0:1619677448.426483:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:30.0:1619677448.426483:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.426483:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:17.0:1619677448.426483:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.426484:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.426484:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96715087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.426484:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677448.426485:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f3800] (22)++
00000800:00000200:30.0:1619677448.426485:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.426486:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.426487:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.14@o2ib3
00000400:00000200:17.0:1619677448.426487:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.426487:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.426488:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.426488:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677448.426489:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.426489:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:2.0:1619677448.426489:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.426490:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.14@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.426490:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677448.426491:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.426491:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.426491:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.426492:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.426492:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a8677f3800] (23)++
00000800:00000200:30.0:1619677448.426493:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a8677f3800] (24)--
00000400:00000200:2.0:1619677448.426493:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:31.0:1619677448.426494:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:30.0:1619677448.426494:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f3800] (23)--
00000400:00000200:13.0:1619677448.426494:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.426494:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:31.0:1619677448.426496:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96715086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.426496:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f3800] (22)--
00000800:00000200:25.2:1619677448.426496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.426496:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.426497:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.426498:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.426499:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.426499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:31.0:1619677448.426500:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.426500:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.426500:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.426501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:13.0:1619677448.426502:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677448.426502:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.426503:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.426503:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:31.0:1619677448.426504:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.426504:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.426504:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677448.426505:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.426505:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.426506:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:13.0:1619677448.426506:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.426506:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.426507:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677448.426507:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:25.2:1619677448.426508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:13.0:1619677448.426508:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.426508:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:31.0:1619677448.426509:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.426509:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.426510:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.426510:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:12.0:1619677448.426510:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.426511:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677448.426511:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.426511:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:12.0:1619677448.426512:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.426513:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.426513:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a8677f3800] (21)++
00000400:00000200:1.0:1619677448.426514:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.426515:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a8677f3800] (22)++
00000800:00000200:6.0:1619677448.426515:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:31.0:1619677448.426516:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a8677f3800] (23)--
00000800:00000200:31.0:1619677448.426516:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f3800] (22)--
00000400:00000200:1.0:1619677448.426516:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.426518:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:1.0:1619677448.426518:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:22.0:1619677448.426519:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.2:1619677448.426519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:6.0:1619677448.426519:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.426519:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96715085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.426521:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:1.0:1619677448.426521:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.426522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a8677f3800] (21)++
00000400:00000200:1.0:1619677448.426523:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.426524:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:1.0:1619677448.426524:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.426527:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.426529:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.426529:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.426530:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f3800] (22)++
00000400:00000200:17.0:1619677448.426530:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.426531:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:17.0:1619677448.426532:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.426533:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.14@o2ib3
00000800:00000200:1.0:1619677448.426534:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.426535:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.426536:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f3800] (23)++
00000400:00000200:30.0:1619677448.426536:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.14@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.426536:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.426536:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.426537:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a8677f3800] (24)--
00000800:00000200:17.0:1619677448.426537:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677448.426538:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f3800] (23)--
00000800:00000200:31.0:1619677448.426538:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f3800] (22)++
00000800:00000200:1.0:1619677448.426538:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:30.0:1619677448.426539:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.426539:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:31.0:1619677448.426540:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.14@o2ib3
00000400:00000200:30.0:1619677448.426540:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:1.0:1619677448.426540:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:30.0:1619677448.426541:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.426541:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:30.0:1619677448.426543:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.426544:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.426544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:30.0:1619677448.426545:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.426548:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.426550:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.426551:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.14@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.426552:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.426552:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:31.0:1619677448.426553:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.426553:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.426553:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:31.0:1619677448.426554:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:5.0:1619677448.426554:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:30.0:1619677448.426555:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:5.0:1619677448.426555:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.426556:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96715083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.426557:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.426557:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.426557:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.426559:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.426559:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.426559:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:30.0:1619677448.426560:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a8677f3800] (23)++
00000800:00000200:25.2:1619677448.426560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.426562:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a8677f3800] (24)--
00000800:00000200:30.0:1619677448.426562:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f3800] (23)--
00000400:00000200:31.0:1619677448.426563:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.426563:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f3800] (22)++
00000800:00000200:31.0:1619677448.426564:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.426565:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.14@o2ib3
00000400:00000200:30.0:1619677448.426566:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.14@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.426567:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.426568:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:30.0:1619677448.426568:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.426569:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.426570:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:30.0:1619677448.426570:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.426571:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96715082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.426572:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.426573:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.426574:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a8677f3800] (23)++
00000400:00000200:30.0:1619677448.426574:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.426575:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a8677f3800] (24)--
00000800:00000200:31.0:1619677448.426576:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f3800] (23)--
00000800:00000200:8.2:1619677448.426576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.426577:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:30.0:1619677448.426577:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.426578:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.426578:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.426579:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.426580:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.426581:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.426581:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677448.426581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.426582:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f3800] (22)--
00000800:00000200:30.0:1619677448.426582:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677448.426583:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.426583:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (23)++
00000800:00000200:30.0:1619677448.426584:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:30.0:1619677448.426585:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a8677f3800] (21)++
00000800:00000200:20.0:1619677448.426585:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)++
00000800:00000200:30.0:1619677448.426586:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a8677f3800] (22)--
00000800:00000200:30.0:1619677448.426586:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f3800] (21)--
00000800:00000200:20.0:1619677448.426588:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (24)--
00000800:00000200:10.0:1619677448.426588:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.2:1619677448.426589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:20.0:1619677448.426589:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.426591:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677448.426591:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.426591:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:5.0:1619677448.426592:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)++
00000800:00000200:5.0:1619677448.426593:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (24)--
00000800:00000200:20.0:1619677448.426594:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000400:00000200:10.0:1619677448.426594:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.426594:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.426595:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677448.426596:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.426596:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.426597:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:10.0:1619677448.426597:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.426598:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:6.0:1619677448.426598:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.0:1619677448.426599:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:10.0:1619677448.426600:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.426601:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.426602:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.426603:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.426604:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.426604:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:22.0:1619677448.426605:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:8.2:1619677448.426605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000400:00000200:22.0:1619677448.426607:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.426609:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.426610:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.426610:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.426611:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.426612:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.426612:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.426615:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.426616:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.426616:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:10.0:1619677448.426616:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.426617:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.426617:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.426617:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.426617:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677448.426618:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.426619:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.426619:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:10.0:1619677448.426620:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.426620:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677448.426621:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.426621:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.426622:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.426622:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.426624:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.426625:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.2:1619677448.426625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677448.426626:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.426627:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.426628:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.426630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.426633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677448.426634:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.426636:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.426637:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.426638:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.426638:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:5.0:1619677448.426639:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677448.426640:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:5.0:1619677448.426640:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.426641:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:6.0:1619677448.426641:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677448.426642:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:6.0:1619677448.426642:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:5.0:1619677448.426642:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.426643:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.426644:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.426645:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.426680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.426687:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.426690:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.426691:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.426692:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.426694:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.426696:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.426697:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.426699:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.426700:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.426701:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.426704:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.426706:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.426710:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.426710:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.426711:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.426713:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.426713:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.426715:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.426716:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.426717:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.426719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677448.426722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.426728:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:8.0:1619677448.426729:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.426730:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:8.0:1619677448.426731:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:1.0:1619677448.426731:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.426732:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.426732:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.426733:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.426733:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677448.426736:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.426738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.426746:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.426748:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.426749:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.426749:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.426750:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.426751:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.426758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.426765:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.426768:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.2:1619677448.426768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:31.0:1619677448.426771:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.426772:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677448.426773:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.426774:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.426775:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.426776:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.426776:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.426777:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677448.426778:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.426779:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.426780:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.426781:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677448.426781:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.426783:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.426785:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.426786:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.426787:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.426788:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.426789:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:17.0:1619677448.426790:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.426793:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.426793:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:17.0:1619677448.426793:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.426794:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.426795:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.426796:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.426796:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.426797:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.426798:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677448.426799:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.426800:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.426800:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:25.2:1619677448.426802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.426802:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.426803:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.426803:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.426804:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.426808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.426809:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.426812:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.426813:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.426813:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:13.0:1619677448.426815:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:22.0:1619677448.426816:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.426818:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:22.0:1619677448.426819:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:13.0:1619677448.426819:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.426820:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.426821:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.426823:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.426835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.426841:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.426844:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.426845:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.426847:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.426850:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.426851:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.426853:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.426854:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.426855:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.426856:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.426860:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.426861:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.426865:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.426866:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.426867:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.426869:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.426869:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.426870:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.426870:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.426875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.426883:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.426885:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.426886:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.426887:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.426889:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.426892:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.426993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677448.426999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:26.0:1619677448.427000:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.427003:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.427004:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.427006:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.427008:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677448.427009:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.427010:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677448.427011:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:26.0:1619677448.427012:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.427013:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.427013:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.427014:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677448.427015:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.427015:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.427016:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.427018:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.427020:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.427020:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.427021:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.427022:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.427022:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.427023:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.427025:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.427026:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.427026:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.427027:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.427028:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.427028:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.427029:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.427030:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:30.0:1619677448.427031:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.427031:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:30.0:1619677448.427032:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.427032:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.427033:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.427034:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:25.2:1619677448.427034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.427035:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.427036:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.427036:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:4.2:1619677448.427037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677448.427041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.427042:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.427045:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.427045:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.427046:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.427046:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.427047:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.427048:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.427048:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.427048:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.427049:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.427051:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:6.0:1619677448.427051:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:20.0:1619677448.427052:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.427054:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.427054:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.427057:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:25.2:1619677448.427059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.427066:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.427069:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.427070:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.427070:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.427071:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.427072:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.427141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.427146:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.427150:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.427150:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.427152:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.427155:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.427157:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.427158:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.427160:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.427160:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.427162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:2.0:1619677448.427162:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.427165:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.427166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677448.427167:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.427168:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.427169:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.427170:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.427170:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.427171:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.427171:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.427173:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.427173:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.427173:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:1.0:1619677448.427173:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.427174:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.427174:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:31.0:1619677448.427176:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:1.0:1619677448.427176:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.427178:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:8.2:1619677448.427179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677448.427180:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.427180:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:1.0:1619677448.427180:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.427182:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.427183:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.427183:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.427184:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677448.427185:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.427187:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96715071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.427188:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:31.0:1619677448.427189:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.427189:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:31.0:1619677448.427190:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.427190:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.427191:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.427192:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.427192:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.427194:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:31.0:1619677448.427196:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.427196:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.427197:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677448.427198:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.427199:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.427199:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.427200:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.427201:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.427202:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.427202:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677448.427203:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.427203:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.427203:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.427204:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.427204:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.427205:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.427206:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.427208:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:25.2:1619677448.427208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.427209:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.427211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.427214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677448.427215:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.427217:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.427218:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:12.0:1619677448.427218:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677448.427218:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.427220:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.427221:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:30.0:1619677448.427222:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:12.0:1619677448.427222:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.427223:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.427223:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:30.0:1619677448.427225:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:13.0:1619677448.427225:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.427226:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.427226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:12.0:1619677448.427226:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.427227:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.427233:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.427236:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.427237:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.427237:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.427238:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.427239:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.427303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.427311:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.427314:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.427315:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.427317:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.427320:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.427321:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.427323:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.427324:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.427325:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.427326:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.427329:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.427331:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.427334:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.427335:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.427335:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.427337:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.427337:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.427338:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.427340:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.427340:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677448.427341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.2:1619677448.427344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.427346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.427351:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.427352:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:17.0:1619677448.427352:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:26.0:1619677448.427354:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.427355:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:10.0:1619677448.427355:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:17.0:1619677448.427356:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.427356:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:1.0:1619677448.427356:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:26.0:1619677448.427357:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.427357:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.427358:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677448.427359:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.427359:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:26.0:1619677448.427360:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.427361:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.427363:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.427364:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.427365:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.427366:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:26.0:1619677448.427368:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.427370:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.427373:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.427374:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.427375:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.427376:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.427377:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.427378:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.427378:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.427383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.427391:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.2:1619677448.427391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:6.0:1619677448.427394:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.427395:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.427395:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.427398:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000400:00000200:6.0:1619677448.427398:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.427400:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677448.427401:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:26.0:1619677448.427402:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:10.0:1619677448.427404:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.427407:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.427408:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.427409:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.427411:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.427412:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.427412:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.427415:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.427416:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.2:1619677448.427417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:10.0:1619677448.427419:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.427420:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.427420:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677448.427422:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.427422:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.427422:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677448.427423:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677448.427423:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:22.0:1619677448.427424:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.427425:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.427427:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.427427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:22.0:1619677448.427430:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.427431:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.427432:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.427434:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.427434:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.427435:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.427436:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.427438:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:22.0:1619677448.427439:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.427439:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.427439:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.427441:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.427441:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.427444:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.427444:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.427444:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:22.0:1619677448.427445:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.427447:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.427448:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.427448:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.427449:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.427453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.427460:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.427463:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.427464:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.427464:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.427466:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.427468:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.427475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.427482:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677448.427483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677448.427484:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.427485:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.427486:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.427487:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.427487:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677448.427491:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.427494:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.427496:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.427499:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.427499:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.427501:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.427502:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.427503:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.427504:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.427505:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.427508:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.427510:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.427513:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.427514:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.427514:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.427516:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.427517:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.427518:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.427520:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.427520:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677448.427521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677448.427523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.427526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.2:1619677448.427527:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.0:1619677448.427532:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.2:1619677448.427533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:17.0:1619677448.427533:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.427534:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.427534:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.427534:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.427536:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:17.0:1619677448.427536:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.427536:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.427537:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:20.0:1619677448.427537:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.427538:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:5.0:1619677448.427538:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.427539:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.427539:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.427539:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:17.0:1619677448.427540:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.427542:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:5.0:1619677448.427542:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.427542:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:17.0:1619677448.427543:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.427544:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.427545:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.427546:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.427546:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.427547:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.427548:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.427549:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.427549:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.427550:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.427550:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.427551:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.427552:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.427552:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.427553:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.427554:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.427555:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:18.0:1619677448.427555:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.427556:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:18.0:1619677448.427556:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.427556:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.427557:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.427558:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.427559:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.427561:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.427562:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.427562:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.427563:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.427563:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.427565:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.427565:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.427565:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.427566:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.427566:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.427566:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.427567:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677448.427567:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.427569:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.427569:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.427570:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.2:1619677448.427570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.427571:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:5.0:1619677448.427571:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.427572:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.427572:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677448.427573:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677448.427573:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.427574:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.427575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.427575:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:1.0:1619677448.427578:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.427580:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.427580:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.427581:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.427582:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.427583:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:1.0:1619677448.427584:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:13.0:1619677448.427585:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677448.427587:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.427588:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.427589:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677448.427590:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677448.427591:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.427591:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.427592:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:12.0:1619677448.427593:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.427594:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.427594:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.427595:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.427595:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677448.427596:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.427597:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.427680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.427687:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.427689:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.427690:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.427693:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.427695:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.427696:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.427698:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.427699:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.427700:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.427701:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.427705:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.427707:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:4.2:1619677448.427708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:26.0:1619677448.427710:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.427710:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.427711:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.427713:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.427713:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.427714:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.427714:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.0:1619677448.427716:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677448.427718:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:25.2:1619677448.427719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677448.427719:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677448.427721:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.427724:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.427725:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:6.0:1619677448.427726:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:8.0:1619677448.427727:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96715061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.427728:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.427729:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.427729:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.427730:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.2:1619677448.427730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:8.0:1619677448.427730:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.427731:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.427733:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.427735:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.427737:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.427737:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677448.427738:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.427740:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:8.0:1619677448.427740:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.427740:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:26.0:1619677448.427741:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.0:1619677448.427741:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:8.0:1619677448.427742:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.427743:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.427743:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677448.427744:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677448.427744:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000400:00000200:10.0:1619677448.427746:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.427747:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:25.2:1619677448.427748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000400:00000200:10.0:1619677448.427749:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.427750:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.427752:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.427753:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.427754:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.427755:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.427756:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.427757:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.427758:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.427759:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.427760:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.427761:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:10.0:1619677448.427762:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.427763:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.427764:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.427766:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.427766:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.427767:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.427768:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677448.427772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.427780:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.427783:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.427784:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.427784:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.427786:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.427790:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.428067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.428070:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.428073:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.428075:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.428076:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.428078:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.428080:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677448.428081:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.428082:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.428084:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.428085:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677448.428088:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.428090:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677448.428092:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.428093:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677448.428094:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.428096:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677448.428096:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.428097:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.428098:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.2:1619677448.428102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.2:1619677448.428104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.428110:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.428112:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.428114:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:17.0:1619677448.428115:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.428116:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.428117:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.428117:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.428118:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.428119:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.428120:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.428121:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.428122:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.428123:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.428126:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.428128:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.428129:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.428130:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.428130:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.428131:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.428131:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677448.428132:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.428133:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.428134:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.428136:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.428137:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.428137:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.428139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.428142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.428147:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.428148:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.428148:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.428149:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:1.0:1619677448.428149:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.428150:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.428151:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.2:1619677448.428151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:1.0:1619677448.428151:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.428151:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677448.428152:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.428158:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.428161:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.428162:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.428164:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.428167:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.428169:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.428170:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.428171:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.428172:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.428173:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.428176:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.428178:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.428180:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.428181:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.428181:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.428183:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.428183:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.428184:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.428184:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.428188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.428194:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.428196:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.428196:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.428197:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.428198:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.428199:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.428258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.428265:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.428267:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.428268:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.428269:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.428271:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.428273:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.428274:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.428275:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.428276:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.428276:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.428279:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.428280:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.428283:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.428284:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.428284:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.428286:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.428286:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.428287:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.428287:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.428292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.428301:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.428304:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.428305:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.428305:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.428308:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.428310:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.428357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.428365:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.428368:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.428371:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.428373:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.428373:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.428375:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.428376:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.428378:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.428379:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.428380:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.428383:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.428384:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.428387:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.428387:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.428388:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.428390:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.428390:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.428391:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.428392:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.428396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.428404:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.428406:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.428407:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.428408:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.428410:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.428411:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.428496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.428502:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.428504:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.428507:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.428507:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.428509:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.428511:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.428512:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.428513:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.428515:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.428516:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.428519:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.428521:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.428523:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.428524:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.428525:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.428526:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.428527:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.428528:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.428528:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.428532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.428540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:6.0:1619677448.428540:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.428542:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.428543:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.428544:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.2:1619677448.428544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:6.0:1619677448.428545:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.428546:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:24.0:1619677448.428547:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:6.0:1619677448.428547:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677448.428549:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:24.0:1619677448.428551:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.428551:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:20.0:1619677448.428552:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.428552:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.428553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000400:00000200:24.0:1619677448.428554:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.428555:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.428557:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.428557:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677448.428558:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.428559:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:22.0:1619677448.428560:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:20.0:1619677448.428560:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.428561:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.428561:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.428562:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.428563:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.428563:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.428563:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677448.428564:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677448.428564:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.428564:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.428565:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.428566:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:20.0:1619677448.428569:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.428570:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.428572:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.428573:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.428575:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.428577:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.428577:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.428577:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677448.428577:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.428578:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.428579:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.428580:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.428580:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:24.0:1619677448.428581:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677448.428581:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.428581:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:24.0:1619677448.428582:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.428583:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.428584:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677448.428585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:21.2:1619677448.428589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:9.2:1619677448.428589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.428595:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:18.0:1619677448.428597:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.0:1619677448.428597:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.428598:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:8.0:1619677448.428598:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.428599:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.428599:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.428600:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:18.0:1619677448.428600:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:8.0:1619677448.428600:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.428601:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.428602:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:18.0:1619677448.428602:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.428604:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.428604:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:18.0:1619677448.428606:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.428719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.2:1619677448.428720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.428726:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.428727:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.428729:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.428730:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:10.0:1619677448.428730:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.428731:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677448.428732:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.428734:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.428735:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.428737:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.428738:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.428738:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.428739:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:6.0:1619677448.428740:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.428741:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.428741:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.428743:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.428744:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.428745:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.428747:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:6.0:1619677448.428748:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.428751:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.428751:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.428754:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677448.428754:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.428755:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.428755:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.428756:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:6.0:1619677448.428756:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.428757:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.428758:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:6.0:1619677448.428758:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.428758:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.428759:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.428760:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677448.428760:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:10.0:1619677448.428761:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677448.428761:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.428764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:25.2:1619677448.428767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.428770:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.428773:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:26.0:1619677448.428773:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:30.0:1619677448.428774:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.428775:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:30.0:1619677448.428776:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.428776:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.428777:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.428777:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.428778:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.428779:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.428780:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.428788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.428794:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.428797:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.428798:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.428800:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.428802:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.428804:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677448.428805:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.428806:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.428807:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.428809:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677448.428812:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.428814:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.428815:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.428816:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.428817:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.428818:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.428819:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.428820:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.428820:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.428824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.428832:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.428833:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.428834:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.428835:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.428836:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.428838:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.429049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.429056:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.429059:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.429060:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.429062:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.429063:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.429064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:1.0:1619677448.429065:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.429066:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.429067:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.429068:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.429069:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.429071:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:1.0:1619677448.429073:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.429074:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.429075:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677448.429075:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.429076:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677448.429077:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.429077:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.429078:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.429079:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.429079:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:24.0:1619677448.429080:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.429080:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.429081:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677448.429082:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.429083:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96715047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.429085:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677448.429085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:24.0:1619677448.429087:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.429088:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.2:1619677448.429088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:24.0:1619677448.429092:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.429092:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677448.429094:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.429094:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:5.0:1619677448.429094:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:24.0:1619677448.429096:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.429096:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.429096:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.429097:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.429097:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.429097:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:17.0:1619677448.429098:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.429099:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.429099:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.429099:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.429100:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.429100:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677448.429100:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:24.0:1619677448.429101:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:5.0:1619677448.429102:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.429104:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:8.2:1619677448.429105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000400:00000200:5.0:1619677448.429105:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.429107:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.429107:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.429108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677448.429109:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.429112:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.429114:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.429114:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.429116:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677448.429116:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.429116:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.429117:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.429117:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.429118:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.429118:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.429118:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.429119:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.429119:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677448.429119:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.429120:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.429120:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.429120:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.0:1619677448.429121:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677448.429121:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.429122:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.429123:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677448.429124:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:25.2:1619677448.429125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.429128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.429131:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:22.0:1619677448.429132:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677448.429133:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.429133:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.429134:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677448.429135:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.429136:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.429137:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.429138:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.429140:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.429276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.429281:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.429284:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.429285:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.429287:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.429290:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.429291:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.429293:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.429297:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.429299:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.429300:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.429303:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.429304:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.429307:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.429308:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.429309:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.429310:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.429311:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.429312:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.429312:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.429317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.429326:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.429328:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.429329:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.429331:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677448.429332:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:20.0:1619677448.429336:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.429337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.429340:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.429340:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.429341:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.429343:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:20.0:1619677448.429343:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.429344:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000400:00000200:31.0:1619677448.429345:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.429345:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.429345:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.429346:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:31.0:1619677448.429347:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.429347:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.429349:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.429350:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.429351:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.429352:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.429354:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:4.2:1619677448.429354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:31.0:1619677448.429357:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.429358:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.429361:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.429361:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.429361:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.429362:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.429364:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.429364:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.429364:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.429364:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:31.0:1619677448.429365:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.429366:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:10.0:1619677448.429366:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.429368:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.429369:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:25.2:1619677448.429370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:10.0:1619677448.429370:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.429372:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.429373:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.429373:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.429376:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:10.0:1619677448.429376:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.429378:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.429379:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.429380:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.429381:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.429381:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.429382:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.429382:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.429383:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.429384:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:10.0:1619677448.429385:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.429385:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.429386:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.429388:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.429388:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677448.429391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677448.429394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.429399:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.429401:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.429402:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:18.0:1619677448.429402:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.429403:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677448.429403:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.429403:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.429405:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:18.0:1619677448.429405:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.429409:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.429413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.429419:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.429422:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.429423:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.429423:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.429424:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.429425:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.429507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.429513:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.429515:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.429516:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.429518:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.429520:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.429521:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.429523:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.429524:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.429525:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.429526:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.429529:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.429531:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.429534:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.429535:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.429535:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.429537:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.429538:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.429538:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.429539:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.429543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.429551:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.429554:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.429554:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.429555:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.429557:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.429559:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.429600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.429607:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.429610:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.429611:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.429613:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.429617:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.429618:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.429619:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.429620:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.429622:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.429622:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.429626:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.429627:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.429630:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.429631:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.429632:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.429634:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.429634:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.429635:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.429635:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.429640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.429647:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.429650:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.429651:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.429651:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.429653:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.429656:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.429974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.429979:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.429983:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.429983:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.429985:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.429988:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.429989:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.429990:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.429992:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.429993:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.429994:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.429997:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.429999:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.430001:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.430002:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.430002:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.430003:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.430004:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.430005:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.430005:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.430009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.430016:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.430018:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.430019:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.430020:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.430021:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.430022:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.430042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.430048:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.430050:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.430052:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.430053:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.430056:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.430057:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.430058:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.430060:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.430061:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.430062:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.430065:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.430066:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.430068:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.430069:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.430070:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.430071:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.430072:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.430073:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.430074:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.430074:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677448.430077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.430080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.430085:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.430086:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.430086:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:18.0:1619677448.430087:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.430087:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:18.0:1619677448.430088:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.430088:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.430089:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.430090:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.430090:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:4.2:1619677448.430122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.430129:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.430132:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.430132:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677448.430135:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.430138:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.430139:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.430140:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677448.430141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:8.0:1619677448.430142:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.430143:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.430144:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.430147:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:8.0:1619677448.430147:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.430149:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.430150:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.430151:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677448.430152:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.430153:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.430153:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.430153:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.430155:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:26.0:1619677448.430156:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.430156:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.430156:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:26.0:1619677448.430157:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:8.0:1619677448.430157:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:26.0:1619677448.430159:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.430160:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.430161:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.430161:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.2:1619677448.430161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677448.430165:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.430166:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.430169:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.430169:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:26.0:1619677448.430170:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.430171:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.430171:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:26.0:1619677448.430172:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:17.0:1619677448.430172:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.430172:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:26.0:1619677448.430173:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.430174:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.430175:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:17.0:1619677448.430175:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.430178:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:9.2:1619677448.430178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.2:1619677448.430179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.430186:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677448.430187:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.430189:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.430190:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.430190:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.430190:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677448.430190:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.430191:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000400:00000200:18.0:1619677448.430192:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.430192:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.430192:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677448.430195:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.430312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.430317:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.430320:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.430322:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.430323:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.430324:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.430326:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677448.430327:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.430329:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.430330:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.430331:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677448.430334:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.430336:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.430338:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.430339:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.430339:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.430341:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.430342:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.430342:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.430343:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.430347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.430355:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.430358:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.430358:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.430360:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.430361:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.430362:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.430427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.430435:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.430437:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.430438:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.430440:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.430443:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.430444:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.430445:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.430447:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.430448:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.430449:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.430451:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.430453:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.2:1619677448.430453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.430456:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.430458:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.430458:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.430460:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.430461:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.430461:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.430462:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:5.2:1619677448.430462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.430463:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.430463:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:10.0:1619677448.430464:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.430465:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.430466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:26.0:1619677448.430467:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.430469:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.430470:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.430471:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.430471:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.430472:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.430472:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.430473:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.430475:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:26.0:1619677448.430477:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.430477:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.430477:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.430478:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.430479:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.430480:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.430480:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000400:00000200:17.0:1619677448.430481:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.430482:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96715033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.430483:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.430484:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.430484:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:17.0:1619677448.430484:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.430485:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.430486:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.430486:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677448.430486:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:26.0:1619677448.430487:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.430489:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.430489:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:17.0:1619677448.430490:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.430490:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.430491:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.430492:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:25.2:1619677448.430493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.430493:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:5.2:1619677448.430495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.430496:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.430496:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.430497:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.430499:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.430499:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677448.430499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:17.0:1619677448.430500:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.430502:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.430502:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.430502:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:12.0:1619677448.430503:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.430504:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:12.0:1619677448.430505:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:9.2:1619677448.430505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:26.0:1619677448.430506:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677448.430506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:12.0:1619677448.430506:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.430506:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:13.0:1619677448.430507:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:5.2:1619677448.430507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:12.0:1619677448.430508:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.430509:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:24.0:1619677448.430511:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:13.0:1619677448.430511:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.430511:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.430512:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677448.430513:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:22.0:1619677448.430513:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:12.0:1619677448.430513:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:1.0:1619677448.430513:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:13.0:1619677448.430514:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.430514:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.430515:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.430516:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:13.0:1619677448.430516:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:1.0:1619677448.430516:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.430516:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677448.430517:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.430517:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:13.0:1619677448.430518:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.430518:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677448.430520:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:22.0:1619677448.430520:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.430520:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.430521:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.430522:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.430522:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.430524:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.430526:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.430527:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.430528:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.430529:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.430530:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:13.0:1619677448.430530:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.430534:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.430534:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.430534:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:13.0:1619677448.430535:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.430536:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677448.430537:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:13.0:1619677448.430537:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677448.430538:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:22.0:1619677448.430539:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.430539:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:13.0:1619677448.430539:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:22.0:1619677448.430540:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.430542:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.430543:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.430543:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.430543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.430544:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.430548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677448.430551:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.430553:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.430555:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.430556:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:20.0:1619677448.430557:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.430557:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.430559:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677448.430559:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.430560:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.430560:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.430562:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.430564:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.430607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.430610:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.430613:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.430615:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.430615:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.430617:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.430619:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677448.430620:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.430622:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.430623:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.430624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677448.430624:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677448.430627:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.430629:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.430632:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.430632:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.430633:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.430633:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.430634:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677448.430635:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.430636:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:31.0:1619677448.430637:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.430637:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.430638:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:31.0:1619677448.430639:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.430639:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.430640:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:25.2:1619677448.430641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:31.0:1619677448.430642:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.430643:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.430644:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.430645:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.430647:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:31.0:1619677448.430648:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.430650:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.430650:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.430651:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.430651:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.430653:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.430653:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:10.0:1619677448.430653:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:31.0:1619677448.430654:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.430656:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.430656:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.430656:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677448.430657:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.430658:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.430661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:25.2:1619677448.430665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.430667:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.430669:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.430670:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.430670:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.430671:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:12.0:1619677448.430671:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677448.430672:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:12.0:1619677448.430673:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.430674:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.430676:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.430678:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.430679:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.430680:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.430681:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.430681:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.430681:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.430682:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.430682:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.430683:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.430779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.430786:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.430789:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.430790:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.430791:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.430794:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.430795:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.430796:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.430798:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.430799:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.430800:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.430803:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.430804:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.430807:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.430808:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.430808:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.430810:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.430811:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.430812:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.430812:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.430816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.430824:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.430827:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.430828:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.430828:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.430830:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.430833:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.430844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.430851:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.430854:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.430856:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.430858:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.430859:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.430860:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.430861:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.430863:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.430864:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.430865:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.430868:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.430869:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.430872:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.430872:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.430873:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.430874:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.430875:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.430876:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.430876:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.430880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.430886:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.430889:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.430889:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.430891:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.430892:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.430893:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.430967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.430973:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.430976:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.430977:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.430979:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.430981:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.430982:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.430984:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.430985:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.430986:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.430987:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.430991:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.430993:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.430996:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.430997:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.430998:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.431000:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.431000:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.431001:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.431001:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.431005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.431012:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.431014:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.431015:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.431015:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.431017:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.431021:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.431125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.431131:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.431134:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.431135:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.431137:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.431140:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.431141:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.431142:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.431144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:17.0:1619677448.431144:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.431145:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.431146:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.431149:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.431150:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.431151:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.431153:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.431153:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677448.431154:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.431155:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:22.0:1619677448.431156:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.431156:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:22.0:1619677448.431158:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.431158:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.431159:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.431159:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677448.431160:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:17.0:1619677448.431160:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677448.431161:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.431162:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.431163:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.431164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:22.0:1619677448.431165:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.431168:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.431170:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:12.0:1619677448.431170:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:22.0:1619677448.431172:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.431173:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:12.0:1619677448.431173:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:22.0:1619677448.431174:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:12.0:1619677448.431174:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.431175:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:12.0:1619677448.431175:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.431176:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.431177:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:13.0:1619677448.431177:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677448.431178:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:12.0:1619677448.431178:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677448.431179:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.431181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.431183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.431189:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:5.0:1619677448.431190:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677448.431191:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.431192:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.0:1619677448.431192:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:6.0:1619677448.431193:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:5.0:1619677448.431193:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.431195:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.431195:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.431197:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.431203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.431210:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.431213:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.431215:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.431217:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.431217:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.431219:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.431220:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.431221:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.431222:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.431223:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.431227:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.431229:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.431231:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.431232:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.431232:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.431234:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.431234:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.431235:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.431236:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677448.431237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:25.2:1619677448.431240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.431244:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677448.431246:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.431247:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.431248:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:13.0:1619677448.431248:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.431249:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.431249:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.431251:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.431251:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.431252:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.431253:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677448.431253:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.431254:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.431255:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.431256:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.431257:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.431260:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.431262:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.431265:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.431266:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.431266:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.431268:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.431268:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.431269:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.431269:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.431274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.431282:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.431284:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.431285:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.431285:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.431288:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.431290:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.431392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.431400:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.431402:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.431403:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.431405:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.431408:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.431410:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.431411:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.431412:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.431413:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.431414:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.431417:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.431419:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.431420:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.431421:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.431422:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.431423:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.431424:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.431424:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.431425:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.431430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.431437:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.431440:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.431441:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.431442:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.431444:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.431446:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.431470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.431473:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.431475:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.431477:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.431479:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.431480:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.431482:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.431483:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.431484:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.431485:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.431486:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.431490:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.431491:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.431494:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.431495:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.431496:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.431498:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.431498:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.431499:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.431499:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.431503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.431511:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.431513:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.431514:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.431516:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.431518:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.431519:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.431608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.431616:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.431619:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.431621:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.431623:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.431623:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.431624:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.431626:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.431627:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.431628:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.431629:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.431632:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.431634:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.431636:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.431637:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.431638:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.431639:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.431640:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.431641:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.431641:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.431645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:21.2:1619677448.431648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:13.0:1619677448.431651:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.431654:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.431654:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.431655:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:13.0:1619677448.431656:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.431658:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:13.0:1619677448.431658:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.431658:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:30.0:1619677448.431659:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.431661:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.431662:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.431663:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.431664:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.431665:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.431666:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.431667:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.431670:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.431672:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.431675:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.431675:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.431676:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.431678:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.431678:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.431680:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.431681:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.431681:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.431684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.2:1619677448.431684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:21.2:1619677448.431687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.431691:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677448.431692:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677448.431694:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:31.0:1619677448.431695:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:18.0:1619677448.431695:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:6.0:1619677448.431695:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.431696:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.431696:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.431697:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.431698:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:6.0:1619677448.431698:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.431699:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.431700:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:2.0:1619677448.431700:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.431701:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:2.0:1619677448.431702:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.431703:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.431704:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.431705:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.431706:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:4.2:1619677448.431708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677448.431710:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.431712:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.431714:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.431715:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.431715:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.431716:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.431718:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:2.0:1619677448.431718:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.431719:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677448.431719:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.431719:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:10.0:1619677448.431720:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.431720:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:10.0:1619677448.431723:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.431724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.431725:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.431726:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.431727:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.431728:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.431729:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:13.0:1619677448.431730:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.431733:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:10.0:1619677448.431733:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677448.431734:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.431736:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.431736:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.431736:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.431738:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.431739:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.431739:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.431739:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.431741:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.431741:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.431743:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.431744:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.431744:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.431747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.431750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.431753:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.431755:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.431756:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.431757:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677448.431758:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.431759:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.431759:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.431760:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.431761:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.431761:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677448.431822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.431828:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.431831:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.431832:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.431834:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.431837:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.431838:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677448.431839:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.431841:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.431842:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.431843:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677448.431847:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.431848:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677448.431851:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.431852:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677448.431852:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.431854:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677448.431854:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.431856:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.431857:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.431857:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:8.2:1619677448.431861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.431863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.431868:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:5.0:1619677448.431870:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677448.431870:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.431871:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677448.431872:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:2.0:1619677448.431872:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:5.0:1619677448.431873:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.431875:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.431875:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.431878:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.431887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.431894:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.431897:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.431898:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.431898:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.431899:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.431900:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.431955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.431961:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.431964:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.431965:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.431967:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.431970:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.431971:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.431973:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715014] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.431974:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.431975:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.431976:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.431979:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.431981:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.431983:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.431984:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.431985:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.431986:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.431987:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.431987:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.431988:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.431992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.431999:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.432001:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.432002:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.432002:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.432004:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.432005:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.432107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.432115:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.432118:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.432120:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.432121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:31.0:1619677448.432122:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.432123:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.432125:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.432126:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.432126:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.2:1619677448.432126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000400:00000200:30.0:1619677448.432127:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.432129:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.432129:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.432129:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:4.2:1619677448.432129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:30.0:1619677448.432130:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:22.0:1619677448.432133:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.432134:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677448.432135:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:30.0:1619677448.432136:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.432136:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.432136:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677448.432137:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:30.0:1619677448.432138:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.432139:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.432139:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.432139:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.432140:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677448.432141:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.432141:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.432141:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.432142:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:22.0:1619677448.432142:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.432142:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.432143:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:22.0:1619677448.432143:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.432144:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:30.0:1619677448.432144:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:31.0:1619677448.432145:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96715011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.432145:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.432147:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.432147:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.432148:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96715011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.432149:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.432149:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.432150:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:25.2:1619677448.432150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:10.0:1619677448.432150:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.432151:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677448.432151:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.432152:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677448.432152:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.432153:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:10.0:1619677448.432153:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677448.432154:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.432155:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:22.0:1619677448.432155:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.432156:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.432156:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.432157:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:22.0:1619677448.432157:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.432158:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:31.0:1619677448.432159:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.432159:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.432159:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:10.0:1619677448.432159:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.432160:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.432161:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:26.0:1619677448.432161:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.432161:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.432161:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.432162:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.432162:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677448.432163:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.432163:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677448.432164:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:26.0:1619677448.432164:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.432165:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:26.0:1619677448.432165:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.432165:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677448.432165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.432166:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.432166:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.432168:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.432169:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:9.2:1619677448.432169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:10.0:1619677448.432170:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.432170:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.2:1619677448.432171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:25.2:1619677448.432174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.0:1619677448.432177:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677448.432177:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:30.0:1619677448.432179:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.432179:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.432180:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.432180:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.432181:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.432181:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.432182:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:20.0:1619677448.432182:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.432182:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:18.0:1619677448.432183:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.432183:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.432184:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:30.0:1619677448.432185:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.432185:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677448.432186:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:22.0:1619677448.432186:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.432186:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.432188:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:6.0:1619677448.432188:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.432190:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.432191:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:6.0:1619677448.432191:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.432192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000400:00000200:30.0:1619677448.432193:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.432194:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.432195:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.432196:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.432199:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.432200:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.432201:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.432202:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.432203:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677448.432204:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.432204:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.432204:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677448.432205:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.432205:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:30.0:1619677448.432206:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.432207:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.432208:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.432210:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.432211:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.432212:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.432214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.432217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.432222:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677448.432225:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:5.0:1619677448.432225:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.432226:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.432228:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:20.0:1619677448.432228:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.432228:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.432229:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.432230:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.432231:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.432243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.432250:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.432253:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.432254:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.432254:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.432255:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.432256:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.432449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.432455:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.432458:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.432459:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.432461:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.432462:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.432464:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.432465:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.432466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:1.0:1619677448.432466:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.432467:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.432468:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.2:1619677448.432470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:1.0:1619677448.432472:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.432473:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.432473:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.432474:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677448.432475:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677448.432476:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.432477:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:1.0:1619677448.432477:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:20.0:1619677448.432478:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.432478:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.0:1619677448.432478:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677448.432478:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.432479:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:26.0:1619677448.432480:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.432480:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.432480:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677448.432481:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.432481:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677448.432483:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.432483:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.432484:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96715007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.432485:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:25.2:1619677448.432486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:20.0:1619677448.432486:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.432487:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96715007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.432487:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.432488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:26.0:1619677448.432489:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.432489:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.432490:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.432491:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:2.0:1619677448.432493:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:20.0:1619677448.432494:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.432494:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:26.0:1619677448.432496:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.432497:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.432497:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677448.432497:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:26.0:1619677448.432498:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677448.432498:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.432498:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:6.0:1619677448.432498:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.432501:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.432501:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.432502:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.432502:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:2.0:1619677448.432502:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.432503:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677448.432503:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.432503:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:26.0:1619677448.432504:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:2.0:1619677448.432505:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.432506:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677448.432506:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:2.0:1619677448.432506:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:26.0:1619677448.432507:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677448.432507:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.432508:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677448.432508:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:2.0:1619677448.432508:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96715005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.432509:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.0:1619677448.432509:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:2.0:1619677448.432509:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.432510:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.432511:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:25.2:1619677448.432512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.2:1619677448.432513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:2.0:1619677448.432514:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.432516:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.432518:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.432519:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.432519:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.432520:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677448.432520:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:2.0:1619677448.432522:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.432522:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677448.432523:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:17.0:1619677448.432523:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:2.0:1619677448.432523:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:31.0:1619677448.432524:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.432524:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:1.0:1619677448.432524:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.432525:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.432525:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:31.0:1619677448.432527:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.432528:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:17.0:1619677448.432528:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.432528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.432530:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.432531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677448.432532:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677448.432536:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.432538:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.432538:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.432538:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:18.0:1619677448.432539:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.432539:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.432540:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.432541:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.432541:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677448.432542:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.432551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.432558:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.432561:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.432562:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.432562:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.432562:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.432563:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.432590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.432598:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.432601:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.432603:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.432604:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.432606:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.432607:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.432608:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.432610:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.432611:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.432612:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.432615:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.432617:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.432619:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.432620:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.432621:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.432623:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.432623:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.432624:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.432625:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.432629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.432637:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.432640:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.432641:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.432642:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.432643:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.432645:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.432652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.432655:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.432657:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.432660:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.432660:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.432662:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.432664:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677448.432665:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.432667:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.432668:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.432669:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677448.432672:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.432674:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677448.432677:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.432678:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677448.432679:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677448.432680:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677448.432681:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.432682:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.432682:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677448.432686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.432693:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.432696:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.432697:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.432697:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.432699:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.432702:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.432706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677448.432711:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.432714:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.432715:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.432715:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.432716:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.432717:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.432771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.432777:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.432780:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.432780:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677448.432782:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.432785:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.432787:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.432788:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96715002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.432789:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.432790:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.432791:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.432795:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.432797:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.432800:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.432801:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.432802:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.432804:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.432804:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.432804:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.432805:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.432809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.432818:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.432820:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:4.2:1619677448.432820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677448.432821:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.432822:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.432824:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.432826:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.432827:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.432830:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.432830:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.432832:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.432835:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.432836:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.432838:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96715001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.432839:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.432840:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.432841:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.432844:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.432845:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.432848:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.432849:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.432850:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.432851:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.432852:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.432852:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.432853:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.432857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.432865:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.432868:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.432869:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.432869:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.432872:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.432874:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.432908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.432914:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.432917:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.432917:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.432919:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.432921:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.432922:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.432924:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96715000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.432925:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.432926:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.432927:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.432930:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.432932:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.432934:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.432934:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.432935:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.432936:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.432937:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.432938:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.432939:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.432940:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.432942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.432945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.432951:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.432952:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.432952:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:17.0:1619677448.432953:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.432953:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.432954:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.432955:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.432955:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.432956:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677448.432956:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:21.2:1619677448.433155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.433160:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.433163:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.433164:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.433166:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.433168:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.433169:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.433171:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.433172:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.433173:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.433174:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.433178:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.433180:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.433183:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.433184:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.433184:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.433186:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.433187:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.433188:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.433188:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.433192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.433199:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.433202:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.433203:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.433205:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.433206:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.433208:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.433213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.433219:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.433222:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.433223:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.433224:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.433224:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.433226:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.433255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.433262:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.433265:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.433266:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.433268:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.433270:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.433272:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.433273:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714998] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.433274:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.433275:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.433276:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.433280:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.433281:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.2:1619677448.433282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:26.0:1619677448.433284:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677448.433284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:26.0:1619677448.433285:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.433286:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.433287:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677448.433287:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:26.0:1619677448.433288:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.433288:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.433289:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.433289:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.433290:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.433291:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677448.433292:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:25.2:1619677448.433293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.0:1619677448.433298:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:2.0:1619677448.433298:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.433300:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.433300:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.433302:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:2.0:1619677448.433302:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.433303:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.433303:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.433305:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:5.0:1619677448.433305:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.433305:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.433306:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.433306:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:2.0:1619677448.433306:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.433307:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.433307:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.433309:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.433309:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.433310:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.433311:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.433311:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.433314:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.433315:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.433317:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677448.433317:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.433318:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.433318:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.433318:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.433320:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.433320:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.433320:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677448.433321:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.433321:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677448.433322:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.433322:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677448.433324:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.433325:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.433326:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:8.2:1619677448.433327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.433327:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.433328:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.433330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.433333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.433335:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677448.433336:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677448.433337:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677448.433338:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:6.0:1619677448.433338:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677448.433339:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677448.433339:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.433339:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.433340:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677448.433340:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.433341:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.433341:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.433342:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.433343:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.433344:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677448.433346:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.433350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.433357:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.433360:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.433361:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.433361:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.433362:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.433362:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.433376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.433384:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.433387:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.433388:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.433390:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.433393:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.433394:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.433396:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.433397:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.433398:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.433399:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.433403:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.433404:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.433408:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.433408:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.433409:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.433411:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.433411:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.433412:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.433412:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677448.433415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:8.2:1619677448.433417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:18.0:1619677448.433421:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.433423:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.433424:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:17.0:1619677448.433425:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:18.0:1619677448.433426:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.433428:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.433428:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:18.0:1619677448.433429:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.433429:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.433430:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.433431:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.433432:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.433433:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.433434:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.433434:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:18.0:1619677448.433435:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.2:1619677448.433435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000400:00000200:18.0:1619677448.433438:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.433440:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.433443:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.433443:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:18.0:1619677448.433444:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.433444:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.433446:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.433446:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.433446:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:18.0:1619677448.433447:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:1.0:1619677448.433447:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:18.0:1619677448.433448:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:1.0:1619677448.433448:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:18.0:1619677448.433449:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:1.0:1619677448.433449:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:25.2:1619677448.433452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.433460:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.433463:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.433464:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.433466:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.433467:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.433468:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.433470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.433476:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.433479:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.433480:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.433481:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.433482:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.433483:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.433512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.433521:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.433523:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.433525:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.433528:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.433528:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.433530:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.433531:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.433532:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.433533:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.433534:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.433537:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.433539:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.433542:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.433543:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.433543:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.433545:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.433546:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.433547:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.433547:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.433562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.433570:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.433572:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.433573:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.433575:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.433576:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.433578:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.433752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.433758:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677448.433759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677448.433761:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.433762:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.433764:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.433765:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.433766:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677448.433766:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.433768:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.433769:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:17.0:1619677448.433770:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.433771:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.433772:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:31.0:1619677448.433773:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.433773:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.433776:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.433776:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.433777:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:30.0:1619677448.433778:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.433779:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.433781:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.433781:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.433782:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.433782:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.433782:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:30.0:1619677448.433783:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.433784:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.433785:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.433785:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.433786:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677448.433787:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.433789:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:9.2:1619677448.433790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:30.0:1619677448.433791:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.433792:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.433793:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.433794:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.433795:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.433796:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.433796:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677448.433797:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:25.2:1619677448.433799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677448.433800:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.433801:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.433801:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677448.433803:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.433806:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:13.0:1619677448.433806:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.433808:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.433809:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.433811:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.433812:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.433813:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.433858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.433866:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.433868:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.433869:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677448.433870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:26.0:1619677448.433871:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.433875:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.433877:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:10.0:1619677448.433878:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:26.0:1619677448.433879:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.433879:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677448.433880:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.433881:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.433882:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.433885:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.433886:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.433888:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.433888:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.433889:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.433890:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.433892:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.433892:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.433893:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.433896:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.433897:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.433898:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.433898:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.433898:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.433900:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.433900:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.433901:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.433901:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.433902:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.433902:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.433904:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.433904:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.433904:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.433905:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.433905:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (23)++
00000800:00000200:26.0:1619677448.433906:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (24)--
00000800:00000200:25.2:1619677448.433906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.433907:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (24)--
00000800:00000200:10.0:1619677448.433907:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.433909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.2:1619677448.433910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:4.2:1619677448.433913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.433916:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.433918:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.433918:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:26.0:1619677448.433919:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:6.0:1619677448.433919:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.433920:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:22.0:1619677448.433920:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.433921:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.433922:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.433922:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:6.0:1619677448.433922:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.433923:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677448.433923:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.433923:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.433924:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.433924:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677448.433925:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.433926:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.433926:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.433927:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.433945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.433951:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.433954:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.433955:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.433957:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.433960:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.433961:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.433963:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.433964:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.433965:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.433966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:2.0:1619677448.433966:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.433970:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.433971:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.433971:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.433974:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677448.433974:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.433975:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.433976:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.433977:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.433977:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.433978:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.433978:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:5.0:1619677448.433979:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.433979:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:5.0:1619677448.433980:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:2.0:1619677448.433980:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:5.0:1619677448.433982:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.433983:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.433984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677448.433984:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.433985:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677448.433988:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.433989:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677448.433990:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.433992:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.433993:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.433993:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.433993:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:12.0:1619677448.433994:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.433995:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:12.0:1619677448.433996:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.433996:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677448.433997:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.433997:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.433997:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.433999:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677448.434002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:21.2:1619677448.434009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:1.0:1619677448.434009:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.434012:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.434013:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.434013:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.434015:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:1.0:1619677448.434015:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.434018:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.434018:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:1.0:1619677448.434018:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:6.0:1619677448.434020:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.434023:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.434024:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.434026:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.434027:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.434028:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.434029:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.434033:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.434034:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.434037:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.434038:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.434039:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.434041:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.434041:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.434042:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.434042:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.434047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.434054:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.434057:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.434058:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.434060:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.434061:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.434062:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.434196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.434202:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.434204:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.434205:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.434207:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.434210:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.434211:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.434212:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.434214:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.434215:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.434216:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.434218:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.434220:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.434222:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.434222:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.434223:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.434224:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.434225:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.434225:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.434226:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.434230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677448.434237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:1.0:1619677448.434237:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.434239:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.434239:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.434240:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.434241:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.434242:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.0:1619677448.434244:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.434247:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.434248:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.434250:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.434253:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.434254:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.434256:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.434257:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.434258:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.434259:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.434262:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.434264:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.434266:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.434267:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.434268:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.434269:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.434270:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.434270:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.434271:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.434275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.434282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:17.0:1619677448.434283:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.434286:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.434287:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.434287:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.434288:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:17.0:1619677448.434289:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.434291:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.434292:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677448.434293:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.434294:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000400:00000200:18.0:1619677448.434301:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.434303:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:18.0:1619677448.434304:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.434305:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:1.0:1619677448.434306:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.434307:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.434307:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.434307:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000400:00000200:18.0:1619677448.434308:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.434308:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.434308:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:18.0:1619677448.434309:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.434310:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.434313:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.434315:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.434317:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.434318:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.434318:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.434320:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.434321:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.434321:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.434322:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.434327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.434334:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.434336:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.434336:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.434338:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.434338:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.434340:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.434478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677448.434481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677448.434487:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.434489:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.434490:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.434492:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.434492:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.434493:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.434494:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.434495:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.434495:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.434496:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.434498:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.434498:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.434499:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.434499:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.434501:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.434501:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.434502:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.434502:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.434503:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.434505:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.434506:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.434509:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.434509:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.434511:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.434511:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.434512:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.434513:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.434513:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.434514:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.434514:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.434514:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.434515:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.434516:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.434516:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:10.0:1619677448.434516:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.434517:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.434517:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.434518:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.434520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:25.2:1619677448.434524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.434526:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.434529:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.434529:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.434531:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.434531:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677448.434533:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:13.0:1619677448.434533:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:31.0:1619677448.434534:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:13.0:1619677448.434534:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.434536:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.434537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:25.2:1619677448.434538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:12.0:1619677448.434538:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.434539:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.434539:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.434540:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.434540:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.434541:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.434542:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677448.434543:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677448.434545:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:18.0:1619677448.434546:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.434547:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:30.0:1619677448.434548:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.434549:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000400:00000200:18.0:1619677448.434549:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:30.0:1619677448.434550:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.434550:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:18.0:1619677448.434551:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.434552:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:18.0:1619677448.434552:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677448.434554:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.434555:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.434556:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.434557:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677448.434561:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.434562:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.434565:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.434566:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677448.434566:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677448.434568:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.434568:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.434569:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.434570:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.434574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.434581:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.434584:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.434585:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.434585:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.434587:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.434590:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.434593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:6.0:1619677448.434600:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.434603:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.434604:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.434604:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.434605:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.434605:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.434611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677448.434618:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.434621:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677448.434622:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677448.434624:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.434624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677448.434627:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.434628:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.434629:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.434630:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.434630:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.434632:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.434632:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:26.0:1619677448.434633:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.434633:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.434636:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.434637:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.434638:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.434639:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.434640:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.434641:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714978] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.434642:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.434643:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:17.0:1619677448.434643:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.434644:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:17.0:1619677448.434644:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.434645:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:17.0:1619677448.434645:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.434646:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.434647:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677448.434648:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000400:00000200:17.0:1619677448.434648:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.434649:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:17.0:1619677448.434649:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.2:1619677448.434651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.434651:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.434652:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.434653:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.434654:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:21.2:1619677448.434655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:17.0:1619677448.434655:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.434655:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.434656:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.434658:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.434660:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:8.2:1619677448.434660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677448.434661:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.434662:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:31.0:1619677448.434663:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.434664:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.434664:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:31.0:1619677448.434665:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.434665:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:10.0:1619677448.434665:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:1.0:1619677448.434667:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.434670:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.434670:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.434671:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.434673:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.434674:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.434755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.434763:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.434765:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.434766:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.434768:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.434771:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.434772:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.434773:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.434775:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.434776:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.434777:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.434780:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.434782:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.434784:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.434785:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.434785:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.434787:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.434787:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.434788:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677448.434790:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.434790:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677448.434793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677448.434796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:17.0:1619677448.434801:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:26.0:1619677448.434803:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:17.0:1619677448.434804:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:26.0:1619677448.434805:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:17.0:1619677448.434805:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.434805:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.434806:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.434807:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:17.0:1619677448.434807:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:17.0:1619677448.434810:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.434812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.434820:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.434822:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.434823:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.434823:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.434824:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.434825:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.434919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.434926:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.434929:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.434930:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.434932:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.434935:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.434937:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.434938:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.434940:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.434940:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.434942:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.434945:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.434946:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.434949:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.434950:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.434950:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.434952:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.434953:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.434953:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.434954:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.434958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.434966:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.434968:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.434970:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.434970:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.434972:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.434975:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.435014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.435020:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.435023:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.435023:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.435025:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.435027:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.435029:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.435030:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.435031:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.435032:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.435033:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.435037:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.435039:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.435042:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.435042:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.435043:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.435045:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.435046:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.435047:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:5.2:1619677448.435047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:17.0:1619677448.435048:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.435049:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.435051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:18.0:1619677448.435053:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677448.435053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.435056:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.435057:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.435058:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.435059:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:1.0:1619677448.435059:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:18.0:1619677448.435061:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.435061:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:30.0:1619677448.435062:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:18.0:1619677448.435062:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:1.0:1619677448.435062:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677448.435063:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.435063:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.435063:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.435065:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.435065:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.435065:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.435065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000400:00000200:18.0:1619677448.435066:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.435067:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.435069:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:18.0:1619677448.435071:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.435072:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677448.435072:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:18.0:1619677448.435074:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.435075:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677448.435075:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:18.0:1619677448.435076:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.435077:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:13.0:1619677448.435077:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.435078:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.435079:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:12.0:1619677448.435079:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:18.0:1619677448.435080:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000400:00000200:13.0:1619677448.435080:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.435081:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:13.0:1619677448.435082:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:13.0:1619677448.435083:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.435083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:13.0:1619677448.435085:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.435086:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.435086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000400:00000200:13.0:1619677448.435087:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:13.0:1619677448.435090:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.435092:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.435092:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.435092:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.435093:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.435093:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:2.0:1619677448.435093:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.435094:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.435095:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.435096:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677448.435096:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.435096:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.435097:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:13.0:1619677448.435098:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.435098:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677448.435099:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:13.0:1619677448.435100:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:9.2:1619677448.435101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677448.435102:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677448.435103:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677448.435104:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:8.2:1619677448.435105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.435108:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.435109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:18.0:1619677448.435111:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.435112:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.435112:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.435113:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677448.435113:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.435114:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677448.435116:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.435117:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.435117:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000400:00000200:20.0:1619677448.435118:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.435119:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:5.0:1619677448.435119:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:12.0:1619677448.435120:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677448.435121:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:20.0:1619677448.435122:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.435191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.435199:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.435201:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.435202:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.435204:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.435207:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.435208:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.435210:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.435211:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.435212:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.435213:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.435217:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.435218:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.435221:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.435222:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.435223:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.435225:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.435225:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.435226:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.435227:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.435231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.435238:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.435241:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.435242:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.435243:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.435245:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.435247:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.435462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.435468:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.2:1619677448.435469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677448.435471:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.435471:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.435474:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.435475:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:17.0:1619677448.435476:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.435478:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:5.0:1619677448.435478:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:17.0:1619677448.435479:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.435480:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:17.0:1619677448.435481:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.435481:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.435482:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.435483:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:5.0:1619677448.435483:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.435485:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677448.435486:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.435488:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.435488:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.435489:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.435490:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:5.0:1619677448.435490:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.435493:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.435494:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:5.0:1619677448.435494:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.435495:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677448.435496:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.435497:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.435497:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.435498:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.435498:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677448.435499:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.435499:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.435500:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.435502:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:25.2:1619677448.435503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677448.435503:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.435503:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.435504:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.435509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:12.0:1619677448.435511:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.435514:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.435515:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.435516:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:12.0:1619677448.435516:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.435518:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:13.0:1619677448.435518:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:20.0:1619677448.435519:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.435519:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:20.0:1619677448.435521:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.435521:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.435523:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.435539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.435546:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.435549:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.435550:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.435552:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.435555:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.435556:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.435557:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.435558:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.435560:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.435561:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.435564:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.435565:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.435567:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.435568:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.435569:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.435570:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.435570:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.435571:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.435572:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.435575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.435581:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.435583:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.435584:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.435585:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.435586:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.435587:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.435597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.435605:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.435608:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.435610:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.435612:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.435613:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:30.0:1619677448.435613:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.435614:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.435615:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.435616:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.435617:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677448.435620:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.435622:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.435625:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.435626:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.435626:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.435628:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.435628:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.435629:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.435630:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.435651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.435659:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.435662:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.435663:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.435664:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.435666:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.435668:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.435675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.435683:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.435685:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.435686:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.435686:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.435687:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.435688:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.435709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.435716:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.435718:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.435719:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.435721:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.435724:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.435725:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.435726:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.435728:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.435729:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.435730:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.435733:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.435735:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.435738:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.435738:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.435739:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.435740:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.435741:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.435742:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.435743:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.435744:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.435746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.435750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.435754:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.435756:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.435757:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.435758:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.435758:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.435759:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.435759:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.435760:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.435760:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.435764:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.436073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.436081:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.436083:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.436085:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.436087:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.436088:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.436090:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.436091:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.436092:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.436093:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.436094:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.436098:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.436100:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.436102:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.436103:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:4.2:1619677448.436103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.436104:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.436106:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.436106:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.436108:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.436109:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.436110:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677448.436111:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677448.436112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.436113:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.436114:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.436115:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.436115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677448.436117:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.436118:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.436119:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.436120:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.436120:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:26.0:1619677448.436121:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.436122:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.436122:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:31.0:1619677448.436123:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:5.0:1619677448.436123:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.436125:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.436125:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.436126:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:5.0:1619677448.436126:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.436127:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677448.436127:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.436128:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.436129:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.436129:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:26.0:1619677448.436130:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.436130:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.436131:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.436132:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.436132:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.436133:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.436137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.436143:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.436145:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.436145:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.436146:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.436147:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.436148:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.436155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.436160:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.436163:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.436164:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.436164:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.2:1619677448.436164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.436165:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.436165:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:17.0:1619677448.436169:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.436172:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.436173:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.436175:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.436178:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.436179:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.436181:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.436182:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.436183:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.436184:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.436187:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.436189:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.436192:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.436193:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.436193:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.436195:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.436196:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.436197:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.436197:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.436201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.436208:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.436210:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.436211:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.436212:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.436214:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.436217:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.436222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.436228:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.436230:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.436231:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.436233:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.436236:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.436238:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.436239:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.436240:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.436241:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.436242:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.436246:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.436248:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.436251:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.436252:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.436252:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.436254:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.436255:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.436256:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.436257:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.436258:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:25.2:1619677448.436260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:21.2:1619677448.436263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.436266:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:6.0:1619677448.436268:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:26.0:1619677448.436269:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.436269:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.436270:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.436270:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.436271:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677448.436272:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.436272:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.436275:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677448.436287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.436290:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.436292:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:5.0:1619677448.436297:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.436298:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.436301:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.436302:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.436304:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.436305:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.436306:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.436307:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.436310:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.436312:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.436315:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.436316:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.2:1619677448.436317:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.436320:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.436322:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.436322:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.436323:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.436323:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.436324:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677448.436324:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.436325:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.436326:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:25.2:1619677448.436329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677448.436329:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.436330:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.436332:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.436333:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.436334:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.436335:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.436336:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:5.0:1619677448.436338:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:13.0:1619677448.436339:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.436339:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.436340:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:13.0:1619677448.436341:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.436343:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.436343:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.436344:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.436345:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.436346:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.436348:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.436349:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.436351:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.2:1619677448.436352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:5.0:1619677448.436352:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:25.2:1619677448.436353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.436353:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.2:1619677448.436358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:12.0:1619677448.436360:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677448.436360:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677448.436361:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:24.0:1619677448.436363:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677448.436363:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:12.0:1619677448.436363:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677448.436363:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:20.0:1619677448.436364:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:12.0:1619677448.436364:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.436365:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:12.0:1619677448.436365:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000400:00000200:8.0:1619677448.436365:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.436366:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.436368:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677448.436368:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.436369:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.436371:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.436372:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.436373:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.436374:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:25.2:1619677448.436377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:8.0:1619677448.436377:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.436378:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.436381:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.436382:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.436383:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:13.0:1619677448.436384:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677448.436385:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.436385:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:13.0:1619677448.436386:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677448.436386:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.436386:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677448.436387:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.436387:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.436388:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.436389:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677448.436390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.436398:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.436401:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.436402:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.436403:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.436404:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.436406:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.436441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.436449:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.436452:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.436454:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.436456:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.436457:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.436458:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.436460:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.436461:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.436462:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.436463:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.436466:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.436467:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.436470:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.436470:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.436471:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.436472:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.436473:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.436474:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.436475:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.436476:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.436478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.436481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.436484:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.436487:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.436488:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.436489:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:13.0:1619677448.436489:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.436491:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:13.0:1619677448.436491:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.436491:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.436492:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.436493:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677448.436515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.436523:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.436526:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.436527:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.436528:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.436530:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.436532:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.436533:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.436535:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.436535:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.436536:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.436539:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.436541:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.436543:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.436543:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.436544:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.436546:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.436546:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.436547:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.436547:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.436551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.436558:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.436560:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.436561:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.436562:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.436564:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:4.2:1619677448.436564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.436565:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:10.0:1619677448.436571:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.436573:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.436574:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.436576:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.436578:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.436580:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.436581:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.436582:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.436583:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.436584:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.436587:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.436588:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.436591:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.436592:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.436592:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.436594:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.436595:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.436595:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.436596:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.436600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.436608:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.436611:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.436612:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.436614:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.436614:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.436617:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.436619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.2:1619677448.436620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:20.0:1619677448.436627:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.436627:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:20.0:1619677448.436630:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.436630:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677448.436630:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:20.0:1619677448.436631:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.436632:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.436633:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000400:00000200:1.0:1619677448.436633:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.436634:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:1.0:1619677448.436636:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.436637:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677448.436639:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.436640:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.436641:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.436642:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677448.436646:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.436647:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677448.436650:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.436651:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:1.0:1619677448.436652:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.436653:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677448.436654:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677448.436655:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677448.436656:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677448.436657:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677448.436659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.436662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677448.436668:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:5.0:1619677448.436668:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677448.436670:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:5.0:1619677448.436670:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:17.0:1619677448.436671:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:5.0:1619677448.436671:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.436671:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.0:1619677448.436673:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.436673:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.436675:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.436749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.436755:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.436758:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.436758:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.436760:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.436763:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.436764:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.436766:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.436767:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.436768:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.436769:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.436772:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.436773:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.436776:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.436776:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.436777:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.436779:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.436779:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.436779:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.436780:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.436785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.436793:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.436795:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.436796:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.436797:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.436799:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.436802:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.436880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.436883:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.436885:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.2:1619677448.436886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677448.436887:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.436890:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.436890:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.436892:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.436897:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677448.436897:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677448.436899:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:20.0:1619677448.436899:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.436900:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:20.0:1619677448.436901:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.436902:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.436903:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.436903:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.436904:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.436905:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.436906:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.436907:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.436908:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.436909:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:30.0:1619677448.436910:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.436910:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.436912:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.436913:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.436914:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:20.0:1619677448.436914:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.436915:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.436915:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.436917:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:20.0:1619677448.436917:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:31.0:1619677448.436918:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96714952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.436918:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.436919:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.436919:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.436919:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:31.0:1619677448.436920:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.436920:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000400:00000200:31.0:1619677448.436921:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.436921:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677448.436921:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.436923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.436924:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.436924:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.436925:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.2:1619677448.436925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.436926:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.436926:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677448.436927:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677448.436928:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.436929:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.436930:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.436930:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677448.436930:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677448.436931:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.436931:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.436931:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677448.436931:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677448.436933:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.436933:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:12.0:1619677448.436933:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.2:1619677448.436933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.436933:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:31.0:1619677448.436934:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:12.0:1619677448.436934:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.436934:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677448.436935:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.436935:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:12.0:1619677448.436936:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.436936:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.436937:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.436938:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677448.436941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.436941:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.436944:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.436945:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.436945:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.436947:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.436948:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.436950:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.436951:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.436951:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:5.0:1619677448.436953:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.436954:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.436956:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.436964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.436971:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.436974:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.436975:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677448.436976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677448.436977:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.436980:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.436981:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.436982:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.436983:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:24.0:1619677448.436983:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.436985:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:24.0:1619677448.436985:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.436986:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.436986:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.436988:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.436989:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.436991:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.436991:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:26.0:1619677448.436992:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.436994:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.436994:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.436995:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.436995:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.436995:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677448.436996:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.436997:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:24.0:1619677448.436997:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.436997:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.436998:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.436998:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:26.0:1619677448.437000:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.437001:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.2:1619677448.437003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.437004:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.437005:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.437005:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.437007:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.437007:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.437009:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:5.2:1619677448.437009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:26.0:1619677448.437010:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.437010:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.437010:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:25.2:1619677448.437013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.437013:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.437014:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.437014:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.437015:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:17.0:1619677448.437016:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.437016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.437018:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.437018:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677448.437019:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:18.0:1619677448.437020:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.437020:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:18.0:1619677448.437022:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.437022:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:21.2:1619677448.437023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.437023:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:18.0:1619677448.437024:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:6.0:1619677448.437024:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.437025:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:18.0:1619677448.437025:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714949] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.437025:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.437026:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677448.437027:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:18.0:1619677448.437027:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.437027:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.437027:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.437028:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.437029:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:18.0:1619677448.437029:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.437031:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:22.0:1619677448.437032:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.437032:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:18.0:1619677448.437034:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.437035:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.437036:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:22.0:1619677448.437037:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.437038:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.437039:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.437040:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.437040:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:22.0:1619677448.437041:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.437041:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677448.437042:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.437043:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:22.0:1619677448.437045:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.437046:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:22.0:1619677448.437047:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.437047:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:25.2:1619677448.437048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.437048:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:22.0:1619677448.437049:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.437050:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.437051:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.437053:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.437053:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.437054:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.437054:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677448.437055:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.437057:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.437058:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.437059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:13.0:1619677448.437060:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.437061:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.437063:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.437066:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.437069:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.437070:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.437070:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.437072:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.437075:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.437079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.437087:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.437090:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.437090:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.437093:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.437095:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.437097:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.437098:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.437099:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.437100:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.437101:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.437105:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.437107:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.437109:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.437110:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.437110:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.437112:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.437113:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.437114:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.437115:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.437115:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.437118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.2:1619677448.437121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.437124:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.437126:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.437127:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.437128:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:30.0:1619677448.437129:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.437130:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.437130:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:30.0:1619677448.437131:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:10.0:1619677448.437131:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.437132:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677448.437233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:5.2:1619677448.437233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677448.437239:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677448.437239:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.437242:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:18.0:1619677448.437243:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:17.0:1619677448.437243:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.437243:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:1.0:1619677448.437246:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.437247:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677448.437248:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.437250:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.437250:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.437251:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677448.437252:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.437253:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.437254:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.437255:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.437255:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.437257:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.437257:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.437258:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677448.437261:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.437263:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677448.437263:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.437265:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.437265:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.437266:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.437267:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.437268:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.437269:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.437269:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677448.437269:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.437269:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.437271:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677448.437271:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:18.0:1619677448.437272:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677448.437272:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.437272:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.437273:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.437274:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677448.437274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.437277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:25.2:1619677448.437278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.437283:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.437284:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.437284:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:17.0:1619677448.437285:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.437285:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:1.0:1619677448.437285:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.437286:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.437287:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.437287:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.437288:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:1.0:1619677448.437288:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.437289:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:17.0:1619677448.437289:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:6.0:1619677448.437289:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.437292:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677448.437299:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.437367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:4.2:1619677448.437367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.437375:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.437376:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.437378:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:26.0:1619677448.437378:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.437378:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677448.437381:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.437381:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.437383:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.437384:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.437384:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.437385:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.437386:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.437387:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.437388:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.437389:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.437390:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.437390:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.437391:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.437391:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.437392:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.437397:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.437397:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.437400:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.437400:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.437402:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.437403:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.437403:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.437404:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.437404:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.437404:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.437405:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.437406:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.437406:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.437407:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.437407:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.437407:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.437408:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.437408:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.437412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:25.2:1619677448.437416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:12.0:1619677448.437419:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.437422:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.437422:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.437423:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:12.0:1619677448.437424:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.437426:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:12.0:1619677448.437426:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.437427:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677448.437427:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:22.0:1619677448.437428:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.437429:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677448.437430:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.437430:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (23)++
00000800:00000200:6.0:1619677448.437431:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (24)--
00000800:00000200:6.0:1619677448.437432:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.437433:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677448.437433:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:21.2:1619677448.437452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.437460:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.437463:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.437465:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.437467:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.437467:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.437468:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.437469:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.437471:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.437472:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.437473:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.437476:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.437477:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.437480:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.437481:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.437482:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.437484:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.437484:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.437485:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.437486:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.437489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.437497:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.437499:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.437500:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.437501:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.437503:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.437506:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.437554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.437557:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.437560:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.437562:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.437564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:5.0:1619677448.437564:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.437565:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.437566:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.437567:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.437569:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.437570:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.437571:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:2.0:1619677448.437571:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.437574:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:20.0:1619677448.437575:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.437575:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.437577:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.437578:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:2.0:1619677448.437580:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.437581:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.437582:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.437582:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:2.0:1619677448.437582:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.437583:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.437584:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:2.0:1619677448.437584:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.437585:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.437585:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:2.0:1619677448.437585:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.437586:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:2.0:1619677448.437586:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.437589:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.437590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:2.0:1619677448.437591:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.437593:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.437594:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.437594:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.437596:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.437597:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:2.0:1619677448.437597:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.437597:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.437598:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:31.0:1619677448.437600:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.437601:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.437602:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.437602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.437603:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.437605:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.437609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:18.0:1619677448.437609:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.437612:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.437612:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.437613:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.437614:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.437616:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677448.437617:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.437619:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.437620:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.437621:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.437622:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.437623:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.437671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.437677:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.437680:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.437680:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.437682:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.437684:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.437685:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.437687:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.437688:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.437689:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.437690:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.437693:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.437695:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.437697:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.437698:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.437699:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.437700:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.437701:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.437702:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.437702:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.437706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.437714:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.437717:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.437718:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.437718:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.437720:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.437722:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.437767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.437774:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.437777:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.437778:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.437780:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.437783:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.437784:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.437786:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.437787:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.437788:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.437789:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.437792:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.437794:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.437796:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.437797:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.437798:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.437799:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.437800:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.437800:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.437801:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.437805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.437812:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.437815:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.437815:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.437817:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.437819:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.437820:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.437851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.437859:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.437861:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.437863:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.437866:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.437866:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.437867:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.437868:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.437869:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.437870:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.437871:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.437874:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.437876:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.437879:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.437879:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.437880:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.437882:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.437882:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.437883:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.437884:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.437888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.437895:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.437898:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.437899:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.437899:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.437902:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.437904:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.437979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.437984:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.437986:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.437987:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.437989:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.437991:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.437992:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.437993:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.437994:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.437995:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.437996:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.437999:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.438001:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.438003:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.438004:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.438005:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.438006:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.438007:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.438007:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.438008:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.438012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.438020:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.438022:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.438023:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.438024:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.438026:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.438028:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.438174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.438179:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.438182:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.438183:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.438185:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.438187:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.438189:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.438190:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.438191:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.438193:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.438194:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.438197:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.438199:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.438202:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.438202:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.438203:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.438205:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.438205:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.438206:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.438207:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.438211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.438218:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.438220:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.438221:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.438223:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.438224:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.438226:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.438230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.2:1619677448.438234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:13.0:1619677448.438236:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.438239:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.438240:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.438240:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.438241:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.438241:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.438242:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.438244:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.438245:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.438247:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.438250:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.438251:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.438252:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.438253:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.438254:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.438255:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.438259:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.438260:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.438262:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.438263:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.438264:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.438265:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.438265:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.438266:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.438266:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.438270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.438276:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.438278:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.438279:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.438280:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.438281:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.438282:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.438364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.438371:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.438374:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.438375:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.438376:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.438378:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.438379:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.438380:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.438382:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.438382:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.438383:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.438387:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.438388:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.438391:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.438392:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.438392:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.438394:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.438395:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.438395:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.438395:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.438401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.438409:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.438412:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.438413:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.438413:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.438415:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.438418:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.438497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.2:1619677448.438497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.438504:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:30.0:1619677448.438506:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.438507:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.438508:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:30.0:1619677448.438509:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677448.438510:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.438511:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.438513:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677448.438513:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.438514:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.438514:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.438515:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.438516:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.438517:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.438517:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.438518:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.438519:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.438519:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.438520:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.438521:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.438525:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.438525:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.438528:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.438528:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.438530:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.438531:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.438531:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.438532:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.438532:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.438533:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.438533:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.438534:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.438534:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.438535:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:10.0:1619677448.438535:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.438536:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.438536:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:30.0:1619677448.438537:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.438537:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.438539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.2:1619677448.438542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.438543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.438546:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.438548:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.438549:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.438551:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:13.0:1619677448.438551:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.438551:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:13.0:1619677448.438553:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.438554:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:12.0:1619677448.438554:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.438554:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:31.0:1619677448.438555:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:5.0:1619677448.438555:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.438556:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:5.0:1619677448.438556:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.438557:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.438560:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.438565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.438572:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.438575:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.438576:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.438577:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.438579:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.438580:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.438582:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.438583:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.438584:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.438585:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.438589:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.438591:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.438593:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.438593:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.438594:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.438596:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.438596:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.438597:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.438598:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.438598:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677448.438602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.438604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.438610:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.438611:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.438611:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.438612:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.438612:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.438612:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.438614:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.438614:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.438615:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677448.438615:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.438787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.438792:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.438795:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.438796:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.438798:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.438800:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.438801:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.438802:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.438804:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.438805:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.438806:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.438810:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.438811:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.438814:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.438815:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.438815:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.438817:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.438817:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.438818:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.438818:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677448.438819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.438823:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:8.2:1619677448.438823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.438826:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.438828:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.438829:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:1.0:1619677448.438830:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:20.0:1619677448.438831:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.438832:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:1.0:1619677448.438833:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:20.0:1619677448.438834:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.438834:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.438834:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.438835:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.438836:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.438837:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.438838:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.438839:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:20.0:1619677448.438842:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.438844:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.438847:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.438847:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:8.2:1619677448.438847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.438848:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.438850:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.438850:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.438851:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.438852:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677448.438854:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:25.2:1619677448.438856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.438857:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.438857:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.438858:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.438859:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.438859:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677448.438862:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.438865:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.438866:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.438868:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:30.0:1619677448.438869:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.438871:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.438896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.438901:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.438903:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.438906:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.438906:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.438908:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.438910:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:5.2:1619677448.438911:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:5.0:1619677448.438915:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.438916:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.438917:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:5.0:1619677448.438917:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.438918:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:18.0:1619677448.438921:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.438921:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:5.0:1619677448.438922:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.438923:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.438925:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:18.0:1619677448.438926:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.438927:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:5.0:1619677448.438928:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.438929:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.438929:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.438929:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:18.0:1619677448.438930:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.438931:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.438931:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:18.0:1619677448.438932:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.438932:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.438933:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.438934:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677448.438936:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.438937:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.2:1619677448.438937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:18.0:1619677448.438940:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.438941:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.438941:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.438943:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.438944:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.438944:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.438945:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677448.438945:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.438947:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.438948:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.438949:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.2:1619677448.438950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:1.0:1619677448.438950:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.438954:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677448.438955:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.438957:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.438957:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.438958:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.438959:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.438961:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.438993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.438998:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.439000:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.439001:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.439003:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.439003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:17.0:1619677448.439005:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.439006:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.439007:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.439008:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.439009:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.439010:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677448.439010:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.439013:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:17.0:1619677448.439014:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.439014:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.2:1619677448.439016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677448.439016:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.439017:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.439018:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.439019:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.439019:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:26.0:1619677448.439020:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.439021:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:17.0:1619677448.439021:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.439021:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.439022:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.439022:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.439022:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.439023:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677448.439024:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.439024:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.439025:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:30.0:1619677448.439026:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:8.2:1619677448.439026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:30.0:1619677448.439029:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.439030:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677448.439030:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.439032:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.439032:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:30.0:1619677448.439033:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:1.0:1619677448.439033:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:30.0:1619677448.439034:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96714923] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.439035:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.439035:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:30.0:1619677448.439036:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.439036:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (23)++
00000400:00000200:30.0:1619677448.439037:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.439037:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (23)++
00000400:00000200:1.0:1619677448.439037:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.439038:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.439038:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (24)--
00000800:00000200:26.0:1619677448.439039:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.439039:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.439040:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.439040:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:30.0:1619677448.439041:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.439041:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:1.0:1619677448.439042:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677448.439043:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.2:1619677448.439045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677448.439046:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.439047:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.439047:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.439049:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.439050:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.439051:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.439051:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:17.0:1619677448.439052:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.439055:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:8.2:1619677448.439055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:17.0:1619677448.439056:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.439056:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.439058:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.439060:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.0:1619677448.439063:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.439065:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:8.2:1619677448.439065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000400:00000200:20.0:1619677448.439066:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.439068:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.439069:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.439071:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677448.439072:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.439075:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.439075:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.439076:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.439077:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.439077:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.439082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.439089:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.439092:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.439093:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.439093:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.439094:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.439095:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.439105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.439112:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.439115:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.439116:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.439118:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.439121:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.439122:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.439124:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.439125:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.439126:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.439127:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.439131:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.439132:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.439135:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.439136:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.439136:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.439139:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.439139:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.439140:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.439142:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.439142:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677448.439145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677448.439148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.439152:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.439155:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:2.0:1619677448.439155:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.439156:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:18.0:1619677448.439156:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.439156:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.439157:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.439158:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:2.0:1619677448.439159:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.439162:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.439167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.439173:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.439176:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.439177:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.439177:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.439178:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.439179:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.439294:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.439300:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.439303:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.439304:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.439305:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.439308:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.439309:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:21.2:1619677448.439310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:2.0:1619677448.439310:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.439312:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.439313:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.439314:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677448.439318:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:2.0:1619677448.439318:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.439319:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677448.439321:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.439322:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:2.0:1619677448.439322:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.439323:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.439323:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:8.0:1619677448.439324:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.439325:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.439325:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.439326:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:8.0:1619677448.439327:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.439327:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:8.0:1619677448.439328:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.439329:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96714920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.439331:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.439332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:8.0:1619677448.439332:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.439333:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.439336:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.439338:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677448.439340:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677448.439341:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.439342:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.439342:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.439342:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.439344:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677448.439344:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:6.0:1619677448.439344:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.439345:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.439346:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000400:00000200:6.0:1619677448.439346:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:8.0:1619677448.439347:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.439348:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:6.0:1619677448.439348:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677448.439351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.439353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:18.0:1619677448.439359:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677448.439360:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:18.0:1619677448.439362:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:24.0:1619677448.439363:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.439363:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:18.0:1619677448.439363:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.439363:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.439364:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:18.0:1619677448.439365:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.439368:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.439530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.439536:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.439539:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.439540:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.439542:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.439543:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.439545:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.439546:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.439547:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.439548:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.439549:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.439552:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.439554:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.439556:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.439557:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.439558:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.439560:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.439560:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.439562:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.439563:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.439563:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.439566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.2:1619677448.439568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.439572:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:1.0:1619677448.439573:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677448.439574:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.439575:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.439575:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:8.0:1619677448.439576:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.439576:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.439577:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.439578:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.439581:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.2:1619677448.439642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.439650:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.439652:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.439653:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.439655:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.439658:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.439660:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.439661:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.439662:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.439663:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.439664:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.439668:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.439670:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.439673:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.439673:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.439674:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:21.2:1619677448.439676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:10.0:1619677448.439676:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.439676:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.439677:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.439677:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.2:1619677448.439678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:22.0:1619677448.439682:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.2:1619677448.439682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677448.439684:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:2.0:1619677448.439684:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677448.439685:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677448.439687:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:22.0:1619677448.439688:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.439688:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:22.0:1619677448.439690:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.439690:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:2.0:1619677448.439690:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.439692:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.439692:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.439693:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.439693:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.439693:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.439694:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.439694:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.439695:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.439695:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.439696:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.439697:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.439697:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.439698:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.439698:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.439699:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.439702:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.439704:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.439704:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:22.0:1619677448.439707:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.439708:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:2.0:1619677448.439708:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.439709:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.439710:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.439710:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.439711:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.439712:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677448.439712:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.439713:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.439713:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677448.439713:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.439714:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.2:1619677448.439716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677448.439716:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:2.0:1619677448.439716:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.439717:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:21.2:1619677448.439718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677448.439718:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:6.0:1619677448.439724:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:1.0:1619677448.439725:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000800:00000200:6.0:1619677448.439726:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.439727:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.439727:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677448.439727:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000800:00000200:17.0:1619677448.439728:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000800:00000200:17.0:1619677448.439728:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000400:00000200:1.0:1619677448.439728:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.439729:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.439730:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.439731:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.439732:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:21.2:1619677448.439733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677448.439733:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.439734:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.439735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.439741:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.439743:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677448.439744:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:31.0:1619677448.439745:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.439746:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.439748:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.439748:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.439750:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.439751:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.439752:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.439753:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.439755:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.439756:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.439758:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.439758:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.439759:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.439760:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.439761:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.439762:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.439762:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.439765:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.439766:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:30.0:1619677448.439766:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.439767:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.439768:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.439768:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.439769:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.439770:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.439770:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.439771:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.439771:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.439772:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677448.439772:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.439773:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.439773:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.439774:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:25.2:1619677448.439774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.439776:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.439777:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.439778:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.439779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:13.0:1619677448.439781:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.2:1619677448.439783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677448.439783:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.439784:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.439785:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.439788:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.439789:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.439789:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677448.439791:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:5.0:1619677448.439791:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.439792:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.439794:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.439794:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677448.439794:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.439794:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.439796:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.439797:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.439824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.439831:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.439834:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.439835:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.439837:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.439839:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.439841:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.439842:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.439844:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.439845:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.439846:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.439849:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.439850:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.439853:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.439854:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.439855:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.439856:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.439857:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.439857:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.439858:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.439862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.439870:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.2:1619677448.439871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:22.0:1619677448.439872:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.439873:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.439874:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.439874:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:22.0:1619677448.439876:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.439876:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:25.2:1619677448.439879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.439879:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:20.0:1619677448.439879:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.439880:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.439881:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.439883:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677448.439884:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.439885:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.439886:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.439887:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:6.0:1619677448.439887:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.439890:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.439890:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.439891:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.439891:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.439892:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.439893:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677448.439893:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677448.439896:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.439896:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677448.439897:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.439899:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677448.439899:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.439900:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.439901:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.439905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.439913:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.439916:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.439916:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.439917:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.439919:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.439921:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.440087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.440092:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.440095:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.440095:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.440097:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.440100:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.440101:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.440103:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.440104:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.440105:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.440106:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.440109:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.440111:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.440113:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.440114:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.440115:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.440116:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.440117:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.440117:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.440118:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.440123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.440130:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.440133:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.440134:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.440134:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.440136:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.440139:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.440140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:6.0:1619677448.440147:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.440150:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.440151:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.440151:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.440152:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.440152:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.440339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.440347:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.440350:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.440351:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.440353:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.440356:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.440357:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.440359:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.440360:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.440361:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.440362:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:4.2:1619677448.440363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:10.0:1619677448.440366:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.440368:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.440371:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.440371:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.440372:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.440372:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.440373:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:26.0:1619677448.440374:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.440375:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:26.0:1619677448.440376:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.440376:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:26.0:1619677448.440377:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.440378:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.440380:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.440380:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.440381:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:25.2:1619677448.440382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.440382:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.440383:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.440384:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.440385:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.440385:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.440387:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:13.0:1619677448.440389:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:26.0:1619677448.440390:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.440390:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.2:1619677448.440390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.440391:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:13.0:1619677448.440391:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:26.0:1619677448.440392:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:13.0:1619677448.440392:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.440393:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:13.0:1619677448.440393:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.440394:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.440394:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:13.0:1619677448.440395:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.440396:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:13.0:1619677448.440396:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.440397:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.440398:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.440398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.440398:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.440405:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.440407:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.440408:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.440410:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.440412:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.440413:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.440415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.440422:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.440425:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.440427:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.440429:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.440429:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.440431:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.440432:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714908] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.440433:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.440434:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.440435:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.440437:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.440439:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.440440:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.440441:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.440442:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.440443:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.440444:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.440445:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.440445:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.440449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.440455:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.440458:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.440459:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.440460:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.440462:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.440463:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.440472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.440479:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.440482:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.440483:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.440484:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.440487:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.440488:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.440489:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.440491:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.440492:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.440492:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.440495:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.440497:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.440500:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.440500:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.440501:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.440503:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.440504:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.440505:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.440505:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.440509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.440517:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.440520:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.440521:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.440522:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.440524:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.440527:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.440685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.440691:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.440694:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.440694:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.440697:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.440699:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.440700:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.440701:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.440703:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.440704:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.440705:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.440708:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.440710:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.440712:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.440713:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.440714:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.440715:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.440716:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.440717:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.440717:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.440722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.440730:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.2:1619677448.440731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:22.0:1619677448.440732:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.440734:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.440734:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.440736:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.440736:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:22.0:1619677448.440738:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677448.440739:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.440740:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.440741:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.440744:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.440746:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.440747:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.440749:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.440749:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.440750:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.440754:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.440756:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.440758:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.440759:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.440760:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.440762:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.440762:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.440763:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.440764:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.440765:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:25.2:1619677448.440768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.440770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677448.440774:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.440775:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:20.2:1619677448.440776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:18.0:1619677448.440777:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:12.0:1619677448.440777:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677448.440778:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:12.0:1619677448.440778:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.440779:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:12.0:1619677448.440779:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.440781:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.440782:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.440782:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.440785:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.440787:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677448.440788:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.440790:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.440792:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:5.0:1619677448.440793:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.440794:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.440795:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.440796:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:5.0:1619677448.440800:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.440802:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:5.0:1619677448.440805:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.440806:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677448.440807:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.0:1619677448.440808:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677448.440809:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.440810:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.440810:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677448.440815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.440821:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.440824:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.440825:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.440825:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.440827:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677448.440830:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.441027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.441035:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.441037:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.441038:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.441040:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.441042:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.441043:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.441045:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.441046:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.441047:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.441048:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.441052:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.441053:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.441055:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.441056:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.441056:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.441058:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.441058:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.441059:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.441059:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.441064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.441064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677448.441071:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:24.0:1619677448.441071:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677448.441073:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:24.0:1619677448.441073:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:31.0:1619677448.441074:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.441074:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:31.0:1619677448.441076:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.441076:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.441077:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.441078:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:24.0:1619677448.441079:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.441081:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:25.2:1619677448.441082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:24.0:1619677448.441082:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.441083:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.441084:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.441085:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.441088:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:24.0:1619677448.441088:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.441090:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.441091:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.441091:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.441092:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.441092:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:24.0:1619677448.441092:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.441093:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:24.0:1619677448.441093:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.441094:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.441096:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.441096:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.441097:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.441097:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.441102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.441110:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.441112:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.441113:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.441114:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.441116:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.441118:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.441121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.441128:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.441131:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.441132:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.441132:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.441133:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.441133:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.441159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.441162:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.441165:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.441167:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.441168:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.441169:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.441171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677448.441171:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.441172:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.441174:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.441175:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.441176:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.441178:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677448.441179:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.441181:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677448.441181:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.441183:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.441185:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.441185:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.441185:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.441185:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.441186:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677448.441187:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.441188:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.441188:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.441188:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677448.441190:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.441190:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.441190:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.441191:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000400:00000200:30.0:1619677448.441192:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.441192:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.441194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677448.441195:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.441197:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.2:1619677448.441197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.441199:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.441200:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.441200:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.441201:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:5.0:1619677448.441202:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:30.0:1619677448.441203:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (25)--
00000800:00000200:30.0:1619677448.441203:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.441203:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (25)--
00000800:00000200:30.0:1619677448.441204:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:13.0:1619677448.441204:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.441204:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:30.0:1619677448.441205:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:13.0:1619677448.441205:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.441205:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677448.441207:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:5.0:1619677448.441207:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.441208:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.441208:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677448.441209:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.441209:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.441210:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.441211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.441214:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.441215:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.441216:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.441217:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.441218:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.441218:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.441219:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.441374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.441382:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.441384:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.441385:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.441387:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.441390:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.441391:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.441392:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.441393:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.441394:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.441395:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.441398:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.441400:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.441403:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.441403:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.441404:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.441406:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.441407:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.441407:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.441408:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677448.441409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:9.2:1619677448.441413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.441416:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.441419:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.441420:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677448.441421:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:6.0:1619677448.441422:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.441424:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.441424:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.441425:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:6.0:1619677448.441425:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677448.441425:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.441427:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.441427:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.441428:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.441430:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.441431:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:2.0:1619677448.441431:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:6.0:1619677448.441434:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.441435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:6.0:1619677448.441436:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.441439:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.441440:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.441441:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:1.0:1619677448.441441:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.441442:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:20.2:1619677448.441443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:6.0:1619677448.441443:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.441444:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.441444:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677448.441444:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:6.0:1619677448.441445:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:1.0:1619677448.441447:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.441448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000400:00000200:1.0:1619677448.441449:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.441450:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:31.0:1619677448.441452:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:1.0:1619677448.441452:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.441453:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.441454:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:1.0:1619677448.441454:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.441455:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:1.0:1619677448.441455:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.441457:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:31.0:1619677448.441458:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.441458:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.441458:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.441459:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.441460:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.441460:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:10.0:1619677448.441461:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.441462:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:1.0:1619677448.441462:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:31.0:1619677448.441464:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.441464:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:1.0:1619677448.441464:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.441465:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.441465:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:31.0:1619677448.441466:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.441466:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:31.0:1619677448.441467:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.441468:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.441468:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.441469:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.441469:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.441470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000400:00000200:31.0:1619677448.441471:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.441472:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:25.2:1619677448.441474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.441475:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.441476:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.441476:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.441476:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:31.0:1619677448.441478:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.441478:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:31.0:1619677448.441479:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.441479:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677448.441479:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.441479:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:31.0:1619677448.441480:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.441480:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.441481:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:22.0:1619677448.441482:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:25.2:1619677448.441483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677448.441484:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.441485:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.441485:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.441488:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.441490:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:13.0:1619677448.441491:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.441493:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.441494:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.441496:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.441497:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.441499:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.441503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:4.2:1619677448.441504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677448.441509:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.441511:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:12.0:1619677448.441512:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.441513:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.441513:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:10.0:1619677448.441513:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.441514:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:12.0:1619677448.441514:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.441515:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:10.0:1619677448.441516:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.441519:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.441520:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.441521:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714895] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.441522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:10.0:1619677448.441522:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.441523:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.441524:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.441527:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.441528:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.441529:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.441531:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:10.0:1619677448.441531:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.441532:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.441532:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.441532:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:10.0:1619677448.441533:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:18.0:1619677448.441534:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.441534:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.441535:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:18.0:1619677448.441536:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.441536:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.441537:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677448.441538:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:25.2:1619677448.441539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:18.0:1619677448.441539:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.441541:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.441542:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677448.441542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:18.0:1619677448.441543:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.441546:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.441547:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:18.0:1619677448.441548:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.441549:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.441550:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.441551:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.441551:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.441551:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:26.0:1619677448.441552:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.441552:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (23)++
00000800:00000200:10.0:1619677448.441553:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.441553:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.441555:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (24)--
00000800:00000200:18.0:1619677448.441555:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (24)--
00000800:00000200:18.0:1619677448.441555:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.441555:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.441556:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:18.0:1619677448.441556:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.441556:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.441560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.441566:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.441569:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.441570:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.441570:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.441572:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.441574:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.441817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.441823:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.441826:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.441827:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.441829:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.441832:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.441833:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.441835:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.441836:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.441837:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.441838:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.441841:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.441843:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.441846:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.441847:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.441847:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.441849:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.441849:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.441850:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.441851:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.441855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.441861:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.441864:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.441865:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.441867:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.441868:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.441869:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.441880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.441886:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.441889:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.441890:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.441891:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.441894:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.441895:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:5.2:1619677448.441895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:18.0:1619677448.441896:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.441897:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.441898:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.441899:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.441900:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677448.441901:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.441903:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.441903:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.441904:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.441905:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.441906:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.441906:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:17.0:1619677448.441907:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.441908:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.441908:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:17.0:1619677448.441909:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.441910:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:17.0:1619677448.441910:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:18.0:1619677448.441911:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.441911:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:17.0:1619677448.441912:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.441913:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.441914:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677448.441914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:17.0:1619677448.441915:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:4.2:1619677448.441915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677448.441916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:17.0:1619677448.441919:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.441920:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.441922:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.441922:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:17.0:1619677448.441923:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.441924:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.441924:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:17.0:1619677448.441924:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.441924:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.441925:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:18.0:1619677448.441925:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677448.441925:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:2.0:1619677448.441925:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.441926:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.441926:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677448.441927:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.441927:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677448.441927:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677448.441927:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:17.0:1619677448.441928:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:26.0:1619677448.441929:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.441929:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.441930:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.441930:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:26.0:1619677448.441931:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.441932:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.441933:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.441933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:9.2:1619677448.441934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:26.0:1619677448.441935:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.441935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.441936:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.441937:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.441940:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.441940:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677448.441941:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.441941:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.441941:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:26.0:1619677448.441943:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.441943:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677448.441943:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:26.0:1619677448.441944:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.441944:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.441944:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:13.0:1619677448.441944:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.441944:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.441945:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.441945:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677448.441945:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.441946:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.441946:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.441946:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000400:00000200:13.0:1619677448.441946:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.441947:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677448.441947:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677448.441947:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.441948:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:13.0:1619677448.441949:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.441951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.2:1619677448.441954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.441959:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.441961:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.441961:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.441962:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:10.0:1619677448.441963:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.441964:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.441965:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.441966:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.441966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677448.441966:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.441967:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.441968:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.441969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:26.0:1619677448.441971:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.441971:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.441972:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677448.441973:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.441974:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.441975:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:25.2:1619677448.441975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677448.441975:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:30.0:1619677448.441978:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.441978:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.441978:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.441981:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:26.0:1619677448.441981:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.441981:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.441982:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.441983:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.441984:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.441984:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.441984:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.441984:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000400:00000200:5.0:1619677448.441984:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:26.0:1619677448.441985:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.441985:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:5.0:1619677448.441985:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.441986:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.441987:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:5.0:1619677448.441987:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.441988:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677448.441988:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.441989:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.441989:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.441990:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.441990:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:5.0:1619677448.441990:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.441992:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.441992:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.441992:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:30.0:1619677448.441993:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.441993:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:30.0:1619677448.441994:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.441994:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:5.0:1619677448.441994:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.441997:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:25.2:1619677448.441998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:30.0:1619677448.441999:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.442001:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.442004:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677448.442004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.0:1619677448.442004:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.442005:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:6.0:1619677448.442005:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.0:1619677448.442005:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.442006:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.442007:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.442008:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:6.0:1619677448.442008:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:30.0:1619677448.442009:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.442009:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:22.0:1619677448.442009:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.442009:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.442009:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.442010:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.442010:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.442011:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.442011:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:6.0:1619677448.442012:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.442012:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.442013:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677448.442014:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.442015:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.442015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.2:1619677448.442015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:6.0:1619677448.442015:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:30.0:1619677448.442017:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.442018:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.442020:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96714886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.442021:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.442022:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.442022:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:30.0:1619677448.442023:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:20.0:1619677448.442023:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:12.0:1619677448.442024:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677448.442025:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:12.0:1619677448.442025:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.442026:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.442027:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677448.442027:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.442028:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677448.442029:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.442029:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.442030:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677448.442031:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.442033:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677448.442033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.2:1619677448.442033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.442034:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.442034:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.442036:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677448.442036:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.442037:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.442037:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.442038:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.442038:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677448.442039:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.442040:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:13.0:1619677448.442040:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.442041:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:9.2:1619677448.442042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677448.442042:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.442043:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677448.442043:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.442044:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.442044:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.442045:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.442046:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.442050:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.442053:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.442054:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.442054:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.442056:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.442059:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.442069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677448.442074:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.442077:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677448.442078:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677448.442080:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.442083:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.442085:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677448.442086:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.442087:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.442089:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.442090:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677448.442093:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.442095:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677448.442098:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.442099:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:17.0:1619677448.442099:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.2:1619677448.442101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:17.0:1619677448.442101:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:17.0:1619677448.442102:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.442102:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677448.442103:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:25.2:1619677448.442107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.442111:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.442113:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.442114:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.442115:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.442116:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.442117:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.442118:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.442119:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.442121:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.442122:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.442123:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.442124:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.442125:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.442126:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.442127:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.442128:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:10.0:1619677448.442129:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.442131:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.442133:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.442134:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.442135:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.442136:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.442137:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.442137:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.442138:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677448.442141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.442148:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.442150:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.442151:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.442153:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677448.442154:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.442155:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.442187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.442193:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.442195:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.442196:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.442198:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.442201:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.442202:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.442204:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.442205:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.442206:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.442207:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.442210:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.442212:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.442215:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.442215:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.442216:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.442218:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.442218:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.442219:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.442219:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.442224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.442230:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.442233:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.442234:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.442234:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.442237:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.442240:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.442290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.442297:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.442300:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.442301:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.442303:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.442305:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.442307:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.442308:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677448.442309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:18.0:1619677448.442310:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.442311:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.442312:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.442315:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.442317:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.442317:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.442319:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:18.0:1619677448.442320:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.442320:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677448.442321:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:26.0:1619677448.442322:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.442322:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.442324:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.442324:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:26.0:1619677448.442325:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.442325:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.442325:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:26.0:1619677448.442326:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.442328:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.442329:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.442330:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.442330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:26.0:1619677448.442331:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.442335:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.442337:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.442337:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.442339:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.442340:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.442342:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.442342:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.442343:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.442343:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.442345:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.442345:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.442346:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.442346:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.442347:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.442348:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.442352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.2:1619677448.442359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.0:1619677448.442366:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:10.0:1619677448.442367:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677448.442368:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:10.0:1619677448.442369:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:5.0:1619677448.442369:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.442370:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677448.442371:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.442371:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.442373:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.442375:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:10.0:1619677448.442377:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.442378:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.442380:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.442381:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.442382:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.442383:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.442386:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.442387:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.442389:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.442390:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.442390:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.442392:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.442392:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.442393:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.442393:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.442397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.442405:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.442407:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.442408:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.442409:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.442412:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.442413:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.442417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.442423:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.442426:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.442427:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.442427:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.442428:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.442429:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.442443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.442450:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.442453:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.442453:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.442455:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.442458:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.442459:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.442460:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.442461:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.442462:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.442463:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.442466:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.442467:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.442470:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.442471:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.442471:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.442473:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.442473:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.442474:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.442474:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.442479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.442487:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.442490:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.442491:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.442491:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.442493:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.442495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:17.0:1619677448.442496:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677448.442501:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.442504:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.442505:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.442507:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.442510:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.442511:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.442513:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.442514:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.442515:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.442516:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.442519:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.442521:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.442524:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.442524:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.442525:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.442527:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.442527:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.442528:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.442528:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.442533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.442540:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.442543:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.442544:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.442544:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.442546:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.442549:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.442553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.442560:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.442563:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.442564:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.442564:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.442565:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.442566:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.442588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.442594:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.442596:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.442597:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.442599:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.442602:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.442603:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.442604:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.442606:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.442607:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.442608:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.442611:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.442613:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.442616:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.442617:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.442617:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.442619:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.442620:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.442620:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.442620:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.442625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.442631:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:21.2:1619677448.442633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677448.442634:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.442635:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.442635:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.442637:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.442640:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677448.442640:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:31.0:1619677448.442643:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.2:1619677448.442644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:9.2:1619677448.442644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000400:00000200:31.0:1619677448.442645:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.442647:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.442647:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.442648:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.442649:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677448.442649:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:24.0:1619677448.442650:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000400:00000200:31.0:1619677448.442651:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.442652:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.442652:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677448.442653:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.442653:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677448.442654:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.442654:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.442654:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:31.0:1619677448.442655:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:24.0:1619677448.442655:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.442656:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:20.0:1619677448.442656:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.442658:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677448.442659:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.442659:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.442660:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.442661:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.442662:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.442663:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.442665:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.442666:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.442666:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:20.0:1619677448.442667:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.442668:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.442669:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.442669:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.442670:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.442672:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.442672:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.442672:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.442673:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.442673:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:25.2:1619677448.442674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.442675:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.442676:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.442677:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.2:1619677448.442678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.442678:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.442681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.442682:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.442684:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.442685:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.442686:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:6.0:1619677448.442686:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.442688:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:6.0:1619677448.442688:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.442689:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.442690:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.442690:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.442691:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.442691:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.442691:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:31.0:1619677448.442693:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.442693:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.442696:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.442705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.442712:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.442714:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.442715:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.442717:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.442719:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.442720:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.442721:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714874] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.442723:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.442724:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.442725:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.442729:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.442730:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.442733:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.442734:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.442735:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.442736:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.442737:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.442738:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.442738:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.442742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.442750:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.442752:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.442753:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.442754:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.442756:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.442758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.442761:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.442762:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.442764:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.442766:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.442766:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.442767:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.442767:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.443110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.443117:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.443120:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.443122:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.443123:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.443125:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.443127:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.443128:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.443130:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.443131:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.443131:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.443134:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.443136:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.443138:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.443139:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.443139:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.443141:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.443141:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.443142:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.443143:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.443146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.443154:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.443157:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.443158:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.443159:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.443161:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.443162:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.443164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.443169:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.443172:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.443173:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.443175:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.443178:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.443180:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.443181:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.443182:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.443183:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.443184:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.443187:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.443189:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.443192:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.443192:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.443193:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.443195:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.443195:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.443196:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.443196:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.443201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.443208:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.443211:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.443212:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.443213:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.443214:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:4.2:1619677448.443214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677448.443216:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.443217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677448.443221:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677448.443222:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.443224:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.443225:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.443225:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.443228:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.443228:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.443229:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.443231:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.443231:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.443234:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.443235:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.443236:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.443236:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.443237:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.443238:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.443240:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.443240:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.443241:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.443241:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.443245:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.443247:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.443250:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.443250:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.443251:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.443252:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.443252:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.443253:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.443254:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.443255:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:5.0:1619677448.443255:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.443256:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:5.0:1619677448.443256:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.443256:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:10.0:1619677448.443257:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.443258:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:25.2:1619677448.443259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.443259:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.443260:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.443260:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677448.443262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.443266:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:31.0:1619677448.443268:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (24)--
00000400:00000200:31.0:1619677448.443269:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.443269:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677448.443271:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.443271:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:30.0:1619677448.443272:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:26.0:1619677448.443272:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.443273:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:26.0:1619677448.443273:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.443274:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (24)--
00000400:00000200:30.0:1619677448.443274:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.443275:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.443276:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.443278:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.443384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.443389:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.443392:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.443393:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.443395:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.443397:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.443399:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.443400:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714869] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.443402:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.443403:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.443404:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.443407:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.443409:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.443412:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.443412:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.443413:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.443415:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.443415:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.443416:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.443417:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.443422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.443430:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.443433:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.443433:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.443435:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.443436:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.443438:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.443490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.443495:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.443498:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.443500:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.443500:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677448.443503:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.443504:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677448.443506:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.443507:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.443508:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.443509:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.443512:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.443514:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.443517:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.443518:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677448.443518:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.443520:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.443521:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.443521:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.443522:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.443526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.443534:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.443536:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.443537:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.443537:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.443540:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:10.0:1619677448.443543:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.443626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.443633:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:4.2:1619677448.443633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677448.443636:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.443636:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.443639:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.443640:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677448.443641:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.443642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677448.443642:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:26.0:1619677448.443643:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:24.0:1619677448.443643:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:26.0:1619677448.443644:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.443645:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.443645:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.443646:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.443647:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:8.0:1619677448.443648:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.443649:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:31.0:1619677448.443650:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677448.443651:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.443651:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96714866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.443653:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677448.443653:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:8.0:1619677448.443653:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.443654:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.443655:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.443656:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.443656:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.443657:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.443657:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.443657:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:31.0:1619677448.443658:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.443658:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.443659:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.443659:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677448.443660:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:26.0:1619677448.443660:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677448.443660:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.443661:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96714865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.443661:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000400:00000200:31.0:1619677448.443662:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.443662:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:31.0:1619677448.443663:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.443663:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.443664:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:25.2:1619677448.443664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:8.0:1619677448.443664:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.443665:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.443666:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:31.0:1619677448.443667:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.443667:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.2:1619677448.443667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677448.443668:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:31.0:1619677448.443669:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677448.443669:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.443670:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677448.443671:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.443672:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.443672:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.443672:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.443673:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:8.2:1619677448.443673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.443674:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.443674:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677448.443674:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.443675:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:4.2:1619677448.443675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677448.443676:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.443676:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:10.0:1619677448.443676:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.443677:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.443678:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.443678:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.443679:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.443679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.443679:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:1.0:1619677448.443681:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.443682:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.2:1619677448.443683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:24.0:1619677448.443684:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:1.0:1619677448.443684:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677448.443685:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677448.443685:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.443685:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.443686:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677448.443687:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.443688:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.443689:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:26.0:1619677448.443690:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000400:00000200:12.0:1619677448.443690:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.443691:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:12.0:1619677448.443692:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.443693:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.443693:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.443694:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.443694:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:13.0:1619677448.443694:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.443695:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:12.0:1619677448.443695:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.443697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.443703:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.443706:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.443707:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.443707:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.443708:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.443709:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.443770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.443776:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.443779:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.443780:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.443782:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.443785:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.443787:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677448.443788:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.443790:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.443791:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.443792:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.443795:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.443797:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.443800:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.443800:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.443801:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.443803:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.443803:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.443804:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.443804:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677448.443809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.443817:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.443820:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.443821:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.443822:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.443823:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.443825:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.444106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.444112:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.444115:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.444116:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.444118:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.444121:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.444122:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.444123:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.444125:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.444126:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.444126:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.444130:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.444131:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.444133:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.444134:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.444134:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.444136:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.444136:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.444137:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.444137:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.2:1619677448.444151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.2:1619677448.444153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.444158:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.444160:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:21.2:1619677448.444161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677448.444162:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:17.0:1619677448.444163:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.444163:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.444164:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.444166:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.444166:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.444167:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.444169:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.444170:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677448.444170:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.444171:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.444172:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:30.0:1619677448.444173:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677448.444177:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.444177:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.444178:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.444180:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.444180:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:30.0:1619677448.444182:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:1.0:1619677448.444182:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:30.0:1619677448.444183:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.444183:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.444183:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.444184:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.444184:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:30.0:1619677448.444185:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.444185:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:30.0:1619677448.444186:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.444187:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.444187:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:30.0:1619677448.444190:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.444191:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.444192:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.444193:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.2:1619677448.444193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677448.444194:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.444194:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:30.0:1619677448.444195:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)++
00000800:00000200:30.0:1619677448.444196:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (23)++
00000800:00000200:17.0:1619677448.444196:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.444198:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (24)++
00000800:00000200:30.0:1619677448.444199:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (24)--
00000800:00000200:30.0:1619677448.444200:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.2:1619677448.444200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.444201:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.444201:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677448.444201:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (24)--
00000400:00000200:17.0:1619677448.444202:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.444202:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.444203:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:17.0:1619677448.444204:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.444204:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.2:1619677448.444205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.444206:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.444207:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.444208:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677448.444209:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.444212:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.444213:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.444213:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.444214:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.444217:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.444219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.444226:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.444228:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.444229:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.444230:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.444231:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.444231:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.444402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.444410:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.444413:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.444415:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.444418:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.444418:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.444420:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.444421:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.444422:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.444423:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.444424:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.444428:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.444429:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.444432:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.444433:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.444433:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.444435:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.444436:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.444437:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.444438:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.444439:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.444442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.444445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.444451:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.444453:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677448.444454:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:5.0:1619677448.444454:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.444456:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:20.0:1619677448.444456:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.444456:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.444457:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.444458:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.444460:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.444511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.444516:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.444519:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.444520:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.444522:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.444524:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.444526:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:20.2:1619677448.444526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:22.0:1619677448.444527:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.444529:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.444529:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.444529:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:22.0:1619677448.444531:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.444532:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.444535:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.444535:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:22.0:1619677448.444536:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.444538:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.444538:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.444539:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:22.0:1619677448.444541:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.444541:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.444542:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.444542:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:20.0:1619677448.444542:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.444543:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.444544:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:20.0:1619677448.444544:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:22.0:1619677448.444545:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.444546:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.444547:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000400:00000200:20.0:1619677448.444547:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.444548:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677448.444549:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.2:1619677448.444551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.444552:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.444552:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:21.2:1619677448.444553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677448.444553:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.444555:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677448.444556:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.444557:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:30.0:1619677448.444558:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.444558:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:6.0:1619677448.444558:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:20.0:1619677448.444559:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677448.444560:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:30.0:1619677448.444561:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:8.2:1619677448.444561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677448.444561:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:30.0:1619677448.444562:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.444562:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:30.0:1619677448.444564:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.444564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.444565:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.444567:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.444568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.444569:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:5.0:1619677448.444569:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:17.0:1619677448.444572:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:5.0:1619677448.444572:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:17.0:1619677448.444573:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.444573:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677448.444574:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.444574:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677448.444574:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.444576:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.444577:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.444578:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677448.444578:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:31.0:1619677448.444579:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.444580:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.444581:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.444616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.444624:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.444627:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.444628:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.444630:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.444631:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.444633:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.444634:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.444635:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.444636:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.444637:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.444641:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.444643:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.444646:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.444646:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.444647:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.444649:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.444649:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.444650:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.444651:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.2:1619677448.444655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.444662:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.444665:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.444666:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.444666:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.444668:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.444671:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.444695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.444702:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.444704:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.444705:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.444707:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.444710:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.444711:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.444713:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.444714:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.444715:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.444715:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.444718:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.444720:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.444722:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.444723:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.444724:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.444726:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.444726:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.444727:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.444727:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.444732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.444740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677448.444740:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.444743:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.444744:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.444744:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.444746:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.444747:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.444749:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677448.444750:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.444752:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.444754:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.444755:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.444756:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.444757:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.444759:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.444759:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.444760:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.444763:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.444765:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.444767:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.444767:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.444768:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.444769:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.444770:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.444771:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.444773:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.444773:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.444775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.444779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.444781:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.444783:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.444784:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.444784:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.444786:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:10.0:1619677448.444786:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.444789:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.444789:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677448.444789:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677448.444790:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677448.444918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.444924:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.444927:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.444928:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.444930:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.444931:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.444933:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.444934:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.444935:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.444937:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.444937:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.444941:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.444942:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.444944:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.444945:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.444946:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.444947:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.444948:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.444948:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.444949:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.444953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.444960:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.444962:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.444963:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.444963:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.444964:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.444965:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.445006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.445011:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.445014:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.445016:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677448.445017:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.445019:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.445020:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.445022:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.445023:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.445024:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.445025:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.445029:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.445030:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.445033:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.445034:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.445034:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.445037:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.445037:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.445038:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.445039:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.445040:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.445042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.445045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.445048:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:31.0:1619677448.445049:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.445050:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.445050:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.445051:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677448.445052:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.445053:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.445054:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:30.0:1619677448.445056:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.445057:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.445152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.445158:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.445160:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.445161:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.445163:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.445165:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.445166:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.445168:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.445169:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.445170:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.445171:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.2:1619677448.445173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:18.0:1619677448.445175:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.445177:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.445179:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.445180:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.445180:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.445181:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677448.445182:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.445182:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.445183:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.445183:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:30.0:1619677448.445184:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.445186:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.445188:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.445188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677448.445189:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.445190:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.445192:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.445193:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.445194:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.445195:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.445195:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.445197:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.445198:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.445198:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:30.0:1619677448.445199:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.445201:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:17.0:1619677448.445201:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.445203:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677448.445204:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.445205:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:8.2:1619677448.445205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.445206:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.445207:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.445208:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.445209:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.445210:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:1.0:1619677448.445211:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.2:1619677448.445214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:1.0:1619677448.445214:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.445215:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.445215:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.445216:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.445217:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677448.445221:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.445223:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.445224:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.445226:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.445227:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.445229:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.445234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.445241:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.445245:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677448.445247:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.445249:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.445250:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.445251:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.445251:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:26.0:1619677448.445252:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.445255:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.445255:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.445257:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.445260:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.445262:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.445263:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.445264:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.445265:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.445266:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.445270:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.445271:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.445274:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.445275:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.445275:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.445277:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.445278:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.445278:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.445279:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.445283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.445291:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.445297:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.445300:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.445301:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.445302:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.445305:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.445337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.445343:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.445345:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.445346:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.445348:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.445350:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.445352:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.445353:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.445354:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.445355:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.445356:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.445359:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.445360:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.445363:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.445364:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.445364:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.445366:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.445366:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.445367:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.445367:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.445373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.445381:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.445384:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.445385:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.445385:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.445388:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.445390:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.445488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677448.445495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:10.0:1619677448.445496:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.445498:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.445499:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.445501:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.445502:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.445505:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:10.0:1619677448.445505:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.445506:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:8.0:1619677448.445506:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.445508:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.445508:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.445509:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.445510:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.445510:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.445511:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677448.445512:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.445513:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96714847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.445514:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.445515:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.445515:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.445516:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.445517:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:24.0:1619677448.445519:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.445519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677448.445520:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.445521:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.445521:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.445521:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.445523:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677448.445524:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.445524:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.445524:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677448.445525:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.445525:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:24.0:1619677448.445526:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.445527:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677448.445528:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.445528:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.445529:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.445529:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:31.0:1619677448.445530:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:25.2:1619677448.445530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:31.0:1619677448.445532:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.445534:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677448.445534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677448.445535:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.445536:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.445537:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96714846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.445538:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:31.0:1619677448.445539:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.445539:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.445540:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.445541:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.445542:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:31.0:1619677448.445543:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.445543:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.445543:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677448.445545:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:22.0:1619677448.445545:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.445545:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:31.0:1619677448.445547:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.445547:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.445547:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.445548:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677448.445549:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.445549:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:2.0:1619677448.445549:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.445550:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.445551:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.445552:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677448.445552:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677448.445553:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.445556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.445564:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.445566:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.445567:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.445569:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.445569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677448.445571:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.445572:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.445574:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.445577:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.445578:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.445580:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.445582:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.445583:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.445585:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.445586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:1.0:1619677448.445586:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.445587:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.445588:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.445589:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.445591:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:1.0:1619677448.445593:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.445594:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.445595:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677448.445595:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.445596:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.445598:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677448.445599:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.445600:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.445600:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.445600:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:20.0:1619677448.445601:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.445602:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.445602:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.445602:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:20.0:1619677448.445603:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.445603:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.445604:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:20.0:1619677448.445606:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.445608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.445608:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.445611:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.445612:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.445613:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.445615:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.445615:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.445615:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.445616:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677448.445617:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677448.445617:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:30.0:1619677448.445619:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.445620:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.445620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.445621:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.445623:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:12.0:1619677448.445627:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.445629:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.445630:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.445632:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.445633:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.445635:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.445678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.445686:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.445688:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.445691:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.445692:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.445693:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.445694:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.445695:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.445697:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.445698:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.445699:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.445702:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.445704:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.445706:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.445707:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.445708:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.445709:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.445710:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.445711:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.445711:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.445715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.445721:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:21.2:1619677448.445721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:30.0:1619677448.445723:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.445724:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.445726:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.445727:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:18.0:1619677448.445727:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677448.445728:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:18.0:1619677448.445730:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.445731:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.445733:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.445735:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.445736:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.445737:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714842] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.445738:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.445739:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.445740:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.445743:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.445745:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.445747:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.445748:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.445748:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.445750:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.445751:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.445751:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.445752:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.445756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.445763:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.445766:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.445767:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.445767:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.445769:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677448.445771:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.445883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.445890:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.445893:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.445894:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.445896:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.445898:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.445900:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.445901:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.445902:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.445903:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.445904:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.445907:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.445909:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.445911:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.445911:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.445912:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.445914:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.445915:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.445915:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.445916:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.445919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.445926:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.445928:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.445929:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.445931:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.445932:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.445933:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.445982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.445989:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.445992:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.445993:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.445995:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.445997:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.445998:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.446000:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.446001:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.446002:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.446002:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.446005:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.446007:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.446010:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.446011:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.446011:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.446013:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.446013:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.446014:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.446014:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.446019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.446026:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.446029:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.446030:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.446030:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.446032:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.446035:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.446088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.446096:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.446098:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.446101:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.446102:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.446103:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.446104:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.446105:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.446106:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.446107:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.446108:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.446111:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.446113:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.446115:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.446115:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.446116:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.446117:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.446118:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.446119:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.446119:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.446123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.446129:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.446131:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.446132:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.446132:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.446134:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.446137:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.446141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.446149:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.446151:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.446153:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.446155:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.446155:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.446157:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.446158:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.446159:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.446160:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.446161:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.446164:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.446165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677448.446166:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.446168:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.446169:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.446170:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.446170:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.446171:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.446172:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.446173:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.446173:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.446173:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.446175:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.446176:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677448.446178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:5.0:1619677448.446178:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.446179:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.446180:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714837] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.446182:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.446183:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.446183:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.446186:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:5.0:1619677448.446186:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.446187:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.446188:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.446188:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.446190:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.446191:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.446192:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.446193:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.446193:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677448.446193:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.446195:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.446195:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.446196:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.446197:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.446200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.446207:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.446210:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.446211:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.446213:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.446214:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.446215:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.446338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.446346:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.446348:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.446349:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.446351:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.446354:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.446355:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.446356:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.446358:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.446358:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.446359:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.446362:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.446364:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.446367:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.446368:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.446368:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.446370:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.446370:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.446372:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.446373:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.446373:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.446377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.446379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.446385:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.446386:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.446388:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677448.446388:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.446388:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.446389:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.446389:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.446390:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.446391:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.446394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.446394:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677448.446402:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.446404:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.446405:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.446405:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.446406:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.446407:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.446413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.446419:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.446421:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.446422:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.446424:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.446427:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.446429:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.446430:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.446432:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.446433:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.446434:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.446437:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.446438:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.446441:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.446442:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.446443:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:21.2:1619677448.446444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:17.0:1619677448.446444:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.446445:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.446445:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.446446:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.446452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.446452:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.446455:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.446456:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.446457:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.446459:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.2:1619677448.446459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:8.0:1619677448.446460:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.446461:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:8.0:1619677448.446461:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677448.446462:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.446463:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96714834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.446464:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.446464:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.446465:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.446466:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:8.0:1619677448.446466:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.446467:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.446468:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.446469:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.446470:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:8.0:1619677448.446470:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.446472:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.446472:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.446475:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.446475:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.446476:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:8.0:1619677448.446476:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:10.0:1619677448.446477:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.446477:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677448.446479:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.446479:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.446479:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.446479:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:10.0:1619677448.446480:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.446480:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.446480:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:10.0:1619677448.446484:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.446485:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:9.2:1619677448.446485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.446488:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.446489:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.446489:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.446491:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.446491:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.446492:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.446493:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:2.0:1619677448.446493:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.446494:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:2.0:1619677448.446495:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.446497:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.2:1619677448.446497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:2.0:1619677448.446497:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.446499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:2.0:1619677448.446499:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.446502:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677448.446505:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:26.0:1619677448.446507:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:1.0:1619677448.446507:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.446508:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.446509:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:1.0:1619677448.446509:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.446510:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.446511:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:1.0:1619677448.446511:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.446514:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.446516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.446523:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.446526:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.446527:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.446527:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.446528:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.446529:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.446538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.446543:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.446546:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.446547:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.446549:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.446551:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.446552:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.446554:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.446555:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.446556:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.446557:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.446560:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.446562:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.446564:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.446565:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.446566:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.446567:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.446568:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.446569:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.446570:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.446570:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677448.446573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.446576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.446581:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.446582:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.446582:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.446583:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:2.0:1619677448.446583:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.446584:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.446585:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.446585:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.446586:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677448.446586:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.446591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.446597:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.446600:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.446601:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.446601:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.446601:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.446602:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.446612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.446621:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.446623:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.446626:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.446628:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.446628:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.446630:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:21.2:1619677448.446630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:30.0:1619677448.446631:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.446633:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.446633:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.446634:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:6.0:1619677448.446635:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:30.0:1619677448.446638:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.446638:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.446639:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677448.446641:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:6.0:1619677448.446641:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.446643:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.446644:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.446645:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:6.0:1619677448.446645:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:30.0:1619677448.446646:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:6.0:1619677448.446646:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714830] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.446647:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.446648:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.446648:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:6.0:1619677448.446648:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.446649:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:30.0:1619677448.446650:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.446651:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.446652:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:6.0:1619677448.446653:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.446654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:6.0:1619677448.446655:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.446657:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.446658:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.446659:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:20.2:1619677448.446660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:6.0:1619677448.446661:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.446661:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.446662:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.446662:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.446666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:20.0:1619677448.446666:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:31.0:1619677448.446668:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.446668:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.446669:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.446670:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.446671:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.446671:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.446672:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.446672:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.446673:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:20.0:1619677448.446674:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:10.0:1619677448.446675:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.446676:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.446677:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.446679:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.446682:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:4.2:1619677448.446700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.446707:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.446710:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.446710:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.446712:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.446715:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.446716:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.446717:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.446719:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.446719:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.446721:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.446724:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.446726:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.446728:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.446729:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.446730:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.446731:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.446732:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.446732:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.446733:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.446737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.446745:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.446748:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.446749:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.446749:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.446751:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.446754:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.446812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.446817:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.446820:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.446821:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.446823:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.446825:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.446827:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.446828:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.446830:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.446831:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.446832:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.446835:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.446837:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.446840:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.446841:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.446842:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.446843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:1.0:1619677448.446843:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.446844:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.446845:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.446846:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.446847:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.446849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.446851:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:5.2:1619677448.446852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.446854:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.446855:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:13.0:1619677448.446856:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677448.446856:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.446857:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.446859:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:13.0:1619677448.446859:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:10.0:1619677448.446859:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.446860:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677448.446860:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.446861:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677448.446861:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677448.446862:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.446862:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.446862:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.446864:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.446865:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677448.446865:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.446866:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:25.2:1619677448.446869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:10.0:1619677448.446869:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.446871:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.446873:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.446874:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.446875:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.446875:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.446877:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:10.0:1619677448.446877:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:12.0:1619677448.446878:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.446878:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:10.0:1619677448.446878:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:12.0:1619677448.446879:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:10.0:1619677448.446879:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677448.446880:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.446881:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.446883:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.446884:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.446885:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.446888:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.446889:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.446928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.446935:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.446938:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.446939:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.446940:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.446943:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.446944:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.446946:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.446947:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.446948:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.446948:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.446952:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.446953:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.446956:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.446957:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.446957:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.446959:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.446960:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.446960:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.446961:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.446965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.446972:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.446975:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.446976:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.446977:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.446979:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.446981:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.447127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.447132:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.447135:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.447136:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.447138:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.447141:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.447142:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.447144:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.447145:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.447146:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.447147:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.2:1619677448.447149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:17.0:1619677448.447150:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.447152:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.447153:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.447154:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.447155:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.447156:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.447157:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.447157:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.447157:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.447158:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.447159:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.447160:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.447162:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.447162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:10.0:1619677448.447165:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.447166:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:20.2:1619677448.447167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:10.0:1619677448.447168:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.447169:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.447170:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.447170:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:10.0:1619677448.447171:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:1.0:1619677448.447171:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.447172:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.447173:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.447175:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:1.0:1619677448.447175:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.447176:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.447176:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677448.447177:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:10.0:1619677448.447179:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.447180:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.447181:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.447181:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.447182:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.447182:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.447182:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:31.0:1619677448.447184:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:10.0:1619677448.447184:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:31.0:1619677448.447185:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.447185:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.447185:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:31.0:1619677448.447186:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.447186:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:31.0:1619677448.447187:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.447188:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:25.2:1619677448.447190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:31.0:1619677448.447191:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.447193:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.447195:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.447196:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.447196:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.447197:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.447197:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.447198:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:30.0:1619677448.447198:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.447199:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.447200:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.447200:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:30.0:1619677448.447200:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.447201:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.447203:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.447204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.447210:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.447213:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.447213:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.447215:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.447217:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.447218:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.447274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.447279:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.447282:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.447284:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.447285:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.447287:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.447289:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.447290:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.447291:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.447292:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.447297:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.447300:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.447302:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.447305:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.447306:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.447307:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.447309:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.447310:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.447311:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.447312:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.447312:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.447316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.447318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.447324:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:12.0:1619677448.447324:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.447326:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:12.0:1619677448.447326:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677448.447327:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:12.0:1619677448.447327:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.447328:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:12.0:1619677448.447329:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.447331:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.447332:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.447349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987d3f2000] (20)++
00000800:00000200:22.0:1619677448.447356:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987d3f2000] (21)++
00000800:00000200:22.0:1619677448.447359:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.67@o2ib3
00000800:00000200:6.0:1619677448.447360:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987d3f2000] (22)--
00000400:00000200:22.0:1619677448.447362:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.67@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.447364:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.447366:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:22.0:1619677448.447367:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.447368:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.447369:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.447370:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677448.447373:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.67@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.447375:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.447377:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.447378:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.447378:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.447380:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.447380:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987d3f2000] (21)++
00000800:00000200:22.0:1619677448.447382:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987d3f2000] (22)--
00000800:00000200:22.0:1619677448.447382:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987d3f2000] (21)--
00000800:00000200:5.2:1619677448.447385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:25.2:1619677448.447386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.447391:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.447393:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677448.447394:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:6.0:1619677448.447395:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.447395:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.447395:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.447396:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.447397:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:17.0:1619677448.447398:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.447398:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.447399:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:17.0:1619677448.447400:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.447401:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.447402:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.447403:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.447404:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.447407:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.447409:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.447412:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.447413:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.447414:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.447416:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.447416:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.447417:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.447418:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.447421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.447428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987d3f2000] (20)++
00000800:00000200:13.0:1619677448.447428:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.447431:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.447432:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.447433:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.447434:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987d3f2000] (21)++
00000800:00000200:12.0:1619677448.447435:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677448.447436:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.67@o2ib3
00000800:00000200:13.0:1619677448.447436:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.447437:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987d3f2000] (22)++
00000800:00000200:6.0:1619677448.447438:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.67@o2ib3
00000400:00000200:22.0:1619677448.447440:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.67@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.447441:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.67@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.447442:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.447443:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.447444:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:21.2:1619677448.447444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:22.0:1619677448.447445:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.447447:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.447448:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.447449:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:6.0:1619677448.447450:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:4.2:1619677448.447450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677448.447450:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:6.0:1619677448.447452:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.447453:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.67@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.447453:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.447453:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.447454:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:6.0:1619677448.447455:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.447456:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.447457:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677448.447457:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677448.447457:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:26.0:1619677448.447460:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:6.0:1619677448.447460:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.67@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.447460:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.447461:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677448.447461:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:6.0:1619677448.447463:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:2.0:1619677448.447463:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96714817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.447464:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.447464:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.447465:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:6.0:1619677448.447465:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:2.0:1619677448.447465:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.447466:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.447466:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:6.0:1619677448.447466:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:2.0:1619677448.447466:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.447467:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.447468:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.447468:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:6.0:1619677448.447468:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:26.0:1619677448.447470:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.447472:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96714816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.447472:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000400:00000200:26.0:1619677448.447474:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.447474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.447474:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987d3f2000] (23)++
00000800:00000200:6.0:1619677448.447474:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987d3f2000] (23)++
00000800:00000200:8.2:1619677448.447475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:6.0:1619677448.447475:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987d3f2000] (25)--
00000800:00000200:2.0:1619677448.447475:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.447476:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.447476:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987d3f2000] (24)++
00000800:00000200:6.0:1619677448.447476:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987d3f2000] (24)--
00000400:00000200:26.0:1619677448.447477:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.447477:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.447478:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987d3f2000] (24)--
00000800:00000200:6.0:1619677448.447478:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987d3f2000] (24)--
00000800:00000200:2.0:1619677448.447478:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.447479:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987d3f2000] (22)--
00000800:00000200:6.0:1619677448.447479:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (23)++
00000800:00000200:2.0:1619677448.447479:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (23)++
00000400:00000200:26.0:1619677448.447480:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.447481:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (25)--
00000800:00000200:26.0:1619677448.447482:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:6.0:1619677448.447482:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (25)--
00000800:00000200:2.0:1619677448.447482:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677448.447483:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.447483:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.447483:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677448.447483:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.447483:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677448.447484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987d3f2000] (21)++
00000800:00000200:26.0:1619677448.447485:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.447486:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (23)++
00000400:00000200:6.0:1619677448.447486:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.447486:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677448.447487:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (23)++
00000800:00000200:25.2:1619677448.447487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (22)++
00000400:00000200:5.0:1619677448.447487:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.447489:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (24)--
00000800:00000200:26.0:1619677448.447489:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.447489:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:5.0:1619677448.447489:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.447490:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677448.447490:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677448.447491:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.447491:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987d3f2000] (22)++
00000800:00000200:22.0:1619677448.447492:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.447492:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987d3f2000] (23)--
00000800:00000200:5.0:1619677448.447492:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:22.0:1619677448.447493:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.447493:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987d3f2000] (22)--
00000400:00000200:22.0:1619677448.447494:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.447494:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987d3f2000] (21)--
00000800:00000200:8.2:1619677448.447495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:6.0:1619677448.447495:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.447496:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.447499:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677448.447502:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.447504:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:25.2:1619677448.447505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000400:00000200:20.0:1619677448.447505:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.447507:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.447508:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.447509:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.447510:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.447510:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (23)++
00000800:00000200:5.0:1619677448.447511:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.447512:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:6.0:1619677448.447513:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.0:1619677448.447513:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:6.0:1619677448.447516:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.447517:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.447517:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.447518:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.447518:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.447549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987d3f2000] (20)++
00000800:00000200:22.0:1619677448.447554:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987d3f2000] (21)++
00000800:00000200:22.0:1619677448.447556:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.67@o2ib3
00000800:00000200:6.0:1619677448.447557:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987d3f2000] (22)--
00000400:00000200:22.0:1619677448.447559:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.67@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.447561:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.447563:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:22.0:1619677448.447564:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.447565:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.447566:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.447567:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:22.0:1619677448.447571:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.67@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.447573:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677448.447576:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.447577:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:22.0:1619677448.447577:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:22.0:1619677448.447580:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:22.0:1619677448.447580:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987d3f2000] (21)++
00000800:00000200:22.0:1619677448.447581:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987d3f2000] (22)--
00000800:00000200:22.0:1619677448.447581:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987d3f2000] (21)--
00000800:00000200:25.2:1619677448.447585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:21.2:1619677448.447588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.447594:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.447596:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.447596:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:31.0:1619677448.447597:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.447598:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.447599:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.447600:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.447602:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.447603:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:30.0:1619677448.447603:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.447604:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.447604:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.447605:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.447607:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.447608:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.447609:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.447612:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.447614:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.447618:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.447618:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.447619:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.447621:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.447621:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.447622:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.447623:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.447626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.447634:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.447636:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.447637:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.447638:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.447640:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.447643:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.447645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.447652:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.447654:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.447655:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.447656:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.447657:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.447657:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.447695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677448.447698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:1.0:1619677448.447701:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.447704:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:1.0:1619677448.447704:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.447705:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.447706:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.447707:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.447708:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:1.0:1619677448.447709:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.447710:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.447711:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.447712:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.447713:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.447714:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677448.447714:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.447715:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.447715:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.447716:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677448.447717:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.447718:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.447720:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.447721:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.447724:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.447724:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677448.447726:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.447727:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.447728:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.447728:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:18.0:1619677448.447729:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:1.0:1619677448.447729:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:18.0:1619677448.447730:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.447731:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:18.0:1619677448.447732:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.447732:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:1.0:1619677448.447732:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.447733:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677448.447733:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.447733:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677448.447734:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.447737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:9.2:1619677448.447741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.447744:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.447747:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.447748:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.447748:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:2.0:1619677448.447748:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:24.0:1619677448.447750:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.447751:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.447751:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.447752:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.447753:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.447754:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677448.447754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677448.447755:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677448.447757:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.447760:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.447762:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.447763:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.447764:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.447766:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.447767:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.447769:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.447770:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.447771:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.447774:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.447776:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.447779:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.447780:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.447781:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.447783:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.447783:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.447784:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.447784:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.2:1619677448.447787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:25.2:1619677448.447789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.447794:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.447795:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677448.447797:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.447797:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:12.0:1619677448.447798:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.447799:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.447800:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677448.447801:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.447802:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.447802:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.447803:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677448.447804:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.447805:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.447806:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.447807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:10.0:1619677448.447807:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.447809:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.447812:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.447813:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.447814:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:13.0:1619677448.447816:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.447816:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.447817:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:10.0:1619677448.447817:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677448.447817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:13.0:1619677448.447818:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.447818:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:13.0:1619677448.447819:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.447819:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.447820:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.447821:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.447821:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.447822:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677448.447824:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.2:1619677448.447826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:8.0:1619677448.447826:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.447827:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.447829:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.447831:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.447833:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.447834:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96714809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.447834:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:8.0:1619677448.447835:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.447836:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.447836:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.447837:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:8.0:1619677448.447837:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.447838:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.447841:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.447842:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.447843:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677448.447844:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.447847:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.447848:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.447849:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.447851:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.447851:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.447852:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.447853:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.447857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.447865:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.447867:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.447868:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.447869:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.447871:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.447874:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.447874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.447881:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.447884:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.447885:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.447887:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.447890:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.447891:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.447892:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.447894:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.447895:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.447896:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.447898:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.447900:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.447902:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.447902:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.447903:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.447904:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.447904:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.447906:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.447907:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.447907:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.447909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:4.2:1619677448.447913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.447918:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.447919:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.447920:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.447921:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:10.0:1619677448.447921:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.447923:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.447923:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.447923:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.447924:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.447924:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.447941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.447947:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.447949:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.447952:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.447952:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677448.447954:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.447956:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677448.447957:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.447958:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.447959:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.447960:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677448.447964:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.447966:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677448.447968:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.447969:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677448.447970:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677448.447971:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.447972:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.447973:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.447973:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.447977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.447984:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.447987:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.447988:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.447990:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677448.447991:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.447992:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.447994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.448001:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.448003:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.448004:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.448005:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.448005:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.448006:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.448124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.448130:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.448133:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.448133:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.448135:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.448138:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.448140:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.448141:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.448142:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.448143:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.448144:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.448147:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.448149:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.448152:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.448153:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.448154:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.448155:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.448156:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.448157:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.448158:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.448159:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.448161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.448163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.448168:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.448169:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677448.448170:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677448.448171:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.448171:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677448.448171:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.448172:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677448.448173:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.448174:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.448176:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.448180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.448185:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.448188:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.448189:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.448191:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.448194:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.448195:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.448196:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714805] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.448197:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.448198:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.448199:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.448203:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.448205:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.448207:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.448208:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.448209:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.448210:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.448211:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.448212:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.448212:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.448217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.448224:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.448227:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.448228:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.448228:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.448230:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.448233:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.448262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.448269:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.448272:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.448274:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.448276:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.448277:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.448278:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.448280:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.448281:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.448282:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.448283:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.448287:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.448289:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.448292:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.448292:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.448298:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.448301:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.448301:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.2:1619677448.448301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:30.0:1619677448.448303:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.448304:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:20.0:1619677448.448304:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.448305:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.448306:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:9.2:1619677448.448307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:20.0:1619677448.448308:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.448311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677448.448311:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.448311:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.448313:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.448314:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.448315:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:20.0:1619677448.448316:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.448316:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.448317:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:20.0:1619677448.448318:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:18.0:1619677448.448319:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.448319:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.448320:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:2.0:1619677448.448321:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.448322:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.448323:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.448324:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.448324:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.448325:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677448.448325:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.0:1619677448.448328:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.448329:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.448329:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.448331:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.448332:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.448333:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.448334:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.448335:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.448337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.448340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.448344:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.448346:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677448.448347:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:21.2:1619677448.448347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:31.0:1619677448.448348:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.448349:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.448349:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:30.0:1619677448.448351:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.448351:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677448.448352:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.0:1619677448.448352:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.448353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:24.0:1619677448.448354:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.448357:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.448358:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:30.0:1619677448.448360:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:24.0:1619677448.448360:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.448363:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:24.0:1619677448.448363:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.448364:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.448365:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.448365:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:24.0:1619677448.448365:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.448366:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.448367:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:24.0:1619677448.448367:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.448368:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.448369:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.448372:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.448373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:24.0:1619677448.448374:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.448377:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.448377:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.448378:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.448379:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:24.0:1619677448.448380:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.448380:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.448381:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.448382:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:17.0:1619677448.448382:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677448.448382:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:24.0:1619677448.448383:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.2:1619677448.448385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:17.0:1619677448.448385:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.448388:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.448389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000400:00000200:17.0:1619677448.448389:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.448390:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.448391:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.448392:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.448392:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:17.0:1619677448.448393:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677448.448395:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.448395:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.448395:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:17.0:1619677448.448397:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677448.448397:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.448397:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.448398:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:17.0:1619677448.448399:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677448.448399:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.448399:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:13.0:1619677448.448400:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677448.448401:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.448401:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.448402:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.448404:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.448404:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.448405:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677448.448405:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.2:1619677448.448410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.448417:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.448419:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.448420:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.448420:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.448422:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.448423:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.448427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.448435:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.448437:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.448438:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.448438:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.448438:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.448439:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.448452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.448459:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.448462:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.448463:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.448465:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.448467:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.448468:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.448470:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.448471:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.448472:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.448473:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.448476:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.448478:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.448480:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.448481:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.448482:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.448483:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.448484:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.448484:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.448484:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.448489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.448496:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.448499:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.448500:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.448500:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.448502:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.448505:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.448626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.448634:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.448637:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.448638:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.448639:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.448641:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.448642:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.448644:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.448645:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.448646:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.448647:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.448649:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.448651:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.448652:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.448653:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.448653:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.448654:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.448655:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.448656:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.448656:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.448660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.448666:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.448668:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.448668:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.448669:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.448670:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.448671:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.448679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.448685:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.448687:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.448688:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.448688:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.448688:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.448689:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.448735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.448742:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.448744:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.448746:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.448748:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.448749:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.448751:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.448752:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.448753:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.448754:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.448755:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.448758:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.448759:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.448762:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.448763:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.448764:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.448766:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.448766:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.448767:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.448767:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.448772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.448780:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.448783:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.448784:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.448784:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.448786:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.448790:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.448815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.448818:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.448820:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.448823:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.448823:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.448826:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.448827:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677448.448828:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.448829:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.448831:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.448831:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677448.448835:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.448836:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.448838:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.448839:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.448839:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.448841:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.448842:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.448842:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.448843:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.448847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.448854:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.448857:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.448857:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.448859:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.448860:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.448861:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.448985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.448993:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.448996:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.448998:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.448999:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.449001:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.449002:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.449004:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.449005:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.449006:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.449007:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.449010:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.449012:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.449014:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.449015:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.449015:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.449017:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.449017:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.449018:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.449019:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.449022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.449029:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.449031:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.449032:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.449033:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.449035:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.449035:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.449063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.449069:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.449071:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.449072:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.449074:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.449076:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.449078:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.449079:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.449080:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.449081:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.449082:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.449085:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.449086:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.449089:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.449089:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.449090:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.2:1619677448.449090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677448.449092:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.449092:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.449093:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.449094:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.449095:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677448.449098:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:9.2:1619677448.449098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:26.0:1619677448.449100:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:21.2:1619677448.449100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.449101:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.449103:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.449105:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.449105:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:26.0:1619677448.449106:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:18.0:1619677448.449106:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:26.0:1619677448.449107:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.449107:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:18.0:1619677448.449108:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:6.0:1619677448.449108:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:26.0:1619677448.449109:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.449109:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.449109:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:26.0:1619677448.449110:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.449110:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:26.0:1619677448.449111:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.449112:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.449115:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.449115:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:26.0:1619677448.449117:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.449119:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.449119:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.449120:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.449122:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.449122:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.449123:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.449123:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.449127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.449133:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.449135:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.449136:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.449136:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.449138:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.449139:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.449197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.449203:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.449206:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.449207:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.449209:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.449210:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.449212:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.449213:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.449214:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.449216:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.449217:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.449220:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.449221:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.449224:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.449224:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.449225:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.449227:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.449228:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.449228:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.449229:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.449233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.449240:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.449243:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.449244:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.449245:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.449246:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.449248:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.449326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.449332:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.449335:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.449336:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.449338:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.449339:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.449340:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:4.2:1619677448.449341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:17.0:1619677448.449342:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.449343:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.449344:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.449345:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.449348:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.449349:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.449350:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.449352:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:17.0:1619677448.449352:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.449352:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.449353:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.449354:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:26.0:1619677448.449355:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.449355:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.449356:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.449356:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.449357:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.449357:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677448.449358:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.449359:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.449361:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.449362:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.449362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:26.0:1619677448.449363:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.449366:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.449368:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.449369:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:26.0:1619677448.449371:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.449371:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:1.0:1619677448.449371:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:26.0:1619677448.449372:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677448.449372:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.449373:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.449374:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.449374:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677448.449374:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.449375:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677448.449375:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:26.0:1619677448.449376:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.449381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.449389:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.449392:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.449393:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.449393:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.449395:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.449399:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.449448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.449456:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.449459:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.449461:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.449463:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.449463:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.449465:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:31.0:1619677448.449466:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.449467:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.449469:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.449469:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677448.449473:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.449474:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.449477:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.449478:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:31.0:1619677448.449478:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677448.449480:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677448.449481:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.449481:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.449482:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.449486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.449494:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.449497:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.449497:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.449498:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.449500:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.449503:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.449738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.449743:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.449745:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.449746:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.449748:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.449749:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.449751:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.449752:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.449753:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.449754:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.449755:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.449758:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.449759:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.449762:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.449762:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.449763:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.449765:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.449765:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.449765:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.449766:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.449771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.449779:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.449781:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.449782:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.449783:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.449785:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.449787:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.449890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.449898:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.449900:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.449903:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.449903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677448.449904:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.449906:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.449907:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.449908:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.449909:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.449910:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:30.0:1619677448.449911:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.449912:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677448.449913:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.449914:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:30.0:1619677448.449915:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.449916:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.449917:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:8.0:1619677448.449919:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.449920:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.449920:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.449921:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:8.0:1619677448.449921:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:30.0:1619677448.449922:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:8.0:1619677448.449922:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96714787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.449923:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:8.0:1619677448.449923:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.449924:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.449925:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677448.449925:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.449926:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.449927:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.449928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:8.0:1619677448.449928:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.449929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.0:1619677448.449929:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.449932:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677448.449933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677448.449933:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.449934:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677448.449935:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677448.449935:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.449936:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.449936:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677448.449937:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677448.449937:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677448.449937:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:13.0:1619677448.449938:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.449939:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:13.0:1619677448.449940:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.449941:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.449941:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:8.2:1619677448.449941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.449942:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.449942:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677448.449942:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:31.0:1619677448.449943:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.449943:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677448.449944:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:30.0:1619677448.449944:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.449945:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:31.0:1619677448.449946:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.449947:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.449948:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.449949:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:17.0:1619677448.449949:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.449951:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:31.0:1619677448.449952:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.449952:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.449953:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:17.0:1619677448.449953:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.449955:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.449956:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.449956:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.449957:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:17.0:1619677448.449958:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677448.449959:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.449959:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.449960:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.449961:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.449964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:21.2:1619677448.449970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:30.0:1619677448.449971:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.449973:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.449974:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.449976:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.449976:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:31.0:1619677448.449977:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.449979:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:24.0:1619677448.449979:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.449980:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.449982:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.449984:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.449986:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.449987:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.449988:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.449989:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.449990:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.449993:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.449995:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.449997:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.449998:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.449999:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.450001:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.450001:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.450002:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.450003:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.450007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.450015:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.450018:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.450019:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.450019:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.450021:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.450025:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.450224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.450231:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.450233:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.450234:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.450236:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.450237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:10.0:1619677448.450239:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.450241:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.450242:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.450243:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.450245:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.450246:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:10.0:1619677448.450246:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.450247:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:26.0:1619677448.450250:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.450250:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.450252:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.450253:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.450254:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:10.0:1619677448.450255:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.450256:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714783] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.450256:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.450256:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:26.0:1619677448.450257:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.450258:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.450258:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:26.0:1619677448.450259:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.450260:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.450261:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.450261:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.450262:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.450263:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:25.2:1619677448.450264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.450265:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.450267:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.450268:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.450269:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.450270:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.450271:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:26.0:1619677448.450271:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.450272:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:31.0:1619677448.450273:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:26.0:1619677448.450273:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:31.0:1619677448.450274:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.450274:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:31.0:1619677448.450275:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.450276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.450277:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.450278:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.450279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:21.2:1619677448.450281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677448.450286:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:5.0:1619677448.450287:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677448.450289:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.450289:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.450289:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:5.0:1619677448.450290:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:10.0:1619677448.450291:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:5.0:1619677448.450291:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.450292:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677448.450293:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.450293:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000400:00000200:30.0:1619677448.450298:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.450298:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677448.450299:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.450300:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.450301:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677448.450302:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:20.0:1619677448.450302:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:5.0:1619677448.450302:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:30.0:1619677448.450303:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.450303:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677448.450305:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.450305:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.450306:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.450307:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677448.450307:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.450309:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.450311:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.450312:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.450313:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.450314:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.450315:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:20.0:1619677448.450315:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.450318:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.450319:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.450319:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:20.0:1619677448.450319:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.450321:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677448.450321:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.450322:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.450323:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.450323:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677448.450323:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.450324:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.2:1619677448.450324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677448.450325:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.450326:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.450327:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.450327:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.450328:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.450328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:1.0:1619677448.450331:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:25.2:1619677448.450333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677448.450334:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.450335:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.450336:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:1.0:1619677448.450336:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.450339:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:1.0:1619677448.450339:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.450340:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.450340:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:1.0:1619677448.450340:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:6.0:1619677448.450341:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:1.0:1619677448.450341:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96714780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.450343:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.450343:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:1.0:1619677448.450343:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.450344:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.450344:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.450345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:1.0:1619677448.450345:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.450346:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:6.0:1619677448.450347:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.450349:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:1.0:1619677448.450349:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.450350:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:1.0:1619677448.450351:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.450352:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.450352:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677448.450353:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.450354:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.450354:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:1.0:1619677448.450354:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.450355:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:1.0:1619677448.450355:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.450355:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.450357:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.450357:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.450358:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.450359:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.450363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.450371:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.450373:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.450374:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.450374:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.450376:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.450378:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.450382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.450387:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.450390:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.450390:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.450392:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.450394:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.450395:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.450396:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.450398:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.450399:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.450400:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.450404:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.450406:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.450409:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.450410:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.450411:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.450413:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.450413:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.450414:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.450414:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.450418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.450425:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.450428:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.450429:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.450431:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.450432:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.450434:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.450448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.450456:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.450459:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.450460:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.450462:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.450464:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.450466:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.450467:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.450469:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.450470:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.450471:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.450474:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.450476:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.450479:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.450480:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.450480:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.450482:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.450483:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.450484:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.450485:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.450486:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.2:1619677448.450489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.450491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.450497:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:10.0:1619677448.450498:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:17.0:1619677448.450500:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.450500:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.450501:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.450501:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.450501:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.450502:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:17.0:1619677448.450503:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.450506:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.450674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.450681:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.450684:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.450686:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.450687:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.450688:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.450689:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.450691:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714777] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.450692:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.450693:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.450694:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.450697:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.450698:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.450701:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.450702:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.450702:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.450704:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.450705:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.450706:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.450706:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.450710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.450718:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.450721:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:20.2:1619677448.450722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677448.450722:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.450722:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.450724:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.450727:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:31.0:1619677448.450730:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.450733:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.450735:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.450737:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.450738:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.450739:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.450740:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.450741:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.450742:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.450743:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.450746:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.450748:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.450750:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.450751:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.450751:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:21.2:1619677448.450752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:31.0:1619677448.450753:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.450753:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.450754:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.450754:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.450758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:18.0:1619677448.450758:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.450761:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.450762:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.450763:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.450765:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:18.0:1619677448.450765:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.450767:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:18.0:1619677448.450767:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677448.450768:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.450768:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714775] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.450770:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.450770:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.450771:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:18.0:1619677448.450771:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.450772:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:18.0:1619677448.450772:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.450775:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.450776:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.450778:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.450779:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.450780:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.450782:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.450782:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.450783:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.450784:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.450788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.450795:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.450797:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.450798:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.450799:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.450800:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.450801:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.450815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.450821:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.450824:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.450824:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.450825:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.450825:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.450826:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.450885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.450890:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.450892:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:20.2:1619677448.450893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677448.450893:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.450894:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:20.2:1619677448.450896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:2.0:1619677448.450896:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.450897:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.450898:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.450899:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:2.0:1619677448.450900:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.450900:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.450901:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.450901:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:2.0:1619677448.450901:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.450903:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.450905:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.450907:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.450907:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000400:00000200:5.0:1619677448.450908:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.450909:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.450910:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.450910:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.450911:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:2.0:1619677448.450911:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.450911:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677448.450913:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714773] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.450913:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:5.0:1619677448.450914:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.450914:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.450914:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:5.0:1619677448.450915:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.450915:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:5.0:1619677448.450916:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.450918:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:9.2:1619677448.450919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677448.450920:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96714772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.450921:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.450921:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.450922:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.450923:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.450923:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.450926:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677448.450926:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.450927:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.450927:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.450928:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:5.0:1619677448.450928:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:18.0:1619677448.450929:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.450930:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.450930:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.450930:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)++
00000800:00000200:2.0:1619677448.450930:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.450931:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.450931:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.450932:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.450933:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.450933:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677448.450933:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677448.450934:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.450934:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.450934:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.450936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.450936:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.450937:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.450937:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.450938:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:5.2:1619677448.450940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:25.2:1619677448.450942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.450943:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.450946:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.450947:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.450947:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.450948:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.450949:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:22.0:1619677448.450949:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.450951:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:22.0:1619677448.450951:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677448.450951:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:5.2:1619677448.450951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:31.0:1619677448.450952:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.450952:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.450954:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.450955:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:30.0:1619677448.450955:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:2.0:1619677448.450956:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677448.450957:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:10.0:1619677448.450957:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.450959:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:2.0:1619677448.450959:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.450960:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:10.0:1619677448.450960:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.450962:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.450963:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.450963:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.450964:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.450964:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.450966:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:10.0:1619677448.450968:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.450968:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.450969:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.450970:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.450971:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.450971:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.450974:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.450974:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.450975:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:2.0:1619677448.450975:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.450977:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677448.450977:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.450978:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.450978:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.450979:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677448.450980:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.450980:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.450981:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.450982:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:25.2:1619677448.450983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677448.450983:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.450984:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.450984:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.450989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677448.450990:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.450993:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.450993:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.450995:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.450996:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:13.0:1619677448.450997:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.450997:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.450998:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.450999:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.451000:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:4.2:1619677448.451000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677448.451002:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.451004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:21.2:1619677448.451004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:1.0:1619677448.451004:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677448.451007:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.451009:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:6.0:1619677448.451009:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.451010:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:12.0:1619677448.451011:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677448.451012:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:26.0:1619677448.451013:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.451013:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677448.451015:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.451016:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.451016:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.451016:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:6.0:1619677448.451016:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.451017:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:12.0:1619677448.451017:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:26.0:1619677448.451018:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.451018:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:6.0:1619677448.451019:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.451020:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.451020:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.451021:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.451022:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.451023:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677448.451024:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.451025:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.451026:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.451032:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.451032:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.451034:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.451036:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.451038:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.451039:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.451039:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.451040:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.451040:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.451041:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.451042:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.451043:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.451043:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.451044:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.451045:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.451045:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.451045:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:26.0:1619677448.451046:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.451047:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.451048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:25.2:1619677448.451052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.451053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.451055:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.451058:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.451059:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.451060:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.451060:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:30.0:1619677448.451061:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.451061:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:31.0:1619677448.451062:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.451062:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677448.451063:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677448.451063:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.451064:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:6.0:1619677448.451064:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:25.2:1619677448.451065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:22.0:1619677448.451065:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.451067:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.451069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.451072:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.451072:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.451074:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.451075:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.451076:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.451076:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.451077:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.451077:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.451077:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.451077:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.451078:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.451079:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:22.0:1619677448.451079:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.451223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda11a00] (20)++
00000800:00000200:8.0:1619677448.451231:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda11a00] (21)++
00000800:00000200:8.0:1619677448.451234:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.3.50@o2ib3
00000800:00000200:24.0:1619677448.451235:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (22)--
00000400:00000200:8.0:1619677448.451237:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.50@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.451241:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.451243:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:8.0:1619677448.451244:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.451246:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.451247:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.451248:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:20.2:1619677448.451252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:8.0:1619677448.451252:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.451254:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:8.0:1619677448.451257:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.451258:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:8.0:1619677448.451258:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:30.0:1619677448.451260:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:8.0:1619677448.451260:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:8.0:1619677448.451261:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda11a00] (21)++
00000800:00000200:30.0:1619677448.451262:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:8.0:1619677448.451262:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda11a00] (22)--
00000800:00000200:8.0:1619677448.451263:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (21)--
00000400:00000200:30.0:1619677448.451264:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.451266:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.451266:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.451266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000400:00000200:30.0:1619677448.451268:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.451269:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.451270:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.451271:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.451272:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.451274:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:30.0:1619677448.451275:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.451277:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.451277:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.451277:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.451279:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.451280:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.451280:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:30.0:1619677448.451281:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.451282:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.451283:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:30.0:1619677448.451284:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.451284:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.451286:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.451287:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.451288:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.451290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.451297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.451304:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:31.0:1619677448.451306:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.451306:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.451307:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.451308:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.451309:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.451309:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.451310:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.451310:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.451312:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.451318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda11a00] (20)++
00000800:00000200:24.0:1619677448.451326:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda11a00] (21)++
00000800:00000200:24.0:1619677448.451329:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.50@o2ib3
00000800:00000200:8.0:1619677448.451330:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (22)--
00000400:00000200:24.0:1619677448.451332:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.50@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.451333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:24.0:1619677448.451334:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.451336:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.451337:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.451339:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.451340:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:24.0:1619677448.451340:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.451341:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.451343:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:24.0:1619677448.451344:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.451345:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.451347:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677448.451347:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:31.0:1619677448.451348:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.451349:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.451350:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:24.0:1619677448.451350:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:31.0:1619677448.451351:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.451351:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677448.451352:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.451353:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.451353:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.451353:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda11a00] (21)++
00000400:00000200:31.0:1619677448.451354:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.451354:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda11a00] (22)--
00000800:00000200:4.2:1619677448.451354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda11a00] (22)++
00000400:00000200:31.0:1619677448.451357:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.451357:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (22)--
00000800:00000200:31.0:1619677448.451358:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.451358:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda11a00] (21)++
00000800:00000200:9.2:1619677448.451359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.451360:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.50@o2ib3
00000800:00000200:31.0:1619677448.451361:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.451361:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677448.451361:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (22)--
00000800:00000200:31.0:1619677448.451362:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:24.0:1619677448.451363:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.50@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.451364:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.451364:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.451365:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.451365:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:24.0:1619677448.451365:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.451367:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:24.0:1619677448.451368:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96714763] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.451368:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:24.0:1619677448.451369:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677448.451369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:24.0:1619677448.451370:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.451371:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:24.0:1619677448.451372:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677448.451373:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.451373:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.451375:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.451376:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:24.0:1619677448.451377:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.451378:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:18.0:1619677448.451379:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:8.0:1619677448.451379:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.451380:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:8.0:1619677448.451381:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.451383:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.451383:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:24.0:1619677448.451384:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:8.0:1619677448.451384:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.451385:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677448.451386:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:24.0:1619677448.451387:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda11a00] (21)++
00000800:00000200:24.0:1619677448.451388:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda11a00] (22)--
00000800:00000200:24.0:1619677448.451388:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (21)--
00000800:00000200:25.2:1619677448.451391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.451398:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.451401:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.451402:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.451402:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.451404:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.451406:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.451416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:4.2:1619677448.451420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda11a00] (20)++
00000800:00000200:10.0:1619677448.451423:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.451425:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.451427:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677448.451427:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda11a00] (21)++
00000400:00000200:10.0:1619677448.451428:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.451430:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (22)--
00000400:00000200:10.0:1619677448.451430:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.451430:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.50@o2ib3
00000400:00000200:10.0:1619677448.451432:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677448.451432:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.50@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.451433:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.451435:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.451435:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.451436:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.451436:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677448.451436:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.451437:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.451439:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.451440:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.451440:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.451441:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.451443:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:8.0:1619677448.451444:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.451445:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.451446:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.451446:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:8.0:1619677448.451446:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.451448:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.451448:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.451449:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.451449:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677448.451449:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.451449:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.451450:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.451452:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.451452:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda11a00] (21)++
00000800:00000200:25.2:1619677448.451453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:8.0:1619677448.451454:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda11a00] (22)++
00000800:00000200:8.0:1619677448.451455:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda11a00] (23)--
00000800:00000200:8.2:1619677448.451457:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677448.451460:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:8.0:1619677448.451460:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (22)--
00000800:00000200:4.2:1619677448.451461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda11a00] (21)++
00000800:00000200:30.0:1619677448.451463:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.451463:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.451465:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.451465:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:31.0:1619677448.451466:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.451468:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:24.0:1619677448.451468:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda11a00] (22)++
00000800:00000200:17.0:1619677448.451469:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:24.0:1619677448.451470:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda11a00] (23)--
00000400:00000200:17.0:1619677448.451470:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.451470:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.451471:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (22)--
00000800:00000200:24.0:1619677448.451472:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (21)--
00000400:00000200:17.0:1619677448.451472:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.451476:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.451477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.451484:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.451487:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.451488:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.451488:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.451489:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.2:1619677448.451490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677448.451490:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.451495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda11a00] (20)++
00000800:00000200:5.0:1619677448.451496:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.451499:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:8.0:1619677448.451501:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda11a00] (21)++
00000400:00000200:5.0:1619677448.451501:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.451502:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.451503:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.451505:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (22)--
00000800:00000200:8.0:1619677448.451505:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.50@o2ib3
00000400:00000200:5.0:1619677448.451505:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677448.451507:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714760] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.451508:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.50@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.451508:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.451510:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.451511:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.451511:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.451514:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.451515:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.451516:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96714759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.451517:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.451518:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.451518:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.451519:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.451521:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.451521:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.451522:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:8.0:1619677448.451523:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.451524:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.451524:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677448.451525:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.451525:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.451527:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:8.0:1619677448.451528:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.451528:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.0:1619677448.451529:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.2:1619677448.451531:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677448.451532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677448.451534:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.451535:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:8.0:1619677448.451535:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.451536:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda11a00] (21)++
00000800:00000200:20.0:1619677448.451537:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:8.0:1619677448.451537:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda11a00] (22)--
00000800:00000200:20.0:1619677448.451538:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.0:1619677448.451538:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (21)--
00000800:00000200:20.0:1619677448.451539:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677448.451539:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.451541:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:8.2:1619677448.451541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:17.0:1619677448.451542:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.451544:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.451544:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.451546:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.451547:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677448.451547:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.451548:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.451548:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.451550:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.451552:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.451566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.451573:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.451576:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.451577:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.451577:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.451578:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.451578:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.451605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.451612:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.451614:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.451615:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.2:1619677448.451616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:26.0:1619677448.451617:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.451619:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.451619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda11a00] (20)++
00000400:00000200:26.0:1619677448.451621:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.451622:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.451622:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:26.0:1619677448.451623:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.451624:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.451626:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:24.0:1619677448.451626:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda11a00] (21)++
00000800:00000200:18.0:1619677448.451626:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.451626:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:24.0:1619677448.451629:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.50@o2ib3
00000800:00000200:8.0:1619677448.451630:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (22)--
00000400:00000200:2.0:1619677448.451630:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.451631:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.451632:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.50@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.451632:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.451633:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.451634:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677448.451635:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.451635:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.451636:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.451637:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.451637:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:24.0:1619677448.451637:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677448.451637:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.451638:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.451639:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.451639:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:24.0:1619677448.451639:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96714756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.451640:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.451640:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:2.0:1619677448.451640:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677448.451641:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.451642:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.451643:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677448.451644:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.451645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.451647:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:24.0:1619677448.451648:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.451650:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.451652:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.451653:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.451653:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677448.451653:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.451653:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677448.451654:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:24.0:1619677448.451655:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.451655:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677448.451656:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.451656:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:24.0:1619677448.451657:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda11a00] (21)++
00000800:00000200:22.0:1619677448.451657:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.451657:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677448.451657:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:24.0:1619677448.451658:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda11a00] (22)--
00000400:00000200:6.0:1619677448.451658:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.451659:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (21)--
00000800:00000200:2.0:1619677448.451659:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.451659:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:6.0:1619677448.451660:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.451662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.451662:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677448.451663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.2:1619677448.451666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677448.451671:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.2:1619677448.451672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677448.451673:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:12.0:1619677448.451673:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.451673:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:12.0:1619677448.451674:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.451674:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.451675:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.451675:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.451676:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.451676:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.451676:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677448.451677:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:1.0:1619677448.451677:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677448.451678:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.451678:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.451679:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.451680:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:25.2:1619677448.451681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.451681:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.451682:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:1.0:1619677448.451683:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.451685:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.451687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:1.0:1619677448.451687:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:13.0:1619677448.451688:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677448.451688:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.451690:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.451691:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.451691:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.451692:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677448.451692:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677448.451693:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.451693:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.451695:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677448.451695:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677448.451695:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.451697:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.451698:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:1.0:1619677448.451699:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.451700:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.451700:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.451701:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:1.0:1619677448.451701:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.451702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda11a00] (20)++
00000800:00000200:1.0:1619677448.451702:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:31.0:1619677448.451703:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.451703:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.451704:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:31.0:1619677448.451705:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:1.0:1619677448.451705:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000400:00000200:31.0:1619677448.451706:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.451706:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677448.451707:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.451708:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.451708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677448.451709:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.2:1619677448.451711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.451713:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.451713:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda11a00] (21)++
00000800:00000200:31.0:1619677448.451714:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.0:1619677448.451716:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.50@o2ib3
00000800:00000200:31.0:1619677448.451717:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.451717:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (22)--
00000800:00000200:17.0:1619677448.451717:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.451718:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:1.0:1619677448.451718:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677448.451719:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677448.451719:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.451719:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.451719:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.50@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.451719:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.451719:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.451721:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.451721:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.451721:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.451721:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.451722:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.451722:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:8.0:1619677448.451722:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.451722:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677448.451724:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:8.0:1619677448.451724:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:8.2:1619677448.451727:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:8.0:1619677448.451730:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96714753] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.451731:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.451732:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.451733:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.451734:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:8.0:1619677448.451736:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.451737:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.451738:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.451738:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.451739:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.451740:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.451741:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.451741:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.451742:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.0:1619677448.451742:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.2:1619677448.451745:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:8.0:1619677448.451748:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:25.2:1619677448.451749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677448.451749:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda11a00] (21)++
00000800:00000200:8.0:1619677448.451750:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda11a00] (22)--
00000800:00000200:8.0:1619677448.451750:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (21)--
00000800:00000200:5.2:1619677448.451752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:12.0:1619677448.451755:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.451757:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.451757:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:12.0:1619677448.451758:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.451759:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677448.451760:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:12.0:1619677448.451761:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.451761:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:13.0:1619677448.451762:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.451762:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:5.0:1619677448.451762:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:26.0:1619677448.451763:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677448.451763:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:12.0:1619677448.451763:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677448.451764:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.451765:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:10.0:1619677448.451767:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.451768:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.451770:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.451771:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.451772:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.451773:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.451776:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.451777:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.451780:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.451780:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.451781:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.451782:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.451783:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.451783:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.451784:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.451788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.451794:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:21.2:1619677448.451795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:13.0:1619677448.451797:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.451797:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.451799:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.451800:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.451801:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677448.451801:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.451803:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.451805:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.451806:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.451809:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.451811:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.451812:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.451813:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.451814:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.451815:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.451819:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.451820:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.451823:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.451824:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.451825:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.451827:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.451827:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.451828:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.451828:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.451833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.451839:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.451842:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.451843:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.451844:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.451845:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.451847:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.451871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.451876:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.451880:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.451880:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.451882:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.451885:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.451886:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.451887:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.451889:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.451890:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.451891:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.451895:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.451896:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.451900:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.451900:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.451901:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.451903:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.451904:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.451904:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.451905:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.451909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.451916:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.451918:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.451919:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.451920:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.451922:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.451925:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.452062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.452068:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.452071:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.452071:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.452072:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.452073:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.452075:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.452076:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.452077:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.452078:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.452080:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.452081:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.452082:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.452086:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.452087:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.452088:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.452089:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.452090:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.452092:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.452092:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.452093:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.452095:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.452096:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.452096:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:17.0:1619677448.452097:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.452098:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.452098:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.452099:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)++
00000800:00000200:1.0:1619677448.452100:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (24)--
00000800:00000200:1.0:1619677448.452101:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.452102:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.452102:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.452103:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.452103:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.2:1619677448.452104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.452105:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.452106:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.452106:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.452108:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677448.452108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677448.452109:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.452110:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:25.2:1619677448.452111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:17.0:1619677448.452111:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.452112:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.452112:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.452113:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677448.452115:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677448.452118:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:10.0:1619677448.452118:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.452119:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:13.0:1619677448.452120:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:10.0:1619677448.452120:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.452121:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.452123:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.452123:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.452124:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:10.0:1619677448.452124:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.452125:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714747] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.452126:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:10.0:1619677448.452127:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.452128:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.452129:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.452132:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.452134:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.452137:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.452137:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.452138:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.452140:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.452140:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.452141:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.452141:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.452146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.452153:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.452156:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.452157:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.452157:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.452159:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.452162:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.452262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.452265:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.452267:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.452269:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.452271:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.452272:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.452273:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677448.452275:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.452276:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.452277:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.452278:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677448.452282:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.452284:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677448.452287:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.452288:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677448.452288:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677448.452290:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677448.452291:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.2:1619677448.452293:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:9.2:1619677448.452299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677448.452300:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.452301:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677448.452303:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.452306:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:24.0:1619677448.452306:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:21.2:1619677448.452307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:30.0:1619677448.452308:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.452309:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.452310:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.452310:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.452311:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.452311:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.452312:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.452313:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.452314:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.452314:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:30.0:1619677448.452315:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.452315:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:30.0:1619677448.452317:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.452317:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:30.0:1619677448.452318:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.452318:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.452320:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:30.0:1619677448.452322:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.452322:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.452324:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677448.452325:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.452326:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.452327:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.452328:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.452329:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.452329:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.452330:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.452330:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.452332:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.452333:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:18.0:1619677448.452333:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:30.0:1619677448.452334:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.452334:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:18.0:1619677448.452334:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.452337:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.452337:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.452338:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.2:1619677448.452338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.452340:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.452341:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.452342:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.452343:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.452343:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677448.452346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.452347:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:21.2:1619677448.452348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.0:1619677448.452350:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.452351:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.452353:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677448.452353:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.452353:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.452354:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:22.0:1619677448.452356:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677448.452356:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677448.452356:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:22.0:1619677448.452357:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.452357:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.452357:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:22.0:1619677448.452359:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.452359:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677448.452362:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.452367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.452375:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.452378:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.452379:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.452381:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.452384:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.452385:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.452386:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.452388:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.452389:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.452390:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.452393:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.452395:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.452398:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.452399:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.452400:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.452401:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.452402:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.452403:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.452404:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.452404:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.452407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.452410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.452415:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677448.452417:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677448.452417:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.452418:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.452418:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.452419:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.452420:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.452421:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677448.452421:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.452423:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.452425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:22.0:1619677448.452432:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.452435:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.452436:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.452436:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.452437:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.452437:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.452451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.452457:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.452460:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.452461:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.452463:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.452466:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.452467:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.452469:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.452470:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.452471:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.452472:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.452475:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.452477:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.452478:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.452479:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.452480:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.452481:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.452482:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.452483:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.452483:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.452487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.452495:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.452497:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.452498:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.452498:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.452499:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.452500:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.452553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.452558:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.452561:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.452562:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.452564:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.452566:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.452567:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.452569:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.452570:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.452571:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.452572:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.452575:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.452577:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.452580:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.452580:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.452581:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.452583:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.452583:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.452584:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.452584:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.452589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.452596:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.452599:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.452600:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.452602:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.452604:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.452604:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.452637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.452644:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.452647:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.452647:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.452650:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.452653:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.452654:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.452655:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.452657:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.452657:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.452658:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.452662:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.452663:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.452665:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.452666:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.452667:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.452668:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.452669:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.452669:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.452670:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.452674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.452681:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.452683:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.452684:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.452685:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.452687:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.452688:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.452763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:6.0:1619677448.452769:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677448.452772:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677448.452772:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000400:00000200:6.0:1619677448.452774:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.452777:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.452779:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:6.0:1619677448.452780:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.452781:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.452782:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.452783:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:6.0:1619677448.452786:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.452788:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677448.452790:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.452791:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677448.452792:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.452794:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677448.452794:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677448.452795:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677448.452795:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:5.2:1619677448.452797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:9.2:1619677448.452800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.452802:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.452805:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.452806:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.452808:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:1.0:1619677448.452808:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.452810:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:1.0:1619677448.452810:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.452811:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.452811:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.452811:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.452813:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.452814:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.452815:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.452816:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.452817:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:1.0:1619677448.452817:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.452820:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.452822:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.452826:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.452826:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.452827:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.452829:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.452829:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.452830:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.452831:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.452832:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.452835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.452837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.452841:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.452842:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.452844:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:13.0:1619677448.452844:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677448.452845:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677448.452845:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.452846:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677448.452847:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.452847:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.452850:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.452853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.452859:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.452862:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.452863:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.452863:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.452864:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.452865:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.452879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.452885:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.452888:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.452888:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.452890:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.452893:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.452894:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.452896:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714737] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.452897:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.452898:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.452899:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.452903:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.452904:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.452906:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.452907:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.452907:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.452909:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.452909:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.452910:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.452910:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.452915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.452922:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.452924:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.452925:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.452926:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.452927:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.452929:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.452934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.452942:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.452944:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.452944:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.452945:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.452945:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.452946:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.452971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.452978:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.452981:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.452981:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677448.452984:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.452986:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.452988:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.452989:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.452990:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.452991:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.452992:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.452995:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.452997:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.453000:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.453000:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.453001:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.453003:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.453003:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.453004:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.453004:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.453009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.453016:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.453019:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.453020:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.453020:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.453022:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:1.0:1619677448.453026:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.453048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.453055:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.453058:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.453060:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.453062:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.453063:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.453065:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.453066:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.453067:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.453069:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.453069:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.453073:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.453074:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.453077:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.453077:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.453078:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.453079:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.453080:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.453081:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.453081:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.453085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.453092:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.453094:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.453095:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.453096:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.453098:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.453099:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.453140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.2:1619677448.453145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677448.453147:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.453150:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677448.453150:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.453152:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.453153:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.453153:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677448.453154:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.453154:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.453155:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677448.453156:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.453157:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.453158:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.453158:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.453159:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.453160:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677448.453160:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.453161:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.453163:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.453164:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.453165:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.453166:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.453168:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:17.0:1619677448.453170:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.453172:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.453172:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:17.0:1619677448.453172:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.453173:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.453175:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.453175:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.453175:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.453176:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.453177:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:17.0:1619677448.453177:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.453178:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.453179:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.453179:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677448.453179:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.453180:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.453181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.453181:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.453185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:21.2:1619677448.453185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677448.453189:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.453192:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.453192:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.453193:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:22.0:1619677448.453193:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.453193:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:22.0:1619677448.453195:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.453195:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:30.0:1619677448.453196:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:12.0:1619677448.453196:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.453197:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.453198:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.453198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:12.0:1619677448.453198:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.453199:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677448.453201:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:12.0:1619677448.453202:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.453203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:22.0:1619677448.453203:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.453205:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.453206:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.453206:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.453207:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.453208:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677448.453210:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.453213:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.453214:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.453214:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.453215:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.453216:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.453343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677448.453349:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677448.453352:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677448.453353:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677448.453355:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.453357:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.453359:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677448.453360:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.453361:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.453363:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.453363:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.2:1619677448.453367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:1.0:1619677448.453367:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.453368:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.453369:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.453371:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.453371:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.453372:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677448.453372:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:20.0:1619677448.453374:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.453374:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677448.453374:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:5.0:1619677448.453375:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677448.453376:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000400:00000200:20.0:1619677448.453377:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.453377:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677448.453378:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:20.0:1619677448.453379:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:25.2:1619677448.453380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:20.0:1619677448.453380:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.453381:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.453383:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.453383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:20.0:1619677448.453384:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.453388:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.453389:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.453389:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:6.0:1619677448.453389:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677448.453391:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:20.0:1619677448.453392:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.453392:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:6.0:1619677448.453392:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.453393:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.453393:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:6.0:1619677448.453393:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.453393:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.0:1619677448.453394:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:6.0:1619677448.453395:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.453396:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.453396:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.453397:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677448.453397:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677448.453398:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.453402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.453409:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.453412:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.453413:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.453415:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.453416:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.453418:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.453456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.453464:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.453467:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.453467:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.453470:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.453472:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.453473:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.453475:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.453476:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.453477:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.453478:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.453481:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.453482:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.453484:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.453485:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.453486:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.453487:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.453487:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.453488:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:21.2:1619677448.453488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:26.0:1619677448.453490:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.453490:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.453492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:8.0:1619677448.453495:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:4.2:1619677448.453496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677448.453498:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.453499:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.453500:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.453501:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.453502:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.453503:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.453503:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.453504:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:8.0:1619677448.453504:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:26.0:1619677448.453505:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.453505:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.453505:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.453506:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:8.0:1619677448.453506:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96714729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.453507:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.453507:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:8.0:1619677448.453507:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.453508:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.453509:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.453512:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.453513:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.453515:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.453516:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.453517:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.453518:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.453519:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.453519:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.453520:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.453524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.453531:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.453534:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.453535:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.453536:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.453538:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:4.2:1619677448.453538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:13.0:1619677448.453539:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677448.453545:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.453547:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.453548:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.453549:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.453551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:24.0:1619677448.453552:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.453554:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.453555:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.453556:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.453557:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.453557:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:24.0:1619677448.453558:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.453560:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.453560:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677448.453561:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.453563:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.453565:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.453567:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.453567:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:17.0:1619677448.453567:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.453568:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.453569:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.453569:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:24.0:1619677448.453570:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:17.0:1619677448.453570:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.453571:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:17.0:1619677448.453571:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.453572:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677448.453572:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.453573:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.453573:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.453577:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.453577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.453578:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:4.2:1619677448.453578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.453582:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.453583:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.453584:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677448.453584:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.453586:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:1.0:1619677448.453586:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.453587:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677448.453587:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677448.453587:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.453588:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.453588:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.453588:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.453589:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.453589:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:1.0:1619677448.453590:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.453591:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:25.2:1619677448.453592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.453594:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.453597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:12.0:1619677448.453599:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.453601:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.453602:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.453604:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:12.0:1619677448.453604:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.453606:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.453607:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.453607:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:12.0:1619677448.453607:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677448.453608:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.453608:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.453609:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.453649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.453657:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.453659:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.453660:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.453662:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.453664:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.453665:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.453666:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.453668:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.453669:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.453670:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.453673:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.453675:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.453677:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.453678:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.453679:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.453680:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.453681:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.453682:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.453682:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.453687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.453694:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.453697:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.453698:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.453698:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.453700:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.453703:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.453994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.454000:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.454002:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.454003:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.454005:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.454008:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.454009:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.454010:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.454012:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.454013:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.454014:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.454017:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.454019:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.454022:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.454022:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.454023:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.454025:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.454025:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.454026:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.454026:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.2:1619677448.454028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:9.2:1619677448.454031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:2.0:1619677448.454034:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:24.0:1619677448.454037:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:2.0:1619677448.454037:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.454038:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.454039:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.454040:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677448.454040:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.454041:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.454041:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.454044:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:24.0:1619677448.454045:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.454045:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.454047:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.454048:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:2.0:1619677448.454048:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.454049:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:9.2:1619677448.454052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000400:00000200:2.0:1619677448.454052:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.454054:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.454057:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.454057:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:8.0:1619677448.454058:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:2.0:1619677448.454058:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.454060:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.454060:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:24.0:1619677448.454061:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.454061:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:2.0:1619677448.454061:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.454061:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.0:1619677448.454062:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.454063:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.454064:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.2:1619677448.454066:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.454075:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.454077:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.454078:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.454080:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.454081:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.454083:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.454092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.454098:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.454100:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.454101:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.454103:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.454105:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.454107:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.454108:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.454109:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.454111:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.454111:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.454115:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.454117:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.454119:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.454120:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.454121:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.454122:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.454123:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.454124:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.454125:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.454125:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677448.454128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.454130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.454135:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677448.454136:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677448.454137:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.454138:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677448.454138:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:22.0:1619677448.454139:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:18.0:1619677448.454139:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.454140:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.454142:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.454144:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.454154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.454157:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.454159:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.454162:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.454163:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.454165:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.454166:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.454168:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714722] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.454169:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.454170:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.454171:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:21.2:1619677448.454174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677448.454174:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.454176:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.454179:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.454180:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.454180:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.454182:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.454182:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.454183:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.454183:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.454184:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677448.454185:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.454186:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.454188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.454189:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.454189:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.454190:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.454192:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.454193:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.454194:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.454195:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.454195:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:31.0:1619677448.454198:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.454198:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.454199:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.454201:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:13.0:1619677448.454201:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.454201:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.454203:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.454204:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.454204:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.454205:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:4.2:1619677448.454205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.454206:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.454207:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.454208:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.454208:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.454212:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:25.2:1619677448.454212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.454215:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.454216:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.454218:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.454218:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:26.0:1619677448.454220:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.454220:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.454221:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.454222:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677448.454222:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.454223:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.454224:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:26.0:1619677448.454225:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.454225:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677448.454226:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.454226:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.454230:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.454231:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.454234:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.454235:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.454236:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.454238:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.454238:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.454239:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.454239:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.454244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.454251:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.454254:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.454255:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.454256:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.454258:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.454261:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.454278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677448.454285:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.454288:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677448.454289:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677448.454291:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.454300:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.454301:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.454303:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.454304:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.454305:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.454306:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.454309:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.454311:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.454313:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.454314:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.454314:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.454315:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.454316:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.454317:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.454317:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677448.454322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.454329:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.454332:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.454332:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.454334:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.454336:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.454337:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.454338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.454345:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.454348:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.454348:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.454349:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.454349:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.454350:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.454358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.454366:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.454369:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.454370:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.454371:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.454374:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.454375:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.454376:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.454377:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.454378:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.454379:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.454382:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.454383:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.454385:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.454385:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.454385:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.454387:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.454387:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.454388:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.454389:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.454393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.454398:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.454400:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.454401:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.454402:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.454403:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.454404:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.454451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.454457:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.454460:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.454461:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.454463:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.454465:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.454467:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.454468:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.454469:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.454470:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.454471:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.454475:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.454476:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.454479:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.454480:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.454480:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.454482:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.454483:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.454484:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.454485:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.454485:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.454488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.2:1619677448.454490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.454495:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:1.0:1619677448.454496:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677448.454498:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677448.454498:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:24.0:1619677448.454499:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.454499:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.454499:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677448.454501:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.454501:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677448.454504:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677448.454555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.454562:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.454565:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.454567:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.454569:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.454570:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.454571:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:31.0:1619677448.454573:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.454574:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.454575:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.454576:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677448.454580:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.454581:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.454584:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.454585:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:31.0:1619677448.454586:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677448.454587:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677448.454588:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.454589:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.454589:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.454594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.454601:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.454604:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.454605:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.454605:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.454607:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.454611:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.454649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.454656:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.454659:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.454660:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.454660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:26.0:1619677448.454661:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.454663:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.454664:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.454666:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.454667:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.454667:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677448.454668:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.454669:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:8.0:1619677448.454669:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.454671:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:26.0:1619677448.454672:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.454672:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.454674:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.454675:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677448.454675:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.454676:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.454676:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:8.0:1619677448.454676:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:26.0:1619677448.454677:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:8.0:1619677448.454677:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96714714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.454678:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.454679:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.454679:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:8.0:1619677448.454679:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.454679:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.454680:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:25.2:1619677448.454683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:8.0:1619677448.454683:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.454685:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.454688:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:8.0:1619677448.454688:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.454688:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.454689:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677448.454690:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.454691:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.454691:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.454691:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677448.454692:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:8.0:1619677448.454692:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.454692:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:10.0:1619677448.454693:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.454694:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:8.2:1619677448.454696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:25.2:1619677448.454703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:1.0:1619677448.454704:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.454707:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.454707:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.454708:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.454709:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000400:00000200:1.0:1619677448.454710:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.454712:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.454712:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.454713:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.454713:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:1.0:1619677448.454713:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677448.454714:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.454737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.454743:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.454746:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.454747:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.454749:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.454751:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.454753:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.454754:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.454755:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.454757:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.454757:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.454761:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.454763:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.454766:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.454767:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.454768:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.454769:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.454770:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.454771:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.454772:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.454772:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.454775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.454778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.454781:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:2.0:1619677448.454783:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:10.0:1619677448.454784:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.454785:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.454785:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.454785:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.454786:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:10.0:1619677448.454787:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.454787:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:10.0:1619677448.454790:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.454804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.454812:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.454815:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.454817:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.454818:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.454820:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.454821:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.454822:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714712] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.454823:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.454824:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.454825:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.454828:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.454830:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.454832:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.454833:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.454833:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.454835:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.454835:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.454837:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.454838:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.454839:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.454840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.2:1619677448.454845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.454849:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.454850:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.454851:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.454853:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.454853:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.454853:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.454854:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.454855:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.454856:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677448.454857:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:21.2:1619677448.454904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.454910:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.454913:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.454913:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.454914:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.454916:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.454917:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.454919:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.454920:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.454920:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.454922:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.454923:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.454925:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.454926:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.454927:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.454929:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.454929:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.454930:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.454931:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.454932:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.454932:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.454935:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.454936:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.454936:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.454937:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.454938:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.454939:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:21.2:1619677448.454940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677448.454940:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)++
00000800:00000200:6.0:1619677448.454941:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (24)--
00000800:00000200:22.0:1619677448.454942:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.454942:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.454943:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.454943:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.454943:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.454945:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:8.2:1619677448.454945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.454946:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.454946:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677448.454947:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677448.454947:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677448.454950:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.454952:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.454952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.454953:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677448.454954:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.454954:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.454956:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:30.0:1619677448.454956:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677448.454956:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.454956:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.454957:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.454957:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96714709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.454958:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.454959:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.454959:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.454959:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.454960:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.454960:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.454961:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.454961:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677448.454965:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.454965:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.454966:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.454967:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.454969:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.454970:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.454971:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.454971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:30.0:1619677448.454972:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.454973:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.454974:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.454974:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677448.454974:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.454977:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:25.2:1619677448.454978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.454979:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.454980:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.454982:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.454983:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677448.454985:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.454985:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:20.0:1619677448.454986:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.454987:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.454987:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:20.0:1619677448.454988:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:13.0:1619677448.454988:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.454991:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.454992:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.454992:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.454994:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677448.454995:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:25.2:1619677448.454996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.454997:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.454998:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.454999:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.455000:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.455001:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.455002:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.455002:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.455003:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.455006:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.455006:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.2:1619677448.455006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:12.0:1619677448.455007:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.455008:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.455009:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.455014:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.455016:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.455017:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.455019:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.455020:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.455021:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.455027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.455035:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.455037:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.455038:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.455039:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.455040:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.455040:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.455161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.455167:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.455169:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.455170:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.455172:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.455175:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.455176:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.455177:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.455178:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.455180:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.455181:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.455184:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.455185:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.455187:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.455188:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.455189:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.455191:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.455191:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.455192:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.455192:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.455197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.455203:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.455206:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.455206:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.455207:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.455208:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.455210:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.455214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.455221:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.455223:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.455224:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.455224:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.455224:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.455225:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.455230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.455237:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.455239:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.455240:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.455242:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.455243:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.455245:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.455246:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.455248:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.455248:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.455250:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.455253:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.455254:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.455257:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.455258:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.455258:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.455260:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.455260:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.455261:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.455262:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.455266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.455269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677448.455274:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:31.0:1619677448.455276:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677448.455277:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.455278:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.455278:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.455279:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:6.0:1619677448.455281:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.455282:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.455283:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.455284:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.455284:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:31.0:1619677448.455286:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.455287:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.455288:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.455289:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.455290:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677448.455299:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.455301:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.455304:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.455305:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.455305:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.455307:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.455308:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.455309:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.455311:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.455311:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677448.455314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.455318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.455322:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.455325:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:30.0:1619677448.455326:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:18.0:1619677448.455326:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.455326:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.455328:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:18.0:1619677448.455328:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.455329:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.455330:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677448.455332:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.455333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.455340:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.455343:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.455344:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.455344:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.455346:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.455346:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.455393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.455398:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.455400:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.455401:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.455403:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.455406:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.455407:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677448.455408:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.455409:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.455411:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.455412:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677448.455415:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.455416:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.455419:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.455419:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677448.455420:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677448.455422:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.455422:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.455423:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.455423:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.455428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.455436:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.455439:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.455440:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.455440:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.455442:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.455444:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.455455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.455463:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.455466:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.455467:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.455469:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.455472:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.455473:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.455475:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.455476:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.455477:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.455478:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.455481:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.455483:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.455486:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.455487:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.455487:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.455489:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.455489:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.455490:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.455490:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.455495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.455501:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.455504:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.455504:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.455506:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.455508:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.455509:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.455714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.455717:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.455719:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.455721:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.455723:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.455724:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.455725:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677448.455727:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714702] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.455728:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.455729:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.455730:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677448.455734:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.455735:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.455738:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.455739:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.455739:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.455741:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.455742:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.455743:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.455744:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.455745:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.455747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.2:1619677448.455750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:4.2:1619677448.455750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677448.455753:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.455756:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.455756:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.455756:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000400:00000200:26.0:1619677448.455757:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.455758:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:26.0:1619677448.455759:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.455759:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.0:1619677448.455759:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.455760:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:24.0:1619677448.455761:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:26.0:1619677448.455762:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:8.0:1619677448.455763:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.455764:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.455767:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.455769:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.455770:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.455771:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.455772:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.455773:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.455777:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.455778:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.455781:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.455782:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.455783:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.455784:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.455785:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.455785:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.455786:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.455790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.455798:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.455801:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.455802:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.455802:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.455805:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.455807:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.455827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.455834:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.455837:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.455839:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.455841:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:4.2:1619677448.455841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:31.0:1619677448.455842:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.455843:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.455845:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.455846:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.455847:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.455848:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.455849:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.2:1619677448.455851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677448.455851:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:31.0:1619677448.455852:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.455852:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.455854:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.455855:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.455856:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.455857:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.455858:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:10.0:1619677448.455858:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.455859:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.455859:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677448.455859:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:31.0:1619677448.455860:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.455860:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.455861:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677448.455861:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:10.0:1619677448.455861:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.455862:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:10.0:1619677448.455862:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.455863:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.455863:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.455864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:10.0:1619677448.455864:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.455868:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.455870:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.455870:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.455871:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:30.0:1619677448.455872:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:26.0:1619677448.455872:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:30.0:1619677448.455873:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96714698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.455873:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.455873:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.455874:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.455874:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:30.0:1619677448.455875:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.455875:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.455876:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.455876:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.455877:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.455877:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.455878:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.455879:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:30.0:1619677448.455880:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.455880:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.455880:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.455880:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.455882:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:25.2:1619677448.455882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.455885:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.455886:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.455886:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:4.2:1619677448.455886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.455888:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.455889:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.455889:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.455890:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.455890:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.455890:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.455890:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.455891:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:31.0:1619677448.455892:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.455893:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:31.0:1619677448.455894:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.455895:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:8.2:1619677448.455895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.455896:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.455896:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.455901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.455903:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.455905:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.455906:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.455908:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.455908:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.455908:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.455911:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.455911:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677448.455912:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.455913:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.455914:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.455915:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.2:1619677448.455915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.455922:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.455925:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.455926:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.455927:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.455927:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.455928:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.455951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.455956:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.455959:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.455959:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.455962:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.455964:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.455966:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.455967:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.455968:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.455970:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.455971:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.455974:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.455976:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.455979:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.455980:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.455981:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.455982:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:21.2:1619677448.455983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:6.0:1619677448.455983:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.455983:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.455984:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.455988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.0:1619677448.455990:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:20.2:1619677448.455993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.0:1619677448.455993:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.455994:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:31.0:1619677448.455995:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.455996:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:8.0:1619677448.455996:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.455998:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677448.455998:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:31.0:1619677448.455999:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.455999:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.456001:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.456001:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.456001:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:5.0:1619677448.456001:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677448.456002:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:8.0:1619677448.456003:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96714696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.456004:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.456004:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.456005:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:8.0:1619677448.456005:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.456006:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677448.456006:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677448.456007:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.456008:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.456009:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.456011:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.456015:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.456016:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.456018:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677448.456019:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.456021:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.456021:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.456022:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677448.456023:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.456024:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677448.456024:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.456024:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.456025:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.456026:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:8.0:1619677448.456026:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.456026:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.456027:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677448.456028:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.456028:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.0:1619677448.456028:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.456030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.2:1619677448.456031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.456034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677448.456037:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677448.456038:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.456039:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:22.0:1619677448.456040:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:20.0:1619677448.456040:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.456040:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:22.0:1619677448.456041:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.456041:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:6.0:1619677448.456042:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.456042:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.456043:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.456044:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.456044:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.456044:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677448.456046:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677448.456047:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.456148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.456156:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.456158:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.456159:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.456161:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.456164:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.456165:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.456166:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.456167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:26.0:1619677448.456168:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.456169:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.456170:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:2.0:1619677448.456172:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:26.0:1619677448.456173:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.456175:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677448.456175:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.456176:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:26.0:1619677448.456178:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.456179:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.456179:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:2.0:1619677448.456179:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.456181:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.456181:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:2.0:1619677448.456181:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.456182:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:2.0:1619677448.456183:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:26.0:1619677448.456184:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.456184:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:2.0:1619677448.456184:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.456185:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.456186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:2.0:1619677448.456186:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.456187:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.2:1619677448.456190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:2.0:1619677448.456191:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.456193:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:2.0:1619677448.456193:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.456195:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.456196:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:2.0:1619677448.456196:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.456196:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677448.456197:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.456197:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:30.0:1619677448.456198:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.456198:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:10.0:1619677448.456199:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:2.0:1619677448.456199:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.456199:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677448.456200:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.456200:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:2.0:1619677448.456200:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:30.0:1619677448.456201:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:26.0:1619677448.456201:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.2:1619677448.456205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.456212:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.456215:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.456216:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.456216:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.456218:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.456221:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.456262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.456270:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.456272:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.456273:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.456275:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.456278:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.456279:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.456280:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.456282:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.456283:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.456284:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.456287:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.456288:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.456291:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.456292:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.456292:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.456298:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.456299:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.456300:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.456300:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:5.2:1619677448.456302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:25.2:1619677448.456305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.456308:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.456311:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.456312:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.456312:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.456314:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:1.0:1619677448.456314:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.456315:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.456316:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.456317:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.456317:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:12.0:1619677448.456319:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:1.0:1619677448.456319:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.456320:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677448.456320:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.456322:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.456323:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.456326:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.456328:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:21.2:1619677448.456330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:1.0:1619677448.456330:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.456331:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.456331:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.456333:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.456334:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.456334:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.456335:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.456338:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:25.2:1619677448.456339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.456340:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677448.456343:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.456345:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.456345:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677448.456347:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:6.0:1619677448.456347:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:31.0:1619677448.456348:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.456350:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.456350:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:31.0:1619677448.456351:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.456351:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.456351:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.456352:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.456353:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.456355:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.456356:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677448.456357:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.456361:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.456361:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.456362:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.456364:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.456364:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677448.456365:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677448.456366:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677448.456371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.456379:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.456382:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.456383:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.456383:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.456385:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.456388:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.456439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.456446:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.456449:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.456452:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.456453:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.456455:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.456456:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.456458:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.456459:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.456460:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.456461:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.456465:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.456466:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.456470:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.456470:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.456471:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.456472:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.456473:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.456474:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.456474:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.456479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.456487:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.456490:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.456491:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.456493:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.456493:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.456496:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.456525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.456533:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.456535:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.456537:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.456539:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.456540:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.456542:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.456543:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714688] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.456544:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.456545:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.456546:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677448.456549:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.456551:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.456554:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.456555:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.456555:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.456557:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.456557:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.456558:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.456559:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.456563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.456570:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.456573:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.456574:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.456574:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.456577:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.456580:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.456592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.456599:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.456602:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.456602:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.456605:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.456607:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.456608:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.456610:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.456611:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.456612:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.456613:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.456616:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.456618:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.456621:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.456621:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.456622:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.456624:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.456624:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.2:1619677448.456624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:26.0:1619677448.456625:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.456626:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.456629:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:25.2:1619677448.456631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.456632:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:22.0:1619677448.456635:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.456635:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.456637:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:22.0:1619677448.456638:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.456639:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.456640:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.456640:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.456641:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.456642:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.456643:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.456644:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.456645:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.456648:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.456650:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.456651:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.456652:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.456653:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.456654:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.456654:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.456654:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.456655:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.456655:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.456656:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.456657:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677448.456658:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.456659:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.456660:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.456661:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.456661:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:25.2:1619677448.456664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:21.2:1619677448.456667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.456670:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:6.0:1619677448.456672:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:26.0:1619677448.456673:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.456673:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.456674:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.456674:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.456675:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.456675:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677448.456676:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.456679:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:25.2:1619677448.456681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000800:00000200:26.0:1619677448.456687:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.456689:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.456690:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.456690:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.456691:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.456692:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677448.456702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.456704:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.456707:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.456709:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.456710:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.456712:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.456713:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677448.456714:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.456716:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.456716:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.456717:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677448.456720:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.456722:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677448.456725:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.456726:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677448.456726:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677448.456728:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677448.456729:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.456730:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.456730:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677448.456734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.456741:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.456743:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.456744:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.456745:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.456747:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677448.456750:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.456754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677448.456759:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.456762:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.456762:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677448.456763:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.456764:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.456764:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677448.456783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.456788:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.456791:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.456793:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.456794:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.456796:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.456798:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:5.0:1619677448.456801:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.456802:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.456804:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:5.0:1619677448.456804:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.456805:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.456806:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.456807:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:5.2:1619677448.456808:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.456808:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.456810:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.456812:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.456814:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:17.0:1619677448.456814:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677448.456814:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.456816:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.456816:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.456817:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.456817:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.456817:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677448.456819:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.456819:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.456820:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.456820:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.456820:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.456821:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:17.0:1619677448.456822:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.456822:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.456823:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:18.0:1619677448.456824:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.456824:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:5.0:1619677448.456824:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:17.0:1619677448.456825:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96714682] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.456826:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:18.0:1619677448.456827:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.456827:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.456828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:17.0:1619677448.456828:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.456829:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.456829:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677448.456830:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.456831:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.456831:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.456833:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.456833:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:17.0:1619677448.456833:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.456833:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677448.456835:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:17.0:1619677448.456835:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.456836:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677448.456836:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:17.0:1619677448.456836:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.456836:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.456837:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677448.456837:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677448.456837:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.456838:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:17.0:1619677448.456838:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:25.2:1619677448.456839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677448.456839:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.456839:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:10.0:1619677448.456839:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.456840:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.456840:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.456840:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.456841:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.456841:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.2:1619677448.456841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000400:00000200:10.0:1619677448.456842:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.456843:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:10.0:1619677448.456843:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.456844:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96714681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.456845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677448.456846:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677448.456846:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.456847:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.456847:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:12.0:1619677448.456848:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:10.0:1619677448.456848:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677448.456849:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.456849:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.456850:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:12.0:1619677448.456851:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.456851:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677448.456852:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.456852:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.456853:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:10.0:1619677448.456854:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677448.456855:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.456855:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.456856:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.456857:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:10.0:1619677448.456857:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.456858:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:1.0:1619677448.456858:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.456859:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.456860:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677448.456861:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.456862:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.456863:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.456863:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.2:1619677448.456866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:9.2:1619677448.456868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.456873:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.456875:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:17.0:1619677448.456876:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.456877:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.456877:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.456878:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.456879:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.456879:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.456879:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677448.456879:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.456881:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.456885:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.456948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677448.456953:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677448.456956:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677448.456957:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677448.456959:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.456962:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.456963:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677448.456964:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.456965:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.456967:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.456968:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677448.456971:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.456973:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677448.456976:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.456977:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677448.456977:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677448.456979:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677448.456979:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677448.456980:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677448.456980:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:25.2:1619677448.456985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.456991:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.456994:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.456995:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.456995:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.456997:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.457000:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:4.2:1619677448.457094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.457100:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.457103:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.457104:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.457105:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.457108:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.457109:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.457110:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.457111:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.457112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:10.0:1619677448.457112:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.457113:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.457116:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.457118:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.457119:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.457120:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.457121:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.457121:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.457122:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:10.0:1619677448.457123:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.457123:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677448.457124:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.457124:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.457124:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.457126:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.457126:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.457128:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:25.2:1619677448.457128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:30.0:1619677448.457129:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.457130:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.457131:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.457132:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.457135:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:30.0:1619677448.457135:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.457136:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.457137:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.457137:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.457138:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.457140:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.457140:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.457141:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.457141:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.457141:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.457143:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.457144:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.457144:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.457145:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.457149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.457158:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.457160:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.457161:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.457163:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.457164:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.457166:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.457263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.457271:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.457274:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.457276:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.457277:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.457279:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.457281:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677448.457282:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.457283:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.457284:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.457285:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.457288:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.457290:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.457293:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.457294:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.457299:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.457300:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.457301:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.457302:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.457302:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.457307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.457316:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.457319:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.457320:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.457320:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.457322:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.457324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.457326:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677448.457327:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.457329:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.457330:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.457332:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.457333:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.457335:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.457336:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.457337:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.457338:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.457339:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.457342:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.457344:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.457346:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.457347:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.457348:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.457350:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.457350:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.457352:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.457353:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.457353:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.457356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.2:1619677448.457358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.457363:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:30.0:1619677448.457364:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677448.457365:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.457365:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.457366:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.457367:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.457368:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.457370:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.457371:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.457373:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.457475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.457483:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.457486:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.457487:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.457489:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.457491:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.457492:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.457494:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.457495:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.457496:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.457497:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.457500:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.457501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:26.0:1619677448.457502:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.457505:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.457505:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.457506:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.457508:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.457508:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677448.457508:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:26.0:1619677448.457509:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.457509:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677448.457511:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.457512:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.457513:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.457514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:24.0:1619677448.457516:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.457518:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.457519:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96714674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.457520:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.457521:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.457521:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:6.0:1619677448.457522:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:24.0:1619677448.457525:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.457525:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:24.0:1619677448.457526:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:22.0:1619677448.457526:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.457526:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.457528:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.457529:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.457530:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.457531:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:6.0:1619677448.457531:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677448.457533:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.457533:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.2:1619677448.457534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:24.0:1619677448.457535:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.457536:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.457537:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.2:1619677448.457538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677448.457541:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:21.2:1619677448.457543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:22.0:1619677448.457544:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.457544:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.457545:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.457546:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.457546:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677448.457548:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.0:1619677448.457550:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:5.0:1619677448.457551:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:8.0:1619677448.457552:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:5.0:1619677448.457552:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.457553:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677448.457553:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:21.2:1619677448.457554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:20.0:1619677448.457554:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.2:1619677448.457555:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:5.0:1619677448.457558:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.457559:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677448.457561:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677448.457561:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.457562:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.457563:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677448.457564:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.457565:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677448.457565:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.457568:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.457569:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.457571:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677448.457571:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.457572:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.457573:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:6.0:1619677448.457574:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.457574:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.457575:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.457576:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677448.457576:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.457577:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.457579:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:6.0:1619677448.457581:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.457583:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.457584:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677448.457585:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:6.0:1619677448.457587:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.457588:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.457588:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:1.0:1619677448.457588:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.457589:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:6.0:1619677448.457590:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:1.0:1619677448.457590:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677448.457591:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.457592:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677448.457592:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.457592:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.457593:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677448.457593:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.457594:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.457596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:25.2:1619677448.457600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.457602:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.457605:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.457606:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.457606:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.457608:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:10.0:1619677448.457608:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.457610:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.457611:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.457611:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:13.0:1619677448.457613:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.457615:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.457615:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:4.2:1619677448.457622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.457629:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.457632:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.457632:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.457634:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.457637:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.457638:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.457639:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.457641:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.457642:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.457643:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.457646:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.457648:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.457651:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.457652:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.457652:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.457654:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.457655:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.457656:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.457656:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.457660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.457668:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.457670:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.457671:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.457673:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.457674:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.457676:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.457677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.457684:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.457687:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.457688:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.457689:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.457689:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.457690:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.457741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.457747:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.457751:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.457751:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.457754:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.457757:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.457758:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.457759:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.457761:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.457762:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.457763:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.457766:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.457768:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.457769:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.457770:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.457771:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.457772:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.457773:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.457773:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.457774:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.457778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.457785:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.457788:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.457788:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.457789:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.457790:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.457792:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.457846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.457853:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.457855:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.457856:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.457858:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.457862:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.457863:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.457864:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.457866:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.457867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:8.0:1619677448.457867:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.457868:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.457871:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.457873:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.457874:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:8.0:1619677448.457875:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.457876:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.457877:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:8.0:1619677448.457877:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677448.457879:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.457879:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.457879:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.457879:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.457880:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:30.0:1619677448.457881:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.457882:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.457883:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.457884:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.457885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677448.457886:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.457887:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.457887:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.457891:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.457893:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.457893:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.457896:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.457896:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.457896:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.457897:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:17.0:1619677448.457897:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.457898:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:17.0:1619677448.457899:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.457900:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.457900:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.457902:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:17.0:1619677448.457902:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677448.457904:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.457905:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.457906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.457910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.457914:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.457916:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.457917:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.457918:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.457919:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.457919:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.457920:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.457921:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.457922:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.457922:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.458055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.458061:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.458063:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.458064:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.458066:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.458069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:1.0:1619677448.458069:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.458070:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677448.458072:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677448.458072:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714667] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.458073:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.458074:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.458075:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:1.0:1619677448.458075:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.458078:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.458078:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:1.0:1619677448.458079:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.458080:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.458081:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.458082:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677448.458083:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.458085:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.458085:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.458086:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.458086:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.458086:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.458087:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.458088:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677448.458091:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.458092:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.458093:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.458093:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.458093:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.458094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:4.2:1619677448.458094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677448.458097:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.458098:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677448.458098:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.458100:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677448.458101:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677448.458101:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.458101:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.458102:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.458102:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.458103:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.458104:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.458104:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.458105:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:12.0:1619677448.458106:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.458106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:13.0:1619677448.458107:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.458108:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.458110:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677448.458110:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.458112:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.458113:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.458114:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:10.0:1619677448.458114:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.458116:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:10.0:1619677448.458116:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.458116:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677448.458117:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.458117:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.458120:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.458121:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.458122:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:10.0:1619677448.458123:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.458126:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.458127:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.458128:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.458130:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.458130:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.458131:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.458133:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.458133:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.458136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.458139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.458143:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.458146:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.458146:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.458147:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.458148:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677448.458148:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.458149:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.458149:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.458150:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.458153:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.458200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.458208:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.458211:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.458213:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.458215:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.458216:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.458218:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677448.458219:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.458221:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.458222:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.458223:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677448.458226:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.458227:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677448.458230:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.458231:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:30.0:1619677448.458231:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:30.0:1619677448.458233:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:30.0:1619677448.458233:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.458235:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.458236:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.458237:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677448.458238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:21.2:1619677448.458242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677448.458245:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.458247:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.458248:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.458249:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.458250:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677448.458251:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.458252:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.458253:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677448.458253:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:31.0:1619677448.458254:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677448.458436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.458444:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.458447:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.458448:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.458451:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.458451:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.458452:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.458453:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.458455:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.458456:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.458457:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.458459:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.458461:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.2:1619677448.458463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677448.458464:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.458465:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.458465:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.458467:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.458467:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.458468:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.458469:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:1.0:1619677448.458469:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.458472:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.458473:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.458474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:1.0:1619677448.458475:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.458478:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.458479:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.458481:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.458482:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:1.0:1619677448.458482:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.458483:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.458484:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:5.0:1619677448.458485:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.458486:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.458488:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.458488:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.458489:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.458491:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677448.458491:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.458494:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.458495:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.458496:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.458497:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.458498:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.458498:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.458499:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.458504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.458510:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.458513:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.458514:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.458514:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.458516:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.458519:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.458833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.458842:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.458844:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.458845:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.458847:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.458849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:26.0:1619677448.458851:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.458852:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.458854:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.458855:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.458857:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.458857:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:26.0:1619677448.458858:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.458858:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.458861:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.458863:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.458863:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.458865:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.458865:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.458866:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.458867:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.458868:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.458869:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.458872:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.458872:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.458873:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.458874:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.458874:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.458876:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.458877:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.458877:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.458877:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.458878:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.458878:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.458878:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (22)++
00000800:00000200:26.0:1619677448.458879:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.458879:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (23)--
00000800:00000200:10.0:1619677448.458880:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.458881:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.458881:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.458882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.458890:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (23)++
00000800:00000200:5.2:1619677448.458890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:22.0:1619677448.458893:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (24)++
00000800:00000200:6.0:1619677448.458893:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000800:00000200:22.0:1619677448.458894:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000400:00000200:6.0:1619677448.458894:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.458895:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.458896:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:6.0:1619677448.458896:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.458897:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.458899:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:6.0:1619677448.458899:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.458900:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677448.458900:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:6.0:1619677448.458901:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:18.0:1619677448.458902:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.458904:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.458905:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.458907:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.458908:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.458909:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.458910:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.458913:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.458915:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.458918:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.458918:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.458919:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.458921:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.458921:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.458922:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.458923:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.2:1619677448.458925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.2:1619677448.458927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677448.458930:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.458933:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.458934:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.458935:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:17.0:1619677448.458936:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.458938:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.458939:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.458939:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.458940:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:20.0:1619677448.458941:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.458941:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.458941:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.458943:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:17.0:1619677448.458943:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.458944:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.458945:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:8.2:1619677448.458946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000400:00000200:17.0:1619677448.458948:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.458949:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.458952:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.458953:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:5.0:1619677448.458953:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677448.458954:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.458956:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.458956:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.458956:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.458957:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:17.0:1619677448.458958:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:5.0:1619677448.458958:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:17.0:1619677448.458959:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.458959:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.458959:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.458960:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:9.2:1619677448.458962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:20.2:1619677448.458964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:5.2:1619677448.458965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.458968:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.458970:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677448.458971:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.458971:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:26.0:1619677448.458972:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:24.0:1619677448.458972:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.458973:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:24.0:1619677448.458974:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.458974:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.458975:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:1.0:1619677448.458975:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.458976:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:24.0:1619677448.458977:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:26.0:1619677448.458978:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:21.2:1619677448.458979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:26.0:1619677448.458981:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.458983:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.458984:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.458985:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.458986:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.458987:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677448.458987:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.458990:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677448.458991:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000400:00000200:30.0:1619677448.458992:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.458993:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.458994:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.458995:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.458995:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.458996:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:26.0:1619677448.458996:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.458997:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.458998:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.458998:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:30.0:1619677448.458999:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.458999:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:30.0:1619677448.459000:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.459000:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.459000:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:30.0:1619677448.459001:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677448.459004:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.459004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.459005:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677448.459008:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.459008:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:30.0:1619677448.459009:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:30.0:1619677448.459010:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:30.0:1619677448.459011:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.459012:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.459012:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677448.459013:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.459015:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.2:1619677448.459016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:9.2:1619677448.459016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:5.0:1619677448.459016:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.459018:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677448.459020:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.459022:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677448.459022:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:5.0:1619677448.459022:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677448.459025:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:8.0:1619677448.459025:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.459026:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.459026:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.459028:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.459028:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.459028:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.459030:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.459031:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:20.0:1619677448.459032:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.459033:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.459035:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.459035:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.459037:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.459040:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.459042:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.459045:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.459045:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.459046:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.459048:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.459049:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.459050:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.459050:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.459056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.459062:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.459065:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.459066:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.459067:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.459069:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.459070:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.459075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.459082:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.459085:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.459086:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.459086:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.459087:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.459088:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.459150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:22.0:1619677448.459155:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677448.459158:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677448.459159:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677448.459161:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.459164:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.459165:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677448.459167:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.459168:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.459169:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.459170:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677448.459174:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.459175:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677448.459178:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.459178:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:22.0:1619677448.459179:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677448.459181:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:22.0:1619677448.459181:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677448.459182:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677448.459182:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:21.2:1619677448.459187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:8.2:1619677448.459187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677448.459193:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677448.459195:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:2.0:1619677448.459196:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.459197:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:17.0:1619677448.459198:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.459199:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.459199:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.459199:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:2.0:1619677448.459201:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.459202:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.459203:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.459204:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.459205:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:2.0:1619677448.459205:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.459206:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.459207:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.459210:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.459212:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.459213:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.459214:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.459215:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.459217:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.459217:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.459218:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.459218:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.459223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.459230:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.459232:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.459233:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.459234:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.459235:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.459236:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.459248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.459255:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.459258:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.459258:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.459259:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.459259:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.459260:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.459306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.459312:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.459314:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:18.0:1619677448.459316:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.459316:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.459318:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.459319:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.459320:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.459321:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.459322:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.459322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:18.0:1619677448.459323:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.459325:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.459326:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.459327:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.459328:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.459328:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.459328:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.459330:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.459330:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.459331:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.459331:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677448.459331:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.459331:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.459333:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.459336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677448.459336:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.459338:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.459339:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.459340:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.459341:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.459342:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:2.0:1619677448.459343:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.459345:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:1.0:1619677448.459345:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.459346:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.459347:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677448.459347:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:2.0:1619677448.459348:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.459350:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677448.459350:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.459351:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.459352:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.459354:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.459354:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.459355:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.459356:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.459360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.459367:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.459369:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677448.459370:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.459371:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.459373:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.459375:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677448.459396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.459404:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.459406:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.459408:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.459411:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.459411:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.459412:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.459413:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.459415:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.459416:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.459417:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.459420:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.459422:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.459424:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.459425:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.459426:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.459427:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.459428:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.459429:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.459429:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.459433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.459441:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.459443:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.459444:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.459445:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.459447:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.459448:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.459451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.459457:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.459459:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.459460:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.459460:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.459461:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.459462:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.459481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.459488:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.459491:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.459492:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.459493:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.459496:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.459498:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.459499:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.459500:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.459501:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.459502:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.459505:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.459506:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.459509:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.459510:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.459511:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.459513:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.459513:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.459514:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.459514:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.459519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.459527:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.459529:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.459531:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.459531:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.459533:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.459536:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.459625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.459633:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.459636:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.459638:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.459640:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.459640:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.459641:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.459643:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.459644:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.459645:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.459646:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.459649:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.459650:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.459652:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.459653:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.459653:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.459655:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.459655:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.459656:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.459657:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.459660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.459667:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.459669:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.459670:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.459671:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.459673:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.459674:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.459679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.459685:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.459688:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.459688:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.459689:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.459689:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.459690:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.459718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.459724:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.459727:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.459727:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.459729:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.459732:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.459733:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.459734:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.459736:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.459737:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.459738:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.459740:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.459742:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.459745:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.459746:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.459746:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.459748:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.459748:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.459749:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.459749:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.459754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.459760:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.459763:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.459764:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.459764:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.459766:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.459768:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677448.459774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:9.2:1619677448.459777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677448.459782:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.459784:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.459784:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.459785:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000400:00000200:24.0:1619677448.459787:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.459787:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.459788:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677448.459788:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.459789:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.459790:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.459790:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:24.0:1619677448.459792:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.459793:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.459794:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.459795:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.459796:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.459800:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.459802:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.459805:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.459805:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.459806:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.459808:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.459808:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.459810:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.459811:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.459811:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.459814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.459817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.459822:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677448.459824:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.459824:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.459825:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.459826:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677448.459826:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.459827:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.459828:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.459828:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.459831:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.459854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.459861:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.459864:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.459865:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.459867:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.459868:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.459869:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.459871:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.459872:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.459873:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.459874:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.459877:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.459879:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.459882:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.459883:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.459883:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.459885:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.459886:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.459887:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.459887:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.459891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.459899:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.459902:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.459903:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.459904:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.459906:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.459909:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.459957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.459964:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.459966:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.459967:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:4.2:1619677448.459968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677448.459969:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.459972:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.459973:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:26.0:1619677448.459975:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:24.0:1619677448.459975:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.459976:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.459977:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:24.0:1619677448.459977:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.459978:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.459978:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.459980:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.459981:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.459983:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.459985:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.459985:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.459986:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:24.0:1619677448.459986:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:24.0:1619677448.459986:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:26.0:1619677448.459987:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.459988:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.459988:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:24.0:1619677448.459988:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:26.0:1619677448.459989:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.459989:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:26.0:1619677448.459990:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:24.0:1619677448.459990:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:26.0:1619677448.459993:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.459993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.459995:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.459998:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.459999:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.460000:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.460001:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.460001:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.460002:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:10.0:1619677448.460002:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.460003:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.460004:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.460004:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.460005:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.460006:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.460006:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.460008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:25.2:1619677448.460012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:22.0:1619677448.460015:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.460018:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:22.0:1619677448.460018:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.2:1619677448.460018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:22.0:1619677448.460019:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.460019:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.460020:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.460021:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.460022:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000400:00000200:22.0:1619677448.460022:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.460022:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.460023:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.0:1619677448.460024:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677448.460025:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677448.460027:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.460029:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.460030:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.460032:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.460033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000400:00000200:5.0:1619677448.460034:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.460035:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.460037:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.460038:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.460039:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.460041:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.2:1619677448.460042:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677448.460044:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.460044:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.460045:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.460045:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.460046:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:5.0:1619677448.460047:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.460049:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.460050:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.460052:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.460053:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.0:1619677448.460053:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.460054:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.460054:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.460055:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.460056:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.460057:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.460058:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:10.0:1619677448.460059:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.460059:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000400:00000200:10.0:1619677448.460060:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:5.0:1619677448.460060:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:10.0:1619677448.460061:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.460062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.460062:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.460063:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.460064:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:20.2:1619677448.460065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.460068:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:10.0:1619677448.460068:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.460069:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.460069:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677448.460070:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.460071:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.460072:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.460072:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:10.0:1619677448.460072:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.460072:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:13.0:1619677448.460073:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.460073:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.460074:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:13.0:1619677448.460075:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.460075:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.460076:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.460076:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:13.0:1619677448.460077:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.460077:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.460078:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.460080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.2:1619677448.460083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.460087:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.460090:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.460090:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.460091:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:31.0:1619677448.460092:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.460092:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.460093:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.460094:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.460094:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677448.460095:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.460147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.460149:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.460152:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.460154:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.460155:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.460157:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.460159:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677448.460160:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.460161:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.460163:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.460164:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677448.460167:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.460169:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.460172:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.460173:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.460173:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.460175:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.460176:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.460177:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.460178:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.460181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.460187:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.460190:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.460191:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.460191:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.460193:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.460196:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.460219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.460226:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.460229:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.460229:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.460231:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.460233:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.460235:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.460236:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.460237:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.460238:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.460239:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.460241:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.460243:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.460245:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.460245:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.460246:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.460247:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.460248:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.460249:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.460250:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.460250:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.460253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:4.2:1619677448.460256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.460261:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.460262:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.460262:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.460264:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:10.0:1619677448.460264:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.460265:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.460265:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.460266:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.460266:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.460267:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677448.460271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.460277:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.460280:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.460281:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.460283:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.460285:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.460287:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.460288:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.460289:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.460290:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.460291:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:21.2:1619677448.460299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:17.0:1619677448.460300:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.460301:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.460304:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.460305:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.460305:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.460306:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677448.460307:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:17.0:1619677448.460307:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.460308:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.460308:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677448.460309:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.460309:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677448.460310:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.460313:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.460314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:22.0:1619677448.460314:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.460315:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.460316:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.460317:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.460318:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.460322:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.460322:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:22.0:1619677448.460324:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:12.0:1619677448.460324:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.460325:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.460327:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.460328:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.460328:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.460329:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:13.0:1619677448.460329:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.460330:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677448.460331:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.460332:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.460332:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.460333:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.460337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.460345:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.460348:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.460349:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.460351:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.460352:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.460354:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.460473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.460482:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.460484:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.460485:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.460487:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.460489:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.460490:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.460491:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.460492:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.460494:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.460495:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.460498:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.460499:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.460502:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.460503:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.460503:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.460505:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.460505:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.460506:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.460506:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.460511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.460517:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.460519:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.460520:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.460521:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.460523:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.460524:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.460614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:21.2:1619677448.460620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:2.0:1619677448.460620:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.460623:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.460624:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:4.2:1619677448.460624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677448.460626:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.460627:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:2.0:1619677448.460628:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.460630:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:2.0:1619677448.460630:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:26.0:1619677448.460631:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677448.460631:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:2.0:1619677448.460632:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.460634:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:8.0:1619677448.460634:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.460634:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.460635:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677448.460635:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.460636:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.460636:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.460637:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.460638:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.460640:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.460640:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96714634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.460640:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.460642:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677448.460642:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.460643:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.460643:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.460643:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:8.0:1619677448.460644:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.460645:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.460645:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.460646:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.460646:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:26.0:1619677448.460647:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.460647:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:8.0:1619677448.460649:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.460649:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.460649:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.460650:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.0:1619677448.460651:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.460651:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:26.0:1619677448.460652:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.460654:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677448.460654:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677448.460655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677448.460655:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.460655:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.460656:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.460657:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677448.460657:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.460658:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (23)++
00000800:00000200:8.0:1619677448.460658:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:26.0:1619677448.460659:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (24)--
00000800:00000200:8.0:1619677448.460659:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.460659:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:26.0:1619677448.460660:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.460661:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.460661:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.460662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677448.460663:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:22.0:1619677448.460666:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (24)++
00000800:00000200:6.0:1619677448.460666:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:22.0:1619677448.460667:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000400:00000200:6.0:1619677448.460667:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.460668:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.460669:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:6.0:1619677448.460669:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.460670:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.460671:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.460672:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.460673:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.460673:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:12.0:1619677448.460674:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.460674:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677448.460675:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.460676:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.460681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:13.0:1619677448.460687:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.460690:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.460690:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.460691:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.460692:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.460692:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.460725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.460732:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.460735:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.460737:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.460738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:30.0:1619677448.460739:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.460740:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.460741:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.460742:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.460743:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.460744:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.460745:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.460746:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.460746:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.2:1619677448.460748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:5.0:1619677448.460750:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.460752:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.460754:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.460754:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.460754:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.460755:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677448.460756:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.460756:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.460757:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.460757:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:5.0:1619677448.460757:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.460758:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.460759:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677448.460759:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.460760:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:30.0:1619677448.460760:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.460760:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.460761:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:5.0:1619677448.460761:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.460763:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.460763:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.460763:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.460765:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.460765:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:25.2:1619677448.460765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677448.460765:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.460767:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96714630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.460768:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.460769:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.460769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:30.0:1619677448.460770:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.460771:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:13.0:1619677448.460772:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.460773:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.460774:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)++
00000400:00000200:30.0:1619677448.460775:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.460775:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:5.0:1619677448.460775:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (24)++
00000400:00000200:13.0:1619677448.460776:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.460777:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.460777:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.460778:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.460778:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:13.0:1619677448.460778:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.460778:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.460779:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.460779:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.460780:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.460780:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.460780:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.460780:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.460781:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.460781:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.460782:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:25.2:1619677448.460783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.460784:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.460784:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.460785:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.460786:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677448.460789:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.2:1619677448.460790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:12.0:1619677448.460792:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.460793:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.460794:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.460796:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.460797:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677448.460798:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:25.2:1619677448.460800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.460801:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.460802:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.460804:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.460804:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:13.0:1619677448.460806:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.460807:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677448.460809:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.460809:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.460810:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.460811:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.460812:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.461040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.461048:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.461051:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.461051:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.461053:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.461054:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.461055:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.461058:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.461058:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.461059:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.461061:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714629] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.461063:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.461064:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.461065:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.461066:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.461068:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.461068:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.461069:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.461070:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.461070:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:4.2:1619677448.461070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:26.0:1619677448.461071:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.461074:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.461075:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.461075:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.461076:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.461077:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.461077:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677448.461078:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.461078:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)++
00000800:00000200:10.0:1619677448.461079:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (24)--
00000800:00000200:26.0:1619677448.461080:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.461080:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:10.0:1619677448.461080:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.461081:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.461081:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677448.461082:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:24.0:1619677448.461082:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.461082:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.461083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.461084:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.461084:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.461085:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:24.0:1619677448.461085:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.461086:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:24.0:1619677448.461086:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.461087:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96714627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.461088:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.461089:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.461090:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677448.461090:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:9.2:1619677448.461090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677448.461092:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:24.0:1619677448.461093:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677448.461093:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.461094:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.461095:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677448.461096:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677448.461097:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.461097:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677448.461098:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.461098:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677448.461099:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.461100:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.461101:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.461101:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677448.461101:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:24.0:1619677448.461102:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677448.461102:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.461102:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.461104:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.461106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.461107:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677448.461114:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.461117:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:21.2:1619677448.461118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:17.0:1619677448.461118:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.461118:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.461121:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.461123:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677448.461124:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.2:1619677448.461124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.461126:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.461127:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.461129:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.461132:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.461133:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:17.0:1619677448.461133:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:22.0:1619677448.461135:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.461135:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.461136:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.461136:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.461136:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:22.0:1619677448.461137:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.461137:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000400:00000200:22.0:1619677448.461138:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.461138:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:22.0:1619677448.461142:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.461143:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.461146:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.461147:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.461147:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.461149:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.461150:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.461150:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.461151:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.461155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.461163:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.461166:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.461166:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.461168:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.461169:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.461170:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.461246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.461252:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.461255:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.461256:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.461257:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.461260:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.461262:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.461263:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.461265:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.461266:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.461267:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.461270:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.461271:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.461274:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.461275:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.461276:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.461278:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.461278:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.461279:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.461279:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.461284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.461290:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.461293:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.461293:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.461300:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.461302:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.461305:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.461321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.461327:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.461330:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.461331:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.461333:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.461335:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.461337:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.461338:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.461339:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.461340:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.461341:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.461345:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.461347:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.461349:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.461350:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.461351:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.461352:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.461353:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.461354:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.461355:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.461356:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.461359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.461361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.461366:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.461367:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677448.461367:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.461368:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.461369:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.461370:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.461371:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.461371:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.461373:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.461375:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.461424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.461432:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.461435:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.461436:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.461437:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.461440:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.461441:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.461442:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.461444:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.461445:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.461446:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.461450:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.461451:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.461454:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.461455:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.461455:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.461457:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.461457:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.461458:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.461459:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.461463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.461470:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.461473:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.461474:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.461474:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.461476:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.461479:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.461487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677448.461494:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.461497:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677448.461497:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677448.461499:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.461502:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.461504:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:18.0:1619677448.461505:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714622] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.461507:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.461507:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.461509:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:18.0:1619677448.461512:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.461514:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:18.0:1619677448.461517:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.461518:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:18.0:1619677448.461518:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:18.0:1619677448.461520:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:18.0:1619677448.461521:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.461522:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677448.461523:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677448.461523:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:9.2:1619677448.461526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:21.2:1619677448.461529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:8.0:1619677448.461533:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:2.0:1619677448.461534:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:8.0:1619677448.461536:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:2.0:1619677448.461536:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:24.0:1619677448.461537:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677448.461537:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.461537:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677448.461539:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:8.0:1619677448.461540:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.461543:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677448.461688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.461693:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.461696:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.461697:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.461699:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.461701:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.461702:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.461703:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.461705:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.461706:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.461707:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.461711:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.461712:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.461715:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.461716:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.461716:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.461718:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.461719:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.461720:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.461721:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.461721:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.2:1619677448.461723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:8.2:1619677448.461724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.461727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677448.461729:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:22.0:1619677448.461732:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677448.461732:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.461733:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677448.461733:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677448.461735:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:18.0:1619677448.461735:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.461736:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677448.461736:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:22.0:1619677448.461737:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677448.461737:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.461737:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.461738:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.461739:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677448.461740:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.461741:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.461742:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.461742:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:18.0:1619677448.461743:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.461744:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.461747:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.461749:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.461751:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.461752:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.461753:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677448.461754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677448.461754:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.461754:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.461755:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.461756:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.461760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.461762:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.461765:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.461767:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.461767:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.2:1619677448.461768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.461769:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:31.0:1619677448.461770:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.461770:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:2.0:1619677448.461770:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.461771:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:31.0:1619677448.461772:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.461772:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.461773:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.461773:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.461774:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:31.0:1619677448.461775:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.461776:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.461776:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:31.0:1619677448.461777:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:9.2:1619677448.461777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:1.0:1619677448.461777:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.461779:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.461781:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.461782:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.461783:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:31.0:1619677448.461784:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:17.0:1619677448.461784:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.461785:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:17.0:1619677448.461785:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.461786:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.461788:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.461788:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677448.461788:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677448.461789:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.461789:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.461789:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.461789:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.461790:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.461791:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.461791:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:17.0:1619677448.461791:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.461792:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.461794:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:17.0:1619677448.461794:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677448.461795:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.461796:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677448.461796:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.461797:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:8.2:1619677448.461797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677448.461798:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.461799:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.461800:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.461800:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.461801:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677448.461802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:25.2:1619677448.461805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:21.2:1619677448.461809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677448.461810:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.461812:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677448.461813:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:20.0:1619677448.461813:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.461813:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:30.0:1619677448.461815:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:20.0:1619677448.461815:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.461815:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:30.0:1619677448.461816:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:22.0:1619677448.461816:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.461817:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.461817:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.461819:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.461819:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:6.0:1619677448.461819:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.461820:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677448.461821:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.461821:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:6.0:1619677448.461821:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:30.0:1619677448.461823:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.461826:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.461826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000400:00000200:30.0:1619677448.461827:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.461828:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.461830:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.461831:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.461832:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:22.0:1619677448.461833:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:30.0:1619677448.461835:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.461836:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.461836:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.461837:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:30.0:1619677448.461838:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:22.0:1619677448.461838:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.461838:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:30.0:1619677448.461840:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.461841:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.461842:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.461843:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.461844:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.461845:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.461846:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.461849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.461855:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.461858:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.461859:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.461859:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.461861:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.461861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.461865:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.461866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.461867:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.461869:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.461869:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.461870:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.461871:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.461872:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.461872:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.461872:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.461873:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.461874:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:10.0:1619677448.461874:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.461876:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.461877:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714616] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.461879:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.461880:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.461881:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.461884:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.461885:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.461888:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.461889:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.461890:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.461892:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.461892:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.461893:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.461895:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.461895:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677448.461898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.461901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.461905:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.461908:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:2.0:1619677448.461908:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.461909:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.461909:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.461910:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.461911:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:2.0:1619677448.461911:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.461912:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677448.461915:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.462087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.462095:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.462098:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.462098:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677448.462100:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.462103:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.462105:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.462106:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.462108:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.462109:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.462109:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.462113:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.462114:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.462117:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.462118:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.462118:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.462120:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.462120:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.462121:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677448.462122:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.462123:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677448.462126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677448.462128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:1.0:1619677448.462133:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:26.0:1619677448.462136:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:1.0:1619677448.462136:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:26.0:1619677448.462137:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:17.0:1619677448.462137:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.462137:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.462138:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.462139:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:1.0:1619677448.462140:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:1.0:1619677448.462142:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.462183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.462188:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.462191:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.462192:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.462194:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.462195:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.462197:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.462198:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.462199:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.462200:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.462201:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.462204:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.462206:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.462209:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.462210:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.462210:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.462212:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.462212:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.462214:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.462215:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.462215:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.462218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.462220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.462225:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:17.0:1619677448.462226:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677448.462227:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:17.0:1619677448.462228:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:2.0:1619677448.462228:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677448.462229:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.462229:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677448.462230:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.462231:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.462233:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.462251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.462257:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.462259:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.462260:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.462262:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.462264:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.462266:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.462267:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.462268:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.462269:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.462270:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.462273:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.462275:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.462278:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.462279:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.462280:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.462282:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.462282:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.462283:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.462283:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.462288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.462301:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.462304:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.462305:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.462307:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.462309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:12.0:1619677448.462309:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.462310:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.462314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.0:1619677448.462319:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.462320:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.462321:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677448.462323:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.0:1619677448.462324:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677448.462324:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677448.462324:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.462327:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.462327:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.462329:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677448.462329:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.462330:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.462331:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.462332:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.462333:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.462334:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.462335:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.462336:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677448.462336:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.462337:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.462341:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.462342:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.462344:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.462345:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.462347:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.462347:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.462348:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:1.0:1619677448.462348:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.462349:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677448.462349:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.462350:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677448.462350:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.462351:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.462351:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.462352:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677448.462352:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.462353:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677448.462353:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.462357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.2:1619677448.462357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.462364:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677448.462366:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677448.462367:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.462367:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.462368:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:1.0:1619677448.462368:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.462369:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.462369:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.462370:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.2:1619677448.462370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:17.0:1619677448.462371:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:6.0:1619677448.462371:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.462374:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.462376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.462378:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677448.462380:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:26.0:1619677448.462381:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:22.0:1619677448.462383:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:10.0:1619677448.462384:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.462385:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.462386:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.462387:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000400:00000200:10.0:1619677448.462387:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.462388:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.462388:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:10.0:1619677448.462389:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677448.462390:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.462391:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.462392:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.462393:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.462397:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.462399:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.462402:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.462402:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.462403:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.462405:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.462405:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677448.462407:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75ca00] (22)++
00000800:00000200:10.0:1619677448.462408:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (23)--
00000800:00000200:10.0:1619677448.462409:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:8.2:1619677448.462411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.462414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:20.0:1619677448.462419:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:26.0:1619677448.462421:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)++
00000800:00000200:20.0:1619677448.462421:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.462422:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.462423:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75ca00] (23)--
00000800:00000200:26.0:1619677448.462423:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:20.0:1619677448.462424:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.462425:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:5.0:1619677448.462425:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.462427:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.462431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:8.2:1619677448.462431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.462437:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.462440:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:5.0:1619677448.462440:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.462441:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.462443:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.462443:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.462444:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.462444:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000400:00000200:6.0:1619677448.462445:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.462445:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.462446:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:6.0:1619677448.462446:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.462448:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.462449:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.462450:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.462451:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.462455:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.462456:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.462460:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.462460:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.462461:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.462463:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.462463:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.462464:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.462465:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.462469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.462475:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.462478:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.462478:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.462479:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.462481:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.462484:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677448.462550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.462555:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.462558:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.462560:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.462560:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.462562:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.462564:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.462565:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.462566:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.462568:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.462568:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.462572:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.462574:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.462577:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.462577:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.462578:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.462580:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.462581:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.462581:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.462582:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677448.462586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.462595:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.462597:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:4.2:1619677448.462598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:18.0:1619677448.462599:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.462599:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.462601:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.462604:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:26.0:1619677448.462605:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.462608:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.462608:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.462610:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.462614:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.462615:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.462617:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.462618:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.462619:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.462620:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.462623:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.462625:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.462628:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.462629:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.462630:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.462632:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.462632:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.462633:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.462633:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.462638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.462645:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.462648:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.462649:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.462650:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.462651:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.462654:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.462807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677448.462815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:31.0:1619677448.462816:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.462818:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.462821:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.462821:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:31.0:1619677448.462823:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.462823:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:2.0:1619677448.462824:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:31.0:1619677448.462825:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:18.0:1619677448.462825:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:31.0:1619677448.462827:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.462828:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.462829:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.462830:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.462831:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.462831:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.462833:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.462834:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.462835:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.462836:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.462837:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.462838:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.462838:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.462841:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.462841:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.462842:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:2.0:1619677448.462842:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.462843:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.462844:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677448.462844:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.462845:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.462845:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:2.0:1619677448.462847:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.462847:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.462848:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:25.2:1619677448.462849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677448.462850:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.462850:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.462851:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.462851:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677448.462855:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.2:1619677448.462856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677448.462858:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.462859:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.462860:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.462862:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.462863:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677448.462863:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.462866:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.462867:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.462869:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.462869:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.462871:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.462886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.462893:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.462896:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.462896:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.462899:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.462902:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.462903:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.462905:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.462906:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.462907:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.462908:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.462911:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.462913:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.462915:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.462916:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.462917:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.462918:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.462919:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.462919:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.462920:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.462924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.462931:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.462933:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.462934:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.462935:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.462937:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.462938:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.462957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.462964:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.462967:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.462969:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.462971:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.462971:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.462972:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677448.462973:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.462974:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.462975:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.462976:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677448.462979:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.462980:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677448.462983:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.462984:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:30.0:1619677448.462984:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:30.0:1619677448.462986:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:30.0:1619677448.462986:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.462987:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.462988:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.462992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.462998:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.463000:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.463001:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.463001:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.463003:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.463005:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.463014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677448.463020:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.463022:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.463023:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.463023:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.463024:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.463024:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.463042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.463048:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.463050:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.463051:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.463053:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.463055:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.463057:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.463058:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714602] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.463059:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.463060:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.463061:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.463064:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.463065:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.463068:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.463069:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.463069:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.463071:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.463071:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.463072:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.463072:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.463077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.463085:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.463088:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.463089:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.463089:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.463091:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.463093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.463097:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.463098:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.463101:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.463101:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.463102:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.463103:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.463103:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.463181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.463186:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.463189:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.463189:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.463192:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.463195:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.463196:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.463198:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.463199:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.463200:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.463201:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.463204:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.463206:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.463208:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.463209:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.463210:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.463211:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.463212:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.463213:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.463213:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.463218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.463225:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.463228:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.463229:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.463229:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.463231:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.463234:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.463252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.463258:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.463260:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.463261:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.463263:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.463264:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.463266:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.463267:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.463268:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.463269:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.463270:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.463273:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.463275:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.463278:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.463279:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.463279:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.463281:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.463282:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.463283:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.463283:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.463287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.463294:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.463302:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.463303:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.463303:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.463304:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.463307:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.463439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:31.0:1619677448.463448:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677448.463451:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:31.0:1619677448.463453:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.463455:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.463456:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:31.0:1619677448.463457:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:31.0:1619677448.463458:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.463459:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.463460:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.463461:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677448.463465:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.463466:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.463469:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.463470:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:31.0:1619677448.463471:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677448.463472:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677448.463473:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677448.463474:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871380600] (22)++
00000800:00000200:31.0:1619677448.463476:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (23)--
00000800:00000200:31.0:1619677448.463476:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:8.2:1619677448.463479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.463482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (21)++
00000800:00000200:1.0:1619677448.463488:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:30.0:1619677448.463490:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (22)++
00000800:00000200:17.0:1619677448.463491:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.463491:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.463492:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.463493:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871380600] (23)--
00000800:00000200:30.0:1619677448.463493:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677448.463494:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000400:00000200:1.0:1619677448.463494:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.463496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.463499:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.463500:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.463502:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.463503:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.463504:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.463504:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.463505:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.463525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.463533:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.463536:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.463536:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.2:1619677448.463537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:26.0:1619677448.463538:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.463541:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.463542:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.463543:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.463544:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677448.463545:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.463546:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.463547:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:26.0:1619677448.463547:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.463550:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.463551:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677448.463551:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.463553:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.463553:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.463555:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.463556:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.463556:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.463557:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:30.0:1619677448.463558:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.463558:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.463558:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.463559:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:30.0:1619677448.463560:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.463560:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.463561:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.463562:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000400:00000200:30.0:1619677448.463563:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.463563:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.463564:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.2:1619677448.463565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.463567:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.463567:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.463568:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.463569:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:4.2:1619677448.463569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.463570:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.463570:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.463571:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677448.463574:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:25.2:1619677448.463576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.463576:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:20.0:1619677448.463577:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.463578:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.463578:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:20.0:1619677448.463579:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.463579:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.463580:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.463581:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.463582:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677448.463583:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.463585:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.463586:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.463587:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.463589:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.463590:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.463699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.463705:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.463709:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.463710:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.463711:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.463714:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.463715:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.463716:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.463718:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.463719:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.463720:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.463723:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.463725:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.463728:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.463729:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.463729:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.463731:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.463731:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.463732:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.463732:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.463737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.463745:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.463747:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.463748:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.463749:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.463750:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.463752:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.463771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.463778:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.463781:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.463783:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.463785:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.463786:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.463787:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.463789:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.463790:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.463791:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.463792:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.463796:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.463797:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.463800:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.463801:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.463801:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.463803:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.463804:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.463805:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.463805:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.463809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.463815:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.463817:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.463818:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.463819:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.463821:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.463822:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.463826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.463832:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.463835:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.463836:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.463836:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.463837:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.463838:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.463984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.463991:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.463994:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.463996:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.463998:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.463998:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.464000:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.464001:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.464002:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.464003:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.464004:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.464007:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.464009:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.464012:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.464013:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.464013:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.464015:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.464016:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.464016:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.464017:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.464021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.464028:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.464031:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.464032:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.464033:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.464034:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.464036:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.464043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.464048:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.464050:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.464053:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.464053:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.464055:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.464057:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.464058:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.464059:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.464061:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.464062:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.464065:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.464066:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.464069:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.464070:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.464071:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.464073:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.464073:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.464074:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.464075:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.464079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.464085:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.464088:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.464089:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.464091:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.464092:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.464094:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.464100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.464106:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.464109:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.464110:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.464112:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.464114:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.464115:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:5.2:1619677448.464115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:18.0:1619677448.464117:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.464118:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.464119:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.464120:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.464121:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677448.464122:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.464124:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.464124:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.464125:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.464126:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.464127:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.464128:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:17.0:1619677448.464128:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.464129:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.464130:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:17.0:1619677448.464130:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.464131:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.464132:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.464132:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:17.0:1619677448.464132:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.464133:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.464134:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.464135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:17.0:1619677448.464135:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.464136:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:9.2:1619677448.464136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.464139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.0:1619677448.464139:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:17.0:1619677448.464140:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.464141:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:17.0:1619677448.464142:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:20.0:1619677448.464144:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.464144:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:17.0:1619677448.464145:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.464145:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:2.0:1619677448.464145:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:17.0:1619677448.464146:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.464146:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:2.0:1619677448.464146:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.464147:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.464147:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.464148:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.464148:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677448.464148:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000400:00000200:20.0:1619677448.464149:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:17.0:1619677448.464149:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677448.464149:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.464150:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96714590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.464150:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.464150:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677448.464150:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.464151:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.464151:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.464151:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:20.0:1619677448.464152:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.464153:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.464154:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:9.2:1619677448.464154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.2:1619677448.464156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:20.0:1619677448.464157:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.464159:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.0:1619677448.464160:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:20.0:1619677448.464162:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.464162:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:1.0:1619677448.464162:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:20.0:1619677448.464163:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:8.0:1619677448.464163:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.464164:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.464164:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.464164:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:20.0:1619677448.464165:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.464165:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677448.464165:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.464165:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677448.464166:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.464167:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.464168:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.464169:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.0:1619677448.464170:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:25.2:1619677448.464171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.2:1619677448.464174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:26.0:1619677448.464177:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.464179:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:5.0:1619677448.464179:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:10.0:1619677448.464180:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.464181:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.464181:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.464182:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:26.0:1619677448.464183:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.464183:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:26.0:1619677448.464186:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.464217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.464224:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.464227:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.464227:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.464229:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.464232:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.464233:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.464234:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.464235:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.464236:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.464237:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.464240:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.464242:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.464245:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.464245:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.464246:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.464248:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.464248:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.464249:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.464249:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.464264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.464272:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.464275:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.464276:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.464276:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.464278:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.464281:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.464341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.464348:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.464351:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.464353:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.464356:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.464356:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.464357:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.464358:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.464359:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.464360:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.464361:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.464364:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.464365:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.464368:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.464368:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.464369:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.464370:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.464371:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.464372:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.464374:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.464374:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.464377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.464381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.464384:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.464387:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.464387:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.464388:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:12.0:1619677448.464389:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.464390:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:12.0:1619677448.464390:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.464391:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.464392:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.464392:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.464511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.464517:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.464520:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.464521:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.464523:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.464525:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.464527:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.464528:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714587] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.464529:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.464530:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.464531:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.464534:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.464535:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.464538:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.464539:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.464540:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.464541:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.464542:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.464542:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.464543:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.464547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.464554:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.464557:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:5.2:1619677448.464557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:24.0:1619677448.464558:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.464558:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677448.464560:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.464563:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:26.0:1619677448.464565:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.464567:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.464568:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.464570:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.464573:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.464574:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.464575:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.464576:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.464577:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.464578:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.464581:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.464582:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.464584:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.464585:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.464585:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.464587:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.464587:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.464588:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.464588:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.464592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.464598:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.464600:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.464600:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.464601:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.464602:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.464603:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.464642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.464648:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.464650:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.464652:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.464653:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.464655:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.464657:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.464658:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.464659:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.464660:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.464661:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.464665:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.464666:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:4.2:1619677448.464667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677448.464669:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.464670:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.464671:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.464673:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677448.464673:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.464673:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.464674:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.464675:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:26.0:1619677448.464676:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:5.0:1619677448.464676:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677448.464677:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.464678:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.464678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:26.0:1619677448.464680:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.464681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677448.464682:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.464683:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.464684:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.464684:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:26.0:1619677448.464685:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.464685:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:26.0:1619677448.464686:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.464686:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.464686:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.464687:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.464688:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.464688:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.464690:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677448.464690:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.464692:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677448.464692:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.464693:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.464695:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.464696:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.464697:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.464699:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.464699:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.464700:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.464700:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.464705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.464713:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.464716:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.464717:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.464717:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.464720:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.464721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.464724:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.464726:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.464728:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.464729:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.464729:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.464730:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.464730:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.464796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.464802:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.464804:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.464805:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.464807:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.464809:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.464811:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.464812:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.464813:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.464814:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.464815:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.464818:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.464820:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.464822:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.464822:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.464823:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.464825:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.464825:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.464826:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.464827:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.464828:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677448.464830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.464833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.464838:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.464839:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.464840:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.464840:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.464840:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.464841:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.464842:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.464842:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.464843:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.464843:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.2:1619677448.464891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.2:1619677448.464897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677448.464897:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.464900:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.464901:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.464903:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.464905:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.464906:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.464907:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:31.0:1619677448.464908:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:17.0:1619677448.464908:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.464910:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.464911:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.464912:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:17.0:1619677448.464912:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.464913:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677448.464915:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.464917:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:17.0:1619677448.464917:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.464918:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.464920:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.464920:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:31.0:1619677448.464921:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.464923:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:17.0:1619677448.464923:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.464923:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.464924:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.464926:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000400:00000200:31.0:1619677448.464927:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.464927:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.464927:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.464928:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.464929:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:9.2:1619677448.464931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:31.0:1619677448.464932:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.464932:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.464933:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.464935:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.464935:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.464937:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:8.0:1619677448.464937:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:31.0:1619677448.464938:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.464939:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.464940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:8.0:1619677448.464940:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.464941:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.464941:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.464943:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.464945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:8.0:1619677448.464946:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:30.0:1619677448.464949:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:9.2:1619677448.464950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:30.0:1619677448.464951:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.464952:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.464953:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:30.0:1619677448.464953:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.464954:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.464954:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.464956:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:24.0:1619677448.464956:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:30.0:1619677448.464957:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677448.464958:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:24.0:1619677448.464959:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677448.464960:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.464960:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.464961:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.464961:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677448.465048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.465055:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.465057:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.465058:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.465060:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.465063:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.465065:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.465066:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.465067:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.465068:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.465069:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.465073:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.465074:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.465077:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.465078:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.465079:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.465081:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.465081:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.465082:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.465082:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.465087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.465094:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.465097:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.465098:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.465098:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.465101:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.465103:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.465139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.465147:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.465150:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.465150:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.465152:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.465155:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.465157:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.465158:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.465159:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.465161:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.465161:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.465165:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.465166:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.465169:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.465170:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.465171:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.465172:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.465173:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.465173:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.465174:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.465179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.465187:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.465189:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.465190:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.465191:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.465193:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.465196:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.465217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677448.465220:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677448.465222:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677448.465224:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.465224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:20.0:1619677448.465227:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.465229:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:5.0:1619677448.465229:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677448.465230:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714578] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.465230:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:20.0:1619677448.465231:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.465233:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.465233:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.465233:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:20.0:1619677448.465234:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677448.465237:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.465238:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.465240:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:1.0:1619677448.465240:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.465242:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:20.0:1619677448.465243:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.465243:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.465244:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:1.0:1619677448.465244:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.465245:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:1.0:1619677448.465245:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.465246:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:20.0:1619677448.465247:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.465248:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677448.465249:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:20.0:1619677448.465250:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000400:00000200:1.0:1619677448.465250:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.465251:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:1.0:1619677448.465252:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:25.2:1619677448.465253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:1.0:1619677448.465254:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677448.465255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:1.0:1619677448.465255:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.465256:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.465258:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.465258:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.465258:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.465259:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:1.0:1619677448.465259:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677448.465261:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000800:00000200:10.0:1619677448.465262:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.465263:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.465263:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.465263:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000800:00000200:9.2:1619677448.465264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.0:1619677448.465264:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:5.0:1619677448.465265:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000400:00000200:10.0:1619677448.465266:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.465268:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.465269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:8.0:1619677448.465270:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.465272:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.465273:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.465274:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.465276:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:8.0:1619677448.465276:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.465278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:8.0:1619677448.465278:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:26.0:1619677448.465279:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.465280:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.465282:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.465284:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.465284:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:26.0:1619677448.465285:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.465287:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.465287:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.465288:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:26.0:1619677448.465290:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.465291:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.465291:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.465292:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.465301:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.465303:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.465303:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.465304:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:26.0:1619677448.465306:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.465306:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:2.0:1619677448.465306:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.465307:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677448.465307:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.465308:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.465309:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.465309:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.465310:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.465310:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.465311:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:2.0:1619677448.465312:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.465314:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:9.2:1619677448.465316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.465317:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.465317:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.465318:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.465321:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.465321:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.465322:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.465323:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677448.465325:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:25.2:1619677448.465327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.465327:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.465327:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.465328:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.465330:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.465333:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677448.465334:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.465337:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.465338:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.465338:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.465340:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677448.465343:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.465348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.465354:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.465356:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.465357:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.465359:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.465361:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.465363:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677448.465364:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.465365:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.465366:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.465367:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677448.465370:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.465371:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.465375:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.465375:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:18.0:1619677448.465376:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.465378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:18.0:1619677448.465378:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677448.465378:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.465379:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.465379:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677448.465383:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:25.2:1619677448.465384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677448.465386:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.465388:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677448.465389:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:12.0:1619677448.465391:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:5.0:1619677448.465391:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.465392:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:12.0:1619677448.465394:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:5.0:1619677448.465394:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714573] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.465395:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.465395:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.465396:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677448.465397:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.465398:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:5.0:1619677448.465398:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.465403:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:25.2:1619677448.465404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677448.465405:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.465406:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.465407:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677448.465408:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.465408:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.465409:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:5.0:1619677448.465409:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.465410:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.465411:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.465411:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:5.0:1619677448.465411:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.465412:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677448.465412:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.465412:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.465413:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.465417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.465424:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.465426:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.465427:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.465429:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677448.465430:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.465432:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.465457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.465464:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.465467:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.465468:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.2:1619677448.465470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:10.0:1619677448.465470:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.465472:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.465474:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.465475:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714572] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.465476:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.465477:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677448.465477:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.465478:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.465480:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.465483:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.465483:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.465484:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.465485:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.465485:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:30.0:1619677448.465486:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.465487:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.465488:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.465489:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.465489:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.465489:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677448.465490:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.465490:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.465491:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.465492:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.465493:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:30.0:1619677448.465494:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.465495:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.465495:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.465495:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677448.465497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.465498:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.465498:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.465499:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.465500:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.465501:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.465501:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:4.2:1619677448.465501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.465502:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.465505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.465505:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.465508:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:2.0:1619677448.465508:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.465509:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:18.0:1619677448.465509:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.465509:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.465510:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.465511:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:12.0:1619677448.465512:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:2.0:1619677448.465512:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.465514:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.465515:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.465515:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.465516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:12.0:1619677448.465517:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.465518:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.465519:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677448.465523:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:25.2:1619677448.465524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.465526:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.465527:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.465527:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.465529:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.465529:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677448.465530:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.465533:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.465534:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.465534:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.465535:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.465537:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.465570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.465573:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.465575:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.465578:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.465578:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.465580:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.465582:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.465583:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.465585:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.465586:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.465587:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.465590:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.465592:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.465595:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.465596:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.465596:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.465598:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.465599:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.465599:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.465600:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.465604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.465611:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.465613:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.465614:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.465616:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.465617:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.465619:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.465762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.465769:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.465772:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.465772:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.465774:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.465778:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.465779:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.465780:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.465781:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.465782:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.465783:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.465786:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.465788:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.465790:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.465791:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.465791:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.465792:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.465793:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.465794:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.465795:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.465796:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.465799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.465802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.465808:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.465809:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.465809:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.465809:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.465810:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.465810:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.465810:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.465812:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.465812:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.465813:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.465855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.465863:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.2:1619677448.465865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:26.0:1619677448.465866:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.465866:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.465868:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.465871:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.465872:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:10.0:1619677448.465873:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:24.0:1619677448.465874:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:10.0:1619677448.465874:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.465875:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:10.0:1619677448.465876:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.465877:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.465877:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.465879:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:24.0:1619677448.465880:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.465882:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.465882:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.465883:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96714567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.465884:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.465885:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:24.0:1619677448.465886:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.465886:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.465888:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.465889:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:24.0:1619677448.465890:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.465890:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.465891:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:24.0:1619677448.465892:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.465892:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.465893:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:24.0:1619677448.465894:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.465894:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:24.0:1619677448.465895:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.465895:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.465895:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677448.465897:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.465897:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.2:1619677448.465898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:24.0:1619677448.465898:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.465899:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.2:1619677448.465900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:25.2:1619677448.465903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677448.465905:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:26.0:1619677448.465907:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677448.465908:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.465909:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.465909:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:26.0:1619677448.465910:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.465910:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677448.465910:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.465911:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.465912:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.465912:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.465912:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.465914:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:13.0:1619677448.465914:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.465915:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.2:1619677448.465916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:13.0:1619677448.465917:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.465924:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.465927:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.465929:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.465931:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.465931:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.465933:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.465934:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.465936:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.465937:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.465938:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.465941:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.465942:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.465946:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.465947:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.465947:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.465948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.465949:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.465950:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.465951:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.465951:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:10.0:1619677448.465955:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677448.465956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.465958:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.465959:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.465960:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.465963:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.465964:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:10.0:1619677448.465965:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.465966:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.465967:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.465967:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:10.0:1619677448.465968:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.465968:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.465969:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.465970:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.465970:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.465972:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.465974:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.465974:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:10.0:1619677448.465976:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.465976:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:8.2:1619677448.465976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.465977:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.465978:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.465979:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.465979:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.465980:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.465984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.465984:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.465986:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.465987:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.465988:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.465989:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.465990:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:5.0:1619677448.465990:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:26.0:1619677448.465992:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.465993:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.465994:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.465995:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.465997:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.466005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.466010:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.466013:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.466013:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.466014:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.466014:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.466015:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.466166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.466172:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.466175:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.466175:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.466177:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.466179:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.466180:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.466181:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.466183:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.466184:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.466185:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.466188:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.466190:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.466192:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.466192:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.466193:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.466194:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.466195:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.466195:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.466196:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.466200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.466208:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.466210:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.466210:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.466211:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.466212:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.466213:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.466239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.466245:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.466247:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.466248:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:21.2:1619677448.466249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:6.0:1619677448.466250:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.466252:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.466254:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.466255:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.466256:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677448.466257:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.466257:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.466258:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.466259:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:6.0:1619677448.466262:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.466263:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677448.466264:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:30.0:1619677448.466266:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.466267:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.466268:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:30.0:1619677448.466269:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.466269:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:30.0:1619677448.466270:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:6.0:1619677448.466271:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.466271:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:30.0:1619677448.466272:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.466272:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.466272:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:30.0:1619677448.466273:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.466274:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.466275:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:25.2:1619677448.466277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:30.0:1619677448.466278:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.466280:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.466283:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.466283:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.466283:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.466284:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.466285:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.466285:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.466286:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.466286:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:31.0:1619677448.466287:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.466287:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.466288:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677448.466289:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.466290:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.466291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.466299:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.466301:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.466302:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.466304:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.466305:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.466307:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.466393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.466399:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.466402:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.466403:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.466405:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.466408:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.466410:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677448.466411:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.466412:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.466413:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.466415:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.466418:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.466420:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.466423:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.466424:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677448.466424:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677448.466426:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677448.466427:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.466427:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.466428:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.466433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.466440:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.466442:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.466443:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.466445:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.466446:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.466448:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.466504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.466510:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.466512:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.466513:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.466515:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.466518:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.466520:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.466521:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.466522:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.466523:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.466524:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.466527:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.466528:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.466530:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.466531:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.466532:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.466534:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.466534:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.466535:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.466535:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:4.2:1619677448.466536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:25.2:1619677448.466539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.466543:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.466546:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:13.0:1619677448.466546:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.466547:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.466548:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.466549:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:26.0:1619677448.466550:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.466550:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.466552:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:13.0:1619677448.466553:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.466553:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:26.0:1619677448.466554:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.466555:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.466556:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.466556:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677448.466557:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.2:1619677448.466560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677448.466561:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.466562:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.466565:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.466566:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.466566:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.466567:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.466568:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.466568:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.466569:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.466569:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677448.466569:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.466570:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.466572:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.466573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:17.0:1619677448.466574:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.466575:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.466576:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.466578:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.466578:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.466580:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.466581:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:17.0:1619677448.466582:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.466583:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.466584:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.466584:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.466586:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.466587:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.466588:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.466588:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.466588:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.466590:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677448.466590:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.466590:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.466591:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.466592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.466592:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.466597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.466599:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.466602:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.466603:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.466604:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.466604:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.466604:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.466606:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:1.0:1619677448.466607:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.466607:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.466608:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.466610:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.466611:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.466671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.466674:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.466676:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.466679:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.466680:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.466681:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.466683:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.466684:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.466685:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.466687:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.466688:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.2:1619677448.466688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:20.0:1619677448.466691:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.466693:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.466696:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.466696:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677448.466697:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.466698:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.466698:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.466699:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677448.466699:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.466700:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.466701:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.466701:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:10.0:1619677448.466701:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.466704:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.466705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.466705:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.466707:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.466708:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.466709:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.466710:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:12.0:1619677448.466711:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677448.466713:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.466714:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.466715:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.466715:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:12.0:1619677448.466716:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.466718:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.466718:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.466719:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.466719:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.466719:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.466721:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.466721:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:25.2:1619677448.466722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.466722:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.466723:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.466727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677448.466728:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.466731:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.466731:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.466732:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.466732:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.466733:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.466735:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.466738:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.466739:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.466739:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.466742:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.466745:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.466865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.466872:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.466875:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.466877:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.466878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:31.0:1619677448.466879:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.466879:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.466881:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.466882:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.466883:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.466884:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.466884:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:30.0:1619677448.466885:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.466887:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.466888:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:30.0:1619677448.466891:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.466891:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.466893:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.466895:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.466895:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.466896:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.466896:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:18.0:1619677448.466896:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:30.0:1619677448.466898:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.466898:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677448.466898:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.466899:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677448.466899:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.466900:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:18.0:1619677448.466900:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.466901:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:25.2:1619677448.466903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:18.0:1619677448.466905:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.466906:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.466909:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:18.0:1619677448.466909:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.466910:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (23)++
00000800:00000200:18.0:1619677448.466911:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (24)++
00000800:00000200:18.0:1619677448.466914:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (25)--
00000800:00000200:18.0:1619677448.466914:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.466914:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (25)--
00000800:00000200:26.0:1619677448.466915:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:18.0:1619677448.466915:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.466916:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:26.0:1619677448.466917:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.466918:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.466919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.466922:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.466924:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.466925:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.466925:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.466926:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.466927:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.466928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.466930:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:17.0:1619677448.466933:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.466936:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.466937:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.466939:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.466940:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.466942:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.466943:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.466944:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.466945:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.466946:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.466949:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.466951:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.466953:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.466954:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.466955:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.466957:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.466957:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.466958:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.466958:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.466963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.466971:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.466973:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.466975:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.466975:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.466976:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.466979:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.467027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677448.467034:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677448.467037:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677448.467038:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677448.467040:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.467042:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.467044:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677448.467045:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.467046:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.467047:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.467048:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677448.467051:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.467052:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677448.467054:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.467055:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677448.467056:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.467057:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.467057:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677448.467058:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677448.467059:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677448.467062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.467068:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.467071:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.467071:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.467072:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.467073:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.467074:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.467248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.467255:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.467258:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.467259:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.467260:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.467262:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.467264:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.467265:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.467266:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.467267:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.467268:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.467271:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.467273:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.467275:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.467275:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.467276:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.467277:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.467278:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.467278:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.467279:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.467283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.467290:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.467293:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.467299:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.467299:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:31.0:1619677448.467300:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.467303:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.467397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.467404:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.467407:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.467407:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.467409:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.467412:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.467413:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.467414:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.467415:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.467416:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.467418:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.467421:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.467423:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.467425:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.467426:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.467426:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.467428:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.467428:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.467429:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.467430:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.467431:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.467435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.467436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.467441:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.467442:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.467443:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.467443:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.467444:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.467446:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.467447:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.467447:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.467449:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.467451:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.467455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:22.0:1619677448.467463:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.467465:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.467466:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.467467:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.467467:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.467468:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.467470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.2:1619677448.467474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:26.0:1619677448.467478:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.467480:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:20.2:1619677448.467481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677448.467481:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:2.0:1619677448.467481:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:21.2:1619677448.467482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:26.0:1619677448.467484:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.467484:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.467484:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:26.0:1619677448.467487:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.467487:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677448.467487:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.467488:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677448.467490:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677448.467490:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.467490:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:2.0:1619677448.467490:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.467491:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677448.467491:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:30.0:1619677448.467493:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:26.0:1619677448.467493:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.467493:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.467495:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.467495:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.467495:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.467496:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677448.467497:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.467497:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.467498:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.467498:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.467498:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.467500:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.467501:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.467503:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.467503:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96714547] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.467504:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.467504:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.467504:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.467505:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96714547] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.467506:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.467507:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.467507:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:6.0:1619677448.467507:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.467507:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.467509:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.467510:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.467510:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.467511:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.467511:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.467511:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.467512:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:6.0:1619677448.467512:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.467512:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.467514:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.467514:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.467515:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.467515:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.467515:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:26.0:1619677448.467516:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.467517:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677448.467517:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:30.0:1619677448.467518:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.467518:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.467518:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.467518:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:6.0:1619677448.467519:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.467519:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (22)++
00000800:00000200:25.2:1619677448.467520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.467521:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.467521:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:8.2:1619677448.467522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (23)++
00000800:00000200:6.0:1619677448.467522:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.467523:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:5.2:1619677448.467523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:30.0:1619677448.467524:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.467524:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:30.0:1619677448.467525:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:30.0:1619677448.467525:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (24)++
00000800:00000200:6.0:1619677448.467525:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677448.467527:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (25)--
00000800:00000200:30.0:1619677448.467528:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677448.467528:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.467529:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:18.0:1619677448.467529:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:30.0:1619677448.467530:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:21.2:1619677448.467530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677448.467530:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677448.467531:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:13.0:1619677448.467531:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.467532:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.467533:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:13.0:1619677448.467533:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.467533:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (24)++
00000800:00000200:13.0:1619677448.467535:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677448.467536:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:12.0:1619677448.467536:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.467536:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (25)--
00000400:00000200:5.0:1619677448.467537:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.467538:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.467539:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677448.467539:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (24)++
00000400:00000200:5.0:1619677448.467539:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.467540:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677448.467540:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (25)--
00000400:00000200:20.0:1619677448.467541:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.467542:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.467542:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (24)--
00000800:00000200:5.0:1619677448.467543:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)++
00000800:00000200:5.0:1619677448.467544:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (24)--
00000800:00000200:20.0:1619677448.467545:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.467545:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.467546:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:5.0:1619677448.467546:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.467547:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.467548:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:20.0:1619677448.467549:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:5.0:1619677448.467549:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.467550:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.467551:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.467683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.467685:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.467688:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.467690:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.467691:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.467692:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.467694:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.467695:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.467696:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.467697:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.467698:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.467702:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.467704:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.467706:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.467707:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.467707:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.467709:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.467709:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.467710:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.467711:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.467715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.467722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:6.0:1619677448.467722:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.467725:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.467726:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.467726:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.467728:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.467729:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:6.0:1619677448.467730:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677448.467732:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677448.467734:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.467735:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677448.467737:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.467739:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677448.467740:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.467741:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.467742:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.467743:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677448.467747:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.467748:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677448.467751:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.467751:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.467752:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.467754:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.467754:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677448.467755:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677448.467756:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677448.467759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.467766:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.467768:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.467769:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.467769:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.467771:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.467773:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.468021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.468028:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.468031:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.468032:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.468033:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.468035:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.468036:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.468037:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.468039:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.468040:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.468041:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.468043:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.468045:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.468048:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.468048:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.468049:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.468050:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.468051:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.468051:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.468052:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.468056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.468064:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.468067:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.468068:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.468068:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.468070:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.468073:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.468076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:21.2:1619677448.468084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677448.468084:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.468086:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.468087:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.468087:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.468088:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:21.2:1619677448.468089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:17.0:1619677448.468089:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677448.468091:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.468094:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677448.468095:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:30.0:1619677448.468096:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.468097:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:31.0:1619677448.468098:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677448.468098:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:30.0:1619677448.468099:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.468101:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:22.0:1619677448.468101:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.468103:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.468103:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.468104:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.468105:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:30.0:1619677448.468106:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.468106:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.468107:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:22.0:1619677448.468108:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.468109:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.468110:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:30.0:1619677448.468112:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.468115:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:22.0:1619677448.468115:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.468117:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:30.0:1619677448.468118:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.468118:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:30.0:1619677448.468119:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:30.0:1619677448.468120:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:22.0:1619677448.468120:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.468121:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677448.468121:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.468122:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:30.0:1619677448.468123:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.468124:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:22.0:1619677448.468124:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.468124:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677448.468125:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677448.468125:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.468126:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.468126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:25.2:1619677448.468130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:21.2:1619677448.468131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677448.468132:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.468134:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.468135:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.468135:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.468136:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:31.0:1619677448.468138:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677448.468138:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.468139:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:31.0:1619677448.468140:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:26.0:1619677448.468140:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.468140:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.468140:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:31.0:1619677448.468141:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.468142:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:10.0:1619677448.468142:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.468143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:10.0:1619677448.468145:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:25.2:1619677448.468149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000800:00000200:24.0:1619677448.468149:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.468151:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.468152:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.468152:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.468153:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.468154:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:26.0:1619677448.468156:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.468158:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.468159:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.468159:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.468160:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.468161:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:4.2:1619677448.468311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677448.468312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:10.0:1619677448.468319:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677448.468320:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:10.0:1619677448.468322:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.468323:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:13.0:1619677448.468323:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677448.468326:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.468326:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.468328:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:10.0:1619677448.468328:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.468329:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.468329:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.468330:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677448.468331:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:5.2:1619677448.468331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677448.468332:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.468333:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.468333:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677448.468334:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.468335:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.468335:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:13.0:1619677448.468336:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.468338:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.468340:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.468341:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677448.468341:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.468341:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:10.0:1619677448.468343:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:13.0:1619677448.468344:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:1.0:1619677448.468344:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.468346:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.468347:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.468347:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:1.0:1619677448.468347:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.468348:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.468348:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:1.0:1619677448.468348:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:13.0:1619677448.468349:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.468349:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:13.0:1619677448.468350:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.468350:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677448.468350:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96714538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.468351:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:10.0:1619677448.468351:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:1.0:1619677448.468351:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.468352:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:10.0:1619677448.468352:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000400:00000200:1.0:1619677448.468352:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.468353:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:10.0:1619677448.468353:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677448.468353:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:25.2:1619677448.468356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.2:1619677448.468357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:1.0:1619677448.468357:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.468358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.468358:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.468362:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.468362:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.468363:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.468364:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.468365:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:1.0:1619677448.468365:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.468366:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:1.0:1619677448.468366:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.468367:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677448.468367:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:1.0:1619677448.468367:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.468367:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.468368:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677448.468368:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677448.468368:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:10.0:1619677448.468369:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.468369:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.468369:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.468371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677448.468371:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.468372:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:6.0:1619677448.468372:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.468374:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:25.2:1619677448.468375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.468378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:12.0:1619677448.468378:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677448.468378:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.468379:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:12.0:1619677448.468380:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.468381:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.468381:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.468382:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.468383:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:12.0:1619677448.468383:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.468383:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.468384:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:26.0:1619677448.468385:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:13.0:1619677448.468385:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.468386:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.468388:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.468388:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.468391:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.468393:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.468395:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.468396:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.468398:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.468399:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.468400:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.468403:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:5.2:1619677448.468403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:26.0:1619677448.468405:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.468408:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.468408:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.468409:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.468409:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:26.0:1619677448.468411:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.468412:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.468412:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:2.0:1619677448.468412:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:26.0:1619677448.468413:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:18.0:1619677448.468413:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.468415:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.468417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:2.0:1619677448.468417:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.468419:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.468420:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.468422:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.468423:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.468424:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.468425:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:2.0:1619677448.468427:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.468428:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:2.0:1619677448.468428:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:20.0:1619677448.468429:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.468431:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.468431:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.468432:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.468432:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.468433:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:2.0:1619677448.468433:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (22)++
00000800:00000200:8.2:1619677448.468435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677448.468436:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.468436:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.468437:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.468438:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677448.468443:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.468446:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.468447:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:5.0:1619677448.468447:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:5.0:1619677448.468448:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.468449:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.468451:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.468452:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.468454:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.468454:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.2:1619677448.468456:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.0:1619677448.468461:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677448.468464:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.468467:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.468470:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.468472:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.468472:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.468474:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.468475:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.468476:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.468477:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.468478:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.468482:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.468483:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.468486:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.468487:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.468487:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.468489:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.468489:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.468491:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.468491:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.468494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.468501:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.468503:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.468504:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.468505:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.468507:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.468508:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.468566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.468572:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.468574:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.468577:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.468577:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.468579:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.468581:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677448.468582:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.468583:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.468584:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.468585:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.468589:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.468591:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.468593:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.468594:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677448.468595:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.468597:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.468598:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.468599:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.468599:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.468603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.468610:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.468612:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.468613:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.468614:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.468616:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.468619:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.468687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677448.468693:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.468696:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677448.468697:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677448.468699:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.468701:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.468702:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.468703:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.468704:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.468705:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.468706:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.468709:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.468710:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.468712:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.468713:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.468714:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.468715:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.468716:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.468716:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677448.468717:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677448.468720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.468726:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.468728:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.468728:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.468729:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.468730:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.468731:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.468738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.468744:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.468746:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.468747:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.468747:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.468748:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.468748:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.468797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.468803:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.468805:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.468806:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.468808:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.468809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:22.0:1619677448.468811:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.468812:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.468813:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.468815:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.468815:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:22.0:1619677448.468816:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.468817:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.468818:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.468819:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:22.0:1619677448.468821:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.468822:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:22.0:1619677448.468824:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.468825:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.468826:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.468826:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:22.0:1619677448.468827:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.468827:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:18.0:1619677448.468828:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.468829:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:18.0:1619677448.468829:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.468830:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.468830:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:18.0:1619677448.468830:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.468831:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:18.0:1619677448.468831:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.468834:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.468836:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.2:1619677448.468836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.468837:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.468838:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.468839:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.468841:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.468841:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.468842:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.468842:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677448.468844:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.468846:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:9.2:1619677448.468847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.468847:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.468848:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.468850:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.468852:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677448.468854:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.468856:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.468857:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.468858:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.468859:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.468860:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.468910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.468918:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.468920:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.468921:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.468923:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.468924:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.468926:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.468927:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.468929:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.468930:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.468931:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.468934:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.468935:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.468938:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.468939:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.468939:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.468941:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.468942:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.468942:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.468943:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.468947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.468954:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.468956:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.468957:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.468958:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.468960:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.468961:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.468964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:20.2:1619677448.468968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:4.2:1619677448.468969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677448.468970:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.468973:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677448.468973:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.468974:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.0:1619677448.468975:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:13.0:1619677448.468975:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.468975:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.468976:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677448.468976:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:20.0:1619677448.468977:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.468978:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:5.0:1619677448.468978:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677448.468979:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677448.468980:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.468981:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.468982:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.468983:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.468984:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.468985:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.468985:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.468986:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.468986:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.468987:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.468989:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.468990:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.468991:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.468992:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.468994:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.468996:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.468997:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:26.0:1619677448.468998:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.469000:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.469001:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.469001:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.469002:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.469002:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.469003:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.469003:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.469003:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.469004:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.469005:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.469005:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677448.469005:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.469006:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.469007:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:30.0:1619677448.469008:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:20.0:1619677448.469008:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:25.2:1619677448.469009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.469009:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677448.469010:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.469012:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:9.2:1619677448.469012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677448.469013:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.469014:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:30.0:1619677448.469014:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:20.2:1619677448.469014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.469015:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.469015:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96714527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.469016:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.469017:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.469017:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.469018:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.469019:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.469019:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677448.469020:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677448.469022:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.469022:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.469022:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:5.0:1619677448.469022:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:31.0:1619677448.469023:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.0:1619677448.469023:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677448.469023:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677448.469024:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:18.0:1619677448.469024:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.469024:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677448.469026:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.469027:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.469028:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:18.0:1619677448.469028:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677448.469029:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.469030:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.469031:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.469032:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.469034:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.469035:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.469036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.469040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.469042:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.469045:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.469045:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.469047:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.469048:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:13.0:1619677448.469049:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.469050:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:12.0:1619677448.469050:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.469051:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.469052:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:4.2:1619677448.469217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.469224:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.469227:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.469228:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.469229:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.469232:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.469233:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.469234:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.469235:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.469237:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.469237:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.469241:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.469242:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.469246:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.469246:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.469247:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.469248:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.469249:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.469250:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.469251:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.469252:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.2:1619677448.469254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.2:1619677448.469255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.469258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677448.469262:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.469263:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.469264:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:24.0:1619677448.469265:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:31.0:1619677448.469266:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.469266:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677448.469267:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677448.469267:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.469267:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.469268:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:31.0:1619677448.469269:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.469269:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677448.469269:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.469269:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.469270:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.469272:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.469272:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:31.0:1619677448.469273:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.469274:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.469275:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.469277:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.469279:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.469281:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.469282:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.469283:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.469284:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.469285:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.469286:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.469286:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.469290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.469298:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.469300:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.469301:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.469303:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.469305:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.469306:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.469440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.469446:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.469450:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.469450:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.469453:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.469455:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.469456:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.469458:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.469459:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.469460:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.469461:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.469464:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.469466:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.469468:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.469469:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.469470:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.469471:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.469472:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.469473:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.469473:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.469478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.469487:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.469489:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.469490:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.469490:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.469492:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.469495:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.469535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.469543:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.469546:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.469547:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.469549:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.469552:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.469554:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.469555:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.469556:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.469557:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.469558:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.469561:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.469563:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.469566:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.469566:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.469567:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.469569:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.469570:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.469570:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.469571:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.469576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.469584:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.469587:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.469588:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.469588:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.469590:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.469592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.469596:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.469597:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.469599:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.469600:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.469601:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.469601:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.469601:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:20.2:1619677448.469659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.469665:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.469667:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.469669:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.469670:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.2:1619677448.469671:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:5.0:1619677448.469675:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.469677:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677448.469677:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677448.469678:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.469680:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677448.469680:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.469680:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:5.0:1619677448.469682:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.469683:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.469683:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.469687:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.469687:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.469689:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.469689:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.469690:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.469691:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.469692:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.469692:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.469693:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.469694:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677448.469694:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:5.0:1619677448.469696:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.469696:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.469697:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:1.0:1619677448.469697:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.469698:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677448.469699:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:25.2:1619677448.469701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.469702:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.469703:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.469704:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.469706:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.469707:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.469707:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.469708:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.469708:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.469711:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:25.2:1619677448.469712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:13.0:1619677448.469712:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.469714:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.469715:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.469716:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.469717:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677448.469718:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.469718:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.469719:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.469720:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.469723:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.469724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.469730:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.469732:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.469733:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.469734:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.469734:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.469735:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.469767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.469775:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.469778:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.469780:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.469782:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.469783:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.469784:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.469785:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.469786:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.469787:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.469788:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.469792:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.469793:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.469796:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.469797:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.469798:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.469799:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.469800:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.469800:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.469801:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.469805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.469811:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.469813:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.469814:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.469816:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.469817:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.469818:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.469889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.469897:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.469899:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.469900:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.469902:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.469905:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.469906:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.469908:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.469909:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.469910:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.469911:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.469914:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.469915:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.469918:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.469919:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.469919:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.469921:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.469921:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.469923:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.469924:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.469924:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.469928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.469930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.469937:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677448.469937:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677448.469939:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:6.0:1619677448.469939:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.469940:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:10.0:1619677448.469940:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677448.469941:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.469942:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677448.469943:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.469947:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.470061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.470068:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.470071:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.470072:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.470074:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.470077:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.470078:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.470079:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.470080:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.470081:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.470082:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.470085:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.470087:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.470090:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.470091:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.470091:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.470093:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.470093:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.470094:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.470094:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.470099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.470106:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.470109:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.470110:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.470110:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.470113:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.470115:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.470139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.470145:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.470148:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.470148:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.470150:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.470153:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.470154:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.470156:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.470157:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.470158:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.470159:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.470162:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.470163:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.470165:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.470166:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.470167:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.470168:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.470169:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.470169:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.470170:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.470176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.470183:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.470185:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.470186:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.470187:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.470188:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.470189:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.470202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.470209:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.470212:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.470213:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.470215:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.470218:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.470220:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.470221:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.470222:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.470223:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.470224:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677448.470227:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.470229:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.470232:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.470233:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.470233:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.470235:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.470236:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.470237:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.470238:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.470238:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.2:1619677448.470241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.470244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:21.2:1619677448.470248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677448.470250:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:24.0:1619677448.470251:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:20.0:1619677448.470252:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:24.0:1619677448.470253:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.470253:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:20.0:1619677448.470253:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.470254:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:31.0:1619677448.470255:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677448.470255:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.470256:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.470258:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677448.470258:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:31.0:1619677448.470261:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.470262:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.470263:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.470265:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.470266:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.470267:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.470268:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.470269:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677448.470273:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.470274:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.470278:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.470278:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.470279:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.470281:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.470281:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.470282:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.470283:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.470287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.470295:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.470301:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.470302:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.470303:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.470306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:2.0:1619677448.470306:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.470310:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.470311:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.470313:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.470314:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.470315:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.470315:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.470316:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.470469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677448.470469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677448.470477:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.470477:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.2:1619677448.470478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.470480:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:26.0:1619677448.470481:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.470481:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:30.0:1619677448.470483:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.470484:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.470485:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677448.470485:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677448.470486:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.470487:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.470488:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:17.0:1619677448.470488:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.470488:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.470489:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.470490:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.470491:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.470492:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.470492:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.470493:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.470493:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.470494:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.470494:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.470495:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677448.470496:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.470496:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:17.0:1619677448.470497:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.470499:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.470500:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.470501:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.470502:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677448.470503:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.470504:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.470506:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:17.0:1619677448.470507:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.470507:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.470508:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.470508:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.470509:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.470510:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:17.0:1619677448.470510:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677448.470510:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.470510:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.470511:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.470511:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.470512:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.470513:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.470513:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677448.470513:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.470514:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677448.470514:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.470514:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:30.0:1619677448.470515:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.470516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:17.0:1619677448.470516:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.470516:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.470518:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.470519:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.470519:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.470520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.470522:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:9.2:1619677448.470522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:31.0:1619677448.470525:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.470526:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.470526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.470528:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.470528:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:30.0:1619677448.470529:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.470530:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:8.0:1619677448.470531:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677448.470531:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677448.470532:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.470532:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.470532:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:8.0:1619677448.470534:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.470534:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.470535:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677448.470535:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.470535:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.470535:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677448.470537:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:5.0:1619677448.470537:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.470538:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.470541:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.470576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.470580:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.470582:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.470584:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.470585:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.470588:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.470589:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677448.470591:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.470592:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.470593:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.470594:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677448.470598:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.470600:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677448.470603:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.470603:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677448.470604:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.470606:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677448.470607:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.470607:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.470608:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.470612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.470619:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.470622:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.470623:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.470625:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677448.470625:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.470628:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.470629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.470635:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.470638:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.470639:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.470640:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.470640:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.470642:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.470677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.470683:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.470685:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.470688:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.470688:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.470691:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.470692:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677448.470694:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.470695:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.470696:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.470697:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677448.470700:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.470701:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.470703:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.470704:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.470704:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.470706:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.470706:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.470707:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.470708:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.470711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.470719:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.470721:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.470722:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.470724:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.470725:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.470726:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.470746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.470754:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.470757:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.470758:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.470760:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.470762:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.470763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677448.470764:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.470765:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.470766:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.470767:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.470768:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.470768:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.470771:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.470772:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.470773:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.470775:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.470777:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.470778:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.470779:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:1.0:1619677448.470779:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.470780:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.470781:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.470781:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:26.0:1619677448.470782:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.470782:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:1.0:1619677448.470782:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.470783:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:1.0:1619677448.470783:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.470784:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.470785:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.2:1619677448.470787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:1.0:1619677448.470789:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.470791:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.470794:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.470795:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.470796:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.470797:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.470798:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.470798:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.470798:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:17.0:1619677448.470799:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.470799:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.470800:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:17.0:1619677448.470801:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:1.0:1619677448.470802:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.2:1619677448.470803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.470807:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677448.470808:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.470809:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.470810:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.470811:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.470812:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.470813:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.470823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.470829:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.470831:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.470832:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.470833:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.470833:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.470834:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.470943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.470950:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.470953:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.470954:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.470956:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.470958:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.470960:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.470961:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.470962:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.470963:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.470964:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.470967:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.470969:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.470972:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.470972:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.470973:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.470974:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.470975:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.470976:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.470977:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.470978:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.470983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:25.2:1619677448.470988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.470990:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.470991:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.470992:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.2:1619677448.470993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677448.470994:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.470995:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:10.0:1619677448.470995:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:30.0:1619677448.470997:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:10.0:1619677448.470997:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:30.0:1619677448.470998:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.470998:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677448.470999:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.471000:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.471001:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:10.0:1619677448.471001:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.471002:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:10.0:1619677448.471004:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677448.471005:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.471006:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.471007:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.471008:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.471011:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.471012:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.2:1619677448.471014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:10.0:1619677448.471014:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.471015:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.471016:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.471017:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.471018:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.471018:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.471019:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:2.0:1619677448.471020:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.471022:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:8.2:1619677448.471023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.471024:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.471025:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.471028:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.471030:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.471031:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.471032:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:2.0:1619677448.471032:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.471033:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.471034:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:2.0:1619677448.471034:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.471036:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.471038:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.471038:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.471039:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.471041:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.471042:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677448.471044:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.471045:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.471046:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.471047:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.471048:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.471049:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.471050:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.471051:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.471053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.471056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677448.471060:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.471062:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:18.0:1619677448.471062:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:26.0:1619677448.471063:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.471063:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:18.0:1619677448.471064:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.471064:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.471065:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:26.0:1619677448.471066:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.471069:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.471093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.471100:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.471103:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.471103:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.471106:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.471107:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.471109:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.471110:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.471111:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.471111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677448.471112:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.471113:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.471116:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.471117:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.471119:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:10.0:1619677448.471119:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.471120:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.471122:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.471123:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.471124:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.471124:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.471124:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:17.0:1619677448.471125:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:10.0:1619677448.471126:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.471126:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677448.471127:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.471127:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:17.0:1619677448.471128:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.471128:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:17.0:1619677448.471129:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.471130:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:25.2:1619677448.471132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:17.0:1619677448.471134:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.471135:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.471138:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.471139:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.471140:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (23)++
00000800:00000200:6.0:1619677448.471140:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (23)++
00000800:00000200:17.0:1619677448.471142:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (25)--
00000800:00000200:17.0:1619677448.471142:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.471143:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.471143:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000800:00000200:22.0:1619677448.471144:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677448.471144:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.471144:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.471146:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.471148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.471151:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.471153:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.471154:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.471155:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.471156:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.471157:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.471159:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.471240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.471248:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.471251:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:5.2:1619677448.471252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:31.0:1619677448.471253:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.471254:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.471256:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.471257:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:31.0:1619677448.471258:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.471259:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.471260:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:31.0:1619677448.471261:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.471261:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:31.0:1619677448.471262:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.471264:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.471264:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.471267:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.471267:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.471269:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.471269:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.471270:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.471271:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.471272:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.471273:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.471274:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.471274:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:2.0:1619677448.471274:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.471276:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.471277:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:2.0:1619677448.471277:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.471278:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.471278:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:2.0:1619677448.471279:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.471280:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.471281:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.2:1619677448.471282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.471282:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.471284:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.471284:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.471285:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.471285:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677448.471290:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:9.2:1619677448.471290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.471292:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.471298:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.471298:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677448.471299:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:20.0:1619677448.471300:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.471301:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.471302:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.471302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.471303:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.471304:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:18.0:1619677448.471304:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.471306:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.471306:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677448.471308:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.471309:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.471310:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.471310:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.471311:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.471343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.471351:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.471353:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.471356:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.471358:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.471359:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.471360:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.471362:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.471363:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.471364:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.471365:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.471368:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.471370:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.471373:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.471374:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.471374:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.471376:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.471376:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.471377:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.471378:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.471382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.471389:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.471391:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.471392:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.471392:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.471394:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.471397:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.471438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.471444:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.471447:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:21.2:1619677448.471447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:8.0:1619677448.471448:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.471450:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.471452:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.471453:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:24.0:1619677448.471454:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.471455:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.471456:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.471457:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:24.0:1619677448.471458:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.471460:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.471460:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.471461:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677448.471462:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.471464:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:24.0:1619677448.471465:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.471466:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.471467:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.471468:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:22.0:1619677448.471468:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.471469:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677448.471471:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.471472:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.471472:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:22.0:1619677448.471473:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.471474:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.471474:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.471475:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677448.471475:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:24.0:1619677448.471476:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677448.471478:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.471479:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.471480:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:9.2:1619677448.471480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677448.471481:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.471482:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.471483:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.471483:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.471488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:2.0:1619677448.471488:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.471490:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.471492:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.471492:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.471494:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:2.0:1619677448.471494:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.471497:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:2.0:1619677448.471497:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:31.0:1619677448.471498:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.471500:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.471501:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.471503:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.471504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.471511:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.471513:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.471514:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.471515:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.471515:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.471517:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.471526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.471532:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.471535:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.471536:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.471537:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.471540:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.471541:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.471542:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714497] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.471544:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.471545:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.471546:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:4.2:1619677448.471548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677448.471549:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.471551:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.471553:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.471554:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.471555:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.471556:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.471556:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.471557:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.471558:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:1.0:1619677448.471558:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.471558:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.471559:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.471561:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.471562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:10.0:1619677448.471564:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.471565:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.471567:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.471568:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.471569:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.471570:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.471570:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.471572:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.471573:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:10.0:1619677448.471573:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.471573:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.471575:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.471576:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.471578:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.471578:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.471578:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:10.0:1619677448.471579:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:25.2:1619677448.471580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.471581:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.471581:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.471582:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.471583:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.471587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.471587:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.471590:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.471591:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.471591:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.471592:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.471593:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677448.471594:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.471596:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.471597:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.471599:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.471600:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.471602:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.471636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:21.2:1619677448.471641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:8.0:1619677448.471643:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677448.471646:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677448.471647:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:18.0:1619677448.471647:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:8.0:1619677448.471649:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000800:00000200:18.0:1619677448.471650:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:8.0:1619677448.471651:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.471651:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:8.0:1619677448.471652:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677448.471653:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.471654:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.471654:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.471655:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.471656:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677448.471657:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.471658:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.471658:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677448.471659:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.471661:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.471662:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677448.471664:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000400:00000200:18.0:1619677448.471666:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677448.471667:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677448.471668:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.471669:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677448.471669:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677448.471669:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:18.0:1619677448.471671:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.471671:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.471671:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:18.0:1619677448.471672:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.471673:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677448.471673:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677448.471674:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:18.0:1619677448.471675:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.471675:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.0:1619677448.471675:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:18.0:1619677448.471676:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:9.2:1619677448.471676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:18.0:1619677448.471677:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.2:1619677448.471681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (21)++
00000800:00000200:25.2:1619677448.471682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677448.471686:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.471687:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.471688:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.471689:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677448.471690:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000400:00000200:24.0:1619677448.471691:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677448.471691:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:6.0:1619677448.471691:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.471692:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:8.0:1619677448.471692:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677448.471693:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:6.0:1619677448.471693:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.471694:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:8.0:1619677448.471694:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:6.0:1619677448.471695:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677448.471697:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.471746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.471753:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.471756:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.471758:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.471759:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.471760:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.471762:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.471763:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.471764:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.471766:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.471767:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.471770:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.471772:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.471774:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.471775:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.471776:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.471777:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.471778:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.471779:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.471781:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.471781:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.471783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.471787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.471790:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.471792:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.471793:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.471795:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:12.0:1619677448.471795:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.471797:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.471797:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677448.471797:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.471798:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.471798:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.471856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.471862:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.471865:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.471866:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.471868:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.471871:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.471872:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.471873:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.471875:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.471876:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.471876:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.471880:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.471881:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.471885:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.471886:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.471886:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.471888:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.471889:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.471889:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.471890:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.471894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.471902:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.471905:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.471906:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.471906:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.471908:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.471911:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.471916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677448.471923:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.471925:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677448.471926:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677448.471928:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.471931:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.471932:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:26.0:1619677448.471933:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.471935:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.471936:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.471936:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:26.0:1619677448.471939:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.471941:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.471942:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.471943:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:26.0:1619677448.471944:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.471945:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.471945:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.471947:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677448.471948:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677448.471948:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:25.2:1619677448.471950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:21.2:1619677448.471955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:5.2:1619677448.471956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.471961:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:17.0:1619677448.471962:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.471962:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.471963:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.471965:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:17.0:1619677448.471965:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:10.0:1619677448.471965:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.471966:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677448.471966:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:1.0:1619677448.471966:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.471967:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.471967:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:26.0:1619677448.471968:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:17.0:1619677448.471968:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.471971:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.471972:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.471974:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.471975:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.471976:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.471977:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.471980:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.471982:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.471985:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.471986:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.471987:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.471988:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.471989:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.471990:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.471991:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.471992:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.471995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.471996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.472002:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.472002:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.472004:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677448.472005:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677448.472005:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677448.472006:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.472006:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.472006:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677448.472008:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.472011:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.472015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.472022:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.472025:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.472026:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.472028:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.472030:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.472031:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.472033:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.472034:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.472035:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.472036:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.472039:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.472041:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.472043:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.472044:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.472045:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.472047:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.472047:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.472048:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.472048:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.472052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.472058:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.472061:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.472061:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.472063:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.472065:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.472066:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.472130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.472137:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.472139:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.472140:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.472142:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.472144:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.472145:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.472146:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.472147:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.472148:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.472149:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.472152:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.472153:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:21.2:1619677448.472154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677448.472155:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.472156:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.472156:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.472158:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.472158:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.472158:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.472159:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.472161:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677448.472163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.472164:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.472166:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.472169:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.472169:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.472170:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677448.472170:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:31.0:1619677448.472171:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.472171:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.472172:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.472172:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.472173:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.472174:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.472174:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.472176:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:31.0:1619677448.472177:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.472177:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.472179:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.472181:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.472182:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.472182:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.472183:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.472184:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.472185:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.472186:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.472189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.472195:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.472197:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.472198:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.472200:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.472202:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.472202:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:21.2:1619677448.472211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.472217:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.472219:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.472220:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.472222:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.472225:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.472226:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.472228:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.472229:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.472230:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.472231:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.472234:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.472236:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.472239:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.472239:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.472240:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.472242:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.472243:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.472244:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.472245:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.472245:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677448.472248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.472251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.472256:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677448.472256:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677448.472258:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.472259:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677448.472259:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:22.0:1619677448.472260:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:18.0:1619677448.472260:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.472260:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.472262:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.472265:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.472286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.472292:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.472299:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.472299:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.472302:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.472304:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.472305:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.472307:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.472308:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.472309:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.472310:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.472314:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.472315:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.472318:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.472319:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.472319:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.472321:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.472321:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.472322:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.472323:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.472328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.472336:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.472339:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.472340:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.472340:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.472342:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.472344:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.472353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:5.2:1619677448.472353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.472359:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:24.0:1619677448.472361:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:2.0:1619677448.472362:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.472363:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:24.0:1619677448.472364:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.472365:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:2.0:1619677448.472365:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.472368:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.472368:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.472369:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:24.0:1619677448.472371:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.472371:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.472372:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677448.472373:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.472374:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.472374:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.472375:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.472375:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:24.0:1619677448.472377:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.472379:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.472380:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.472383:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:24.0:1619677448.472384:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.472386:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.472386:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.472386:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.472387:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:24.0:1619677448.472389:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.472389:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.472389:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:24.0:1619677448.472390:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.472390:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677448.472391:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:24.0:1619677448.472392:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.472392:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:2.0:1619677448.472392:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:24.0:1619677448.472393:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:2.0:1619677448.472393:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:24.0:1619677448.472394:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.472398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.472398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.472403:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:12.0:1619677448.472405:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:18.0:1619677448.472406:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.472406:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.472407:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677448.472408:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.472408:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.472410:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.472412:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.472413:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.472418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.2:1619677448.472418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677448.472425:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.472426:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:12.0:1619677448.472427:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.472428:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:12.0:1619677448.472428:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:20.0:1619677448.472429:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.472429:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.472430:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.472430:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:20.0:1619677448.472431:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.472432:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.472434:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.472538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.472543:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.472546:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.472548:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.472549:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.472551:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.472552:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677448.472554:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.472555:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.472556:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.472557:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.472561:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.472563:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.472566:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.472566:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677448.472567:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.472569:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.472569:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.472571:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.472572:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.472572:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.472575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.2:1619677448.472577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.472580:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.472582:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:10.0:1619677448.472582:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:20.0:1619677448.472583:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.472584:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:10.0:1619677448.472584:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.472585:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.472586:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.472586:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677448.472588:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.472590:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:17.0:1619677448.472593:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.472595:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.472596:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.472598:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.472601:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.472602:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.472603:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.472605:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.472606:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.472607:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.472610:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.472612:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.472615:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.472615:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.472616:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.472618:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.472619:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.472619:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.472620:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.472624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.472630:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.472633:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.472634:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.472635:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.472637:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.472638:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.472655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.472663:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.472665:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.472666:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.472668:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.472670:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.472671:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.472672:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.472674:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.472675:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.472676:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.472679:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.472681:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.472684:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.472685:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.472685:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.472687:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.472687:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.472688:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.472688:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.472693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.472700:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.472703:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.472704:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.472704:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.472706:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.472709:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.472711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.472718:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.472721:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.472722:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.472722:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.472723:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.472724:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.472733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.472741:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.472743:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.472746:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.472748:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.472748:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.472749:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.472751:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.472752:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.472753:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.472754:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.472758:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.472759:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.472763:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.472764:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.472764:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.472766:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.472766:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.472767:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.472768:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.472772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.472780:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.472783:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.472784:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.472784:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.472786:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.472789:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.472935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.472942:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.472945:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.472946:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.472948:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.472951:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.472953:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.472954:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.472955:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.472956:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.472957:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.472961:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.472962:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.472965:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.472966:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.472966:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.472967:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.472968:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.472969:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.472970:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.472971:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.472973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.472976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677448.472980:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.472982:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:26.0:1619677448.472983:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:13.0:1619677448.472983:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.472985:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:13.0:1619677448.472985:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.472986:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:12.0:1619677448.472986:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.472987:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.472988:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.472992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.472998:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.473000:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.473001:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.473002:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.473002:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.473003:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.473054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.473059:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.473062:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.473064:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.473065:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.473067:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.473067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000400:00000200:20.0:1619677448.473068:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.473070:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.473071:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.473072:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.473073:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.473075:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000400:00000200:20.0:1619677448.473076:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.473078:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.473078:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677448.473080:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.473081:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.473082:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.473082:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:20.0:1619677448.473083:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:12.0:1619677448.473083:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.473085:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.473085:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:12.0:1619677448.473085:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:20.0:1619677448.473086:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:12.0:1619677448.473086:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.473087:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.473088:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.473088:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:12.0:1619677448.473088:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.473090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:12.0:1619677448.473090:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.2:1619677448.473093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:12.0:1619677448.473093:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.473095:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.473096:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.473098:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.473098:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677448.473099:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.473099:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.473099:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:13.0:1619677448.473100:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.473100:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677448.473101:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.473101:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:12.0:1619677448.473102:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:5.0:1619677448.473102:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677448.473103:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.473103:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:5.0:1619677448.473103:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.473104:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:13.0:1619677448.473105:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677448.473107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.473115:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.473118:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.473119:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.473120:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.473121:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.473123:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.473125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.473133:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.473136:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.473137:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.473138:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.473138:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.473139:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.473155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.473161:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.473164:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.473164:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.473167:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.473170:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.473171:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.473172:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.473173:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.473174:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.473175:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.473179:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.473180:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.473183:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.473184:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.473185:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.473186:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.473187:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.473188:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.473188:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.473192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.473199:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.473201:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.473202:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.473204:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.473205:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.473207:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.473212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.473220:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.473222:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.473225:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.473226:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.473228:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.473229:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.473230:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.473231:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.473233:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.473234:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.473237:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.473238:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.473241:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.473242:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.473242:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.473244:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.473244:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.473246:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.473247:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.473248:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.473249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.473253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677448.473256:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.473258:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.473259:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.473260:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.473261:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:12.0:1619677448.473262:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.473263:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.473264:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.473265:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.473266:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:21.2:1619677448.473356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.473361:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.473364:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.473365:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.473367:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.473369:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.473371:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.473372:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.473374:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.473374:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.473375:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.473378:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.473380:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.473382:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677448.473382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677448.473383:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.473384:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.473386:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.473386:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.473387:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.473387:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.473389:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.473392:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:8.2:1619677448.473392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.473393:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.473395:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.473397:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.473399:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:20.0:1619677448.473400:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:10.0:1619677448.473400:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.473401:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.473402:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:10.0:1619677448.473402:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.473403:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.473403:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677448.473405:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.473406:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:10.0:1619677448.473407:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.473409:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.473410:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:10.0:1619677448.473412:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.473413:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.473414:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.473416:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.473416:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.473417:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.473418:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.473419:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.473422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.473424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677448.473429:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677448.473431:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:5.0:1619677448.473431:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.473432:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.473433:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.473434:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:5.0:1619677448.473434:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.473435:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.473435:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.473437:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.473462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.473469:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.473471:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.473472:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.473474:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.473476:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.473478:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.473479:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.473480:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.473482:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.473482:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.473486:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.473488:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.473490:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.473491:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.473492:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.473493:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.473494:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.473495:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.473496:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.473497:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677448.473500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.473502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.473507:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:6.0:1619677448.473508:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677448.473509:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.473509:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677448.473510:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677448.473510:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:6.0:1619677448.473511:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.473512:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.473514:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.473516:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.473537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.473544:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677448.473545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:17.0:1619677448.473546:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.473547:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.473549:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.473551:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:17.0:1619677448.473552:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.473553:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:18.0:1619677448.473554:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:17.0:1619677448.473555:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.473555:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.473557:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.473557:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.473558:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.473559:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677448.473560:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.473561:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.473563:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.473563:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.473564:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.473566:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.473566:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:18.0:1619677448.473567:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.473569:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.473570:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:18.0:1619677448.473571:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.473571:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677448.473573:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.473573:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.473573:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.473574:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.473575:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677448.473576:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.473577:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:25.2:1619677448.473578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677448.473578:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.473579:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.473580:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.473580:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.473581:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.473585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677448.473586:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.473589:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.473589:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.473591:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.473593:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:13.0:1619677448.473593:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.473594:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677448.473595:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.473596:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.473597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:20.0:1619677448.473598:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.473599:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.473600:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:13.0:1619677448.473604:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.473607:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.473608:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.473609:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.473609:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.473610:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.473657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.473662:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.473665:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.473667:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.473668:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.473670:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.473672:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.473673:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.473674:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.473675:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.473677:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.473680:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.473683:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:21.2:1619677448.473684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:5.0:1619677448.473687:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.473688:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.473688:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.473689:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677448.473690:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.473691:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.473692:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.473692:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677448.473692:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.473693:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.473694:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.473696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:2.0:1619677448.473697:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.473699:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.473700:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.473701:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.473702:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:2.0:1619677448.473702:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.473703:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.473705:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.473706:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.473706:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677448.473708:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.473708:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.473709:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.473710:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.473711:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.473712:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.473713:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.473715:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.473715:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.473716:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.473717:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.473720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.473727:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.473729:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.473730:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.473732:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.473733:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.473735:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.473946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.473953:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.473956:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.473957:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.473959:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.473962:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.473963:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.473964:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.473966:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.473966:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.473968:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.473971:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.473973:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.473976:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.473977:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.473978:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.473979:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.473980:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.473981:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.473981:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.473985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.473993:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.473996:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.473996:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.473998:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.473999:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.474001:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.474002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.474010:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.474013:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.474015:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.474018:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.474018:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.474019:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.474021:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.474022:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.474023:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.474024:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.474027:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.474029:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.474031:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.474032:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.474032:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.474034:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.474035:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.474035:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.474036:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.474039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.474046:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.474048:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.474049:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.474050:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.474052:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.474052:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.474058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:21.2:1619677448.474062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:13.0:1619677448.474064:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.474067:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.474068:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677448.474068:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.474069:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.474069:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677448.474071:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:13.0:1619677448.474071:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.474071:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.474073:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.474076:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.474077:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.474078:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.474080:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.474081:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.474082:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.474085:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.474087:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.474090:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.474090:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.474091:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.474093:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.474093:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.474093:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.474094:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.474099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.474106:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.474109:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.474110:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.474111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:5.0:1619677448.474111:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.474114:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.474115:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677448.474119:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.474122:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.474124:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.474125:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.474126:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.474128:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.474129:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.474130:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.474132:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.474132:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.474136:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.474138:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.474140:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.474141:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.474142:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.474143:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.474144:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.474145:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.474145:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.474149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.474155:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.474157:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.474158:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.474160:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.474161:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.474163:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.474194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.474202:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.474204:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.474205:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.474207:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.474207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:10.0:1619677448.474210:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.474211:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.474212:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.474214:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.474214:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677448.474215:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.474215:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:8.0:1619677448.474217:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.474218:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:10.0:1619677448.474219:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.474220:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.474221:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:8.0:1619677448.474223:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.474224:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.474224:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:8.0:1619677448.474224:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:10.0:1619677448.474225:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:8.0:1619677448.474225:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96714461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.474226:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:8.0:1619677448.474226:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.474227:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.474227:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:8.0:1619677448.474227:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.474228:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:8.0:1619677448.474228:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.474231:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.474232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:8.0:1619677448.474233:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.474236:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.474237:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.474237:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.474238:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:8.0:1619677448.474239:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.474239:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.474240:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.474240:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:30.0:1619677448.474241:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.474242:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.474243:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.474244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.474245:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.474246:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:1.0:1619677448.474252:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.474255:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.474256:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.474256:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.474258:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.474261:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.474343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.474349:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.474351:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.474354:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.474354:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.474356:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.474358:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.474359:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.474361:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.474362:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.474363:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.474366:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.474368:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.474371:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.474372:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.474372:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.474374:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.474375:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.474376:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.474377:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.474381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.474389:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.474391:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.474393:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.474394:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.474396:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.474398:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.474458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.474464:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.474467:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.474468:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.474469:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.474472:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.474473:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.474475:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.474476:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.474477:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.474478:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.474481:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.474483:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.474486:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.474488:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.474488:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.474490:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.474491:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.474492:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.474493:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.474494:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.474496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.2:1619677448.474499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.474503:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:1.0:1619677448.474504:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677448.474506:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677448.474506:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.0:1619677448.474507:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677448.474507:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677448.474508:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.474509:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677448.474510:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.474513:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.2:1619677448.474832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919065782400] (20)++
00000800:00000200:18.0:1619677448.474839:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (21)++
00000800:00000200:18.0:1619677448.474842:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:2.0:1619677448.474843:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000400:00000200:18.0:1619677448.474845:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.474848:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.474849:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:18.0:1619677448.474851:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.474852:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.474853:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.474854:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677448.474857:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.474859:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677448.474862:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.474862:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:18.0:1619677448.474863:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:4.2:1619677448.474863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677448.474865:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:18.0:1619677448.474865:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (21)++
00000800:00000200:18.0:1619677448.474866:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919065782400] (22)++
00000800:00000200:18.0:1619677448.474867:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (23)--
00000800:00000200:18.0:1619677448.474868:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000800:00000200:26.0:1619677448.474871:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677448.474871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:26.0:1619677448.474873:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:5.2:1619677448.474873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919065782400] (21)++
00000800:00000200:10.0:1619677448.474874:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.474876:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.474878:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (22)++
00000400:00000200:26.0:1619677448.474879:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.474879:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:26.0:1619677448.474880:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:2.0:1619677448.474880:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919065782400] (23)--
00000800:00000200:20.0:1619677448.474881:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:2.0:1619677448.474881:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000400:00000200:26.0:1619677448.474882:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.474882:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.474882:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (21)--
00000400:00000200:26.0:1619677448.474883:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.474884:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.474884:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.474885:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.474885:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.474887:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:26.0:1619677448.474889:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.474891:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.2:1619677448.474892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000800:00000200:26.0:1619677448.474894:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.474895:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.474895:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.474897:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.474898:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.474898:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.474899:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.474899:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.474902:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:25.2:1619677448.474903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.474903:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.474904:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677448.474904:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.474906:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.2:1619677448.474908:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:6.0:1619677448.474911:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.2:1619677448.474912:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677448.474913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:6.0:1619677448.474914:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:30.0:1619677448.474915:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:22.0:1619677448.474915:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.474916:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.474917:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.474918:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.474920:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.474920:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.474921:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.474921:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677448.474922:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.474922:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.474923:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:5.0:1619677448.474923:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:30.0:1619677448.474924:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:26.0:1619677448.474924:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:30.0:1619677448.474925:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.474925:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.474926:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.474927:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.474928:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.474928:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.474929:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:5.0:1619677448.474929:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.474931:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.474932:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.474933:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.474934:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.474936:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.474936:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.474936:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.474937:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.474937:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.474938:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.474939:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.474940:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.474940:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.474941:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:5.0:1619677448.474941:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.474942:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.474942:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.474943:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.474943:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.474944:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.474945:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:10.0:1619677448.474945:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.474946:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.474947:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.474948:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.474948:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.474948:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.474949:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:8.2:1619677448.474949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.474950:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.474950:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.474951:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.0:1619677448.474951:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.474952:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.474952:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.474954:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:25.2:1619677448.474955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.474955:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.474955:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.474956:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677448.474957:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:25.2:1619677448.474960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.474960:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:26.0:1619677448.474961:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:20.0:1619677448.474961:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.474963:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:20.0:1619677448.474963:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.474963:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:26.0:1619677448.474964:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.474964:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.474965:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.474966:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:26.0:1619677448.474967:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:13.0:1619677448.474968:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.2:1619677448.474969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.474970:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:13.0:1619677448.474970:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677448.474971:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:13.0:1619677448.474971:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.474973:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:13.0:1619677448.474973:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.474974:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677448.474975:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.474975:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677448.474976:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:13.0:1619677448.474976:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.474976:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.474977:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.474978:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:20.0:1619677448.474979:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.474979:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.474980:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677448.474980:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:25.2:1619677448.474982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000400:00000200:20.0:1619677448.474982:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.474983:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.474984:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.474985:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.474988:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.474988:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.474990:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.474991:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.474992:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.474992:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:20.0:1619677448.474993:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.474993:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.474994:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.474994:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.0:1619677448.474995:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.474997:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.474997:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.474998:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.474999:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.475002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.475009:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.475011:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.475012:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.475014:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677448.475015:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.475017:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.475039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.475045:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.475047:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.475048:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.475050:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.475053:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.475054:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.475056:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.475057:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.475058:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.475059:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.475062:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.475064:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.475067:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.475068:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.475068:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.475070:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.475070:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.475071:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.475071:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.475076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.475082:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.475085:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.475085:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677448.475086:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.475088:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.475091:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677448.475273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.475279:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.475282:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.475282:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.475284:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.475286:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.475288:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.475289:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.475291:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.475292:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.475293:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.475301:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.475303:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.475305:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.475306:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.475307:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.475308:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.475309:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.475309:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.475310:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.475315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.475324:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:21.2:1619677448.475327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.0:1619677448.475327:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.475328:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.475330:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.475331:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.475333:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677448.475335:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.475338:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.475340:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.475342:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.475343:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.475345:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.475346:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.475348:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.475349:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.475350:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.475353:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.475355:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:4.2:1619677448.475356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677448.475357:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.475358:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.475359:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.475360:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.475361:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.2:1619677448.475361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:31.0:1619677448.475362:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:24.0:1619677448.475363:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677448.475364:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.475364:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677448.475365:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.2:1619677448.475366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677448.475366:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.475368:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.475369:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:24.0:1619677448.475370:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.475370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.475372:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:24.0:1619677448.475372:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:10.0:1619677448.475372:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:24.0:1619677448.475373:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96714449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.475374:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:26.0:1619677448.475375:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.475375:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.475376:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.475376:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677448.475376:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:26.0:1619677448.475377:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.475377:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.475378:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:26.0:1619677448.475379:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:13.0:1619677448.475379:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.475380:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:26.0:1619677448.475380:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96714448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.475381:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677448.475381:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.475381:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.475381:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.475382:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:26.0:1619677448.475382:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.475382:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677448.475383:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.475383:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.475386:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.475387:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.475387:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.475387:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.475389:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.475389:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.475389:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.475390:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.475391:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:26.0:1619677448.475392:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.475392:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.475393:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.475394:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.475395:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:8.2:1619677448.475395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.475396:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677448.475397:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.475398:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677448.475400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677448.475404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:1.0:1619677448.475406:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.475408:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.475410:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:1.0:1619677448.475410:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.475411:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:10.0:1619677448.475411:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:17.0:1619677448.475412:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.475412:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.475413:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.475413:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.475413:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.475414:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:5.2:1619677448.475414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.475415:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.475416:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.475417:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677448.475418:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677448.475418:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.475421:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.475422:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.475423:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.475425:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.475427:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.475428:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.475429:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.475430:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.475431:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.475434:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.475436:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.475438:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.475439:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.475439:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.475441:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.475441:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.475442:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.475442:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.475446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.475453:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.475455:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.475456:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.475456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:1.0:1619677448.475457:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.475458:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.475459:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677448.475462:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:8.2:1619677448.475463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:18.0:1619677448.475466:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.475466:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.475468:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.475471:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.475471:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:18.0:1619677448.475472:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:1.0:1619677448.475473:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:18.0:1619677448.475474:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.475474:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.475474:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000400:00000200:18.0:1619677448.475475:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.475475:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.475475:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:18.0:1619677448.475476:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.475477:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.475480:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.475482:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.475484:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.475485:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.475486:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.475487:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.475488:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.475488:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.475489:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.475493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.475501:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.475503:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.475504:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.475504:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.475506:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.475508:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.475519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.475526:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.475529:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.475530:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.475532:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.475535:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.475536:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.475538:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.475539:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.475540:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.475541:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.475544:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.475546:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.475549:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.475550:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.475550:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.475552:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.475552:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.475553:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.475553:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.475558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.475566:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.475569:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.475570:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.475570:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.475572:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.475575:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.475605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.475612:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.475615:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.475617:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.475619:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.475619:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.475621:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.475622:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.475623:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.475624:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.475625:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:21.2:1619677448.475625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:30.0:1619677448.475629:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.475630:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677448.475632:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:30.0:1619677448.475633:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.475633:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.475634:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:8.0:1619677448.475635:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:30.0:1619677448.475636:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.475636:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:24.0:1619677448.475636:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:30.0:1619677448.475637:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.475637:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:8.0:1619677448.475638:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.475641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:8.0:1619677448.475641:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.475642:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.475643:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96714443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.475645:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.475646:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.475646:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.475648:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:8.0:1619677448.475649:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.475650:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.475651:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.475651:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:31.0:1619677448.475653:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.475654:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:8.0:1619677448.475654:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.475655:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:8.0:1619677448.475655:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.2:1619677448.475655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.0:1619677448.475656:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.475658:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.475658:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.475659:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.475661:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.475661:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:10.0:1619677448.475663:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:9.2:1619677448.475664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.475665:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.475666:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:21.2:1619677448.475667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000400:00000200:10.0:1619677448.475668:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.475671:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.475672:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:2.0:1619677448.475672:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677448.475673:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:10.0:1619677448.475673:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.475674:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.475674:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.475675:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:10.0:1619677448.475675:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.475676:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.475676:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:10.0:1619677448.475676:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.475676:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.475678:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:2.0:1619677448.475678:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.475681:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.475682:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.475683:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:9.2:1619677448.475683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.475684:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.475685:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.475685:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.475687:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.475687:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.475688:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.475688:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:18.0:1619677448.475690:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:25.2:1619677448.475692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.475693:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.475694:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.475694:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.475695:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.475696:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:26.0:1619677448.475698:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.475700:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.475701:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.475702:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.475703:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.475705:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.475743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677448.475749:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.475752:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677448.475752:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677448.475755:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.475756:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.475758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:1.0:1619677448.475758:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677448.475759:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.475761:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.475762:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.475763:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.475764:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.475765:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677448.475768:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.475768:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.475770:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.475770:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:17.0:1619677448.475771:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.475773:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.475774:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.475775:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.475776:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.475779:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.475780:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.475780:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:1.0:1619677448.475781:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.475782:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.475783:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677448.475783:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:20.2:1619677448.475784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677448.475784:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677448.475785:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.475785:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:17.0:1619677448.475786:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.475786:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.475786:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.475787:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:25.2:1619677448.475788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.475788:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.475789:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.475789:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.475790:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:17.0:1619677448.475790:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:20.0:1619677448.475792:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.475793:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:9.2:1619677448.475794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:20.0:1619677448.475795:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.475796:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:20.0:1619677448.475796:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677448.475797:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96714439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.475798:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.475799:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:20.0:1619677448.475799:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.475800:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.475800:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:6.0:1619677448.475800:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677448.475801:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:22.0:1619677448.475802:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.475803:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.475804:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:20.0:1619677448.475804:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.475804:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.475805:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.475806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.475806:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:8.0:1619677448.475807:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.475809:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.475810:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.475810:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.475810:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.0:1619677448.475812:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.475812:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.475813:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.475814:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.475816:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.475816:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677448.475816:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.475817:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.475817:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.2:1619677448.475818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:6.0:1619677448.475818:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.475819:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.475821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.475824:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.475826:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.475826:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:20.0:1619677448.475827:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.475827:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.475828:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.475829:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.475829:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.475830:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677448.475831:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.476093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.476101:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.476104:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.476106:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.476108:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.476108:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.476110:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.476111:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.476113:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.476114:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.476115:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.476118:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.476119:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.476122:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.476123:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.476124:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.476126:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.476126:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.476128:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.476129:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.476130:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.476132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.476135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.476139:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.476142:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677448.476143:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:5.0:1619677448.476143:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.476144:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.476145:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:20.0:1619677448.476145:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.476146:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.476147:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.476148:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.476210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.476213:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.476215:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.476218:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.476218:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.476219:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.476221:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677448.476222:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.476223:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.476224:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.476225:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.476229:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.476230:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.476233:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.476234:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.476234:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.476236:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.476237:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.476237:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.476238:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677448.476242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.476251:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.476254:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.476255:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.476255:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.476257:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.476259:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.476267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.476274:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.476276:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.476277:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.476278:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:4.2:1619677448.476278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677448.476279:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.476279:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677448.476285:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.476288:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.476289:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.476290:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.476292:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.476299:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.476300:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.476302:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.476303:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.476304:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.476307:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.476308:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.476311:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.476312:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.476312:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.476314:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.476315:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.476316:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.476317:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.476317:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.476321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.476324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.476329:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.476331:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.476332:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.476333:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.476334:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.476334:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.476334:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.2:1619677448.476334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:22.0:1619677448.476335:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.476335:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.476338:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677448.476341:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.476343:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.476344:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.476346:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.476348:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.476350:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.476351:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.476352:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.476353:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.476355:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.476358:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.476360:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.476363:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.476363:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.476364:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.476366:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.476366:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.476368:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:21.2:1619677448.476369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:1.0:1619677448.476369:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.476369:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.476372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.476374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.476377:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.476380:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:17.0:1619677448.476380:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677448.476380:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677448.476381:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:24.0:1619677448.476382:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.476382:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677448.476382:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:17.0:1619677448.476383:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.476383:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.476385:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.476386:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.476388:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.476389:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.476391:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.476392:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.476393:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.476394:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.476395:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.476396:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:24.0:1619677448.476397:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.476399:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.476402:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.476403:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.476404:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.476406:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.476406:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.476407:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.476407:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.476412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.476420:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.476423:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.476424:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.476424:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.476426:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.476429:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.476467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.476474:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.476477:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.476479:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.476482:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.476482:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.476483:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.476485:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.476486:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.476487:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.476488:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.476491:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.476493:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.476495:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.476496:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.476496:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.476498:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.476498:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.476499:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.476500:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.476504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.476511:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.476513:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.476514:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.476515:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.476517:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.476518:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.476522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.476528:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.476530:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.476531:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.476532:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.476532:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.476533:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.476565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.476571:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.476574:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.476575:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.476577:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.476579:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.476580:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.476582:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.476583:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.476584:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.476585:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.476588:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.476590:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.476592:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.476592:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.476593:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.476595:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.476595:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.476596:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.476596:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.476601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.476607:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.476609:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.476610:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.476610:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.476611:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.476613:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.476666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677448.476674:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677448.476677:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677448.476678:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677448.476680:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.476682:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.476684:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677448.476685:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.476686:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.476688:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.476689:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677448.476692:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.476693:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677448.476696:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.476697:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:8.0:1619677448.476698:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677448.476699:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:8.0:1619677448.476700:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677448.476701:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677448.476701:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.2:1619677448.476705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.476713:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.476716:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.476717:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.476717:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.476719:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.476722:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.476751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.476756:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.476759:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.476760:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.476762:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.476764:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.476766:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.476767:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.476768:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.476769:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.476770:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.476774:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.476776:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.476778:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.476779:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.476780:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.476782:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.476783:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.476784:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.476785:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.476786:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:25.2:1619677448.476788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:21.2:1619677448.476791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677448.476795:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677448.476796:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:30.0:1619677448.476798:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:22.0:1619677448.476798:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.476799:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.476799:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:30.0:1619677448.476800:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.476802:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.476802:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.476805:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.476806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677448.476813:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.476815:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.476816:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.476817:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.476818:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.2:1619677448.476818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677448.476819:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.476826:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.476828:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.476829:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.476831:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.476833:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.476835:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.476836:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.476837:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.476838:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.476839:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.476842:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.476844:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.476847:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.476848:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.476848:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.476850:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.476850:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.476852:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.476853:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.476853:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.476856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.476859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:4.2:1619677448.476859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677448.476863:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.476866:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:10.0:1619677448.476866:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:24.0:1619677448.476867:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:22.0:1619677448.476867:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.476867:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.476868:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:5.2:1619677448.476868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.476869:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677448.476870:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:22.0:1619677448.476870:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.476870:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677448.476871:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.476873:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.476873:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677448.476874:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:24.0:1619677448.476876:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.476877:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:24.0:1619677448.476878:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:17.0:1619677448.476878:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677448.476880:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.476881:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.476881:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.476882:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.476883:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.476884:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.476887:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.476889:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.476889:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.476891:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.476892:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.476892:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:1.0:1619677448.476892:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.476893:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:1.0:1619677448.476893:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.476894:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.476894:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677448.476895:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.476895:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.476896:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.476898:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.476900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.476900:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.476903:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.476904:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.476905:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.476907:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.476908:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.476909:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.476909:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.476909:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677448.476910:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.476911:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.476911:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:25.2:1619677448.476913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:17.0:1619677448.476913:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.476916:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677448.476920:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.476923:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.476924:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.476925:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.476927:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.476928:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.476937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.476944:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.2:1619677448.476945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677448.476947:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.476948:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.476950:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.476952:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677448.476953:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.476954:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:31.0:1619677448.476955:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677448.476955:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714426] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.476957:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.476957:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.476958:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.476959:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677448.476959:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.476960:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.476961:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.476963:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.476963:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.476964:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.476965:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.476966:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.476966:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.476968:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.476969:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.476969:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.476970:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.476971:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.476971:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.476972:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.476973:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.476973:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.476973:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.476974:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.476974:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.476976:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.476976:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.476977:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.476978:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.476978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:25.2:1619677448.476981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:6.0:1619677448.476985:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.476987:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:6.0:1619677448.476988:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.476989:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.476989:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.476990:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.476991:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.476991:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.476992:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.476993:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.476994:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677448.476996:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.477168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.477176:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.477178:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:21.2:1619677448.477179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:30.0:1619677448.477180:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.477182:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.477182:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.477184:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.477185:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.477186:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.477186:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:30.0:1619677448.477187:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.477188:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.477189:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.477190:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:30.0:1619677448.477192:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.477193:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.477194:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.477195:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.477196:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.477197:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:8.0:1619677448.477197:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:30.0:1619677448.477198:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:8.0:1619677448.477198:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96714423] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.477199:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:8.0:1619677448.477199:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.477200:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.477200:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:8.0:1619677448.477200:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.477201:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:8.0:1619677448.477201:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.477204:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.477205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.477206:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.477208:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.477209:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.477210:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677448.477211:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.0:1619677448.477212:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.477212:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:12.0:1619677448.477213:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:8.0:1619677448.477213:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:12.0:1619677448.477214:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.477214:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:12.0:1619677448.477216:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.477218:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.477218:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:9.2:1619677448.477218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.477225:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.477228:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.477229:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.477229:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.477232:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.477234:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.477309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.477315:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.477317:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.477318:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.477320:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.477322:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.477324:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677448.477325:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.477327:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.477328:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.477329:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677448.477332:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.477334:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.477336:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.477337:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677448.477338:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677448.477339:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.477340:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.477340:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.477341:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.477346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.477354:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.477357:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.477358:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.477358:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.477360:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:21.2:1619677448.477360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677448.477362:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677448.477367:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.477370:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.477372:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.477375:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.477375:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.477376:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.477378:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.477379:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.477380:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.477381:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.477384:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.477386:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.477388:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.477389:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.477390:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.477391:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.477392:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.477393:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.477393:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.477397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.477404:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.477406:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.477407:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.477407:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:21.2:1619677448.477408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:26.0:1619677448.477409:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.477411:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:6.0:1619677448.477413:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.477416:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.477416:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.477418:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.477421:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.477423:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.477424:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.477425:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.477426:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.477427:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.477430:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.477432:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.477434:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.477435:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.477436:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.477437:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.477437:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.477438:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.477439:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.477443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.477451:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.477453:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.477454:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.477455:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.477456:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.477459:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.477461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:9.2:1619677448.477466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:4.2:1619677448.477467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677448.477467:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.477470:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.477471:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.477473:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.477474:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:10.0:1619677448.477474:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677448.477476:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.477477:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:26.0:1619677448.477478:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677448.477478:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:10.0:1619677448.477478:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:2.0:1619677448.477478:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.477479:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (21)++
00000400:00000200:1.0:1619677448.477479:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.477480:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000400:00000200:1.0:1619677448.477480:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.477481:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:1.0:1619677448.477481:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.477482:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.477482:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.477485:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.477487:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677448.477487:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.477488:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.477489:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.477490:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.477490:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.477492:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:1.0:1619677448.477492:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.477493:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.477494:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:10.0:1619677448.477495:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.477496:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.477496:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.477497:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677448.477497:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.477497:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.477499:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.477500:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.477500:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:25.2:1619677448.477502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.477502:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.477502:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.477503:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.477503:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677448.477503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:25.2:1619677448.477507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.477508:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:2.0:1619677448.477509:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:30.0:1619677448.477511:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.477512:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.477512:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.477512:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:26.0:1619677448.477513:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:30.0:1619677448.477514:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.477514:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:31.0:1619677448.477515:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.477515:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.477516:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.477516:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.477516:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:2.0:1619677448.477516:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.477517:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:26.0:1619677448.477518:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.477518:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.477519:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.477520:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:2.0:1619677448.477521:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.477522:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.477524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000400:00000200:2.0:1619677448.477524:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.477525:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.477526:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.477527:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.477529:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714416] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.477530:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.477530:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000400:00000200:2.0:1619677448.477530:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.477531:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.477532:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.477532:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.477533:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.477534:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.477534:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.477535:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.477535:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:18.0:1619677448.477536:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.477538:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.477539:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.477539:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.477541:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.477541:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.477541:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.477541:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.477544:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.477544:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)++
00000800:00000200:2.0:1619677448.477544:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.477544:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)++
00000800:00000200:2.0:1619677448.477546:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (25)++
00000800:00000200:18.0:1619677448.477547:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (26)++
00000800:00000200:2.0:1619677448.477547:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (26)--
00000800:00000200:18.0:1619677448.477548:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (26)--
00000800:00000200:2.0:1619677448.477548:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (25)--
00000800:00000200:18.0:1619677448.477549:0:19933:0:(o2iblnd_cb.c:929:kiblnd_post_tx_locked()) 10.51.3.43@o2ib3(1): redundant or enough NOOP
00000800:00000200:2.0:1619677448.477549:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (24)--
00000800:00000200:18.0:1619677448.477550:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:9.2:1619677448.477550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.2:1619677448.477551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.477554:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677448.477555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.477557:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.477558:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.477558:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.477559:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:18.0:1619677448.477560:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.477560:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.477560:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.477562:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.477562:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.477563:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677448.477563:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677448.477565:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.477566:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.477568:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:9.2:1619677448.477568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:5.0:1619677448.477568:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.477571:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677448.477576:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.477579:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.477580:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.477580:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.477580:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.477581:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.477851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.477858:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.477861:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.477862:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.477863:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.477864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677448.477866:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.477867:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.477869:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.477870:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.477870:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.477871:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.477873:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.477873:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.477873:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.477877:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.477877:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.477879:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:17.0:1619677448.477880:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.477881:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:26.0:1619677448.477883:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.477883:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.477884:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.477885:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.477885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:17.0:1619677448.477885:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.477885:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.477887:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.477887:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677448.477887:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.477888:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.477889:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:17.0:1619677448.477890:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.477892:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.477892:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.2:1619677448.477893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.477895:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677448.477895:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.477896:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:31.0:1619677448.477897:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.477897:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.477899:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677448.477899:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.477899:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.477900:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.477900:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.477901:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677448.477901:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677448.477902:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.477903:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96714413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.477903:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:31.0:1619677448.477904:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.477904:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.477905:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.477905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:31.0:1619677448.477906:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.477906:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.477907:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:31.0:1619677448.477910:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.477910:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677448.477912:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.477912:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.477913:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.477914:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.477914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.477916:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.477916:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.477917:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.477918:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.477919:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.477919:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.477920:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.477920:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.477921:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:5.0:1619677448.477922:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.477923:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.477923:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.0:1619677448.477924:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.477925:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.477926:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:25.2:1619677448.477927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.477927:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:21.2:1619677448.477929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.477929:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677448.477934:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:30.0:1619677448.477937:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677448.477937:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.477938:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.477938:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.477940:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.477941:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677448.477941:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.477942:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.477944:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.478035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.478041:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.478043:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.478044:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.478046:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.478048:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.478049:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.478051:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.478052:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.478053:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.478054:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.478057:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.478059:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.478061:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.478062:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.478063:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.478064:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.478065:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.478065:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.478066:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.2:1619677448.478067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:9.2:1619677448.478071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:10.0:1619677448.478075:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.478077:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.478078:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:2.0:1619677448.478078:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:10.0:1619677448.478081:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.478081:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.478082:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.478082:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.478084:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.478084:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.478085:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.478086:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714411] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.478087:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:10.0:1619677448.478088:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.478089:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.478090:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.478093:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.478094:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.478097:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.478098:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.478099:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.478100:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.478100:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.478101:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.478102:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.478106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.478113:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.478115:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.478116:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.478118:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.478120:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.478121:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.478231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.478236:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.478239:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.478241:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.478241:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.478244:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.478246:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.478247:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.478248:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.478249:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.478250:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.478254:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.478255:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.478258:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.478259:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.478259:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.478261:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.478262:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.478262:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.478263:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.478267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.478274:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.478277:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.478278:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.478278:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.478280:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.478282:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.478286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.478293:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.478299:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.478300:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.478301:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.478302:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.478303:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.478306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.478313:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.478316:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.478317:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.478319:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.478322:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.478324:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.478325:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.478326:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.478327:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.478328:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.478332:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.478333:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.478336:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.478337:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.478338:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.478340:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.478340:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.478341:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.478342:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.478346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.478355:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.478358:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.478359:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.478359:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.478361:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.478364:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.478381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.478389:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.478392:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.478393:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.478394:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.478397:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.478399:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.478400:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.478401:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.478402:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.478403:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:4.2:1619677448.478403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:26.0:1619677448.478407:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.478409:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.478411:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.478412:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.478412:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.478413:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.478414:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:10.0:1619677448.478414:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.478416:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.478417:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:10.0:1619677448.478417:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.478418:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.478418:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:10.0:1619677448.478418:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:26.0:1619677448.478420:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.478420:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.478421:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.478422:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.478422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:10.0:1619677448.478423:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.478426:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.478427:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.478429:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.478430:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:10.0:1619677448.478430:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.478430:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.478432:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.478432:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.478433:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:10.0:1619677448.478433:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:20.0:1619677448.478434:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.478434:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000400:00000200:20.0:1619677448.478435:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:10.0:1619677448.478435:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.478436:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:25.2:1619677448.478438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.478438:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.478441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.478445:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.478448:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:26.0:1619677448.478448:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:30.0:1619677448.478449:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.478449:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.478450:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:30.0:1619677448.478451:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.478451:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.478453:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.478453:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.478455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677448.478462:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.478465:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.478465:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.478466:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.478467:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.478468:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.478520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.478527:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.478530:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.478531:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.478534:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.478534:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.478535:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.478537:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.478538:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.478539:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.478540:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.478543:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.478544:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.478547:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.478548:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.478548:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.478549:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.478550:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.478551:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.478551:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677448.478553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:25.2:1619677448.478555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.478559:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.478562:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.478562:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.478563:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677448.478565:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.478565:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.478565:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677448.478567:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.478568:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.478569:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:1.0:1619677448.478569:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:13.0:1619677448.478570:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677448.478570:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714405] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.478572:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.478573:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.478574:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.478577:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.478578:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.478582:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.478582:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.478583:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.478585:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.478586:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.478587:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.478588:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.478589:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.478591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.478594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.478597:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.478599:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677448.478600:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677448.478601:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:12.0:1619677448.478601:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.478602:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.478603:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677448.478603:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.478603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:13.0:1619677448.478604:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.478605:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.478610:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.478612:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.478614:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.478615:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.478617:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.478619:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.478620:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.478621:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.478622:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.478623:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.478627:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.478628:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.478632:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.478632:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.478633:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.478635:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.478635:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.478636:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.478637:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.478641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.478649:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.478652:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.478653:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.478654:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.478655:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.478657:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.478658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.478665:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.478669:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677448.478670:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.478672:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.478674:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.478674:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.478675:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677448.478676:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.478679:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.478681:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.478683:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.478683:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.478685:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.478686:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.478688:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.478689:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.478690:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.478693:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.478695:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.478698:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.478698:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.478699:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.478700:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.478701:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.478702:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.478703:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.478706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.478712:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.478715:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.478716:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.478717:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.478719:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.478720:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.478803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.478809:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.478811:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.478813:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.478814:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.478816:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.478817:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.478818:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.478820:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.478821:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.478822:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.478826:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.478827:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.478830:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.478831:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.478832:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.478835:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.0:1619677448.478838:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:25.2:1619677448.478839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.478839:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.478839:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.478840:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:10.0:1619677448.478843:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.478845:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.478845:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.478846:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:12.0:1619677448.478848:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:10.0:1619677448.478848:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677448.478849:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.478850:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.478851:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.478851:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.478852:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.478853:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.478854:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677448.478854:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.478855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:10.0:1619677448.478855:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.478856:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.478859:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.478861:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.478861:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.478863:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.478864:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.478864:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:13.0:1619677448.478865:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.478865:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:10.0:1619677448.478865:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:13.0:1619677448.478866:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.478866:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:13.0:1619677448.478867:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.478867:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.478867:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.478868:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.478872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.478879:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.478881:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.478882:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.478883:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.478885:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.478885:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:21.2:1619677448.479027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.479032:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.479035:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.479036:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.479038:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.479040:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.479042:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677448.479043:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.479045:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.479046:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.479047:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677448.479050:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.479052:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677448.479055:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.479056:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677448.479056:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677448.479058:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677448.479059:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.479059:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.479060:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.479064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.479070:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.479073:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.479074:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.479074:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.479076:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:26.0:1619677448.479079:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:4.2:1619677448.479143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.479150:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.479152:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.479153:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.479155:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.479157:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.479158:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.479160:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.479161:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.479162:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.479163:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.479166:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.479168:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.479171:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.479172:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.479173:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.479175:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.479175:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.479176:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.479176:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.479181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.479188:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.2:1619677448.479189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.479191:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.479192:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.479192:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.479194:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677448.479195:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.479196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677448.479197:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.479198:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.479198:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677448.479199:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677448.479200:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.479202:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:1.0:1619677448.479203:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.479204:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.479205:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:1.0:1619677448.479205:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.479206:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.479207:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.479208:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.479209:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.479209:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.479210:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:20.0:1619677448.479211:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.479212:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.479214:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.479214:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.479215:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.479217:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:20.0:1619677448.479219:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.479221:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.479221:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.479222:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.479223:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.479224:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.479225:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.479225:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677448.479226:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.479226:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.479227:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.479227:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677448.479228:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.479229:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.479230:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.479230:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.479231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:25.2:1619677448.479233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.479238:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:21.2:1619677448.479240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:12.0:1619677448.479240:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.479241:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:12.0:1619677448.479241:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.479242:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.479243:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.479243:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:13.0:1619677448.479244:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.479245:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677448.479246:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:12.0:1619677448.479246:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677448.479247:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677448.479248:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:12.0:1619677448.479249:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.479249:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:25.2:1619677448.479250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:22.0:1619677448.479251:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.479254:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.479255:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.479257:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.479257:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:22.0:1619677448.479258:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.479259:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.479260:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677448.479260:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.479261:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.479261:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.479262:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.479263:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:22.0:1619677448.479264:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.479265:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.479268:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.479269:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.479269:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.479271:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.479271:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.479272:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.479273:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.479277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.479284:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.479287:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.479288:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.479288:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.479290:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.479298:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.479418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.479424:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.479427:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.479429:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677448.479430:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.479432:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.479434:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677448.479435:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.479436:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.479437:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.479438:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677448.479442:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:5.2:1619677448.479445:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.0:1619677448.479449:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677448.479451:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.479452:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.479453:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677448.479453:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677448.479455:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.479455:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:26.0:1619677448.479456:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:5.0:1619677448.479456:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:10.0:1619677448.479457:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.0:1619677448.479457:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:26.0:1619677448.479459:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.479461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:26.0:1619677448.479462:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.479464:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.479465:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.479466:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.479467:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.479468:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.479469:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.479472:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:26.0:1619677448.479472:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.479473:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.479473:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.479475:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:26.0:1619677448.479476:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.479477:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:26.0:1619677448.479477:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.479477:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.479478:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.479478:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.479479:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:26.0:1619677448.479479:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.479480:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.479480:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.479480:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (24)--
00000800:00000200:26.0:1619677448.479480:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.479481:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.479481:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.479481:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.479483:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.479484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.479492:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.479495:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.479495:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.479497:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.479499:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.479500:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.479535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.479538:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.479540:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.479543:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.479543:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.479546:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.479547:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.479549:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.479550:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.479551:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.479552:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.479555:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.479557:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.479560:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.479561:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.479561:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.479563:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.479564:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.479565:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.479567:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.479567:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.479569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.479572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.479576:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.479577:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677448.479578:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.479579:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.479580:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.479580:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677448.479581:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.479581:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.479582:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.479584:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.479589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.479596:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.479598:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.479599:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.479601:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.479604:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.479605:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.479607:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.479608:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.479609:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.479610:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.479613:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.479615:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.479618:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.479619:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.479620:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.479622:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.479622:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.479623:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.479623:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.479627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.479634:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.479636:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.479637:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.479639:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.479641:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.479641:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.479643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.479650:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.479652:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.2:1619677448.479653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38ba00] (20)++
00000400:00000200:30.0:1619677448.479655:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.479657:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.479657:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.479658:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38ba00] (21)++
00000400:00000200:30.0:1619677448.479658:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.479659:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.479660:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.45@o2ib3
00000400:00000200:30.0:1619677448.479661:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.479662:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.479663:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.479665:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.5.45@o2ib3 : GET - for me
00000400:00000200:30.0:1619677448.479667:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.479668:0:19945:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.5.45@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000800:00000200:30.0:1619677448.479669:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.479670:0:19945:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.5.45@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:31.0:1619677448.479671:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38ba00] (22)++
00000800:00000200:31.0:1619677448.479672:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38ba00] (23)--
00000800:00000200:30.0:1619677448.479672:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.479673:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.479673:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.479674:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.5.45@o2ib3 local destination
00000800:00000200:30.0:1619677448.479675:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:31.0:1619677448.479678:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.5.45@o2ib3(10.51.5.45@o2ib3:10.51.5.45@o2ib3) : REPLY try# 0
00000800:00000200:30.0:1619677448.479679:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.479680:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.5.45@o2ib3
00000800:00000200:30.0:1619677448.479680:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.479680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.479681:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677448.479682:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a867b82f00] -> 10.51.5.45@o2ib3 (2) version: 12
00000800:00000200:30.0:1619677448.479682:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38ba00] (22)--
00000800:00000200:31.0:1619677448.479683:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a86b38ba00] (21)++
00000800:00000200:31.0:1619677448.479683:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38ba00] (22)++
00000800:00000200:31.0:1619677448.479685:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a86b38ba00] (23)--
00000800:00000200:31.0:1619677448.479685:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38ba00] (22)--
00000800:00000200:13.0:1619677448.479686:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.479689:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.479689:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.479691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38ba00] (21)++
00000400:00000200:13.0:1619677448.479691:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.479693:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.479694:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.479699:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38ba00] (22)++
00000800:00000200:30.0:1619677448.479701:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38ba00] (23)--
00000400:00000200:30.0:1619677448.479702:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.479704:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.5.45@o2ib3: REPLY: OK
00000800:00000200:31.0:1619677448.479705:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38ba00] (22)--
00000800:00000200:30.0:1619677448.479706:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38ba00] (21)--
00000800:00000200:4.2:1619677448.479772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.479779:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.479782:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.479782:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.479784:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.479786:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.479788:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.479789:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.479791:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.479792:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.479793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:10.0:1619677448.479793:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.479796:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.479797:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.479800:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.479801:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.479801:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.479801:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.479803:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:10.0:1619677448.479803:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.479804:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.479804:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.479804:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:31.0:1619677448.479805:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.479807:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.479807:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.479809:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:25.2:1619677448.479809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:31.0:1619677448.479810:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.479812:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.479812:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.479813:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.479816:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.479817:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:31.0:1619677448.479818:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.479820:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677448.479821:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.479821:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.479821:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.479822:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.479823:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:6.0:1619677448.479823:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.479824:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.479825:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.479826:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.479826:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:6.0:1619677448.479826:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.2:1619677448.479831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.479839:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.479842:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.479843:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.479845:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.479845:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.479848:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.479928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.479934:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.479937:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.479937:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.479940:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.479943:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.479944:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.479945:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.479947:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.479948:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.479949:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.479952:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.479953:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.479955:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.479956:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.479957:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.479958:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.479958:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.479959:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.479959:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.479964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.479971:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.479973:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.479974:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.479975:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.479975:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.479977:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.480022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.480029:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.480032:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.480033:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.480035:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.480038:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.480039:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.480040:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.480041:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.480042:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.480044:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.2:1619677448.480045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:26.0:1619677448.480047:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.480047:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.480049:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.480050:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:5.2:1619677448.480050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.480052:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.480052:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.480053:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.480054:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.480056:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677448.480056:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.480057:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:20.0:1619677448.480057:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:5.0:1619677448.480057:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677448.480058:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:20.0:1619677448.480058:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.480058:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.480059:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:26.0:1619677448.480060:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:20.0:1619677448.480060:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.480060:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:25.2:1619677448.480061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.480062:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.480063:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.480064:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.480067:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.480067:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.480068:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677448.480068:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:20.0:1619677448.480069:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.480069:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96714385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.480070:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.480071:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.480071:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.480072:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.480073:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:12.0:1619677448.480073:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000400:00000200:10.0:1619677448.480073:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.480074:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677448.480074:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.480074:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.480076:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.480077:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.480077:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:10.0:1619677448.480077:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.480078:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.480079:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.480079:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.480082:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.480082:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:9.2:1619677448.480082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.480083:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.480084:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.480085:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.480086:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.480087:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.480087:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.480090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.480090:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.480093:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:5.2:1619677448.480093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:18.0:1619677448.480094:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.480094:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.480096:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.480098:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.480099:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:18.0:1619677448.480099:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:26.0:1619677448.480100:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.480101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:10.0:1619677448.480102:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:26.0:1619677448.480103:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.480104:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.480104:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.480105:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.480105:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.480106:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677448.480107:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.480109:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.480110:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.480112:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.480115:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.480117:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.480118:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.480120:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.480121:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.480122:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.480126:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.480128:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.480130:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.480131:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.480131:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.480133:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.480133:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.480134:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.480135:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.480136:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.480139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.480141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.480146:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677448.480146:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.480148:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.480148:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677448.480149:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:6.0:1619677448.480149:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:17.0:1619677448.480150:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.480151:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.480152:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.480155:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.480156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.480164:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.480167:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.480167:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.480168:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.480168:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.480169:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.480274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.2:1619677448.480277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:17.0:1619677448.480280:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.480282:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:2.0:1619677448.480283:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:1.0:1619677448.480283:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.480285:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.480286:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.480287:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:17.0:1619677448.480287:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.480288:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:2.0:1619677448.480289:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.480290:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.480291:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.480292:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.480292:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.480297:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:2.0:1619677448.480298:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.480300:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.480301:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.480302:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.480302:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.480303:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:2.0:1619677448.480304:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:17.0:1619677448.480305:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.480306:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.480307:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.480308:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:2.0:1619677448.480308:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.480309:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.480310:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:2.0:1619677448.480310:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677448.480311:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.480312:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.480313:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.480314:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:8.2:1619677448.480315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677448.480315:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.480316:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.480317:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.2:1619677448.480318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.480318:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.480318:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.480323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:17.0:1619677448.480324:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.480324:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.480324:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.480325:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.480326:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.480326:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.480327:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.480328:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.480329:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.480329:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:10.0:1619677448.480330:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.480333:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.480333:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.480334:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.480336:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.480340:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.480348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.480357:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.480359:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.480362:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.480364:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.480364:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.480366:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.480367:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.480368:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.480370:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.480371:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.480374:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.480374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.480376:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.480379:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.480380:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.480380:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.480381:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.480383:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.480383:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.480383:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.480384:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.480385:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:26.0:1619677448.480385:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.480386:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.480387:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677448.480387:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.480389:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:8.2:1619677448.480389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:26.0:1619677448.480390:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.480391:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.480392:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.480392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677448.480393:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.480395:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.480397:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.480397:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.480399:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677448.480400:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.480400:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.480401:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:26.0:1619677448.480401:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.480401:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.480402:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.480402:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.480403:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:26.0:1619677448.480403:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.480403:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.480404:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677448.480404:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.480405:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.480405:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.480405:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:25.2:1619677448.480409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.480417:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.480420:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.480420:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.480421:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.480423:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.480426:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.480608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.480615:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.480618:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.480619:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.480621:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.480624:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.480625:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.480626:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.480628:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.480629:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.480629:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.480632:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.480634:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.480636:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.480637:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.480637:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.480639:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.480639:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.480640:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.480640:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.480644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.480652:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.480654:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.480655:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.480657:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.480659:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.480660:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.480784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.480787:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.480790:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.480792:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.480793:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.480794:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.480796:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677448.480797:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.480798:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.480800:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.480800:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.480803:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.480805:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.480807:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.480808:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.480809:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.480810:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.480811:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.2:1619677448.480814:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.0:1619677448.480817:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:8.2:1619677448.480817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.480818:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.480819:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677448.480821:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.2:1619677448.480824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677448.480824:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.480825:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.480827:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.480827:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.480828:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.480828:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.480829:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.2:1619677448.480829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:31.0:1619677448.480830:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.480830:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000400:00000200:31.0:1619677448.480832:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.480833:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.480833:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677448.480833:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000400:00000200:31.0:1619677448.480835:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.480835:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.480836:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.480836:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.480836:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:31.0:1619677448.480837:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:17.0:1619677448.480837:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:5.0:1619677448.480837:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:1.0:1619677448.480837:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.480840:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.480841:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.480842:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.480844:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:17.0:1619677448.480844:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.480845:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.480846:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.480847:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.480847:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:17.0:1619677448.480847:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.480848:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.480849:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.480849:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.480849:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.480850:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.480851:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:17.0:1619677448.480852:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.480854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:17.0:1619677448.480854:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.480857:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.480858:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.480859:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.480861:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:17.0:1619677448.480861:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.480861:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.480862:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.480862:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.480864:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.480865:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.480866:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.480866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.480868:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.480869:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:13.0:1619677448.480873:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.480875:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.480876:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.480878:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.480879:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.480880:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.480886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:12.0:1619677448.480893:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.480896:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.480897:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.480897:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.480898:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.480899:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.480976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.480983:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.480986:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.480987:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.480989:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.480992:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.480993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:8.0:1619677448.480994:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.480995:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.480996:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.480997:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.480998:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.480999:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:8.0:1619677448.481001:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.481002:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:8.0:1619677448.481003:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.481003:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.481005:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.481006:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.481007:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.481007:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:18.0:1619677448.481008:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.481009:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:18.0:1619677448.481010:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:8.0:1619677448.481010:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:18.0:1619677448.481011:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.481012:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.481012:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:18.0:1619677448.481013:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.481013:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.481013:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:18.0:1619677448.481014:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.2:1619677448.481016:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:18.0:1619677448.481017:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.481019:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:4.2:1619677448.481019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677448.481022:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.481022:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677448.481023:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (23)++
00000800:00000200:1.0:1619677448.481024:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000800:00000200:18.0:1619677448.481025:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (25)--
00000800:00000200:24.0:1619677448.481026:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677448.481026:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.481026:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.481027:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677448.481027:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000800:00000200:24.0:1619677448.481028:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.481028:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677448.481028:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.481028:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.481030:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.481030:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.481031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.481034:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.481036:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.481038:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.481038:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.481039:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.481040:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.481042:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.481133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.481140:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.481143:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.481143:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.481146:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.481148:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.481150:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.481151:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.481152:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.481153:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.481154:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.481157:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.481159:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.481162:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.481163:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.481163:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.481165:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.481165:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.481166:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.481168:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.481168:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.481171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.481174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.481179:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.481180:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.481182:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.481183:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677448.481183:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.481184:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.481184:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.481184:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.481186:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.481188:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.481228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.481235:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.481238:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.481239:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.481240:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.481243:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.481245:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.481246:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.481247:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.481248:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.481249:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677448.481252:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.481254:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.481256:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.481257:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.481258:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.481260:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.481260:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.481261:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.481262:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.481263:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.2:1619677448.481265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.481269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677448.481275:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:24.0:1619677448.481276:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:5.0:1619677448.481277:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:24.0:1619677448.481278:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.481279:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.481279:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:5.0:1619677448.481279:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.481280:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.481280:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.481283:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.481341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.481349:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.481352:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.481354:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.481356:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.481356:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.481357:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.481359:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.481360:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.481361:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.481362:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.481365:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.481367:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.481370:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.481371:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.481371:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.481373:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.481374:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.481375:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.481375:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.481380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.481389:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.481391:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.481392:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.481394:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.481394:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.481397:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.481400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.481408:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.481410:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.481411:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.481412:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.481412:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.481413:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.481431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.481439:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.481442:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.481443:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.481445:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.481448:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.481449:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:5.2:1619677448.481449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677448.481451:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.481452:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.481453:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.481454:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.481456:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.481457:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.481459:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:10.0:1619677448.481459:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.481460:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.481462:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.481463:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.481463:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:17.0:1619677448.481464:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.481464:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:17.0:1619677448.481465:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:10.0:1619677448.481466:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.481466:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677448.481467:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.481467:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:17.0:1619677448.481468:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.481468:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:17.0:1619677448.481469:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.481470:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.481473:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.481473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677448.481475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:17.0:1619677448.481475:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.481478:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.481479:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.481479:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.481481:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.481481:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.481481:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:2.0:1619677448.481481:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:17.0:1619677448.481482:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.481483:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677448.481484:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:2.0:1619677448.481484:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.481485:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.481485:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.481487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.481487:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:5.0:1619677448.481487:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.481488:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.481490:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.481491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677448.481491:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:5.0:1619677448.481493:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.481494:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.481495:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:5.0:1619677448.481495:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.481495:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:5.0:1619677448.481496:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.481497:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:2.0:1619677448.481497:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:13.0:1619677448.481498:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.481500:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:5.0:1619677448.481500:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.481500:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:13.0:1619677448.481501:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.481501:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677448.481501:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.481502:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.481502:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677448.481503:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677448.481503:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677448.481505:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.481506:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.481507:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.481508:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.481509:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:5.0:1619677448.481510:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:5.0:1619677448.481511:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677448.481516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.481524:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.481526:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.481527:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.481528:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.481529:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.481530:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.481766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.481772:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.481775:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.481776:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.481778:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.481780:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.481782:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.481783:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.481784:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.481785:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.481786:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.481789:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.481791:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.481793:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.481794:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.481795:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.481796:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.481796:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.481797:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.481797:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.481802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.481809:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.481811:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.481811:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.481812:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.481813:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.481814:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.481965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.481972:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.481975:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.481976:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.481978:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.481981:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.481983:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.481984:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.481985:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.481986:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.481987:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.481991:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.481992:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.481995:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.481996:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.481997:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.481999:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.481999:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.482000:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.482000:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.482004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.482010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677448.482012:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.482015:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.482016:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.482016:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677448.482017:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677448.482018:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.482020:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:18.0:1619677448.482021:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:30.0:1619677448.482022:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.482024:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.482025:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.482027:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.482028:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.482029:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.482030:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.482031:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.482034:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.482036:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.482038:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.482039:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.482039:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.482041:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.482041:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.482042:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.482043:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.482047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.482053:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.482055:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.482056:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.482057:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.482059:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.482059:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.482065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.482071:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.482073:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.482074:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.482075:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.482075:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.482076:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.482084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.482089:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.482092:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.482093:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.482095:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.482097:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.482098:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.482100:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.482101:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.482102:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.482103:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.482106:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.482108:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.482110:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.482110:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.482111:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.482113:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.482113:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.482114:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.482114:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.482119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.482126:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.482128:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.482129:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.482129:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.482130:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.482133:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.482145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.482151:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.482153:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.482156:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.2:1619677448.482156:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:5.0:1619677448.482159:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.482161:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.482162:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677448.482163:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677448.482164:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.482165:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.482165:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:5.0:1619677448.482166:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.482167:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.482169:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677448.482169:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.482172:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.482172:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.482174:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:5.0:1619677448.482175:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:1.0:1619677448.482175:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.482176:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.482177:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.482177:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.482178:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:1.0:1619677448.482178:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:5.0:1619677448.482179:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.482181:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.482181:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.482182:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.482182:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677448.482182:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.482184:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.2:1619677448.482187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.482187:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.482187:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.482188:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.482190:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.482190:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.482191:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.482192:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677448.482194:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:9.2:1619677448.482196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677448.482197:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.482197:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.482199:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.482200:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.482201:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.0:1619677448.482202:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.482205:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.482205:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.482206:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.482208:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.482211:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.482214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677448.482220:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.482223:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677448.482223:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.482224:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.482225:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.482225:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.482332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.482339:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.482342:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.482343:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.482345:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.482348:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.482349:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.482350:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.482351:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.482352:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.482354:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.482357:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.482358:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.482361:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.482362:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.482362:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.482364:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.482365:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.482365:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.482366:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.482371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.482380:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.482383:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.482384:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.482384:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.482386:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.482388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.482391:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.482393:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.482395:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.482396:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.482396:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.482397:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.482397:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.482713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.482721:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.482724:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.2:1619677448.482725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:31.0:1619677448.482726:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.482728:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.482729:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:4.2:1619677448.482729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:31.0:1619677448.482730:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:6.0:1619677448.482730:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:31.0:1619677448.482731:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.482732:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.482733:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.482733:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:31.0:1619677448.482734:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.482736:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677448.482737:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:6.0:1619677448.482737:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.482739:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.482739:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:6.0:1619677448.482739:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.482740:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677448.482741:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:31.0:1619677448.482742:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:8.0:1619677448.482743:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.482743:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.482744:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.482745:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.482746:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.482746:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:8.0:1619677448.482746:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.482746:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.482746:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:31.0:1619677448.482748:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.482748:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:8.0:1619677448.482748:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.482749:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96714358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.482750:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:6.0:1619677448.482750:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.482751:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:8.0:1619677448.482751:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.482752:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:8.0:1619677448.482752:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.482753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:8.0:1619677448.482753:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.482753:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.482756:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677448.482757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:8.0:1619677448.482757:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.482757:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.482758:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:8.0:1619677448.482759:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.482760:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.482760:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677448.482761:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677448.482761:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677448.482762:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.482762:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:13.0:1619677448.482763:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677448.482763:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:13.0:1619677448.482764:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.482764:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.482765:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:25.2:1619677448.482765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000400:00000200:13.0:1619677448.482765:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.482765:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.482766:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.482766:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677448.482767:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:12.0:1619677448.482767:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677448.482767:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:30.0:1619677448.482768:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.482769:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677448.482769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:13.0:1619677448.482769:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677448.482771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677448.482772:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.482774:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.482775:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.482775:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.482775:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:10.0:1619677448.482778:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.482779:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:17.0:1619677448.482779:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677448.482779:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:10.0:1619677448.482781:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:17.0:1619677448.482782:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:2.0:1619677448.482782:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.482783:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.482783:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.482783:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:2.0:1619677448.482785:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677448.482786:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.482786:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.482787:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677448.482788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.482788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677448.482788:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.482789:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.482792:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.2:1619677448.482793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:2.0:1619677448.482793:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.482795:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.482795:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.482795:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.482797:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.482797:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:26.0:1619677448.482798:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:17.0:1619677448.482798:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.482798:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.482799:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:10.0:1619677448.482799:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677448.482799:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.482799:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677448.482800:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677448.482801:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677448.482801:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.482801:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.482801:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.482802:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.482803:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:30.0:1619677448.482804:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:26.0:1619677448.482804:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.482806:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.482806:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.482807:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.482807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.482808:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677448.482808:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.482809:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.482809:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.482811:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.482811:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677448.482812:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.482813:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.482814:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.482815:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:5.0:1619677448.482815:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.482818:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.482819:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.482820:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.482821:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.482821:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.482821:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.482824:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.482824:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.482824:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:26.0:1619677448.482826:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.482827:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.482827:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.482828:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.482828:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.482828:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.482830:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.482830:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.482831:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.482831:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.482832:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677448.482832:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.482833:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.482833:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.482836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:25.2:1619677448.482840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.482842:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.482845:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.482846:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.482846:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:31.0:1619677448.482848:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.482849:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:12.0:1619677448.482849:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.482849:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.482850:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:12.0:1619677448.482851:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.482853:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.482854:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.482894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.482900:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.482904:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.482904:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.482906:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.482908:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.482910:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.482911:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.482912:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.482913:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.482914:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.482917:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.482919:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.482921:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.482922:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.482923:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.482924:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.482925:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.482926:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.482927:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.482928:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677448.482930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.482933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.482939:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.482940:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.482940:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.482940:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.482941:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.482941:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.482942:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.482942:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.482943:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677448.482944:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.482950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.482958:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.482961:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.482962:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.482962:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.482962:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.482963:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.483101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.483107:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.483110:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.483111:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.483113:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.483115:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.483117:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.483118:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.483119:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.483120:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.483121:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.483125:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.483126:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.483130:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.483131:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.483131:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.483133:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.483134:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.483135:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.483137:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.483137:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.483139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.483142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.483146:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.483147:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.483149:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:12.0:1619677448.483149:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.2:1619677448.483150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:17.0:1619677448.483150:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677448.483150:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.483151:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677448.483151:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.483153:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:13.0:1619677448.483153:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.483154:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677448.483155:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677448.483158:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.483158:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677448.483160:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.483161:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:20.0:1619677448.483163:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.483164:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.483165:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.483166:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:20.0:1619677448.483169:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.483171:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.483174:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.483175:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:20.0:1619677448.483176:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.483178:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677448.483178:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677448.483179:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677448.483180:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:21.2:1619677448.483181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:25.2:1619677448.483184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.483189:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.483190:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677448.483191:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:13.0:1619677448.483193:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:30.0:1619677448.483194:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.483194:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.483195:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.483195:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.483196:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.483197:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:12.0:1619677448.483197:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:30.0:1619677448.483198:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.483199:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.483199:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:30.0:1619677448.483201:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.483201:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.483205:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.483206:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.483209:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.483210:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.483210:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.483212:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.483212:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.483213:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.483214:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.483217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.483219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:12.0:1619677448.483224:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.483226:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.483227:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.483227:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:12.0:1619677448.483228:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.483229:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.483230:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:12.0:1619677448.483230:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.483231:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677448.483232:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.483234:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.483236:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.483237:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.483238:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.483239:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.483240:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.483243:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.483245:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.483247:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.483248:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.483249:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.483251:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.483251:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.483252:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.483252:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.483257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.483264:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.483267:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.483268:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.483268:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.483270:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.483273:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.483278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:4.2:1619677448.483282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677448.483285:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.483288:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.483288:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.483289:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.483289:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.483290:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.483290:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:26.0:1619677448.483291:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.483292:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.483299:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.483303:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.483305:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.483306:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.483307:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.483308:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.483309:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.483312:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.483313:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.483316:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.483317:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.483317:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.483319:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.483319:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.483320:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.483321:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.483325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.483334:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.483336:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.483337:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.483339:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.483340:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.483342:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.483416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.483422:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.483425:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.483426:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.483427:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.483430:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.483431:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.483433:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.483434:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.483435:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.483436:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.483439:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.483441:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.483444:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.483445:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.483445:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.483447:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.483448:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.483449:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.483449:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.483454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.483462:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.483465:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.483466:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.483466:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.483468:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.483471:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.483515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.483518:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.483520:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.483523:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.483523:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.483525:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.483527:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.483528:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.483530:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.483531:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.483532:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.483535:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.483537:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.483539:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677448.483540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:20.0:1619677448.483540:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.483541:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.483542:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.483543:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.483543:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.483544:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677448.483546:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:25.2:1619677448.483548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.483549:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.483550:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.483551:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.483553:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.483555:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677448.483555:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.483556:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:18.0:1619677448.483557:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.483558:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.483558:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:18.0:1619677448.483559:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:22.0:1619677448.483560:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.483560:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.483563:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.483563:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.483566:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:6.0:1619677448.483566:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677448.483568:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.483569:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677448.483569:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.483571:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677448.483572:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.483573:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.483574:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.483575:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:8.2:1619677448.483577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.483580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677448.483585:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677448.483586:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:5.0:1619677448.483588:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:2.0:1619677448.483588:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:5.0:1619677448.483589:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.483589:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:5.0:1619677448.483590:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.483590:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677448.483591:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.483592:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.483595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:21.2:1619677448.483603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677448.483603:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.483605:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.483606:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.483607:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.483607:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.483608:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677448.483612:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.483615:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.483617:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.483619:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.483620:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.483621:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.483623:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.483624:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.483625:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.483626:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.483629:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.483631:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.483635:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.483636:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.483636:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.483638:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.483639:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.483640:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.483640:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.483644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.483651:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.483655:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.483655:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.483656:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.483658:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.483661:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.483675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.483680:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.483682:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.483685:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:4.2:1619677448.483685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677448.483686:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.483687:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.483688:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677448.483689:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.483691:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.483692:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.483693:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677448.483693:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.483696:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.483696:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:5.0:1619677448.483697:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000400:00000200:10.0:1619677448.483699:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.483699:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.483702:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.483702:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.483703:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:10.0:1619677448.483704:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:5.0:1619677448.483704:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:10.0:1619677448.483705:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.483705:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:10.0:1619677448.483706:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.483706:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:10.0:1619677448.483707:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.483707:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.483707:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:10.0:1619677448.483708:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:25.2:1619677448.483711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:10.0:1619677448.483711:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.483713:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.483716:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.483717:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.483718:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:10.0:1619677448.483718:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.483719:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.483720:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.483721:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.483721:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.483721:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.483722:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000400:00000200:30.0:1619677448.483723:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:10.0:1619677448.483723:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.483724:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:8.2:1619677448.483725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.483726:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.483729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.483734:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.483736:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677448.483737:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:20.0:1619677448.483737:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.483738:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.483739:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677448.483739:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.483739:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.483741:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677448.483742:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.483866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.483872:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.483875:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.483876:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.483878:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.483881:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.483882:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.483884:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.483885:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.483886:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.483887:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.483890:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.483892:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.483895:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.483896:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.483896:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.483898:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.483899:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.483899:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.483899:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.483904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.483911:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.2:1619677448.483914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:12.0:1619677448.483914:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.483915:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.483917:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.483919:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.483919:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.483922:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.483924:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.483926:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.483928:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.483930:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.483931:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:20.2:1619677448.483931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:31.0:1619677448.483932:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.483932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:31.0:1619677448.483934:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.483935:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.483936:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:5.0:1619677448.483936:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677448.483938:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.483939:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:31.0:1619677448.483940:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.483941:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677448.483942:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.483942:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.483942:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.483943:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.483943:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.483944:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.483945:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.483945:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:13.0:1619677448.483945:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677448.483945:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.483946:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:5.0:1619677448.483947:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677448.483948:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.483948:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:5.0:1619677448.483948:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.483950:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:5.0:1619677448.483950:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.483950:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.483951:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.483952:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:5.0:1619677448.483952:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:25.2:1619677448.483953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:5.0:1619677448.483955:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.483957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.483957:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.483959:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:21.2:1619677448.483961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:5.0:1619677448.483961:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.483961:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.483962:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.483962:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.483962:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:12.0:1619677448.483964:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.483965:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:13.0:1619677448.483965:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677448.483965:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.483966:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677448.483966:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677448.483966:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.483967:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:30.0:1619677448.483968:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.483968:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:6.0:1619677448.483969:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:5.0:1619677448.483969:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.483969:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677448.483970:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.483970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.483970:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.483972:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.483973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:20.2:1619677448.483974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:6.0:1619677448.483975:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.483976:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:20.0:1619677448.483977:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677448.483977:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677448.483977:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.483979:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:13.0:1619677448.483979:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.483979:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:6.0:1619677448.483979:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.483980:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:13.0:1619677448.483980:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.483980:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:6.0:1619677448.483980:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.483981:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.483981:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:6.0:1619677448.483981:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:13.0:1619677448.483982:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.483982:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.483983:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.483984:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.483985:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:6.0:1619677448.483985:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.483986:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.483987:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.483991:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.483992:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.483992:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.483994:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.483994:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.483995:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.483996:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.484000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.484007:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.484009:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.484010:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.484012:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.484013:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.484015:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.484027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.484033:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.484036:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.484037:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.484039:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.484041:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.484043:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.484044:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.484045:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.484046:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.484047:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.484051:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.484053:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.484055:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.484056:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.484057:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.484058:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.484058:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.484059:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.484060:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.484064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.484071:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.484074:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.484074:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.484076:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.484077:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.484079:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.484149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.484156:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:5.2:1619677448.484158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.484159:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.484162:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.484162:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:26.0:1619677448.484163:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.484164:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.484165:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.484167:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.484167:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.484169:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.484170:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.484171:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.484172:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.484175:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.484176:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000400:00000200:26.0:1619677448.484177:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.484178:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.484178:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.484179:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.484180:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.484181:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.484184:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.484184:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.484185:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.484186:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.484186:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.484189:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.484189:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.484190:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.484190:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.484190:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.484191:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.484191:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:26.0:1619677448.484192:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.484192:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.484193:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.484194:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.484195:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:9.2:1619677448.484195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.484196:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.484198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.484201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.484203:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:12.0:1619677448.484205:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.484205:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.484207:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.484207:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.484208:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.484209:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.484209:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:2.0:1619677448.484209:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000400:00000200:12.0:1619677448.484211:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.484211:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:13.0:1619677448.484212:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.484212:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.484213:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677448.484213:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677448.484215:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.484246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.484253:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.484255:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.484256:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.484258:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.484261:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.484262:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.484263:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.484265:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.484266:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.484266:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.484269:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.484271:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.484272:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.484273:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.484273:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.484275:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.484275:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.484276:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.484276:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.484280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.484286:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.484288:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.484289:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.484289:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.484290:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.484292:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.484303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.484309:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.484312:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.484313:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.484315:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.484318:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.484319:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.484320:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.484321:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.484322:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.484323:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.484327:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.484329:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.484332:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.484332:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.484333:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.484335:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.484335:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.484336:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.484336:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.484341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.484348:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:21.2:1619677448.484348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:24.0:1619677448.484352:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.484353:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.484353:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:18.0:1619677448.484354:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:24.0:1619677448.484355:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.484357:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:24.0:1619677448.484358:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:2.0:1619677448.484358:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.484360:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:9.2:1619677448.484361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000400:00000200:18.0:1619677448.484362:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.484363:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.484365:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.484366:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.484367:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.484367:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000400:00000200:18.0:1619677448.484368:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677448.484370:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677448.484371:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.484371:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000400:00000200:18.0:1619677448.484372:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677448.484372:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.484372:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:18.0:1619677448.484374:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.484376:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.484377:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.484378:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.484379:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.484380:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.484381:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.484382:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.484383:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677448.484385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.484388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677448.484393:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677448.484393:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:22.0:1619677448.484395:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677448.484395:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:6.0:1619677448.484396:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.484396:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:22.0:1619677448.484397:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.484397:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:22.0:1619677448.484399:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.484402:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.484412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.484417:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.484420:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.484422:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.484423:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.484425:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.484427:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.484428:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.484429:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.484430:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.484431:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.484434:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.484436:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.484439:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.484440:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.484441:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.484442:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.484443:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.484444:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.484444:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.484449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.484456:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.484459:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.484460:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.484462:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.484463:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.484464:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.484551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.484558:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.484560:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.484561:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.484563:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.484565:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.484567:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.484568:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.484569:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.484570:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.484571:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.484574:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.484576:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.484579:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.484580:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.484581:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.484583:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.484583:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.484584:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.484584:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.2:1619677448.484588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:9.2:1619677448.484589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.484594:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.484597:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677448.484597:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.484598:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.484599:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.484600:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:1.0:1619677448.484600:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.484601:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.484602:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.484603:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.484604:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.484605:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.484606:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:1.0:1619677448.484606:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.484607:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.484608:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.484611:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.484612:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.484615:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.484616:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.484616:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.484618:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.484618:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.484619:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.484620:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.484624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.484632:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.484634:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.484635:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.484635:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.484637:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.484640:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.484726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.484733:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.484735:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.484736:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.484738:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.484741:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.484743:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.484744:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.484745:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.484746:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.484747:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.484751:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.484752:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.484755:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.484756:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.484757:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.484758:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.484759:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.484760:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.484760:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.484765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.484773:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.484776:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.484777:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.484777:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.484779:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.484781:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.484792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677448.484795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.484798:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.484801:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:10.0:1619677448.484801:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677448.484803:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.484804:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.484804:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:5.0:1619677448.484804:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.484806:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.484806:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.484807:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.484808:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.484809:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.484810:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.484811:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.484811:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.484812:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.484813:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.484814:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.484815:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.484817:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.484817:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.484821:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.484821:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.484824:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.484824:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.484824:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.484825:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.484827:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.484828:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.484828:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.484828:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.484829:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677448.484829:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.484830:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:10.0:1619677448.484830:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.484831:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.484832:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.484832:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.484833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:25.2:1619677448.484837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:12.0:1619677448.484840:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.484842:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.484843:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.484845:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:12.0:1619677448.484845:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.484846:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677448.484848:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:12.0:1619677448.484848:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:22.0:1619677448.484849:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.484849:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:25.2:1619677448.484850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:22.0:1619677448.484851:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.484854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:13.0:1619677448.484856:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.484858:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.484859:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:13.0:1619677448.484859:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.484860:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.484860:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.484861:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677448.484862:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:13.0:1619677448.484862:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.484862:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.484863:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.484864:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.484864:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.484883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.484890:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.484893:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.484894:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.484896:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.484899:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.484901:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.484902:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714326] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.484903:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.484904:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.484905:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.484908:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.484910:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.484913:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.484913:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.484914:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.484915:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.484916:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.484917:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.484917:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.2:1619677448.484921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.484930:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.484932:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.484933:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.484934:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.484936:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.484939:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.484978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.484983:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.484986:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.484986:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.484988:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.484990:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.484992:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.484993:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.484994:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.484995:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.484996:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.484999:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.485001:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.485004:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.485004:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.485005:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.485007:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.485007:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.485008:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.485009:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.485010:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.485013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.485015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.485020:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.485021:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.485022:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677448.485023:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677448.485023:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677448.485024:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.485024:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.485025:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.485026:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.485028:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.485055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.2:1619677448.485059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:31.0:1619677448.485064:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.485065:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:31.0:1619677448.485067:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677448.485068:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677448.485068:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:31.0:1619677448.485070:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.485071:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677448.485071:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.485073:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.485073:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.485075:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.485076:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.485076:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677448.485077:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.485078:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.485079:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.485080:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.485080:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.485082:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.485083:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677448.485088:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.485088:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.485091:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:17.0:1619677448.485092:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.485095:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.485095:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.485096:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:17.0:1619677448.485096:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.485097:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:17.0:1619677448.485097:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.485099:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:17.0:1619677448.485099:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.485100:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.485100:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677448.485101:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677448.485101:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:31.0:1619677448.485102:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.485102:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.485106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:9.2:1619677448.485106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:10.0:1619677448.485113:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:2.0:1619677448.485114:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:10.0:1619677448.485115:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.485116:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.485117:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.485117:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.485118:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.485118:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.485119:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.485121:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.485122:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:2.0:1619677448.485124:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:25.2:1619677448.485128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.485134:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.485136:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.485137:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.485138:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.485139:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.485139:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.485142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.485147:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:21.2:1619677448.485150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:18.0:1619677448.485150:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.485151:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.485153:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.485155:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:18.0:1619677448.485155:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.485156:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:22.0:1619677448.485158:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:18.0:1619677448.485158:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.485159:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:18.0:1619677448.485161:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.485162:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.485162:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.485163:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:22.0:1619677448.485165:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.485167:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.485167:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.485169:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.485169:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:22.0:1619677448.485170:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.485171:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.485172:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.485172:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.485173:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.485174:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:22.0:1619677448.485176:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.485176:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.485176:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.485177:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:22.0:1619677448.485178:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.485178:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.485181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:22.0:1619677448.485181:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.485182:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.485183:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.485184:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.485185:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.485186:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.485187:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:10.0:1619677448.485187:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:22.0:1619677448.485188:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:10.0:1619677448.485190:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:8.2:1619677448.485190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.485191:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.485192:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.485193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677448.485194:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.485197:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:6.0:1619677448.485199:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:5.0:1619677448.485200:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677448.485201:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.485202:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.0:1619677448.485202:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:6.0:1619677448.485203:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:5.0:1619677448.485203:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.485205:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.485206:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.485207:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.485208:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:5.0:1619677448.485208:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.485209:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.485210:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.485211:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.485213:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.485214:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.485217:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.485225:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.485228:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.485230:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.485231:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.485233:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.485235:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.485236:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.485237:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.485238:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.485239:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.485243:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.485245:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.485247:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.485247:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.485248:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.485249:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.485250:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.485251:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.485252:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.485255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.485261:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.485264:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.485265:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.485266:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.485268:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.485269:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.485305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.485313:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.485316:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.485317:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.485319:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.485322:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.485323:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.485324:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.485325:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.485326:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.485327:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.485330:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.485331:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.485334:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.485334:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.485335:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.485336:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.485337:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.485338:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.485338:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.485343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.485350:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.485353:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.485353:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.485355:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.485357:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.485358:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.485358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.485365:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.485367:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.485368:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.485370:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.485373:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.485374:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.485376:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.485377:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.485378:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.485379:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.485383:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.485384:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.485387:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.485387:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.485388:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.485390:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.485390:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.485391:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.485391:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.485396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.2:1619677448.485400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:22.0:1619677448.485403:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.0:1619677448.485405:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:22.0:1619677448.485406:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.485407:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.485407:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:5.0:1619677448.485408:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:22.0:1619677448.485410:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.485410:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.485411:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.485412:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:5.0:1619677448.485413:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.485415:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677448.485416:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.485418:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.485419:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.485420:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.485423:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.485425:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:4.2:1619677448.485425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677448.485428:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.485429:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677448.485429:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.485431:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.485432:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677448.485432:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.485433:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.485433:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:10.0:1619677448.485435:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.485436:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.485437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:10.0:1619677448.485438:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.485441:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.485443:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:26.0:1619677448.485444:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:10.0:1619677448.485444:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.485445:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.485445:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.485446:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.485447:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.485448:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.485449:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.485452:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.485452:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.485453:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.485453:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.485457:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.485458:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.485459:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.485460:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.485461:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.485461:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.485462:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.485466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.485474:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.485477:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.485477:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.485478:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.485480:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.485483:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.485586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.485589:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.485592:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.485594:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.485595:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.485597:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.485599:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.485600:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.485601:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.485602:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.485603:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.485606:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.485608:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.485611:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.485612:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.485612:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.485614:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.485615:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.485615:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.485616:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.485620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.485624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:13.0:1619677448.485627:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.485629:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.485630:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.485631:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:13.0:1619677448.485632:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.485633:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.485634:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:13.0:1619677448.485635:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.485635:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.485637:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.485640:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.485641:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.485642:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.485643:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.485644:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.485645:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.485649:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.485650:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.485652:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.485653:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.485654:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.485655:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.485655:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.485656:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.485657:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.485660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.485666:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.485668:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.485668:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.485670:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.485670:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.485672:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.485698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.485706:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.485709:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.485710:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.485712:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.485713:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.485714:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.485715:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.485716:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.485717:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.485718:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.485721:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.485723:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.485726:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.485727:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.485727:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.485729:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.485730:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.485730:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.485731:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.485736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.485744:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.485747:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.485748:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.485749:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.485750:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.485753:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.485773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.485780:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.485783:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.485784:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.485785:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.485788:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.485790:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.485791:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.485792:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.485793:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.485794:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.485797:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.485798:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.485801:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.485802:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.485803:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.485804:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.485805:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.485805:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.485806:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.485810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.485818:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.485821:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.485821:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.485822:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.485824:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.485827:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.485966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.485971:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.485974:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.485975:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.485977:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.485979:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.485981:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.485982:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.485983:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.485984:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.485985:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.485989:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.485990:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.485993:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.485994:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.485994:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.485994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677448.485996:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.485996:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.485997:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.485998:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:26.0:1619677448.486002:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.2:1619677448.486002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.486004:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.486005:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.486007:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.486009:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.486010:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:20.0:1619677448.486010:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:26.0:1619677448.486011:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.486012:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:26.0:1619677448.486013:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.486013:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.486014:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.486014:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.486015:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.486015:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.486017:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:26.0:1619677448.486018:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.486020:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.486022:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.486022:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.486023:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.486024:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.486025:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.486026:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.486027:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.486028:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.486029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.2:1619677448.486033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.486038:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.486040:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.486040:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.486042:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:10.0:1619677448.486042:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.486043:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.486043:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.486044:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.486044:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.486045:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.486049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.486055:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.486058:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.486058:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.486058:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.486059:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.486059:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.486085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.486091:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677448.486094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:17.0:1619677448.486094:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.486094:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.486096:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.486099:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.486100:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:8.0:1619677448.486101:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:1.0:1619677448.486102:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714309] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.486103:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:1.0:1619677448.486103:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.486104:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:1.0:1619677448.486105:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.486106:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.486107:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.486109:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.486109:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.486111:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:1.0:1619677448.486111:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.486113:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96714308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.486114:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.486115:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.486115:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677448.486116:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.486116:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.486116:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.486118:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.486118:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.486119:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677448.486120:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.486120:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677448.486122:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:25.2:1619677448.486124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.486125:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.486125:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.0:1619677448.486126:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:8.0:1619677448.486128:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:8.0:1619677448.486128:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.486129:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677448.486130:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:8.0:1619677448.486130:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677448.486133:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:12.0:1619677448.486133:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.486135:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.486137:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677448.486138:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677448.486138:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.486138:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:12.0:1619677448.486139:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677448.486140:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.486142:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.486323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.486332:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.486334:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.486337:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.486339:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.486340:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.486341:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.486342:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.486343:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.486344:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.486345:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.486348:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.486350:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.486351:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.486352:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.486353:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.486354:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.486355:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.486356:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.486356:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.486361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.486367:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.486370:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.486370:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.486372:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.486373:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.486374:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.486379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.486385:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.486388:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.486389:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.486389:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.486390:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.486391:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.486479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:5.2:1619677448.486482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:2.0:1619677448.486485:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.486488:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.486490:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:12.0:1619677448.486490:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000400:00000200:2.0:1619677448.486491:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:12.0:1619677448.486493:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:2.0:1619677448.486494:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.486496:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.486496:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.486497:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.486498:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:2.0:1619677448.486498:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.486499:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.486499:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.486501:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:12.0:1619677448.486502:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:12.0:1619677448.486503:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.486506:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.486506:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:12.0:1619677448.486507:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677448.486509:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.486509:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.486512:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.486513:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:12.0:1619677448.486514:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.486514:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.486515:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:12.0:1619677448.486516:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.486516:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.486517:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.486517:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677448.486518:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.486519:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:12.0:1619677448.486520:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:25.2:1619677448.486521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:12.0:1619677448.486521:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.486522:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677448.486523:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677448.486523:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:25.2:1619677448.486527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677448.486529:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.486532:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.486533:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:13.0:1619677448.486533:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:6.0:1619677448.486533:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.486535:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:13.0:1619677448.486536:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.486537:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.486538:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.486538:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677448.486539:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.486541:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.486543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:25.2:1619677448.486544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.486550:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.486550:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.486551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677448.486553:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:12.0:1619677448.486553:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.486554:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:31.0:1619677448.486555:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.486555:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.486555:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.486557:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677448.486557:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:31.0:1619677448.486558:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.486559:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:26.0:1619677448.486559:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677448.486560:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.486562:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.486562:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:31.0:1619677448.486563:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.486563:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:31.0:1619677448.486564:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.486565:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.486567:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.486568:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.486569:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.486570:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.486570:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:31.0:1619677448.486571:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:26.0:1619677448.486571:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.486572:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.486573:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.486573:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.486574:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677448.486574:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.486574:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.486575:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.486577:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:26.0:1619677448.486577:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.486578:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.486579:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:25.2:1619677448.486579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.486582:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.486583:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.486583:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (23)++
00000800:00000200:21.2:1619677448.486583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.486585:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (24)--
00000800:00000200:26.0:1619677448.486586:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.486586:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (24)++
00000800:00000200:10.0:1619677448.486586:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (24)--
00000800:00000200:26.0:1619677448.486587:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.486587:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.486588:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677448.486588:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.486589:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.486591:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:25.2:1619677448.486591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (22)++
00000800:00000200:30.0:1619677448.486593:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:10.0:1619677448.486593:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:30.0:1619677448.486594:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677448.486594:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:30.0:1619677448.486595:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.486596:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.486596:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.486597:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.486598:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.486599:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.486667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.486673:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.486676:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.486677:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.486679:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.486681:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.486683:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.486684:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.486685:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.486686:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.486687:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:4.2:1619677448.486687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:18.0:1619677448.486690:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.486690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677448.486692:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.486694:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.486694:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.486695:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.486696:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:18.0:1619677448.486696:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.486696:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.486697:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.486697:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677448.486698:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.486698:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.486699:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677448.486699:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:26.0:1619677448.486700:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.486700:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.486702:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.486703:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.486703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677448.486704:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.486705:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.486706:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.486706:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.486708:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.486708:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.486709:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.486710:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.486711:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.486711:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:17.0:1619677448.486712:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.486712:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.486713:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.486714:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.486715:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.486716:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.486717:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.486718:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.486718:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:26.0:1619677448.486720:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.486721:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.486722:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.486723:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.486724:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.486724:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.486725:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.486726:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.486726:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.486727:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.486728:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:17.0:1619677448.486728:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.486728:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.486729:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:17.0:1619677448.486729:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.486730:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.486730:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.486732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.486735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:25.2:1619677448.486736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.486738:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.486741:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.486742:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.486742:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:30.0:1619677448.486743:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.486743:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:31.0:1619677448.486744:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.486744:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.486745:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.486746:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:13.0:1619677448.486746:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.486746:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.486746:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:13.0:1619677448.486748:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.486749:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.486751:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.486753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677448.486760:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.486763:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.486764:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.486764:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.486765:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.486766:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.486795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.486802:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.486805:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.486806:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.486807:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.486810:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.486811:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.486812:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.486814:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.486815:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.486816:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.486819:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.486821:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.486824:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.486825:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.486825:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.486827:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.486827:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.486829:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.486830:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.486831:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.486833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.486836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.486841:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677448.486843:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.486844:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.486844:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677448.486845:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677448.486845:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.486846:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.486847:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.486847:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.486850:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.487088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.487096:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.487098:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.487099:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.487101:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.487103:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.487105:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.487106:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.487107:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.487108:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.487109:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.487112:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.487114:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.487117:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.487117:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.487118:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.487120:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.487120:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.487121:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677448.487122:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.487123:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677448.487126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.487128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:5.0:1619677448.487134:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:10.0:1619677448.487135:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677448.487137:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:5.0:1619677448.487137:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:10.0:1619677448.487138:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:5.0:1619677448.487138:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.487139:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:5.0:1619677448.487139:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677448.487140:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.487142:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.487163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.487169:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.487172:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:5.2:1619677448.487172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.487173:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.487174:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.487176:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.487177:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:17.0:1619677448.487178:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677448.487179:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.487180:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.487181:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.487181:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.487181:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677448.487182:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677448.487184:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.487186:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.487187:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.487188:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:17.0:1619677448.487188:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.487189:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.487191:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.487192:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.487192:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.487193:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:17.0:1619677448.487193:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.487194:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.487195:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677448.487196:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.487196:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.487197:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:17.0:1619677448.487197:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.487198:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.487201:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677448.487201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677448.487202:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.487202:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.487204:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.487205:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.487206:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.487207:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:20.0:1619677448.487208:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677448.487208:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.487210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.487210:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.487211:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.487213:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.487213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.487215:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.487216:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:12.0:1619677448.487217:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677448.487218:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677448.487220:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677448.487220:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:12.0:1619677448.487221:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.487221:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677448.487222:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.487222:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.487224:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.487225:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.487248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.487254:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.487257:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.487258:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.487260:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.487262:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.487264:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.487265:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.487266:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.487267:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.487268:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.487272:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.487273:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.487275:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.487276:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.487276:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.487278:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.487278:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.487279:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.487279:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.487284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.487291:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.487297:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.487299:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.487299:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.487301:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.487302:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.487305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:9.2:1619677448.487305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.487313:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.487314:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.487315:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.487316:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.487316:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677448.487317:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:2.0:1619677448.487317:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.487317:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:31.0:1619677448.487319:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.487321:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.487322:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.487323:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.487325:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.487326:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.487327:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.487328:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.487331:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.487333:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.487336:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.487337:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.487337:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.487339:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.487339:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.487340:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.487341:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.487344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.487352:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.487354:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.487355:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.487356:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.487359:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.487359:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.487596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.487604:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.487606:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.487607:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.487609:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.487612:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.487613:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.487614:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.487615:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.487616:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.487617:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.487620:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.487621:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.487625:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.487625:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.487626:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.487628:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.487628:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.487629:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.487629:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.487634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.487642:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.487645:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.487646:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.487646:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.487648:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.487652:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.487688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.487694:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.487697:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.487698:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.487700:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.487703:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.487704:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.487705:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.487706:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.487707:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.487708:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.487711:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.487713:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.487715:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.487716:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.487717:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.487718:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.487718:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.487719:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.487719:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.487724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.487732:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.487734:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.487735:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.487736:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.487737:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.487740:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.487743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:22.0:1619677448.487750:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.487753:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.487753:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.487754:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.487754:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.487755:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.487895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.487902:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.487905:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.487907:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.487909:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.487910:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.487911:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.487912:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.487914:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.487915:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.487916:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.487919:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.487921:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.487923:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.487924:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.487925:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.487926:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.487927:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.487928:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.487928:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.487932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.487939:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.487941:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.487942:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.487943:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.487945:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.487946:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.487982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.487988:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.487991:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.487992:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.487993:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.487995:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.487996:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.487998:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.487999:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.488000:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.488001:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.488004:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.488005:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.488007:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.488008:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.488008:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.488010:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.488010:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.488012:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.488013:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.488013:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677448.488016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.488019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.488024:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.488025:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.488025:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.488025:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.488026:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.488027:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.488027:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.488028:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.488029:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677448.488029:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.488146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.488151:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.488154:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.488154:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.488156:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.488158:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.488160:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.488161:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.488163:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.488164:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.488165:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:20.2:1619677448.488168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000400:00000200:6.0:1619677448.488169:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.488170:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.488173:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.488174:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:1.0:1619677448.488174:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:6.0:1619677448.488175:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.488177:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.488177:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677448.488177:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677448.488178:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:6.0:1619677448.488178:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.488179:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:1.0:1619677448.488180:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.488182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000400:00000200:1.0:1619677448.488182:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.488184:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677448.488185:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.488186:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.488187:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.488188:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.488189:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.488191:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:1.0:1619677448.488191:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.488192:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.488193:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.488193:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.488195:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.488196:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.488197:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:1.0:1619677448.488197:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.488198:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:1.0:1619677448.488199:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677448.488199:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677448.488200:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677448.488201:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:25.2:1619677448.488206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000800:00000200:8.2:1619677448.488206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.488212:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:20.0:1619677448.488214:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677448.488215:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.488216:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (21)++
00000800:00000200:20.0:1619677448.488216:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:10.0:1619677448.488216:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.488217:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000400:00000200:20.0:1619677448.488217:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.488218:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:20.0:1619677448.488219:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.488220:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.488221:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.488223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.488231:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.488233:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.488234:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.488235:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.488236:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.488237:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.488316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677448.488322:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677448.488325:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677448.488326:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677448.488328:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.488329:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.488331:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677448.488332:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.488334:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.488334:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.488336:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677448.488339:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.488340:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.488342:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.488343:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.488343:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.488345:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.488345:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:4.2:1619677448.488346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677448.488346:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677448.488346:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:9.2:1619677448.488352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:10.0:1619677448.488354:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.488357:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.488357:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:18.0:1619677448.488359:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:10.0:1619677448.488359:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.488361:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:10.0:1619677448.488361:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.488362:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.488363:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.488364:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.488364:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.488365:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.488366:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:10.0:1619677448.488367:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.488368:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.488369:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.488372:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.488373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:10.0:1619677448.488374:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.488377:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.488378:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.488379:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.488380:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:10.0:1619677448.488381:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.488381:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.488382:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.488383:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677448.488383:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.488384:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.488384:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.488385:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.488385:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:8.2:1619677448.488387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.488396:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.488398:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.488399:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.488401:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.488401:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.488404:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.488440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:21.2:1619677448.488443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:18.0:1619677448.488445:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.488448:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.488449:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:24.0:1619677448.488450:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:18.0:1619677448.488451:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.488453:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:18.0:1619677448.488453:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.488454:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:8.0:1619677448.488454:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:18.0:1619677448.488456:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.488458:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.488458:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.488459:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.488460:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677448.488461:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.488462:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.488464:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96714284] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.488464:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.488466:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.488467:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.488467:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:24.0:1619677448.488468:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:18.0:1619677448.488470:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.488471:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:24.0:1619677448.488472:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.488472:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.488473:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677448.488474:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677448.488474:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.488474:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.488475:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:24.0:1619677448.488477:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.488478:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:25.2:1619677448.488479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677448.488479:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.488480:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.488481:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.488482:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.488483:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.488484:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677448.488487:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.2:1619677448.488487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.488489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:12.0:1619677448.488490:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.488490:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.488492:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.488494:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:13.0:1619677448.488494:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.488495:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677448.488496:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:20.0:1619677448.488497:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.488498:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.488498:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:25.2:1619677448.488499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:20.0:1619677448.488499:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.488499:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.488500:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677448.488500:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.488502:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677448.488506:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.488509:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.488509:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.488510:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.488511:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.488512:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.488670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.488678:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.488680:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.488683:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.488685:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.488685:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.488686:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.488688:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.488689:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.488690:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.488691:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.488694:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.488696:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.488698:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.488699:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.488699:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.488699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:31.0:1619677448.488701:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.488702:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.488702:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.488703:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677448.488704:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:25.2:1619677448.488706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.488707:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.488709:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.488709:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.488712:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.488713:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:5.0:1619677448.488714:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:12.0:1619677448.488715:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:5.0:1619677448.488715:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.488716:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.488717:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.488718:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.488718:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.488719:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677448.488719:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:12.0:1619677448.488720:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677448.488723:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.488725:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.488727:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.488728:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.488728:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.488730:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.488731:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.488731:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.488732:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677448.488736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.488745:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.488747:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.488748:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.488748:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.2:1619677448.488750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:17.0:1619677448.488751:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.488753:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677448.488753:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677448.488755:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.488757:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.488758:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.488760:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.488762:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.488763:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.488765:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.488766:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.488767:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.488770:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.488772:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.488775:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.488776:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.488777:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.488779:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.488779:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.488781:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.488782:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.488782:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.488784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.488787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677448.488790:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.488793:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.488793:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:13.0:1619677448.488794:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.488795:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:13.0:1619677448.488796:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.488796:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677448.488797:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.488797:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.488798:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.488843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.488848:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.488851:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.488852:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.488854:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.488856:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.488857:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.488858:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.488860:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.488861:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.488862:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.488865:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.488867:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.488868:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.488869:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.488870:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.488871:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.488872:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.488872:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.488873:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.488877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.488884:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.488886:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.488887:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.488888:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.488888:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.488890:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.488897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.488904:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.488906:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.488907:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.488907:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:4.2:1619677448.488908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:1.0:1619677448.488908:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.488908:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:24.0:1619677448.488915:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.488918:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.488919:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.488921:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.488924:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.488925:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.488927:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.488928:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.488929:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.488930:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.488933:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.488935:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.488938:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.488939:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.488939:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.488941:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.488941:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.488942:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.488942:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.488947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.488954:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.488957:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.488958:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.488958:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.488960:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.488963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.488966:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.488968:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.488970:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.488971:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.488971:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.488972:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.488973:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.488974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.488981:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.488983:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.488984:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.488986:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.488989:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.488991:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.488992:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.488994:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.488995:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.488996:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:5.2:1619677448.488997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:22.0:1619677448.488999:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.489001:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.489003:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.489004:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677448.489004:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.489005:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.489006:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:26.0:1619677448.489007:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:22.0:1619677448.489007:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.489007:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.489008:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.489008:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.489010:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.489013:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.489013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:26.0:1619677448.489014:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.489015:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.489017:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.489018:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.489019:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.489020:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:26.0:1619677448.489022:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.489023:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:26.0:1619677448.489024:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.489024:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.489024:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.489026:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.489026:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.489027:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.489028:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.489029:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:26.0:1619677448.489030:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.489030:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.489031:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.489032:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.489035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.489042:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.489044:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.489045:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.489046:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.489048:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.489049:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.489100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.489107:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.489110:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.489111:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.489112:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.489115:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.489117:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.489118:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714276] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.489119:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.489120:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.489121:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.489124:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.489126:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.489128:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.489129:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.489130:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.489131:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.489132:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.489133:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.489133:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.489137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.489145:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.489148:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.489149:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.489150:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.489151:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.489153:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.489268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.489271:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.489274:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.489276:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.489277:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.489279:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.489280:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677448.489281:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.489283:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.489284:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.489285:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677448.489288:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.489289:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.489292:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.489292:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.489297:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.489298:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.489299:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.489300:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.489301:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.489305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.489313:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.489315:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.489316:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.489317:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.489319:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.489320:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.489323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.489331:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.489333:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.489334:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.489335:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.489335:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.489337:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.489366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.489373:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.489376:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.489377:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.489379:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.489382:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.489383:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.489384:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.489385:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.489387:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.489387:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.2:1619677448.489387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677448.489391:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.489392:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.489394:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.489395:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.489396:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:1.0:1619677448.489396:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:26.0:1619677448.489397:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.489397:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.489398:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.489399:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677448.489399:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.489400:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.489401:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000400:00000200:1.0:1619677448.489401:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.489402:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677448.489403:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.489404:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.489405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:1.0:1619677448.489405:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677448.489407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677448.489407:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.489407:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.489411:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.489413:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:1.0:1619677448.489413:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.489415:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.489416:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:1.0:1619677448.489416:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.489416:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.489417:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.489417:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:1.0:1619677448.489417:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.489418:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.489418:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.489418:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.489419:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.489419:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:6.0:1619677448.489420:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.489420:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.489421:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.489422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.489425:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:25.2:1619677448.489426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.489426:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.489428:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.489429:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.489430:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.489430:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.489431:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677448.489433:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.489436:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.489437:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.489439:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.489441:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.489442:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.489474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.489482:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.489484:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.489486:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.489486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:31.0:1619677448.489488:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.489488:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.489490:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.489491:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.489491:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:31.0:1619677448.489492:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.489494:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.489494:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.489495:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:31.0:1619677448.489496:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:22.0:1619677448.489497:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.489500:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.489500:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.489501:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:31.0:1619677448.489502:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:22.0:1619677448.489503:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.489504:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.489505:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.489506:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.489506:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:22.0:1619677448.489506:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.489507:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.489509:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:22.0:1619677448.489510:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.489511:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:22.0:1619677448.489512:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.489513:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:25.2:1619677448.489514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.489515:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.489515:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:22.0:1619677448.489515:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.489516:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.489517:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.489519:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.489519:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.489520:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.489521:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:20.2:1619677448.489521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:22.0:1619677448.489522:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:13.0:1619677448.489522:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:25.2:1619677448.489524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:13.0:1619677448.489525:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.489526:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.489527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:13.0:1619677448.489527:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.489529:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.489530:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:13.0:1619677448.489530:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.489530:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.489531:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.489532:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:6.0:1619677448.489532:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:30.0:1619677448.489533:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677448.489535:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.489535:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:6.0:1619677448.489535:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:31.0:1619677448.489536:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.489536:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677448.489536:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:31.0:1619677448.489538:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.489541:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.489687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.489692:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.489695:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.489697:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.489698:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.489699:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.489700:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.489702:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.489703:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.489704:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.489705:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.489709:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.489711:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.489713:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.489714:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.489715:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.489717:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.489717:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.489718:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.489719:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.489722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.489729:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.489732:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:4.2:1619677448.489732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:30.0:1619677448.489733:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.489735:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.489736:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.489738:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.489739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.489742:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.489744:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.489747:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:10.0:1619677448.489747:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.489748:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677448.489750:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.489751:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.489753:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.489754:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.489755:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.489756:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.489757:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.489759:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.489760:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.489761:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.489761:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.489763:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.489763:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.489764:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.489765:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.489768:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.489769:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:26.0:1619677448.489770:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.489770:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.489771:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.489772:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.489773:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.489774:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.489774:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.489775:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.489775:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.489776:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.489776:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.489777:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.489777:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.489778:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:25.2:1619677448.489778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.489779:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.489779:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.2:1619677448.489781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:25.2:1619677448.489783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.489785:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677448.489787:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.489788:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.489789:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677448.489789:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:13.0:1619677448.489789:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677448.489790:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.489790:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.489792:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:6.0:1619677448.489792:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.489793:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.489793:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.489795:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.489795:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:17.0:1619677448.489796:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.489796:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:17.0:1619677448.489797:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:13.0:1619677448.489798:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:17.0:1619677448.489799:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.489800:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.489801:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.489802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:17.0:1619677448.489802:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.489806:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.489807:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:12.0:1619677448.489808:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.489810:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.489811:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.489811:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:12.0:1619677448.489811:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.489812:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.489812:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677448.489813:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:12.0:1619677448.489813:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.489814:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.489815:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677448.489815:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.489816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677448.489816:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.489817:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.489819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.489821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.489823:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.489826:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677448.489827:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.489827:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:31.0:1619677448.489828:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.489829:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:31.0:1619677448.489830:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.489830:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677448.489830:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:31.0:1619677448.489831:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:22.0:1619677448.489831:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.489831:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.489831:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.489832:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677448.489833:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.489833:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.489834:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.489836:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.489836:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.489836:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:31.0:1619677448.489840:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.489842:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.489844:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.489845:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.489846:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.489847:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.489848:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.489850:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.489851:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.489852:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.489853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.489857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.489859:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.489862:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.489863:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.489864:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.489865:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677448.489865:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.489866:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.489867:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.489868:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.489869:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677448.489896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.489901:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.489904:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.489906:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.489907:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.489909:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.489910:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.489912:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.489913:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.489914:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.489915:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.489918:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.489920:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.489922:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.489923:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.489924:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.489926:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.489927:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.489928:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.489929:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.489930:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:9.2:1619677448.489932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677448.489935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:18.0:1619677448.489940:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677448.489940:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:18.0:1619677448.489943:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:5.0:1619677448.489943:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.489944:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:2.0:1619677448.489944:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.489945:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.489945:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:18.0:1619677448.489947:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.489950:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.490042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.490049:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.490052:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.490053:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.490055:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.490058:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.490059:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.490061:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.490062:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.490063:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.490064:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.490067:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.490069:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.490071:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.490072:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:21.2:1619677448.490073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:10.0:1619677448.490073:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.490074:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.490075:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.490075:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.490075:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677448.490079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.490079:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.490081:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.490082:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.490084:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.490086:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:2.0:1619677448.490087:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.490088:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:2.0:1619677448.490088:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:12.0:1619677448.490089:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.490089:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.490091:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.490092:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.490092:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000400:00000200:2.0:1619677448.490092:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.490093:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:12.0:1619677448.490094:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677448.490097:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.490099:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.490102:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.490103:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.490104:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.490106:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.490106:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.490107:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.490108:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.490109:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.490111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.490114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677448.490118:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:18.0:1619677448.490119:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:26.0:1619677448.490121:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:18.0:1619677448.490121:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:10.0:1619677448.490121:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.490122:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.490122:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.490123:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:26.0:1619677448.490124:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.490127:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.490164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.490171:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.490173:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.490174:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.490176:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.490179:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.490180:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.490181:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.490182:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.490183:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.490184:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.490188:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.490189:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.490193:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.490194:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.490194:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.490196:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.490196:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.490197:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.490197:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.490202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.490207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:6.0:1619677448.490210:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.490213:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.490213:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.490214:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.490215:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:6.0:1619677448.490216:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.490218:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:6.0:1619677448.490218:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:30.0:1619677448.490220:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.490222:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.490222:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.490224:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.490225:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.490226:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.490228:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.490228:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.490232:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.490233:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.490235:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.490236:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.490237:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.490238:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.490239:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.490240:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.490240:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.2:1619677448.490241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:25.2:1619677448.490244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.490249:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677448.490250:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:26.0:1619677448.490251:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.490252:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:13.0:1619677448.490253:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.490253:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.490254:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.490255:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.490257:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.490257:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:26.0:1619677448.490258:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:13.0:1619677448.490258:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:26.0:1619677448.490260:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.490261:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.490262:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.490262:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:25.2:1619677448.490263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:26.0:1619677448.490265:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.490267:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:12.0:1619677448.490269:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.490270:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.490270:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.490271:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:12.0:1619677448.490272:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.490273:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.490273:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.490273:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:26.0:1619677448.490274:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:12.0:1619677448.490274:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.490274:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.490276:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.490276:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:12.0:1619677448.490276:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.490279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.490281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.2:1619677448.490285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677448.490286:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.490289:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:10.0:1619677448.490289:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677448.490289:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.490290:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.490291:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.490291:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.490291:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677448.490292:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.490293:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677448.490299:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.490299:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677448.490300:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:17.0:1619677448.490302:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.490305:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.490306:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.490308:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.490310:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.490311:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.490312:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.490315:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.490317:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.490319:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.490319:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.490320:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.490321:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.490322:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.490323:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.490323:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.490328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.490335:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.490336:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.490337:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.490338:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.490339:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.490340:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.490382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.490389:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.490391:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.490393:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.490394:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.490397:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.490398:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677448.490400:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.490401:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.490402:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.490403:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677448.490406:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.490407:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677448.490410:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.490410:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677448.490411:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677448.490413:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677448.490413:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.490414:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.490414:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.490419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.490427:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.490429:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.490430:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.490431:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.490433:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:6.0:1619677448.490434:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.490554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.490561:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.490564:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.490565:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.490566:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.490569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:26.0:1619677448.490570:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.490572:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.490573:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.490575:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.490576:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.490577:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:26.0:1619677448.490578:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.490579:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.490581:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.490583:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.490584:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.490585:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.490585:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.490586:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.490588:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.490588:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.490589:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.490593:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.490593:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.490594:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.490595:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.490595:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.490597:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.490597:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.490598:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.490598:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.490599:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.490599:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.490600:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.490601:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.490601:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.490602:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:25.2:1619677448.490603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.490603:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.490603:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.490607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:22.0:1619677448.490611:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:31.0:1619677448.490614:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:22.0:1619677448.490614:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.490614:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.490615:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.490617:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:22.0:1619677448.490617:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.490618:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.490619:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:31.0:1619677448.490620:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.490622:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.490623:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.490641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677448.490645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677448.490647:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.490649:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:30.0:1619677448.490652:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.490652:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.490652:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.490654:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.490655:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:5.0:1619677448.490656:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.490657:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.490658:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.490659:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:5.0:1619677448.490659:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.490660:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.490660:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.490661:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.490662:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.490663:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714254] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.490665:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.490665:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.490666:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.490667:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.490668:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.490670:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.490672:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.490674:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.490675:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.490676:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.490676:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.490677:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.490678:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.490678:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.490678:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.490679:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.490679:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677448.490680:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.490680:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.490681:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.490681:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.490684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.2:1619677448.490685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:13.0:1619677448.490691:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.490693:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:13.0:1619677448.490694:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.490694:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.490695:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.490696:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.490696:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.490698:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.490698:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.490698:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:13.0:1619677448.490699:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677448.490702:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.490933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.490941:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.490944:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.490945:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.490947:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.490949:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.490951:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.490952:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.490953:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.490954:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.490955:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.490958:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.490960:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.490962:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.490963:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.490964:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.490965:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.490966:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.490966:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.490967:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.490971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.490976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677448.490977:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.490979:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.490980:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.490982:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:12.0:1619677448.490982:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.490984:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677448.490985:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:12.0:1619677448.490985:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677448.490986:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.490988:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.490990:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.490992:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.490993:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.490995:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.490995:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.490997:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.491000:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.491002:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.491005:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.491006:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.491006:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.491008:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.491008:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.491009:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.491010:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.491014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.491016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:13.0:1619677448.491020:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.491022:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:13.0:1619677448.491023:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.491024:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.491024:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.491025:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:13.0:1619677448.491026:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.491026:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677448.491027:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.491029:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677448.491031:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.491033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:2.0:1619677448.491033:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.491034:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.491036:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.491036:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.491037:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:12.0:1619677448.491039:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:2.0:1619677448.491040:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.491042:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.491043:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.491043:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.491043:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677448.491044:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.491044:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.491045:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.491045:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.491045:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.491047:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.491047:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.491048:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.491049:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:21.2:1619677448.491051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:9.2:1619677448.491053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.491057:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677448.491060:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677448.491060:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.491061:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677448.491062:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.491063:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.491063:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.491064:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.491065:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.491066:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.491067:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:6.0:1619677448.491067:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.491069:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.491070:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.491071:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.491072:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.491076:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.491077:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.491080:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.491081:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.491081:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.491083:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.491083:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.491084:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.491085:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.491089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.491096:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.491099:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.491100:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.491100:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.491102:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.491105:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.491133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.491141:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.491144:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.491147:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.491148:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.491150:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.491151:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.491153:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.491154:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.491155:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.491156:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.491160:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.491162:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.491165:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.491166:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.491166:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.491168:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.491169:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.491170:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.491170:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.491174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.491182:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.491184:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.491185:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.491187:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.491187:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.491190:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.491225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.491233:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.491235:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.491238:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.491239:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.491241:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.491242:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.491244:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.491245:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.491246:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.491247:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.491250:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.491252:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.491255:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.491256:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.491257:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.491258:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.491259:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.491260:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.491260:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.491265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.491272:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.491275:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.491276:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.491276:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.491278:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.491281:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.491344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.491350:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.491353:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.491354:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.491356:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.491357:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.491359:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.491360:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714247] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.491361:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.491362:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.491363:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.491367:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.491368:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.491371:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.491372:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.491372:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.491374:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.491374:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.491375:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.491376:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.491381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.491390:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.491392:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.491393:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.491393:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.491396:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.491398:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.491400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:4.2:1619677448.491402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:6.0:1619677448.491407:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.491410:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:8.0:1619677448.491410:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677448.491410:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.491411:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.491412:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.491412:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677448.491413:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.491414:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.491416:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.491419:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.491421:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.491422:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.491424:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.491425:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.491426:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.491429:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.491431:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.491434:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.491434:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.491435:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.491437:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.491437:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.491438:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.491438:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.491443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.491450:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.491453:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.491454:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.491454:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.491456:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.491459:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.491498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.491506:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.491509:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.491510:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.491512:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.491514:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.491516:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.491517:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714245] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677448.491517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:10.0:1619677448.491518:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.491519:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.491521:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.491524:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.491525:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.491526:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.491526:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.491528:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.491530:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.491530:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.491531:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.491531:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677448.491532:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.491533:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.491533:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:26.0:1619677448.491535:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.491535:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.491536:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.491537:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.491538:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.491538:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.491539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677448.491540:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.491540:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.491542:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.491544:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.491544:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.491545:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.491547:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.491547:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.491547:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.491548:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.491549:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.491550:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677448.491550:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.491551:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.491551:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:25.2:1619677448.491552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:18.0:1619677448.491553:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.491555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.491557:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.491558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:31.0:1619677448.491560:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:10.0:1619677448.491562:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:31.0:1619677448.491563:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.491564:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.491564:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.491565:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.491565:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677448.491565:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677448.491566:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.491568:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:2.0:1619677448.491568:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.491569:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677448.491569:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.491569:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.491570:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.491571:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:25.2:1619677448.491572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.491579:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.491582:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.491583:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.491584:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.491585:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.491586:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.491615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:20.2:1619677448.491616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.491618:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:24.0:1619677448.491621:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677448.491621:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.491623:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.491624:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:5.0:1619677448.491624:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.0:1619677448.491625:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:20.0:1619677448.491626:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.491627:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.491628:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677448.491629:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.491629:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.491630:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.491631:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.491631:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.491632:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.491633:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677448.491634:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.491635:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.491637:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.491637:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.491640:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:24.0:1619677448.491642:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.491643:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.491643:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677448.491644:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.491644:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.491646:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677448.491647:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.491647:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.491647:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:24.0:1619677448.491648:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.491649:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.491650:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:24.0:1619677448.491651:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.491651:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677448.491651:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.491652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677448.491652:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.491653:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.2:1619677448.491656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:9.2:1619677448.491657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677448.491660:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.491661:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677448.491662:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.491663:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.491664:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677448.491664:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.491664:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:13.0:1619677448.491665:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.491665:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.491666:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.491667:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:13.0:1619677448.491667:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:18.0:1619677448.491668:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.491668:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.491670:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.491674:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.491785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677448.491787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.491792:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:21.2:1619677448.491793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:1.0:1619677448.491793:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.491796:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.491796:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:30.0:1619677448.491797:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677448.491799:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:22.0:1619677448.491799:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:1.0:1619677448.491800:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.491801:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.491802:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.491802:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677448.491803:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.491805:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.491805:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.491805:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.491806:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.491807:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:22.0:1619677448.491808:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.491808:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.491809:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.491809:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.491809:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.491811:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.491811:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.491811:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677448.491812:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.491813:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:22.0:1619677448.491813:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.491814:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.491815:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677448.491820:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.491820:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.491821:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.491823:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.491824:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.491825:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.491827:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.491827:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.491828:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:22.0:1619677448.491828:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.491828:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.491829:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677448.491829:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.491829:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.491830:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.491831:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.491831:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.491832:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677448.491832:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.491832:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.491833:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:22.0:1619677448.491833:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677448.491833:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.491834:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677448.491835:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:22.0:1619677448.491835:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677448.491836:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:25.2:1619677448.491836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.2:1619677448.491838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.491839:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.491840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.491841:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.491842:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.2:1619677448.491843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.491844:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.491844:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.491845:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.491845:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.491846:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.491847:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:12.0:1619677448.491847:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:31.0:1619677448.491848:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.491848:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677448.491848:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:31.0:1619677448.491849:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:17.0:1619677448.491849:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.491849:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677448.491849:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:12.0:1619677448.491850:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.491852:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:12.0:1619677448.491852:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.491853:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.491854:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:12.0:1619677448.491855:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.491863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.491871:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.491873:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.491876:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.491877:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.491878:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.491880:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.491881:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.491882:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.491883:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.491884:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.491888:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.491889:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.491892:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.491893:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.491893:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.491895:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.491896:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.491896:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.491897:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.491900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.491907:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.491909:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.491910:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.491911:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.491913:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.491914:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.492102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.492108:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.492111:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.492112:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.492114:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.492116:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.492117:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.492118:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.492120:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.492121:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.492122:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.492125:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.492126:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.492128:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.492129:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.492130:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.492131:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.492132:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.492132:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.492133:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.492137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.492144:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.492146:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.492147:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.492148:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.492149:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.492150:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.492174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.492180:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677448.492182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:1.0:1619677448.492183:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.492184:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.492185:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.492188:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.492189:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:1.0:1619677448.492189:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.492190:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.492192:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.492192:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:1.0:1619677448.492192:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.492193:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.492195:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:8.0:1619677448.492197:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.492199:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.492199:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.492201:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:8.0:1619677448.492202:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.492203:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96714235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.492204:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.492204:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677448.492205:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.492205:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.492205:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:8.0:1619677448.492206:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.492207:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.492207:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.492208:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.492209:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000400:00000200:8.0:1619677448.492210:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.492210:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677448.492212:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:25.2:1619677448.492213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:8.0:1619677448.492214:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.492215:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.492215:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.492215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.492217:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.492217:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.492218:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.492219:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:17.0:1619677448.492220:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677448.492221:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:25.2:1619677448.492222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.492223:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.492223:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.492223:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677448.492224:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.492224:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.492225:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.492226:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.492229:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:6.0:1619677448.492229:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677448.492231:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.492232:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.492234:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.492235:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.492236:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.492318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.492325:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.492328:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.492329:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.492330:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.492332:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.492334:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677448.492335:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.492337:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.492338:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.492339:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677448.492342:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.492344:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.492345:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.492346:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.492346:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.492348:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.492348:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.492349:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.492349:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.492355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.492362:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.492364:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.492364:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.492365:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.492366:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.492367:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.492474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.492480:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.492483:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.492484:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.492486:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.492488:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.492490:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.492491:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.492493:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.492493:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.492494:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.492498:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.492499:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.492502:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.492503:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.492504:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.492506:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.492506:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.492507:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.492507:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:21.2:1619677448.492509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:25.2:1619677448.492511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.492517:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.492519:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:31.0:1619677448.492520:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.492522:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.492522:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.492523:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.492524:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.492524:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.492525:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.492526:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:12.0:1619677448.492526:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:31.0:1619677448.492527:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.492527:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:31.0:1619677448.492529:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.492529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:31.0:1619677448.492530:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.492531:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677448.492534:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.492536:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:12.0:1619677448.492536:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.492538:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.492539:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.492539:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:31.0:1619677448.492540:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.492540:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:12.0:1619677448.492540:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.492540:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.492541:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.492542:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.492542:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.492543:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.492544:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.492548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.492554:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.492556:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.492558:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.492558:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.492560:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.492562:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.492587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677448.492593:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677448.492596:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677448.492597:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677448.492599:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.492601:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.492603:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:22.0:1619677448.492604:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.492606:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.492607:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.492608:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677448.492611:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.492613:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.492617:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.492617:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:22.0:1619677448.492618:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.492620:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677448.492621:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677448.492622:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677448.492622:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:4.2:1619677448.492623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:25.2:1619677448.492626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677448.492631:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:12.0:1619677448.492632:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677448.492634:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.492634:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:12.0:1619677448.492635:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.492636:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.492636:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677448.492638:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.492638:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.492639:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:8.0:1619677448.492640:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:12.0:1619677448.492641:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677448.492641:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.492642:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.492644:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.492645:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.492648:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.492650:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.492652:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.492653:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.492654:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.492655:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.492656:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.492657:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.492657:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.492662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.492670:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.492672:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.492673:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.492674:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.492676:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.492679:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.492710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.492716:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.492719:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.492720:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.492722:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.492724:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.492725:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677448.492726:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.492728:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.492729:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.492730:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.492733:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.492735:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.492737:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.492738:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.492739:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.492740:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.492741:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.492741:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.492742:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677448.492746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.492755:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.492757:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.492758:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.492760:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.492761:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.492763:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.492927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.492935:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.492937:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.492940:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.492941:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.492942:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.492943:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.492945:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.492946:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.492947:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.492948:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.492951:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.492953:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.492956:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.492956:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.492957:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.492959:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.492959:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.492961:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.492963:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.492963:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.492964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.492969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.492970:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.492973:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.492974:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.492975:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.492976:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.492977:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.492977:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.492979:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.492979:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.492980:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.492984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.492990:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.492993:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.492994:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.492994:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.492995:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.492996:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.492996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.493002:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.493005:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.493006:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.493008:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.493010:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.493011:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.493013:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.493014:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.493015:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.493016:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.493020:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.493022:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.493024:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.493025:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.493026:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.493028:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.493028:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.493029:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.493029:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.493033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.493040:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.493043:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.493044:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.493045:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.493047:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.493048:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.493118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.493124:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.493128:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.493128:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.493131:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.493133:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.493135:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.493136:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.493138:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.493139:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.493140:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.493144:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.493146:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.493149:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.493150:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.493150:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.493152:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.493152:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.493153:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.493154:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.493158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.493166:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.493168:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.493169:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.493169:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.493172:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.493174:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.493178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.493186:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.493188:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.493189:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.493189:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.493190:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.493191:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.493247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677448.493252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.493255:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:10.0:1619677448.493255:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.493257:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:26.0:1619677448.493259:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.493259:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:20.0:1619677448.493260:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.493261:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.493262:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.493262:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.493264:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.493264:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.493266:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.493267:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.493269:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.493269:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.493270:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.493271:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.493271:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.493272:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.493274:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.493276:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.493279:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.493279:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.493281:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.493282:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.493283:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.493284:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.493284:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.493285:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.493286:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.493286:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:10.0:1619677448.493286:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.493287:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.493288:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:10.0:1619677448.493288:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.493288:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.493289:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.493290:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.493292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:25.2:1619677448.493298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.493301:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.493304:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.493304:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.493306:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:6.0:1619677448.493306:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:30.0:1619677448.493308:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.493309:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:6.0:1619677448.493309:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.493310:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.493310:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.493312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:6.0:1619677448.493312:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.493317:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.493318:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.493320:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.493320:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.493322:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.493323:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.493323:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.493364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.493371:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.493374:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.493375:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.493377:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.493380:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.493381:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.493383:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.493384:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.493385:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.493386:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.493389:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.493391:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.493394:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.493395:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.493395:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.493397:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.493398:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.493399:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.493400:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.493401:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.493403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.493406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.493412:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677448.493413:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.493414:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.493415:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.493415:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.493416:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677448.493416:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.493417:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677448.493417:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.493420:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.493426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.493434:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.493437:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.493438:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.493439:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.493443:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.493444:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.493446:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.493447:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.493448:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.493449:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.493452:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.493452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677448.493454:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.493457:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.493458:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.493458:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.493460:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.493460:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.493461:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.493461:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:30.0:1619677448.493462:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677448.493462:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:30.0:1619677448.493464:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.493466:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677448.493466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.493467:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.493468:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677448.493469:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.493471:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.493471:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.493472:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:5.0:1619677448.493474:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:30.0:1619677448.493475:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.493476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:30.0:1619677448.493477:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:5.0:1619677448.493478:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677448.493479:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.493479:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.493480:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:30.0:1619677448.493480:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677448.493480:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.493481:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:5.0:1619677448.493481:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.493482:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:30.0:1619677448.493482:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.493483:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.493484:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.493484:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:5.0:1619677448.493484:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:6.0:1619677448.493485:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.493487:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.493488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:22.0:1619677448.493489:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.493491:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:8.2:1619677448.493491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000400:00000200:22.0:1619677448.493492:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.493493:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.493494:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.493496:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:20.0:1619677448.493498:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.0:1619677448.493498:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:22.0:1619677448.493499:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.493500:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.493501:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:22.0:1619677448.493501:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:20.0:1619677448.493501:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.493502:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000400:00000200:8.0:1619677448.493502:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.493503:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677448.493504:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.493504:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:8.0:1619677448.493504:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.493504:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677448.493505:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.493506:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:8.0:1619677448.493506:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677448.493508:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.493508:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.493509:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.493509:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.493513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.493520:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.493523:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.493523:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.493524:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.493526:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.493529:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.493541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.493547:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.493550:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.493550:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.493552:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.493555:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.493557:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.493558:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.493559:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.493560:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.493561:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.493565:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.493567:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.493569:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.493570:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.493571:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.493573:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.493573:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.493574:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.493574:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.493579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.493586:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.493588:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.493589:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.493591:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.493593:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.493594:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.493893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.493895:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.493898:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.493900:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.493901:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.493903:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.493905:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.493906:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.493907:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.493908:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.493909:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.493912:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.493914:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.493917:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.493918:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.493919:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.493920:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.493921:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.493922:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.493922:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.493926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.493933:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.493935:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.493936:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.493938:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.493939:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.493940:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.493941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.493948:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.493951:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.493952:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.493955:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.493955:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.493956:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.493958:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.493959:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.493960:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.493961:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.493964:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.493966:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.493969:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.493970:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.493970:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.493972:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.493973:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.493974:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.493974:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.493978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.493986:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.493989:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.493989:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.493990:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.493992:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.493995:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.493996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.494002:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.494005:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.494005:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.494008:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.494011:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.494012:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.494013:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.494015:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.494016:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.494017:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.494020:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.494021:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.494023:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.494024:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.494025:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.494026:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.494027:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.494028:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.494029:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.494030:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.494032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.494035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.494041:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.494042:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.494042:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.494042:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.494042:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.494043:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.494044:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.494044:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.494045:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.494045:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.494115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.494122:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.494125:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.494126:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.494128:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.494131:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.494132:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.494134:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.494135:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.494136:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.494137:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.494140:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.494141:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.494145:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.494145:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.494146:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.494148:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.494149:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.494149:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.494150:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.494154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.494163:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.494166:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.494167:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.494168:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.494169:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.494172:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.494221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.494228:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.494231:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.494232:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.494233:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.494236:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.494237:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.494238:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.494240:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.494241:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.494241:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677448.494244:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.494246:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.494249:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.494250:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.494250:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.494252:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.494252:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.494253:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.494254:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.494258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.494266:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.494268:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.494269:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.494271:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.494271:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.494274:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.494275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.494282:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.494285:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.494286:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.494286:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.494287:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.494288:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.494404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.494410:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.494413:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.494414:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.494415:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.494417:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.494418:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.494420:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.494421:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.494422:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.494423:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.494426:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.494428:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.494429:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.494430:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.494431:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.494432:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.494432:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.494433:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.494433:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.494439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.494446:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.494448:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.494449:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.494449:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.494450:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.494452:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.494472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.494479:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.494481:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.494482:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.494484:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.494487:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.494488:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.494489:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.494491:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.494492:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.494493:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.494496:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.494497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677448.494498:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.494501:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.494501:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.494502:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.494503:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.494504:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.494505:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:10.0:1619677448.494505:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.494506:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.494507:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677448.494508:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.494508:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.494510:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.494510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.494512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677448.494513:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.494515:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.494516:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.494517:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.494517:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:10.0:1619677448.494518:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.494519:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:2.0:1619677448.494519:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:10.0:1619677448.494520:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:6.0:1619677448.494520:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.494521:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677448.494521:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.494523:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:10.0:1619677448.494523:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.494523:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.494525:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.494525:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.494527:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:10.0:1619677448.494528:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.494529:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.494530:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:9.2:1619677448.494531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:10.0:1619677448.494532:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.494532:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.494533:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.494533:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.494537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.494538:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.494541:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.494542:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.494543:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.494543:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.494544:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677448.494545:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.494548:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.494549:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.494551:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.494551:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.494555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.0:1619677448.494555:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:30.0:1619677448.494564:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.494567:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.494569:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.494571:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.494572:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.494573:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.494575:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.494576:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.494577:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.494578:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.494581:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.494583:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.494586:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.494587:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.494587:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.494589:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.494590:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.494591:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.494591:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.494595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.494603:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.494606:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.494606:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.494607:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.494609:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.494612:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.494839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.494846:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.494848:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.494849:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.494851:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.494854:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.494856:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.494857:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.494858:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.494859:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.494860:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.494863:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.494864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.494865:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.494867:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.494867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.494868:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.494869:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677448.494870:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.494871:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.494872:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.494872:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:26.0:1619677448.494873:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:17.0:1619677448.494873:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.494874:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.494875:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677448.494875:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.494876:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:25.2:1619677448.494877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:1.0:1619677448.494877:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.494878:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:26.0:1619677448.494879:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.494879:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:1.0:1619677448.494879:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677448.494880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677448.494881:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.494882:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.494882:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.494883:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:6.0:1619677448.494884:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:26.0:1619677448.494885:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.494886:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.494887:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:6.0:1619677448.494887:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.494888:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677448.494888:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.494889:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96714207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.494889:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:6.0:1619677448.494889:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.494890:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.494891:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.494891:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.494891:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.494892:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.494892:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677448.494892:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.494892:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:26.0:1619677448.494894:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.494894:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.494894:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.494895:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.494895:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.494896:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.494897:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:26.0:1619677448.494899:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.494900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.494901:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.494903:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.494904:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.494905:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.494906:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.494907:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.494907:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:26.0:1619677448.494908:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.494909:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:17.0:1619677448.494909:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:26.0:1619677448.494910:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:17.0:1619677448.494910:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.494911:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:25.2:1619677448.494912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:17.0:1619677448.494912:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.494914:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.494915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677448.494919:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.494922:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:10.0:1619677448.494922:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:13.0:1619677448.494923:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.494924:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:13.0:1619677448.494925:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.494925:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.494926:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.494926:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.494927:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.495089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.495095:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.495098:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.495099:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.495101:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.495103:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.495104:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.495106:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.495107:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.495108:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.495109:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:4.2:1619677448.495112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677448.495112:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.495114:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.495116:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.495116:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.495117:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.495119:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.495119:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.495119:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.495120:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:10.0:1619677448.495121:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:2.0:1619677448.495121:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:26.0:1619677448.495122:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677448.495122:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:10.0:1619677448.495124:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.495125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677448.495127:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.495128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:10.0:1619677448.495129:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.495130:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.495131:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.495132:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.495133:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:10.0:1619677448.495133:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:4.2:1619677448.495133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:18.0:1619677448.495135:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.495135:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.495135:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.495135:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.495136:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:10.0:1619677448.495137:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.495137:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.495138:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.495138:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677448.495140:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.495140:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.0:1619677448.495141:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677448.495142:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.495143:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:8.0:1619677448.495143:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.495144:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:10.0:1619677448.495144:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.495146:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.495146:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:8.0:1619677448.495146:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.495147:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.495147:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:8.0:1619677448.495149:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.495151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:8.0:1619677448.495151:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.495152:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96714204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.495153:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.495154:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.495155:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.495158:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:8.0:1619677448.495158:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.495160:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:8.0:1619677448.495160:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.495161:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.495163:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.495164:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.495164:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:8.0:1619677448.495164:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.495164:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.495166:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.495166:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677448.495167:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.0:1619677448.495167:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.495167:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.495172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677448.495178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677448.495180:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677448.495181:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.495182:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677448.495183:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:17.0:1619677448.495183:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.495184:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.495186:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.495186:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:1.0:1619677448.495186:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.495189:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.495189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677448.495190:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677448.495192:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.495193:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.495193:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:20.0:1619677448.495194:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.495195:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:1.0:1619677448.495195:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.495197:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.495198:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.495198:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000400:00000200:20.0:1619677448.495199:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.495199:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.495200:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677448.495201:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.495203:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.495204:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.495205:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.495207:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.495207:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.495208:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.495208:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.495213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.495220:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.495222:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.495223:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.495225:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.495225:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.495227:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.495360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.495368:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.495371:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.495372:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.495374:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.495376:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.495377:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.495379:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.495380:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.495381:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.495382:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.495385:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.495387:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.495389:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.495390:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.495390:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.495392:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.495392:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.495393:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.495393:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.495398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.495405:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.495408:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.495409:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.495410:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.495413:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.495413:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.2:1619677448.495441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.495449:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.495451:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.495453:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.495455:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.495455:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.495457:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:4.2:1619677448.495457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:30.0:1619677448.495458:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.495459:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.495460:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.495461:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.495464:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.495464:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.495465:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.495466:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.495467:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.495469:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.495469:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.495470:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.495470:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:10.0:1619677448.495471:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.495472:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.495472:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677448.495472:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677448.495473:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:10.0:1619677448.495473:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.495474:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:10.0:1619677448.495474:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.495475:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.495476:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.2:1619677448.495478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:10.0:1619677448.495479:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.495481:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.495484:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.495484:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.495485:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.495486:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:10.0:1619677448.495487:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.495487:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.495488:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.495488:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677448.495489:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.495489:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.495491:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.495492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.495492:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.495494:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677448.495500:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.495503:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:21.2:1619677448.495503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:22.0:1619677448.495504:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.495504:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.495506:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.495509:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677448.495510:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.2:1619677448.495512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677448.495513:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.495514:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.495515:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.495518:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.495519:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:24.0:1619677448.495520:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.495521:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.495522:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677448.495523:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.495523:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:24.0:1619677448.495524:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.495524:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000400:00000200:24.0:1619677448.495525:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:6.0:1619677448.495525:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.495525:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:24.0:1619677448.495528:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.495530:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.495533:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.495534:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.495535:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.495536:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.495537:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.495538:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.495539:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.495540:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.2:1619677448.495542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.495546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677448.495551:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.0:1619677448.495553:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.495555:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:5.2:1619677448.495555:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:24.0:1619677448.495556:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677448.495556:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677448.495557:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (23)++
00000800:00000200:20.0:1619677448.495558:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.495558:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (24)--
00000800:00000200:20.0:1619677448.495559:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (23)++
00000400:00000200:5.0:1619677448.495559:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.495560:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (24)--
00000800:00000200:20.0:1619677448.495561:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.495561:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.495561:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677448.495562:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.495564:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:2.0:1619677448.495564:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.495565:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.495566:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.495569:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.495570:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.495572:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.495573:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.495574:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.495575:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.495579:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.495581:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.495584:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.495584:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.495585:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.495587:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.495587:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.495588:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.495589:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.495590:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:25.2:1619677448.495593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.495595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677448.495600:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.495601:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.495603:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:12.0:1619677448.495603:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677448.495604:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.495604:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:12.0:1619677448.495605:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.495606:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.495608:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.495609:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.495613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.495619:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.495622:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.495623:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.495624:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.495624:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.495626:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.495713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.495720:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.495723:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.495724:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.495726:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.495729:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.495730:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.495732:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.495733:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.495734:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.495735:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.495738:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.495740:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.495743:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.495743:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.495744:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.495746:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.495746:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.495747:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.495747:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.495751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.495757:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.495760:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.495760:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.495762:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.495763:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.495764:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.495776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.495779:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.495781:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.495784:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.495785:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.495786:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.495787:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.495789:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.495790:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.495791:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.495792:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.495795:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.495797:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.495799:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.495800:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.495801:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.495802:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.495803:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.495804:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.495804:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.495808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.495816:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.495818:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.495819:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.495819:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.495822:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.495823:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.495831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.495838:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.495841:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.495842:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.495844:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.495846:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.495848:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.495849:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.495850:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.495852:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.495853:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.495856:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.495858:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.495861:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.495862:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.495863:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.495865:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.495865:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.495866:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.495866:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.495871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.495879:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.495881:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.495882:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.495883:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.495885:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.495888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.495891:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.495892:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.495894:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.495895:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.495896:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.495896:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.495897:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.495929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.495937:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.495940:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.495942:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.495944:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.495945:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.495947:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.495948:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.495949:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.495950:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.495951:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.495954:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.495955:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.495958:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.495958:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.495959:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.495961:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.495961:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.495962:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.495963:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.495966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.495973:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.495976:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.495976:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.495978:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.495979:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.495980:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.496062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.496070:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.496072:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.496074:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.496076:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.496077:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.496078:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.496079:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.496080:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.496081:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.496082:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.496086:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.496087:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.496089:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.496090:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.2:1619677448.496090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.2:1619677448.496090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677448.496091:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.496092:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.496092:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.496094:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.496096:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.496096:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.0:1619677448.496096:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:26.0:1619677448.496097:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:25.2:1619677448.496098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.496099:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:26.0:1619677448.496100:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:21.2:1619677448.496102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.496102:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:10.0:1619677448.496102:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:5.0:1619677448.496103:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.496104:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.496104:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.496105:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.496106:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.496107:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.496108:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.496108:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.496110:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:5.0:1619677448.496110:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.496111:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:31.0:1619677448.496113:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.496113:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677448.496113:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.496113:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.496114:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:5.0:1619677448.496114:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.496116:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.496116:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.496117:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.496117:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.496118:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.496122:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000400:00000200:26.0:1619677448.496124:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.496125:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.496127:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.496128:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.496128:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.496129:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.496129:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.496129:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.496130:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.496130:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.496130:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.496131:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (23)++
00000800:00000200:5.0:1619677448.496132:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.496132:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:26.0:1619677448.496133:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (24)--
00000800:00000200:26.0:1619677448.496133:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.496133:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (24)++
00000800:00000200:5.0:1619677448.496133:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:26.0:1619677448.496134:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.496134:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (24)--
00000800:00000200:5.0:1619677448.496134:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:26.0:1619677448.496135:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.496135:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.496136:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:25.2:1619677448.496137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:25.2:1619677448.496141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.496144:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.496146:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.496147:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.496147:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:31.0:1619677448.496149:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:26.0:1619677448.496149:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.496149:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.496150:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.496151:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:26.0:1619677448.496151:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.496152:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:26.0:1619677448.496153:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.496241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.496248:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.496251:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.496252:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.496253:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.496255:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.496257:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.496258:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.496260:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.496261:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.496262:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.496265:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.496267:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.496270:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.496270:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.496271:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.496273:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.496273:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.496274:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.496274:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.496279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.496287:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.496289:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.496290:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.496290:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.496293:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.496300:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.496321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.496329:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.496332:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:21.2:1619677448.496334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:30.0:1619677448.496335:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.496336:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.496337:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.496338:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.496340:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.496340:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:30.0:1619677448.496341:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.496343:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.496343:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.496343:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:30.0:1619677448.496344:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:22.0:1619677448.496347:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.496349:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.496350:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.496351:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:22.0:1619677448.496352:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.496353:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.496354:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.496355:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:22.0:1619677448.496355:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.496356:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:22.0:1619677448.496356:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.496357:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:22.0:1619677448.496357:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.496358:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.496359:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.496359:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:22.0:1619677448.496360:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.496362:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:25.2:1619677448.496363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677448.496364:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.496365:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.496366:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.496367:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.496368:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.496368:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.496369:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677448.496371:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.496373:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.496374:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.496374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:12.0:1619677448.496376:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.496378:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.496379:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.496382:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.496386:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.496387:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.496388:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.496388:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.496389:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.496390:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.496392:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.496392:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.496394:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:5.0:1619677448.496394:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.496395:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.496396:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.496397:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.496399:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.496400:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.496401:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.496404:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.496406:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.496409:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.496410:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.496411:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.496412:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.496413:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.496414:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.496415:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.496416:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.496418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.496421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.496424:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677448.496425:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.496426:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.496427:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.496427:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.496427:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.496428:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.496430:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.496431:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.496433:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.496487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.496495:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.496498:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.496498:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.496500:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.496502:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.496503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000400:00000200:10.0:1619677448.496504:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:20.0:1619677448.496505:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000400:00000200:10.0:1619677448.496505:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.496506:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.496507:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.496508:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:10.0:1619677448.496508:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:20.0:1619677448.496511:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.496511:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:10.0:1619677448.496513:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.496514:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.496515:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:20.0:1619677448.496516:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677448.496517:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.496518:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.496519:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.496519:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.496520:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.496520:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.496520:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.496522:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:20.0:1619677448.496524:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.496525:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677448.496526:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.496527:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:25.2:1619677448.496528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.496528:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.496528:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677448.496529:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.496530:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.496530:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.496532:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.496532:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677448.496533:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677448.496534:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:5.2:1619677448.496534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.496536:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:25.2:1619677448.496538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:6.0:1619677448.496539:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.496540:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.496540:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.496542:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.496543:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.496544:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.496545:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.496545:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.496546:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677448.496546:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:26.0:1619677448.496547:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.496547:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.496548:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.496550:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.496553:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.496563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.496570:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.496572:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.496573:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.496575:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.496577:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.496578:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.496580:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.496581:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.496582:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.496583:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.496586:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.496586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:26.0:1619677448.496587:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.496589:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.496590:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.496590:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.496592:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.496592:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.496592:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677448.496593:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.496594:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.496595:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677448.496595:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.496596:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.496597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:2.0:1619677448.496598:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.496600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677448.496601:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.496602:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.496604:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.496605:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:2.0:1619677448.496605:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.496606:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:2.0:1619677448.496606:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.496607:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.496607:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.496609:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.496609:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.496610:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.496610:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.496611:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:2.0:1619677448.496611:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.496612:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.496612:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:2.0:1619677448.496613:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.496615:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.496616:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.496616:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.496618:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.496618:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.496619:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.496620:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.496624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.496631:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.496633:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.496634:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.496635:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.496636:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.496637:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.496660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.496668:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.496671:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.496673:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.496675:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.496676:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.496677:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.496678:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.496680:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.496681:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.496682:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.496685:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.496687:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.496690:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.496690:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.496691:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.496693:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.496693:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.496694:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.496695:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.496698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.496704:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.496707:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.496707:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.496709:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.496711:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.496712:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.496847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.496853:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.496855:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.496857:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.496858:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.496860:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.496861:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.496862:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.496863:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.496864:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.496865:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.496868:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.496870:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.496872:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.496872:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.496873:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.496874:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.496875:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.496876:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.496876:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.496880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.496887:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.496889:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.496890:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.496891:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.496892:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.496893:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.496921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.496929:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.496932:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.496934:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.496936:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.496936:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.496937:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.496939:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.496940:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.496941:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.496942:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.496945:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.496946:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.496949:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.496949:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.496950:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.496951:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.496952:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.496954:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.496955:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.496956:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.496957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:21.2:1619677448.496960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:20.2:1619677448.496962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677448.496964:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:22.0:1619677448.496965:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677448.496966:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.496967:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.496968:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.496968:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:13.0:1619677448.496969:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.496970:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:22.0:1619677448.496970:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.496970:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.496971:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.496973:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.496973:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:22.0:1619677448.496973:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.496974:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:22.0:1619677448.496974:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.496976:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.496977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:22.0:1619677448.496977:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.496978:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.496979:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.496982:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.496984:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:12.0:1619677448.496984:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.496986:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.496987:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.496987:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:22.0:1619677448.496988:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.496988:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:12.0:1619677448.496988:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.496989:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677448.496990:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:13.0:1619677448.496990:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677448.496991:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.496992:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.496992:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.496997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.497004:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.497006:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.497007:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.497009:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.497009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.497010:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.497012:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.497015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.497015:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.497018:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.497019:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.497021:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.497022:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:1.0:1619677448.497023:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.497025:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.497025:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:30.0:1619677448.497027:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.497027:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:1.0:1619677448.497027:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.497028:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:1.0:1619677448.497028:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.497029:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:1.0:1619677448.497029:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.497030:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.497033:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.497035:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.497037:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.497038:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.497039:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.497040:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.497040:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.497041:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.497042:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.497043:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.497048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.497048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.497053:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.497056:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.497056:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.497056:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.497057:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.497058:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.497060:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.497060:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.497062:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.497064:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.497065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:22.0:1619677448.497072:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.497075:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.497075:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.497076:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.497077:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.497077:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.497108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.497113:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.497116:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.497118:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.497119:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.497121:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.497123:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.497124:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.497126:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.497127:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.497128:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.497131:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.497133:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.497136:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.497137:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.497137:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.497139:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.497140:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.497141:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.497141:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.497145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.497151:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.497154:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.497155:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.497157:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.497158:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.497159:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.497163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.497170:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.497172:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.497173:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.497174:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.497174:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.497175:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.497281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.497289:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.497291:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677448.497299:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.497301:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.497303:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.497304:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.497306:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.497307:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.497308:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.497309:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.497313:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.497314:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.497318:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.497319:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.497319:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.497321:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.497322:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.497323:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.497323:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.497328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.497336:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.497339:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.497341:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.497341:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.497343:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.497346:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.497384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.497392:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.497396:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.497397:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.497399:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.497401:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.497403:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.497404:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.497405:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.497406:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.497407:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.497410:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.497412:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.497414:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.497415:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.497416:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.497417:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.497418:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.497418:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.497419:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.497423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.497430:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.497432:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:5.2:1619677448.497432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:12.0:1619677448.497433:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.497435:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.497437:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.497438:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.497440:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.497443:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.497444:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.497445:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.497448:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.497449:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.497450:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.497452:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.497453:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.497454:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.497457:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.497458:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.497460:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.497461:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.497461:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.497463:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.497463:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.497464:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.497465:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.497468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.497475:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.497478:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.497478:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.497480:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.497482:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.497483:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.497556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.497563:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.497566:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.497567:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.497568:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.497569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:26.0:1619677448.497571:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.497572:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.497573:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.497575:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.497575:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.497575:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:26.0:1619677448.497576:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.497578:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.497579:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:26.0:1619677448.497580:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.497582:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.497584:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:26.0:1619677448.497585:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.497585:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.497586:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.497587:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677448.497587:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.497588:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.497589:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.497589:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:2.0:1619677448.497589:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.497590:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:2.0:1619677448.497590:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.497591:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:2.0:1619677448.497591:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:9.2:1619677448.497595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677448.497595:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.497596:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.497600:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.497601:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.497602:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.497604:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.497604:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.497605:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.497605:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:18.0:1619677448.497606:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.497606:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.497607:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:18.0:1619677448.497608:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.497608:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:25.2:1619677448.497610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.497611:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.497612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677448.497617:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:24.0:1619677448.497619:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677448.497620:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.497620:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.497621:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.497622:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.497622:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677448.497623:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:24.0:1619677448.497625:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.497625:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:24.0:1619677448.497627:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.497628:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.497630:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.497631:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.497632:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.497633:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.497636:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.497637:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.497640:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.497641:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.497642:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.497643:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.497644:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.497645:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.497645:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.2:1619677448.497646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:8.2:1619677448.497649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.497651:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.497654:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.497655:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.497657:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.497657:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:6.0:1619677448.497660:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.497660:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.497661:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:6.0:1619677448.497661:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677448.497661:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.497663:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.497664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:1.0:1619677448.497664:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.497665:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.497666:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.497667:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.497667:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677448.497668:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.497670:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.497670:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:6.0:1619677448.497671:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.497673:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:1.0:1619677448.497673:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.497676:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.497676:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.497677:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.497677:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:1.0:1619677448.497677:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:6.0:1619677448.497679:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.497679:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:1.0:1619677448.497679:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.497680:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677448.497680:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.497681:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:1.0:1619677448.497681:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.497682:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:8.2:1619677448.497685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:1.0:1619677448.497685:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.497687:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.497690:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.497691:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.497692:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.497693:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:1.0:1619677448.497694:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.497694:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.497695:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.497695:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677448.497696:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.497696:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.497698:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.497699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677448.497699:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.497701:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677448.497706:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.497709:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.497710:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.497711:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.497712:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.497714:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.497812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.497818:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.497821:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.497821:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.497824:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.497826:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.497827:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.497828:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.497830:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.497831:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.497832:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.497835:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.497837:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.497838:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.497839:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.497840:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.497841:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.497842:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.497843:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.497843:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.497847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.497854:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.497857:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.497857:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.497858:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.497859:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.497860:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.497866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:21.2:1619677448.497869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:2.0:1619677448.497873:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.497875:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.497876:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:2.0:1619677448.497876:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.497877:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.497877:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:8.0:1619677448.497878:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:2.0:1619677448.497878:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677448.497879:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.497881:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.497884:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.497886:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.497887:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.497888:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.497889:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.497890:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.497894:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.497895:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.497898:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.497898:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.497899:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.497901:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.497901:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.497902:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.497902:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.497908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.497915:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.497918:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.497918:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.497920:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.497922:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.497923:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.497925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:20.2:1619677448.497925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.497931:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:12.0:1619677448.497932:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677448.497934:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:12.0:1619677448.497935:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.497936:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:12.0:1619677448.497936:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.497936:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:5.0:1619677448.497936:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:12.0:1619677448.497937:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.497938:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:5.0:1619677448.497939:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.497940:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677448.497942:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714166] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.497943:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.497944:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.497945:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.497949:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.497950:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.497953:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.497954:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677448.497955:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.497957:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.497957:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.497958:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.497959:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.497963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.497969:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.497971:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.497972:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.497973:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.497975:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:10.0:1619677448.497978:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.497982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.497988:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.497990:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.497991:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.497991:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.497992:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.497993:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.498054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677448.498057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:20.0:1619677448.498063:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.498065:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:20.0:1619677448.498065:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:30.0:1619677448.498068:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:20.0:1619677448.498068:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.498068:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677448.498070:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.498071:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.498072:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677448.498073:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.498073:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.498074:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.498075:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.498076:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.498076:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.498077:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.498079:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.498079:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.498081:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.498082:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677448.498084:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.498086:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.498087:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.498089:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.498090:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.498091:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.498092:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.498092:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.498093:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.498094:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.498094:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.498094:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.498095:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.498095:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677448.498096:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:20.0:1619677448.498096:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.498098:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87b3e5800] (22)++
00000800:00000200:30.0:1619677448.498099:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:30.0:1619677448.498100:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:25.2:1619677448.498100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.2:1619677448.498102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.498105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:13.0:1619677448.498106:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.498109:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.498110:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.498110:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:13.0:1619677448.498111:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.498112:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.498112:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677448.498114:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)++
00000800:00000200:13.0:1619677448.498114:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677448.498114:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.498115:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.498116:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:20.0:1619677448.498116:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.498117:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677448.498118:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:5.0:1619677448.498119:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.498305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.498311:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.498314:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.498315:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.498317:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.498320:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.498321:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.498323:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.498324:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.498325:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.498326:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.498330:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.498331:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.498334:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.498335:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.498335:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.498337:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.498338:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.498338:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.498339:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.498344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.498351:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.498354:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.498355:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.498356:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.498358:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.498359:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.498383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:20.2:1619677448.498387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:21.2:1619677448.498393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.498395:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677448.498397:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000800:00000200:30.0:1619677448.498397:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.498398:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:4.2:1619677448.498398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:31.0:1619677448.498399:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.498401:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.498403:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.498404:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.498405:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:10.0:1619677448.498405:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.498406:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.498408:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.498408:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:31.0:1619677448.498409:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.498409:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677448.498411:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:4.2:1619677448.498411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:10.0:1619677448.498412:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.498413:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.498414:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.498414:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.498415:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.498415:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.498416:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.498416:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:31.0:1619677448.498417:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.498417:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.498418:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:24.0:1619677448.498418:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677448.498419:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.498420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:10.0:1619677448.498420:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.498421:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.498421:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:10.0:1619677448.498421:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.498422:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:30.0:1619677448.498422:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.498422:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677448.498423:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:24.0:1619677448.498424:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.498425:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.498425:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:30.0:1619677448.498425:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.498426:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (23)--
00000400:00000200:10.0:1619677448.498426:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.498427:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:24.0:1619677448.498427:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.498427:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:31.0:1619677448.498428:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.498428:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.498429:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677448.498429:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677448.498429:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:30.0:1619677448.498430:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.498430:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:24.0:1619677448.498431:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96714159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.498431:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:10.0:1619677448.498431:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677448.498431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677448.498432:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.498432:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.498432:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (22)++
00000800:00000200:10.0:1619677448.498432:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.498432:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.498433:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000400:00000200:24.0:1619677448.498433:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.498434:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.498434:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.498434:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:31.0:1619677448.498435:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.498435:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.498435:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.498435:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.498435:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.498436:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:10.0:1619677448.498436:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:31.0:1619677448.498437:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677448.498437:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:24.0:1619677448.498437:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.498437:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:31.0:1619677448.498438:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96714158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.498438:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.498438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.498438:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:30.0:1619677448.498439:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:2.0:1619677448.498439:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.498440:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.498440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:18.0:1619677448.498440:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.498442:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.498442:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:508, d:32, s:-96714158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.498442:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.498444:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.498444:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.498445:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.498446:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:18.0:1619677448.498446:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.498448:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.498448:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677448.498448:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:18.0:1619677448.498448:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.498449:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.498450:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.498451:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.498451:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.498452:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.498452:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:20.0:1619677448.498452:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.498452:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.498452:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.498453:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:22.0:1619677448.498453:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.498453:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.498453:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677448.498454:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.498454:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.498455:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:18.0:1619677448.498455:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.498456:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.498456:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677448.498456:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677448.498457:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.498457:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677448.498457:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.498457:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.2:1619677448.498458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.498459:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:25.2:1619677448.498459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.498460:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.498460:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677448.498460:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.498461:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677448.498461:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.498462:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.498462:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:22.0:1619677448.498463:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.2:1619677448.498463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.498464:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:25.2:1619677448.498465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:18.0:1619677448.498465:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:22.0:1619677448.498466:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.498466:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.498467:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677448.498467:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:6.0:1619677448.498467:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.498468:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.498468:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:22.0:1619677448.498469:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677448.498469:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677448.498469:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.498470:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.498470:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:1.0:1619677448.498470:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677448.498471:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.498472:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:13.0:1619677448.498472:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:1.0:1619677448.498473:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:21.2:1619677448.498474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:13.0:1619677448.498474:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:17.0:1619677448.498475:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.498475:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.498476:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.498478:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.498478:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000400:00000200:1.0:1619677448.498478:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.498479:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677448.498479:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.498480:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:12.0:1619677448.498481:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.498481:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:13.0:1619677448.498482:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.498482:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:13.0:1619677448.498483:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.498483:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.498484:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677448.498485:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.498489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.498496:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.498499:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.498500:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.498500:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.498501:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.498502:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.498626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.498632:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.498635:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.498635:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.498638:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.498640:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.498641:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.498643:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714156] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.498644:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.498645:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.498646:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.498649:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.498651:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.498654:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.498655:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.498656:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.498658:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.498658:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.498660:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.498661:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.498661:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:25.2:1619677448.498664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:21.2:1619677448.498666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.498670:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:22.0:1619677448.498671:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:26.0:1619677448.498672:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:22.0:1619677448.498673:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:10.0:1619677448.498673:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.498674:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.498674:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.498675:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:26.0:1619677448.498676:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.498679:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.498706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.498712:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.498715:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.498715:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.498717:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.498720:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.498721:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.498723:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.498724:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.498725:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.498726:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.498730:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.498731:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.498734:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.498735:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.498736:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.498738:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.498738:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.498739:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.498739:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.498744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.498750:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.498753:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.498754:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.498756:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.498757:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.498760:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.499014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.499021:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.499023:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.499024:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.499026:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.499029:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.499029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677448.499030:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.499031:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.499032:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.499033:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.499034:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.499034:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.499037:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.499037:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.499038:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.499039:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:1.0:1619677448.499041:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.499042:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.499043:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.499043:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:1.0:1619677448.499043:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.499045:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.499045:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677448.499045:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:10.0:1619677448.499046:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677448.499046:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.499047:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:1.0:1619677448.499047:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.499048:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.499049:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:25.2:1619677448.499051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:1.0:1619677448.499053:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.499054:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.499057:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.499058:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:6.0:1619677448.499059:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:1.0:1619677448.499059:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.499061:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.499061:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.499062:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.499062:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.499063:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.499063:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.499064:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.499064:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677448.499065:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.499067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:6.0:1619677448.499068:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.499069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.499073:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677448.499075:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677448.499076:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.499077:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.499077:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.499077:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677448.499077:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.499079:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.499079:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.499083:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.499204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.499210:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.499213:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.499213:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.499215:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.499218:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.499219:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.499220:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.499221:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.499222:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.499223:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.499227:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.499228:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.499231:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.499232:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.499233:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.499234:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.499235:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.499236:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.499237:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.499238:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.499241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.499243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.499249:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677448.499250:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:5.2:1619677448.499251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:1.0:1619677448.499251:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.499252:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.499252:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.499253:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.499253:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.499254:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.499254:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.499257:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:1.0:1619677448.499257:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677448.499260:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.499261:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.499263:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.499266:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.499268:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.499269:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.499271:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.499272:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.499273:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.499276:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.499278:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.499280:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.499281:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.499282:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.499283:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.499284:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.499285:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.499285:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.499289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.499300:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.499303:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.499304:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.499305:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.499306:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.499308:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.499310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.499317:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.499320:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.499320:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.499321:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.499322:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.499322:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.499333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.499338:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.499341:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.499342:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.499344:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.499346:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.499347:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.499348:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.499350:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.499350:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.499352:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.499354:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.499356:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.499358:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.499358:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.499359:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.499360:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.499361:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.499361:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.499362:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.499366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.499373:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.499375:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.499375:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.499376:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.499377:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.499378:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.499407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.499414:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.499417:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.499418:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.499420:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.499423:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.499425:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.499426:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714149] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.499427:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.499428:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.499429:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.499432:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.499434:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.499436:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.499437:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.499437:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.499439:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.499439:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.499440:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.499441:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677448.499442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:25.2:1619677448.499446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.499451:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.499452:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.499454:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.499455:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.499456:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.499456:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.499457:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.499458:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.499459:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677448.499460:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.499461:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.499462:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.499463:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.499465:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:31.0:1619677448.499466:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.499467:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.499467:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.499468:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.499471:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.499471:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.499472:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.499474:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.499474:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.499476:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.499477:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.499478:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677448.499480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.499483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.499487:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.499490:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:30.0:1619677448.499491:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:17.0:1619677448.499491:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.499491:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.499493:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:17.0:1619677448.499493:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.499494:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.499495:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677448.499496:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.499641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.499646:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.499649:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.499650:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.499651:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.499653:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.499654:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.499655:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.499657:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.499658:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.499659:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.499662:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.499664:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.499667:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.499668:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.499669:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.499670:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.499671:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.499671:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.499672:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.499676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.499683:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.499685:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.499687:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.499687:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.499689:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.499692:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677448.499756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.499763:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.499766:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.499767:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.499769:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.499771:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.499773:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.499774:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.499775:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.499776:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.499777:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.499781:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.499782:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.499785:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.499786:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.499786:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.499788:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.499788:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.499790:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.499791:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.499791:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.499794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.499797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.499802:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.499804:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.499805:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.499806:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677448.499806:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.499806:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.2:1619677448.499806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.499807:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.2:1619677448.499807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:22.0:1619677448.499808:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.499809:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.499810:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677448.499812:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677448.499812:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:6.0:1619677448.499812:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.499813:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677448.499814:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.499814:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:10.0:1619677448.499816:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.499817:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.499820:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.499820:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.499821:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:6.0:1619677448.499822:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:10.0:1619677448.499824:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.499824:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.499826:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.499826:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.499827:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.499828:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.499829:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:10.0:1619677448.499830:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:6.0:1619677448.499834:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.499835:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.499836:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:10.0:1619677448.499838:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677448.499839:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.499839:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:10.0:1619677448.499840:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.499840:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:10.0:1619677448.499841:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.499842:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:6.0:1619677448.499842:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.499843:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.499844:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.499844:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.499844:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.499844:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.499845:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.499846:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.499848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:25.2:1619677448.499851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:2.0:1619677448.499857:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:30.0:1619677448.499859:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:2.0:1619677448.499859:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.499860:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.499861:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:18.0:1619677448.499861:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:30.0:1619677448.499862:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.499863:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.499865:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.499866:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:2.0:1619677448.499866:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:31.0:1619677448.499868:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.499868:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.499869:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.499869:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.499870:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.499870:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.499958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.499964:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.499967:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.499967:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.499969:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.499970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:18.0:1619677448.499972:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.499973:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677448.499975:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.499976:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.499977:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.499978:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:18.0:1619677448.499978:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.499981:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:18.0:1619677448.499982:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.499985:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.499985:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.499985:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.499988:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.499988:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.499989:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677448.499989:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:30.0:1619677448.499990:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.499991:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.499991:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677448.499991:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:30.0:1619677448.499992:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.499992:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.499993:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:30.0:1619677448.499994:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.499995:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:8.2:1619677448.499997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:30.0:1619677448.499998:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.500000:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.500003:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.500004:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)++
00000800:00000200:30.0:1619677448.500005:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (23)++
00000800:00000200:5.0:1619677448.500006:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (24)++
00000800:00000200:30.0:1619677448.500008:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (25)--
00000800:00000200:30.0:1619677448.500008:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.500009:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677448.500009:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (24)--
00000800:00000200:30.0:1619677448.500010:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:5.0:1619677448.500010:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.500012:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (23)++
00000400:00000200:5.0:1619677448.500012:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.500013:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (24)--
00000400:00000200:20.0:1619677448.500014:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.500015:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.500016:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.500016:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.500018:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.500031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.500036:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.500039:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.500040:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.500043:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.500045:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.500046:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.500047:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.500049:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.500050:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.500051:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:4.2:1619677448.500052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:17.0:1619677448.500054:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.500056:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677448.500059:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.500059:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.500060:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.500060:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:26.0:1619677448.500062:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:17.0:1619677448.500062:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.500062:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.500063:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.500064:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.500064:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.500065:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.500068:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.500068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:26.0:1619677448.500069:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.500070:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.500072:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.500072:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.500073:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677448.500074:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:21.2:1619677448.500074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:26.0:1619677448.500077:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.500077:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:26.0:1619677448.500078:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:24.0:1619677448.500078:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.500078:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:26.0:1619677448.500080:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677448.500080:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.500081:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.500081:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.500082:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.500083:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677448.500083:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:24.0:1619677448.500083:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:26.0:1619677448.500084:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.500084:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.500085:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:31.0:1619677448.500086:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.500087:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.500089:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.500089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:31.0:1619677448.500090:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677448.500091:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.500092:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.500093:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.500094:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:30.0:1619677448.500096:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:31.0:1619677448.500097:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.500097:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.500098:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:30.0:1619677448.500098:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.500099:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.500101:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.500101:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.500102:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.500102:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:30.0:1619677448.500102:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.500104:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.500104:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.500105:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.500106:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.500109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.500116:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.500118:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.500119:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.500119:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.500121:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.500123:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.500129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677448.500135:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.500138:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.500139:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:21.2:1619677448.500139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:8.0:1619677448.500139:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.500140:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.500140:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:2.0:1619677448.500145:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.500148:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.500149:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.500151:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.500153:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.500155:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.500156:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.500157:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.500158:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.500159:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.500162:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.500164:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.500165:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.500166:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.500166:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.500168:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.500168:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.500169:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.500170:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.500174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.500181:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.500183:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.500183:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.500184:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.500185:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.500187:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.500440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.500448:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.500451:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.500452:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.500453:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.500456:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.500457:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.500458:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.500460:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.500461:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.500462:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.500465:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.500467:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.500469:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.500470:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.500471:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.500472:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.500473:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.500473:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.500474:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.500479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.500487:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.500489:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.500490:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.500491:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.500493:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.500494:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.500496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.500503:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.500505:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.500506:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.500507:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.500507:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.500508:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.500540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.500548:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.500550:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.500552:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.500554:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.500555:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.500556:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.500557:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.500558:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.500559:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.500560:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.500563:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.500564:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.500567:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.500567:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.500568:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.500569:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.500570:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.500571:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.500571:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.500576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.500583:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.500585:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.500586:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.500587:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.500589:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.500590:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.500593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.500599:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.500602:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.500604:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.500604:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.500607:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:5.0:1619677448.500611:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.500613:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:26.0:1619677448.500614:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:5.0:1619677448.500614:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.500615:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.500617:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.500617:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:5.0:1619677448.500617:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.500618:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.500621:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.500621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:5.0:1619677448.500622:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.500623:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.500624:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.500624:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.500625:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:20.0:1619677448.500625:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:26.0:1619677448.500627:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.500627:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.500628:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.500628:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:26.0:1619677448.500629:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.500629:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.500630:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.500630:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:26.0:1619677448.500631:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:20.0:1619677448.500631:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.500631:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.500631:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.500632:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:20.0:1619677448.500633:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:5.0:1619677448.500633:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.500635:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96714133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.500636:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.500636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:20.0:1619677448.500637:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.500638:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:20.0:1619677448.500638:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.500639:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.500642:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.500643:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:20.0:1619677448.500643:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.500644:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.500644:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.500645:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:20.0:1619677448.500645:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.500646:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.500646:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677448.500646:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.500647:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677448.500647:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.500648:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.500648:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.500649:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:22.0:1619677448.500650:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.500650:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:25.2:1619677448.500651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.0:1619677448.500652:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.500652:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:22.0:1619677448.500654:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677448.500654:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.500655:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.500655:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:22.0:1619677448.500656:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:25.2:1619677448.500658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.500658:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.500659:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.500659:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.500661:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.500661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677448.500664:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677448.500664:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677448.500666:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:30.0:1619677448.500667:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.500668:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.500668:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.500669:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:30.0:1619677448.500670:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.500670:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677448.500671:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.500673:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.500701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.500708:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.500711:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.500713:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.500715:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.500715:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.500717:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677448.500718:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.500719:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.500720:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.500721:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677448.500724:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.500726:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677448.500728:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.500729:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.500730:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.500731:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.500732:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.500733:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.500733:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.500737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.500745:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.500747:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.500748:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.500748:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.500750:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.500752:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.500906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677448.500913:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.500915:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677448.500916:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677448.500918:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.500921:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.500922:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677448.500924:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.500925:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.500926:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.500926:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.500930:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.500931:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.500934:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.500935:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.500936:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.500937:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.500938:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.500938:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.500939:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677448.500943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.500951:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.500954:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.500955:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.500957:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.500957:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.500960:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.500961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.500967:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.500970:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.500971:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.500973:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.500975:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.500977:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.500978:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.500979:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.500981:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.500981:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.2:1619677448.500982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:24.0:1619677448.500985:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.500986:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.500989:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.500990:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:24.0:1619677448.500990:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.500990:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.500992:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:24.0:1619677448.500992:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.500992:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.500993:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.500993:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:30.0:1619677448.500994:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.500996:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.500996:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.500998:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:8.2:1619677448.500998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677448.500999:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.501000:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.501001:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.501002:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.501005:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.501006:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.501006:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:30.0:1619677448.501008:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.501009:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.501009:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:30.0:1619677448.501010:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.501010:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.501010:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.501011:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.501012:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.501013:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:1.0:1619677448.501013:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.501014:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.501015:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:1.0:1619677448.501016:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:25.2:1619677448.501017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.501021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.501023:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.501026:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.501026:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.501028:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.501029:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:12.0:1619677448.501030:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.501031:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:13.0:1619677448.501031:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.501032:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.501033:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.2:1619677448.501048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.501055:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.501057:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.501058:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.501060:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.501063:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.501064:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.501066:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.501067:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.501068:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.501069:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.501072:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.501073:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.501075:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.501076:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.501076:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.501077:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.501078:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.501079:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.501080:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.501081:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.501083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.2:1619677448.501086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.501091:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.501093:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.501093:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.501095:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:10.0:1619677448.501095:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.501096:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.501097:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.501097:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.501098:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.501099:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677448.501191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.501193:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.501196:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.501198:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.501199:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.501201:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.501202:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.501204:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.501205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677448.501205:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.501206:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.501207:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.501210:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.501212:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.501213:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.501214:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.501215:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677448.501215:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.501215:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.501217:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.501217:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.501218:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.501219:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.501219:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:31.0:1619677448.501220:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.501220:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.501221:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677448.501222:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.501223:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.501223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:31.0:1619677448.501224:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.501225:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.501225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.501226:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.501229:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.501230:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:31.0:1619677448.501231:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.501231:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677448.501233:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.501233:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677448.501233:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:31.0:1619677448.501234:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.501234:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:6.0:1619677448.501234:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.501234:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677448.501235:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.501235:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677448.501236:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:6.0:1619677448.501236:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.501237:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.501238:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.501239:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.501239:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.501240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:25.2:1619677448.501244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.501247:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.501250:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.501251:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:12.0:1619677448.501251:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677448.501251:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.501252:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.501252:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677448.501253:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.501254:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.501256:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.501257:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.501258:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.501259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.501266:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.501269:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.501270:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.501271:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.501273:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.501275:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.501276:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.501277:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.501277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677448.501278:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.501279:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.501282:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.501282:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.501284:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.501285:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.501286:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.501287:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.501288:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.501288:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:17.0:1619677448.501288:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.501290:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.501290:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.501291:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.501291:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:17.0:1619677448.501291:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.501292:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:25.2:1619677448.501297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:17.0:1619677448.501299:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.501300:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.501302:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.501303:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.501305:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:17.0:1619677448.501306:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.501308:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:17.0:1619677448.501308:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.501309:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.501310:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.501311:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.501312:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.501312:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.501312:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.501313:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677448.501314:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.501315:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.501317:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.501318:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.501318:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.501320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.501324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.501327:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.501330:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677448.501330:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:13.0:1619677448.501331:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.501332:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.501332:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.501332:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.501333:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.501335:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.501335:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:21.2:1619677448.501363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.501370:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.501373:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.501374:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.501375:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.501378:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.501380:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677448.501381:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.501382:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.501383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:2.0:1619677448.501383:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.501385:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.501388:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.501390:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677448.501390:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.501392:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.501393:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:2.0:1619677448.501393:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.501393:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677448.501395:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.501395:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.501396:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.501396:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:31.0:1619677448.501397:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.501397:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:2.0:1619677448.501397:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:31.0:1619677448.501399:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.501400:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.501402:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677448.501402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677448.501403:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.501404:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.501407:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.501409:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.501411:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.501411:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:31.0:1619677448.501412:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.501412:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677448.501413:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:31.0:1619677448.501414:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:5.0:1619677448.501414:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.501415:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.501416:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677448.501416:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.501416:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.501417:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.501418:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:25.2:1619677448.501420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.501426:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.501429:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.501430:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.501431:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.501433:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.501434:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.501437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:13.0:1619677448.501444:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.501447:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.501448:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.501448:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.501449:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.501450:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.501462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.501468:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.501471:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.501471:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.501474:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.501476:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.501478:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.501479:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.501481:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.501482:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.501483:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.501486:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.501488:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.501491:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.501492:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.501493:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.501495:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.501495:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.501496:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.501497:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.501501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.501507:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.2:1619677448.501509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:30.0:1619677448.501510:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.501511:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.501513:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.501514:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:18.0:1619677448.501515:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:30.0:1619677448.501516:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677448.501518:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.501519:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.501521:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.501524:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.501525:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.501526:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.501527:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.501529:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.501530:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.501532:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.501534:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.501536:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.501537:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.501537:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.501539:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.501539:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.501540:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.501541:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.501541:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677448.501547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.501552:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.501554:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.501554:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677448.501555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.501557:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.501557:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677448.501559:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.501559:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.501560:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.501561:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.501563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677448.501563:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677448.501571:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.501574:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.501576:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.501578:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.501579:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.501580:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677448.501582:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.501583:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.501584:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.501585:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.501588:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.501590:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.501593:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.501594:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.501594:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.501594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677448.501596:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.501597:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.501598:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.501600:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.501600:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677448.501602:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.2:1619677448.501602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677448.501604:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.501605:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:21.2:1619677448.501606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677448.501607:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.501610:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:10.0:1619677448.501610:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.501611:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.501612:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.501613:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:10.0:1619677448.501613:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.501614:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:17.0:1619677448.501614:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.501614:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.501614:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:10.0:1619677448.501615:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.501616:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:17.0:1619677448.501616:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.501617:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.501618:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:10.0:1619677448.501619:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.501620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677448.501621:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.501622:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.501623:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:10.0:1619677448.501623:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.501623:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.501624:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:17.0:1619677448.501625:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677448.501625:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.501625:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.501626:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677448.501627:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.501628:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.501629:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:25.2:1619677448.501630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:17.0:1619677448.501630:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.501630:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677448.501636:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.501638:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.501639:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.501639:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.501641:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.501642:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.501708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.501715:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.501717:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.501718:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.501719:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.501721:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.501722:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677448.501723:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.501724:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.501725:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.501726:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.501729:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.501730:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.501732:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.501733:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.501734:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.501736:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.501736:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.501737:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677448.501738:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.501739:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677448.501742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.501745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.501752:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677448.501752:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:26.0:1619677448.501754:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.501754:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:20.0:1619677448.501755:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.2:1619677448.501755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.501756:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:20.0:1619677448.501756:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.501757:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677448.501760:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:1.0:1619677448.501760:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.501762:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677448.501763:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.501764:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.501765:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.501768:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.501770:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.501771:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.501773:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.501773:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.501775:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.501778:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.501780:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.501782:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.501783:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.501784:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.501785:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.501786:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.501787:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.501787:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.501791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.501799:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.501801:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.501802:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.501802:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.501804:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.501807:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.501835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.501837:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.501840:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.501842:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.501843:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.501844:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.501846:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.501847:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.501849:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.501850:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.501851:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.501855:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.501857:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.501859:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.501860:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.501861:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.501863:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.501863:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.501864:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.501865:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.501869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.501875:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.501878:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.501879:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.501881:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.501882:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.501884:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.501892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677448.501899:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.501902:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.501903:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.501904:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.501904:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.501906:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.501960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.501967:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.501970:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.501972:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.501974:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.501974:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.501976:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.501977:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.501978:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.501979:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.501980:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.501983:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.501985:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.501988:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.501989:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.501989:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.501991:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.501991:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.501992:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.501993:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.501996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.502003:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.2:1619677448.502003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677448.502005:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.502006:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.502007:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.502009:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.502009:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:13.0:1619677448.502010:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.502013:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.502014:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.502015:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.502017:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.502019:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.502020:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.502021:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.502022:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.502023:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.502026:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.502028:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.502031:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.502031:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.502032:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.502034:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.502034:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.502034:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.502035:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.502039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.502047:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.502049:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.502050:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.502052:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.502053:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.502055:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.502096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677448.502103:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.502106:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677448.502107:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677448.502109:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.502112:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.502113:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.502115:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714112] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.502116:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.502117:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.502118:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.502121:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.502123:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.502125:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.502126:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.502126:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.502128:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.502128:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.502129:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.502129:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677448.502133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.502140:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.502142:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.502143:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.502145:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.502147:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.502148:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.502250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.502257:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.502260:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.502261:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.502262:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.502264:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.502265:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.502267:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.502268:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.502269:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.502269:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.502272:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.502274:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.502276:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677448.502276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:26.0:1619677448.502277:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.502277:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.502279:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.502279:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.502280:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.502281:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:20.0:1619677448.502281:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:26.0:1619677448.502282:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.502282:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677448.502283:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.502284:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.502285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:20.0:1619677448.502287:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.502288:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:4.2:1619677448.502288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677448.502291:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.502292:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.502292:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.502297:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.502298:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:20.0:1619677448.502300:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.502301:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (23)--
00000400:00000200:20.0:1619677448.502301:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.502301:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.502302:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.502304:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.502305:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.502305:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.502306:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.502308:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677448.502309:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677448.502311:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.502312:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677448.502313:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677448.502314:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677448.502315:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.502316:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.502316:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.502321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.502330:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.502333:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.502334:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.502334:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.502336:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.502339:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.502371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:5.2:1619677448.502371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.502377:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.502379:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:17.0:1619677448.502380:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.502380:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677448.502382:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.502383:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:17.0:1619677448.502383:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.502385:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.502386:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.502387:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:8.0:1619677448.502388:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.502389:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.502390:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677448.502391:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.502391:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.502392:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.502393:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:8.0:1619677448.502393:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.502394:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.502395:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677448.502397:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.502399:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.502400:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:8.0:1619677448.502402:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.502403:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.502403:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.502404:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:8.0:1619677448.502405:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.502406:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.502406:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.502406:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:17.0:1619677448.502407:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677448.502407:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677448.502408:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677448.502409:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.502409:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.502410:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:25.2:1619677448.502411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:8.0:1619677448.502411:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.502415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677448.502419:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.502422:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.502423:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.502423:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:5.0:1619677448.502423:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:22.0:1619677448.502425:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.502425:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.502426:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.502427:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677448.502428:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.502428:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.502431:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.502433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.502441:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.502444:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.502445:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.502446:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.502446:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.502447:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.502497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:5.2:1619677448.502498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.2:1619677448.502502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677448.502503:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.502506:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677448.502506:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.502506:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:26.0:1619677448.502509:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:6.0:1619677448.502509:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.502510:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.502510:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.502512:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.502512:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.502513:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:31.0:1619677448.502514:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677448.502515:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.502515:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.502516:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.502517:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.502517:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677448.502518:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.502518:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.502519:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.502520:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.502520:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.502520:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677448.502521:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.502522:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.502522:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.502524:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.502525:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.502525:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.502527:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.502527:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.502528:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.502528:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.502530:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:6.0:1619677448.502531:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.502532:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:26.0:1619677448.502533:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.502533:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677448.502534:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.502534:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.502535:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677448.502535:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.502536:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.502536:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677448.502536:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.502536:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677448.502537:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.502537:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677448.502538:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.502538:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.502539:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.502539:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677448.502540:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.502541:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.2:1619677448.502541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.502542:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677448.502542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.502543:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.502543:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.502547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.502549:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:13.0:1619677448.502551:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677448.502551:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.502552:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.502553:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:13.0:1619677448.502553:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.502554:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.502554:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.502555:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:1.0:1619677448.502555:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:13.0:1619677448.502556:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:12.0:1619677448.502556:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.502558:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.502558:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:13.0:1619677448.502559:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.502560:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.502560:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.502561:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.502768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.502775:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.502778:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.502780:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.502782:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.502783:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.502785:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.502786:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.502787:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.502788:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.502789:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.502793:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.502793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:30.0:1619677448.502794:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.502796:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.502797:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.502798:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.502799:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.502799:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:30.0:1619677448.502801:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.502801:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:20.0:1619677448.502801:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.502802:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677448.502802:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677448.502803:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.502803:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.502805:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677448.502806:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.502807:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677448.502807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:20.0:1619677448.502808:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.502809:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677448.502813:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.502815:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.502815:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.502816:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.502817:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.502817:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.502818:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:5.0:1619677448.502818:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.502819:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.502820:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.502821:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.502821:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.502822:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.502822:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677448.502823:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.2:1619677448.502825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.502825:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.502827:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.502827:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.502828:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.502829:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677448.502830:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.502832:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.502833:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.502835:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.502836:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.502837:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.503024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.503029:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.503032:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.503033:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.503035:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.503037:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.503039:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.503040:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.503041:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.503043:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.503043:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.503047:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.503049:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.2:1619677448.503051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:17.0:1619677448.503052:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.503053:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.503054:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.503055:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.503056:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.503056:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.503057:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677448.503057:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:25.2:1619677448.503061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.503061:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.503062:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.503063:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.503065:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.503066:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:12.0:1619677448.503068:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:2.0:1619677448.503068:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.503069:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.503070:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.503071:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:2.0:1619677448.503071:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677448.503072:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.503074:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.503075:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677448.503075:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.503078:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.503078:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.503079:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677448.503080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.503080:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.503081:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.503083:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.503083:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.503084:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.503084:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677448.503086:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.503089:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:9.2:1619677448.503089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677448.503090:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.503090:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.503091:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.503092:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677448.503096:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.503098:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.503099:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.503100:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.503101:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.503103:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.503111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.503118:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.503121:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.503122:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.503122:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.503123:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.503123:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.503220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.503227:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.503230:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.503231:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.503233:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.503235:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.503237:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.503238:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.503239:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.503240:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.503241:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.503245:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.503246:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.503248:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.503249:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.503250:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.503251:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.503252:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.503252:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.503253:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.503257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.503264:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:21.2:1619677448.503264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:31.0:1619677448.503266:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.503267:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.503269:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.503269:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:30.0:1619677448.503270:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.503271:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677448.503272:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.503273:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.503275:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.503276:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.503277:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.503279:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.503280:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.503281:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.503282:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.503285:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.503287:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.503288:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.503289:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.503290:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.503291:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.503292:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.503292:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677448.503298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.503301:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677448.503303:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.503304:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.503305:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.503306:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.503306:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.503308:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.503382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.503388:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.503390:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.503391:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.503392:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.503394:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.503395:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677448.503396:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.503397:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.503398:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.503399:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677448.503401:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.503403:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677448.503405:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.503405:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677448.503406:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677448.503407:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677448.503408:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.503408:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.503409:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.503414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.503422:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.503424:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.503425:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.503425:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.503428:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.503430:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.503430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:25.2:1619677448.503432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:5.0:1619677448.503436:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.503438:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:6.0:1619677448.503440:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:21.2:1619677448.503441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:20.0:1619677448.503441:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.503441:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.503442:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.503444:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.503444:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000400:00000200:5.0:1619677448.503444:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.503445:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.503445:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.503446:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:5.0:1619677448.503446:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:22.0:1619677448.503447:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.503447:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:5.0:1619677448.503447:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.503449:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.503450:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.503450:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.503452:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:22.0:1619677448.503453:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.503454:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.503455:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.503456:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.503457:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.503458:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.503458:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:22.0:1619677448.503459:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.503460:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.503461:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.503462:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:22.0:1619677448.503463:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.503464:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:22.0:1619677448.503465:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.503465:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.503466:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:22.0:1619677448.503467:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.503467:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:22.0:1619677448.503468:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.503468:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677448.503469:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.503470:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.2:1619677448.503470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677448.503471:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.503472:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.503473:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:20.2:1619677448.503473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677448.503474:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.503476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.503479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.2:1619677448.503479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.503481:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.503483:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.503484:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:5.0:1619677448.503484:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.503484:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:5.0:1619677448.503485:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.503486:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:6.0:1619677448.503486:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:4.2:1619677448.503486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:5.0:1619677448.503487:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677448.503487:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677448.503488:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677448.503488:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.503488:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:1.0:1619677448.503488:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.503489:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.503490:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677448.503490:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:1.0:1619677448.503490:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.503491:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677448.503491:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.503492:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677448.503493:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.503493:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:1.0:1619677448.503493:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:24.0:1619677448.503494:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:5.0:1619677448.503495:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.503497:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.503497:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:5.0:1619677448.503497:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.503498:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.503500:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.503501:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.503501:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.503502:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.503504:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.503506:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677448.503506:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.503507:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.503508:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96714094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.503509:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.503510:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677448.503511:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.503511:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:24.0:1619677448.503512:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.503512:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.503514:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.503514:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.503515:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000400:00000200:24.0:1619677448.503516:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.503516:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:24.0:1619677448.503517:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.503520:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677448.503520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.503521:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.503522:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.503523:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.503524:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.503525:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.503525:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677448.503527:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.2:1619677448.503529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:18.0:1619677448.503531:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.503532:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.503532:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.503534:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.503537:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677448.503537:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.503540:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.503541:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.503542:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.503543:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.503546:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.503548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.503555:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.503558:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.503559:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.503559:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.503560:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.503561:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.503588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.503594:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.503596:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.503597:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.503599:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.503601:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.503603:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.503604:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.503605:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.503606:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.503607:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.503611:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.503612:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.503615:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.503616:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.503617:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.503619:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.503619:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.503621:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.503622:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.503622:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.503625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.503627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.503632:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677448.503633:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.503634:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:12.0:1619677448.503635:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677448.503635:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.503635:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677448.503636:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.503638:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.503640:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.503641:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.503751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.503759:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.503762:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.503763:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.503765:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.503767:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.503769:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.503770:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.503771:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.503772:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.503773:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.503776:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.503777:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.503780:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.503780:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.503781:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.503783:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.503783:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.503784:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.503784:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677448.503788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.503794:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.503797:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.503797:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.503799:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677448.503800:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.503801:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.503909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.503916:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.503919:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.503920:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.503921:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.503923:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.503925:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.503926:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.503927:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.503928:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.503928:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.503931:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.503933:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.503936:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.503937:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.503937:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.503939:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.503939:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.503940:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.503940:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.503945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.503953:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.503956:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.503957:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.503958:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.503960:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.503963:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.503977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.503983:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.503985:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.503986:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.503988:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.503990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:17.0:1619677448.503991:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.503993:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.503994:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.503995:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.503996:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:17.0:1619677448.503997:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.503997:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:21.2:1619677448.503998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:2.0:1619677448.503999:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.504000:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:17.0:1619677448.504002:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.504003:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677448.504003:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.504005:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.504006:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.504006:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:2.0:1619677448.504006:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.504007:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:17.0:1619677448.504008:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.504009:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.504009:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714089] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.504010:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.504011:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:6.0:1619677448.504011:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.504011:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.504012:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:6.0:1619677448.504012:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677448.504012:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.504013:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.504013:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:2.0:1619677448.504013:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.504014:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.504014:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.504016:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.504017:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.504017:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.504018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:6.0:1619677448.504019:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:2.0:1619677448.504020:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.504022:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.504023:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.504023:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:12.0:1619677448.504024:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.504024:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.504025:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677448.504026:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677448.504026:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:12.0:1619677448.504027:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.504027:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000400:00000200:12.0:1619677448.504028:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.504028:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.504028:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:6.0:1619677448.504029:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.504030:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:13.0:1619677448.504031:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.504031:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.504031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.504031:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:12.0:1619677448.504033:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.504033:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.504033:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:21.2:1619677448.504034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:6.0:1619677448.504034:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.504035:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.504039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677448.504040:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.504041:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.504041:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677448.504042:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.504043:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.504043:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.504044:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.504045:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:2.0:1619677448.504045:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.504046:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.504046:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:26.0:1619677448.504048:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.504048:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.504049:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.504051:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.504054:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.504103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.504110:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.504112:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.504113:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.504115:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.504118:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.504119:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.504120:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.504122:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.504123:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.504124:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.504127:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.504129:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.504132:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.504132:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:21.2:1619677448.504133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677448.504133:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.504134:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.504135:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.504136:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.504136:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.504140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.504144:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.504145:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.504147:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.504148:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.504149:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.504150:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.504150:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.504152:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.504152:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.504153:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.504154:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.504155:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.504156:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.504159:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.504160:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.504160:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.504161:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.504162:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.504164:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.504165:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.504165:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.504167:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.504167:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.504168:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.504169:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.504172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.504179:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.504181:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.504182:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.504184:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.504185:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.504186:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.504190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.504196:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.504198:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.504199:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.504200:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.504200:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.504202:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.504219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.504225:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.504228:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.504228:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.504231:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.504234:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.504235:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.504237:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.504238:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.504239:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.504240:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.504243:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.504245:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.504247:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.504248:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.504249:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.504250:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.504251:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.504252:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.504252:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.504256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.504263:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.504266:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.504266:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.504267:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.504269:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.504272:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.504310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677448.504315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677448.504317:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.504320:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.504320:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677448.504322:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677448.504323:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.504324:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:26.0:1619677448.504325:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.504325:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:26.0:1619677448.504327:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.504328:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.504328:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.504329:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.504330:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.504330:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.504331:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677448.504332:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.504333:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96714083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.504335:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.504335:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.504336:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.504337:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.504338:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:24.0:1619677448.504339:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.504341:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.504341:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677448.504341:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.504342:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.504344:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.504344:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677448.504344:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.504345:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.504345:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:24.0:1619677448.504345:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.504345:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.504347:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.504347:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.504348:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.504349:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677448.504350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.2:1619677448.504353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.504359:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677448.504361:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677448.504362:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.504363:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.504363:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677448.504364:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:22.0:1619677448.504365:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.504365:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.504365:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.504367:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.504368:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677448.504370:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.504392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.504400:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.504403:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.504405:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.504408:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.504408:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.504409:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.504411:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.504412:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.504413:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.504414:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.504417:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.504419:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.504422:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.504423:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.504423:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.504425:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.504426:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.504427:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.504427:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.504431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.504439:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.504442:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.504443:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.504444:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.504446:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.504447:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.504534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.504540:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.504544:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.504545:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.504546:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.504548:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.504549:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.504551:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.504552:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.504553:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.504554:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.504557:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.504559:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.504561:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.504561:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.504562:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.504563:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.504564:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.504565:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.504565:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.504570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.504576:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.504578:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.504579:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.504580:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.504581:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.504582:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.504589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.504596:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.504598:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.504599:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.504599:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.504600:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.504600:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.504602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.504608:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.504610:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.504612:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.504613:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.504616:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.504617:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.504618:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.504619:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.504620:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.504621:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.504625:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.504627:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.504630:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.504631:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.504631:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.504633:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.504633:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.504634:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.504634:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.504639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.504645:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.504648:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.504648:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677448.504649:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.504651:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.504654:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677448.504710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.504717:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.504719:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.504720:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.504722:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.504725:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.504726:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.504727:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.504728:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.504730:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.504731:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.504734:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.504736:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.504739:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.504740:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.504740:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.504742:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.504742:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.504743:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.504745:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.504745:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.504748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.504751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.504756:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.504758:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.504758:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.504759:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.504760:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677448.504760:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.504761:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.504762:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.504762:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.504765:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.504766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.504773:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.504775:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.504776:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.504777:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.504778:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.504778:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.504817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677448.504823:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.504826:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677448.504826:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677448.504829:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.504830:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.504832:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677448.504833:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.504834:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.504835:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.504836:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677448.504840:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.504841:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677448.504844:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.504845:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:18.0:1619677448.504846:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:18.0:1619677448.504847:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:18.0:1619677448.504848:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.504849:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677448.504849:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:25.2:1619677448.504853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.504860:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.504863:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.504864:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.504865:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.504866:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.504869:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.504999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.505004:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.505007:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.505008:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.505010:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.505013:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.505014:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.505015:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.505017:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.505018:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.505019:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.505022:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.505024:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.505027:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.505028:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.505029:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.505030:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.505031:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.505032:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.505033:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.505034:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.505036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.2:1619677448.505039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.505042:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:1.0:1619677448.505044:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677448.505045:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.505046:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.505046:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.505046:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.505046:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.505047:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677448.505048:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.505051:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677448.505052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677448.505056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:10.0:1619677448.505059:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.505061:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:8.0:1619677448.505061:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:26.0:1619677448.505062:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677448.505064:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677448.505065:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:10.0:1619677448.505065:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.505065:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.505066:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.505066:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:10.0:1619677448.505067:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.505068:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.505070:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.505071:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.505072:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.505073:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.505076:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.505078:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.505081:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.505082:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.505083:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.505084:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.505085:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.505085:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.505086:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.505090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.505098:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.505100:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.505101:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.505102:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.505104:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.505106:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.505190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.505197:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.505199:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.505200:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.505202:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.505205:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.505206:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.2:1619677448.505206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:24.0:1619677448.505207:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.505208:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.505209:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.505210:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.505212:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:24.0:1619677448.505214:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.505214:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:24.0:1619677448.505216:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.505217:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.505217:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:24.0:1619677448.505219:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.505219:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.505220:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677448.505220:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.505221:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:5.0:1619677448.505221:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:24.0:1619677448.505222:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.505223:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:5.0:1619677448.505223:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.505224:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.505225:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.505225:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:5.0:1619677448.505225:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.505226:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:8.2:1619677448.505227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677448.505230:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.505231:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:4.2:1619677448.505231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.0:1619677448.505234:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677448.505235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677448.505235:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677448.505235:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677448.505236:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677448.505237:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:17.0:1619677448.505238:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677448.505238:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:5.0:1619677448.505238:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:17.0:1619677448.505239:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.505239:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.505239:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:1.0:1619677448.505239:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.505240:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.505241:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677448.505241:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.505241:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677448.505243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:8.2:1619677448.505244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.505246:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677448.505247:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.505248:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:5.2:1619677448.505248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:17.0:1619677448.505249:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677448.505250:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.505251:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:17.0:1619677448.505251:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.505252:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:17.0:1619677448.505252:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000400:00000200:31.0:1619677448.505253:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.505253:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.505253:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.505253:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.505254:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677448.505254:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:31.0:1619677448.505255:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.505255:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000400:00000200:31.0:1619677448.505256:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.505257:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:31.0:1619677448.505258:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.505258:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:31.0:1619677448.505260:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.505261:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.505261:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.505264:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:2.0:1619677448.505264:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.505265:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.505266:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.505266:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:2.0:1619677448.505266:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:31.0:1619677448.505267:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.505268:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.505269:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.505269:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.505270:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:2.0:1619677448.505270:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.505271:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:2.0:1619677448.505271:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.505272:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.505273:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.505274:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:2.0:1619677448.505274:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.505276:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:2.0:1619677448.505276:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.505277:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.505278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677448.505278:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.505279:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.505279:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.505281:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.505281:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.2:1619677448.505282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677448.505282:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.505283:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677448.505286:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.2:1619677448.505287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677448.505288:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.505289:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.505290:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:13.0:1619677448.505290:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.505292:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:12.0:1619677448.505292:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.505293:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677448.505297:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.505301:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677448.505302:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:30.0:1619677448.505304:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:1.0:1619677448.505306:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.505307:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.505309:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.505311:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.505351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.505357:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.505360:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.505360:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:4.2:1619677448.505361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:22.0:1619677448.505362:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.505365:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.505366:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.505368:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.505369:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677448.505369:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.505370:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.505371:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:26.0:1619677448.505372:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.505372:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.505376:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.505377:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.505379:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:26.0:1619677448.505380:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.505381:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.505382:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:22.0:1619677448.505382:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.505382:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:26.0:1619677448.505383:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.505384:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000400:00000200:26.0:1619677448.505385:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.505385:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:26.0:1619677448.505386:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.505386:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.505386:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:26.0:1619677448.505387:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.505390:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.505391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:26.0:1619677448.505392:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.505396:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.505396:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.505397:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.505399:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.505399:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.505400:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.505400:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:18.0:1619677448.505400:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.505402:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.505403:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.505403:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.505405:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.505405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.505408:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.0:1619677448.505414:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.505416:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.505417:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.505419:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.505420:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.505422:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.505484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.505490:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.505493:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.505494:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.505495:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.505498:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.505499:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677448.505500:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.505502:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.505502:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.505504:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677448.505507:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.505509:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.505510:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.505511:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.505511:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.505513:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.505513:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.505514:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.505515:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.505516:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:9.2:1619677448.505519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.505521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.2:1619677448.505525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:18.0:1619677448.505527:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.505528:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.505528:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.505528:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.505529:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:18.0:1619677448.505530:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.505530:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.505531:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.505532:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:10.0:1619677448.505533:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.505533:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677448.505536:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.505537:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677448.505538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000400:00000200:10.0:1619677448.505539:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.505542:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.505544:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:18.0:1619677448.505545:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000400:00000200:10.0:1619677448.505545:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.505547:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.505548:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000400:00000200:10.0:1619677448.505548:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.505549:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.505549:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000400:00000200:10.0:1619677448.505549:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.505549:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.505550:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:10.0:1619677448.505552:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.505554:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.505558:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.505558:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.505559:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.505561:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.505562:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.505563:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.505564:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.505565:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677448.505567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.505570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.505575:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.505577:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.505578:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:18.0:1619677448.505578:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.505579:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.505580:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.505580:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.505581:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:2.0:1619677448.505581:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.505584:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.505620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.505626:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.505629:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.505630:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.505632:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.505634:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.505636:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.505637:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.505639:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.505640:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.505641:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.505644:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.505645:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.2:1619677448.505649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677448.505649:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.505649:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.505650:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.505652:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.505652:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.505653:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.505653:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.505657:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:25.2:1619677448.505659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.505660:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.505662:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.505664:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.505664:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.505665:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:30.0:1619677448.505665:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:31.0:1619677448.505667:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.505667:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.505667:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.505668:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.505669:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.505669:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.505670:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.505673:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.505673:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.505674:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.505675:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.505677:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.505678:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.505679:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.505680:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.505681:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.505682:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.505682:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.505686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.505692:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.505695:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.505696:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.505697:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.505699:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.505700:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.505705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:13.0:1619677448.505711:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.505713:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.505714:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.505715:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.505715:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.505716:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.505988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.505994:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.505996:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.505998:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.505999:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.506001:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.506003:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.506004:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.506005:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.506006:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.506007:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.506011:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.506013:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.506015:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.506016:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.506016:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.506018:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.506018:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.506019:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.506019:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.506024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.506031:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.506034:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.506035:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.506035:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.506037:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.506039:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.506074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.506082:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.506084:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.506087:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.506088:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.506089:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.506089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:30.0:1619677448.506090:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.506092:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.506093:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.506094:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.506095:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.506096:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677448.506098:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.506098:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.506099:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.506100:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.506101:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.506102:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.506103:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.506104:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.506104:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.506105:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.506105:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677448.506106:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.506107:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:10.0:1619677448.506107:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.506108:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.506109:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.506109:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677448.506109:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.506109:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:25.2:1619677448.506111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.506112:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.506114:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:21.2:1619677448.506115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.506116:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.506116:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:13.0:1619677448.506117:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.506117:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:13.0:1619677448.506119:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:10.0:1619677448.506119:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.506119:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:13.0:1619677448.506120:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.506120:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.506120:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:13.0:1619677448.506122:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.506123:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.506123:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.506124:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.506125:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.506125:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.506126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.506128:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677448.506129:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.506132:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.506133:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.506134:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.506136:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.506137:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.506163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.506170:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.506173:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.506175:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.506177:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.506177:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.506178:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.506179:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.506181:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.506181:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.506182:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.506185:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.506187:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.506188:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.506189:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.506190:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.506191:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.506192:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.506193:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.506193:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.506197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.506203:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.506205:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.506206:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.506207:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.506209:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.506210:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.506215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.506221:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.506224:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.506225:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.506225:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.506226:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.506227:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.506340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.506348:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.506350:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.506351:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.506353:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.506355:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.506356:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.506357:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.506359:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.506360:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.506360:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.506363:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.506365:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.506367:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.506368:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.506368:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.506370:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.506370:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.506371:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.506371:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.506376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.506383:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.506385:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.506386:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.506388:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.506390:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.506391:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.506428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.506436:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.506438:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.506439:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.506441:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.506444:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.506445:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.506447:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.506448:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.506449:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.506449:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.506452:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.506454:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:21.2:1619677448.506456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677448.506456:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.506457:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.506457:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.506459:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.506459:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.506460:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.506461:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.506462:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:31.0:1619677448.506464:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677448.506464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.506466:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.2:1619677448.506467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000400:00000200:31.0:1619677448.506468:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.506470:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.506471:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677448.506471:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677448.506471:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:31.0:1619677448.506472:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.506473:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.506473:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:31.0:1619677448.506474:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.506474:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:13.0:1619677448.506474:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.506476:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677448.506476:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:13.0:1619677448.506476:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.506477:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677448.506477:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:31.0:1619677448.506479:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.506479:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:31.0:1619677448.506480:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.506481:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.506483:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.506483:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.506484:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.506485:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.506486:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.506487:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.506487:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.506491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.506497:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.506500:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.506500:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.506502:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.506504:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.506505:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.506535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.506538:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.506541:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.506543:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.506544:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.506545:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.506547:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677448.506548:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.506550:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.506551:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.506552:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677448.506555:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.506557:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677448.506560:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.506560:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677448.506561:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677448.506563:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677448.506563:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.506564:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.506565:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.506569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.506577:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.506580:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.506581:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.506581:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.506583:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.506585:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.506634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.506640:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.506643:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.506644:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.506646:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.506648:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.506650:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.506651:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.506652:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.506654:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.506655:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.506658:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.506660:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.506663:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.506664:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.506665:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.506667:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.506667:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.506668:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.506668:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.506673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.506680:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.506683:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.506684:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.506686:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.506686:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.506689:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.506689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:25.2:1619677448.506693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:2.0:1619677448.506694:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.506697:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.506698:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677448.506699:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:2.0:1619677448.506700:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.506702:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677448.506702:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.506703:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.506704:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.506704:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:2.0:1619677448.506704:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.506705:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.506706:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:2.0:1619677448.506706:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.506707:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.506708:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.506712:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.506714:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.506716:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.506717:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.506718:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.506720:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.506720:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.506721:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.506723:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.506723:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:25.2:1619677448.506726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.506728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:5.2:1619677448.506728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:13.0:1619677448.506732:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.506735:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:13.0:1619677448.506735:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.506736:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.506736:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:18.0:1619677448.506737:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.506737:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:13.0:1619677448.506738:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.506738:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.506738:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.506739:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:18.0:1619677448.506739:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677448.506741:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677448.506741:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:25.2:1619677448.506743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:10.0:1619677448.506743:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.506745:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.506746:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.506747:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.506748:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.506749:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.506749:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:12.0:1619677448.506752:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.506752:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.506753:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:10.0:1619677448.506753:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:12.0:1619677448.506754:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.506755:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.506755:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.506758:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.506759:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.506759:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.506761:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.506761:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.506762:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.506763:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.506767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.506776:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.506778:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.506779:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.506780:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.506782:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:1.0:1619677448.506785:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.506853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.506860:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.506863:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.506865:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.506867:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.506868:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.506869:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.506871:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.506872:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.506873:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.506874:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.506878:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.506879:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.506882:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.506883:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.506884:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.506886:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.506886:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.506887:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.506888:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.506892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.506899:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.506902:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.506903:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.506904:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.506906:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.506909:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.506995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.507002:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.507005:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.507006:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.507008:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.507011:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.507012:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.507013:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.507014:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.507015:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.507016:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.507019:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.507020:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.2:1619677448.507020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.507022:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.507023:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.507023:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.507025:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.507025:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.507025:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.507026:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.507026:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677448.507028:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.507029:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.507030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:17.0:1619677448.507031:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.507033:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.507034:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.507036:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.507036:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:17.0:1619677448.507037:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.507038:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.507038:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:17.0:1619677448.507039:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.507039:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.507040:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.507042:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.507043:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.507043:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:17.0:1619677448.507046:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.507048:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.507049:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.507049:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.507051:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.507052:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.507052:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.507053:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.507057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.507065:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.507067:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.507068:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.507068:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.507071:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.507073:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.507120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.507125:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.507128:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.507130:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.507130:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.507132:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.507134:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.507135:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.507137:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.507138:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.507139:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.507142:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.507144:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.507147:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.507147:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.507148:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.507150:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.507150:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.507151:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.507152:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.507156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:4.2:1619677448.507160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:13.0:1619677448.507163:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.507165:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.507166:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.507167:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:13.0:1619677448.507168:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.507169:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677448.507169:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.507170:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677448.507171:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677448.507172:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.507174:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.507176:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.507177:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.507179:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.507180:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.507181:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.507184:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.507186:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.507189:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.507190:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.507191:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.507192:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.507193:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.507193:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.507194:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.2:1619677448.507194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:8.2:1619677448.507198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.507200:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.507203:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.507204:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.507206:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.507206:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:18.0:1619677448.507209:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.507209:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:18.0:1619677448.507210:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:17.0:1619677448.507210:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.507210:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.507212:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.507213:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.507214:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.507215:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.507216:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.507216:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:18.0:1619677448.507219:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.507221:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.507224:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.507225:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.507225:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.507227:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.507227:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.507228:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.507229:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.507233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.507242:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.507244:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.507245:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.507246:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.507247:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.507249:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.507267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.507275:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.507278:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.507280:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.507282:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.507283:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.507285:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.507286:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.507288:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.507289:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.507290:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.507298:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.507300:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.507303:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.507304:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.507305:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.507306:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.507307:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.507308:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.507308:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.507313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.507321:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.507324:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.507325:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.507325:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.507328:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.507331:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.507385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.507394:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.507397:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.507399:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.507401:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.507402:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.507403:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.507404:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.507406:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.507407:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.507408:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.507411:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.507413:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.507415:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.507416:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.507416:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.507418:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.507418:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.507419:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.507420:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.507424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.507431:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.507433:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.507434:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.507435:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.507437:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.507438:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.507600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:4.2:1619677448.507600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677448.507601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.507608:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677448.507608:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:17.0:1619677448.507609:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.507611:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:17.0:1619677448.507612:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:8.0:1619677448.507612:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:1.0:1619677448.507612:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.507613:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.507613:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:24.0:1619677448.507616:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.507616:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.507617:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.507618:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.507619:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.507620:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.507621:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677448.507621:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.507622:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.507623:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.507623:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.507624:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.507625:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.507626:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.507626:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.507626:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.507627:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.507627:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.507628:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:5.2:1619677448.507628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:26.0:1619677448.507629:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677448.507629:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677448.507633:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.507636:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.507637:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.507637:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.507637:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677448.507638:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.507639:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:17.0:1619677448.507640:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.507641:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.507641:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.507642:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.507642:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:17.0:1619677448.507642:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.507643:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:10.0:1619677448.507643:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.507644:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.507644:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.507644:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.507645:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677448.507645:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677448.507645:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.507645:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.507646:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677448.507646:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:17.0:1619677448.507646:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677448.507647:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.507647:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:10.0:1619677448.507647:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:26.0:1619677448.507648:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:10.0:1619677448.507648:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96714043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.507649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:10.0:1619677448.507650:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.507651:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.507651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:10.0:1619677448.507652:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.507653:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.507654:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.507655:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:25.2:1619677448.507655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.507656:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:12.0:1619677448.507656:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:10.0:1619677448.507656:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.507657:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.507658:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.507659:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:5.0:1619677448.507660:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:12.0:1619677448.507661:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.507662:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.507662:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:6.0:1619677448.507662:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.507662:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:4.2:1619677448.507662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:12.0:1619677448.507663:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.507663:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677448.507663:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.507664:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.507664:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.507665:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.507665:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.507666:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.507666:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:12.0:1619677448.507666:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.507666:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:5.0:1619677448.507666:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.507667:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677448.507667:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.507669:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:8.2:1619677448.507669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:6.0:1619677448.507669:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.507670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.507670:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.507671:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.507671:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.507672:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677448.507672:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.507673:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:25.2:1619677448.507675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:13.0:1619677448.507676:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.507676:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.507677:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.507678:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.507678:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.507678:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677448.507679:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.507680:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.507682:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:13.0:1619677448.507682:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.507682:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.507684:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677448.507685:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.507686:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.507686:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.507687:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.507688:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.507980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.507988:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.507991:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.507993:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.507995:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.507996:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.507997:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.507999:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.508001:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.508002:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.508003:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.2:1619677448.508003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:31.0:1619677448.508006:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.508007:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.508008:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.508009:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:31.0:1619677448.508010:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.508011:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.508012:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.508012:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.508012:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677448.508014:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.508015:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677448.508015:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.508016:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:20.0:1619677448.508016:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677448.508017:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.508018:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.508018:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.508019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.508019:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.508020:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.508021:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:21.2:1619677448.508024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677448.508024:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.508026:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:12.0:1619677448.508026:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.508028:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.508028:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677448.508029:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.508029:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:12.0:1619677448.508029:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.508031:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:12.0:1619677448.508031:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.508032:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677448.508032:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677448.508032:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.508033:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677448.508033:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.508034:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:20.0:1619677448.508034:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:30.0:1619677448.508035:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.508035:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677448.508036:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.508037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.508040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.508045:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.508046:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.508047:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.508047:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.508048:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.508048:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.508049:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.508050:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:5.0:1619677448.508050:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.508052:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.508088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.508094:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.508097:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.508098:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.508100:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.508103:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.508104:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.508105:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.508107:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.508108:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.508109:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.508112:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.508114:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.508115:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.508116:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.508117:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.508118:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.508119:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.508119:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.508120:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.508124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.508132:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.508134:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.508134:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.508135:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.508136:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.508138:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.508168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.508173:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.508176:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.508177:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.508179:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.508182:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.508184:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.508185:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.508186:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.508187:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.508188:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.508191:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.508193:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.508195:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.508196:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.508196:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.508198:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.508198:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.508199:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.508200:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.508201:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.508204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.508206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.508212:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.508213:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.508213:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.508214:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.508214:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.508214:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.508215:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:17.0:1619677448.508216:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.508216:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.508217:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.508226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.508234:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.508236:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.508237:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.508239:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.508242:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.508243:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.508244:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.508246:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.508247:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.508248:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.508251:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.508253:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.508256:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.508257:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.508257:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.508259:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.508259:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.508260:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.508260:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.508265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.508272:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.508275:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.508276:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:21.2:1619677448.508277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:6.0:1619677448.508277:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.508279:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.508283:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.508283:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677448.508284:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.508284:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.508287:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.508289:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.508290:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.508292:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.508293:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.508300:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.508301:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.508304:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.508306:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.508309:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.508310:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.508310:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.508312:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.508313:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.508314:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.508315:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.508315:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.508319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.508321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677448.508327:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677448.508327:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:20.0:1619677448.508329:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:6.0:1619677448.508329:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:20.0:1619677448.508330:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.508330:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.508331:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:20.0:1619677448.508332:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.508333:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.508334:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.508374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.508382:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:21.2:1619677448.508383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:30.0:1619677448.508385:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.508387:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.508389:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.508390:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.508390:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:30.0:1619677448.508391:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.508393:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.508393:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.508393:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:30.0:1619677448.508395:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.508396:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.508396:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.508397:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:8.0:1619677448.508399:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.508400:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.508401:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:30.0:1619677448.508402:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:8.0:1619677448.508402:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96714035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.508404:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.508404:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.508405:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.508406:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.508407:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.508407:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:8.0:1619677448.508408:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.508409:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.508409:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:8.0:1619677448.508410:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.508411:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.508412:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.508413:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.0:1619677448.508413:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.508414:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.2:1619677448.508416:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.508418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:8.0:1619677448.508419:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.508421:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.508421:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.508422:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.508423:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677448.508424:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677448.508426:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:8.2:1619677448.508427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.508427:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.508428:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.508429:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.508429:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.508430:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:5.0:1619677448.508430:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.508431:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.508433:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.508433:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:5.0:1619677448.508433:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.508434:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.508434:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.508435:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.508436:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677448.508436:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.508437:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.508438:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.508438:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.508440:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.508449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.508456:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.508459:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.508460:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.508461:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:20.2:1619677448.508462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:2.0:1619677448.508463:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.508465:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:2.0:1619677448.508465:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.508466:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.508467:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:2.0:1619677448.508468:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.508470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:2.0:1619677448.508470:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.508471:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.508471:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.508474:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.508474:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:20.0:1619677448.508475:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:5.0:1619677448.508475:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:2.0:1619677448.508476:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.508477:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.508478:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.508478:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:20.0:1619677448.508479:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.508479:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.508480:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:20.0:1619677448.508481:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.508481:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:2.0:1619677448.508481:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.508482:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:2.0:1619677448.508482:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.508483:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.508483:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.508484:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:20.0:1619677448.508485:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000400:00000200:10.0:1619677448.508485:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.508488:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.508488:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.508489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:10.0:1619677448.508490:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:20.0:1619677448.508491:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.508492:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:10.0:1619677448.508492:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96714032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.508493:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:10.0:1619677448.508493:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.508494:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.508495:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.508495:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:10.0:1619677448.508495:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.508496:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677448.508496:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.508497:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:22.0:1619677448.508499:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:10.0:1619677448.508499:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.508500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:22.0:1619677448.508500:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.508500:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.508501:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:22.0:1619677448.508503:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:10.0:1619677448.508503:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.508504:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:22.0:1619677448.508505:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677448.508505:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.508507:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:10.0:1619677448.508507:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.508507:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.508508:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.508509:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:31.0:1619677448.508510:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.508510:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.508510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677448.508510:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:31.0:1619677448.508512:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:25.2:1619677448.508512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.508514:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.508515:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.508515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.508516:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.508519:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:13.0:1619677448.508520:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:30.0:1619677448.508521:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.508522:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677448.508522:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:30.0:1619677448.508523:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.508523:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:13.0:1619677448.508523:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.508525:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:13.0:1619677448.508525:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.508526:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:26.0:1619677448.508526:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:30.0:1619677448.508527:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.508527:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.508527:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:30.0:1619677448.508528:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.508528:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:30.0:1619677448.508529:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.508530:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:25.2:1619677448.508532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:30.0:1619677448.508534:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.508535:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.508538:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.508538:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.508539:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.508540:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677448.508540:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.508541:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:13.0:1619677448.508541:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677448.508542:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.508542:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:30.0:1619677448.508543:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.508543:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.508543:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.508544:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.2:1619677448.508547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.508556:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.508558:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.508559:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.508560:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.508562:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.508565:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.508567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.508574:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.508577:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.508577:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.508578:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.508579:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.508579:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.508612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.508618:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.508621:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.508623:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.508623:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.508625:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.508627:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677448.508628:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.508629:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.508630:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.508631:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677448.508635:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.508637:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.508638:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.508639:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.508640:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.508641:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.508642:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.508642:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.508643:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.508647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.508654:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.508656:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.508657:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.508659:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.508660:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.508661:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.508765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.508771:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.508774:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.508775:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.508777:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.508778:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.508780:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.508781:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.508783:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.508783:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.508785:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.508788:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.508790:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.508793:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.508793:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.508794:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.508796:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.508796:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.508797:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.508798:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.508802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.508808:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.508811:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.508812:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.508813:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.508815:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.508816:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.508819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:4.2:1619677448.508823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677448.508825:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.508828:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.508829:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.508830:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.508830:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.508830:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677448.508832:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.508833:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.508834:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.508836:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.508838:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.508840:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.508841:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.508842:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.508843:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.508844:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.508848:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.508850:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.508853:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.508854:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.508855:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.508856:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.508857:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.508857:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.508858:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.508863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.508871:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.508874:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.508874:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.508875:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.508877:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.508880:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.508947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.508952:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.508954:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.508955:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.508957:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.508959:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.508960:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.508961:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.508962:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.508963:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.508964:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.508967:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.508969:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.508971:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.508972:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.508973:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.508974:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.508975:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.508975:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.508976:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677448.508980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.2:1619677448.508981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:24.0:1619677448.508988:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.0:1619677448.508989:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:24.0:1619677448.508991:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:5.0:1619677448.508991:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:8.0:1619677448.508992:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:5.0:1619677448.508992:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.508994:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.508994:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.508995:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.508996:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:24.0:1619677448.508997:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.508998:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.509000:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.509001:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.509002:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.509003:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.509007:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.509008:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.509011:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.509012:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.509013:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.509014:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.509015:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.509015:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.509016:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.509020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.509028:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.509031:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.509031:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.509032:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.509034:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.509037:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.509152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.2:1619677448.509156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677448.509157:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.509159:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.509160:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.509161:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.509164:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677448.509164:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.509165:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.509166:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.509167:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:1.0:1619677448.509168:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.509169:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.509171:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.509171:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:1.0:1619677448.509171:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.509174:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.509175:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.509176:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:1.0:1619677448.509176:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:31.0:1619677448.509178:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.509178:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.509179:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.509179:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:31.0:1619677448.509180:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.509180:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677448.509181:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.509181:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.509182:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.509182:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.509183:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677448.509185:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.509187:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.2:1619677448.509187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677448.509189:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.509190:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.509191:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.509193:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.509193:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.509194:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677448.509194:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:31.0:1619677448.509195:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677448.509195:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.509197:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.509197:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.2:1619677448.509198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:17.0:1619677448.509199:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.509202:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677448.509204:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.509205:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.509206:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.509206:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.509208:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.509210:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.509222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.509229:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.509232:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.509233:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.509235:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.509238:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.509239:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.509241:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.509242:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.509243:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.509244:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.509247:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.509248:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.509251:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.509252:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.509253:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.509254:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.509255:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.509256:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.509256:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.509260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.509268:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.509271:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.509271:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.509272:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.509274:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.509277:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.509280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.509287:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.509289:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.509290:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.509290:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.509291:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.509292:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.509296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.509303:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.509306:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.509307:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.509309:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.509311:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.509312:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.509314:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.509315:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.509316:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.509317:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.509321:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.509322:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.509325:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.509326:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.509327:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.2:1619677448.509328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677448.509328:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.509329:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.509330:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.509330:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677448.509331:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.509333:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:25.2:1619677448.509335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:20.0:1619677448.509335:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.509336:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.509337:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.509339:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.509340:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.509341:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.509342:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.509342:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:20.0:1619677448.509343:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.509345:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.509346:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.509346:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.509347:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.509349:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.509349:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.509351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.509352:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.509353:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.509354:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.509354:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.509355:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.509355:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.509356:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.509357:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677448.509358:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.509358:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:10.0:1619677448.509358:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.509359:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.509360:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.509361:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.509362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.509370:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.509373:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.509373:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.509375:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.509377:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.509378:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.509457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.509464:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.509467:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.509467:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.509469:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.509471:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.509473:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.509474:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.509475:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.509476:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.509477:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.509481:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.509483:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.509486:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.509487:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.509487:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.509489:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.509490:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.509490:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.509491:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.509495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.509502:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.509504:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.509505:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.509507:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.509508:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.509510:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.509581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.2:1619677448.509584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:26.0:1619677448.509589:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.509590:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.509592:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.509592:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677448.509593:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.509593:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:26.0:1619677448.509595:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.509597:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.509598:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.509599:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.509600:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:6.0:1619677448.509601:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:26.0:1619677448.509602:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.509603:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.509604:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.509606:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.509606:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.509608:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.509609:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:6.0:1619677448.509609:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:26.0:1619677448.509615:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.509615:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.509617:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677448.509618:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.509620:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.509621:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:6.0:1619677448.509621:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.509622:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:6.0:1619677448.509622:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.509623:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.509624:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.509624:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.509625:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677448.509625:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.509625:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.509626:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677448.509626:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.509627:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.509629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:25.2:1619677448.509633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:30.0:1619677448.509637:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.509639:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:10.0:1619677448.509639:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:30.0:1619677448.509640:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.509642:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.509642:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.509643:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.509643:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.509644:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.509644:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:10.0:1619677448.509645:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.509648:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:25.2:1619677448.509650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000800:00000200:26.0:1619677448.509656:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.509659:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.509659:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.509660:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.509661:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.509661:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:4.2:1619677448.509662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.509669:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.509672:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.509672:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.509674:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.509676:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.509678:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.509679:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.509681:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.509682:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.509683:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.509686:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.509688:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.509690:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.509691:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.509692:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.509693:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.509694:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.509695:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.509696:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.509696:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.509699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.509702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.509706:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.509708:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.509709:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.509709:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677448.509710:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.509710:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.509711:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.509712:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.509712:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.509713:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.509828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.509834:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.509837:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.509838:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.509840:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.509843:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.509844:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.509845:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.509847:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.509848:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.509849:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.509852:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.509853:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.509855:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.509856:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.509857:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.509858:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.509858:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.509860:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.509861:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.509861:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.509865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.509868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.509873:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.509874:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.509874:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.509874:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.509875:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.509875:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.509876:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.509876:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.509877:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.509878:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.509960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.509968:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.509970:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.509973:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.509974:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.509975:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.509977:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.509978:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.509979:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.509980:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.509981:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.509985:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.509987:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.509990:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.509991:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.509991:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.509993:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.509994:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.509995:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.509995:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.509999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.510007:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.510010:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.510011:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.510013:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.510013:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.510016:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.510022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.510029:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.510031:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.510032:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.510033:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.510034:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.510035:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.510042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.510049:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.510051:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.510052:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.510054:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.510056:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.510057:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.510059:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714014] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.510060:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.510061:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.510062:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.510065:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.510067:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.510069:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.510069:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.510070:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.510071:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.510072:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.510073:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.510073:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.510077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.510084:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.510086:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.510087:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.510089:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.510090:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.510091:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.510127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.510134:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.510136:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.510138:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.510139:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.510142:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.510143:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.510144:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96714013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.510145:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.510146:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.510147:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.510150:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.510152:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.510154:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.510155:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.510156:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.510157:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.510158:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.510158:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.510159:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.510163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.510171:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.510174:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.510174:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.510175:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.510177:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.510180:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.510181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.510188:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.510191:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.510192:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.510192:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.510193:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.510193:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.510268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.510273:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.510275:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.510276:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.510278:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.510279:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.510280:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.510281:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.510283:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.510283:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.510284:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.510287:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.510289:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.510292:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.510293:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:5.2:1619677448.510297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:17.0:1619677448.510300:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.510302:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.510302:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.510303:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.510303:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677448.510304:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.510304:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677448.510305:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677448.510306:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.510308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:1.0:1619677448.510308:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.510310:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.510311:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.510312:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.510313:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.510313:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.510316:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:1.0:1619677448.510316:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.510317:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.510319:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677448.510319:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677448.510320:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.510320:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.510320:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:12.0:1619677448.510322:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.510322:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.510323:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677448.510323:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:13.0:1619677448.510324:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.510324:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:12.0:1619677448.510325:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.510328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.510335:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.510337:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.510338:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.510339:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.510341:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.510342:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.510508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.510516:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.510518:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.510519:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.510521:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.510523:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.510525:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.510526:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.510527:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.510528:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.510529:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.510533:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.510535:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.510537:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.510538:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.510539:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.510541:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.510541:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.510542:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.510542:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.2:1619677448.510548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.2:1619677448.510548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.510554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:30.0:1619677448.510557:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.510557:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.510559:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:30.0:1619677448.510560:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:20.0:1619677448.510560:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.510560:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:20.0:1619677448.510562:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.510563:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.510563:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.510564:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:18.0:1619677448.510564:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.510564:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:30.0:1619677448.510566:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.510566:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:2.0:1619677448.510567:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.510568:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.510569:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.510570:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.510570:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.510571:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.510572:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.510573:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.510573:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.510574:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.510576:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.510577:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.510578:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.510581:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.510582:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.510584:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.510584:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.510585:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.510585:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.510587:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.510587:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677448.510587:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.510588:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:2.0:1619677448.510588:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.510588:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.510589:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:2.0:1619677448.510590:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.510590:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.510591:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.510592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677448.510592:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677448.510596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:12.0:1619677448.510598:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.510601:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.510601:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.510602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:12.0:1619677448.510603:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.510604:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:13.0:1619677448.510605:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.510605:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677448.510606:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.510607:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.510609:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.510609:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.510610:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:5.0:1619677448.510611:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:10.0:1619677448.510612:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.510613:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.510615:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.510618:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.510620:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677448.510621:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.510623:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.510624:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.510625:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.510628:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.510630:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.510633:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.510634:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.510635:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.510636:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.510637:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.510638:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.510638:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.510642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.510645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.510652:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.510653:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.510654:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:5.0:1619677448.510655:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.510656:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.510656:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.510657:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.510659:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.510662:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677448.510663:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.510664:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.510665:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.510666:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.510668:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.510669:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.510670:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.510670:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677448.510672:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.510675:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.510675:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.510676:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.510678:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.510678:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.510680:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.510681:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.510682:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.510683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.2:1619677448.510687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:26.0:1619677448.510690:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.510692:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:5.0:1619677448.510692:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:10.0:1619677448.510693:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.510694:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.510694:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.510695:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:26.0:1619677448.510696:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.510696:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:26.0:1619677448.510699:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.510740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.510746:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.510749:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.510750:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.510752:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.510754:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.510756:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677448.510757:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.510758:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.510759:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.510760:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677448.510764:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.510766:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677448.510769:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.510770:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677448.510770:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677448.510772:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677448.510772:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.510773:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.510774:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.510778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.510784:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.510787:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.510787:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.510788:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.510790:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:10.0:1619677448.510793:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.510832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.510838:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.510841:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.510842:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.510843:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.510846:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.510847:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.510848:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96714004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.510850:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.510851:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.510851:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.510854:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.510855:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.510857:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.510858:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.510858:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.510860:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.510861:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.510861:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.510862:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:21.2:1619677448.510867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:8.2:1619677448.510867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.510874:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:17.0:1619677448.510874:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.510877:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000800:00000200:17.0:1619677448.510877:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.510878:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.510878:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:31.0:1619677448.510880:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.510881:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:17.0:1619677448.510881:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.510883:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.510884:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:17.0:1619677448.510884:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:31.0:1619677448.510886:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96714003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.510887:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.510888:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.510889:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677448.510893:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.510895:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677448.510897:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.510898:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.510899:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.510900:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.510901:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677448.510902:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:31.0:1619677448.510904:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:21.2:1619677448.510904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (23)++
00000800:00000200:9.2:1619677448.510906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.510907:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:31.0:1619677448.510908:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:31.0:1619677448.510910:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000800:00000200:20.2:1619677448.510911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:31.0:1619677448.510912:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.510913:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:31.0:1619677448.510914:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.510914:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (23)++
00000400:00000200:31.0:1619677448.510915:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:30.0:1619677448.510915:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc2e00] (24)--
00000800:00000200:30.0:1619677448.510915:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:24.0:1619677448.510915:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:31.0:1619677448.510916:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.510916:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.510916:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:30.0:1619677448.510917:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:31.0:1619677448.510918:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.510918:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.510919:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.510919:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.510920:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677448.510922:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.510922:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:31.0:1619677448.510924:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.510925:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.510926:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:30.0:1619677448.510926:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.510928:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96714001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.510929:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.510930:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.510930:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.510934:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.510934:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:30.0:1619677448.510934:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.510935:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677448.510935:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.510936:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677448.510937:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:31.0:1619677448.510938:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:31.0:1619677448.510938:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677448.510938:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.510939:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.510940:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:30.0:1619677448.510940:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.510941:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.510941:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:21.2:1619677448.510941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:30.0:1619677448.510942:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.510943:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.510943:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.510943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677448.510947:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.2:1619677448.510947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677448.510950:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:1.0:1619677448.510950:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:6.0:1619677448.510951:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.510953:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.510953:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.510954:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.510954:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:1.0:1619677448.510954:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.510956:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.510956:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.510957:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:5.0:1619677448.510957:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:22.0:1619677448.510959:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96714000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.510959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:5.0:1619677448.510959:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.510959:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:22.0:1619677448.510960:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.510960:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:22.0:1619677448.510962:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.510962:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.510963:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:21.2:1619677448.510964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000400:00000200:22.0:1619677448.510966:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.510966:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677448.510968:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677448.510968:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.510969:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677448.510971:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:22.0:1619677448.510971:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.510972:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:31.0:1619677448.510973:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.510973:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000800:00000200:22.0:1619677448.510973:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.510974:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000400:00000200:30.0:1619677448.510975:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.510975:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.510975:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:31.0:1619677448.510976:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.510976:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:30.0:1619677448.510978:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.510980:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:9.2:1619677448.510980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:31.0:1619677448.510981:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.510982:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.510983:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.510984:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.510986:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:31.0:1619677448.510988:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.510988:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96713998] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.510988:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:30.0:1619677448.510989:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.510990:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.510990:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:31.0:1619677448.510991:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.510991:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:2.0:1619677448.510991:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.510992:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.510994:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.510995:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.510996:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.510996:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677448.510996:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.510997:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:31.0:1619677448.510998:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.510998:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.510998:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:31.0:1619677448.511000:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.511000:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.511001:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.511002:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677448.511002:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:30.0:1619677448.511002:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:31.0:1619677448.511003:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:25.2:1619677448.511003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.511004:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:30.0:1619677448.511004:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677448.511005:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677448.511006:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:25.2:1619677448.511009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:13.0:1619677448.511010:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.511013:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.511013:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.511015:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:13.0:1619677448.511015:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.511016:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.511017:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:13.0:1619677448.511017:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677448.511018:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.511018:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.511020:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.511022:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.511188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.511196:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.511199:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.511199:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.511201:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.511203:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.511205:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.511206:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.511207:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.511208:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.511209:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.511212:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.511213:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.511215:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.511215:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.511216:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.511217:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.511218:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.511218:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.511219:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.511223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.511229:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.511230:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.511231:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.511232:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.511233:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.511234:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.511281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.511289:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.511291:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.511298:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.511298:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.511301:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.511302:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.511304:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.511305:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.511306:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.511307:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.511310:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.511312:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.511314:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.511314:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.511315:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.511316:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.511317:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.511318:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.511320:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.511320:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.511322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.2:1619677448.511327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.511331:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.511333:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.511333:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.511335:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:30.0:1619677448.511335:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.511336:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.511336:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.511337:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.511338:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.511339:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.2:1619677448.511364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.511367:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.511369:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.511372:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.511373:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.511374:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.511376:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.511377:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.511379:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.511380:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.511381:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.511384:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.511386:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.511389:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.511390:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.511390:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.511392:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.511393:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.511394:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.511394:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.511398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.511405:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.511407:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.511408:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.511410:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.511411:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.511413:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.511421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:21.2:1619677448.511426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.511430:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:21.2:1619677448.511431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:30.0:1619677448.511432:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.511433:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.511434:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.511434:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.511434:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.511435:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677448.511435:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:2.0:1619677448.511436:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:31.0:1619677448.511437:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.511437:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.511439:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.511439:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.511440:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:31.0:1619677448.511441:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.511442:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.511443:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.511444:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.511445:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.511445:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.511446:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.511447:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677448.511448:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.511450:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.511451:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.511452:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.511453:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.511455:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.511457:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.511458:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.511459:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.511459:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.511460:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.511461:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.511461:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.511462:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:2.0:1619677448.511463:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.511463:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.511464:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.511464:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677448.511464:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:25.2:1619677448.511466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.511466:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.511466:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.511467:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.511468:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.511469:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.2:1619677448.511470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.511473:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:8.2:1619677448.511473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.511474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.511475:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.511476:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.511476:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.511477:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.511479:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:10.0:1619677448.511479:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.511480:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677448.511480:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:30.0:1619677448.511481:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.511481:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.0:1619677448.511481:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:18.0:1619677448.511482:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.511483:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677448.511483:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:18.0:1619677448.511484:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:5.0:1619677448.511484:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.511486:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.511487:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.511489:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.511490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.511497:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.511500:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.511501:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.511501:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.511503:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.511503:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.511525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.511532:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.511534:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.511535:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.511537:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.511540:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.511541:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.511542:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.511544:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.511545:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.511546:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.511550:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.511552:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.511555:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.511555:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.511556:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.511558:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.511558:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.511560:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.511561:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.511561:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.511564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.511567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.511572:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.511574:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.511575:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677448.511576:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677448.511576:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.511576:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.511577:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.511578:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677448.511578:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.511581:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.511583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.511590:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.511593:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.511593:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.511594:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.511595:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.511595:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.511603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.511609:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.511612:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.511613:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.511615:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.511617:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.511619:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.511620:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.511621:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.511622:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.511623:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.511626:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.511628:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.2:1619677448.511628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.511630:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.511631:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.511632:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.511633:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.511634:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.511634:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.511635:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.511635:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677448.511636:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.511637:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.511639:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.511640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:17.0:1619677448.511641:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.511643:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.511644:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.511645:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.511646:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.511647:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677448.511648:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.511650:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:17.0:1619677448.511651:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.511652:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677448.511652:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:2.0:1619677448.511652:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.511654:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.511655:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.511656:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.511656:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677448.511657:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677448.511658:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.511659:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.511659:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.511660:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.2:1619677448.511662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:25.2:1619677448.511664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677448.511668:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.511671:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:6.0:1619677448.511672:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677448.511672:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.511674:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.511674:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.511675:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.511675:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.511676:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.511678:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:6.0:1619677448.511678:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.511680:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.511681:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.511681:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:18.0:1619677448.511682:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.511683:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.511687:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.511688:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.511691:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.511692:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.511693:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.511695:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.511696:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.511697:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.511698:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.511699:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:25.2:1619677448.511701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.511703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677448.511709:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.511709:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.511711:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:12.0:1619677448.511712:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.511712:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:12.0:1619677448.511713:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.511713:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:12.0:1619677448.511715:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.511717:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.511718:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.511719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.511725:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.511726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:13.0:1619677448.511728:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.511729:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.511729:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.511730:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.511731:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.511734:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677448.511737:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:30.0:1619677448.511739:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.511741:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677448.511741:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.511743:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.511744:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.511745:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.511746:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.511747:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.511751:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.511753:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.511755:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.511756:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.511757:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.511758:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.511759:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677448.511760:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677448.511760:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:25.2:1619677448.511764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.511770:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.511773:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.511773:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.511775:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.511777:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.511778:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.511783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:12.0:1619677448.511790:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.511792:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.511793:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.511794:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.511794:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.511796:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.511865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.511868:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.511870:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.511873:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.511874:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.511876:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.511877:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.511878:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.511880:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.511881:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.511882:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.511885:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.511887:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.511890:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.511890:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.511891:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.511893:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.511893:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.511895:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.511896:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.511897:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.511899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.511902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677448.511907:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.511907:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:22.0:1619677448.511909:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677448.511909:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:22.0:1619677448.511910:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.511910:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677448.511911:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.511911:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:22.0:1619677448.511912:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.511915:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.511959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.511966:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.511969:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.511972:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.511973:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.511974:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.511975:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.511977:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.511978:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.511979:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.511980:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.511984:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.511985:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.511988:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.511989:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.511990:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.511991:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.511992:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.511993:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.511993:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.511997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.512003:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.512005:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.512006:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.512007:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.512009:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.512010:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.512125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.512131:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.512134:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.512135:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.512137:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.512140:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.512141:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.512142:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.512144:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.512145:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.512146:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.512149:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.512150:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.512152:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.512153:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.512154:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.512155:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.512156:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.512156:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.512157:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.512162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.512168:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.512171:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.512171:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.512172:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.512173:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.512174:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.512182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.512188:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.512191:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.512192:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.512192:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.512192:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.512193:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.512235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.512240:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.512243:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.512244:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.512246:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.512248:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.512250:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.512251:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.512252:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.512252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.2:1619677448.512253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:6.0:1619677448.512253:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.512254:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.512258:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.512258:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.512260:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:6.0:1619677448.512260:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.512261:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.512261:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:6.0:1619677448.512263:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.512264:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.512264:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:6.0:1619677448.512264:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.512264:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:1.0:1619677448.512264:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.512266:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:8.0:1619677448.512267:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.512267:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:1.0:1619677448.512267:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.512268:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:8.0:1619677448.512269:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.512269:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.512269:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677448.512269:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.512270:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.512271:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:25.2:1619677448.512272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:1.0:1619677448.512272:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.512273:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96713983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.512273:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.512274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:1.0:1619677448.512274:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677448.512275:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.512276:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.512277:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:5.2:1619677448.512278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677448.512279:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677448.512279:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.512281:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:30.0:1619677448.512282:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:22.0:1619677448.512282:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677448.512282:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.512283:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.512283:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:8.0:1619677448.512283:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.512285:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.512285:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:30.0:1619677448.512286:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.512286:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677448.512286:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.512289:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677448.512289:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:1.0:1619677448.512289:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.512290:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677448.512290:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.512290:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677448.512291:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:1.0:1619677448.512291:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:10.0:1619677448.512292:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.512292:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677448.512300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (22)++
00000800:00000200:1.0:1619677448.512301:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.512302:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677448.512302:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.512303:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.512303:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:1.0:1619677448.512303:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.512304:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:8.0:1619677448.512304:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:1.0:1619677448.512304:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677448.512305:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:10.0:1619677448.512306:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96713981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.512307:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677448.512307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.512307:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:10.0:1619677448.512308:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.512309:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:10.0:1619677448.512309:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:8.0:1619677448.512309:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.512310:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.512312:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.512313:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.512315:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.0:1619677448.512315:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:10.0:1619677448.512316:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.512316:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.512318:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.512318:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:9.2:1619677448.512318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:2.0:1619677448.512318:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.512320:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.512320:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:2.0:1619677448.512320:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.512321:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.512321:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.512322:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:2.0:1619677448.512322:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.512324:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:2.0:1619677448.512325:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:25.2:1619677448.512326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:24.0:1619677448.512327:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.512328:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.512328:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.512329:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.512329:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:26.0:1619677448.512332:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.512334:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.512335:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.512335:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.512337:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.512338:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.512343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.512348:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.512351:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.512351:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.512352:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.512352:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.512353:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.512386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.512392:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.512394:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.512396:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.512397:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.512399:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.512401:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.512402:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.512403:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.512404:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.512405:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.512409:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.512411:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.512414:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.512415:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.512416:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.512418:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.512418:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.512419:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.512420:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.512424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.512431:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.512434:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.512435:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.512436:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.512439:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.512439:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.512524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677448.512531:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677448.512534:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677448.512536:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.512539:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.512539:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677448.512540:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.512541:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.512543:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.512544:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.512545:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677448.512548:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.512550:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.512553:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.512554:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.512555:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.512556:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.512557:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677448.512558:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677448.512558:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677448.512562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.512570:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.512573:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.512574:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.512574:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.512576:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.512579:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.512580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.512587:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.512589:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.512590:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.512590:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.512591:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.512592:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.512646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:4.2:1619677448.512651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677448.512652:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.512655:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.512655:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.512658:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677448.512658:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.512660:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:17.0:1619677448.512661:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.512661:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677448.512662:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.512664:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713978] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.512665:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.512666:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.512667:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.512667:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.512668:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677448.512668:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:26.0:1619677448.512670:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.512672:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.512673:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.512673:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.512675:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:17.0:1619677448.512676:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:26.0:1619677448.512678:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.512679:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.512680:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677448.512680:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.512681:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:26.0:1619677448.512682:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.512682:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:26.0:1619677448.512683:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:17.0:1619677448.512683:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.512684:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:17.0:1619677448.512684:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:26.0:1619677448.512685:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.512685:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.512685:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:26.0:1619677448.512686:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:17.0:1619677448.512686:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.512688:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.512688:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677448.512688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:25.2:1619677448.512690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.2:1619677448.512691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:4.2:1619677448.512694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677448.512696:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:1.0:1619677448.512696:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677448.512698:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:1.0:1619677448.512698:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:24.0:1619677448.512699:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:10.0:1619677448.512699:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:1.0:1619677448.512699:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.512700:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.512700:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:8.0:1619677448.512700:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.512701:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.512702:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:8.0:1619677448.512702:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.512703:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.512703:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.512703:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.512704:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:8.0:1619677448.512704:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:10.0:1619677448.512705:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.512706:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.2:1619677448.512940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.512948:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.512951:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.512953:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.512955:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.512956:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.512957:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.512958:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.512960:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.512961:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.512962:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.512965:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.512967:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.512969:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.512970:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.512970:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.512972:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.512972:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.512973:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.512974:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.512977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.512984:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.512986:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.512987:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.512989:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.512990:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.512992:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.513034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.2:1619677448.513036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:20.0:1619677448.513037:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.513039:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.513041:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:18.0:1619677448.513042:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677448.513043:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.513044:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.513045:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:20.0:1619677448.513046:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:2.0:1619677448.513046:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:20.0:1619677448.513048:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.513049:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.513050:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.513051:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.513051:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.513052:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.513053:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677448.513054:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.513056:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.513057:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000400:00000200:18.0:1619677448.513057:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.513058:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.513060:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:18.0:1619677448.513062:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.513063:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.513064:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:18.0:1619677448.513064:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.513065:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677448.513066:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:18.0:1619677448.513066:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.513067:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:18.0:1619677448.513067:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.2:1619677448.513067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677448.513068:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:18.0:1619677448.513068:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.513069:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677448.513069:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.513070:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.513071:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.513072:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.513072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.513072:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.513075:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.513076:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.513077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.513078:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:1.0:1619677448.513078:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.513081:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:1.0:1619677448.513081:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.513082:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.513082:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:1.0:1619677448.513082:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.513083:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96713973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.513084:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:24.0:1619677448.513085:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000400:00000200:1.0:1619677448.513085:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.513086:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:1.0:1619677448.513086:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.513087:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.513087:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:24.0:1619677448.513088:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:18.0:1619677448.513088:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.513090:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.513092:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.513093:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677448.513094:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.513097:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.513098:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.513098:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.513100:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.513100:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.513101:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.513102:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.513106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.513114:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.513117:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.513118:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.513118:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.513120:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.513120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:6.0:1619677448.513123:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:10.0:1619677448.513127:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.513130:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.513131:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.513133:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.513134:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.513135:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.513137:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.513138:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.513139:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.513140:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.513144:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.513144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.513145:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:21.2:1619677448.513149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:10.0:1619677448.513149:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.513150:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.513151:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.513152:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.513153:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:21.2:1619677448.513153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677448.513153:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.513154:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677448.513155:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.513155:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:26.0:1619677448.513156:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.513156:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.513156:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.513157:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.513158:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.513158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677448.513158:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.513159:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:26.0:1619677448.513160:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:31.0:1619677448.513161:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677448.513161:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.513162:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.513162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:26.0:1619677448.513163:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.513163:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.513164:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677448.513164:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.513165:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:26.0:1619677448.513166:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677448.513166:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.513167:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.513167:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677448.513168:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.513168:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.513168:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.513170:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.513170:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:18.0:1619677448.513171:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.513171:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677448.513171:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.513171:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:26.0:1619677448.513172:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.513172:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.513172:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:12.0:1619677448.513174:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.513174:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:31.0:1619677448.513176:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.513176:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:18.0:1619677448.513176:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.513177:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:26.0:1619677448.513178:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.513179:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96713969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.513179:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.513179:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:18.0:1619677448.513179:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.513181:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.513181:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.513181:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.513182:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.513182:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677448.513182:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.513183:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.513183:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:18.0:1619677448.513183:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677448.513184:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.513185:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:31.0:1619677448.513186:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.513186:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:25.2:1619677448.513187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:18.0:1619677448.513187:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.513187:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:31.0:1619677448.513188:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.513191:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.513192:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.2:1619677448.513192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.513193:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.513193:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.513194:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.513195:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.513195:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.513196:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.513196:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:30.0:1619677448.513196:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.513197:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.513198:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:25.2:1619677448.513199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.513200:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.0:1619677448.513200:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.513203:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.513204:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.513205:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.513206:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:13.0:1619677448.513206:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.513208:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.513208:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:13.0:1619677448.513209:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.513211:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.513212:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.513213:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.513619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:4.2:1619677448.513625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.513630:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.513631:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.513633:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.513634:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.513634:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.513637:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.513637:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.513638:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.513640:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.513641:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.513641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:10.0:1619677448.513642:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.513643:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.513645:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.513647:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.513647:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000400:00000200:26.0:1619677448.513648:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.513650:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.513650:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.513651:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.513652:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.513655:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.513657:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.513658:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.513658:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.513658:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.513659:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.513659:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.513659:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.513661:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.513661:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.513662:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.513662:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677448.513663:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.513663:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (23)++
00000800:00000200:26.0:1619677448.513664:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.513665:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (24)--
00000800:00000200:10.0:1619677448.513665:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (24)--
00000800:00000200:26.0:1619677448.513666:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.513666:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677448.513667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.513667:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.2:1619677448.513668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:10.0:1619677448.513669:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.513670:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.2:1619677448.513671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:10.0:1619677448.513671:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.513673:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.513674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:12.0:1619677448.513674:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677448.513675:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:2.0:1619677448.513675:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:10.0:1619677448.513676:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96713966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.513677:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.513678:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.513678:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.513678:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:26.0:1619677448.513679:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:18.0:1619677448.513679:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:26.0:1619677448.513680:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:10.0:1619677448.513680:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.513681:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:12.0:1619677448.513681:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000400:00000200:10.0:1619677448.513681:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.513682:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:13.0:1619677448.513682:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677448.513682:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:26.0:1619677448.513683:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.513684:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.513684:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.513685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:2.0:1619677448.513685:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.513686:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.513687:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.513687:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.513688:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96713965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.513689:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.513690:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:10.0:1619677448.513690:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:2.0:1619677448.513690:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.513691:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:13.0:1619677448.513691:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:2.0:1619677448.513691:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.513692:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.513693:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.513694:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.513694:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.513694:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:13.0:1619677448.513695:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.513695:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.513696:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.513696:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.513696:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:2.0:1619677448.513696:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:13.0:1619677448.513697:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.513697:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.513698:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677448.513698:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.513699:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.513700:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:25.2:1619677448.513701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:2.0:1619677448.513701:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.513703:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.513703:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.513704:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.513705:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677448.513709:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:9.2:1619677448.513709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.513711:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.513712:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.513714:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.513716:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:18.0:1619677448.513716:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677448.513717:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:18.0:1619677448.513718:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.513719:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.513720:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.513721:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.513723:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.513784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.513791:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.513794:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.513795:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.513797:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.513800:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.513802:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.513803:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.513804:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.513805:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.513806:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.513810:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.513812:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.513815:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.513816:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.513816:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.513818:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.513818:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.513819:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.513819:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.513824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.513830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:17.0:1619677448.513832:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.513835:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:22.0:1619677448.513836:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:17.0:1619677448.513836:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.513836:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.513838:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:17.0:1619677448.513839:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.513840:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.513841:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.513842:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:22.0:1619677448.513844:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.513846:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.513847:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.513848:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.513849:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.513850:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.513853:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.513855:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.513857:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.513858:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.513859:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.513860:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.513861:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.513862:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.513862:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.513867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.513870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.0:1619677448.513877:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.2:1619677448.513878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677448.513878:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:5.0:1619677448.513878:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.513879:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.513880:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.513880:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.513882:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.513883:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.513885:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:30.0:1619677448.513886:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677448.513886:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.513887:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.513889:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:20.0:1619677448.513889:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.513890:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.513892:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.513892:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.513894:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.513894:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.513896:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.513896:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:30.0:1619677448.513897:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:5.0:1619677448.513897:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:30.0:1619677448.513898:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.513898:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677448.513899:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.513900:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.513900:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:30.0:1619677448.513901:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.513901:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:30.0:1619677448.513902:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.2:1619677448.513903:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:30.0:1619677448.513905:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.513906:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.513906:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:30.0:1619677448.513907:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.513907:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:5.0:1619677448.513908:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.513909:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:20.0:1619677448.513910:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.513910:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:30.0:1619677448.513912:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.513912:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.2:1619677448.513912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:5.0:1619677448.513912:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.513913:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.513915:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.513915:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:5.0:1619677448.513915:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.513916:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:5.0:1619677448.513916:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:30.0:1619677448.513917:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:5.0:1619677448.513918:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96713960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.513919:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:5.0:1619677448.513919:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.513920:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.513920:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.513921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:20.0:1619677448.513921:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.513921:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677448.513923:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.513924:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677448.513925:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.513926:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:6.0:1619677448.513928:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:5.0:1619677448.513929:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.513929:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.513931:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677448.513931:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.513932:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.513932:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.513932:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.513933:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.513934:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.513935:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.513935:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677448.513936:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:1.0:1619677448.513936:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.513938:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677448.513938:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677448.513938:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.513939:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677448.513939:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.513940:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677448.513940:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.513941:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.513942:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.513943:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677448.513944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:1.0:1619677448.513944:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.513948:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.513949:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.513951:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:1.0:1619677448.513951:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.513952:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.513953:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.513954:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:1.0:1619677448.513954:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.513955:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.513955:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.513955:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.513955:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.513956:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:2.0:1619677448.513957:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.513959:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:8.2:1619677448.513960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.513968:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.513970:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.513971:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.513972:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.513973:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.513974:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.514042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.514049:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.514052:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.514053:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.514055:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.514057:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.514059:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.514060:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.514061:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.514062:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.514063:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.514066:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.514067:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.514070:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.514071:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.514072:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.514074:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.514074:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.514074:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.514075:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.514079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.514087:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.514090:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.514091:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.514091:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.514093:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.514097:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.514121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.514129:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.514132:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.514134:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.514135:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.514137:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.514139:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.514140:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.514141:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.514142:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.514143:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.514146:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.514148:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.514151:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.514151:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.514152:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.514154:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.514154:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.514155:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.514156:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.514160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.514168:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.514170:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.514171:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.514173:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.514174:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.514176:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.514179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.514188:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.514190:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.514191:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.514192:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.514193:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677448.514193:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:21.2:1619677448.514236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.514243:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.514246:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.514247:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.514249:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.514252:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.514253:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.514255:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.514256:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.514257:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.514258:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677448.514261:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.514263:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.514266:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.514267:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.514267:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.514269:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.514269:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.514271:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.514272:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.514272:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.2:1619677448.514276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.514278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677448.514284:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.0:1619677448.514285:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.514287:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:5.0:1619677448.514287:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.514287:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.514288:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.514289:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:5.0:1619677448.514289:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.514290:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.514292:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.514335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.514341:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.514343:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.514344:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.514346:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.514348:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.514349:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.514350:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.514352:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.514353:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.514354:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.514357:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.514359:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.514362:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.514362:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.514363:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.514365:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.514365:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.514366:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.514366:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.514371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.514379:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.514381:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.514382:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.514384:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.514386:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.514387:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.514425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.514433:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.514436:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.514437:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.514439:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.514442:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.514443:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.514445:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.514446:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.514447:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.514448:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.514451:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.514453:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.514455:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.514456:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.514456:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.514458:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.514458:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.514459:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.514460:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.514463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.514471:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.514474:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.514475:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.514476:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.514478:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.514480:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.514482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.514489:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.514491:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.514492:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.514492:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.514493:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.514494:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.514514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.514522:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.514524:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:4.2:1619677448.514524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677448.514525:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.514527:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.514530:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.514531:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:24.0:1619677448.514531:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.514532:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.514533:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:24.0:1619677448.514534:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.514534:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:24.0:1619677448.514535:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.514536:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677448.514537:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.514537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:26.0:1619677448.514539:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.514539:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.514541:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:24.0:1619677448.514541:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:26.0:1619677448.514542:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.514543:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.514543:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:26.0:1619677448.514544:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.514544:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.514545:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:24.0:1619677448.514545:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.514545:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.514546:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:24.0:1619677448.514547:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.514547:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677448.514547:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:26.0:1619677448.514548:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.514548:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.514549:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:26.0:1619677448.514550:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:6.0:1619677448.514552:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.514553:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677448.514553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.514554:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.514554:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:6.0:1619677448.514554:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.514555:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:26.0:1619677448.514556:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.514556:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677448.514556:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96713951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.514557:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:6.0:1619677448.514558:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.514559:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.514559:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.514559:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.514560:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.514561:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:25.2:1619677448.514562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:6.0:1619677448.514563:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.514564:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.514564:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.514565:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.514565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.514566:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:17.0:1619677448.514567:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.514569:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.514570:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677448.514570:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.514571:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.514571:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.514572:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677448.514572:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.514573:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:22.0:1619677448.514574:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.514574:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677448.514574:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.514575:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677448.514575:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.514575:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:22.0:1619677448.514576:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.514576:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.514577:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.514579:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.2:1619677448.514579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:25.2:1619677448.514580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.514586:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.514587:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.514589:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:6.0:1619677448.514589:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.514590:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:20.0:1619677448.514590:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.514590:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.514591:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.514591:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:20.0:1619677448.514592:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.514593:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.514594:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.514611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.514616:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.514619:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.514621:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.514621:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.514624:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.514625:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677448.514627:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.514628:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.514629:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.514630:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.514634:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.514635:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.514638:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.514639:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677448.514640:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.514641:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.514642:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.514643:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.514643:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.514649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.514655:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.514657:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.514658:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.514659:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.514661:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.514664:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.514739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.514744:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.514747:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.514748:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.514750:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.514752:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.514753:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.514755:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713949] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.514756:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.514757:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.514758:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.514761:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.514763:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.514766:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.514767:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.514767:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.514769:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.514770:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.514771:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.514772:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.514773:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.514775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.514778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.514782:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677448.514783:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.514785:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:13.0:1619677448.514785:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:17.0:1619677448.514786:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677448.514786:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.514787:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677448.514788:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.514789:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.514791:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.514794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:12.0:1619677448.514800:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.514803:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.514804:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.514804:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.514805:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.514806:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.514896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.514903:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.514906:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.514907:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.514908:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.514911:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.514912:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.514913:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.514915:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.514916:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.514917:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.514920:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.514921:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.514924:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.514925:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.514926:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.514927:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.514928:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.514928:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.514929:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.514933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.514941:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.514944:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.514945:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.514946:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.514948:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.514950:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.515025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.515032:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.515035:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.515035:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677448.515037:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.515038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:8.0:1619677448.515040:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.515041:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:8.0:1619677448.515042:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:20.0:1619677448.515043:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:8.0:1619677448.515043:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.515045:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.515046:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.515046:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.515046:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:8.0:1619677448.515047:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677448.515049:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.515052:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.515054:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:8.0:1619677448.515054:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.2:1619677448.515056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677448.515056:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.515056:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.515057:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.515057:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:20.0:1619677448.515058:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.515058:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:20.0:1619677448.515059:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:8.0:1619677448.515059:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.515060:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.515061:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.515061:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:20.0:1619677448.515062:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.515064:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.515064:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.2:1619677448.515065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.515066:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677448.515067:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.515068:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677448.515068:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:31.0:1619677448.515069:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.515070:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.515070:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677448.515070:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.515071:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.515071:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.515072:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:31.0:1619677448.515073:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.515074:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96713945] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.515074:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:31.0:1619677448.515075:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.515076:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.515076:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:9.2:1619677448.515076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:31.0:1619677448.515077:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.515077:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.515079:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.515080:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:31.0:1619677448.515082:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.515082:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677448.515084:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.515084:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:31.0:1619677448.515086:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.515086:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:8.2:1619677448.515086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.515087:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:21.2:1619677448.515087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677448.515088:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677448.515088:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.515088:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.515089:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.515090:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677448.515090:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.515092:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:18.0:1619677448.515092:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:31.0:1619677448.515093:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.515094:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.515095:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:25.2:1619677448.515095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.515095:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.515099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.515099:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.515100:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:5.0:1619677448.515100:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.515101:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:13.0:1619677448.515101:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.515101:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000400:00000200:31.0:1619677448.515102:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.515102:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:30.0:1619677448.515103:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677448.515104:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.515104:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.515104:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:30.0:1619677448.515105:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:13.0:1619677448.515105:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.515106:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:30.0:1619677448.515106:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.515107:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:13.0:1619677448.515107:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.515107:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:31.0:1619677448.515108:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.515109:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.515110:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.515111:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:21.2:1619677448.515111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:13.0:1619677448.515113:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677448.515114:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.515116:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.515116:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.515119:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:31.0:1619677448.515120:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.515120:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:31.0:1619677448.515121:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.515121:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:6.0:1619677448.515122:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.515123:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.515123:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677448.515124:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000400:00000200:6.0:1619677448.515124:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.515124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677448.515125:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:6.0:1619677448.515126:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.515127:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.515128:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677448.515129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:6.0:1619677448.515129:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.515130:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.515130:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.515133:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.515134:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677448.515134:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.515136:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:20.2:1619677448.515137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:6.0:1619677448.515137:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.515138:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.515139:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.515139:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.515140:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.515140:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:2.0:1619677448.515140:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.515141:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:1.0:1619677448.515141:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.515143:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.515143:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677448.515143:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.515143:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:30.0:1619677448.515144:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:6.0:1619677448.515144:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677448.515144:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96713942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.515145:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:1.0:1619677448.515145:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.515146:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:30.0:1619677448.515147:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:1.0:1619677448.515147:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.515148:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:8.2:1619677448.515149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.515151:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.515151:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.515153:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.515154:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.515155:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.515156:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.515157:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96713941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.515158:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:30.0:1619677448.515159:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.515159:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.515160:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.515160:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.515160:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:30.0:1619677448.515161:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677448.515161:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.515161:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.515163:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:17.0:1619677448.515164:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.515164:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.515165:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.515165:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677448.515166:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.515166:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.515167:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.515168:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.515168:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.515169:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:25.2:1619677448.515169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.515169:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.515169:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.515170:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677448.515172:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.515173:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.515174:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.515175:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:12.0:1619677448.515175:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677448.515176:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.515177:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.515177:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677448.515178:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.515179:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.515180:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.515181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:13.0:1619677448.515182:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.515183:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677448.515189:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.515192:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.515193:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.515194:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.515195:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.515197:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.515259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.515266:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.515269:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.515270:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.515272:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.515275:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.515276:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.515278:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.515279:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.515280:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.515281:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.515284:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.515286:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.515289:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.515289:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.515290:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.515292:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.515292:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.515293:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.515297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.515301:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677448.515302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677448.515306:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.515309:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.515309:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:26.0:1619677448.515310:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677448.515310:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.515312:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.515313:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.515313:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.515315:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.515315:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:26.0:1619677448.515318:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.515320:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.515321:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.515322:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.515323:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.515324:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.515327:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.515329:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.515332:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.515332:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.515333:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.515335:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.515336:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.515336:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.515337:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.515342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.515350:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.515353:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.515354:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.515354:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.515356:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.515360:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.515361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.515368:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.515371:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.515372:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.515372:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.515373:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.515374:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.515568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.515574:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.515577:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.515577:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.515579:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.515582:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.515583:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.515585:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.515586:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.515587:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.515588:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.515591:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.515593:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.515596:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.515597:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.515598:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.515599:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.515600:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.515600:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.515601:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.515605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.515612:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.515614:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.515615:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.515616:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.515618:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.515621:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677448.515629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.515637:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.515639:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.515641:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.515644:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.515644:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.515645:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.2:1619677448.515645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:31.0:1619677448.515647:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.515648:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.515649:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.515650:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.515652:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:31.0:1619677448.515653:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.515654:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:31.0:1619677448.515655:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.515655:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:31.0:1619677448.515657:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.515657:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677448.515658:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.515659:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.515660:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.515660:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.515661:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677448.515661:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:31.0:1619677448.515662:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.515662:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:10.0:1619677448.515662:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.515664:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.515664:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.515665:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:25.2:1619677448.515666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.515668:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.515670:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.515673:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.515673:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.515673:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.515674:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:13.0:1619677448.515675:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:10.0:1619677448.515675:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:13.0:1619677448.515676:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.515676:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.515676:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.515677:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:13.0:1619677448.515678:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.515680:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.515681:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677448.515681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.515690:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.515693:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.515694:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.515696:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.515696:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.515699:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.515705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.515713:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.515716:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.515718:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.515719:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.515720:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.515722:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.515723:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.515724:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.515725:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.515726:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.515729:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.515731:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.515732:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.515733:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.515734:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.515735:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.515736:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.515737:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.515738:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.515739:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.515741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.2:1619677448.515745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.515749:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.515750:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.515751:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.515753:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.515753:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.515753:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.515754:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.515755:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.515756:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677448.515757:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.2:1619677448.515800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.515807:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.515810:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.515811:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.515812:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.515815:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.515816:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.515818:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.515819:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.515820:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.515821:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.515824:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.515825:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.515828:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.515829:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.515830:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.515832:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.515832:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.515833:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.515833:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.515838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.515846:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.515849:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.515850:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.515850:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.515852:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.515855:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.515872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.515875:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.515877:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.515880:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.515880:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.515882:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.515884:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.515885:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.515887:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.515888:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.515889:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.515892:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.515894:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.515897:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.515898:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.515899:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.515900:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.515901:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.515903:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.515904:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.515905:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.515906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.515909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677448.515913:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.515915:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.515915:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:13.0:1619677448.515916:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.515917:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:13.0:1619677448.515918:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.515918:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677448.515919:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.515919:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.515921:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.515923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.515930:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.515933:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.515933:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.515935:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.515938:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.515939:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.515941:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.515942:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.515943:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.515944:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.515947:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.515949:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.515952:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.515952:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.515953:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.515955:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.515955:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.515956:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.515958:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.515958:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.515961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.515964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.515969:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677448.515971:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:21.2:1619677448.515972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:17.0:1619677448.515972:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.515973:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.515973:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.515974:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677448.515975:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.515975:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.515976:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.515976:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677448.515978:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:24.0:1619677448.515979:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:21.2:1619677448.515979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677448.515979:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.515981:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.515984:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.515985:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:30.0:1619677448.515986:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:24.0:1619677448.515987:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.515988:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.515989:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:24.0:1619677448.515989:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.515991:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.515992:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.515993:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.515994:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.515994:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.515996:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:24.0:1619677448.515996:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.515997:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.515998:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.515999:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.515999:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.516000:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:24.0:1619677448.516000:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.516000:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.516002:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.516002:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:30.0:1619677448.516003:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.516003:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.516004:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:30.0:1619677448.516005:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.516008:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.516008:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:8.2:1619677448.516008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.516009:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.516010:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.516011:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.516012:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.516012:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.516016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.516016:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.516019:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.516019:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.516021:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.516022:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.516022:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.516024:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:5.0:1619677448.516024:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:26.0:1619677448.516025:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.516026:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.516028:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.516030:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.516035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.516041:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.516043:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.516044:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.516044:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.516045:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.516045:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.516090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.516097:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.516100:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.516101:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.516102:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.516105:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.516106:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.516107:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.516109:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.516109:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.516110:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.516113:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.516115:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.516118:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.516119:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.516119:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.516121:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.516122:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.516122:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.516122:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.516127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.516135:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.516138:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.516139:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.516139:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.516141:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.516144:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.516302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.516311:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.2:1619677448.516312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677448.516314:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:20.0:1619677448.516315:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.516316:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.516318:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:31.0:1619677448.516319:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.516319:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.516320:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.516321:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:5.0:1619677448.516321:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677448.516322:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.516323:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.516323:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.516324:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677448.516325:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.516326:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677448.516326:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.516327:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.516328:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.516330:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677448.516331:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.516333:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:20.0:1619677448.516335:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.516335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:20.0:1619677448.516337:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.516338:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.516339:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.516339:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.516341:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677448.516341:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.516341:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:18.0:1619677448.516341:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677448.516342:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.516342:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.516343:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.516344:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677448.516344:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:18.0:1619677448.516344:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:20.0:1619677448.516345:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.516345:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677448.516346:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.516346:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.516347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:18.0:1619677448.516347:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.516349:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.516350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:18.0:1619677448.516351:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.516352:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96713926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.516353:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.516354:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.516355:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:18.0:1619677448.516355:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.516356:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.516357:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.516358:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.516359:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:18.0:1619677448.516359:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.516360:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.516360:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.516361:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.516362:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.516363:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.516364:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:18.0:1619677448.516364:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.516365:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:18.0:1619677448.516365:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.516366:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677448.516366:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.516367:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.516368:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.516369:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.516369:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.516374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.516383:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.516385:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.516386:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.516386:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.516388:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.516390:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.516468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.516473:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.516476:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.516477:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.516479:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.516481:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.516483:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.516484:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.516485:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.516486:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.516487:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.2:1619677448.516488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:2.0:1619677448.516490:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.516492:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.516493:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.516494:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.516495:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.516495:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.516496:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:2.0:1619677448.516497:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.516497:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677448.516497:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.516498:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.516498:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.516499:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.516500:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:17.0:1619677448.516501:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.516502:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:25.2:1619677448.516503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:17.0:1619677448.516504:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.516505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:17.0:1619677448.516506:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.516507:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.516508:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:21.2:1619677448.516509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677448.516510:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677448.516511:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:17.0:1619677448.516511:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.516513:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:18.0:1619677448.516513:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:17.0:1619677448.516513:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:22.0:1619677448.516514:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.516514:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677448.516514:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.516515:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677448.516516:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.516516:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.516517:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.516517:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.516518:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.516519:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677448.516520:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677448.516520:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.516520:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.516521:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.516521:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.516522:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677448.516523:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.516523:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.516525:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:25.2:1619677448.516525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:31.0:1619677448.516526:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713923] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.516527:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.516528:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.516530:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.516533:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.516533:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:31.0:1619677448.516535:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.516536:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:31.0:1619677448.516537:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677448.516537:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.516538:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:12.0:1619677448.516538:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.516539:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.516540:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.516541:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.516541:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.516541:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.516542:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.516543:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.516546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:4.2:1619677448.516547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:13.0:1619677448.516552:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677448.516554:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677448.516555:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.516555:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.516557:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.516557:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:13.0:1619677448.516558:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.516558:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.516560:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677448.516560:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.516563:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.516564:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:25.2:1619677448.516565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:8.0:1619677448.516566:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.516567:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.516568:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.516569:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:12.0:1619677448.516572:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:8.0:1619677448.516572:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.516574:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677448.516574:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677448.516575:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.516576:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.516576:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.516577:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677448.516577:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.516577:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.516578:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.516580:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.516580:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.516581:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.516581:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.516586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.516593:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.516596:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.516597:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.516597:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.516600:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.516602:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.516831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.516839:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.516842:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.516845:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.516846:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.516847:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.516848:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:20.2:1619677448.516848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:30.0:1619677448.516849:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.516851:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.516852:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.516853:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:5.0:1619677448.516854:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677448.516856:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.516856:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:30.0:1619677448.516859:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.516859:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.516859:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.516861:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.516862:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:5.0:1619677448.516862:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.516863:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.516864:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:5.0:1619677448.516864:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677448.516865:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:5.0:1619677448.516865:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.516866:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.516866:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:5.0:1619677448.516867:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.516868:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.516869:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:25.2:1619677448.516870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:5.0:1619677448.516872:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.516874:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.516876:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.516877:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677448.516877:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.516878:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:12.0:1619677448.516879:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.516880:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.516880:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.516880:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.516881:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:12.0:1619677448.516882:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.516882:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.516884:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.516885:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.516886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.516893:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.516896:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.516897:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.516897:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.516899:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.516901:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.516931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.516937:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.516940:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.516941:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.516943:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.516945:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.516946:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.516948:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.516949:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.516950:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.516951:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.516954:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.516956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:2.0:1619677448.516956:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.516958:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.516959:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.516960:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.516961:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.516962:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677448.516962:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.516963:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:22.0:1619677448.516964:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:2.0:1619677448.516964:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.516964:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:6.0:1619677448.516965:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.516967:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.516968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:22.0:1619677448.516969:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.516969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000400:00000200:22.0:1619677448.516971:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.516972:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.516973:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.516974:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.516975:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:18.0:1619677448.516975:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:17.0:1619677448.516975:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677448.516977:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:22.0:1619677448.516978:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.516978:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677448.516978:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677448.516979:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:17.0:1619677448.516979:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.516979:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.516981:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.516981:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.516984:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.516984:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677448.516985:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.516986:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.516988:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.516989:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.516990:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:8.2:1619677448.516990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.516991:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.516992:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:21.2:1619677448.516993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:25.2:1619677448.516994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:21.2:1619677448.516996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677448.516997:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.516998:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:1.0:1619677448.517000:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.517001:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677448.517001:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.517001:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:1.0:1619677448.517001:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677448.517002:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:6.0:1619677448.517002:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:1.0:1619677448.517002:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.517003:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677448.517004:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:6.0:1619677448.517004:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:2.0:1619677448.517004:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.517005:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.517005:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.517006:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:2.0:1619677448.517006:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.517007:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.517007:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:2.0:1619677448.517008:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677448.517009:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.517010:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:2.0:1619677448.517011:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.517012:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.517013:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:25.2:1619677448.517015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:2.0:1619677448.517016:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.517018:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.517021:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:2.0:1619677448.517021:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.517021:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.517022:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.517024:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:2.0:1619677448.517024:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.517024:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:30.0:1619677448.517025:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:2.0:1619677448.517025:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:30.0:1619677448.517026:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:2.0:1619677448.517026:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:30.0:1619677448.517027:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.517028:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.2:1619677448.517030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.517038:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.517040:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.517041:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.517043:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.517044:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.517045:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.517054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.517059:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.517061:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.517064:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.517064:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677448.517066:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.517068:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:5.0:1619677448.517069:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713916] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.517070:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.517072:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.517072:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:5.0:1619677448.517076:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.517078:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:5.0:1619677448.517081:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.517082:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:5.0:1619677448.517083:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:5.0:1619677448.517084:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:5.0:1619677448.517085:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.517086:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.517087:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677448.517090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.517096:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.517099:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.517100:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.517100:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677448.517103:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:24.0:1619677448.517105:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.2:1619677448.517272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.517277:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.517280:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.517281:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.517283:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.517286:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.517287:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.517289:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.517290:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.517291:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.517292:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.517300:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.517301:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.517304:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.517305:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.517306:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.517308:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.517308:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.517310:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.517311:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.517311:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.517314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.517317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.517322:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677448.517323:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.517325:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.517325:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.517326:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677448.517326:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677448.517327:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.517328:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.517330:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.517331:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.517331:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.517332:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.517332:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.517333:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.517334:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.517348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677448.517354:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.517357:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677448.517357:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677448.517359:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.517362:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.517363:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.517365:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.517366:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.517367:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.517368:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.517370:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.517372:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.517375:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.517375:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.517376:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.517377:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.517378:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.517379:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677448.517379:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677448.517383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.517390:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.517392:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.517393:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.517394:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.517395:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.517399:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677448.517449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.517452:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.517455:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.517457:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.517458:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.517459:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.517461:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.517462:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.517464:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.517465:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.517466:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.517469:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.517471:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.517473:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.517474:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.517474:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.517476:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.517476:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.517477:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.517478:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677448.517480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:25.2:1619677448.517482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.517488:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.517490:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.517491:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.517491:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677448.517492:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:26.0:1619677448.517493:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.517493:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.517494:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.517495:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.517496:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.517498:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677448.517498:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:26.0:1619677448.517499:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.517501:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.517502:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.517503:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:25.2:1619677448.517504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000400:00000200:26.0:1619677448.517506:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.517508:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.2:1619677448.517510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.517511:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.517511:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.517511:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.517513:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (23)++
00000800:00000200:22.0:1619677448.517515:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (24)--
00000800:00000200:6.0:1619677448.517515:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.517516:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.517516:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (23)++
00000800:00000200:26.0:1619677448.517517:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.517518:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.517518:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.517519:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.517519:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.517520:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.517520:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.517521:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.517521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.517522:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.517523:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.517525:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.517526:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:4.2:1619677448.517526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.517528:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:10.0:1619677448.517528:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.517530:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.517531:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:10.0:1619677448.517531:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.517532:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.517532:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:6.0:1619677448.517532:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.517534:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.517535:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677448.517535:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.517536:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.517536:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.517537:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.517539:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:22.0:1619677448.517540:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677448.517540:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.517541:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.517541:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.517543:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.517544:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.517545:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.517546:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.517547:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.517548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.2:1619677448.517552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.517557:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.517558:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.517559:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.517561:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.517561:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.517562:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.517562:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.517563:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.517564:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.517564:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.517714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677448.517720:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.517723:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677448.517724:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677448.517726:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.517729:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.517731:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.517732:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.517733:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.517735:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.517735:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.517739:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.517741:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.517743:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.517744:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.517745:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.517746:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.517747:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.517747:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677448.517748:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:25.2:1619677448.517752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.517760:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.517763:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.517764:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.517764:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.517766:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.517768:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.517836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.517842:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.517845:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.517846:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.517847:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.517850:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.517851:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677448.517852:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.517853:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.517855:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.517855:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.517858:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.517860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.517860:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.517862:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.517863:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.517864:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.517866:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.517866:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.517867:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.517868:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.517868:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.517868:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677448.517869:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.517870:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.517872:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.517872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:18.0:1619677448.517874:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.517875:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713908] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.517876:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.517877:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.517878:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.517880:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:18.0:1619677448.517881:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.517882:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.517883:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.517884:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.517886:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677448.517886:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.517886:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:5.0:1619677448.517886:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.517887:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.517889:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.517889:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677448.517889:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677448.517890:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.517891:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.517895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.517902:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.517904:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.517905:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.517906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:13.0:1619677448.517907:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.517908:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.517910:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.517912:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.517915:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.517916:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.517919:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.517921:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.517923:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.517924:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.517926:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.517927:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.517928:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.517931:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.517932:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.517935:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.517936:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.517937:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.517939:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.517939:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.517940:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.517940:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.517952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.517960:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.517963:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.517964:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.517965:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.517967:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.517968:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.518054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.518060:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.518063:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.518064:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.518066:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.518068:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.518070:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:21.2:1619677448.518070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:24.0:1619677448.518071:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.518072:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.518073:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.518074:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.518075:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:24.0:1619677448.518077:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.518078:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:24.0:1619677448.518079:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.518079:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.518081:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.518082:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.518083:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.518084:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:22.0:1619677448.518084:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.518085:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.518086:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:22.0:1619677448.518086:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:24.0:1619677448.518087:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.518087:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677448.518087:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.518088:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.518089:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.518090:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.2:1619677448.518091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:22.0:1619677448.518094:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.518096:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.518099:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.518099:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677448.518100:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.518100:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.518102:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.518102:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:22.0:1619677448.518103:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:17.0:1619677448.518103:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.518103:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.518104:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.518104:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:17.0:1619677448.518106:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.518109:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:25.2:1619677448.518114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.518121:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.518124:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.518125:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.518126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:30.0:1619677448.518127:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.518129:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.518130:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.518131:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.518132:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677448.518132:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.518134:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.518136:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.518137:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677448.518139:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.518140:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.518141:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.518142:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677448.518145:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.518146:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677448.518149:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.518150:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.518151:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.518152:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.518153:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.518154:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.518156:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.518156:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677448.518158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:21.2:1619677448.518162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677448.518164:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.518167:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.518167:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.518168:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.518170:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677448.518170:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.518172:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677448.518172:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:30.0:1619677448.518173:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.518174:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.518175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677448.518181:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.518183:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.518184:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.518184:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.518185:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.518185:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.518412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.518419:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.518422:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.518423:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.518424:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.518426:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.518428:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.518429:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.518430:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.518431:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.518432:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.518435:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.518436:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.518439:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.518440:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.518440:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.518442:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.518443:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.518443:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.518443:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.518449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.2:1619677448.518450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:6.0:1619677448.518457:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.518458:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.518460:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:22.0:1619677448.518460:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.518460:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677448.518461:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677448.518461:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.518463:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.518463:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.518464:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.518466:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.518467:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.518468:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.518469:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.518470:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.518472:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:26.0:1619677448.518473:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.518475:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.518477:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.518478:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.518478:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.518480:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.518480:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.518481:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.518482:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677448.518484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:25.2:1619677448.518486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.518486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.518492:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.518493:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.518493:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.518495:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:17.0:1619677448.518496:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677448.518496:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677448.518496:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:12.0:1619677448.518497:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.518499:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677448.518499:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.518500:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:1.0:1619677448.518500:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.518501:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.518501:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.518502:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:30.0:1619677448.518503:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.518503:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.518504:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.518505:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677448.518506:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.518506:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.518507:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.518508:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.518509:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.518509:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.518511:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677448.518514:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.518515:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.518516:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.518518:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.518518:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677448.518519:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.518520:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.518521:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.518521:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.518522:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:1.0:1619677448.518522:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.518522:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:30.0:1619677448.518523:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.518523:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:1.0:1619677448.518524:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.518525:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.518526:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.518526:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.518527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:9.2:1619677448.518531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:13.0:1619677448.518533:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.518536:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.518536:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.518538:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.518538:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:12.0:1619677448.518539:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.518540:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677448.518540:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.518541:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.518542:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.518544:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.518546:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677448.518617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.518619:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.518622:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.518624:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.518625:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.518627:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.518629:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.518630:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.518632:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.518633:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.518634:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.518637:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.518639:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.518642:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.518643:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.518644:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.518646:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.518646:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.518648:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.518649:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.518650:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.518652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.518655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:12.0:1619677448.518658:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.518660:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:12.0:1619677448.518661:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:21.2:1619677448.518662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:12.0:1619677448.518662:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.518663:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.518664:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:13.0:1619677448.518665:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.518665:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:12.0:1619677448.518666:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.518667:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:21.2:1619677448.518668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.518672:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.518674:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.518675:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.518677:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677448.518677:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.518679:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.518682:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.518682:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.518683:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.518685:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.518686:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.518687:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.518689:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.518692:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.518694:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.518694:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.518696:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.518697:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.518698:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.518699:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.518701:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.518702:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.518702:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677448.518703:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.518704:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.518705:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.518705:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.518708:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.518708:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.518708:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.518709:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.518709:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:9.2:1619677448.518710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.518710:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.518711:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677448.518712:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.518713:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.518713:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.518714:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.518714:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.518714:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.518715:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000400:00000200:18.0:1619677448.518716:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.518716:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.518717:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:25.2:1619677448.518718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.518718:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.518721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.518724:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:2.0:1619677448.518726:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:26.0:1619677448.518727:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.518727:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.518728:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.518728:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.518729:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:10.0:1619677448.518730:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.518730:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:10.0:1619677448.518733:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.518756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.518762:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.518765:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.518766:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.518768:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.518770:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.518771:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.518772:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.518773:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.518774:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.518775:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.518778:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.518780:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.518783:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.518784:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.518784:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.518786:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.518786:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.518787:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.518787:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.518792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.518800:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.518802:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.518803:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.518804:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.518806:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.518808:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.518811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.518818:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.518820:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.518821:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.518822:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.518823:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.518823:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.518851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.518857:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.518860:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.518861:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.518863:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.518866:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.518867:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.518868:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713895] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.518870:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.518871:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.518872:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.518875:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.518877:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.518879:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.518879:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.518880:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.518881:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.518882:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.518882:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.518882:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.518887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.518894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677448.518895:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.518897:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.518897:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.518898:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.518899:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.518901:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677448.518901:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:10.0:1619677448.518904:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.518905:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.518907:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.518910:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.518911:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.518913:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.518914:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.518915:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.518916:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.518920:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.518921:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.518924:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.518924:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.518925:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.518926:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.518927:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.518928:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.518928:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.518932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.518939:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.518941:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.518942:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.518944:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.518945:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.518946:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.519150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.519155:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.519158:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.519159:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.519161:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.519163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:18.0:1619677448.519164:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.519165:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.519167:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.519168:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.519169:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.519170:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.519171:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.519173:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:18.0:1619677448.519174:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.519174:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677448.519176:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.519178:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.519179:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.519180:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.519180:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.519180:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:26.0:1619677448.519182:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:18.0:1619677448.519182:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.519182:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:26.0:1619677448.519183:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.519183:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:26.0:1619677448.519184:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.519184:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:26.0:1619677448.519185:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.519186:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:25.2:1619677448.519188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:26.0:1619677448.519189:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.519191:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.519195:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.519195:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.519196:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.519196:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.519196:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.519197:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.519198:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.519199:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:10.0:1619677448.519199:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.519200:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.519201:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.519202:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.519202:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.519204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.519212:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.519215:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.519216:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.519216:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.2:1619677448.519218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:22.0:1619677448.519219:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.519221:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.519222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.519226:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:17.0:1619677448.519227:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.519228:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.519230:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:17.0:1619677448.519231:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.519231:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.519233:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.519234:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.519236:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.519236:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.519238:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677448.519239:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677448.519240:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.519241:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.519241:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.519243:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.519243:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.519245:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.519245:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677448.519247:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677448.519249:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.519251:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.519252:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.519254:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.519255:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.519255:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.519255:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.519257:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.519257:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677448.519257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:26.0:1619677448.519258:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.519259:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.519259:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677448.519259:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.519260:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.519260:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.519261:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.519261:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.519262:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:2.0:1619677448.519262:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:26.0:1619677448.519263:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.519263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.519265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.519265:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.519266:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.519267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:2.0:1619677448.519268:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.519271:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.519271:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.519272:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.519272:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.519273:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:2.0:1619677448.519273:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677448.519273:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.519274:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:2.0:1619677448.519274:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96713889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.519275:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:17.0:1619677448.519275:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.519275:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.519276:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:2.0:1619677448.519276:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.519277:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:2.0:1619677448.519277:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.519277:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:20.0:1619677448.519278:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.519278:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.519281:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.519281:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:2.0:1619677448.519282:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.519284:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677448.519284:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.2:1619677448.519285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.519286:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.519287:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.519288:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.519289:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.519290:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.519290:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.519291:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:21.2:1619677448.519297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:9.2:1619677448.519297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.519301:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.519304:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677448.519304:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.519305:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677448.519305:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.519305:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.519306:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.519307:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:5.0:1619677448.519307:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677448.519308:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677448.519308:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.519308:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.519309:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.519310:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.519310:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.519313:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:6.0:1619677448.519313:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.519314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:6.0:1619677448.519314:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.519316:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.519317:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.519318:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.519319:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:2.0:1619677448.519321:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677448.519323:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.519324:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.519325:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677448.519325:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677448.519325:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.519326:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.519326:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677448.519328:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.519329:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.519330:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.519332:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.519332:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.519333:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.519334:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.519338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:21.2:1619677448.519338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677448.519345:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:30.0:1619677448.519346:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.519348:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:30.0:1619677448.519349:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677448.519349:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.519349:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.519351:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.519352:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.519353:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.519354:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.519355:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:10.0:1619677448.519356:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:30.0:1619677448.519357:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.519358:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.519359:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.519360:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677448.519363:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.519365:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.519368:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.519369:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.519370:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.519371:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.519372:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.519373:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.519373:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.519377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.519386:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.519388:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.519389:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.519390:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.519392:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.519395:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.519544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.519551:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.519553:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.519554:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.519556:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.519559:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.519560:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.519561:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.519563:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.519564:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.519565:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.519568:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.519569:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.519572:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.519573:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.519573:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.519575:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.519575:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.519577:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.519578:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.519578:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.519581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.519584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.519588:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.519591:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677448.519591:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:22.0:1619677448.519592:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.519592:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.519593:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.519593:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.519595:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.519595:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.519598:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.519611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.519618:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.519621:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.519621:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.519623:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.519626:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.519627:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.519629:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.519630:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.519631:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.519631:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.519634:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.519636:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.519638:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.519639:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.519639:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.519641:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.519641:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.519642:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.519642:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.519646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.519654:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.519656:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.519657:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.519658:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.519661:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.519662:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.519692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.519698:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.519700:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.519701:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.519703:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.519706:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.519707:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677448.519708:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.519709:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.519710:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.519711:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677448.519714:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.519716:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677448.519718:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.519719:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677448.519720:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.519722:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677448.519722:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.519723:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.519724:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677448.519728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.2:1619677448.519731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677448.519736:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:1.0:1619677448.519737:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.519738:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.519739:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.519740:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.519740:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.519740:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:5.0:1619677448.519742:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.519743:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:1.0:1619677448.519743:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.519744:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.519745:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.519747:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.519748:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.519749:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.519750:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:8.2:1619677448.519752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000400:00000200:1.0:1619677448.519752:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.519754:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.519757:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.519758:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.519758:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:5.0:1619677448.519759:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.519760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:1.0:1619677448.519760:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.519760:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.519761:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.519762:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:1.0:1619677448.519762:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677448.519763:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.519764:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.519764:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.519765:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:9.2:1619677448.519766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.519769:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.519772:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.519773:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:24.0:1619677448.519775:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.519775:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677448.519776:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.519778:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.519778:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.519781:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.519782:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.519783:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.519784:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.519786:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.519786:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.519788:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.519789:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:24.0:1619677448.519790:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.519792:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.519794:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.519795:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.519796:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.519797:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.519798:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.519799:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.519799:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677448.519803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.2:1619677448.519803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.519810:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677448.519811:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.519813:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:17.0:1619677448.519814:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:31.0:1619677448.519815:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.519815:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.519815:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.519817:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:17.0:1619677448.519817:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.519818:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.519819:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:17.0:1619677448.519820:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:31.0:1619677448.519821:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.519821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000400:00000200:31.0:1619677448.519822:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.519823:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.519824:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.519827:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.519828:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677448.519828:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.519831:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.519831:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.519831:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.519832:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:1.0:1619677448.519832:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.519833:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.519833:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.519833:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677448.519835:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.519835:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.519837:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.519838:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.519839:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.519840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.519847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.519852:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:30.0:1619677448.519854:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.519855:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.519856:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.519857:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.519857:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.519857:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.519858:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.519859:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677448.519860:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.519876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.519883:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.519886:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.519887:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.519889:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.519891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:10.0:1619677448.519891:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.519892:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.519894:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.519895:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.519896:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.519896:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:10.0:1619677448.519897:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.519899:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:10.0:1619677448.519901:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.519902:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.519902:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.519903:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.519905:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.519906:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.519907:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.519907:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:5.0:1619677448.519907:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677448.519908:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.519909:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:5.0:1619677448.519909:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.519910:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.519910:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:5.0:1619677448.519910:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.519911:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:5.0:1619677448.519911:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:25.2:1619677448.519915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:5.0:1619677448.519915:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.519916:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677448.519919:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.519920:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677448.519920:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677448.519922:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.519923:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.519923:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.519924:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.519925:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:22.0:1619677448.519926:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.519926:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677448.519926:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:6.0:1619677448.519927:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.519928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:6.0:1619677448.519929:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.519931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.519932:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677448.519933:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:2.0:1619677448.519935:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:20.0:1619677448.519936:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.519936:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.519937:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677448.519938:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.519939:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.519939:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.519941:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.519944:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.519946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.2:1619677448.519948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:22.0:1619677448.519953:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:18.0:1619677448.519954:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.519957:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:18.0:1619677448.519957:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.519957:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:2.0:1619677448.519957:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.519958:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000400:00000200:22.0:1619677448.519959:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.519959:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.519960:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:22.0:1619677448.519962:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.519963:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:22.0:1619677448.519965:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.519966:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.519967:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.519968:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677448.519971:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.519973:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.519975:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.519976:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.519976:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.519978:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.519978:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677448.519980:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:22.0:1619677448.519981:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:22.0:1619677448.519982:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:25.2:1619677448.519983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.519987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677448.519992:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.519993:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000800:00000200:22.0:1619677448.519993:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:6.0:1619677448.519993:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.519993:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.519994:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677448.519995:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.519995:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.519996:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:6.0:1619677448.519996:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.520005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.520011:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.520014:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.520014:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.520015:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.520015:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.520016:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.520041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.520049:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.520052:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.520054:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.520056:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.520057:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.520058:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.520060:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.520061:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.520062:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.520063:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.520067:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.520068:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.520071:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.520072:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.520072:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.520074:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.520075:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.520076:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.520076:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.520080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.520087:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:21.2:1619677448.520089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.520090:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.520092:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.520092:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.520093:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.520094:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.520094:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.520097:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.520097:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677448.520098:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.520099:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.520101:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.520102:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.520104:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.520105:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.520105:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.520108:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.520110:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.520113:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.520113:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.520114:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.520116:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.520116:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.520117:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.520117:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.520122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.520129:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.520131:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.520132:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.520134:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.520135:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.520136:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.520322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.520328:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.520331:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.520332:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.520334:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.520336:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.520338:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.520339:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.520340:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.520341:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.520342:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.520345:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.520347:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.520350:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.520351:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.520351:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.520353:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.520353:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.520354:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.520355:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.520360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.520367:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.520369:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.520370:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677448.520370:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.520373:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.520376:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677448.520379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677448.520384:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.520387:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677448.520387:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.520388:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.520388:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.520389:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677448.520490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.520498:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.520500:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.520501:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.520503:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.520506:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.520508:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.520509:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713874] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.520510:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.520511:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.520512:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.520515:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.520517:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.520520:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.520521:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.520521:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.520523:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.520523:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.520524:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.520525:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677448.520525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.2:1619677448.520529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.520533:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.520536:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.0:1619677448.520537:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:31.0:1619677448.520538:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.520540:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:30.0:1619677448.520541:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.2:1619677448.520541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:31.0:1619677448.520542:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.520542:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.520544:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.2:1619677448.520544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:5.0:1619677448.520544:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.520545:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.520547:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.520548:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.520548:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:31.0:1619677448.520549:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.520549:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677448.520550:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.520552:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.520552:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.520553:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:31.0:1619677448.520554:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.520554:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:20.0:1619677448.520556:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677448.520557:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.520557:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.520558:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.520558:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677448.520558:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.520560:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.520560:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:30.0:1619677448.520560:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.520560:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.520561:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.520562:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:5.0:1619677448.520562:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677448.520563:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.520563:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:5.0:1619677448.520563:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.520564:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.520565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677448.520565:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.520567:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.520567:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.520568:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.520568:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.520570:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.520572:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:5.0:1619677448.520574:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.520575:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.520575:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.520576:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.520576:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:13.0:1619677448.520577:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.520578:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:12.0:1619677448.520579:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.520579:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.520580:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.520581:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.520581:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.520582:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:25.2:1619677448.520582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.520582:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.520583:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.520584:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.520584:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.520585:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.520585:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.520585:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677448.520586:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677448.520586:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.520587:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677448.520588:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.520591:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:8.2:1619677448.520591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677448.520592:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677448.520593:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.520593:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.520595:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.520597:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.520598:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.520599:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:12.0:1619677448.520599:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.520601:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:13.0:1619677448.520601:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677448.520602:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.520603:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:20.0:1619677448.520604:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.520604:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.520607:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.520755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.520761:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.520764:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.520764:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.520767:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.520769:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.520770:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.520772:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.520773:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677448.520773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000400:00000200:6.0:1619677448.520775:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.520776:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.520779:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.520780:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:6.0:1619677448.520781:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:24.0:1619677448.520783:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677448.520783:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:6.0:1619677448.520783:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.520784:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.520785:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:8.0:1619677448.520786:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.520787:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.520788:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.520788:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:8.0:1619677448.520789:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.520789:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000400:00000200:8.0:1619677448.520790:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:6.0:1619677448.520790:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.520793:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.520795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.2:1619677448.520795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:8.0:1619677448.520796:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96713869] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.520798:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.520799:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.520800:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:8.0:1619677448.520800:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:22.0:1619677448.520802:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.520802:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.520803:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.520803:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:8.0:1619677448.520803:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.520804:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:8.0:1619677448.520805:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.520806:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:5.0:1619677448.520806:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:26.0:1619677448.520807:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677448.520807:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.520807:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.520808:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677448.520809:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:10.0:1619677448.520809:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.520809:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:5.0:1619677448.520809:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.520810:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:8.0:1619677448.520811:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000400:00000200:10.0:1619677448.520812:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.520812:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677448.520812:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:5.0:1619677448.520812:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:10.0:1619677448.520813:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.520815:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.520816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:10.0:1619677448.520816:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.520817:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.520818:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.520821:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.520822:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.520823:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.520825:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:10.0:1619677448.520825:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.520825:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:31.0:1619677448.520826:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.520826:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677448.520827:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.520828:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.520828:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.520829:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.520830:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:10.0:1619677448.520830:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.520831:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.520831:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677448.520834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.520837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:25.2:1619677448.520838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:2.0:1619677448.520842:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:30.0:1619677448.520844:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.520844:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:2.0:1619677448.520845:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.520846:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:18.0:1619677448.520846:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.520846:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.520847:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.520848:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.520848:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.520849:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.520849:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000400:00000200:2.0:1619677448.520849:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.520850:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.520851:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:2.0:1619677448.520852:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.520887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.520894:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.520897:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.520898:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.520899:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.520902:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.520903:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.520905:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.520906:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.520907:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.520908:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.520911:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.520912:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.520915:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.520916:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.520916:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.520918:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.520918:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.520920:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.520921:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.520921:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.2:1619677448.520922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.2:1619677448.520924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.520927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.520929:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.520931:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.520933:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677448.520934:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:1.0:1619677448.520934:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.520935:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.520936:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.520936:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.520937:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677448.520937:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677448.520937:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.520938:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.520939:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.520940:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.520941:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.520942:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.520943:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.520948:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.520949:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.520950:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.520951:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.520952:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677448.520952:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.520953:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:21.2:1619677448.520955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677448.520955:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.520956:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.520957:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.520958:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.520958:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.520962:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:9.2:1619677448.520962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:31.0:1619677448.520965:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.520967:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.520968:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:30.0:1619677448.520969:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.520970:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.520971:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:8.0:1619677448.520971:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.520972:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677448.520972:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.520973:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.520974:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.520974:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.520975:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.520975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:31.0:1619677448.520976:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:8.0:1619677448.520977:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:31.0:1619677448.520979:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.520981:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.520983:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.520984:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.520984:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.520984:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677448.520985:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:30.0:1619677448.520986:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.520987:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.520988:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:30.0:1619677448.520988:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.520990:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677448.520990:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:31.0:1619677448.520991:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:30.0:1619677448.520991:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.520992:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.520992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.520993:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.520993:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.520993:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.520994:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.520997:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.520997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.520998:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:30.0:1619677448.520999:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.521001:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:30.0:1619677448.521002:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.521002:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.521002:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.521003:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.521003:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:10.0:1619677448.521003:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.521005:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.521005:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.521005:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.521006:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.521006:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.521006:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677448.521007:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677448.521007:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.521008:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.521011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:25.2:1619677448.521013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.521018:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677448.521019:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.521021:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677448.521022:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677448.521022:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.521023:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.521023:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.521024:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:20.0:1619677448.521024:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.521025:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.0:1619677448.521025:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.2:1619677448.521026:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.521030:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677448.521031:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.521034:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.521036:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.521037:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.521038:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.521040:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677448.521041:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.521043:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.521044:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.521045:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.521048:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.521050:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.521053:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.521053:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.521054:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.521056:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.521057:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.521057:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.521058:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677448.521062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.521070:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.521072:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.521073:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.521073:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.521075:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677448.521078:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.521122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.521128:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.2:1619677448.521129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677448.521131:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.521131:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.521133:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.521134:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:18.0:1619677448.521135:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.521137:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:5.0:1619677448.521137:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:18.0:1619677448.521138:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.521139:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.521140:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:18.0:1619677448.521140:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.521141:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.521142:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.521142:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.521144:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.521145:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.521146:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:5.0:1619677448.521147:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.521148:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.521149:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.521149:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.521151:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.521153:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.521153:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:5.0:1619677448.521153:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.521155:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.521155:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.521156:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:5.0:1619677448.521156:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.521157:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.0:1619677448.521159:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.521160:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.521160:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:25.2:1619677448.521161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.521162:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.521163:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.521164:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.521164:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.521168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.521169:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.521171:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.521172:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.521173:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:13.0:1619677448.521175:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677448.521175:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:13.0:1619677448.521177:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.521178:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.521178:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:13.0:1619677448.521180:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.521181:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.521183:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.521291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.521300:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.521303:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.521304:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.521306:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.521309:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.521310:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.521311:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.521313:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.521313:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.521314:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.521317:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.521319:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.521321:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.521322:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.521323:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.521324:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.521325:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.521325:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.521326:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.521331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.521338:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.521341:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.521341:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.521343:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.521346:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.521347:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.521372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.521378:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.521381:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.521382:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.521383:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.521386:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.521387:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.521389:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.521390:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.521391:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.521392:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.521395:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.521397:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.521399:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.521400:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.521401:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.521402:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.521403:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.521404:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.521405:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.521406:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.521408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.521411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.521416:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.521416:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.521418:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677448.521419:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677448.521419:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677448.521420:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.521420:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.521420:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:21.2:1619677448.521421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:22.0:1619677448.521422:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.521425:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677448.521427:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:25.2:1619677448.521429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677448.521430:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.521431:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.521433:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.521435:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.521436:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:2.0:1619677448.521436:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.521438:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.521439:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677448.521439:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.521440:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.521440:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000400:00000200:2.0:1619677448.521440:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.521441:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.521441:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:2.0:1619677448.521441:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.521444:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.521446:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.521448:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.521449:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.521449:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.521451:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.521451:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.521452:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.521453:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.521457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.521464:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.521467:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.521467:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.521468:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.521469:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.521471:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.521676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.521684:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.521687:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.521689:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.521691:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.521692:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.521693:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.521694:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.521696:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.521697:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.521698:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.521702:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.521703:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.521706:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.521707:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.521708:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.521710:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.521710:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.521711:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.521712:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.521716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.521723:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.521726:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.521727:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.521728:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.521730:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.521731:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.521733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:8.2:1619677448.521734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.521738:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677448.521741:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.521741:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.521742:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677448.521744:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.521744:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.521745:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000400:00000200:2.0:1619677448.521746:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.521747:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.2:1619677448.521747:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1200] (20)++
00000400:00000200:2.0:1619677448.521748:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.521749:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.521750:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.521750:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:2.0:1619677448.521751:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.521752:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.521752:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000400:00000200:2.0:1619677448.521753:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.521756:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:1.0:1619677448.521756:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000400:00000200:2.0:1619677448.521757:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.521759:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.521760:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.521762:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.521762:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:1.0:1619677448.521762:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.521763:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:1.0:1619677448.521764:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:2.0:1619677448.521765:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.521765:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:1.0:1619677448.521765:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.521766:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.521766:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:1.0:1619677448.521767:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.521768:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.521768:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:8.2:1619677448.521771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:1.0:1619677448.521772:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.521774:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:1.0:1619677448.521778:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.521779:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.521779:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:17.0:1619677448.521780:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.521780:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:17.0:1619677448.521781:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.521782:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:1.0:1619677448.521782:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000400:00000200:17.0:1619677448.521783:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.521783:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:1.0:1619677448.521784:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:1.0:1619677448.521785:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.521786:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.521788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.521795:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.521798:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.521798:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.521800:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.521801:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.521802:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:21.2:1619677448.521852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.521860:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.521862:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.521865:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.521867:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.521867:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.521869:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.521870:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.521871:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.521873:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.521874:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.521877:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.521878:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.521881:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.521881:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.521882:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.521884:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.521884:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.521885:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.521886:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.521890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.521897:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.521900:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.521900:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.521902:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.521904:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.521905:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.521915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.521921:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.521924:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.521925:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.521927:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.521929:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.521931:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.521932:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.521934:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.521935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000400:00000200:1.0:1619677448.521935:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.521936:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.521940:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.521941:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.521942:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677448.521943:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.521944:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.521946:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.521947:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.521947:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:17.0:1619677448.521948:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:1.0:1619677448.521948:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:17.0:1619677448.521950:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.521951:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.521951:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:17.0:1619677448.521952:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.521953:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.521953:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.521955:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.521955:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.521956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:17.0:1619677448.521956:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.521956:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677448.521958:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.521960:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.521961:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:17.0:1619677448.521961:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.521963:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677448.521963:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:12.0:1619677448.521963:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677448.521964:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677448.521964:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:12.0:1619677448.521965:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.521966:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.521968:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.521969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:13.0:1619677448.521969:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.521971:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677448.521977:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.521979:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.521980:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.521980:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.521982:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.521985:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.522155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.522161:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.522164:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.522166:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.522166:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.522169:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.522171:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.522172:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.522174:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.522175:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.522176:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.522180:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.522182:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.522185:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.522186:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.522187:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.522188:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.522189:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.522189:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.522190:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.522194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.522201:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.522204:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.522205:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.522205:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.522207:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.522209:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.522209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:25.2:1619677448.522212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.522216:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.522219:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:22.0:1619677448.522219:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.522220:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677448.522222:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.522223:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.522223:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.522223:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.522224:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.522225:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:26.0:1619677448.522226:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.522227:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.522228:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.522230:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.522231:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.522232:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.522235:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.522237:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.522238:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.522239:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.522240:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.522241:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.522241:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.522242:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.522243:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.522246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.522252:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.522254:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.522254:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.522255:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.522256:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.522257:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.522259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.522266:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.522269:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.522271:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.522272:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.522275:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.522276:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.522277:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.522279:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.522279:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.522280:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.522284:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.522285:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.522288:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.522289:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.522289:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.522291:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.522291:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.522292:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.522292:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.522298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677448.522303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:5.2:1619677448.522304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:1.0:1619677448.522306:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.522309:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.522310:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.522310:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.522311:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:18.0:1619677448.522311:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:1.0:1619677448.522312:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.522314:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:8.2:1619677448.522314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.522315:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:10.0:1619677448.522315:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:2.0:1619677448.522316:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:26.0:1619677448.522318:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.522318:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:18.0:1619677448.522319:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.522320:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.522320:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677448.522322:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.522322:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.522322:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.522323:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.522323:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000400:00000200:26.0:1619677448.522324:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.522324:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677448.522324:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.522324:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:18.0:1619677448.522326:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.522327:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.522328:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.522329:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.522330:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.522331:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.522331:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.522336:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.522337:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.522339:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.522340:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.522342:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.522343:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.522344:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.522344:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.522345:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.522345:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.522346:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.522346:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:26.0:1619677448.522348:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.522348:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:18.0:1619677448.522348:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:26.0:1619677448.522349:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:18.0:1619677448.522349:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:26.0:1619677448.522350:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:18.0:1619677448.522350:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:25.2:1619677448.522353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:9.2:1619677448.522354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.522358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:13.0:1619677448.522361:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.522364:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.522364:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.522365:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:13.0:1619677448.522365:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.522366:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.522366:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.522367:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:13.0:1619677448.522367:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.522367:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.522368:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:12.0:1619677448.522369:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:2.0:1619677448.522369:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.522370:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677448.522370:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.522370:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.522371:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.522372:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.522372:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.522372:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.522373:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677448.522374:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.522375:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.522494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.522500:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.522503:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.522503:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.522505:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.522507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:17.0:1619677448.522508:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.522509:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.522510:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.522512:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.522513:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:17.0:1619677448.522513:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.522514:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.522515:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.522516:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.522519:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.522519:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.522521:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:18.0:1619677448.522522:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.522524:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:17.0:1619677448.522524:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.522524:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:18.0:1619677448.522525:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.522525:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:18.0:1619677448.522527:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.522527:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.522527:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.522527:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.522528:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.522528:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:18.0:1619677448.522529:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:25.2:1619677448.522532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:18.0:1619677448.522532:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.522534:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677448.522536:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.522537:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677448.522538:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.522539:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:12.0:1619677448.522539:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:18.0:1619677448.522540:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.522541:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:12.0:1619677448.522541:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:18.0:1619677448.522542:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.522542:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:12.0:1619677448.522542:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.522544:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.522545:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.2:1619677448.522545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.522547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:12.0:1619677448.522547:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.522552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:20.0:1619677448.522553:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677448.522553:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.522555:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:20.0:1619677448.522556:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:2.0:1619677448.522556:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:20.0:1619677448.522557:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.522557:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:20.0:1619677448.522558:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.522559:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677448.522560:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.522561:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677448.522562:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:8.2:1619677448.522562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:13.0:1619677448.522563:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.522563:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.522564:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.522565:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677448.522569:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.522572:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.522573:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.522573:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.522574:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.522575:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.522613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.522621:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:4.2:1619677448.522622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.522623:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.522625:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.522628:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.522628:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.522629:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:10.0:1619677448.522629:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.522631:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.522631:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:31.0:1619677448.522632:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.522632:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677448.522633:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.522634:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.522634:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.522637:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.522637:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.522639:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.522639:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.522640:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.522641:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.522642:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.522642:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.522643:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:10.0:1619677448.522643:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.522644:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.522646:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:10.0:1619677448.522646:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.522647:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.522648:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:21.2:1619677448.522648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:10.0:1619677448.522648:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.522649:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.522650:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.522650:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.522650:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.522651:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.522652:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:8.2:1619677448.522652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.522653:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.522654:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.522655:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.522655:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677448.522655:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.2:1619677448.522656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:6.0:1619677448.522658:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:25.2:1619677448.522659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.522659:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.0:1619677448.522659:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:6.0:1619677448.522660:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.522662:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:4.2:1619677448.522662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677448.522663:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.522663:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.522664:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.522665:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:6.0:1619677448.522665:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677448.522665:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.522666:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.522666:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:6.0:1619677448.522666:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96713842] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.522667:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.522667:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.522668:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.522668:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:6.0:1619677448.522668:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.522669:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:6.0:1619677448.522669:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.522669:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:31.0:1619677448.522670:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.522670:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:6.0:1619677448.522670:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:26.0:1619677448.522671:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:31.0:1619677448.522672:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.522672:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.522673:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.522673:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.522674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.522675:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.522678:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.522679:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.522679:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.522681:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.522681:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.522682:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.522682:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.522683:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677448.522684:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.522684:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.522685:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.522686:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.522687:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:9.2:1619677448.522687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.522695:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.522698:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.522699:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.522699:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.522701:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.522704:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.522932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.522935:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.522938:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.522940:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.522941:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.522942:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.522944:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.522945:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.522946:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.522948:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.522949:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.522952:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.522954:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.522956:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.522957:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.522958:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.522960:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.522960:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.522961:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.522963:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.522963:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.522966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.522968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677448.522973:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.522974:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:22.0:1619677448.522976:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677448.522976:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:22.0:1619677448.522977:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.522977:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.522977:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.522978:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:22.0:1619677448.522979:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.522981:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.523065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.523071:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.523074:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.523075:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.523077:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.523079:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.523080:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.523081:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.523083:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.523084:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.523085:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.523088:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.523090:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.523092:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.523093:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.523093:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.523095:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.523096:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.523096:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.523097:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.523101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.523109:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.523111:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.523112:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.523112:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.523114:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.523117:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.523276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.523283:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.523286:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.523287:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.523289:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.523292:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.523298:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.523299:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.523301:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.523302:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.523303:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677448.523306:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.523308:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.523311:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.523312:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.523312:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.523314:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.523315:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.523315:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.523316:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.523320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.523328:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.523330:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.523331:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.523333:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.523334:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.523336:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.523350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.523358:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.523361:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.523363:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.523365:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.523366:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.523368:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.523369:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.523371:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.523372:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.523373:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.523376:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.523378:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.523380:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.523380:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.523381:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.523382:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.523383:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.523384:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.523384:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.523389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.523396:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.523399:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.523399:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.523401:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.523403:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.523404:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.523423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.523430:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.523433:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.523434:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.523436:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.523438:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.523439:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.523441:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713837] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.523442:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.523443:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.523443:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.523447:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.523448:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.523450:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.523451:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.523451:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.523453:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.523453:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.523454:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.523455:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.523459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.523466:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.523468:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.523469:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.523470:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.523472:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.523474:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.523533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.523540:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.523543:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.523543:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.523546:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.523548:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.523549:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:20.2:1619677448.523550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:6.0:1619677448.523550:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.523551:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.523552:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.523553:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:6.0:1619677448.523553:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:20.0:1619677448.523556:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:6.0:1619677448.523557:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.523559:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.523559:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:6.0:1619677448.523560:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:4.2:1619677448.523560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:6.0:1619677448.523562:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.523563:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.523563:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.523563:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:20.0:1619677448.523565:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:6.0:1619677448.523565:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.523565:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:20.0:1619677448.523566:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.523566:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.523567:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:24.0:1619677448.523568:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:20.0:1619677448.523568:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.523569:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.523570:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677448.523571:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.2:1619677448.523571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677448.523572:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:20.0:1619677448.523574:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.523576:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:24.0:1619677448.523578:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.523579:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:24.0:1619677448.523580:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.523580:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.523580:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:20.0:1619677448.523581:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:18.0:1619677448.523581:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:24.0:1619677448.523582:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.523583:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96713834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.523583:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:20.0:1619677448.523583:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:18.0:1619677448.523583:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.523583:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677448.523584:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.523585:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:18.0:1619677448.523585:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.523586:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.523587:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.523588:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.523588:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.523589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:24.0:1619677448.523592:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.523594:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.523596:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.523597:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.523598:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.523599:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:8.0:1619677448.523599:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.523600:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:8.0:1619677448.523601:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.523602:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.523603:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.523603:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.523604:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677448.523604:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.523605:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.2:1619677448.523608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.523615:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.523618:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.523619:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.523619:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.523621:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.523624:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.523631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.523638:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.523641:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:5.2:1619677448.523641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.523642:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.523643:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.523646:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.523646:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.523647:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.523649:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.523649:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.523650:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.523651:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.523653:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.523653:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.523654:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.523656:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.523657:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:26.0:1619677448.523658:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.523659:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.523660:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.523660:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.523661:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.523662:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:26.0:1619677448.523663:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.523664:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.523664:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.523666:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.523666:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677448.523666:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.523667:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.523668:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677448.523668:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:25.2:1619677448.523671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.523671:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.523672:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.523673:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.523674:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.523675:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.523676:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.523676:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.523678:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.523680:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:9.2:1619677448.523680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:31.0:1619677448.523681:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.523683:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.523684:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.523686:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:24.0:1619677448.523687:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.523689:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.523690:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.523691:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.523692:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.523696:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.524015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.524022:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.524025:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.524026:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.524028:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.524030:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.524031:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.524032:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.524034:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.524035:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.524036:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.524039:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.524041:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.524043:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.524044:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.524045:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.524046:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.524046:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.524047:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.524047:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.524052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.524059:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.524061:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.524062:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.524064:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.524065:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.2:1619677448.524065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.524066:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677448.524071:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.524073:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.524074:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.524077:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.524079:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.524081:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.524082:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713830] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.524083:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.524084:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.524085:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.524088:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.524090:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.524091:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.524092:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.524093:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.524094:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.524095:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.524096:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.524097:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.524098:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.524100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.524103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.524109:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.524110:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.524110:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.524111:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.524111:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.524111:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.524112:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.524113:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:1.0:1619677448.524113:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.524114:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.524121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.524128:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.524130:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.524131:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.524131:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.524131:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.524132:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.524171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.524177:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.524181:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.524181:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.524184:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.524187:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.524188:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.524190:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.524191:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.524192:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.524193:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.524197:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.524199:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:20.2:1619677448.524200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:22.0:1619677448.524202:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.524203:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.524204:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.524206:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:31.0:1619677448.524207:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:22.0:1619677448.524207:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.524208:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.524208:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677448.524210:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.524212:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.524212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:30.0:1619677448.524214:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.524215:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.524216:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.524217:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.524218:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:31.0:1619677448.524219:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.524220:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.524221:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.524221:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.524221:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.524222:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.524224:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.524225:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.524227:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.524227:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:31.0:1619677448.524229:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.524230:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:25.2:1619677448.524230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000800:00000200:31.0:1619677448.524231:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.524233:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.524233:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.524234:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.524235:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:10.0:1619677448.524236:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:25.2:1619677448.524238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.524239:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:21.2:1619677448.524239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:10.0:1619677448.524239:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.524240:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.524240:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.524241:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:12.0:1619677448.524245:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.0:1619677448.524246:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:12.0:1619677448.524247:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.524248:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.524249:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.524249:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.524250:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677448.524251:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:8.0:1619677448.524251:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.524252:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:8.0:1619677448.524254:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.524255:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.524257:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.524258:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.524259:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.524260:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.524263:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.524265:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.524268:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.524269:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.524270:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.524271:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.524272:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.524273:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.524275:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.524275:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677448.524278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.524281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:2.0:1619677448.524285:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677448.524287:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:2.0:1619677448.524288:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.524289:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.524289:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.524290:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.524290:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.524291:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:2.0:1619677448.524292:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.524298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.524303:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.524305:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.524307:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.524307:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.524308:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.524309:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.524309:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.524330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.524337:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.524340:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.524341:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.524342:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.524345:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.524347:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.524348:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.524349:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.524350:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.524351:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.524355:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.524357:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.524360:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.524360:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.524361:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.524362:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.524363:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.524364:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677448.524365:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.524365:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677448.524369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677448.524371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:5.2:1619677448.524377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677448.524378:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.524381:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.524381:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:10.0:1619677448.524382:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:1.0:1619677448.524382:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.524383:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.524384:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:1.0:1619677448.524384:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.524385:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.524386:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:26.0:1619677448.524387:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.524387:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:1.0:1619677448.524387:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:10.0:1619677448.524389:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.524392:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.524393:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.524395:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.524396:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.524397:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.524397:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:4.2:1619677448.524398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:10.0:1619677448.524400:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.524402:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.524405:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.524405:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.524406:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.524407:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.524408:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.524408:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.524409:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.524410:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.524410:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:26.0:1619677448.524411:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.524412:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.524413:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.524414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:26.0:1619677448.524415:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.524415:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.524417:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.524417:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.524418:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.524421:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.524421:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:26.0:1619677448.524422:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:12.0:1619677448.524423:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.524424:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.524425:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677448.524425:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.524426:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.524426:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.524428:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:13.0:1619677448.524428:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.524428:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.524429:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.524429:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.524430:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.524434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.2:1619677448.524434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.524440:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.524443:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:6.0:1619677448.524443:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:1.0:1619677448.524444:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.524445:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.524446:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.524446:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:17.0:1619677448.524447:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.524447:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.524449:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.524450:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.524451:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.524452:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.524453:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.524453:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:17.0:1619677448.524454:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.524458:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.524459:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.524462:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.524463:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.524463:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.524465:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.524466:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.524466:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.524467:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:21.2:1619677448.524468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:25.2:1619677448.524471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.524477:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.524479:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.524479:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677448.524481:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.524481:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.524482:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.524483:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.524484:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.524484:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.524486:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.524487:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.524488:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.524489:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.524492:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.524493:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.524494:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.524494:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.524495:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.524497:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.524498:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.524498:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.524499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:30.0:1619677448.524500:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.524501:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.524501:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.524502:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677448.524504:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:25.2:1619677448.524505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.524507:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.524508:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.524510:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.524512:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.524512:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:18.0:1619677448.524513:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.524514:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.524514:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:18.0:1619677448.524515:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.524515:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.524516:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677448.524517:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.524518:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:13.0:1619677448.524518:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.524519:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:18.0:1619677448.524521:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.524523:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:21.2:1619677448.524525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.524525:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.524526:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.524527:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.524529:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.524530:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.524531:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.524531:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.524532:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677448.524533:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.524533:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:9.2:1619677448.524535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677448.524535:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.524538:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.524539:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677448.524541:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.524542:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.524543:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:2.0:1619677448.524543:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.524544:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.524544:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677448.524545:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.524546:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.524547:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.524549:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.524549:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.524550:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.524550:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.524551:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.524551:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.524553:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.524554:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.524554:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.524555:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.524559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.524566:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.524568:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.524568:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.524570:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.524570:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.524571:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.524711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.524717:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.524720:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.524720:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:4.2:1619677448.524722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:1.0:1619677448.524722:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.524725:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.524727:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.524728:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.524729:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:1.0:1619677448.524729:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.524730:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.524731:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.524732:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.524733:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677448.524735:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.524737:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.524738:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.524739:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.524740:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.524740:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:24.0:1619677448.524741:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:1.0:1619677448.524741:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:24.0:1619677448.524742:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96713818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.524743:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.524743:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:24.0:1619677448.524744:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.524744:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.524744:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677448.524745:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.524746:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.2:1619677448.524749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:24.0:1619677448.524750:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.524753:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.524755:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.524756:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.524756:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677448.524757:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (23)++
00000800:00000200:24.0:1619677448.524759:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (25)--
00000800:00000200:24.0:1619677448.524759:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.524760:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677448.524760:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (24)--
00000800:00000200:1.0:1619677448.524760:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:24.0:1619677448.524761:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677448.524761:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.524763:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.524765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (22)++
00000800:00000200:4.2:1619677448.524768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677448.524769:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.524771:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.524772:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.524773:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.524773:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.524775:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.524775:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.524777:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:10.0:1619677448.524778:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.524779:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.524780:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.524783:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.524785:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.524786:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.524787:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.524788:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.524789:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.524792:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.524793:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.524795:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.524796:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.524797:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.524798:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.524799:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.524800:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.524801:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.524802:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.524804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.524807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.524811:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.524813:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.524814:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.524814:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677448.524815:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.524816:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:30.0:1619677448.524817:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.524817:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.524818:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.524819:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.524820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.524827:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.524830:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.524831:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.524831:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.524832:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.524833:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.524885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677448.524892:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677448.524895:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677448.524897:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.524899:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.524899:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677448.524901:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:13.0:1619677448.524902:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677448.524903:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.524904:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677448.524905:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:13.0:1619677448.524908:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.524910:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.524913:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.524914:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:13.0:1619677448.524914:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:13.0:1619677448.524916:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:13.0:1619677448.524917:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677448.524918:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677448.524920:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677448.524920:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:8.2:1619677448.524922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.524927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677448.524934:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:5.0:1619677448.524935:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:12.0:1619677448.524936:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677448.524937:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:5.0:1619677448.524937:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:12.0:1619677448.524938:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000400:00000200:5.0:1619677448.524938:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.524940:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.2:1619677448.524940:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:5.0:1619677448.524945:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.524948:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677448.524948:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:26.0:1619677448.524951:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.524951:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.524953:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.524955:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.524957:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.524958:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.524960:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.524961:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.524961:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.524965:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.524966:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.524969:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.524970:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.524971:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.524973:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.524973:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.524974:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.524975:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.524976:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677448.524979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.524981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.524986:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.524988:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:2.0:1619677448.524989:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.524990:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.524990:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.524991:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.524991:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.524992:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:2.0:1619677448.524992:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.524996:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.525019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.525025:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.525027:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.525028:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.525030:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.525032:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.525033:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.525035:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.525036:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.525037:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.525038:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.525041:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.525043:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.525045:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677448.525046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677448.525046:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.525047:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.525049:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.525049:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.525050:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.525050:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.525054:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677448.525055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.525058:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.525060:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.525061:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.525062:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.525062:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:30.0:1619677448.525064:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.525065:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.525065:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:30.0:1619677448.525066:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.525066:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.525066:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:30.0:1619677448.525068:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.525069:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:22.0:1619677448.525069:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.525071:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:30.0:1619677448.525073:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.525075:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.525077:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.525078:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.525079:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.525080:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.525081:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.525082:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.525084:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.525084:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.525086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.525090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.525092:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.525094:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.525095:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.525095:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.525097:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:10.0:1619677448.525097:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.525100:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.525100:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677448.525100:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677448.525101:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677448.525103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.525106:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.525109:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.525111:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.525112:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.525114:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.525115:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.525117:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.525119:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677448.525119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:20.0:1619677448.525120:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.525122:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.525125:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.525126:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.525127:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.525129:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:20.0:1619677448.525130:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.525130:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677448.525131:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:26.0:1619677448.525132:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.525133:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:26.0:1619677448.525135:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.525137:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:20.0:1619677448.525137:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:26.0:1619677448.525138:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.525138:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:26.0:1619677448.525139:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.525139:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.525140:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677448.525140:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.525140:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677448.525141:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.525141:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677448.525144:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:26.0:1619677448.525145:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.525146:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.525147:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.525148:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.525148:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.525150:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:26.0:1619677448.525151:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.525152:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96713810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.525152:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.525152:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677448.525153:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.525154:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.525154:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.525154:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.525155:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.525155:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.525156:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:31.0:1619677448.525157:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.525159:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.2:1619677448.525160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.525162:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677448.525162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.525163:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)++
00000800:00000200:31.0:1619677448.525164:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (23)++
00000800:00000200:31.0:1619677448.525165:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (24)--
00000800:00000200:31.0:1619677448.525166:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.525167:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.525167:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.525167:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (23)++
00000800:00000200:12.0:1619677448.525168:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.525170:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (24)--
00000800:00000200:12.0:1619677448.525171:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:5.0:1619677448.525171:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.525172:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.525173:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (23)++
00000400:00000200:5.0:1619677448.525173:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.525174:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (24)--
00000400:00000200:12.0:1619677448.525174:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.525175:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.525175:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.525176:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.525177:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.525178:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.525178:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.525180:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.525186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.525193:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.525196:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.525197:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.525199:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.525200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:8.0:1619677448.525201:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.525203:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.525204:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.525205:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.525205:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:8.0:1619677448.525206:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.525207:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:8.0:1619677448.525208:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.525210:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.525210:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677448.525212:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.525213:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.525214:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:5.0:1619677448.525215:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:5.0:1619677448.525216:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.525217:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.525217:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:5.0:1619677448.525217:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.525218:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:5.0:1619677448.525219:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.525219:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677448.525220:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:5.0:1619677448.525223:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677448.525224:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.2:1619677448.525225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.525225:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677448.525225:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677448.525226:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677448.525228:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.525229:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677448.525229:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.0:1619677448.525231:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:13.0:1619677448.525232:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677448.525232:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.525233:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.525233:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:13.0:1619677448.525234:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.525235:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.525237:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.525237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:12.0:1619677448.525238:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.525239:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677448.525243:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.525246:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.525247:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.525247:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.525249:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677448.525252:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.525385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.525393:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.525396:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.525398:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.525401:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.525401:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.525403:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.525404:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.525405:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.525406:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.525407:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.525410:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.525412:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.525415:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.525416:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.525417:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.525418:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.525419:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.525420:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.525421:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.525425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.525434:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.525437:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.525438:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.525438:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.525440:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.525443:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.525451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.525457:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.525459:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.525460:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.525462:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.525465:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.525466:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.525468:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.525469:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.525470:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.525471:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.525475:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.525476:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.525479:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.525480:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.525481:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.525483:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.525483:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.525485:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.525486:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.525486:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.525489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.2:1619677448.525491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.525496:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:1.0:1619677448.525496:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.525498:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.0:1619677448.525499:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677448.525499:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677448.525500:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.525500:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.525500:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.525503:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.525506:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677448.525556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.525564:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.525566:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.525567:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.525569:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.525572:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.525573:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.525575:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713805] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.525576:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.525577:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.525578:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.525581:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.525583:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.525585:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.525585:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.525586:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.525588:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.525588:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.525588:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.525589:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.525593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.525600:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.525602:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.525603:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.525605:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.525606:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.525608:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.525708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.525715:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.525717:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.525718:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.525720:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.525722:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.525723:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.525724:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.525726:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.525726:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.525727:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.525730:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.525732:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.525735:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.525736:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.525736:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.525738:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.525738:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.525740:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.525741:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.525741:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.525744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:21.2:1619677448.525745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677448.525747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677448.525752:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.525752:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:8.0:1619677448.525754:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:30.0:1619677448.525755:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677448.525755:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.525756:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.525756:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:5.2:1619677448.525756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677448.525756:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.525757:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:30.0:1619677448.525758:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.525758:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.525758:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:31.0:1619677448.525759:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.525761:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.525762:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:17.0:1619677448.525762:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:26.0:1619677448.525763:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:30.0:1619677448.525764:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.525764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000400:00000200:30.0:1619677448.525765:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.525765:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:30.0:1619677448.525766:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.525766:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:30.0:1619677448.525767:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:26.0:1619677448.525769:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.525771:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.525771:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.525772:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677448.525773:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:26.0:1619677448.525773:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.525774:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.525774:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.525775:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677448.525776:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.525776:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.525776:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.525776:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677448.525777:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:26.0:1619677448.525777:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.525777:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677448.525778:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:26.0:1619677448.525778:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.525779:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.525780:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.525781:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.525781:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:26.0:1619677448.525781:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.525782:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.525785:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677448.525785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:26.0:1619677448.525786:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.525787:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.525789:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.525789:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.525790:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.525790:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:18.0:1619677448.525793:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:9.2:1619677448.525795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.525796:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.525798:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.525799:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.525800:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.525801:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.525802:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.525803:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:2.0:1619677448.525803:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.525804:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:9.2:1619677448.525805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000400:00000200:2.0:1619677448.525805:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.525808:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.525809:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.525811:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.525812:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.525813:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.525814:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.525814:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.525847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:21.2:1619677448.525854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:31.0:1619677448.525855:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677448.525858:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677448.525860:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.525860:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677448.525862:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677448.525863:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.525863:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.525863:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:31.0:1619677448.525865:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.525866:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.525866:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.525867:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.525869:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.525869:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.525869:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.525870:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.525872:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.525874:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.525875:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.525876:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.525877:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.525878:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.525881:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.525881:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.525882:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.525883:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.525883:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.525884:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.525885:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:22.0:1619677448.525885:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.525886:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:22.0:1619677448.525886:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.525887:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:22.0:1619677448.525887:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.525889:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.525889:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.525890:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:9.2:1619677448.525890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.525891:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.525892:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.525895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.525897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677448.525899:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.525902:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:6.0:1619677448.525902:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:20.0:1619677448.525903:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:18.0:1619677448.525903:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.525903:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.525905:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.525905:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.525905:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677448.525906:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.525906:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.525907:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:20.0:1619677448.525908:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.525908:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677448.525909:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.525911:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.525934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.525942:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.525945:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.525947:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.525948:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.525950:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.525951:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:4.2:1619677448.525951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:30.0:1619677448.525952:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.525954:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.525955:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.525956:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.525958:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677448.525960:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.525960:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:30.0:1619677448.525961:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.525961:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.525963:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.525964:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.525964:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.525965:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:10.0:1619677448.525966:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.525967:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.525967:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677448.525967:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677448.525968:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:10.0:1619677448.525968:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.525969:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:10.0:1619677448.525969:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.525970:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.525971:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:25.2:1619677448.525972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:10.0:1619677448.525974:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.525976:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.525978:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.525979:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:10.0:1619677448.525979:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.525979:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.525981:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:10.0:1619677448.525981:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.525981:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.525982:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.525982:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.525982:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:31.0:1619677448.525983:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.525985:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.525986:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.525986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.525993:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.525995:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.525996:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.525997:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.525999:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.526000:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.526032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.526037:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.526040:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.526042:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.526042:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.526044:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.526046:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.526047:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.526049:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.526050:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.526051:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.526054:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.526056:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.526059:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.526060:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.526060:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.526062:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.526063:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.526064:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.526064:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.526068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.526076:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.526079:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.526080:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.526082:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.526083:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.526084:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.526090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.526097:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.526099:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.526100:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.526101:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.526102:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.526103:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.526189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.526197:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.526199:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.526201:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.526203:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.526204:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.526205:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.526206:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.526208:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.526209:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.526210:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.526213:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.526215:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.526217:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.526218:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.526219:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.526220:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.526221:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.526221:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.526222:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.526226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.526232:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.526234:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.526235:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.526235:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.526237:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.526240:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.526292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.526299:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.526302:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.526303:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.526305:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.526308:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.526309:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.526311:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.526312:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.526313:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.526314:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.526317:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.526319:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.526322:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.526323:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.526324:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.526325:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.526326:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.526326:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.526327:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.526332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.526339:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.526342:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.526343:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.526344:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.526346:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.526347:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.526397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:21.2:1619677448.526405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:10.0:1619677448.526405:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.526408:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677448.526409:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677448.526410:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.526411:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:10.0:1619677448.526413:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.526414:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.526415:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:10.0:1619677448.526415:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.526416:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.526417:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.526418:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.526419:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.526419:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.526420:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:2.0:1619677448.526421:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.526423:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.526424:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.526425:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.526426:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.526427:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:2.0:1619677448.526427:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.526430:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.526432:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.526434:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.526435:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:2.0:1619677448.526435:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.526436:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677448.526436:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.526437:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.526438:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.526438:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.526439:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:2.0:1619677448.526439:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.526440:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:2.0:1619677448.526440:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.526441:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.526442:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677448.526444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:25.2:1619677448.526445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.526451:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.526453:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:1.0:1619677448.526453:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:26.0:1619677448.526455:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.526455:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:17.0:1619677448.526456:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.526456:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:26.0:1619677448.526457:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.526457:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.526459:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.526461:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:1.0:1619677448.526463:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:25.2:1619677448.526465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.526472:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.526474:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.526475:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.526476:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.526477:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.526477:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.526623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.526630:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.526633:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.526635:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.526637:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.526638:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.526639:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.526640:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.526641:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.526643:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.526644:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.526647:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.526649:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.526651:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.526652:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.526653:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.526655:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.526655:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.526657:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.526659:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.526659:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.526660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.526660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.526663:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.2:1619677448.526665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.526666:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:13.0:1619677448.526667:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:20.0:1619677448.526668:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.526669:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.526670:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.526670:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.526671:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.526672:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.526673:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:20.0:1619677448.526673:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:12.0:1619677448.526673:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677448.526674:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.526674:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:20.0:1619677448.526675:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.526676:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.526676:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677448.526676:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.526677:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:20.0:1619677448.526677:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.526680:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.526682:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.526685:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.526686:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.526686:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.526688:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.526689:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.526690:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.526690:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.526694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.526697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.526702:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.526704:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.526705:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.526706:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.526706:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.526707:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677448.526709:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.526709:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.526710:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:30.0:1619677448.526711:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.526712:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.526713:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.526714:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.526715:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.526716:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.526717:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.526720:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.526722:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.526724:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.526725:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.526726:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.526727:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.526728:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.526729:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.526730:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.526731:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.526732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.2:1619677448.526737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.526741:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:21.2:1619677448.526741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:31.0:1619677448.526742:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.526743:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.526744:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.526744:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.526745:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:21.2:1619677448.526745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:31.0:1619677448.526746:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.526747:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:18.0:1619677448.526747:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677448.526748:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.526749:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:22.0:1619677448.526750:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677448.526750:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.526750:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677448.526753:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:18.0:1619677448.526753:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:6.0:1619677448.526754:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.526756:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.526756:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.526757:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:22.0:1619677448.526759:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.526759:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.526760:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.526761:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.526761:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.526762:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.526763:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.526763:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677448.526765:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.526766:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.526767:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000400:00000200:22.0:1619677448.526770:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.526770:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677448.526772:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.526772:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.526772:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.526773:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677448.526775:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.526775:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.526775:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.526775:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677448.526776:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.526777:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:4.2:1619677448.526777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677448.526778:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.526778:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677448.526778:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:22.0:1619677448.526779:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677448.526779:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677448.526780:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.526781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.526784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.2:1619677448.526784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:24.0:1619677448.526785:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.526788:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:2.0:1619677448.526788:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.526789:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:8.0:1619677448.526790:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677448.526790:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.526791:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.526791:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.526792:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.526793:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:5.0:1619677448.526793:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:2.0:1619677448.526793:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.526794:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:24.0:1619677448.526795:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.526795:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.0:1619677448.526795:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:2.0:1619677448.526795:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:24.0:1619677448.526796:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.526796:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.526798:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96713787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.526798:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.526799:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:9.2:1619677448.526799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:24.0:1619677448.526800:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.526801:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:24.0:1619677448.526802:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.526803:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.2:1619677448.526803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.526805:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:24.0:1619677448.526807:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.526808:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.526809:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.526809:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.526810:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.526810:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:20.0:1619677448.526811:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.526811:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677448.526812:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.526813:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.526813:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.526814:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:24.0:1619677448.526815:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.526815:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677448.526815:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:24.0:1619677448.526816:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677448.526816:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.526816:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:24.0:1619677448.526817:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.0:1619677448.526817:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.526821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.526828:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.526831:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.526832:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.526832:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.526834:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.526837:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.526886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.526893:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.526896:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.526897:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.526899:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.526902:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.526903:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.526905:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.526906:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.526907:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.526908:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.526912:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.526913:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.526915:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.526916:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.526916:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.526918:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.526919:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.526920:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.526921:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.526922:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.526924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.526927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.526930:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.526932:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.526933:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.526934:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677448.526935:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.526936:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:30.0:1619677448.526937:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.526937:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.526938:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677448.526938:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.526949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.526956:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.526959:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.526959:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.526960:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.526960:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.526961:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.527091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677448.527097:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677448.527100:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677448.527100:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:1.0:1619677448.527102:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.527105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:1.0:1619677448.527105:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.527107:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677448.527108:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.527110:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:1.0:1619677448.527110:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.527111:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.527112:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.527112:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677448.527115:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.527116:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.527117:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.527118:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:1.0:1619677448.527119:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:17.0:1619677448.527120:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.527121:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.527121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:17.0:1619677448.527122:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.527123:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.527126:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:1.0:1619677448.527126:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.527127:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.527127:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:1.0:1619677448.527128:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.527130:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:17.0:1619677448.527130:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.527130:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677448.527130:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677448.527131:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677448.527132:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:18.0:1619677448.527133:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.527133:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.527133:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:1.0:1619677448.527134:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677448.527135:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.527135:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.527135:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:18.0:1619677448.527136:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:9.2:1619677448.527136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677448.527137:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96713783] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.527138:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.527138:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677448.527139:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.527139:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.527139:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:18.0:1619677448.527140:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.527142:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:25.2:1619677448.527144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.527144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:2.0:1619677448.527144:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:18.0:1619677448.527145:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.527145:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.527146:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.527148:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.527148:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.527150:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:18.0:1619677448.527151:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.527151:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.527152:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:18.0:1619677448.527152:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.527152:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.527152:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677448.527153:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.527154:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.527155:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:18.0:1619677448.527155:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.527155:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.527156:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:12.0:1619677448.527156:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.527156:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:18.0:1619677448.527157:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677448.527157:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:26.0:1619677448.527158:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:12.0:1619677448.527158:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:26.0:1619677448.527160:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.527161:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:8.2:1619677448.527161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:26.0:1619677448.527163:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.527163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:26.0:1619677448.527164:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.527165:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.527166:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.527168:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:26.0:1619677448.527169:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:13.0:1619677448.527169:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.527170:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.527170:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.527171:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.527172:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.527173:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.527173:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.527173:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:26.0:1619677448.527174:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:13.0:1619677448.527174:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.527174:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677448.527174:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.527175:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.527175:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:26.0:1619677448.527176:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:13.0:1619677448.527176:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.527177:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.527178:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.527178:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.527182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.527190:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.527192:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.527193:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.527195:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.527195:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.527198:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.527203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.527210:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.527211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:20.0:1619677448.527213:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.527214:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.527215:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.527215:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.527216:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:2.0:1619677448.527217:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.527220:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.527221:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.527223:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.527226:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.527227:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.527228:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.527230:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.527231:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.527232:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.527235:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.527236:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.527238:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.527238:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.527239:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.527241:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.527241:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.527242:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.527242:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.527247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.527249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:18.0:1619677448.527253:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.527255:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.527256:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.527256:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.527257:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.2:1619677448.527257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000400:00000200:18.0:1619677448.527258:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.527260:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.527260:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.527261:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677448.527262:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:10.0:1619677448.527263:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.527265:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.527265:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677448.527266:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:10.0:1619677448.527267:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.527268:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.527269:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.527270:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.527271:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.527271:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.527272:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677448.527273:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:10.0:1619677448.527275:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.527275:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.527276:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.527278:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:1.0:1619677448.527278:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.527279:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.527281:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.527282:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.527282:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:1.0:1619677448.527283:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.527284:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.527284:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.527285:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677448.527285:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.527286:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:1.0:1619677448.527288:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677448.527289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:1.0:1619677448.527289:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:1.0:1619677448.527289:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:1.0:1619677448.527291:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:1.0:1619677448.527292:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:20.2:1619677448.527293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:25.2:1619677448.527299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:1.0:1619677448.527301:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:30.0:1619677448.527302:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:1.0:1619677448.527302:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677448.527302:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:26.0:1619677448.527303:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:30.0:1619677448.527305:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:26.0:1619677448.527305:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.527306:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.527307:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.527307:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:26.0:1619677448.527308:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.527308:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:31.0:1619677448.527309:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.527309:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.527309:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.527310:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:30.0:1619677448.527311:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:26.0:1619677448.527311:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:10.0:1619677448.527311:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.527312:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.527312:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:30.0:1619677448.527314:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.527315:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.527315:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:30.0:1619677448.527316:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.527320:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.527321:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.527325:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.527326:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.527327:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.527327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:30.0:1619677448.527328:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.527329:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.527330:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.527331:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:17.0:1619677448.527333:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:8.2:1619677448.527335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677448.527336:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677448.527337:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677448.527337:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:5.0:1619677448.527343:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.527345:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.527346:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.527348:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.527348:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.527351:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.527501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677448.527508:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:21.2:1619677448.527510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677448.527511:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677448.527512:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677448.527513:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.527515:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.527516:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:31.0:1619677448.527517:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677448.527518:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713777] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.527519:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.527520:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677448.527520:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.527521:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677448.527523:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.527524:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677448.527525:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.527526:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.527527:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:26.0:1619677448.527527:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:31.0:1619677448.527528:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.527530:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.527530:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.527530:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.527530:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:31.0:1619677448.527531:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.527531:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.527533:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.527533:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000400:00000200:31.0:1619677448.527534:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.527534:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:31.0:1619677448.527536:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.527536:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677448.527536:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:31.0:1619677448.527538:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.527539:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.527539:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.2:1619677448.527539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677448.527541:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.527541:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.527542:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.527542:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677448.527543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:25.2:1619677448.527546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.527546:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.527548:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.527549:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.527549:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.527550:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:17.0:1619677448.527551:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.527552:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:12.0:1619677448.527553:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.527553:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677448.527554:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:17.0:1619677448.527555:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:12.0:1619677448.527556:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.527557:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.527558:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.527561:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.527562:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.527647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.527650:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.527652:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.527655:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.527655:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.527657:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.527659:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.527660:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713775] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.527662:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.527663:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.527664:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.527667:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.527669:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.527671:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.527672:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.527673:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.527674:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.527675:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.527676:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.527678:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.527678:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.527681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.527683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.527689:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.527689:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.527691:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:6.0:1619677448.527692:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677448.527692:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677448.527693:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.527693:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.527694:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:6.0:1619677448.527695:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.527696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:6.0:1619677448.527697:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.527699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:1.0:1619677448.527702:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.527705:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.527705:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:22.0:1619677448.527707:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:1.0:1619677448.527707:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.527709:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.527710:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.527710:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:1.0:1619677448.527710:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.527711:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000400:00000200:1.0:1619677448.527711:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:22.0:1619677448.527712:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:1.0:1619677448.527712:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.527713:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.527715:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.527716:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.527719:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.527721:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.527723:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.527724:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.527725:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.527727:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.527727:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.527728:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.527728:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.527732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.527739:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.527742:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.527743:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.527744:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.527745:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.527747:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.527750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.527756:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.527759:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.527760:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.527762:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.527764:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.527765:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.527767:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713773] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.527768:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.527769:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.527770:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.527773:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.527775:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.527778:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.527778:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.527779:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.527781:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.527782:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.527783:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.527784:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.527784:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:25.2:1619677448.527787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.527789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677448.527793:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.527795:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:12.0:1619677448.527796:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677448.527797:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:12.0:1619677448.527797:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.527798:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:12.0:1619677448.527798:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.527799:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677448.527800:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.527801:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.527805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.2:1619677448.527810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:13.0:1619677448.527811:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.527813:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.527814:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.527815:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.527815:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.527815:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:13.0:1619677448.527816:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.527818:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.527819:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677448.527820:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.527822:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.527824:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.527825:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.527826:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.527827:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.527828:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.527831:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.527833:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.527835:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.527836:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.527837:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.527839:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.527839:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.527840:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.527842:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.527842:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677448.527844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677448.527847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.527850:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.527852:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:17.0:1619677448.527852:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677448.527853:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.527854:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677448.527854:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.527855:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.527856:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.527858:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677448.527860:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.527869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.527876:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.527878:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.527879:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.527880:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.527880:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.527881:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.527882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.527888:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.527890:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.527891:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.527893:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.527896:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.527898:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677448.527899:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.527900:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.527902:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.527903:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.527906:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.527908:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.527910:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.527911:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.527912:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.527913:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.527914:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.527915:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.527915:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677448.527919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.527927:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.527929:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.527930:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.527932:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.527933:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.527934:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.527979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.527987:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.527990:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.527992:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.527993:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.527994:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.527996:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.527997:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.527998:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.527999:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.528000:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.528004:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.528005:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.528009:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.528009:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.528010:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.528012:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.528012:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.528013:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.528014:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.528018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.528025:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.528028:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.528029:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.528031:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.528031:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.528033:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.528152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.528160:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.528163:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.528163:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.528165:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.528168:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.528170:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.528171:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.528172:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.528173:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.528174:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.528177:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.528179:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.528181:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.528181:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.528182:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.528183:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.528184:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.528184:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.528185:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.528189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.528196:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.528198:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.528199:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.528200:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.528202:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.528203:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.528208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.528215:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.528217:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.528218:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.528219:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.528219:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.528220:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.528277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.528283:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.528285:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.528286:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.528288:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.528291:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.528293:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.528298:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.528299:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.528300:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.528301:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.528305:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.528307:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:5.2:1619677448.528307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:17.0:1619677448.528310:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.528311:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.528311:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:21.2:1619677448.528312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:18.0:1619677448.528313:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677448.528313:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.528313:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.528315:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:18.0:1619677448.528316:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:17.0:1619677448.528316:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.528316:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.528317:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:6.0:1619677448.528318:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:18.0:1619677448.528319:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.528320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677448.528320:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.528321:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:18.0:1619677448.528321:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.528321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677448.528323:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:6.0:1619677448.528324:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.528325:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.528326:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.528326:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.528327:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.528327:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677448.528327:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:18.0:1619677448.528328:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:6.0:1619677448.528328:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:1.0:1619677448.528329:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:13.0:1619677448.528330:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:6.0:1619677448.528330:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.528330:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677448.528331:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.528331:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.528332:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.528333:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.528334:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677448.528334:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.528334:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:12.0:1619677448.528335:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677448.528337:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.528338:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:6.0:1619677448.528339:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.528340:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.528341:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:6.0:1619677448.528341:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677448.528342:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.528344:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.528344:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677448.528344:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.528345:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:6.0:1619677448.528345:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:18.0:1619677448.528346:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:6.0:1619677448.528346:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.528348:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.528348:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:25.2:1619677448.528349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.528349:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.528350:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.528353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:12.0:1619677448.528356:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.528359:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.528359:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.528361:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:12.0:1619677448.528361:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.528362:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.528363:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:12.0:1619677448.528363:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:30.0:1619677448.528364:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.528366:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.528368:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.528369:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.528421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.528428:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.528431:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.528432:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.528434:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.528436:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.528438:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.528439:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.528440:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.528441:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.528442:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.528446:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.528448:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.528450:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.528451:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.528451:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.528453:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.528453:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.528454:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.528454:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.528458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.528465:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.528468:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.528468:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.528470:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.528472:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.528473:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.528591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.2:1619677448.528595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.528598:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.528600:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:8.0:1619677448.528600:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:20.0:1619677448.528603:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.528603:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.528604:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677448.528604:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.528606:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.528606:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.528607:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.528608:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.528609:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.528610:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.528611:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.528611:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677448.528612:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677448.528613:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713763] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.528615:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.528616:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.528616:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.528617:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.528619:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:8.0:1619677448.528621:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.528622:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.528622:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.528623:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677448.528623:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.528625:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.528625:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.528626:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.528626:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.0:1619677448.528626:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.528627:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.528628:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.528629:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.528630:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.528630:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677448.528631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677448.528631:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.528635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.528638:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.528641:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.528642:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.528642:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677448.528643:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:22.0:1619677448.528644:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.528646:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677448.528646:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.528647:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.528647:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.528650:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.528653:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.528970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.528975:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.528978:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.528979:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:4.2:1619677448.528979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677448.528980:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.528983:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.528984:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.528985:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.528986:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677448.528987:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.528988:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.528989:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.528989:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:1.0:1619677448.528989:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.528993:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.528994:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.528996:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.528996:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.528998:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:1.0:1619677448.528998:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.528999:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.528999:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.529000:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:10.0:1619677448.529001:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.529001:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:10.0:1619677448.529002:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.529002:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.529002:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.529003:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.529003:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:10.0:1619677448.529006:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.529007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.529008:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.529011:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.529012:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.529013:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.529014:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677448.529015:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:10.0:1619677448.529015:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.529015:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.529016:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:10.0:1619677448.529016:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:17.0:1619677448.529017:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.529018:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.529019:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.529020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.529021:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:6.0:1619677448.529027:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.529031:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.529031:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.529032:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.529034:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.529037:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.529077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.529082:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.529085:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.529085:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.529087:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.529090:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.529091:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.529092:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713760] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.529094:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.529095:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.529096:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.529099:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.529101:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.529103:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.529104:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.529105:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.529107:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.529107:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.529108:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.529108:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.529113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.529121:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.529123:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.529124:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.529125:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.529126:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.529127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.529130:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677448.529131:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.529133:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.529133:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677448.529135:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.529138:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.529139:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.529140:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.529142:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.529143:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.529144:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.529147:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.529149:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.529152:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.529153:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.529154:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.529156:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.529156:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.529158:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.529158:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.529159:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.529162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.529164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:13.0:1619677448.529169:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.529170:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:13.0:1619677448.529171:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677448.529172:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.529172:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:13.0:1619677448.529172:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.529173:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:13.0:1619677448.529174:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.529175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:12.0:1619677448.529175:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.529177:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.529182:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.529184:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.529185:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.529188:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.529190:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.529192:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.529193:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.529195:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.529196:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.529197:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.529200:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.529202:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.529205:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.529206:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.529206:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.529208:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.529209:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.529209:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.529210:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.529214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.529223:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.529226:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.529227:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.529228:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.529229:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.529231:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.529232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.529240:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.529242:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.529243:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.529244:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.529245:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.529245:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.529311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.529314:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.529316:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.529318:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.529319:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.529320:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.529322:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.529323:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.529324:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.529326:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.529327:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.529330:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.529332:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.529334:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.529335:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.529336:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.529338:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.529339:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.529339:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.529340:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.529345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.529353:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.529355:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.529356:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.529358:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:31.0:1619677448.529360:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.529361:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.529440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.529446:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.529449:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.529450:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.529452:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.529454:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.529455:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.529456:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.529458:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.529459:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.529459:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.529462:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.529464:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.529467:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.529467:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.529468:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.529469:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.529470:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.529471:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.529471:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.529476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.529483:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.529486:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.529487:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.529487:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.529489:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.529491:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.529503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.529509:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.529512:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.529513:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.529515:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.529517:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.529519:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.529520:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.529521:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.529522:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.529523:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.529526:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.529528:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.529530:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.529531:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.529531:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.529533:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.529534:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.529534:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.529535:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.529539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.529544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677448.529547:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.529549:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.529550:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.529550:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.529552:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677448.529553:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:10.0:1619677448.529554:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.529555:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677448.529555:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:10.0:1619677448.529557:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.529560:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.529562:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.529563:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.529564:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.529565:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.529566:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.529570:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.529571:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.529575:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.529575:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.529576:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.529578:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.529578:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.529579:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.529580:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.2:1619677448.529581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:25.2:1619677448.529584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.529589:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.529591:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:6.0:1619677448.529591:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:31.0:1619677448.529593:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.529594:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:30.0:1619677448.529595:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:22.0:1619677448.529595:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.529595:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.529596:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.529598:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:6.0:1619677448.529598:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.529600:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713753] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.529601:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.529601:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:31.0:1619677448.529602:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.529602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000400:00000200:31.0:1619677448.529603:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.529606:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.529608:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:22.0:1619677448.529609:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.529611:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.529612:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:22.0:1619677448.529612:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.529612:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.529613:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677448.529614:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.529614:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.529615:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.529615:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:22.0:1619677448.529615:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677448.529616:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.529617:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.529621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.529630:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.529632:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.529633:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.529635:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.529635:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.529638:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.529655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.529661:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.529664:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.529665:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.529666:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.529669:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.529671:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.529672:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.529673:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.529674:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.529675:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.529679:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.529681:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:5.2:1619677448.529683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:6.0:1619677448.529684:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.529684:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.529685:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.529687:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.529688:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677448.529689:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677448.529689:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.529690:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.529690:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677448.529692:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.529692:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:25.2:1619677448.529693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000400:00000200:18.0:1619677448.529694:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.529695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:18.0:1619677448.529697:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.529698:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:26.0:1619677448.529699:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:22.0:1619677448.529700:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:18.0:1619677448.529700:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.529701:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.529702:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:18.0:1619677448.529702:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.529703:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.529703:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.529703:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:18.0:1619677448.529703:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.529703:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:22.0:1619677448.529704:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:26.0:1619677448.529706:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.529707:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.529710:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:18.0:1619677448.529710:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.529711:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677448.529712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677448.529712:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.529713:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.529714:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.529715:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.529716:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.529716:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:30.0:1619677448.529719:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:9.2:1619677448.529721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.529722:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.529724:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.529725:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.529726:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.529728:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:2.0:1619677448.529728:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:30.0:1619677448.529729:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.529729:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.529730:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.529731:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.529731:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:30.0:1619677448.529732:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.529733:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.529733:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.529735:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:30.0:1619677448.529737:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.529739:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.529742:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.529743:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.529743:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.529745:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.529746:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.529747:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.529747:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.529751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.529759:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.529762:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.529762:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.529763:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.529765:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.529768:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.529806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.529812:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.529814:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:4.2:1619677448.529815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:1.0:1619677448.529815:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.529817:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.529820:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.529821:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.529822:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.529823:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:17.0:1619677448.529824:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.529825:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:17.0:1619677448.529825:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.529826:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677448.529827:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.529829:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.529831:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:24.0:1619677448.529832:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.529834:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677448.529835:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.529835:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.529835:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:24.0:1619677448.529837:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:17.0:1619677448.529837:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677448.529838:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96713748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.529838:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.529838:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677448.529839:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.529839:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677448.529840:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.529841:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:5.2:1619677448.529842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:9.2:1619677448.529843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:24.0:1619677448.529845:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.529847:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.529849:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677448.529851:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.529851:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.529852:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.529852:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:8.0:1619677448.529852:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:26.0:1619677448.529853:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677448.529853:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:8.0:1619677448.529853:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.529854:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.529855:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:8.0:1619677448.529855:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.529856:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.529857:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.529857:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:10.0:1619677448.529857:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.529858:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:8.0:1619677448.529858:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.529859:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:10.0:1619677448.529859:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713747] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.529860:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.529861:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.529861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:10.0:1619677448.529862:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.2:1619677448.529865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:10.0:1619677448.529865:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.529866:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.529868:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.529868:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.529869:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.529869:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:10.0:1619677448.529870:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.529871:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:10.0:1619677448.529871:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.529871:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.529872:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.529872:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:20.0:1619677448.529873:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.529873:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:1.0:1619677448.529873:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.529874:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:5.0:1619677448.529874:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.529875:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.529875:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677448.529875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000400:00000200:1.0:1619677448.529875:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.529877:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677448.529877:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:25.2:1619677448.529878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.529878:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.529880:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.529880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:20.0:1619677448.529881:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.529882:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677448.529882:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.529885:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:20.0:1619677448.529885:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.529885:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:12.0:1619677448.529886:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.529887:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.529888:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:12.0:1619677448.529888:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.529889:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.529889:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:26.0:1619677448.529890:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.529890:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677448.529890:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.529890:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.529890:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:20.0:1619677448.529891:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.529891:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000400:00000200:12.0:1619677448.529891:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.529891:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:26.0:1619677448.529892:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677448.529892:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.529893:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:20.0:1619677448.529894:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677448.529894:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:20.0:1619677448.529895:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677448.529895:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:12.0:1619677448.529895:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.529896:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677448.529896:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.529896:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.529897:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.529898:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:25.2:1619677448.529899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.529905:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.529908:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.529909:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.529911:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.529912:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.529913:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.529928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:4.2:1619677448.529931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677448.529935:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.529937:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.529938:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.529938:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:12.0:1619677448.529939:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.529939:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.529940:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.529941:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:12.0:1619677448.529941:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677448.529943:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.529945:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.529947:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.529948:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.529949:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.529950:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.529951:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.529954:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.529956:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.529959:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.529959:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.529960:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.529962:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.529962:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.529963:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.529964:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.529965:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.529968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.529970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.529975:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.529977:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.529978:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.529978:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.529979:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677448.529979:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.529980:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.529981:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.529981:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.529984:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.530205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.530210:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.530214:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.530214:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.530216:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.530219:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.530220:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.530222:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.530223:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.530224:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.530225:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.530229:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.530230:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.530233:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.530234:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.530235:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.530236:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.530237:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.530238:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.530239:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.530240:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.530242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.530245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.530249:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677448.530250:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677448.530252:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:17.0:1619677448.530252:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:24.0:1619677448.530253:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.530253:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677448.530253:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:17.0:1619677448.530255:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677448.530256:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.530259:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.530260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677448.530265:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.530268:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.530268:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677448.530269:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.530270:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.530270:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.530288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.530297:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.530300:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:4.2:1619677448.530300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:31.0:1619677448.530302:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.530304:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.530305:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.530306:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677448.530307:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.530308:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.530309:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.530310:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.530310:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:31.0:1619677448.530311:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.530311:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.530313:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.530314:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.530316:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.530316:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.530317:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:31.0:1619677448.530318:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.530318:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.530319:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:10.0:1619677448.530319:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.530320:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.530320:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.530321:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.530321:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.530322:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.530323:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.530323:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:10.0:1619677448.530323:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.530325:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.530327:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.530327:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:25.2:1619677448.530328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.530328:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.530329:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.530329:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.530330:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.530331:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.530334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:13.0:1619677448.530335:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.530338:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.2:1619677448.530339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:13.0:1619677448.530339:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.530340:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.530341:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:13.0:1619677448.530342:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.530343:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.530343:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.530343:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.530344:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.530345:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.530347:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.530347:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:30.0:1619677448.530350:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.530352:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.530354:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.530354:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.530356:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.530357:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.530358:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.530359:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.530360:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.530363:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.530365:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.530368:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.530369:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.530370:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.530371:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.530372:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.530373:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.530375:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.530375:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.530378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.530381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.530387:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:31.0:1619677448.530389:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:5.0:1619677448.530390:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:31.0:1619677448.530391:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:5.0:1619677448.530391:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.530392:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:5.0:1619677448.530392:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.530393:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677448.530393:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.530395:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.530410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.530418:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.530420:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.530421:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.530423:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.530424:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.530426:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.530427:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.530428:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.530429:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.530430:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.530433:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.530434:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.530437:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.530437:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.530438:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.530440:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.530440:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.530441:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.530441:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.530445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.530452:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.530454:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.530455:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.530456:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.530458:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.530459:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.530634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.530640:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.530643:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.530644:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.530646:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.530649:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.530650:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.530651:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.530653:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.530654:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.530655:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.530658:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.530660:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:4.2:1619677448.530662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677448.530663:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.530663:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.530664:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.530666:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.530666:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.530667:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.530667:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.530669:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.530672:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:25.2:1619677448.530672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.530672:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.530675:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.530677:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.530678:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.530679:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.530680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677448.530680:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:26.0:1619677448.530681:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.530682:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.530682:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:26.0:1619677448.530683:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:22.0:1619677448.530683:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.530683:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.530686:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.530687:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677448.530688:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.530689:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677448.530690:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677448.530690:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.530692:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.530693:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.530694:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.530694:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:31.0:1619677448.530695:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.530695:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677448.530697:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:26.0:1619677448.530697:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.530697:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.530698:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713737] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.530698:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.530698:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:31.0:1619677448.530699:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.530700:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.530701:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:8.2:1619677448.530702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677448.530704:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.530706:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.530708:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.530709:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.530709:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.530711:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.530711:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.530711:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:31.0:1619677448.530713:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677448.530713:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:31.0:1619677448.530714:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:20.0:1619677448.530714:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.530715:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.530716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:20.0:1619677448.530716:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.530717:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.530719:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.530720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:21.2:1619677448.530721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.530722:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.530725:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.530726:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:21.2:1619677448.530726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:10.0:1619677448.530726:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.530728:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.530729:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:10.0:1619677448.530730:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.0:1619677448.530730:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.530731:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.530732:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.530733:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677448.530733:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677448.530733:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.530734:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.530735:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.530735:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.530736:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.530736:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677448.530737:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:24.0:1619677448.530738:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.530741:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.530743:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.530744:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.530745:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.530746:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.530747:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.530750:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.530751:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.530754:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.530754:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.530755:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.530757:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.530757:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.530758:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.530758:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.530762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.530769:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.530772:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.530773:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.530774:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.530776:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.530777:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.530780:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.530781:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.530781:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.530782:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.530783:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.530799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.530804:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.530807:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.530809:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.530810:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.530812:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.530813:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.530815:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.530816:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.530817:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.530818:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.530822:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.530823:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.530825:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.530826:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.530827:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.530829:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.530830:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.530830:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.530831:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.530835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.530843:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.530845:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.530847:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.530847:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.530848:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.530851:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.530855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.530862:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.530865:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.530866:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.530866:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.530867:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.530868:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.530926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.530934:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.530937:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.530939:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.530941:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.530942:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.530943:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.530944:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.530946:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.530947:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.530948:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.530951:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.530952:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.530955:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.530956:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.530956:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.530958:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.530959:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.530959:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.530960:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.530963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.530970:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.530972:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.530973:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.530974:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.530976:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.530977:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.530979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.530987:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.530989:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.530990:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.530992:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.530995:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.530996:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.530997:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.530998:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.530999:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.531000:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.531003:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.531004:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.531007:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.531007:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.531008:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.531009:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.531010:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.531011:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.531011:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.531015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.531021:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.531023:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.531024:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.531025:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.531027:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.531028:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.531047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.531054:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.531056:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.531057:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.531059:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.531062:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.531063:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.531064:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.531065:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.531066:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.531067:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.531070:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.531071:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.531074:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.531075:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.531075:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.531077:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.531077:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.531078:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.531078:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.531083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.531090:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.531093:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.531094:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.531095:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.531096:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.531098:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.531127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677448.531129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677448.531134:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677448.531135:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677448.531138:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.0:1619677448.531138:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.531139:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:31.0:1619677448.531141:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.531141:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:8.0:1619677448.531142:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.531143:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.531143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:8.0:1619677448.531144:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.531145:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.531146:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.531146:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.531147:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.531148:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.531149:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.531149:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677448.531149:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.531150:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.531151:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.531151:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:8.0:1619677448.531152:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.531152:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.531155:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.531157:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.531157:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.531158:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.531160:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:17.0:1619677448.531160:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.531161:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:8.0:1619677448.531161:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.531162:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96713729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.531162:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:31.0:1619677448.531163:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.531163:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677448.531164:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:17.0:1619677448.531164:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.531164:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677448.531165:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:17.0:1619677448.531165:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.531165:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677448.531166:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:17.0:1619677448.531166:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677448.531166:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:31.0:1619677448.531167:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677448.531167:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:31.0:1619677448.531168:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677448.531168:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677448.531169:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:17.0:1619677448.531169:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.531170:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.531171:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:25.2:1619677448.531173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.531174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.531175:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.531176:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.531176:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.531179:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:22.0:1619677448.531180:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677448.531180:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.531180:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677448.531181:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.531181:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.531181:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.531182:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:1.0:1619677448.531182:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.531183:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:22.0:1619677448.531183:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:17.0:1619677448.531183:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.531184:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677448.531184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:6.0:1619677448.531184:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:1.0:1619677448.531184:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.531185:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.531187:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677448.531187:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.531188:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:24.0:1619677448.531190:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:22.0:1619677448.531190:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677448.531191:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.531192:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.531193:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.531195:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.531198:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677448.531301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.531306:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.531309:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.531310:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.531312:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.531315:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.531316:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.531317:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.531319:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.531320:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.531321:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.531324:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.531326:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.531329:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.531330:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.531330:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.531332:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.531332:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.531333:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.531334:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.531339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.531347:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.531350:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.531351:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.531351:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.531353:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.531356:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.531701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.531707:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.531709:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.531712:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.531712:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.531714:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.531716:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.531717:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.531718:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.531720:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.531721:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.531724:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.531726:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.531729:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.531730:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.531731:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.531733:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.531733:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.531734:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.531736:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.531736:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.531739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.531741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.2:1619677448.531743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:20.0:1619677448.531744:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.531746:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:30.0:1619677448.531747:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.531747:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.531748:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.531750:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:8.0:1619677448.531750:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:30.0:1619677448.531751:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.531753:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.531753:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.531753:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:31.0:1619677448.531754:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.531756:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:8.0:1619677448.531756:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.531758:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.531760:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:25.2:1619677448.531761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:8.0:1619677448.531761:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.531762:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.531763:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.531764:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.531767:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.531768:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.0:1619677448.531769:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.531770:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.531771:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:8.0:1619677448.531771:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.531772:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:8.0:1619677448.531772:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.531773:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:8.0:1619677448.531773:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.531774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.0:1619677448.531774:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.531774:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.531775:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.531776:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:31.0:1619677448.531777:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.531778:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.531780:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:25.2:1619677448.531780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:31.0:1619677448.531782:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.531783:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.531785:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.531786:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.531787:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.531787:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:31.0:1619677448.531789:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.531789:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.531789:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:31.0:1619677448.531790:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:12.0:1619677448.531790:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.531792:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.531794:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.531794:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.531795:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.531796:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.531799:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.531800:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.531800:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.531802:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.531803:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.531804:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.531804:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.531808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.531816:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.531819:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.531820:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.531821:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.531822:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.531824:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.531880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.531886:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.531888:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.531889:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.531891:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.531893:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.531895:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.531896:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.531898:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.531899:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.531900:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.531904:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.531906:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.531908:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.531909:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.531910:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.531912:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.531912:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.531913:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.531913:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.531917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.531924:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.531927:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.531928:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.531930:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.531931:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.531933:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.531980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.531987:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.531989:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.531992:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.531992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:30.0:1619677448.531993:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.531994:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.531996:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.531997:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.531998:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.531998:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:31.0:1619677448.531999:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.532000:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.532001:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.532002:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:31.0:1619677448.532004:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.532006:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:18.0:1619677448.532008:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.532009:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.532009:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.532010:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.532011:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:18.0:1619677448.532011:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.532012:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713722] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.532013:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.532013:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.532014:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677448.532014:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.532014:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.532015:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:18.0:1619677448.532016:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.532019:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.532019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.532021:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.532024:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.532025:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.532027:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.532027:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.532028:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:4.2:1619677448.532028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677448.532029:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.532029:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.532030:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.532030:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.532031:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677448.532032:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.532032:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.532034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.532035:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:26.0:1619677448.532036:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.532038:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.532039:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.532041:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.532041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000400:00000200:26.0:1619677448.532044:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.532045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:26.0:1619677448.532046:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.532047:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.532048:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.532048:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:26.0:1619677448.532049:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.532049:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:13.0:1619677448.532049:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:5.0:1619677448.532049:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:26.0:1619677448.532050:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677448.532050:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.532051:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:13.0:1619677448.532052:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:5.0:1619677448.532052:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.532055:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677448.532055:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.532055:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:26.0:1619677448.532057:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:13.0:1619677448.532058:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.532059:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:13.0:1619677448.532059:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:5.0:1619677448.532059:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.532060:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.532060:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:26.0:1619677448.532061:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:13.0:1619677448.532061:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.532061:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:26.0:1619677448.532062:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:20.0:1619677448.532062:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.532062:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.532063:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:20.0:1619677448.532063:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:26.0:1619677448.532064:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:13.0:1619677448.532064:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.532065:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:20.0:1619677448.532065:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677448.532065:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.532066:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000400:00000200:20.0:1619677448.532066:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.532067:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677448.532067:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.532068:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.2:1619677448.532069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.532073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:13.0:1619677448.532074:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.532075:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:13.0:1619677448.532076:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.532077:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.532078:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.532078:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.532079:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.532080:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.532080:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.532080:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:13.0:1619677448.532081:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:5.0:1619677448.532081:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.532082:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.532082:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.532082:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.532082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.0:1619677448.532083:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:13.0:1619677448.532083:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.532083:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677448.532084:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:13.0:1619677448.532084:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677448.532084:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:5.0:1619677448.532084:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:13.0:1619677448.532085:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.0:1619677448.532085:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677448.532087:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.532088:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.532089:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677448.532089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:24.0:1619677448.532090:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677448.532092:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.532093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:24.0:1619677448.532093:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:20.0:1619677448.532094:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.532095:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:8.0:1619677448.532095:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.532096:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.532096:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.532097:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:20.0:1619677448.532098:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.532099:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:24.0:1619677448.532099:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.532100:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.532101:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96713718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.532101:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677448.532102:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.532103:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.532103:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.532104:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.532105:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000400:00000200:24.0:1619677448.532105:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.532106:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:21.2:1619677448.532106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:30.0:1619677448.532108:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:24.0:1619677448.532108:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.532110:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:18.0:1619677448.532112:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:24.0:1619677448.532113:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.532114:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.532114:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:18.0:1619677448.532115:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.532115:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:24.0:1619677448.532116:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.532116:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.532117:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:18.0:1619677448.532117:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:24.0:1619677448.532118:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677448.532120:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.532122:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:8.2:1619677448.532122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:18.0:1619677448.532123:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.532124:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.532125:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.532126:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.532130:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677448.532130:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:18.0:1619677448.532131:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.532132:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:18.0:1619677448.532133:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.532133:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:18.0:1619677448.532134:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:17.0:1619677448.532134:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.532135:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.532136:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:17.0:1619677448.532136:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.532137:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.532137:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.532138:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677448.532139:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:25.2:1619677448.532142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.532150:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.532152:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.532153:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.532154:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.532156:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.532158:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.532160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.2:1619677448.532163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:6.0:1619677448.532167:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.532170:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.532170:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.532171:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.532171:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.532172:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.532172:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:26.0:1619677448.532173:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.532174:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.532176:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.532180:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.532181:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.532182:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.532184:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.532185:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.532186:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:21.2:1619677448.532188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:26.0:1619677448.532189:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.532191:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677448.532193:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.532194:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.532194:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.532195:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:22.0:1619677448.532196:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:26.0:1619677448.532197:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.532197:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.532197:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677448.532198:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.532199:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677448.532199:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.532201:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.532202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:22.0:1619677448.532202:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.532203:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.532204:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.532205:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.532206:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.532209:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:22.0:1619677448.532210:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.532211:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:22.0:1619677448.532211:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.532212:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.532214:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.532214:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.532215:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:22.0:1619677448.532215:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.532216:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:22.0:1619677448.532216:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.532218:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.532218:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.532219:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.532220:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.532224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.532232:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.532235:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.532236:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.532236:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.532239:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.532241:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.532327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.532335:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.532337:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.532338:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.532340:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.532343:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.532344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:10.0:1619677448.532344:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.532346:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.532347:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.532348:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.532349:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.532351:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:10.0:1619677448.532353:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.532354:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.532354:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.532355:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.532356:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.532357:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.532358:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.532359:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677448.532359:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.532361:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.532361:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:8.0:1619677448.532361:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:10.0:1619677448.532362:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.532362:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:8.0:1619677448.532362:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96713713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.532363:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.532364:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.532365:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:9.2:1619677448.532367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677448.532368:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.532370:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.532372:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.532373:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.0:1619677448.532374:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000800:00000200:2.0:1619677448.532377:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (24)++
00000800:00000200:8.0:1619677448.532378:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (24)--
00000800:00000200:8.0:1619677448.532378:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.532379:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.532380:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:2.0:1619677448.532380:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000800:00000200:18.0:1619677448.532381:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:8.0:1619677448.532381:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:2.0:1619677448.532381:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.532383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (22)++
00000400:00000200:2.0:1619677448.532384:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:21.2:1619677448.532386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:2.0:1619677448.532388:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.532390:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.532391:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.532391:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.532392:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.532393:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:2.0:1619677448.532393:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.532395:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677448.532396:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.532397:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.532397:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.2:1619677448.532533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.532539:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.532542:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.532543:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.532545:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.532548:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.532549:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.532551:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713712] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.532552:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.532553:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.532554:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.532557:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.532559:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.532562:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.532563:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.532564:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.532566:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.532566:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.532567:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.532568:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.532569:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.532572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.532574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.532579:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677448.532579:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.532581:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:12.0:1619677448.532582:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677448.532582:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.532582:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677448.532583:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.532585:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.532587:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.532588:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.532675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.532683:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.532685:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.532687:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.532690:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.532690:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.532691:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.532692:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.532694:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.532694:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.532695:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.532699:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.532700:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.532703:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.532703:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.532704:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.532705:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.532706:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.532707:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.532708:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.532711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.532717:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.532719:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.532720:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.532722:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.532723:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.532724:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.532803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.532811:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.532813:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.532815:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.532817:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.532818:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.532820:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.532821:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.532822:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.532823:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.532824:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.532827:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.532828:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.532831:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.532831:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.532832:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.532834:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.532834:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.532835:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.532835:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.532839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.532846:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.532848:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.532849:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.532850:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.532852:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.532853:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.532858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:13.0:1619677448.532864:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.532867:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.532868:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.532868:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.532869:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.532870:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.532890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.532897:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.2:1619677448.532899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:26.0:1619677448.532900:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.532900:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.532902:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.532905:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.532905:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677448.532906:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:5.0:1619677448.532907:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:26.0:1619677448.532908:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.532910:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.532910:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.532910:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.532911:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.532912:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677448.532914:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.532916:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.532917:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.532918:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.532919:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:5.0:1619677448.532919:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.532921:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.532922:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.532922:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:5.0:1619677448.532922:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.532923:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.532924:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.532925:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:5.0:1619677448.532925:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.532926:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.532927:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:5.0:1619677448.532927:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.532928:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.532930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677448.532930:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.532931:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.2:1619677448.532933:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.532936:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:5.0:1619677448.532936:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.532938:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.532938:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.532939:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:5.0:1619677448.532939:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677448.532940:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.532940:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:5.0:1619677448.532940:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:30.0:1619677448.532942:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.532942:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:31.0:1619677448.532943:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.532943:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.532944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.532945:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.532946:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.532948:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.532950:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.532950:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.532950:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.532952:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.532954:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.532973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.532979:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.532982:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.532982:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.532984:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.532987:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.532989:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.532990:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.532991:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.532992:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.532993:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.532997:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.532999:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.533001:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.533002:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.533003:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.533005:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.533005:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.533006:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.533006:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.533011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.533017:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.533019:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.533020:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.533021:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.533023:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.533026:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.533032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.533038:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.533041:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.533042:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.533042:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.533043:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.533044:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.533168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.533175:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.533177:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.533178:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.533180:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.533182:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.533184:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.533185:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.533186:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.533187:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.533188:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.533192:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.533194:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.533197:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.533198:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.533199:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.533201:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.533201:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.533201:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.533202:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.533207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.533215:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.533218:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.533218:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.533219:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.533221:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.533224:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.533243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.533248:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.533251:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.533252:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.533254:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.533257:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.533259:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.533260:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.533261:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.533262:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.533263:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.533267:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.533269:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.533272:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.533273:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.533273:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.533275:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.533276:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.533277:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.533278:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.533279:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:25.2:1619677448.533281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.533284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:13.0:1619677448.533289:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.533290:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:13.0:1619677448.533291:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677448.533292:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.533292:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:13.0:1619677448.533292:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.533299:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:25.2:1619677448.533302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.533302:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:13.0:1619677448.533302:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.533305:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.533307:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.533308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677448.533309:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.533309:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.533309:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.533310:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.533311:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677448.533314:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.533318:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.533318:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.533320:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.533323:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.533325:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.533326:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.533328:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.533329:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.533330:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.533333:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.533335:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.533337:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.533338:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.533339:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.533340:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.533341:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.533341:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.533342:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.533347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.533354:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.533356:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.533357:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.533359:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.533360:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.533361:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.533378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.533386:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.533389:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.533389:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.533391:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.533393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000400:00000200:24.0:1619677448.533394:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.533396:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.533397:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.533398:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.533399:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.533399:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000400:00000200:24.0:1619677448.533400:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.533402:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677448.533402:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:24.0:1619677448.533405:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.533406:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.533407:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.533409:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.533410:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.533410:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:24.0:1619677448.533411:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:22.0:1619677448.533411:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713702] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.533412:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.533413:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:22.0:1619677448.533413:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.533414:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:22.0:1619677448.533414:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.533415:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.533415:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677448.533415:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.533419:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.533419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.533420:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.533423:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.533424:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677448.533424:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (23)++
00000800:00000200:22.0:1619677448.533426:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (24)--
00000800:00000200:22.0:1619677448.533427:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:1.0:1619677448.533427:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000800:00000200:22.0:1619677448.533428:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677448.533429:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677448.533430:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:1.0:1619677448.533430:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000800:00000200:17.0:1619677448.533431:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (24)++
00000400:00000200:1.0:1619677448.533431:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.533432:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000400:00000200:17.0:1619677448.533432:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.533433:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.533434:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.533435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (21)++
00000800:00000200:1.0:1619677448.533437:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.533438:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.533438:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.533440:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:17.0:1619677448.533441:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.533441:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.533441:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677448.533442:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:1.0:1619677448.533442:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.533443:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:6.0:1619677448.533443:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:1.0:1619677448.533443:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.533667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.533671:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.533673:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.533675:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.533677:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.533678:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.533680:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.533681:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.533682:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.533683:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.533684:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.533687:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.533689:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.533692:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.533692:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.533693:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.533694:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.533695:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.533696:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.533696:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.533701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.533708:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.533710:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.533711:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.533711:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.533713:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.533716:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.533722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.533727:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.533729:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.533732:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.533732:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677448.533734:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.533735:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.533737:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.533738:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.533739:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.533740:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.533743:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.533744:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.533746:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677448.533747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677448.533747:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.533748:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.533749:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.533750:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.533750:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.533751:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677448.533755:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:8.2:1619677448.533755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.533758:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.533760:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.533762:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.533763:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.533763:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:31.0:1619677448.533764:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.533766:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.533766:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:31.0:1619677448.533767:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.533767:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.533767:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:31.0:1619677448.533769:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.533769:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000400:00000200:31.0:1619677448.533770:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.533772:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:31.0:1619677448.533773:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.533775:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.533777:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.533778:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.533779:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.533780:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.533781:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.533782:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.533784:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.533784:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.533786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.533790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:21.2:1619677448.533792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:13.0:1619677448.533793:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.533795:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.533796:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.533798:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:22.0:1619677448.533798:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:13.0:1619677448.533798:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.533799:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.533800:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.533801:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:22.0:1619677448.533801:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.533801:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677448.533802:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.533803:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:22.0:1619677448.533804:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.533806:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.533807:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.533809:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.533810:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.533811:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.533812:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.533815:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.533817:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.533820:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.533821:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.533821:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.533823:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.533823:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.533824:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.533825:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.533829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.533837:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.533839:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.533840:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.533841:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.533842:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.533845:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.533850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.533857:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.533859:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.533860:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.533860:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.533861:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.533862:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.533914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.533922:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.533925:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.533926:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.533928:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.533931:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.533932:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.533934:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.533935:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.533936:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.533937:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.533940:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.533942:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.533945:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.533946:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.533946:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.533948:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.533948:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.533949:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.533949:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.533954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.533961:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.533964:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.533965:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.533966:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.533968:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.533970:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.533999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.534005:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.534008:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.534009:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.534011:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.534014:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.534015:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.534016:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.534018:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.534019:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.534020:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.534023:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.534025:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.534028:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.534029:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.534029:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.534031:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.534031:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.534032:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.534032:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.534037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.534044:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.534047:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.534048:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.534048:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.534050:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.534052:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.534055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.534062:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.534065:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.534067:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.534069:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.534069:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.534071:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.534072:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.534073:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.534074:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.534076:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.534079:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.534081:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.534083:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.534084:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.534085:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.534086:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.534087:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.534088:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.534090:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.534090:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.534092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.534096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.534098:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.534101:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.534101:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.534103:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:12.0:1619677448.534103:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.534105:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.534106:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:12.0:1619677448.534106:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.534107:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.534107:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677448.534230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.534236:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.534239:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.534239:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.534241:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.534244:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.534246:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.534247:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.534248:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.534249:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.534250:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.534254:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.534256:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.534259:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.534259:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.534260:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.534262:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.534263:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.534263:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.534264:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.534268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.534275:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.534277:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.534278:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.534280:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.534281:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.534283:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.534305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.534312:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.534315:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677448.534318:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.534321:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.534321:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.534324:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.534325:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.534326:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.534327:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.534328:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677448.534332:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.534333:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.534336:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.534337:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.534338:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.534339:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.534340:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.534340:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.534341:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.534345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.534354:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.534356:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.534357:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.534359:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.534360:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.534362:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.534468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.534474:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.534477:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:20.2:1619677448.534477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:6.0:1619677448.534477:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.534479:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.534482:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.534482:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:22.0:1619677448.534483:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:5.0:1619677448.534484:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:22.0:1619677448.534485:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.534486:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.534487:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:22.0:1619677448.534488:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.534488:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.534489:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677448.534490:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.534492:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:22.0:1619677448.534493:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.534493:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.534495:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.534496:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:5.0:1619677448.534496:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.534497:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.534498:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.534499:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.534499:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.534501:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000400:00000200:5.0:1619677448.534501:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.534502:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.534503:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:5.0:1619677448.534503:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.534504:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.534505:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.0:1619677448.534505:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.534506:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.534506:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:9.2:1619677448.534508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677448.534509:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.534509:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.2:1619677448.534510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677448.534510:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.534511:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.534514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.534516:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.534519:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:6.0:1619677448.534519:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:18.0:1619677448.534520:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.534520:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.534521:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.534521:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:18.0:1619677448.534522:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.534523:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677448.534523:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.534524:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.534525:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.534525:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:6.0:1619677448.534525:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.534527:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.534530:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.534530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919065782400] (20)++
00000800:00000200:25.2:1619677448.534534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.534535:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (21)++
00000800:00000200:2.0:1619677448.534538:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:18.0:1619677448.534539:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000400:00000200:2.0:1619677448.534540:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.534541:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.534543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:2.0:1619677448.534543:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.534544:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.534544:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:22.0:1619677448.534545:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.534545:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.534546:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000400:00000200:2.0:1619677448.534546:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.534547:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:2.0:1619677448.534547:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.534548:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677448.534549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677448.534549:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:31.0:1619677448.534551:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:2.0:1619677448.534553:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.534554:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:2.0:1619677448.534555:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677448.534556:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.534557:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.534558:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:2.0:1619677448.534558:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.534558:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000400:00000200:31.0:1619677448.534559:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.534559:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:2.0:1619677448.534559:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:10.0:1619677448.534560:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677448.534561:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:2.0:1619677448.534561:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:2.0:1619677448.534561:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (21)++
00000800:00000200:2.0:1619677448.534562:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (22)--
00000800:00000200:2.0:1619677448.534562:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (21)--
00000400:00000200:31.0:1619677448.534563:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.534563:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.534564:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.534565:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.534565:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.534566:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.534567:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:8.2:1619677448.534567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000400:00000200:26.0:1619677448.534569:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96713688] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.534570:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.534570:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.534571:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.534572:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.534573:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.534575:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.534575:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:31.0:1619677448.534576:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.534577:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.534577:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.534577:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:31.0:1619677448.534578:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:20.0:1619677448.534578:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.534579:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.534580:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.534580:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.534580:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.534580:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:31.0:1619677448.534581:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677448.534581:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.534581:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.534582:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677448.534582:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:26.0:1619677448.534583:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.534584:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.534585:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.534585:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.534586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:25.2:1619677448.534590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.534593:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.534596:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:6.0:1619677448.534597:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:5.0:1619677448.534597:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.534599:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:5.0:1619677448.534599:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.534600:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.534601:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677448.534601:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.534601:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.534602:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000400:00000200:6.0:1619677448.534602:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.534603:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.534603:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.534604:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:6.0:1619677448.534604:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.534606:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.534653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.534660:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.534663:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.534663:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.534666:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.534668:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.534670:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.534671:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.534672:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.534673:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.534675:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.534678:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.534680:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.534681:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.534682:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.534683:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.534684:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.534685:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.534686:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.534686:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:21.2:1619677448.534691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:8.2:1619677448.534691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:18.0:1619677448.534697:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677448.534698:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:18.0:1619677448.534700:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:17.0:1619677448.534700:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.534701:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.534701:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677448.534702:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:18.0:1619677448.534704:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677448.534704:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.534706:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.534706:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:18.0:1619677448.534708:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677448.534709:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.534710:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.534711:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.534712:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677448.534716:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.534718:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.534720:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.534721:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677448.534721:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677448.534723:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.534723:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.534724:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.534725:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.534729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.534736:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.534739:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.534740:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.534740:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.534742:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.534744:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.534747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:6.0:1619677448.534753:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.534756:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.534757:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.534757:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.534758:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.534759:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.534759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.534765:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.534767:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.534768:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.534770:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.534772:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.534773:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.534775:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.534776:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.534777:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.534778:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.534780:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.534782:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.534783:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.534784:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.534784:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.534786:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.534787:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.534788:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.534788:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.534792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.534800:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.534802:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.534803:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.534803:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.534805:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.534807:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.534810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.534817:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.534819:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.534820:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.534820:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.534821:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.534822:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.534833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.534840:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.534843:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.534845:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.534847:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.534848:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.534850:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677448.534851:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.534853:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.534854:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.534855:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677448.534858:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.534860:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677448.534862:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.534863:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:30.0:1619677448.534864:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:30.0:1619677448.534865:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:30.0:1619677448.534866:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.534867:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.534867:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.534871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.534877:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.534879:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.534881:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.534881:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.534883:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.534885:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.2:1619677448.534894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.534901:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.534904:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.534905:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.534906:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.534909:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.534911:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.534912:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.534913:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.534914:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.534915:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.534918:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.534919:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.534922:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.534922:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.534923:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.534925:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.534925:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.534926:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.534926:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.2:1619677448.534927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:25.2:1619677448.534930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.534932:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.534935:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.534936:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677448.534938:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:1.0:1619677448.534938:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.534939:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.534940:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.534941:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.534941:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.534942:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713682] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.534943:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.534944:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.534945:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:1.0:1619677448.534945:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.534946:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:1.0:1619677448.534946:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:25.2:1619677448.534950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:1.0:1619677448.534950:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.534951:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.534954:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.534955:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.534955:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:13.0:1619677448.534956:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.534957:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.534958:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.534959:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.534959:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677448.534960:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.534960:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677448.534960:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.534961:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.534962:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.534963:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:9.2:1619677448.534963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.534966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.534970:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677448.534971:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677448.534972:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.534973:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:17.0:1619677448.534973:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:8.0:1619677448.534973:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.534974:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.534975:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.534975:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.534978:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.534978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.534985:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.534987:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.534988:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.534990:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.534992:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.534993:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:5.2:1619677448.534994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1200] (20)++
00000400:00000200:26.0:1619677448.534995:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.534996:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.534997:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.534998:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.534999:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000400:00000200:26.0:1619677448.535001:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.535002:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:1.0:1619677448.535002:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:26.0:1619677448.535003:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:1.0:1619677448.535005:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.535006:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.535007:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.535008:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:1.0:1619677448.535008:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.535009:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:26.0:1619677448.535010:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.535010:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.535011:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:1.0:1619677448.535011:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.535012:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.535013:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.535013:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677448.535013:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.535014:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:25.2:1619677448.535016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:1.0:1619677448.535017:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.535019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.535019:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 984 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:1.0:1619677448.535022:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.535023:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:1.0:1619677448.535023:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:1.0:1619677448.535024:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:1.0:1619677448.535025:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:22.0:1619677448.535026:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677448.535026:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.535026:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:1.0:1619677448.535026:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000400:00000200:22.0:1619677448.535027:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.535027:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:1.0:1619677448.535027:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdbb1200] (22)++
00000800:00000200:10.0:1619677448.535028:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677448.535028:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (23)--
00000800:00000200:1.0:1619677448.535028:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:22.0:1619677448.535029:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.535030:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.535032:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.2:1619677448.535032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.2:1619677448.535034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677448.535039:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)++
00000800:00000200:17.0:1619677448.535042:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdbb1200] (23)--
00000800:00000200:17.0:1619677448.535042:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:5.0:1619677448.535042:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:17.0:1619677448.535043:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:5.0:1619677448.535044:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.535045:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.535047:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.535047:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.535049:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:21.2:1619677448.535123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.535128:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.535131:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.535132:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.535134:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.535136:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.535137:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.535139:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.535140:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.535141:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.535142:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.535145:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.535147:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.535150:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.535151:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.535152:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.535154:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.535154:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.535155:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.535156:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.535160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:4.2:1619677448.535163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677448.535166:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.535169:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:24.0:1619677448.535169:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677448.535169:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.535170:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.535172:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.535172:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.535173:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677448.535175:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:24.0:1619677448.535175:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.535177:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.535179:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.535180:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.535181:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.535182:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.535183:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.535187:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.535188:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.535191:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.535192:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.535193:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.535194:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.535195:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.535195:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.535196:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.535200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.535208:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.535211:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.535212:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.535212:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.535214:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.535217:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.535218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.535225:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.535228:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.535229:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.535229:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.535230:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.535231:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.535235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.535241:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.535244:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.535245:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.535247:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.535249:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.535250:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.535251:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.535253:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.535254:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.535255:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.535258:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.535260:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.535262:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.535263:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.535264:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.535265:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.535266:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.535266:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.535267:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.535271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.535278:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.535280:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.535280:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.535281:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.535282:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.535283:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.535363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.535371:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.535374:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.535375:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.535376:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.535379:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.535381:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.535382:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.535383:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.535384:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.535385:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.535389:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.535391:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:4.2:1619677448.535392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677448.535393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677448.535393:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.535395:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.535396:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.535396:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.535398:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.535399:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:10.0:1619677448.535399:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.535400:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677448.535401:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.535401:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677448.535402:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.535402:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:20.0:1619677448.535403:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.535403:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:5.0:1619677448.535403:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677448.535404:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.535405:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677448.535405:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677448.535405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677448.535406:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.535407:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.535408:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:26.0:1619677448.535409:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.535410:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.535412:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.535413:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.535413:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:26.0:1619677448.535415:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.535415:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.535416:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.535416:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:26.0:1619677448.535417:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.535417:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.535417:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:2.0:1619677448.535417:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.535418:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.535419:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.535420:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:18.0:1619677448.535421:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.535422:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.535423:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.535424:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:26.0:1619677448.535426:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.2:1619677448.535426:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:26.0:1619677448.535428:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.535429:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.535429:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.535430:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.535431:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.535432:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.535432:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677448.535432:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.535433:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.535433:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677448.535433:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.535434:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.535436:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:20.0:1619677448.535436:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:25.2:1619677448.535437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.535437:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.535438:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.535438:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.535439:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.535440:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677448.535441:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.535441:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.535442:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:25.2:1619677448.535442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.535443:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96713673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.535445:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.535445:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.2:1619677448.535445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.535446:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.535447:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:22.0:1619677448.535448:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.535449:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.535450:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.535450:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.535450:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:31.0:1619677448.535452:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.535452:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.535452:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:13.0:1619677448.535453:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.535453:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677448.535454:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:13.0:1619677448.535454:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.535454:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677448.535456:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.535456:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:13.0:1619677448.535456:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.535457:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:12.0:1619677448.535457:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.535458:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:13.0:1619677448.535459:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.535460:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.535460:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.535461:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.535462:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.535465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.2:1619677448.535469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677448.535472:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.535474:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.535475:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.535476:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:30.0:1619677448.535477:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.535478:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.535479:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:10.0:1619677448.535479:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.535480:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.535481:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.535484:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.535485:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.535487:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.535488:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.535489:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.535490:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.535493:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.535494:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.535496:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.535497:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.535498:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.535499:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.535500:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.535500:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.535501:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.535505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.535511:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.2:1619677448.535512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677448.535514:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.535514:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.535516:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.535518:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.535519:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677448.535519:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.535521:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.535522:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.535524:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.535526:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.535527:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.535529:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.535530:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.535531:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.535532:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.535535:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.535537:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:21.2:1619677448.535540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677448.535540:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.535541:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.535541:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.535543:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.535543:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.535544:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.535545:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.535548:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:9.2:1619677448.535549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:31.0:1619677448.535550:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.535553:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.535554:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.535555:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.535556:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:31.0:1619677448.535557:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.535558:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.535559:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.535559:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677448.535559:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:31.0:1619677448.535560:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.535560:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.535562:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:8.0:1619677448.535563:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.535565:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.535566:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:31.0:1619677448.535567:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.535570:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.535571:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.535571:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.535573:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.535574:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.535575:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.535575:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.535579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.535585:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.535587:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.535588:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.535588:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.535590:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.535593:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.535598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.535604:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.535607:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.535609:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.535609:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677448.535611:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.535613:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677448.535614:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.535616:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.535617:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.535618:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677448.535622:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.535624:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677448.535626:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.535627:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677448.535628:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677448.535630:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.535631:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.535632:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.535633:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.535634:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.535636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.2:1619677448.535639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677448.535642:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:20.0:1619677448.535642:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.535644:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:30.0:1619677448.535645:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:20.0:1619677448.535645:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:30.0:1619677448.535646:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.535646:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:30.0:1619677448.535648:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677448.535649:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.535650:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.535653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.535660:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.535663:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.535664:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.535664:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.535665:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.535666:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.535671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.535677:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.535679:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.535681:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.535682:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.535685:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.535687:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.535688:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.535689:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.535690:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.535691:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.535694:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.535696:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.535699:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.535700:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.535701:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.535703:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.535703:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.535704:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.535704:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.535709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.535715:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.535718:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.535719:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.535720:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.535721:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.535723:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.535849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.535856:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.535859:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.535860:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.535862:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.535864:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.535865:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.535866:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713667] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.535868:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.535869:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.535870:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.535873:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.535875:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:5.2:1619677448.535875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.535877:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.535878:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.535879:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.535880:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.535880:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.535881:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.535881:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.535882:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677448.535884:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.535884:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.535885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:17.0:1619677448.535887:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.535890:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.535891:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.535892:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.535892:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:17.0:1619677448.535893:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.535894:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.535894:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:17.0:1619677448.535895:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:13.0:1619677448.535895:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.535898:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.535898:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:17.0:1619677448.535899:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.535901:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677448.535902:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.535905:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.535906:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.535906:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.535908:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.535909:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.535910:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.535910:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.535914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.535920:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.535923:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677448.535923:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677448.535924:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.535926:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.535928:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.2:1619677448.535929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:4.2:1619677448.535930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677448.535933:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.535936:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:26.0:1619677448.535937:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677448.535938:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.535939:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677448.535940:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:20.0:1619677448.535941:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.535941:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677448.535943:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.535944:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.535945:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.535946:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.535946:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.535947:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.535948:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677448.535948:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.535949:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.535951:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.535953:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.535953:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.535954:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.535955:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.535958:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.535960:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.535962:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.535963:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.535963:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.535963:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.535964:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.535965:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.535965:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.535966:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.535967:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.535967:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.535967:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.535967:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677448.535968:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.535969:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.535971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.2:1619677448.535973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:12.0:1619677448.535978:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.535980:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.535981:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.535981:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:12.0:1619677448.535983:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.535984:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.535984:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.535985:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.535986:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677448.535986:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.535987:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.535990:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:25.2:1619677448.535991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.2:1619677448.535993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:13.0:1619677448.535997:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.536000:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.536001:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:13.0:1619677448.536001:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.536001:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.536002:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.536003:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:13.0:1619677448.536003:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677448.536004:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.536005:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.536006:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.536006:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.536022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.536029:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.536032:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.536033:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.536035:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.536037:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.536039:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.536040:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.536041:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.536042:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.536043:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.536047:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.536048:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.536051:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.536052:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.536053:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.536054:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.536055:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.536056:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.536058:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.536058:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.536060:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.536064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.2:1619677448.536066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677448.536068:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.536071:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.536071:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.536071:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.536072:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.536073:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:30.0:1619677448.536074:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:24.0:1619677448.536074:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677448.536074:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.536075:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:30.0:1619677448.536077:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:1.0:1619677448.536077:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:30.0:1619677448.536079:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.536081:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.536081:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.536083:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.536084:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.536085:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.536086:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.536087:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.536090:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.536091:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.536094:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.536094:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.536095:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.536096:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.536097:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.536098:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:21.2:1619677448.536098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.536101:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.536102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.536103:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.536104:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.536106:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.536107:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.536107:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.536108:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:30.0:1619677448.536109:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.536110:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.536111:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.536111:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.536111:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.536112:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.536112:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:12.0:1619677448.536113:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.536115:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.536115:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.536116:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.536117:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.536119:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.536120:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.536120:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.536122:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.536122:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.536123:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.536124:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.536127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.536134:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.536136:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.536137:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.536137:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.536139:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.536141:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.536163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677448.536169:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.536172:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677448.536173:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677448.536175:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.536178:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.536179:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.536180:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.536182:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.536183:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.536184:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.536187:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.536189:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.536192:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.536193:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.536193:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.536195:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.536196:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.536197:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677448.536198:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677448.536199:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:25.2:1619677448.536201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.536204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:26.0:1619677448.536207:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:2.0:1619677448.536209:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:26.0:1619677448.536210:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.536210:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.536211:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.536211:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677448.536212:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:26.0:1619677448.536213:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.536213:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:26.0:1619677448.536216:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.536283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.536290:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.536294:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.536299:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.536301:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.536304:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.536305:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.536306:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.536307:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.536309:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.536309:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.536313:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.536314:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.536316:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.536317:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.536317:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.536319:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.536319:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.536320:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.536321:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.536322:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.536324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.2:1619677448.536328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.536333:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.536334:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.536335:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.536336:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.536338:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.536338:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.536339:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.536339:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.536340:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.536340:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.536343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.536349:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.536351:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.536352:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.536352:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.536352:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.536353:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.536475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.536482:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.536485:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.536486:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.536487:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.536489:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.536490:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.536491:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.536493:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.536493:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.536494:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.536497:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.536499:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.536500:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.536500:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.536501:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.536502:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.536502:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.536503:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.536503:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.536507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.536513:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.536514:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.536515:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.536516:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.536517:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.536518:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.536527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.536534:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.536536:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.536537:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.536539:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.536542:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.536544:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.536545:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.536546:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.536547:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.536548:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.536551:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.536553:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.536556:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.536557:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.536557:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.536559:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.536560:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.536560:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.536561:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.536566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.536573:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.536576:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.536577:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.536577:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.536580:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.536583:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.536585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.536592:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.536595:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.536595:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.536596:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.536597:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.536597:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.536760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.536763:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.536766:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.536768:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.536769:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.536771:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.536772:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.536773:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.536775:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.536776:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.536777:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.536780:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.536782:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.536785:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.536785:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.536786:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.536788:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.536789:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.536789:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.536790:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.536794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.536802:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.536804:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.536805:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.536805:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.536807:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.536810:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.536844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.536851:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.536854:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.536855:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.536857:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.536858:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.536859:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.536861:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.536862:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.536863:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.536864:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.536867:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.536869:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.536870:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.536871:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.536872:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.536873:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.536873:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.536874:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.536875:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.536878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.536884:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.536886:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.536887:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.536888:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.536889:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.536889:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.536937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.536943:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.536946:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.536947:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.536949:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.536952:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.536953:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.536955:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.536956:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.536957:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.536958:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.536961:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.536963:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.536966:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.536967:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.536968:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.536969:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.536970:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.536971:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.536972:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.536972:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.536975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.536977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.536983:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.536983:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.536985:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.536985:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.536986:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677448.536986:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677448.536987:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.536989:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.536990:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.536991:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.537025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.537031:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.537034:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.537035:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.537037:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.537040:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.537041:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.537043:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.537044:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.537045:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.537046:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.537049:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.537050:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.537053:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.537054:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.537055:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.537056:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.537057:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.537057:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.537058:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.537063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.537071:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.537073:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.537074:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.537076:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.537077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677448.537078:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.537079:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:26.0:1619677448.537085:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.537087:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.537088:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.537090:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.537092:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.537093:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.537094:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.537095:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.537097:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.537098:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.537101:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.537103:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.537104:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.537105:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.537106:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.537107:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.537107:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.537108:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.537109:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.537110:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.537112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:4.2:1619677448.537115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.537120:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.537122:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.537122:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.537124:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:10.0:1619677448.537124:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.537125:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.537125:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.537126:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.537126:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.537127:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677448.537152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.537159:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.537161:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.537162:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.537164:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.537167:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.537168:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.537169:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.537171:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.537172:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.537173:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.537175:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.537177:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.537180:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.537180:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.537181:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.537183:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.537183:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.537184:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.537185:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.537189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.537197:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.537199:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.537200:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.537201:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.537203:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.537205:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.537205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.537213:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.537216:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.537216:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.537219:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.537222:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.537223:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.537224:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.537225:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.537227:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.537227:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:21.2:1619677448.537227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:24.0:1619677448.537231:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.537233:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.537235:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677448.537235:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.537236:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.537237:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.537238:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:24.0:1619677448.537238:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.537239:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.537239:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:31.0:1619677448.537240:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.537240:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:31.0:1619677448.537242:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.537242:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.537244:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:8.2:1619677448.537244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677448.537245:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.537246:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.537247:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.537248:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.537250:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.537252:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.537252:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.537254:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.537254:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:31.0:1619677448.537255:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.537255:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.537255:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.537256:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.537257:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.537257:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677448.537258:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.537259:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.537260:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.537261:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677448.537261:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:25.2:1619677448.537262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.537266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.537269:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.537271:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.537272:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.537274:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:12.0:1619677448.537274:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.537276:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:13.0:1619677448.537276:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.537276:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.537277:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.537278:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677448.537280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.537287:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.537289:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.537290:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:21.2:1619677448.537292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:8.0:1619677448.537292:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.537300:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:8.0:1619677448.537300:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.537302:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:18.0:1619677448.537303:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:8.0:1619677448.537303:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.537304:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:8.0:1619677448.537306:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.537307:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677448.537307:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.537308:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.537311:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.537313:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.537314:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.537315:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:8.0:1619677448.537316:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.537317:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.537317:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.537317:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677448.537318:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.537319:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.537319:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.537320:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.537320:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.537320:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.537321:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:18.0:1619677448.537324:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.537326:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:9.2:1619677448.537326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.537328:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.537330:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.537331:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000800:00000200:18.0:1619677448.537334:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (24)--
00000800:00000200:18.0:1619677448.537335:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.537335:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:18.0:1619677448.537336:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.537336:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000400:00000200:2.0:1619677448.537337:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.537338:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.537339:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.537339:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000400:00000200:2.0:1619677448.537339:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.537340:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000400:00000200:18.0:1619677448.537341:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.537343:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.537343:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:21.2:1619677448.537344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.537344:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.537347:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677448.537348:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.537349:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.537350:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.537351:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.2:1619677448.537412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.537418:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.2:1619677448.537419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:1.0:1619677448.537421:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.537422:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.537424:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.537426:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.537427:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:1.0:1619677448.537427:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.537429:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.537430:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:1.0:1619677448.537430:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.537431:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:1.0:1619677448.537432:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.537433:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.537434:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.537436:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.537437:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.537438:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.537439:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.537439:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.537440:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.537441:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.537442:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.537442:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.537442:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.537443:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.537445:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:10.0:1619677448.537446:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677448.537446:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.537446:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.537447:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.537448:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.2:1619677448.537451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.537451:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.537452:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.537452:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.537454:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.537455:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.537455:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.537456:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:30.0:1619677448.537458:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:9.2:1619677448.537460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.537461:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.537462:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.537464:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.537466:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.537467:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:2.0:1619677448.537467:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:25.2:1619677448.537469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:2.0:1619677448.537470:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.537471:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.537471:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.537474:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:31.0:1619677448.537476:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:2.0:1619677448.537477:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.537478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:31.0:1619677448.537479:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.537480:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.537480:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.537481:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.537483:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677448.537485:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.537488:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.537489:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.537489:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.537490:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.537491:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.537500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.537508:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.537511:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.537513:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.537514:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.537515:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.537517:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.537518:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.537519:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.537520:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.537521:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.537524:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.537526:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.537529:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.537530:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.537531:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.537533:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.537533:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.537535:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.537536:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.537537:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.537539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.537542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.537547:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.537550:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677448.537551:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:5.0:1619677448.537551:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.537553:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:20.0:1619677448.537553:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.537553:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.537554:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.537555:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.537556:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.537677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.537680:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.537683:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.537685:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.537686:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.537688:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.537689:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677448.537691:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.537692:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.537693:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.537694:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.537697:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.537699:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.537702:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.537702:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.537703:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.537705:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.537705:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.537706:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.537706:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677448.537711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.537719:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.537721:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.537722:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.537722:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.537725:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.537727:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.537865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.537872:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.537875:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.537875:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.537878:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.537880:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.537882:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.537883:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.537885:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.537886:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.537887:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.2:1619677448.537887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:24.0:1619677448.537889:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.537891:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:5.0:1619677448.537892:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:24.0:1619677448.537893:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.537894:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.537895:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677448.537895:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:24.0:1619677448.537896:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.537897:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:5.0:1619677448.537897:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.537898:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:20.0:1619677448.537898:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:24.0:1619677448.537899:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.537900:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:5.0:1619677448.537900:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.537901:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:25.2:1619677448.537902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.537904:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.2:1619677448.537905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000400:00000200:5.0:1619677448.537907:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.537908:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.537909:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:5.0:1619677448.537909:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.537910:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.537910:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.537912:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.537912:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.537912:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:17.0:1619677448.537913:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.537913:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:8.0:1619677448.537914:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.537915:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:13.0:1619677448.537915:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.537915:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677448.537916:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:5.0:1619677448.537916:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.537917:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.537918:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677448.537920:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.537920:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.537921:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:25.2:1619677448.537922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:5.0:1619677448.537922:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.537923:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:1.0:1619677448.537923:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.537924:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:1.0:1619677448.537924:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.537925:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.537926:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:1.0:1619677448.537926:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.537927:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.537927:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677448.537928:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677448.537928:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677448.537929:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.537931:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.2:1619677448.537932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:12.0:1619677448.537932:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.537932:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.537933:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677448.537933:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.537934:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.537934:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:12.0:1619677448.537935:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677448.537935:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.537936:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.537937:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.537937:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.537938:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.537939:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.537942:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.537942:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.537942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677448.537944:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.537946:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.537947:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677448.537949:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.537952:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.537952:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.537953:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.537954:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.537956:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.537959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:21.2:1619677448.537963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:1.0:1619677448.537966:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.537969:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677448.537969:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.537970:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.537970:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.537971:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.537971:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677448.537972:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.537973:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.537974:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.537977:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.537978:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.537980:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.537981:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.537982:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.537983:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.537986:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.537986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677448.537988:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.537991:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.537992:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.537992:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.537993:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.537994:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.537994:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.537995:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677448.537996:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:22.0:1619677448.537996:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.537997:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.537999:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.537999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677448.538002:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.538003:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.538004:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.538006:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.538007:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.538007:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:26.0:1619677448.538008:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.538010:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:26.0:1619677448.538011:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.538011:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.538011:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:26.0:1619677448.538013:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.538013:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.538015:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677448.538016:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.538017:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.538017:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.538019:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.538020:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.538020:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.538021:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.538025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.538031:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.538034:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.538035:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.538037:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.538038:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.538039:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.538328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677448.538334:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677448.538337:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677448.538338:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677448.538340:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.538342:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.538344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:1.0:1619677448.538344:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.538346:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.538347:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.538348:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.538349:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:8.0:1619677448.538351:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:1.0:1619677448.538352:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.538354:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.2:1619677448.538354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677448.538354:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:1.0:1619677448.538354:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:8.0:1619677448.538357:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.538357:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.538358:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.538358:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:8.0:1619677448.538359:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.538360:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.538360:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000400:00000200:8.0:1619677448.538361:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:1.0:1619677448.538361:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdf63800] (22)++
00000800:00000200:31.0:1619677448.538362:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:8.0:1619677448.538362:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96713636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.538362:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (23)--
00000400:00000200:8.0:1619677448.538363:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.538363:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:31.0:1619677448.538364:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:8.0:1619677448.538365:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.538366:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.538367:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.538367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.538369:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:8.0:1619677448.538369:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.538369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (21)++
00000400:00000200:31.0:1619677448.538370:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.538371:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.538372:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.538373:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96713635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.538374:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.538374:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.538375:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.538375:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)++
00000800:00000200:8.0:1619677448.538375:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.538375:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:31.0:1619677448.538376:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:8.0:1619677448.538376:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.538377:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdf63800] (23)--
00000800:00000200:8.0:1619677448.538377:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.538378:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677448.538378:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:31.0:1619677448.538379:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.538379:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:8.0:1619677448.538379:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.538379:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677448.538379:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.538379:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:22.0:1619677448.538380:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677448.538382:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.538382:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.538384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.538385:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.538385:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677448.538386:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.538386:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.2:1619677448.538386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677448.538388:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.538389:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.538390:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.538390:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677448.538393:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.538393:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.538394:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.538395:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:9.2:1619677448.538395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:17.0:1619677448.538396:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.538397:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.538398:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.538399:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.538401:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:2.0:1619677448.538402:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:1.0:1619677448.538403:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.538404:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.538405:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:1.0:1619677448.538405:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.538406:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.538406:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:18.0:1619677448.538407:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:17.0:1619677448.538407:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.538409:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:2.0:1619677448.538409:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.538410:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:1.0:1619677448.538412:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.538414:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677448.538415:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:1.0:1619677448.538417:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.538418:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.538418:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.538421:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.538421:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.538421:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.538422:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.538427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.538433:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.538436:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.538437:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.538439:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.538440:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.538442:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.538487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.538493:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.538496:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.538496:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.538498:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.538500:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.538502:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.538503:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.538505:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.538506:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.538507:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.538510:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.538512:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.538515:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.538516:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.538516:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.538518:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.538519:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.538520:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.538520:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.538524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.538531:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.538534:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.538535:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.538535:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.538538:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.538541:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.538542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.2:1619677448.538547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:26.0:1619677448.538548:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.538550:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.538551:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.538551:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.538552:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.538553:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:2.0:1619677448.538553:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.538556:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.538556:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.538558:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.538562:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.538563:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.538564:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.538566:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.538567:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.538567:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.538570:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.538572:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.538573:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.538574:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.538574:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.538576:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.538577:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.538577:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.538578:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.538579:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677448.538582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.538585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.538591:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.538592:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.538592:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.538592:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.538592:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.538593:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.538594:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.538594:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.538595:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677448.538596:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.538635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.538643:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.538646:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.538648:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.538649:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.538651:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.538652:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.538654:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.538655:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.538656:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.538657:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.538660:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.538662:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.538666:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.538666:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.538667:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.538669:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.538669:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.538670:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.538671:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.538675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.538683:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.538685:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.538686:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.538688:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.538689:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.538691:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.538700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.538708:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.538711:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.538712:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.538712:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.538713:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.538714:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.538733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.538740:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.538743:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.538744:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.538746:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.538748:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.538749:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.538751:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.538752:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.538753:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.538754:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.538757:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.538759:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.538761:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.538761:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.538762:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.538764:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.538764:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.538765:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.538765:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.538769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.538776:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.538778:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.538779:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.538781:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.538782:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.538783:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.538785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677448.538792:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.538794:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.538795:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.538796:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.538796:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.538797:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.538964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919065782400] (20)++
00000800:00000200:2.0:1619677448.538969:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (21)++
00000800:00000200:2.0:1619677448.538972:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:18.0:1619677448.538973:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000400:00000200:2.0:1619677448.538975:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.538977:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.538979:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677448.538980:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713629] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.538981:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.538982:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.538983:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677448.538987:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.538988:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677448.538991:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.538992:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:2.0:1619677448.538993:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:2.0:1619677448.538995:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:2.0:1619677448.538995:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (21)++
00000800:00000200:2.0:1619677448.538996:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (22)--
00000800:00000200:2.0:1619677448.538996:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (21)--
00000800:00000200:25.2:1619677448.539000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.539008:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.539011:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.539012:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.539013:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.539015:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.539016:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.539143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.539149:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.539152:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.539153:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.539155:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.539157:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.539158:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.539160:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.539162:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.539163:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.539164:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.2:1619677448.539164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:22.0:1619677448.539167:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.539169:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.539172:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677448.539172:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.539173:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.539174:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.539175:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:22.0:1619677448.539176:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.539176:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.539176:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677448.539177:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:26.0:1619677448.539178:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.539178:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.539179:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:26.0:1619677448.539180:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.539181:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:25.2:1619677448.539181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:26.0:1619677448.539182:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.539184:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.539184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:26.0:1619677448.539185:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.539186:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.539188:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:26.0:1619677448.539189:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.539189:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:30.0:1619677448.539191:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:26.0:1619677448.539191:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677448.539191:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:30.0:1619677448.539192:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.539192:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.539193:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:30.0:1619677448.539194:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.539194:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.539195:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.539195:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.539195:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.539197:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.539197:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.539197:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.539198:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.539199:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.539202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.539209:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.539211:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.539212:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.539213:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.539215:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.539216:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.539281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.539288:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.539291:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.539293:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.539293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.539295:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.539301:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.539301:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677448.539303:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.539304:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.539304:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.539304:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:30.0:1619677448.539306:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.539307:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.539307:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.539308:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.539309:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.539311:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677448.539312:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.539312:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.539314:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.539314:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.539315:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.539316:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.539317:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.539317:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.539318:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.539319:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.539319:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.539321:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.539321:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677448.539321:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.539322:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.539323:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.539324:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.539325:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:25.2:1619677448.539326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.539326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:10.0:1619677448.539326:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.539327:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.539327:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.539328:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.539332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:13.0:1619677448.539334:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677448.539336:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:13.0:1619677448.539336:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.539337:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.539338:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:13.0:1619677448.539339:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.539340:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.539340:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.539341:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.539341:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.539341:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.539342:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677448.539343:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.539343:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.539344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:31.0:1619677448.539345:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677448.539346:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.539347:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.539348:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.539349:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:30.0:1619677448.539351:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:18.0:1619677448.539351:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:31.0:1619677448.539352:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.539352:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677448.539353:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.539354:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677448.539354:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:20.2:1619677448.539355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677448.539355:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.539357:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.539358:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.539358:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.539359:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:18.0:1619677448.539360:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.539361:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.539361:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000400:00000200:18.0:1619677448.539361:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:31.0:1619677448.539362:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677448.539363:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:30.0:1619677448.539363:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:18.0:1619677448.539363:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.539364:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.539365:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.539366:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:9.2:1619677448.539366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:30.0:1619677448.539367:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677448.539370:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.539370:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.539371:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.539372:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.539373:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:18.0:1619677448.539373:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:30.0:1619677448.539374:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:24.0:1619677448.539375:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:18.0:1619677448.539375:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.539376:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96713622] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.539376:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.539376:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.539376:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:18.0:1619677448.539377:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677448.539378:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.539379:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.539379:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.539379:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.539379:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.539380:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:30.0:1619677448.539381:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.539381:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:24.0:1619677448.539383:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:30.0:1619677448.539384:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.539385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.539386:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.539389:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.539389:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.539390:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.539392:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.539392:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677448.539392:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:30.0:1619677448.539393:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:2.0:1619677448.539393:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:30.0:1619677448.539394:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:2.0:1619677448.539394:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.539395:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.539396:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.539397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677448.539398:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677448.539404:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.539406:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.539407:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.539409:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.539410:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.539411:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.539530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677448.539536:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677448.539539:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677448.539540:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677448.539542:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.539545:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.539547:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677448.539548:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.539549:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.539550:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.539551:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677448.539555:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.539557:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.539559:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.539560:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:17.0:1619677448.539561:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677448.539563:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677448.539563:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677448.539564:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677448.539564:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677448.539569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.539577:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.539580:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.539581:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.539582:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.539583:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.539585:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.539642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.539648:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.539650:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.539651:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.539653:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.539655:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.539657:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.539658:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.539659:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.539660:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.539661:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.539664:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.539666:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.539667:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.539668:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.539668:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.539670:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.539670:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.539671:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.539672:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.539673:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.539676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.539678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.539684:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.539685:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.539685:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:17.0:1619677448.539686:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.539686:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.539686:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.539687:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.539688:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.539688:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677448.539689:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.539779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.539785:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:4.2:1619677448.539786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677448.539787:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.539788:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.539790:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.539793:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677448.539793:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.539795:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:26.0:1619677448.539796:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:6.0:1619677448.539796:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.539797:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.539798:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.539799:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.539800:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.539800:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.539802:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.539804:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:6.0:1619677448.539805:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.539806:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.539808:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.539808:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.539809:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.539810:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677448.539811:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.539812:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.539813:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:26.0:1619677448.539814:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.539815:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.539815:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.539816:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677448.539816:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.539816:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:21.2:1619677448.539817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:26.0:1619677448.539819:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.539820:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.539821:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:25.2:1619677448.539821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.539822:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.539823:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.539823:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:26.0:1619677448.539824:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.539825:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.539826:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677448.539826:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:31.0:1619677448.539827:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:2.0:1619677448.539827:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:9.2:1619677448.539828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677448.539829:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677448.539830:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:18.0:1619677448.539830:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.539831:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.539831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.539833:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.539833:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:30.0:1619677448.539834:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:18.0:1619677448.539835:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96713617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.539837:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:18.0:1619677448.539837:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.539837:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:25.2:1619677448.539838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:18.0:1619677448.539838:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.539838:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:18.0:1619677448.539839:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.539839:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:10.0:1619677448.539840:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:2.0:1619677448.539840:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.539841:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.539842:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:2.0:1619677448.539842:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.539844:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:30.0:1619677448.539845:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:18.0:1619677448.539846:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.2:1619677448.539847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.539848:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:18.0:1619677448.539848:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.539848:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677448.539849:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:18.0:1619677448.539849:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.539849:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677448.539850:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:18.0:1619677448.539850:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.539851:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.539852:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:18.0:1619677448.539852:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.539853:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677448.539853:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.539853:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.539854:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677448.539856:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.539858:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.539858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677448.539860:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.539860:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.539862:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.539863:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713616] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.539864:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.539866:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.539866:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.539866:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.539868:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.539869:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.539870:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.539870:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.539871:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:31.0:1619677448.539873:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.539873:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677448.539877:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.539877:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:9.2:1619677448.539877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:31.0:1619677448.539878:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.539880:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.539880:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.539881:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.539882:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.2:1619677448.539884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677448.539885:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.2:1619677448.539886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.539888:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.539888:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.539889:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677448.539890:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677448.539890:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.539890:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677448.539892:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.539894:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.539895:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.539896:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.539897:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.539897:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.539899:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.539900:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.539901:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.539902:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.539903:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.539904:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.539907:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.539907:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.539908:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.539909:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677448.539910:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.539912:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.539913:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.539913:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.539915:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.539915:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.539917:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.539918:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.539919:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.539921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.539924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677448.539928:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.539929:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:22.0:1619677448.539931:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677448.539931:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:22.0:1619677448.539932:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.539932:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.539932:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.539933:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:22.0:1619677448.539934:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.539936:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.539940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.539947:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.2:1619677448.539948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:6.0:1619677448.539950:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.539951:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.539951:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.539952:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.539952:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:26.0:1619677448.539956:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.539959:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.539959:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.539962:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.539965:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.539966:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.539967:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.539968:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.539970:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.539971:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.539974:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.539976:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.539978:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.539979:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.539980:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.539981:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.539982:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.539982:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.539983:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.539987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.539993:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.539996:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.539996:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.539998:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.540000:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.540001:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.540021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.540028:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.540030:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.540033:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.540035:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.540035:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.540037:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.540038:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.540039:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.540040:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.540041:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.540045:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.540046:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.540049:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.540049:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.540050:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.540052:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.540052:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.540053:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.540054:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.540057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.540063:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.540066:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.540067:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.540067:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.540069:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.540071:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.540101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.540106:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.540109:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.540110:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.540112:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.540113:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.540115:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.540116:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.540117:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.540118:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.540119:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.540122:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.540123:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.540125:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.540125:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.540126:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.540128:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.540128:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.540129:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.540129:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.540134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.540140:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.540143:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.540143:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.540144:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.540145:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.540145:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.540242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.540249:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.540252:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.540253:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.540255:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.540257:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.540259:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.540260:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.540261:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.540263:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.540263:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.540267:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.540269:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.540272:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.540273:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.540274:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.540275:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.540276:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.540277:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.540277:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.540281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.540289:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.540292:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.540293:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.540300:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.540302:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.540305:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.540392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.540398:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677448.540401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677448.540401:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.540402:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.540404:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.540406:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.540407:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:10.0:1619677448.540409:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:1.0:1619677448.540409:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.540410:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.540411:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.540412:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.540412:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:1.0:1619677448.540412:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.540415:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.540416:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.540418:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.540419:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.540420:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:1.0:1619677448.540421:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.540422:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.540422:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:10.0:1619677448.540423:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.540423:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:10.0:1619677448.540424:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.540424:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:10.0:1619677448.540425:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.540425:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.540425:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.540426:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:10.0:1619677448.540428:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.540430:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.2:1619677448.540431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.540434:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.540434:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.540435:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.540437:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.540438:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.540439:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677448.540439:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677448.540440:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.540441:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677448.540442:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.540443:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:9.2:1619677448.540443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677448.540443:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.540445:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.540446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.540448:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677448.540451:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.540453:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:2.0:1619677448.540454:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.540455:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.540456:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677448.540456:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.540456:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.540458:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:2.0:1619677448.540458:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.540461:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.540480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.540488:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.540490:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.540491:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.540493:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.540496:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.540498:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.540499:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.540500:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.540501:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.540502:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.540505:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.540507:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.540510:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.540511:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.540511:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.540513:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.540513:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.540514:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.540514:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.540520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.540527:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.540530:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.540531:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.540533:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.540533:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.540536:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.540609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.540615:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.540618:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.540619:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.540621:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.540623:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.540624:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.540626:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.540627:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.540628:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.540629:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.540632:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.540633:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.540636:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.540637:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.540637:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.540639:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.540639:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.540640:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.540641:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.540645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.540652:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.540655:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.540655:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.540657:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.540657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677448.540659:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.540660:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677448.540663:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.540665:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.540666:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.540668:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.540670:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.540672:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.540673:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.540675:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.540676:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.540677:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.540680:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.540682:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.540685:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.540685:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.540686:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.540688:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.540689:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.540689:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.540690:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.540694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.540702:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.540705:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.540706:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.540708:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.540709:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.540710:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.540910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.540918:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.540920:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.540921:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.540923:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.540926:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.540928:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.540929:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.540929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:26.0:1619677448.540930:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.540931:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.540932:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.540932:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.540934:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.540937:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.540938:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.540938:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:20.0:1619677448.540939:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.540940:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:20.0:1619677448.540941:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:5.0:1619677448.540941:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.540942:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.540943:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.540943:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.540944:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:20.0:1619677448.540944:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.540944:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:20.0:1619677448.540945:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.540946:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.540946:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:20.0:1619677448.540946:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:1.0:1619677448.540946:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:26.0:1619677448.540947:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.540947:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677448.540947:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677448.540949:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.540950:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.540951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.540952:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.540953:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.540953:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.540954:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:1.0:1619677448.540954:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:20.0:1619677448.540955:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:1.0:1619677448.540956:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96713603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.540957:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.540957:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677448.540957:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.540958:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677448.540958:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:1.0:1619677448.540958:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.540959:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677448.540959:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:12.0:1619677448.540960:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.540961:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.540962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:1.0:1619677448.540963:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.540964:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677448.540964:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.540965:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:12.0:1619677448.540966:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677448.540968:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.540969:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:5.0:1619677448.540970:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:1.0:1619677448.540970:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:21.2:1619677448.540972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677448.540972:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.540972:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.540973:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:1.0:1619677448.540973:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.540974:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000400:00000200:5.0:1619677448.540975:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.540975:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.540976:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.540977:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.540978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.2:1619677448.540980:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.540981:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.540983:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:30.0:1619677448.540984:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.0:1619677448.540984:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:17.0:1619677448.540985:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:30.0:1619677448.540986:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.540987:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.0:1619677448.540987:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:31.0:1619677448.540988:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.540988:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.540988:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:17.0:1619677448.540989:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677448.540989:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.540990:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:1.0:1619677448.540990:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.540991:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.540992:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713602] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.540993:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.540994:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.540994:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:30.0:1619677448.540995:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.540999:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.541000:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:9.2:1619677448.541001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:30.0:1619677448.541003:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.541004:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.541004:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.541006:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.541006:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.541008:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:24.0:1619677448.541008:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:30.0:1619677448.541009:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.541010:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677448.541010:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:25.2:1619677448.541011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:24.0:1619677448.541011:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.541011:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.541012:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.541013:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677448.541016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.541017:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.541020:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.541020:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.541021:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.541023:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:10.0:1619677448.541023:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.541025:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:10.0:1619677448.541025:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677448.541026:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.541027:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677448.541099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.541105:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.541108:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.541109:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.541111:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.541114:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.541115:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.541117:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.541118:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.541119:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.541120:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.541123:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.541125:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.541128:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.541128:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.541129:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.541131:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.541131:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.541133:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.541134:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.541134:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.541137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.541140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:13.0:1619677448.541143:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.541145:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:13.0:1619677448.541146:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.541147:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.541148:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.541148:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.541149:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:13.0:1619677448.541149:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.541150:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.541151:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.541200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.541203:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.541205:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.541208:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.541208:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.541210:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.541211:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.541212:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.541214:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.541215:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.541216:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.541220:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.541222:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.541224:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.541225:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.541226:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.541228:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.541228:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.541229:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.541230:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.541234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.541241:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.541243:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.541244:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.541246:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.541247:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.541249:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.541305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677448.541312:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.541315:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677448.541316:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:4.2:1619677448.541317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:26.0:1619677448.541318:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.541321:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.541323:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.541324:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.541326:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.541326:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:26.0:1619677448.541327:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.541327:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677448.541328:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.541331:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.541332:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.541333:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.541334:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.541335:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.541336:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.541337:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.541338:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.541338:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.541342:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.541342:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.541343:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.541343:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.541344:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.541346:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.541346:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.541347:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.541347:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.541348:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.541348:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.541349:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:26.0:1619677448.541350:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.541350:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.541351:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.541351:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.541352:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.541353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.2:1619677448.541356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.541362:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.541364:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.541365:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:5.0:1619677448.541365:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.541366:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.541366:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.541367:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.541369:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.541369:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.541371:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677448.541372:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.541372:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.541375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.541383:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.541386:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.541387:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.541387:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.541388:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.541389:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.541457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.541463:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.541466:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.541467:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.541469:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.541472:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.541473:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.541475:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.541476:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.541477:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.541478:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.541481:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.541482:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.541485:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.541486:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.541487:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.541489:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.541489:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.541490:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.541490:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.541495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.541500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:12.0:1619677448.541503:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.541505:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.541506:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.541508:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:12.0:1619677448.541508:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.541510:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.541511:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:12.0:1619677448.541511:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.541511:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.541513:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.541513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:26.0:1619677448.541516:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.541518:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.541519:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.541519:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:26.0:1619677448.541520:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.541521:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.541522:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:13.0:1619677448.541522:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.541523:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.541523:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.541524:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:26.0:1619677448.541525:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.541525:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.541527:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.541529:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.541529:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.541530:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.541532:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.541532:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.541533:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.541533:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.541537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.541543:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.541545:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.541546:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.541547:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.541548:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.541549:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.541611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.541620:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.541622:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.541624:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.541626:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.541627:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.541628:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.541629:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.541630:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.541631:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.541632:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.541635:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.541636:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.541639:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.541639:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.541640:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.541641:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.541642:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.541643:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.541645:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.541645:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.541647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.541651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677448.541653:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.541655:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.541656:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.541657:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.541659:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:12.0:1619677448.541659:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.541660:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.541661:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.541662:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.541663:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:4.2:1619677448.541720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.541727:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.541730:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.541731:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.541733:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.541735:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.541737:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.541738:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.541739:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.541740:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.541741:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.541744:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.541745:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.541748:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.541749:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.541749:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.541751:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.541751:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.541752:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.541753:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.541757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.541766:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.541768:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.541769:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.541769:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.541772:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.541774:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.541778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.541785:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.541788:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.541789:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.541789:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.541790:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.541791:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.541967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.541974:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.541977:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.541979:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.541981:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.541982:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.541983:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677448.541984:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.541986:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.541987:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.541988:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677448.541991:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.541992:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677448.541995:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.541995:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.541996:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.541997:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.541998:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.541999:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.542000:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.542003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.542009:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.542012:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.542013:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.542013:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.542015:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.542017:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.542018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677448.542022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:26.0:1619677448.542025:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.542027:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:8.0:1619677448.542027:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:10.0:1619677448.542028:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.542030:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.542030:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.542031:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.542031:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000400:00000200:26.0:1619677448.542032:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.542032:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000400:00000200:26.0:1619677448.542033:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:8.0:1619677448.542033:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:26.0:1619677448.542034:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.542036:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.542037:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.542037:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.542040:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.542042:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.542043:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.542044:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.542044:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.542046:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.542046:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.542047:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.542047:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.542051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.542057:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.542059:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.542059:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.542060:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.542061:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.542062:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.542070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.542076:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.542078:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.542079:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.542079:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.542080:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.542080:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.542190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.542196:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.542199:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.542200:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.542202:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.542205:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.542206:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.542207:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.542208:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.542209:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.542209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:22.0:1619677448.542210:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.542213:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.542215:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.542217:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.542217:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677448.542218:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.542219:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.542219:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.542220:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677448.542220:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.542221:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.542222:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.542222:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677448.542222:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.542223:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.542224:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.542226:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.542227:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677448.542227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:10.0:1619677448.542228:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.542229:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.542232:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.542234:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.542235:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.542235:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:10.0:1619677448.542236:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.542237:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.542237:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:10.0:1619677448.542238:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:5.0:1619677448.542238:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.542239:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.542239:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677448.542240:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.542240:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:5.0:1619677448.542240:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.542242:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:25.2:1619677448.542244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:8.2:1619677448.542246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.542249:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.542252:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.542252:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.542253:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.542254:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.542254:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.542256:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.0:1619677448.542256:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.542257:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.542258:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.542259:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.542260:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.542292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.542299:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.542302:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.542303:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.542305:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.542307:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.542309:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.542310:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.542311:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.542312:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.542313:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.542316:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.542317:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.542319:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.542320:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.542321:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.542322:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.542323:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.542323:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.542324:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.542329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.542330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:17.0:1619677448.542336:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677448.542336:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677448.542339:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.542339:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.542339:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.542340:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677448.542340:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.542342:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.542342:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:17.0:1619677448.542344:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:2.0:1619677448.542345:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.542347:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677448.542348:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.542350:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.542350:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.542352:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677448.542355:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.542357:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677448.542360:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.542361:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677448.542361:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677448.542363:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677448.542363:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.542364:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.542365:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.542369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.542376:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.542379:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.542380:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.542380:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.542382:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:10.0:1619677448.542385:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677448.542387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:25.2:1619677448.542391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000800:00000200:30.0:1619677448.542395:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.542397:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:30.0:1619677448.542398:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:26.0:1619677448.542399:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000400:00000200:30.0:1619677448.542400:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.542400:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.542400:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.542401:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.542402:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:31.0:1619677448.542403:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.542403:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.542404:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.542405:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713587] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.542407:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.542408:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.542408:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.542411:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.542413:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.542415:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.542416:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.542417:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.542418:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.542419:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.542420:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.542420:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.542424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.542432:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.542434:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.542435:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.542437:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.542439:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.542439:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:4.2:1619677448.542490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.542497:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.542500:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.542500:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:20.2:1619677448.542502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:10.0:1619677448.542502:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.542504:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.542505:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:10.0:1619677448.542505:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:21.2:1619677448.542506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:10.0:1619677448.542506:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.542507:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:10.0:1619677448.542508:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.542509:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.542511:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.542511:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.542511:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677448.542515:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677448.542515:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.542516:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:20.0:1619677448.542517:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.542518:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:20.0:1619677448.542519:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:10.0:1619677448.542520:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.542520:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:31.0:1619677448.542521:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.542521:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.542521:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.542522:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677448.542522:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.542523:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.542523:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.542523:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.542524:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.542524:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:10.0:1619677448.542524:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.542526:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.542526:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677448.542527:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.542528:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96713584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.542528:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:31.0:1619677448.542529:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.542530:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.542530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677448.542531:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:4.2:1619677448.542532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677448.542535:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.542535:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.542537:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.542537:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.542538:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677448.542539:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:5.0:1619677448.542539:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.542540:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.542541:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:20.0:1619677448.542541:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.542541:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000400:00000200:5.0:1619677448.542541:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.542542:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.542542:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677448.542542:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:31.0:1619677448.542543:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.542543:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:5.0:1619677448.542543:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:31.0:1619677448.542544:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.542544:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000400:00000200:5.0:1619677448.542544:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96713583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.542546:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.542546:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:5.0:1619677448.542546:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.542547:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:5.0:1619677448.542547:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.542548:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:5.0:1619677448.542548:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:9.2:1619677448.542550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:25.2:1619677448.542551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677448.542551:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.542552:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.542553:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.542553:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.542554:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677448.542555:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.542555:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.542556:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (23)++
00000800:00000200:24.0:1619677448.542557:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:20.0:1619677448.542557:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.542557:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.542557:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (24)++
00000400:00000200:20.0:1619677448.542559:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.542559:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.542559:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (24)--
00000800:00000200:24.0:1619677448.542560:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677448.542560:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:5.0:1619677448.542560:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.542560:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000400:00000200:24.0:1619677448.542561:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.542561:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.542561:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.542562:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000400:00000200:12.0:1619677448.542562:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.542562:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:24.0:1619677448.542563:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.542563:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:12.0:1619677448.542564:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.542565:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677448.542566:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:12.0:1619677448.542566:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.542566:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.542566:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.542568:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.542569:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.542570:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.542660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.542668:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.542670:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.542671:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.542673:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.542676:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.542677:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677448.542679:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.542680:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.542681:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.542682:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.542685:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.542687:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.542690:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.542691:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.542692:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.542693:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.542694:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.542695:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.542695:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.542718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.542726:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.542729:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.542729:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.542731:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.542732:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.542734:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.542786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677448.542792:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:25.2:1619677448.542794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.542795:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677448.542796:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677448.542798:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.542800:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677448.542800:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.542801:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.542802:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.542803:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.542804:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.542804:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:1.0:1619677448.542804:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.542805:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:1.0:1619677448.542805:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.542806:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.542807:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677448.542809:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.542811:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.542814:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.542815:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.542816:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.542818:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.542818:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677448.542820:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677448.542821:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677448.542822:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:25.2:1619677448.542823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.542827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:12.0:1619677448.542830:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.542832:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:12.0:1619677448.542833:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.542833:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.542834:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677448.542834:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677448.542835:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000400:00000200:12.0:1619677448.542835:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.542836:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.542838:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.542880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.542887:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.542889:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.542890:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.542892:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.542895:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.542896:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.542897:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.542899:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.542900:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.542901:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.542904:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.542906:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.542909:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.542910:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.542911:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.542912:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.542913:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.542914:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.542914:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:4.2:1619677448.542917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677448.542919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.542924:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.2:1619677448.542924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:26.0:1619677448.542926:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.542927:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677448.542928:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:26.0:1619677448.542929:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.542930:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.542930:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:26.0:1619677448.542932:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.542932:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.542932:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:26.0:1619677448.542934:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.542935:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.542936:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.542936:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.542937:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.542938:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.542939:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.542939:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.542940:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.542941:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677448.542942:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.542942:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.542943:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713578] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.542944:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.542945:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.542946:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.542946:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.542947:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.542950:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.542950:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.542951:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:2.0:1619677448.542951:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.542953:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.542953:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.542954:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.542954:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677448.542954:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.542957:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.542957:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.542958:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:25.2:1619677448.542959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.542960:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.542960:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.542961:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.542962:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.542966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.542967:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.542970:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.542970:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.542971:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.542972:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677448.542973:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.542975:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.542976:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.542977:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.542977:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677448.542979:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.542980:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.542992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.542997:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.543000:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.543001:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.543002:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.543005:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.543006:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.543007:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.543009:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.543010:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.543011:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.543014:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.543016:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.543018:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.543019:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.543020:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.543021:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.543022:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.543022:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.543023:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.543028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.543035:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.543038:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.543039:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.543039:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.543041:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.543044:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.543047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.543054:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.543056:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.543057:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.543057:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.543058:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.543059:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.543126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.543133:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.543136:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.543137:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.543139:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.543142:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.543143:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.543144:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.543146:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.543147:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.543148:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.543151:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.543153:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.543156:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.543157:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.543158:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.543159:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.543160:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.543161:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.543162:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.543163:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.543165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677448.543168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.543173:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:8.0:1619677448.543175:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.543176:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.543176:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:8.0:1619677448.543177:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677448.543177:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.543178:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.543179:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.543179:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.543182:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.543200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.543207:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.543210:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.543212:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.543214:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.543215:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.543216:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.543218:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.543218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1200] (20)++
00000400:00000200:30.0:1619677448.543219:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.543219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:30.0:1619677448.543220:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.543221:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.543224:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.543224:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:5.0:1619677448.543224:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.543226:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.543226:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:5.0:1619677448.543227:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677448.543227:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:30.0:1619677448.543229:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.543230:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.543230:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.543230:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:17.0:1619677448.543230:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.543231:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.543232:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.543232:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.543233:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677448.543233:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.543233:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.543234:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:5.0:1619677448.543235:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.543236:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:25.2:1619677448.543237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:17.0:1619677448.543237:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.543237:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.543239:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.543239:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.543240:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.543240:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.543241:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.543242:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.543243:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:5.0:1619677448.543245:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.543246:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.543247:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.543247:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.543250:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.543250:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.543250:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.543251:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.543252:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.543252:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.543253:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.543254:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.543255:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.543255:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:25.2:1619677448.543256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.543256:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.543256:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677448.543257:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:5.0:1619677448.543257:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677448.543258:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.543260:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.543260:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677448.543261:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:17.0:1619677448.543262:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:13.0:1619677448.543262:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.543265:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:25.2:1619677448.543266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.543266:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.543267:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677448.543267:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.543269:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.543270:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.543270:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:13.0:1619677448.543271:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.543271:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.543271:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.543273:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:12.0:1619677448.543274:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.543276:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.543278:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.543279:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.543280:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.543301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.543307:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.543310:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.543311:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.543313:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.543316:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.543318:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.543319:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713572] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.543321:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.543322:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.543323:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.543326:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.543328:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.543331:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.543332:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.543333:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.543335:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.543335:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.543336:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.543336:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.543341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.543349:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.543351:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.543352:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.543354:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.543356:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.543357:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.543397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.543405:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.543408:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.543410:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.543411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:31.0:1619677448.543412:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.543412:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.543414:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.543415:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.543416:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.543417:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.543418:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.543419:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677448.543422:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.543422:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.543423:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.543424:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.543425:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.543427:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.543428:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:10.0:1619677448.543428:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.543429:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:10.0:1619677448.543429:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.543430:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.543431:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.543431:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677448.543431:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.543432:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:10.0:1619677448.543432:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.543433:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:10.0:1619677448.543433:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.543436:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.543438:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.2:1619677448.543439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.543440:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.543441:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.543441:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.543443:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.543443:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.543444:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.543446:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.543446:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677448.543447:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:25.2:1619677448.543448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.543450:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.543450:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.543452:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.543452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.543454:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.543455:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.543456:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677448.543457:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.543458:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.543459:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:12.0:1619677448.543459:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.543461:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:13.0:1619677448.543461:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.543462:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:12.0:1619677448.543462:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.543463:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.2:1619677448.543532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.543538:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.543541:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.543542:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.543544:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.543546:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.543548:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.543549:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.543550:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.543551:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.543552:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.543556:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.543557:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.543560:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.543561:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.543562:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.543564:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.543564:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.543565:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.543565:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.543569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.543576:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.543579:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.543580:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.543581:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.543582:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.543584:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.543586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:4.2:1619677448.543592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677448.543593:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.543596:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.543596:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.543597:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.543597:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.543599:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:12.0:1619677448.543599:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.543602:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.543603:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.543605:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.543608:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.543609:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.543610:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.543612:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.543613:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.543614:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.543617:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.543619:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.543621:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.543622:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.543623:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.543625:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.543625:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.543626:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.543626:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.543630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.543637:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.543640:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.543640:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.543642:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.543644:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.543644:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.543660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.543666:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.543669:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.543670:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.543672:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.543674:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.543676:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.543677:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.543678:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.543679:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.543680:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.2:1619677448.543682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:6.0:1619677448.543683:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.543685:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.543685:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.543687:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:6.0:1619677448.543688:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.543689:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.543689:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677448.543690:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.543690:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677448.543691:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.543691:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:20.0:1619677448.543692:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.543692:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.543693:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000400:00000200:20.0:1619677448.543694:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:6.0:1619677448.543694:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677448.543695:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.543696:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.543697:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.543697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677448.543698:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:21.2:1619677448.543699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:20.0:1619677448.543701:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.543703:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.543704:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:17.0:1619677448.543705:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677448.543706:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.543706:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677448.543706:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.543707:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677448.543707:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.543708:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.543708:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.543709:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.543709:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.543710:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.543711:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:17.0:1619677448.543711:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.543712:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.543714:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.543714:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677448.543714:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:25.2:1619677448.543716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.543719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.2:1619677448.543720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:13.0:1619677448.543722:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.543725:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.543726:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.543726:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:2.0:1619677448.543726:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:13.0:1619677448.543728:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.543728:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:12.0:1619677448.543729:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.543729:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677448.543729:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.543730:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:13.0:1619677448.543730:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.543730:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:2.0:1619677448.543732:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.543734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:2.0:1619677448.543736:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.543737:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.543739:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.543740:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:2.0:1619677448.543740:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.543741:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.543742:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:12.0:1619677448.543744:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.543745:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.543746:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.543746:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677448.543746:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.543747:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.543748:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.543751:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.543752:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.543753:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.543755:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.543755:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.543756:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.543756:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.543760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.543767:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.543770:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.543770:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.543771:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.543773:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.543776:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.543865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.543871:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.543874:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.543874:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.543876:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.543879:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.543880:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.543882:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.543883:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.543884:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.543884:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.543887:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.543889:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.543891:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.543892:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.543893:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.543894:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.543895:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.543896:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.543897:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.543898:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:25.2:1619677448.543900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.543903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677448.543907:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.543908:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:12.0:1619677448.543909:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.543910:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.543911:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.543911:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:12.0:1619677448.543912:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.543912:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677448.543914:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.543915:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.543997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.544002:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.544005:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.544006:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.544008:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.544010:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.544012:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.544013:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.544014:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.544015:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.544016:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.544020:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.544022:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.544025:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.544026:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.544026:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.544028:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.544029:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.544030:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.544031:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.544031:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.544034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.2:1619677448.544036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.544041:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677448.544042:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677448.544044:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:17.0:1619677448.544044:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.544045:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.544045:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677448.544046:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.544047:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.544049:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.544051:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.544062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.544070:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.544073:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.544075:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.544076:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.544077:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.544079:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.544080:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.544081:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.544082:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.544083:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.544086:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.544088:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.544091:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.544092:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.544093:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.544094:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.544095:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.544096:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.544097:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.544100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.544109:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.544112:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.544113:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.544114:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.544115:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.544118:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.544194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.544200:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.544203:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.544204:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.544206:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.544209:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.544210:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.544211:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.544213:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.544214:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.544215:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.544218:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.544220:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.544223:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.544224:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.544224:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.544226:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.544227:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.544227:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.544228:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.544232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.544239:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.544241:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.544242:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.544244:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.544245:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.544247:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.544261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.544268:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.544271:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.544271:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.544274:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.544276:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.544277:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.544278:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.544280:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.544281:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.544282:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.544286:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.544287:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.544290:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.544290:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.544291:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.544293:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:25.2:1619677448.544299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:26.0:1619677448.544300:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.544301:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.544301:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.544307:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.544309:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.544310:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.544311:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.544314:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.544314:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.544315:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.544316:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.544316:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.544317:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.544317:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.544410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:5.2:1619677448.544413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:24.0:1619677448.544417:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.544420:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:10.0:1619677448.544420:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677448.544421:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.544423:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.544423:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.544424:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:24.0:1619677448.544426:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.544426:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.544427:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.544428:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.544428:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.544430:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.544430:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:24.0:1619677448.544431:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.544431:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.544432:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.544432:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.544433:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.544434:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:24.0:1619677448.544435:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.544437:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.544438:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.544440:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.544440:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:24.0:1619677448.544441:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.544441:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.544443:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.544443:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677448.544443:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.544444:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:10.0:1619677448.544444:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.544444:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:24.0:1619677448.544445:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677448.544446:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.544447:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.544447:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.544448:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.2:1619677448.544449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:25.2:1619677448.544453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.544458:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.544460:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:6.0:1619677448.544460:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:17.0:1619677448.544461:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.544461:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:6.0:1619677448.544463:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.544464:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:17.0:1619677448.544464:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.544464:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.544467:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:6.0:1619677448.544467:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.544470:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.544517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.544525:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.544528:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.544530:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.544532:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.544533:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.544534:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.544536:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.544537:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.544538:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.544539:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.544542:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.544544:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.544546:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.544546:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.544547:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.544548:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.544549:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.544550:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.544552:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.544552:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.544554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.544558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.544560:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.544562:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.544563:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.544563:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.544566:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:26.0:1619677448.544566:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.544568:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:26.0:1619677448.544568:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:30.0:1619677448.544569:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.2:1619677448.544569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.544570:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677448.544575:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.544578:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.544579:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.544581:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.544583:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.544585:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.544586:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.544587:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.544589:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.544590:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.544593:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.544595:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.544598:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.544599:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.544599:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.544601:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.544602:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.544602:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.544602:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.544607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.544614:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.544616:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.544617:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.544619:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.544621:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.544622:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.544686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.544693:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.544696:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.544697:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.544699:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.544701:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.544702:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.544703:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.544705:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.544706:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.544707:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.544710:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.544712:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.544715:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.544715:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.544716:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.544718:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.544718:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.544719:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.544720:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.544721:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.544724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.544726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.544731:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.544733:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.544734:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.544735:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677448.544735:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.544736:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.544736:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.544737:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.544738:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.544740:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.544746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.544753:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.544756:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.544758:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.544760:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.544761:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.544762:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.544764:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.544765:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.544766:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.544767:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.544771:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.544772:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.544776:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.544777:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.544777:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.544779:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.544780:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.544781:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.544783:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.544783:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.544785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.544790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:20.2:1619677448.544791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.544797:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.0:1619677448.544798:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:30.0:1619677448.544799:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.544799:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.544800:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.544800:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:30.0:1619677448.544801:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:24.0:1619677448.544801:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:30.0:1619677448.544802:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.544802:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.544803:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.544803:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:8.0:1619677448.544804:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.544805:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:8.0:1619677448.544807:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.544808:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.544810:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.544811:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.544812:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.544813:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.544816:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.544817:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.544820:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.544820:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.544821:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.544822:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.544823:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.544823:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.544824:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.544839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.544846:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.544848:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.544849:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.544851:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.544853:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.544854:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.544884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:21.2:1619677448.544890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.544893:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.544895:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.544896:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.544897:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677448.544898:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.544900:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.544902:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.544903:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.544905:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.544906:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.544907:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.544908:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.544909:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.544911:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.544912:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.544913:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.544914:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.544914:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.544915:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.544916:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.544919:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.544920:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.544920:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.544921:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.544922:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.544923:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.544924:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.544925:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.544925:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.544926:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.544926:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.544926:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.544927:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677448.544928:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.544928:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.544929:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.544929:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677448.544929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.544930:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.544934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.544937:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.544940:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.544941:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.544941:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:1.0:1619677448.544941:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.544943:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.544944:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.544945:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.544945:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.544946:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:6.0:1619677448.544947:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677448.544949:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.544951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.544958:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.544961:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.544962:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.544962:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.544963:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.544963:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.544975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.544980:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.544983:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.544984:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.544986:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.544989:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.544990:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.544991:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.544993:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.544994:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.544995:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.544999:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.545000:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.545003:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.545004:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.545005:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.545006:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.545007:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.545007:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.545008:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.545012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.545019:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.545022:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.545022:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.545024:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.545025:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.545027:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.545033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.545039:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.545041:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.545042:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.545044:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.545046:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.545048:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.545049:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.545051:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.545051:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.545053:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.545056:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.545058:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.545060:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.545061:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.545062:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.545064:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.545064:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.545065:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.545065:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.545070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.545077:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.545080:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.545081:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.545081:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.545083:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.545086:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.545140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.545146:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.545148:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.545151:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.545151:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.545154:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.545155:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.545157:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.545158:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.545159:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.545160:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:4.2:1619677448.545161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677448.545163:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.545169:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677448.545169:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.545171:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.545172:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.545172:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.545173:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:5.0:1619677448.545174:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.545174:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:10.0:1619677448.545175:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.545175:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:26.0:1619677448.545176:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.545177:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:10.0:1619677448.545178:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.545178:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.545178:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677448.545179:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.545179:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:10.0:1619677448.545180:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.545181:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713547] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.545182:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.545183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:10.0:1619677448.545183:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.545184:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.545186:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.545187:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96713546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.545188:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.545188:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.545189:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.545190:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:22.0:1619677448.545190:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.545190:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677448.545193:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:26.0:1619677448.545194:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.545194:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.545194:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.545194:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.545195:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:22.0:1619677448.545196:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.545196:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.545197:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677448.545198:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677448.545198:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.545199:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.545200:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.545200:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.545201:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.545201:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.545201:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.545202:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.545203:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.545203:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.545204:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:9.2:1619677448.545204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.545205:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.545205:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.545209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:18.0:1619677448.545212:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.545214:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.545215:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.545216:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:18.0:1619677448.545216:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.545218:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:18.0:1619677448.545218:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.545219:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.545221:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.545221:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677448.545222:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.545224:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.545229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.545232:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.545234:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.545237:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.545237:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.545239:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.545240:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677448.545242:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.545243:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.545244:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.545245:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.545248:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.545250:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.545252:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.545253:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.545254:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.545255:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.545256:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.545257:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.545258:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.545259:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677448.545261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677448.545264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677448.545269:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677448.545270:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:17.0:1619677448.545271:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.545272:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.545272:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.545272:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:1.0:1619677448.545272:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.545273:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:17.0:1619677448.545274:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677448.545277:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.545383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.545391:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.545394:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.545396:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.545398:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.545399:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.545400:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.545402:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.545403:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.545404:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.545405:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.545408:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.545409:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.2:1619677448.545409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:30.0:1619677448.545412:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.545413:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.545414:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.545416:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.545416:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.545417:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.545417:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:30.0:1619677448.545418:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.545420:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.545420:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677448.545422:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.545422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:10.0:1619677448.545425:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.545426:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.545427:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.545429:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.545429:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.545430:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.545431:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:10.0:1619677448.545433:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.545433:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.545434:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.545436:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:5.0:1619677448.545436:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.545437:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.545439:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.545439:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:10.0:1619677448.545440:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.545440:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.2:1619677448.545441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.545442:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.545443:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.545443:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.545444:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677448.545448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.545449:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.545451:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.545452:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.545453:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.545454:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.545454:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677448.545456:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.545459:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.545460:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.545460:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.545463:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.545466:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.545481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.545486:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.545489:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.545489:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.545491:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.545494:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.545495:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.545497:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.545498:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.545499:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.545500:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.545503:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.545505:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.545508:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.545508:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.545509:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.545510:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.545511:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.545511:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.545512:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.545518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.545526:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.545528:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.545529:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.545530:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.545532:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.545534:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.545539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:22.0:1619677448.545546:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.545549:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.545550:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.545550:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.545551:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.545551:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.545601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.545608:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.545611:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.545613:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.545614:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.545616:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.545618:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.545619:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.545620:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.545621:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.545623:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677448.545626:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.545628:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.545630:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.545631:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.545632:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.545634:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.545634:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.545635:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.545636:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.545640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.545647:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.545650:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.545651:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.545651:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.545653:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.545657:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.545658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.545664:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.545667:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.545668:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.545668:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.545669:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.545670:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.545838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.545844:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.545846:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.545847:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.545849:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.545851:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.545853:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.545854:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.545855:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.545856:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.545857:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.545860:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.545862:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.545864:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.545865:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.545865:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.545867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:2.0:1619677448.545867:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.545868:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.545868:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.545869:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.545873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:1.0:1619677448.545873:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677448.545876:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677448.545877:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677448.545879:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.545881:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:1.0:1619677448.545881:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.545883:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:1.0:1619677448.545883:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.545884:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.545884:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.545885:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.545886:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.545887:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:1.0:1619677448.545887:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.545888:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:1.0:1619677448.545888:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.545892:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.545894:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.545897:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.545898:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.545898:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.545900:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.545901:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677448.545901:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677448.545902:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:25.2:1619677448.545906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.545911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:12.0:1619677448.545914:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.545917:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:26.0:1619677448.545918:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.2:1619677448.545918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:12.0:1619677448.545918:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.545919:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.545921:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:13.0:1619677448.545921:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.545922:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.545922:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.545924:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.545924:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.545926:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:26.0:1619677448.545927:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.545927:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:26.0:1619677448.545928:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:6.0:1619677448.545929:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.545930:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.545931:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.545932:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.545932:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.545933:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:26.0:1619677448.545934:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:6.0:1619677448.545935:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.545936:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.545938:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.545939:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.545939:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:26.0:1619677448.545942:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:6.0:1619677448.545944:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.545945:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.545945:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.545946:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677448.545946:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:26.0:1619677448.545948:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.545948:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.545948:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.545949:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.545949:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677448.545949:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.545950:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.545951:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.545952:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.545952:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:25.2:1619677448.545953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677448.545953:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.545958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:13.0:1619677448.545959:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.545962:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.545963:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.545965:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:13.0:1619677448.545965:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.545966:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.545967:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677448.545968:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.545969:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.545969:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.545972:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.545974:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.546064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.546070:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.546072:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.546073:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.546075:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.546078:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.546079:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.546081:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.546082:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.546083:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.546084:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.546087:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.546089:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.546092:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.546092:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.546093:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.546095:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.546095:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.546096:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.546097:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.546098:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.546101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.546103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.546108:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.546108:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.546110:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677448.546111:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677448.546111:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677448.546112:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.546112:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.546112:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677448.546115:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.546117:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.546441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.546449:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.546451:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.546452:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.546454:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.546457:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.546458:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.546459:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.546461:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.546462:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.546463:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.546466:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.546467:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.546470:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.546471:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.546472:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.546473:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.546474:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.546475:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.546476:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.546477:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.2:1619677448.546480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.546483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.2:1619677448.546486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:17.0:1619677448.546488:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.546491:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:26.0:1619677448.546492:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:17.0:1619677448.546492:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.546492:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.546493:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.546494:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:17.0:1619677448.546494:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.546495:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677448.546496:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.546498:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:17.0:1619677448.546498:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:10.0:1619677448.546499:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.546500:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.546503:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.546503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000400:00000200:26.0:1619677448.546505:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.546506:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.546507:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.546508:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.546509:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.546510:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:26.0:1619677448.546512:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.546513:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.546514:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.546514:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:26.0:1619677448.546515:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.546515:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.546516:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:26.0:1619677448.546517:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.546518:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.546518:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.546519:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.546520:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.546521:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677448.546522:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.546523:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677448.546526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.546528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:12.0:1619677448.546534:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.546535:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:12.0:1619677448.546536:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.546537:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.546538:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.546539:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:12.0:1619677448.546539:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.546539:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:12.0:1619677448.546541:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.546542:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.546543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.546550:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.546553:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.546553:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.546554:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.546554:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.546555:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.546603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.546611:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.546613:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.546615:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.546618:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.546618:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.546620:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.546621:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.546622:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.546623:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.546624:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.546627:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.546628:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.546630:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.546631:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.546631:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.546633:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.546634:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.546634:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.546635:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.546639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.546645:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.546647:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.546648:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.546650:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.546650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.546651:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.546653:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:17.0:1619677448.546655:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:25.2:1619677448.546657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:17.0:1619677448.546658:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.546659:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.546661:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.546663:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000400:00000200:17.0:1619677448.546663:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.546664:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:30.0:1619677448.546666:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000400:00000200:17.0:1619677448.546666:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.546667:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000400:00000200:17.0:1619677448.546667:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.546668:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.546668:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:17.0:1619677448.546668:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.546669:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.546670:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:17.0:1619677448.546672:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.546674:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.546676:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.546677:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.546677:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.546679:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.546679:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.546680:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.546681:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:21.2:1619677448.546684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:25.2:1619677448.546685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.546691:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677448.546693:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:31.0:1619677448.546694:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.546696:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.546696:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.546697:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.546697:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.546698:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.546698:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:6.0:1619677448.546699:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.546700:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.546701:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.546701:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:31.0:1619677448.546702:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.546703:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.546704:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.546707:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.546709:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.546712:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.546713:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.546714:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.546715:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.546716:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.546717:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.546718:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.546722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.546729:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.546732:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.546733:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.546733:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.546735:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.546738:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.546791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.546798:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:4.2:1619677448.546799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677448.546801:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.546802:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.546804:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.546806:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.546806:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:24.0:1619677448.546808:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.546809:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.546809:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.546810:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:24.0:1619677448.546811:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.546812:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.546812:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.546813:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.546815:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.546817:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.546817:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.546818:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.546819:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.546819:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.546820:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.546821:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.546821:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.546822:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.546823:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.546824:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:10.0:1619677448.546824:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.546825:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.546826:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.546826:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677448.546826:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.546829:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.546830:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.546830:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.2:1619677448.546830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.546832:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.546832:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.546833:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.546833:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677448.546838:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:8.2:1619677448.546839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.546840:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.546841:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.546841:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.546844:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.546846:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.0:1619677448.546847:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.546850:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.546851:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.546852:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.546853:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.2:1619677448.546856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.546856:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677448.546863:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.546866:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.546867:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.546868:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.546868:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.546869:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.546882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677448.546889:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.546892:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677448.546893:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677448.546895:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.546898:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.546899:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.546901:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.546902:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.546903:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.546903:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.546907:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.546908:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.546911:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.546912:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.546912:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.546914:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.546915:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.546915:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677448.546916:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677448.546920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.546927:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.546930:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.546931:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.546931:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.546933:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.546936:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.546948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:8.0:1619677448.546955:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677448.546958:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677448.546959:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677448.546961:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.546963:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.546965:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:8.0:1619677448.546966:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.546967:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.546968:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.546969:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:8.0:1619677448.546972:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.546973:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677448.546975:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.546976:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:8.0:1619677448.546976:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:5.2:1619677448.546976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.0:1619677448.546978:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.546978:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677448.546979:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677448.546979:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:10.0:1619677448.546983:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:9.2:1619677448.546983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:10.0:1619677448.546985:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.546986:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.546988:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.546990:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:10.0:1619677448.546991:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.546992:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.546992:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.546992:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.546993:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.546993:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:5.2:1619677448.546993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:24.0:1619677448.546995:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.546995:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.546995:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.546996:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:10.0:1619677448.546996:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.546999:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.546999:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.547001:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.547001:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.547002:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.547004:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.547004:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.547005:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.547005:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:1.0:1619677448.547006:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.547007:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.547007:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.547008:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.547008:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:1.0:1619677448.547008:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.547009:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.547010:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.547011:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.547012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:1.0:1619677448.547012:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.547016:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.547017:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.547019:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.547021:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677448.547021:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677448.547022:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.547022:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:1.0:1619677448.547023:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (23)++
00000400:00000200:12.0:1619677448.547024:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.547025:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:1.0:1619677448.547025:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.547026:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.547026:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.547026:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.547027:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.547030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.547037:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.547040:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.547040:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.547042:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.547043:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.547045:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.547048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:12.0:1619677448.547055:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.547057:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.547058:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.547059:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.547059:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.547061:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.547125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.2:1619677448.547128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:30.0:1619677448.547134:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.547136:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:13.0:1619677448.547136:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000400:00000200:30.0:1619677448.547138:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.547138:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:31.0:1619677448.547140:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:13.0:1619677448.547140:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.547141:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.547142:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:30.0:1619677448.547143:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:13.0:1619677448.547143:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.547144:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.547145:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.547145:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677448.547147:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677448.547147:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.547148:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:13.0:1619677448.547148:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.547150:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677448.547150:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677448.547156:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677448.547156:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.547158:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.547158:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677448.547161:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.547161:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.547162:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.547162:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:30.0:1619677448.547163:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.547163:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:30.0:1619677448.547165:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.547165:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.547165:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:13.0:1619677448.547165:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:30.0:1619677448.547166:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677448.547166:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:30.0:1619677448.547167:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.547167:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:25.2:1619677448.547171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:9.2:1619677448.547171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:12.0:1619677448.547177:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677448.547177:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.547180:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:12.0:1619677448.547180:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677448.547180:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:12.0:1619677448.547181:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.547181:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.547183:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.547183:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.547184:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.547186:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677448.547186:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.2:1619677448.547203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.547205:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.547208:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.547210:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.547211:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.547213:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.547214:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677448.547216:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.547217:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.547218:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.547219:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677448.547223:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.547225:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.547227:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.547228:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.547229:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.547231:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.547232:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.547233:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.547233:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.547237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.547243:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.547246:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.547246:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.547247:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.547249:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:26.0:1619677448.547252:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.547258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.547263:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.547266:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.547267:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.547267:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.547268:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.547268:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.547340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.547346:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.547348:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.547351:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.547351:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.547353:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.547354:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677448.547356:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.547357:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.547358:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.547359:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677448.547362:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.547364:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.547366:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.547367:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.547368:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.547369:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.547370:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.547370:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.547371:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.547376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.2:1619677448.547378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677448.547384:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677448.547385:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.547387:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.547387:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.547388:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.547388:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677448.547389:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.547390:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:17.0:1619677448.547391:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.547392:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:17.0:1619677448.547394:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.547396:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.547397:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.547398:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.547399:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.547400:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.547404:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.547405:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.547408:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.547409:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.547410:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.547412:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.547412:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.547413:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.547414:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.547418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.547425:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.547428:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.547428:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.547429:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.547431:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.547434:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.547511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.547520:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.547522:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.547524:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.547527:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.547527:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.547528:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.547529:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.547530:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.547530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:31.0:1619677448.547531:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.547532:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.547535:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.547537:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.547538:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.547539:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.547540:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.547541:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.547541:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:31.0:1619677448.547542:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.547542:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.547543:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677448.547543:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.547544:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.547546:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:26.0:1619677448.547546:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.547547:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677448.547547:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.547548:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.547548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:26.0:1619677448.547549:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.547550:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.547551:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:21.2:1619677448.547552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677448.547553:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.547555:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:13.0:1619677448.547555:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677448.547557:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.547557:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:13.0:1619677448.547557:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:26.0:1619677448.547558:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:13.0:1619677448.547558:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.547559:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.547560:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:26.0:1619677448.547560:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.547560:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:13.0:1619677448.547560:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.547561:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.547562:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:13.0:1619677448.547562:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:4.2:1619677448.547562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.547563:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677448.547563:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.547564:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.547564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.547570:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.547571:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.547572:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.547572:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677448.547573:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.547574:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.547575:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.547576:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.547577:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.547577:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.547578:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:10.0:1619677448.547578:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.547579:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.547581:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.547582:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.547583:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.547585:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.547586:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.547587:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.547588:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.547588:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.547589:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.547590:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.547590:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.547591:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.547594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.547600:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.547602:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.547603:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.547604:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.547604:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.547606:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.547664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:21.2:1619677448.547670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:6.0:1619677448.547670:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.547673:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.547674:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.547676:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.547677:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:6.0:1619677448.547678:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.547680:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:6.0:1619677448.547680:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:24.0:1619677448.547681:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:6.0:1619677448.547682:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.547684:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.547684:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.547685:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.547686:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.547686:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677448.547688:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.547689:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96713515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.547690:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.547691:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.547693:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.547693:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:8.0:1619677448.547694:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:6.0:1619677448.547696:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.547697:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000400:00000200:8.0:1619677448.547698:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.547698:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.547700:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.547700:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677448.547701:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:6.0:1619677448.547701:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.547702:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.0:1619677448.547703:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.547704:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:25.2:1619677448.547705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:8.0:1619677448.547705:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.547707:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.547707:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.547708:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.547708:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:10.0:1619677448.547712:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:8.2:1619677448.547712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.547714:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.547715:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.547715:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.547717:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.547720:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.0:1619677448.547721:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.547723:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.547724:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.547726:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.547726:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.547729:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.547762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.547768:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.547771:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.547772:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.547774:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.547776:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.547778:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677448.547779:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.547780:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.547782:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.547783:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677448.547786:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.547788:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677448.547790:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.547791:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677448.547792:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.547793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.547793:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677448.547794:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.547795:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.547796:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.547797:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677448.547799:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:25.2:1619677448.547800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.547801:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:21.2:1619677448.547802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677448.547802:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.547804:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.547807:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:1.0:1619677448.547807:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.547808:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:1.0:1619677448.547808:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:22.0:1619677448.547809:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:1.0:1619677448.547809:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.547810:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.547810:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:1.0:1619677448.547810:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.547811:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677448.547811:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:1.0:1619677448.547811:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.547812:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:22.0:1619677448.547813:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000400:00000200:1.0:1619677448.547813:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:22.0:1619677448.547816:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:1.0:1619677448.547816:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.547818:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.547821:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.547821:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.547822:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.547824:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.547824:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.547826:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.547827:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.547827:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:21.2:1619677448.547829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:9.2:1619677448.547830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.547833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.547836:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:30.0:1619677448.547837:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.547838:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677448.547839:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:30.0:1619677448.547840:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677448.547840:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.0:1619677448.547840:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.547841:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.547841:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.547842:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677448.547843:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.547843:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.547844:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.547845:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.547846:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:24.0:1619677448.547846:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:30.0:1619677448.547847:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.547848:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.547849:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.547850:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.547853:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.547855:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.547858:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.547859:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.547859:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.547861:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.547861:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.547862:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.547863:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.547866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.547873:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.547875:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.547876:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.547877:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.547877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:13.0:1619677448.547879:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.547880:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677448.547884:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.547887:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.547887:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677448.547889:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.547892:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.547893:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.547894:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.547896:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.547896:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.547897:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.547900:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.547902:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.2:1619677448.547903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:8.0:1619677448.547905:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.547906:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.547906:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.547908:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.547908:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677448.547908:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:8.0:1619677448.547909:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.547909:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677448.547911:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.547912:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.547914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:2.0:1619677448.547914:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.547916:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.547918:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.547919:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.547921:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.547922:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.547922:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:2.0:1619677448.547923:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.547925:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.547926:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:2.0:1619677448.547926:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.547926:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.547928:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.547929:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.547931:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.547932:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.547932:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.547932:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:2.0:1619677448.547934:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.547935:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.547935:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.547936:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.547940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.547947:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.547950:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.547951:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.547952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:13.0:1619677448.547953:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.547953:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.547955:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677448.547955:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.547957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.547958:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.547960:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.547961:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.547962:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.547963:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:12.0:1619677448.547963:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.547965:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.547966:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.547966:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.547967:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.547967:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.547967:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:20.0:1619677448.547968:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:12.0:1619677448.547968:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.547969:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:20.0:1619677448.547971:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.547973:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.547975:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.547976:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.547977:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.547979:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.547979:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.547980:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.547981:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.547984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.547989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677448.547992:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.547995:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677448.547996:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677448.547996:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.547997:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.547998:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:20.0:1619677448.547998:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.547999:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.547999:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:26.0:1619677448.548001:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.548002:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.548004:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.548005:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.548006:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.548007:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.548008:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.548012:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.548013:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.548015:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.548016:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.548017:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.548018:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.548019:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.548020:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.548021:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.548022:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.548023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:4.2:1619677448.548027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.548032:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.548033:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.548034:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.548036:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:10.0:1619677448.548036:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.548037:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.548037:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.548037:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.548038:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.548038:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.548234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.548242:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.548245:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.548247:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.548249:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.548249:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.548250:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.548251:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.548253:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.548253:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.548254:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.548257:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.548259:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.548261:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.548262:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.548262:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.548264:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.548264:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.548265:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.548266:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.548270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.548276:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.548278:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.548279:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.548280:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.548282:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.548283:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.548286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.548293:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.548300:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.548300:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.548303:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.548304:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.548306:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.548307:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.548308:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.548309:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.548310:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.548313:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.548314:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.548317:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.548318:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.548319:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.548321:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.548321:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.548322:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.548322:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.548327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.548329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.548332:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.548335:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:6.0:1619677448.548335:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:20.0:1619677448.548337:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.548338:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677448.548338:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677448.548339:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:20.0:1619677448.548339:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.548339:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.548341:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:6.0:1619677448.548342:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.548343:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.548344:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.548345:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.548346:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.548346:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:20.0:1619677448.548349:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.548351:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.548354:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.548354:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.548355:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.548357:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.548357:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.548359:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.548360:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.548361:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.548363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.548365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677448.548370:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.548371:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677448.548373:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.548373:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.548373:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.548374:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:12.0:1619677448.548375:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.548375:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.548377:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.548378:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.548467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.548475:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.548478:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.548480:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.548482:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.548482:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.548483:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.548484:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.548486:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.548487:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.548488:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.548491:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.548493:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.548495:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.548496:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.548497:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.548498:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.548499:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.548500:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.548500:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.548505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.548511:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.548513:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.548514:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.548516:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.548518:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.548519:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.548564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.548571:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.548574:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.548575:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.548576:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.548579:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.548581:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.548582:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.548583:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.548584:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.548585:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.548588:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.548590:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.548592:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.548592:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.548593:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.548595:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.548595:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.548595:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.548596:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.548600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.548607:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.548609:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.548610:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.548611:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.548613:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.548614:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.548687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.548694:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.548697:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.548698:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.548700:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.548702:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.548704:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.548705:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.548707:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.548708:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.548709:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.548712:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.548714:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.548717:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.548718:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.548719:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.548721:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.548721:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.548722:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.548722:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.548726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.548733:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.548735:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.548736:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.548738:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.548739:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.548741:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.548763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.2:1619677448.548764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.548770:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.548772:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677448.548773:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.548774:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:26.0:1619677448.548775:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.548775:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:18.0:1619677448.548777:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.548778:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.548780:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.548780:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.548783:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.548783:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:21.2:1619677448.548785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:18.0:1619677448.548785:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.548786:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.548786:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.548788:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.548788:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.548789:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.548789:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.548790:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677448.548791:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:22.0:1619677448.548793:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.548794:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.548797:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.548797:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.548799:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.548800:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.548800:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:22.0:1619677448.548801:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:10.0:1619677448.548802:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:22.0:1619677448.548803:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96713499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.548804:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.548804:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.548805:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.548805:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.548805:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.548806:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.548806:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.548806:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.548807:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:18.0:1619677448.548808:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.548808:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.548809:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.548809:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:22.0:1619677448.548810:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.548810:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:10.0:1619677448.548810:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677448.548811:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.548811:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:10.0:1619677448.548811:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677448.548812:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:25.2:1619677448.548814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:22.0:1619677448.548814:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.548815:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.548816:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.548816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:25.2:1619677448.548817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:22.0:1619677448.548817:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.548818:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.548819:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:26.0:1619677448.548820:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:22.0:1619677448.548820:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.548820:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677448.548822:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:2.0:1619677448.548822:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:10.0:1619677448.548823:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.548824:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:26.0:1619677448.548824:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.548824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.548824:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:21.2:1619677448.548825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677448.548825:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:26.0:1619677448.548826:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.548826:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:30.0:1619677448.548827:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.548827:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.548829:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.548829:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677448.548830:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.548831:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.548831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:6.0:1619677448.548831:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:5.0:1619677448.548833:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677448.548834:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.548835:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677448.548835:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.0:1619677448.548835:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.548836:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.548837:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.548838:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.548838:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.548840:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.548841:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.548841:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:5.2:1619677448.548841:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:10.0:1619677448.548842:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.548843:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.0:1619677448.548845:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.548846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:18.0:1619677448.548847:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677448.548849:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:18.0:1619677448.548850:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.548850:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677448.548852:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:18.0:1619677448.548854:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.548854:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.548855:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.2:1619677448.548857:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:18.0:1619677448.548857:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.548859:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.548861:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.548861:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.548862:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.548862:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:20.0:1619677448.548863:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677448.548863:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.548864:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.548864:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.548865:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713497] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.548867:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.548869:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.548869:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.548870:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:18.0:1619677448.548872:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.2:1619677448.548872:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1200] (20)++
00000400:00000200:20.0:1619677448.548874:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.548875:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.548876:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:5.0:1619677448.548876:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.548877:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677448.548877:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.548877:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:18.0:1619677448.548879:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.548879:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677448.548880:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.548880:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:18.0:1619677448.548880:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:5.0:1619677448.548880:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.548881:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:18.0:1619677448.548881:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:17.0:1619677448.548881:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:5.0:1619677448.548881:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:1.0:1619677448.548881:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:18.0:1619677448.548882:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677448.548883:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:5.0:1619677448.548883:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96713496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.548884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677448.548884:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.548885:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.548886:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000400:00000200:1.0:1619677448.548887:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.548888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:1.0:1619677448.548888:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:5.0:1619677448.548889:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.548890:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677448.548890:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96713495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.548891:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677448.548891:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.548892:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:1.0:1619677448.548892:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.548894:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.548894:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:1.0:1619677448.548894:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:31.0:1619677448.548895:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:13.0:1619677448.548895:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.548895:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.548895:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677448.548896:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:2.0:1619677448.548896:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.0:1619677448.548897:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000400:00000200:13.0:1619677448.548897:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.548897:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.548897:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.548898:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:20.0:1619677448.548898:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:31.0:1619677448.548899:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.548899:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677448.548899:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677448.548900:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.548901:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677448.548901:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:25.2:1619677448.548902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.548903:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.548903:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 984 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:31.0:1619677448.548904:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.0:1619677448.548904:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.548904:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.548906:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.548906:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.548907:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.548907:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.548907:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:12.0:1619677448.548908:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.548909:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.548909:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677448.548909:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:1.0:1619677448.548909:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:1.0:1619677448.548910:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:12.0:1619677448.548911:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.548911:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.548911:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:12.0:1619677448.548912:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.2:1619677448.548912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:12.0:1619677448.548913:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.548913:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677448.548916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.548921:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.548923:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.548924:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.548925:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.548925:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.548926:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.548927:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.548928:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.548929:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:5.0:1619677448.548929:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.548931:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:4.2:1619677448.548931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677448.548932:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:8.2:1619677448.548933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.548938:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677448.548940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000800:00000200:26.0:1619677448.548941:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.548941:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.548943:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.548944:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000400:00000200:26.0:1619677448.548946:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.548946:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.548947:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:20.0:1619677448.548947:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.548947:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000400:00000200:26.0:1619677448.548948:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.548948:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.548948:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.548949:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:26.0:1619677448.548950:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.548950:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.548950:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677448.548951:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.548951:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (21)++
00000400:00000200:26.0:1619677448.548952:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.548952:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.548953:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:26.0:1619677448.548956:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.548958:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.548960:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.548961:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.548962:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.548963:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.548964:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.548965:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.548965:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.548969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.548976:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.548978:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.548979:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.548981:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.548982:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.548983:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.548987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677448.548993:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.548996:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.548997:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.548998:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.548998:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.549000:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.549042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.549050:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.549053:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.549055:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.549057:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.549057:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.549058:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.549059:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.549061:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.549062:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.549063:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.549067:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.549068:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.549072:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.549072:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.549073:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.549075:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.549075:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.549077:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.549078:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.549079:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.549080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.549086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.549092:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.549094:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:31.0:1619677448.549095:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:20.0:1619677448.549095:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.549097:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:20.0:1619677448.549097:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.549098:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677448.549098:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.549100:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677448.549100:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.549149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.549156:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.549159:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.549160:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.549162:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.549164:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.549166:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.549168:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.549169:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.549170:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.549171:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.549174:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.549176:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.549179:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.549179:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.549180:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.549182:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.549182:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.549184:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.549185:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.549185:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.549188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.549191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.549195:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.549198:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677448.549198:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:17.0:1619677448.549199:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.549199:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.549200:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677448.549201:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.549201:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.549202:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677448.549205:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.549211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.549218:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.549220:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.549221:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.549221:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.549222:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.549223:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.549302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.549310:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.549313:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.549314:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.549316:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.549319:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.549320:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.549321:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.549322:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.549323:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.549324:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.549327:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.549329:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.549332:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.549332:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.549333:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.549334:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.549335:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.549336:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.549336:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:4.2:1619677448.549340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.2:1619677448.549341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.549347:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.549349:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.549350:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.549350:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:26.0:1619677448.549351:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.549353:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.549353:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.549354:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.549356:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677448.549356:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.549356:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.549357:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.549359:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.549359:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.549360:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.549362:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:26.0:1619677448.549364:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.549365:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.549368:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.549369:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.549369:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.549371:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.549371:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.549373:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.549374:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.549375:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.549378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.549380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.549386:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.549387:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.549389:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677448.549389:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.549390:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677448.549390:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.549390:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.549392:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.549392:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.549396:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.549398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.549405:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.549407:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.549408:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.549409:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.549410:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.549410:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.549448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.549457:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.549459:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.549461:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.549464:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.549465:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.549466:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.549467:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.549469:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.549470:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.549470:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.549473:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.549475:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.549478:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.549479:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.549480:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.549481:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.549482:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.549483:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.549484:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.549488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.549496:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.549498:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.549499:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.549501:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.549501:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.549503:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.549504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.549511:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.549514:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.549516:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.549517:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.549520:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.549522:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:24.0:1619677448.549523:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.549524:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.549525:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.549526:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:24.0:1619677448.549529:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.549531:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:24.0:1619677448.549533:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.549534:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:24.0:1619677448.549535:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:24.0:1619677448.549536:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:24.0:1619677448.549537:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:5.2:1619677448.549537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:24.0:1619677448.549538:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.549539:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.549539:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.2:1619677448.549542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.549545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:10.0:1619677448.549546:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.549548:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.549549:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.549550:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.549550:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.549551:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.549552:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:26.0:1619677448.549553:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.549553:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.549554:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:10.0:1619677448.549555:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:8.0:1619677448.549555:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:10.0:1619677448.549556:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.549556:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:10.0:1619677448.549557:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.549558:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.549559:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.549560:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.549561:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.549562:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:10.0:1619677448.549563:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.549565:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.549567:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.549568:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.549568:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.549570:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.549570:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.549571:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.549573:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.549573:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.549576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.2:1619677448.549579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.549582:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.549585:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.549586:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.549586:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:30.0:1619677448.549587:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.549588:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:31.0:1619677448.549589:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.549589:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.549590:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.549590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.549591:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677448.549596:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.549599:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.549600:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.549602:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.549604:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.549605:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.549607:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.549608:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.549609:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.549610:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.549613:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.549614:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.549617:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.549618:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.549618:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.549620:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.549621:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.549622:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.549622:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.549626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.549634:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.549636:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.549637:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.549639:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.549641:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.549642:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.549831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.549836:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.549839:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.549840:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.549841:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.549844:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.549845:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.549846:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.549848:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.549849:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.549849:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.549852:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.549854:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.549857:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.549857:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.549858:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.549860:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.549860:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.549861:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.549861:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.549866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.549874:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.549876:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.549877:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.549879:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.549880:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.549881:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.549884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.549892:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.549894:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.549895:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.549896:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:21.2:1619677448.549897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677448.549897:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.549898:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677448.549904:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.549907:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.549909:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.549911:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.549912:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.549914:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.549915:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.549916:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.549917:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.549919:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.549922:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.549924:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.549926:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.549927:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.549927:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.549929:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.549930:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.549931:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.549931:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.549934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.549940:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.549943:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:4.2:1619677448.549943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:10.0:1619677448.549944:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.549946:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.549946:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.549947:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.549949:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.549949:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.549950:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.549951:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.549954:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.549955:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.549956:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.549958:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.549958:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.549959:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.549962:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.549964:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.549967:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.549967:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.549968:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.549969:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.549970:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.549970:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.549971:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.549975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.549983:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.549986:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.549987:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.549987:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.549989:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.549992:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.549997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.550005:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.550008:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.550010:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.550012:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.550012:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.550014:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.550015:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.550016:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.550017:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.550018:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.550021:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.550023:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.550025:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.550026:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.550027:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.550029:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.550029:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.550030:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.550031:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.550034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.550042:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.550045:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.550046:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.550047:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.550048:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.550050:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.550101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.550107:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.550110:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.550110:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.550112:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.550115:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.550117:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.550118:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.550120:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.550120:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.550122:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.550125:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.550127:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.550130:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.550130:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.550131:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.550133:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.550133:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.550134:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.550134:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677448.550135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.550138:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:9.2:1619677448.550139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:20.0:1619677448.550140:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.550142:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.550143:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.550144:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.550145:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:20.0:1619677448.550146:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.550147:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.550148:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:20.0:1619677448.550148:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.550149:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.550149:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.550149:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:20.0:1619677448.550151:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677448.550153:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.550155:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.550156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:20.0:1619677448.550157:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677448.550158:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.550160:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:20.0:1619677448.550160:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.550160:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.550161:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677448.550162:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677448.550162:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.550163:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (21)++
00000800:00000200:20.0:1619677448.550163:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.550164:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.550164:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:24.0:1619677448.550165:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:20.0:1619677448.550165:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:24.0:1619677448.550166:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:25.2:1619677448.550169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.550176:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.550179:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.550180:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.550180:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.550182:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.550184:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.550499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.550505:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.550508:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.550509:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.550511:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.550514:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.550515:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.550517:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.550518:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.550519:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.550520:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.550523:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.550525:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.550528:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.550529:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.550529:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:4.2:1619677448.550530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677448.550531:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:21.2:1619677448.550532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:17.0:1619677448.550532:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.550533:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.550534:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.550534:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.550536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:9.2:1619677448.550537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:22.0:1619677448.550538:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.550539:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.2:1619677448.550539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.550542:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:10.0:1619677448.550542:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:6.0:1619677448.550542:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677448.550543:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677448.550543:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:8.0:1619677448.550545:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:1.0:1619677448.550545:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:22.0:1619677448.550546:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.550547:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:10.0:1619677448.550547:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.550547:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677448.550547:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:24.0:1619677448.550548:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:22.0:1619677448.550548:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.550548:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677448.550548:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.550549:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.550549:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.550550:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677448.550550:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.550551:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.550551:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.550552:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.550552:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.550553:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.550554:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:22.0:1619677448.550555:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.550555:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:10.0:1619677448.550555:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.550556:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.550557:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677448.550557:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.550557:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.550559:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.550559:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.550560:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.550560:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.550561:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.550566:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.550566:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.550567:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.550568:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.550569:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.550570:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.550572:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.550572:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.550573:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.550573:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:10.0:1619677448.550573:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.550574:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.550574:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.550575:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:10.0:1619677448.550575:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.550576:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.550577:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.550577:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:10.0:1619677448.550577:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.550578:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677448.550578:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.550578:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:10.0:1619677448.550578:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.550579:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:10.0:1619677448.550579:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677448.550580:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.550580:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.550582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:9.2:1619677448.550583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:25.2:1619677448.550584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677448.550590:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677448.550591:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677448.550591:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.550593:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:1.0:1619677448.550593:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.550594:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.550594:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:2.0:1619677448.550594:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.550594:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.550595:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.550595:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:6.0:1619677448.550595:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.550597:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.550598:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.550599:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.550601:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677448.550602:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677448.550602:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.550613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.550620:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.550623:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.550624:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.550626:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.550629:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.550630:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.550632:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.550633:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.550634:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.550635:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.550638:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.550640:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.550643:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.550644:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.550645:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.550647:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:4.2:1619677448.550647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677448.550648:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.550648:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.550649:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.550653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.550654:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.550657:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.550658:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.550660:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.550661:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:24.0:1619677448.550662:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.550663:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:18.0:1619677448.550664:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:24.0:1619677448.550665:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96713474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.550665:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.550665:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:24.0:1619677448.550666:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.550667:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.550667:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.550668:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.550671:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.550671:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.550672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.550673:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.550677:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.550677:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.550678:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.550679:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.550680:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.550680:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.550681:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.550681:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677448.550681:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.550683:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.550683:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.550684:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.550684:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.2:1619677448.550687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.550695:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:21.2:1619677448.550697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:17.0:1619677448.550698:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.550699:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.550699:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.550701:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.550703:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677448.550703:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677448.550706:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.550707:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.550708:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.550710:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.550712:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:21.2:1619677448.550713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:18.0:1619677448.550713:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.550715:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.550716:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.550717:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.550720:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677448.550720:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:18.0:1619677448.550722:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677448.550723:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.550724:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:18.0:1619677448.550725:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.550726:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.550726:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:8.0:1619677448.550726:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.550728:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.550728:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.550729:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:8.0:1619677448.550729:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.550730:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:8.0:1619677448.550731:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.550732:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96713472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.550734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:8.0:1619677448.550734:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.550735:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.550736:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.550739:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.550740:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677448.550741:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677448.550743:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.550744:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:8.0:1619677448.550744:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.550744:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:22.0:1619677448.550745:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.550745:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677448.550746:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:22.0:1619677448.550747:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:8.0:1619677448.550747:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.550748:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.550748:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.550749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.550753:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:9.2:1619677448.550753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677448.550755:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.550757:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.550758:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.550758:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.550759:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.550760:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677448.550761:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.550764:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.550765:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.550765:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.550767:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.550771:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.550994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.551000:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.551003:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.551003:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.551006:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.551008:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.551010:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.551011:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.551013:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.551014:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.551015:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.551018:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.551020:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.551023:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.551024:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.551024:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.551026:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.551027:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.551028:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.551028:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.551033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.551041:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.551044:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.551044:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.551046:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.551048:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.551049:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.551216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.551221:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.551224:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.551225:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.551227:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.551229:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.551230:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.551232:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.551233:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.551234:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.551235:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.551238:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.551240:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.551242:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.551243:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.551244:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.551245:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.551246:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.551246:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.551247:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.551251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.551259:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.551262:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.551263:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.551263:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.551265:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.551267:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.551336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.551344:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.551346:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:5.2:1619677448.551348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:30.0:1619677448.551349:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.551351:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.551352:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.551353:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.551354:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.551354:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:30.0:1619677448.551356:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.551357:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:30.0:1619677448.551358:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.551358:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:30.0:1619677448.551359:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.551361:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.551362:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.551363:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.551364:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.551365:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.551366:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.551367:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.551368:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.551369:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.551369:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:2.0:1619677448.551369:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.551370:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.551371:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.551372:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.551373:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.551373:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:2.0:1619677448.551373:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.551375:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.2:1619677448.551378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.551378:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.551379:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677448.551380:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (23)++
00000800:00000200:2.0:1619677448.551382:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:2.0:1619677448.551382:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.551383:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.551384:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677448.551387:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)++
00000800:00000200:5.0:1619677448.551389:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (24)--
00000400:00000200:5.0:1619677448.551390:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.551392:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.551393:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)++
00000800:00000200:20.0:1619677448.551394:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (24)--
00000400:00000200:20.0:1619677448.551395:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.551395:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.551396:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.551396:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.551399:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.551550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.551558:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.551561:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.551562:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.551564:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.551567:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.551569:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.551570:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.551571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:10.0:1619677448.551571:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.551572:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.551573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:10.0:1619677448.551573:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.551577:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.551579:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.551580:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.551580:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.551582:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:31.0:1619677448.551583:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:10.0:1619677448.551583:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.551584:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677448.551584:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.551584:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:31.0:1619677448.551586:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.551586:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.551586:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.551587:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677448.551587:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.551588:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.551589:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.551590:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.551590:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.551591:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.551591:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.551592:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.551593:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.551593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.551593:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.551593:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:31.0:1619677448.551594:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.551594:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:31.0:1619677448.551595:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.551595:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.551597:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.551599:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.551601:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.551601:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.551601:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:26.0:1619677448.551602:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.551602:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.551603:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.551604:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.551604:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:26.0:1619677448.551605:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:13.0:1619677448.551605:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.551606:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.551606:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.551606:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:31.0:1619677448.551607:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.551607:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:13.0:1619677448.551607:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.551608:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.551609:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.551609:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.551610:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677448.551610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677448.551611:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.551613:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.551613:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.2:1619677448.551613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677448.551617:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:21.2:1619677448.551619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.551619:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.551619:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677448.551620:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.551621:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:17.0:1619677448.551622:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:13.0:1619677448.551624:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677448.551624:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:1.0:1619677448.551624:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.551625:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.551625:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677448.551625:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.551626:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677448.551626:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677448.551627:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.551629:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:17.0:1619677448.551629:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.551630:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677448.551630:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:13.0:1619677448.551630:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.551631:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:17.0:1619677448.551631:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.551633:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.551634:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.551635:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.551638:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.551640:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.551642:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.551642:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.551643:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.551644:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.551645:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.551645:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.551646:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.551651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.551658:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.551660:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.551661:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.551662:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.551663:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.551664:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.551756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.551762:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.551765:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.551766:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.551768:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.551771:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.551772:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.551774:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.551775:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.551776:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.551777:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.551780:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.551782:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.551785:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.551785:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.551786:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.551788:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.551789:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.551789:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.551790:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.551794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677448.551795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677448.551802:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.551802:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.551805:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:10.0:1619677448.551805:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.551806:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677448.551806:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.551806:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:10.0:1619677448.551807:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.551809:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.551811:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.551812:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:10.0:1619677448.551812:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.551814:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.551815:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.551816:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.551817:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.551820:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.551822:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.551825:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.551826:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.551826:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.551828:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.551828:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.551829:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.551830:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.551834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.551837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:22.0:1619677448.551842:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.551843:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677448.551845:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.551846:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.551846:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.551846:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:22.0:1619677448.551848:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.551848:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.551849:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.551851:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:5.0:1619677448.551851:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.551853:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.551854:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.551855:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.551857:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.551857:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.551861:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.551863:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.551866:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.551867:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.551868:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.551870:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.551871:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.551871:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.551872:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.2:1619677448.551874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:25.2:1619677448.551875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.551877:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.551880:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.551882:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.551882:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.551882:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.551885:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.551885:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:20.0:1619677448.551886:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677448.551886:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.551888:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.551888:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.551889:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.551889:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677448.551890:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.551891:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:12.0:1619677448.551893:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:20.0:1619677448.551895:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.551897:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.551900:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.551901:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.551901:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.551903:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.551903:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.551904:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.551905:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677448.551909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.551917:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.551919:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.551920:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.551921:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.551923:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.551925:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.551927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.551935:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.551937:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.551938:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.551939:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.551940:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.551940:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.552091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.552096:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.552098:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.552099:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.552101:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.552102:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.552104:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.552105:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.552107:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.552107:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.552109:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.552112:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.552114:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.552116:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.552117:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.552118:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.552120:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.552120:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.552121:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.552121:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.552126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.552133:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.552136:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.552137:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.552137:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.552139:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.552141:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.552146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:20.2:1619677448.552150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:22.0:1619677448.552153:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.2:1619677448.552155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:22.0:1619677448.552156:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.552156:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:22.0:1619677448.552157:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.552157:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.552158:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.552159:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677448.552159:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677448.552160:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:30.0:1619677448.552162:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:1.0:1619677448.552162:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.552165:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:1.0:1619677448.552165:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.552166:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677448.552168:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.552169:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.552169:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.552170:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.552171:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.552172:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.552172:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.552174:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.552176:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.552176:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.552178:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.552179:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.552179:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.552180:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.552181:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.552182:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:1.0:1619677448.552182:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:30.0:1619677448.552184:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.552184:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677448.552184:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677448.552185:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:30.0:1619677448.552186:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.2:1619677448.552186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677448.552186:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:30.0:1619677448.552188:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.552189:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.552190:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.2:1619677448.552190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.552192:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.552192:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.552193:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.552194:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.552195:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.552196:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677448.552196:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.552197:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.2:1619677448.552197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:26.0:1619677448.552199:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.552200:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:26.0:1619677448.552201:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.552202:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:26.0:1619677448.552203:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:5.0:1619677448.552203:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.552204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677448.552205:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96713456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.552205:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.552206:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.552208:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.552208:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.552208:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.552209:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.552210:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.552210:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.552211:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.552212:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.552212:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.552213:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.552213:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.552213:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.552213:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.552214:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:20.0:1619677448.552214:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.552215:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677448.552215:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.0:1619677448.552215:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.552216:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.552218:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.552218:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677448.552218:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:26.0:1619677448.552219:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.552220:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.552222:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.552222:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.552223:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.552224:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.552228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.552235:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.552238:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.552239:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.552239:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.552241:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.552244:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.552353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.552361:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.2:1619677448.552363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:31.0:1619677448.552364:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.552366:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.552368:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.552369:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.552370:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:31.0:1619677448.552371:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:2.0:1619677448.552372:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:31.0:1619677448.552373:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.552374:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.552374:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:31.0:1619677448.552376:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.552377:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.552377:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.552379:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.552381:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.552381:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.552382:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.552383:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.552384:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.552384:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.552385:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.552387:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.552388:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.552388:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:2.0:1619677448.552389:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.552390:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.552391:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.552391:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677448.552391:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.552392:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677448.552393:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.552394:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.552394:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.552395:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:25.2:1619677448.552396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.552396:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.552397:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.552397:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.552403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677448.552404:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.552407:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.552408:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.552409:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.552410:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:13.0:1619677448.552412:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.552412:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.552413:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.552413:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.552414:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:25.2:1619677448.552415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:18.0:1619677448.552415:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.552417:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677448.552421:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.552424:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.552425:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.552425:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.552426:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.552427:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.552510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.552516:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.552519:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.552520:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.552522:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.552525:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.552526:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.552527:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.552528:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.552530:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.552531:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.552534:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.552536:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.552539:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.552540:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.552540:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.552542:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.552542:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.552544:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.552545:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.552545:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.552548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.552550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.552555:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:1.0:1619677448.552555:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677448.552558:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:1.0:1619677448.552558:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.552558:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677448.552559:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:17.0:1619677448.552559:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.552560:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.552562:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.552565:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.2:1619677448.552770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.552777:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.552780:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.552781:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.552783:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.552786:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.552787:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.552789:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.552790:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.552791:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.552792:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.552794:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.552796:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.552797:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.552798:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.552799:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.552800:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.552801:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.552802:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.552803:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.552804:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677448.552804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:25.2:1619677448.552807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.2:1619677448.552810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.552813:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.552815:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.552815:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.552816:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.552817:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.552817:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.552819:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.552819:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.552820:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.552821:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:10.0:1619677448.552821:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.552822:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:26.0:1619677448.552823:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.552823:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.552824:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.552824:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.552825:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.552825:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:26.0:1619677448.552826:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.552826:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:26.0:1619677448.552830:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.552831:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.552834:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.552835:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.552835:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.552837:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.552837:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.552838:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.552839:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.552840:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.552843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.552845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.552851:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.552852:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.552853:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677448.552854:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677448.552854:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.552855:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.552855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:10.0:1619677448.552855:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.552856:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.552858:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.552860:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.552861:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677448.552862:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.552862:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.552864:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.552866:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.552868:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.552869:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.552870:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.552871:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.552872:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.552876:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.552878:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.552881:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.552881:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.552882:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.552884:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.552884:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.552885:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.552887:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.552887:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:25.2:1619677448.552889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:21.2:1619677448.552892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.552896:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:22.0:1619677448.552897:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:26.0:1619677448.552898:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:22.0:1619677448.552899:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:10.0:1619677448.552899:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.552900:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.552900:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.552901:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:26.0:1619677448.552902:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.552905:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.552998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677448.553005:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677448.553007:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677448.553008:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677448.553010:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677448.553012:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.553013:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677448.553015:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.553016:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.553017:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.553018:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.2:1619677448.553018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:24.0:1619677448.553021:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677448.553023:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677448.553025:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.553026:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677448.553026:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:4.2:1619677448.553026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.553027:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.553029:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:24.0:1619677448.553029:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677448.553030:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677448.553030:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000400:00000200:30.0:1619677448.553031:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.553031:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:24.0:1619677448.553032:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677448.553033:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.553033:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.553034:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:25.2:1619677448.553034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677448.553034:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.553035:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:30.0:1619677448.553036:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.553037:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.553038:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.553039:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.553039:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.553040:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.553041:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:24.0:1619677448.553042:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.553043:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96713447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.553044:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.553044:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:24.0:1619677448.553045:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.553045:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.553046:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.553046:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.553046:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.553047:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:24.0:1619677448.553048:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.553050:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.553051:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.553051:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677448.553052:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:24.0:1619677448.553052:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.553053:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.553054:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677448.553054:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.553055:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.553055:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677448.553057:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.553057:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.553059:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.2:1619677448.553059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (23)++
00000800:00000200:24.0:1619677448.553061:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (24)--
00000800:00000200:24.0:1619677448.553061:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.553062:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.553063:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677448.553068:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000800:00000200:17.0:1619677448.553071:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (24)++
00000800:00000200:1.0:1619677448.553071:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000800:00000200:17.0:1619677448.553072:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000400:00000200:1.0:1619677448.553072:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.553073:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.553074:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.553075:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.553078:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.553079:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.553079:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.553104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.553109:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.553112:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.553113:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.553115:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.553117:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.553118:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.553119:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.553121:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.553122:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.553123:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.553126:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.553128:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.553131:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.553132:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.553132:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.553134:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.553135:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.553135:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.553136:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.553140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.553147:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.553149:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.553150:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.553151:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.553153:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.553156:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.553157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:9.2:1619677448.553160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:2.0:1619677448.553163:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:24.0:1619677448.553166:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:2.0:1619677448.553166:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.553167:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:5.2:1619677448.553168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:2.0:1619677448.553169:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:24.0:1619677448.553170:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.553170:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.553170:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:2.0:1619677448.553171:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.553172:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.553172:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:2.0:1619677448.553173:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:18.0:1619677448.553174:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:2.0:1619677448.553174:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.553176:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.553176:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.553177:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.553178:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.553181:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000400:00000200:18.0:1619677448.553184:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.553184:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.553185:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.553186:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.553186:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.553187:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677448.553188:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.553189:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.553189:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.553191:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.553191:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.553192:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.553195:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.553196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.553196:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.553197:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.553197:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.553198:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677448.553199:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.553200:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.553200:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.553201:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000800:00000200:2.0:1619677448.553201:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:18.0:1619677448.553202:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (25)--
00000800:00000200:18.0:1619677448.553203:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.553203:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (25)--
00000400:00000200:2.0:1619677448.553203:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.553204:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.553204:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677448.553205:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.553205:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:20.2:1619677448.553207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677448.553207:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:9.2:1619677448.553208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.553213:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:4.2:1619677448.553214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677448.553216:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677448.553216:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:18.0:1619677448.553217:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.553218:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.553218:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.553219:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677448.553219:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:18.0:1619677448.553220:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.553220:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.553221:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.553221:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (23)++
00000800:00000200:18.0:1619677448.553222:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:10.0:1619677448.553222:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677448.553222:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:2.0:1619677448.553222:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.553223:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:5.0:1619677448.553223:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.553223:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:5.0:1619677448.553224:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.553225:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.553226:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.553226:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:5.0:1619677448.553227:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.553229:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.553231:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.553232:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.553233:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:5.0:1619677448.553233:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.553235:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.553236:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.553236:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.553238:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.553238:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.553238:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.553239:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.553240:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.553241:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:10.0:1619677448.553242:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.553242:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.553243:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:10.0:1619677448.553244:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.553244:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.553246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.553247:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.553248:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.2:1619677448.553249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.553249:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.553250:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.553251:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.553251:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677448.553252:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:10.0:1619677448.553252:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:12.0:1619677448.553253:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.553254:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.553254:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.553255:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.553255:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:25.2:1619677448.553256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:12.0:1619677448.553256:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.553258:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.553260:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.553260:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677448.553264:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.553267:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.553268:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.553268:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.553271:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.553273:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.553275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.553282:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.553284:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.553285:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.553286:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.553287:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.553287:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.553398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380000] (20)++
00000800:00000200:31.0:1619677448.553405:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380000] (21)++
00000800:00000200:31.0:1619677448.553408:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.48@o2ib3
00000400:00000200:31.0:1619677448.553411:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.48@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.553413:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380000] (22)--
00000400:00000200:31.0:1619677448.553414:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.553416:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677448.553417:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.553418:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.553419:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.553420:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677448.553424:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.48@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.553425:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677448.553427:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.553428:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.553429:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.553430:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.553431:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380000] (21)++
00000800:00000200:31.0:1619677448.553432:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380000] (22)--
00000800:00000200:31.0:1619677448.553432:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380000] (21)--
00000800:00000200:9.2:1619677448.553436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.553444:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.553446:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.553447:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.553448:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.553449:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.553452:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.553457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380000] (20)++
00000800:00000200:30.0:1619677448.553464:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380000] (21)++
00000800:00000200:30.0:1619677448.553467:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.48@o2ib3
00000400:00000200:30.0:1619677448.553469:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.48@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.553471:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380000] (22)--
00000400:00000200:30.0:1619677448.553471:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.553473:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.553474:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.553475:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.553475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380000] (21)++
00000400:00000200:30.0:1619677448.553476:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.553477:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.553480:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.48@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.553482:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380000] (22)++
00000800:00000200:30.0:1619677448.553482:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.553484:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.48@o2ib3
00000800:00000200:30.0:1619677448.553484:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.553485:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.553485:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:31.0:1619677448.553486:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.48@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.553487:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:31.0:1619677448.553488:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.553488:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380000] (23)++
00000400:00000200:31.0:1619677448.553490:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:31.0:1619677448.553491:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.553492:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.553492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:31.0:1619677448.553493:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.553494:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:30.0:1619677448.553495:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871380000] (24)++
00000400:00000200:31.0:1619677448.553497:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.48@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.553497:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380000] (25)--
00000800:00000200:30.0:1619677448.553497:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380000] (24)--
00000800:00000200:31.0:1619677448.553498:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677448.553499:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380000] (23)--
00000800:00000200:13.0:1619677448.553499:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:31.0:1619677448.553501:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.553502:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:31.0:1619677448.553502:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:13.0:1619677448.553502:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:21.2:1619677448.553503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380000] (22)++
00000400:00000200:13.0:1619677448.553503:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.553504:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:13.0:1619677448.553504:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.553505:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380000] (23)++
00000800:00000200:31.0:1619677448.553506:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380000] (24)--
00000800:00000200:31.0:1619677448.553506:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380000] (23)--
00000800:00000200:13.0:1619677448.553506:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.553507:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.553508:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380000] (22)++
00000800:00000200:31.0:1619677448.553510:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871380000] (23)--
00000800:00000200:31.0:1619677448.553510:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380000] (22)--
00000800:00000200:8.2:1619677448.553510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677448.553512:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380000] (21)--
00000800:00000200:25.2:1619677448.553513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:13.0:1619677448.553519:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.553519:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:13.0:1619677448.553522:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.553522:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.553523:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:13.0:1619677448.553523:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:1.0:1619677448.553523:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.553524:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.553524:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:1.0:1619677448.553525:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.553526:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677448.553529:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.553531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.553539:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.553542:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.553543:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.553543:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.553544:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.553544:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.553548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:21.2:1619677448.553552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380000] (20)++
00000800:00000200:31.0:1619677448.553557:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.553559:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:30.0:1619677448.553560:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380000] (21)++
00000400:00000200:31.0:1619677448.553561:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.553561:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.48@o2ib3
00000400:00000200:30.0:1619677448.553563:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.48@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.553564:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.553565:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.553566:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.553568:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.553568:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.553569:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.553571:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.553572:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.553573:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:31.0:1619677448.553574:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.553574:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.553575:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.553576:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.553577:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.553580:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.553580:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.48@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.553581:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.553581:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.553582:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.553583:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.553583:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.553584:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.553585:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.553585:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.553585:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.553586:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.553586:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.553588:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380000] (22)--
00000800:00000200:30.0:1619677448.553588:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.553588:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380000] (21)++
00000800:00000200:30.0:1619677448.553589:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380000] (22)--
00000800:00000200:8.2:1619677448.553589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.553590:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380000] (21)--
00000800:00000200:9.2:1619677448.553594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677448.553597:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.553599:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.553600:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.553602:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.553602:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:5.0:1619677448.553603:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677448.553604:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:20.0:1619677448.553605:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:18.0:1619677448.553606:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.553606:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.553608:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.553610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380000] (20)++
00000800:00000200:18.0:1619677448.553611:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.553612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.553618:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380000] (21)++
00000800:00000200:2.0:1619677448.553620:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.553621:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.48@o2ib3
00000400:00000200:31.0:1619677448.553623:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.48@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.553623:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.553623:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.553624:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380000] (22)--
00000800:00000200:2.0:1619677448.553624:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.553625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677448.553625:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.553626:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:31.0:1619677448.553627:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.553628:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.553630:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.553631:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.553632:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.553632:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:31.0:1619677448.553633:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.553634:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.553637:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.48@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.553637:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.553639:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.553639:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.553641:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.553641:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:31.0:1619677448.553642:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:30.0:1619677448.553642:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.553643:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.553644:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:30.0:1619677448.553644:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.553644:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.553645:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:31.0:1619677448.553646:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380000] (21)++
00000800:00000200:31.0:1619677448.553647:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380000] (22)--
00000800:00000200:31.0:1619677448.553648:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380000] (21)--
00000400:00000200:30.0:1619677448.553648:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.553649:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.553650:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:25.2:1619677448.553650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.553652:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.553653:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:30.0:1619677448.553654:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:30.0:1619677448.553655:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:30.0:1619677448.553656:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.553656:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677448.553656:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.553657:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.553659:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.553660:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.553660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:12.0:1619677448.553661:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.553663:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.553664:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677448.553667:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.553669:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.553670:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.553670:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.553672:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.553672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380000] (20)++
00000800:00000200:24.0:1619677448.553674:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:31.0:1619677448.553679:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380000] (21)++
00000800:00000200:31.0:1619677448.553682:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.48@o2ib3
00000400:00000200:31.0:1619677448.553684:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.48@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.553686:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.553686:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380000] (22)--
00000400:00000200:31.0:1619677448.553688:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.553689:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.553690:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.553691:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.553692:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.553695:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.48@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.553696:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.553698:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.553699:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.553699:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.553701:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.553702:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380000] (21)++
00000800:00000200:31.0:1619677448.553703:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871380000] (22)++
00000800:00000200:31.0:1619677448.553705:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380000] (23)--
00000800:00000200:31.0:1619677448.553705:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380000] (22)--
00000800:00000200:25.2:1619677448.553707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:21.2:1619677448.553710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380000] (21)++
00000800:00000200:30.0:1619677448.553715:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.553717:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.553717:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.553719:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380000] (22)++
00000800:00000200:31.0:1619677448.553719:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871380000] (23)--
00000400:00000200:30.0:1619677448.553719:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.553720:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380000] (22)--
00000800:00000200:30.0:1619677448.553721:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.553722:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.553723:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380000] (21)++
00000800:00000200:30.0:1619677448.553725:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.48@o2ib3
00000400:00000200:30.0:1619677448.553727:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.48@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.553728:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.553729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.553730:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380000] (22)--
00000400:00000200:30.0:1619677448.553730:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:31.0:1619677448.553731:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000400:00000200:30.0:1619677448.553731:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.553732:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.553733:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.553733:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.553734:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.553734:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.553734:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.553735:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.553736:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:30.0:1619677448.553737:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.48@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.553738:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.553741:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.553742:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.553743:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.553744:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.553745:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380000] (21)++
00000800:00000200:30.0:1619677448.553746:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380000] (22)--
00000800:00000200:30.0:1619677448.553746:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380000] (21)--
00000800:00000200:8.2:1619677448.553750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.553750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.553757:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677448.553759:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677448.553760:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.553761:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.553761:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:26.0:1619677448.553762:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.553762:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.553764:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.553765:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:26.0:1619677448.553766:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.553767:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:5.0:1619677448.553767:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:26.0:1619677448.553768:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.553770:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.553771:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.553772:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.553775:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.553776:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.553779:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.553780:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.553780:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.553782:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.553783:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.553783:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.553784:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.553788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.553790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919078fb5400] (20)++
00000800:00000200:20.0:1619677448.553795:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:18.0:1619677448.553796:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919078fb5400] (21)++
00000800:00000200:21.2:1619677448.553798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380000] (20)++
00000800:00000200:20.0:1619677448.553798:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.553799:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.553799:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.6@o2ib3
00000800:00000200:2.0:1619677448.553800:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (22)--
00000400:00000200:20.0:1619677448.553801:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.553802:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.6@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.553803:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.553804:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:18.0:1619677448.553804:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.553805:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:31.0:1619677448.553806:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380000] (21)++
00000400:00000200:18.0:1619677448.553807:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.553808:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.553809:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.48@o2ib3
00000400:00000200:18.0:1619677448.553809:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.553810:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:31.0:1619677448.553812:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.48@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.553813:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380000] (22)--
00000400:00000200:18.0:1619677448.553814:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.553815:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.553816:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:31.0:1619677448.553817:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677448.553818:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.553819:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.553820:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.553820:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:18.0:1619677448.553820:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:31.0:1619677448.553821:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.553822:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.553822:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000400:00000200:31.0:1619677448.553826:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.48@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.553827:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919078fb5400] (21)++
00000800:00000200:31.0:1619677448.553828:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.553828:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919078fb5400] (22)--
00000800:00000200:8.2:1619677448.553828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:18.0:1619677448.553829:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (21)--
00000800:00000200:31.0:1619677448.553831:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.553831:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:4.2:1619677448.553831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677448.553832:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.553834:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.553835:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380000] (21)++
00000800:00000200:31.0:1619677448.553836:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380000] (22)--
00000800:00000200:31.0:1619677448.553836:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380000] (21)--
00000800:00000200:5.0:1619677448.553836:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:8.0:1619677448.553838:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.0:1619677448.553838:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.553839:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.553840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.553841:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:8.0:1619677448.553841:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:5.0:1619677448.553841:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.553842:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.553843:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.553844:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:8.0:1619677448.553846:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.553847:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:8.0:1619677448.553848:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:5.2:1619677448.553848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:8.0:1619677448.553849:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96713429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.553850:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.553850:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.553851:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:8.0:1619677448.553851:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.553851:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.553852:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:2.0:1619677448.553854:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.553855:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:8.0:1619677448.553857:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.553857:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:26.0:1619677448.553858:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.553858:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:8.0:1619677448.553859:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.553861:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.553862:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.553863:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.553863:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.553863:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:10.0:1619677448.553865:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:8.0:1619677448.553865:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.553865:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677448.553866:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.553866:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:10.0:1619677448.553867:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.553867:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:10.0:1619677448.553868:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.553868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677448.553869:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:8.2:1619677448.553871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:21.2:1619677448.553872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380000] (20)++
00000400:00000200:10.0:1619677448.553872:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.553874:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.553876:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.553876:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:1.0:1619677448.553876:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.553877:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.553878:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:1.0:1619677448.553878:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:30.0:1619677448.553879:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380000] (21)++
00000800:00000200:10.0:1619677448.553879:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.553879:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.553880:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677448.553881:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:1.0:1619677448.553881:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.553883:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.48@o2ib3
00000800:00000200:17.0:1619677448.553883:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:25.2:1619677448.553884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:17.0:1619677448.553884:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.553884:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.553885:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.48@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.553886:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380000] (22)--
00000400:00000200:17.0:1619677448.553886:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.553886:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.553887:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.553887:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96713427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.553888:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.553889:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.553889:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677448.553890:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.553890:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.553891:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96713427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.553891:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.553892:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.553893:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.553893:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.553893:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.553894:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.553896:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.553897:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:1.0:1619677448.553899:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.553900:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.48@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.553902:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.553902:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.553903:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.553904:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.553905:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677448.553906:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.553907:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.2:1619677448.553907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:1.0:1619677448.553907:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.553908:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.553908:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.553909:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.553910:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.553910:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380000] (21)++
00000800:00000200:20.0:1619677448.553910:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677448.553911:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380000] (22)--
00000800:00000200:1.0:1619677448.553911:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.553911:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.553912:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380000] (21)--
00000800:00000200:1.0:1619677448.553912:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.553913:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:1.0:1619677448.553913:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:20.0:1619677448.553915:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:8.2:1619677448.553915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.553916:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.553917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:20.0:1619677448.553918:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.553920:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.553921:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96713425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.553922:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.553923:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.553923:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:13.0:1619677448.553923:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:20.0:1619677448.553924:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.553926:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:13.0:1619677448.553926:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:17.0:1619677448.553927:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.553927:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.553928:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677448.553928:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.553930:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.553930:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.553930:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:13.0:1619677448.553931:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.553933:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.553934:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:13.0:1619677448.553934:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677448.553935:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.553935:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677448.553937:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.553937:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.553938:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.553939:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.553940:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.553942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.553944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677448.553947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:31.0:1619677448.553949:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.553950:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.553952:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677448.553952:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.553952:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:31.0:1619677448.553953:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.553953:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.553953:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.553954:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.553954:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.553955:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:30.0:1619677448.553955:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.553955:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:5.0:1619677448.553957:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.553958:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:5.0:1619677448.553959:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.553961:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.553962:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.553963:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.553964:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.553967:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.553969:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.553972:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.553973:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.553973:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.553975:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.553976:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.553977:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.553977:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.553981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.553988:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.553991:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.553992:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.553994:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.553995:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.553997:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.554103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.554109:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.554112:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.554113:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.554115:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.554117:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.554119:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.554120:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713423] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.554121:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.554122:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.554123:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.554127:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.554129:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.554132:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.554133:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.554133:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.554135:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.554136:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.554137:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.554138:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.554138:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:25.2:1619677448.554141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.554144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677448.554147:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.554149:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:12.0:1619677448.554150:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.554151:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.554151:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.554152:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:12.0:1619677448.554153:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.554153:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677448.554154:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.554156:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.554174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.554181:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.554183:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.554184:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.554186:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.554188:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.554190:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.554191:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.554192:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.554193:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.554194:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.554197:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.554199:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.554201:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.554201:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.554202:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.554203:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.554203:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.554204:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.554205:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.554208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.554214:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.554216:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.554217:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.554218:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.554219:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.554220:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.554227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.554233:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.554235:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.554236:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.554236:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.554237:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.554237:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.554332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.554341:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.554343:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.554345:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.554347:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.554347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:31.0:1619677448.554349:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:30.0:1619677448.554349:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.554350:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.554350:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.554351:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.554352:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.554353:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.554354:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.554356:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.554357:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.554359:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.554359:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.554360:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.554361:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.554362:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.554363:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.554364:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.554367:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.554367:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.554368:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.554369:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.554369:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.554370:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.554371:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.554371:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.554372:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.554372:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.554372:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.554372:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.554374:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.554374:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.554374:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.554376:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.554377:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.2:1619677448.554377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.554378:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.554380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.554384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.554385:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:10.0:1619677448.554387:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:20.0:1619677448.554388:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.554389:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.554390:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.554391:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:20.0:1619677448.554391:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.554391:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.554391:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.554392:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:10.0:1619677448.554393:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.554394:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:20.0:1619677448.554394:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677448.554395:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.554396:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.554396:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:8.2:1619677448.554397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.554404:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.554407:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.554408:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.554409:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.554409:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.554411:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.554461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.554467:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677448.554470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:1.0:1619677448.554470:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.554471:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.554473:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.554475:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.554476:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:1.0:1619677448.554477:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.554478:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.554479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:22.0:1619677448.554480:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.554480:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677448.554480:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.554481:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.554483:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:22.0:1619677448.554484:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.554484:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:22.0:1619677448.554486:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.554487:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:1.0:1619677448.554487:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.554488:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:1.0:1619677448.554488:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:22.0:1619677448.554489:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.554489:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.554490:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.554491:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.554492:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.554493:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.554493:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:22.0:1619677448.554494:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:5.0:1619677448.554494:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.554494:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.554496:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.554496:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677448.554497:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.554497:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.554498:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.554499:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:5.0:1619677448.554500:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.554501:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96713417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.554502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677448.554502:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.554503:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.554503:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:5.0:1619677448.554503:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.554504:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.554505:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.554505:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.554506:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.554506:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.554507:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:5.0:1619677448.554508:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.554509:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677448.554510:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.2:1619677448.554511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:13.0:1619677448.554512:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.554513:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.554513:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.554514:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:13.0:1619677448.554515:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.554515:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.554516:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.554517:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677448.554517:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.554518:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.554518:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.554518:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.554519:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677448.554520:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.554521:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.554521:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:20.0:1619677448.554522:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.554523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.554525:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677448.554529:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.554532:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.554533:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.554534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000400:00000200:12.0:1619677448.554535:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.554536:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.554537:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.554538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.554541:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:4.2:1619677448.554543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677448.554544:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.554546:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677448.554547:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:13.0:1619677448.554547:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.554548:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.554549:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.554549:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.554550:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.554551:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.554551:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.554551:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.554552:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677448.554552:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:10.0:1619677448.554554:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.554555:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.554556:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677448.554557:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.554558:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.554559:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.554561:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.554562:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713416] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.554564:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.554565:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.554566:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.554567:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.554569:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.554570:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.554570:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.554572:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.554572:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.554572:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.554573:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.554576:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.554576:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.554577:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.554578:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.554578:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.554580:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.554580:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.554581:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.554581:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.554581:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.554582:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (23)++
00000800:00000200:10.0:1619677448.554583:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.554584:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (24)--
00000800:00000200:10.0:1619677448.554584:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.554584:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.554585:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.554585:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.554586:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.554586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.554586:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.554588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:21.2:1619677448.554588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:4.2:1619677448.554590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.554593:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.554595:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.554595:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.554596:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:24.0:1619677448.554596:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:31.0:1619677448.554597:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.554597:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.554598:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.554598:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:31.0:1619677448.554599:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:24.0:1619677448.554599:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:31.0:1619677448.554600:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:30.0:1619677448.554600:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.554600:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.554600:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677448.554600:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:31.0:1619677448.554601:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.554602:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:26.0:1619677448.554602:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:24.0:1619677448.554602:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.554603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:24.0:1619677448.554605:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.554606:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.554607:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.554608:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.554609:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.554610:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:24.0:1619677448.554610:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.554612:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.554613:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.554613:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:24.0:1619677448.554613:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.554614:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.554615:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:24.0:1619677448.554615:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.554618:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.554619:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.554620:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.554621:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.554622:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.554623:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.554623:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.554627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.554635:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.554637:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.554638:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.554640:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.554640:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.554643:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.554842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677448.554844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.554849:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.554850:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677448.554852:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677448.554853:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:1.0:1619677448.554853:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677448.554856:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.554856:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:17.0:1619677448.554856:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.554859:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.554859:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.554860:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677448.554861:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677448.554862:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.554863:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.554864:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.554865:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.554866:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.554866:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677448.554867:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.554868:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677448.554871:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.554873:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.554874:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677448.554876:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.554877:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.554878:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:31.0:1619677448.554879:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.554879:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:31.0:1619677448.554880:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.554881:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677448.554881:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.554881:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.554883:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:17.0:1619677448.554883:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677448.554884:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677448.554884:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.554884:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.554885:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.554885:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:9.2:1619677448.554887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.2:1619677448.554889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.554889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.554895:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.554896:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.0:1619677448.554897:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:5.0:1619677448.554897:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:1.0:1619677448.554897:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.554898:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677448.554899:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.554900:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:5.0:1619677448.554900:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:8.0:1619677448.554901:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.554901:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.554903:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:8.0:1619677448.554904:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.554904:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.554906:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.0:1619677448.554908:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.2:1619677448.554939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.554942:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.554944:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.554947:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.554947:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.554949:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.554951:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.554952:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713411] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.554953:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.554954:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.554955:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.554959:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.554961:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.554964:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.554965:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.554965:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.554967:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.554968:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.554968:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.554969:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.554973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.554980:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.554983:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.554983:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.554985:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.554986:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.554988:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.555098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.555106:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.555109:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.555111:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.555113:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.555114:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.555116:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.555117:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.555118:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.555120:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.555121:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.555124:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.555125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677448.555126:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.555128:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.555129:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.555130:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.555131:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.555132:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677448.555132:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677448.555133:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.555133:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677448.555135:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.555136:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677448.555137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:24.0:1619677448.555137:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.555140:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.555141:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.555142:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96713409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.555143:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.555143:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:24.0:1619677448.555144:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.555145:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.555145:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.555146:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.555146:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.555148:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.555148:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.555149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677448.555150:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.555152:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:24.0:1619677448.555153:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.555153:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677448.555154:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.555154:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.555155:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:24.0:1619677448.555156:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.555156:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677448.555157:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677448.555157:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:10.0:1619677448.555157:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.555159:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.555159:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:10.0:1619677448.555159:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.555161:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.555162:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.555162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:10.0:1619677448.555163:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.555164:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.555165:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:4.2:1619677448.555165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677448.555168:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.555169:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.555170:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:8.0:1619677448.555172:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.555173:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:10.0:1619677448.555173:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.555173:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:1.0:1619677448.555173:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.555174:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.555174:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.555174:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.555175:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.555176:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.555176:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.555176:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677448.555176:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677448.555177:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.555177:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677448.555179:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:9.2:1619677448.555182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.555190:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.555193:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.555194:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.555194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677448.555194:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.555197:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.555200:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:26.0:1619677448.555202:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.555204:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.555205:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.555207:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.555210:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.555210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:26.0:1619677448.555211:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.555213:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.555214:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.555216:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.555217:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.555219:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.555220:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:26.0:1619677448.555221:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.555222:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:26.0:1619677448.555223:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.555224:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.555225:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:26.0:1619677448.555226:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.555227:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:10.0:1619677448.555227:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.555228:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.555228:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.555229:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.555230:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.555230:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.555231:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.555231:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.555232:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677448.555233:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.555234:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.555235:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.2:1619677448.555236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.555238:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.555239:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.555239:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.555241:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:12.0:1619677448.555242:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.555242:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.555242:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.555243:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677448.555244:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.555245:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.555246:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.555247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677448.555248:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.555249:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.555255:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.555257:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.555259:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.555259:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.555261:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.555264:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.555265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.2:1619677448.555269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677448.555272:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.555274:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.555275:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.555275:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677448.555275:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.555276:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.555277:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677448.555277:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.555278:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.555280:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.555283:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.555284:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.555285:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713405] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.555287:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.555288:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.555289:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.555292:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.555299:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.555302:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.555302:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.555303:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.555305:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.555305:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.555306:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.555307:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.555312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.555319:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.555322:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.555323:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.555324:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.555326:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.555327:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.555345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.555353:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.555356:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.555356:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.555359:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.555361:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.555363:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.555364:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.555365:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.555366:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.555367:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677448.555371:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.555373:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.555376:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.555377:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.555377:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.555379:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.555379:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.555380:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.555382:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.555382:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.2:1619677448.555385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.555387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677448.555395:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.0:1619677448.555396:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.555397:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:21.2:1619677448.555398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:8.0:1619677448.555398:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677448.555398:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:8.0:1619677448.555399:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:5.0:1619677448.555399:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.555401:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.555402:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.555405:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:26.0:1619677448.555406:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.555409:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677448.555410:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677448.555412:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.555415:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.555416:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.555418:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.555419:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.555420:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.555421:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:5.2:1619677448.555421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000400:00000200:26.0:1619677448.555424:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.555425:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.555428:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.555429:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.555429:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:13.0:1619677448.555429:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:26.0:1619677448.555431:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.555432:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:21.2:1619677448.555432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:13.0:1619677448.555432:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:26.0:1619677448.555433:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677448.555434:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677448.555434:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:13.0:1619677448.555434:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.555436:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.555436:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:8.2:1619677448.555437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:13.0:1619677448.555438:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:6.0:1619677448.555438:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:13.0:1619677448.555439:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.555440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000400:00000200:13.0:1619677448.555440:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.555441:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.555441:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.555442:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:13.0:1619677448.555442:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:6.0:1619677448.555444:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.555445:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:13.0:1619677448.555447:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.555447:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:6.0:1619677448.555447:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.555448:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:6.0:1619677448.555448:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677448.555449:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.555449:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.555449:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:1.0:1619677448.555449:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.555450:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:6.0:1619677448.555450:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96713401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.555451:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:17.0:1619677448.555452:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.555452:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.555453:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.555453:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.555454:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:6.0:1619677448.555454:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.555455:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:13.0:1619677448.555455:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677448.555457:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677448.555458:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677448.555458:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000400:00000200:6.0:1619677448.555458:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.555459:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:6.0:1619677448.555460:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.555462:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677448.555463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.555463:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.555463:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.555465:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.555466:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.555466:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.555467:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.555471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.555471:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.555474:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.555475:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.555475:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.555478:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.555479:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677448.555480:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677448.555481:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.555482:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.555484:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.555485:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.555486:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.555562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.555571:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.555574:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.555576:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.555578:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.555578:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.555580:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.555581:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.555582:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.555583:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.555585:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.2:1619677448.555587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:30.0:1619677448.555588:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.555590:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.555592:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.555592:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.555593:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.555594:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677448.555594:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:30.0:1619677448.555595:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.555596:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:5.0:1619677448.555596:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.555597:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.555597:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677448.555599:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000400:00000200:5.0:1619677448.555599:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.555600:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.555601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:5.0:1619677448.555601:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677448.555602:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.555604:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.555604:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.555605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:5.0:1619677448.555606:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:12.0:1619677448.555607:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.555609:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:5.0:1619677448.555609:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677448.555610:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.555611:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:12.0:1619677448.555612:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.555613:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:13.0:1619677448.555613:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677448.555614:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.555615:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:12.0:1619677448.555615:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677448.555615:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677448.555615:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.555616:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.555617:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.555617:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:25.2:1619677448.555618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:5.0:1619677448.555618:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.555619:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.555620:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.555621:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:9.2:1619677448.555623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:13.0:1619677448.555625:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.555626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677448.555628:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.555629:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677448.555629:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.555629:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.555630:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677448.555631:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.555631:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677448.555631:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:13.0:1619677448.555632:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677448.555633:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677448.555633:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.555634:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.555635:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.555637:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.555639:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.555722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.555728:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.555731:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.555732:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.555734:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.555736:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.555738:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.555739:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.555740:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.555742:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.555742:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.555746:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.555747:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.555749:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.555750:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.555751:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.555752:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.555753:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.555753:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.555754:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.555758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:21.2:1619677448.555760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:17.0:1619677448.555765:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:2.0:1619677448.555766:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677448.555767:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.555768:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.555769:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:1.0:1619677448.555769:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:18.0:1619677448.555770:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:17.0:1619677448.555770:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.555771:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:17.0:1619677448.555772:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:2.0:1619677448.555774:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.555775:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.555776:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.555777:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.555778:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.555779:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.2:1619677448.555780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:4.2:1619677448.555780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677448.555782:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.555784:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.555786:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.555786:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.555787:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.555787:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.555788:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677448.555788:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677448.555789:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:26.0:1619677448.555790:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:2.0:1619677448.555790:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.555790:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677448.555791:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677448.555791:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.555792:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:26.0:1619677448.555793:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.555793:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:9.2:1619677448.555794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.555794:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.555794:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:26.0:1619677448.555796:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.555798:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.555799:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.555800:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.555801:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.555802:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:18.0:1619677448.555802:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.555804:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:26.0:1619677448.555805:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.555805:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.555805:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.555807:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:18.0:1619677448.555807:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.555808:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:26.0:1619677448.555810:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.555811:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.555811:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.2:1619677448.555811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:26.0:1619677448.555813:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.555813:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.555814:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.555815:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677448.555817:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:25.2:1619677448.555819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:18.0:1619677448.555820:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.555820:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.555822:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.555824:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.555825:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:2.0:1619677448.555826:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.555827:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.555828:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:2.0:1619677448.555828:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.555829:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.555829:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.555830:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.555831:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.555832:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.555833:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:2.0:1619677448.555833:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.555835:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.555838:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.555839:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.555839:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.555841:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.555841:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.555842:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.555843:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.555847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.555855:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.555857:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.555858:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.555860:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.555861:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.555863:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.555980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.555986:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.555989:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.555990:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.555991:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.555994:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.555995:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.555996:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.555997:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.555998:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.555999:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.556002:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.556003:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.556005:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.556005:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.556006:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.556007:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.556007:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.556008:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.556008:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.556013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.556020:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.556023:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.556023:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.556024:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.556025:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.556026:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.556080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.2:1619677448.556087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:30.0:1619677448.556088:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.556091:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.0:1619677448.556092:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677448.556093:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.556095:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.556095:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.556095:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:30.0:1619677448.556097:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677448.556098:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.556098:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.556099:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.556100:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.556101:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.556101:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.556102:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.556103:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.556104:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.556106:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.556107:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.556109:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.556109:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:21.2:1619677448.556111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:30.0:1619677448.556112:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.556113:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.556114:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.556115:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.556116:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.556116:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.556117:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677448.556117:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677448.556118:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.556118:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.556119:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.0:1619677448.556119:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.556119:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.556120:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.556120:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.556120:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.0:1619677448.556121:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.556121:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:22.0:1619677448.556122:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.556122:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.556123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.556123:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:22.0:1619677448.556125:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.556127:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:8.2:1619677448.556127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:22.0:1619677448.556128:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96713391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.556129:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.556129:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:22.0:1619677448.556130:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.556131:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:12.0:1619677448.556132:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.556133:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.556134:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:12.0:1619677448.556135:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.556136:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.556136:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.556137:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.556137:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.556139:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.556139:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.556139:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.556140:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:22.0:1619677448.556142:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.556142:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677448.556143:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.556144:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.556145:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.556146:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.556147:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.556147:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.556151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.556158:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.556161:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.556162:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.556163:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.556164:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.556166:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.556264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.556270:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.2:1619677448.556273:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677448.556275:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.556277:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.556279:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677448.556280:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:5.0:1619677448.556281:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.556283:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:26.0:1619677448.556284:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.556284:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:5.0:1619677448.556285:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.556287:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.556288:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.556288:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.556289:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:10.0:1619677448.556291:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.556292:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.556298:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.556300:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.556301:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.556301:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000400:00000200:10.0:1619677448.556302:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.556303:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:10.0:1619677448.556305:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.556306:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.556306:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.556307:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677448.556308:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.556309:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.556309:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:10.0:1619677448.556310:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.556310:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677448.556310:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:10.0:1619677448.556311:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.556311:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:10.0:1619677448.556312:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.556313:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.556314:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.556315:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:8.2:1619677448.556315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677448.556316:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.556319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.2:1619677448.556321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.556324:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.556326:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:26.0:1619677448.556327:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:17.0:1619677448.556327:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.556327:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:26.0:1619677448.556329:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.556329:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.556329:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:10.0:1619677448.556330:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:26.0:1619677448.556331:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.556331:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677448.556331:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.556332:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.556333:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.556334:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.556334:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.2:1619677448.556335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.556342:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.556345:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.556345:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.556346:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.556347:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.556347:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.556386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.556392:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.556394:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.556395:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.556397:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.556398:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.556399:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.556401:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.556402:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.556403:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.556404:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.556407:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.556409:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.556411:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.556412:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.556412:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.556414:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.556414:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.556415:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.556415:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.556420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.556426:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.556428:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.556429:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.556430:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.556431:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.556432:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.556563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.556570:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.556573:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.556574:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.556576:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.556578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:24.0:1619677448.556579:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.556580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677448.556581:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.556582:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.556583:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.556584:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.556584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:30.0:1619677448.556585:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:24.0:1619677448.556585:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.556588:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677448.556588:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.556589:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:24.0:1619677448.556589:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.556590:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:24.0:1619677448.556591:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.556592:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.556592:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.556592:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.556593:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677448.556593:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.556593:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677448.556594:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.556594:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.556595:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:30.0:1619677448.556596:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677448.556596:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.556596:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:20.0:1619677448.556596:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.556597:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:30.0:1619677448.556598:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.556598:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.556598:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.556598:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:20.0:1619677448.556598:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:24.0:1619677448.556599:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:30.0:1619677448.556600:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.556600:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.556600:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.556601:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.556602:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.556602:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.556603:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677448.556603:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.556603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:20.0:1619677448.556604:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.556605:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.556606:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.556608:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.556608:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.556610:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.556610:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.556610:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:26.0:1619677448.556612:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.556612:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:30.0:1619677448.556613:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.556613:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:13.0:1619677448.556613:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.556614:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.556615:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.556615:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:13.0:1619677448.556615:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.556616:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.556616:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.556617:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.556617:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.556617:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.556617:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.556618:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.556618:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.556618:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.556618:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.556619:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.556620:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.556620:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.556621:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.556621:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.556622:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.556622:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.556622:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.556622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.556623:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.556623:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.556626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:25.2:1619677448.556630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.556630:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:12.0:1619677448.556632:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.556633:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.556634:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.556634:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677448.556635:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.556636:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.556637:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:12.0:1619677448.556638:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.556638:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:13.0:1619677448.556639:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.556640:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677448.556640:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.556641:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:12.0:1619677448.556641:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:6.0:1619677448.556642:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.556644:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.556647:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.556770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.556778:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.556781:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.556783:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.556785:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.556786:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.556788:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.556789:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.556791:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.556792:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.556793:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.556797:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.556798:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.556801:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.556801:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.556802:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.556804:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.556804:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.556805:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.556806:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.556809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.556816:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.556819:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.556819:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.556821:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.556822:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.556823:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.556917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.556923:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.556927:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.556927:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.556929:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.556931:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.556932:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.556934:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.556935:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.556936:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.556937:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.556940:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.556942:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.556945:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.556946:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.556946:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.556948:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.556948:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.556949:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.556950:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.556954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.556962:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.556964:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.556965:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.556967:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.556968:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.556970:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.556971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.556978:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.556981:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.556982:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.556982:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.556983:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.556984:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.556993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.556998:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.557001:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.557002:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.557004:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.557005:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.557007:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.557008:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.557010:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.557011:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.557012:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.557015:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.557017:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.557020:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.557021:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.557021:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.557024:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.557024:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.557026:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.557027:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.557027:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.557030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.2:1619677448.557032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.557036:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677448.557037:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677448.557039:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:17.0:1619677448.557039:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:31.0:1619677448.557040:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.557040:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.557041:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677448.557042:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.557042:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.557045:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.557048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.557055:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.557058:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.557059:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.557060:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.557060:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.557061:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.557172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.557179:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.557181:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.557183:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.557184:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.557187:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.557188:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.557190:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.557191:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.557192:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.557193:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:21.2:1619677448.557194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:10.0:1619677448.557196:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.557198:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.557199:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.557200:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.557201:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.557201:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.557202:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.557203:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.557203:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.557204:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:10.0:1619677448.557204:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:31.0:1619677448.557206:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.557208:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.557208:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.557208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:31.0:1619677448.557210:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.557211:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.557212:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.557213:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.557214:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.557214:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.557216:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.557216:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.557217:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.557217:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.557218:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.557219:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.557219:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677448.557221:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.557222:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.557223:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.557224:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.557225:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.557226:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.557228:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.557228:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.557228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.2:1619677448.557231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:25.2:1619677448.557232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.557234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.557234:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.557236:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.557237:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:22.0:1619677448.557237:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:10.0:1619677448.557237:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:13.0:1619677448.557238:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.557238:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:4.2:1619677448.557238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677448.557239:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:10.0:1619677448.557239:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:6.0:1619677448.557240:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:13.0:1619677448.557241:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.557241:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.557242:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:22.0:1619677448.557242:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.557243:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.557244:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.557245:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677448.557245:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.557245:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677448.557245:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677448.557246:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:22.0:1619677448.557246:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:13.0:1619677448.557246:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.557248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:22.0:1619677448.557248:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.557248:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.557249:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677448.557250:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.557250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:22.0:1619677448.557251:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.557252:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.557252:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.557254:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:8.0:1619677448.557255:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.557256:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.557257:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677448.557258:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:22.0:1619677448.557258:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677448.557258:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.557258:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:8.0:1619677448.557258:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:24.0:1619677448.557259:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:12.0:1619677448.557259:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.557260:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677448.557260:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96713377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.557261:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677448.557261:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.557262:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:8.0:1619677448.557262:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.557263:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677448.557264:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.557265:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677448.557265:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000400:00000200:24.0:1619677448.557266:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.557266:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677448.557267:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000400:00000200:8.0:1619677448.557268:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.557269:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.557270:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:8.0:1619677448.557270:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:9.2:1619677448.557271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:24.0:1619677448.557272:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96713376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.557273:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.557274:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.557275:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.557278:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:24.0:1619677448.557279:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.557279:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.557280:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.557281:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.557281:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.557281:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.557282:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.557282:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.557283:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.557283:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:2.0:1619677448.557284:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.557285:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.557285:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.557285:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677448.557286:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.557286:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.0:1619677448.557287:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:2.0:1619677448.557287:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:24.0:1619677448.557289:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.2:1619677448.557289:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:9.2:1619677448.557290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677448.557291:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.557291:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.557292:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:1.0:1619677448.557298:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677448.557299:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677448.557301:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.557302:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.557302:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.557303:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.557303:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (23)++
00000800:00000200:2.0:1619677448.557303:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:18.0:1619677448.557304:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (25)--
00000800:00000200:2.0:1619677448.557304:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (25)--
00000400:00000200:1.0:1619677448.557304:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.557305:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.557305:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.557306:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.557307:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.557308:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677448.557311:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.557550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919861b98600] (20)++
00000800:00000200:6.0:1619677448.557557:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919861b98600] (21)++
00000800:00000200:20.2:1619677448.557560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:22.0:1619677448.557561:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b98600] (22)--
00000800:00000200:6.0:1619677448.557561:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.10@o2ib3
00000400:00000200:6.0:1619677448.557563:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.6.10@o2ib3 : GET - for me
00000400:00000200:6.0:1619677448.557566:0:19937:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.6.10@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000800:00000200:30.0:1619677448.557568:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:6.0:1619677448.557568:0:19937:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.6.10@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:6.0:1619677448.557570:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919861b98600] (21)++
00000800:00000200:30.0:1619677448.557571:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:6.0:1619677448.557571:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919861b98600] (22)--
00000400:00000200:30.0:1619677448.557573:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.557573:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.6.10@o2ib3 local destination
00000800:00000200:31.0:1619677448.557576:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.557576:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.557577:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.6.10@o2ib3(10.51.6.10@o2ib3:10.51.6.10@o2ib3) : REPLY try# 0
00000400:00000200:30.0:1619677448.557578:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.557579:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.557580:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.6.10@o2ib3
00000400:00000200:30.0:1619677448.557581:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.557582:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.557582:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919876b8ff00] -> 10.51.6.10@o2ib3 (2) version: 12
00000800:00000200:6.0:1619677448.557582:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919861b98600] (21)++
00000400:00000200:30.0:1619677448.557583:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:6.0:1619677448.557583:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919861b98600] (22)++
00000800:00000200:21.2:1619677448.557584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.557584:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919861b98600] (23)--
00000400:00000200:30.0:1619677448.557586:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.557586:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b98600] (22)--
00000800:00000200:30.0:1619677448.557588:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677448.557588:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:21.2:1619677448.557589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919861b98600] (21)++
00000800:00000200:22.0:1619677448.557590:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.557590:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:30.0:1619677448.557591:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.557591:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:22.0:1619677448.557591:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919861b98600] (22)++
00000800:00000200:30.0:1619677448.557592:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:22.0:1619677448.557592:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919861b98600] (23)--
00000400:00000200:6.0:1619677448.557592:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.557593:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:22.0:1619677448.557593:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.557594:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.557595:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:22.0:1619677448.557595:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.6.10@o2ib3: REPLY: OK
00000400:00000200:6.0:1619677448.557595:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.557596:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:22.0:1619677448.557597:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b98600] (22)--
00000800:00000200:22.0:1619677448.557598:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b98600] (21)--
00000400:00000200:6.0:1619677448.557598:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.557599:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.557600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:6.0:1619677448.557601:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.557602:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.557603:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.557606:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.557607:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:6.0:1619677448.557607:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:31.0:1619677448.557610:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.557610:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.557610:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.557611:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000400:00000200:31.0:1619677448.557612:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.557612:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:30.0:1619677448.557613:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.557614:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:6.0:1619677448.557614:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.557614:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.557615:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.557616:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.557617:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:25.2:1619677448.557619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:21.2:1619677448.557622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:25.2:1619677448.557623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.557627:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:22.0:1619677448.557627:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:26.0:1619677448.557629:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:22.0:1619677448.557629:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:30.0:1619677448.557630:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:22.0:1619677448.557630:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:26.0:1619677448.557631:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.557631:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.557631:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:30.0:1619677448.557633:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.557633:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000400:00000200:26.0:1619677448.557633:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.557634:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.557634:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.557635:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:26.0:1619677448.557636:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.557659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.557665:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.557668:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.557669:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.557671:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.557673:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.557675:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.557676:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.557677:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.557678:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.557680:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.557682:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.557684:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.557687:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.557687:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.557688:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.557690:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.557690:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.557691:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.557691:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.557696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.557704:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.557706:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.557707:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.557707:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.557709:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.557712:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.557915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.557922:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.557924:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.557925:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.557927:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.557929:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.557930:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.557932:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.557933:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.557934:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.557935:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.557939:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.557940:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.557943:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.557944:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.557945:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.557946:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.557947:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.557948:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.557949:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.557950:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.557952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.557955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.557960:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:21.2:1619677448.557960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:26.0:1619677448.557962:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.557963:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.557963:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.557964:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677448.557964:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.557965:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677448.557965:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677448.557966:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.557966:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.557968:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:22.0:1619677448.557969:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677448.557969:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.557971:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.557974:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.557975:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.557977:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.557978:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.557979:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.557980:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.557984:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.557985:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.557989:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.557989:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.557990:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.557992:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.557992:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.557993:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.557994:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.557998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.558006:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.558009:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.558009:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.558011:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.558013:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.558014:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.558099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.558105:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.558108:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.558109:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.558111:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.558113:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.558114:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.558116:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.558117:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.558118:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.558119:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.558122:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.558124:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.558127:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.558128:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.558128:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.558130:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.558130:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.558131:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.558131:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.558136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.558143:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.558146:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.558147:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.558147:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.558149:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.558151:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.558176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.558184:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.2:1619677448.558186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677448.558186:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.558187:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.558189:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.558192:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.558194:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677448.558194:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.558195:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.558197:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677448.558197:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.558198:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.558199:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.558200:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.558200:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.558202:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.558203:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.558203:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.558204:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.558205:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.558206:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.558207:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.558207:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.558208:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.558208:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.558209:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.558210:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.558211:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.558211:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:30.0:1619677448.558212:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.558212:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.558214:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.558216:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677448.558216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.558217:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.558217:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.558219:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.558219:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.558220:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.558221:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.558222:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:25.2:1619677448.558224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.558225:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.558225:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.558227:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.558229:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.558230:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.558231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:12.0:1619677448.558231:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.558233:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.558234:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.558235:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:12.0:1619677448.558235:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.558237:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.558238:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:13.0:1619677448.558238:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:20.0:1619677448.558240:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.558241:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.558242:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.558244:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.558245:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.558247:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.558248:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.558249:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.558253:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.558254:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.558258:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.558258:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.558259:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.558261:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.558261:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.558262:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.558262:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.558266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.558273:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.558275:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.558276:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.558278:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.558280:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.558280:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.558364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.558371:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.558374:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.558375:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.558376:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.558379:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.558380:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.558382:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.558383:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.558384:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.558385:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.558388:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.558390:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.558393:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.558394:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.558395:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.558397:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.558397:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.558398:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.558398:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.558405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.558413:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.558415:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.558416:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.558418:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.558419:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.558421:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.558422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.558429:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.558432:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.558433:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.558434:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.558434:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.558435:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.558476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.558483:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.558486:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.558487:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.558489:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.558492:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.558493:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.558494:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.558495:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.558496:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.558497:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:5.2:1619677448.558498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:10.0:1619677448.558500:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.558501:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.558503:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:10.0:1619677448.558504:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.558505:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.558505:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.558507:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:10.0:1619677448.558507:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.558507:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:10.0:1619677448.558508:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.558509:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:2.0:1619677448.558510:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.558511:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.558511:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:2.0:1619677448.558512:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.558514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677448.558514:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.558515:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.558516:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.558517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:2.0:1619677448.558518:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.558519:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.558523:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:2.0:1619677448.558523:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.558524:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:18.0:1619677448.558525:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.558525:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.558526:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.558527:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:18.0:1619677448.558527:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.558528:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:18.0:1619677448.558528:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.558529:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.558530:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.558531:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.558532:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.558533:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.558533:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.558534:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.558534:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.558535:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.558539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.558547:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.558550:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.558551:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.558553:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.558555:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.558556:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.558556:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.558557:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.558558:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.558558:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.558558:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.558559:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.558560:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.558564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.558570:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.558573:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.558574:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.558576:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.558579:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.558581:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.558582:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.558583:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.558584:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.558585:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.558589:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.558591:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.558594:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.558595:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.558596:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.558597:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.558598:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.558599:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.558599:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.558603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.558609:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.558612:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.558613:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.558614:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.558616:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.558617:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.558671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.558679:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.558681:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.558683:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.558686:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.558686:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.558687:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.558688:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.558690:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.558691:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.558692:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.558695:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.558697:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.558700:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.558700:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.558701:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.558703:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.558703:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.558704:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.558705:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.558708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.558716:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.558718:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.558719:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.558721:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.558721:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.558724:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.558775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.558781:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.558783:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.558784:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.558786:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:5.2:1619677448.558788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000400:00000200:18.0:1619677448.558789:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.558790:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.558791:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.558793:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.558794:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.558795:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677448.558796:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000400:00000200:18.0:1619677448.558798:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.558799:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677448.558799:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677448.558800:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:18.0:1619677448.558802:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.558803:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:8.0:1619677448.558803:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000800:00000200:18.0:1619677448.558804:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.558806:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.558806:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:8.0:1619677448.558806:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.558807:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:8.0:1619677448.558807:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:18.0:1619677448.558808:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:8.0:1619677448.558808:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96713360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.558809:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:8.0:1619677448.558810:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.558811:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.558812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:8.0:1619677448.558812:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:21.2:1619677448.558814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:8.0:1619677448.558815:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677448.558817:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.2:1619677448.558818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677448.558818:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.558819:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:2.0:1619677448.558819:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:22.0:1619677448.558820:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677448.558820:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.558821:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.558821:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677448.558822:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:2.0:1619677448.558822:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.558822:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677448.558823:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:8.0:1619677448.558823:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:2.0:1619677448.558823:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:22.0:1619677448.558824:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.558824:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.558825:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:22.0:1619677448.558826:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:9.2:1619677448.558827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:30.0:1619677448.558828:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677448.558828:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:30.0:1619677448.558830:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.558832:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.558833:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.558833:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:30.0:1619677448.558835:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:24.0:1619677448.558835:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:30.0:1619677448.558836:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.558836:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.558836:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:30.0:1619677448.558837:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.558838:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.558838:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000400:00000200:30.0:1619677448.558839:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677448.558840:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:30.0:1619677448.558842:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.558844:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.558846:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.558847:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.558848:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.558849:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.558850:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.558851:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.558851:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.558855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.558861:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.558863:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.558864:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.558865:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.558867:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.558868:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.558881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677448.558882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:10.0:1619677448.558888:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.558890:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:10.0:1619677448.558891:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.558892:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.558893:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:10.0:1619677448.558894:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.558895:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.558896:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:10.0:1619677448.558897:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.558898:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.558899:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.558900:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.558900:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.558901:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.558902:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.558903:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.558904:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.558904:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.558905:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.558905:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.558910:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.558910:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.558913:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.558913:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.558916:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.558916:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.558917:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.558917:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.558918:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.558918:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.558920:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.558921:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.558921:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:10.0:1619677448.558921:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.558921:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.558922:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677448.558922:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:10.0:1619677448.558922:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.558926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:9.2:1619677448.558928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.558935:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677448.558936:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.558939:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:6.0:1619677448.558939:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.558940:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.558940:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.558940:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.558941:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.558942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:18.0:1619677448.558944:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.558945:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.558947:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677448.558949:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.558950:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.558952:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.558953:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.558954:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.558955:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.558955:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.559010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.559015:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.559018:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.559020:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.559021:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.559023:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.559024:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.559026:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.559027:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.559029:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.559030:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.559033:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.559035:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.559038:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.559039:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.559040:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.559042:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.559042:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.559044:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.559045:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.559046:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.559049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.559050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.559053:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677448.559055:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.559056:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.559056:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.559057:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.559058:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.559059:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.559061:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.559061:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.559063:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.559212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.559219:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.559222:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.559223:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.559225:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.559227:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.559229:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.559230:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.559231:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.559232:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.559233:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.559237:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.559239:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.559241:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.559242:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.559243:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.559245:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.559245:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.559246:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.559246:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.559251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.559259:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.559262:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.559263:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.559263:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.559265:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.559268:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.559315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.559321:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.559323:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.559324:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.559326:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.559328:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.559329:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.559330:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.559331:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.559332:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.559333:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:4.2:1619677448.559334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:17.0:1619677448.559337:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.559338:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.559340:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.559341:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.559342:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.559342:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.559343:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.559344:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.559345:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:17.0:1619677448.559345:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.559346:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:10.0:1619677448.559346:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.559347:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.559348:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.559350:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.559350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:26.0:1619677448.559352:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.559353:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.559353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.559354:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.559355:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.559356:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.559359:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.559359:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:26.0:1619677448.559360:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.559360:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:5.2:1619677448.559360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:1.0:1619677448.559360:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.559360:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.559361:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.559362:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.559362:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.559362:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.559363:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.559364:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:26.0:1619677448.559365:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.559365:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.559366:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.559366:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:26.0:1619677448.559368:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.559368:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.559369:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677448.559369:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:26.0:1619677448.559370:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:18.0:1619677448.559370:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.559372:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.559374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:2.0:1619677448.559375:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.559376:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.559377:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.559378:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.559379:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.559380:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677448.559383:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:2.0:1619677448.559383:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.559385:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677448.559386:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.559386:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.559387:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.559388:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.559388:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:22.0:1619677448.559389:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.559389:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.559390:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.559391:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:22.0:1619677448.559392:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677448.559392:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.559393:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.559394:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677448.559396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.559399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.559405:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.559406:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.559406:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:18.0:1619677448.559407:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.559407:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.559407:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.559408:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.559409:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.559409:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677448.559410:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.559415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:9.2:1619677448.559417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:30.0:1619677448.559423:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.2:1619677448.559423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677448.559424:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.559427:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:2.0:1619677448.559427:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.559428:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.559428:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000400:00000200:30.0:1619677448.559429:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.559430:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.559430:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677448.559431:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.559431:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:30.0:1619677448.559432:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.559434:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:26.0:1619677448.559434:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.559434:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:30.0:1619677448.559435:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.559437:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.559437:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.559438:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.559439:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.559440:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.559442:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.559442:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.559443:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.559445:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.559445:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.559446:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.559447:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.559447:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.559448:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.559449:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.559450:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:10.0:1619677448.559450:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.559451:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.559452:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:10.0:1619677448.559452:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.559454:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.559454:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.559454:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.559454:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.559455:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:25.2:1619677448.559456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.559456:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.559457:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.559457:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.559458:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677448.559460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:25.2:1619677448.559463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:13.0:1619677448.559464:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.559466:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.559467:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.559468:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:13.0:1619677448.559468:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.559469:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.559470:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:13.0:1619677448.559470:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.559471:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:12.0:1619677448.559471:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.559472:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.559472:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.559473:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677448.559474:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677448.559474:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.559476:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.559556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.559564:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.559567:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.559568:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.559570:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.559571:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.559572:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.559573:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.559575:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.559575:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.559576:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.559580:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.559581:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.559583:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.559584:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.559585:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.559586:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.559587:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.559587:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.559588:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.559592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.559598:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.559600:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.559601:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.559601:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.559603:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.559605:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.559645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:4.2:1619677448.559645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677448.559649:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.559651:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:10.0:1619677448.559652:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677448.559654:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677448.559655:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:5.0:1619677448.559655:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:26.0:1619677448.559656:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677448.559656:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.559658:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.559658:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.559659:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.559660:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.559660:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.559661:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.559662:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.559663:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.559664:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.559665:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.559666:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.559667:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:21.2:1619677448.559671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:20.0:1619677448.559672:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000400:00000200:10.0:1619677448.559673:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.559674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:20.0:1619677448.559675:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.559676:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677448.559676:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677448.559678:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.559678:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.559679:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:6.0:1619677448.559679:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.559680:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:10.0:1619677448.559680:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.559681:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.559682:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.559682:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:6.0:1619677448.559682:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.559683:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:20.0:1619677448.559683:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.559683:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:10.0:1619677448.559683:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.559684:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677448.559685:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:6.0:1619677448.559685:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.559686:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677448.559686:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:25.2:1619677448.559687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.559688:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:6.0:1619677448.559688:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96713346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.559689:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.559689:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.559690:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.559690:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.559691:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:9.2:1619677448.559691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677448.559691:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.559692:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.559692:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:10.0:1619677448.559693:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:26.0:1619677448.559694:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677448.559694:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.559695:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96713345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.559695:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.559695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677448.559696:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.559697:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.559698:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.559698:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.559699:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677448.559699:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:26.0:1619677448.559700:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.559702:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:6.0:1619677448.559702:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.559703:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.559703:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:6.0:1619677448.559703:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.559703:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.559703:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:26.0:1619677448.559705:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.559705:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:6.0:1619677448.559705:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.559706:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.559706:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.559706:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.559706:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:10.0:1619677448.559707:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677448.559707:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:26.0:1619677448.559708:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.559708:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677448.559709:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.559709:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.559711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:26.0:1619677448.559712:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.559713:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.559713:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.559715:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.559715:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.559716:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677448.559717:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:2.0:1619677448.559719:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:25.2:1619677448.559721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.559721:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.559722:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.559722:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.559724:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.559726:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:13.0:1619677448.559728:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.559730:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.559731:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.559733:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.559734:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.559735:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.559736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.559742:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.559745:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.559746:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.559747:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.559747:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.559748:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.559826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.559832:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.559835:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.559837:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.559837:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.559840:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.559841:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.559843:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.559844:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.559845:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.559846:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.559850:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.559851:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.559854:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.559855:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.559855:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.559857:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.559858:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.559859:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.559859:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.559863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.559871:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.559873:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.559875:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.559875:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.559877:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.559879:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.559901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.559907:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.559910:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.559911:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.559913:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.559914:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.559916:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.559917:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.559918:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.559919:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.559920:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.559924:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.559926:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.559929:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.559930:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.559930:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.559933:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.559933:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.559934:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.559934:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.559938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.559945:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.559948:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.559949:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.559950:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.559951:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.559953:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.559956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:4.2:1619677448.559956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677448.559963:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.559963:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:12.0:1619677448.559966:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.559966:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.559967:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:12.0:1619677448.559967:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.559968:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.559968:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:10.0:1619677448.559969:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.559970:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:10.0:1619677448.559971:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.559973:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.559974:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.559975:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.559976:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.559977:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.559981:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.559983:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.559985:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.559986:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.559986:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.559988:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.559988:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.559989:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.559990:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.559994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.560002:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.560004:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.560005:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.560006:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.560008:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.560010:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.560187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.560194:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.560197:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.560198:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.560199:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.560202:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.560204:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.560205:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.560206:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.560207:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.560209:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.560212:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.560213:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.560216:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.560217:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.560217:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.560219:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.560219:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.560220:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.560220:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.560225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.560233:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.560235:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.560236:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.560237:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.560239:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.560241:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.560297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.560302:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.560305:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.560306:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.2:1619677448.560308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:6.0:1619677448.560308:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.560310:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.560311:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.560312:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.560313:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.560314:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.560316:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:4.2:1619677448.560316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.560317:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:6.0:1619677448.560319:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.560320:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:6.0:1619677448.560321:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.560322:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.560323:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.560323:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.560324:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.560324:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.560324:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.2:1619677448.560324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:6.0:1619677448.560325:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:30.0:1619677448.560326:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:26.0:1619677448.560326:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:6.0:1619677448.560326:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:10.0:1619677448.560327:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677448.560327:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.560327:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:30.0:1619677448.560328:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.560328:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:30.0:1619677448.560329:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.560330:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.560330:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.560330:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677448.560332:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.560333:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.560333:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:9.2:1619677448.560333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:1.0:1619677448.560334:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.560335:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.560336:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96713338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.560337:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.560338:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.560338:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.560339:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.560340:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.560340:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.560340:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.560341:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:2.0:1619677448.560341:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:17.0:1619677448.560343:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96713337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.560344:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.560345:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:17.0:1619677448.560345:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.560345:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.560346:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.560346:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.560347:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:2.0:1619677448.560347:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.560349:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.560350:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.560350:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:30.0:1619677448.560351:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.560351:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.560352:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.560352:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:17.0:1619677448.560352:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.560353:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.560354:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.560354:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.560354:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.560354:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677448.560355:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677448.560355:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.560356:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.560356:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.560356:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677448.560356:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.560357:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.560358:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:25.2:1619677448.560359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.560359:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:21.2:1619677448.560360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fdae00] (20)++
00000800:00000200:17.0:1619677448.560360:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.560361:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.560362:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.560363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.560365:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fdae00] (21)++
00000800:00000200:8.2:1619677448.560366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:13.0:1619677448.560367:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.0:1619677448.560368:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.38@o2ib3
00000800:00000200:31.0:1619677448.560369:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:13.0:1619677448.560369:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:20.0:1619677448.560370:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.38@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.560370:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.560371:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (22)--
00000800:00000200:31.0:1619677448.560372:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.560372:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.560372:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.560372:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.560374:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:13.0:1619677448.560374:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.560374:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677448.560374:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:31.0:1619677448.560375:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.560376:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:20.0:1619677448.560376:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96713336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.560376:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:20.0:1619677448.560377:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.560377:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.560378:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:20.0:1619677448.560378:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.560378:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:20.0:1619677448.560379:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677448.560379:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.560382:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:20.0:1619677448.560383:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.38@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.560385:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 504 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.560387:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677448.560387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677448.560388:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.2:1619677448.560390:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.560392:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.560394:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.560394:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fdae00] (21)++
00000800:00000200:17.0:1619677448.560394:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.560395:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fdae00] (22)--
00000800:00000200:5.0:1619677448.560395:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.560396:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (21)--
00000800:00000200:20.0:1619677448.560397:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677448.560397:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.560397:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677448.560398:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.560399:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000400:00000200:5.0:1619677448.560399:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.560400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.560400:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.560400:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:5.0:1619677448.560402:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.560404:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.560405:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.560406:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.560407:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:5.0:1619677448.560407:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.560408:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.560409:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.560411:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.560411:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.560413:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.560413:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.560416:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677448.560416:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.560419:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.560420:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.560420:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.560422:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.560423:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.560424:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.560425:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.560428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.560435:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.560438:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.560439:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.560441:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.560442:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.560444:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.560611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.560617:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.560620:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.560621:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.560623:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.560625:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.560626:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.560628:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.560629:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.560630:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.560631:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.560634:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.560635:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.560638:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.560639:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.560640:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.560641:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.560642:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.560642:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.560643:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.560647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.560655:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.560657:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.560658:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.560660:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.560660:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.560663:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.560669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.560675:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.560678:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.560678:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.560681:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.560682:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.560684:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.560685:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.560686:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.560687:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.560688:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.560692:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.560693:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.560697:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.560697:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.560698:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.560700:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.560700:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.560701:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.560701:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677448.560703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:25.2:1619677448.560706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.560710:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677448.560712:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:10.0:1619677448.560713:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.560714:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:13.0:1619677448.560715:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.560716:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.560716:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.560718:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.560718:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.560719:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.560720:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:10.0:1619677448.560720:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.560721:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.560722:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.560723:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.560724:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.560727:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.560729:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.560732:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.560733:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.560733:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.560735:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.560736:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.560736:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.560737:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.560741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.560749:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.560752:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.560752:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.560753:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.560755:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.560758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.560761:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.560762:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.560764:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.560765:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.560765:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.560766:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.560766:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.560821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.560829:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.560831:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.560832:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.560834:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.560837:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.560838:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.560839:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.560840:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.560841:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.560842:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.560845:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.560846:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.560848:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.560849:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.560849:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.560850:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.560851:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.560852:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.560853:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.560854:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.560856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.560859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.560862:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.560865:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.560866:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.560866:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:12.0:1619677448.560867:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.560868:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:13.0:1619677448.560869:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.560869:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.560870:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.560870:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.560887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.560893:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.560896:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.560896:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.560898:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.560901:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.560903:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.560904:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.560905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:17.0:1619677448.560905:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.560906:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.560907:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.560910:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.560911:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.560912:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.560914:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:17.0:1619677448.560914:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.560914:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677448.560915:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.560915:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:22.0:1619677448.560916:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.560916:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.560917:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.560918:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:22.0:1619677448.560919:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.560919:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000400:00000200:22.0:1619677448.560920:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:17.0:1619677448.560920:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677448.560922:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.560922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:22.0:1619677448.560923:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.560924:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.560925:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.2:1619677448.560925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677448.560928:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.560930:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.560930:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.560932:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.560932:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.560932:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.560932:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.560933:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.560933:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:17.0:1619677448.560933:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677448.560934:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677448.560934:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.560935:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:1.0:1619677448.560935:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.560936:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:1.0:1619677448.560936:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677448.560937:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.560937:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.560938:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.560942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.2:1619677448.560947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:8.2:1619677448.560948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677448.560949:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.560952:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.560953:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.560953:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.560954:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:10.0:1619677448.560954:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:17.0:1619677448.560955:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677448.560956:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:5.0:1619677448.560956:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:26.0:1619677448.560957:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:5.0:1619677448.560958:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:10.0:1619677448.560959:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677448.560959:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.560961:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.560962:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:10.0:1619677448.560962:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.560963:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677448.560964:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:10.0:1619677448.560965:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.560966:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.560967:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.560968:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.560971:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.560973:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.560975:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.560976:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.560976:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.560978:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.560978:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.560979:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.560979:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677448.560983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.560989:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.560992:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.560992:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.560994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:13.0:1619677448.560994:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677448.560996:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.560996:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.561001:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.561004:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.561006:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.561008:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.561008:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.561009:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.561011:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.561012:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.561013:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.561013:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.561016:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.561018:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.561020:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.561021:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.561021:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.561023:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.561023:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.561025:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.561026:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.561027:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.561028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.561032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.561034:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.561036:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.561037:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.561039:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.561040:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.561041:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.561042:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:12.0:1619677448.561042:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.561043:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.561044:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677448.561086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.561093:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.561096:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.561098:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.561100:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.561100:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.561101:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.561103:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713326] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.561104:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.561105:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.561106:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.561109:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.561110:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.561114:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.561114:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.561115:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.561117:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.561117:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.561118:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.561119:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.561123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.561130:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.2:1619677448.561133:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.561135:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.561136:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.561136:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.561137:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:20.0:1619677448.561138:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.561139:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.561141:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:5.0:1619677448.561141:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.561142:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.561144:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.561145:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.561147:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.561148:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.561149:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.561150:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.561153:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.561155:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.561157:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.561158:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.561159:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.561161:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.561161:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.561162:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.561163:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.561167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.561174:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.561177:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.561178:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.561178:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.561180:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.561183:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.561302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.561308:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.561311:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.561312:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.561314:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.561316:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.561317:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.561318:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.561319:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.561321:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.561321:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.561325:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.561326:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.561328:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.561329:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.561330:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.561331:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.561331:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.561332:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.561333:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.561338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.561345:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.561346:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.561347:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.561348:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.561349:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.561350:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.561358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.561364:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.561367:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.561368:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.561370:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.561372:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.561373:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.561374:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.561376:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.561377:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.561378:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.561381:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.561383:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.561384:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.561385:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.561386:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.561387:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.561388:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.561388:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.561389:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.561394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.561401:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.561402:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.561403:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.561404:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.561405:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.561407:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.561411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.561417:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.561420:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.561420:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.561421:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.561421:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.561422:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.561487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.561494:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.561497:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.561498:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.561500:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.561503:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.561505:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.561506:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.561507:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.561508:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.561509:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677448.561512:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.561514:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.2:1619677448.561517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:8.0:1619677448.561517:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.561517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:8.0:1619677448.561518:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.561518:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.561520:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:8.0:1619677448.561520:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.561521:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.561521:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677448.561522:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:8.0:1619677448.561522:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:1.0:1619677448.561523:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:20.0:1619677448.561524:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.561525:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677448.561526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.561526:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:20.0:1619677448.561527:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.561527:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:20.0:1619677448.561528:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.561529:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.561530:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.561531:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.561531:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.561532:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677448.561533:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.561534:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.561534:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:1.0:1619677448.561534:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.561535:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:1.0:1619677448.561535:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.561536:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.561536:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.561538:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.561538:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677448.561539:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.561541:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.561542:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.561542:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:1.0:1619677448.561543:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.561545:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.561545:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.561546:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.561546:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.561548:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.561549:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:1.0:1619677448.561549:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.561549:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.561550:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:1.0:1619677448.561550:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.561551:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:1.0:1619677448.561552:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:1.0:1619677448.561552:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677448.561553:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:25.2:1619677448.561554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.561554:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:25.2:1619677448.561558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.561562:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.561565:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:26.0:1619677448.561565:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:31.0:1619677448.561566:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.561567:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:31.0:1619677448.561568:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:10.0:1619677448.561568:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.561569:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:26.0:1619677448.561569:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.561571:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.561572:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.561574:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.561625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.561631:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.561634:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.561635:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.2:1619677448.561635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:6.0:1619677448.561637:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.561639:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.561641:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:10.0:1619677448.561642:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:6.0:1619677448.561642:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.561643:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.561644:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.561645:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.561646:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677448.561646:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.561649:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.561650:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.561651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:10.0:1619677448.561652:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.561652:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.561654:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:6.0:1619677448.561656:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.561657:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.561657:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.561658:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.561659:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:10.0:1619677448.561659:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.561659:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.561660:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.561660:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.561660:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677448.561661:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677448.561661:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.561662:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.561662:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:26.0:1619677448.561664:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.561665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:10.0:1619677448.561665:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.561666:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.561667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.561667:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.561668:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.561669:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96713317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.561670:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.561671:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.561672:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.561672:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:30.0:1619677448.561673:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677448.561674:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.561675:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677448.561676:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.561676:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.561676:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.561676:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.561676:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.561677:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:30.0:1619677448.561678:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.561678:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.561679:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.561680:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.561680:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.561680:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.561681:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.561682:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.561682:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.561682:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677448.561683:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.561683:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:25.2:1619677448.561683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.561683:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.561685:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.561686:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.561686:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:9.2:1619677448.561686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.561688:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.561688:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.2:1619677448.561689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677448.561691:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.561693:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.561694:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:4.2:1619677448.561694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.561695:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.561695:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:25.2:1619677448.561695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.561695:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.561695:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677448.561697:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.0:1619677448.561698:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677448.561698:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.561699:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.561699:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.561701:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677448.561701:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:5.0:1619677448.561701:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.561702:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.561703:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677448.561703:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:5.0:1619677448.561703:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.561704:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.561705:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:5.0:1619677448.561705:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:2.0:1619677448.561705:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677448.561706:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.561706:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:5.0:1619677448.561706:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.561707:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.561707:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.561709:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.561709:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.561710:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.561713:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:5.0:1619677448.561714:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.561716:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.561719:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.561720:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.561721:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.561723:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.561723:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.561725:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.561726:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.561727:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.561729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.561732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.561734:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.561736:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:13.0:1619677448.561736:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.561737:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.561738:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.561739:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.561740:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.561742:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.561743:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.561745:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.561748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.561754:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.561756:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.561757:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.561758:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.561759:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.561760:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.561804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677448.561810:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677448.561813:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677448.561813:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677448.561816:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.561819:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.561821:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677448.561822:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.561823:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.561824:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.561825:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677448.561829:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.561831:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.561833:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.561834:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:17.0:1619677448.561835:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677448.561836:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677448.561837:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677448.561837:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677448.561839:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677448.561839:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677448.561842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.561845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677448.561850:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:20.0:1619677448.561852:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:1.0:1619677448.561852:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677448.561852:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677448.561853:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.0:1619677448.561854:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.561855:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.561857:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.561858:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.561859:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.561984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.561991:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.561994:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.561996:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.561998:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.561999:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.562001:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677448.562002:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.562004:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.562005:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.562006:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677448.562009:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.562011:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.562014:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.562015:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.562015:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.562017:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.562018:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.562019:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.562019:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.562023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.562031:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.562034:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.562034:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.562035:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.562037:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.562040:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.562095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:5.2:1619677448.562095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1200] (20)++
00000800:00000200:1.0:1619677448.562100:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:18.0:1619677448.562101:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:1.0:1619677448.562103:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:18.0:1619677448.562104:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:17.0:1619677448.562104:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:2.0:1619677448.562105:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:1.0:1619677448.562106:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.562108:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.562109:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.562110:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.562110:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677448.562111:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.562112:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.562113:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.562113:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.562114:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.562115:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.562115:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677448.562116:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.562118:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677448.562119:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.562121:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:18.0:1619677448.562123:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.562124:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.562125:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677448.562126:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.562126:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.562127:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677448.562127:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:18.0:1619677448.562128:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.562128:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:18.0:1619677448.562129:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:1.0:1619677448.562129:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:18.0:1619677448.562130:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.562131:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.562132:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.562132:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677448.562133:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677448.562133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.562135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:8.2:1619677448.562136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.562141:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677448.562141:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677448.562143:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.562144:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:5.0:1619677448.562144:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.562144:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:20.0:1619677448.562145:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:18.0:1619677448.562145:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:20.0:1619677448.562146:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.562147:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.562147:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.562148:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.562150:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:2.0:1619677448.562150:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.562151:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.562152:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:2.0:1619677448.562152:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:20.0:1619677448.562153:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:2.0:1619677448.562153:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.562155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000400:00000200:2.0:1619677448.562155:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.562156:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.562157:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.562160:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.562161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.562161:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.562163:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.562163:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.562164:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.562164:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.562165:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.562166:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.562166:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677448.562167:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677448.562167:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:5.0:1619677448.562168:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:2.0:1619677448.562168:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:20.0:1619677448.562169:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.562169:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.562169:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.0:1619677448.562170:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.562171:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.562171:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677448.562172:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.562172:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.562173:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:9.2:1619677448.562173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.562176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.562182:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.562183:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.562183:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677448.562184:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.562184:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.562185:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.562186:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.562186:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.562187:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677448.562188:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.562256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:20.2:1619677448.562260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677448.562262:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677448.562265:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.562265:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.562266:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.562267:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.562268:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:2.0:1619677448.562269:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.562270:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.562270:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.562271:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.562272:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.562272:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.562273:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.562274:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.562275:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713309] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.562275:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.562276:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.562277:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.562278:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.562280:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.562281:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.562282:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.562283:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.562284:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.562285:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.562285:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.562286:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677448.562287:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.562287:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.562288:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677448.562289:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:5.0:1619677448.562289:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.562289:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.562289:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677448.562290:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.562292:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.562293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.562293:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.562294:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.562300:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:17.0:1619677448.562302:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.562302:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.562302:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677448.562303:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.562303:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677448.562303:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.562304:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:5.0:1619677448.562304:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677448.562304:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.562306:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96713308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.562306:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.562307:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.562308:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.562309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:17.0:1619677448.562310:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.562312:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.562313:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:17.0:1619677448.562314:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.562316:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:13.0:1619677448.562316:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.562319:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.562319:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:13.0:1619677448.562319:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677448.562320:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:13.0:1619677448.562320:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.562321:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.562322:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.562322:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.562323:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.562323:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.562323:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.562324:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.562328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.562336:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.562339:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.562340:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.562340:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.562342:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.562344:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.562347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:6.0:1619677448.562354:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.562357:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.562358:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.562358:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.562359:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.562359:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.562366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.562374:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.562377:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.562379:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.562381:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.562382:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.562384:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.562385:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.562386:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.562387:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.562388:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.562392:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.562393:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.562396:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.562397:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.562397:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.562399:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.562399:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.562401:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.562401:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.562404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.562411:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.562413:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.562414:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.562416:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.562417:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.562418:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.562551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.562559:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.562561:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.562562:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.562565:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.562567:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.562569:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.562570:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.562571:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.562572:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.562573:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.562576:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.562577:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.562579:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.562580:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.562580:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.562582:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.562582:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.562583:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.562583:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.562587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.562593:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.562595:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.562596:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.562596:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.562597:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.562599:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.562605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.562611:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.562613:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.562614:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.562614:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.562614:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.562615:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.562632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.562640:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:4.2:1619677448.562641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.562642:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:24.0:1619677448.562645:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.562646:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.562647:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:24.0:1619677448.562648:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.562648:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677448.562650:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.562650:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.562651:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.562653:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.562654:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.562655:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.562656:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.562658:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.562658:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.562659:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96713304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.562660:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.562661:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.562662:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.562662:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.562666:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.562667:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:8.0:1619677448.562667:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.562668:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.562669:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.562670:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.562671:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.562672:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.562672:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.562673:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.562674:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.562674:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.562674:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.562675:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677448.562676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.562676:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.562676:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.562677:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.562679:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.562679:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.2:1619677448.562680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:8.2:1619677448.562682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.562684:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:4.2:1619677448.562685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677448.562687:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.562688:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.562688:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.562690:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:18.0:1619677448.562690:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.562690:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.562692:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.562693:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.562693:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.562693:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677448.562694:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:18.0:1619677448.562694:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677448.562694:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.562695:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677448.562695:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.562696:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677448.562696:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677448.562697:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.562700:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.562701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.562708:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.562711:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.562712:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.562712:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.562713:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.562714:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.562785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.562793:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.562796:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.562798:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.562799:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.562800:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.562802:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.562803:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.562804:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.562805:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.562806:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.562809:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.562811:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.562814:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.562815:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.562815:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.562817:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.562818:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.562819:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.562819:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.562823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.562831:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.562834:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.562835:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.562835:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.562838:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.562840:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.563031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.563038:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.563041:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.563042:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.563044:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.563045:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.563047:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.563048:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.563050:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.563051:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.563052:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.563055:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.563056:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.563058:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.563059:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.563059:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.563061:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.563062:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.563062:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.563063:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.563067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.563073:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.563076:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.563076:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.563078:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.563080:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.563081:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.563162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:21.2:1619677448.563166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:5.2:1619677448.563166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.0:1619677448.563169:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677448.563171:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677448.563172:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:2.0:1619677448.563172:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:1.0:1619677448.563172:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677448.563174:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000800:00000200:1.0:1619677448.563175:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.563176:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677448.563176:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.563176:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.563177:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:8.0:1619677448.563178:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.563179:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.563180:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.563180:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.563181:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.563182:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.563182:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.563183:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.563183:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:8.0:1619677448.563184:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677448.563184:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677448.563185:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.563186:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.563187:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.563188:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.563188:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.563190:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000400:00000200:2.0:1619677448.563190:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.563190:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677448.563191:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.563194:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677448.563195:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.563195:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.563196:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000400:00000200:2.0:1619677448.563196:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.563197:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.563197:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:8.0:1619677448.563198:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.563199:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677448.563199:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:2.0:1619677448.563199:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.563200:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:1.0:1619677448.563200:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.563201:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.563201:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677448.563202:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:1.0:1619677448.563202:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677448.563203:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677448.563204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:2.0:1619677448.563204:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677448.563204:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677448.563205:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:1.0:1619677448.563205:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.563205:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.563206:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:1.0:1619677448.563206:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677448.563207:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.563210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:24.0:1619677448.563210:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:9.2:1619677448.563211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.563212:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.563213:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.563214:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.563215:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677448.563217:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:18.0:1619677448.563218:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677448.563219:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:18.0:1619677448.563220:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.563221:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.563221:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677448.563221:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677448.563222:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.563222:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.563223:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.563223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:18.0:1619677448.563225:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:6.0:1619677448.563225:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.563227:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677448.563229:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:9.2:1619677448.563232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.563232:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.563233:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.563233:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.563234:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.563235:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:2.0:1619677448.563239:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.563242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677448.563242:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.563243:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.563243:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.563244:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.563244:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.563249:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.563252:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.563253:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.563255:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.563258:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.563259:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:20.2:1619677448.563260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:10.0:1619677448.563261:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.563262:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.563263:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.563264:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.563268:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677448.563268:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.563269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677448.563269:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.563271:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:20.0:1619677448.563271:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.563271:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.563272:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:31.0:1619677448.563273:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.563273:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.563274:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:10.0:1619677448.563274:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:31.0:1619677448.563275:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.563275:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.563275:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.563275:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:20.0:1619677448.563276:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.563276:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:31.0:1619677448.563277:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.563278:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.563278:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677448.563279:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.563279:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.563280:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.563280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:20.0:1619677448.563281:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.563282:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677448.563282:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.563284:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.563285:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.563286:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:20.0:1619677448.563286:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.563288:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.563288:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.563289:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.563290:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.563291:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.563293:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.563302:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:31.0:1619677448.563303:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.563305:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.563306:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.563307:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.563308:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.563308:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.563309:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.563309:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.563310:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.563310:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.563311:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677448.563312:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.563312:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677448.563313:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.563314:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.563316:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.563316:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.563316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.2:1619677448.563319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.563322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.563325:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.563326:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.563328:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.563329:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.563330:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677448.563331:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.563332:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.563332:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.563333:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:20.0:1619677448.563333:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.563333:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:30.0:1619677448.563334:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.563334:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.563334:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.563335:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677448.563337:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.563431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.563437:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.563440:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.563441:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.563443:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.563445:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.563447:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.563448:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.563449:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.563451:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.563452:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.563455:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.563457:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.563460:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.563460:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.563461:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.563463:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.563463:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.563464:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.563465:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.563469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.563477:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.563479:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.563480:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.563481:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.563483:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.563486:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677448.563487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:25.2:1619677448.563489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000800:00000200:5.0:1619677448.563493:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:10.0:1619677448.563495:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:5.0:1619677448.563495:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:10.0:1619677448.563497:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.563498:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:20.0:1619677448.563498:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:10.0:1619677448.563498:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000400:00000200:5.0:1619677448.563498:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677448.563500:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.563500:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:5.0:1619677448.563501:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.563502:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677448.563504:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.563505:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.563507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:5.0:1619677448.563507:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.563508:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.563512:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.563513:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.563514:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.563517:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:5.0:1619677448.563517:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.563517:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677448.563518:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.563518:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.563519:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.563520:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.563520:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:18.0:1619677448.563521:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.563522:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:5.0:1619677448.563522:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000400:00000200:18.0:1619677448.563523:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.563523:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.563524:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.563525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:18.0:1619677448.563525:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.563526:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.563527:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.2:1619677448.563529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000400:00000200:18.0:1619677448.563530:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.563531:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:26.0:1619677448.563532:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:18.0:1619677448.563532:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.563534:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.563534:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:18.0:1619677448.563534:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.563535:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:20.0:1619677448.563535:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677448.563535:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.563535:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.563535:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.563536:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.563537:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.563537:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:26.0:1619677448.563538:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.563538:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.563539:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677448.563541:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.563543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.563550:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.563553:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.563554:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.563554:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.563556:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677448.563559:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.563561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:21.2:1619677448.563565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677448.563566:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677448.563569:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677448.563569:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677448.563571:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.563572:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:22.0:1619677448.563574:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.563575:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:22.0:1619677448.563576:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:22.0:1619677448.563577:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.563578:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.563579:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677448.563579:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.563580:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.563580:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.563581:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.563582:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.563583:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.563585:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.563586:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.563587:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.563592:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.563593:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.563595:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.563596:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.563598:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.563599:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:22.0:1619677448.563599:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.563600:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.563600:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.563601:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677448.563601:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.563602:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.563603:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677448.563603:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:22.0:1619677448.563603:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:30.0:1619677448.563604:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.563604:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677448.563605:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:25.2:1619677448.563607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:25.2:1619677448.563611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:12.0:1619677448.563614:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677448.563617:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:12.0:1619677448.563617:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.563617:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.563619:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.563620:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.563621:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.563621:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.563622:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677448.563623:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.563624:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.563626:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.563627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.563634:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.563637:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.563638:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.563638:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.563639:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.563640:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.563807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.563813:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.563816:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.563817:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.563819:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.563821:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.563823:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.563824:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.563826:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.563827:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.563828:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.563831:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.563832:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.563836:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.563836:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.563837:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.563839:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.563840:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.563841:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.563841:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.563845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.563852:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.563855:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.563855:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.563857:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.563858:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.563860:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.563885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.563893:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.563896:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.563898:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.563899:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.563900:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.563902:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.563903:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.563903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:31.0:1619677448.563904:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.563905:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.563906:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.563906:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.563908:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:31.0:1619677448.563911:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.563911:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677448.563914:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:20.0:1619677448.563915:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.563917:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.563918:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:20.0:1619677448.563918:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.563919:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:20.0:1619677448.563919:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677448.563920:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.2:1619677448.563920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677448.563921:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:20.0:1619677448.563921:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.563922:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.563922:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.563923:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.563923:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.563924:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.563926:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.2:1619677448.563926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:20.0:1619677448.563928:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.563929:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.0:1619677448.563930:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.563930:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.563932:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.563933:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.563934:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:17.0:1619677448.563934:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.563934:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.563935:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:17.0:1619677448.563936:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677448.563936:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.563937:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.563937:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:17.0:1619677448.563937:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96713287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.563937:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.563938:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.563939:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677448.563939:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.563939:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.563940:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:17.0:1619677448.563940:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.563941:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.563942:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:25.2:1619677448.563943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:17.0:1619677448.563945:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.563947:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.2:1619677448.563947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:12.0:1619677448.563949:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.563950:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.563951:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:17.0:1619677448.563952:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (24)++
00000800:00000200:17.0:1619677448.563955:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (25)--
00000800:00000200:17.0:1619677448.563955:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.563955:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.563956:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:17.0:1619677448.563957:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677448.563957:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677448.563957:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.563957:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.563958:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.563958:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.563958:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677448.563959:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.563960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677448.563961:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.563963:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:10.0:1619677448.563963:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.563963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.563964:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677448.563964:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:12.0:1619677448.563965:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.563966:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:12.0:1619677448.563966:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.563966:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.563967:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.563968:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:12.0:1619677448.563968:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.563969:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000400:00000200:10.0:1619677448.563969:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.563969:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677448.563970:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677448.563970:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:10.0:1619677448.563970:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:12.0:1619677448.563971:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.563971:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:13.0:1619677448.563972:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.563972:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.563973:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:10.0:1619677448.563974:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.563976:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.2:1619677448.563978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:10.0:1619677448.563978:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.563979:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.563979:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.563981:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.563981:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.563982:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.563983:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:18.0:1619677448.563984:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.563984:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.563986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:18.0:1619677448.563987:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.563988:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.563990:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.563990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.563993:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:18.0:1619677448.563993:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.563994:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:30.0:1619677448.563995:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:18.0:1619677448.563995:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.563996:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.563997:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:18.0:1619677448.563997:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.563998:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.563999:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677448.563999:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.563999:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:18.0:1619677448.563999:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.564000:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.564001:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:26.0:1619677448.564001:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:18.0:1619677448.564002:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.564004:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.564005:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.564006:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.564007:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.564008:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.564009:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.564010:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.564010:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.564015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.564022:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.564024:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.564025:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.564026:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.564027:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.564028:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.564054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677448.564056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.564062:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.564064:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.564065:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.564066:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677448.564067:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.564068:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.564070:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.564071:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.564072:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.564074:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713284] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.564075:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.564076:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.564077:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.564079:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.564081:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.564081:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.564082:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.564083:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.564083:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.564084:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.564087:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.564088:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:26.0:1619677448.564089:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.564089:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.564090:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.564091:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.564092:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.564093:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.564093:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.564094:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.564094:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.564095:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.564095:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.564096:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.564097:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.2:1619677448.564097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.564098:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.564099:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.564099:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677448.564103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.564105:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.564107:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.564108:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.564109:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.564111:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677448.564111:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.564113:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677448.564114:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.564115:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.564115:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.564117:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.564120:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.564237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.564244:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.564247:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.564249:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.564251:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.564252:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.564253:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.564255:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.564256:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.564257:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.564258:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.564260:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.564262:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.564264:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.564265:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.564266:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.564267:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.564268:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.564269:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.564271:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.564272:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.564273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.564277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.564279:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.564282:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.564282:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.564282:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.564285:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:10.0:1619677448.564285:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.564287:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:10.0:1619677448.564287:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:30.0:1619677448.564288:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.564289:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677448.564485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.564493:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.564495:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.564496:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.564497:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.564500:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.564502:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.564502:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.564504:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.564505:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.564505:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.564508:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.564510:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.564513:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.564513:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.564514:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.564516:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.564516:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.564517:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.564517:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.564522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.564530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.564533:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.564535:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:20.2:1619677448.564536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677448.564536:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:1.0:1619677448.564536:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.564538:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.564538:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.564540:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.564542:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.564543:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.564543:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677448.564544:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:1.0:1619677448.564544:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.564545:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.564546:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.564547:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:17.0:1619677448.564548:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.564550:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.564550:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.564551:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:17.0:1619677448.564551:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677448.564553:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.564556:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.564558:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:17.0:1619677448.564558:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:30.0:1619677448.564560:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.564560:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.564561:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.564561:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:30.0:1619677448.564562:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.564562:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:30.0:1619677448.564563:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.564564:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.564564:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.564564:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.564565:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677448.564567:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.564568:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:25.2:1619677448.564570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.564571:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.564572:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.564572:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.564574:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.564575:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.564576:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.564576:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.564578:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.564580:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:25.2:1619677448.564580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:30.0:1619677448.564581:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.564583:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.564584:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.564586:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:12.0:1619677448.564587:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.564590:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.564590:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.564592:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.564594:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.564595:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.564604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.564610:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.564612:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.564614:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.564615:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.564617:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.564619:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677448.564620:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.564622:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.564623:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.564624:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677448.564627:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.564629:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:4.2:1619677448.564630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677448.564631:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.564632:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677448.564633:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677448.564634:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677448.564635:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.564637:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:26.0:1619677448.564638:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677448.564638:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.564638:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677448.564640:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.564641:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677448.564641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:26.0:1619677448.564643:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.564643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677448.564646:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.564646:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:26.0:1619677448.564648:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677448.564648:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:26.0:1619677448.564649:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.564649:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677448.564649:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:20.0:1619677448.564650:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:26.0:1619677448.564651:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.564651:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:26.0:1619677448.564652:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.564652:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.564653:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.564653:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.564655:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.564657:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.564658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.564658:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:26.0:1619677448.564659:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.564662:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.564663:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.564664:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.564665:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.564665:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:26.0:1619677448.564666:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.564667:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.564667:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:18.0:1619677448.564668:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.564669:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.564669:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.564670:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.564670:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:25.2:1619677448.564671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.564679:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.564682:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.564683:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.564684:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.564686:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.564689:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.564690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.564698:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.564700:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.564701:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.564702:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.564703:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.564703:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.564708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.564715:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.564718:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.564720:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.564722:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.564722:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.564724:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.564725:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713276] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.564727:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.564727:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.564728:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.564731:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.564733:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.564735:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.564735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:31.0:1619677448.564736:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.564737:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.564738:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.564739:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.564740:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.564740:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677448.564741:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:25.2:1619677448.564744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.564744:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.564745:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.564746:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.564747:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.564748:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.564749:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.564750:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.564750:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:2.0:1619677448.564751:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.564752:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.564753:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.564754:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.564754:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.564755:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677448.564756:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.564756:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.564757:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:18.0:1619677448.564758:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.564760:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96713274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.564761:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.564761:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.564762:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.564763:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.564764:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.564765:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:18.0:1619677448.564766:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.564768:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.564770:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.564770:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.564771:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.564771:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.564772:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.564772:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.564775:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.564775:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)++
00000800:00000200:2.0:1619677448.564775:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.564775:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)++
00000800:00000200:2.0:1619677448.564777:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (25)++
00000800:00000200:18.0:1619677448.564778:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (26)++
00000800:00000200:5.2:1619677448.564778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677448.564778:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (26)--
00000800:00000200:2.0:1619677448.564779:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (26)--
00000800:00000200:18.0:1619677448.564780:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (26)--
00000800:00000200:2.0:1619677448.564780:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (24)--
00000800:00000200:25.2:1619677448.564781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.564781:0:19933:0:(o2iblnd_cb.c:929:kiblnd_post_tx_locked()) 10.51.3.43@o2ib3(1): redundant or enough NOOP
00000800:00000200:18.0:1619677448.564782:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:9.2:1619677448.564782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.564783:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677448.564784:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.564784:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677448.564785:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.2:1619677448.564785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.564786:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.564786:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.564787:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.564788:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:18.0:1619677448.564789:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.564789:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.564790:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:13.0:1619677448.564791:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.564792:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.564792:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96713273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677448.564794:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.564794:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677448.564794:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.564795:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.564796:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:18.0:1619677448.564798:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.564799:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:13.0:1619677448.564800:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677448.564801:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:2.0:1619677448.564801:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.564802:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.564803:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.564803:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677448.564804:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677448.564805:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677448.564807:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.564808:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:2.0:1619677448.564809:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:2.0:1619677448.564810:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:2.0:1619677448.564811:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677448.564812:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677448.564812:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677448.564816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.564823:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.564826:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.564827:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.564827:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.564829:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.564832:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.2:1619677448.565026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:4.2:1619677448.565026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.565033:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.565033:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.565036:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:26.0:1619677448.565037:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.565037:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.565037:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:1.0:1619677448.565040:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.565041:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.565044:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.565045:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.565045:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.565047:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677448.565047:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.565048:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.565049:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.565050:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.565051:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.565051:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.565052:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.565054:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.565056:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.565059:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.565059:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.565061:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.565062:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.565063:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.565064:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.565064:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.565065:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.565066:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.565066:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.565066:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.565067:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.565068:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.565068:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.565068:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.565069:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.565071:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.565071:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.565072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:9.2:1619677448.565074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677448.565077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:12.0:1619677448.565080:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.565081:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:12.0:1619677448.565083:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:26.0:1619677448.565084:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:18.0:1619677448.565084:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:12.0:1619677448.565084:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.565085:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.565085:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.565085:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.565086:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.565087:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:13.0:1619677448.565087:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.565087:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:18.0:1619677448.565088:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.565088:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.565091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:9.2:1619677448.565091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.565095:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.565097:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677448.565098:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.565099:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.565100:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.565100:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.565101:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677448.565101:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.565102:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677448.565102:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.565103:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.565103:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.565104:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.565131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.565137:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.565140:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.565141:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.565142:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.565145:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.565146:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.565148:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.565149:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.565150:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.565151:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.565154:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.565156:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.565158:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.565159:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.565160:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.565161:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.565161:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.565162:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.565162:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.565167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.565174:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.565176:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.565176:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.565177:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.565178:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.565180:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.565311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.565314:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:4.2:1619677448.565317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677448.565318:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:5.0:1619677448.565320:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.565322:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.565324:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.565325:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677448.565326:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677448.565327:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.565328:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.565328:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677448.565329:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.565330:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.565332:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.565332:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:21.2:1619677448.565333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:26.0:1619677448.565335:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.565335:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.565336:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.565337:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.565337:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.565339:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.565339:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:26.0:1619677448.565340:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.565340:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.565341:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.565341:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677448.565341:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677448.565342:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.565342:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:20.0:1619677448.565343:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677448.565344:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677448.565345:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.565345:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.565345:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677448.565347:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:6.0:1619677448.565348:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.565350:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677448.565350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.565351:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.565351:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:6.0:1619677448.565351:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.565353:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.565353:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677448.565353:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:26.0:1619677448.565354:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.565354:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96713267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.565355:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.565355:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.565356:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.565357:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.565359:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:25.2:1619677448.565360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.565361:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:6.0:1619677448.565361:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.565362:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.565363:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.565363:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.565365:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.565366:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.565367:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:22.0:1619677448.565368:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.565368:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.565368:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.565370:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.565370:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:22.0:1619677448.565371:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.565371:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.565372:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.565373:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.565373:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677448.565374:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.565376:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.2:1619677448.565376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.565385:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.565387:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.565388:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.565390:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.565391:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.565392:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.565448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.565454:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.565456:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.565457:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.565459:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.565461:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.565462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:1.0:1619677448.565463:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.565464:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.565465:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.565466:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.565467:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:30.0:1619677448.565470:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677448.565470:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.565472:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677448.565473:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:1.0:1619677448.565474:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.565475:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.565475:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.565475:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.565477:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.565477:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.565478:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.565478:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.565478:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677448.565479:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677448.565479:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677448.565480:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.565482:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.565483:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.565483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:30.0:1619677448.565484:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.565487:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.565488:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.565490:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:30.0:1619677448.565491:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.565492:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.565492:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.565492:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.565493:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.565494:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.565494:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.565494:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677448.565495:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677448.565495:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.565496:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677448.565497:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:25.2:1619677448.565499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.565506:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.565509:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.565510:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.565512:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.565513:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.565515:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.565578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.565585:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.565588:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.565589:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.565591:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.565594:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.565596:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.565597:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.565598:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.565599:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.565600:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.565604:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.565605:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.565607:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.565608:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.565609:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.565610:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.565611:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.565611:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.565612:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.565616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.565622:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.565625:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.565626:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.565627:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.565629:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.565630:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.565660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.565668:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.565670:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.565672:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.565674:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.565674:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.565675:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.565677:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.565678:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.565679:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.565679:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.565682:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.565684:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.565686:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.565687:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.565687:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.565689:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.565689:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.565690:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.565691:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.2:1619677448.565691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:25.2:1619677448.565694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.565694:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677448.565696:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677448.565699:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.565700:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677448.565701:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.565701:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:20.0:1619677448.565703:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:12.0:1619677448.565703:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:20.0:1619677448.565704:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.565704:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.565706:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.565706:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.565707:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.565708:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:13.0:1619677448.565708:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.565708:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.565712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:20.0:1619677448.565712:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.565713:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.565716:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.565717:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.565718:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:13.0:1619677448.565718:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.565720:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.565720:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677448.565721:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:13.0:1619677448.565721:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.565722:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:13.0:1619677448.565722:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.565723:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.565724:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:25.2:1619677448.565725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:13.0:1619677448.565725:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.565728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:26.0:1619677448.565732:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:5.0:1619677448.565733:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.565734:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.565735:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:5.0:1619677448.565735:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:26.0:1619677448.565736:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.565738:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.565738:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.565739:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.565741:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.565743:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:5.0:1619677448.565744:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.565746:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.565747:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.565748:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.565749:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.565753:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.565755:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.565758:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.565758:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.565759:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.565761:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.565761:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.565763:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.565763:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.565767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.565770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677448.565774:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.565776:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.565777:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.565777:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:31.0:1619677448.565779:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.565780:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:24.0:1619677448.565780:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.565780:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677448.565782:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:24.0:1619677448.565782:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.565783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:24.0:1619677448.565785:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.565786:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.565787:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.565789:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.565790:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:24.0:1619677448.565790:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.565791:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:30.0:1619677448.565793:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.565794:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.565795:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:24.0:1619677448.565795:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.565796:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.565797:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:24.0:1619677448.565797:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.565800:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.565800:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.565801:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.565802:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.565803:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.565804:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.565804:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.565808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.565817:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.565819:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.565820:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.565821:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.565823:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.565825:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.565832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.565839:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.565841:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.565842:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.565842:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.565843:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.565844:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.565896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677448.565900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.565903:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.565905:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.565906:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677448.565906:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.565908:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.565908:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.565909:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.565910:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.565911:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.565912:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677448.565912:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.565913:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.565914:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.565915:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.565915:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.565916:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.565917:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677448.565917:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.565918:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.565919:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.565921:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.565923:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.565926:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.565926:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.565927:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.565928:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677448.565928:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.565930:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.565930:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.565930:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677448.565930:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.565931:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677448.565931:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.565932:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.565933:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.565934:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.565935:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:25.2:1619677448.565936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677448.565936:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.565937:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.565938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.565941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.565945:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.565947:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.565947:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677448.565947:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.565948:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.565949:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.565949:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.565949:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.565950:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677448.565951:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.565951:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.565951:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.565955:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.565955:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.565956:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677448.565956:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.565957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.565962:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.565965:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.565966:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.565968:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.565969:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.565971:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.565972:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.565973:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.565974:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.565975:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.565979:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.565981:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.565983:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.565984:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.565985:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.565986:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.565987:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.565987:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.565988:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.565993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.566001:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.566003:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.566004:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.566005:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.566006:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.566009:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.566145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.566148:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.566151:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.566153:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.566154:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.566156:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.566158:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677448.566159:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.566160:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.566161:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.566163:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.566166:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.566168:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.566170:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.566171:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.566172:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.566173:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.566174:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.566175:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.566175:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677448.566179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.566187:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.566190:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.566190:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.566191:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.566193:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677448.566196:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.566243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.566249:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.566253:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.566253:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.566255:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.566257:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.566258:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.566260:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.566261:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.566263:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.566264:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.566267:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.566269:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.566271:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.566272:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.566273:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.566275:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.566276:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.566276:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.566277:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.566281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.566289:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.2:1619677448.566292:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677448.566294:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:8.2:1619677448.566300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.566301:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.566302:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.566303:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:5.0:1619677448.566304:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.566305:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.566306:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.0:1619677448.566306:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:26.0:1619677448.566308:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.566308:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:26.0:1619677448.566311:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.566311:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.566311:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.566312:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000400:00000200:26.0:1619677448.566313:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677448.566313:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:26.0:1619677448.566314:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713254] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.566314:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:26.0:1619677448.566315:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.566316:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.566318:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:4.2:1619677448.566319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:26.0:1619677448.566321:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.566324:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.566327:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.566328:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.566329:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.566329:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.566330:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.566331:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.566331:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.566333:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.566333:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.566333:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.566334:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:10.0:1619677448.566334:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:26.0:1619677448.566335:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.566336:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.566338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.566338:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.566338:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.566339:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.566342:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.566343:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:12.0:1619677448.566345:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.566346:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.566347:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.566347:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:12.0:1619677448.566348:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.566348:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.566349:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.566349:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:12.0:1619677448.566350:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.566350:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.566351:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:12.0:1619677448.566352:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.566352:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:12.0:1619677448.566353:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.566355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.566357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.566363:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.566364:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.566366:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677448.566366:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.566367:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.566368:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.566368:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.566368:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.566370:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.566373:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.566386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.566389:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.566391:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.566393:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.566394:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.566396:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.566398:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.566399:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.566400:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.566401:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.566402:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.566406:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.566407:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.566410:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.566410:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:21.2:1619677448.566411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677448.566411:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.566413:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.566413:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.566414:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.566415:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.566419:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:9.2:1619677448.566419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.566422:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.566424:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.566426:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.566427:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677448.566427:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:30.0:1619677448.566428:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.566429:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.566429:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:30.0:1619677448.566430:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.566430:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.566431:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:30.0:1619677448.566432:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.566433:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.566434:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.566436:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:30.0:1619677448.566437:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.566439:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.566442:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.566443:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.566443:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.566445:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.566445:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.566446:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.566447:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.566450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.566457:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.566459:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.566460:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.566461:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.566463:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.566463:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:21.2:1619677448.566497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.566502:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.566505:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.566506:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.566508:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.566510:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.566512:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.566513:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.566515:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.566516:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.566517:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.566521:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.566522:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.566525:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.566526:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.566527:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.2:1619677448.566527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:6.0:1619677448.566528:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.566529:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.566530:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.566531:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.566532:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677448.566533:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:25.2:1619677448.566535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.566536:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:21.2:1619677448.566537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677448.566537:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.566539:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.566541:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.566542:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677448.566542:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:17.0:1619677448.566542:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:31.0:1619677448.566544:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:22.0:1619677448.566544:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:17.0:1619677448.566544:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.566545:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.566545:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:17.0:1619677448.566545:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.566546:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:17.0:1619677448.566546:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.566548:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.566548:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:17.0:1619677448.566548:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.566551:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:17.0:1619677448.566551:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.566553:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.566556:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.566557:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.566558:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.566560:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.566560:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.566561:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.566561:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.566567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.566574:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.566576:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.566577:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.566579:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.566580:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.566582:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.566653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.566661:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.566664:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.566666:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.566668:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.566668:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.566670:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.566671:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.566672:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.566673:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.566674:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.566678:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.566679:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.566681:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.566682:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.566683:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.566684:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.566685:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.566686:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.566687:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.566688:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.566690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.2:1619677448.566694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.566698:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.566699:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.566700:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.566701:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.566702:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.566703:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.566703:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.566705:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.566706:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.566707:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:21.2:1619677448.566715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.566721:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.566723:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.566724:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.566726:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.566728:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.566730:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.566731:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713247] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.566732:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.566733:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.566734:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.566736:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.566738:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.566739:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.566740:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.566741:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.566742:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.566743:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.566743:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.566744:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.566748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.566755:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.566757:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.566757:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.566759:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.566759:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.566761:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.566789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.566796:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.566799:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.566800:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.566802:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.566805:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.566806:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.566807:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.566809:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.566810:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.566811:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.566814:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.566816:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.2:1619677448.566818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.566819:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.566820:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.566820:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.566823:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.566824:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.566825:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.566825:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.566826:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.566828:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:25.2:1619677448.566829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.566829:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.566830:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.566832:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.566833:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.566834:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713245] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.566835:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.566836:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.566837:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677448.566837:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:26.0:1619677448.566840:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.566840:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.566840:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.566841:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:6.0:1619677448.566841:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.566843:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.566844:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.566845:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.566846:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.566846:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:26.0:1619677448.566848:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.566848:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.566849:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.566849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.566850:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.566854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677448.566856:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.566859:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.566860:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.566860:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.566861:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.566861:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:22.0:1619677448.566862:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677448.566864:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.566865:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.566865:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.566867:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.566871:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.566872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.566878:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.566881:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.566882:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.566882:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.566883:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.566884:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.566921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.2:1619677448.566924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:5.0:1619677448.566928:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677448.566930:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677448.566931:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:18.0:1619677448.566933:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:20.0:1619677448.566934:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.566934:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.566934:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.566937:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.566937:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.566939:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.566939:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.566941:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677448.566941:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.566943:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.566943:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.566944:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.566945:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.566946:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.566946:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.566947:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.566953:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.566953:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.566956:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.566957:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.566959:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.566960:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677448.566960:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.566961:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.566961:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.566961:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.566963:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.566963:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.566964:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677448.566964:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677448.566965:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:5.0:1619677448.566966:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:18.0:1619677448.566967:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.566967:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677448.566967:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.566968:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.566969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:9.2:1619677448.566969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.566973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.2:1619677448.566973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.566976:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:10.0:1619677448.566976:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:2.0:1619677448.566978:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677448.566979:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:10.0:1619677448.566979:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.566980:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:20.0:1619677448.566980:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677448.566980:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.566980:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:20.0:1619677448.566981:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677448.566981:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:10.0:1619677448.566981:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.566981:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.566982:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.566983:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.566984:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:10.0:1619677448.566984:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.566984:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.566987:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.566988:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.566992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.566998:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.567001:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.567001:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.567002:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.567003:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.567003:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.567070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.567075:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.567078:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.567079:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.567081:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.567083:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.567085:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.567086:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.567087:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.567088:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.567089:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.567093:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.567095:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.567098:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.567099:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.567099:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.567101:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.567102:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.567102:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.567103:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.567107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.567114:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.567116:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.567117:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.567119:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.567120:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.567122:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.567161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.567167:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.567170:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.567171:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.567173:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.567176:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.567177:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.567178:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.567179:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.567180:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.567182:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.567185:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.567187:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.567190:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.567191:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.567191:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.567193:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.567193:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.567194:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.567194:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.567199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.567206:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.567209:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.567210:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.567212:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.567213:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.567214:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.567286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.567293:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.567299:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.567301:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.567303:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.567306:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.567307:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.567308:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.567309:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.567310:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.567311:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.567315:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.567317:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.567319:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.567320:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.567320:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.567322:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.567322:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.567323:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.567323:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.567328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.567334:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.567336:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.567337:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.567337:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.567338:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.567340:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.567427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.567436:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.567439:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.567441:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.567443:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.567443:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.567444:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.567446:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.567447:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.567448:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.567448:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.567452:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.567453:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.567456:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.567457:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.567457:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.567459:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.567460:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.567460:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.567461:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.567466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.567474:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.567476:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.567477:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.567479:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.567480:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.567482:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.567485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.567492:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.567495:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.567497:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.567499:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.567500:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.567502:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.567503:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.567504:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.567505:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.567506:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.567509:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.567511:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.567514:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.567515:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.567515:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.567517:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.567518:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.567520:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.567521:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.567522:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.567523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.567527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677448.567531:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.567534:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:30.0:1619677448.567535:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:22.0:1619677448.567535:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.567535:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.567537:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:22.0:1619677448.567537:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.567538:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.567539:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677448.567540:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.567795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.567800:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.567803:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.567805:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677448.567806:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.567807:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.567809:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677448.567810:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.567812:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.567813:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.567814:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.567817:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.567818:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.567821:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.567822:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677448.567823:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.567824:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.567825:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.567826:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.567826:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.567830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.567837:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.567839:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.567840:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.567840:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.567843:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:10.0:1619677448.567846:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.567861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.567867:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.567870:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.567870:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.567872:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.567875:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.567877:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.567878:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.567879:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.567880:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.567881:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.567885:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.567887:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.567890:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.567891:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.567891:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.567893:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.567893:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.567894:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.567895:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.567899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.567907:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.567910:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.567911:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.567912:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.567914:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.567915:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.567942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.567949:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.567952:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.567952:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.567954:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.567956:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.567956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:26.0:1619677448.567958:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.567959:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.567961:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.567962:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.567962:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:26.0:1619677448.567963:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.567965:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.567966:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:26.0:1619677448.567967:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.567968:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.567969:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.567972:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.567972:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.567973:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.567974:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:18.0:1619677448.567974:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:26.0:1619677448.567975:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:18.0:1619677448.567975:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.567976:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:18.0:1619677448.567976:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.567977:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.567977:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:18.0:1619677448.567977:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.567978:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:25.2:1619677448.567981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:18.0:1619677448.567981:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.567983:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.567986:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.567987:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.567988:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.567989:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:6.0:1619677448.567989:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:18.0:1619677448.567990:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.567991:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.567991:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:22.0:1619677448.567992:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.567992:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.567993:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.567995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:6.0:1619677448.567995:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.567997:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:10.0:1619677448.568001:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:20.2:1619677448.568002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.568004:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.568006:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.568007:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.568008:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:10.0:1619677448.568008:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.568009:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.568011:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:26.0:1619677448.568012:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677448.568012:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.568015:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.568017:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.568018:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.568019:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.568020:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.568021:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.568024:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.568025:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.568028:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.568029:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.568030:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.568032:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.568032:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.568033:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.568033:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.568038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.568046:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.568048:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.568049:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.568050:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.568051:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.568054:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.568135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.568140:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.568143:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.568144:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.568146:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.568148:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.568150:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.568151:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.568153:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.568153:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.568155:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.568158:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.568159:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.568162:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.568163:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.568164:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.568165:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.568166:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.568167:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.568167:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.568171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.568177:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.568180:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.568181:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.568181:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.568183:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.568186:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.568213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.568218:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.568221:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.568222:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.568224:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.568227:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.568229:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.568230:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.568231:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.568232:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.568233:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.568237:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.568239:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.568241:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.568242:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.568242:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.568244:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.568245:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.568245:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.568246:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.568251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.568258:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.568260:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.568261:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.568261:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.568264:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.568266:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.568273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.568280:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.568283:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.568285:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.568287:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.568288:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.568290:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.568291:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.568292:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.568299:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.568300:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.568303:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.568305:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.568309:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.568309:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.568310:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.568312:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.568313:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.568314:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.568314:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.568319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.568327:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.568330:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.568330:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.568331:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.568333:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.568336:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.568377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.568383:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.568386:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.568388:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.568388:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.568390:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.568392:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677448.568393:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.568395:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.568395:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.568397:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677448.568400:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.568402:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677448.568404:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.568405:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677448.568406:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677448.568407:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677448.568408:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.568409:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.568409:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.568414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.568421:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.568424:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.568425:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.568425:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.568427:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.568429:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.568433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.568441:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.568444:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.568445:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.568447:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.568450:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.568451:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.568453:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.568454:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.568455:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.568456:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.568459:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.568461:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.568463:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.568464:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.568464:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.568466:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.568466:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.568467:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.568469:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.568469:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.568472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.2:1619677448.568475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.568479:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.568482:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:10.0:1619677448.568482:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:31.0:1619677448.568483:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.568484:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.568484:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.568484:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.568485:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.568486:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.568488:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.568660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.568668:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.568671:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.568672:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.568674:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.568677:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.568678:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.568679:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.568680:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.568681:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.568682:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.568685:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.568686:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.568689:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.568690:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.568691:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.568692:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.568693:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.568693:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.568694:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.568698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.568706:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.2:1619677448.568707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677448.568709:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.568710:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.568710:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.568712:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.568714:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677448.568715:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677448.568717:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.568719:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.568721:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.568722:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.568724:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.568725:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.568726:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.568727:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.568728:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.568731:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.568733:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.568736:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.568737:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.568737:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.568739:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.568740:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.568741:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.568741:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.568745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.568753:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.568756:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.568757:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.568758:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.568759:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.568762:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.568832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.568838:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.568840:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.568841:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.568843:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.568845:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.568847:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.568848:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.568850:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.568851:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.568852:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.568855:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.568856:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.568859:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.568860:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.568861:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.568862:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.568863:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.568864:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.568865:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.568866:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.568868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.568871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.568875:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677448.568876:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677448.568878:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:17.0:1619677448.568878:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:24.0:1619677448.568879:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.568879:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677448.568879:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:17.0:1619677448.568880:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677448.568881:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.568885:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.568975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.568981:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.568984:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.568985:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.568986:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.568988:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.568990:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677448.568991:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.568992:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.568993:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.568994:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677448.568998:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.569000:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677448.569003:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.569003:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677448.569004:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677448.569006:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677448.569006:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.569008:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.569009:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.569009:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677448.569012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:21.2:1619677448.569014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:26.0:1619677448.569018:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:18.0:1619677448.569020:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:26.0:1619677448.569021:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.569021:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.569022:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.569022:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.569023:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:26.0:1619677448.569024:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.569024:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677448.569027:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.569189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.569196:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.569199:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.569199:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.569201:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.569203:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.569204:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.569205:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.569206:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.569207:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.569208:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.569211:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.569213:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.569215:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.569216:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.569217:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.569218:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.569219:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.569220:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.569220:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.569224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:4.2:1619677448.569226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.569232:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677448.569232:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:26.0:1619677448.569235:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:13.0:1619677448.569235:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.569235:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.569236:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:13.0:1619677448.569237:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.569238:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.569238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:12.0:1619677448.569240:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:26.0:1619677448.569241:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.569241:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:26.0:1619677448.569242:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:25.2:1619677448.569242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:26.0:1619677448.569244:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.569245:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677448.569245:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.569246:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.569246:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.569248:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:12.0:1619677448.569248:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:26.0:1619677448.569250:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.569251:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.569252:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.569252:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:12.0:1619677448.569252:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.569252:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:31.0:1619677448.569253:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.569253:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.569254:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.569254:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.569255:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:31.0:1619677448.569256:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.569256:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.569257:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:31.0:1619677448.569258:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:26.0:1619677448.569258:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.569259:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.569259:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.569259:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:31.0:1619677448.569260:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.569261:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.569262:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:25.2:1619677448.569263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:31.0:1619677448.569265:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.569267:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.569269:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.569269:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.569270:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.569270:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.569271:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:30.0:1619677448.569271:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.569271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:31.0:1619677448.569272:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.569272:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.569273:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.569273:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.569274:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677448.569275:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.569276:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.569277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.569277:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.569280:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.569281:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:10.0:1619677448.569283:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:2.0:1619677448.569283:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.569286:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:2.0:1619677448.569286:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.569287:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.569287:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.569287:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.569289:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.569289:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.569290:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.569291:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.569292:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:2.0:1619677448.569292:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:25.2:1619677448.569298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000400:00000200:2.0:1619677448.569302:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.569303:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.569304:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.569306:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.569307:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.569307:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.569308:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.569308:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.569309:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.569310:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.569310:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:2.0:1619677448.569310:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.569311:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.569311:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.569312:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677448.569313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:25.2:1619677448.569316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.569319:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.569322:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:12.0:1619677448.569323:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:5.0:1619677448.569323:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.569324:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677448.569326:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:5.0:1619677448.569326:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.569327:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.569328:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.569329:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.569330:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.569331:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677448.569331:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.569332:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.569333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.569333:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677448.569333:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.569337:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.569339:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.569340:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.569342:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.569342:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.569343:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.569343:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:5.0:1619677448.569346:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:5.0:1619677448.569346:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677448.569347:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677448.569347:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.569347:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677448.569348:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.569348:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.569349:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:25.2:1619677448.569351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.569357:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.569359:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.569360:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.569362:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.569363:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.2:1619677448.569365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:12.0:1619677448.569365:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.569373:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.569375:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.569378:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.569380:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.569380:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.569382:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.569383:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.569384:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.569385:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.569386:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.569390:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.569391:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.569394:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.569395:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.569395:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.569397:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.569398:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.569399:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.569399:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.569402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.569408:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.569411:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.569411:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.569413:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.569415:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.569415:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.2:1619677448.569549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.569555:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.569558:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.569559:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.569561:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.569564:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.569566:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.569567:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.569568:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.569569:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.569570:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.569574:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.569575:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.569579:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.569579:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.569580:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.569582:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.569582:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.569583:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.569583:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.569587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.569594:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.569596:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.569597:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.569599:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.569600:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.569602:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.569821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.569827:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.569829:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.569830:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.569832:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.569835:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.569836:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.569837:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.569839:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.569840:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.569840:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.569843:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.569845:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.569847:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.569847:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.569848:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.569849:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.569850:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.569851:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.569851:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.569856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.569864:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.569866:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.569866:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.569867:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.569868:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.569870:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.569886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.569889:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.569892:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.569894:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.569895:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:4.2:1619677448.569897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:20.0:1619677448.569898:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.569901:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.569902:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.569903:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.569904:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.569904:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677448.569905:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.569908:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.569908:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:20.0:1619677448.569909:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.569910:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.569912:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.569913:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.569915:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.569915:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677448.569916:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.569917:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:10.0:1619677448.569917:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.569918:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.569919:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.569919:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:10.0:1619677448.569919:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.569920:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:10.0:1619677448.569920:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.569921:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:10.0:1619677448.569924:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.569925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.569925:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.569928:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.569929:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.569930:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.569931:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.569932:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:10.0:1619677448.569932:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.569933:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.569934:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:30.0:1619677448.569935:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:10.0:1619677448.569935:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:30.0:1619677448.569936:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.569937:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.569937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.569939:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:4.2:1619677448.569940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.569941:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.569943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:4.2:1619677448.569944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677448.569945:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.569947:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.569948:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677448.569949:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.569949:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677448.569949:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.569950:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.569950:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.569951:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.0:1619677448.569951:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677448.569952:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.569952:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.569954:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:24.0:1619677448.569954:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.569954:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:31.0:1619677448.569955:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.569955:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.569955:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:8.0:1619677448.569956:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.569957:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:8.0:1619677448.569959:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.569960:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.569962:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.569963:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.569964:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.569965:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.569968:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.569969:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.569972:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.569973:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.569973:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.569975:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.569975:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.569977:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.569978:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.569979:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.569981:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677448.569984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.569988:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677448.569991:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.569991:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.569992:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.569992:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.569993:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.569994:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.569994:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.569995:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677448.569997:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.570043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.570049:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.570052:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.570052:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.570054:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.570057:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.570059:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.570060:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.570062:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.570063:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.570064:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.2:1619677448.570066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:18.0:1619677448.570067:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.570069:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.570072:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.570073:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677448.570073:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.570074:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.570076:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:18.0:1619677448.570076:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.570076:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.570077:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:31.0:1619677448.570078:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.570078:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.570079:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:30.0:1619677448.570080:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.570081:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.570082:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:25.2:1619677448.570082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.570084:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.570084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000400:00000200:31.0:1619677448.570085:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.570086:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.570087:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.570088:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.570089:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:31.0:1619677448.570091:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.570091:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.570091:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:13.0:1619677448.570092:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.570092:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677448.570093:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:13.0:1619677448.570093:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.570093:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677448.570094:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.570096:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.570096:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.570097:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.570098:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.570100:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.570100:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:21.2:1619677448.570100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.570102:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.570103:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.570104:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.570105:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.570107:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.2:1619677448.570107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.570108:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.570109:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.570109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:31.0:1619677448.570111:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.570112:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.570113:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.570115:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.570115:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.570116:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.570116:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677448.570117:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.570118:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:5.0:1619677448.570118:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677448.570119:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:5.0:1619677448.570119:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.570120:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.570120:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677448.570121:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.570121:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.570122:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.570124:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.570124:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677448.570125:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.570125:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.570127:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.570127:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.570128:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.570129:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.570132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.570137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677448.570138:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.570141:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.570142:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.570143:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.570143:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.570145:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.570145:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.570146:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677448.570146:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677448.570148:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.570150:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.570152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:1.0:1619677448.570152:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.570153:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.570155:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.570155:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.570157:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677448.570159:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.570160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000400:00000200:1.0:1619677448.570160:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.570161:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677448.570162:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.570162:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.570163:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.570164:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.570165:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.570165:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.570166:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.570167:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677448.570167:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.570168:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677448.570168:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.570169:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.570170:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.570171:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.570171:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.570172:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677448.570173:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.570174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:17.0:1619677448.570175:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.570176:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.570177:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.570178:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.570179:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.570182:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.570182:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.570183:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.570184:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.570184:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.570185:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.570185:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.570186:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.570187:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.570187:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.570188:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.570188:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677448.570189:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677448.570189:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677448.570190:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677448.570194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.2:1619677448.570197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.570202:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.570204:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.570204:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.570205:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.570205:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.570205:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.570206:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.570206:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.570207:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:17.0:1619677448.570208:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.570208:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.570209:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.570242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.570244:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.570247:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.570249:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.570250:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.570251:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.570253:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.570254:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.570256:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.570257:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.570258:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.570261:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.570262:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.570265:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.570266:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.570267:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.570269:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.570270:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.570271:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.570272:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.570273:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.570274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.570277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677448.570281:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.570283:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.570283:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:12.0:1619677448.570284:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.570285:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:12.0:1619677448.570286:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.570286:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.570286:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.570287:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.570289:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.570303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.570311:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.570314:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.570315:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.570317:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.570319:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.570320:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.570322:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.570323:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.570324:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.570325:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.570328:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.570329:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.570332:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.570332:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.570333:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.570335:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.570335:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.570336:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677448.570338:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.570338:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677448.570341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.570344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:13.0:1619677448.570348:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.570350:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.570351:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.570352:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:13.0:1619677448.570352:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:26.0:1619677448.570354:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.570355:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:13.0:1619677448.570355:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.570356:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677448.570356:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.570425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677448.570429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.570432:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.570434:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.570435:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677448.570435:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.570437:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.570438:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.570439:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.570439:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.570441:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677448.570441:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.570442:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.570443:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.570444:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.570444:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.570445:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.570446:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.570447:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677448.570447:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.570448:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.570450:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677448.570453:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.570454:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.570455:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677448.570456:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677448.570458:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.570459:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.570460:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.570460:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.570460:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.570461:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:10.0:1619677448.570462:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.570462:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.570463:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677448.570463:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:10.0:1619677448.570464:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677448.570464:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.570465:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.570466:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.570467:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.570469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.2:1619677448.570469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.2:1619677448.570471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.570476:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:17.0:1619677448.570477:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:5.0:1619677448.570478:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677448.570479:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.0:1619677448.570479:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.570480:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:17.0:1619677448.570480:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677448.570480:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.570481:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677448.570481:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.570482:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.570483:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.570484:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.570484:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.570486:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.0:1619677448.570487:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.570491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.570499:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.570501:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.570502:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.570503:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.570504:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.570505:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.570538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.570545:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.570547:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.570548:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.570550:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.570553:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.570555:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.570556:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.570557:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.570558:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.570559:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.570563:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.570565:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.570567:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.570568:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.570569:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.570570:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.570571:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.570572:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.570572:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.570576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.570584:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.570587:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.570588:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.570588:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.570590:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.570592:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.570596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.570598:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.570601:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.570603:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.570604:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.570606:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.570608:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677448.570609:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.570610:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.570611:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.570612:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677448.570616:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.570618:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677448.570620:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.570621:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677448.570622:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.570624:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677448.570624:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.570625:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.570625:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.570630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.570637:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.570640:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.570641:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.570642:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.570644:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.570645:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.570650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.570657:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.570659:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.570660:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.570661:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.570662:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.570663:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.570683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.570690:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.570693:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.570695:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.570697:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.570697:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.570698:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.570700:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.570701:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.570702:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.570703:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.570706:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.570707:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.570710:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.570710:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.570711:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.570713:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.570713:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.570715:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.570716:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.570717:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.570718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.570722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.570725:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.570727:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.570728:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.570729:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.570730:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.570731:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.2:1619677448.570731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677448.570732:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:21.2:1619677448.570732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:12.0:1619677448.570732:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.570733:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.570734:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677448.570737:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.570738:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677448.570741:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.570741:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:1.0:1619677448.570741:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:18.0:1619677448.570742:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.570746:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.570746:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.570749:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.570749:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.570750:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.570751:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.570752:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.570753:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.570754:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.570755:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.570756:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.570756:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.570758:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.570758:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.570762:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.570763:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.570765:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.570766:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.570767:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.570768:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.570769:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.570770:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.570770:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.570770:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.570771:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677448.570771:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.570771:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.570772:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677448.570773:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677448.570774:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.570774:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.570775:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677448.570776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:25.2:1619677448.570779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.570784:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.570785:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.570786:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.570786:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677448.570787:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.570788:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.570789:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.570789:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.570791:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:13.0:1619677448.570791:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.570792:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.570794:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.570796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.570803:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.570806:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.570807:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.570809:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.570812:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.570813:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.570815:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.570816:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.570817:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.570819:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.570822:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.570822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:26.0:1619677448.570824:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.570826:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.570827:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.570827:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.570828:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.570829:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.570829:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.570830:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.570830:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:26.0:1619677448.570831:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:5.0:1619677448.570832:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.570833:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.570835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:5.0:1619677448.570835:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.570837:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.570838:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.570839:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.570840:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.570841:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:5.0:1619677448.570841:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.570844:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:5.0:1619677448.570844:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.570845:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.570846:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.570847:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.570848:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.570849:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.0:1619677448.570850:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.570850:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.570851:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.570853:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.570854:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.570855:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.570855:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.570859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.570865:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.570868:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.570869:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.570870:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.570872:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.570873:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.570878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.570884:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.570887:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.570888:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.570888:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.570889:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.570890:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.570908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677448.570915:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677448.570918:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677448.570920:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.570922:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677448.570922:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.570924:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.570925:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.570926:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.570927:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.570928:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.570932:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.570933:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.570936:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.570936:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.570937:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.570939:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.570939:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677448.570941:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:30.0:1619677448.570942:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:30.0:1619677448.570943:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:25.2:1619677448.570944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:21.2:1619677448.570948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677448.570953:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.570954:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.570955:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.570956:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.570957:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:30.0:1619677448.570957:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:31.0:1619677448.570958:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.570958:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677448.570959:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677448.570960:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:20.2:1619677448.571011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.571019:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.571022:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.571024:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.571026:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.571026:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.571027:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.571029:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.571030:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.571031:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.571031:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.571034:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.571035:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.571038:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.571038:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:20.2:1619677448.571038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:30.0:1619677448.571039:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.571039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:30.0:1619677448.571040:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.571041:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.571041:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.571042:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.571043:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:18.0:1619677448.571044:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:30.0:1619677448.571045:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000800:00000200:31.0:1619677448.571046:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:25.2:1619677448.571046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:30.0:1619677448.571047:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.571047:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:30.0:1619677448.571048:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.571048:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:30.0:1619677448.571050:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.571051:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.571052:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.571052:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:30.0:1619677448.571053:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.571054:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.571054:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.571054:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:30.0:1619677448.571055:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.571055:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:12.0:1619677448.571055:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.571057:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96713193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.571057:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.571058:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.571058:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:18.0:1619677448.571059:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.571060:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.571060:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:18.0:1619677448.571061:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.571064:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.571065:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.571066:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.571067:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.571067:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.571068:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.571069:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.571069:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677448.571070:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87b3e5800] (22)++
00000800:00000200:18.0:1619677448.571070:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.571071:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.571072:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:30.0:1619677448.571072:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:18.0:1619677448.571072:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:25.2:1619677448.571074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.571074:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.571074:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.571075:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.571076:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677448.571078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:10.0:1619677448.571080:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:8.2:1619677448.571080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.571083:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.571084:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.571084:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.571086:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)++
00000400:00000200:10.0:1619677448.571086:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.571088:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:20.0:1619677448.571088:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:10.0:1619677448.571088:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677448.571089:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677448.571090:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:20.0:1619677448.571090:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.571091:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.571092:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.571094:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.571095:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.571097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.571105:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.571108:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.571109:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.571110:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.571110:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.571111:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.571142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.571148:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.571150:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.571151:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.571153:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.571155:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.571156:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.571157:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.571159:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.571160:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.571161:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.571164:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.571166:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.571167:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.571168:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.571169:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.571170:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.571170:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.571171:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.571171:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.571176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.571178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677448.571183:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:26.0:1619677448.571185:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.571185:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.571185:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.571186:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:26.0:1619677448.571188:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:17.0:1619677448.571188:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.571188:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.571189:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:26.0:1619677448.571190:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.571193:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.571194:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.571195:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.571197:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.571198:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.571199:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.571203:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.571204:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.571207:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.571207:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.571208:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.571209:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.571210:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.571211:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.571211:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.571215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.571222:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.571224:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.571225:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.571227:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.571228:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.571230:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.571243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.571249:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.571252:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.571252:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.571254:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.571256:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.571258:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.571259:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.571261:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.571262:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.571263:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.571266:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.571268:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.571270:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.571271:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.571271:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.571273:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.571274:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.571275:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.571276:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.571276:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.2:1619677448.571277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677448.571279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.571282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677448.571285:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.571287:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677448.571287:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:5.0:1619677448.571287:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.571288:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.571289:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.571289:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:5.0:1619677448.571289:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.571290:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677448.571291:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.571291:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:5.0:1619677448.571292:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.571299:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.571300:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.571302:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.571302:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.571304:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.571305:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.571306:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.571307:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:5.0:1619677448.571307:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.571309:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.571312:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.571313:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.571313:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.571315:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.571316:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.571316:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.571317:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.571322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.571330:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.571333:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.571334:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.571335:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.571337:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.571338:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.571641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.571649:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.571652:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.571652:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.571654:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.571657:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.571658:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.571659:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.571661:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.571661:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677448.571661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677448.571663:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.571666:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.571668:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.571668:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677448.571670:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.571671:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.571671:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.571672:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677448.571672:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.571673:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:10.0:1619677448.571673:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.571674:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.571675:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.571675:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:10.0:1619677448.571676:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.571677:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.571678:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.571679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:10.0:1619677448.571680:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.571680:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.571681:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.571685:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.571686:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:13.0:1619677448.571687:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.571689:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:10.0:1619677448.571689:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677448.571690:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.571690:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.571690:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:13.0:1619677448.571691:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.571692:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:13.0:1619677448.571693:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.571693:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:12.0:1619677448.571694:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.571694:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.571695:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.571696:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.571698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.571701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:25.2:1619677448.571703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:22.0:1619677448.571706:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.571708:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.571709:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:13.0:1619677448.571709:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.571710:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677448.571710:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.571710:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.571711:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.571712:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.571712:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.571712:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.571713:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.571713:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.571714:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.571715:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677448.571716:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.571718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.571725:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.571728:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.571729:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.571729:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.571730:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.571730:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.571857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.571863:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.571866:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.571867:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.571869:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.571871:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.571873:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.571874:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.571875:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.571876:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.571877:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.571881:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.571882:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.571885:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.571886:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.571886:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.571888:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.571888:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.571889:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.571889:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.571894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.571895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677448.571901:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677448.571902:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677448.571904:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.571905:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.571905:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:6.0:1619677448.571905:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677448.571906:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677448.571907:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.571908:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.571910:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:8.0:1619677448.571911:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.571913:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.571914:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.571915:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.571916:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.571917:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.571920:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.571922:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.571925:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.571925:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.571926:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.571927:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.571928:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.571929:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.571929:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.571933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.571942:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.571945:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.571945:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.571946:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.571948:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.571950:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.571953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.571960:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.571962:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.571963:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.571963:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.571964:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.571964:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.572057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.572063:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.572065:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.572066:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.572068:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.572069:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.572071:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.572072:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.572073:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.572074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:1.0:1619677448.572074:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.572075:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.572079:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.572080:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677448.572080:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:6.0:1619677448.572082:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.572083:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677448.572084:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.572084:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:6.0:1619677448.572085:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.572085:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:6.0:1619677448.572087:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.572087:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.572087:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.572088:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677448.572089:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:1.0:1619677448.572089:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.572090:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.572091:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.572093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:6.0:1619677448.572093:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.572093:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.572097:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.572099:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677448.572100:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:6.0:1619677448.572101:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.572102:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:12.0:1619677448.572103:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.572103:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.572103:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:12.0:1619677448.572105:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.572105:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.572105:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677448.572106:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677448.572106:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:21.2:1619677448.572107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677448.572107:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677448.572108:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.2:1619677448.572112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.572115:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.572117:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.572120:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.572120:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:30.0:1619677448.572121:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.572122:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.572122:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:31.0:1619677448.572123:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.572123:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.572123:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:31.0:1619677448.572125:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.572126:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.572127:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.572128:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.572129:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.572129:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:31.0:1619677448.572132:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.572134:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:4.2:1619677448.572134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.572137:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.572137:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.572138:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.572140:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.572141:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.572141:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677448.572141:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.572142:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.572144:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.572145:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.572146:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.572146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:10.0:1619677448.572149:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.572151:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:24.0:1619677448.572152:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:10.0:1619677448.572152:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.572152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:10.0:1619677448.572153:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.572154:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:10.0:1619677448.572154:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.572155:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.572155:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677448.572155:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.572158:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.572160:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:10.0:1619677448.572160:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.572161:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.572162:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.572162:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.572164:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.572166:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.572166:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.572167:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.572167:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:26.0:1619677448.572168:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.572169:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.572169:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:26.0:1619677448.572171:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.572171:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.572172:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:21.2:1619677448.572172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:10.0:1619677448.572173:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.572174:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.572175:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.572175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.572175:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.572176:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.572177:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677448.572179:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:26.0:1619677448.572180:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.572181:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.572181:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677448.572181:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.572182:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:26.0:1619677448.572183:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.572183:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677448.572183:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.572184:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.572184:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:8.0:1619677448.572184:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.572185:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:8.0:1619677448.572186:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.572187:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.572187:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:6.0:1619677448.572187:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:25.2:1619677448.572188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677448.572189:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.572189:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96713179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.572190:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.572192:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:8.0:1619677448.572192:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.572192:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.572194:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:8.0:1619677448.572195:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.572197:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677448.572197:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:13.0:1619677448.572199:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.572199:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677448.572200:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.572200:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:12.0:1619677448.572201:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677448.572201:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.572202:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.572203:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677448.572203:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.572203:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.572204:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.572208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.572214:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.572216:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.572217:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.572219:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.572220:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.572222:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.572285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.572288:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.572290:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.572293:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.572294:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.572299:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.572301:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.572302:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.572303:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.572304:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.572305:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.572308:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.572310:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.572312:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.572313:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.572314:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.572316:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.572316:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.572317:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.572318:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.572322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.572325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:6.0:1619677448.572330:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.572330:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:6.0:1619677448.572333:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677448.572333:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:6.0:1619677448.572334:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.572335:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.572336:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:6.0:1619677448.572336:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.572336:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:6.0:1619677448.572338:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:5.0:1619677448.572340:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.572342:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.572343:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.572345:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.572346:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.572347:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.572350:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.572352:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.572355:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.572356:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.572357:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.572359:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.572359:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.572360:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.572362:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.572362:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.572364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.572367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677448.572372:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.572372:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:30.0:1619677448.572375:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677448.572375:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.572375:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:30.0:1619677448.572376:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.572376:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:30.0:1619677448.572378:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:31.0:1619677448.572380:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.572381:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.572384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677448.572391:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.572394:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.572395:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.572395:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.572396:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.572397:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.572549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:4.2:1619677448.572551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677448.572554:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.572557:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.572558:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:10.0:1619677448.572559:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677448.572560:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.572562:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.572562:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.572563:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.572564:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.572566:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.572566:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.572568:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.572569:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.572569:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.572570:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:10.0:1619677448.572570:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.572572:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.572573:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.572574:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.572575:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.572579:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.572580:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:22.0:1619677448.572583:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.572585:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.572586:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.572587:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.572587:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:22.0:1619677448.572588:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.572589:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:10.0:1619677448.572589:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.572589:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.572590:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.572590:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.572591:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.572592:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.572592:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.572593:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.572593:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.572594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:25.2:1619677448.572597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.572602:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.572603:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.572604:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.572604:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.572605:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.572606:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.572606:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.572608:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.572608:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.572609:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.572610:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.572612:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677448.572643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.572651:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.572654:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.572656:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.572658:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.572658:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.572660:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.572661:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.572662:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.572663:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.572664:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.572667:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.572669:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.572672:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.572673:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.572673:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.572675:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.572675:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.572676:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.572677:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.572680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.572688:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.572690:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.572691:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.572692:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.572694:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.572695:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.572722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.572729:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.572732:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.572733:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.572734:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.572736:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.572737:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.572738:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.572739:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.572740:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.572741:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.572744:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.572745:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.572747:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677448.572747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:26.0:1619677448.572748:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.572748:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.572749:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.572749:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.572750:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.572750:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.572754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.572754:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.572757:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.572758:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.572760:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.572760:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:18.0:1619677448.572762:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.572762:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.572763:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.572764:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:18.0:1619677448.572764:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.572766:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.572767:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.572768:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.572768:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:18.0:1619677448.572769:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.572770:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.572773:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.572775:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.572777:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.572778:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.572778:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.572780:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.572781:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.572782:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.572782:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.572787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.572794:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:21.2:1619677448.572796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677448.572796:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.572797:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.572797:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.572799:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.572800:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677448.572803:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.572806:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.572808:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.572810:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.572811:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.572812:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.572814:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.572815:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.572816:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.572817:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.572821:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.572822:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.572826:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.572827:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.572827:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.572829:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.572829:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.572830:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.572831:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.572835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.572842:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.572845:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.572846:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.572847:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.572849:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.572852:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.572853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.572860:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.572863:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.572864:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.572864:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.572865:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.572866:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.573053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.573061:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.573064:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.573066:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.573068:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.573069:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.573070:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.573072:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.573073:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.573074:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.573075:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.573078:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.573079:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.573082:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.573082:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.573083:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.573084:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.573085:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.573086:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.573086:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.573090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.573096:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.573099:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.573100:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.573101:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.573103:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.573104:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.573110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:13.0:1619677448.573117:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.573120:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.573121:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.573122:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.573122:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.573124:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.573168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.573174:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.573176:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.573177:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.573179:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.573181:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.573183:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.573184:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.573186:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.573187:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.573188:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.573191:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.573192:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.573195:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.573196:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.573197:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.573198:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.573199:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.573200:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.573200:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.573204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.573211:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.573213:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:20.2:1619677448.573213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:26.0:1619677448.573215:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.573215:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.573216:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.573217:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.573218:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:21.2:1619677448.573219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677448.573220:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:26.0:1619677448.573221:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677448.573221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000400:00000200:10.0:1619677448.573221:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.573222:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000400:00000200:10.0:1619677448.573223:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.573224:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.573225:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000400:00000200:10.0:1619677448.573225:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:31.0:1619677448.573226:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.573226:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.573226:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.573226:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.573227:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:31.0:1619677448.573228:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677448.573229:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.573231:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.573231:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.573232:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677448.573232:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:4.2:1619677448.573232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:31.0:1619677448.573234:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.573235:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.573236:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.573237:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.573238:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.573239:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.573239:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:31.0:1619677448.573240:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.573241:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.573242:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:31.0:1619677448.573243:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.573244:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.573245:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.573245:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.573245:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.573245:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:26.0:1619677448.573247:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.573248:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.573248:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677448.573249:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.573249:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:26.0:1619677448.573249:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.573250:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96713166] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.573251:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.573251:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677448.573251:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.573252:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677448.573252:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.573253:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:26.0:1619677448.573253:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.573254:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.573255:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.573255:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.573255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.573256:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.573257:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.573257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.573259:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.573262:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.573263:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.573263:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (23)++
00000800:00000200:1.0:1619677448.573263:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:26.0:1619677448.573266:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (24)--
00000800:00000200:22.0:1619677448.573266:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (24)++
00000800:00000200:1.0:1619677448.573266:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:26.0:1619677448.573267:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.573268:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.573268:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677448.573268:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.573268:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.573269:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000400:00000200:1.0:1619677448.573270:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.573271:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.573271:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000400:00000200:22.0:1619677448.573273:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.573273:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:1.0:1619677448.573273:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:6.0:1619677448.573274:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.573275:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.573277:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.573278:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.2:1619677448.573278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677448.573279:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677448.573286:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.573289:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.573289:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.573291:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.573299:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.573300:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.573302:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.573303:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.573304:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.573305:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.573308:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.573309:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.573312:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.573313:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.573314:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.573315:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.573316:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.573317:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.573318:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.573319:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677448.573322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.573325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.2:1619677448.573328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677448.573330:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677448.573331:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.573333:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:18.0:1619677448.573333:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:20.0:1619677448.573334:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:18.0:1619677448.573334:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.573334:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.573335:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.573336:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.573337:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:20.0:1619677448.573337:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.573337:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.573337:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.573340:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.573340:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:20.0:1619677448.573341:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677448.573343:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.573344:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.573345:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.573346:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677448.573349:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.573351:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677448.573354:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.573354:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677448.573355:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.573356:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677448.573357:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.573358:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.573359:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.573360:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677448.573363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677448.573365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.573370:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:17.0:1619677448.573371:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:5.0:1619677448.573372:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.573373:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677448.573374:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:5.0:1619677448.573374:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677448.573375:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.573375:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.573377:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.573379:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.573405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.573410:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.573413:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.573414:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.573416:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.573418:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.573420:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.573421:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.573422:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.573423:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.573424:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.573427:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.573429:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.573432:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.573433:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.573433:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.573435:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.573436:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.573437:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.573438:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.573439:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.573441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.573444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.573448:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677448.573449:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677448.573450:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:17.0:1619677448.573451:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:24.0:1619677448.573452:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.573452:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677448.573452:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:1.0:1619677448.573453:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677448.573454:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.573457:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.573461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677448.573466:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.573469:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.573469:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677448.573470:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.573471:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.573471:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.573543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.573548:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.573551:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.573552:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.573554:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.573556:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.573557:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.573559:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.573560:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.573561:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.573562:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.573566:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.573567:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.573570:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.573571:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.573572:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.573574:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.573574:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.573575:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.573576:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.573577:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.573580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.573582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.573586:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:18.0:1619677448.573587:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:26.0:1619677448.573589:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:18.0:1619677448.573589:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:10.0:1619677448.573589:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:18.0:1619677448.573590:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:10.0:1619677448.573590:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.573591:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:10.0:1619677448.573592:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.573592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:10.0:1619677448.573595:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.573596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:24.0:1619677448.573599:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:26.0:1619677448.573602:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:24.0:1619677448.573602:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677448.573603:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:26.0:1619677448.573604:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000400:00000200:24.0:1619677448.573605:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.573605:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.573606:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.573607:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.573607:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:24.0:1619677448.573607:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.573609:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.573610:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.573611:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.573612:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.573613:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.573617:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.573619:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.573622:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.573622:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.573623:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.573625:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.573625:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677448.573626:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:24.0:1619677448.573627:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:24.0:1619677448.573628:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.2:1619677448.573631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677448.573634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (21)++
00000800:00000200:17.0:1619677448.573639:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.573641:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:8.0:1619677448.573641:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000400:00000200:17.0:1619677448.573642:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.573642:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.573643:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:20.2:1619677448.573644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.0:1619677448.573644:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:17.0:1619677448.573645:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.573645:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:17.0:1619677448.573648:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677448.573652:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.573655:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.573657:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.573660:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.573660:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.573662:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.573663:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.573665:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.573666:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.573667:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.573670:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.573672:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.573675:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.573676:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.573677:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.573678:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.573679:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.573680:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.573682:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.573682:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.573684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.573688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:21.2:1619677448.573691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.0:1619677448.573695:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677448.573696:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.573698:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:5.0:1619677448.573698:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677448.573699:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:5.0:1619677448.573699:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.573700:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.573701:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677448.573701:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677448.573701:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.573701:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.573704:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.573704:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.0:1619677448.573704:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:31.0:1619677448.573707:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.573709:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677448.573710:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.573712:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.573713:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.573714:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.2:1619677448.573716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:31.0:1619677448.573717:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.573719:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.573722:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.573722:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677448.573723:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.573723:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.573725:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.573725:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.573725:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:31.0:1619677448.573726:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:18.0:1619677448.573726:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677448.573728:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.573728:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:2.0:1619677448.573728:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.573731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677448.573731:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.573732:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:21.2:1619677448.573734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:2.0:1619677448.573734:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.573735:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.573736:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.573737:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.573738:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:2.0:1619677448.573740:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.573741:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:30.0:1619677448.573742:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:17.0:1619677448.573742:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.573742:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.573743:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.573744:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.573745:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677448.573745:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.573745:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000400:00000200:1.0:1619677448.573745:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.573746:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677448.573746:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677448.573748:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:2.0:1619677448.573748:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:1.0:1619677448.573748:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:2.0:1619677448.573749:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.573750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.573750:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677448.573758:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.573761:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.573761:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000800:00000200:17.0:1619677448.573762:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (23)++
00000800:00000200:1.0:1619677448.573762:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000800:00000200:17.0:1619677448.573763:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (25)--
00000800:00000200:17.0:1619677448.573763:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.573763:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.573765:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.573765:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.573768:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.573812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.573820:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.573823:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.573824:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.573825:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.573828:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.573830:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.573831:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.573832:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.573833:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.573834:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.573837:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.573839:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.573841:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.573842:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.573843:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.573844:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.573844:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.573845:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.573845:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.573850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.573856:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.573859:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.573859:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.573861:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.573863:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.573863:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.573980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.573985:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.573988:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.573989:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.573990:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.573993:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.573994:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.573996:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713156] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.573997:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.573998:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.573999:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.574002:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.574004:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.574006:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.574007:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.574008:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.574009:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.574010:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.574011:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.574011:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.574015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.574022:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.574024:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.574025:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.574026:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.574028:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.574030:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.574032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.574039:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.574042:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.574042:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.574045:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.574047:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.574048:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.574049:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.574051:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.574052:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.574053:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.574056:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.574058:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.574061:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.574061:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.574062:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.574064:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.574064:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.574065:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.574067:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.574067:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.574070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:4.2:1619677448.574073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.574078:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677448.574079:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.574081:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677448.574081:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.574082:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:10.0:1619677448.574082:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.574082:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.574083:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.574085:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.574087:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.574092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:22.0:1619677448.574099:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.574102:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.574103:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.574103:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.574104:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.574104:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.574145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.574152:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.574155:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.574155:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677448.574157:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.574160:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.574161:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.574163:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.574164:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.574165:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.574166:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.574169:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.574170:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.574173:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.574174:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.574175:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.574176:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.574177:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.574177:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.574178:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.574182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.574190:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.574193:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.574193:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.574194:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.574196:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.574199:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.574200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:21.2:1619677448.574208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:17.0:1619677448.574208:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.574210:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.574211:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.574211:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.574212:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.574212:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677448.574214:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.574217:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.574217:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.574219:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.574222:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.574223:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.574224:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.574226:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.574227:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.574228:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.574230:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.574232:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.574233:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.574234:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.574235:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.574236:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.574236:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.574237:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.574238:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.574242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.574249:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.574251:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.574252:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.574253:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.574253:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.574255:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.574261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.574267:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.574270:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.574271:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.574271:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.574271:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.574272:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.574345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.574351:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.574354:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.574354:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.574356:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.574359:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.574360:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.574361:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.574362:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.574363:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.574364:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.574368:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.574369:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.574372:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.574373:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.574374:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.574375:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.574376:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.574377:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.574377:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.574382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.574389:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.574392:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.574392:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.574393:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.574395:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.574398:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.574434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.574442:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.574445:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.574447:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.574449:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.574450:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.574451:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.574452:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.574453:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.574455:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.574455:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.574459:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.574461:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.574463:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.574464:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.574465:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.574466:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.574467:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.574468:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.574469:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.574472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.574479:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.574482:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.574483:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.574484:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.574486:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.574487:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.574514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.574517:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.574520:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.574522:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.574524:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.574525:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.574526:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.574528:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.574529:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.574530:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.574531:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.574534:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.574536:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.574540:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.574540:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.574541:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.574543:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.574543:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.574544:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.574545:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.574549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.574555:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.574558:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.574559:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.574561:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.574562:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.574563:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.574625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.574631:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.574634:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.574635:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.574637:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.574639:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.574640:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.574642:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713149] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.574643:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.574644:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.574645:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.574648:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.574650:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.574653:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.574654:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.574654:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.574656:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.574657:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.574658:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.574658:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.574662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.574669:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.574672:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.574672:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.574674:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.574675:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.574677:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.574680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:21.2:1619677448.574684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:12.0:1619677448.574686:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.574689:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.574690:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.574690:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.574691:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677448.574691:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:12.0:1619677448.574692:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677448.574694:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.574695:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.574696:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.574699:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.574701:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.574702:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.574703:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.574704:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.574705:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:5.2:1619677448.574705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:8.0:1619677448.574708:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.574709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.0:1619677448.574710:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.574712:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677448.574713:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.574714:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:17.0:1619677448.574715:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.574715:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:8.0:1619677448.574715:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.574716:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677448.574716:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.574717:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:17.0:1619677448.574718:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:8.0:1619677448.574718:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.574718:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:1.0:1619677448.574718:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.574719:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.574721:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.574722:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.574722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:17.0:1619677448.574724:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.574724:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.574725:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677448.574725:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.574727:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.574727:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.574728:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.574729:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.574729:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.574730:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.574731:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:5.0:1619677448.574731:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.574733:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:10.0:1619677448.574734:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.574734:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.574736:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:17.0:1619677448.574736:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.574736:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.574737:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.574739:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677448.574739:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.574739:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:10.0:1619677448.574740:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.574741:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.574742:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.574742:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.574743:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:10.0:1619677448.574743:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.574744:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:10.0:1619677448.574744:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.574745:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:10.0:1619677448.574745:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677448.574746:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.574747:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.574748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.574748:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.574752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:13.0:1619677448.574754:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.574757:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.574758:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.574759:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:13.0:1619677448.574759:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.574760:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.574761:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.574762:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.574762:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677448.574762:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677448.574764:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.574767:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677448.574770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:8.0:1619677448.574776:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.574779:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.574779:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677448.574780:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.574781:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.574781:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677448.574920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.574926:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.574929:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.574930:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.574932:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.574935:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.574936:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.574937:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.574938:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.574938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000400:00000200:26.0:1619677448.574939:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.574940:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.574945:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.574946:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.574947:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.574948:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677448.574951:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.574951:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.574952:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.574953:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:10.0:1619677448.574953:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.574954:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:26.0:1619677448.574955:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:10.0:1619677448.574956:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.574957:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.574958:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.574958:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:25.2:1619677448.574961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:10.0:1619677448.574961:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.574962:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.574963:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.574963:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.574964:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.574965:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.574965:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.574966:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.574966:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.574968:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.574969:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.574969:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:6.0:1619677448.574969:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.574970:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:6.0:1619677448.574972:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:25.2:1619677448.574973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:22.0:1619677448.574973:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.574973:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.574976:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.574978:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:26.0:1619677448.574979:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.574982:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.574983:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.574983:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.574985:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.574987:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.575008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.575016:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.575018:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.575020:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.575022:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.575023:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.575024:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.575025:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.575027:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.575028:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.575029:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.575032:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.575033:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.575036:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.575036:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.575037:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.575038:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.575039:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.575040:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.575040:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.575044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.575050:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.2:1619677448.575053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:12.0:1619677448.575053:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.575054:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.575055:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:12.0:1619677448.575055:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.575057:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.575058:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:13.0:1619677448.575058:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:20.0:1619677448.575060:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.575061:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.2:1619677448.575063:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:20.0:1619677448.575066:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.575068:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:5.0:1619677448.575068:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:20.0:1619677448.575069:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.575070:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.575071:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.575072:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.575073:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677448.575076:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.575078:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.575079:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.575080:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:5.0:1619677448.575080:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677448.575081:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.575083:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.575084:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.575085:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:20.2:1619677448.575086:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:5.0:1619677448.575088:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.575090:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677448.575092:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.575092:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.575093:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677448.575093:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.575094:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.575094:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677448.575094:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677448.575096:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.575096:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677448.575096:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677448.575097:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:20.0:1619677448.575097:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.575098:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.575098:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.575098:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677448.575099:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.575099:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:5.0:1619677448.575100:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.575101:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.575102:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.575102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:9.2:1619677448.575102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:31.0:1619677448.575103:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.575104:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96713140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.575106:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.575107:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.575107:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.575110:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:31.0:1619677448.575111:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.575111:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:31.0:1619677448.575112:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.575112:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.575113:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.575114:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:31.0:1619677448.575115:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.575115:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.575115:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.575115:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.575116:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.575117:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:18.0:1619677448.575117:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.575118:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.575119:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.575119:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.575119:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677448.575121:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.575121:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:18.0:1619677448.575121:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.575122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:8.2:1619677448.575125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.575130:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677448.575133:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677448.575133:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.575134:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.575134:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.575135:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677448.575136:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:2.0:1619677448.575136:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:20.0:1619677448.575137:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.575138:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.575139:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.575141:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.575141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.575147:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.575150:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.575150:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.575153:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.575155:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.575157:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.575158:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.575159:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.575160:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.575162:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.575165:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.575167:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.575169:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.575170:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.575171:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.575172:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.575173:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.575174:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.575175:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.575176:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.575178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.2:1619677448.575181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.575184:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:17.0:1619677448.575186:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677448.575187:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.575188:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.575188:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.0:1619677448.575188:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.575189:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677448.575190:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.575190:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677448.575193:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.2:1619677448.575329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.575335:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.575338:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.575339:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.575341:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.575343:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.575344:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.575346:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.575347:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.575348:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.575349:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.575352:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.575354:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.575356:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.575357:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.575357:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.575359:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.575359:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.575360:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.575361:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.575362:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677448.575367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:9.2:1619677448.575369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.575374:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.575376:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.575376:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.575378:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677448.575378:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.575378:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.575379:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.575379:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.575381:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.575382:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.575436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.575441:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.575444:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.575446:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.575447:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.575449:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.575450:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677448.575451:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.575453:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.575454:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.575455:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677448.575458:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.575460:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677448.575462:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.575463:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677448.575464:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677448.575465:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677448.575466:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:4.2:1619677448.575466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:5.0:1619677448.575467:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.575468:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.575472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677448.575474:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.575477:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.575477:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:18.0:1619677448.575479:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:8.0:1619677448.575479:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.575482:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:8.0:1619677448.575482:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.575483:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.575483:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:2.0:1619677448.575483:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:8.0:1619677448.575485:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96713136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.575486:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.575487:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.575488:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:8.0:1619677448.575488:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.575489:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.575493:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.575495:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.575498:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.575498:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.575499:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.575501:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.575501:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.575502:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.575503:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.575507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.575515:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.575518:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.575519:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.575519:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.575521:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.575524:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.575544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.575550:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.575552:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.575553:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.575555:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.575556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:17.0:1619677448.575558:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.575559:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.575560:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.575561:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.575562:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.575563:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:10.0:1619677448.575563:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.575566:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.575567:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677448.575567:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.575569:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:10.0:1619677448.575569:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.575571:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.575571:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.575572:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000400:00000200:10.0:1619677448.575572:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:17.0:1619677448.575573:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:10.0:1619677448.575573:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.575575:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.575575:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.575575:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.575575:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.575576:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.575576:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:10.0:1619677448.575577:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.575580:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.575580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:10.0:1619677448.575581:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.575584:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.575585:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.575585:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677448.575586:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:10.0:1619677448.575587:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.575587:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.575588:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677448.575589:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:10.0:1619677448.575589:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:24.0:1619677448.575590:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.575590:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677448.575592:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.575593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677448.575595:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677448.575601:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.575604:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.575605:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.575606:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.575607:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.575610:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.575611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.575618:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.575621:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.575621:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.575622:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.575623:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.575623:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.575655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.575663:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.575666:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.575668:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.575671:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.575671:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.575673:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:30.0:1619677448.575674:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.575675:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.575676:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.575677:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677448.575680:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.575682:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677448.575685:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.575686:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:30.0:1619677448.575686:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677448.575688:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677448.575689:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.575689:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.575690:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.575694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.575702:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.575704:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.575705:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.575706:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.575708:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.575711:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.575890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.575897:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.2:1619677448.575898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:31.0:1619677448.575900:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.575902:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.575903:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:30.0:1619677448.575904:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.575906:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.575907:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:18.0:1619677448.575907:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.575907:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:31.0:1619677448.575909:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.575910:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.575911:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.575912:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.575912:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.575913:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.575913:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.575915:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.575917:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.575918:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.575919:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.575920:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.575922:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.575923:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.575925:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.575926:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.575926:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.575927:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.575928:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.575928:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:31.0:1619677448.575929:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.575929:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677448.575929:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.575930:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.575931:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:2.0:1619677448.575931:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.575931:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.575932:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.575933:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.575934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:9.2:1619677448.575937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.575942:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677448.575944:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677448.575945:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.575946:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.575946:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.575946:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.575947:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.575948:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:20.0:1619677448.575949:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.575950:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.575951:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677448.575952:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.575953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.575961:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.575964:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677448.575966:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.2:1619677448.575968:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.575968:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.575969:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.575970:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677448.575970:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.575970:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.575971:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677448.575972:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:5.0:1619677448.575972:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677448.575973:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.575974:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677448.575975:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.575977:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.575978:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.575979:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.575980:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.575980:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.575981:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677448.575982:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.575983:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.575983:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.575985:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.575985:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.575986:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.575987:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.575990:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.575992:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.575993:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.575995:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.575996:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.575997:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.575997:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.575998:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.575999:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.575999:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.576000:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.576001:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.576001:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.576002:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677448.576003:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.576003:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677448.576004:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.576005:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.576005:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677448.576006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:25.2:1619677448.576007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.576010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.576014:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:1.0:1619677448.576014:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677448.576016:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677448.576016:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.576016:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.576017:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.576018:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.576018:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.576018:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.576019:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.576019:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677448.576019:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.576020:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.576021:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.576022:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677448.576024:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.576113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.576121:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.576124:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.576124:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.576126:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.576129:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.576131:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.576132:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.576134:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.576135:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.576136:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.576139:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.576141:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.576144:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.576145:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.576146:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.576148:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.576148:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.576149:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.576149:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.576153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.576161:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.576163:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.576164:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.576166:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.576166:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.576169:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.576200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.576207:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.576210:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.576212:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.576213:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.576215:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.576217:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.576218:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.576220:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.576220:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.576222:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.576225:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.576227:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.576229:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.576230:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.576231:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.576233:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.576233:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.576235:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.576236:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.576237:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677448.576239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.576242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:2.0:1619677448.576246:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677448.576249:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:2.0:1619677448.576249:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.576250:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.576250:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.576251:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.576251:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:2.0:1619677448.576252:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.576253:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:2.0:1619677448.576256:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.576342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.576348:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:4.2:1619677448.576349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677448.576351:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.576351:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.576353:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.576356:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.576357:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:10.0:1619677448.576357:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677448.576359:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.576359:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.576360:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.576360:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.576362:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.576363:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.576364:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.576367:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.576367:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.576368:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.576369:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.576370:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.576371:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.576372:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:10.0:1619677448.576372:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.576373:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:10.0:1619677448.576373:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.576374:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:10.0:1619677448.576374:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.576375:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.576376:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.576377:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.576378:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677448.576378:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.576380:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.2:1619677448.576381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.576382:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677448.576383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.576383:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.576383:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.576385:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.576385:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.576387:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.576388:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677448.576388:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:20.0:1619677448.576389:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:10.0:1619677448.576389:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677448.576390:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.576390:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:25.2:1619677448.576391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.576391:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:6.0:1619677448.576391:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:20.0:1619677448.576393:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.576394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677448.576395:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.576395:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.576398:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677448.576399:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:26.0:1619677448.576401:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:31.0:1619677448.576402:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.576403:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.576403:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.576404:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677448.576405:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.576405:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.576407:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.576408:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.576443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.576449:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.576452:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.576453:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.576455:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.576457:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.576458:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.576460:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.576461:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.576462:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.576463:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.576467:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.576468:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.576471:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.576472:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.576472:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.576474:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.576474:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.576475:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.576475:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.576480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.576488:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.576490:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.576491:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.576491:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.576494:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.576496:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.576621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1200] (20)++
00000800:00000200:1.0:1619677448.576627:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677448.576630:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:1.0:1619677448.576630:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000400:00000200:1.0:1619677448.576632:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.576634:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.576636:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:1.0:1619677448.576637:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.576638:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.576639:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.576640:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:1.0:1619677448.576643:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.576644:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:1.0:1619677448.576646:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.576647:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:1.0:1619677448.576647:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:20.2:1619677448.576648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677448.576649:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:1.0:1619677448.576650:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:1.0:1619677448.576651:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdbb1200] (22)++
00000800:00000200:1.0:1619677448.576652:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (23)--
00000800:00000200:1.0:1619677448.576652:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:5.0:1619677448.576654:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.2:1619677448.576656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.2:1619677448.576657:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:20.0:1619677448.576660:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.576660:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.576662:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:5.0:1619677448.576662:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.576663:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:17.0:1619677448.576663:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)++
00000400:00000200:20.0:1619677448.576664:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.576664:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.576665:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.576665:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdbb1200] (23)--
00000800:00000200:17.0:1619677448.576665:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:17.0:1619677448.576666:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000400:00000200:5.0:1619677448.576666:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677448.576667:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.576669:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.576670:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.576671:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:20.0:1619677448.576674:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.576674:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.576675:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.0:1619677448.576676:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677448.576679:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.576680:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677448.576680:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677448.576682:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677448.576683:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.576684:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.576685:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.576688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.576696:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.576699:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.576700:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.576700:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.576702:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.576704:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.576745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677448.576752:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.576755:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677448.576756:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677448.576758:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.576760:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.576762:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.576763:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.576764:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.576765:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.576766:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.576770:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.576772:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.576774:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.576775:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.576775:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.576777:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.576777:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.576778:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.576779:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677448.576783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:4.2:1619677448.576784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.576789:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:26.0:1619677448.576791:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.576792:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.576793:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.576793:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:21.2:1619677448.576793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:10.0:1619677448.576794:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:30.0:1619677448.576795:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.576796:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:26.0:1619677448.576796:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.576797:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:26.0:1619677448.576798:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.576799:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:26.0:1619677448.576800:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.576801:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.576801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000400:00000200:26.0:1619677448.576802:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.576802:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.576802:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:26.0:1619677448.576803:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.576805:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.576805:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677448.576807:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000400:00000200:2.0:1619677448.576807:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.576809:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.576809:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:31.0:1619677448.576810:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.576811:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.576811:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000400:00000200:2.0:1619677448.576811:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.576812:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.576812:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.576812:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.576813:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:2.0:1619677448.576813:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.576814:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.576815:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.576816:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.576816:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.576818:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:2.0:1619677448.576818:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.576819:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.576819:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.576820:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677448.576820:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.576823:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.576824:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (21)++
00000800:00000200:25.2:1619677448.576825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.576825:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677448.576827:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (24)--
00000800:00000200:2.0:1619677448.576827:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.576828:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.576829:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677448.576832:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:22.0:1619677448.576835:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (24)++
00000800:00000200:6.0:1619677448.576835:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:22.0:1619677448.576836:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000400:00000200:22.0:1619677448.576837:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.576837:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.576839:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:6.0:1619677448.576839:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.576842:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.576843:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.576843:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.576896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.576901:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.576903:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.576904:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.576906:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.576908:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.576909:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.576910:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.576912:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.576913:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.576914:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.576917:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.576919:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.576922:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.576922:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.576923:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.576925:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.576925:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.576926:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.576926:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.576931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.576939:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.576941:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.576942:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.576942:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.576945:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.576947:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.576950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.576957:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.576959:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.576960:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.576960:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.576961:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.2:1619677448.576961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.576964:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677448.576964:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.576966:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.576967:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.576969:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.576971:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.576972:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.576973:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.576975:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.576976:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.576977:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.576980:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.576982:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.576984:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.576985:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.576985:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.576987:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.576987:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.576988:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.576989:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.576990:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.576993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.576995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.577000:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.577000:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.577002:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677448.577003:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677448.577003:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677448.577004:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.577004:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.577005:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.577007:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.577009:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.577016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.577022:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.577024:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.577025:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.577027:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.577029:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.577031:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.577032:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.577034:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.577035:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.577036:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.577039:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.577041:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.577044:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.577045:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.577046:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.577047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:18.0:1619677448.577048:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.577048:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.577049:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.577049:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677448.577050:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.577053:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:25.2:1619677448.577054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.577055:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.577056:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.577057:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.577059:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.577060:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.577061:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:20.0:1619677448.577062:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.577063:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.577064:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.577064:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.577065:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.577067:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.577068:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:13.0:1619677448.577068:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.577070:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.577072:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677448.577073:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.577074:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.577075:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.577077:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.577077:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.577078:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.577079:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.577082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.577089:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.577091:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.577092:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.577094:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677448.577095:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.577097:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.577103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:22.0:1619677448.577109:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677448.577112:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677448.577113:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677448.577114:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.577115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:22.0:1619677448.577117:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.577118:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.577120:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.577121:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.577122:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.577122:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:22.0:1619677448.577123:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:26.0:1619677448.577126:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:22.0:1619677448.577126:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.577126:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:22.0:1619677448.577128:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:10.0:1619677448.577129:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:22.0:1619677448.577131:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.577131:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.577132:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.577132:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:10.0:1619677448.577133:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:22.0:1619677448.577134:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.577134:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000400:00000200:10.0:1619677448.577134:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.577135:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677448.577135:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000400:00000200:10.0:1619677448.577135:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.577136:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.577137:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.577140:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:9.2:1619677448.577140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:10.0:1619677448.577141:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.577144:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.577145:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.577146:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.577148:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.577148:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:2.0:1619677448.577148:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:10.0:1619677448.577149:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.577150:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:2.0:1619677448.577150:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.577151:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.577152:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.577154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677448.577154:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.577157:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677448.577160:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.577161:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.577162:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.577162:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.577164:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.577166:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.577171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.577178:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.577181:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.577182:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.577182:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.577183:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.577184:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.577208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.577215:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.577217:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.577218:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.2:1619677448.577219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:26.0:1619677448.577220:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.577224:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.577225:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.577227:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713112] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.577227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677448.577228:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:26.0:1619677448.577229:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.577229:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677448.577230:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.577231:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.577233:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.577234:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677448.577235:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.577236:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.577236:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.577237:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:10.0:1619677448.577238:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.577239:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.577239:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.577241:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:10.0:1619677448.577241:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.577241:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.577242:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.577242:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.577243:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.577245:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96713110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.577245:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.577246:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.577246:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:10.0:1619677448.577246:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.577247:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.577247:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:31.0:1619677448.577248:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.577248:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.577249:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.577250:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.577251:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.577251:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.577251:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.577252:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.577252:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.577252:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.577253:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.577253:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:25.2:1619677448.577254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.577255:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.577255:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.577255:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.577255:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.577256:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.577256:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.577257:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.577257:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677448.577258:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.577258:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.577259:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677448.577260:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:25.2:1619677448.577260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.577263:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.577263:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.577264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:31.0:1619677448.577265:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.577267:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:12.0:1619677448.577267:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.577268:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.577269:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:12.0:1619677448.577269:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.577270:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.577271:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:12.0:1619677448.577271:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.577272:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.577273:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.577273:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.577274:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.577275:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.577276:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.577278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.577284:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.577287:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.577287:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.577288:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.577289:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.577290:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.577367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.577373:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.577376:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.577377:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.577379:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.577381:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.577382:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.577384:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.577385:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.577386:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.577387:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.577390:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.577391:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.577394:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.577395:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.577396:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.577398:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.577398:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.577399:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.577399:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.577404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.577412:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.577415:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.577416:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.577418:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.577420:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.577421:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.577421:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.577422:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.577422:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.577423:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:30.0:1619677448.577423:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:30.0:1619677448.577424:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.577425:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.577642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:21.2:1619677448.577646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677448.577648:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.577651:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.577652:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:30.0:1619677448.577653:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:2.0:1619677448.577654:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.577656:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:2.0:1619677448.577657:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.577658:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677448.577659:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.577659:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.577660:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:2.0:1619677448.577660:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.577661:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.577662:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.577663:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.577663:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677448.577665:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.577667:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.577667:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.577668:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.577669:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.577670:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.577672:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.577673:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.577673:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.577674:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.577675:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.577676:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.577676:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.577677:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.577677:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:30.0:1619677448.577678:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.577679:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.577680:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.577681:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.577682:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:9.2:1619677448.577682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.577683:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.577685:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.577685:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.577687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.577689:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:21.2:1619677448.577691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.577691:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.577692:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.577693:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.577694:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.577694:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.577696:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677448.577697:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.577697:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.577698:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:12.0:1619677448.577699:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.577701:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:13.0:1619677448.577701:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.577702:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677448.577702:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.577703:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677448.577719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.577727:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.577729:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.577730:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.577732:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.577734:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.577735:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.577737:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.577738:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.577739:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.577740:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.577743:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.577745:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.577746:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.577747:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.577747:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.577749:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.577749:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.577750:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.577750:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.577754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.577760:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.577762:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:5.2:1619677448.577762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:10.0:1619677448.577764:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.577765:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.577766:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.577766:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.577767:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.577769:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.577770:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.577771:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.577773:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.577774:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.577775:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.577776:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.577777:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.577778:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.577780:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.577782:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.577784:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.577785:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.577785:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.577786:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.577787:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.577788:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.577789:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.577790:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.577792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.577795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677448.577798:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:21.2:1619677448.577800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:13.0:1619677448.577800:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.577801:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.577802:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.577803:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:12.0:1619677448.577804:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677448.577805:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677448.577805:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.577805:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.577806:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.577806:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677448.577808:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.577809:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.577810:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.577812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:22.0:1619677448.577814:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.577815:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.577816:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.577818:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.577818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:22.0:1619677448.577819:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.577820:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:8.0:1619677448.577820:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.577823:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.577823:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:22.0:1619677448.577824:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.577825:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:22.0:1619677448.577826:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:8.0:1619677448.577826:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.577828:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:22.0:1619677448.577829:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.577829:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000400:00000200:31.0:1619677448.577830:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.577830:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:8.0:1619677448.577830:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.577831:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:31.0:1619677448.577832:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.577832:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:22.0:1619677448.577832:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.577832:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:8.0:1619677448.577832:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96713103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.577833:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:31.0:1619677448.577834:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:22.0:1619677448.577834:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:8.0:1619677448.577834:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.577835:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.577835:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.577836:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.577836:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677448.577837:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.577837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000400:00000200:31.0:1619677448.577838:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677448.577840:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.577842:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.577842:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.577843:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:31.0:1619677448.577844:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.577845:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.577846:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:8.0:1619677448.577846:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:31.0:1619677448.577847:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.577847:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.577847:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.577847:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677448.577848:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.577849:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:10.0:1619677448.577849:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.577849:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.577849:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.577850:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:31.0:1619677448.577851:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.577851:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.577852:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:10.0:1619677448.577852:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:8.0:1619677448.577852:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.577852:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677448.577854:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.577854:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.577855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677448.577858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.2:1619677448.577859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.577860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677448.577863:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.577866:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:24.0:1619677448.577867:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:17.0:1619677448.577867:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.577867:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.577868:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:30.0:1619677448.577869:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:24.0:1619677448.577869:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.577870:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.577870:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.577871:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.0:1619677448.577871:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:30.0:1619677448.577872:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:5.0:1619677448.577872:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.577873:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677448.577873:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677448.577874:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677448.577874:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.577874:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.577877:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.577902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.577904:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.577907:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.577909:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.577910:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.577912:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.577913:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.577914:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.577916:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.577917:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.577918:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.577921:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.577923:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.577926:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.577926:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.577927:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.577929:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.577929:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.577930:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.577931:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.577935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.577942:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.577945:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.577946:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.577946:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.577948:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.577951:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.578110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.578115:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.578118:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.578119:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.578121:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.578123:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.578125:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.578126:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.578127:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.578128:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.578129:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.578133:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.578134:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.578137:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.578138:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.578139:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.578141:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.578141:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.578142:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.578142:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.578147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.578153:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.578156:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.578157:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.578158:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.578160:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.578161:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.578272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.578277:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.578281:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.578282:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.578284:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.578287:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.578288:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.578290:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.578291:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.578292:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.578298:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.578303:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.578305:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.578308:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.578309:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.578310:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.578312:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.578312:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.578313:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.578313:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.578318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.578326:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.578328:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.578329:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.578329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:13.0:1619677448.578331:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.578333:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.578334:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.578337:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.578340:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.578341:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.578342:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.578344:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.578345:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.578347:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.578348:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.578349:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.578350:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.578352:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.578354:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.578356:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.578357:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.578357:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.578359:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.578359:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.578360:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.578361:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.578362:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.578364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:4.2:1619677448.578367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.578372:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:20.2:1619677448.578374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677448.578374:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.578374:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.578376:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.578376:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:10.0:1619677448.578376:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.578377:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.578378:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.578378:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.578379:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.578382:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.578385:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.578387:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.578389:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.578390:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.578391:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.578392:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.578393:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.578394:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.578395:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.578398:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.578400:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.578403:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.578403:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.578404:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.578405:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.578406:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.578407:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.578407:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.578411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.578412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:12.0:1619677448.578417:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.578419:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:26.0:1619677448.578420:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:12.0:1619677448.578420:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.578421:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.578422:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:13.0:1619677448.578423:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.578423:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:12.0:1619677448.578424:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:26.0:1619677448.578425:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.578427:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.578428:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.578429:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.578431:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.578432:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.578432:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.578436:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.578438:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.578440:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.578441:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.578441:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.578443:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.578444:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.578444:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.578445:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.578449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.578456:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.578458:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.578459:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.578460:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.578462:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.578463:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.578477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.578483:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.578485:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.578486:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.578488:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.578491:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.578492:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.578494:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.578495:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.578496:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.578497:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.578500:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.578502:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.578505:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.578505:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.578506:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.578508:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.578508:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.578509:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.578510:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.578510:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:25.2:1619677448.578513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:21.2:1619677448.578515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.578520:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:6.0:1619677448.578520:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:10.0:1619677448.578522:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.578523:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:6.0:1619677448.578523:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.578524:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677448.578524:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.578524:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677448.578526:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.578529:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.578534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:17.0:1619677448.578541:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677448.578543:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677448.578544:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677448.578546:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.578549:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.578551:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677448.578552:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.578553:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.578554:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.578555:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677448.578559:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.578560:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.578563:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.578564:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:17.0:1619677448.578565:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677448.578567:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677448.578567:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677448.578568:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677448.578568:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677448.578573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.578582:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:21.2:1619677448.578584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.0:1619677448.578586:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.578587:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.578588:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.578589:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.578592:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.578592:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.578593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.578595:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.578597:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.578599:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.578599:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.578601:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677448.578601:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000400:00000200:30.0:1619677448.578602:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.578603:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.578604:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.578605:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.578605:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000400:00000200:30.0:1619677448.578606:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.578606:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.578606:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.578607:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:30.0:1619677448.578609:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.578610:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.578614:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.578614:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.578615:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.578617:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.578617:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.578618:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.578619:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.578623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.578630:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.578633:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.578634:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.578634:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.578636:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.578639:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.578641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.578648:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.578650:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.578651:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.578651:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.578652:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.578653:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.578983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.578990:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.578993:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.578994:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.578995:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.578998:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.578999:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.579001:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.579002:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.579003:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.579004:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.579007:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.579009:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.579011:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.579011:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.579012:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.579013:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.579014:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.579014:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.579015:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.579019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.579026:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.579028:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.579029:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.579030:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.579032:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.579033:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.579064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.579070:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.579073:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.579074:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.579076:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.579078:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.579080:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.579081:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.579082:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.579084:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.579085:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.579087:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.579089:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.579092:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.579093:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.579094:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.579095:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.579096:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.579097:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.579097:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.579101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:21.2:1619677448.579108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:13.0:1619677448.579109:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.579111:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.579112:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.579114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:13.0:1619677448.579114:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.579115:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.579117:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.579118:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.579120:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:25.2:1619677448.579121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:30.0:1619677448.579122:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.579123:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.579124:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.579126:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.579127:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.579127:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:12.0:1619677448.579127:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:30.0:1619677448.579129:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.579130:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.579130:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.579131:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.579131:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.579131:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:30.0:1619677448.579132:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.579132:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.579133:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:31.0:1619677448.579134:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.579135:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.579136:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.579137:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713089] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.579137:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.579138:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.579139:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.579140:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.579142:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.579143:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.579143:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.579144:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.579145:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.579145:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.579146:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.579147:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.579147:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.579148:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.579148:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677448.579149:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.579149:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.579150:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.579151:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:25.2:1619677448.579151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.579152:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.579153:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.579153:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677448.579157:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.2:1619677448.579158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:13.0:1619677448.579159:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.2:1619677448.579160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:13.0:1619677448.579160:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.579162:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.579163:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.579164:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.579167:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.579168:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:25.2:1619677448.579169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.579169:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.579169:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.579170:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.579171:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.579172:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.579173:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.579175:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677448.579175:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.579176:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.579178:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.579178:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.579179:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:5.0:1619677448.579179:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.579180:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:5.0:1619677448.579180:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.579181:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:12.0:1619677448.579181:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.579182:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677448.579182:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677448.579183:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:5.0:1619677448.579183:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.579185:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.579189:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.579189:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.579190:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.579192:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.579193:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.579194:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.579194:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.579198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.579205:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.579207:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.579208:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.579210:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.579211:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.579213:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.579337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.579346:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.579349:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.579349:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.579351:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.579353:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.579355:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.579356:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.579357:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.579359:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.579359:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.579363:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.579364:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.579367:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.579367:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.579368:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.579370:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.579370:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.579371:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.579371:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.579376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.579383:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.579386:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.579386:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.579388:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.579390:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.579391:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.579459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.579466:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.579469:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.579469:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677448.579471:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.579474:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.579476:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.579477:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.579478:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.579479:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.579480:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.579483:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.579484:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.579487:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.579488:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.579489:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.579490:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.579491:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.579491:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.579492:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.579496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.579503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.579508:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.579510:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.579510:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:17.0:1619677448.579511:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.579512:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677448.579513:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.579514:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.579517:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.579517:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.579518:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677448.579519:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.579519:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.579520:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.579521:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.579522:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.579523:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.579526:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.579528:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.579531:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.579532:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.579532:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.579534:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.579535:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.579535:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.579536:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.579540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.579547:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.579550:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.579551:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.579553:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.579554:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.579555:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.579604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.2:1619677448.579606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677448.579607:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.579610:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.579612:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.579613:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:30.0:1619677448.579614:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677448.579615:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.579616:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:20.0:1619677448.579617:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677448.579618:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.579619:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.579620:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.579621:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677448.579621:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.579622:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.579622:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.579624:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.579626:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.579626:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.579627:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.579628:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.579629:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.579629:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.579632:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.579633:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.579633:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.579633:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.579635:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.579635:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.579636:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.579637:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.579637:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677448.579639:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.579639:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.579640:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:25.2:1619677448.579641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.579642:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.579642:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.579643:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.579644:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.579647:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:25.2:1619677448.579648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.579650:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.579650:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.579651:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.579653:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.579656:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:6.0:1619677448.579656:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.579659:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.579660:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.579660:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.579662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000400:00000200:6.0:1619677448.579663:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.579664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:6.0:1619677448.579666:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677448.579668:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.579670:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:26.0:1619677448.579671:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.579671:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.579672:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.579673:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.579673:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:18.0:1619677448.579674:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.579674:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.579677:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.579679:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.579681:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.579682:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.579684:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.579685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:18.0:1619677448.579685:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.579686:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.579689:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.579691:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.579693:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:18.0:1619677448.579694:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.579695:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:18.0:1619677448.579695:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.579696:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.579696:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.579698:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:18.0:1619677448.579698:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.579698:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.579699:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.579699:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:26.0:1619677448.579700:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.579702:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.579703:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.579704:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.579704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:26.0:1619677448.579705:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.579706:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.579710:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:12.0:1619677448.579710:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677448.579712:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.579713:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.579714:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.579715:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.579716:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:12.0:1619677448.579716:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.579717:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.579717:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:26.0:1619677448.579719:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:26.0:1619677448.579719:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:12.0:1619677448.579719:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:26.0:1619677448.579720:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.579720:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677448.579724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.579730:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.579733:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.579733:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.579735:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677448.579737:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.579738:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.579757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.579762:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.2:1619677448.579764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677448.579765:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.579767:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.579768:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.579769:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:2.0:1619677448.579771:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.579772:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:2.0:1619677448.579773:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:20.0:1619677448.579774:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:2.0:1619677448.579774:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.579775:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.579776:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.579777:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.579778:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.579778:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.579780:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677448.579781:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.579782:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000400:00000200:5.0:1619677448.579783:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.579784:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.579784:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.579785:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:2.0:1619677448.579786:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.579786:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.579787:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677448.579789:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.579789:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.579789:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.0:1619677448.579791:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:2.0:1619677448.579791:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.579792:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.579792:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:5.0:1619677448.579794:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.579794:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:9.2:1619677448.579795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.579795:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.579797:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.579798:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.2:1619677448.579799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.0:1619677448.579800:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.579801:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.579802:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677448.579803:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.2:1619677448.579803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:18.0:1619677448.579805:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.579805:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.579806:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:20.2:1619677448.579807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.579807:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:18.0:1619677448.579808:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.579808:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.579809:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.579810:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:18.0:1619677448.579810:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677448.579810:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:2.0:1619677448.579810:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.0:1619677448.579812:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.579813:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677448.579813:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:20.0:1619677448.579814:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677448.579814:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.579814:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.579816:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.579816:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.579819:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677448.579820:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.579822:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.579822:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.579825:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.579827:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.579829:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.579830:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.579831:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.579832:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.579833:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:4.2:1619677448.579836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:17.0:1619677448.579837:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.579838:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.579842:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.579842:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.579843:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.579843:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.579845:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.579845:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.579846:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.579847:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.579847:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.579848:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.579848:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.579849:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.579851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.579851:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.579853:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:5.2:1619677448.579853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.579854:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.579855:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.579856:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.579857:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677448.579857:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.579859:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677448.579860:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677448.579860:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:12.0:1619677448.579861:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.579861:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.579861:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677448.579862:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.579862:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.579863:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:13.0:1619677448.579864:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.579865:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.579866:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.579867:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.579867:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.579869:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.579869:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:25.2:1619677448.579870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.579870:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.579870:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.579875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:13.0:1619677448.579876:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.579879:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.579879:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.579880:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.579880:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677448.579882:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:13.0:1619677448.579882:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677448.579885:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.579886:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.579886:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.579889:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.579892:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.579909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677448.579915:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677448.579918:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677448.579919:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677448.579921:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.579924:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.579925:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677448.579926:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.579928:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.579929:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.579930:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:5.2:1619677448.579930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000400:00000200:17.0:1619677448.579933:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.579935:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.579937:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.579937:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:17.0:1619677448.579938:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:17.0:1619677448.579939:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:12.0:1619677448.579940:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:17.0:1619677448.579941:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677448.579941:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677448.579942:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677448.579942:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:12.0:1619677448.579942:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.579944:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677448.579945:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.579947:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:8.2:1619677448.579947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:12.0:1619677448.579948:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.579949:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.579950:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677448.579951:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:12.0:1619677448.579954:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.579955:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:12.0:1619677448.579956:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677448.579957:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.579958:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.579959:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.579960:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.579960:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:12.0:1619677448.579960:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677448.579961:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.579962:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:12.0:1619677448.579962:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:12.0:1619677448.579963:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677448.579964:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677448.579965:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677448.579966:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:9.2:1619677448.579968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.2:1619677448.579972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:18.0:1619677448.579976:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.579978:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.579979:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.579979:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:2.0:1619677448.579979:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.579982:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.579982:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677448.579982:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677448.579983:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:18.0:1619677448.579985:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.579989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.579996:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.579999:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.580000:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.580000:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.580001:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.580002:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.580074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.580080:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.580083:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.580083:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.580085:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.580088:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.580089:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.580090:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.580092:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.580092:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.580094:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.580097:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.580098:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.580101:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.580102:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.580103:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.580104:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.580105:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.580105:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.580106:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.580110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.580117:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.580120:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.580121:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.580122:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.580123:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.580125:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.580130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.580138:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.580140:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.580142:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.580144:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.580145:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.580147:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.580148:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.580149:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.580150:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.580151:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.580154:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.580155:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.580158:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.580159:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.580159:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.580161:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.580161:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.580163:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.580165:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.580165:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.580166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.580171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.580173:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.580175:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.580176:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.580177:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.580178:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.580179:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.580179:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.580181:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.580182:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.580183:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677448.580189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.580196:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.580199:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.580201:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.580203:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.580203:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.580205:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.580206:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.580207:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.580208:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.580209:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.580212:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.580214:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.580217:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.580217:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.580218:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.580220:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.580220:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.580221:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.580221:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.580226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.580234:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.580236:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.580238:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.580239:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.580239:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.580243:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.580258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.580264:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.580267:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:5.2:1619677448.580267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677448.580267:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.580269:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.580271:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.580273:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.580274:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.580275:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:18.0:1619677448.580275:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.580276:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.580278:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:18.0:1619677448.580278:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.580278:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.580283:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.580283:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.580285:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.580286:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.580287:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:18.0:1619677448.580289:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.580290:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.580290:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:26.0:1619677448.580291:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.580291:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:26.0:1619677448.580292:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.580292:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:25.2:1619677448.580299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.580300:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:26.0:1619677448.580301:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.580301:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.580302:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:26.0:1619677448.580305:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.580307:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.580307:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677448.580310:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.580310:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.580310:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.580311:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.580311:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:13.0:1619677448.580312:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.580313:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.580313:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.580314:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:12.0:1619677448.580314:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.580315:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.580315:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.580319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.580325:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.580327:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.580328:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.580329:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.580331:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.580332:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.580336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:13.0:1619677448.580343:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.580346:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.580346:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.2:1619677448.580347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:13.0:1619677448.580347:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.580347:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.580348:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.0:1619677448.580351:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.580353:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.580355:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.580356:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.580358:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.580360:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.580361:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.580363:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.580364:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.580365:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.580368:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.580368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677448.580370:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.580373:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.580374:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.580374:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.580375:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.580377:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.580377:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.0:1619677448.580378:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.580378:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.580379:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.580379:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677448.580380:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.580382:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.580383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:17.0:1619677448.580384:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.580385:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.580386:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.580387:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.580388:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677448.580389:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:17.0:1619677448.580392:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.580392:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.580392:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.580393:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:13.0:1619677448.580394:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.580395:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.580396:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:12.0:1619677448.580396:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.580397:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677448.580397:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677448.580398:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.580399:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.580399:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.580400:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.580405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.580412:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.580414:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.580414:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.580415:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.580416:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.580418:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.580425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.580432:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.580434:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.580435:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.580437:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.580440:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.580441:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.580442:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.580444:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.580445:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.580446:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.580449:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.580450:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.580453:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.580454:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.580455:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.580456:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.580457:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.580457:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.580458:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.580462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.580471:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.580474:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.580474:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.580475:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.580477:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.580480:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.580667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.580674:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.580677:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.580678:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.580679:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.580682:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.580684:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.580685:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.580686:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.580687:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.580688:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.580691:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.580693:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.580696:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.580696:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:21.2:1619677448.580696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:24.0:1619677448.580697:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.580699:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.580699:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.580700:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.580700:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:31.0:1619677448.580704:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.2:1619677448.580705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677448.580707:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.2:1619677448.580707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:31.0:1619677448.580709:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.580712:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.580712:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.580714:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677448.580714:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.580715:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.580715:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.580716:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.580716:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:31.0:1619677448.580717:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.580717:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.580718:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.580719:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.580720:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.580720:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.580722:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.580723:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.580724:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:31.0:1619677448.580725:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.580725:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.580727:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.580728:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.580728:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:1.0:1619677448.580728:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.580729:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677448.580729:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.580730:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.580731:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.580732:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.580732:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:1.0:1619677448.580733:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.580734:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.580735:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.580735:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.2:1619677448.580736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.580736:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.580737:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.580738:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.580738:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.580738:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.580739:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (24)--
00000800:00000200:1.0:1619677448.580739:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (23)++
00000800:00000200:17.0:1619677448.580740:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (24)--
00000800:00000200:17.0:1619677448.580741:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.580741:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (25)--
00000800:00000200:1.0:1619677448.580741:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.580742:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.580742:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.580743:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.580745:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.580746:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.580746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:12.0:1619677448.580747:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.580749:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.580750:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.580753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.580753:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.580756:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.580756:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.580756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:1.0:1619677448.580757:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.580758:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.580759:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.580760:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:13.0:1619677448.580762:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.580762:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:13.0:1619677448.580763:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.580764:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.580764:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.580765:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.580766:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:13.0:1619677448.580766:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677448.580768:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.580770:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.580772:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.580773:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.580774:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.580775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:2.0:1619677448.580775:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.580776:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.580780:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.580782:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.580784:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.580784:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.580786:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.580786:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.580787:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:18.0:1619677448.580788:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.580788:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677448.580790:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:2.0:1619677448.580790:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.580791:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.580793:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.580793:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.580794:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.580795:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:9.2:1619677448.580796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.580796:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:18.0:1619677448.580798:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.580798:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.580798:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.580800:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.580800:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.580803:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.580803:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.580804:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.2:1619677448.580804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.580804:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.580805:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:2.0:1619677448.580805:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.580806:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.580807:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.580808:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.580808:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.580809:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.580809:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677448.580810:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.580810:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.580811:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.580811:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.580812:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677448.580812:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:25.2:1619677448.580813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.580815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677448.580819:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:2.0:1619677448.580821:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:26.0:1619677448.580822:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.580823:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.580823:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:2.0:1619677448.580823:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.580824:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:26.0:1619677448.580825:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.580825:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:26.0:1619677448.580829:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.580993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.581000:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.581003:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.581003:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.581005:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.581008:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.581010:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.581011:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.581012:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.581013:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.581014:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.581017:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.581019:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.581022:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.581022:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.581023:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.581025:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.581025:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.581026:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.581027:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.581028:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677448.581031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677448.581033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.581039:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.581040:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:18.0:1619677448.581041:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:5.2:1619677448.581041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:26.0:1619677448.581042:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.581043:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677448.581043:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.581044:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677448.581044:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.581045:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.581045:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677448.581046:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:9.2:1619677448.581047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.581049:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.581049:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.581051:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677448.581051:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.581052:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.581053:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.581053:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:2.0:1619677448.581054:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.581055:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.581056:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.581056:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.581056:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.581057:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:2.0:1619677448.581057:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.581059:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:20.2:1619677448.581062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:2.0:1619677448.581062:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.581064:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.581067:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.581067:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.581067:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:2.0:1619677448.581068:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677448.581069:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.581070:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677448.581070:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677448.581071:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000400:00000200:5.0:1619677448.581072:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.581072:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:20.0:1619677448.581073:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677448.581073:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:8.2:1619677448.581075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:5.0:1619677448.581075:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.581078:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (21)++
00000400:00000200:5.0:1619677448.581081:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.581082:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.581083:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.581083:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:18.0:1619677448.581084:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000400:00000200:5.0:1619677448.581084:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.581085:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.581085:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:18.0:1619677448.581086:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:17.0:1619677448.581086:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.581086:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.581087:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677448.581088:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000400:00000200:1.0:1619677448.581089:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.581090:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.581092:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.0:1619677448.581093:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.2:1619677448.581095:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.0:1619677448.581099:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.581100:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.581100:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.581100:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.581102:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:5.0:1619677448.581102:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.581103:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.581103:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.581104:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:17.0:1619677448.581105:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.581105:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677448.581106:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.581108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:17.0:1619677448.581108:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.581109:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.581110:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.581111:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.581112:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:30.0:1619677448.581115:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:17.0:1619677448.581116:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.581118:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:17.0:1619677448.581118:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:30.0:1619677448.581119:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.581121:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.581122:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.581122:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.581122:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.581123:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.581124:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:30.0:1619677448.581125:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677448.581125:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.581127:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.581128:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.581128:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.581130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.2:1619677448.581133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.581137:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:1.0:1619677448.581138:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677448.581139:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.581140:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.581140:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.581140:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.581141:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677448.581142:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.581142:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677448.581146:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.2:1619677448.581337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.581343:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.581346:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.581347:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.581348:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.581351:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.581353:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.581354:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.581355:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.581356:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.581357:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.581360:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.581362:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.581364:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.581365:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.581365:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.581367:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.581367:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.581368:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.581368:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.581374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.581381:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.581383:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.581383:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.581384:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.581385:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.581387:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.581535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.581538:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.581540:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.581543:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.581543:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.581545:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.581547:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.581548:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.581550:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.581551:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.581552:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.581555:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.581557:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.581560:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.581561:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.581562:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.581563:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.581564:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.581565:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.581566:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.581567:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.581569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.581572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:31.0:1619677448.581577:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.581577:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.581580:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:31.0:1619677448.581581:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:5.0:1619677448.581581:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.581581:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:31.0:1619677448.581582:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.581583:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:30.0:1619677448.581585:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.581586:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.581836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.581844:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.581846:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.581847:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.581849:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.581851:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.581853:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.581854:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.581856:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.581856:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.581857:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:21.2:1619677448.581858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:24.0:1619677448.581861:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.581863:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677448.581864:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677448.581866:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.581867:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.581867:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677448.581867:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.581868:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677448.581869:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.581870:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:22.0:1619677448.581870:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.581871:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.581871:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:22.0:1619677448.581873:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.581874:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.581875:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.581875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:22.0:1619677448.581877:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.581878:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.581879:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.581883:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.581883:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:22.0:1619677448.581884:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677448.581886:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.581887:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.581887:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.581888:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.581888:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.581889:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:2.0:1619677448.581890:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.581891:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.581892:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.581892:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677448.581892:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677448.581893:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.581897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.581904:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.581907:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.581908:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.581910:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.581911:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.581913:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.581915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677448.581922:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.581925:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.581925:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.581926:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.581927:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.581928:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.581939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.581947:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.581949:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.581950:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.581953:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.581955:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.581957:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.581958:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.581960:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.581961:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.581962:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.581965:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.581966:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.581969:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.581970:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.581970:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.581972:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.581972:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.581973:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.581973:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677448.581975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:25.2:1619677448.581977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.581982:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.581984:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677448.581984:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.581985:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.581986:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.581987:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.581988:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.581989:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.581990:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.581991:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.581992:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:17.0:1619677448.581992:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.581993:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.581994:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.581995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:17.0:1619677448.581996:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.581996:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.582000:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.582002:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.582003:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.582004:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.582005:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:17.0:1619677448.582005:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.582006:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.582006:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.582008:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.582008:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.582009:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.582009:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.582010:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.582010:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677448.582012:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.582013:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.582014:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.582014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:26.0:1619677448.582015:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.582016:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.582019:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.582021:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677448.582021:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:26.0:1619677448.582022:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.582023:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.582024:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:6.0:1619677448.582024:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:26.0:1619677448.582025:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:22.0:1619677448.582025:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.582025:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.582026:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.582027:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:6.0:1619677448.582027:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.582028:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.582029:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.582030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:6.0:1619677448.582030:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.582032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.2:1619677448.582034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677448.582036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.582040:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.582040:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:30.0:1619677448.582042:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:10.0:1619677448.582042:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.582043:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.582043:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677448.582045:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.582045:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:31.0:1619677448.582046:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.582046:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:5.0:1619677448.582046:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:30.0:1619677448.582047:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.582047:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.582047:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.582048:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.582048:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677448.582049:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:10.0:1619677448.582049:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:5.0:1619677448.582049:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.582050:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:30.0:1619677448.582051:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.582052:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.582052:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.582053:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.582054:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.582054:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677448.582055:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.582057:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.582058:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.582060:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.582060:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.582064:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.582064:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.582067:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.582067:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.582068:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.582069:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.582069:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.582069:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.582070:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.582070:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.582071:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.582072:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.582072:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677448.582072:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.582073:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.582074:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.582074:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.582076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.2:1619677448.582081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.582085:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.582087:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677448.582088:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677448.582088:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.582089:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.582091:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.582091:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.582091:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.582093:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.582094:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677448.582094:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677448.582096:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.582188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.582195:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.582198:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.582201:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.582202:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.582204:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.582205:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.582207:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.582208:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.582209:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.582210:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.582213:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.582215:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.582218:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.582219:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.582220:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.582222:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.582222:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.582224:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.582225:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.582226:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.582227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.582232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.582236:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:21.2:1619677448.582238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:5.0:1619677448.582239:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:30.0:1619677448.582240:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:5.0:1619677448.582241:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.582242:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:20.0:1619677448.582242:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.582243:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:5.0:1619677448.582243:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.582244:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:18.0:1619677448.582244:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.0:1619677448.582245:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677448.582247:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.582248:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.582249:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.582251:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.582253:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677448.582254:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.582255:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.582256:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.582257:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677448.582261:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.582262:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.582265:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.582265:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677448.582266:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677448.582268:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.582268:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.582269:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.582269:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.582274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.582281:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.582283:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.582284:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.582285:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.582287:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.582289:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.582294:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:6.0:1619677448.582302:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.582304:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.582305:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.582305:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.582306:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.582307:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.582395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.582401:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.582403:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.582404:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.582406:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.582409:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.582409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:22.0:1619677448.582410:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.582411:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.582412:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.582413:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.582415:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.582416:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677448.582418:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.582419:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.582420:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677448.582420:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.582422:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.582422:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.582423:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.582424:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:10.0:1619677448.582424:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.582425:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:22.0:1619677448.582426:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.582426:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677448.582426:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.582427:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.582427:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677448.582428:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.582429:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.582430:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:8.2:1619677448.582432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:10.0:1619677448.582433:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.582434:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.582437:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.582438:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.582439:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.582440:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:10.0:1619677448.582441:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.582441:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.582442:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:10.0:1619677448.582442:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677448.582443:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.582443:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:20.0:1619677448.582445:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.582446:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:8.2:1619677448.582447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.582450:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.582452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.582453:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.582456:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.582457:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:5.0:1619677448.582457:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.582458:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.582459:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.582459:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.582460:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.582461:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.582461:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:20.0:1619677448.582462:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.582462:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:26.0:1619677448.582464:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677448.582465:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:26.0:1619677448.582466:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.582467:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.582469:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:8.2:1619677448.582471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000400:00000200:26.0:1619677448.582472:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.582473:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.582474:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.582476:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.582477:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.582477:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.582478:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.582480:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.582481:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.582481:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.582482:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.582483:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677448.582483:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.582484:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.582487:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.582487:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.582488:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.582489:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.582490:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.582491:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677448.582492:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.582493:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677448.582495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.582498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:5.0:1619677448.582502:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.582504:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.582505:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.582506:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677448.582507:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:5.0:1619677448.582507:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677448.582508:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.582508:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.582510:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:5.0:1619677448.582510:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.582681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.582689:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.582691:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.582692:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.582694:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.582697:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.582699:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.582700:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.582701:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.582702:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.582703:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.582706:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.582708:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.582711:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.582711:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.582712:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.582713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.0:1619677448.582714:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.582714:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.582715:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.582715:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.582720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.582721:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.582723:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.582724:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.582726:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.582728:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.582728:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:26.0:1619677448.582729:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.582730:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.582731:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:26.0:1619677448.582732:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.582732:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.582732:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.582732:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:26.0:1619677448.582734:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677448.582735:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.582738:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.582738:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677448.582739:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.2:1619677448.582739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:26.0:1619677448.582741:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.582741:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.582742:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.582743:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.582744:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.582744:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.582745:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:1.0:1619677448.582746:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:25.2:1619677448.582749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:17.0:1619677448.582749:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.582749:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.582750:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.582751:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.582751:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677448.582755:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.582757:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.582758:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.582758:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.582760:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.582761:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.582790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.582796:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.582799:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.582800:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.582802:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.582804:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.582805:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677448.582807:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.582808:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.582809:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.582810:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.582813:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.582815:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.582818:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.582819:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.582819:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.582821:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.582821:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.582822:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.582823:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677448.582827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.582835:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.582837:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.582838:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.582840:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.582841:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.582842:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.582910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.582918:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:21.2:1619677448.582918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:30.0:1619677448.582921:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.582923:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.582923:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:4.2:1619677448.582924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.582926:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.582926:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.582926:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.582927:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:30.0:1619677448.582928:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.582930:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.582930:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.582931:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.582932:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.582932:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677448.582933:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:6.0:1619677448.582933:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.582934:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:26.0:1619677448.582935:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.582935:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.582936:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.582939:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.582939:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.582939:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.582940:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.582941:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.582941:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:30.0:1619677448.582942:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.582944:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.582945:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.582945:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:6.0:1619677448.582945:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.582947:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:26.0:1619677448.582947:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:6.0:1619677448.582947:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:30.0:1619677448.582948:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.582949:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.582949:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:26.0:1619677448.582949:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96713038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.582949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:6.0:1619677448.582950:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.582951:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.582951:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000400:00000200:26.0:1619677448.582952:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.582952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:6.0:1619677448.582952:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:26.0:1619677448.582953:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.582953:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:5.0:1619677448.582955:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677448.582956:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.582957:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:6.0:1619677448.582958:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.582958:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:31.0:1619677448.582959:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:26.0:1619677448.582959:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:25.2:1619677448.582959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.582959:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677448.582960:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.582960:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.582962:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:5.0:1619677448.582962:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.582963:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.582963:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.582964:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:5.0:1619677448.582964:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.582965:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.582965:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:5.0:1619677448.582965:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96713037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.582966:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.582966:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:31.0:1619677448.582967:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:26.0:1619677448.582967:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:5.0:1619677448.582967:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.582968:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.582968:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:5.0:1619677448.582968:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.582969:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.582969:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.582969:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:26.0:1619677448.582970:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.582970:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.582971:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.582973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:5.0:1619677448.582973:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.582975:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.0:1619677448.582975:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677448.582977:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.582978:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677448.582979:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.582980:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.582980:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677448.582981:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.582982:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:22.0:1619677448.582983:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677448.582983:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:6.0:1619677448.582984:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.582984:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:22.0:1619677448.582985:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.582987:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.582987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.2:1619677448.582989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677448.582990:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677448.582992:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.582994:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:18.0:1619677448.582994:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:20.0:1619677448.582995:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.582995:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677448.582996:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.582998:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.582998:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:25.2:1619677448.582999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000400:00000200:18.0:1619677448.583000:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.583003:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:6.0:1619677448.583006:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.583009:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.583010:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.583010:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.583011:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.583012:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.583230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.583235:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.583238:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.583239:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.583241:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.583243:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.583245:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.583246:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.583248:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.583249:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.583250:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:4.2:1619677448.583250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:17.0:1619677448.583253:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.583255:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.583257:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.583258:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.583259:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.583260:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:17.0:1619677448.583260:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.583261:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.583262:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.583262:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.583263:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.583263:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.583263:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677448.583267:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.583268:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:25.2:1619677448.583268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:26.0:1619677448.583269:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.583271:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.583272:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.583273:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.583274:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:26.0:1619677448.583276:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.583277:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.583278:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.583278:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:30.0:1619677448.583279:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.583280:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.583280:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.583281:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.583282:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.583282:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:26.0:1619677448.583284:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.583284:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.583286:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.583287:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.583287:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.583289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.583293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.583298:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.583300:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:10.0:1619677448.583300:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677448.583301:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.583302:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.583302:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.583303:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.583304:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.583305:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.583305:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.583336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.583342:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.583345:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.583346:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.583348:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.583351:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.583352:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.583353:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.583355:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.583356:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.583357:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.583360:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.583362:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.583365:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.583366:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.583366:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.583368:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.583369:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.583370:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.583371:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.583372:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:25.2:1619677448.583375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:21.2:1619677448.583377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.583381:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:6.0:1619677448.583382:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:26.0:1619677448.583384:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:6.0:1619677448.583384:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.583384:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:10.0:1619677448.583385:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.583386:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.583386:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:26.0:1619677448.583388:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.583389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:26.0:1619677448.583391:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.0:1619677448.583392:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:25.2:1619677448.583393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000800:00000200:20.0:1619677448.583394:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.583396:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.583397:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.583399:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.583399:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000400:00000200:20.0:1619677448.583400:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:10.0:1619677448.583401:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.583402:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:20.0:1619677448.583402:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.583402:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000400:00000200:20.0:1619677448.583403:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.583403:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000400:00000200:20.0:1619677448.583404:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.583404:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:20.0:1619677448.583405:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.583409:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.583410:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.583413:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.583414:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.583414:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.583416:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.583417:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.583418:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.583418:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677448.583423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.583431:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.583434:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.583435:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.583435:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.583437:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677448.583440:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.583531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.583536:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.583539:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.583541:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.583541:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.583543:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.583545:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.583546:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.583547:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.583548:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.583549:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.583552:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.583554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677448.583554:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.583557:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.583557:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.583558:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.583560:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.583560:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.583561:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677448.583562:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.583562:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677448.583565:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:25.2:1619677448.583566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:31.0:1619677448.583567:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.583568:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.583569:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.583570:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.583571:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.583572:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.583573:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.583573:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:31.0:1619677448.583574:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.583576:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:31.0:1619677448.583577:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.583577:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.583577:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.583579:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:22.0:1619677448.583579:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.583581:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677448.583582:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.583583:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.583583:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.583585:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.583585:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.583587:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.583587:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.583590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.583598:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.583600:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.583601:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.583602:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.583604:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.583606:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.583657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.583664:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.583667:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.583669:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.583671:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.583671:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.583673:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.583674:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.583675:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.583676:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.583677:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.583680:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.583681:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.583684:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.583685:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.583685:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.583687:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.583687:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.583688:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.583689:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.583692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.583698:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.583701:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.583702:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.583702:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.583704:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.583706:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.583715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.583721:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.583723:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.583724:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.583724:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.583725:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.583726:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.583817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:4.2:1619677448.583818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.583825:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677448.583825:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:26.0:1619677448.583828:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:13.0:1619677448.583828:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:10.0:1619677448.583828:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:13.0:1619677448.583831:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.583832:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.583832:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:26.0:1619677448.583833:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.583834:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.583835:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:13.0:1619677448.583836:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.583837:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677448.583837:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.583839:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.583839:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.583840:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677448.583840:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.583841:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:13.0:1619677448.583842:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.583847:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677448.583847:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.583850:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:13.0:1619677448.583850:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:13.0:1619677448.583852:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.583853:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.583853:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.583854:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.583854:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:13.0:1619677448.583854:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.583856:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:13.0:1619677448.583856:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:13.0:1619677448.583856:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:26.0:1619677448.583857:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677448.583857:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:26.0:1619677448.583858:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.583858:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:13.0:1619677448.583858:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:25.2:1619677448.583861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:25.2:1619677448.583865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.583867:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.583869:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.583870:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.583870:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.583872:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.583872:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.583874:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:6.0:1619677448.583875:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.583876:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.583876:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.583878:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.583882:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.583951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.583956:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.583959:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.583960:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.583962:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.583964:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.583966:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.583967:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.583969:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.583970:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.583971:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.583975:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.583977:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.583979:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.583980:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.583981:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.583983:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.583984:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.583984:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.583985:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.583989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.583996:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.583999:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.584000:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.584001:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.584002:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.584005:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.584005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:25.2:1619677448.584007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.584012:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.584014:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.584015:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.584016:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.584017:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.584018:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:26.0:1619677448.584018:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.584019:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.584019:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.584021:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:26.0:1619677448.584021:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.584022:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.584023:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.584025:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.584026:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.584027:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.584030:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.584031:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.584034:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.584035:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.584036:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.584037:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.584038:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.584039:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.584039:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.584043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.584050:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.584052:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.584053:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.584054:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.584056:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.584056:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.584114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.584116:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.584119:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.584121:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.584122:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.584124:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.584125:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.584126:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.584127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:20.0:1619677448.584128:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.584129:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.584130:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.584132:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:20.0:1619677448.584133:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.584134:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.584135:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.584135:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677448.584137:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.584138:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.584138:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.584139:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:22.0:1619677448.584141:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.584141:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.584141:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:22.0:1619677448.584142:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.0:1619677448.584142:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.584143:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:22.0:1619677448.584144:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.584145:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.584146:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.584147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:22.0:1619677448.584147:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.584151:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.584153:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.584155:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.584156:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.584157:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.584157:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.584158:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.584158:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:6.0:1619677448.584159:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.584161:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.584161:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.584161:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.584162:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.584162:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677448.584163:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.584167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.584175:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.584177:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.584178:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.584178:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.584180:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.584183:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.584194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.584202:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.584205:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.584207:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.584209:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.584210:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.584211:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.584213:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.584214:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.584215:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.584216:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.584219:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.584220:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.584224:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.584225:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.584226:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.584227:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.584228:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.584229:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.584229:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.584233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.584241:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.584244:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.584245:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.584246:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.584247:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.584249:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.584270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.584276:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.584279:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.584280:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.584282:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.584285:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.584286:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.584288:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.584289:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.584290:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.584291:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.584299:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.584300:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.584303:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.584303:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.584304:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.584305:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.584306:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.584307:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.584307:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.584312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.584319:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.584321:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.584322:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.584324:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.584326:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.584327:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.584347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.584355:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.584358:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.584358:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.584360:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.584363:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.584364:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.584366:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.584367:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.584368:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.584368:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.584371:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.584373:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.584375:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.584376:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.584377:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.584378:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.584379:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.584379:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.584380:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.584384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.584391:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.584394:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.584395:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.584396:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.584399:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.584399:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.584421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.584428:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.584431:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.584432:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.584434:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.584436:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.584437:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:5.2:1619677448.584437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:8.0:1619677448.584439:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.584440:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.584441:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.584442:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.584443:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677448.584445:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.584446:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:8.0:1619677448.584447:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.584447:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.584449:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.584450:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.584450:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.584451:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:17.0:1619677448.584452:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.584453:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:8.0:1619677448.584453:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.584453:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:17.0:1619677448.584455:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.584455:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:17.0:1619677448.584456:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.584456:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.584456:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.584457:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.584458:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.2:1619677448.584459:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:17.0:1619677448.584462:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.584462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.584464:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.584468:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.584468:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677448.584469:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.584469:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.584469:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.584469:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:24.0:1619677448.584470:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677448.584470:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.584471:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677448.584471:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.584472:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677448.584472:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:1.0:1619677448.584472:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.584473:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.584474:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.584475:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.584476:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677448.584476:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.2:1619677448.584477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.584480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.584485:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.584487:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.584487:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.584487:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.584487:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.584488:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.584489:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.584489:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.584490:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.584491:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.584527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.584534:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.584537:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.584539:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.584540:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.584542:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.584543:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.584545:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.584546:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.584547:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.584548:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.584551:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.584553:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.584555:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.584556:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.584557:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.584558:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.584559:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.584560:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.584562:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.584562:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.584564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.584568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.584570:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.584572:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.584573:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.584574:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.584576:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677448.584576:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.584577:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.584578:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.584579:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.584580:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677448.584653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.584660:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.584663:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.584665:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.584667:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.584667:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.584668:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.584670:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.584671:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.584672:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.584673:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.584676:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.584677:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.584680:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.584681:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.584682:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.584684:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.584684:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.584685:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.584687:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.584687:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.584690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.584693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.584701:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.584701:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677448.584704:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:20.0:1619677448.584704:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677448.584705:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.584705:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.584706:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.584707:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.584707:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.584710:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.584735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.584741:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.584744:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.584745:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.584746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:17.0:1619677448.584747:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.584749:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.584750:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.584751:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.584752:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.584753:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.584754:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677448.584754:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:26.0:1619677448.584756:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.584757:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:17.0:1619677448.584758:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.584760:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.584761:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:26.0:1619677448.584762:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.584763:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.584764:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:17.0:1619677448.584764:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.584765:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:26.0:1619677448.584766:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.584767:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.584767:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.584767:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.584768:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.584768:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.584769:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.584769:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677448.584772:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.584773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:26.0:1619677448.584774:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.584777:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.584777:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.584778:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677448.584779:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:26.0:1619677448.584780:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.584780:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.584781:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.584781:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677448.584781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.0:1619677448.584781:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.584782:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.584783:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.584785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:8.0:1619677448.584785:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.584786:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677448.584788:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.0:1619677448.584788:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.584791:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.584791:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.584792:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:5.0:1619677448.584793:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.584794:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.584795:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.584795:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.584796:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713014] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.584797:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.584798:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677448.584798:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.584799:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677448.584799:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.584800:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.584803:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.584805:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.584808:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.584809:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.584810:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.584812:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.584812:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.584813:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.584814:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.584817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.584824:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.584826:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.584827:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.584829:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.584830:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.584832:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.584833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.584839:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.584842:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.584843:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.584844:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.584844:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.584846:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.584880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.584886:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.584888:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.584890:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.584891:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:4.2:1619677448.584893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677448.584893:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.584895:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.584896:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.584897:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.584898:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.584899:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.584900:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.584902:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.584903:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677448.584903:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.584906:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.584907:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.584907:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.584908:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.584909:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677448.584910:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.584911:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.584911:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:10.0:1619677448.584912:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:2.0:1619677448.584912:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:10.0:1619677448.584913:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96713012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.584913:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:21.2:1619677448.584914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:10.0:1619677448.584914:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.584915:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.584916:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:9.2:1619677448.584917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677448.584920:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.584921:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.584922:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.584923:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.584924:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:10.0:1619677448.584924:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.584925:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:18.0:1619677448.584926:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.584926:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.584926:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.584926:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.584927:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677448.584927:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:10.0:1619677448.584928:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:2.0:1619677448.584928:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.584929:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.584930:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.584930:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:25.2:1619677448.584931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:18.0:1619677448.584932:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96713011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.584933:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.584934:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.584935:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.584937:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:18.0:1619677448.584938:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.584939:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.584940:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.584940:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.584940:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.584941:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.584942:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.584942:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:26.0:1619677448.584943:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.584943:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.584943:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.584943:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:26.0:1619677448.584944:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:18.0:1619677448.584946:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.584947:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.584948:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.584948:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.584952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.584959:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.584962:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.584963:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.584963:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.584965:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.584966:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.584969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.584976:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.584978:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.584979:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.584980:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.584980:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.584981:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.585134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.585140:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.585143:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.585144:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.585146:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.585148:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.585150:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.585151:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.585152:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.585153:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.585154:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.585157:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.585158:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.585161:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.585161:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.585162:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.585164:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.585164:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.585165:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.585166:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.585170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.585176:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.585179:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.585179:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.585180:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.585182:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.585182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:24.0:1619677448.585185:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:2.0:1619677448.585188:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:9.2:1619677448.585191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:2.0:1619677448.585191:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.585192:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.585194:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.585195:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.585197:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000400:00000200:2.0:1619677448.585197:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.585198:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.585199:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.585200:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677448.585200:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.585201:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.585201:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (21)++
00000400:00000200:2.0:1619677448.585202:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677448.585203:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.585203:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:2.0:1619677448.585205:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.585207:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.585210:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.585211:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.585212:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.585214:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.585214:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.585215:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.585215:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.585220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.585226:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.585229:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.585230:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.585231:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.585233:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.585234:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.585244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.585252:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.585255:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.585257:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.585259:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.585260:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.585262:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.585263:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.585264:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.585265:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.585266:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.585270:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.585272:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.585275:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.585276:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.585276:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.585278:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.585279:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.585280:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.585281:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.585285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.585293:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.585302:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.585302:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)++
00000400:00000200:20.0:1619677448.585303:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.585304:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677448.585305:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.585305:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:5.0:1619677448.585305:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:5.0:1619677448.585306:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.585307:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.585308:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.585379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.585385:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.585387:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.585390:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.585390:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.585393:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.585394:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.585396:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.585397:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.585398:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.585399:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.585403:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.585404:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.585407:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.585408:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.585409:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.585411:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.585411:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.585413:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.585414:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.585414:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.585417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.585419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.585422:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.585424:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:30.0:1619677448.585425:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.585425:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.585426:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.585428:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.585429:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.585431:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.585432:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.585435:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.585468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.585475:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.585478:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.585480:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.585481:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.585483:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.585484:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.585485:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.585486:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.585487:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.585488:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.585491:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.585493:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.585495:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.585496:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.585497:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.585498:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.585499:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.585500:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.585500:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677448.585501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:25.2:1619677448.585504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.585507:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.585510:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:12.0:1619677448.585511:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.585511:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.585513:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.585513:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.585514:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.585516:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.585516:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.585518:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:13.0:1619677448.585518:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:18.0:1619677448.585519:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96713005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.585519:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:18.0:1619677448.585520:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.585521:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.585522:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.585525:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.585527:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.2:1619677448.585528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677448.585530:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.585531:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.585531:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.585533:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.585534:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.585535:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.585535:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677448.585535:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.585537:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:25.2:1619677448.585539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.585539:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.585540:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.585544:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.585545:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:13.0:1619677448.585545:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:17.0:1619677448.585546:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96713004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.585547:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.585548:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:17.0:1619677448.585549:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677448.585549:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.585550:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:13.0:1619677448.585551:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.585552:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:17.0:1619677448.585554:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.585555:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677448.585556:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.585559:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.585560:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.585561:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.585562:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.585563:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.585563:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.585564:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.585568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.585575:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.585578:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.585579:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.585579:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.585581:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.585584:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.585598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.585606:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.585608:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.585609:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.585611:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.585613:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.585614:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.585616:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.585617:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.585618:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.585619:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.585622:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.585624:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.585627:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.585628:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.585629:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.585630:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.585631:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.585632:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.585633:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.585634:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.585637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.585639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.585645:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.585646:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.585648:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677448.585648:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.585648:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.585649:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.585649:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.585650:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.585651:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.585653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.585657:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.585658:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.585659:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.585660:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.585661:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.585662:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.585662:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.585677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:4.2:1619677448.585681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677448.585685:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.585687:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.585688:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677448.585688:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677448.585690:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.585690:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.585691:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:26.0:1619677448.585693:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.585693:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.585694:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.585695:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96713002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.585695:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.585696:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.585697:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.585698:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.585698:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96713002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.585699:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.585700:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.585701:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.585702:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.585703:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.585705:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.585706:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.585707:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.585708:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.585708:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.585709:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.585709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:26.0:1619677448.585710:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.585710:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.585711:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677448.585711:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.585712:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677448.585712:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.585713:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:8.0:1619677448.585713:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.585714:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677448.585714:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.585715:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.585715:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677448.585715:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:25.2:1619677448.585716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677448.585718:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.585719:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677448.585719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.2:1619677448.585720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:2.0:1619677448.585721:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.585722:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:2.0:1619677448.585724:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.585725:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.585725:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.585725:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.585726:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96713000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.585727:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.585727:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:2.0:1619677448.585728:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.585728:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:13.0:1619677448.585729:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.585729:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.585729:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.585729:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:2.0:1619677448.585729:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.585730:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.585731:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:1.0:1619677448.585731:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.585732:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.585732:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.585733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:2.0:1619677448.585733:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.585735:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.585735:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.585737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677448.585737:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.585738:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.585739:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.585739:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.585741:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677448.585741:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.585741:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:1.0:1619677448.585741:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.585742:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677448.585743:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:2.0:1619677448.585743:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677448.585743:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.585744:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.585745:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.585745:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.585745:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.585746:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.585747:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:9.2:1619677448.585747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677448.585748:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.585750:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.585751:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.585752:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.585753:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.585754:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:1.0:1619677448.585754:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:18.0:1619677448.585756:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.585757:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.585758:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:1.0:1619677448.585758:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.585759:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.585760:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677448.585761:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677448.585764:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.585765:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.585765:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.585767:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.585767:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.585769:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.585770:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.585771:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.585772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.585775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.585779:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.585781:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:17.0:1619677448.585781:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677448.585782:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.585783:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.585783:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.585783:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.585784:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677448.585785:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.585788:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677448.585823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677448.585826:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677448.585829:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677448.585831:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.585832:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677448.585834:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.585836:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:20.0:1619677448.585837:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712998] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.585839:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.585840:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.585841:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:20.0:1619677448.585843:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.585845:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:20.0:1619677448.585848:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.585849:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:20.0:1619677448.585849:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:20.0:1619677448.585851:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:20.0:1619677448.585852:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677448.585853:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:20.0:1619677448.585855:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:20.0:1619677448.585855:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:25.2:1619677448.585857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:20.2:1619677448.585860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:26.0:1619677448.585863:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:5.0:1619677448.585865:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000800:00000200:26.0:1619677448.585866:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.585867:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.585867:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:5.0:1619677448.585868:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000800:00000200:5.0:1619677448.585868:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:5.0:1619677448.585869:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000400:00000200:26.0:1619677448.585870:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.585872:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677448.585895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.585902:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.585904:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.585905:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.585907:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.585909:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.585910:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.585911:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.585913:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.585914:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.585915:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.585918:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.585920:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.585923:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.585923:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.585924:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.585926:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.585927:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.585927:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.585928:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677448.585932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.585940:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.585942:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.585944:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.585944:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.585946:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.585949:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.585972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.585980:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.585983:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.585983:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677448.585985:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.585988:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.585989:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.585991:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.585992:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.585993:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.585994:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.585997:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.585998:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.586001:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.586002:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.586002:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.586004:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.586004:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.586005:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.586005:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.586009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.586016:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.586018:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.586019:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.586021:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.586022:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.586024:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.586025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:12.0:1619677448.586031:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.586034:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.586035:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.586035:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.586036:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.586037:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.586194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.586200:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.586203:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.586204:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.586206:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.586207:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.586209:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.586210:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.586211:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.586212:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.586213:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.586217:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.586219:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.586221:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.586222:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.586222:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.586224:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.586225:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.586226:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.586227:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.586227:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.586230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.586233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.586238:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:5.0:1619677448.586240:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677448.586241:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.586241:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.586242:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677448.586243:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.586244:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.586245:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.586246:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.586248:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.586437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.586443:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.586446:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.586447:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.586449:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.586452:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.586453:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.586454:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.586455:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.586456:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.586457:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.586461:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.586462:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.586465:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.586466:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.586466:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.586468:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.586469:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.586469:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.586469:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.586475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.586482:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.586484:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677448.586485:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677448.586486:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.586488:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.586491:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677448.586492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:8.0:1619677448.586498:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.586500:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677448.586501:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.586501:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.586503:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.586503:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.2:1619677448.586505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.586508:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.586511:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.586513:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.586514:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.586516:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.586517:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.586519:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.586520:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.586521:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.586522:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.586526:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.586527:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.586531:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.586531:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.586532:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.586534:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.586534:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.586535:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.586536:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.586539:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.586540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.586540:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.586541:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:5.0:1619677448.586542:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.586543:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677448.586545:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.586547:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677448.586548:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:20.0:1619677448.586548:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.586549:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.586549:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.586550:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:20.0:1619677448.586550:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.586551:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.586553:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.586554:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:31.0:1619677448.586555:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.586556:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.0:1619677448.586557:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.586560:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.586561:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.586561:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.586563:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.586564:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.586564:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.586565:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.586569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.586576:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.586579:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.586580:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.586582:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677448.586583:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.586584:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.586591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.586597:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.586600:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.586600:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.586601:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.586602:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.586603:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.586613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.586620:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.586623:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.586624:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.586626:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.586628:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.586630:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.586631:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.586632:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.586633:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.586634:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.586637:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.586639:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.586642:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.586643:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.586643:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.586646:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.586646:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.586647:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.586647:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.586652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.586660:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.586663:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.586664:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.586664:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.586666:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.586670:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.586728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.586735:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.586738:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.586739:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.586740:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.586743:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.586744:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.586745:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.586746:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.586747:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.586748:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:5.2:1619677448.586748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:10.0:1619677448.586750:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.586752:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.586753:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.586753:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:10.0:1619677448.586754:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.586754:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.586756:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.586756:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.586756:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.586757:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:10.0:1619677448.586757:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.586757:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:2.0:1619677448.586758:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.586761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:2.0:1619677448.586761:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.586762:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.586763:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.586765:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.586765:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.586767:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:2.0:1619677448.586767:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.586769:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.586770:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.586770:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:2.0:1619677448.586770:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.586772:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.586772:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.586774:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:2.0:1619677448.586775:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.586776:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.586776:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.586778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677448.586778:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.586779:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.586779:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.586780:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.586784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.586786:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.586788:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.586791:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.586792:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.0:1619677448.586792:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:31.0:1619677448.586793:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.586794:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677448.586795:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.586795:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.586795:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.586797:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.586798:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.586798:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.586799:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.586800:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.586800:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:31.0:1619677448.586803:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.586805:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.586808:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.586809:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.586809:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.586811:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.586812:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.586813:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.586813:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.586817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.586825:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.586828:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.586829:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.586830:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.586832:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.586835:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.586836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.586843:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.586846:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.586847:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.586847:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.586848:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.586849:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.587109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe1000] (20)++
00000800:00000200:18.0:1619677448.587115:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe1000] (21)++
00000800:00000200:18.0:1619677448.587119:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.17@o2ib3
00000800:00000200:2.0:1619677448.587119:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe1000] (22)--
00000400:00000200:18.0:1619677448.587122:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.3.17@o2ib3 : GET - for me
00000400:00000200:18.0:1619677448.587124:0:19933:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.3.17@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:18.0:1619677448.587127:0:19933:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.3.17@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:18.0:1619677448.587129:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe1000] (21)++
00000800:00000200:18.0:1619677448.587130:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe1000] (22)++
00000800:00000200:18.0:1619677448.587132:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe1000] (23)--
00000400:00000200:18.0:1619677448.587133:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.3.17@o2ib3 local destination
00000400:00000200:18.0:1619677448.587137:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.3.17@o2ib3(10.51.3.17@o2ib3:10.51.3.17@o2ib3) : REPLY try# 0
00000800:00000200:5.2:1619677448.587138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe1000] (22)++
00000800:00000200:18.0:1619677448.587140:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.3.17@o2ib3
00000800:00000200:21.2:1619677448.587141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:18.0:1619677448.587142:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91907bfcf100] -> 10.51.3.17@o2ib3 (2) version: 12
00000800:00000200:18.0:1619677448.587143:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91907dbe1000] (23)++
00000800:00000200:18.0:1619677448.587143:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe1000] (24)++
00000800:00000200:18.0:1619677448.587144:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91907dbe1000] (26)--
00000800:00000200:2.0:1619677448.587144:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe1000] (24)++
00000800:00000200:18.0:1619677448.587145:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe1000] (25)--
00000800:00000200:2.0:1619677448.587145:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe1000] (26)--
00000800:00000200:18.0:1619677448.587146:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe1000] (22)--
00000800:00000200:2.0:1619677448.587146:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe1000] (23)--
00000800:00000200:22.0:1619677448.587147:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.587150:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:5.2:1619677448.587150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe1000] (21)++
00000800:00000200:6.0:1619677448.587151:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.587153:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.587155:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.587156:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe1000] (22)++
00000400:00000200:22.0:1619677448.587157:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:2.0:1619677448.587157:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe1000] (23)--
00000400:00000200:22.0:1619677448.587158:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.587158:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.587159:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.587159:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe1000] (22)--
00000400:00000200:22.0:1619677448.587161:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.587161:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.3.17@o2ib3: REPLY: OK
00000400:00000200:22.0:1619677448.587163:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.587163:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe1000] (21)--
00000400:00000200:22.0:1619677448.587166:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.587168:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.587170:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.587171:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.587172:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.587173:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.587174:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.587175:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.587175:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:4.2:1619677448.587177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:9.2:1619677448.587180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:24.0:1619677448.587185:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677448.587187:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:24.0:1619677448.587188:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.587189:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.587190:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.587190:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.587191:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.587191:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:24.0:1619677448.587193:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.587193:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.587195:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.587196:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.587196:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:24.0:1619677448.587197:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.587198:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.587199:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.587203:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.587205:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.587208:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.587208:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.587209:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.587211:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.587211:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.587212:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.587213:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.587216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.587225:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.2:1619677448.587226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677448.587227:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.587228:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.587229:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.587231:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.587233:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677448.587233:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677448.587237:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.587239:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.587240:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.587241:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.587243:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.587244:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.587246:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.587247:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.587248:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.587251:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.587252:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.587256:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.587256:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.587257:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.587259:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.587259:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.587260:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.587261:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.587265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.587273:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.587275:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.587276:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.587278:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.587279:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.587281:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.587327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.587333:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.587336:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.587336:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.587338:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.587341:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.587343:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.587344:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.587345:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.587346:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.587347:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.587351:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.587353:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.587355:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.587356:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.587356:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.587358:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.587359:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.587359:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.587360:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.587365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.587373:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.587376:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.587377:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.587377:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.587379:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.587381:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.587449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.587457:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.587460:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.587461:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.587463:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.587464:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.587466:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.587467:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.587468:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.587469:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.587470:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.587473:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.587475:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.587477:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.587477:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.587478:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.587479:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.587480:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.587481:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.587481:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.587485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.587491:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.587493:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.587493:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.587494:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.587495:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.587496:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.587505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.587511:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.587513:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.587514:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.587514:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.587515:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.587515:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.587628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:4.2:1619677448.587630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677448.587636:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:26.0:1619677448.587637:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677448.587638:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:26.0:1619677448.587639:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:24.0:1619677448.587639:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:10.0:1619677448.587640:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.587642:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.587642:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.587644:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.587645:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.587646:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677448.587646:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.587647:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.587648:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.587648:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.587649:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.587649:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.587650:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.587651:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677448.587651:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.587655:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.587656:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.587657:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677448.587658:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.587660:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.587661:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.587661:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677448.587661:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.587662:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.587663:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.587663:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677448.587663:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.587664:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.587664:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.0:1619677448.587664:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.587665:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.587666:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.587667:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.587668:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.2:1619677448.587669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.2:1619677448.587670:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.587674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:22.0:1619677448.587677:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.587679:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:22.0:1619677448.587680:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:24.0:1619677448.587681:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:22.0:1619677448.587681:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.587681:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.587682:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:22.0:1619677448.587683:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.587683:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.587684:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.587684:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677448.587685:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.587685:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.587686:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:22.0:1619677448.587687:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677448.587688:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.587777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.587782:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.587785:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.587785:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.587788:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.587790:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.587791:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.587793:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.587794:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.587796:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.587796:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.587800:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.587802:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.587804:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.587805:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.587806:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.587807:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.587808:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.587808:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.587809:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.587813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.587821:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.587824:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.587825:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.587825:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.587827:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.587830:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.587862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.587865:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.587867:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.587869:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.587870:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.587872:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.587874:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.587875:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.587877:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.587878:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.587879:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.587882:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.587884:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.587887:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.587888:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.587889:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.587891:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.587891:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.587892:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.587893:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.587897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.587904:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.587907:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.587908:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.587908:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.587910:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.587912:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.587915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.587922:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.587925:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.587925:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.587926:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.587927:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.587927:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.588003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.588011:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.588014:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.588017:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.588019:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.588020:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.588022:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.588023:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712978] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.588024:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.588025:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.588026:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.588029:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.588031:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.588034:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.588035:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.588036:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.588038:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.588038:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.588039:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.588039:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.588044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.588052:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:4.2:1619677448.588052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677448.588054:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.588055:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.588057:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.588058:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.588059:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677448.588060:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:10.0:1619677448.588062:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:8.2:1619677448.588062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.588063:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.588065:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.588068:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.588069:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677448.588070:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000400:00000200:10.0:1619677448.588070:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.588071:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.588072:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.588072:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.588073:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000400:00000200:10.0:1619677448.588073:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.588074:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.588074:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.588075:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:10.0:1619677448.588077:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.588078:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.588081:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.588082:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.588083:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.588084:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.588085:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.588086:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.588086:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.588091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.588099:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.588102:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.588102:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.588103:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.588105:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.588108:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.588223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.588228:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.588231:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.588232:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.588234:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.588236:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.588237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:6.0:1619677448.588237:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.588238:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.588239:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.588240:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.588241:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.588245:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677448.588245:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.588247:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.588248:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:6.0:1619677448.588250:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.588251:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.588251:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677448.588252:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677448.588252:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:30.0:1619677448.588254:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.588254:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.588254:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:30.0:1619677448.588255:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:6.0:1619677448.588255:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.588256:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000400:00000200:30.0:1619677448.588257:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.588257:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:30.0:1619677448.588258:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.588259:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.588260:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:25.2:1619677448.588260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:21.2:1619677448.588262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:30.0:1619677448.588263:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.588265:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.588266:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677448.588267:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:31.0:1619677448.588268:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.588268:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.588269:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.588269:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.588269:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.588269:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.588269:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:31.0:1619677448.588270:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.588270:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677448.588271:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.588272:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.588273:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.588273:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.588274:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677448.588275:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.588277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:25.2:1619677448.588281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.588283:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.588285:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.588286:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.588287:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.588287:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.588288:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:6.0:1619677448.588288:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.588290:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.588291:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.588292:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.588299:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.588301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:6.0:1619677448.588302:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677448.588308:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.588311:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.588312:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.588314:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.588316:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.588317:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.588318:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.588320:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.588321:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.588322:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.588325:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.588327:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.588328:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.588329:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.588330:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.588332:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.588332:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.588333:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.588334:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.588339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.588346:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.588348:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.588348:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.588349:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.588350:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.588351:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.588391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.588399:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.588402:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.588402:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.588404:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.588407:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.588409:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.588410:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.588411:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.588412:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.588413:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.588417:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.588419:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.588421:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.588422:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.588422:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.588424:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.588424:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.588425:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.588427:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.588427:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.588429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.588433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.588437:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.588439:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:10.0:1619677448.588439:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:30.0:1619677448.588440:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.588441:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:30.0:1619677448.588442:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.588442:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.588443:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.588444:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.588445:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.588588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.588593:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.588596:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.588598:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.588599:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.588600:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.588602:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677448.588603:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.588604:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.588605:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.588606:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677448.588609:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.588611:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677448.588613:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.588614:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677448.588615:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677448.588617:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677448.588617:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.588618:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.588618:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.588623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.588626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.588631:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677448.588631:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.588633:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:22.0:1619677448.588634:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:2.0:1619677448.588634:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.588635:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.588635:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.588638:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.588638:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.588640:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:22.0:1619677448.588641:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.588643:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.588644:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.588644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000400:00000200:22.0:1619677448.588645:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.588646:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.588647:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.588651:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.588651:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.588652:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677448.588654:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.588655:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.588655:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677448.588656:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.588656:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:2.0:1619677448.588656:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.588657:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:2.0:1619677448.588657:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:22.0:1619677448.588659:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.588660:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.588660:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.588661:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.588665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.588672:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:21.2:1619677448.588674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.588676:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.588677:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.588678:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.588680:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.588681:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.588682:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677448.588682:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.588683:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.588685:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.588687:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.588689:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.588690:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.588691:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.588693:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.588693:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.588697:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.588698:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.588701:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.588702:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.588703:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.588704:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.588705:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.588707:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.588708:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.588709:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.588710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.588714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677448.588718:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.588721:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.588721:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:30.0:1619677448.588722:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:6.0:1619677448.588722:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.588724:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:6.0:1619677448.588724:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.588725:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.588726:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.588727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.588730:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.588732:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.588734:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.588735:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.588735:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.588736:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.588736:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.588838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.588845:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.588848:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.588849:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.588850:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.588853:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.588854:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.588855:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.588857:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.588858:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.588859:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.588862:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.588864:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.588866:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.588867:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.588867:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.588869:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.588869:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.588870:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.588870:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.588874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.588881:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.588883:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.588884:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.588885:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.588887:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.588888:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.588899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.588904:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.588907:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.588909:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.588910:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.588912:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.588914:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.588915:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.588916:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.588917:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.588918:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.588922:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.588924:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.588926:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.588927:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.588928:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.588929:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.588930:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.588931:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.588932:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677448.588936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.588945:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.588948:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.588949:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.588949:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.588951:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.588953:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.588976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.588979:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.588982:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.588984:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.588984:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.588986:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.588987:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.588989:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.588990:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.588991:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.588992:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.588996:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.588997:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.589000:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.589001:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.589002:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.589004:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.589004:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.589005:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.589006:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.589007:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.589010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.589012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677448.589018:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.589018:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.589020:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:13.0:1619677448.589021:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.589021:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:13.0:1619677448.589022:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.589022:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:13.0:1619677448.589024:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.589025:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.589026:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.589047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.589053:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.589056:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.589057:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.589058:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.589061:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.589062:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.589063:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.589065:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.589066:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.589067:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.589070:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.589072:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.589073:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.589074:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.589075:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.589076:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.589077:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.589077:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.589078:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.589082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677448.589085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:17.0:1619677448.589089:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.589091:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.589091:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.589092:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:1.0:1619677448.589092:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.589093:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.589094:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677448.589095:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.589096:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.589097:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.589100:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.589102:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.589103:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.589104:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.589105:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.589106:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.589109:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.589111:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.589113:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.589114:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.589115:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.589116:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.589117:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.589118:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.589118:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677448.589122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.589128:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.589130:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.589131:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.589133:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677448.589134:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.589135:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.589204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.589212:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.589215:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.589217:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.589219:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.589220:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.589221:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677448.589222:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.589224:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.589225:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.589226:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.589229:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.589231:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:4.2:1619677448.589232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.589233:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.589234:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.589235:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.589236:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.589237:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.589238:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.589238:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.589239:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.589242:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:8.2:1619677448.589242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677448.589243:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.589245:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.589247:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.589248:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.589249:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.589250:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:26.0:1619677448.589251:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.589251:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.589252:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.589252:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.589253:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.589254:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.589256:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.589257:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.589259:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.589259:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.589260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:26.0:1619677448.589262:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.589263:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.589263:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.589265:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.589265:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.589266:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.589266:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677448.589267:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.589270:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:25.2:1619677448.589271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.589271:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.589271:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.589272:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.589272:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677448.589278:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.589281:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.589282:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.589283:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.589285:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.589288:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.589373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.589378:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.589380:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.589381:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.589383:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.589386:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.589387:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.589388:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.589389:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.589391:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.589391:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.589395:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.589397:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.589399:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677448.589400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:6.0:1619677448.589400:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.589401:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.589403:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.589403:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.589404:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.589404:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677448.589406:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:25.2:1619677448.589409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:5.0:1619677448.589409:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.589411:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.589411:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.589414:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.589415:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:10.0:1619677448.589416:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:5.0:1619677448.589417:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.589418:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.589419:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:5.0:1619677448.589419:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.589420:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.589420:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.589420:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.589422:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.589423:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:5.0:1619677448.589427:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.589429:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.589429:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.589430:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:17.0:1619677448.589432:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.589432:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.589432:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:5.0:1619677448.589433:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.589434:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.589434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:1.0:1619677448.589434:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.589435:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.589436:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677448.589436:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.589437:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.589437:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677448.589437:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.589439:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.589440:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.589441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:1.0:1619677448.589441:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.589442:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:1.0:1619677448.589442:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:24.0:1619677448.589445:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.589445:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677448.589446:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.589448:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:1.0:1619677448.589448:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:24.0:1619677448.589449:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.589450:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677448.589451:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.589451:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:1.0:1619677448.589451:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:24.0:1619677448.589452:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677448.589452:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.589452:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677448.589452:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.589453:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:24.0:1619677448.589454:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96712959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.589454:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.589454:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:24.0:1619677448.589455:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.589455:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:24.0:1619677448.589456:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.589456:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677448.589456:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000400:00000200:24.0:1619677448.589457:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.589457:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.589460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:24.0:1619677448.589461:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.589462:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.2:1619677448.589462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.589465:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.589466:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.589467:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.589467:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677448.589467:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:24.0:1619677448.589468:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.589469:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.589470:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.589470:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677448.589470:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677448.589470:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677448.589471:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677448.589471:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.589472:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.589472:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.589473:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.589474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677448.589476:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:17.0:1619677448.589482:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.589485:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.589485:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.589486:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.589488:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.589491:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.589523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.589530:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.589533:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.589534:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.589536:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.589539:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.589540:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.589541:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.589543:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.589544:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.589545:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.589548:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.589550:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.589552:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.589553:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.589553:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.589555:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.589555:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.589556:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.589556:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.589561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.589568:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.589570:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.589571:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.589572:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.589575:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.589575:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.589614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.589621:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.589624:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.589626:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.589628:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.589629:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.589631:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.589632:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.589633:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.589634:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.589635:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.589638:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.589639:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.589643:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.589643:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.589644:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.589646:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.589646:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.589647:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.589649:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.589650:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.2:1619677448.589651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.2:1619677448.589652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.589656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:1.0:1619677448.589657:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.589660:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.589660:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.589662:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.589663:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677448.589664:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:1.0:1619677448.589665:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.589666:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:20.0:1619677448.589666:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:1.0:1619677448.589666:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677448.589667:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.589667:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.589668:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.589668:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:1.0:1619677448.589668:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.589669:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.589670:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:1.0:1619677448.589670:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.589670:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.589672:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:1.0:1619677448.589674:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.589676:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.589678:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.589679:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.589679:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.589681:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.589681:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.589682:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.589683:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.589687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.589694:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.589696:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.589697:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.589697:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.589699:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.589700:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.589712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.589719:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.589722:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.589724:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.589726:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.589727:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.589729:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677448.589730:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.589732:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.589732:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.589733:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677448.589737:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.589738:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677448.589742:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.589743:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:31.0:1619677448.589743:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.589745:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677448.589746:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.589747:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.589747:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.589751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.589759:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.589762:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.589763:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.589763:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.589766:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.589768:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.589791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.589798:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.589801:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.589801:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.589803:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.589806:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.589807:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.589808:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.589809:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.589810:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.589811:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.589814:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.589816:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.589817:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.589818:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.589819:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.589820:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.589820:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.589821:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.589821:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.589825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.589831:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.589833:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.589833:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.589834:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.589835:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.589837:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.589854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.589859:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.589862:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.589864:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.589865:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.589866:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.589868:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.589869:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.589870:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.589871:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.589872:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.589876:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.589877:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.589880:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.589881:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.589882:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.589884:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.589884:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.589885:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.589885:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.589889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.589896:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.589899:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.589900:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.589902:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.589902:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.589905:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.589945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.589950:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.589953:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.589954:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.589956:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.589957:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.589959:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.589960:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.589961:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.589963:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.589963:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.589967:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.589969:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.589971:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.589972:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.589973:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.589974:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.589975:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.589975:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.589976:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.589980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.589988:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.589990:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.589991:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.589991:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.589994:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.589996:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.589998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:22.0:1619677448.590005:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.590008:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.590008:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.590009:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.590010:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.590010:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.590069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.590076:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.590078:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.590079:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.590081:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.590083:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.590085:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.590086:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.590087:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.590088:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.590089:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.590093:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.590095:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.590097:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.590098:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.590098:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.590100:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.590100:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.590101:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.590102:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.590103:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677448.590106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.590108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:6.0:1619677448.590113:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677448.590113:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.590115:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:6.0:1619677448.590116:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677448.590116:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677448.590117:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.590117:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.590117:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:6.0:1619677448.590119:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677448.590122:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.590128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.590136:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.590139:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.590141:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.590142:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.590144:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.590146:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.590147:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.590149:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.590149:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.590151:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.590154:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.590156:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.590158:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.590158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677448.590159:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.590160:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.590161:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.590162:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.590163:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.590164:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:21.2:1619677448.590165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677448.590165:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:25.2:1619677448.590167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.590168:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.590169:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.590170:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.590172:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677448.590172:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.590173:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677448.590174:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:12.0:1619677448.590174:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:10.0:1619677448.590175:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712949] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.590176:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:10.0:1619677448.590176:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.590177:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677448.590177:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.590177:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.590178:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677448.590178:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.590179:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.590179:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.590180:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:31.0:1619677448.590181:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677448.590182:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96712948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.590182:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.590184:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.590184:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.590185:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.590186:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:10.0:1619677448.590186:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.590187:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.590187:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.590188:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:31.0:1619677448.590189:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.590189:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.590190:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.590191:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:10.0:1619677448.590191:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.590192:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.590192:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.590193:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.590194:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.590194:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.590196:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.590196:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.590197:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.590198:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677448.590198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:9.2:1619677448.590202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:25.2:1619677448.590203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.590205:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.590207:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.590208:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.590209:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677448.590209:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:13.0:1619677448.590211:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677448.590211:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.590212:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:13.0:1619677448.590213:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:8.0:1619677448.590213:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.590213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:13.0:1619677448.590214:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.590215:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:13.0:1619677448.590216:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.590217:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.590218:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.590218:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677448.590218:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:17.0:1619677448.590221:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.590222:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.590224:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.590226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:17.0:1619677448.590226:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.590228:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.590229:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.590230:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.590231:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.590232:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:12.0:1619677448.590232:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.590235:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.590236:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.590236:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.590236:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.590237:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.590238:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677448.590238:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677448.590240:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.590241:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.590241:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.590243:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.590244:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.590244:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.590245:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.590249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.590257:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.590259:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.590260:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.590261:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.590262:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.590264:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.590283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.590286:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.590289:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.590291:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.590292:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.590297:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.590299:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.590300:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.590302:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.590303:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.590304:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.590307:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.590309:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.590312:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.590313:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.590313:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.590315:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.590316:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.590317:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.590317:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.2:1619677448.590327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.590335:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.590337:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.590338:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.590340:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.590343:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.590344:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.590345:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96712945] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.590347:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.590348:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.590349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677448.590349:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.590352:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.590353:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.590356:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.590356:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677448.590357:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.590358:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677448.590359:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:24.0:1619677448.590360:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.590360:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:13.0:1619677448.590360:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.590361:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.590361:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:13.0:1619677448.590361:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.590363:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.590364:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:9.2:1619677448.590366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677448.590370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:2.0:1619677448.590374:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:17.0:1619677448.590376:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:2.0:1619677448.590377:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:4.2:1619677448.590378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677448.590378:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.590379:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677448.590379:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677448.590379:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:2.0:1619677448.590381:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.590382:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.590384:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.590385:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:17.0:1619677448.590386:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:10.0:1619677448.590386:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677448.590387:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.590388:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.590389:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677448.590389:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.590389:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.590391:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.590392:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.590394:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.590394:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.590396:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.590396:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.590397:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.590398:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.590399:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.590399:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.590400:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:10.0:1619677448.590400:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:17.0:1619677448.590401:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677448.590402:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677448.590403:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:10.0:1619677448.590403:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.590404:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677448.590404:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:10.0:1619677448.590405:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.590407:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.590408:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:8.2:1619677448.590408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.590409:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.590410:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.590410:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.590411:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.590413:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.590413:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.590415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.590417:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.590419:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:4.2:1619677448.590419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677448.590420:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.590422:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.590424:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.590425:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.590426:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:5.0:1619677448.590426:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:26.0:1619677448.590427:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.590429:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.590429:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.590429:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.590430:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.590431:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.590431:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.590432:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677448.590491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.590497:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.590500:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.590500:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.590503:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.590505:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.590506:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.590507:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.590509:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.590510:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.590511:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.590514:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.590516:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.590518:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.590519:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.590520:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.590521:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.590522:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.590522:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.590523:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.590527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677448.590533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677448.590535:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.590538:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.590539:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.590539:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.590541:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.2:1619677448.590541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:1.0:1619677448.590541:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.590543:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:1.0:1619677448.590544:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:30.0:1619677448.590545:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.590546:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.590547:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.590548:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.590548:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.590548:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:30.0:1619677448.590550:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.590551:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.590551:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.590552:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.590553:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.590553:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.590554:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677448.590555:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.590556:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.590558:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.590559:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.590560:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.590560:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.590563:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:4.2:1619677448.590564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:1.0:1619677448.590564:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.590567:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.590567:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.590568:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.590568:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.590570:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:1.0:1619677448.590570:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.590571:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:8.0:1619677448.590571:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.590571:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.590571:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.590572:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.590572:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:1.0:1619677448.590573:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677448.590574:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.590574:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:1.0:1619677448.590574:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.590575:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.2:1619677448.590576:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:1.0:1619677448.590576:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.590576:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.590581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677448.590582:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.590584:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:8.0:1619677448.590584:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.590586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.590586:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:8.0:1619677448.590586:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:20.0:1619677448.590587:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:8.0:1619677448.590587:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96712939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.590588:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.590588:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:8.0:1619677448.590588:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.590589:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.590589:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.590590:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.590590:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.590590:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:8.0:1619677448.590591:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:20.0:1619677448.590592:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677448.590593:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.2:1619677448.590593:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:12.0:1619677448.590595:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.590596:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.590598:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.590599:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:8.0:1619677448.590599:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.590600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:12.0:1619677448.590602:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677448.590602:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.590605:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.590605:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.590606:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.590606:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.590607:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677448.590608:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.590608:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.0:1619677448.590609:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.590610:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:8.0:1619677448.590610:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:5.0:1619677448.590610:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.590610:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:26.0:1619677448.590611:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677448.590611:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.590612:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677448.590612:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.590613:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:10.0:1619677448.590613:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.590614:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:10.0:1619677448.590615:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.590616:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.590617:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677448.590617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677448.590619:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.590620:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.590621:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.590622:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677448.590624:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:10.0:1619677448.590624:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.590625:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:24.0:1619677448.590626:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677448.590626:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.590626:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.590626:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.590627:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.590628:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677448.590629:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.590629:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.590630:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.590630:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.590631:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:10.0:1619677448.590632:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.590632:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.590633:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.590634:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.590645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.590651:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.590653:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.590654:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.590656:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.590657:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.590658:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.590667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.590674:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.590677:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.590679:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.590681:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.590682:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.590683:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.590684:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.590686:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.590687:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.590688:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.590691:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.590692:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.590695:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.590695:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.590696:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.590697:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.590698:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.590699:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.590701:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.590701:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.590703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.590707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.590709:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.590711:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.590712:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.590713:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.590714:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.590715:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.590716:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.590717:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.590717:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.590718:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.590721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.590727:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.590730:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.590731:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.590731:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.590732:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.590733:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.590927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.590934:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.590936:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.590937:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.590939:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.590941:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.590942:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.590944:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.590945:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.590946:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.590947:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.590950:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.590951:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.590954:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.590955:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.590956:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.590958:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.590958:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.590959:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.590959:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.590964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.590971:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.590974:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.590975:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.590975:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.590978:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.590981:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.590984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.590990:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.590993:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.590994:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.590996:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.590998:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.591000:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.591001:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.591002:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.591003:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.591004:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.591008:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.591010:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.591012:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.591013:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.591013:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.591015:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.591016:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.591016:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.591017:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.591021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.591028:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.591031:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.591032:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.591033:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.591035:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.591037:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.591039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.2:1619677448.591041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:31.0:1619677448.591046:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:18.0:1619677448.591046:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677448.591050:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:18.0:1619677448.591050:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.591050:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677448.591051:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.591051:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.591052:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:18.0:1619677448.591052:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.591053:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:18.0:1619677448.591054:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.591056:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.591057:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.591059:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.591060:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.591061:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.2:1619677448.591062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:18.0:1619677448.591064:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.591065:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.591067:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677448.591068:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.591069:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.591069:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.591070:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:18.0:1619677448.591071:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.591071:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.591072:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:5.0:1619677448.591072:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.591073:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677448.591073:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:5.0:1619677448.591075:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.591076:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:8.2:1619677448.591077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:5.0:1619677448.591077:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.591079:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.591080:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.591081:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677448.591084:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.591085:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.591086:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.591086:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:20.0:1619677448.591087:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.591089:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.591089:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.591090:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.591091:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.591091:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.591092:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677448.591092:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.591093:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.591094:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.591096:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.591096:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.591098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.2:1619677448.591099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.591105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.591106:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.591109:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.591111:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.591111:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.591112:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.591112:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:31.0:1619677448.591113:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.591113:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.591113:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:5.0:1619677448.591113:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.591114:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677448.591115:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:20.0:1619677448.591115:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.591115:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.591116:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.591116:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677448.591117:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:30.0:1619677448.591118:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.591119:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.591120:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.591123:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.591125:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.591127:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.591128:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.591129:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.591130:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.591131:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.591132:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.591133:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.591136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.591143:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.591145:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.591146:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.591147:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.591149:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.591150:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.591183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.591191:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.591193:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.591194:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.591196:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.591199:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.591200:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.591201:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.591202:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.591203:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.591204:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.591207:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.591209:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.591211:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.591212:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.591212:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.591214:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.591214:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.591214:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.591215:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.591219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.591225:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.591228:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.591228:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.591230:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.591231:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.591233:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.591413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.591415:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.591418:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.591420:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.591421:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.591423:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.591425:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:20.0:1619677448.591426:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.591427:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.591427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:20.0:1619677448.591428:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.591430:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:20.0:1619677448.591433:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.591433:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677448.591434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:20.0:1619677448.591435:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.591436:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.591436:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.0:1619677448.591438:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.591439:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000400:00000200:1.0:1619677448.591439:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.591440:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:2.0:1619677448.591440:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677448.591441:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:20.0:1619677448.591442:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:1.0:1619677448.591442:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.591443:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:2.0:1619677448.591443:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:1.0:1619677448.591443:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:20.0:1619677448.591444:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:18.0:1619677448.591444:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677448.591445:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:1.0:1619677448.591445:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.591447:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.591447:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.591448:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.591449:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.591449:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:20.2:1619677448.591450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000400:00000200:2.0:1619677448.591450:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.591452:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96712928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.591454:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.591454:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.591455:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000400:00000200:2.0:1619677448.591455:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.591456:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.591457:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:1.0:1619677448.591457:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:9.2:1619677448.591458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.0:1619677448.591458:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.591459:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:2.0:1619677448.591460:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.591460:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.591461:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.591461:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:2.0:1619677448.591462:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.591463:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.591463:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.591464:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677448.591465:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:2.0:1619677448.591465:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.591465:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:1.0:1619677448.591465:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677448.591466:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.591467:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:8.0:1619677448.591468:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:2.0:1619677448.591468:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:24.0:1619677448.591469:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:9.2:1619677448.591469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:8.0:1619677448.591469:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.591469:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.591470:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000400:00000200:8.0:1619677448.591471:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.591471:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.591474:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.591476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:24.0:1619677448.591477:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.591478:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.591479:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.591481:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.591481:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:18.0:1619677448.591482:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:8.0:1619677448.591482:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.591483:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:18.0:1619677448.591484:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:5.0:1619677448.591484:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677448.591485:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.591486:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677448.591486:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.591487:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.591489:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:9.2:1619677448.591489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000400:00000200:5.0:1619677448.591489:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.591491:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.0:1619677448.591495:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.591498:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.591499:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.591499:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.591501:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.591501:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.591541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.591547:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.591550:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.591551:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.591553:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.591555:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.591557:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.591558:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.591559:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.591560:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.591561:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.591565:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.591567:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.591570:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.591570:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.591571:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.591573:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.591574:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.591574:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.591575:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.591579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.591587:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.591590:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.591591:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.591591:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.591593:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.591596:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.591599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.591606:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.591608:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.591609:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.591610:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.591611:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.591611:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.591645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.591653:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.591656:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.591657:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.591658:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.591661:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.591662:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.591663:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.591665:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.591666:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.591667:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.591670:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.591672:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.591674:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.591675:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.591675:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.591677:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.591677:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.591678:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.591678:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.591682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.591689:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.591692:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.591692:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.591694:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.591696:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.591697:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.591700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.591706:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.591709:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.591710:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.591710:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.591711:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.591712:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.591763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.591770:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.591772:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.591773:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.591775:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.591778:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.591780:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.591781:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.591782:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.591783:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.591784:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.591787:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.591789:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.591791:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.591792:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.591793:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.591794:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.591795:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.591795:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.591796:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.591800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.591808:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.591811:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.591812:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.591812:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.591815:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.591818:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.591834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.591840:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.591843:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.591843:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.591846:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.591848:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.591849:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.591851:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.591852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000400:00000200:2.0:1619677448.591852:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.591853:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.591854:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.591857:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.591859:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.591860:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677448.591863:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:2.0:1619677448.591863:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.591864:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.591864:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:12.0:1619677448.591865:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.591866:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.591867:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677448.591867:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.591867:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.591867:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.591868:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:12.0:1619677448.591869:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:12.0:1619677448.591870:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712923] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.591872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:12.0:1619677448.591873:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.591874:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677448.591875:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:12.0:1619677448.591878:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.591879:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.591880:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.591880:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:13.0:1619677448.591881:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.591882:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677448.591883:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.591883:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:12.0:1619677448.591884:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:13.0:1619677448.591886:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.591886:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:12.0:1619677448.591886:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677448.591887:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.591887:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677448.591888:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:25.2:1619677448.591891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.591898:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.591900:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.591901:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.591903:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.591904:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.591906:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.592019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.592025:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.592027:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.592028:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.592030:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.592033:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.592034:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.592036:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.592037:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.592038:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.592039:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.592042:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.592044:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.592046:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.592047:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.592047:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.592049:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.592049:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.592050:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.592051:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.592052:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677448.592055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.592057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.592063:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.592064:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.592064:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.592065:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.592065:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.592065:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.592066:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.592066:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.592067:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677448.592068:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.592081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.592089:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.592092:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.592094:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.592095:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.592096:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.592098:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.592099:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.592100:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.592101:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.592102:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.592106:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.592108:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.592110:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.592111:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.592111:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.592113:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.592113:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.592115:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.592116:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.592117:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.592118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.2:1619677448.592123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.592127:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.592128:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.592129:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.592131:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:30.0:1619677448.592131:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.592132:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.592132:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.592133:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.592133:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.592134:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:21.2:1619677448.592300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.592308:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.592310:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.592313:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.592315:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.592315:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.592316:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.592318:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.592319:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.592320:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.592320:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.592324:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.592324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677448.592325:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.592328:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.592328:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.592329:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.592330:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.592330:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.592331:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.592331:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.592332:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677448.592333:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.592334:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.592336:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.592337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677448.592339:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.592340:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.592341:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.592342:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.592343:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.592344:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677448.592344:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:13.0:1619677448.592347:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:1.0:1619677448.592347:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677448.592348:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.592350:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.592350:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:12.0:1619677448.592351:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.592353:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.592353:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.592353:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.592354:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.592356:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.592356:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.592357:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.592357:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.592362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.592369:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.592371:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677448.592372:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677448.592373:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.592375:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.592377:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.2:1619677448.592381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.592388:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.592391:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.592392:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.592394:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.592396:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.592397:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.592399:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.592400:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.592401:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.592402:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.592405:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.592407:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.592409:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.592410:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.592411:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.592412:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.592412:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.592414:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.592415:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.592415:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.592417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.2:1619677448.592421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.592424:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.592426:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.592427:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.592428:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.592428:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:31.0:1619677448.592430:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.592430:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:30.0:1619677448.592431:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:26.0:1619677448.592431:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.592432:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677448.592576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.592583:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:4.2:1619677448.592585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677448.592586:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.592587:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.592589:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.592590:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.592592:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:10.0:1619677448.592592:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:18.0:1619677448.592593:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.592594:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.592596:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677448.592596:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.592596:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:18.0:1619677448.592597:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.592599:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.592601:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.592601:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.592603:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.592603:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.592604:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712916] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.592605:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.592606:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.592606:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677448.592606:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.592607:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.592608:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.592608:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:10.0:1619677448.592608:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:18.0:1619677448.592609:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.592610:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:10.0:1619677448.592612:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.592613:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:9.2:1619677448.592614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.592617:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.592617:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.592618:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.592620:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.592620:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.592621:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.592621:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677448.592621:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.592623:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.592624:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.592625:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:21.2:1619677448.592626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:8.2:1619677448.592626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:2.0:1619677448.592626:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.592628:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677448.592632:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677448.592634:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:22.0:1619677448.592635:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.592635:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677448.592637:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:22.0:1619677448.592638:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.592638:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.592640:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.592640:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.592641:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:22.0:1619677448.592642:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.592643:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.592643:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:22.0:1619677448.592644:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.592645:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.592646:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.592650:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.592652:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.592655:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.592655:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.592656:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.592658:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.592659:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.592659:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.592660:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.592664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.592671:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.592674:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.592674:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.592676:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.592677:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.592679:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.592828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.592836:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.592839:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.592841:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.592843:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.592843:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.592844:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.592846:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.592847:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.592848:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.592849:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.592852:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.592854:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.592856:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.592857:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.592857:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.592859:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.592859:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.592860:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.592861:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.592864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.592871:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.592873:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.592874:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.592875:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.592877:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.592878:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.593029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.593034:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.593037:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.593039:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677448.593040:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.593042:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.593043:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677448.593045:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.593046:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.593047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:5.0:1619677448.593047:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.593048:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:20.0:1619677448.593049:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.593051:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.593052:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000400:00000200:20.0:1619677448.593053:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.593054:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:20.0:1619677448.593056:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.593058:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:5.0:1619677448.593058:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.593059:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.593059:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677448.593060:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:20.0:1619677448.593061:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.593062:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.593062:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:20.0:1619677448.593063:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.593063:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.593064:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.593065:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:20.0:1619677448.593066:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.593066:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.593067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.593068:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677448.593071:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.593071:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677448.593072:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.593074:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677448.593074:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.593075:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:20.0:1619677448.593075:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.593076:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677448.593077:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.593078:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.593080:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:8.2:1619677448.593080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677448.593081:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.593083:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:17.0:1619677448.593088:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:25.2:1619677448.593090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:17.0:1619677448.593090:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.593091:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.593091:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.593094:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.593097:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:17.0:1619677448.593097:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.593099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:30.0:1619677448.593100:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.593101:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.593102:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.593102:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.593104:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:1.0:1619677448.593106:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.593109:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.593110:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.593110:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.593112:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.593112:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.593128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677448.593131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:26.0:1619677448.593135:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.2:1619677448.593135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:6.0:1619677448.593136:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.593138:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.593138:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677448.593139:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.593140:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677448.593140:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:26.0:1619677448.593142:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.593143:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:6.0:1619677448.593144:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.593144:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:26.0:1619677448.593145:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.593147:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677448.593147:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:6.0:1619677448.593147:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.593149:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.593149:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.593150:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.593150:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.593151:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.593153:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.593153:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.593153:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.593154:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.593155:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.593155:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.593156:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.593157:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:6.0:1619677448.593157:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.593158:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.593163:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.593164:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000400:00000200:26.0:1619677448.593165:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.593166:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.593167:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:26.0:1619677448.593168:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.593168:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.593169:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.593169:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.593170:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.593170:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.593171:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.593171:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.593172:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677448.593172:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.593173:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:6.0:1619677448.593173:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.593174:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677448.593174:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.593174:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677448.593174:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.593175:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.593175:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:26.0:1619677448.593176:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.593176:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.593176:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677448.593177:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.593177:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677448.593178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:8.2:1619677448.593179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:9.2:1619677448.593181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.593182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:25.2:1619677448.593185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677448.593186:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677448.593188:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677448.593188:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:5.0:1619677448.593190:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677448.593191:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677448.593191:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:5.0:1619677448.593191:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.593191:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.593192:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:18.0:1619677448.593192:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:22.0:1619677448.593193:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:18.0:1619677448.593193:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.593193:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.593193:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.593194:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.593195:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.593195:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:18.0:1619677448.593196:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.593196:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:2.0:1619677448.593196:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.593197:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96712908] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.593198:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.593198:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.593199:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.593199:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.593200:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.593201:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.593202:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.593204:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.593205:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677448.593206:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.593207:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:2.0:1619677448.593207:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.593209:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.593213:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.593213:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.593214:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.593216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:2.0:1619677448.593216:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.593217:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.593218:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.593218:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.593222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.593224:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.593226:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.593227:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:12.0:1619677448.593229:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677448.593230:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.593232:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.593232:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.593232:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.593234:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.593234:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.593235:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.593236:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.593236:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.593237:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677448.593238:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.593238:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:25.2:1619677448.593242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:21.2:1619677448.593242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:10.0:1619677448.593242:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.593244:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.593247:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.593248:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.593248:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.593249:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:10.0:1619677448.593249:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.593250:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:13.0:1619677448.593251:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.593251:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.593251:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:31.0:1619677448.593252:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:13.0:1619677448.593252:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.593252:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:10.0:1619677448.593252:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677448.593253:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:31.0:1619677448.593254:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.593254:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677448.593256:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.593256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.2:1619677448.593256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.593257:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677448.593258:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:30.0:1619677448.593258:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:31.0:1619677448.593259:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.593260:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.593260:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.593261:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.593262:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677448.593264:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:30.0:1619677448.593265:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.593266:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.593266:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.593267:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.593268:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.593269:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:20.0:1619677448.593270:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.593270:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:30.0:1619677448.593271:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.593272:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96712905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.593272:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.593273:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000400:00000200:30.0:1619677448.593274:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.593274:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (23)++
00000400:00000200:30.0:1619677448.593275:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.593275:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.593275:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000400:00000200:30.0:1619677448.593276:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.593276:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677448.593280:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.593280:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:30.0:1619677448.593280:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.593281:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.593282:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.593283:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677448.593283:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677448.593284:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677448.593284:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.593285:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:30.0:1619677448.593285:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.593286:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.593287:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.593287:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.593288:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.593289:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.593289:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.593289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:25.2:1619677448.593293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:6.0:1619677448.593298:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:26.0:1619677448.593300:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:6.0:1619677448.593301:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:26.0:1619677448.593302:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.593303:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.593303:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.593303:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:6.0:1619677448.593303:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.593306:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.593306:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.593309:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:26.0:1619677448.593310:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.593584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677448.593588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1200] (20)++
00000800:00000200:10.0:1619677448.593591:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.593593:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.593594:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.593594:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000400:00000200:10.0:1619677448.593596:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.593597:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000400:00000200:10.0:1619677448.593598:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.593598:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:10.0:1619677448.593599:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677448.593600:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.593601:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.593602:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.593603:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.593603:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.593604:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677448.593604:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677448.593606:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.593608:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.593609:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.593609:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.593610:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:10.0:1619677448.593613:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:17.0:1619677448.593614:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.593616:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.593617:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677448.593617:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.593617:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.593619:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677448.593620:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.593620:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.593621:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:17.0:1619677448.593621:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:10.0:1619677448.593621:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.593622:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:17.0:1619677448.593623:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:17.0:1619677448.593623:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:10.0:1619677448.593623:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.593624:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:17.0:1619677448.593625:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:8.2:1619677448.593626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.593629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677448.593630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:21.2:1619677448.593632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b4c00] (20)++
00000800:00000200:20.0:1619677448.593635:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677448.593636:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:20.0:1619677448.593636:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.593637:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.593638:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.593639:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677448.593639:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.593639:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.593639:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:1.0:1619677448.593639:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b4c00] (21)++
00000800:00000200:5.0:1619677448.593640:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.593641:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.593642:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.593643:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:5.0:1619677448.593643:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.593643:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.68@o2ib3
00000800:00000200:17.0:1619677448.593644:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4c00] (22)--
00000800:00000200:20.0:1619677448.593645:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.593646:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:1.0:1619677448.593646:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.68@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.593649:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.593650:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:1.0:1619677448.593651:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.593653:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.593654:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.593654:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:1.0:1619677448.593658:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.68@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.593659:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:1.0:1619677448.593662:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.593662:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:1.0:1619677448.593663:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:1.0:1619677448.593665:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:1.0:1619677448.593665:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b4c00] (21)++
00000800:00000200:1.0:1619677448.593666:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b4c00] (22)--
00000800:00000200:1.0:1619677448.593667:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4c00] (21)--
00000800:00000200:8.2:1619677448.593670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.593678:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.593680:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.593681:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.593682:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.593683:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.593685:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:8.2:1619677448.593692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000800:00000200:5.0:1619677448.593699:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.593701:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.593702:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.593703:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.593704:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.0:1619677448.593704:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:21.2:1619677448.593718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.593724:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.593726:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.593727:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.593729:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.593731:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.593733:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.593734:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.593735:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.593736:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.593737:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.593741:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.593743:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.593746:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.593747:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.593747:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.593749:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.2:1619677448.593749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:22.0:1619677448.593750:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.593750:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.593751:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677448.593755:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:25.2:1619677448.593756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.593757:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.593759:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.593760:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.2:1619677448.593762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:5.0:1619677448.593762:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.593763:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:5.0:1619677448.593765:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:31.0:1619677448.593766:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.593767:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.593767:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.593768:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.593769:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:31.0:1619677448.593770:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.593770:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.593771:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:5.0:1619677448.593771:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.593772:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.593772:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:31.0:1619677448.593773:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:8.0:1619677448.593775:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.593775:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.593778:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.593778:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.593779:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.593781:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96712899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.593781:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.593782:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.593782:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:8.0:1619677448.593783:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.593784:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.593784:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:8.0:1619677448.593785:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.593785:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.593786:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.593786:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:8.0:1619677448.593788:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.593790:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.593794:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.593797:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.593798:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:8.0:1619677448.593798:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.593799:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677448.593800:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:8.0:1619677448.593800:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:17.0:1619677448.593801:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.593801:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.593801:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:1.0:1619677448.593801:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.593802:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:17.0:1619677448.593804:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.593806:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.593806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.593814:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.593816:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.593817:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.593819:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.593820:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.593821:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.593838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.593844:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.593847:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.593847:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.593849:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.593851:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.593853:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.593854:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.593855:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.593856:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.593858:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.593861:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.593862:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.593864:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.593865:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.593866:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.593867:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.593867:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.593869:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.593870:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.593870:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.593873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.593876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.593882:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.593883:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.593883:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.593883:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.593883:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.593884:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.593885:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.593885:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.593886:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677448.593887:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.593890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.593897:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.593900:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.593901:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.593901:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.593901:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.593902:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.593926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.593933:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.593936:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.593937:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.593939:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.593942:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.593943:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.593945:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.593946:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.593947:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.593948:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.593951:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.593953:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.593956:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.593957:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.593958:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.593960:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.593960:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.593961:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.593961:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.593966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.593973:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.593976:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.593977:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.593977:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.593979:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.593982:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.593989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:25.2:1619677448.593996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677448.593996:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.593998:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.593999:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.594001:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.594001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677448.594003:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:24.0:1619677448.594004:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.594006:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:22.0:1619677448.594006:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.594006:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:24.0:1619677448.594007:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.594007:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.594008:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:24.0:1619677448.594008:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.594009:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.594009:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.594009:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:24.0:1619677448.594010:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.594011:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.594013:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.594015:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.594015:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.594015:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.594017:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:24.0:1619677448.594017:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:30.0:1619677448.594019:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712895] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.594019:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.594020:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.594020:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:30.0:1619677448.594021:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.594021:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677448.594022:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.594023:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:30.0:1619677448.594024:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.594026:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.594026:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.594026:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677448.594028:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.594028:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.594028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.594029:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:30.0:1619677448.594030:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:30.0:1619677448.594031:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:30.0:1619677448.594032:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.594032:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.594033:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677448.594036:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:9.2:1619677448.594037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.594038:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.594039:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.594040:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.594042:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.594043:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677448.594044:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.0:1619677448.594045:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.594046:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.594046:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.594049:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.594051:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677448.594110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.594115:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.594118:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.594120:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.594120:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.594122:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.594124:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.594125:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.594127:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.594128:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.594129:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.594132:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.594134:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.594136:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.594137:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.594138:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.594139:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:21.2:1619677448.594141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:5.0:1619677448.594141:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.594142:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.594143:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.594145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.594150:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.594152:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.594154:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.594155:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.594155:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.594156:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.594158:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.594158:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.594159:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.594160:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.594162:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.594163:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.594164:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.594168:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.594170:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.594170:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.594171:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.594172:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677448.594173:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.594174:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.594175:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.594177:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.594177:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.594178:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.594178:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.594183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.594191:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.594193:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.594194:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.594195:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.594196:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.594198:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.594201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.594209:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.594212:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.594213:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.594213:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.594214:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.594215:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.594307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.594315:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.594318:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.594319:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.594321:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.594323:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.594325:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.594326:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.594328:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.594329:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.594330:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.594333:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.594334:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.594337:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.594338:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.594338:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.594340:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.594340:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.594341:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.594342:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.594347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.594355:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.594358:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.594359:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.594359:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.594361:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.594364:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.594475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677448.594481:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677448.594484:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677448.594485:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677448.594487:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.594489:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.594490:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.594492:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.594493:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.594494:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.594495:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.594498:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.594500:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.594503:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.594504:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.594504:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.594506:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.594507:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677448.594508:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677448.594509:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677448.594509:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:9.2:1619677448.594512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.2:1619677448.594514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:24.0:1619677448.594519:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:17.0:1619677448.594520:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:24.0:1619677448.594521:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:17.0:1619677448.594521:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677448.594522:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:8.0:1619677448.594522:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.594523:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.594523:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000400:00000200:24.0:1619677448.594525:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.594528:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.2:1619677448.594622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.594627:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.594630:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.594630:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.594632:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.594635:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.594636:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:4.2:1619677448.594638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677448.594638:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.594639:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.594640:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.594641:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.594644:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.594645:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.594645:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.594646:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.594647:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.594647:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.594648:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.594649:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677448.594649:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.594649:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.594650:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.594650:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677448.594651:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.594654:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.594655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677448.594656:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.594657:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.594658:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.594659:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.594660:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.594662:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:26.0:1619677448.594663:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.594664:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:26.0:1619677448.594665:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.594665:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.594665:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.594667:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.594668:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.594668:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677448.594669:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.594669:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.594671:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.594672:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.594672:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.594673:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.2:1619677448.594678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.594683:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:25.2:1619677448.594684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.594686:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.594688:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.594689:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.594690:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.594692:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:5.0:1619677448.594692:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677448.594693:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.594694:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.594695:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.594695:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.594695:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.594697:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677448.594698:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.594699:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.594700:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:5.0:1619677448.594701:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.594703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.594703:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.594705:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.594706:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.594707:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.594708:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677448.594709:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.594709:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.594710:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.594711:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:30.0:1619677448.594712:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.594712:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677448.594713:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.594714:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.594714:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:8.2:1619677448.594714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677448.594716:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.594717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.594720:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.594722:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.594722:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677448.594722:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:20.0:1619677448.594723:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677448.594725:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.594726:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.594726:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.594728:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.594730:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.594904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.594911:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.594914:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.594915:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.594917:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.594919:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.594921:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.594921:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.594923:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.594924:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.594925:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.594928:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.594930:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.594932:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.594933:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.594934:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.594935:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.594936:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.594936:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.594937:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.594941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.594948:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.594951:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.594951:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.594953:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.594955:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.594956:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.594962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:12.0:1619677448.594968:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.594970:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.594971:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.594972:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.594972:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.594973:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.594999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.595005:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.595008:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.595009:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.595010:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.595012:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.595014:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.595015:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.595016:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.595017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:17.0:1619677448.595018:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.595018:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.595022:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.595023:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.595024:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.595026:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.595027:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677448.595027:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.595028:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.595030:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.595030:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.595031:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677448.595031:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.595031:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677448.595032:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.595032:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677448.595033:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.595035:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.595036:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.595036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.595037:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.595038:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.595041:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.595043:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.595043:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677448.595046:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.595046:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677448.595047:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677448.595047:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000400:00000200:12.0:1619677448.595047:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.595048:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.595049:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:31.0:1619677448.595050:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.595050:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677448.595051:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677448.595051:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.595053:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.595053:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.595054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.595059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.595060:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.595063:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.595064:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.595065:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.595066:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677448.595067:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.595068:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.595069:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.595070:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.595070:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677448.595161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.595169:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.595171:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.595172:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.595174:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.595176:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.595177:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.595178:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.595180:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.595180:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.595181:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.595184:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.595186:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.595188:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.595189:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.595190:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.595192:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.595192:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.595193:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.595195:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.595195:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.595198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.595201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.595205:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.595208:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.595208:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.595209:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677448.595209:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677448.595209:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.595210:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.595211:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.595212:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.595215:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.595244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677448.595244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.595250:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.595251:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:1.0:1619677448.595253:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:30.0:1619677448.595254:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:17.0:1619677448.595254:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.595257:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.595258:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.595258:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.595260:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.595261:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.595262:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677448.595263:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677448.595264:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.595265:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.595266:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.595267:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.595267:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.595269:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.595269:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677448.595271:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677448.595273:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.595275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:30.0:1619677448.595276:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.595276:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.0:1619677448.595278:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.595278:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.595279:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.595280:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.595280:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.0:1619677448.595281:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677448.595282:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.595282:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.595283:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.595283:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.595283:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.595284:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.595284:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.595284:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677448.595284:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.595286:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:20.0:1619677448.595286:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.595287:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:20.0:1619677448.595287:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677448.595288:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.595288:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.595288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:20.0:1619677448.595289:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96712881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.595290:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.595291:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.595292:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.2:1619677448.595292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.595299:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:20.0:1619677448.595300:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.595300:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.595301:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.595302:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:5.0:1619677448.595302:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:22.0:1619677448.595303:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.595303:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.595304:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:20.0:1619677448.595305:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.595306:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.595306:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677448.595306:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.595307:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.595309:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.595309:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677448.595310:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677448.595310:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.595310:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.595311:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677448.595312:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:25.2:1619677448.595315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.595316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:12.0:1619677448.595322:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677448.595322:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.595325:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.595326:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677448.595326:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.595327:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.595328:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677448.595329:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.595330:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:6.0:1619677448.595330:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.595332:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:6.0:1619677448.595332:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.595333:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.595334:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.595335:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.595336:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.595339:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.595341:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.595344:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.595344:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.595345:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.595347:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.595347:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.595348:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.595348:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.595353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.595361:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.595364:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.595364:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.595365:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.595367:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.595370:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.595487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.595494:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.595497:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.595498:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.595500:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.595502:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.595503:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.595505:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.595506:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.595507:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.595508:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.595512:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.595513:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.595516:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.595517:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.595518:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.595519:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.595520:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.595521:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.595521:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.595526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.595533:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.595536:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.595537:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.595537:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.595539:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.595542:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.595583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.595591:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.2:1619677448.595592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677448.595594:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.595596:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.595598:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677448.595598:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.595599:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.595600:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:17.0:1619677448.595601:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.595601:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677448.595602:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.595604:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.595604:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.595605:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.595606:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.595606:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.595608:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.595610:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.595611:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.595612:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.595613:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.595614:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.595614:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.595617:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.595618:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.595618:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:17.0:1619677448.595618:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.595620:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.595620:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.595620:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.595621:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.595622:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677448.595624:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677448.595625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.595625:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.595625:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.595627:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:17.0:1619677448.595628:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677448.595628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:17.0:1619677448.595629:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.595630:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.595631:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.595632:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.595635:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677448.595635:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.595636:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.2:1619677448.595636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:13.0:1619677448.595637:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.595638:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:26.0:1619677448.595639:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:12.0:1619677448.595639:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:10.0:1619677448.595639:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:13.0:1619677448.595640:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677448.595640:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.595641:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.595641:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:26.0:1619677448.595642:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677448.595642:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.595643:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:26.0:1619677448.595644:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.595644:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677448.595645:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.595645:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677448.595646:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:25.2:1619677448.595646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:26.0:1619677448.595647:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.595648:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.595649:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.595650:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:13.0:1619677448.595652:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:26.0:1619677448.595653:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.595655:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677448.595655:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.595656:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.595656:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.595657:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.595658:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.595658:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.595659:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.595660:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.595662:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.595662:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.595663:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.595664:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.595668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.595676:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.595679:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.595681:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.595681:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.595683:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.595686:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.595697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:20.2:1619677448.595698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677448.595703:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677448.595706:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677448.595706:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.595706:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677448.595708:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:30.0:1619677448.595709:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:2.0:1619677448.595710:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.595711:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.595712:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:31.0:1619677448.595713:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.595713:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.595713:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.595714:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.595715:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.595716:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.595717:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712874] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.595717:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:30.0:1619677448.595718:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.595719:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.595721:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.595721:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000400:00000200:30.0:1619677448.595725:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.595725:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:30.0:1619677448.595728:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.595728:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.595729:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677448.595729:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677448.595731:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677448.595731:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677448.595732:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.595732:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.595732:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:30.0:1619677448.595733:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.595733:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:30.0:1619677448.595735:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.595735:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.595736:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.595737:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.595737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:8.2:1619677448.595741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.595743:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.595746:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.595746:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.595747:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.595749:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:10.0:1619677448.595749:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:20.0:1619677448.595751:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.595752:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.595752:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:20.0:1619677448.595754:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.595755:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.595757:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.595762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.595769:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.595772:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.595772:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.595774:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.595777:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.595778:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.595779:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.595780:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.595781:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.595782:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.595786:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.595787:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.595789:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.595790:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.595790:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.595792:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.595792:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.595793:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.595793:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.595797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.595803:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.595804:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.595805:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.595806:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.595807:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.595808:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.595817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.595822:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.595825:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.595825:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.595826:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.595826:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.595827:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.595828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.595833:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.595836:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.595837:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.595839:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.595842:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.595843:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.595844:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.595846:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.595847:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.595848:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.595851:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.595853:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.595854:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.595855:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.595856:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.595857:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.595858:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.595858:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.595859:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.595863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.595870:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.595873:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.595873:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.595874:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.595875:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.595876:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.595882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.595889:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.595891:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.595892:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.595893:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.595893:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.595893:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:5.2:1619677448.595910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.595915:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.595918:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.595918:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.595920:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.595921:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.595923:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.595924:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.595925:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.595926:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.595926:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.595929:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.595931:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.595934:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.595934:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.595935:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.595937:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.595937:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.595938:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.595938:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.595943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.595949:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.595952:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.595953:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.595954:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.595955:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.595957:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.595974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.595980:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.595983:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.595984:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.595986:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.595989:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.595991:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.595992:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.595993:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.595994:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.595995:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.595999:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.596000:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.596003:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.596004:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.596005:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.596007:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.596007:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.596008:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.596008:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.596013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.596019:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.596022:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.596023:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.596025:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.596026:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.596028:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.596039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.596047:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.596049:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.596052:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.596053:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.596054:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.596056:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677448.596057:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712869] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.596059:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.596060:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.596061:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.596064:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.596066:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.596069:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.596070:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.596070:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.596072:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.596073:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.596074:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.596074:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.596078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.596086:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.596089:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.596090:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.596090:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.596092:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.596095:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.596097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.596104:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.596106:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.596107:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.596107:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.596109:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.596109:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.596124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.596132:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.596134:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.596135:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.596137:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.596139:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.596140:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.596141:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.596142:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.596143:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.596144:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.596147:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.596148:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.596151:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.596151:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.596152:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.596153:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.596154:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.596155:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.596155:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.596159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.596165:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.596168:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.596169:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.596170:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.596172:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.596173:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.596230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.596236:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.596239:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.596241:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.596241:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.596244:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.596245:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.596247:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.596248:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.596249:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.596250:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.596253:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.596255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677448.596255:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.596259:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.596260:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.596260:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.596262:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.596263:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.596263:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.596263:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.596264:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.596266:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.596268:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.596268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.596270:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.596271:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.596272:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.596274:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.596274:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:30.0:1619677448.596275:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.596276:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.596277:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.596277:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.596278:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.596280:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.596281:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.596281:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.596283:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.596283:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.596285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.596286:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.596287:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.596288:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.596290:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.596290:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.596291:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.596292:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:8.2:1619677448.596298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677448.596299:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.596300:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:13.0:1619677448.596300:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.596300:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.596300:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.2:1619677448.596301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (23)++
00000800:00000200:12.0:1619677448.596301:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.596304:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.596305:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.596307:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.596307:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.596308:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677448.596309:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.596311:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.596312:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.596314:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.596315:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.596317:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.596423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.596430:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.596433:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.596433:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.596435:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.596435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:24.0:1619677448.596439:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.596440:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.596441:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.596442:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:24.0:1619677448.596443:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.596444:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.596445:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.596446:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.596446:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:24.0:1619677448.596448:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.596450:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.596451:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.596452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:24.0:1619677448.596453:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.596454:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.596454:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:1.0:1619677448.596454:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.596455:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:24.0:1619677448.596456:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.596456:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:1.0:1619677448.596456:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.596458:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677448.596458:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:1.0:1619677448.596458:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.596458:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.596459:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.596459:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677448.596460:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.596462:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:8.2:1619677448.596462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677448.596463:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:1.0:1619677448.596463:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.596465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.596465:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:18.0:1619677448.596466:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.596468:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.596468:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.596469:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:1.0:1619677448.596469:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.596470:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:18.0:1619677448.596471:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96712863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.596471:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:18.0:1619677448.596472:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.596472:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:8.0:1619677448.596472:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:18.0:1619677448.596473:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.596473:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.596473:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:18.0:1619677448.596474:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:8.0:1619677448.596474:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.596474:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677448.596474:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.596475:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.596475:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:17.0:1619677448.596476:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.596476:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677448.596477:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:18.0:1619677448.596478:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.596479:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:25.2:1619677448.596480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677448.596480:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.596483:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.596483:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.596484:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.2:1619677448.596485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:18.0:1619677448.596486:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.596486:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.596487:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.596487:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:22.0:1619677448.596488:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.2:1619677448.596489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677448.596491:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.596492:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.596492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.596493:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.596493:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:22.0:1619677448.596494:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.596496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.596496:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.596497:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.596497:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.596497:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.596498:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.596498:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:26.0:1619677448.596499:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:22.0:1619677448.596500:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.596500:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.0:1619677448.596500:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:26.0:1619677448.596502:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.596502:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.0:1619677448.596502:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.596503:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.596504:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.596505:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.596505:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:20.0:1619677448.596505:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.596505:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:5.0:1619677448.596505:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.596506:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:22.0:1619677448.596506:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.596507:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:26.0:1619677448.596508:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.596508:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:26.0:1619677448.596509:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677448.596509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000400:00000200:26.0:1619677448.596510:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.596511:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.596515:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.596516:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677448.596517:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.596519:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.596519:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677448.596519:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.596520:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.596520:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:26.0:1619677448.596521:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677448.596521:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:26.0:1619677448.596522:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677448.596522:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.596523:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:5.0:1619677448.596523:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:26.0:1619677448.596524:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.596525:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.596527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.2:1619677448.596530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.596534:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.596537:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.596537:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.596538:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:30.0:1619677448.596539:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.596540:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.596540:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.596540:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.596541:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.596542:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.596620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.596628:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.596631:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.596631:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677448.596633:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.596635:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.596636:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677448.596638:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.596639:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.596640:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.596641:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.596644:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.596646:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.596649:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.596650:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.596650:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.596652:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.596652:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.596653:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677448.596654:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.596655:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677448.596658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.596661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.596668:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677448.596669:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:26.0:1619677448.596671:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.596671:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:20.0:1619677448.596671:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:26.0:1619677448.596672:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:20.0:1619677448.596672:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.596674:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.596674:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.596677:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.596682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.596690:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.596693:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.596695:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.596696:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.596698:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.596699:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:21.2:1619677448.596699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:31.0:1619677448.596701:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.596702:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.596703:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.596704:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.596706:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.596707:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.596708:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.596710:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.596710:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.596713:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.596713:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:30.0:1619677448.596713:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.596714:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:30.0:1619677448.596714:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:31.0:1619677448.596715:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:30.0:1619677448.596715:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.596717:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.596718:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.596719:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.596720:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.596720:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.596721:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.596721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:30.0:1619677448.596722:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.596723:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.596723:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:4.2:1619677448.596724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.596727:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.596728:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.596728:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:13.0:1619677448.596728:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677448.596730:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:13.0:1619677448.596730:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:30.0:1619677448.596731:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:13.0:1619677448.596731:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.596731:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.596732:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.596732:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:13.0:1619677448.596733:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.596734:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:12.0:1619677448.596734:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.596734:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:13.0:1619677448.596735:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.596736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:10.0:1619677448.596736:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.596739:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.596740:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.596742:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.596743:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.596744:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.596744:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:10.0:1619677448.596745:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:22.0:1619677448.596747:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:10.0:1619677448.596747:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.596747:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.596748:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.596749:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:6.0:1619677448.596750:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.596751:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.596752:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.596752:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:6.0:1619677448.596753:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677448.596754:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.596754:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:25.2:1619677448.596755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.596755:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.596755:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.596759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:22.0:1619677448.596762:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.596765:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:22.0:1619677448.596765:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.596765:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.596766:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.596767:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:22.0:1619677448.596767:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.596767:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:26.0:1619677448.596768:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.596768:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.596769:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.596771:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.596774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.596780:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.596783:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.596785:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.596785:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677448.596787:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.596789:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.596790:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.596791:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.596792:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.596793:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.596797:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.596798:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.596801:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.596802:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.596802:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.596804:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.596805:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.596805:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.596806:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.596810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.596816:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.596819:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.596820:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.596821:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677448.596823:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.596824:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.596860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.596866:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.596869:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.596869:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.596871:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.596874:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.596875:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.596876:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.596878:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.596879:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.596880:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.596883:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.596885:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.596888:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.596888:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.596889:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.596891:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.596891:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.596892:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.596892:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.596897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.596903:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.596906:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:5.2:1619677448.596906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:24.0:1619677448.596907:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.596907:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677448.596909:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.596911:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:24.0:1619677448.596912:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:2.0:1619677448.596914:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.596915:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.596917:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.596919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000400:00000200:2.0:1619677448.596919:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.596920:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.596922:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.596923:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.596924:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.596925:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000400:00000200:2.0:1619677448.596925:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.596927:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677448.596928:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.596928:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.596929:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000400:00000200:2.0:1619677448.596929:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.596930:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:2.0:1619677448.596931:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.596933:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.596933:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.596934:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.596935:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.596936:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.596937:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.596938:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.596939:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677448.596940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:9.2:1619677448.596942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.596944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:10.0:1619677448.596948:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677448.596950:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.596951:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677448.596951:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:10.0:1619677448.596951:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:2.0:1619677448.596951:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:18.0:1619677448.596952:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.596952:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.596953:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.596954:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.596954:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.596954:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:10.0:1619677448.596955:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.596955:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677448.596956:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:10.0:1619677448.596957:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.596958:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.596959:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.596960:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.596961:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.596964:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.596966:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.596968:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.596968:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.596969:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.596970:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.596971:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.596972:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.596972:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.596976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.596982:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.596983:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.596984:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.596985:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.596986:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.596988:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.597136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677448.597139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.0:1619677448.597143:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.597145:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:31.0:1619677448.597147:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:20.0:1619677448.597148:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.597149:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:5.0:1619677448.597149:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.597150:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.597152:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.597152:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677448.597153:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.597154:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.597155:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.597155:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.597156:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.597157:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.597158:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.597158:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677448.597160:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.597161:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.597162:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.597166:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.597167:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:20.0:1619677448.597170:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.597172:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.597174:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.597174:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.597175:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.597175:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.597176:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677448.597177:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677448.597177:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.597178:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.597179:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.597179:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677448.597179:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.597180:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.597180:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.597181:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.597183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:25.2:1619677448.597185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.597190:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.597192:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.597193:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:30.0:1619677448.597193:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.597194:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.597195:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.597195:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.597197:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.597198:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.597199:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677448.597200:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.597201:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.597366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:4.2:1619677448.597369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:10.0:1619677448.597374:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677448.597376:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677448.597377:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.597378:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677448.597379:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:10.0:1619677448.597379:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.597380:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:10.0:1619677448.597381:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.597382:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.597382:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:24.0:1619677448.597384:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.597384:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.597385:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.597386:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.597386:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.597387:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.597388:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:24.0:1619677448.597389:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.597390:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.597392:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.597392:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.597394:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:24.0:1619677448.597395:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.597397:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.597397:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.597398:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.597399:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:24.0:1619677448.597400:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.597401:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.597401:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.597401:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.597401:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.597402:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677448.597403:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.597403:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677448.597403:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677448.597404:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.597404:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677448.597407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:8.2:1619677448.597409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677448.597415:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.597417:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:6.0:1619677448.597418:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.597419:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.597419:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.597419:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.597420:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.597421:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:6.0:1619677448.597422:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.597423:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.597426:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677448.597427:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.597466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.2:1619677448.597472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:17.0:1619677448.597472:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.597474:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.597475:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.597477:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.597477:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:17.0:1619677448.597480:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.597480:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.597481:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:17.0:1619677448.597482:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.597483:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.597484:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.597485:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.597486:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.597486:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.597487:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677448.597487:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.597489:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.597491:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.597491:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.597492:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.597493:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.597494:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.597497:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.597497:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.597498:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.597499:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:2.0:1619677448.597499:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.597500:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.597501:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.597501:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.597501:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:17.0:1619677448.597502:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:2.0:1619677448.597502:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.597504:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.597504:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.597504:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.597504:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.597505:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:9.2:1619677448.597506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:2.0:1619677448.597506:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.597507:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.2:1619677448.597509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:9.2:1619677448.597510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.597513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:8.0:1619677448.597513:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:1.0:1619677448.597514:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677448.597515:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.597516:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.597516:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.597516:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.597517:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.597518:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677448.597519:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:8.0:1619677448.597519:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.597519:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.597520:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.597520:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.597520:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:8.0:1619677448.597521:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:2.0:1619677448.597521:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.597523:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.597523:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.597524:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.597524:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:21.2:1619677448.597597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.597605:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.597607:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.597610:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.597612:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.597612:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.597613:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.597615:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.597616:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.597617:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.597618:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.597621:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.597623:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.597626:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.597627:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.597627:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.597629:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.597630:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.597631:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.597633:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.597633:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.597635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.597639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.597642:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.597645:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.597645:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.597646:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:13.0:1619677448.597647:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.597649:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:12.0:1619677448.597649:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.597650:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677448.597650:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.597651:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677448.597667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.597674:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.597677:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.597679:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.597681:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.597681:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.597683:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.597684:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.597685:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.597686:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.597687:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.597691:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.597692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.597693:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.597696:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.597697:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.597698:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.597699:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.597699:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.597700:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.597701:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.597701:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.597701:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.597702:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.597704:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.597705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:26.0:1619677448.597707:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.597708:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.597709:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.597711:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.597712:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.597713:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.597714:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:26.0:1619677448.597716:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.597716:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.597717:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.597718:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:5.0:1619677448.597719:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.597720:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.597721:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.597722:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.597722:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.0:1619677448.597722:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:26.0:1619677448.597724:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.597724:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.597725:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.597726:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.597730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.597738:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.597741:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.597741:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.597742:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.597744:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.597747:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.597889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.597894:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.597897:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.597898:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.597900:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.597902:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.597903:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.597904:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.597905:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.597907:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.597907:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.597911:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.597913:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.597915:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.597916:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.597916:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.597918:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.597918:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.597919:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.597920:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.597924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.597932:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.597935:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.597936:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.597936:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.2:1619677448.597938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:17.0:1619677448.597938:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.597941:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677448.597941:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677448.597944:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.597946:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.597947:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.597949:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.597950:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677448.597952:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.597953:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.597954:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.597955:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677448.597958:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.597959:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.597962:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.597963:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.597963:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.597964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:20.0:1619677448.597965:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.597965:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.597967:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.597968:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.597969:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677448.597970:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.2:1619677448.597970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.597972:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:20.2:1619677448.597973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.597973:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.597975:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677448.597977:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.597978:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:18.0:1619677448.597978:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.597978:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:20.0:1619677448.597979:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.597979:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.597980:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712842] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.597980:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.597980:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.597981:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.597982:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.597982:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.597983:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:18.0:1619677448.597983:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.597983:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677448.597984:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.597987:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.597988:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.597990:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677448.597990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677448.597991:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.597991:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.597993:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.597994:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.597994:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.597995:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.0:1619677448.597997:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:9.2:1619677448.597999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.598000:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.598001:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.598002:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.598002:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.598003:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:2.0:1619677448.598006:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.598008:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.598009:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.598010:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.598011:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.598014:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.598111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.598116:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.598119:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.598120:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.598122:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.598124:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.598126:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.598127:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.598128:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.598129:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.598130:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.598134:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.598136:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.598138:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.598139:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.598139:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.598141:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.598142:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.598143:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.598144:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.598144:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.598147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.598151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677448.598155:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:22.0:1619677448.598156:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.598158:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:5.0:1619677448.598158:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.598158:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.598159:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.598160:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:5.0:1619677448.598160:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.598161:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.598162:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.598204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.598211:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.598214:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.598215:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.598217:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.598220:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.598221:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.598222:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.598224:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.598225:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.598226:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.598229:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.598231:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.598234:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.598235:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.598236:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.598238:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.598238:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.598239:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.598241:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.598241:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677448.598243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.598247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:4.2:1619677448.598247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.598252:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.598254:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:18.0:1619677448.598254:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.598254:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677448.598256:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.598256:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677448.598256:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.598256:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:26.0:1619677448.598257:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:18.0:1619677448.598258:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.598259:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.598261:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.598261:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.598262:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.598263:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.598263:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:18.0:1619677448.598264:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.598265:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.598265:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.598266:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.598266:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.598267:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.598268:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:2.0:1619677448.598268:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.598269:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.598272:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.598274:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.598277:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.598278:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.598279:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.598281:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.598281:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.598282:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.598282:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.598287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.598294:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.598301:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.598302:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.598303:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.598303:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.598306:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.598346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.598354:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.598357:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.598359:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.598362:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.598362:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.598364:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.598365:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.598366:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.598367:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.598368:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.598372:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.598373:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.598376:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.598376:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.598377:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.598378:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.598379:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.598380:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.598380:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.598385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.598391:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.598393:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.598394:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.598396:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.598397:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.598399:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.598402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.598408:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.598410:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.598411:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.598412:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.598412:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.598413:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.598490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677448.598497:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.598500:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677448.598500:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677448.598502:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.598504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:1.0:1619677448.598505:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.598506:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:1.0:1619677448.598506:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677448.598508:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712837] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.598509:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:1.0:1619677448.598510:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.598511:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.598512:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.598512:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:1.0:1619677448.598512:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.598514:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.598515:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.598516:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.598517:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.598518:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.598519:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.598519:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.598520:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.598521:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.598522:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:1.0:1619677448.598523:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:20.0:1619677448.598524:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677448.598525:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.598526:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.598526:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677448.598526:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677448.598527:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.0:1619677448.598529:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.598529:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.598530:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:25.2:1619677448.598531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.598532:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.598533:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.598533:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.598534:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.598538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.598539:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.598542:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.598543:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.598543:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.598545:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:6.0:1619677448.598545:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.598547:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677448.598548:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.598549:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.598551:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.598551:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:4.2:1619677448.598551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.598554:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677448.598559:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.598562:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.598562:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.598565:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.598568:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.598570:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.598571:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.598572:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.598573:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.598574:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.598578:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.598579:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.598582:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.598582:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.598583:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.598585:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.598585:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.598586:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.598586:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.598590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.598597:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.598600:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.598600:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.598602:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.598604:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.598604:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.2:1619677448.598669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.598675:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.598677:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.598679:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.598680:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.598682:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.598684:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.598685:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.598686:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.598687:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.598688:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.598692:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.598694:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.598697:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.598697:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.598698:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.598700:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.598700:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.598701:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.598702:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.598706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.598712:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.598715:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.598716:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.598717:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.598719:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:21.2:1619677448.598720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:12.0:1619677448.598720:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677448.598725:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.598728:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.598728:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.598730:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.598733:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.598734:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.598735:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.598737:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.598737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:22.0:1619677448.598738:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.598739:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.598742:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.598745:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:22.0:1619677448.598745:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.598747:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.598748:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:22.0:1619677448.598748:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.598748:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.0:1619677448.598749:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677448.598750:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:24.0:1619677448.598751:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.598751:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.598751:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.598752:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:24.0:1619677448.598754:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.598756:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.598757:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96712832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.598757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:24.0:1619677448.598760:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.598761:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.598762:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.598765:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:24.0:1619677448.598766:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.598767:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:24.0:1619677448.598768:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:20.0:1619677448.598768:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.598770:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.598771:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.598771:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:5.0:1619677448.598771:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:24.0:1619677448.598772:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.598773:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:24.0:1619677448.598774:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.598774:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.598776:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.598777:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.598777:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.2:1619677448.598779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:21.2:1619677448.598783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:13.0:1619677448.598786:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.598788:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.598789:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.598790:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:13.0:1619677448.598791:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.598791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677448.598792:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.598792:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.598792:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677448.598794:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677448.598794:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:17.0:1619677448.598797:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.598799:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.598800:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.598802:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.598805:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.598806:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.598808:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.598809:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.598810:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.598811:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.598815:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.598816:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.598819:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.598820:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.598821:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.598823:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.598823:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.598824:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.598824:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.598828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.598835:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.598838:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.598839:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.598840:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.598842:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.598843:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.598843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:25.2:1619677448.598847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.598850:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.598853:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:13.0:1619677448.598853:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.598853:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.598856:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.598856:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.598857:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:26.0:1619677448.598858:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.598858:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.598858:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:26.0:1619677448.598859:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:13.0:1619677448.598860:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:26.0:1619677448.598861:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712830] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.598862:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.598863:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.598864:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.598867:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.598869:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.598872:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.598873:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.598874:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.598876:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.598876:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.598877:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.598877:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.598882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.598890:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.598892:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.598894:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.598895:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.598895:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.598898:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.599246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:4.2:1619677448.599248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677448.599252:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.599255:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:10.0:1619677448.599255:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.599256:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:10.0:1619677448.599258:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.599259:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677448.599259:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.599262:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.599262:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.599263:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.599264:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.599264:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.599266:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.599266:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677448.599267:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.599267:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.599268:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.599269:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.599270:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.599273:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.599273:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:18.0:1619677448.599276:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.599277:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.599278:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.599278:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.599279:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.599279:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677448.599280:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.599281:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.599281:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.599282:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:10.0:1619677448.599282:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.599283:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.599284:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.599286:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.599286:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.599286:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.599287:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.599287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.2:1619677448.599291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.599294:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.599299:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:20.0:1619677448.599301:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677448.599301:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.599302:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.599303:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.599304:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.599304:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.599306:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.599306:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677448.599307:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.599310:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.599355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.2:1619677448.599359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:5.0:1619677448.599362:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.599364:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677448.599365:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.599367:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.599367:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.599368:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.599368:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.599370:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.599370:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.599371:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:6.0:1619677448.599372:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.599373:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.599374:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.599375:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.599375:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.599376:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.599377:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.599377:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677448.599378:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.599379:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.599385:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.599385:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.599387:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:5.0:1619677448.599388:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.599391:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.599391:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:5.0:1619677448.599391:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.599392:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.599393:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:5.0:1619677448.599393:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.599395:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.599395:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677448.599395:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.599396:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.599397:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.599397:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677448.599397:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.599398:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.599401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:25.2:1619677448.599405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.599408:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.599411:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.599412:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.599412:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.599413:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:26.0:1619677448.599415:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.599415:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.599416:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.599416:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.599418:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:22.0:1619677448.599418:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.599420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.599421:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677448.599426:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.599429:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.599429:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.599430:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.599431:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.599432:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.599440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.599446:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.599449:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.599450:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.599452:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.599454:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.599456:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.599457:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.599458:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.599459:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.599460:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.599464:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.599466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677448.599466:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.599468:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.599469:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.599469:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.599471:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.599472:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.599472:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.599473:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.599474:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.599477:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.599479:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.599481:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.599481:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.599483:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.599484:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.599485:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.599486:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.599487:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:25.2:1619677448.599488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:30.0:1619677448.599491:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.599492:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.599495:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.599495:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:30.0:1619677448.599496:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.599496:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.599498:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.599498:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677448.599498:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.599499:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.599499:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.599500:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.599500:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:6.0:1619677448.599501:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.599503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.599503:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677448.599511:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.599514:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.599514:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.599516:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.599518:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.599519:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.599684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.599691:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.599694:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.599694:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.599696:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.599699:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.599700:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.599701:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.599703:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.599704:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.599705:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.599708:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.599709:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.599711:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.599712:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.599712:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.599714:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.599714:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.599716:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.599717:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.599717:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.599719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.599723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.599726:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.599728:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.599729:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.599730:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.599730:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.599731:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:30.0:1619677448.599732:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.599732:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.599733:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.599734:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.599737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.599743:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.599746:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.599747:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.599747:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.599748:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.599749:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.599780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.599787:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.599790:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.599791:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.599792:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.599794:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.599796:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.599797:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.599798:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.599799:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.599800:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.599803:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.599804:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.599807:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.599808:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.599808:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.599810:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.599811:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.599812:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.599812:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.599817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.599825:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.599828:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.599829:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.599829:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.599832:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:17.0:1619677448.599835:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.599863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.599868:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.599871:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.599871:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.599873:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.599875:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.599877:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.599878:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.599879:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.599880:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.599881:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.599884:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.599885:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.599887:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.599888:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.599889:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.599890:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.599891:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.599891:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.599892:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.599897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.599904:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.599907:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.599908:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.599908:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.599910:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.599912:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.599917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:6.0:1619677448.599925:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.599927:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.599928:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.599928:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.599929:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.599930:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.599963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.599966:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.599968:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.599970:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.599972:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.599973:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.599974:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.599976:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.599977:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.599978:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.599979:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.599982:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.599984:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.599987:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.599988:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.599989:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.599991:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.599991:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.599992:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.599993:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.599997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.600003:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.600006:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.600007:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.600008:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.600009:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.600011:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.600109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.600117:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.600119:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.600122:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.600123:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.600124:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.600126:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.600127:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.600128:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.600129:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.600130:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.600134:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.600135:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.600138:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.600139:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.600140:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.600142:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.600142:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.600144:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.600145:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.600146:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.600148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.600152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.600156:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.600158:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.600159:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.600160:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.600161:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.600161:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.600162:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.600163:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.600164:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.600164:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.600166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.600173:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.600176:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.600177:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.600177:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.600178:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.600179:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.600188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.600196:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.2:1619677448.600198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:10.0:1619677448.600199:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.600200:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.600202:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.600203:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677448.600205:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.600206:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:10.0:1619677448.600206:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:6.0:1619677448.600207:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677448.600208:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.600209:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.600210:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.600211:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.600212:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.600212:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677448.600214:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.600215:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.600216:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.600217:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.600217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:22.0:1619677448.600218:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.600219:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.600219:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.600222:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.600223:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.600223:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.600223:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.600224:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677448.600225:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.600225:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.600225:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.600226:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.600227:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:10.0:1619677448.600227:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677448.600228:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.600229:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:31.0:1619677448.600230:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.600230:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.600231:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677448.600231:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:9.2:1619677448.600231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677448.600232:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:31.0:1619677448.600233:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.600233:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.600233:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:31.0:1619677448.600234:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.600236:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96712816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.600237:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677448.600237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677448.600238:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.600239:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.600239:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.600242:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.600242:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:31.0:1619677448.600243:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.600243:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.600245:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.600245:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:18.0:1619677448.600246:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.600247:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:31.0:1619677448.600248:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.600248:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.600248:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.600249:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.600249:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.600249:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677448.600251:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:17.0:1619677448.600251:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.600252:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.600253:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.600253:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677448.600254:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:25.2:1619677448.600256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.600263:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.600266:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.600267:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.600269:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.600270:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.600271:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.600271:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.600272:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.600272:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.600273:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.600273:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.600300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.600306:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.600309:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.600309:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.600311:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.600314:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.600315:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.600316:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.600318:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.600319:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.600320:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.600323:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.600324:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.600327:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.600328:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.600328:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.600330:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.600330:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.600332:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.600333:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.600333:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.600337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.600339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.600344:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677448.600345:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.600347:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.600347:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.600347:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677448.600348:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.600348:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.600349:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.600351:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.600353:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.600487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.600494:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.600497:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.600498:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.600500:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.600503:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.600504:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.600505:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.600507:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.600508:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.600509:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.600512:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.600512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:26.0:1619677448.600514:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.600515:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.600517:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.600517:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:26.0:1619677448.600518:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.600519:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.600520:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:20.0:1619677448.600520:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.600520:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677448.600521:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.600521:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.600522:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:20.0:1619677448.600523:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.600524:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:25.2:1619677448.600526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:20.0:1619677448.600526:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.600527:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.600528:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.600529:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.600532:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.600534:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677448.600534:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.600537:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677448.600537:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.600537:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.600538:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.600538:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677448.600538:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (23)++
00000400:00000200:22.0:1619677448.600540:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.600540:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677448.600541:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677448.600542:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.600542:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.600543:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.600544:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.600546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.600549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.600553:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.600554:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:6.0:1619677448.600555:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.600556:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.600557:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.600557:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.600557:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.600557:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.600558:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677448.600560:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.600568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.600575:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.600577:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.600578:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.600578:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.600579:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.600580:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.600704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.600712:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.600715:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.600717:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.600720:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.600720:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.600721:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.600722:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.600724:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.600725:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.600726:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.600728:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.600730:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.2:1619677448.600731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.600733:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.600733:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.600734:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.600735:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.600736:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.600736:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.600737:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.600739:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:1.0:1619677448.600739:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:30.0:1619677448.600740:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677448.600740:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.600741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677448.600742:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.600745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677448.600745:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.600746:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.600747:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.600748:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677448.600748:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.600750:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.600751:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.600751:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.600751:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677448.600753:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:13.0:1619677448.600753:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.600755:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:13.0:1619677448.600755:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.600755:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:1.0:1619677448.600755:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.600756:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.600757:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677448.600757:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.600760:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.600760:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.600761:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.600763:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.600763:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.600764:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.600765:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.600769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.600776:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.600779:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.600780:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.600780:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.600782:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.600784:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.600801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.600808:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.600811:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.600812:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.600814:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.600817:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.600818:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.600820:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.600821:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.600822:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.600823:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.600826:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.600828:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.600830:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.600831:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.600832:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.600833:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.600834:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.600835:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.600836:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.600837:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.600839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.2:1619677448.600842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.600846:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.600848:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.600849:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.600849:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:31.0:1619677448.600850:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.600851:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:30.0:1619677448.600852:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.600852:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.600853:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677448.600854:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.600912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.600921:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.600923:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.600924:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.600926:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.600928:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.600930:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.600931:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.600932:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.600933:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.600934:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.600937:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.600939:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.600941:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.600942:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.600943:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.600944:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.600945:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.600945:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.600945:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.600950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.600958:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.600960:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.600961:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.600962:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.600964:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.600967:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.601126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.601132:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.2:1619677448.601134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:2.0:1619677448.601135:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.601136:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.601138:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.601140:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.601140:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:2.0:1619677448.601141:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.601142:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.601143:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677448.601144:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:2.0:1619677448.601144:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.601145:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.601146:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.601147:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.601149:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.601149:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.601151:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.601151:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677448.601152:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.601153:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.601154:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.601155:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.601155:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677448.601155:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.601156:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.601157:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.601157:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.601158:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.601158:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:1.0:1619677448.601159:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.601161:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:9.2:1619677448.601163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.601163:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.601164:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:1.0:1619677448.601165:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.601166:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677448.601167:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:21.2:1619677448.601168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:1.0:1619677448.601168:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677448.601168:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677448.601172:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.2:1619677448.601172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.601174:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.601174:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.601175:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:18.0:1619677448.601176:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.601176:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:20.2:1619677448.601177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677448.601179:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.601179:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.601180:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677448.601181:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677448.601181:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.601181:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.601182:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.601183:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:5.0:1619677448.601183:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:2.0:1619677448.601183:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.601184:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.601185:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.601185:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.601187:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.601187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677448.601187:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.601188:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:2.0:1619677448.601188:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.601189:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:2.0:1619677448.601189:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.601190:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:20.0:1619677448.601191:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.601192:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.601193:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.601194:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712805] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.601194:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.601195:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677448.601196:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.601196:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.601197:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.601197:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.601198:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.601198:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:2.0:1619677448.601198:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677448.601199:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.601199:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:10.0:1619677448.601202:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.601203:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.601205:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.601205:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.601205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.601205:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.601205:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:10.0:1619677448.601206:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.601207:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96712804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.601208:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.601208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677448.601208:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677448.601209:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000400:00000200:10.0:1619677448.601209:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.601209:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000400:00000200:10.0:1619677448.601210:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.601211:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677448.601211:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000400:00000200:10.0:1619677448.601212:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.601212:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.601213:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:20.0:1619677448.601214:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (22)++
00000400:00000200:2.0:1619677448.601215:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.601216:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (23)--
00000800:00000200:20.0:1619677448.601216:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.601217:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677448.601217:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:10.0:1619677448.601217:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.601217:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677448.601218:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.601218:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:20.0:1619677448.601218:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:10.0:1619677448.601220:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.601221:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:26.0:1619677448.601221:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.601222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.601223:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.601224:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.601224:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.601224:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.601225:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.601225:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:30.0:1619677448.601226:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.601226:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.601227:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96712803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.601227:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:30.0:1619677448.601228:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:18.0:1619677448.601229:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:30.0:1619677448.601230:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96712803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.601230:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.601231:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.601232:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.601232:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.601232:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000400:00000200:30.0:1619677448.601233:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.601233:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:25.2:1619677448.601233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.601233:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (24)++
00000800:00000200:2.0:1619677448.601233:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000400:00000200:30.0:1619677448.601234:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.601234:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (24)--
00000800:00000200:10.0:1619677448.601234:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677448.601234:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.601235:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:10.0:1619677448.601235:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677448.601236:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.601236:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:2.0:1619677448.601237:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.601238:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:30.0:1619677448.601239:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.601240:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.601241:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.601241:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.601242:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.601243:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.601244:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.601244:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677448.601244:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677448.601244:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:26.0:1619677448.601245:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.601245:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.601245:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:30.0:1619677448.601246:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.601246:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.601246:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.601246:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.601247:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:22.0:1619677448.601247:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.601248:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.601248:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.601250:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677448.601250:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:26.0:1619677448.601251:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.601251:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.601251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:9.2:1619677448.601255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677448.601258:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.601261:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.601261:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.601262:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:12.0:1619677448.601263:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.601265:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:13.0:1619677448.601265:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.601265:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:18.0:1619677448.601266:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.601266:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.601268:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.601271:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.601282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.601288:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.601291:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.601292:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.601297:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.601298:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.601300:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677448.601301:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.601302:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.601303:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.601304:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677448.601308:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.601310:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677448.601312:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.601313:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677448.601313:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677448.601315:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677448.601316:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.601317:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.601318:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.601318:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677448.601322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:21.2:1619677448.601324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:21.2:1619677448.601329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677448.601330:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:18.0:1619677448.601331:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:6.0:1619677448.601332:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:18.0:1619677448.601333:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:22.0:1619677448.601334:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.601334:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:6.0:1619677448.601334:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.601335:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:6.0:1619677448.601336:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.601337:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677448.601338:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677448.601339:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.601342:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.601344:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.601344:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.601346:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.601347:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.601349:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.601350:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.601351:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677448.601355:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.601357:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.2:1619677448.601358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677448.601360:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.601361:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.601362:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.601363:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.601364:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.601364:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.601365:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.601365:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677448.601367:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.601368:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.601369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:17.0:1619677448.601370:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.601372:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.601374:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.601375:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.601376:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.601377:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.601378:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:2.0:1619677448.601378:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.601381:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.601382:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:17.0:1619677448.601382:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.601382:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.601384:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:2.0:1619677448.601384:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.601387:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.601388:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:2.0:1619677448.601388:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:17.0:1619677448.601389:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.601391:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.601391:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.601392:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.601392:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.601396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.601404:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.601407:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.601407:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677448.601408:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.601410:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.601413:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677448.601515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.601520:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.601523:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.601523:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.601525:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.601528:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.601529:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.601530:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.601532:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.601533:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.601534:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.601537:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.601538:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.601541:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.601542:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.601542:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.601544:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.601545:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.601546:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.601547:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.601547:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677448.601550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.601553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677448.601558:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:6.0:1619677448.601558:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.601560:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.601560:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677448.601561:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:6.0:1619677448.601561:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:18.0:1619677448.601562:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.601562:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.601564:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.601565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.601568:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:18.0:1619677448.601569:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.601571:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.601572:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.601574:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.601575:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.601578:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.601578:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.601580:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.601581:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.601582:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.601585:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.601587:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.601590:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.601591:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.601592:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.601594:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.601594:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.601595:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.601595:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.601600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.601607:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.2:1619677448.601609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:13.0:1619677448.601610:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.601611:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.601612:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.601614:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.601615:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.601615:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.601618:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.601620:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.601620:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677448.601622:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.601624:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.601625:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.601627:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.601628:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.601629:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.601632:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.601634:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.601637:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.601638:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.601639:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.601640:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.601641:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.601642:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.601644:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.601644:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.601646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.601649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.601652:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:12.0:1619677448.601653:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.601655:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.601655:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.601656:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:12.0:1619677448.601656:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.601657:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.601658:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677448.601660:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.601661:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.601674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.601681:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.601684:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.601685:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.601686:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.601689:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.601691:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.601692:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.601693:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.601694:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.601695:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.601698:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.601700:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.601703:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.601704:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.601704:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.601706:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.601706:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.601707:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.601708:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.601712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.601720:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.601723:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.601723:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.601724:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.601726:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.601729:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.601754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.601762:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.601765:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.601766:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.601768:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.601771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:10.0:1619677448.601771:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.601773:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.601774:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.601775:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.601776:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.601777:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.601778:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677448.601780:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.601781:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:10.0:1619677448.601781:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.601783:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.601784:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.601785:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.601785:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.601785:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:30.0:1619677448.601786:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.601787:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:10.0:1619677448.601787:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.601787:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:30.0:1619677448.601788:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.601788:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:30.0:1619677448.601789:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.601789:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:30.0:1619677448.601790:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.601791:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:25.2:1619677448.601792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:30.0:1619677448.601794:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.601796:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.601799:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.601799:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677448.601800:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.601800:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:13.0:1619677448.601801:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:30.0:1619677448.601802:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:13.0:1619677448.601802:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.601803:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.601803:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:13.0:1619677448.601803:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.601804:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677448.601805:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.601806:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.2:1619677448.601808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.601816:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.601819:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.601819:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.601820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677448.601821:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.601824:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.601825:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:31.0:1619677448.601828:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.2:1619677448.601828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.601831:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.601833:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.601835:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.0:1619677448.601835:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677448.601836:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.601838:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:5.0:1619677448.601838:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.601839:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.601839:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.601840:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000400:00000200:31.0:1619677448.601841:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.601841:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:31.0:1619677448.601842:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.601842:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.601842:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:31.0:1619677448.601846:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.601848:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.601851:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.601852:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.601852:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.601854:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.601855:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.601855:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.601856:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.601860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.601868:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.601871:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.601872:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.601872:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.601874:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.601877:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.601994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.602001:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.602003:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.602004:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.602006:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.602009:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.602010:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.602011:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.602013:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.602014:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.602015:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.602018:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.602019:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.602023:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.602023:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.602024:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.602025:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.602026:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.602027:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.602027:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.602031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.602039:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.602042:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.602043:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.602043:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.602045:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.602048:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.602089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.602096:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.602099:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.602101:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.602103:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.602105:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.602105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:30.0:1619677448.602106:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.602108:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.602109:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.602110:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.602111:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:2.0:1619677448.602111:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.602114:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.602115:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:30.0:1619677448.602116:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.602117:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.602118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.602119:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:2.0:1619677448.602120:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.602121:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.602121:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.602122:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:2.0:1619677448.602122:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.602123:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.602124:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.602124:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677448.602124:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677448.602124:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.602125:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:2.0:1619677448.602125:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.602126:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:2.0:1619677448.602126:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.602127:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.602128:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.602131:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.602131:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.602133:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.602134:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.602136:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:2.0:1619677448.602136:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.602137:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96712788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.602137:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:17.0:1619677448.602138:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.602138:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:25.2:1619677448.602139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:17.0:1619677448.602139:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.602140:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:2.0:1619677448.602140:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.602140:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.602141:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.602143:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.602143:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677448.602144:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.602145:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677448.602146:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:25.2:1619677448.602146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.602148:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:17.0:1619677448.602148:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.602148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000400:00000200:31.0:1619677448.602149:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.602149:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.602149:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:31.0:1619677448.602151:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.602151:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.602151:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.602152:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:17.0:1619677448.602152:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.602153:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.602153:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677448.602154:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:18.0:1619677448.602154:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.602156:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:12.0:1619677448.602156:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.602156:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.602157:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677448.602157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:18.0:1619677448.602158:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:12.0:1619677448.602158:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.602159:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.602161:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.602163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:8.0:1619677448.602164:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:25.2:1619677448.602166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677448.602166:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.602167:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.602167:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.602169:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000400:00000200:8.0:1619677448.602170:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.602172:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:8.0:1619677448.602172:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:30.0:1619677448.602173:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.602173:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:13.0:1619677448.602173:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.602174:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.602175:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:13.0:1619677448.602175:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.602176:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.602177:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.602177:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:21.2:1619677448.602179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:13.0:1619677448.602179:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677448.602186:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.602189:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.602190:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.602192:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.602194:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.602196:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.602197:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.602198:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.602199:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.602200:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.602203:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.602205:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.602208:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.602209:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.602210:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.602212:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.602212:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.602213:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.602213:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.602218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.602225:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.602228:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.602229:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.602229:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.602232:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.602235:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.602248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.602254:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.602256:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.602259:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.602259:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.602261:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.602263:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.602264:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.602265:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.602267:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.602268:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.602271:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.602273:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.602276:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.602277:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.602278:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.602279:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.602280:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.602281:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.602281:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.602285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.602292:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.602301:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.602301:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.602302:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.602304:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.602306:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.602363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.602371:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.602374:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.602375:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.602376:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.602379:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.602381:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.602382:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.602383:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.602384:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.602385:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.602388:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.602390:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.602392:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.602393:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.602393:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.602395:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.602395:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.602396:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.602396:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.602401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.602409:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.602411:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.602412:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.602413:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.602415:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.602416:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.602457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.602465:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.602467:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.602468:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.602470:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.602473:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.602474:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.602475:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.602477:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.602477:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.602478:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.602481:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.602482:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.602485:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.602486:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.602486:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.602488:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.602489:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.602490:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.602491:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.602491:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.602494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677448.602497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.602502:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:21.2:1619677448.602503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:24.0:1619677448.602504:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.602505:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:24.0:1619677448.602506:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.602506:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677448.602506:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.602506:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.602507:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.602509:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.602511:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.602511:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677448.602513:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.602516:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.602517:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677448.602517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000400:00000200:30.0:1619677448.602518:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.602519:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.602521:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712783] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.602522:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.602523:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.602524:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:17.0:1619677448.602525:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:30.0:1619677448.602527:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.602527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677448.602528:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.602529:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.602529:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677448.602530:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:17.0:1619677448.602530:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.602531:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677448.602533:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.602533:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.602534:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.602535:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.602536:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:30.0:1619677448.602536:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.602536:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.602537:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.602538:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.602538:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:31.0:1619677448.602540:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.602540:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.602542:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:9.2:1619677448.602542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:31.0:1619677448.602543:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.602544:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.602545:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.602546:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.602549:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.602549:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:31.0:1619677448.602551:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677448.602552:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:31.0:1619677448.602553:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.602553:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:18.0:1619677448.602553:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.602553:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.602554:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.602555:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:18.0:1619677448.602555:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.602556:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.602557:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:18.0:1619677448.602558:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:31.0:1619677448.602559:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.602559:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:9.2:1619677448.602560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:25.2:1619677448.602561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.2:1619677448.602565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677448.602567:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.602570:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:2.0:1619677448.602570:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.602571:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.602571:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:30.0:1619677448.602572:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:2.0:1619677448.602572:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.602573:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:30.0:1619677448.602573:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.602573:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:31.0:1619677448.602574:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.602575:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.602575:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.602576:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.602577:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.602578:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.602587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.602594:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.602597:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.602598:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.602600:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.602602:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.602604:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.602605:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.602606:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.602607:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.602608:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.602611:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.602612:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.602614:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.602615:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.602616:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.602617:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.602618:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.602619:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.602620:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.602620:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.602623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.602626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.602629:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.602631:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.602632:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.602633:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677448.602634:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.602635:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.602635:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:31.0:1619677448.602636:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677448.602636:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.602637:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677448.602689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.602695:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.602698:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.602699:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.602700:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.602703:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.602704:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.602706:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.602707:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.602708:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.602709:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.602712:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.602714:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.602717:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.602717:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.602718:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.602720:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.602720:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.602722:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.602723:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.602723:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.602726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.602728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.602732:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677448.602734:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677448.602735:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.602735:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.0:1619677448.602735:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:17.0:1619677448.602736:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677448.602736:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.602737:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.602738:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.602741:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.602744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677448.602749:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.602752:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677448.602752:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.602753:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.602754:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.602754:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.602910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.602918:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.602920:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.602922:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.602924:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.602924:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.602926:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.602927:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.602928:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.602929:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.602930:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.602933:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.602935:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.602938:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.602939:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.602939:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.602941:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.602942:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.602943:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.602945:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.602945:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677448.602947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.602951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.602955:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:31.0:1619677448.602958:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:18.0:1619677448.602958:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.602958:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.602959:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.602961:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:18.0:1619677448.602961:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.602962:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.602962:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677448.602964:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.602988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.602995:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.602998:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.602998:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.603000:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.603002:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.603003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:10.0:1619677448.603004:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.603005:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.603007:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.603007:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.603008:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677448.603011:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677448.603012:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.603013:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:10.0:1619677448.603013:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:31.0:1619677448.603016:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.603016:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.603017:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677448.603018:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.603018:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:31.0:1619677448.603019:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.603019:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.603020:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677448.603021:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:10.0:1619677448.603021:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:31.0:1619677448.603022:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712777] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.603022:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.603022:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:31.0:1619677448.603023:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.603024:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.603025:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.603027:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.603028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.603029:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.603031:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.603032:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.603032:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.603034:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.603034:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.603035:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677448.603035:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:31.0:1619677448.603036:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.603036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.603037:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.603037:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.603038:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.603040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677448.603044:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.603046:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:13.0:1619677448.603046:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:22.0:1619677448.603048:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.603048:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:13.0:1619677448.603049:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.603049:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.603050:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:13.0:1619677448.603051:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.603053:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677448.603053:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.603054:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.603057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:6.0:1619677448.603065:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.603068:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.603068:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.603069:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.603070:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.603071:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.603178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.603185:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.603188:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.603188:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.603191:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.603194:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.603195:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.603196:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.603197:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.603198:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.603199:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.603203:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.603204:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.603207:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.603207:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.603208:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.603209:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.603210:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.603210:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.603211:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.603215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.603222:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.603225:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.603225:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.603227:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.603228:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.603229:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.603263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677448.603270:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.603272:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677448.603273:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677448.603275:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.603277:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.603278:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.603279:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712775] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.603280:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.603281:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.603282:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.603284:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.603286:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.603287:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.603288:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.603288:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.603290:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.603290:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.603291:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677448.603292:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677448.603293:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:25.2:1619677448.603298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.603300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.603306:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.603307:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.603308:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.603309:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:10.0:1619677448.603309:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.603310:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677448.603310:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677448.603311:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.603311:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.603312:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:20.2:1619677448.603366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.603369:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.603372:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.603374:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.2:1619677448.603374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:20.0:1619677448.603377:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.603378:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:5.0:1619677448.603379:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.603380:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.603380:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:20.0:1619677448.603381:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.603382:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.603383:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.603383:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.603384:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:21.2:1619677448.603385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:20.2:1619677448.603386:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:1.0:1619677448.603386:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.603391:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:20.0:1619677448.603392:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.603392:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.603394:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.603394:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677448.603394:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.603395:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:1.0:1619677448.603395:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.603397:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.603397:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.603397:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:20.0:1619677448.603398:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.603398:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677448.603398:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677448.603399:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.603399:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712773] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.603400:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:5.0:1619677448.603401:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677448.603401:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.603401:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.603402:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.603403:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.603403:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677448.603403:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677448.603405:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.603406:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677448.603407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677448.603407:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.603407:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.603408:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.603408:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.603409:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.603410:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.603411:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.603414:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.603415:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.603416:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:2.0:1619677448.603416:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677448.603416:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.603417:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:17.0:1619677448.603417:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677448.603417:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677448.603417:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.603418:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677448.603418:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677448.603419:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:17.0:1619677448.603419:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.603419:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.603419:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.603420:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.603420:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.603421:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.603421:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.603421:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.603422:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.603422:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.603422:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:1.0:1619677448.603422:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677448.603423:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.603424:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677448.603424:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:25.2:1619677448.603425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.603425:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:5.0:1619677448.603425:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.603425:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.603425:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.0:1619677448.603426:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.603426:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677448.603426:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677448.603426:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.603427:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677448.603427:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.603428:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.603428:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (24)--
00000800:00000200:1.0:1619677448.603428:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.603429:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.603430:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.603430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:25.2:1619677448.603431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.603433:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.603436:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.603437:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.603438:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:31.0:1619677448.603439:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.603439:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677448.603440:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:30.0:1619677448.603441:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.603441:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:20.0:1619677448.603441:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.603442:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:22.0:1619677448.603442:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.603442:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:20.0:1619677448.603443:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.603444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.603444:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:22.0:1619677448.603445:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:20.0:1619677448.603445:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677448.603448:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677448.603451:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:25.2:1619677448.603452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.603454:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.603454:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.603455:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.603456:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.603457:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:6.0:1619677448.603460:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.603462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:22.0:1619677448.603463:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.603463:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.603464:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.603465:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.603465:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677448.603468:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.603471:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.603472:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.603474:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.603477:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.603478:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677448.603480:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.603481:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.603482:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.603483:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677448.603487:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.603489:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677448.603491:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.603491:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677448.603492:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.603494:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677448.603494:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.603496:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.603497:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.603497:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:8.2:1619677448.603499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.603503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677448.603509:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677448.603509:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:21.2:1619677448.603511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:2.0:1619677448.603511:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.603511:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.603512:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677448.603513:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.603514:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.603516:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677448.603516:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.603516:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.603518:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677448.603519:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.603520:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.603522:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.603524:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.603526:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.603527:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.603528:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.603529:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.603530:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.603534:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.603536:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.603539:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.603540:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.603540:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:5.2:1619677448.603541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:22.0:1619677448.603542:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.603543:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.603543:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.603544:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677448.603547:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:25.2:1619677448.603548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677448.603550:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.603551:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.603553:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.603554:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:18.0:1619677448.603555:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.603557:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:18.0:1619677448.603557:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:10.0:1619677448.603557:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.603558:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.603558:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.603560:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.603562:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.603562:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.603563:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.603565:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:18.0:1619677448.603567:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.603568:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:21.2:1619677448.603570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677448.603571:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.603572:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.603572:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.603574:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.603574:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.603575:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.603576:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:31.0:1619677448.603578:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.2:1619677448.603580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.603581:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.603583:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.603584:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.603585:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.603586:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.603588:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.603588:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:31.0:1619677448.603589:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.603590:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.603590:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:31.0:1619677448.603591:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.603591:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.603592:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.603594:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.603596:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.603597:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677448.603599:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.603601:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.603602:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.603603:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.603604:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.603604:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.603605:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.603606:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.603611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:4.2:1619677448.603614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677448.603619:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:4.2:1619677448.603619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:12.0:1619677448.603621:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:10.0:1619677448.603621:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:12.0:1619677448.603622:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.603624:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:12.0:1619677448.603624:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.603624:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:13.0:1619677448.603626:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.603626:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.603627:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.0:1619677448.603627:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677448.603628:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.603629:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:8.0:1619677448.603629:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.603630:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:10.0:1619677448.603631:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.603632:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.603632:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.603633:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.603634:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.603635:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.603637:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:10.0:1619677448.603638:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.603638:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96712765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.603639:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.603640:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.603640:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.603641:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.603642:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.603643:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.603644:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677448.603644:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.603646:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.603646:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677448.603646:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.603647:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.603647:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.0:1619677448.603649:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.603649:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.603650:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:9.2:1619677448.603652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.603652:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.603652:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.603652:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.603653:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.603657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.603659:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.603662:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.603663:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.603663:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.603665:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:2.0:1619677448.603666:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.603668:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.603669:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.603669:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677448.603669:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.603671:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.603674:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:9.2:1619677448.603675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.603682:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.603685:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.603686:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.603686:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.603687:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.603688:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.604027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677448.604032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.604034:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.604037:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.604038:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677448.604038:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.604040:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.604041:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.604042:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.604043:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.604044:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677448.604044:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.604046:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.604046:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.604047:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.604048:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.604048:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.604049:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712763] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.604050:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677448.604051:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.604052:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.604054:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.604056:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.604059:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:1.0:1619677448.604059:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.604062:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.604062:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.604063:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.604064:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677448.604064:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.604065:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.604066:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677448.604066:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.604067:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.604068:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.604068:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677448.604068:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.604069:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.604070:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.604070:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.604072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:25.2:1619677448.604076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.604079:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.604083:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.604083:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:30.0:1619677448.604084:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:6.0:1619677448.604084:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.604086:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:21.2:1619677448.604086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:6.0:1619677448.604086:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.604087:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.604089:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.604090:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.604090:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677448.604092:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:6.0:1619677448.604092:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.604093:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.604093:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.604096:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.604099:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.604100:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.604101:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.604102:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.604104:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.604105:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.604108:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.604109:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.604112:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.604113:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.604113:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.604115:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.604116:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.604117:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.604117:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.604121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.604129:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.604132:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.604132:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.604134:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.604135:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.604137:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.604167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.604172:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.604174:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.604177:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.604177:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.604179:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.604180:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677448.604182:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.604183:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.604184:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.604185:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677448.604188:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.604190:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.604192:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.604193:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.604194:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.604195:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.604196:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.604197:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.604198:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.604199:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677448.604201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677448.604204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.604206:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.604209:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.604209:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677448.604209:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:20.0:1619677448.604210:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677448.604212:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.604213:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.604213:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.604215:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.604218:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.604417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.604424:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.604427:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.604428:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.604430:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.604433:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.604434:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.604436:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712760] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.604437:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.604438:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.604439:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.604442:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.604444:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.604447:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.604448:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.604449:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.604451:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.604451:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.604452:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.604454:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.604454:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.604457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.604461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.604467:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677448.604468:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:20.0:1619677448.604469:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677448.604470:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:20.0:1619677448.604470:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.604471:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677448.604471:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.604472:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.604473:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.604475:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.604493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.604499:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.604502:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.604503:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.604505:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.604507:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.604509:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.604510:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.604512:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.604513:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.604514:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.604517:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.604519:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.604521:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.604522:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.604522:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.604524:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.604524:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.604525:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.604525:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.604531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.604539:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.604541:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.604542:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.604543:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.604545:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.604547:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.604833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.604838:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.604841:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.604841:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.604843:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.604845:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.604846:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.604848:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.604849:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.604850:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.604851:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.604855:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.604856:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.604859:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.604860:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.604861:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.604863:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.604863:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.604863:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.604864:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.604868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.604875:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.604877:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.604878:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.604879:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.604881:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.604884:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677448.604915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.604921:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.604923:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.604925:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.604926:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.604928:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.604929:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677448.604930:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.604932:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.604933:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.604934:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677448.604937:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.604939:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.604941:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.604942:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.604943:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.604944:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.604945:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.604945:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.604946:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.604950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.604957:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.604959:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.604960:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.604962:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.604962:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.604964:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.604970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.604977:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.604981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:5.0:1619677448.604981:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.604982:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.604983:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.604984:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.604985:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:2.0:1619677448.604987:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.604990:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.604991:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.604992:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.604993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:2.0:1619677448.604995:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.604997:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.604998:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.604999:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.605000:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.605001:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.2:1619677448.605001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:2.0:1619677448.605001:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.605004:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:2.0:1619677448.605006:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.605007:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.605008:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.605008:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.605009:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.605010:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.605011:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.605011:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:2.0:1619677448.605012:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.605013:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:2.0:1619677448.605013:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:31.0:1619677448.605014:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.605014:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.605014:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:31.0:1619677448.605016:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.605016:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.605016:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:31.0:1619677448.605017:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.605017:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.605017:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.605017:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:31.0:1619677448.605018:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:26.0:1619677448.605018:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.605020:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96712754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.605021:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.605021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:31.0:1619677448.605022:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.605022:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.605024:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:26.0:1619677448.605024:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.605027:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.605028:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:10.0:1619677448.605028:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:31.0:1619677448.605029:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:26.0:1619677448.605029:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.605030:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:10.0:1619677448.605030:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:31.0:1619677448.605031:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.605031:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.605031:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.605032:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.605032:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:10.0:1619677448.605033:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.605034:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.605035:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.605036:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.605036:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:9.2:1619677448.605036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:4.2:1619677448.605036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.605038:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.605038:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.605039:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.605039:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.605040:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.605041:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.605041:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.605042:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677448.605043:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:10.0:1619677448.605043:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.605044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.605045:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:25.2:1619677448.605046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000400:00000200:24.0:1619677448.605046:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.605046:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.605048:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.605048:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.605050:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96712753] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.605050:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.605051:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:10.0:1619677448.605051:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.605052:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:10.0:1619677448.605052:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.605053:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000400:00000200:10.0:1619677448.605053:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.605054:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.605055:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:17.0:1619677448.605055:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:26.0:1619677448.605056:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:1.0:1619677448.605056:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:26.0:1619677448.605057:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:17.0:1619677448.605057:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.605057:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.605058:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:17.0:1619677448.605059:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.605060:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.605062:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:10.0:1619677448.605064:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.605065:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.605065:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.605067:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.605067:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.605068:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.605069:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.605073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.2:1619677448.605078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677448.605081:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.605083:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:1.0:1619677448.605083:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:20.0:1619677448.605084:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.605086:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.605086:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.605087:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.605087:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.605089:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:1.0:1619677448.605089:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.605091:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.605093:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.605094:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.605096:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.605097:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.605098:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.605102:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.605103:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.605106:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.605106:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.605107:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.605108:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.605109:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.605110:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.605111:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.605112:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.605115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.605117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.605122:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.605122:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.605124:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677448.605125:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677448.605125:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677448.605126:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.605126:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.605126:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.605128:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.605131:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.605330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.605339:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.605342:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.605344:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.605346:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.605347:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.605349:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.605350:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.605351:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.605352:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.605353:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.605357:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.605358:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.605362:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.605362:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.605363:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.605365:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.605366:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.605367:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.605367:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.605372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.605380:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.605383:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.605384:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.605386:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.605386:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.605389:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.605493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.605499:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.605501:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.605502:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.605504:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.605505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677448.605506:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.605507:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.605509:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.605510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:1.0:1619677448.605510:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.605511:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.605512:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.605513:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.605513:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.605515:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:1.0:1619677448.605515:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.605516:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.605517:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677448.605517:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.605518:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.605519:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.605519:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.605520:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:26.0:1619677448.605521:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.605521:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.605521:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.605522:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:20.0:1619677448.605523:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677448.605523:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.605524:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.605524:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.605524:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677448.605525:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.605525:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.605527:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.605527:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.605528:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.605529:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.605529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:20.0:1619677448.605529:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.605530:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.605531:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.605534:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.605536:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.605536:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.605537:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:26.0:1619677448.605538:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.605540:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.605540:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:26.0:1619677448.605541:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.605541:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:20.0:1619677448.605541:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:6.0:1619677448.605541:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.605542:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.605542:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.605542:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:6.0:1619677448.605543:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.605544:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.605544:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.605545:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.605545:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.605545:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:26.0:1619677448.605546:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677448.605546:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677448.605547:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677448.605547:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.605550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:25.2:1619677448.605553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.605556:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.605558:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.605559:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.605561:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.605561:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.605562:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:13.0:1619677448.605563:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677448.605564:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:13.0:1619677448.605564:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.605566:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.605568:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.605569:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.605806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.605813:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.605816:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.605817:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.605819:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.605822:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.605823:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.605824:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712747] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.605825:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.605826:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.605827:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.605831:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.605832:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.605835:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.605836:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.605837:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.605839:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.605840:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.605840:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.605841:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.605845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.605853:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.605856:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.605857:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.605857:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.605859:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.605863:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.605870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.605875:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.605878:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.605879:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.605881:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.605882:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.605884:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.605885:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.605886:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.605887:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.605888:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.605890:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.605892:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.605894:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.605895:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.605895:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.605897:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.605897:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.605898:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.605898:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.605903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.605909:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.605911:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.605912:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.605913:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.605913:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.605914:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.605971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.605976:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.605979:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.605979:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.605981:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.605982:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.605983:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.605984:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.605986:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.605987:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.605987:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.605990:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.605991:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.605992:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.605993:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.605993:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.605995:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.605995:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.605996:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.605996:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.606000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.606008:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.606010:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.606010:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.606011:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.606012:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.606013:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.606036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.606041:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.606044:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.606045:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.606047:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.606050:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.606051:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.606052:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.606054:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.606055:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.606056:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.606059:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.606061:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.606064:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.606064:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.606065:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.606067:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.606068:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.606069:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.606070:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.606071:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.606073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.606076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.606081:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677448.606081:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:17.0:1619677448.606083:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:6.0:1619677448.606083:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:17.0:1619677448.606084:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.606084:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677448.606084:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.606085:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:17.0:1619677448.606087:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.606090:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.606097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.606104:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.606107:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.606108:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.606110:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.606113:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.606114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:26.0:1619677448.606115:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.606116:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.606117:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.606118:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.606119:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.606120:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:26.0:1619677448.606123:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.606123:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.606123:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:26.0:1619677448.606126:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.606126:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.606128:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.606129:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:26.0:1619677448.606130:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.606131:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:2.0:1619677448.606131:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.606132:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:2.0:1619677448.606132:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.606133:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:2.0:1619677448.606133:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.606134:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.606134:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:2.0:1619677448.606134:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:26.0:1619677448.606135:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:2.0:1619677448.606137:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:8.2:1619677448.606139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.606139:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.606141:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.606142:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.606142:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.606144:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.606144:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.606145:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.606145:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.2:1619677448.606149:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:9.2:1619677448.606150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677448.606151:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.606154:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.606155:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.606155:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.606157:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.606157:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:5.0:1619677448.606157:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.606157:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.606158:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.606159:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.606160:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.606160:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:1.0:1619677448.606160:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.606161:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.606162:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000400:00000200:1.0:1619677448.606162:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.606164:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:1.0:1619677448.606164:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.606165:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.606166:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.606167:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.606168:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.606171:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.606173:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.606176:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.606177:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.606177:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.606179:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.606179:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.606180:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.606181:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.606185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.606192:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.606194:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.606195:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.606197:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.606198:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.606200:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.606201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:13.0:1619677448.606208:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.606210:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.606211:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.606212:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.606212:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.606213:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.606230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.606237:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.606240:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.606242:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.606244:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.606244:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.606246:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677448.606247:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.606248:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.606249:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.606250:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677448.606254:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.606256:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677448.606258:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.606259:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:30.0:1619677448.606259:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:30.0:1619677448.606261:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:30.0:1619677448.606261:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.606262:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.606263:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.606267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.606273:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.606275:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.606276:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.606277:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.606278:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.606280:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.2:1619677448.606297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.606305:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.606307:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.606308:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.606310:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.606313:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.606314:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.606315:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.606317:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.606318:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.606318:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.606321:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.606323:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.606324:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.606325:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.606325:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.606327:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.606327:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.606328:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.606329:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.606333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.606339:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.606341:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.606342:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.606343:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.606344:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.606345:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.606392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.606395:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.606398:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.606400:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.606401:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.606403:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.606405:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.606406:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.606407:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.606408:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.606409:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.606413:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.606415:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.606417:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.606418:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.606418:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.606420:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.606421:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.606422:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.606422:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.606427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.606435:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.606437:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.606438:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.606439:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.606440:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.606443:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.606444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.2:1619677448.606446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:5.0:1619677448.606451:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:18.0:1619677448.606452:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.606454:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:18.0:1619677448.606455:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.606456:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.606457:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:18.0:1619677448.606457:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677448.606457:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.606458:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.606458:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:5.0:1619677448.606459:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.606460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:5.0:1619677448.606461:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677448.606462:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712737] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.606463:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.606464:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.606465:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.606468:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677448.606469:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.606470:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.606471:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.606471:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.606473:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.606474:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.606475:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.606475:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:10.0:1619677448.606476:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:5.0:1619677448.606476:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:10.0:1619677448.606477:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.606477:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.606478:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000400:00000200:10.0:1619677448.606479:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.606479:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:5.0:1619677448.606479:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000400:00000200:10.0:1619677448.606480:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.606481:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:25.2:1619677448.606483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:10.0:1619677448.606484:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.606486:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.606489:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.606490:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.606490:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.606491:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.606492:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.606493:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.606493:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:26.0:1619677448.606494:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.606494:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.606495:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.606496:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.606497:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.606498:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.606499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.606507:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.606510:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.606510:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.606511:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.606513:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.606516:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.606558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.606564:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.606567:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.606568:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.606569:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.606571:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.606573:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677448.606574:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.606575:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.606576:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.606577:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.606580:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.606582:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.606585:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.606586:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677448.606586:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677448.606588:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677448.606588:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.606589:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.606590:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.606594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.606602:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.606604:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.606605:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.606607:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.606608:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.606610:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.606659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.606667:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.606670:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.606672:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.606675:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.606675:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.606676:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.606678:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.606679:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.606680:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.606681:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.606684:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.606686:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.2:1619677448.606686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:31.0:1619677448.606689:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.606690:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.606691:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.606692:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677448.606693:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.606693:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.606694:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.606695:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:18.0:1619677448.606695:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.606696:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.606698:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.606700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:18.0:1619677448.606701:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.606702:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.606704:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.606705:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.606706:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.606707:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.606708:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.606710:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:18.0:1619677448.606710:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.606711:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.606712:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:20.0:1619677448.606713:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.606714:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.606714:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677448.606715:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.606716:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677448.606716:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.606718:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.606718:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.606719:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.606720:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.606724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.606731:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.606734:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.606735:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.606737:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.606737:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.606739:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.606743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.2:1619677448.606744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.606750:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.606751:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.606754:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.606754:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.606754:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.606755:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.606757:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000400:00000200:17.0:1619677448.606757:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.606757:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:4.2:1619677448.606757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.0:1619677448.606758:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:17.0:1619677448.606759:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.606761:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.606762:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.606763:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.606764:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.606765:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:17.0:1619677448.606765:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677448.606768:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:17.0:1619677448.606769:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.606769:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677448.606770:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:24.0:1619677448.606772:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.606773:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.606774:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000400:00000200:24.0:1619677448.606775:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.606775:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:24.0:1619677448.606776:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.606777:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96712731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.606777:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.606777:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.606778:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677448.606779:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.606779:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677448.606780:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.606781:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:9.2:1619677448.606782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:24.0:1619677448.606784:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.606786:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.606789:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.606790:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.606790:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.606791:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.606791:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677448.606792:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.606793:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.606794:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:8.0:1619677448.606794:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.606795:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.606796:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677448.606797:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.606797:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:8.2:1619677448.606799:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.606807:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.606809:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.606810:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.606811:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.606813:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.606816:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.606818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.606825:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.606827:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.606828:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.606828:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.606829:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.606830:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.606851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.606859:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.606861:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.606862:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.606864:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.606867:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.606868:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.606869:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.606870:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.606871:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.606872:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.606874:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.606876:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.606877:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.606878:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.606879:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.606880:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.606880:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.606881:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.606883:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.606883:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.606885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.2:1619677448.606889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.606894:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.606895:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.606895:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.606897:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.606898:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.606899:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.606899:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.606900:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.606900:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.606901:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677448.607008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:4.2:1619677448.607012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:30.0:1619677448.607016:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677448.607018:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000800:00000200:24.0:1619677448.607019:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000400:00000200:30.0:1619677448.607020:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.607022:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:24.0:1619677448.607022:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000400:00000200:30.0:1619677448.607023:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.607023:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:30.0:1619677448.607025:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677448.607025:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.607026:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.607027:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.607028:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.607028:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.607029:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677448.607029:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:24.0:1619677448.607030:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.607032:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.607033:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.607033:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.607034:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:30.0:1619677448.607035:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:24.0:1619677448.607037:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.607039:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.607039:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677448.607039:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677448.607040:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677448.607041:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.607041:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:30.0:1619677448.607042:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.607042:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:24.0:1619677448.607042:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:30.0:1619677448.607043:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:24.0:1619677448.607043:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:30.0:1619677448.607044:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:24.0:1619677448.607044:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677448.607044:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677448.607045:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:20.2:1619677448.607045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:25.2:1619677448.607048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:9.2:1619677448.607049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:31.0:1619677448.607053:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.607055:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:8.0:1619677448.607055:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:6.0:1619677448.607056:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:31.0:1619677448.607057:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.607057:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677448.607058:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.607059:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:6.0:1619677448.607059:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:31.0:1619677448.607060:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.607060:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:22.0:1619677448.607060:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:8.0:1619677448.607060:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.607061:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.607062:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:31.0:1619677448.607063:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:6.0:1619677448.607063:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.607064:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.607066:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.607067:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.607067:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:6.0:1619677448.607069:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:31.0:1619677448.607071:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.607073:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.607076:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.607077:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.607077:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.607079:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.607080:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.607081:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.607082:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.607083:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:21.2:1619677448.607085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.2:1619677448.607085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.607089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:24.0:1619677448.607093:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677448.607094:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:24.0:1619677448.607095:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.607096:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:30.0:1619677448.607097:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:24.0:1619677448.607097:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.607097:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.607098:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.607099:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.607100:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:24.0:1619677448.607100:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.607100:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.607100:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.607101:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:24.0:1619677448.607101:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:5.0:1619677448.607102:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:24.0:1619677448.607103:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.607104:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.607105:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.607106:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.607109:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.607111:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.607113:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.607114:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.607115:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.607116:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.607116:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.607117:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.607118:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.607122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.607128:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.607131:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.607132:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.607133:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.607135:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.607136:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.607285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.607287:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.607290:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.607292:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.607293:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.607300:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.607302:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677448.607303:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.607305:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.607306:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.607307:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.607310:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.607311:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.607314:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.607315:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.607315:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.607317:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.607317:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.607318:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.607318:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677448.607324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.607327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677448.607332:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.607335:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:30.0:1619677448.607336:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.607336:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.607336:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.607338:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.607339:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.607341:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.607341:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677448.607343:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.607344:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.607346:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.607347:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.607349:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.607350:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.607351:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.607354:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.607356:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.607358:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.607359:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.607360:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.607361:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.607362:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.607363:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.607365:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.607365:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.607368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.2:1619677448.607368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:21.2:1619677448.607371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.607375:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:31.0:1619677448.607376:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.607378:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:10.0:1619677448.607378:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.607379:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.2:1619677448.607379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:10.0:1619677448.607379:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.607380:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.607380:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.607380:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.607381:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677448.607381:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.607382:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.607382:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.607383:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.607384:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:31.0:1619677448.607385:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:25.2:1619677448.607385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677448.607385:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.607386:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.607387:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:31.0:1619677448.607388:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.607389:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.607390:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.607390:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.607390:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.607391:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.607394:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.607394:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:31.0:1619677448.607395:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.607395:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.607396:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.607396:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677448.607397:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:5.0:1619677448.607398:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.607400:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.607400:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:31.0:1619677448.607401:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:5.0:1619677448.607401:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712722] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.607402:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.607403:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.607403:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.607404:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:5.0:1619677448.607404:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.607405:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.607405:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:5.0:1619677448.607405:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.2:1619677448.607409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:5.0:1619677448.607409:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.607410:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.607413:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.607413:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.607414:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.607416:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.607417:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.607417:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.607418:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.607419:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.607420:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.607420:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.607420:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.607421:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.607422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.607422:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.2:1619677448.607425:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.607428:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677448.607429:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.607430:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.607431:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677448.607431:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677448.607432:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677448.607434:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.607435:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.607435:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.607437:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.607439:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.607526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.607532:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.607534:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.607535:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.607537:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.607540:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.607542:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.607543:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.607544:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.607545:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.607546:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.607550:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.607552:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.607554:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.607555:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.607556:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.607558:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.607558:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.607560:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.607561:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.607561:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.607564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.2:1619677448.607566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.607570:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.607572:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677448.607572:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677448.607573:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.607573:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:8.0:1619677448.607574:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.607574:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.607575:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.607576:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.607578:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.2:1619677448.607579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:9.2:1619677448.607580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677448.607586:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:10.0:1619677448.607586:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677448.607589:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:10.0:1619677448.607589:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.607590:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677448.607590:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.607590:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.607591:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.607592:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:10.0:1619677448.607592:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.607594:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.607595:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.607596:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.607597:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.607598:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.607599:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.607602:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.607604:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.607606:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.607607:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.607607:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.607609:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.607609:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.607610:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.607611:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.607615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.607620:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.607622:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.607623:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.607624:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.607625:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.607626:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.607686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.607694:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.607696:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.607698:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.607700:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.607701:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.607703:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.607704:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.607705:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.607706:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.607707:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.607710:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.607711:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.607714:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.607714:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.607715:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.607716:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.607717:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.607718:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.607718:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.607722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:21.2:1619677448.607723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.607729:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677448.607730:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:22.0:1619677448.607732:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:13.0:1619677448.607732:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.607733:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.607733:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.607734:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.607735:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.607737:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.607737:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.607737:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:22.0:1619677448.607739:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.607740:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.607741:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.607742:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677448.607742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:22.0:1619677448.607743:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.607747:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.607748:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.607749:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:10.0:1619677448.607751:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.607752:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677448.607752:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.607752:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:21.2:1619677448.607752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:22.0:1619677448.607753:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:10.0:1619677448.607754:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.607755:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.607755:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.607756:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.607756:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677448.607756:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.607757:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.607759:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.607759:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:10.0:1619677448.607760:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.607761:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677448.607761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:24.0:1619677448.607762:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:10.0:1619677448.607762:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:8.0:1619677448.607762:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677448.607765:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.607766:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.607767:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.607768:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:10.0:1619677448.607768:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.607769:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677448.607769:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:10.0:1619677448.607770:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:8.0:1619677448.607770:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96712716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.607771:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:10.0:1619677448.607771:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:8.0:1619677448.607771:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.607772:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.607772:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.607772:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.607772:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:10.0:1619677448.607773:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:8.0:1619677448.607773:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.607774:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.607774:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000400:00000200:8.0:1619677448.607774:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.607775:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.607777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:8.0:1619677448.607777:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.607779:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:4.2:1619677448.607780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.607781:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:8.0:1619677448.607783:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.607783:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.607784:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.607785:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:8.0:1619677448.607786:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.607787:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:8.0:1619677448.607787:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.607787:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:26.0:1619677448.607788:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.607788:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:10.0:1619677448.607789:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:26.0:1619677448.607790:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.607790:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.607791:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.607791:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.607792:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:9.2:1619677448.607792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.607793:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677448.607800:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.607802:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.607803:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:4.2:1619677448.607804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:2.0:1619677448.607804:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.607806:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.607808:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677448.607811:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.607814:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.607814:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.607816:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.607819:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.607820:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.607821:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.607823:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.607824:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.607824:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.607827:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.607828:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.607831:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.607832:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.607832:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.607834:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.607834:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.607835:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.607836:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.607840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.607848:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.607851:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.607851:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.607852:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.607854:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.607857:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.607936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.607941:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.607943:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.607944:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.607946:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.607948:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.607949:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.607950:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.607952:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.607952:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.607953:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.607956:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.607958:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.607959:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.607960:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.607961:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.607962:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.607963:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.607963:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.607964:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.607968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.607975:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.607977:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.607977:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.607978:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.607979:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.607980:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.607989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.2:1619677448.607989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.607996:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677448.607997:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.607998:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.607999:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677448.608000:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:1.0:1619677448.608000:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.608000:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000400:00000200:31.0:1619677448.608001:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.608001:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677448.608003:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.608004:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.608006:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.608007:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.608008:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.608009:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.608010:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.608013:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.608015:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.608018:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.608019:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.608020:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.608021:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.608022:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.608023:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.608024:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.608028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.608035:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.608038:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.608039:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.608039:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.608041:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.608044:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.608048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.608055:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.608058:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.608058:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.608059:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.608060:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.608060:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.608129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.608137:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.608140:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.608142:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.608143:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.608145:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.608146:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.608147:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712712] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.608149:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.608150:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.608150:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.608154:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.608155:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.608157:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.608158:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.608158:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.608160:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.608160:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.608161:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.608162:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.608165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.608172:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.608174:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.608175:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.608176:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.608178:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.608179:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.608330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.608337:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.608340:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.608341:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.608342:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.608344:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.608345:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.608346:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.608348:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.608349:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.608349:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.608352:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.608354:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.608355:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.608355:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.608356:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.608357:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.608358:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.608358:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.608359:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.608363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.608369:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.608371:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.608371:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.608372:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.608373:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.608374:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.608515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.608521:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.608524:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.608524:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.608526:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.608528:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.608529:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.608530:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.608531:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677448.608531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:17.0:1619677448.608532:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.608533:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.608536:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.608537:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.608539:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.608539:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.608540:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.608540:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.608542:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.608542:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.608542:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.608543:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677448.608543:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.608544:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.608545:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.608548:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:8.0:1619677448.608552:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.608553:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.608554:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96712709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.608555:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:8.0:1619677448.608556:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.608557:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.608557:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:8.0:1619677448.608558:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.608558:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.608559:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.608561:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.608563:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.608565:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.608565:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.0:1619677448.608568:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.608568:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.608569:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.608571:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.608571:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.608573:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.608574:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.608574:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.608576:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.608580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.608585:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.608587:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.608587:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.608588:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.608588:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.608589:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.608590:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.608590:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.608591:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677448.608592:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.608604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.608610:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.608613:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.608614:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.608616:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.608618:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.608620:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.608621:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.608622:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.608624:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.608625:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.608628:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.608630:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.608632:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.608633:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.608634:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.608635:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.608636:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.608636:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.608637:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.608642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.608650:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.608653:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.608653:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.608655:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.608656:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.608658:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.608664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.608672:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.608674:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.608675:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.608677:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.608679:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.608680:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.608682:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.608683:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.608684:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.608685:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.608688:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.608690:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.608693:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.608693:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.608694:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.608696:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.608696:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.608697:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.608697:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.608702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.608710:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.608713:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.608714:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.608714:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.608716:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.608719:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.608725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:6.0:1619677448.608732:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.608735:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.608736:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.608736:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.608737:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.608738:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.608787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.608795:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.608798:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.608800:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.608802:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.608802:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.608804:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.608805:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.608806:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.608807:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.608808:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.608811:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.608813:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.608816:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.608817:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.608817:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.608819:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.608819:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.608820:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.608821:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.608825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.608833:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.608836:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.608837:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.608837:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.608839:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.608842:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.608844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.608851:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.608854:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.608855:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.608855:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.608856:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.608856:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.608913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.608918:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.608921:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.608922:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.608924:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.608926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:18.0:1619677448.608926:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.608927:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.608929:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.608930:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.608932:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.608932:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:18.0:1619677448.608933:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.608933:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677448.608935:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.608937:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.608937:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.608939:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.608939:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677448.608940:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.608942:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.608942:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.608943:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:18.0:1619677448.608947:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.608947:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.608948:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.608949:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.608949:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677448.608951:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.608951:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.608952:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.608953:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.608953:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677448.608954:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.608954:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.608955:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.608956:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.608956:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.2:1619677448.608957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.608957:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.608957:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.608959:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.608959:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.2:1619677448.608960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:8.2:1619677448.608962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.608964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:1.0:1619677448.608964:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.608967:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677448.608968:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:17.0:1619677448.608968:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.608968:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.608969:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:18.0:1619677448.608970:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:1.0:1619677448.608970:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.608971:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:18.0:1619677448.608971:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.608971:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.608972:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677448.608972:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:1.0:1619677448.608972:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:20.0:1619677448.608973:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.608973:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677448.608973:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:20.0:1619677448.608975:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.608975:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.608977:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.608979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.2:1619677448.608983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:17.0:1619677448.608986:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.608989:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.608990:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677448.608990:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.608990:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.608991:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.608992:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.608992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677448.608993:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.608994:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.608994:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.608995:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.608996:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:26.0:1619677448.608999:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.609002:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.609003:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.609005:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.609008:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.609009:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.609010:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.609012:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.609013:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.609014:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.609017:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.609019:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.609021:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.609022:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.609023:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.609025:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.609025:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.609026:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.609026:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.609031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.609039:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.609041:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.609042:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.609043:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.609045:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.609047:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.609095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.609098:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.609101:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.609103:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.609104:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.609106:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.609107:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.609108:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712702] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.609110:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.609111:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.609112:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.609115:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.609117:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.609120:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.609121:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.609121:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.609123:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.609124:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.609125:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.609125:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.609129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.609136:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.609139:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.609140:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.609141:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.609142:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.609145:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.609192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.609198:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.609201:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.609202:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.609204:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.609206:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.609207:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677448.609209:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.609210:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.609211:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.609212:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677448.609215:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.609216:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677448.609219:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.609220:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677448.609221:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677448.609223:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677448.609223:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.609224:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.609224:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.609228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.609235:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.609237:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.609238:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.609239:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.609241:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:10.0:1619677448.609244:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677448.609250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.609255:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.609258:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.609259:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.609261:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.609263:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.609264:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.609266:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.609267:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.609268:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.609269:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.609272:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.609274:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.609277:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.609278:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.609279:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.609281:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.609281:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.609281:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.609282:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.609286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.609292:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.609299:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.609299:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.609301:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.609303:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.609304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.609308:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.609309:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.609311:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.609311:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.609312:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.609313:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.609314:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:4.2:1619677448.609337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.609345:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.609347:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.609348:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.609350:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.609352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677448.609353:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.609354:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.609355:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.609357:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.609358:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.609359:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.609359:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.609362:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.609362:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:10.0:1619677448.609363:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.609364:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.609365:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.609365:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.609367:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:10.0:1619677448.609368:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.609368:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.609369:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.609369:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677448.609370:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.609371:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:1.0:1619677448.609371:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.609372:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677448.609372:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.609373:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.609374:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.609375:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677448.609375:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.609377:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:9.2:1619677448.609378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.609379:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677448.609380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.609380:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.609381:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.609383:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.609383:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.609384:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.609384:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677448.609386:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.609387:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:18.0:1619677448.609388:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.609389:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:25.2:1619677448.609389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677448.609389:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.609390:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677448.609390:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.609391:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:18.0:1619677448.609392:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.609395:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677448.609396:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.609399:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.609400:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.609400:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.609402:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.609404:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.609458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.609466:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.609468:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.609469:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.609471:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.609474:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.609476:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.609477:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.609479:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.609480:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.609481:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.609484:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.609486:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.609489:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.609490:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.609490:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.609492:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.609492:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.609493:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.609493:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.609498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.609505:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.609508:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.609509:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.609509:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.609511:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.609514:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.609706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.609713:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.609716:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.609717:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.609718:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.609722:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.609723:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.609725:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.609726:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.609727:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.609728:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.609730:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.609732:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.609735:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.609735:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.609736:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.609738:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.609738:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.609739:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.609739:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.609744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.609752:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.609754:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.609755:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.609755:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.609757:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.609760:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.609826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.609833:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.609836:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.609837:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.609838:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.609841:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.609843:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.609844:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.609845:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.609846:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.609847:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.609850:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.609851:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.609854:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.609855:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.609856:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.609857:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.609858:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.609858:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.609859:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.609863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.609872:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.609874:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.609875:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.609877:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.609877:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.609880:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.609937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.609939:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.609942:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.2:1619677448.609943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:20.0:1619677448.609944:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.609946:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.609947:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.609948:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:20.0:1619677448.609948:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677448.609949:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.609951:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:20.0:1619677448.609951:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.609952:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.609952:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677448.609953:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.609955:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.609957:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.609957:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.609959:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:20.0:1619677448.609959:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.609960:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.609961:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.609962:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.609962:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.609963:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:22.0:1619677448.609964:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.609964:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677448.609965:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000400:00000200:22.0:1619677448.609967:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.609968:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677448.609969:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.609969:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.609970:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677448.609971:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677448.609971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.609972:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.609973:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.609974:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.609975:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.609976:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.609977:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.609978:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677448.609979:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:8.2:1619677448.609980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.609981:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:21.2:1619677448.609983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677448.609983:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.609983:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.609984:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.609987:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677448.609988:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677448.609989:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:5.0:1619677448.609990:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:6.0:1619677448.609991:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:5.0:1619677448.609991:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.609992:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.609993:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:5.0:1619677448.609993:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.609994:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.609995:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.610023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.610029:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.610032:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.610033:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.610035:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.610037:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.610039:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.610040:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.610042:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.610042:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.610043:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.610046:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.610048:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.610051:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.610052:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.610052:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.610054:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.610055:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.610056:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.610056:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.610060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.610067:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.610069:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.610070:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.610071:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.610073:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.610076:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.610081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.610087:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.610089:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.610090:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.610090:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.610091:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.610091:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.610114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.610122:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.610125:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.610127:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.610129:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.610130:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.610131:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.610132:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.610133:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.610134:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.610135:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.610138:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.610140:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.610143:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.610144:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.610145:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.610147:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.610147:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.610149:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.610150:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.610151:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.610153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.610159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.610164:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677448.610167:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.610167:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.610168:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.610169:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.610170:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.610170:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.610171:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.610171:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.610173:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.610173:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.610174:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:20.0:1619677448.610175:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.610175:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.610175:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.610209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.610216:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.610219:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.610219:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.610221:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.610224:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.610226:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.610227:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.610229:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.610230:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.610231:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.610234:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.610235:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.610238:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.610239:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.610239:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.610241:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.610241:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.610242:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.610242:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.610247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.2:1619677448.610254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:6.0:1619677448.610254:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.2:1619677448.610254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677448.610257:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.610257:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.610258:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.610258:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.610260:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677448.610260:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:6.0:1619677448.610261:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.610263:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.610263:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677448.610263:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.610264:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.610264:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:20.0:1619677448.610266:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.610266:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.610267:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677448.610268:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.610268:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.610270:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:20.0:1619677448.610271:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.610271:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.610272:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.610273:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677448.610274:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.610275:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.610276:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:25.2:1619677448.610277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000400:00000200:20.0:1619677448.610277:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.610281:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:1.0:1619677448.610281:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.610283:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.610283:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:1.0:1619677448.610283:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.610284:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.0:1619677448.610284:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.610286:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.610286:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.610286:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.610286:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.610287:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677448.610287:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.610287:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677448.610287:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (23)++
00000800:00000200:6.0:1619677448.610288:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677448.610289:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.610290:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (23)++
00000800:00000200:1.0:1619677448.610290:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.610291:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:8.2:1619677448.610291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.610291:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677448.610292:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.610292:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.610293:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.610298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.610300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.610305:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677448.610306:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677448.610307:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.610308:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:5.0:1619677448.610308:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:17.0:1619677448.610309:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.610309:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:5.0:1619677448.610309:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.610310:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.610310:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.610311:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:20.0:1619677448.610311:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.2:1619677448.610311:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:6.0:1619677448.610312:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.610315:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:5.0:1619677448.610315:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.610317:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:2.0:1619677448.610317:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.610320:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.610321:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.610323:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.610326:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.610328:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.610329:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.610330:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.610332:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.610333:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.610336:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.610338:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.610342:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.610343:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.610343:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.610346:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.610346:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.610347:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:21.2:1619677448.610348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677448.610348:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.610352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.610356:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.610359:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:12.0:1619677448.610359:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:31.0:1619677448.610361:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.610362:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.610363:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.610364:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.610364:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:12.0:1619677448.610365:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.610366:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.610367:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.610367:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:31.0:1619677448.610368:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.610368:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677448.610370:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.610370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:31.0:1619677448.610371:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.610374:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.610376:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.610376:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.610379:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.610379:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.610380:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677448.610380:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.610380:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.610381:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (25)--
00000800:00000200:31.0:1619677448.610382:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677448.610382:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:31.0:1619677448.610383:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.610383:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.610384:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.610384:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.610387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.610393:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.610395:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.610396:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.610398:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.610399:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.610400:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.610451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.610459:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.610461:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.610462:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.610464:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.610467:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.610468:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.610469:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.610471:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.610471:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.610472:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.610475:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.610477:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.610480:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.610481:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.610481:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.610483:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.610484:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.610484:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.610485:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.610489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.610497:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.610499:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.610500:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.610502:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.610503:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.610505:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.610589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.610594:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.610597:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.610599:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.610599:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.610600:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.610602:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677448.610603:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.610605:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.610605:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.610607:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677448.610610:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.610612:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.610614:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.610615:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.610615:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.610617:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.610618:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.610619:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.610619:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.610623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.610630:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.610632:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.610633:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.610634:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.610636:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.610637:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.610640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.610648:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.610650:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.610651:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.610652:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.610652:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.610653:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.610760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.610767:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.610769:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.610770:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.610772:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.610775:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.610777:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.610778:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.610779:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.610781:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.610782:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.610785:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.610786:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:21.2:1619677448.610786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:26.0:1619677448.610789:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.610790:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.610791:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.610792:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.610792:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677448.610793:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.610794:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:18.0:1619677448.610794:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:26.0:1619677448.610795:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:2.0:1619677448.610795:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:26.0:1619677448.610796:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:18.0:1619677448.610798:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.610799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:18.0:1619677448.610801:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.610801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:18.0:1619677448.610802:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.610803:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712682] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.610805:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.610806:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:18.0:1619677448.610806:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.610807:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:22.0:1619677448.610809:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:10.0:1619677448.610809:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:22.0:1619677448.610810:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.610810:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.610810:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.610811:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.610812:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:18.0:1619677448.610812:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.610812:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677448.610813:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.610815:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677448.610815:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.610816:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.610817:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.610818:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.610819:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.610820:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.610820:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.610824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.610830:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.610833:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.610834:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.610834:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.610837:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.610839:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.610921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677448.610927:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677448.610930:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677448.610930:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677448.610932:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.610935:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.610936:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677448.610938:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.610939:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.610940:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.610941:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:21.2:1619677448.610942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:1.0:1619677448.610944:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.610945:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677448.610948:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.610949:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.610949:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:1.0:1619677448.610949:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.610951:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.610952:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:1.0:1619677448.610952:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677448.610953:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000400:00000200:30.0:1619677448.610954:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.610954:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677448.610954:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:31.0:1619677448.610956:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.610956:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.610957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:30.0:1619677448.610958:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.610959:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.610959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:30.0:1619677448.610960:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.610961:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.610962:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.610965:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:5.0:1619677448.610965:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:30.0:1619677448.610966:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.610967:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.610967:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:5.0:1619677448.610967:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:17.0:1619677448.610968:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:5.0:1619677448.610968:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.610969:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:30.0:1619677448.610970:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.610970:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.610970:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.610971:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.610972:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.610972:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:30.0:1619677448.610973:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.610974:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.610976:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.610977:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.610978:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.610979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.610983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.610986:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.610988:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.610989:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.610990:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.610991:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677448.610992:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.610993:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:13.0:1619677448.610993:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.610994:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.610995:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677448.611033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.611038:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.611041:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.611042:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.611044:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.611047:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.611049:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.611050:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.611051:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.611052:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.611053:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.611056:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.611058:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.611061:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.611062:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.611062:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.611064:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.611064:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.611065:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.611065:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.611070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.611077:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.611080:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.611081:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.611083:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.611083:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.611085:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.611088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.611093:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.611096:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.611098:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.611098:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.611100:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.611102:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.611103:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.611104:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.611105:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.611106:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.611109:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.611111:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.611113:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.611114:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.611115:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.611117:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.611117:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.611118:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.611119:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.611120:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.611123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.611125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677448.611128:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:6.0:1619677448.611130:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677448.611132:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:6.0:1619677448.611133:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.611134:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.611134:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:6.0:1619677448.611134:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.611135:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677448.611135:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.611136:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:6.0:1619677448.611136:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.611138:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:6.0:1619677448.611138:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677448.611139:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.611141:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.611144:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.611145:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.611147:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.611148:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.611149:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.611150:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.611153:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.611155:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.611158:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.611158:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.611159:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.611161:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.611162:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.611162:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.611163:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.611167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.611174:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.611177:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.611178:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.611179:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.611181:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.611183:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.611316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.611322:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.611325:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.611326:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.611328:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.611330:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.611332:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.611333:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.611334:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.611335:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.611336:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.611340:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.611341:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.611344:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.611344:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.611345:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.611346:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.611347:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.611347:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.611348:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.611353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.611361:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.611364:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.611364:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.611365:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.611367:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.611367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677448.611369:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:10.0:1619677448.611376:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.611378:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.611379:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.611381:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.611384:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.611385:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.611386:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.611388:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.611389:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.611390:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.611393:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.611395:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.611398:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.611398:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.611399:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.611401:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.611402:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.611402:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.611403:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.611407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.611416:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.611418:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.611420:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.611420:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.611422:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.611425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.611428:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.611429:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.611431:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.611432:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.611432:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.611433:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.611433:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.611475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677448.611483:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677448.611486:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677448.611488:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.611490:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677448.611490:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.611492:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:12.0:1619677448.611493:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.611495:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.611496:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677448.611497:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:12.0:1619677448.611500:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.611502:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677448.611504:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.611505:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:12.0:1619677448.611506:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:12.0:1619677448.611508:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:12.0:1619677448.611509:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677448.611509:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677448.611510:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677448.611515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.611522:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.611525:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.611526:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.611526:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.611528:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.611531:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.611555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.611561:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.611564:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.611564:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.611566:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.611568:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.611570:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.611571:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.611573:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.611574:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.611575:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.611578:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.611580:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.611583:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.611583:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.611584:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.611586:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.611586:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.611587:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.611587:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677448.611589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677448.611592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.611595:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.611598:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677448.611599:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677448.611600:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.611600:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.611601:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:5.0:1619677448.611602:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.611602:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.611603:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:5.0:1619677448.611604:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:1.0:1619677448.611605:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.611606:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.611607:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.611608:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.611608:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:5.0:1619677448.611609:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677448.611613:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.611615:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677448.611617:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.611618:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677448.611618:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677448.611620:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677448.611621:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.611622:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.611622:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.611626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.611634:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.611636:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.611638:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.611638:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.611640:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.611642:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.611651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.611657:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.2:1619677448.611658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.611659:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.611661:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.611662:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:2.0:1619677448.611662:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.611664:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:18.0:1619677448.611665:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.611666:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.611667:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.611667:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.611669:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.611670:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.611671:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.611675:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.611677:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.611677:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.611678:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:18.0:1619677448.611679:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.611679:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.611680:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.611681:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677448.611681:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.611682:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.611682:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.611683:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.611686:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.611687:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.611687:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:9.2:1619677448.611688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.611689:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.611690:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.611690:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.611691:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.611691:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.611691:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000800:00000200:18.0:1619677448.611693:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.611693:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (24)--
00000800:00000200:18.0:1619677448.611694:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (24)++
00000800:00000200:2.0:1619677448.611694:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.611695:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000400:00000200:18.0:1619677448.611695:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.611695:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:21.2:1619677448.611696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.611696:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:18.0:1619677448.611697:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.611697:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.611698:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.611699:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.611699:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:9.2:1619677448.611700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.611701:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.611702:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677448.611703:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.611704:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.611704:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.611706:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.611706:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.611707:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.611790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.611798:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.611801:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.611803:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.611805:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.611805:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.611807:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.611808:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.611809:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.611810:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.611811:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.611814:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.611816:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.611818:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.611819:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.611820:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.611821:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.611822:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.611823:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.611824:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.611827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.611834:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:4.2:1619677448.611835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677448.611836:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:5.2:1619677448.611836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:13.0:1619677448.611837:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.611839:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.611840:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.611841:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.611842:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.611842:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.611846:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:17.0:1619677448.611846:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:10.0:1619677448.611846:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677448.611846:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.611850:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.611850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:17.0:1619677448.611850:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.611853:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.611853:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.611854:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.611856:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677448.611856:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.611856:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:26.0:1619677448.611857:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.611858:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.611859:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.611859:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.611860:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.611860:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.611860:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:17.0:1619677448.611861:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.611861:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.611861:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:26.0:1619677448.611862:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:12.0:1619677448.611863:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:17.0:1619677448.611867:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.611868:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.611870:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.611871:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.611873:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.611874:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.611874:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.611875:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:17.0:1619677448.611875:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.611876:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.611877:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.611877:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.611878:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.611878:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.611878:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.611879:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.611879:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.611880:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.611882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:25.2:1619677448.611886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:13.0:1619677448.611889:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.611892:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.611893:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.611894:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:13.0:1619677448.611894:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.611895:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677448.611897:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:13.0:1619677448.611897:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677448.611897:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.611898:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.611900:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.611903:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.611939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.611946:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.611949:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.611950:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.611952:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.611955:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.611956:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.611957:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.611959:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.611960:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.611961:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.611964:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.611966:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.611969:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.611970:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.611971:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.611972:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.611973:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.611973:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.611974:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.611978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.611986:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.611988:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.611989:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.611990:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.611992:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.611994:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.612035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.612038:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.612041:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.612043:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.612044:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.612046:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.612047:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.612049:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.612050:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.612051:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.612052:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.612055:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.612057:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.612059:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.612060:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.612061:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.612063:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.612063:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.612064:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.612066:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.612066:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.612069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.612071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677448.612076:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.612077:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:30.0:1619677448.612078:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:5.0:1619677448.612079:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:30.0:1619677448.612080:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.612080:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:30.0:1619677448.612081:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.612081:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677448.612082:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.612084:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.612165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.612170:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.612173:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.612175:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.612176:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.612178:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.612179:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.612181:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.612182:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.612183:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.612184:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.612187:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.612189:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.612191:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.612192:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.612193:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.612195:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.612195:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.612196:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.612196:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.612201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.612208:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.612211:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.612212:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.612212:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.612215:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.612217:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.612242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.612250:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.612253:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.612255:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.612256:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.612257:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.612259:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.612260:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.612262:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.612263:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.612264:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.612267:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.612269:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.612272:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.612273:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.612273:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.612275:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.612276:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.612277:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.612277:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.612281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.612288:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.612290:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.612292:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.612294:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677448.612298:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.612299:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.612301:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:30.0:1619677448.612302:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.612305:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.612308:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.612310:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.612310:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.612312:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677448.612314:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.612315:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.612316:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.612317:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677448.612320:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.612321:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.612325:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.612325:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.612326:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.612328:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.612328:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.612329:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677448.612330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677448.612333:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677448.612334:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677448.612335:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:8.2:1619677448.612335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677448.612337:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.612338:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:5.2:1619677448.612338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677448.612339:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.612340:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677448.612341:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.612342:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.612343:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.612344:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.612345:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.612345:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.612346:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:30.0:1619677448.612347:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.612347:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.612347:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677448.612349:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.612350:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.612351:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.612353:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.612353:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.612354:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:17.0:1619677448.612354:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:1.0:1619677448.612354:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.612355:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.612355:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.612356:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:30.0:1619677448.612357:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677448.612358:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:1.0:1619677448.612358:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.612359:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000400:00000200:1.0:1619677448.612359:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.612360:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.612361:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000400:00000200:1.0:1619677448.612361:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:30.0:1619677448.612362:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677448.612364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677448.612364:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.612366:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.612368:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677448.612369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:1.0:1619677448.612369:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.612370:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:2.0:1619677448.612372:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677448.612372:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.612372:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.612373:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.612373:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677448.612375:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.612376:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.612376:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.612377:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:9.2:1619677448.612377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:2.0:1619677448.612378:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.612379:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677448.612380:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677448.612381:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:2.0:1619677448.612381:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.612384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.612384:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.612386:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.612387:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.612388:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.612390:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.612391:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.612393:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:18.0:1619677448.612394:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.612394:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.612395:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.612395:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.612396:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.612578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.612586:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.612589:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.612590:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.612591:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.612594:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.612596:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.612597:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.612598:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.612599:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.612600:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.612604:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.612605:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.612608:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.612608:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.612609:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.612611:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.612611:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.612612:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.612612:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.612616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.612624:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.612627:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.612627:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.612629:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.612631:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.612632:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.612757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.612765:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.612768:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.612770:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.612772:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.612773:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.612774:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.612775:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.612777:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.612777:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.612778:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.612781:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.612783:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.612785:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.612786:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.612786:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.612788:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.612788:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.612789:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.612789:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:4.2:1619677448.612790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:25.2:1619677448.612793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.612797:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.612799:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:30.0:1619677448.612800:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:10.0:1619677448.612800:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.612802:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:26.0:1619677448.612802:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.612803:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.612804:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.612805:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.612806:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.612806:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:26.0:1619677448.612806:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.612807:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.612809:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.612809:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.612810:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.612813:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.612815:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.612817:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.612817:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.612818:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.612819:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.612820:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.612821:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.612822:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.612823:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.612825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.612828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:4.2:1619677448.612829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.612831:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.612834:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.612834:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.612834:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.612836:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.612836:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:5.0:1619677448.612837:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:31.0:1619677448.612838:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.612838:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.612838:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.612839:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:5.0:1619677448.612839:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.612840:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677448.612840:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:5.0:1619677448.612841:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.612843:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.612844:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.612846:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.612846:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.612847:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.612851:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.612853:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.612855:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.612856:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.612857:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.612859:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.612859:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.612860:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.612861:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.612865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.612872:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.612875:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.612876:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.612876:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.612878:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.612880:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.612882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.612887:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.612890:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.612891:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.612893:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.612896:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.612897:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.612899:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.612900:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.612901:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.612902:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.612905:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.612907:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.612909:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.612910:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.612910:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.612912:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.612913:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.612914:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.612915:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.612916:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.612918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.612921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677448.612925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677448.612926:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.612927:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.612928:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:1.0:1619677448.612928:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.612929:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.612929:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.612930:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.612931:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.612931:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677448.612932:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.612932:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.612935:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.2:1619677448.612935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000400:00000200:30.0:1619677448.612937:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.612939:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.612939:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.612941:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.612942:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.612942:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677448.612943:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.612945:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.612945:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.612945:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.612945:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000400:00000200:30.0:1619677448.612946:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.612946:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.612947:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:30.0:1619677448.612949:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.612951:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.612955:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.612955:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.612956:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.612958:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.612958:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.612959:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.612960:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.612964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.612972:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.612975:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.612976:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.612977:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.612979:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.612981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:6.0:1619677448.612981:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677448.612987:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.612990:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.612991:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.612993:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.612994:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.612996:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677448.612997:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.612999:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.613000:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.613001:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677448.613004:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.613005:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.613009:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.613009:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:18.0:1619677448.613010:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677448.613012:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677448.613012:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.613013:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.613014:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.613018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.613024:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.613027:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.613028:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.613030:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.613031:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.613032:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.613034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.613039:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.613042:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.613043:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.613045:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.613047:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.613048:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.613050:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.613051:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.613052:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.613053:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.613056:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.613058:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.613060:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.613061:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.613062:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.613063:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.613064:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.613065:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.613065:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.613070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.613077:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.613079:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.613080:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.613080:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.613082:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.613085:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.613283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.613288:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.613291:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.613292:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.613298:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.613300:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.613301:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.613303:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.613304:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.613305:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.613306:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.613309:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.613311:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.613312:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.613313:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.613314:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.613315:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.613315:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.613316:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.613317:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.613322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.613329:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.613331:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.613332:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.613333:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.613333:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.613334:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:25.2:1619677448.613342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677448.613349:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.613352:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.613353:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.613354:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.613355:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.613356:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.613503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.613509:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.613512:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:5.2:1619677448.613513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677448.613513:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.613515:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.613518:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.613519:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.613520:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.613521:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:18.0:1619677448.613522:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.613523:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:18.0:1619677448.613523:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.613524:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.613524:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.613528:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.613529:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.613530:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.613531:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:18.0:1619677448.613531:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.613533:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.613534:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.613534:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.613535:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677448.613536:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.613537:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.613537:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:26.0:1619677448.613538:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.613538:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.613539:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.613540:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000400:00000200:26.0:1619677448.613541:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.613541:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:26.0:1619677448.613543:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:9.2:1619677448.613543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.613545:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.613545:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.613546:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.613546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677448.613548:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.613548:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.613549:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.613549:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677448.613549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677448.613551:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.613552:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:25.2:1619677448.613553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.613553:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:2.0:1619677448.613553:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.613554:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.613554:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.613555:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.613555:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.613557:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677448.613557:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677448.613557:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.613559:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:31.0:1619677448.613560:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:10.0:1619677448.613561:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:31.0:1619677448.613562:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.613562:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.613562:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.613564:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:10.0:1619677448.613564:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.613565:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.613566:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:10.0:1619677448.613566:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.613566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:31.0:1619677448.613567:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.613569:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.613570:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.613571:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.613573:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:31.0:1619677448.613574:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.613575:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:31.0:1619677448.613576:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.613576:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677448.613578:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677448.613578:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.613579:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.613580:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.613581:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:24.0:1619677448.613581:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.613582:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:24.0:1619677448.613582:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:31.0:1619677448.613583:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:24.0:1619677448.613583:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96712647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.613585:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.613585:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:24.0:1619677448.613585:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.613585:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.613586:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:25.2:1619677448.613587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:24.0:1619677448.613590:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.613591:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.2:1619677448.613591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:24.0:1619677448.613594:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.613595:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.613595:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677448.613595:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:24.0:1619677448.613597:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.613597:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677448.613597:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:30.0:1619677448.613598:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:24.0:1619677448.613598:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.613598:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:13.0:1619677448.613598:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.613599:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.613601:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:13.0:1619677448.613601:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.613602:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:12.0:1619677448.613602:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.613603:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.613603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:25.2:1619677448.613609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.613611:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.613614:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.613615:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.613615:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.613615:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.613617:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.613618:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.613619:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:13.0:1619677448.613619:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.613620:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.613620:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.613622:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.613707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.613714:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.613716:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.613718:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.613720:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.613721:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.613722:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.613723:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.613725:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.613726:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.613726:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.2:1619677448.613727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677448.613729:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.613731:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.613733:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.613734:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.613735:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.613735:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677448.613735:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.613736:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.613737:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.613737:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.613738:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:1.0:1619677448.613738:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.613739:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:1.0:1619677448.613739:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.613741:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.613742:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.613743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:1.0:1619677448.613743:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.613744:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.613745:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.613748:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.613749:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677448.613751:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:1.0:1619677448.613751:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.613752:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.613753:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677448.613754:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:1.0:1619677448.613754:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.613754:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.613755:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.613755:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.613755:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.613756:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.613757:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.613760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677448.613760:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.613767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:17.0:1619677448.613767:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.613770:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.613770:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.613771:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.613773:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.613774:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.613774:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:22.0:1619677448.613777:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.613778:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.613778:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.613779:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.613779:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.613828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.613833:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.613836:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.613836:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.613838:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.613840:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.613842:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.613843:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.613844:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.613845:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.613847:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.613850:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.613851:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.613854:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.613855:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.613856:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.613858:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.613858:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.613859:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.613859:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.613863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.613869:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.613872:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.613873:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.613873:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.613875:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.613879:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:4.2:1619677448.613897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.613903:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.613906:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.613907:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.613908:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.613910:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.613912:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.613913:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.613914:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.613915:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.613916:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.613920:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.613922:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.613924:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.613924:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.613925:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.613927:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.613927:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.613928:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.613928:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.613933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.613939:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.613942:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.613943:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.613944:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.613946:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.613947:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.614041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.614047:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.614050:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.614051:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.614052:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.614054:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.614055:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.614057:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.614058:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.614059:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.614060:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.614063:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.614064:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.614066:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.614066:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.614067:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.614068:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.614069:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.614070:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.614070:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.614075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.614081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.614084:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.614086:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.614087:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:2.0:1619677448.614087:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.614089:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677448.614089:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.614090:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.614091:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.614091:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.614092:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.614093:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.614094:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677448.614095:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.614097:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.614097:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.614098:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677448.614101:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.614102:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.614105:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.614105:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677448.614106:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677448.614107:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.614108:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.614108:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.614109:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.614114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.614121:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.614123:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.614124:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.614125:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.614127:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.614129:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.614137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:6.0:1619677448.614144:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.614147:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.614147:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.614148:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.614149:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.614149:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.614206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.614212:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.614214:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.614215:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.614217:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.614218:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.614220:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.614221:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.614223:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.614224:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.614225:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.614227:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.614229:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.614232:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.614233:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.614233:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.614235:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.614236:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.614236:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.614237:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.614241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.614248:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.614251:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.614252:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.614254:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.614255:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.614257:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.614283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.614290:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.614293:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.614293:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.614301:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.614303:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.614305:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.614306:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.614307:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.614308:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.614309:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.614313:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.614315:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.614317:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.614318:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.614318:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.614320:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.614320:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.614321:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.614321:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.614326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.614334:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.614336:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.614337:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.614338:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.614340:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.614341:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.614344:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.614345:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.614346:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.614346:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.614348:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.614395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.614403:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.614405:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.614407:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.614409:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.614410:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.614411:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677448.614412:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.614414:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.614415:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.614416:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677448.614419:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.614420:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677448.614423:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.614424:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.614424:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.614426:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.614426:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.614428:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.614429:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.614430:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677448.614431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:21.2:1619677448.614435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677448.614438:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.614441:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.614442:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.614442:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.614443:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677448.614444:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.614445:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.614446:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677448.614446:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:30.0:1619677448.614447:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677448.614521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.614529:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.614531:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.614532:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.614534:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.614536:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.614537:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.614538:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.614539:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.614540:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.614541:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.614544:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.614545:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.614547:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.614547:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.614548:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.614549:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.614550:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.614550:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.614551:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.614555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.614561:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.614563:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.614563:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.614564:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.614565:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.614566:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.614597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.2:1619677448.614603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:17.0:1619677448.614603:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.2:1619677448.614606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677448.614606:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.614606:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.614608:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.614609:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.614609:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.0:1619677448.614611:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:17.0:1619677448.614611:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.614612:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.614613:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:17.0:1619677448.614614:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.614615:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.614615:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.614616:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.614616:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.614617:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.614618:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.614618:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.614619:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.614619:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.614621:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.614621:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.614622:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.614623:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.614624:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.614624:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.614625:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.614626:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.614626:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.614627:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.614627:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677448.614628:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.614631:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.614632:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.614632:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.614633:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.614633:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.614634:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.614634:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.614636:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.614636:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:2.0:1619677448.614636:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677448.614637:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.614637:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.614639:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.614639:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677448.614640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.614640:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.614641:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.614641:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.614642:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:5.2:1619677448.614642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.614643:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.614644:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:2.0:1619677448.614644:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.0:1619677448.614645:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.614645:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.614645:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677448.614646:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.614647:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:12.0:1619677448.614647:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:1.0:1619677448.614647:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:25.2:1619677448.614648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.614648:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677448.614650:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:1.0:1619677448.614650:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:12.0:1619677448.614651:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.614651:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.614652:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677448.614653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:12.0:1619677448.614653:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.614655:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:10.0:1619677448.614655:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:13.0:1619677448.614656:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.614656:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677448.614657:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.614657:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.614657:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.614658:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:13.0:1619677448.614658:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.614658:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:10.0:1619677448.614659:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.614660:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:13.0:1619677448.614661:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677448.614661:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.614661:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.614662:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:10.0:1619677448.614664:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.614666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:25.2:1619677448.614669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.614672:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.614674:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.614675:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:12.0:1619677448.614675:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.614675:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.614677:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.614677:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:12.0:1619677448.614678:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.614678:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.614679:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.614680:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.614680:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.614734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.614741:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.614744:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.614745:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.614746:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.614749:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.614751:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.614752:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.614753:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.614754:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.614756:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.614759:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.614761:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.614764:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.614765:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.614765:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.614767:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.614767:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.614769:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.614770:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.614770:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.614773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.614776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.614782:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677448.614783:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.614784:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.614785:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.614785:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:1.0:1619677448.614785:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.614786:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.614787:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.614787:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677448.614790:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.614797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.614804:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.614806:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.614807:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.614809:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.614811:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.614812:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.614814:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.614815:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.614816:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.614817:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.614820:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.614821:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.614824:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.614825:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.614826:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.614827:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.614828:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.614829:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.614830:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.614830:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.614833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:4.2:1619677448.614836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.614842:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:26.0:1619677448.614843:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.614844:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:26.0:1619677448.614845:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677448.614845:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:26.0:1619677448.614846:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.614846:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.614846:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.614848:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.614851:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.614853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.614859:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.614862:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.614863:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.614865:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.614868:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.614869:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.614871:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.614872:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.614873:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.614874:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.614877:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.614879:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.614882:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.614883:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.614884:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.614885:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.614886:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.614887:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.614888:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.614888:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:25.2:1619677448.614891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.614893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.2:1619677448.614893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677448.614898:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.614899:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:12.0:1619677448.614900:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.614900:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:12.0:1619677448.614901:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.614901:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.614902:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.614902:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677448.614903:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:12.0:1619677448.614904:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.614905:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:13.0:1619677448.614905:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677448.614906:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.2:1619677448.614907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:12.0:1619677448.614907:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677448.614909:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.614911:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.614912:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.614914:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.614915:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:5.0:1619677448.614915:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.614916:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.614917:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677448.614921:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.614921:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.614921:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.614923:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:30.0:1619677448.614924:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.614926:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:5.0:1619677448.614926:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.614926:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:30.0:1619677448.614927:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712629] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.614927:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677448.614929:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.614929:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:30.0:1619677448.614930:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.614930:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.614930:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000400:00000200:30.0:1619677448.614931:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.614931:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:30.0:1619677448.614934:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.614935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.614936:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.614939:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.614940:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.614941:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.614942:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.614943:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.614943:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:30.0:1619677448.614944:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.614945:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677448.614946:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.614947:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.614947:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.2:1619677448.614948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:18.0:1619677448.614949:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.614952:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.614953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:1.0:1619677448.614956:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.614959:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.614960:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.614960:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677448.614960:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.614962:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.614963:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.614964:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.614964:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.614965:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677448.614965:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.614966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.614966:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677448.614974:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.614977:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.614977:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.614978:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.614979:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.614980:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.614994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.615000:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.615003:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.615003:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.615005:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.615008:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.615010:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.615011:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.615013:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.615013:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.615014:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.615017:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.615019:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.615022:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.615023:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.615023:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.615025:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.615026:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.615027:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.615028:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.615029:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:25.2:1619677448.615031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:21.2:1619677448.615034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.615037:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:22.0:1619677448.615039:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:26.0:1619677448.615040:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.615041:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.615041:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.615041:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:10.0:1619677448.615041:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:22.0:1619677448.615042:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:26.0:1619677448.615043:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.615046:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:4.2:1619677448.615117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.615124:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.615126:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.615127:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.615129:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.615132:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.615133:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.615134:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.615135:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.615136:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.615137:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.615141:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.615143:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.615145:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.615146:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.615147:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.615148:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.615149:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.615150:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.615150:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.615154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.615162:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.615165:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.615166:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.615166:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.615168:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.615171:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.615177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.615183:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.615186:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.615187:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.615188:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.615191:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.615192:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.615193:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.615194:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.615195:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.615196:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.615200:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.615202:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.615204:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.615205:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.615205:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.615207:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.615208:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.615208:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.615209:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.615213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.615221:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.615223:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.615224:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.615224:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.615226:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.615229:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.615279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.615282:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.615284:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.615286:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.615287:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.615289:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.615291:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.615292:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.615297:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.615298:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.615299:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.615302:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.615304:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.615307:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.615308:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.615309:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.615311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677448.615311:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.615311:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.615312:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.615313:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.615317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.615320:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.615323:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.615325:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.615325:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.615327:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677448.615327:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:31.0:1619677448.615328:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.615328:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.615330:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677448.615330:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.615331:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.615332:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:31.0:1619677448.615333:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.615333:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677448.615334:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.615335:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.615338:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.615340:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.615343:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.615343:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.615344:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.615346:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.615347:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.615348:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.615348:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.615353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.615361:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.615364:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.615364:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.615365:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.615367:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.615370:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.615439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.615447:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.615450:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.615450:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.615452:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.615455:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.615457:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.615457:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.615459:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.615460:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.615461:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.615463:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.615465:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.615467:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.615467:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.615468:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.615469:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.615470:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.615470:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.615471:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.615474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.615480:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.615482:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.615483:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.615484:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.615485:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.615486:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.615626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.615633:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.615636:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.615636:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.615638:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.615640:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.615642:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.615644:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712622] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.615645:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.615646:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.615647:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.615651:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.615652:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.615655:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.615656:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.615657:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.615659:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.615659:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.615660:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.615660:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.2:1619677448.615663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:25.2:1619677448.615665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.615671:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.615671:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677448.615674:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:12.0:1619677448.615674:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.615675:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.615675:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.615677:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677448.615677:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.615678:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:26.0:1619677448.615679:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.615680:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677448.615681:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.615682:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.615683:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.615684:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.615685:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.615689:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.615690:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.615694:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.615694:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.615695:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.615697:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.615698:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.615699:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.615700:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.615701:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677448.615704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.615706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.615711:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.615713:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:2.0:1619677448.615714:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.615715:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.615715:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:2.0:1619677448.615715:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.615716:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.615717:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:2.0:1619677448.615717:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.615720:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.615815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.615822:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.615825:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.615827:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.615827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677448.615829:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.615830:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.615831:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.615832:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.615833:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.615833:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677448.615834:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.615836:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.615836:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.615836:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.615840:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.615841:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.615842:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.615844:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:30.0:1619677448.615845:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.615846:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.615847:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.615847:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.615848:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:1.0:1619677448.615848:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.615849:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677448.615849:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.615850:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.615851:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.615852:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.615852:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:1.0:1619677448.615853:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.615854:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:25.2:1619677448.615856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.615857:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.615857:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.615858:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.615860:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.615860:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.615861:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.615861:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.615863:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.615865:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:25.2:1619677448.615866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:13.0:1619677448.615866:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.615868:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.615869:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.615870:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.615874:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:25.2:1619677448.615876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:6.0:1619677448.615876:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.615877:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.615877:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.615879:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.615881:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677448.615882:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.615885:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.615886:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.615886:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.615887:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.615888:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.615917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.615923:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.615926:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:4.2:1619677448.615927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:5.0:1619677448.615928:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.615930:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.615931:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.615932:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:10.0:1619677448.615934:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677448.615934:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.615935:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.615936:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.615937:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.615938:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:5.0:1619677448.615938:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.2:1619677448.615939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:10.0:1619677448.615941:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.615942:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.615943:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.615945:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:10.0:1619677448.615945:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:5.0:1619677448.615945:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.615946:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.615947:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.615948:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:10.0:1619677448.615948:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.615948:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.615948:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:10.0:1619677448.615949:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.615949:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677448.615950:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.615950:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:18.0:1619677448.615951:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.615951:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.615951:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.615952:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677448.615954:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.615956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:18.0:1619677448.615956:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.615956:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.615958:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96712616] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.615958:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:18.0:1619677448.615959:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.615960:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.615961:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.615962:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.615963:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.615963:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677448.615964:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:18.0:1619677448.615965:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.615965:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:22.0:1619677448.615966:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677448.615966:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.615966:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.615967:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.615967:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.615967:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.615968:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.615969:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.615970:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.615970:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.615971:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:8.2:1619677448.615971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677448.615972:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677448.615973:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.615973:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.615974:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.615975:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:4.2:1619677448.615978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:25.2:1619677448.615979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.615981:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.615984:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677448.615985:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:24.0:1619677448.615985:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:20.0:1619677448.615985:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.615987:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.615987:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.615988:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:24.0:1619677448.615988:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:10.0:1619677448.615988:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.615989:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.615989:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677448.615990:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:26.0:1619677448.615991:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.615992:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.615993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.615994:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:24.0:1619677448.615996:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.615998:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.616000:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.616001:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.616002:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.616002:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000400:00000200:24.0:1619677448.616003:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.616006:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000400:00000200:24.0:1619677448.616007:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.616008:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.616009:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:5.0:1619677448.616009:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.616010:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.616010:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.616011:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.616012:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.616012:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677448.616012:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:10.0:1619677448.616012:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:26.0:1619677448.616013:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:24.0:1619677448.616013:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:10.0:1619677448.616014:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.616015:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.616015:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.616016:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.616016:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:10.0:1619677448.616017:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.616018:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677448.616019:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.616020:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.616021:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.616021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:10.0:1619677448.616022:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677448.616025:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.616027:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677448.616028:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:10.0:1619677448.616030:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.616031:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:10.0:1619677448.616031:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:10.0:1619677448.616031:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:1.0:1619677448.616032:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.616033:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.616033:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:10.0:1619677448.616033:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677448.616034:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677448.616034:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000400:00000200:17.0:1619677448.616035:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.616037:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.616038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:8.2:1619677448.616043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.616046:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.616049:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.616050:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.616051:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.616052:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.616052:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.616054:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.616054:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.616055:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:1.0:1619677448.616055:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.616056:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.616057:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.616128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.616136:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.616139:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.616141:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.616143:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.616143:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.616145:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.616146:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.616147:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.616148:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.616149:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.616152:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.616154:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.616157:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.616158:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.616159:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.616160:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.616161:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.616162:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.616162:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.616166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.616173:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.616176:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.616177:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.616178:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.616179:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.616181:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.616192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677448.616199:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.616202:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677448.616203:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677448.616205:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.616208:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.616209:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:26.0:1619677448.616211:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.616212:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.616214:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.616214:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:26.0:1619677448.616218:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.616219:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:26.0:1619677448.616222:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.616223:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:26.0:1619677448.616224:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:26.0:1619677448.616225:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:26.0:1619677448.616226:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.616226:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677448.616227:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:5.2:1619677448.616235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.616242:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.616244:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.616246:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.616246:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.616249:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.616250:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:9.2:1619677448.616250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:10.0:1619677448.616251:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.616252:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.616253:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.616253:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.616256:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.616257:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:10.0:1619677448.616258:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.616260:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.616260:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:2.0:1619677448.616260:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.616261:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:10.0:1619677448.616261:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:2.0:1619677448.616261:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.616262:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.616263:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:2.0:1619677448.616263:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.616264:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.616264:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677448.616266:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:25.2:1619677448.616268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:9.2:1619677448.616271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.616276:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.616278:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:18.0:1619677448.616279:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000400:00000200:30.0:1619677448.616280:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.616281:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.616282:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:18.0:1619677448.616282:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.616283:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.616283:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.616284:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.616285:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:18.0:1619677448.616285:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:25.2:1619677448.616287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.616294:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.616301:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.616302:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.616302:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.616303:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.616303:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.616410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.616416:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.616419:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.616420:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.616422:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.616425:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.616426:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.616427:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.616428:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.616430:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.616430:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.616434:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.616435:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.616437:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.616438:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.616439:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.616440:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.616441:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.616441:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.616442:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.616447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.616454:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.616456:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.616457:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.616458:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.616459:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.616460:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.616470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.616476:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.616478:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.616481:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.616481:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.616483:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.616484:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.616486:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.616487:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.616488:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.616489:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.616492:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.616494:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.616497:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.616498:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.616498:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.616500:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.616501:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.616501:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.616502:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.616506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.616514:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.616516:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.616517:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.616519:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.616521:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.616522:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.616634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:20.2:1619677448.616639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677448.616639:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.0:1619677448.616641:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.616642:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.616643:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.0:1619677448.616644:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:2.0:1619677448.616645:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677448.616646:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.616647:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:2.0:1619677448.616647:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.616649:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:21.2:1619677448.616650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:20.0:1619677448.616650:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.616650:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.616651:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677448.616652:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.616653:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.616653:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.616654:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.616655:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.616655:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.616657:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.616658:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:2.0:1619677448.616659:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:30.0:1619677448.616660:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:21.2:1619677448.616662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000400:00000200:20.0:1619677448.616662:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.616662:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.616663:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.616665:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:20.0:1619677448.616665:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.616666:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.616667:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.616668:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:20.0:1619677448.616668:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.616668:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.616669:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.616669:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677448.616669:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:30.0:1619677448.616670:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96712606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.616670:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.616671:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677448.616671:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:30.0:1619677448.616672:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.616672:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000400:00000200:30.0:1619677448.616673:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.616673:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.616673:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:30.0:1619677448.616674:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:20.0:1619677448.616674:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677448.616674:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.0:1619677448.616675:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677448.616675:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.616675:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.616677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.616677:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:30.0:1619677448.616678:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.616678:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.616680:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:18.0:1619677448.616680:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677448.616681:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96712605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.616682:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.616683:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000400:00000200:18.0:1619677448.616683:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.616684:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:18.0:1619677448.616684:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.616685:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.616685:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.2:1619677448.616685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.616686:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:30.0:1619677448.616686:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677448.616687:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677448.616688:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677448.616688:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.616689:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:18.0:1619677448.616689:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.616689:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.616691:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677448.616691:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:9.2:1619677448.616691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:6.0:1619677448.616691:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677448.616693:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677448.616694:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677448.616695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677448.616695:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677448.616695:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:17.0:1619677448.616695:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.616695:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.616697:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000400:00000200:17.0:1619677448.616697:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.616698:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.616698:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:8.0:1619677448.616698:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:18.0:1619677448.616699:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:17.0:1619677448.616700:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.616701:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:8.0:1619677448.616701:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.616702:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:17.0:1619677448.616702:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96712604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.616702:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.616703:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677448.616703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000400:00000200:17.0:1619677448.616704:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677448.616705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:8.0:1619677448.616705:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.616706:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:17.0:1619677448.616706:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.616707:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677448.616708:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:31.0:1619677448.616709:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.616709:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.616711:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.616711:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:17.0:1619677448.616712:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.616712:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:31.0:1619677448.616713:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:26.0:1619677448.616713:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:17.0:1619677448.616713:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:31.0:1619677448.616714:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96712603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.616714:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.616715:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.616715:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:31.0:1619677448.616716:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.616716:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.616716:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.616717:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.616718:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.616718:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.616718:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.616718:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:2.0:1619677448.616719:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.616720:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:17.0:1619677448.616720:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.616721:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.616722:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.616722:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677448.616725:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.616726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:31.0:1619677448.616727:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:26.0:1619677448.616728:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:2.0:1619677448.616729:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:31.0:1619677448.616730:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.616731:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.616732:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:24.0:1619677448.616732:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:31.0:1619677448.616734:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.616734:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677448.616735:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:31.0:1619677448.616736:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677448.616736:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.616736:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:31.0:1619677448.616737:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.616738:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:24.0:1619677448.616738:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.616741:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677448.616744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:9.2:1619677448.616746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.616751:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.616754:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.616754:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677448.616754:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:30.0:1619677448.616755:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677448.616757:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.616757:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.616758:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.616760:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.616763:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.616850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.616857:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.616860:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.616860:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.616862:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.616864:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.616866:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.616867:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712602] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.616868:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.616869:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.616870:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.616873:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.616875:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.616878:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.616879:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.616879:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.616881:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.616881:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.616883:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.616884:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.616884:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.616887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.616890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.616895:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.616897:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.616898:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.616898:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.616899:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.616899:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.616899:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.616901:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.616901:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.616904:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.616908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.616915:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.616918:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.616918:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.616919:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.616920:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.616920:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.616976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.616983:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.616986:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.616987:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.616989:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.616991:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.616993:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.616994:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.616996:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.616997:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.616997:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.617001:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.617002:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.617004:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.617005:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.617005:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.617007:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.617008:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.617008:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.617009:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.617012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.617019:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.617021:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.617022:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.617023:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.617025:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.617026:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.617049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.617057:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.617060:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.617062:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.617064:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.617065:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.617066:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.617067:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.617068:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.617069:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.617070:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.617073:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.617075:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.617077:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.617077:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.617078:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.617079:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.617080:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.617081:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.617081:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.617085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.617091:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.617094:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.617095:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.617096:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.617098:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.617099:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.617270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.617275:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.617278:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.617279:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.617281:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.617284:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.617285:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.617287:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.617288:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.617289:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.617291:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.617299:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.617301:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.617303:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.617304:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.617305:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.617307:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.617307:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.617308:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.617309:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.617314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.617322:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.617325:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.617326:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.617326:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.617328:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.617329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:1.0:1619677448.617331:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.0:1619677448.617336:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.617339:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.617340:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.617342:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.617344:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.617346:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.617347:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.617348:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.617349:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.617350:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.617354:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.617356:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.2:1619677448.617356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.0:1619677448.617359:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.617359:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.617360:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.617361:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.617362:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.617363:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.617363:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677448.617364:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:8.0:1619677448.617364:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677448.617365:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.617367:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.617367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:17.0:1619677448.617370:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.617372:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.617374:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.617375:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.617376:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.617376:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.617377:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.617378:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.617379:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.617381:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.617382:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.617383:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.617384:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.617386:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.617386:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677448.617387:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.617387:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.617389:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.617389:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.617391:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.617392:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.617392:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.617394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.617397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.617403:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.617403:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.617404:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.617404:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.617404:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.617405:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.617406:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.617406:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.617407:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.617407:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.617413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.617419:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.617422:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.617423:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.617423:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.617423:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.617424:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.617493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.617498:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.617501:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.617503:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.617504:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.617505:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.617507:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677448.617508:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.617509:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.617511:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.617512:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677448.617515:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.617517:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.617519:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.617520:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.617521:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.617523:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.617523:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.617524:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.617526:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.617526:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.617527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:8.2:1619677448.617529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677448.617531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677448.617534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:26.0:1619677448.617535:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:17.0:1619677448.617536:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:26.0:1619677448.617538:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:17.0:1619677448.617538:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.617539:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.617539:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:20.0:1619677448.617540:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677448.617540:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:26.0:1619677448.617541:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.617542:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.617542:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:17.0:1619677448.617542:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.617543:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677448.617544:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.617544:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677448.617544:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677448.617545:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:26.0:1619677448.617546:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.617547:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.617548:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.617549:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.617549:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.617549:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.617550:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.617551:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.617551:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.617552:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.617554:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.617554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:30.0:1619677448.617555:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.617555:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.617556:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.617558:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.617559:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.617561:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.617562:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.617563:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.617563:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.617564:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.617564:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.617564:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:30.0:1619677448.617565:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.617566:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.617566:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.617567:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:10.0:1619677448.617567:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.617568:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.617568:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.617568:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:30.0:1619677448.617569:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:10.0:1619677448.617569:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.617570:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.617570:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.617570:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.617571:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.617571:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:8.2:1619677448.617572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:10.0:1619677448.617573:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96712593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.617574:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.617575:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.617575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:10.0:1619677448.617576:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.2:1619677448.617578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:10.0:1619677448.617579:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.617581:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.617584:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.617584:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.617585:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:10.0:1619677448.617585:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.617586:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.0:1619677448.617586:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677448.617587:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677448.617587:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.617587:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.617587:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.617588:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.617588:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.617588:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.617589:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:10.0:1619677448.617589:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:5.0:1619677448.617589:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.617590:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.617590:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.617591:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.0:1619677448.617592:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.617593:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:25.2:1619677448.617595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.0:1619677448.617595:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677448.617596:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:6.0:1619677448.617603:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.617606:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.617607:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.617607:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.617609:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.617612:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.617616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:22.0:1619677448.617624:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.617626:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.617627:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.617627:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.617628:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.617629:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.617812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.617819:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.617822:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.617824:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.617825:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.617827:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.617827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:31.0:1619677448.617829:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.617830:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.617832:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.617833:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.617833:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.617834:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:1.0:1619677448.617835:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.617836:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677448.617839:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.617839:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.617841:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:1.0:1619677448.617841:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.617843:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:31.0:1619677448.617844:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.617845:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:1.0:1619677448.617845:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.617846:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:1.0:1619677448.617846:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.617847:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.617848:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.617848:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677448.617848:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.617849:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.617850:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:1.0:1619677448.617852:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.617853:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:9.2:1619677448.617854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:1.0:1619677448.617856:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.617857:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.617858:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.617860:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.617860:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.617861:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.617861:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677448.617862:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.617864:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.617865:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:25.2:1619677448.617866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:2.0:1619677448.617866:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.617868:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.617871:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:13.0:1619677448.617873:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.617876:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.617877:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.617878:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.617880:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.617881:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.617934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.617940:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.617943:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.617944:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.617946:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.617948:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.617950:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.617951:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.617952:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.617953:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.617954:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.617958:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.617959:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.617962:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.617963:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.617964:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.617966:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.617966:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.617968:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.617969:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.617969:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.617972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.617974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:12.0:1619677448.617978:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.617980:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:12.0:1619677448.617981:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.617982:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.617982:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.617982:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.617983:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:12.0:1619677448.617984:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.617985:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.617987:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.618046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.2:1619677448.618052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:26.0:1619677448.618053:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.618056:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.618057:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677448.618057:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:26.0:1619677448.618059:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.618060:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.618061:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:26.0:1619677448.618062:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.618063:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.618064:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.618065:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.618065:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.618066:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.618067:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:26.0:1619677448.618068:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.618068:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.618069:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:6.0:1619677448.618070:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.618071:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.618072:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:26.0:1619677448.618074:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.618077:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.618078:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:6.0:1619677448.618079:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:26.0:1619677448.618080:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.618081:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.618082:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677448.618082:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.618082:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.618083:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:26.0:1619677448.618084:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.618084:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.618085:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.618085:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.618086:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677448.618086:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677448.618087:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:6.0:1619677448.618087:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:26.0:1619677448.618088:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.618090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:9.2:1619677448.618091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.2:1619677448.618093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677448.618097:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:18.0:1619677448.618099:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:13.0:1619677448.618100:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:10.0:1619677448.618100:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:18.0:1619677448.618101:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:13.0:1619677448.618101:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.618102:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.618102:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.618102:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.618103:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:2.0:1619677448.618103:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:26.0:1619677448.618104:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:18.0:1619677448.618104:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.618104:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.618105:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677448.618107:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.618111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.618118:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.618121:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.618122:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.618122:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.618123:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.618124:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.618148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.618155:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.618158:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.618161:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.618162:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.618164:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.618164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:30.0:1619677448.618165:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.618166:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712587] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.618168:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.618169:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.618170:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.618170:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.618172:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:30.0:1619677448.618174:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.618175:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677448.618177:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:5.0:1619677448.618178:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.618180:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.618180:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.618181:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.618182:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:5.0:1619677448.618182:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:30.0:1619677448.618183:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.618183:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.618184:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:5.0:1619677448.618184:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.618185:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:5.0:1619677448.618185:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.618186:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:30.0:1619677448.618187:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.618187:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.618189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:5.0:1619677448.618189:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.618191:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.618193:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.618194:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.2:1619677448.618195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.618195:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.618196:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.618197:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.618198:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.618198:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677448.618200:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.618201:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.618202:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:8.2:1619677448.618202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:5.0:1619677448.618202:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.618203:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:5.0:1619677448.618204:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.618206:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.618207:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677448.618207:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677448.618208:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:17.0:1619677448.618210:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.618213:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.618214:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.618214:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.618216:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.618219:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.618400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.618405:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.618408:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.618409:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.618411:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.618413:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.618414:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.618415:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.618417:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.618418:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.618419:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.618422:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.618423:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.618425:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.618426:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.618426:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.618428:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.618428:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.618429:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.618429:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.618435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.618441:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.618443:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.618444:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.618445:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.618445:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.618447:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.618476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.618482:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.618485:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.618486:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.618488:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.618490:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.618492:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.618493:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.618494:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.618496:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.618496:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.618500:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.618501:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.618504:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.618505:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.618506:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.618508:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.618508:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.618509:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.618509:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.618514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.618521:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.618524:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.618524:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.618526:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.618527:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.618529:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.618531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.618538:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.618541:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.618542:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.618543:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.618543:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.618544:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.618605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.618610:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.618612:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.618615:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.618615:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.618617:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.618619:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.618620:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.618621:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.618622:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.618623:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.618626:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.618628:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.618631:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677448.618632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677448.618632:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.618632:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.618634:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.618635:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.618635:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.618636:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677448.618640:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:25.2:1619677448.618642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.618643:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.618645:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.618648:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.618648:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.618649:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:31.0:1619677448.618650:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.618651:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.618651:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.618652:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.618653:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.618653:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.618654:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.618655:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.618658:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.618658:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.618658:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.618660:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.618662:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.618663:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.618664:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.618666:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.618666:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.618667:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.618668:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.618671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.618679:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:21.2:1619677448.618680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:12.0:1619677448.618681:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.618682:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.618683:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.618685:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.618686:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677448.618686:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677448.618689:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677448.618690:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677448.618691:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.618694:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.618696:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677448.618697:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.618698:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.618699:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.618700:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677448.618704:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.618706:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677448.618709:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.618710:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677448.618710:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677448.618712:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677448.618713:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677448.618713:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677448.618714:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:25.2:1619677448.618718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.618725:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.618727:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.618728:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.618729:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.618730:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.618733:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:25.2:1619677448.618738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000800:00000200:5.2:1619677448.618741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.618746:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:5.2:1619677448.618746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.618748:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.618749:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.618750:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.618750:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.618751:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:4.2:1619677448.618751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.618752:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:1.0:1619677448.618752:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.618753:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:10.0:1619677448.618754:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.618755:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677448.618755:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:26.0:1619677448.618756:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.618756:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.618756:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.618758:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.618758:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677448.618758:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.618760:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.618760:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.618761:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.618761:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.618763:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.618763:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.618765:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677448.618765:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.618766:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.618767:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.618771:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.618772:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.618773:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.618774:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.618775:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.618776:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.618776:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.618777:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.618778:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:1.0:1619677448.618778:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.618779:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:1.0:1619677448.618779:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.618780:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:1.0:1619677448.618780:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:26.0:1619677448.618781:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96712578] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.618782:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.618782:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.618782:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.618783:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.618783:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.618783:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677448.618784:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:25.2:1619677448.618785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.618785:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.618786:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.618787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.618787:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:26.0:1619677448.618788:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.618788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:10.0:1619677448.618788:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.618790:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.618790:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.618791:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.618791:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.618793:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.618794:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.618794:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:12.0:1619677448.618794:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:26.0:1619677448.618795:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.618795:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.618796:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:12.0:1619677448.618796:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:10.0:1619677448.618796:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:18.0:1619677448.618797:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:12.0:1619677448.618797:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.618798:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.618798:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:26.0:1619677448.618799:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.618799:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.618800:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:13.0:1619677448.618800:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:18.0:1619677448.618801:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677448.618801:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.618804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:18.0:1619677448.618804:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.618804:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:18.0:1619677448.618806:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.618807:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.618808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:18.0:1619677448.618808:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.618809:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.618810:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:21.2:1619677448.618812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:18.0:1619677448.618813:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.618814:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:13.0:1619677448.618814:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677448.618815:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677448.618817:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:18.0:1619677448.618817:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.618818:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.618818:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.618818:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677448.618818:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:6.0:1619677448.618818:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:13.0:1619677448.618819:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:22.0:1619677448.618820:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.618820:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677448.618820:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.618820:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677448.618820:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:18.0:1619677448.618821:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.618822:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.618822:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677448.618822:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:6.0:1619677448.618822:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.618824:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.618825:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:6.0:1619677448.618825:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.618826:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677448.618826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:6.0:1619677448.618827:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.618828:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.618830:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.618830:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.618831:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.618833:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:6.0:1619677448.618834:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.618836:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.618836:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.618836:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.618837:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677448.618839:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.618839:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.618840:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.618841:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.618841:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677448.618842:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.618843:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.618844:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.618844:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:21.2:1619677448.618845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:8.2:1619677448.618848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.618851:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.618854:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.618854:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.0:1619677448.618855:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:18.0:1619677448.618856:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677448.618857:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.618858:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.618858:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.618860:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:20.0:1619677448.618861:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.618861:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:18.0:1619677448.618862:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.618863:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.618864:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:18.0:1619677448.618864:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.618865:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677448.618869:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.618870:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.618873:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.618873:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677448.618874:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677448.618876:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.618876:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.618877:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.618878:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.618882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.618890:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.618892:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.618893:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.618893:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.618895:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.618898:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.618992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.618998:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.619000:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.619002:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.619003:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.619006:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.619007:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.619008:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.619010:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.619011:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.619012:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.619016:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.619017:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.619020:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.619021:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.619022:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.619024:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.619024:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.619025:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.619025:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.619030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.619036:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.619039:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.619040:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.619042:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.619043:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.619045:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.619174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.619181:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.619184:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.619185:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.619187:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.619190:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.619191:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.619192:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712573] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.619194:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.619195:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.619196:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.619199:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.619201:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.619204:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.619205:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.619206:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.619207:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.619208:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.619209:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.619209:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.619213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.619221:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.619224:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.619225:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.619225:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.619227:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.619230:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.619262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.619267:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.619270:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.619271:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.619273:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.619276:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.619277:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.619278:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712572] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.619280:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.619281:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.619282:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.619285:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.619287:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:21.2:1619677448.619290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:17.0:1619677448.619290:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.619290:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.619291:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677448.619298:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.619298:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.619298:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:25.2:1619677448.619300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.619300:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677448.619301:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:17.0:1619677448.619301:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.619301:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.619302:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.619303:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.619306:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.619307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677448.619308:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:12.0:1619677448.619308:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:22.0:1619677448.619309:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.619310:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.619311:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.619311:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.619312:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.619313:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:1.0:1619677448.619313:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:12.0:1619677448.619314:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.619315:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.619315:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.619316:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677448.619317:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.619317:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677448.619317:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.619319:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.619321:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.619322:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.619323:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.619325:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.619325:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.619326:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.619327:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.619328:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:25.2:1619677448.619331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:21.2:1619677448.619333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.619338:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:6.0:1619677448.619338:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.619340:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:26.0:1619677448.619341:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.619341:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.619341:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:26.0:1619677448.619342:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.619342:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:26.0:1619677448.619344:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.619348:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677448.619398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.619405:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.619408:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.619409:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.619410:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.619413:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.619415:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.619416:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.619417:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.619418:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.619419:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.619422:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.619424:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.619427:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.619428:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.619428:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.619430:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.619431:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.619431:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.619432:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.619436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.619442:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.619445:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.619446:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.619448:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.619448:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.619450:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.619456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.619463:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.619465:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.619466:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.619467:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.619467:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.619469:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.619497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.619504:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.619507:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.619509:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.619512:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.619512:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.619513:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.619514:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.619516:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.619517:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.619518:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.619521:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.619523:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.619525:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.619526:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.619527:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.619528:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.619529:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.619530:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.619530:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.619534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.619540:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.619542:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.619543:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.619544:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.619546:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.619546:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.2:1619677448.619553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.619556:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.619559:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:4.2:1619677448.619561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:20.0:1619677448.619562:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.619562:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.619565:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.619567:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.619568:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.619569:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677448.619570:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.619571:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.619572:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677448.619572:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.619572:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.619575:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.619576:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.619578:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.619578:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.619579:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.619580:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.619581:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.619582:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:10.0:1619677448.619582:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.619583:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.619583:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.619584:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.619585:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.619585:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.619587:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:10.0:1619677448.619587:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.619588:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.619589:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677448.619589:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:25.2:1619677448.619590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.619591:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.619592:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.619593:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.619594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.619594:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.619595:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.619596:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.619596:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:12.0:1619677448.619597:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.619599:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.619599:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:25.2:1619677448.619600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:12.0:1619677448.619600:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.619601:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.619601:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.619602:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677448.619603:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.619603:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.619604:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677448.619608:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.619611:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.619611:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.619612:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.619614:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.619617:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.619679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.619686:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.619689:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.619691:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.619693:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.619693:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.619695:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.619696:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.619697:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.619698:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.619699:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.619702:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.619704:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.619706:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.619707:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.619707:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.619708:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.619709:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.619710:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.619710:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.619714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.619721:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.619723:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.619724:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.619725:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.619727:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.619727:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.2:1619677448.619782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.619788:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.619790:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.619791:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.619793:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.619796:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.619797:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.619798:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.619799:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.619800:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.619801:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.619804:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.619806:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.619809:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.619810:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.619810:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.619812:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.619813:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.619813:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.619813:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.619818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.619825:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.619828:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.619829:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.619830:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.619832:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.619835:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.619837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677448.619844:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.619846:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.619847:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.619847:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.619848:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.619849:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.619879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.619886:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.619888:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.619889:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.619891:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.619893:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.619895:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.619896:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.619897:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.619898:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.619899:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.619902:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.619904:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.619907:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.619908:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.619908:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.619910:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.619911:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.619911:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.619911:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.619916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.619923:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.619927:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.619928:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.619928:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.619930:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.619933:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.619937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.619944:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.619946:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.619947:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.619947:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.619948:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.619949:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.619949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.619956:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.619959:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.619960:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.619962:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.619965:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.619967:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.619968:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.619969:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.619970:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.619971:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.619974:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.619975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677448.619976:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.619978:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.619979:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.619980:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.619981:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677448.619982:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.619982:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.619983:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.619984:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:31.0:1619677448.619985:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677448.619985:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677448.619987:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.619988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:31.0:1619677448.619989:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.619989:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.619990:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:4.2:1619677448.619990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.619991:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.619993:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.619993:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.619994:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:6.0:1619677448.619995:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:31.0:1619677448.619997:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.619997:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:31.0:1619677448.619999:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:10.0:1619677448.619999:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677448.619999:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.620000:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.620000:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.620000:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.620001:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.620001:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.620002:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.620002:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:6.0:1619677448.620002:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.620004:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.620004:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.620005:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677448.620005:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677448.620006:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.620009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.620016:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.620018:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.620019:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.620019:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.620022:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.620024:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.620034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.620040:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.620043:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.620045:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.620046:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.620049:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.620051:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.620052:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.620053:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.620054:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.620055:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.620058:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.620060:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.620062:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.620063:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.620063:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.620065:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.620065:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.620066:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.620066:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.620071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.620078:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.620080:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.620080:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.620082:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.620082:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.620084:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.620090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.620097:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.620099:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.620100:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.620100:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.620101:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.620101:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:21.2:1619677448.620201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.620206:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.620209:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.620209:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.620211:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.620214:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.620216:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.620217:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.620218:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.620219:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.620221:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.620224:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.620226:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.620229:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.620230:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.620231:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.620233:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.620233:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.620234:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.620234:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.620239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.620239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.620245:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677448.620246:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.620248:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:30.0:1619677448.620249:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.620250:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.620250:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677448.620251:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677448.620252:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.620252:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.620253:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.620255:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:5.0:1619677448.620255:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677448.620256:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.620257:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.620258:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.620259:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677448.620263:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.620265:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677448.620267:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.620268:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677448.620269:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677448.620271:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.620271:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.620272:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.620273:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.620276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:21.2:1619677448.620283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.620286:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.620288:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.620289:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.620290:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677448.620290:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.620292:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677448.620299:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.620300:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:30.0:1619677448.620300:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.620302:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.620302:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.620304:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.620305:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.620307:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.620308:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.620309:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.620312:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.620314:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.620316:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.620317:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.620318:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.620319:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.620320:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.620321:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.620323:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.620323:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.2:1619677448.620323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:25.2:1619677448.620325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.620329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.620329:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.620332:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.620333:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:12.0:1619677448.620334:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:2.0:1619677448.620335:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.620336:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.620336:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.620337:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:2.0:1619677448.620337:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.620338:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.620339:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:2.0:1619677448.620339:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677448.620340:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:2.0:1619677448.620340:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.620341:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.620341:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.620342:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677448.620342:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.620343:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.620344:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.620348:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.620350:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.2:1619677448.620352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677448.620353:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.620354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.620354:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.620354:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.620355:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.620356:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.620357:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677448.620358:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677448.620358:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.620359:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000400:00000200:20.0:1619677448.620360:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.620360:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677448.620361:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677448.620361:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:25.2:1619677448.620362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.620363:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.620364:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:20.0:1619677448.620365:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:17.0:1619677448.620365:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.620365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000400:00000200:20.0:1619677448.620366:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.620367:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.620368:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.620368:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:20.0:1619677448.620369:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.620369:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.620370:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.620371:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:13.0:1619677448.620371:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:1.0:1619677448.620371:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:13.0:1619677448.620372:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.620372:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96712555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.620373:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.620374:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:13.0:1619677448.620374:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.620374:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:1.0:1619677448.620374:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.620375:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.620375:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:1.0:1619677448.620376:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.620376:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.620377:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:25.2:1619677448.620381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.620381:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.620382:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677448.620382:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.620383:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:20.0:1619677448.620384:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677448.620384:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:1.0:1619677448.620384:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.620385:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677448.620385:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.620386:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.620387:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.620388:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:1.0:1619677448.620388:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.620389:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.620389:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:25.2:1619677448.620390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:1.0:1619677448.620390:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:1.0:1619677448.620392:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (25)--
00000800:00000200:13.0:1619677448.620393:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)++
00000800:00000200:1.0:1619677448.620393:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.620394:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.620395:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (25)--
00000800:00000200:13.0:1619677448.620395:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:1.0:1619677448.620395:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.620397:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.620400:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677448.620401:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.620402:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.620404:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.620405:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.620405:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677448.620406:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.620406:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.620407:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.620410:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.620434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.620442:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.620445:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.620446:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.620448:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.620451:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.620453:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.620454:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.620455:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.620456:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.620457:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.620460:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.620461:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.620464:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.620465:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.620466:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.620467:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.620468:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.620469:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.620469:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.620474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.620483:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.620485:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.620486:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.620488:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.620488:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.620491:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.620721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.620729:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.620732:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.620733:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.620735:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.620737:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.620738:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.620740:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.620741:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.620742:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.620743:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677448.620746:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.620748:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.620751:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.620752:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.620752:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.620753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:24.0:1619677448.620754:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.620754:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.620756:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.620757:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.620757:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:31.0:1619677448.620760:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.2:1619677448.620760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.620762:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.2:1619677448.620763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000400:00000200:31.0:1619677448.620764:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.620766:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.620767:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.620768:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:20.0:1619677448.620768:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:31.0:1619677448.620769:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.620770:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.620770:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:8.0:1619677448.620770:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:31.0:1619677448.620771:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.620771:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.620772:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677448.620772:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.620773:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:20.0:1619677448.620773:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.620773:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677448.620774:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:31.0:1619677448.620775:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.620776:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.2:1619677448.620777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.620778:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.620779:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.620780:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677448.620780:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.620781:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677448.620781:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.620782:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.620783:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.620783:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677448.620783:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.620784:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.620785:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.620785:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.620786:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:9.2:1619677448.620787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.620793:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.620796:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.620796:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.620797:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.620799:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.620802:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.2:1619677448.620817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.620825:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.620827:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.620828:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.620830:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.620833:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.620834:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.620835:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.620837:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.620838:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.620839:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.620842:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.620843:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.620846:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.620847:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.620847:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.620849:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.620849:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.620850:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.620850:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.620855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.620863:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.620866:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.620866:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.620867:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.620869:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.620872:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.621011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.621018:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.621020:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.621021:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.621023:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.621024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677448.621026:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.621027:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.621028:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.621029:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.621030:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.621030:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.621031:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.621032:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.621033:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.621035:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.621036:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.621037:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.621039:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.621040:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.621040:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:17.0:1619677448.621041:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:10.0:1619677448.621041:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:17.0:1619677448.621042:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.621042:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:17.0:1619677448.621043:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.621043:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.621043:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677448.621044:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.621044:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:17.0:1619677448.621045:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:25.2:1619677448.621048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:17.0:1619677448.621048:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.621050:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.621053:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.621054:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.621055:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (23)++
00000800:00000200:22.0:1619677448.621056:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (24)++
00000800:00000200:17.0:1619677448.621057:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (25)--
00000800:00000200:17.0:1619677448.621057:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.621058:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677448.621059:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000800:00000200:17.0:1619677448.621059:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677448.621060:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.621060:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.621062:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.621063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.621066:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.621068:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.2:1619677448.621068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:22.0:1619677448.621069:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.621070:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.621070:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.621071:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000400:00000200:22.0:1619677448.621072:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.621072:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (23)++
00000800:00000200:6.0:1619677448.621073:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (24)--
00000800:00000200:22.0:1619677448.621074:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (24)--
00000800:00000200:6.0:1619677448.621074:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677448.621074:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:22.0:1619677448.621075:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677448.621077:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.621078:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.621080:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.621082:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.621084:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.621085:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.621086:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.621087:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.621088:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.621092:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.621094:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.621097:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.621098:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.621098:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.621100:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.621100:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.621101:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.621101:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.621106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.621113:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.621115:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.621116:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.621118:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.621118:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.621120:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.621305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.621313:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.621316:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.2:1619677448.621316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:30.0:1619677448.621318:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.621321:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.621321:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.621322:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.0:1619677448.621322:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677448.621324:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712547] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.621324:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:30.0:1619677448.621325:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.621327:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.621327:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.621327:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.621328:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.621330:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.621332:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.621332:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.621333:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.621335:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.621336:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.621336:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.621338:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.621338:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.621339:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.621340:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.621341:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677448.621341:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.621342:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.621343:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.621343:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677448.621343:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.621346:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.621346:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:25.2:1619677448.621347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.621347:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.621349:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.621349:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.621350:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.621351:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.621354:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:25.2:1619677448.621355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:12.0:1619677448.621357:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.621358:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.621360:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.621362:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677448.621362:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:12.0:1619677448.621363:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.621364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:6.0:1619677448.621365:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.621366:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.621366:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.621367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:6.0:1619677448.621368:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.621370:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.621370:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677448.621373:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.621374:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677448.621375:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:25.2:1619677448.621375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:13.0:1619677448.621375:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.621375:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.621377:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.621378:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.621380:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.621382:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.621382:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:22.0:1619677448.621383:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:31.0:1619677448.621384:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:22.0:1619677448.621385:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.621386:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.621386:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:31.0:1619677448.621387:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.621387:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000400:00000200:31.0:1619677448.621388:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.621388:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.621388:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:31.0:1619677448.621389:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.621392:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.621392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:31.0:1619677448.621394:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.621398:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.621398:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677448.621398:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:31.0:1619677448.621399:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.621400:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:31.0:1619677448.621401:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.621401:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.621402:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:2.0:1619677448.621402:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:31.0:1619677448.621403:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:18.0:1619677448.621404:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.621407:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.621407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:18.0:1619677448.621408:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.621409:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.621411:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.621412:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.621413:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.621415:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:18.0:1619677448.621416:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.621418:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:18.0:1619677448.621418:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:20.0:1619677448.621419:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.621420:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.621421:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677448.621422:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.621423:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677448.621423:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.621423:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.621425:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.621425:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.621426:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.621427:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.621428:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.621431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.621433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677448.621438:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:2.0:1619677448.621438:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:26.0:1619677448.621440:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:2.0:1619677448.621440:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:10.0:1619677448.621441:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:2.0:1619677448.621441:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:26.0:1619677448.621442:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.621443:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:26.0:1619677448.621445:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.621448:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.621451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.621457:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.621459:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.621460:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.621460:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.621461:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.621462:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.621516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.621523:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.621525:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.621526:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.621528:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.621531:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.621532:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.621533:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.621534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:26.0:1619677448.621535:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.621536:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.621537:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.621540:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.621540:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:26.0:1619677448.621541:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.621542:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:26.0:1619677448.621545:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.621545:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.621545:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:26.0:1619677448.621546:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.621547:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677448.621548:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.621549:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:5.0:1619677448.621549:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:26.0:1619677448.621550:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.621550:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:5.0:1619677448.621550:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.621551:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:5.0:1619677448.621552:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.621553:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.621554:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:9.2:1619677448.621556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677448.621558:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.621560:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.621562:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.621563:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677448.621563:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.621563:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.621565:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:18.0:1619677448.621566:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:5.0:1619677448.621566:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:18.0:1619677448.621567:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.621567:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:2.0:1619677448.621567:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.621568:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.621569:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:18.0:1619677448.621570:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.621571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:18.0:1619677448.621573:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.621574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.621577:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:30.0:1619677448.621578:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.621579:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.621580:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.621580:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677448.621581:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.621582:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.621583:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:4.2:1619677448.621583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677448.621584:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.621587:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.621588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:25.2:1619677448.621589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:24.0:1619677448.621590:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.621593:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.621594:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.621595:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.621595:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.621596:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:24.0:1619677448.621598:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.621598:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:31.0:1619677448.621599:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.621599:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:24.0:1619677448.621599:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:31.0:1619677448.621600:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.621600:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.621601:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:24.0:1619677448.621601:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.621602:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.621602:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.621603:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:24.0:1619677448.621603:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.621604:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.621604:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.621606:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.621607:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.621608:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.621609:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.621609:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.621610:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.621610:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.621613:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.621614:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.621614:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:10.0:1619677448.621614:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.621616:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.621616:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677448.621616:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.621617:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.621619:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.621619:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:10.0:1619677448.621619:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.621619:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.621620:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.621622:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.621622:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677448.621622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.621623:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.621623:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:4.2:1619677448.621625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.2:1619677448.621628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.621630:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.621633:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.621634:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.621634:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.621635:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:20.0:1619677448.621636:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.0:1619677448.621637:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677448.621637:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.621638:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677448.621638:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:8.0:1619677448.621638:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:20.0:1619677448.621639:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.621640:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:20.0:1619677448.621641:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.621642:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.621645:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.622067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.622074:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.622077:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.622080:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.622081:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.622082:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.622084:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.622085:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.622087:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.622088:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.622089:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.622092:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.622094:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.622096:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.622097:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.622098:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.622099:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.622100:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.622100:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.622101:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.622105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.622111:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.622113:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:20.2:1619677448.622113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:26.0:1619677448.622114:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.622114:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.2:1619677448.622114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:26.0:1619677448.622116:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.622118:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677448.622120:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677448.622120:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677448.622123:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:18.0:1619677448.622123:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.622123:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.622126:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.622127:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.622127:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:2.0:1619677448.622128:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.622130:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.622130:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677448.622131:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677448.622132:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.622133:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.622134:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.622135:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.622136:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.622137:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.622137:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677448.622139:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677448.622141:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.622144:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.622144:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.622146:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677448.622146:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.622146:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.622147:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.622149:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.622149:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.622149:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677448.622150:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.622150:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.2:1619677448.622150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.622150:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.622150:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:31.0:1619677448.622152:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.622152:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.622154:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.622155:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:17.0:1619677448.622155:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:9.2:1619677448.622155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.622156:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.622158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:17.0:1619677448.622158:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.622159:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.2:1619677448.622161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.622161:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.622163:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:17.0:1619677448.622163:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.622164:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:18.0:1619677448.622165:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:30.0:1619677448.622166:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:18.0:1619677448.622166:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.622166:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96712536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.622167:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.622167:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677448.622168:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:18.0:1619677448.622168:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.622168:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.622169:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.622169:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.622170:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:18.0:1619677448.622170:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677448.622171:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:30.0:1619677448.622171:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.622172:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.622173:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:17.0:1619677448.622173:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.622175:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.622175:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:17.0:1619677448.622175:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.622177:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.622178:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.622179:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.622180:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.622181:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.622182:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.622183:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.622184:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.622186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.622189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.622195:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.622196:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.622196:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.622197:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.622197:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.622198:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:1.0:1619677448.622198:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.622199:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.622199:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.622201:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.622306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.622314:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.622317:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.622317:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677448.622319:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.622322:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.622323:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.622325:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.622326:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.622327:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.622328:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.622331:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.622333:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.622336:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.622337:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.622338:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.622339:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.622340:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.622340:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.622341:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.622346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.622355:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.622358:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.622359:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.622361:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.622361:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.622363:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.622458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.622465:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.2:1619677448.622466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677448.622468:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.622469:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.622471:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.622474:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677448.622474:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.622475:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677448.622477:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677448.622477:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.622479:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.622479:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.622480:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.622480:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.622481:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.622482:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.622483:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.622484:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.622484:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.622486:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.622486:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.622486:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.622487:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.622488:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.622489:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.622489:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677448.622490:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.622491:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.622491:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.622492:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.622492:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.622493:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:30.0:1619677448.622494:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.622494:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.622495:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.622495:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.622497:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:30.0:1619677448.622497:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677448.622498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.622499:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:5.2:1619677448.622499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.622500:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.622501:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:4.2:1619677448.622504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:12.0:1619677448.622506:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:21.2:1619677448.622507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.622507:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:12.0:1619677448.622508:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677448.622509:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.622509:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.622509:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.622510:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:12.0:1619677448.622510:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.622511:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:12.0:1619677448.622512:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.622513:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677448.622514:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:31.0:1619677448.622515:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:24.0:1619677448.622515:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677448.622515:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.622516:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.622517:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:8.0:1619677448.622517:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.622518:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:13.0:1619677448.622518:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.622518:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.622519:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:8.0:1619677448.622519:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.622520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:13.0:1619677448.622520:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:8.0:1619677448.622521:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.622522:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.622524:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.622525:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.622526:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.622528:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.622529:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677448.622530:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.622531:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.622532:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.622534:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.622536:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:8.0:1619677448.622536:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677448.622537:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96712531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.622537:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.622538:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:24.0:1619677448.622539:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.622540:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.622540:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.622540:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.622541:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.622542:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.622543:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:24.0:1619677448.622544:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.622546:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.2:1619677448.622546:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.622550:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.622550:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.622552:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.622552:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.622554:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.622554:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.622555:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:17.0:1619677448.622555:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.622556:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:17.0:1619677448.622558:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.622558:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.622559:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.622560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:17.0:1619677448.622561:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.622564:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677448.622568:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.622571:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.622572:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.622573:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.622575:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.622578:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.622633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677448.622641:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677448.622644:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677448.622646:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.622647:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677448.622648:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.622650:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677448.622651:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.622652:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.622654:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.622654:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677448.622658:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.622659:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677448.622662:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.622663:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.622664:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.622665:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.622666:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677448.622667:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677448.622667:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677448.622671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.622677:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.622680:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.622680:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.622681:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.622683:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.622685:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.622713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.622719:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.622722:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.622722:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.622725:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.622727:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.622729:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.622730:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.622731:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.622732:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.622733:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.622736:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.622738:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.622741:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.622742:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.622742:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.622744:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.622745:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.622745:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.622746:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.622750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.622759:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.622761:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.622762:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.622762:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.622765:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.622767:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.622774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.622779:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.622782:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.622782:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.622785:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.622786:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.622787:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.622789:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.622790:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.622791:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.622792:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.622795:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.622796:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.622798:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.622799:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.622799:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.622801:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.622801:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.622802:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.622802:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.622806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.622813:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.622815:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.622816:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.622817:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.622817:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.622818:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.622825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.622833:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.622835:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.622836:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.622836:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.622836:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.622837:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.622884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.622891:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.622894:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.622895:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.622897:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.622899:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.622901:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.622902:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.622903:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.622904:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.622905:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.622909:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.622910:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.622913:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.622914:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.622915:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.622916:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.622917:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.622917:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.622918:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.622933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.622940:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.622943:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.622944:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.622944:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.622946:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.622949:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.622996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.622999:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.623001:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.623003:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.623005:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.623006:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.623007:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.623009:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.623010:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.623011:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.623012:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.623015:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.623017:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.623020:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.623021:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.623021:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.623023:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.0:1619677448.623028:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.623029:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.623030:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.623030:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.623031:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.623034:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:25.2:1619677448.623034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:30.0:1619677448.623036:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.623038:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.623039:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.623040:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.623041:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.623041:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.623042:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.623042:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.623043:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.623044:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.623045:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.623046:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.623049:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.623051:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.623052:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.623053:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.623054:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.623055:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.623056:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.623056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:30.0:1619677448.623058:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.623058:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:21.2:1619677448.623058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:30.0:1619677448.623059:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.623060:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:18.0:1619677448.623063:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:21.2:1619677448.623064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.2:1619677448.623064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.623065:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677448.623065:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.623066:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:18.0:1619677448.623068:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.623068:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.623070:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.623071:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.623072:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:2.0:1619677448.623072:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:5.0:1619677448.623074:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:2.0:1619677448.623074:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.623075:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:2.0:1619677448.623076:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.623077:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:2.0:1619677448.623077:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.623078:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:5.0:1619677448.623078:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.623079:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.623079:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:31.0:1619677448.623080:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.623080:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:18.0:1619677448.623081:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.623083:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.623083:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:18.0:1619677448.623084:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.623084:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.623085:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.623086:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.623086:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:31.0:1619677448.623087:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:18.0:1619677448.623087:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.623088:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.623090:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.623091:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.623092:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.623092:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677448.623093:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:31.0:1619677448.623094:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:18.0:1619677448.623094:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.623095:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677448.623096:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.623097:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.623098:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.623098:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (23)++
00000800:00000200:2.0:1619677448.623099:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (24)--
00000800:00000200:25.2:1619677448.623100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677448.623100:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:31.0:1619677448.623101:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:18.0:1619677448.623101:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.623101:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677448.623102:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.623102:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.623103:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.623104:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.623105:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:31.0:1619677448.623105:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:21.2:1619677448.623105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.623105:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.623106:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:31.0:1619677448.623107:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.623107:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:18.0:1619677448.623107:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:31.0:1619677448.623108:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.623108:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:31.0:1619677448.623109:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.623109:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:18.0:1619677448.623109:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:31.0:1619677448.623110:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:26.0:1619677448.623110:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.623110:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:10.0:1619677448.623110:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:25.2:1619677448.623111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.623111:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:26.0:1619677448.623112:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.623112:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677448.623112:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:8.2:1619677448.623113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:21.2:1619677448.623114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:26.0:1619677448.623116:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:4.2:1619677448.623117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677448.623118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:12.0:1619677448.623118:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.623120:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.623120:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000400:00000200:12.0:1619677448.623121:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.623121:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:2.0:1619677448.623122:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000400:00000200:12.0:1619677448.623123:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.623123:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677448.623124:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:10.0:1619677448.623124:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.623124:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.623125:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:13.0:1619677448.623125:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677448.623125:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000400:00000200:1.0:1619677448.623125:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.623126:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.623127:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.623127:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:1.0:1619677448.623128:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.623129:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677448.623129:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000400:00000200:10.0:1619677448.623131:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.623131:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:6.0:1619677448.623132:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.623134:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.623135:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:6.0:1619677448.623135:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.623137:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.623137:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677448.623138:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.623139:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.623140:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.623140:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.623141:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.623143:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:6.0:1619677448.623143:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.623149:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.623149:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.623152:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.623152:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677448.623154:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.623156:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.623156:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.623157:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.623157:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.623157:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.623159:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.623159:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.623160:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.623160:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677448.623160:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:10.0:1619677448.623161:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.623161:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:10.0:1619677448.623163:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.623163:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.623165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677448.623169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.623172:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:25.2:1619677448.623175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.623175:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:26.0:1619677448.623176:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:17.0:1619677448.623176:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.623176:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.623178:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:17.0:1619677448.623178:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.623179:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.623179:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677448.623180:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:22.0:1619677448.623182:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.623185:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.623186:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.623186:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:5.2:1619677448.623187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:22.0:1619677448.623189:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.623192:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677448.623193:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.623195:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.623196:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.623198:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.623200:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.623201:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.623203:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.623204:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677448.623205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:1.0:1619677448.623205:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.623206:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.623210:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.623211:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.623211:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:24.0:1619677448.623214:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.623215:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677448.623215:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.623215:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.623216:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:24.0:1619677448.623217:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.623218:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.623218:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.623219:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677448.623220:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.623220:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677448.623221:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.623223:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96712518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.623224:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677448.623224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:24.0:1619677448.623225:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.623227:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.623231:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.623231:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.623232:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.623233:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.623233:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.623235:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.623237:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.623237:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.623238:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.623239:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.623240:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.623240:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:24.0:1619677448.623241:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.623242:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677448.623243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677448.623245:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677448.623246:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.623248:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.2:1619677448.623248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.623249:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.623250:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.623250:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.623251:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:17.0:1619677448.623256:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.623259:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.623259:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.623260:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.623262:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.623265:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.623325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.623331:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.623334:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.623335:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.623337:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.623340:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.623342:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.623343:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.623344:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.623345:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.623346:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.623350:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.623351:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.623354:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.623354:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.623355:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.623357:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.623357:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.623358:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.623359:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.623364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.623372:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.623375:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.623375:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.623377:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.623378:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.623380:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.623407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.623415:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.623417:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.623418:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.623420:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.623423:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.623425:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.623426:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.623427:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.623429:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.623429:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.623433:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.623435:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.623437:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.623438:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.623438:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.623440:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.623441:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.623441:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.623442:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.623446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.623454:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.623456:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.623457:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.623459:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.623461:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.623462:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.623508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.623514:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.623517:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.623517:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.623519:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.623522:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.623523:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.623525:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.623526:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.623527:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.623528:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.623531:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.623533:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.623536:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.623537:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.623537:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.623539:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.623539:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.623540:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.623540:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.2:1619677448.623541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:9.2:1619677448.623545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:1.0:1619677448.623547:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.623550:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.623551:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.623552:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.623553:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:1.0:1619677448.623555:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.623556:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:1.0:1619677448.623556:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.623557:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.623557:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:1.0:1619677448.623557:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.623559:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.623559:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.623560:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.623561:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:1.0:1619677448.623561:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.623565:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.623567:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.623569:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.623569:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.623570:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.623572:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.623572:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.623573:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.623573:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.623578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.623585:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.623586:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.623587:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.623588:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.623589:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.623590:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.623680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.623686:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.623688:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.623691:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.623691:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.623693:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.623695:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.623696:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.623698:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.623699:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677448.623699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:5.0:1619677448.623700:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.623703:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.623705:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.623706:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677448.623708:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.623709:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:5.0:1619677448.623709:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.623710:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.623710:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.623712:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.623712:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.623713:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.623714:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:10.0:1619677448.623715:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.623715:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000400:00000200:10.0:1619677448.623716:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:5.0:1619677448.623716:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.623718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.623718:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.623719:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.623720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:10.0:1619677448.623720:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.623721:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.623723:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677448.623724:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677448.623724:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.623725:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.623726:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677448.623726:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.623727:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.623727:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.623728:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.623728:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.623729:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.623729:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:12.0:1619677448.623730:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.623731:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.623731:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.623731:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:12.0:1619677448.623732:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.623732:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.623733:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.623736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:25.2:1619677448.623739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.623740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:13.0:1619677448.623742:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.623744:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.623745:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.623746:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.623746:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.623747:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:24.0:1619677448.623747:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677448.623747:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677448.623749:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:30.0:1619677448.623750:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:8.0:1619677448.623750:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:30.0:1619677448.623751:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.623753:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.623753:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.623754:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.623755:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:24.0:1619677448.623756:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.623757:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.623758:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.623760:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.623761:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.623761:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.623765:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.623766:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.623769:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.623769:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.623770:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.623771:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.623772:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.623772:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.623773:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.623777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.623785:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.623788:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.623789:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.623789:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.623791:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.623794:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.623982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.623989:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.623992:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.623993:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.623994:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.623997:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.623998:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.623999:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.624000:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.624001:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.624002:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.624005:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.624005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677448.624006:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.624009:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.624009:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.624010:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.624011:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.624012:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.624013:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.624013:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.624013:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677448.624015:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.624017:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.624017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.624019:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.624020:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.624021:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.624022:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.624023:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:31.0:1619677448.624024:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.624024:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.624024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:31.0:1619677448.624025:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:13.0:1619677448.624026:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.624026:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.624028:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.624029:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.624030:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.624030:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.624031:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677448.624031:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677448.624033:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.624034:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.624035:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677448.624035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.624036:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.624036:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:17.0:1619677448.624036:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.624038:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000400:00000200:17.0:1619677448.624038:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.624039:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.624040:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.624040:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:17.0:1619677448.624040:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.624041:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.624042:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.624042:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:17.0:1619677448.624043:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.624044:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:9.2:1619677448.624044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:12.0:1619677448.624045:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.624046:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.624047:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.624047:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:17.0:1619677448.624048:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.624048:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677448.624050:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677448.624052:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:17.0:1619677448.624053:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.624053:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.624054:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:2.0:1619677448.624055:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.624056:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:17.0:1619677448.624056:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:2.0:1619677448.624056:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.624057:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.624058:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:2.0:1619677448.624058:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.624059:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.624060:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.624061:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:25.2:1619677448.624062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.2:1619677448.624064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.624069:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:1.0:1619677448.624070:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:30.0:1619677448.624071:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:1.0:1619677448.624072:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:30.0:1619677448.624073:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.624073:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.624073:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677448.624074:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.624075:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.624078:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.624084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.624091:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.624093:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.624094:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.624096:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.624099:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.624100:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.624101:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.624103:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.624104:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.624105:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.624108:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.624110:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.624113:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.624113:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.624114:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.624116:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.624116:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.624117:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.624117:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.624122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.624129:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.624132:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.624133:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.624134:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.624136:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.624137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:22.0:1619677448.624139:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677448.624139:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.624142:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.624144:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.624145:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.624147:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.624148:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:21.2:1619677448.624149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:20.0:1619677448.624150:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.624151:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.624152:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.624153:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.624154:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:20.0:1619677448.624157:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.624157:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.624158:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677448.624159:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.624160:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.624162:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.624162:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.624163:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.624163:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:6.0:1619677448.624164:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:20.0:1619677448.624165:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:6.0:1619677448.624165:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.624166:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:6.0:1619677448.624166:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.624167:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.624167:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:6.0:1619677448.624167:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.624168:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:25.2:1619677448.624171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:6.0:1619677448.624171:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.624173:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.624176:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.624176:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.624177:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.624178:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:6.0:1619677448.624179:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.624179:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.624180:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:31.0:1619677448.624181:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:6.0:1619677448.624181:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000400:00000200:31.0:1619677448.624182:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.624182:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:31.0:1619677448.624183:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.624185:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:8.2:1619677448.624185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.624186:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.624187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:25.2:1619677448.624192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.624192:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:5.0:1619677448.624193:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:22.0:1619677448.624195:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.624195:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.0:1619677448.624195:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677448.624196:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:5.0:1619677448.624196:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.624198:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.624199:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.624199:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.624201:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677448.624202:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:8.2:1619677448.624202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.624203:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.624204:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.624205:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.624206:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.0:1619677448.624209:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.624212:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.624213:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.624214:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.624214:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.624216:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.624273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.624279:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.624282:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.624283:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.624285:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.624287:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.624288:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.624290:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.624291:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.624292:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.624298:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.624302:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.624304:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.2:1619677448.624305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:18.0:1619677448.624307:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.624308:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:18.0:1619677448.624308:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.624309:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.624311:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.624311:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:4.2:1619677448.624311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677448.624312:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:18.0:1619677448.624312:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.624313:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:20.0:1619677448.624315:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.624315:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.624317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.624318:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677448.624318:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.624320:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:26.0:1619677448.624321:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:20.0:1619677448.624321:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.624322:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677448.624323:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.624324:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.624325:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:26.0:1619677448.624326:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.624326:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:13.0:1619677448.624328:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:26.0:1619677448.624329:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.624329:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677448.624329:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.624331:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:13.0:1619677448.624331:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.624332:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96712502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.624332:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:12.0:1619677448.624332:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:26.0:1619677448.624334:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.624334:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677448.624335:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.624335:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.624336:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.624336:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677448.624336:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.624338:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:26.0:1619677448.624339:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.624339:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.624340:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.624340:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:26.0:1619677448.624341:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.624344:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677448.624344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.624345:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.624346:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.624347:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.624348:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.624348:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.624349:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.624351:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.624354:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:25.2:1619677448.624354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:30.0:1619677448.624355:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.624357:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.624359:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.624360:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677448.624361:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.624363:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:25.2:1619677448.624363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000400:00000200:31.0:1619677448.624364:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.624365:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.624365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.624368:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.624370:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.624371:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.624372:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.624372:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.624373:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:30.0:1619677448.624373:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.624374:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.624374:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:31.0:1619677448.624375:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.624375:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:31.0:1619677448.624377:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.624377:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.624379:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.624380:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.624382:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.624383:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.624384:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.624387:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.624389:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.624392:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.624393:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.624393:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.624395:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.624396:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.624397:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.624397:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.624402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.624409:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.624412:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.624413:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.624414:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.624415:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.624417:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.624562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.624568:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:4.2:1619677448.624570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677448.624571:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.624572:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.624574:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.624576:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.624577:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677448.624577:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.624579:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.624580:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.624580:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:6.0:1619677448.624580:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.624582:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.624583:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.624584:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.624587:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.624587:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.624588:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:6.0:1619677448.624589:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.624590:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.624591:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.624591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:6.0:1619677448.624592:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.624593:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.624593:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:10.0:1619677448.624594:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:6.0:1619677448.624594:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.624595:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.624596:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677448.624597:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.624597:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.624597:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677448.624597:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.624599:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677448.624600:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.624601:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.624601:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677448.624601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.624602:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.624602:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:1.0:1619677448.624603:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.624604:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.624604:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.624605:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.624606:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000400:00000200:1.0:1619677448.624606:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.624607:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677448.624607:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:20.0:1619677448.624608:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:1.0:1619677448.624608:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96712498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.624610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.624610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:1.0:1619677448.624610:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.624611:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:1.0:1619677448.624611:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.624612:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.624612:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:4.2:1619677448.624613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677448.624615:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.624615:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:1.0:1619677448.624616:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.624618:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:30.0:1619677448.624619:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.624619:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677448.624619:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.624620:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.624621:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.624621:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.624621:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:30.0:1619677448.624622:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:10.0:1619677448.624622:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:1.0:1619677448.624622:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:26.0:1619677448.624623:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.624623:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:20.2:1619677448.624624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677448.624624:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677448.624624:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.624624:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677448.624625:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.624625:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:1.0:1619677448.624625:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.624626:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677448.624626:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.624627:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.624627:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:30.0:1619677448.624628:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.624629:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.624630:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712497] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.624630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.624630:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677448.624632:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.624632:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:30.0:1619677448.624633:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.624634:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.624635:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.624635:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.624637:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:30.0:1619677448.624638:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.624638:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.624639:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:30.0:1619677448.624640:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.624640:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.624640:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677448.624641:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96712496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.624641:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677448.624643:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.624643:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.624643:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.624644:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:5.0:1619677448.624644:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.624645:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.624645:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:5.0:1619677448.624645:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:30.0:1619677448.624646:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.624647:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.624648:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677448.624648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:30.0:1619677448.624649:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:5.0:1619677448.624649:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.624650:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.2:1619677448.624652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.624653:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.624653:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.624654:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.624655:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:5.0:1619677448.624657:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (24)--
00000800:00000200:12.0:1619677448.624658:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.624658:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.624658:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677448.624658:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.624659:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:5.0:1619677448.624659:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677448.624659:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (23)++
00000800:00000200:1.0:1619677448.624660:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:12.0:1619677448.624661:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.624661:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.624661:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.2:1619677448.624662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:12.0:1619677448.624663:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.624665:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.624665:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677448.624670:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.624672:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.624673:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.624673:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.624675:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.624677:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.624884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.624891:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.624894:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.624894:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.624896:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.624898:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.624899:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.624901:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.624902:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.624903:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.624904:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.624907:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.624909:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.624912:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.624913:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.624914:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.624916:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.624916:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.624917:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.624917:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.624922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.624929:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:21.2:1619677448.624929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:22.0:1619677448.624932:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.624932:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.624933:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.624935:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.624935:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:22.0:1619677448.624938:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677448.624938:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.624939:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.624941:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.624943:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.624944:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.624946:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.624947:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.624948:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.624949:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.624952:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.624954:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.624956:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.624957:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.624957:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.624959:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.624959:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.624960:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.624960:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.624965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.624973:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.624974:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.624975:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.624976:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.624977:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.624979:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.624985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:9.2:1619677448.624986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:1.0:1619677448.624990:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.624993:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.624993:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.624994:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.624996:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.624996:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.624997:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.624997:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.624998:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.624998:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:1.0:1619677448.624998:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.624999:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.625000:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.625002:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.625003:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.625004:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.625006:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.625008:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.625010:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.625011:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.625011:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.625012:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.625013:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.625014:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.625014:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.625018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.625025:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.625027:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.625028:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.625028:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.625029:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.625030:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.625064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.625071:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.625074:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.625074:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.625077:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.625079:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.625081:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.625082:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.625083:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.625084:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.625085:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.625089:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.625091:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.625093:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.625094:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.625094:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.625096:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.625096:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.625097:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.625098:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.625099:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.625102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.2:1619677448.625104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.625108:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.625110:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.625111:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.625111:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:31.0:1619677448.625113:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.625114:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.625114:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.625115:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.625115:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677448.625116:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.625117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.625124:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.625127:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:4.2:1619677448.625128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:30.0:1619677448.625129:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.625131:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.625131:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.625132:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.625134:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.625135:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.625135:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:30.0:1619677448.625136:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.625137:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677448.625138:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.625139:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.2:1619677448.625139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:30.0:1619677448.625141:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.625142:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.625143:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.625145:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677448.625145:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.625146:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.625146:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:8.0:1619677448.625146:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:31.0:1619677448.625147:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:21.2:1619677448.625147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:8.0:1619677448.625147:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96712490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.625148:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:30.0:1619677448.625148:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.625148:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.625149:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677448.625149:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.625150:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.625151:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.625151:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:8.0:1619677448.625151:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:30.0:1619677448.625152:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.625153:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.625153:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.625153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.625153:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:8.0:1619677448.625154:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.625155:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.625156:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96712489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.625156:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:8.0:1619677448.625156:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.2:1619677448.625157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.625157:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:31.0:1619677448.625158:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.625159:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.625159:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:31.0:1619677448.625160:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.625160:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677448.625160:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.625161:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.625161:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.625161:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:13.0:1619677448.625162:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:18.0:1619677448.625163:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:13.0:1619677448.625163:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.625163:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677448.625164:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:18.0:1619677448.625164:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96712488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.625164:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:31.0:1619677448.625165:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.625165:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:13.0:1619677448.625165:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.625165:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:30.0:1619677448.625166:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677448.625166:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.625166:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677448.625166:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:30.0:1619677448.625167:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:18.0:1619677448.625167:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.625167:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677448.625168:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:18.0:1619677448.625168:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:13.0:1619677448.625168:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677448.625169:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677448.625170:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.625171:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.625171:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:18.0:1619677448.625172:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:31.0:1619677448.625173:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.625173:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:4.2:1619677448.625173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677448.625174:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:18.0:1619677448.625174:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.625175:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:18.0:1619677448.625176:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.625177:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.625177:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:25.2:1619677448.625178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.625178:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677448.625179:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677448.625179:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.625179:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.625180:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677448.625180:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:18.0:1619677448.625181:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677448.625181:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.625182:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.625182:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677448.625182:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.625183:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677448.625184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677448.625184:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.625185:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.625187:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677448.625187:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:12.0:1619677448.625188:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.625189:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.625191:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677448.625191:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:12.0:1619677448.625192:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677448.625194:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.625195:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.625196:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.625197:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.625199:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.625233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.625239:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.625242:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.625242:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.625244:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.625247:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.625249:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.625250:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.625251:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.625252:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.625253:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.625257:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.625259:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.2:1619677448.625261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:6.0:1619677448.625262:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.625263:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.625263:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.625264:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.625265:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.625266:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:6.0:1619677448.625266:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.625267:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.625267:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:20.0:1619677448.625268:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.625269:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.625271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:20.0:1619677448.625271:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.625272:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677448.625274:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.625275:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.625276:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.625277:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:30.0:1619677448.625278:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:20.0:1619677448.625280:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.625281:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.625282:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.625282:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.625284:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.625284:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.625285:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.625286:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:20.0:1619677448.625286:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677448.625287:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.0:1619677448.625288:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677448.625289:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.625290:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.625290:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.2:1619677448.625295:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.2:1619677448.625299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.625305:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.625306:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.625308:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:17.0:1619677448.625309:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.625309:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.625310:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.625312:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.625312:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.625314:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.625315:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.625316:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.625317:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.625319:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.625321:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.625322:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.625323:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.625323:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.625324:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.625325:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.625325:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.625326:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.625327:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.625328:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.625328:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677448.625329:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.625330:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.625332:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.625332:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.625333:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.625333:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.625339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.625346:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.625349:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.625350:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.625350:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.625352:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.625355:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.625361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:6.0:1619677448.625368:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.625371:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.625372:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.625372:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.625373:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.625373:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.625405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.625413:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.625415:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.625416:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.625418:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.625420:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.625422:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.625423:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.625425:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.625426:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.625427:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.625430:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.625432:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.625434:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.625435:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.625436:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.625437:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.625438:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.625439:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.625439:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.625443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.625450:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.625453:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.625453:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.625455:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.625457:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.625458:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.625486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.625493:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.625496:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.625496:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.625498:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.625500:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.625502:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.625503:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.625504:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.625505:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.625506:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.625509:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.625511:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.625514:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.625515:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.625516:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.625518:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.625518:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.625519:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.625519:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.625523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.625531:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.625534:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.625534:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.625536:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.625538:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.625539:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.625543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.625550:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.625553:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.625553:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.625554:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.625555:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.625556:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.625621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.625628:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.625631:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.625633:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.625635:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.625635:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.625636:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.625638:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.625639:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.625640:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.625641:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.625644:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.625646:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.625649:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.625650:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.625650:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.625652:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.625652:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.625654:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.625655:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.625656:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.625657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.625661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677448.625663:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.625666:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.625666:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.625668:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.625669:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:12.0:1619677448.625670:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.625671:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:13.0:1619677448.625671:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.625671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:30.0:1619677448.625672:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.625673:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:2.0:1619677448.625677:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.625680:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.625681:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.625683:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.625685:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.625687:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.625688:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.625689:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.625690:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.625691:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.625694:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.625696:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.625698:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.625699:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.625699:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.625701:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.625701:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.625703:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.625704:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.625704:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.625707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.625709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677448.625713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677448.625715:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.625717:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.625717:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677448.625717:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677448.625718:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677448.625718:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.625719:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:1.0:1619677448.625720:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.625721:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.625722:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.625724:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.625725:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.625726:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.625726:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.625727:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.625727:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000400:00000200:17.0:1619677448.625728:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:1.0:1619677448.625728:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.625729:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.625729:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.625730:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.625731:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.625731:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:17.0:1619677448.625732:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.625733:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.625737:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.625739:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.625742:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.625743:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.625743:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.625745:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.625746:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.625747:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.625748:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.625748:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.625751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.625753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.625758:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677448.625758:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677448.625760:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677448.625760:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:12.0:1619677448.625761:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.625761:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.625762:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677448.625763:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.625764:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.625766:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.625844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677448.625852:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677448.625854:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677448.625856:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.625858:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677448.625859:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.625861:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677448.625862:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677448.625863:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.625864:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677448.625865:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677448.625868:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.625870:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677448.625873:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.625874:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:13.0:1619677448.625874:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677448.625876:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:13.0:1619677448.625877:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677448.625878:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677448.625880:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677448.625880:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:8.2:1619677448.625882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.625886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:1.0:1619677448.625890:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.625892:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.625893:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677448.625893:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000400:00000200:1.0:1619677448.625894:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.625896:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677448.625896:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:1.0:1619677448.625896:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.625897:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:1.0:1619677448.625898:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.625979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.625987:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.625990:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.625991:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.625992:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.625995:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.625997:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.625998:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.625999:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.626000:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.626001:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677448.626004:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.626006:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.626009:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.626009:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.626010:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.626012:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.626012:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.626013:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.626013:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.626018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.626026:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.626029:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.626030:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.626032:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.626032:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.626034:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.626041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.626048:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.626050:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.626051:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.626052:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.626052:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.626054:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.626223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.626231:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.626234:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.626236:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.626238:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.626238:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.626240:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.626241:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.626243:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.626244:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.626245:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.626248:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.626250:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.626252:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.626253:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.626253:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.626255:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.626256:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.626257:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.626257:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.626261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.626267:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.626270:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.626271:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.626272:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.626274:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.626275:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.626277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:13.0:1619677448.626284:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.626286:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.626287:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.626288:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.626288:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.626289:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.626313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.626321:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.626323:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.626324:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.626326:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.626328:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.626330:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.626331:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.626333:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.626334:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.626335:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.626338:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.626339:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:21.2:1619677448.626341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677448.626342:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.626343:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.626343:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.626345:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.626346:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.626346:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.626347:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677448.626349:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.626352:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:9.2:1619677448.626352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677448.626354:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.626356:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.626357:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:30.0:1619677448.626357:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.626359:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.626360:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.626360:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:31.0:1619677448.626361:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.626362:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:2.0:1619677448.626363:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.626364:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.626364:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.626365:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.626366:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:2.0:1619677448.626367:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.626369:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.626370:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.626370:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:2.0:1619677448.626370:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677448.626372:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.626373:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.626373:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.626374:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.626378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.626385:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.626387:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.626388:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.626388:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.626390:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.626393:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.626397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677448.626402:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.626403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:24.0:1619677448.626405:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.626405:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677448.626406:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.626407:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.626407:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:18.0:1619677448.626408:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.626411:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.626412:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.626414:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.626416:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.626417:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.626419:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.626420:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.626421:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.626422:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.626425:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.626427:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.626430:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.626430:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.626431:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.626433:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.626433:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.626434:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.626434:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.626439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.626447:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.626450:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.626451:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.626451:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.626453:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.626455:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.626497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.626505:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.626507:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.626508:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.626510:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.626513:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.626514:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.626516:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.626517:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.626518:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.626519:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:5.2:1619677448.626521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677448.626522:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.626524:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.626526:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.626527:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.626528:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.626528:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.626529:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:26.0:1619677448.626530:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.626530:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.626530:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.626531:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.626532:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:1.0:1619677448.626532:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.626535:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.626536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:1.0:1619677448.626536:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.626537:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.626538:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.626539:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.626540:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.626543:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:1.0:1619677448.626543:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.626545:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.626546:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677448.626546:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.626547:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.626547:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.626547:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.626547:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.626549:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.626549:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677448.626550:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.626550:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.626551:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.626553:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.2:1619677448.626555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.626562:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.626564:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.626565:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.626566:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.626567:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.626568:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.626601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.626608:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.626611:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.626612:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.626614:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.626616:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.626618:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.626619:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.626620:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.626622:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.626623:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.626626:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.626628:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.626631:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.626631:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.626632:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.626634:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.626634:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.626635:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.626635:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.626640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.626647:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.626650:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.626650:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.626651:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.626653:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.626655:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.626664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.626671:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.626674:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.626675:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.626677:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.626680:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.626681:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.626682:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.626684:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.626685:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.626685:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.626689:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.626690:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.626693:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.626694:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.626695:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.626696:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.626697:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.626697:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.626698:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.626702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.626710:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:21.2:1619677448.626712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:5.0:1619677448.626713:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.626715:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.626716:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.626717:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.626718:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677448.626719:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:2.0:1619677448.626720:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.626721:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.626723:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.626726:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.626727:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677448.626729:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.626730:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.626731:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.626732:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.626735:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.626737:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.626739:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677448.626740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677448.626740:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.626741:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.626742:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.626743:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.626743:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.626744:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:20.0:1619677448.626745:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677448.626745:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.626746:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:20.0:1619677448.626748:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.626748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:20.0:1619677448.626750:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.626750:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.2:1619677448.626751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000400:00000200:20.0:1619677448.626751:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:5.0:1619677448.626752:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:20.0:1619677448.626753:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.626753:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.626754:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.626754:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.626755:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.626756:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.626757:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:5.0:1619677448.626757:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.626759:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:5.0:1619677448.626759:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:20.0:1619677448.626760:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.626760:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.0:1619677448.626760:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:2.0:1619677448.626761:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677448.626762:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.626764:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.626765:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.626766:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.626768:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.626768:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.626770:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.626771:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.626772:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.626774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.626776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.626781:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.626782:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:6.0:1619677448.626783:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677448.626784:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:22.0:1619677448.626785:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.626785:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.626785:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.626786:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:6.0:1619677448.626787:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.626789:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.626883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.626888:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.626891:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.626893:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.626893:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.626896:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.626897:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.626899:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.626900:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.626901:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.626902:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.626905:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.626907:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.626910:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.626910:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.626911:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.626912:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.626913:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.626914:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.626914:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677448.626918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.626926:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.626929:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.626929:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.626930:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.626932:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.626934:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.626938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.626945:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.626948:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.626948:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.626949:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.626950:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.626950:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.626953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.626960:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.626963:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.626965:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.626967:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.626968:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.626969:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.626970:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.626972:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.626972:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.626973:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.626977:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.626978:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.626980:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.626981:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.626982:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.626983:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.626984:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.626985:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.626985:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.626989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:21.2:1619677448.626993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.626998:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.626999:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.627000:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.627001:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.627001:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.627002:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677448.627003:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.627004:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:30.0:1619677448.627004:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.627005:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.627006:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.627008:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.627009:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.627010:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.627011:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.627012:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.627015:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.627016:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.627019:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.627020:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.627020:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.627022:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.627022:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.627023:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.627024:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.627028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.627035:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.627038:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.627039:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.627040:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.627042:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.627045:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.627050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.627057:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.627060:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.627060:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.627061:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.627062:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.627063:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.627190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.627197:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.627200:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.627201:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.627203:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.627205:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.627207:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.627208:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.627209:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.627210:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.627211:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.627214:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.627216:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.627219:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.627220:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.627220:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.627222:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.627222:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.627224:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.627225:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.627226:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.627228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.627232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.627236:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.627239:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:10.0:1619677448.627239:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:20.0:1619677448.627240:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.627241:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.627241:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677448.627242:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.627242:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.627243:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677448.627245:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.627281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.627287:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.627290:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.627291:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.627293:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.627300:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.627301:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.627302:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.627303:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.627305:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.627306:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.627309:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.627310:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.627312:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.627313:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.627313:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.627315:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.627316:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.627317:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.627317:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.627322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.627329:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.627331:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.627331:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.627332:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.627333:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.627334:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.627363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:5.0:1619677448.627368:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:5.0:1619677448.627371:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:5.0:1619677448.627373:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.627374:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:5.0:1619677448.627376:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.627377:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:5.0:1619677448.627378:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.627380:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.627381:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.627382:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:5.0:1619677448.627385:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.627387:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.627389:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.627390:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.627390:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.627392:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.627393:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:5.0:1619677448.627393:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:5.0:1619677448.627394:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677448.627398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.627404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677448.627405:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.627407:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.627408:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.627410:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.627411:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.627412:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677448.627412:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:26.0:1619677448.627415:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.627416:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.627418:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.627421:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.627422:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.627423:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.627424:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.627426:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.627427:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.627430:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.627431:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.627434:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.627435:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.627436:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.627437:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.627438:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.627438:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.627439:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.627444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.627452:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.627455:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.627456:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.627456:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.627458:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.627461:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.627468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.627475:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.627478:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.627479:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.627481:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.627483:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.627485:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.627486:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.627487:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.627488:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.627489:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677448.627492:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.627493:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.627496:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.627497:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.627497:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.627499:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.627499:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.627500:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.627501:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.627505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.627513:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.627515:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.627516:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.627518:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.627518:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.627520:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.627523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.627530:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.627532:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.627533:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.627534:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.627534:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.627536:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.627561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.627567:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.627570:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.627571:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.627573:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.627575:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.627577:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677448.627578:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.627579:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.627580:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.627581:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677448.627585:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.627586:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677448.627589:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.627590:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677448.627591:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677448.627593:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677448.627593:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.627594:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.627594:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.627598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.627604:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.627607:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.627608:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.627608:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.627610:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.627613:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677448.627614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.627622:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.627625:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.627627:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.627629:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.627629:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.627631:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.627632:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.627634:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.627634:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.627636:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.627639:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.627641:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.627643:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.627644:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.627645:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.627646:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.627647:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.627648:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.627648:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.627652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.627659:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.627661:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.627662:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.627664:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.627666:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.627667:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.627832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.627839:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.627842:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.627843:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.627844:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.627847:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.627849:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.627850:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.627851:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.627852:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.627853:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.627856:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.627857:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.627860:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.627861:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.627861:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.627863:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.627864:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.627864:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.627865:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.627869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.627877:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.627880:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.627881:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.627881:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.627883:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.627886:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.628008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.628014:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.628017:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.628017:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.628019:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.628022:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.628024:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.628025:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.628026:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.628027:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.628028:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.628031:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.628033:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.628035:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.628036:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.628036:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.628038:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.628038:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.628039:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.628039:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.628045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.628052:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.628054:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.628055:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.628056:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.628057:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.628058:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.628093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.628101:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.628103:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.628104:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.628106:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.628109:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.628110:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.628112:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.628113:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.628114:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.628115:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.628119:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.628120:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.628123:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.628124:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.628125:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.628126:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.628127:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.628127:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.628128:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.628133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.628140:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.2:1619677448.628141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677448.628143:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.628143:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.628144:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.628146:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.628147:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.628149:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677448.628150:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.628151:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.628153:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.628154:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.628156:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.628157:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.628158:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.628159:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.628160:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.628164:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.628166:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.628169:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.628170:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.628170:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:21.2:1619677448.628172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677448.628172:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.628173:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.628173:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.628174:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.628178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.628179:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.628182:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.628184:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.628184:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:31.0:1619677448.628186:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.628187:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.628187:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.628188:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.628189:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677448.628190:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.628190:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.628191:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:30.0:1619677448.628192:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.628193:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.628193:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:30.0:1619677448.628194:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677448.628197:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.628199:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.628202:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.628203:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.628204:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.628205:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.628206:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.628207:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.628209:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.628209:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677448.628211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.628215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.628219:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.628222:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:31.0:1619677448.628223:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:17.0:1619677448.628223:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.628223:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.628225:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:17.0:1619677448.628225:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.628226:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.628227:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677448.628229:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.628287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677448.628294:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.628300:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.628303:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677448.628305:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.628309:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.628310:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677448.628311:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.628313:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.628314:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.628315:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.628318:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.628320:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.628323:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.628324:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.628325:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.628326:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.628327:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.628327:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.628328:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677448.628333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.628341:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.628344:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.628345:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.628347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:20.0:1619677448.628347:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.628347:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.628351:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.628356:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:8.0:1619677448.628360:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.628362:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.628363:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677448.628363:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:8.0:1619677448.628365:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.628366:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.628367:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.628368:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.628368:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:8.0:1619677448.628368:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.628369:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:8.0:1619677448.628370:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.628371:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.628372:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.628373:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.628374:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.628377:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.628379:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.628381:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677448.628382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.0:1619677448.628382:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.628383:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.628385:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677448.628385:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.628385:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.628386:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:20.0:1619677448.628387:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:8.0:1619677448.628387:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.628388:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:20.0:1619677448.628389:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.628390:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.628391:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.628392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677448.628393:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:21.2:1619677448.628394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000400:00000200:20.0:1619677448.628395:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677448.628395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:20.0:1619677448.628397:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.628399:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.628400:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:20.0:1619677448.628400:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:2.0:1619677448.628400:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.628402:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.2:1619677448.628402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:24.0:1619677448.628403:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:20.0:1619677448.628403:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.628403:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:24.0:1619677448.628404:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:18.0:1619677448.628404:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.628404:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.628405:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:20.0:1619677448.628405:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.0:1619677448.628405:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:10.0:1619677448.628406:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677448.628407:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.628408:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677448.628408:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.628409:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.628409:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.628409:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:2.0:1619677448.628410:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677448.628411:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.628411:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:20.0:1619677448.628412:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.628412:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.628412:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:26.0:1619677448.628413:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.628413:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.628413:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677448.628414:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677448.628414:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.628415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:6.0:1619677448.628415:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.628416:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.628417:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.628418:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.628418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:6.0:1619677448.628418:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.628419:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.628419:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.628421:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677448.628421:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.628423:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.628423:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:22.0:1619677448.628424:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.628425:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.628426:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.628427:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:18.0:1619677448.628427:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.628427:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.628428:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.628429:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677448.628430:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.628430:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.628432:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.628432:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.628432:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.628434:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:26.0:1619677448.628435:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.628435:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.628436:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.628436:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.628436:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:31.0:1619677448.628437:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.628438:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.628438:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.628438:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.628438:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96712447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.628439:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.628439:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.628440:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:6.0:1619677448.628440:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.628441:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.628442:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.628442:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677448.628442:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.628443:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.628443:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.628445:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.628446:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:6.0:1619677448.628446:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.628447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.628447:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.628448:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.628448:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.628449:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)++
00000800:00000200:22.0:1619677448.628451:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (24)--
00000800:00000200:6.0:1619677448.628451:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.628452:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.628452:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.628452:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.628453:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.628454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.628454:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.628455:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.628455:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.628456:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.628457:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.628458:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.628458:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.628458:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.628459:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.628460:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.628461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.628463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677448.628463:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:22.0:1619677448.628466:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677448.628466:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677448.628467:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:5.0:1619677448.628467:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.628468:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.628469:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677448.628469:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.628469:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.628469:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677448.628470:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677448.628472:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:1.0:1619677448.628472:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.628473:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.628473:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.628475:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.628477:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.628478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.628486:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.628488:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.628489:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.628489:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.628490:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.628491:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.628549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb3800] (20)++
00000800:00000200:1.0:1619677448.628555:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:17.0:1619677448.628558:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (22)--
00000800:00000200:1.0:1619677448.628558:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.65@o2ib3
00000400:00000200:1.0:1619677448.628560:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.65@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.628562:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.628564:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.628565:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.628567:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.628568:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.628569:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.628572:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.65@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.628574:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.628577:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.628578:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.628578:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.628580:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.628580:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:1.0:1619677448.628581:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb3800] (22)--
00000800:00000200:1.0:1619677448.628581:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)--
00000800:00000200:25.2:1619677448.628586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.628593:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.628596:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.628597:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.628597:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.628599:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.628601:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.628607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:6.0:1619677448.628614:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.628616:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.628617:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.628617:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.628618:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.628619:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.628632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.628639:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.628641:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.628642:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.628644:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.628647:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.628649:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.628650:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.628651:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.628652:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.628653:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.628657:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.628658:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.628661:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.628662:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.628663:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.628664:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.628665:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.628666:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.628666:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.628670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.628678:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.628680:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:5.2:1619677448.628681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677448.628681:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.628682:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.628683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:17.0:1619677448.628684:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.628686:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677448.628687:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677448.628689:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:10.0:1619677448.628689:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:20.0:1619677448.628691:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.628692:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.628692:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:5.0:1619677448.628692:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.628694:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.628695:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.628696:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677448.628697:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.628697:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.628698:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.628699:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:20.0:1619677448.628700:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.628701:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.628701:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.628704:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.628705:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.628706:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.628706:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.628708:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.628710:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.628711:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.628712:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.628712:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.628713:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.628714:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.628714:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.628715:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:10.0:1619677448.628715:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:20.0:1619677448.628716:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:10.0:1619677448.628716:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.628717:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:10.0:1619677448.628717:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.628718:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.628719:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:25.2:1619677448.628720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.628720:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.628721:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.628724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.628726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.628727:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.628729:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.628730:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.628732:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.628732:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677448.628733:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.628734:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:13.0:1619677448.628734:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:31.0:1619677448.628735:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:13.0:1619677448.628735:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.628736:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.628736:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.628737:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:13.0:1619677448.628737:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.628739:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.628740:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.628793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.628799:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.628802:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.628803:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.628805:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.628807:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.628808:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.628809:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.628810:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.628811:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.628812:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.628815:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.628817:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.628819:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.628820:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.628821:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.628822:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.628823:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.628823:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.628824:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.628828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.628835:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.628837:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.628838:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.628839:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.628840:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.628841:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.628842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.628849:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.628852:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.628853:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.628855:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.628858:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.628859:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.628860:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.628861:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.628862:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.628863:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.2:1619677448.628866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677448.628867:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.628869:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.628871:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.628872:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.628872:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.628873:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677448.628874:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.628875:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.628875:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.628875:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.628876:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.628877:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:1.0:1619677448.628877:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.628879:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.628881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:1.0:1619677448.628881:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.628882:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.628883:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.628884:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.628885:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:22.0:1619677448.628888:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:1.0:1619677448.628888:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.628890:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:22.0:1619677448.628891:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.628892:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.628892:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677448.628894:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.628895:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.628895:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.628895:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.628897:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:22.0:1619677448.628898:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677448.628898:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.628899:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.628900:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.628901:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.628903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.2:1619677448.628905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.628909:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:17.0:1619677448.628911:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677448.628912:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.628913:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.628913:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:8.0:1619677448.628913:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.628914:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.628914:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.628915:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.628919:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.2:1619677448.628934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.628940:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.628942:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.628945:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.628945:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.628948:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.628950:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.628951:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.628952:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.628953:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.628954:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.628958:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.628960:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.628963:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.628963:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.628964:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.628966:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.628967:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.628968:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.628968:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.628972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.628979:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.628982:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.628983:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.628985:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.628985:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.628988:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.628991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.628997:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.628999:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.629000:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.629002:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.629004:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.629006:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.629008:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.629009:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.629010:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.629011:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.629014:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.629016:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.629019:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.629019:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.629020:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.629022:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.629023:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.629023:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.629024:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.629028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.629034:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.629037:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:21.2:1619677448.629037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677448.629037:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.629038:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.629040:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.629043:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:31.0:1619677448.629044:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.629047:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.629049:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.629050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000800:00000200:30.0:1619677448.629051:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.629052:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.629053:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.629055:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.629056:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.629056:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000400:00000200:31.0:1619677448.629057:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.629058:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.629059:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.629060:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.629060:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000400:00000200:31.0:1619677448.629061:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.629061:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.629061:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:31.0:1619677448.629063:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.629066:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.629066:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.629067:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.629068:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.629069:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.629070:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.629070:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677448.629072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:25.2:1619677448.629074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.629080:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677448.629080:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.629082:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:13.0:1619677448.629083:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.629083:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.629084:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.629085:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.629086:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.629087:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.629087:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.629088:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:30.0:1619677448.629089:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.629090:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.629092:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.629093:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.629094:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.2:1619677448.629095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:30.0:1619677448.629097:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.629098:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.629101:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.629102:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.629102:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.629103:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.629104:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:30.0:1619677448.629105:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.629105:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.629105:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.629106:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.629107:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:26.0:1619677448.629107:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.629110:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.629111:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:8.2:1619677448.629111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:26.0:1619677448.629112:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.629113:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.629114:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.629115:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.629118:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.629119:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.0:1619677448.629119:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.629121:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:26.0:1619677448.629122:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.629122:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.629123:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.629123:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:20.0:1619677448.629124:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.629125:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.629126:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.629126:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.0:1619677448.629126:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.629127:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.0:1619677448.629127:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:25.2:1619677448.629131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.629138:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.629141:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.629142:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.629142:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.629144:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.629148:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.629325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.629333:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.629335:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.629336:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.629338:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.629341:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.629342:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.629344:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.629345:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.629346:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.629346:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.629349:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.629350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.629351:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.629353:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.629354:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.629354:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.629356:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.629356:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.629356:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.629357:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.629358:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677448.629359:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.629360:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.629361:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.629363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:1.0:1619677448.629364:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.629366:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.629367:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.629368:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.629369:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.629370:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:6.0:1619677448.629371:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.629373:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:1.0:1619677448.629373:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.629374:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.629374:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.629375:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:6.0:1619677448.629377:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.629377:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.629378:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.629379:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677448.629380:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677448.629381:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.629381:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.629382:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.629383:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.629388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.629395:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.629397:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.629398:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.629399:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.629400:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.629401:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.629407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.629412:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.629415:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.629416:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.629418:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.629420:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.629422:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.629423:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.629425:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.629426:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.629427:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.629431:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.629433:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.629435:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.629436:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.629437:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.629439:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.629439:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.629440:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.629440:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.629446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.629454:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.629457:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.629458:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.629459:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.629461:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.629462:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.629494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.629501:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.629503:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.629504:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.629506:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.629509:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.629509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:26.0:1619677448.629510:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.629511:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.629512:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.629513:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.629514:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:5.0:1619677448.629515:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677448.629517:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.629518:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:26.0:1619677448.629519:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677448.629520:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.2:1619677448.629521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:5.0:1619677448.629521:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.629522:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.629522:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.629523:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.629524:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:5.0:1619677448.629524:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.629525:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.629525:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.629526:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:5.0:1619677448.629526:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.629527:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.629529:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:5.0:1619677448.629529:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.629529:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.629531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:5.0:1619677448.629531:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.629532:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:5.0:1619677448.629535:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.629536:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.0:1619677448.629536:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.629537:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:30.0:1619677448.629538:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.629539:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:5.0:1619677448.629539:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.629540:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.629540:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.629540:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:31.0:1619677448.629541:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.629541:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:30.0:1619677448.629542:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.629543:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96712429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.629543:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.629543:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.629544:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677448.629545:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.629545:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:30.0:1619677448.629546:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.629547:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.629549:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677448.629550:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.629550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.629551:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.629552:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:4.2:1619677448.629552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677448.629553:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.629553:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.629554:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.629554:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.629555:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.629556:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.629556:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.629557:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.629557:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:31.0:1619677448.629558:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.629559:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.629559:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.629559:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677448.629559:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677448.629560:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.629561:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677448.629562:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:24.0:1619677448.629562:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:30.0:1619677448.629563:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:8.0:1619677448.629563:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677448.629564:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:24.0:1619677448.629564:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.629565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:24.0:1619677448.629567:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.629568:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.629569:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96712428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.629570:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.629571:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.629572:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.629572:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:24.0:1619677448.629575:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.629575:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.629576:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.629576:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.629577:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.629578:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.629580:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.629580:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.629581:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.629581:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:24.0:1619677448.629583:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.629583:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.629585:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.629586:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.629586:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.629589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.629592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.629597:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677448.629599:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.629600:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.629601:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.629601:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677448.629601:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.629602:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.629603:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.629603:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.629606:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.629641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fe8667000] (20)++
00000800:00000200:24.0:1619677448.629649:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667000] (21)++
00000800:00000200:24.0:1619677448.629653:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.67@o2ib3
00000800:00000200:8.0:1619677448.629653:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667000] (22)--
00000800:00000200:21.2:1619677448.629654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000400:00000200:24.0:1619677448.629655:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.67@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.629658:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.629659:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:24.0:1619677448.629660:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.629660:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:24.0:1619677448.629662:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.629663:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.629663:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000400:00000200:24.0:1619677448.629664:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:2.0:1619677448.629664:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677448.629667:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.629669:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.67@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.629669:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.629671:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:18.0:1619677448.629671:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677448.629672:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712426] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.629674:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.629675:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.629675:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000400:00000200:18.0:1619677448.629675:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.629676:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:18.0:1619677448.629676:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:24.0:1619677448.629678:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:24.0:1619677448.629678:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667000] (21)++
00000800:00000200:24.0:1619677448.629680:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667000] (22)--
00000400:00000200:18.0:1619677448.629680:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.629681:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667000] (21)--
00000800:00000200:18.0:1619677448.629682:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677448.629684:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677448.629684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:18.0:1619677448.629685:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:18.0:1619677448.629686:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:18.0:1619677448.629688:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:18.0:1619677448.629688:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.629689:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677448.629690:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:25.2:1619677448.629693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.629693:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.629695:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.629696:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.629698:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.629699:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:30.0:1619677448.629700:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:20.0:1619677448.629701:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:30.0:1619677448.629703:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.629704:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.629706:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.629707:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.629709:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.629743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.629748:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.629751:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.629752:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.629754:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.629755:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.629756:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.629758:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.629759:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.629760:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.629761:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.629764:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.629765:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.629767:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.629768:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.629769:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.629770:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.629770:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.629771:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.629771:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.629776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.629783:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.629786:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.629786:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.629787:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.629788:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.629789:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.629814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677448.629816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:10.0:1619677448.629821:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.629822:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.629824:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.629824:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:22.0:1619677448.629825:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.629825:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:10.0:1619677448.629828:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.629829:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.629831:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.629832:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:6.0:1619677448.629832:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.629834:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.629835:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.629835:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.629836:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.629837:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.629839:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.629839:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.629840:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677448.629840:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.629847:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.629847:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.629849:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.629850:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.629852:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.629853:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.629853:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.629854:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.629854:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.629855:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.629855:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.629856:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.629857:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.629857:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.629857:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.629858:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677448.629858:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:10.0:1619677448.629859:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677448.629859:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.629861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.629864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:25.2:1619677448.629865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.629868:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.629871:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.629871:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677448.629872:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:22.0:1619677448.629872:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.629872:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.629873:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.629874:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.629874:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.629875:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:26.0:1619677448.629875:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:31.0:1619677448.629876:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.629877:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.629877:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677448.629879:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.629881:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.629882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.629888:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.629891:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.629892:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.629892:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.629893:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.629895:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.630129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.2:1619677448.630133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:31.0:1619677448.630136:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.630138:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:2.0:1619677448.630138:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:31.0:1619677448.630140:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.630141:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:30.0:1619677448.630142:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677448.630142:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:31.0:1619677448.630143:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.630144:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.630145:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.630146:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.630147:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.630148:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.630149:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.630149:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.630150:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.630150:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.630152:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.630153:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.630154:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.630155:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.630158:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.630160:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.630161:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.630161:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.630162:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.630162:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.630163:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.630164:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.630165:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.630165:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677448.630165:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.630166:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.630167:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:25.2:1619677448.630169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:2.0:1619677448.630169:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.630170:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.630170:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.630171:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677448.630177:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677448.630180:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677448.630181:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.630183:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.630185:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.630186:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.630186:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677448.630187:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.630187:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677448.630188:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.630191:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.630195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:25.2:1619677448.630201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.630203:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677448.630205:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:26.0:1619677448.630206:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:12.0:1619677448.630208:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.630209:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.630211:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.630211:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.630212:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.630212:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.630213:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.630213:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:12.0:1619677448.630214:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677448.630214:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.630216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:10.0:1619677448.630216:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.630217:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.630218:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.630221:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.630223:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.630223:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.630226:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:10.0:1619677448.630226:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.630227:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.630228:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.630229:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.630230:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.630230:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.630230:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000400:00000200:30.0:1619677448.630231:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.630231:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677448.630231:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000400:00000200:30.0:1619677448.630232:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.630233:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.630235:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.630236:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.630236:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:8.2:1619677448.630236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.630240:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.630241:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.630244:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.630245:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.630245:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677448.630246:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (24)++
00000800:00000200:30.0:1619677448.630248:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (25)--
00000800:00000200:30.0:1619677448.630249:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.630249:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (24)--
00000800:00000200:30.0:1619677448.630250:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:5.0:1619677448.630250:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.630251:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677448.630252:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.630252:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:5.0:1619677448.630252:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.630254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.630257:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.630258:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.2:1619677448.630260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.630260:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.630261:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.630262:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.630265:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677448.630266:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.630268:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:8.2:1619677448.630268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.630268:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677448.630270:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.630271:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677448.630271:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.630272:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:17.0:1619677448.630273:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.630274:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.630276:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:17.0:1619677448.630276:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.630278:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677448.630278:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.630279:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000400:00000200:17.0:1619677448.630279:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:20.0:1619677448.630280:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.630281:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:17.0:1619677448.630281:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.630282:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.630282:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:17.0:1619677448.630283:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.630284:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.630288:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.630290:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.630292:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.630293:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.630298:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.630300:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.630301:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.630301:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.630302:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.630307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.630314:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.630317:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.630318:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.630320:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.630321:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.630323:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.630334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.630342:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.630344:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.630345:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.630347:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.630350:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.630351:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.630352:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.630354:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.630355:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.630356:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.630359:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.630361:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.630363:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.630364:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.630365:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.630366:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.630367:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.630367:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.630368:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.630372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.630380:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.630383:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.630384:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.630384:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.630386:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.630389:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.630468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.630475:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.630478:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.630479:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.630480:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.630483:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.630484:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.630486:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712416] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.630487:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.630488:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.630488:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.630491:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.630492:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.630494:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.630495:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.630495:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.630497:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.630497:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.630498:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.630499:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.630503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.2:1619677448.630503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.630507:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.630509:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:20.0:1619677448.630509:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.630511:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.630512:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.630512:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.630512:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677448.630514:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.630514:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.630516:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:20.0:1619677448.630516:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677448.630517:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:20.0:1619677448.630517:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.630519:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.630520:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.630521:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.630524:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.630526:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.630529:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.630530:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.630531:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.630533:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.630534:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.630535:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.630536:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.630537:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.630539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.630542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677448.630546:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.630547:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677448.630549:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.630549:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:12.0:1619677448.630550:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.630550:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.630551:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:12.0:1619677448.630552:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.630554:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.630554:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.630556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.630562:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.630565:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.630566:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.630568:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.630571:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.630572:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677448.630574:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.630575:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.630576:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.630577:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677448.630580:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.630582:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.630584:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.630585:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.630585:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.630587:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.630587:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.630588:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.630588:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.630593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.630600:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.630602:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.630603:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.630603:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.630605:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.630606:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.630613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.630620:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.630622:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.630623:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.630624:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.630624:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677448.630624:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (23)--
00000800:00000200:21.2:1619677448.630662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.630667:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.630670:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.630671:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.630673:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.630676:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.630677:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.630678:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.630680:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.630681:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.630682:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.630686:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.630688:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.630691:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.630692:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.630692:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.630694:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.630695:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.630696:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.630697:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.630698:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:25.2:1619677448.630700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:21.2:1619677448.630703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677448.630707:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:6.0:1619677448.630708:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:31.0:1619677448.630710:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:6.0:1619677448.630710:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.630710:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:31.0:1619677448.630711:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.630711:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:31.0:1619677448.630713:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.630713:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.630716:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.630756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.630764:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.630767:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.630769:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.630770:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.2:1619677448.630770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677448.630772:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.630773:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.630774:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.630776:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.630776:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677448.630777:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.630778:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.630779:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.630779:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:30.0:1619677448.630782:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.630783:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.630785:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.630786:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.630788:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.630788:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:30.0:1619677448.630790:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.630790:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:1.0:1619677448.630790:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712411] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.630791:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.630792:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.630792:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:1.0:1619677448.630792:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.630793:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:1.0:1619677448.630793:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:30.0:1619677448.630794:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:1.0:1619677448.630796:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.630797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.630798:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.630801:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.630802:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.630803:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:13.0:1619677448.630804:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677448.630804:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.630805:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.630806:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677448.630806:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:13.0:1619677448.630807:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.630807:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.630808:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677448.630809:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.630810:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:9.2:1619677448.630810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:13.0:1619677448.630811:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.630812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.630817:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677448.630818:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.630820:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.0:1619677448.630820:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.630821:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.630821:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677448.630821:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.630822:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.630823:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.630826:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.630894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.630900:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.630903:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.630904:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.630905:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.630907:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.630909:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.630910:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.630912:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.630913:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.630914:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.630917:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.630919:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.630922:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.630923:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.630924:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.630925:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.630926:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.630927:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.630927:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.630931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.630938:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.630940:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.630941:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.630943:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.630945:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.630946:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.631021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:20.2:1619677448.631024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.631030:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.631032:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677448.631033:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:26.0:1619677448.631035:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.631035:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677448.631037:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.631039:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.631040:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.631041:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.631042:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.631043:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.631044:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.631045:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.631047:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.631049:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.631049:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.631050:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.631051:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.631051:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.631052:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:5.2:1619677448.631053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fe8667000] (20)++
00000800:00000200:26.0:1619677448.631055:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.631056:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:10.0:1619677448.631056:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.631057:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.631057:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.631059:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.631060:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (22)++
00000800:00000200:24.0:1619677448.631060:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667000] (21)++
00000800:00000200:26.0:1619677448.631061:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677448.631061:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.631062:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:24.0:1619677448.631062:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.67@o2ib3
00000800:00000200:10.0:1619677448.631062:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.631062:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.631063:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.0:1619677448.631063:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667000] (22)--
00000800:00000200:26.0:1619677448.631064:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677448.631064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.631064:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:24.0:1619677448.631065:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.67@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.631065:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.631066:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:24.0:1619677448.631067:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.631067:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:24.0:1619677448.631069:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:24.0:1619677448.631070:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96712407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.631070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.631071:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:24.0:1619677448.631072:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.631073:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:24.0:1619677448.631073:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.631074:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.631074:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:31.0:1619677448.631076:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.631077:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.631078:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:24.0:1619677448.631078:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.67@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.631079:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.631079:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:24.0:1619677448.631082:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.631082:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:24.0:1619677448.631083:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000400:00000200:5.0:1619677448.631083:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.631084:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:5.0:1619677448.631084:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677448.631085:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.631086:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:24.0:1619677448.631086:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667000] (21)++
00000800:00000200:24.0:1619677448.631087:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fe8667000] (22)++
00000800:00000200:5.0:1619677448.631087:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:24.0:1619677448.631089:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667000] (23)--
00000800:00000200:24.0:1619677448.631090:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667000] (22)--
00000800:00000200:8.2:1619677448.631092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.2:1619677448.631095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fe8667000] (21)++
00000800:00000200:20.0:1619677448.631099:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.631101:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:20.0:1619677448.631102:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.631102:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667000] (22)++
00000800:00000200:21.2:1619677448.631103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677448.631104:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.631105:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fe8667000] (23)--
00000800:00000200:8.0:1619677448.631105:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667000] (22)--
00000800:00000200:8.0:1619677448.631106:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667000] (21)--
00000800:00000200:5.0:1619677448.631106:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.631107:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:30.0:1619677448.631111:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.631114:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.631116:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.631117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:31.0:1619677448.631118:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.631119:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.631120:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.631122:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.631123:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.631123:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:30.0:1619677448.631124:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.631125:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:20.2:1619677448.631126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677448.631126:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.631127:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:30.0:1619677448.631129:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.631131:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:18.0:1619677448.631132:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.631134:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.631134:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.631135:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:30.0:1619677448.631135:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.631135:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:18.0:1619677448.631135:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.631136:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677448.631137:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.631137:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:31.0:1619677448.631138:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.631138:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712405] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.631139:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:18.0:1619677448.631140:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.631141:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.631141:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.631141:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677448.631141:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.631142:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.631143:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:9.2:1619677448.631143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:31.0:1619677448.631144:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.631145:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96712404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.631146:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.631146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:31.0:1619677448.631147:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.631147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:31.0:1619677448.631148:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.631149:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.631149:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.631152:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:31.0:1619677448.631154:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.631154:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:26.0:1619677448.631154:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.631154:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.631154:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:30.0:1619677448.631155:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.631155:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677448.631155:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:2.0:1619677448.631155:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.631156:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.631156:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.631156:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:18.0:1619677448.631156:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.631157:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677448.631158:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.631158:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:2.0:1619677448.631158:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.631159:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.631160:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.631160:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:26.0:1619677448.631160:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.631161:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:31.0:1619677448.631162:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:26.0:1619677448.631162:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.631162:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.631162:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.631163:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677448.631163:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.631163:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:31.0:1619677448.631164:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677448.631164:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:25.2:1619677448.631164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.631165:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:26.0:1619677448.631165:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96712403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.631166:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.631167:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.631168:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:25.2:1619677448.631168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.631168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000400:00000200:26.0:1619677448.631172:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.631173:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:13.0:1619677448.631173:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.631174:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:13.0:1619677448.631174:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.631175:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.631175:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:18.0:1619677448.631176:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:12.0:1619677448.631176:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:26.0:1619677448.631177:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.631177:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:13.0:1619677448.631177:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.631177:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:26.0:1619677448.631178:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:12.0:1619677448.631178:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.631178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677448.631179:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:12.0:1619677448.631179:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.631180:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.631181:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.631181:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:13.0:1619677448.631181:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.631182:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:12.0:1619677448.631182:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.631183:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.631184:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677448.631185:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.2:1619677448.631186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:24.0:1619677448.631188:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.631189:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677448.631190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:24.0:1619677448.631190:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.631193:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.631194:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:24.0:1619677448.631195:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.631196:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96712402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.631196:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:13.0:1619677448.631196:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:24.0:1619677448.631197:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.631198:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.631198:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.631198:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:24.0:1619677448.631199:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677448.631200:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:22.0:1619677448.631201:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.631201:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.631201:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.631202:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:24.0:1619677448.631203:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.631203:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677448.631204:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.631205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:24.0:1619677448.631205:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.631208:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.631209:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.631209:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.631211:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.631211:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.631212:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.631212:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677448.631213:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.631216:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.631216:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:8.2:1619677448.631217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.631217:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.631218:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.631218:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:17.0:1619677448.631225:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.631228:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.631229:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.631229:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.631231:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.631234:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.631279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.631284:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677448.631287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.631287:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.631289:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.631291:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.631292:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:5.0:1619677448.631292:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.631297:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:20.0:1619677448.631298:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.631298:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.631300:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.631301:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.631302:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677448.631305:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.631307:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:20.0:1619677448.631309:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.631310:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.631311:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.631311:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.631312:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:20.0:1619677448.631313:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:5.0:1619677448.631313:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:20.0:1619677448.631314:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.631316:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.631317:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.631318:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.631318:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.2:1619677448.631319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.631319:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.631321:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.631321:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677448.631322:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.631323:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.631323:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.631324:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.631325:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.631326:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.631327:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.631327:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:5.0:1619677448.631327:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.631328:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.631329:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.631329:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677448.631330:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.631331:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.631333:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.631333:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.631336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.631339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.2:1619677448.631341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.631344:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.631346:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:5.0:1619677448.631346:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:1.0:1619677448.631347:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677448.631348:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.631349:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.631349:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:30.0:1619677448.631350:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.631350:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.631350:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:1.0:1619677448.631350:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677448.631351:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.631353:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.631353:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.631354:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:30.0:1619677448.631354:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.631355:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.631355:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.631356:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:1.0:1619677448.631356:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.631358:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.631359:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.631360:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.631361:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.631362:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.631366:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.631368:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.631370:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.631371:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.631372:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.631373:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.631374:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.631374:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.631375:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.631379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.631388:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.631390:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.631391:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.631391:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.631393:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.631395:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.631455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.631463:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.631466:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.631467:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.631468:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.631471:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.631473:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.631474:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.631476:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.631477:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.631478:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.631481:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.631483:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.631485:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.631486:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.631487:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.631488:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.631489:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.631490:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.631490:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.631494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.631501:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.631504:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.631505:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.631506:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.631508:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.631509:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.631536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.631543:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.2:1619677448.631543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:31.0:1619677448.631546:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.631548:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.631549:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:31.0:1619677448.631550:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.631550:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.631552:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:17.0:1619677448.631552:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677448.631552:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:31.0:1619677448.631555:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.631556:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.631557:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.631557:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.631558:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.631558:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.631560:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677448.631562:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.631563:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.631564:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.631565:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.631566:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:31.0:1619677448.631567:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.631571:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.631571:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:17.0:1619677448.631571:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.631572:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.631573:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.631573:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:31.0:1619677448.631574:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.631575:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.631575:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:17.0:1619677448.631576:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.631577:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:17.0:1619677448.631577:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:25.2:1619677448.631579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.631579:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:17.0:1619677448.631580:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.631581:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677448.631581:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:25.2:1619677448.631585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:12.0:1619677448.631585:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.631587:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.631588:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.631590:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.631592:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.631592:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:13.0:1619677448.631593:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.631595:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.631596:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.631596:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.631598:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.631601:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:4.2:1619677448.631719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.631725:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.631728:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.631728:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.631730:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.631733:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.631734:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.631735:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.631737:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.631738:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.631739:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.631742:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.631744:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.631747:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.631748:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.631748:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.631750:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.631750:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.631751:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.631752:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677448.631755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:25.2:1619677448.631756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.631761:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.631763:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.631764:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.631765:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.631766:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.631767:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.631767:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.631767:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.631769:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.631770:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.631772:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:1.0:1619677448.631772:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.631773:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.631774:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.631775:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.631776:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.631780:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.631782:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.631784:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.631785:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.631785:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.631787:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.631787:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.631788:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.631789:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.631793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.631800:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.631803:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.631804:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.631804:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.631806:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.631808:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.631825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.631832:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.631835:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.631835:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.631838:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.631840:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.631842:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.631843:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.631844:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.631845:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.631846:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.631850:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.631851:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.631854:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.631855:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.631855:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.631857:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.631858:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.631858:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.631859:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.631863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.631870:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.631872:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.631873:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.631875:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.631876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:13.0:1619677448.631877:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.631878:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.631880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.631884:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.631886:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:13.0:1619677448.631886:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677448.631889:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.631889:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.631890:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677448.631890:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:30.0:1619677448.631891:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.631891:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.631891:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:30.0:1619677448.631893:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:13.0:1619677448.631893:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:30.0:1619677448.631894:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.631895:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.631896:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.631897:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677448.631900:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.631902:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677448.631905:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.631906:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:30.0:1619677448.631906:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:30.0:1619677448.631908:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:30.0:1619677448.631908:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.631909:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.631910:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.631914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.631920:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.631922:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.631923:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.631924:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.631925:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.631928:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.632020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.632027:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.632029:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.632030:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.632032:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.632034:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.632035:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.632037:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.632038:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.632039:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.632040:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677448.632043:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.632044:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.632047:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.632048:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.632048:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.632050:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.632050:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.632051:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.632051:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.632056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.632064:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.632067:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.632068:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.632070:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.632070:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.632073:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.632160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.632167:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.632170:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.632171:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.632173:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.632175:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.632177:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.632178:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.632179:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.632180:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.632181:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.632184:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.632186:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.2:1619677448.632188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.0:1619677448.632189:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.632190:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.632190:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.632192:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.632192:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.632192:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.632193:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677448.632194:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.632197:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.632197:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.632198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:17.0:1619677448.632200:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.632203:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.632204:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.632206:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.632207:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.632208:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:17.0:1619677448.632209:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.632209:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.632210:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.632210:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.632212:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.632213:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.632215:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.632215:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.632216:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677448.632218:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.632219:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.632219:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.632221:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:8.2:1619677448.632221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.632222:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.632222:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.632224:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677448.632225:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:25.2:1619677448.632227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.632227:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.632228:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.632228:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.632229:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.632229:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.632234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:6.0:1619677448.632234:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:20.0:1619677448.632237:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.632237:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.632238:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.632238:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.632240:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:6.0:1619677448.632241:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.632242:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.632243:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.0:1619677448.632243:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.632245:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.632246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000400:00000200:20.0:1619677448.632247:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677448.632248:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.632249:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.632250:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.632251:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:22.0:1619677448.632253:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:20.0:1619677448.632254:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.632256:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.632256:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.632257:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:20.0:1619677448.632257:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:22.0:1619677448.632258:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.632258:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677448.632259:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.632260:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.632260:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.632262:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.632263:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.632263:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.632264:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.632268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.632276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:5.0:1619677448.632277:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.632280:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.632280:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.632282:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.632282:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.632282:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.0:1619677448.632284:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:2.0:1619677448.632285:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.632286:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677448.632286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000400:00000200:2.0:1619677448.632288:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.632290:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.632291:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677448.632293:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.632298:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000400:00000200:2.0:1619677448.632299:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.632300:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.632301:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.632301:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:20.0:1619677448.632302:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.632302:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.632303:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.632304:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:2.0:1619677448.632304:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.632305:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677448.632308:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.632309:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677448.632310:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677448.632312:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677448.632312:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.2:1619677448.632314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677448.632314:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.632315:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.632315:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677448.632319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:21.2:1619677448.632321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677448.632322:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.632325:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:10.0:1619677448.632326:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:31.0:1619677448.632327:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.632327:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:31.0:1619677448.632329:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.632329:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:18.0:1619677448.632329:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:10.0:1619677448.632329:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:30.0:1619677448.632330:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:18.0:1619677448.632330:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:10.0:1619677448.632330:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.632331:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:18.0:1619677448.632331:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:31.0:1619677448.632332:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.632333:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000400:00000200:31.0:1619677448.632334:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.632334:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.632335:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.632336:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:31.0:1619677448.632339:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.632340:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.632343:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.632344:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.632344:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.632346:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.632346:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.632348:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.632349:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.632350:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.632351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.632356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.632358:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.632361:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.632361:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.632363:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.632364:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:12.0:1619677448.632364:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.632366:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.632367:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.632368:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.632369:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:21.2:1619677448.632396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.632403:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.632406:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.632408:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.632410:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.632410:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.632411:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677448.632412:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.632414:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.632415:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.632415:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677448.632418:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.632420:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677448.632422:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.632423:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:30.0:1619677448.632423:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:30.0:1619677448.632425:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:30.0:1619677448.632425:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.632426:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.632427:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.632430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.632438:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.632440:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.632441:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.632441:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.632443:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.632445:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.632452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.632459:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.632462:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.632463:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.632464:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.632467:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.632468:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.632469:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.632471:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.632472:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.632472:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.632476:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.632477:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.632480:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.632481:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.632482:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.632484:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.632484:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.632485:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.632487:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.632487:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.632489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.632493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677448.632498:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:10.0:1619677448.632500:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:5.0:1619677448.632500:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.632501:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.632502:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.632503:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:5.0:1619677448.632503:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.632504:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677448.632504:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.632506:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.632591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.632597:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.632600:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.632600:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.632602:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.632605:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.632606:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.632607:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.632609:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.632610:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.632611:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.632614:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.632616:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.632618:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.632619:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.632619:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.632621:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.632621:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.632623:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.632624:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.632624:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.632627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.632629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.632634:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.632635:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.632636:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677448.632637:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677448.632637:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677448.632638:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.632638:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.632638:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.632640:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.632643:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.632732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.632739:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.632742:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.632743:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.632745:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.632748:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.632749:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.632750:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.632752:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.632753:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.632753:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.632756:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.632758:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.632761:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.632762:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.632763:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.632765:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.632765:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.632766:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.632768:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.632768:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677448.632771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.632774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.632779:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.632781:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:2.0:1619677448.632781:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.632783:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:18.0:1619677448.632783:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.632783:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.632784:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.632784:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:2.0:1619677448.632785:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.632789:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.632861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.632865:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.632867:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.632869:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.632870:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.632872:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.632874:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677448.632875:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.632876:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.632877:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.632878:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.632881:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.632883:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.632886:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.632886:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.632887:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.632888:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.632889:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.632890:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.632890:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677448.632895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.632898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:17.0:1619677448.632902:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.632904:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.632905:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.632906:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.632906:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.632908:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.632908:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.632909:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.632911:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.632911:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:24.0:1619677448.632913:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.632915:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.632916:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.632918:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.632919:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.632920:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.632923:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.632923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:24.0:1619677448.632925:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.632928:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.632928:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.632929:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677448.632931:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677448.632931:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.632931:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.632932:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.632932:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:31.0:1619677448.632934:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.632936:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.632936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677448.632938:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.632938:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.632940:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677448.632941:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.632942:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.632943:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.632944:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.632944:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:31.0:1619677448.632946:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.632947:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:31.0:1619677448.632948:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.632948:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.632948:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.632951:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.632951:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.632952:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.632952:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.632953:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:31.0:1619677448.632954:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.632954:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.632955:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.632956:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.632956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:9.2:1619677448.632959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.632964:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.632966:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.632966:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677448.632967:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.632967:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.632968:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.632968:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.632969:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.632969:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.632969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000400:00000200:8.0:1619677448.632971:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.632972:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677448.632973:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:8.0:1619677448.632974:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:17.0:1619677448.632975:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677448.632975:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677448.632978:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.632979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000400:00000200:17.0:1619677448.632980:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.632981:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677448.632982:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.632984:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.632985:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000400:00000200:17.0:1619677448.632985:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.632986:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:24.0:1619677448.632988:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.632989:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.632989:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.632990:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000400:00000200:17.0:1619677448.632990:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.632991:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.632991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:17.0:1619677448.632992:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.632994:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.632995:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:17.0:1619677448.632996:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.632997:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677448.632998:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:8.0:1619677448.632998:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:17.0:1619677448.632999:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677448.632999:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.0:1619677448.633001:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.633002:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.2:1619677448.633004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:8.0:1619677448.633004:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.633006:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.633008:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.633009:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96712377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.633010:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.633011:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:8.0:1619677448.633011:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.633012:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.633014:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.633014:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.633015:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.633016:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.633017:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.633018:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.633020:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677448.633021:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.633022:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.633022:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.633024:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.633024:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.633025:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.633026:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.633030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:25.2:1619677448.633034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677448.633037:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.633039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:6.0:1619677448.633040:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.633041:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:13.0:1619677448.633041:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:6.0:1619677448.633041:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.633042:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.633043:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677448.633044:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.633045:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.633045:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:13.0:1619677448.633047:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.633047:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677448.633047:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:12.0:1619677448.633049:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.633050:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677448.633050:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.633050:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000400:00000200:5.0:1619677448.633050:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.633051:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.633051:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.633052:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:5.0:1619677448.633052:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.633054:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.633055:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.633057:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.633058:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.633059:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.633062:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.633064:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.633067:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.633068:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.633069:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.633071:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.633072:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.633072:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.633073:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.633077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.633084:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.633086:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.633087:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.633089:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.633089:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.633092:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.633166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.633172:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.633175:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.633176:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.633178:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.633180:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.633181:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.633183:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.633184:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.633185:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.633186:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.633189:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.633191:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.633194:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.633195:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.633195:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.633197:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.633198:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.633198:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.633199:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.633203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.633210:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.633212:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.633213:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.633215:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.633216:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.633218:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.633317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.633325:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.633328:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.633328:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.633330:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.633333:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.633334:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.633336:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.633337:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.633338:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.633339:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.633343:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.633344:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.633347:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.633348:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.633349:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.633351:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.633351:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.633352:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.633352:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.633358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.633366:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.633369:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.633370:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.633370:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.633373:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.633375:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.633484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.633492:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.633495:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.633496:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.633497:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.633500:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.633501:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.633503:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.633504:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.633505:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.633506:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.633509:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.633511:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.633514:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.633515:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.633515:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.633517:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.633517:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.633519:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.633520:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.633521:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.633523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.633526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.633532:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677448.633533:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.633534:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.633535:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.633535:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:1.0:1619677448.633535:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.633536:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.2:1619677448.633536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:8.0:1619677448.633537:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677448.633538:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.633541:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677448.633542:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.633545:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.633545:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.633547:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.633550:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.633551:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.633552:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.633554:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.633555:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.633556:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.633559:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.633561:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.633563:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.633563:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.633564:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.633566:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.633566:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.633567:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.633567:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.633571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.633578:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.633580:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.633580:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.633582:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.633583:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.633584:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.633589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:21.2:1619677448.633596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677448.633596:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.633598:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.633599:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.633599:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.633600:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.633600:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.633603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:30.0:1619677448.633604:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.633607:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:18.0:1619677448.633608:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:30.0:1619677448.633609:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.633611:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.633611:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.633611:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677448.633612:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:30.0:1619677448.633613:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.633614:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.633615:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.633616:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.633616:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.633618:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.633618:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.633620:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.633620:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.633621:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.633623:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.633624:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.633626:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.633628:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.633629:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.633631:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.633632:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.633633:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.633633:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.633634:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.633634:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.633635:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.633635:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.633636:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.633636:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.633637:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:18.0:1619677448.633637:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.633638:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677448.633638:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:30.0:1619677448.633639:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.633639:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.633641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:9.2:1619677448.633642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.633645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.633649:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.633650:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:13.0:1619677448.633651:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677448.633652:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:13.0:1619677448.633652:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.633652:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.633653:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.633654:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:13.0:1619677448.633654:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.633655:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.633656:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677448.633656:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677448.633656:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.633657:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.633657:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.633658:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.633731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.633738:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.633741:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.633741:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.633744:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.633746:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.633748:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.633749:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.633750:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.633752:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.633753:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.633756:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.633758:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.633761:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.633762:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.633762:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.633764:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.633765:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.633765:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.633766:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.633770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.633778:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.633781:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.633782:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.633782:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.633785:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.633788:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.633814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.633817:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.633819:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.633822:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.633823:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.633825:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.633826:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.633828:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.633829:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.633830:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.633831:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.633835:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.633836:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.633839:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.633840:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.633841:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.633843:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.633843:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.2:1619677448.633844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677448.633845:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.633846:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.633846:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.633849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.633851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.633852:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.633855:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.633855:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:12.0:1619677448.633856:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.633857:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:26.0:1619677448.633858:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.633859:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.633859:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.633860:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677448.633861:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.633861:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.633861:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:26.0:1619677448.633863:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:13.0:1619677448.633863:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.633863:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.633864:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.633866:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.633866:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677448.633867:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.633867:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.633870:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.633872:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.633874:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.633875:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.633876:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.633877:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.633878:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.633878:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.633879:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.633883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.633890:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.633892:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.633893:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.633894:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.633896:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.633897:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.633902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.633908:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.633911:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.633911:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.633912:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.633912:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.633913:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.633941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.633947:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.633950:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.633951:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.633952:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.633955:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.633956:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.633958:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.633959:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.633960:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.633960:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.633963:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.633965:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.633967:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.633968:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.633969:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.633971:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.633972:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.633973:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.633974:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.633975:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677448.633977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.633980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:18.0:1619677448.633985:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.0:1619677448.633987:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:18.0:1619677448.633988:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.633989:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.633989:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:2.0:1619677448.633989:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.633990:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677448.633990:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:21.2:1619677448.633991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:18.0:1619677448.633991:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.633995:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677448.633996:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.633999:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.634000:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.634002:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.634004:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.634006:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.634007:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.634009:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.634010:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.634011:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.634015:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.634017:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.634020:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.634021:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.634022:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.634024:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.634024:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.634025:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.634026:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.634029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.634036:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.634038:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.634039:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.634040:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.634042:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.634042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.634045:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:1.0:1619677448.634047:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.634050:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.634051:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.634053:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.634055:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.634057:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.634058:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.634059:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.634060:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.634061:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.634065:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.634067:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.634070:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.634070:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.634071:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.634073:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.634073:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.634074:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.634074:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.634079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.634085:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.634088:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.634089:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.634089:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.634091:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.634094:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677448.634239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.634246:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.634249:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.634249:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.634251:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.634254:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.634256:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.634257:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.634259:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.634260:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.634261:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.634265:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.634266:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.634269:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.634270:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.634270:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.634272:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.634272:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.634273:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.634273:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.634278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.634286:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.634288:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.634289:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.634289:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.634291:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.634296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:17.0:1619677448.634298:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677448.634305:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.634307:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.634308:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.634310:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.634313:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.634314:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.634315:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.634316:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.634317:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.634318:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:4.2:1619677448.634320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:26.0:1619677448.634321:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.634324:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.634326:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.634327:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.634328:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.634328:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.634329:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.634330:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.634330:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.634331:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.634331:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.634332:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677448.634333:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.634334:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.634335:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:25.2:1619677448.634336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:10.0:1619677448.634336:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.634337:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.634338:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.634339:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.634342:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.634343:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:10.0:1619677448.634343:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.634346:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.634346:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.634346:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.634347:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.634347:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:31.0:1619677448.634348:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.634349:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.634350:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.634350:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.634350:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.634351:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:10.0:1619677448.634351:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.634355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.634363:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.634366:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.634367:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.634367:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.634369:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.634372:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.634390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.634395:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.634398:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.634399:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.634400:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.634403:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.634405:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.634405:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.634407:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.634408:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.634409:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.634412:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.634413:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.634416:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.634417:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.634418:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.634419:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.634420:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.634421:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.634421:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.634425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.634432:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.634435:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.634436:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.634437:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.634438:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.634441:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.634515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.634522:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.634525:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.634527:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.634529:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.634529:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.634530:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.634531:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.634533:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.634534:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.634535:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.634538:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.634540:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.634543:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.634543:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.634544:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.634545:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.634546:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.634547:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.634547:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.634551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.634558:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.634561:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.634561:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.634563:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.634565:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.634566:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.634724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.2:1619677448.634724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.0:1619677448.634730:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:1.0:1619677448.634730:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.634733:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.634734:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.634734:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.634736:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.634738:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.634738:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.634741:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.634742:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.634743:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:5.0:1619677448.634744:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.634745:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.634746:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.634748:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.634748:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.634749:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.634750:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.634751:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.634751:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.634755:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.634757:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:5.0:1619677448.634759:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.634760:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.634761:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.634761:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.634761:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.634763:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:5.0:1619677448.634764:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.634764:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.634764:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.634765:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.634765:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677448.634766:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.0:1619677448.634768:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677448.634768:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.634769:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.634770:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677448.634774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:9.2:1619677448.634780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.634784:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.634786:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.634787:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:8.0:1619677448.634787:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.634787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:24.0:1619677448.634788:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.634789:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.634789:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.634791:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.634793:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:26.0:1619677448.634794:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677448.634794:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.634794:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677448.634795:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:26.0:1619677448.634797:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.634798:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.634800:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.634803:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.634804:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.634805:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.634807:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.634808:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.634809:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.634812:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.634814:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.634816:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.634817:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.634818:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.634819:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.634820:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.634820:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.634822:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.634822:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:21.2:1619677448.634823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:25.2:1619677448.634825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.634828:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.2:1619677448.634828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677448.634831:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:13.0:1619677448.634831:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677448.634832:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.634834:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.634834:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.634834:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.634835:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:13.0:1619677448.634836:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.634837:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.634837:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.634837:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:22.0:1619677448.634838:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:13.0:1619677448.634838:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.634838:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.634839:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677448.634840:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.634841:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.634842:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.634843:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.634847:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.634849:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.634852:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.634853:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.634853:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.634855:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.634855:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.634856:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.634857:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.634862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.634869:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.634871:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.634872:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.634874:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.634875:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.634877:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.634909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.634916:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.634919:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.634920:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.634921:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.634924:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.634925:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.634926:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.634927:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.634928:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.634929:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.634933:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.634935:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.634938:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.634938:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.634939:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.634941:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.634942:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.634942:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.634943:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.634947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.634956:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.634959:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.634960:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.634960:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.634962:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.634965:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.635033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.635040:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.635043:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.635044:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.635045:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.635048:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.635050:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.635051:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.635052:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.635053:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.635054:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.635057:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.635058:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.635061:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.635062:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.635063:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.635065:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.635065:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.635066:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.635066:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.635071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.635078:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.635081:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.635082:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.635082:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.635084:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.635088:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.635128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677448.635135:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677448.635138:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677448.635139:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677448.635141:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.635143:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.635145:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677448.635146:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.635147:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.635148:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.635149:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:20.2:1619677448.635152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:8.0:1619677448.635152:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.635154:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677448.635155:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677448.635157:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.635158:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:8.0:1619677448.635158:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:8.0:1619677448.635158:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:20.0:1619677448.635160:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.635160:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:8.0:1619677448.635160:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677448.635161:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677448.635161:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:5.0:1619677448.635161:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.635163:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.635164:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.635165:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.635166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:20.0:1619677448.635167:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.635168:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.635169:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.635173:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.635173:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:20.0:1619677448.635174:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.635176:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:20.0:1619677448.635177:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.635177:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.635177:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.635178:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.635179:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:2.0:1619677448.635179:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.635181:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.635181:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.635182:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677448.635182:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.635183:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:25.2:1619677448.635186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.635186:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677448.635187:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677448.635189:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:5.0:1619677448.635189:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677448.635191:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.635192:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.635193:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:20.0:1619677448.635194:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677448.635195:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.635196:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.635196:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:20.0:1619677448.635197:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677448.635197:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.635198:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:13.0:1619677448.635199:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.635199:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.635202:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.635202:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677448.635204:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.635207:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.635208:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.635208:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.635210:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.635210:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677448.635211:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677448.635212:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:25.2:1619677448.635216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.635222:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.635225:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.635225:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.635226:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.635228:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.635231:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.635241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.635248:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.635250:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.635251:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.635253:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.635255:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.635257:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.635258:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.635259:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.635260:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.635261:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.635264:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.635266:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.635269:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.635270:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.635270:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.635272:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.635272:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.635273:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677448.635275:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.635275:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677448.635278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677448.635281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:17.0:1619677448.635286:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:26.0:1619677448.635288:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:17.0:1619677448.635289:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:26.0:1619677448.635290:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:17.0:1619677448.635290:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.635290:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.635291:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.635292:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:17.0:1619677448.635292:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:17.0:1619677448.635302:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.635410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.635415:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.635418:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.635419:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.635421:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.635423:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.635424:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.635425:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.635427:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.635428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:1.0:1619677448.635428:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.635429:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.635432:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.635434:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677448.635436:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677448.635437:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.635438:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:1.0:1619677448.635438:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.635438:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:26.0:1619677448.635439:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677448.635440:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.635440:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.635441:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.635442:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:21.2:1619677448.635443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:1.0:1619677448.635443:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.635443:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.635444:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.635446:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.635447:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.635448:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.635448:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.2:1619677448.635448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.635449:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.635451:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:6.0:1619677448.635451:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.635452:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677448.635454:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:6.0:1619677448.635455:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.635456:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.635456:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.635456:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.635456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.635457:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.635458:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:6.0:1619677448.635458:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.635459:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:10.0:1619677448.635461:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.635461:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96712346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.635462:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.635463:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:10.0:1619677448.635463:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:6.0:1619677448.635463:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.635463:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.635464:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.635465:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.635465:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677448.635465:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677448.635466:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:10.0:1619677448.635466:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:24.0:1619677448.635467:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.635467:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:6.0:1619677448.635469:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.635470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:24.0:1619677448.635470:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.635471:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:24.0:1619677448.635473:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677448.635474:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.635475:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.635476:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:13.0:1619677448.635477:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:9.2:1619677448.635477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:6.0:1619677448.635478:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.635478:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.635479:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:13.0:1619677448.635480:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.635480:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.635480:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.635481:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:13.0:1619677448.635482:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.635483:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:25.2:1619677448.635484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:12.0:1619677448.635484:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.635485:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677448.635485:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677448.635486:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:21.2:1619677448.635486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677448.635486:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.635487:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.635488:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:22.0:1619677448.635491:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:30.0:1619677448.635492:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677448.635494:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.635494:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677448.635495:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:22.0:1619677448.635495:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:30.0:1619677448.635496:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.635498:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.635500:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.635501:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.635529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.635536:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.635539:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.635540:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.635542:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.635544:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.635546:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.635547:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.635548:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.635549:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.635550:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.635554:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.635556:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.635558:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.635559:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.635560:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.635561:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.635562:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.635563:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.635564:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.635564:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.635568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:4.2:1619677448.635570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.635575:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677448.635577:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.635578:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.635579:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.635579:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677448.635579:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.635580:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.635580:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.635581:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.635585:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.635622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.635630:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.635632:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.635635:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.635636:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.635637:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.635638:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.635640:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.635641:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.635642:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.635643:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.635646:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.635648:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.635650:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.635651:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.635651:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.635653:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.635653:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.635654:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.635655:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.635658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.635665:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.635667:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.635668:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.635670:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.635671:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.635673:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.635784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677448.635790:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.635793:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677448.635793:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677448.635796:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.635798:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.635800:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.635801:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.635802:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.635803:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.635804:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.635807:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.635809:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.635811:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.635812:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.635813:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.635815:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.635815:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677448.635816:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677448.635817:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677448.635818:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:8.2:1619677448.635821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.635827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.635831:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.635832:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.635833:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.635833:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677448.635834:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677448.635834:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677448.635835:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.635836:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677448.635836:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.635837:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.2:1619677448.635842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.635850:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.635853:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.635854:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.635854:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.635854:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.635855:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.635888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.635894:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.635897:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.635898:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.635899:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.635901:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.635903:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.635904:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.635905:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.635906:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.635907:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.635911:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.635912:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.635915:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.635916:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.635917:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.635919:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.635919:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.635920:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.635920:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.635925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.635930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:13.0:1619677448.635931:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.635934:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.635935:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.635936:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.635936:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677448.635937:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.635939:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.635939:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.635940:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.635941:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.635942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:2.0:1619677448.635945:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.635946:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.635947:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.635948:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:2.0:1619677448.635949:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.635949:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.635950:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:21.2:1619677448.635951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:12.0:1619677448.635951:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.635952:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.635953:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.635953:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.635955:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677448.635955:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.635956:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.635958:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.635960:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.635960:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.635961:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:18.0:1619677448.635962:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:2.0:1619677448.635962:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.635964:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:18.0:1619677448.635965:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.635965:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:18.0:1619677448.635966:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:2.0:1619677448.635966:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.635967:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:18.0:1619677448.635968:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.635968:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.635969:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.635970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677448.635970:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.635971:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.2:1619677448.635973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:18.0:1619677448.635974:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.635975:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:13.0:1619677448.635976:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:18.0:1619677448.635977:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.635978:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.635978:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677448.635978:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.635979:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677448.635980:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:13.0:1619677448.635980:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.635981:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.635981:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.635981:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677448.635981:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.635982:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677448.635982:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.635982:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.635984:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677448.635984:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.635986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:1.0:1619677448.635986:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.635988:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.635989:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.635990:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.635991:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.635992:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:22.0:1619677448.635993:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.635996:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:1.0:1619677448.635996:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.635997:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.635997:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677448.635998:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:22.0:1619677448.635999:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:1.0:1619677448.636001:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.636002:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677448.636002:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.636002:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.636004:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.636005:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.636005:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.636006:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.636010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.636017:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.636019:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.636020:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.636021:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.636023:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.636025:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677448.636045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.636051:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.636053:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.636056:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.636056:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.636059:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.636060:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.636062:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.636063:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.636064:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.636065:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.636069:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.636070:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.636073:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.636074:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.636075:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.636077:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.636077:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.636078:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.636078:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.636082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.636089:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.636091:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.636092:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.636094:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.636095:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.636097:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.636122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.636129:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.636132:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.636133:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.636134:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.636137:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.636138:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.636140:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.636141:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.636142:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.636143:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.636146:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.636148:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.636150:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.636151:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:4.2:1619677448.636151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677448.636152:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.636153:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.636154:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.636154:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.636155:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677448.636158:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677448.636159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677448.636160:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.636162:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.636163:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.636166:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.636167:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:10.0:1619677448.636168:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.636169:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.636170:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:10.0:1619677448.636170:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.636171:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.636171:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.636171:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:10.0:1619677448.636172:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.636173:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.636175:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.636176:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677448.636177:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.636181:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.636181:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.636182:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.636183:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.636184:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.636185:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.636186:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.636190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.636197:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.636200:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.636200:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.636201:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.636203:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.636206:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.636209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.636217:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.636220:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.636222:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.636224:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.636224:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.636226:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.636227:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.636228:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.636229:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.636230:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.636233:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.636235:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.636237:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.636238:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.636238:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.636240:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.636240:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.636241:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.636242:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.636245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.636252:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.636255:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.636256:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.636257:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.636259:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.636260:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.636265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:12.0:1619677448.636272:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.636275:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.636275:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.636276:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.636277:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.636277:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.636371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.636377:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.636379:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.636380:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.636382:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.636385:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.636386:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.636387:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.636389:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.636390:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.636391:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.636394:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.636395:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.636398:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.636399:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.636399:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.636401:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.636401:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.636402:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.636402:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.636408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.636415:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.636418:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.636418:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.636419:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.636421:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.636424:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677448.636555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:21.2:1619677448.636558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.636564:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.636565:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:30.0:1619677448.636566:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.636567:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.636568:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.636570:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.636571:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.636572:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.636574:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.636575:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.636577:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.636578:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.636579:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.636581:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.636582:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.636582:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.636584:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.636584:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.636585:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.636585:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.636588:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.636589:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.636590:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.636590:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.636590:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.636592:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.636592:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.636593:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.636593:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.636593:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.636594:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.636594:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.636595:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.636595:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.636596:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.636597:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.636597:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.636598:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.636598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:25.2:1619677448.636601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.636606:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:26.0:1619677448.636608:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:20.0:1619677448.636609:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:26.0:1619677448.636610:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:20.0:1619677448.636610:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.636611:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.636611:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:20.0:1619677448.636612:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.636612:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:26.0:1619677448.636614:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.636615:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:26.0:1619677448.636617:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.636621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.636626:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:4.2:1619677448.636628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.636629:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.636631:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.636631:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.636631:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.636632:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.636633:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.636634:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.636635:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.636637:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.636639:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.636641:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.636642:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.636643:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.636644:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.636645:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.636649:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.636650:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.636653:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.636654:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.636654:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.636656:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.636656:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.636657:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.636657:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677448.636658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:25.2:1619677448.636662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.636665:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.636668:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.636669:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:22.0:1619677448.636670:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:2.0:1619677448.636671:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.636673:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:2.0:1619677448.636673:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.636674:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.636674:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:2.0:1619677448.636675:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.636676:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.636677:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.636678:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.636679:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:2.0:1619677448.636679:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.636680:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:25.2:1619677448.636681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000400:00000200:2.0:1619677448.636684:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.636685:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.636688:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.636688:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.636689:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.636690:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.636691:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.636692:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.636692:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677448.636692:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.636692:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677448.636694:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.636694:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677448.636694:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.636695:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.636695:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.636697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.636701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.636706:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:5.0:1619677448.636707:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677448.636708:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.636709:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.636710:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677448.636710:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.636711:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.636712:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.636713:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.636715:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.636744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.636750:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.636753:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.636754:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.636756:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.636759:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.636760:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.636761:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.636763:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.636764:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.636765:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:21.2:1619677448.636766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:17.0:1619677448.636768:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.636770:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.636773:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.636773:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:17.0:1619677448.636774:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.636775:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.636775:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.636776:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:17.0:1619677448.636776:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.636777:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.636778:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.636778:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.636778:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.636781:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.636781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:9.2:1619677448.636782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:8.0:1619677448.636783:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.636784:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96712328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.636786:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.636787:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.636789:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:24.0:1619677448.636789:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:8.0:1619677448.636789:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.636791:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000800:00000200:24.0:1619677448.636791:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.636792:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.636793:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.636794:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.636794:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.636796:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:8.0:1619677448.636796:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.636797:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.636797:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:31.0:1619677448.636798:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:24.0:1619677448.636799:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000400:00000200:31.0:1619677448.636800:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.636800:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.636801:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.636801:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.636801:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:31.0:1619677448.636802:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.636802:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.636802:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000400:00000200:31.0:1619677448.636803:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:24.0:1619677448.636803:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.636803:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.636804:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677448.636805:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.636805:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:31.0:1619677448.636806:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.636806:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.636807:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:31.0:1619677448.636808:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.2:1619677448.636810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.636811:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.636811:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.636812:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.636814:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.636815:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677448.636816:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:31.0:1619677448.636816:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:20.0:1619677448.636818:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.2:1619677448.636820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.636821:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.636822:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.636823:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.636826:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.636827:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677448.636827:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.636830:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.636831:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.636832:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.636833:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.636835:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.636852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.636860:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.636863:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.636865:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.636867:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.636868:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.636869:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.636871:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712326] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.636872:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.636873:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.636874:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.636877:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.636879:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.636881:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.636881:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.636882:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.636883:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.636884:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.636886:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.636887:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.636888:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.636889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.2:1619677448.636893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.636898:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.636899:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.636900:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.636901:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:31.0:1619677448.636902:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.636903:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.636903:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.636904:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.636905:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.636905:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.636907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.636914:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.636916:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.636917:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.636918:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.636918:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.636919:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.637065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.637068:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.637071:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.637073:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.637074:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.637075:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.637077:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.637078:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.637080:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.637081:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.637082:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.637085:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.637087:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.637089:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.637090:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.637090:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.637092:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.637093:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.637094:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.637094:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.637098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.637106:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.637108:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.637109:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.637109:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.637111:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.637113:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.637143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.637151:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.637154:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.637156:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.637158:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.637158:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.637160:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.637161:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.637162:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.637163:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.637164:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.637168:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.637170:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.637172:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.637172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.637173:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.637173:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.637175:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.637176:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.637177:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.637177:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677448.637178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677448.637178:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.637180:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:25.2:1619677448.637181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.637181:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.637183:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.637186:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:1.0:1619677448.637186:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.637187:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:12.0:1619677448.637188:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677448.637189:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:1.0:1619677448.637189:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.637190:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.637191:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.637191:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.637193:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.637193:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:12.0:1619677448.637193:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.637193:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.637194:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:13.0:1619677448.637195:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:31.0:1619677448.637196:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.637196:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677448.637197:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.637198:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.637199:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.637199:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:31.0:1619677448.637200:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.637202:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.637202:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.637203:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.637203:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.637204:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.637205:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.637206:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.637207:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.637207:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.637208:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:31.0:1619677448.637209:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.637209:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.637211:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677448.637211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.637212:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.637213:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.637213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.637215:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.637215:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.637216:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.637217:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:17.0:1619677448.637219:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677448.637219:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.637221:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.2:1619677448.637221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.637221:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677448.637222:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677448.637223:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.637223:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.637223:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.637225:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.637227:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.637228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:5.0:1619677448.637229:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.637232:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.637232:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.637234:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.637234:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.637235:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.0:1619677448.637237:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677448.637238:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.637239:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.2:1619677448.637239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.637239:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.637240:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.637241:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677448.637246:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.637249:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.637250:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.637251:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.637251:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.637252:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.637528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.637536:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.637539:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.637541:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.637544:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.637544:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.637546:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.637547:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.637548:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.637549:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.637550:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.637554:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.637556:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.637559:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.637559:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.637560:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.637562:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.637562:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.637564:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.637565:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.637566:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.637568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.637572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677448.637576:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.637579:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677448.637580:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677448.637580:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.637580:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.637582:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:6.0:1619677448.637582:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.637583:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.637584:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.637585:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.637597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.637600:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.637602:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.637605:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.637605:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.637607:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.637609:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.637610:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.637612:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.637613:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.637614:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.637618:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.637620:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.637622:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.637623:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.637624:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.637626:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.637626:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.637627:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.637628:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.637632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.637639:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.637642:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.637643:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.637644:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677448.637646:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.637647:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.637651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.637657:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.637660:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:20.2:1619677448.637661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:12.0:1619677448.637661:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.637661:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.637662:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.637663:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.637666:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.637669:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.637671:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.637672:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:5.0:1619677448.637674:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.637675:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:20.0:1619677448.637676:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.637677:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.637678:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.637679:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.637680:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:5.0:1619677448.637680:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.637684:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:5.0:1619677448.637684:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.637686:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.637686:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.637687:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677448.637688:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.637689:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.637689:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.637690:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677448.637691:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:5.0:1619677448.637691:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:31.0:1619677448.637692:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.637692:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.637693:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.637693:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.637694:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000400:00000200:31.0:1619677448.637695:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.637695:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.637695:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677448.637698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677448.637699:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.637701:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.2:1619677448.637701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.637704:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.637705:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.637705:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.637707:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.637707:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.637708:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677448.637708:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:31.0:1619677448.637709:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677448.637709:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.637709:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.637710:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:5.0:1619677448.637710:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.637711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:20.0:1619677448.637711:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.637712:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.637713:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.637714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.637715:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677448.637716:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:22.0:1619677448.637717:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677448.637717:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.637719:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677448.637720:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:20.0:1619677448.637720:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.637721:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677448.637722:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.637723:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.637723:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.637725:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:22.0:1619677448.637727:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.637728:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.637729:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.637731:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.637732:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.637733:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.637736:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.637738:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.637741:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.637742:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.637742:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.637744:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.637745:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.637746:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.637746:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.637750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.637758:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.637761:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.637761:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.637763:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.637764:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.637765:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.637781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.637787:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.637790:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.637790:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.637793:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.637795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677448.637795:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.637796:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.637798:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.637799:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.637800:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.637801:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.637803:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677448.637804:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.637806:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.637806:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:1.0:1619677448.637806:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.637808:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.637809:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.637810:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.637810:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:10.0:1619677448.637811:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.637812:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.637812:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.637813:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:1.0:1619677448.637813:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.637814:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.637814:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:10.0:1619677448.637815:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.637817:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.637817:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:25.2:1619677448.637818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:10.0:1619677448.637821:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.637823:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.637825:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.637825:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677448.637826:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.637826:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.637827:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.637828:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.637828:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.637829:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.637829:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.637829:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.637830:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.637831:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.637833:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.637834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.637840:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.637843:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.637844:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.637846:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.637847:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.637849:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.637851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677448.637858:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.637861:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.637861:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.637862:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.637863:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.637864:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.637864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.637871:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.637874:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.637874:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.637876:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.637879:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.637880:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.637881:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.637882:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.637883:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.637884:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.637887:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.637889:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.637892:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.637892:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.637893:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.637894:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.637895:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.637895:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.637896:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.637900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.637908:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.637911:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.637911:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.637912:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.637914:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.637917:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.637924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.637932:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.637934:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.637936:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.637938:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.637939:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.637941:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.637942:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.637943:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.637944:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.637945:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.637948:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.637950:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.637953:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.637954:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.637954:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.637956:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.637957:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.637958:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.637958:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.637962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.637969:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.637972:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.637973:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.637974:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.637976:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.637978:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.638154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.638160:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.638163:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.638164:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.638166:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.638168:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.638170:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.638171:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.638173:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.638174:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.638175:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.638178:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.638180:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.638184:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.638184:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.638185:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.638187:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.638188:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.638189:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.638190:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.638190:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.638193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.638195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677448.638199:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:18.0:1619677448.638201:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:26.0:1619677448.638202:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.638202:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.638203:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.638203:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.638204:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:26.0:1619677448.638205:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.638205:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:26.0:1619677448.638208:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.638264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.638267:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.638269:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.638271:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.638272:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.638274:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.638275:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677448.638277:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.638278:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.638279:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.638280:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677448.638284:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.638285:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.638288:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.638289:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.638289:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.638291:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.638291:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.638292:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.638293:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.638297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.638305:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.2:1619677448.638309:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677448.638310:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.638312:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.638313:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.638315:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.638316:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.2:1619677448.638316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.638318:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.638319:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:5.0:1619677448.638319:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.638320:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.0:1619677448.638322:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.638323:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:10.0:1619677448.638323:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.638323:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.638324:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.638324:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:10.0:1619677448.638326:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.638327:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.638328:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.638329:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.638330:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.638331:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.638335:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.638337:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.638340:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.638340:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.638341:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.638343:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.638344:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.638344:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.638345:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.638350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.638358:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.638361:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.638362:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.638362:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.638365:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.638367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.638370:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.638371:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.638373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.638373:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.638374:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:4.2:1619677448.638375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677448.638375:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.638376:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.638376:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677448.638379:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.638382:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.638382:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.638382:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.638384:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.638385:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.638386:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677448.638386:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.638387:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.638388:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.638389:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712309] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.638391:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.638391:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.638392:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.638393:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.638393:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.638395:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.638397:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.638397:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.638399:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.638400:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.638401:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.638404:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.638405:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:26.0:1619677448.638406:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.638406:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.638407:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.638408:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.638408:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.638409:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.638409:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.638410:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.638411:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.638412:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.638413:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.638413:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:25.2:1619677448.638413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.638415:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.638416:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.638416:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.638419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:13.0:1619677448.638421:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:4.2:1619677448.638422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677448.638424:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.638425:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.638426:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.638427:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:12.0:1619677448.638428:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.638429:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:13.0:1619677448.638429:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.638429:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:30.0:1619677448.638430:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.638431:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.638431:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.638432:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.638433:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.638433:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.638434:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.638455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.638461:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.638464:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.638465:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.638466:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.638469:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.638470:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.638472:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.638473:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.638474:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.638475:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.638479:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.638480:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.638483:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.638484:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.638484:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.638486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677448.638486:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.638486:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.638487:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.638487:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.638492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.638494:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.638497:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.638499:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.638499:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.638501:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.638502:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.638502:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:30.0:1619677448.638503:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.638503:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.638503:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:30.0:1619677448.638505:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.638505:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.638507:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.638508:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.638508:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:30.0:1619677448.638509:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677448.638512:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.638514:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677448.638517:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.638517:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:30.0:1619677448.638518:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:30.0:1619677448.638520:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:30.0:1619677448.638520:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.638522:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.638522:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677448.638522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:9.2:1619677448.638526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.638530:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.638532:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.638533:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.638534:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.638535:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.638536:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.638537:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.638538:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.638540:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.638541:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.638542:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.638543:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.638544:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.638545:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.638546:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.638547:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:8.0:1619677448.638548:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.638550:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.638552:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.638553:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.638554:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.638555:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.638556:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.638557:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.638558:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.638559:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.638561:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.638565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.638568:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.638571:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.638571:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.638572:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.638572:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.638574:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677448.638574:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.638575:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.638576:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677448.638577:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.638580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.638588:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.638590:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.638591:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.638591:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.638592:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.638593:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.638721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.638726:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.638729:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.638730:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.638732:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.638734:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.638736:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.638737:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.638738:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.638739:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.638740:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.638743:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.638745:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.638746:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.638747:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.638748:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.638750:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.638750:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.638751:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.638751:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.638756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.638763:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.638765:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.638766:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.638766:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.638767:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.638769:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.638791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677448.638793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.638800:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.638802:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.638802:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.638803:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677448.638805:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.638806:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.638808:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.638809:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.638810:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.638812:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.638813:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.638814:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.638815:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.638817:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.638818:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.638819:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.638820:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.638821:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.638821:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.638822:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.638826:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.638826:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.638827:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.638827:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.638827:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.638829:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.638830:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:20.2:1619677448.638830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.638830:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.638830:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.638831:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.638831:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.638832:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.638833:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.638833:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.638833:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.638835:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.638835:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.638835:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677448.638836:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:25.2:1619677448.638836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.638838:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.638839:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.638840:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:25.2:1619677448.638840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:26.0:1619677448.638842:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.638843:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.638844:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96712301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.638844:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:26.0:1619677448.638845:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.638846:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.638847:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:26.0:1619677448.638847:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.638848:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.638848:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677448.638849:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:6.0:1619677448.638849:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.638850:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.638850:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:21.2:1619677448.638850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:6.0:1619677448.638851:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.638852:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.638853:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.638854:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.638854:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677448.638855:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.638855:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.638855:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.638856:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.638856:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677448.638856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.638857:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.638858:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.638858:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:30.0:1619677448.638859:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677448.638859:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:30.0:1619677448.638861:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.638863:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.638863:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.638863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.638863:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:30.0:1619677448.638865:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.638866:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.638866:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.638866:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.638867:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000400:00000200:30.0:1619677448.638868:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.638868:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.638868:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:30.0:1619677448.638869:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.638869:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:30.0:1619677448.638870:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:12.0:1619677448.638872:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:30.0:1619677448.638873:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677448.638873:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.638874:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:30.0:1619677448.638875:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677448.638876:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.638876:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.638878:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.638879:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.638879:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.638881:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.638882:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.638883:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.638883:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677448.638884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.638887:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:9.2:1619677448.638887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.0:1619677448.638889:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.638891:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.638892:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.638893:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.638895:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:2.0:1619677448.638895:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:20.0:1619677448.638896:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.638897:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:20.0:1619677448.638898:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.638898:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.638899:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.638899:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.638900:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.638901:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.638904:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.638905:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.0:1619677448.638906:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:9.2:1619677448.638906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677448.638908:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.638909:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.638910:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.638911:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.638912:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.638913:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.638913:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677448.638914:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.638916:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:8.2:1619677448.638917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.638917:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.638918:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.638918:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.638919:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677448.638925:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.638928:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.638928:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.638930:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.638931:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.638932:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.638987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.638992:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.638995:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.638996:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.638998:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.639001:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.639002:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.639003:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.639005:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.639006:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.639007:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.639010:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.639012:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:4.2:1619677448.639015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:1.0:1619677448.639015:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.639016:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.639017:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.639019:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.639019:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.639020:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.639020:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677448.639022:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.2:1619677448.639025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.639025:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.639025:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677448.639027:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.639031:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.639032:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:8.0:1619677448.639032:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.639033:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96712297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.639034:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.639035:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.639035:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.639036:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677448.639037:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.639037:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:12.0:1619677448.639038:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:8.0:1619677448.639041:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.639043:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677448.639043:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.639046:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.639046:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.639047:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.639049:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.639049:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.639050:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.639051:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.639055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.639063:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.639066:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.639066:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.639067:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.639069:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.639072:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.639200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.639205:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.639209:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.639209:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.639211:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.639213:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.639215:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.639216:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.639217:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.639219:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.639220:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.639223:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.639225:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.639227:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.639228:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.639229:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.639231:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.639232:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.639232:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.639233:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.639237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.639244:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.639247:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.639248:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.639250:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.639252:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.639253:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.639272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.639278:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.639281:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.639282:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.639284:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.639286:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.639287:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.639289:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.639290:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.639291:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.639292:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.639300:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.639302:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.639305:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.639306:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.639307:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.639309:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.639309:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.639310:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.639310:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.639315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.639322:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.639325:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.639326:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.639327:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.639329:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.639331:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.639332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.2:1619677448.639338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:30.0:1619677448.639340:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.639343:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:18.0:1619677448.639344:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:30.0:1619677448.639345:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.639347:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.639347:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.639347:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.639348:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:30.0:1619677448.639350:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677448.639351:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.639351:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.639352:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.639354:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.639354:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.639354:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.639355:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.639357:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.639359:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.639360:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.639360:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.639361:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.639363:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.639365:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.639366:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.639367:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.639367:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.639367:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.639369:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.639369:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.639369:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.639369:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.639370:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.639371:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:18.0:1619677448.639372:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.639372:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:30.0:1619677448.639373:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.639373:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677448.639373:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.639374:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.639375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:9.2:1619677448.639378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.639379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:4.2:1619677448.639382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677448.639385:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677448.639385:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.639387:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:2.0:1619677448.639387:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:17.0:1619677448.639388:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:2.0:1619677448.639388:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.639388:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:31.0:1619677448.639389:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:26.0:1619677448.639389:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.639389:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:1.0:1619677448.639389:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.639390:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:2.0:1619677448.639390:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.639391:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.639392:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.639392:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677448.639392:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:1.0:1619677448.639392:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.639395:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.639395:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:26.0:1619677448.639397:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.639398:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.639400:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.639401:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.639402:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.639403:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.639407:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.639408:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.639412:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.639412:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.639413:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.639415:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.639415:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.639416:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.639416:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.639421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.639428:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.639431:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.639432:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.639432:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.639435:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.639437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677448.639438:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677448.639445:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.639447:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.639448:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.639450:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.639453:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.639454:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.639456:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.639457:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.639458:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.639458:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.639461:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.639462:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.639465:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.639466:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.639466:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.639467:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.639468:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.639469:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.639470:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.639471:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.639473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.639476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.639480:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677448.639483:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:12.0:1619677448.639483:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.639483:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.639485:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:12.0:1619677448.639485:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.639486:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.639487:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.639487:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.639488:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.639589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.639595:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.639598:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.639598:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.639601:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.639603:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.639604:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.639606:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.639607:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.639608:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.639609:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.639612:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.639614:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.639617:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.639618:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.639619:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.639620:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.639621:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.639622:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.639623:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.639624:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.639626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.639629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.639633:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677448.639634:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677448.639635:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.639636:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.639636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:1.0:1619677448.639636:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.639637:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677448.639638:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.639638:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.639639:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.639641:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677448.639642:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.639645:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.639646:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.639648:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.639651:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.639652:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.639653:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.639654:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.639656:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.639656:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.639660:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.639662:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.639665:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.639666:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.639666:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.639668:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.639668:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.639669:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.639669:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.639674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.639682:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.639685:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.639685:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.639686:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.639688:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.639691:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.639728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.639734:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.639737:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.639737:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.639739:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.639742:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.639743:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.639745:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.639746:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.639747:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.639748:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:20.2:1619677448.639751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:6.0:1619677448.639751:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.639753:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:6.0:1619677448.639753:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.639755:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.639756:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:6.0:1619677448.639756:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.639757:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:20.0:1619677448.639758:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.639758:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.639759:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.639759:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.0:1619677448.639759:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:6.0:1619677448.639760:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:20.0:1619677448.639761:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.639762:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.639763:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.639765:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677448.639765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:20.0:1619677448.639766:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.639767:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.639770:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.639772:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677448.639772:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:20.0:1619677448.639774:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.639775:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:18.0:1619677448.639775:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:20.0:1619677448.639776:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:18.0:1619677448.639776:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.639776:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677448.639778:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.639778:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:18.0:1619677448.639778:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.639779:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.639781:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.639781:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:18.0:1619677448.639781:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:25.2:1619677448.639784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.639786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:31.0:1619677448.639791:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.639792:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:31.0:1619677448.639794:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:5.0:1619677448.639794:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:31.0:1619677448.639795:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.639795:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.639796:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:31.0:1619677448.639797:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.639798:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.639800:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.639894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.639901:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.639904:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.639905:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.639907:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.639909:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.639911:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.639912:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.639913:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.639914:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.639915:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.639918:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.639920:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.639922:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.639923:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.639924:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.639925:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.639925:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.639926:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.639927:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.639931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.639938:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.639941:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.639941:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.639943:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.639945:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.639945:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:25.2:1619677448.639947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:12.0:1619677448.639954:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.639957:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.639957:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.639958:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.639959:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.639960:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.640173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:21.2:1619677448.640177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:8.0:1619677448.640181:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:22.0:1619677448.640182:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677448.640183:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.640184:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677448.640185:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:8.0:1619677448.640186:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.640186:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.640189:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.640190:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.640191:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.640192:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:22.0:1619677448.640193:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.640194:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.640194:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.640194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:8.0:1619677448.640196:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.640197:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.640198:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.640198:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.640199:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677448.640200:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677448.640201:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:1.0:1619677448.640202:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.640203:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677448.640206:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.640206:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.640207:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.640208:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.640209:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:1.0:1619677448.640209:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:8.0:1619677448.640210:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:1.0:1619677448.640211:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96712283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.640212:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.640212:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.640213:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.640213:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.0:1619677448.640214:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.640214:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.640215:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:8.0:1619677448.640215:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:1.0:1619677448.640215:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:22.0:1619677448.640216:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677448.640216:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677448.640218:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:8.0:1619677448.640218:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.640218:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:1.0:1619677448.640218:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.640219:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.640219:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677448.640219:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.640220:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:1.0:1619677448.640220:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:9.2:1619677448.640221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:1.0:1619677448.640222:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677448.640223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.640223:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.640223:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:21.2:1619677448.640224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677448.640225:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.640226:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.640226:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.640227:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.640229:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677448.640229:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:8.2:1619677448.640231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677448.640231:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:5.0:1619677448.640232:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677448.640232:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:6.0:1619677448.640233:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.640233:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:2.0:1619677448.640233:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.640234:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677448.640235:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:2.0:1619677448.640235:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.640236:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.640238:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.640238:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.640238:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:17.0:1619677448.640239:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.640241:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:5.0:1619677448.640241:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:17.0:1619677448.640242:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.640242:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.640244:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.640245:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.640251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.640258:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.640261:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.640262:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.640262:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.640263:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.640263:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.640275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.640278:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.640281:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.640283:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.640283:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.640286:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.640287:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.640289:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.640290:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.640291:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.640292:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.640299:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.640301:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.640304:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.640305:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.640306:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.640308:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.640308:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.640309:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.640310:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.640314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.640322:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.640325:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.640325:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.640327:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.640329:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.640330:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.640336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:4.2:1619677448.640340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.640346:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677448.640348:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:10.0:1619677448.640349:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.640350:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677448.640351:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.640353:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.640353:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.640355:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.640357:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.640358:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.640359:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.640360:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.640362:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.640363:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.640364:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.640364:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.640366:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.640366:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.640367:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.640367:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.640371:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.640371:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.640372:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.640373:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.640373:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.640375:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.640375:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.640376:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)++
00000800:00000200:10.0:1619677448.640376:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.640377:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (23)--
00000800:00000200:10.0:1619677448.640377:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.640378:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:26.0:1619677448.640379:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:10.0:1619677448.640379:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.640380:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.640380:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.640380:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.640381:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.640381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:25.2:1619677448.640385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.640390:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:30.0:1619677448.640393:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677448.640393:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.640394:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.640394:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677448.640395:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.640396:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.640397:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.640399:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.640400:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.640400:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677448.640402:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.640404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677448.640411:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.640414:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.640415:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.640415:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.640416:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.640417:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.640636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.640644:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.640647:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.640648:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.640649:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.640652:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.640653:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.640654:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.640656:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.640656:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.640657:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.640660:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.640661:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.640663:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.640664:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.640665:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.640666:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.640667:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.640668:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.640668:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.640672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.640678:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.640681:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.640681:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.640683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:12.0:1619677448.640683:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.640685:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.640686:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.640690:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.640693:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.640695:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.640697:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.640698:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.640699:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677448.640700:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.640702:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.640702:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.640703:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677448.640706:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.640708:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.640711:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.640712:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.640712:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.640714:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.640714:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.640715:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.640715:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.640720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.640728:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.640731:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.640731:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.640732:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.640734:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.640736:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.640753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.640758:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.640761:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.640761:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.640763:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.640764:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.640766:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.640767:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.640769:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.640769:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.640771:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.640774:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.640776:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.640779:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.640780:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.640780:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.640782:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.640783:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.640784:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.640784:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.640788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:4.2:1619677448.640794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:13.0:1619677448.640795:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.640797:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.640798:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.640800:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.640801:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:12.0:1619677448.640801:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.640803:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677448.640804:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.640805:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.640807:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.640808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:24.0:1619677448.640809:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.640811:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.640812:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712276] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.640814:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.640814:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677448.640815:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.640816:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:12.0:1619677448.640817:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.640818:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.640818:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:24.0:1619677448.640819:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.640819:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.640820:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677448.640821:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.640824:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.640825:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.640825:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.640827:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.640828:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.640828:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.640829:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.640833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.640840:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.640843:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.640844:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.640844:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.640847:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.640849:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.640866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.640874:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.640877:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.640879:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.640880:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.640882:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.640883:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.640885:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.640886:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.640887:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.640888:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.640891:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.640893:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.640896:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.640897:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.640898:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.640899:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.640900:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.640901:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.640901:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.640905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.640914:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.640916:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.640917:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.640919:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.640919:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.640922:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.640971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.640978:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.640981:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.640982:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.640984:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.640986:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.640987:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.640989:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.640990:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.640991:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.640992:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.640995:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.640997:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:21.2:1619677448.641000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.641000:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.641001:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.641002:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.641005:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.641005:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.641006:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.641007:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.641007:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.641008:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677448.641009:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.641009:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:25.2:1619677448.641010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:2.0:1619677448.641011:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.641013:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677448.641014:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.641015:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.641015:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.641016:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:12.0:1619677448.641017:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:2.0:1619677448.641019:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:12.0:1619677448.641020:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.641020:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:12.0:1619677448.641021:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.641022:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.641023:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.641023:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:13.0:1619677448.641024:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.641024:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:12.0:1619677448.641025:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.641026:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:2.0:1619677448.641026:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.641027:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.641028:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.641032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.641039:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.641042:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.641043:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.641043:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.641045:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.641048:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.641091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.641096:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.641098:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.641099:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.641101:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.641103:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.641104:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:20.2:1619677448.641105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:6.0:1619677448.641105:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.641107:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.641108:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.641109:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:10.0:1619677448.641112:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:6.0:1619677448.641112:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.641113:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:10.0:1619677448.641115:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.641116:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:6.0:1619677448.641116:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.641117:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.641118:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:10.0:1619677448.641119:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:6.0:1619677448.641119:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.641120:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677448.641121:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.641121:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.641121:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677448.641122:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677448.641124:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.641125:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.641126:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677448.641126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:10.0:1619677448.641127:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.641130:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.641132:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.641133:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:10.0:1619677448.641135:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.641135:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.641136:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:10.0:1619677448.641136:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:18.0:1619677448.641137:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.641137:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:10.0:1619677448.641138:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.641138:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:18.0:1619677448.641139:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.641139:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:5.2:1619677448.641139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:10.0:1619677448.641140:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:18.0:1619677448.641142:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:18.0:1619677448.641144:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:8.2:1619677448.641144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.641145:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.641146:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.641148:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.641150:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.641151:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:18.0:1619677448.641152:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:20.0:1619677448.641153:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:18.0:1619677448.641153:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.641154:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.641155:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.641155:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.641156:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677448.641157:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.641157:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:18.0:1619677448.641160:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.641162:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.641164:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677448.641165:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.641166:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.641166:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.641168:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.641169:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.641169:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.641170:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.641174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.641181:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.641184:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.641185:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.641186:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.641187:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.641190:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.641196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.641203:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.641206:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.641207:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.641207:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.641208:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.641209:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.641270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.641277:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.641280:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.641281:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.641283:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.641283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:26.0:1619677448.641286:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.641286:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677448.641287:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:20.0:1619677448.641288:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:26.0:1619677448.641289:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.641291:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.641291:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.641291:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677448.641292:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.641298:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:20.0:1619677448.641299:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.641301:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677448.641302:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.641303:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.641304:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.641305:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.641306:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:20.0:1619677448.641306:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.641309:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.641310:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:20.0:1619677448.641310:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.641311:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.641312:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:20.0:1619677448.641312:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.641313:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.641314:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:20.0:1619677448.641314:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.641315:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:20.0:1619677448.641315:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.641316:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677448.641316:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.641317:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677448.641318:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:25.2:1619677448.641319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.0:1619677448.641320:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:4.2:1619677448.641321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.641322:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.641323:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677448.641324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677448.641327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677448.641328:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677448.641329:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.641330:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:10.0:1619677448.641331:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677448.641331:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.641332:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.641332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677448.641332:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.641333:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677448.641333:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:5.0:1619677448.641333:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:22.0:1619677448.641334:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.641335:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:17.0:1619677448.641336:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:5.0:1619677448.641336:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677448.641337:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:17.0:1619677448.641337:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.641337:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.641338:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677448.641339:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.641341:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677448.641341:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677448.641344:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.641346:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.641348:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.641349:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.641350:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.641351:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.641353:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.641354:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.641355:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.641358:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.641360:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.641362:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.641363:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.641363:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.641365:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.641365:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.641367:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.641369:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.641369:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.641371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.2:1619677448.641375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.641379:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.641381:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.641381:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.641383:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.641383:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.641384:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.641384:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.641385:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.641386:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:31.0:1619677448.641387:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:21.2:1619677448.641457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.641464:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.641467:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.641467:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677448.641470:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.641472:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.641474:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.641475:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.641476:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.641477:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.641478:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.641481:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.641483:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.641485:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.641486:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.641487:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.641488:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.641488:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.641490:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.641491:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.641491:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.2:1619677448.641494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.641499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677448.641501:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.641503:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.641504:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.641505:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.641505:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.641508:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.641508:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.641508:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:12.0:1619677448.641509:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:1.0:1619677448.641510:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.641513:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.641515:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.641516:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.641517:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.641518:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.641519:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.641523:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.641524:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.2:1619677448.641526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:1.0:1619677448.641527:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.641528:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.641528:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.641530:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.641531:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.641531:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.641532:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677448.641533:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.641535:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:9.2:1619677448.641536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.641537:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.641538:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:24.0:1619677448.641539:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.641541:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.641542:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.641543:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.641545:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.641548:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677448.641834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.641842:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.641845:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.641847:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.641848:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.641849:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.641851:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.641852:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.641853:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.641854:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.641855:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.641858:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.641860:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.641863:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.641863:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.641864:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.641865:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.641866:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.641867:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.641867:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.641871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.641877:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.2:1619677448.641877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:13.0:1619677448.641880:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.641880:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.641882:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.641884:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677448.641884:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677448.641885:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677448.641887:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.641888:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.641889:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.641892:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.641893:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.641894:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.641896:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.641897:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.641898:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.641901:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.641903:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.641905:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.641906:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.641907:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.641909:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.641909:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.641910:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.641911:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.641912:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.641915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.641917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.641923:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.641924:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.641925:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677448.641925:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677448.641926:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677448.641927:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.641928:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.641930:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.641930:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.641932:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.642063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.642069:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.642072:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.642073:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.642074:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.642077:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.642078:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.642080:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.642081:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.642082:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.642083:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.642086:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.642087:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.642088:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.642089:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.642089:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.642091:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.642091:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.642092:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.642092:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.642097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.642104:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.642106:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.642106:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.642107:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.642108:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.642109:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.642162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677448.642168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.642170:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.642173:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.642174:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.642174:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.642176:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:10.0:1619677448.642176:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.642177:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.642178:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.642179:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.642180:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.642181:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.642182:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.642183:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.642183:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.642184:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.642185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:17.0:1619677448.642185:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.642185:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.642186:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.642188:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:17.0:1619677448.642188:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.642189:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.642190:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.642191:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.642193:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.642193:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.642194:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.642196:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.642196:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.642196:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.642197:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.642197:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:20.0:1619677448.642198:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:17.0:1619677448.642198:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:20.0:1619677448.642199:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96712259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.642199:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.642199:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677448.642200:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.642200:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.642201:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:20.0:1619677448.642202:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.642202:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.642203:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.642203:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.642204:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (22)++
00000800:00000200:25.2:1619677448.642205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (23)++
00000800:00000200:17.0:1619677448.642205:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (23)--
00000400:00000200:20.0:1619677448.642206:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.642206:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.642207:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:20.0:1619677448.642208:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.642208:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.642209:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677448.642211:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.642212:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.642212:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.642213:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (23)++
00000800:00000200:20.0:1619677448.642214:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.642214:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.2:1619677448.642214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.642215:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677448.642215:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000800:00000200:22.0:1619677448.642216:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (24)++
00000800:00000200:20.0:1619677448.642216:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677448.642217:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000400:00000200:22.0:1619677448.642217:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.642217:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.642219:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.642219:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.642220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677448.642220:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.642222:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677448.642223:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677448.642223:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.642224:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.642224:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.642225:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.642226:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.642227:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.642227:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.642228:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000400:00000200:6.0:1619677448.642228:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.642229:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.642230:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.642230:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.642231:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.642232:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.642232:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.642363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.642368:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.642371:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.642373:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.642373:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.642375:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.642377:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.642378:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.642379:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.642380:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.642381:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.642385:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.642386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677448.642387:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.642390:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.642391:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.642392:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.642394:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.642394:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.642394:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.642395:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.642396:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677448.642397:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.642399:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.642400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.642402:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.642402:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.642403:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.642404:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.642406:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.642407:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.642408:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.642408:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:31.0:1619677448.642410:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.642410:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.642411:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.642412:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:13.0:1619677448.642413:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.642415:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.642415:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.642415:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.642416:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.642416:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.642418:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.642418:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.642419:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.642420:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.642423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.642430:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.642432:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.642433:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.642434:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.642436:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.642437:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.642442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:13.0:1619677448.642448:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.642451:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.642451:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.642452:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.642453:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.642454:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.642631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.642638:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.642640:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.642641:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.642643:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.642646:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.642647:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.642648:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.642650:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.642651:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.642652:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.642655:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.642657:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.642659:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.642660:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.642661:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.642663:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.642663:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.642664:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.642664:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.2:1619677448.642668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:25.2:1619677448.642669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.642676:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:26.0:1619677448.642676:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:30.0:1619677448.642679:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:26.0:1619677448.642679:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000400:00000200:30.0:1619677448.642680:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.642680:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:30.0:1619677448.642682:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.642683:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:26.0:1619677448.642683:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.642685:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:26.0:1619677448.642685:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.642687:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.642688:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.642689:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.642690:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.642691:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.642695:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.642697:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:4.2:1619677448.642699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.642700:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.642700:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.642701:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.642703:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.642704:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677448.642705:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:26.0:1619677448.642706:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:26.0:1619677448.642707:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.642709:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.642710:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.642710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:26.0:1619677448.642711:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.642713:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.642715:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.642716:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712254] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.642717:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.642718:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.642718:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.642718:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.642720:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:26.0:1619677448.642721:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.642721:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.642722:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.642723:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:17.0:1619677448.642724:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.642726:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.642727:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.642727:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.642727:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:26.0:1619677448.642729:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.642730:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.642731:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.642731:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.642735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.642743:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.642746:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.642747:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.642747:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.642749:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.642752:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.642772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.642774:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.2:1619677448.642777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:20.0:1619677448.642777:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.642779:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.642781:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.642782:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.642782:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:20.0:1619677448.642784:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.642785:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.642785:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.642786:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677448.642787:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.642788:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.642789:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677448.642789:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.642793:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.642794:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:6.0:1619677448.642794:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.642796:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:20.0:1619677448.642797:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.642797:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.642798:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.642798:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:6.0:1619677448.642799:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.642800:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.642800:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.642801:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.642801:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:6.0:1619677448.642801:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.642802:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.2:1619677448.642804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:6.0:1619677448.642804:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.642806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.642806:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.642809:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.642810:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:8.0:1619677448.642811:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:6.0:1619677448.642811:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.0:1619677448.642813:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:6.0:1619677448.642813:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:24.0:1619677448.642814:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:22.0:1619677448.642814:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.642814:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.642815:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.642816:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.642816:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.642816:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.642816:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677448.642817:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.642818:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.642819:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.642821:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.642823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000400:00000200:8.0:1619677448.642823:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.642825:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.642826:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96712251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.642827:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.642828:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.642828:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:8.0:1619677448.642829:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677448.642830:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.642830:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.642831:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.642833:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.642833:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.642833:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.642833:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.642834:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.642834:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.642835:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.642836:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677448.642836:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.0:1619677448.642837:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.642840:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.642841:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.642842:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.642843:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.642844:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:21.2:1619677448.642845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677448.642845:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.642845:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.642850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.642852:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.642855:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.642857:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.642857:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:31.0:1619677448.642859:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.642859:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677448.642860:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:31.0:1619677448.642861:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:18.0:1619677448.642861:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.642861:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.642863:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.642864:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.642865:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.642866:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.642867:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.642868:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.642869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:31.0:1619677448.642871:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.642872:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.642875:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.642876:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.642876:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.642877:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.642879:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.642879:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.642879:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.642880:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677448.642880:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.642881:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677448.642881:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.642882:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.642882:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:25.2:1619677448.642885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.642892:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.642895:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.642895:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:4.2:1619677448.642895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:22.0:1619677448.642896:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.642898:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.642901:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677448.642902:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.642904:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.642905:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.642907:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.642909:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.642911:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.642912:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.642913:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.642914:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.642915:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.642919:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.642920:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.642923:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.642924:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.642924:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.642926:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.642926:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.642927:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.642928:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.642929:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.642932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.642934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.642940:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677448.642941:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.642942:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.642943:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.642943:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677448.642943:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.642944:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.642945:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.642946:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.642948:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.642949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.642957:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.642960:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.642961:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.642961:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.642962:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.642963:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.642981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.2:1619677448.642985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.642989:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677448.642990:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.642991:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.642992:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677448.642993:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.642994:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.642995:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.642997:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.642999:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.643000:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.643001:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677448.643002:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.643003:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.643004:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.643005:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.643006:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.643006:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.643007:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.643008:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677448.643008:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.643012:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.643013:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.643015:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677448.643016:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.643017:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.643018:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.643019:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:1.0:1619677448.643019:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.643020:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.643020:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677448.643020:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.643020:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.643022:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.643022:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:1.0:1619677448.643022:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.643023:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.643024:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.643024:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.643025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:25.2:1619677448.643029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.643031:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.643033:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.643034:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.643035:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.643036:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.643036:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:26.0:1619677448.643038:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:6.0:1619677448.643039:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.643040:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.643040:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.643042:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.643045:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.643102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.643109:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.643111:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.643112:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.643114:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.643116:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.643118:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.643119:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.643120:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.643122:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.643123:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.643126:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.643128:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.643131:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.643131:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.643132:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.643134:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.643134:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.643135:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.643136:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.643140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.643147:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.643150:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.643151:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.643151:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.643153:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.643156:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.643266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.643274:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.643277:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.643279:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.643281:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.643282:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.643284:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.643285:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712245] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.643286:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.643287:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.643288:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.643292:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.643299:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.643302:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.643303:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.643304:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.643305:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.643306:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.643307:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.643307:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.643312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.643320:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.643323:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.643324:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.643324:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.643327:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.643329:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.643359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.643366:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.643368:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.643369:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.643371:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.643374:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.643375:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.643376:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.643377:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.643379:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.643379:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.643383:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.643385:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.643387:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.643388:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.643389:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.643391:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.643391:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.643392:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.643392:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.643397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.643404:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.643406:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.643407:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.643407:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.643409:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.643411:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.643417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.643420:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.643423:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.643425:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.643426:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.643427:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.643429:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.643430:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.643431:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.643432:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.643433:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.643437:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.643438:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.643441:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.643442:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.643442:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.643444:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.643445:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.643446:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.643446:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.643450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.2:1619677448.643457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677448.643458:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.643461:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.643462:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.643463:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.643465:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.643465:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.643467:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.643468:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.643469:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.643471:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.643473:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.643475:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.643476:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.643477:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.643478:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.643479:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.643482:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.643484:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.643487:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.643488:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.643488:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.643490:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.643491:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.643492:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.643493:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.643493:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677448.643497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.643499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.643505:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.643506:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.643508:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:2.0:1619677448.643508:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.643509:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.643509:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:2.0:1619677448.643509:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.643511:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:2.0:1619677448.643512:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.643515:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.643617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.643625:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.643628:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.643630:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.643632:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.643633:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.643634:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.643636:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.643637:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.643638:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.643639:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.643642:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.643644:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.643647:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.643648:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.643648:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.643650:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.643651:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.643652:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.643652:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.643657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.643666:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.643668:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.643669:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.643671:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.643671:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.643675:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.643714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.643721:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.643724:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.643725:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.643727:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.643730:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.643731:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.643732:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.643734:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.643735:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.643736:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.643739:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.643740:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.643742:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.643743:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.643744:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.643745:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.643746:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.643746:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.643748:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.643748:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.643751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.643754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.643757:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.643759:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.643760:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.643761:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677448.643762:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.643763:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.643763:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.643764:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.643764:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.643765:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677448.643836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.643842:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.643844:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.643845:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.643847:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.643849:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.643851:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.643852:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.643853:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.643855:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.643855:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.643858:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.643860:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.643862:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.643863:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.643864:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.643866:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.643866:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.643867:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:21.2:1619677448.643868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677448.643868:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.643869:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677448.643872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.643874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677448.643875:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.643878:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:6.0:1619677448.643879:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:30.0:1619677448.643880:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.643880:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:6.0:1619677448.643881:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677448.643881:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:31.0:1619677448.643882:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677448.643882:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:30.0:1619677448.643883:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.643883:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.643883:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.643884:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:30.0:1619677448.643885:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:6.0:1619677448.643886:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:30.0:1619677448.643887:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.643888:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.643888:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.643889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000400:00000200:30.0:1619677448.643890:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.643890:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.643894:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.643896:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.643897:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.2:1619677448.643897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677448.643899:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.643900:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.643900:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.643900:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.643902:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.643902:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.643903:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (24)--
00000800:00000200:22.0:1619677448.643904:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:30.0:1619677448.643905:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.643905:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.643905:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.643907:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:10.0:1619677448.643907:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:30.0:1619677448.643908:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:26.0:1619677448.643908:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.643909:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.643910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.643910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000400:00000200:10.0:1619677448.643910:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.643912:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.643913:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:21.2:1619677448.643914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677448.643914:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.643916:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.643916:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:10.0:1619677448.643917:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.643917:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:6.0:1619677448.643918:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:1.0:1619677448.643919:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677448.643920:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:6.0:1619677448.643920:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677448.643922:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677448.643922:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.643922:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.643923:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.643924:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.643924:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.643925:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.643925:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.643926:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.643926:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.643927:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.643927:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:1.0:1619677448.643927:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.643928:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.643928:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:1.0:1619677448.643928:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677448.643929:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.643930:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.643930:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.643931:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:1.0:1619677448.643931:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.643932:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:1.0:1619677448.643932:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.643933:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:25.2:1619677448.643935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:1.0:1619677448.643936:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.643938:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.2:1619677448.643939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:26.0:1619677448.643941:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:1.0:1619677448.643941:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.643941:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:1.0:1619677448.643942:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.643943:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.643944:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.643944:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:10.0:1619677448.643944:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:1.0:1619677448.643944:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677448.643944:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677448.643945:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677448.643945:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:26.0:1619677448.643946:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.643947:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:26.0:1619677448.643948:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:2.0:1619677448.643948:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.643949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:18.0:1619677448.643950:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.643952:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.643954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000400:00000200:18.0:1619677448.643954:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.643955:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.643956:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.643957:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:18.0:1619677448.643957:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.643958:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.643960:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.643960:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.643960:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000400:00000200:18.0:1619677448.643961:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.643963:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:20.0:1619677448.643963:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.643963:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.643964:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.643964:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.643965:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.643965:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.643966:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677448.643966:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.643966:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:18.0:1619677448.643967:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.643967:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.2:1619677448.643968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677448.643969:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.643970:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.643970:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.643971:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.643975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.643976:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.643979:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.643980:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.643980:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.643981:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.643982:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677448.643983:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.643986:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:4.2:1619677448.643986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:17.0:1619677448.643987:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.643987:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.643989:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.643991:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:24.0:1619677448.643993:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.643996:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.643997:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.643999:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.644002:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.644003:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.644005:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.644006:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.644007:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.644008:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.644011:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.644013:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.644016:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.644017:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.644017:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.644018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.0:1619677448.644019:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.644019:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.644020:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.644020:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677448.644024:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.2:1619677448.644025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.644027:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.644029:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.644029:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.644031:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.644032:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.644032:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.644033:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.644034:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.644035:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.644035:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.644037:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.644038:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.644039:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.644042:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.644043:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:1.0:1619677448.644043:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.644045:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.644049:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.644049:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.644050:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.644052:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.644052:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.644054:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.644055:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.644055:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.644058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.644060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.644065:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677448.644066:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.644067:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.644068:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677448.644068:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:17.0:1619677448.644069:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677448.644069:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.644071:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.644073:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.644074:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.644212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.644219:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.644222:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.644222:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.644225:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.644227:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.644228:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.644229:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.644231:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.644232:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.644233:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.644236:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.644237:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.644240:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.644241:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.644242:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.644244:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.644244:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.644245:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.644245:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.644250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.644257:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.644260:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.644261:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.644261:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.644262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:2.0:1619677448.644263:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.644266:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677448.644268:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.644271:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.644271:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.644273:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.644275:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.644277:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.644278:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.644280:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.644281:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.644282:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.644285:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.644287:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.644290:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.644291:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.644291:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.644299:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:25.2:1619677448.644300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (22)++
00000800:00000200:22.0:1619677448.644300:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.644301:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.644301:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.644307:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.644310:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.644310:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.644311:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.644313:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.644316:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:25.2:1619677448.644323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000800:00000200:26.0:1619677448.644329:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.644331:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.644332:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.644332:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.644333:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.644334:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677448.644376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677448.644379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677448.644386:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:5.0:1619677448.644387:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677448.644389:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677448.644390:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:5.0:1619677448.644390:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.644392:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:8.0:1619677448.644393:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677448.644393:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.644396:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.644396:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.644398:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.644398:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.644399:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.644400:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.644401:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.644402:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.644403:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.644403:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.644404:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.644404:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.644409:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.644411:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.644412:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.0:1619677448.644413:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.644415:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.644416:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:8.0:1619677448.644417:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.644417:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.644417:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.0:1619677448.644418:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.644419:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:21.2:1619677448.644420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677448.644420:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.644420:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677448.644421:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677448.644422:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:5.0:1619677448.644422:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:8.0:1619677448.644423:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677448.644424:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:5.0:1619677448.644424:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:25.2:1619677448.644425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.644425:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677448.644429:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677448.644429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.644430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.2:1619677448.644430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (21)++
00000800:00000200:31.0:1619677448.644431:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677448.644433:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.644433:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:31.0:1619677448.644434:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.644435:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677448.644435:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:31.0:1619677448.644436:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.644436:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.644436:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:24.0:1619677448.644437:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:20.0:1619677448.644437:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677448.644437:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:31.0:1619677448.644439:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.644439:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.644440:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96712228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.644440:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677448.644440:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:6.0:1619677448.644440:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.644441:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677448.644441:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000400:00000200:31.0:1619677448.644442:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.644442:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.644443:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.644445:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:6.0:1619677448.644445:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:30.0:1619677448.644448:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.644448:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:31.0:1619677448.644449:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.644449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:30.0:1619677448.644450:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.644451:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.644451:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.644452:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.644453:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.644453:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.644454:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.644454:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.644454:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.644455:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.644455:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.644456:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.644456:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:31.0:1619677448.644457:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.644458:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.644458:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.644459:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.644459:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677448.644461:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.644462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.644464:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.644465:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:10.0:1619677448.644466:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.644468:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.644469:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677448.644469:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.644469:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:13.0:1619677448.644471:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.644472:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.644472:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677448.644473:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.644474:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677448.644476:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.644476:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.644477:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.644477:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:10.0:1619677448.644477:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.644478:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.644479:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.644480:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677448.644481:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677448.644481:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:25.2:1619677448.644484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:21.2:1619677448.644489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.644493:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.644494:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.644495:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.644497:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.644497:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677448.644498:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677448.644498:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677448.644499:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.644499:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:26.0:1619677448.644500:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:5.2:1619677448.644529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.644535:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.644538:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.644539:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.644541:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.644544:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.644545:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.644546:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.644548:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.644549:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.644550:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.644553:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.644555:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.644557:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.644557:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.644558:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.644560:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.644560:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.644561:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.644561:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.644566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.644573:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.644575:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.644575:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.644576:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.644577:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.644579:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.644583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.644590:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.644592:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.644593:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.644593:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.644594:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.644594:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.644661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.644664:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.644667:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.644669:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.644670:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.644672:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.644674:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677448.644675:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.644676:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.644677:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.644678:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677448.644682:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.644684:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.644687:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.644687:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.644688:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.644690:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.644691:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.644691:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.644692:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.644696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.644702:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.644705:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.644705:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.644706:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.644708:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.644711:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.644775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.644782:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.644785:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.644785:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.644787:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.644789:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.644791:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.644792:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.644793:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.644794:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.644795:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.644798:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.644800:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.644803:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.644803:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.644804:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.644806:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.644807:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.644807:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.644808:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.2:1619677448.644809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:8.2:1619677448.644812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.644816:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.644818:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.644819:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.644820:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.644820:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:10.0:1619677448.644822:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.644823:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:5.0:1619677448.644823:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.644823:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.644824:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.644825:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.644826:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:10.0:1619677448.644826:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.644827:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.644828:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.644832:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.644833:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:10.0:1619677448.644834:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.644836:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.644837:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.644838:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.644839:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.644840:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.644840:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.644841:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.644845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.644853:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.644855:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.644856:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.644858:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.644858:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.644861:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.644929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.644935:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.644937:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.644938:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.644940:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.644942:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.644943:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.644945:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.644946:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.644947:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.644948:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.644951:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.644952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:17.0:1619677448.644953:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.644955:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.644956:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.644956:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.644957:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.644958:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.644958:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677448.644959:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.644959:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677448.644963:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.2:1619677448.644963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:24.0:1619677448.644966:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.644968:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.644970:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.644970:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:24.0:1619677448.644971:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:1.0:1619677448.644972:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:24.0:1619677448.644973:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96712221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.644973:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.644974:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:24.0:1619677448.644975:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.644975:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.644976:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.644977:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.644978:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:24.0:1619677448.644980:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.644982:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.644985:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.644986:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.644986:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.644988:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.644989:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.644989:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.644990:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.644994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.645001:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.645004:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.645004:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.645004:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.645007:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.645009:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.645240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.645246:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:21.2:1619677448.645246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:22.0:1619677448.645249:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.645249:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.645252:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.645253:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:22.0:1619677448.645254:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.645254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:22.0:1619677448.645256:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:8.0:1619677448.645256:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.645257:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:22.0:1619677448.645257:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.645259:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.645260:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.645260:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.645260:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:22.0:1619677448.645261:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.645262:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.645263:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.645265:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:8.0:1619677448.645265:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:22.0:1619677448.645266:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.645266:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.645267:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96712219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.645269:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.645269:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.645269:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.645270:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.645271:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677448.645271:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:22.0:1619677448.645272:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.645272:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.645273:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.645274:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677448.645274:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677448.645274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677448.645275:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:5.0:1619677448.645275:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.645276:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:5.0:1619677448.645276:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677448.645277:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.645278:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.645279:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:5.0:1619677448.645280:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.645281:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677448.645281:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.645283:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.645284:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.645285:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:8.0:1619677448.645286:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.645287:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.645289:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677448.645289:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:26.0:1619677448.645290:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.645290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.0:1619677448.645291:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.645291:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:1.0:1619677448.645291:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:26.0:1619677448.645292:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:17.0:1619677448.645292:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.645292:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.645292:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.645293:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.645293:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.645301:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677448.645301:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.645301:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:1.0:1619677448.645301:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.645302:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96712217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.645302:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677448.645303:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.645303:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.645303:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677448.645303:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.645303:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.0:1619677448.645304:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:1.0:1619677448.645304:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677448.645305:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677448.645305:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.645305:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:26.0:1619677448.645306:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677448.645307:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.645308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.645309:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677448.645309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:30.0:1619677448.645310:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.645310:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.645311:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.2:1619677448.645311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:30.0:1619677448.645312:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96712216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.645312:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:30.0:1619677448.645314:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.645314:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.645316:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.645316:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.645317:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.645317:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.0:1619677448.645317:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:18.0:1619677448.645318:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:30.0:1619677448.645319:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.645319:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.645319:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.645319:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:2.0:1619677448.645319:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:30.0:1619677448.645320:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.645320:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.2:1619677448.645320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:20.0:1619677448.645320:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.645320:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.645321:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:18.0:1619677448.645321:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.645321:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:20.0:1619677448.645322:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.645324:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.645324:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.645324:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.645325:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:25.2:1619677448.645325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.0:1619677448.645325:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:2.0:1619677448.645325:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.645326:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.645327:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.645327:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.645327:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677448.645328:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677448.645328:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:2.0:1619677448.645328:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.645329:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677448.645329:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:6.0:1619677448.645330:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:2.0:1619677448.645330:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.645331:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96712215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.645332:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:25.2:1619677448.645333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.645333:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677448.645333:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.645334:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:2.0:1619677448.645334:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.645335:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.2:1619677448.645335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:2.0:1619677448.645335:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677448.645336:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.645337:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.645338:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.645338:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.645339:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.645340:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.645340:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.645341:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.645342:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677448.645342:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.645343:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.645343:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.645343:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.645343:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.645344:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:25.2:1619677448.645345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677448.645345:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.645345:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:1.0:1619677448.645345:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.645346:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677448.645347:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.645347:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.645347:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.645347:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677448.645349:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:17.0:1619677448.645350:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.645351:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:9.2:1619677448.645351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:25.2:1619677448.645352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:22.0:1619677448.645352:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:17.0:1619677448.645352:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.645354:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.645355:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.645355:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.645356:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:6.0:1619677448.645356:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.645357:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.645358:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.645358:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677448.645359:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.645360:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:17.0:1619677448.645360:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.645362:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:12.0:1619677448.645362:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.645363:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.645363:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.645363:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.645364:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.645364:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.645364:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677448.645365:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.645366:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.645366:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:12.0:1619677448.645366:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677448.645367:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:18.0:1619677448.645368:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677448.645369:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.645369:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.645370:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.645370:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.645370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:9.2:1619677448.645374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:2.0:1619677448.645377:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.645380:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.645381:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:18.0:1619677448.645381:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.645381:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.645382:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.645383:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:2.0:1619677448.645383:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.0:1619677448.645384:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.645385:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.645387:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.645390:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.2:1619677448.645443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:10.0:1619677448.645451:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677448.645453:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:26.0:1619677448.645454:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:10.0:1619677448.645456:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.645459:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.645461:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677448.645462:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.645463:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.645464:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.645465:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677448.645468:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.645470:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677448.645473:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.645474:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:10.0:1619677448.645475:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:10.0:1619677448.645476:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:10.0:1619677448.645477:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677448.645477:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677448.645478:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:8.2:1619677448.645482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.645491:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.645493:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.645494:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.645496:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.645496:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.645499:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:8.2:1619677448.645502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (20)++
00000800:00000200:20.0:1619677448.645509:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.645512:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677448.645513:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.645514:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.645514:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.645515:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:20.2:1619677448.645610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.645617:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.645620:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.645623:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.645624:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.645625:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.645626:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.645628:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.645629:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.645630:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.645631:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.645634:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.645636:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.645638:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.645639:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.645639:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.645641:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.645641:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.645643:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.645644:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.645645:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.645646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.2:1619677448.645651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.645655:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.645657:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.645658:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.645659:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:30.0:1619677448.645659:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.645660:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.645660:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.645661:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.645662:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.645663:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.2:1619677448.645664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.645671:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.645674:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.645675:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.645677:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.645680:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.645681:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.645682:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.645683:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.645684:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.645685:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.645688:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.645690:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.645692:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.645693:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.645693:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.645695:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.645695:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.645696:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.645696:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.645700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.645707:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.645709:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.645710:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.645712:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.645713:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.645714:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.645915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.645920:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:21.2:1619677448.645922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:6.0:1619677448.645923:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.645924:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.645926:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.645928:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:6.0:1619677448.645928:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.645929:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:18.0:1619677448.645931:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:6.0:1619677448.645931:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.645931:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:6.0:1619677448.645932:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.645934:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.645935:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.645935:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.645939:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.645940:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.645941:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:18.0:1619677448.645942:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:6.0:1619677448.645943:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.645944:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.645944:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:18.0:1619677448.645945:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.645945:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.2:1619677448.645946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:18.0:1619677448.645946:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.645946:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.645947:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:6.0:1619677448.645947:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.645948:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.645948:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677448.645949:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:18.0:1619677448.645950:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.645951:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:18.0:1619677448.645953:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:9.2:1619677448.645953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:20.0:1619677448.645954:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.645954:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.645956:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.645957:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.645958:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:18.0:1619677448.645958:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:20.0:1619677448.645959:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96712208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.645959:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:20.0:1619677448.645961:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.645961:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:20.0:1619677448.645962:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.645962:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.645962:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.645962:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:20.0:1619677448.645963:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.645963:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000400:00000200:2.0:1619677448.645963:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.645964:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.645965:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.645966:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.645966:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.645967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:20.0:1619677448.645967:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.645969:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.645969:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.645970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.0:1619677448.645972:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677448.645972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677448.645973:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.645973:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:20.0:1619677448.645974:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.645976:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.645976:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.645977:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.645977:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.645977:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:10.0:1619677448.645977:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.645978:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677448.645978:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.645979:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:10.0:1619677448.645979:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.645980:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.645981:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.645982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.645982:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:10.0:1619677448.645982:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:18.0:1619677448.645984:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.645985:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.645985:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.645986:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.645987:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:12.0:1619677448.645989:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.645991:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.645992:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.645994:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.645995:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.645997:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.646034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.646042:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.646045:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.646047:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.646048:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.646049:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.646051:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.646052:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.646054:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.646054:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.646056:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.646059:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.646060:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.646062:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.646063:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.646064:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.646066:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.646066:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.646067:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.646068:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.646071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.646076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677448.646077:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:21.2:1619677448.646079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:13.0:1619677448.646080:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.646080:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.646081:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:13.0:1619677448.646082:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.646083:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.646084:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.646085:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.646085:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.646085:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:1.0:1619677448.646087:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.646088:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.646089:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:1.0:1619677448.646090:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.646091:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.646092:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677448.646093:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.646094:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.646094:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.646095:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.646096:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677448.646096:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.646098:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.646100:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.646101:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.646101:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.646102:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:1.0:1619677448.646103:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.646105:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677448.646106:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.646107:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677448.646107:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.646108:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.646110:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.646110:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.646110:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.646111:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677448.646112:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:1.0:1619677448.646112:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.646113:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:2.0:1619677448.646114:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677448.646114:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677448.646114:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.646115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.646115:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.646116:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.2:1619677448.646118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:25.2:1619677448.646119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:12.0:1619677448.646122:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.646124:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:26.0:1619677448.646125:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:12.0:1619677448.646125:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.646125:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.646126:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.646127:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.646127:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677448.646127:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.646128:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:13.0:1619677448.646128:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:26.0:1619677448.646129:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.646129:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:12.0:1619677448.646130:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677448.646131:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:26.0:1619677448.646134:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:25.2:1619677448.646139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.2:1619677448.646139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:13.0:1619677448.646146:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.646147:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677448.646149:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.646150:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:13.0:1619677448.646150:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:21.2:1619677448.646151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37e800] (20)++
00000800:00000200:13.0:1619677448.646151:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.646151:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:30.0:1619677448.646152:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.646152:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.646154:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.646154:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.646155:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.646157:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.646158:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.646158:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37e800] (21)++
00000400:00000200:30.0:1619677448.646159:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.646160:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.646161:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.56@o2ib3
00000800:00000200:26.0:1619677448.646162:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)--
00000400:00000200:10.0:1619677448.646164:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.56@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.646165:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.646166:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.646167:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.646168:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677448.646169:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.646169:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:10.0:1619677448.646169:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.646170:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:10.0:1619677448.646170:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.646171:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:10.0:1619677448.646171:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.646172:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677448.646172:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.646173:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.646173:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:10.0:1619677448.646175:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.646177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.646177:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.646180:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.646180:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.646181:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.646183:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.646183:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37e800] (21)++
00000800:00000200:31.0:1619677448.646184:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:10.0:1619677448.646184:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37e800] (22)--
00000800:00000200:10.0:1619677448.646184:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (21)--
00000800:00000200:31.0:1619677448.646186:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.646187:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.646188:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.646189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.646190:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.646191:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.646194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:22.0:1619677448.646196:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.646199:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677448.646200:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000400:00000200:22.0:1619677448.646200:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.646200:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.646203:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000400:00000200:22.0:1619677448.646203:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.646204:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.646205:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.646205:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.646206:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:22.0:1619677448.646206:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.646288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.646294:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.646300:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.646301:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.646303:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.646306:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.646308:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677448.646309:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.646311:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.646312:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.646313:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677448.646316:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.646318:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677448.646320:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.646321:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677448.646322:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677448.646324:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.646325:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.646325:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.646326:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.646330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.646339:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.646342:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.646343:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.646344:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.646346:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.646347:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.646348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37e800] (20)++
00000800:00000200:26.0:1619677448.646355:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37e800] (21)++
00000800:00000200:26.0:1619677448.646358:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.56@o2ib3
00000800:00000200:10.0:1619677448.646359:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)--
00000400:00000200:26.0:1619677448.646361:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.56@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.646363:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.646365:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.646366:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.646367:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.646368:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.646369:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.646372:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.646374:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.646376:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.646377:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.646377:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.646379:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.646379:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37e800] (21)++
00000800:00000200:26.0:1619677448.646380:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37e800] (22)--
00000800:00000200:26.0:1619677448.646380:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (21)--
00000800:00000200:25.2:1619677448.646384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.646390:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.646392:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.646392:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.646393:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.646394:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.646395:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.646418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.646425:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.646428:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.646429:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.646430:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.646431:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.646433:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.646434:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.646435:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.646435:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.646436:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.646439:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.646440:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.646442:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.646443:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.646443:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.646445:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.646445:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.646446:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.646446:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.646450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.646458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37e800] (20)++
00000800:00000200:12.0:1619677448.646458:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.646460:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.646461:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.646462:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.646462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677448.646464:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.646465:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.646465:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37e800] (21)++
00000800:00000200:10.0:1619677448.646467:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.56@o2ib3
00000800:00000200:26.0:1619677448.646468:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)--
00000800:00000200:18.0:1619677448.646469:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:10.0:1619677448.646469:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.56@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.646472:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:10.0:1619677448.646472:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.646472:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:21.2:1619677448.646473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:10.0:1619677448.646473:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677448.646476:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.646476:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.646477:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.646478:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.646479:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.646479:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:2.0:1619677448.646479:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.646481:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:18.0:1619677448.646482:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.646484:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.646484:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.646486:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.646487:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.646488:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:18.0:1619677448.646489:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.646491:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.646491:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.646492:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.646492:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:18.0:1619677448.646494:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.646494:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.646494:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37e800] (21)++
00000400:00000200:2.0:1619677448.646494:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.646495:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37e800] (22)--
00000400:00000200:2.0:1619677448.646495:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:18.0:1619677448.646496:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.646496:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (21)--
00000400:00000200:2.0:1619677448.646497:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96712197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.646499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:2.0:1619677448.646499:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.646500:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.646501:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.646504:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.646505:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.646505:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:2.0:1619677448.646505:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.646506:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:18.0:1619677448.646507:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.646507:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.646508:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:18.0:1619677448.646508:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:31.0:1619677448.646509:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.646509:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.646510:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677448.646510:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.646511:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.646511:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.646511:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.646511:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.646512:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.646513:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:2.0:1619677448.646513:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:8.2:1619677448.646514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.646514:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.646515:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.646515:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.646519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.646522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37e800] (20)++
00000800:00000200:20.0:1619677448.646522:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.646524:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.646525:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.646526:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.646529:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:5.0:1619677448.646529:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.646530:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:26.0:1619677448.646531:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:5.0:1619677448.646531:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.646532:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.646532:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:10.0:1619677448.646532:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37e800] (21)++
00000800:00000200:5.0:1619677448.646532:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.646533:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (24)--
00000800:00000200:10.0:1619677448.646533:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.5.56@o2ib3
00000800:00000200:5.0:1619677448.646533:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (24)--
00000400:00000200:26.0:1619677448.646534:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.646534:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.646535:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:10.0:1619677448.646536:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.56@o2ib3 : REPLY - routed
00000800:00000200:26.0:1619677448.646538:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.646539:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.646539:0:19940:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 49626
00000800:00000200:26.0:1619677448.646540:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.646541:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.646541:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37e800] (22)++
00000800:00000200:26.0:1619677448.646542:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.646542:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.646543:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (24)--
00000800:00000200:10.0:1619677448.646543:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37e800] (23)++
00000800:00000200:10.0:1619677448.646543:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37e800] (24)--
00000800:00000200:26.0:1619677448.646544:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.646544:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)--
00000800:00000200:21.2:1619677448.646549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37e800] (21)++
00000800:00000200:10.0:1619677448.646556:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)++
00000800:00000200:10.0:1619677448.646558:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (23)--
00000800:00000200:10.0:1619677448.646559:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)--
00000800:00000200:21.2:1619677448.646572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37e800] (21)++
00000800:00000200:26.0:1619677448.646579:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)++
00000800:00000200:26.0:1619677448.646582:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.5.56@o2ib3
00000800:00000200:10.0:1619677448.646582:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (23)--
00000800:00000200:26.0:1619677448.646583:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37e800] (23)--
00000400:00000200:26.0:1619677448.646584:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.646585:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677448.646587:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.646588:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.646589:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.646590:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677448.646593:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : REPLY try# 0
00000800:00000200:26.0:1619677448.646595:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 49626 bytes in 256 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677448.646596:0:19941:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 49626
00000800:00000200:26.0:1619677448.646598:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.646599:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:26.0:1619677448.646600:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:26.0:1619677448.646601:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:26.0:1619677448.646601:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37e800] (21)++
00000800:00000200:26.0:1619677448.646602:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37e800] (22)--
00000800:00000200:26.0:1619677448.646602:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (21)--
00000800:00000200:9.2:1619677448.646606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.646613:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.646615:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677448.646616:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:4.2:1619677448.646620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677448.646626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:10.0:1619677448.646627:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.2:1619677448.646628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37e800] (20)++
00000800:00000200:10.0:1619677448.646629:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.646631:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.646632:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677448.646632:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:10.0:1619677448.646633:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.646634:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37e800] (21)++
00000800:00000200:26.0:1619677448.646635:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.56@o2ib3
00000800:00000200:24.0:1619677448.646635:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[1] from 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677448.646635:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:8.0:1619677448.646636:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:10.0:1619677448.646637:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.646638:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000400:00000200:10.0:1619677448.646638:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.646639:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.646640:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:24.0:1619677448.646641:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.646641:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:26.0:1619677448.646642:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.56@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.646643:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.646644:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.646645:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.646646:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.646647:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96712194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.646648:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.646649:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.646649:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.646652:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.646653:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.646653:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:9.2:1619677448.646653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:10.0:1619677448.646654:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.646655:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.646656:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.646657:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.646658:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.646658:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.646659:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.646659:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:8.0:1619677448.646659:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:26.0:1619677448.646660:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.646660:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.646661:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)--
00000800:00000200:26.0:1619677448.646662:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:25.2:1619677448.646662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:8.0:1619677448.646662:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:26.0:1619677448.646663:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37e800] (21)++
00000800:00000200:24.0:1619677448.646663:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.646663:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.646664:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37e800] (22)--
00000800:00000200:26.0:1619677448.646664:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (21)--
00000400:00000200:8.0:1619677448.646664:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: REPLY: OK
00000800:00000200:4.2:1619677448.646665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677448.646668:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677448.646670:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677448.646672:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:8.0:1619677448.646672:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:6.0:1619677448.646673:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.646674:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.646674:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677448.646674:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.646675:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.646676:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.646676:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.646676:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:6.0:1619677448.646679:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:1.0:1619677448.646679:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.646680:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.646680:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.646682:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.646686:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.646687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.646694:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.646697:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.646698:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.646699:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.646702:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.646703:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:21.2:1619677448.646704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37e800] (20)++
00000400:00000200:24.0:1619677448.646705:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.646706:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.646707:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.646708:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.646711:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37e800] (21)++
00000400:00000200:24.0:1619677448.646711:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.646712:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.646714:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.5.56@o2ib3
00000800:00000200:10.0:1619677448.646715:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)--
00000400:00000200:26.0:1619677448.646716:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.56@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.646716:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.646716:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.646717:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.646719:0:19941:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 10896
00000800:00000200:24.0:1619677448.646719:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.646719:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677448.646720:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37e800] (21)++
00000800:00000200:24.0:1619677448.646720:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.646720:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:26.0:1619677448.646722:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37e800] (22)++
00000800:00000200:26.0:1619677448.646722:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37e800] (23)--
00000800:00000200:26.0:1619677448.646723:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)--
00000800:00000200:8.2:1619677448.646725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.646728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37e800] (21)++
00000800:00000200:17.0:1619677448.646733:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:10.0:1619677448.646734:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)++
00000800:00000200:17.0:1619677448.646735:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.646736:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.646737:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.646737:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (23)--
00000800:00000200:10.0:1619677448.646738:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)--
00000400:00000200:17.0:1619677448.646739:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.646741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37e800] (21)++
00000800:00000200:20.2:1619677448.646742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677448.646742:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.646743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:26.0:1619677448.646748:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)++
00000800:00000200:30.0:1619677448.646750:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.646751:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.5.56@o2ib3
00000800:00000200:1.0:1619677448.646751:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.646752:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37e800] (23)--
00000800:00000200:10.0:1619677448.646752:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (23)--
00000800:00000200:30.0:1619677448.646753:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:26.0:1619677448.646753:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.646754:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677448.646755:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:17.0:1619677448.646755:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.646755:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677448.646756:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.646756:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.646756:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.646757:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:1.0:1619677448.646757:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:30.0:1619677448.646758:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.646758:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.646759:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.646760:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.646760:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.646762:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.646763:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.646764:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.646764:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.646765:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.646766:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 10896 bytes in 256 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.646768:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.646768:0:19941:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 10896
00000800:00000200:30.0:1619677448.646770:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.646770:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.646771:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.646771:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.646772:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.646773:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.646773:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.646773:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:21.2:1619677448.646773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:26.0:1619677448.646774:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37e800] (21)++
00000800:00000200:26.0:1619677448.646774:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37e800] (22)--
00000800:00000200:30.0:1619677448.646775:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.646775:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.646775:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (21)--
00000800:00000200:30.0:1619677448.646776:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.646777:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:22.0:1619677448.646778:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:25.2:1619677448.646781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:22.0:1619677448.646781:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.646782:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.646784:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.646786:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.646787:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:22.0:1619677448.646787:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.646789:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96712190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.646790:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.646790:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.646790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:22.0:1619677448.646790:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.646791:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.646792:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.646793:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:22.0:1619677448.646793:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.646795:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:22.0:1619677448.646798:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.646798:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.646800:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.646800:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.646801:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.646803:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.646804:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.646804:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.646806:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:25.2:1619677448.646807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.646807:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.646807:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:21.2:1619677448.646808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37e800] (20)++
00000800:00000200:22.0:1619677448.646810:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677448.646811:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:9.2:1619677448.646812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.646813:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[3] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.646815:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (23)++
00000800:00000200:10.0:1619677448.646815:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37e800] (21)++
00000800:00000200:6.0:1619677448.646815:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.646817:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:21.2:1619677448.646817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677448.646817:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.5.56@o2ib3
00000800:00000200:26.0:1619677448.646818:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)--
00000800:00000200:22.0:1619677448.646818:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.646820:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:10.0:1619677448.646820:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.56@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.646822:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:10.0:1619677448.646822:0:19940:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 9608
00000800:00000200:10.0:1619677448.646823:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37e800] (21)++
00000800:00000200:30.0:1619677448.646824:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677448.646824:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.646824:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37e800] (22)++
00000800:00000200:2.0:1619677448.646824:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:10.0:1619677448.646825:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37e800] (23)--
00000400:00000200:18.0:1619677448.646826:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.646826:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)--
00000800:00000200:30.0:1619677448.646827:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:18.0:1619677448.646828:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:30.0:1619677448.646829:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.646830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37e800] (21)++
00000800:00000200:31.0:1619677448.646831:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.646832:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.646833:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.646834:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.646835:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.646837:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.646837:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)++
00000400:00000200:30.0:1619677448.646838:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.646839:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (23)--
00000800:00000200:26.0:1619677448.646840:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)++
00000400:00000200:30.0:1619677448.646841:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.646841:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.5.56@o2ib3
00000800:00000200:10.0:1619677448.646841:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (23)--
00000800:00000200:26.0:1619677448.646842:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37e800] (22)--
00000800:00000200:30.0:1619677448.646843:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.646843:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.646845:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.646845:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:30.0:1619677448.646846:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:26.0:1619677448.646846:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96712188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.646847:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:26.0:1619677448.646847:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.646848:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:26.0:1619677448.646848:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.646849:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677448.646849:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.646850:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:25.2:1619677448.646850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.646852:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.646852:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677448.646853:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.646853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.646854:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 9608 bytes in 256 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.646856:0:19941:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 9608
00000800:00000200:6.0:1619677448.646857:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:21.2:1619677448.646858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.646859:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.646859:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.646859:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.646860:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677448.646860:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:22.0:1619677448.646861:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.646861:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.646862:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:26.0:1619677448.646862:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37e800] (21)++
00000800:00000200:13.0:1619677448.646862:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:26.0:1619677448.646863:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37e800] (22)--
00000800:00000200:26.0:1619677448.646863:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (21)--
00000400:00000200:13.0:1619677448.646863:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.646863:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:13.0:1619677448.646865:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.646866:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:6.0:1619677448.646866:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677448.646867:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.646867:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.646868:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.2:1619677448.646868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.646869:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.646870:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677448.646876:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.646879:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.646880:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.2:1619677448.646889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.646897:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.646899:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[0] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.646900:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.646902:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.646904:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.646904:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:21.2:1619677448.646906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37e800] (20)++
00000800:00000200:10.0:1619677448.646913:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37e800] (21)++
00000800:00000200:8.2:1619677448.646913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677448.646916:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.5.56@o2ib3
00000800:00000200:26.0:1619677448.646917:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)--
00000400:00000200:10.0:1619677448.646918:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.56@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.646920:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:10.0:1619677448.646921:0:19940:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 9600
00000800:00000200:10.0:1619677448.646922:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37e800] (21)++
00000800:00000200:17.0:1619677448.646923:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:10.0:1619677448.646923:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37e800] (22)++
00000400:00000200:17.0:1619677448.646924:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.646924:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37e800] (23)--
00000800:00000200:1.0:1619677448.646924:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:10.0:1619677448.646925:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)--
00000400:00000200:17.0:1619677448.646926:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.646929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37e800] (21)++
00000800:00000200:17.0:1619677448.646929:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677448.646936:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)++
00000800:00000200:26.0:1619677448.646938:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (23)--
00000800:00000200:26.0:1619677448.646939:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)--
00000800:00000200:21.2:1619677448.646942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37e800] (21)++
00000800:00000200:10.0:1619677448.646948:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)++
00000800:00000200:26.0:1619677448.646951:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (23)--
00000800:00000200:10.0:1619677448.646951:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.5.56@o2ib3
00000800:00000200:10.0:1619677448.646952:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37e800] (23)--
00000400:00000200:10.0:1619677448.646953:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.646955:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.646956:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.646957:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.646958:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.646959:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.646963:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.646964:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 9600 bytes in 256 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.646965:0:19940:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 9600
00000800:00000200:10.0:1619677448.646967:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.646967:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.646968:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.646969:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.646970:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37e800] (21)++
00000800:00000200:10.0:1619677448.646970:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37e800] (22)--
00000800:00000200:10.0:1619677448.646971:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (21)--
00000800:00000200:25.2:1619677448.646975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.2:1619677448.646975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.646981:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.646982:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.646984:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:1.0:1619677448.646984:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.2:1619677448.646985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677448.646985:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.646986:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:1.0:1619677448.646987:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.646989:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.646991:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677448.646991:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:25.2:1619677448.646992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:1.0:1619677448.646992:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.646993:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.646994:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:1.0:1619677448.646995:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.646996:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.646997:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.646997:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.646999:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:5.0:1619677448.647000:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.647000:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.647002:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[1] from 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.647002:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:1.0:1619677448.647002:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.647003:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:1.0:1619677448.647004:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.647005:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:21.2:1619677448.647005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:5.0:1619677448.647005:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96712185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.647005:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.647006:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:31.0:1619677448.647006:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:5.0:1619677448.647006:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.647006:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.647007:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.647007:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677448.647008:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.647008:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677448.647009:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.647009:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:5.0:1619677448.647012:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.647013:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.647014:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.2:1619677448.647015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.0:1619677448.647016:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677448.647017:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.647018:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.647018:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.647019:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.647020:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:17.0:1619677448.647020:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677448.647020:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.647020:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.647021:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677448.647022:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677448.647022:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:8.0:1619677448.647022:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.647022:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677448.647023:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.647023:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.647025:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:17.0:1619677448.647025:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.647026:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.647026:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:30.0:1619677448.647028:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.647029:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.647029:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:8.0:1619677448.647029:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.647030:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:30.0:1619677448.647031:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677448.647031:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:8.0:1619677448.647031:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96712184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.647033:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:8.0:1619677448.647033:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.647034:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.647034:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.647035:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:1.0:1619677448.647035:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.647036:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.647038:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:8.0:1619677448.647038:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.647040:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.0:1619677448.647042:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.647043:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:8.0:1619677448.647044:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:8.0:1619677448.647045:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:8.0:1619677448.647046:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.647047:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.647047:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.647051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.647057:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.647059:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.647060:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.647061:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.647063:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.647065:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:8.2:1619677448.647156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.647164:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.647167:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.647168:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.647168:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.647169:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.647170:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.647193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:17.0:1619677448.647198:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677448.647201:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677448.647201:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677448.647203:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.647205:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.647207:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.647208:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.647210:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.647211:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.647212:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.647215:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.647217:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.647218:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.647218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:17.0:1619677448.647219:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.647220:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.647221:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.647222:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677448.647222:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677448.647223:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:2.0:1619677448.647224:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:8.2:1619677448.647227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:2.0:1619677448.647227:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.647228:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.647230:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.647233:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.647234:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:2.0:1619677448.647235:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.647236:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.647236:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:2.0:1619677448.647237:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.647237:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.647238:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:2.0:1619677448.647238:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.647239:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677448.647239:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.647241:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:2.0:1619677448.647243:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.647245:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.647247:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.647248:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.647248:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.647250:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.647250:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.647252:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.647253:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.647253:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.647256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.647258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677448.647264:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677448.647264:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.647266:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:20.0:1619677448.647267:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:18.0:1619677448.647267:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:20.0:1619677448.647268:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.647268:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:20.0:1619677448.647269:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.647270:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.647272:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.647409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.647415:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.647418:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.647419:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.647421:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.647422:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.647424:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.647425:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.647426:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.647427:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.647428:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.647432:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.647433:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.647435:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.647436:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.647436:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.647438:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.647438:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.647439:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.647439:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.647444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.647451:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.647453:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.647454:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.647455:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.647456:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.647457:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.647493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.647501:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.647503:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.647504:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.647506:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.647508:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.647510:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.647511:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.647512:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.647513:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.647514:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.647517:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.647519:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.647522:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.647523:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.647523:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.647525:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.647526:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.647527:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677448.647528:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.647529:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677448.647532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.647534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:2.0:1619677448.647540:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.0:1619677448.647541:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.647543:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:2.0:1619677448.647543:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.647544:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.647544:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:2.0:1619677448.647544:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.647545:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:2.0:1619677448.647546:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.647550:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.647593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.647601:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.647604:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.647606:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.647608:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.647609:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.647610:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.647611:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.647613:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.647614:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.647615:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.647618:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.647619:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.647621:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.647622:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.647623:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.647624:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.647625:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.647626:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.647627:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.647630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:21.2:1619677448.647634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:12.0:1619677448.647638:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677448.647641:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.647641:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.647641:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.647643:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.647644:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:4.2:1619677448.647644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677448.647645:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:30.0:1619677448.647646:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.647646:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.647648:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.647648:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.647649:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.647650:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.647652:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.647652:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677448.647653:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.647653:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.647654:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.647655:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:30.0:1619677448.647657:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.647657:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.647659:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.647660:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.647661:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:30.0:1619677448.647662:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.647662:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.647663:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.647664:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.647664:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.647664:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.647665:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.647665:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677448.647665:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.647666:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.647667:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:10.0:1619677448.647668:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.647670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.647670:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.647671:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.647672:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.647672:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.647674:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.647674:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.647675:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.647675:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:12.0:1619677448.647677:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:25.2:1619677448.647679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:12.0:1619677448.647679:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.647680:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.647681:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.647683:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.647684:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.647685:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.647687:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.647688:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.647688:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.647689:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.647691:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.647722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.647728:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.647730:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.647732:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.647733:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.647735:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.647737:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677448.647738:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.647739:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.647740:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.647741:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677448.647745:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.647746:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.647749:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.647749:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.647750:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.647751:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.647752:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.647753:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.647755:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.647755:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677448.647758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677448.647760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.647763:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.647765:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:17.0:1619677448.647765:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:20.0:1619677448.647766:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.647767:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677448.647768:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.647769:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.647770:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.647771:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.647773:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.647803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.647809:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.647812:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.647812:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.647814:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.647815:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.647817:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.647818:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.647819:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.647820:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.647821:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.647825:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.647826:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.647829:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.647830:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.647831:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.647832:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.647833:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.647833:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.647834:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.647838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.647845:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.647847:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.647848:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.647848:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.647851:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.647854:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.2:1619677448.648000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.648008:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.648010:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.648011:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.648013:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.648014:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.648016:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.648017:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.648018:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.648019:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.648020:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.648023:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.648024:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.648027:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.648028:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.648029:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.648031:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.648032:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.648033:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.648034:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.648034:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677448.648037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.648039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677448.648045:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.648047:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:20.2:1619677448.648048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:18.0:1619677448.648048:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.648049:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:18.0:1619677448.648049:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.648049:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.648050:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:18.0:1619677448.648051:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.648052:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677448.648053:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:18.0:1619677448.648054:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:26.0:1619677448.648057:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.648058:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.648059:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.648060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:26.0:1619677448.648062:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.648064:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.648066:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.648068:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.648068:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:26.0:1619677448.648069:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.648069:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677448.648070:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.648072:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.648073:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.648074:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.648075:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.648075:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.648076:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.648078:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.648078:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.648079:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.648082:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.648083:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.648084:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.648085:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.648085:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.648086:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.648086:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.648086:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.648087:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.648087:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.648088:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.648088:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.648089:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.648089:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.648090:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677448.648090:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.648091:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.648092:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677448.648092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:25.2:1619677448.648096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.648097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.648101:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:12.0:1619677448.648102:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.648103:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.648103:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.648104:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.648105:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.648105:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.648106:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677448.648106:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.648107:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.648107:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:12.0:1619677448.648107:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.648108:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.648108:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.648109:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677448.648110:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.648112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.648118:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.648121:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.648122:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.648122:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:21.2:1619677448.648123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:13.0:1619677448.648123:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.648124:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677448.648129:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.648132:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.648133:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.648135:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.648137:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.648139:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.648140:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.648141:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.648143:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.648143:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.648147:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.648148:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.648151:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.648151:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.648152:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.648154:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.648154:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.648155:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.648156:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.648160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.648168:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.648170:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.648171:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.648173:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.648174:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677448.648175:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.648176:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677448.648179:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.648182:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.648183:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.648186:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.648188:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.648190:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.648191:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.648192:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.648193:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.648194:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.648197:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.648199:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.2:1619677448.648200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677448.648201:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.648202:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.648203:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.648205:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.648206:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.648206:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.648207:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.648208:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.648208:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677448.648209:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.648210:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677448.648210:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.648211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:1.0:1619677448.648213:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.648214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:1.0:1619677448.648215:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.648216:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.648218:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.648219:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.648220:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.648220:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.648220:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.648221:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.648223:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.648224:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.648224:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.648225:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.648226:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.648227:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.648227:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.648228:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677448.648229:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.648230:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.648231:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.648231:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.2:1619677448.648233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.648233:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.648234:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677448.648234:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677448.648236:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:1.0:1619677448.648237:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:9.2:1619677448.648239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.648239:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.648240:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.648240:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.648241:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677448.648241:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.648246:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.648248:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.648249:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.648250:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.648252:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.648255:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.648257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677448.648263:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.648266:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.648266:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677448.648267:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.648268:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.648269:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.648315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.648323:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.648326:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.648328:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.648331:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.648331:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.648332:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677448.648333:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.648335:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.648336:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.648337:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677448.648340:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.648342:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677448.648345:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.648345:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.648346:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.648347:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.648348:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.648349:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.648349:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.648354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.648361:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.648363:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.648364:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.648364:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.648366:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.648368:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.648414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.648420:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:4.2:1619677448.648420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677448.648423:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.648424:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.648425:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.648427:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:8.0:1619677448.648428:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.648430:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:8.0:1619677448.648430:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:26.0:1619677448.648431:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:8.0:1619677448.648431:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.648432:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.648433:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.648433:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.648434:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.648435:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.648436:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.648437:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712166] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.648437:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.648438:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.648439:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.648439:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.648440:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:20.2:1619677448.648441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:10.0:1619677448.648443:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.648443:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.648444:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677448.648444:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.648444:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.648445:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.648446:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.648446:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.648446:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677448.648447:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:10.0:1619677448.648447:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.648447:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:8.0:1619677448.648448:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:21.2:1619677448.648449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:20.0:1619677448.648449:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.648449:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.648449:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677448.648449:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:10.0:1619677448.648450:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.648450:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:20.0:1619677448.648451:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.648452:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677448.648452:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.648453:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:25.2:1619677448.648455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:20.0:1619677448.648455:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96712165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.648456:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677448.648456:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677448.648456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:20.0:1619677448.648457:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.648458:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.648459:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:20.0:1619677448.648459:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.648460:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:24.0:1619677448.648460:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.648460:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.648461:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677448.648462:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.648462:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:26.0:1619677448.648463:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:20.0:1619677448.648463:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.648464:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.648464:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.648465:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:8.0:1619677448.648465:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.648465:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:26.0:1619677448.648467:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.648467:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:26.0:1619677448.648468:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.0:1619677448.648468:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.648468:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:8.0:1619677448.648468:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.648469:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:8.0:1619677448.648469:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677448.648469:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.648470:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:8.0:1619677448.648471:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96712164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.648472:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.648472:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677448.648472:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.648473:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:8.0:1619677448.648473:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.648474:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:8.0:1619677448.648474:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.648475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000400:00000200:8.0:1619677448.648475:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.648477:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:25.2:1619677448.648478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:8.0:1619677448.648479:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.648481:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:8.0:1619677448.648481:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.648483:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.648484:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.648484:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:8.0:1619677448.648484:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.648485:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:8.0:1619677448.648485:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.648486:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:8.0:1619677448.648486:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.648486:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677448.648488:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.648488:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.648489:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.648489:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677448.648489:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.648490:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.648490:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.648492:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.648494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.648494:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677448.648501:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.648504:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.648506:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.648506:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.648508:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.648511:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.648573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.648581:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.648583:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.648584:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.648586:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.648588:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.648589:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.648591:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.648592:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.648593:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.648594:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.648597:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.648598:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.648600:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.648601:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.648602:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.648603:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.648604:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.648604:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.648605:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.648609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.648616:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.648619:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.648620:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.648621:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.648623:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.648623:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.648707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.648714:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.648716:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.648717:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.648719:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.648721:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.648723:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.648724:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.648725:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.648726:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.648727:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.648730:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.648731:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.2:1619677448.648732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677448.648734:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.648734:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.648735:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.648737:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.648737:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.648738:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:31.0:1619677448.648739:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.648739:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.648740:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.648742:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:25.2:1619677448.648742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:31.0:1619677448.648744:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.648745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.648746:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.648746:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.648747:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.648749:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.648750:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.648750:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:31.0:1619677448.648751:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.648752:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.648752:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.648753:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.648753:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:31.0:1619677448.648754:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.648754:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677448.648754:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.648755:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.648756:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.648756:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.648757:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.648759:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.648760:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.648760:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.648760:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677448.648762:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.648762:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.648763:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.648764:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.648767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.648775:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.648777:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.648778:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.648779:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.648781:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.648782:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.648970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.648975:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.648978:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.648980:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.648980:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.648983:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.648985:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:5.0:1619677448.648986:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.648987:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.648988:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.648989:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:5.0:1619677448.648993:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.648995:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.648997:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.648998:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677448.648999:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.649001:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.649002:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.649003:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.649004:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.649005:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.649007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.2:1619677448.649010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.649012:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:10.0:1619677448.649013:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:20.0:1619677448.649015:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.649015:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:26.0:1619677448.649016:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:20.0:1619677448.649016:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:10.0:1619677448.649016:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.649017:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.649019:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.649022:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.649046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.649054:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.649057:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.649059:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.649061:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.649061:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.649063:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.649064:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.649065:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.649066:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.649067:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.649071:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.649073:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.649076:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.649077:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.649077:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.649079:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.649080:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.649081:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.649083:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.649083:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.649086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.649090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.649098:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.649098:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677448.649100:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.649101:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677448.649101:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677448.649102:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:4.2:1619677448.649102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:20.0:1619677448.649103:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.649104:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:20.0:1619677448.649105:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.649108:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:26.0:1619677448.649109:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.649111:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.649112:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.649114:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.649117:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.649118:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.649119:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.649120:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.649121:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.649122:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.649126:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.649127:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.649130:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.649131:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.649131:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.649133:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.649133:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.649134:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.649135:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.649139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.649146:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.649149:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.649150:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.649150:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.649152:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.649155:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.649161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.649168:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.649171:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.649171:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.649172:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.649173:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.649174:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.649205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.649211:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.649214:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.649215:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.649217:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.649219:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.649221:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677448.649222:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.649223:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.649224:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.649225:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677448.649228:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.649230:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.649232:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.649233:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.649233:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.649235:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.649235:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.649236:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.649236:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.649241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.649248:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.649250:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.649251:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.649252:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.649253:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.649254:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.649309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.649315:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.649318:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.649319:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.649321:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.649323:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.649325:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.649326:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712156] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.649327:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.649329:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.649330:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.649333:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.649335:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.649338:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.649339:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.649340:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.649341:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.649342:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.649343:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.649343:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.649348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.649355:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.649358:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.649359:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.649361:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.649363:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.2:1619677448.649363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:12.0:1619677448.649364:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.649372:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.649374:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.649375:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:21.2:1619677448.649375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:10.0:1619677448.649377:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.649380:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.649381:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:31.0:1619677448.649383:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677448.649383:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.649384:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.649385:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677448.649385:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.649386:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677448.649388:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.649390:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677448.649390:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.649391:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.649391:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:31.0:1619677448.649393:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.649394:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.649395:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.649395:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.649396:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.649396:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:31.0:1619677448.649397:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.649397:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.649399:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.649399:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677448.649400:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.649400:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.649400:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677448.649402:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.649405:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.649405:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:9.2:1619677448.649405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.649406:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.649408:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.649408:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.649409:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.649411:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.649411:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.649413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.649413:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.649416:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:21.2:1619677448.649417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677448.649417:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.649417:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.649420:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.649421:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677448.649423:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.649424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677448.649424:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:30.0:1619677448.649425:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677448.649425:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.649426:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.649427:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.649428:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:6.0:1619677448.649428:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.649429:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.649431:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677448.649432:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677448.649433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677448.649434:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.649435:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.649436:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.649437:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.649437:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677448.649438:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.649441:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.649443:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.649444:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.649446:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.649448:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.649449:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.649450:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.649451:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.649453:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.649456:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.649458:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.649461:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.649462:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.649462:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.649464:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.649465:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.649466:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.649467:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.649470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.649478:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.649481:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.649482:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.649483:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.649484:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.649486:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.649538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.649543:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.649546:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.649547:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.649548:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.649551:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.649553:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.649554:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.649556:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.649557:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.649558:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.649561:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.649563:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.649566:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.649567:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.649567:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.649569:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.649570:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.2:1619677448.649570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:22.0:1619677448.649571:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.649572:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.649573:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:25.2:1619677448.649575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677448.649576:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:21.2:1619677448.649578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677448.649578:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.649579:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:26.0:1619677448.649581:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:18.0:1619677448.649581:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.649583:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.649583:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:26.0:1619677448.649584:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.649584:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.649585:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.649585:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:6.0:1619677448.649585:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.649586:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:26.0:1619677448.649587:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.649587:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.649587:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:18.0:1619677448.649588:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.649589:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.649590:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:18.0:1619677448.649590:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.649594:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.649596:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.649598:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.649599:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.649600:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.649601:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.649602:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.649602:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.649603:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.649608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.649616:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.649619:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.649620:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.649620:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.649622:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.649625:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.649686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677448.649690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:10.0:1619677448.649692:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.649695:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.649696:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677448.649697:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677448.649698:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.649699:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.649700:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:10.0:1619677448.649701:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.649702:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.649702:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.649703:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.649705:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.649705:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.649705:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.649707:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:10.0:1619677448.649707:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677448.649709:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712149] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.649710:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.649711:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.649711:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.649712:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.649713:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:24.0:1619677448.649716:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.649717:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.649717:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.649717:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.649718:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677448.649720:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.649720:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.649720:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677448.649721:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.649721:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677448.649721:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.649722:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:24.0:1619677448.649723:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.649723:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.649724:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.649725:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677448.649726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.2:1619677448.649729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.649733:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.649736:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.649737:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.649737:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677448.649737:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.649739:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.649740:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.649741:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.649741:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.649742:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:17.0:1619677448.649743:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.649746:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.649777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677448.649784:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.649786:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677448.649787:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677448.649789:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.649792:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.649794:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.649795:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.649796:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.649797:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.649798:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.649802:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.649803:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.649806:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.649807:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.649807:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.649809:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.649810:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.649811:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677448.649811:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677448.649815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.649823:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.649825:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.649827:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.649827:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.649829:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.649832:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.649839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.649845:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.649847:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.649848:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.649850:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.649852:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.649854:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.649855:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.649857:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.649857:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.649859:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.649861:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.649863:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.649866:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.649867:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.649868:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.649869:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.649870:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.649870:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.649871:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.649875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.649882:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.649884:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.649885:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677448.649885:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.649888:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.649891:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.2:1619677448.649986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.649994:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.649997:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.649999:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.650001:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.650002:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.650003:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.650005:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.650006:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.650007:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.650008:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.650011:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.650012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.650013:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.650017:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.650017:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.650018:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.650019:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.650020:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.650020:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677448.650021:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.650022:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.650022:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:30.0:1619677448.650022:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.650024:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.650025:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.650026:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:8.2:1619677448.650026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.650027:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.650029:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.650029:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.650030:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.650033:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.650035:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.650035:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677448.650037:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.650037:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677448.650038:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.650038:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.650038:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.650040:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.650040:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677448.650040:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.650041:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.0:1619677448.650041:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.650042:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677448.650043:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.650044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:25.2:1619677448.650045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.650052:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.650053:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:13.0:1619677448.650055:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.650055:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.650056:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:13.0:1619677448.650057:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.650057:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.650058:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.650058:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.650059:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677448.650059:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.650060:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.650082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907b38d400] (20)++
00000800:00000200:18.0:1619677448.650089:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907b38d400] (21)++
00000800:00000200:18.0:1619677448.650092:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.11@o2ib3
00000800:00000200:2.0:1619677448.650092:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907b38d400] (22)--
00000400:00000200:18.0:1619677448.650094:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.11@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.650097:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.650098:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:18.0:1619677448.650100:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.650101:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.650102:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.650103:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677448.650106:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.650108:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677448.650110:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.650111:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:18.0:1619677448.650112:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:18.0:1619677448.650114:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:18.0:1619677448.650114:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907b38d400] (21)++
00000800:00000200:5.2:1619677448.650114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:18.0:1619677448.650115:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907b38d400] (22)--
00000800:00000200:18.0:1619677448.650115:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907b38d400] (21)--
00000800:00000200:8.2:1619677448.650120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:24.0:1619677448.650123:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677448.650125:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677448.650126:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677448.650128:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.650129:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000400:00000200:24.0:1619677448.650130:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.650131:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:24.0:1619677448.650132:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:5.0:1619677448.650132:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.650133:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96712143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.650134:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.650134:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.650135:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.650136:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.650137:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:5.0:1619677448.650137:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:24.0:1619677448.650140:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677448.650142:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.650144:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.650145:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677448.650145:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.650147:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.650147:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677448.650148:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677448.650148:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677448.650152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.650158:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.650160:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.650160:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.650161:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.650162:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677448.650164:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.650216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.650222:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.650225:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.650226:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.650227:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.650230:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.650232:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.650233:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.650234:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.650235:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.650236:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.650240:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.650242:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.2:1619677448.650243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.650245:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.650246:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.650247:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.650248:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.650248:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.650249:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.650250:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.650250:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677448.650251:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.650252:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.650254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:17.0:1619677448.650254:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.650257:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.650258:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.650259:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.650260:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.650261:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:17.0:1619677448.650262:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.650262:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:12.0:1619677448.650263:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.650264:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.650266:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.650267:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.650267:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.650269:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:12.0:1619677448.650270:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677448.650273:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.650273:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.650274:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.650276:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.650276:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.650277:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.650278:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.650281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.650288:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.650290:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677448.650291:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.650292:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.650297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:24.0:1619677448.650300:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.650303:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:18.0:1619677448.650303:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.650307:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.650307:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.650310:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.650312:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.650314:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.650315:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.650316:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.650317:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.650318:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.650322:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.650323:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.650325:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.650326:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.650327:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.650328:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.650329:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.650330:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.650330:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.650335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.650342:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.650344:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.650344:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.650346:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.650346:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.650348:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.650462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.650470:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.650473:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.650473:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677448.650476:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.650479:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.650480:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.650482:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.650483:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.650484:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.650485:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.650488:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.650490:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.650493:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.650494:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.650494:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.650496:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.650497:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.650497:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.650498:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677448.650502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.650510:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.650512:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.650513:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.650514:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.650516:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.650519:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.650557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.650560:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.650563:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.650565:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.650566:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.650568:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.650570:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.650571:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.650572:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.650573:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.650574:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.650578:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.650580:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.650582:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.650583:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.650584:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.650586:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.650586:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.650587:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.650588:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.650593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.650600:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.650603:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.650604:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.650604:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.650606:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.650609:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.650610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.650616:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.650619:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.650620:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.650620:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.650621:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.650622:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.650657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.650663:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.650666:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.650667:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.650669:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.650672:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.650673:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.650674:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.650676:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.650677:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.650678:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.650681:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.650683:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.650686:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.650687:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.650687:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.650689:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.650690:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.650691:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.650692:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.650693:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.650695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.650698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.650703:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677448.650703:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677448.650705:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:13.0:1619677448.650705:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:17.0:1619677448.650706:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677448.650706:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.650707:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677448.650708:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.650710:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.650711:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.650713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:12.0:1619677448.650719:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.650722:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.650723:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.650724:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.650724:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.650726:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.650729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.650735:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.650738:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.650739:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.650741:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.650744:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.650745:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.650746:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.650748:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.650749:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.650750:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.650753:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.650754:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.650757:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.650758:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.650758:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.650760:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.650761:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.650761:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.650762:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.650766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.650774:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.650776:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.650777:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.650777:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.650779:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.650781:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.650910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.650917:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.650920:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.650921:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.650923:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.650926:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.650927:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.650929:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.650930:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.650931:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.650932:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.650935:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.650937:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.650940:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.650941:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.650942:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.650944:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.650944:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.650944:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.650945:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.650949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.650954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677448.650957:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.650959:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.650960:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.650961:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.650962:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677448.650963:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.650965:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:18.0:1619677448.650966:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:31.0:1619677448.650967:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.650969:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.650969:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.650970:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.650972:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.650973:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.650974:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.650975:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.650978:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.650979:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.650981:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.650982:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.650983:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.650984:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.650985:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.650986:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.650986:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.650990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.650991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:13.0:1619677448.650996:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.650999:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:13.0:1619677448.650999:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.651000:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.651001:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:13.0:1619677448.651001:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.651003:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.651003:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:30.0:1619677448.651004:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.651005:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677448.651006:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.651007:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:30.0:1619677448.651009:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.651009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:30.0:1619677448.651010:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.651011:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.651011:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:30.0:1619677448.651014:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.651015:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.651016:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:30.0:1619677448.651018:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.651018:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.651019:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:12.0:1619677448.651019:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.651019:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:30.0:1619677448.651020:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:12.0:1619677448.651020:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.651021:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:12.0:1619677448.651021:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.651022:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677448.651022:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677448.651023:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677448.651026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.651033:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.651035:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:4.2:1619677448.651035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677448.651036:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.651036:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.651038:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.651041:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:8.0:1619677448.651042:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.651044:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.2:1619677448.651045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000800:00000200:24.0:1619677448.651045:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.651047:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.651049:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.651050:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:26.0:1619677448.651051:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000400:00000200:8.0:1619677448.651051:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.651053:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677448.651053:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.651054:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.651054:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:8.0:1619677448.651054:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.651055:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000400:00000200:8.0:1619677448.651055:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.651056:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:8.0:1619677448.651058:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.651060:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.651062:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.651063:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.651064:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.651065:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.651066:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.651066:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.651067:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.651071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.651079:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.651082:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.651083:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.651083:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.651086:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.651089:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.651142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677448.651146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:17.0:1619677448.651147:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.651149:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.651150:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.651152:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.651154:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.651154:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677448.651156:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677448.651156:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.651157:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:17.0:1619677448.651157:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.651159:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.651159:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.651160:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.651161:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.651162:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.651164:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.651165:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.651166:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.651167:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.651168:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.651168:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.651170:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.651171:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.651172:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.651173:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.651173:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.651174:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.651175:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.651175:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.651175:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.651176:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.651178:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.651179:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:25.2:1619677448.651180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.651180:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.651182:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.651182:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.651183:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.651183:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.651187:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:25.2:1619677448.651189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.651190:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.651190:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.651192:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.651193:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.651194:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.651195:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.651197:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.651198:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.651200:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.651202:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.651202:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.651270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.651276:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.651278:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.651280:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.651281:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677448.651283:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000400:00000200:5.0:1619677448.651283:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.651285:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.651286:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.651287:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.651289:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.651289:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.651289:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677448.651292:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677448.651292:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:5.0:1619677448.651299:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.651300:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.651302:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.651303:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.651305:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.651305:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:5.0:1619677448.651306:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:1.0:1619677448.651306:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.651307:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677448.651308:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.651309:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.651309:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677448.651309:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.651310:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.651311:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.651312:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.651313:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:1.0:1619677448.651313:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.651315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.651315:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677448.651317:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677448.651318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.651318:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:1.0:1619677448.651319:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.651320:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.651321:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677448.651321:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677448.651322:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677448.651322:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.0:1619677448.651323:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.651324:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677448.651324:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677448.651325:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.651327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.651327:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.651328:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.651328:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.651330:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.651332:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677448.651335:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.651338:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.651338:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.651340:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.651341:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.651343:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.651347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.651355:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.651357:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.651358:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.651359:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.651360:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677448.651360:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:21.2:1619677448.651507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.651513:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.651516:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.651516:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.651519:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.651521:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.651522:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.651523:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.651525:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.651526:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.651527:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.651531:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.651532:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.651535:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.651536:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.651536:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.2:1619677448.651536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:22.0:1619677448.651538:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.651538:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.651539:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.651539:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677448.651544:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:9.2:1619677448.651544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.651547:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.651549:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.651549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.651551:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.651552:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.651552:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:31.0:1619677448.651553:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:18.0:1619677448.651554:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:31.0:1619677448.651555:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.651555:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.651555:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.651556:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.651556:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:31.0:1619677448.651557:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.651558:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.651558:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:31.0:1619677448.651559:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:1.0:1619677448.651559:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.651560:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:17.0:1619677448.651562:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.651563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000400:00000200:31.0:1619677448.651564:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.651565:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.651566:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:17.0:1619677448.651567:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.651568:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.651569:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.651569:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.651570:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:17.0:1619677448.651570:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.651570:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000400:00000200:17.0:1619677448.651571:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.651572:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.651572:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.651572:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.651573:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:2.0:1619677448.651573:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.651574:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:18.0:1619677448.651574:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.651574:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000400:00000200:17.0:1619677448.651575:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.651575:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.651576:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:25.2:1619677448.651577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:17.0:1619677448.651577:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.651580:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.651581:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.651582:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.651584:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:17.0:1619677448.651584:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.651584:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.651585:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.651585:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.651587:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.651588:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.651589:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.651590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.651591:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.651592:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.651597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:13.0:1619677448.651597:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.651599:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.651600:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.651602:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.651603:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:12.0:1619677448.651604:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.651605:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.651606:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.651607:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.651608:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.651609:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.651610:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.651669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.651674:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.651677:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.651679:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677448.651680:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.651682:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.651684:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.651685:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.651686:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.651687:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.651688:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.651692:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.651694:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.651696:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.651697:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.651697:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.651699:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.651700:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.651701:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.651701:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677448.651706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.651713:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.651716:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.651717:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.651717:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.651719:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.651722:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.651783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.651786:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.651788:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.651790:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.651791:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.651793:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.651794:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.651796:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.651797:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.651798:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.651799:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.651802:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.651804:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.651807:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.651808:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.651808:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.651810:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.651811:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.651812:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.651812:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.651816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.651818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:12.0:1619677448.651822:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.651824:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:12.0:1619677448.651825:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.651826:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.651827:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677448.651827:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:12.0:1619677448.651828:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.651829:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:1.0:1619677448.651830:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.651831:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677448.651833:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.651835:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.651836:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.651837:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.651838:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.651839:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.651842:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.651844:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.651847:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.651848:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.651848:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.651850:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.651850:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677448.651851:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677448.651852:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:25.2:1619677448.651856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.651862:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.651865:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.651866:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.651867:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.651869:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.651870:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.651931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:5.2:1619677448.651934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677448.651936:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677448.651939:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677448.651940:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:26.0:1619677448.651942:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:2.0:1619677448.651942:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.651943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:2.0:1619677448.651944:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.651945:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.651946:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:2.0:1619677448.651946:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677448.651948:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.651949:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.651949:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677448.651949:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.651950:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.651951:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.651951:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:17.0:1619677448.651952:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.651952:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.651956:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.651956:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.651957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:17.0:1619677448.651958:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.651958:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:26.0:1619677448.651959:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.651961:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.651961:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.651962:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.651963:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.651964:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.651964:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:17.0:1619677448.651964:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.651964:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000400:00000200:26.0:1619677448.651965:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.651965:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677448.651966:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:17.0:1619677448.651966:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.651967:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.651967:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.651967:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677448.651968:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000400:00000200:18.0:1619677448.651969:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.651969:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677448.651970:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:2.0:1619677448.651971:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:26.0:1619677448.651972:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.651973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000400:00000200:18.0:1619677448.651973:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.651974:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.651974:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.651976:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96712118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.651977:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.651978:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.651979:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.651979:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.651980:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.651980:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.651980:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.651980:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:18.0:1619677448.651981:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.651982:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.651982:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.651983:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.651984:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.651984:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677448.651985:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.651985:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677448.651985:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.651987:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677448.651987:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677448.651987:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:26.0:1619677448.651988:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:25.2:1619677448.651988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.651988:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.651988:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.651989:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.651989:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.651990:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.651991:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677448.651991:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.651991:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.651991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.651992:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.651993:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:12.0:1619677448.651994:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:18.0:1619677448.651995:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.651996:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.2:1619677448.651996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677448.651997:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:12.0:1619677448.651997:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:2.0:1619677448.651997:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.651998:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:12.0:1619677448.651998:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.651999:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.652000:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:25.2:1619677448.652001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:13.0:1619677448.652001:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000400:00000200:12.0:1619677448.652001:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.652002:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677448.652003:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.652003:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:2.0:1619677448.652003:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.652004:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.652004:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677448.652005:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.652006:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.652006:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.652007:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:2.0:1619677448.652007:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:25.2:1619677448.652008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.652008:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:2.0:1619677448.652008:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.652009:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.652009:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:2.0:1619677448.652009:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.652010:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.652010:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:2.0:1619677448.652010:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.652011:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.652012:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.652014:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.652014:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:2.0:1619677448.652015:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.652017:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:12.0:1619677448.652017:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.652017:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.652018:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.652018:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.652019:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.652020:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.652021:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.652021:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.652022:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.652024:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.652024:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.652025:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.652026:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.652029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.652036:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.652039:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.652040:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.652042:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.652043:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.652045:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.652189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.652197:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.652200:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.652202:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.652203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:31.0:1619677448.652204:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.652204:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.652205:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.652206:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.652208:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.652209:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.652209:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:30.0:1619677448.652210:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.652212:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.652213:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:30.0:1619677448.652214:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.652215:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.652217:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:18.0:1619677448.652218:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.652219:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.652220:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:18.0:1619677448.652220:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677448.652221:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:18.0:1619677448.652221:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.652222:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.652223:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:18.0:1619677448.652223:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.652224:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.652224:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:18.0:1619677448.652224:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.652225:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:25.2:1619677448.652227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:18.0:1619677448.652228:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.652230:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.652232:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.652233:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.652234:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:13.0:1619677448.652234:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:18.0:1619677448.652235:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.652236:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.652237:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.652237:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677448.652237:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.652237:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.652239:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.652240:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:4.2:1619677448.652240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677448.652241:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.652242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.652244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:10.0:1619677448.652247:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.652249:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.652250:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677448.652250:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.652251:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677448.652252:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.652252:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.652253:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.652253:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.652254:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.652255:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.652256:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.652257:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.652258:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.652259:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.652260:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.652260:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.652260:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:10.0:1619677448.652261:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.652261:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.652262:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677448.652262:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:22.0:1619677448.652263:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.652266:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.652266:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.652267:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.652268:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.652268:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:22.0:1619677448.652271:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.652272:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.652273:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.652273:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.652274:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677448.652276:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.652276:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.652276:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.652277:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.652277:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.652278:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.652278:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677448.652279:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.652279:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677448.652280:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.652281:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.652282:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.652282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.652284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677448.652287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.652291:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:5.0:1619677448.652291:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.652296:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:10.0:1619677448.652298:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (23)--
00000800:00000200:5.0:1619677448.652300:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677448.652301:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677448.652301:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.652301:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.652302:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:20.0:1619677448.652302:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.652303:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.652304:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.652307:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.652308:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.652310:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677448.652311:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.652393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.652399:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.652402:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.652403:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.652405:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.652408:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.652409:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.652411:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712112] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.652412:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.652413:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.652414:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.652418:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.652419:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.652422:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.652423:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.652423:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.652425:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.652425:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.652427:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.652428:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.652428:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.652432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.652433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.652438:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.652440:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677448.652440:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.652441:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.652441:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.652442:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.652443:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.652444:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.652445:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.652448:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.652478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.652483:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.652486:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.652488:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677448.652489:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.652491:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.652492:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.652494:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.652495:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.652496:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.652497:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.652500:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.652502:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.652505:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.652505:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.652506:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.652508:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.652508:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.652509:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.652510:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677448.652514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.652515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.652523:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.652523:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.652525:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:31.0:1619677448.652526:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677448.652526:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.652526:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.652529:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000400:00000200:31.0:1619677448.652530:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.652531:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677448.652532:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:31.0:1619677448.652533:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.652535:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:8.2:1619677448.652535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000400:00000200:31.0:1619677448.652536:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.652537:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.652538:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.652539:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.2:1619677448.652539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:31.0:1619677448.652542:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.652542:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677448.652542:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.652544:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.652546:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:17.0:1619677448.652546:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.652546:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.652547:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.652547:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.652548:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)++
00000400:00000200:20.0:1619677448.652548:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.652548:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.652548:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677448.652549:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.652549:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677448.652551:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:20.0:1619677448.652551:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.652552:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.652553:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677448.652553:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677448.652554:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:20.0:1619677448.652554:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.652555:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.652556:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.652557:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.2:1619677448.652557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677448.652560:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.652562:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.652565:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.652565:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677448.652566:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.652567:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.652567:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:20.0:1619677448.652568:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.652568:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.652568:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.652569:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.652569:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.652570:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677448.652570:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.652574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.652574:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.652577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:12.0:1619677448.652582:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.652584:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:24.0:1619677448.652585:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000400:00000200:12.0:1619677448.652585:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.652587:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000400:00000200:12.0:1619677448.652587:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.652588:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:13.0:1619677448.652589:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677448.652590:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.652590:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:24.0:1619677448.652593:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.652594:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677448.652595:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.652597:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.652598:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.652599:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677448.652601:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.652603:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.652605:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.652606:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677448.652606:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.652607:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.652608:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677448.652609:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed9d3400] (22)++
00000800:00000200:24.0:1619677448.652611:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (23)--
00000800:00000200:24.0:1619677448.652611:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:9.2:1619677448.652613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.652617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (21)++
00000800:00000200:21.2:1619677448.652621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:8.0:1619677448.652622:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.652624:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.652624:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.652626:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)++
00000400:00000200:8.0:1619677448.652626:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.652627:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed9d3400] (23)--
00000800:00000200:8.0:1619677448.652627:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:2.0:1619677448.652627:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:24.0:1619677448.652628:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677448.652628:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:24.0:1619677448.652629:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:2.0:1619677448.652630:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.652631:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.652633:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.652636:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.652637:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.652638:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.652640:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.652641:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.652642:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.652645:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.652647:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.652649:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.652650:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.652650:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.652652:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.652652:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.652654:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.652655:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.652655:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677448.652658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.652661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.652666:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.652667:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.652668:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.652668:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.652668:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.652669:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.652670:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.652670:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.652671:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.652671:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.652676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.652683:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.652685:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.652686:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.652686:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.652687:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.652687:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.652724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.652731:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.652734:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.652735:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.2:1619677448.652736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:10.0:1619677448.652737:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.652739:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.652741:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.652742:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.652743:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.652744:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677448.652745:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.652746:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.652747:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:10.0:1619677448.652750:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.652751:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.652752:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:30.0:1619677448.652753:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.652754:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.652755:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.652755:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.652756:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:30.0:1619677448.652757:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.652758:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.652758:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.652758:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677448.652759:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.652759:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.652759:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:30.0:1619677448.652760:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.652761:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.652764:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.652764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.652766:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.652768:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.652769:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.652770:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.652770:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.652771:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.652771:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:31.0:1619677448.652772:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.652772:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.652773:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:31.0:1619677448.652774:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.652774:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.652775:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.652776:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.652777:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.652777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.652781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677448.652783:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.652785:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.652786:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.652788:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:13.0:1619677448.652788:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.652789:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.652790:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.652791:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:13.0:1619677448.652791:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.652792:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.652796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:4.2:1619677448.652796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:12.0:1619677448.652802:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.652803:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:12.0:1619677448.652805:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.652806:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:12.0:1619677448.652806:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.652806:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:8.0:1619677448.652806:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:12.0:1619677448.652807:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:8.0:1619677448.652808:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.652809:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:8.0:1619677448.652810:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.652811:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.652812:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.652814:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.652815:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.652816:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.652819:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.652820:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.652823:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.652824:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.652824:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.652826:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.652826:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.652827:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.652827:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.652832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.652840:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.652843:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.652843:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.652844:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.652846:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.652848:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.652856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.652862:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.652864:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.652865:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.652867:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.652869:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.652871:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.652872:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.652873:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.652874:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.652875:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.652878:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.652880:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.652883:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.652884:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.652884:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.652886:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.652886:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.652887:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.652887:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.652892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.652898:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.652901:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.652902:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.652902:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.652904:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.652908:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.652911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677448.652917:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.652920:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.652920:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677448.652921:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.652922:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.652922:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.653004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.653010:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.653013:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.653013:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677448.653015:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.653017:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.653018:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.653020:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.653021:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.653022:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.653023:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.653027:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.653028:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.653031:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.653032:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.653033:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.653035:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.653035:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.653036:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.653036:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.653041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.653047:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.653050:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.653051:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.653051:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.653053:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.653056:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.653057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.653064:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.653067:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.653067:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.653070:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.653072:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.653073:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.653074:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.653076:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.653077:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.653078:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.653081:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.653083:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.653085:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.653086:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.653087:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.653089:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.653090:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.653090:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.653090:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.653095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677448.653102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:18.0:1619677448.653102:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.653105:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.653106:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.653106:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.653108:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.653109:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.653112:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:18.0:1619677448.653112:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.653113:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.653114:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.653117:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.653119:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.653120:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.653121:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.653122:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.653123:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.653126:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.653127:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.653129:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.653130:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.653131:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.653132:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.653133:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.653133:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.653134:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677448.653138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.653144:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.653147:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.653147:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.653149:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677448.653150:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.653151:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.653158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.653164:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.653166:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.653167:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.653168:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.653168:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.653169:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.653170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.653175:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:4.2:1619677448.653176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677448.653178:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.653180:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.653182:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677448.653183:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677448.653183:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.653185:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:10.0:1619677448.653186:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.653187:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.2:1619677448.653187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:5.0:1619677448.653187:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.653190:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.653190:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.653191:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.653192:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.653192:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.653195:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677448.653195:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.653197:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.653197:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.653198:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677448.653198:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.653199:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.653200:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.653201:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.653201:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.0:1619677448.653201:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.653203:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.653204:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.653204:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:31.0:1619677448.653205:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.653205:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.653205:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:31.0:1619677448.653206:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96712097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.653206:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.653207:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.653207:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.653208:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.653208:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.653208:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:31.0:1619677448.653209:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.653210:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.653210:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.653211:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.653211:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (22)++
00000400:00000200:31.0:1619677448.653213:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.653213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (23)++
00000800:00000200:10.0:1619677448.653213:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (24)--
00000800:00000200:10.0:1619677448.653214:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.653214:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.653215:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.653215:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.653217:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.653218:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.653218:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.653219:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.653220:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677448.653220:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (23)++
00000800:00000200:31.0:1619677448.653221:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.653223:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.653223:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677448.653223:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:6.0:1619677448.653223:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (24)++
00000400:00000200:22.0:1619677448.653224:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.653224:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (24)--
00000800:00000200:25.2:1619677448.653225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:6.0:1619677448.653225:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.653226:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.653227:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.653229:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:21.2:1619677448.653229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677448.653230:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.653231:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677448.653232:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.653234:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.653235:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.653236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000400:00000200:12.0:1619677448.653236:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.653237:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.653238:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.653239:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:12.0:1619677448.653240:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.653241:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.653242:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.653243:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.653246:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.653247:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.653247:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.653248:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.653249:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.653441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.653447:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.653449:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.653450:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.653452:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.653454:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.653455:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.653457:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.653458:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.653459:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.653460:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.653463:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.653465:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.653467:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.653468:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.653469:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.653470:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.653471:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.653472:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.653473:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.653473:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.653477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.653479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.653484:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:20.0:1619677448.653485:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677448.653486:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:20.0:1619677448.653487:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:6.0:1619677448.653487:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677448.653488:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.653488:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:20.0:1619677448.653490:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.653491:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.653492:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.653512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.653520:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.653523:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.653523:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.653525:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.653529:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.653530:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.653531:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.653532:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.653533:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.653534:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.653538:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.653539:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.653541:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.653542:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.653543:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.653544:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.653545:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.653546:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.653546:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.653551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.653559:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.2:1619677448.653559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.653561:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.653562:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.653563:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.653565:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.653566:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.653567:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.653569:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677448.653570:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.653570:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.653571:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.653572:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:17.0:1619677448.653572:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.653573:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.653574:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:17.0:1619677448.653575:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.653577:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.653578:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.653579:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.653580:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.653582:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:5.2:1619677448.653584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:17.0:1619677448.653585:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.653586:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.653589:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.653590:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.653590:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.653591:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.653592:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.653592:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.2:1619677448.653593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677448.653593:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.653594:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.653594:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.653594:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.653596:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.653598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:10.0:1619677448.653599:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.653600:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677448.653600:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.653601:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.653603:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:10.0:1619677448.653603:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.653605:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.653605:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.653606:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:10.0:1619677448.653606:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.653607:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:22.0:1619677448.653608:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.653609:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.653609:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.653609:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:31.0:1619677448.653611:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.653611:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:22.0:1619677448.653612:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.653613:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.653614:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.653614:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:10.0:1619677448.653614:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.653614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:10.0:1619677448.653615:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.653615:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:31.0:1619677448.653616:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.653616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:31.0:1619677448.653617:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.653617:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.653617:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677448.653618:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.653618:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.653619:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:31.0:1619677448.653620:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.653621:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677448.653622:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:25.2:1619677448.653623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:18.0:1619677448.653623:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.653624:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.653624:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:31.0:1619677448.653625:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.653625:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.653626:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:18.0:1619677448.653627:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.653627:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.653628:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.653628:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.653629:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.653629:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:18.0:1619677448.653629:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.653630:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.653630:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:2.0:1619677448.653630:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.653631:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677448.653631:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.653631:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:25.2:1619677448.653633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:18.0:1619677448.653633:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96712091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.653634:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.653635:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.653636:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:18.0:1619677448.653636:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.653637:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.653640:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.653640:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:2.0:1619677448.653642:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:18.0:1619677448.653643:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677448.653643:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.653644:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.653644:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96712090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677448.653646:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.653646:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.653647:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.653648:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:2.0:1619677448.653648:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.653649:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677448.653651:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.653652:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:2.0:1619677448.653652:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.653653:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.653654:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.653655:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.653656:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.653656:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.653657:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.653657:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.653658:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.653659:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.653659:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.653660:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.653661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.653661:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.653662:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.653664:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.653665:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.653666:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.653667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.653668:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.653669:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.653669:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.653670:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.653672:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.653673:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.653674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677448.653679:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677448.653680:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:5.0:1619677448.653681:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:18.0:1619677448.653682:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:5.0:1619677448.653682:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.653683:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677448.653684:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.653684:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:5.0:1619677448.653684:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.653686:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.653820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.653827:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.2:1619677448.653828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677448.653830:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677448.653830:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.653832:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.653833:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.653833:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:31.0:1619677448.653834:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.653835:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.653836:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:5.0:1619677448.653836:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677448.653837:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712089] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.653837:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.653839:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.653839:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677448.653840:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.653840:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.653841:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.653842:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.653843:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.653845:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677448.653846:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.653849:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.653849:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.653852:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.653853:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.653854:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.653855:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.653855:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.653856:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677448.653857:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.653857:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.653858:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.653858:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.653859:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.653859:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677448.653859:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.653860:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.653862:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.653862:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677448.653863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:25.2:1619677448.653864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.653867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.653871:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:30.0:1619677448.653872:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.653873:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:30.0:1619677448.653874:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:1.0:1619677448.653874:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.653875:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.653875:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:1.0:1619677448.653875:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.653876:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.653876:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.653877:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677448.653878:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.653878:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.653878:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.653882:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:1.0:1619677448.653882:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.653964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.653971:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.653974:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.653974:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.653976:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.653979:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.653980:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.653981:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.653982:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.653983:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.653984:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.653988:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.653990:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.653993:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.653994:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.653994:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.653996:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.653996:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.653998:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.653999:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.654000:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.654002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.654005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.654009:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.654012:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.654012:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.654013:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.654013:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.654014:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.654015:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.654016:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.654016:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.654019:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.654189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:4.2:1619677448.654190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:2.0:1619677448.654195:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:24.0:1619677448.654197:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677448.654198:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.654199:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:24.0:1619677448.654200:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.654201:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677448.654201:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.654204:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.654204:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.654206:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.654206:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.654207:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.654208:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:2.0:1619677448.654209:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.654210:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.654211:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.654211:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677448.654212:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.654214:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.654215:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:2.0:1619677448.654216:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.654219:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.654219:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677448.654221:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:2.0:1619677448.654222:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.654223:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677448.654224:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.654224:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677448.654225:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:2.0:1619677448.654225:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677448.654226:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.654226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:2.0:1619677448.654226:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:24.0:1619677448.654227:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:2.0:1619677448.654227:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.654227:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:24.0:1619677448.654229:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.654230:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.2:1619677448.654231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677448.654231:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.654232:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.654234:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.2:1619677448.654234:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.654236:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000400:00000200:24.0:1619677448.654238:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000800:00000200:12.0:1619677448.654238:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:4.2:1619677448.654238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:24.0:1619677448.654240:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.654240:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677448.654241:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:12.0:1619677448.654241:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:24.0:1619677448.654242:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96712084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.654242:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:12.0:1619677448.654242:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.654242:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:12.0:1619677448.654243:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.654243:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:24.0:1619677448.654244:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.654244:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.654245:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.654245:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:13.0:1619677448.654245:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677448.654245:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677448.654246:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.654246:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.654246:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:24.0:1619677448.654247:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.654249:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.654251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:24.0:1619677448.654252:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677448.654255:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.654256:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.654256:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.654257:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677448.654257:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.654258:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.654259:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.654259:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677448.654260:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:17.0:1619677448.654260:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.654260:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.654261:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:17.0:1619677448.654261:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.654262:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.654263:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:9.2:1619677448.654264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.654270:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.654273:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.654273:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.654274:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.654276:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677448.654278:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.2:1619677448.654285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.654292:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.654298:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.654299:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677448.654302:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.654305:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.654306:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.654307:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.654308:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.654309:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.654310:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.654313:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.654315:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.654318:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.654318:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.654319:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.654321:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.654321:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.654322:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.654322:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.654328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.654336:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.654339:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.654340:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.654340:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.654342:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.654346:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.654442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.654447:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.654450:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.2:1619677448.654452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:20.0:1619677448.654453:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.654453:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.654457:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:5.0:1619677448.654457:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.654458:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:22.0:1619677448.654460:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:5.0:1619677448.654460:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.654461:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:5.0:1619677448.654462:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.654463:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.654464:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.654465:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.654466:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.654468:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.654469:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.654469:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.654471:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.654471:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.654472:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.654473:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:5.0:1619677448.654474:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.654475:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.654476:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:22.0:1619677448.654477:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.654478:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.654478:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677448.654479:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:5.0:1619677448.654479:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.654480:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677448.654482:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.654483:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.654483:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:25.2:1619677448.654484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.654485:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.654486:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.654486:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.654487:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.654491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:31.0:1619677448.654492:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.654494:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.654496:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.654497:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.654498:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:30.0:1619677448.654499:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.654501:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677448.654501:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.654502:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.654502:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.654504:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.654507:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:4.2:1619677448.654596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.654603:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.654605:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.654606:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.654608:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.654610:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.654612:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.654613:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.654614:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.654615:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.654616:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.654619:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.654621:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.654624:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.654625:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.654626:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.654627:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.654628:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.654628:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.654629:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.654633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.654641:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.654643:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.654644:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.654645:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.654647:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.654650:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.654680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677448.654688:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677448.654690:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677448.654693:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.654695:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.654695:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677448.654697:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:13.0:1619677448.654698:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677448.654700:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.654701:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677448.654702:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:21.2:1619677448.654703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:13.0:1619677448.654705:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.654707:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.654708:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:13.0:1619677448.654710:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.654711:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.654711:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:13.0:1619677448.654712:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.654713:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:13.0:1619677448.654714:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:13.0:1619677448.654714:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000400:00000200:2.0:1619677448.654714:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.654715:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677448.654716:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000400:00000200:2.0:1619677448.654716:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.654718:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.654719:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.654720:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677448.654721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:2.0:1619677448.654721:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.654722:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.654726:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.654727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.654727:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.654729:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677448.654730:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.654731:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:2.0:1619677448.654731:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:20.0:1619677448.654732:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.654732:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.654734:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.654734:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.654734:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.654734:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:1.0:1619677448.654734:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.654735:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.654736:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:20.0:1619677448.654737:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677448.654737:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:1.0:1619677448.654737:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.654738:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.654739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677448.654740:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.654742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000400:00000200:1.0:1619677448.654742:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.654743:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.654745:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.654746:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677448.654746:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.654747:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.654748:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:1.0:1619677448.654748:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.654749:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677448.654750:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:12.0:1619677448.654750:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.654751:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.654752:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:12.0:1619677448.654752:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.654753:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:1.0:1619677448.654753:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.654755:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.654756:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.654758:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.654759:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.654759:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.654761:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.654762:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.654762:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.654763:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.654767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.654773:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.654776:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.654777:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.654779:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.654780:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.654782:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.654794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.654799:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.654802:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.654804:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.654804:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677448.654806:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.654808:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677448.654810:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.654811:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.654812:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.654813:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677448.654816:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.654818:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677448.654821:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.654822:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677448.654822:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:21.2:1619677448.654825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.0:1619677448.654825:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.654826:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.654827:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.654828:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.654831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.654835:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.654837:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677448.654838:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:30.0:1619677448.654839:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.654840:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.654841:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.654841:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.654843:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000400:00000200:30.0:1619677448.654843:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.654844:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.654846:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.654847:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.654848:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.654851:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677448.654851:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.654852:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.654852:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.654853:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.654855:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.654856:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.654857:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.654859:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.654859:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.654860:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.654861:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.654864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.654872:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.654875:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.654876:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.654876:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.654878:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.654881:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.654882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.654889:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.654891:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.654892:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.654892:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.654893:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.654894:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.654906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.654911:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.654915:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.654915:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.654917:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.654920:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.654921:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.654923:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.654924:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.654925:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.654926:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.654929:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.654931:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.654933:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.654934:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.654934:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.654935:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.654936:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.654937:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.654937:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.654942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.654949:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.654951:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.654952:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.654952:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.654954:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.654955:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.654960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.654967:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.654970:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.654971:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.654971:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.654972:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.654972:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.655135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.655143:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.655146:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.655148:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.655150:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.655151:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.655153:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.655154:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.655156:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.655157:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.655158:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.655161:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.655163:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.655166:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.655167:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.655167:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.655169:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.655170:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.655171:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.655171:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.655175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.655183:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.655185:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.655186:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.655188:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.655189:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.655191:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.655212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.655217:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.655220:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.655221:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.655223:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.655225:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.655226:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677448.655228:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.655229:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.655230:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.655231:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677448.655235:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.655236:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677448.655239:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.655240:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:18.0:1619677448.655241:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:18.0:1619677448.655243:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677448.655243:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.655244:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.655245:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.655246:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677448.655248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:21.2:1619677448.655251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:10.0:1619677448.655255:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:2.0:1619677448.655256:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:10.0:1619677448.655257:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.655258:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:2.0:1619677448.655258:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:10.0:1619677448.655259:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.655259:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.655260:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:10.0:1619677448.655261:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.655264:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.655315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.655321:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.655324:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.655325:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.655327:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.655330:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.655331:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.655332:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.655333:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.655334:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.655335:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.655338:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.655340:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.655343:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.655344:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.655345:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.655346:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.655347:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.655348:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.655349:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.655350:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.655353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.655355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.655361:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677448.655361:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.655363:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.655363:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677448.655364:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677448.655364:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677448.655365:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.655367:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.655369:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.655370:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.655503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.655511:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.655514:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.655515:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.655516:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.655519:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.655520:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.655522:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.655523:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.655524:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.655525:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.655529:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.655530:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.655533:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.655534:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.655534:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.655536:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.655536:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.655537:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.655537:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.655542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.655549:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.655552:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.655553:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.655553:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.655556:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.655559:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.655561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.655568:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.655571:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.655572:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.655572:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.655573:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.655573:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.655647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.655655:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.655658:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.655660:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.655662:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.655663:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.655664:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.655665:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.655667:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.655668:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.655668:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.655672:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.655674:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.655677:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.655677:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.655678:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.655680:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.655680:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.655681:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.2:1619677448.655681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:30.0:1619677448.655682:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.655686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.655687:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.655691:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.655691:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.655693:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.655694:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:18.0:1619677448.655696:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.655697:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:18.0:1619677448.655697:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:6.0:1619677448.655697:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.655698:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.655698:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.655700:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.655700:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.655701:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.655702:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677448.655703:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:18.0:1619677448.655705:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.655707:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.655708:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.655709:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.655710:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.655711:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.655712:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.655712:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.655713:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:21.2:1619677448.655716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:9.2:1619677448.655718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.655724:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.655726:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.655727:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.655728:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.655728:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.655729:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.655730:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.655731:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.655732:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.655733:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.655734:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.655735:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.655736:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.655737:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.655739:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.655740:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:2.0:1619677448.655740:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.655742:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.655745:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.655745:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.655746:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.655748:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.655748:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.655749:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.655749:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.655754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.655754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:31.0:1619677448.655761:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:13.0:1619677448.655761:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677448.655764:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000800:00000200:13.0:1619677448.655764:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.655764:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.655767:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.655767:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.655767:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.655768:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:31.0:1619677448.655770:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.655770:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677448.655772:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:25.2:1619677448.655772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:31.0:1619677448.655773:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.655774:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.655775:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.655776:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:12.0:1619677448.655778:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.655779:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.655781:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677448.655781:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.655782:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.655782:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.655783:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.655784:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.655784:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.655785:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:31.0:1619677448.655786:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677448.655788:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677448.655788:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677448.655789:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:31.0:1619677448.655790:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:8.2:1619677448.655794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.655801:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.655804:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.655804:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.655805:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.655807:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.655810:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.655847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.655854:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.655857:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.655858:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.655860:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.655862:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.655864:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.655865:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.655867:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.655868:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.655869:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:5.2:1619677448.655871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677448.655872:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.655873:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.655876:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.655877:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:1.0:1619677448.655877:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.655878:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.655879:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:26.0:1619677448.655880:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.655880:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.655880:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.655881:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.655882:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:1.0:1619677448.655882:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.655884:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.655886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677448.655886:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.655887:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.655888:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.655889:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.655890:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.655893:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.655894:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677448.655894:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:18.0:1619677448.655896:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:1.0:1619677448.655897:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.655898:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.655898:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677448.655898:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.655899:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:18.0:1619677448.655900:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.655901:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.655901:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.655902:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.655903:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677448.655904:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.655907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.655913:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:5.2:1619677448.655914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.0:1619677448.655916:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.655917:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.655917:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.655919:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.655921:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.655923:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:8.0:1619677448.655923:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:26.0:1619677448.655924:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.655927:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.655927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000400:00000200:10.0:1619677448.655929:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.655931:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.655932:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.655933:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000400:00000200:10.0:1619677448.655933:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.655934:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.655936:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677448.655936:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.655936:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.655937:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:20.2:1619677448.655937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:24.0:1619677448.655938:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.655938:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:10.0:1619677448.655939:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.655940:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.655943:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.655943:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.655944:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.655944:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.655946:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677448.655946:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:10.0:1619677448.655947:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.655948:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:5.0:1619677448.655948:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.655949:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677448.655949:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.655949:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:5.0:1619677448.655951:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.655952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:5.0:1619677448.655952:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:5.2:1619677448.655955:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.655959:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:5.0:1619677448.655960:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.655961:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:5.0:1619677448.655961:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.655962:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:12.0:1619677448.655962:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.655962:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.655964:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.655964:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:12.0:1619677448.655964:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.655964:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.655965:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.655966:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:5.0:1619677448.655967:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.655969:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.655971:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.2:1619677448.655971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.655973:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.655974:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.655974:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.655976:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.655977:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.655977:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.655978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:5.0:1619677448.655978:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677448.655979:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:25.2:1619677448.655981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.655981:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.655982:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.655983:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.655983:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.655985:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:13.0:1619677448.655986:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.655988:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.655988:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.655989:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.655989:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.655989:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.655990:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677448.655990:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.655991:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:12.0:1619677448.655991:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.655992:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.655994:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.656106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.656112:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.656115:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.656115:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.656117:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.656120:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.656121:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.656123:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.656124:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.656125:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.656126:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.656129:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.656131:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.656133:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.656134:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.656134:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.656136:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.656136:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.656137:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.656137:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.656142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.656150:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.656151:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.656152:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.656153:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.656154:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.656155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:1.0:1619677448.656155:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677448.656161:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.656163:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.656164:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.656166:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.656169:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.656170:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.656171:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.656173:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.656174:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.656175:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.656179:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.656180:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.656183:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.656184:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.656185:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.656187:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.656187:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.656188:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.656189:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.656193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.656200:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.656202:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.656204:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.656206:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.656206:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.656209:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.656251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.656258:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.656261:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.656263:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.656265:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.656266:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.656267:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.656269:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.656270:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.656271:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.656272:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.656276:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.656278:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.656280:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.656281:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.656282:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.656284:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.656284:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.656286:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.656288:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.656288:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.656289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.656296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.656298:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.656300:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.656301:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.656302:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.656304:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.656305:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677448.656305:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.656307:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.656308:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.656309:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677448.656634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.656642:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.656644:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.656645:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.656647:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.656649:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.656651:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677448.656652:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.656653:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.656654:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.656655:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.656658:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.656660:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.656663:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.656664:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.656664:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.656666:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.656666:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.2:1619677448.656667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677448.656667:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.656667:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:5.2:1619677448.656667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677448.656670:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.656673:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:17.0:1619677448.656673:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.2:1619677448.656673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:20.0:1619677448.656675:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.656676:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.2:1619677448.656677:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677448.656677:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677448.656677:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.656678:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.656679:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:17.0:1619677448.656680:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.656680:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.656681:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.656682:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.656682:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.656683:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.656684:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.656684:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.656685:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:20.0:1619677448.656686:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.656686:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.656687:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.656688:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:20.0:1619677448.656688:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.656688:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.656689:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.656690:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.656692:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:17.0:1619677448.656693:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.656694:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:20.0:1619677448.656695:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.656695:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677448.656696:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:30.0:1619677448.656697:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.656698:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.656699:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.656700:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.656700:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.656701:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.656701:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.656701:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:30.0:1619677448.656702:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:20.0:1619677448.656702:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:30.0:1619677448.656703:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96712055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.656703:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.656703:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.656704:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.656704:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677448.656705:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.656705:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677448.656705:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677448.656706:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.656706:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.656706:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:30.0:1619677448.656707:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:25.2:1619677448.656709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:30.0:1619677448.656710:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.656712:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:25.2:1619677448.656713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.656715:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.656716:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.656716:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.656717:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.656718:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.656718:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:31.0:1619677448.656719:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.656719:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.656720:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:13.0:1619677448.656720:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:31.0:1619677448.656721:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.656721:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.656722:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:13.0:1619677448.656722:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.656723:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.656724:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.2:1619677448.656724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:13.0:1619677448.656725:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.656726:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.656727:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.656728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.656732:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677448.656735:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.656735:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.656736:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.656737:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.656738:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.656738:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.656738:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.656739:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.656740:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.656740:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677448.656741:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.656766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.656773:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.656775:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.656776:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.656778:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.656781:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.656782:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.656783:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.656785:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.656786:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.656786:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.656790:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.656791:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.656794:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.656795:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.656795:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.656797:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.656797:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.656798:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.656799:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.656803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.656811:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.656814:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.656815:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.656815:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.656817:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.656820:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.656831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.656838:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.656841:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.656842:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.656844:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.656847:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.656849:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.656850:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.656851:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.656852:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.656853:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.656856:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.656858:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.656860:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.656861:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.656861:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.656863:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.656863:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.656864:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.656865:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.656869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.656875:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.656878:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.656878:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.656880:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.656882:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.656883:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.656941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.656947:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.656951:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.656951:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.656953:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.656956:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.656957:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.656959:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.656960:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.656961:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.656962:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.656965:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.656966:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.656969:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.656970:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.656971:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.656973:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.656973:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.656974:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.656974:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.656978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.656985:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.656988:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.656989:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.656990:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.656991:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.656993:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.656994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.657000:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.657003:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.657004:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.657005:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.657005:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.657007:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.657039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.657047:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.657050:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.657052:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.657053:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.657054:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.657056:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.657057:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.657058:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.657059:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.657060:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.657063:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.657064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.657065:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.657067:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.657068:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.657069:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:17.0:1619677448.657069:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.657070:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.657071:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.657072:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.657072:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677448.657072:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.657073:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.657075:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.657076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:17.0:1619677448.657077:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.657079:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.657080:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.657081:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.657082:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.657082:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:17.0:1619677448.657083:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.657084:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.657085:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.657085:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.657087:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.657088:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.657089:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.657090:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:17.0:1619677448.657092:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.657093:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.657094:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.657096:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.657096:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:25.2:1619677448.657098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:17.0:1619677448.657098:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.657099:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.657100:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.657102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.657104:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.2:1619677448.657104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.657106:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.657107:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.657107:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.657108:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:26.0:1619677448.657108:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.657109:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:1.0:1619677448.657110:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677448.657111:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.657112:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.657112:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.657112:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677448.657113:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.657113:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.657115:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.657116:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.657157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.657163:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.657166:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.657167:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.657169:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.657172:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.657173:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.657175:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.657176:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.657177:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.657178:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.657182:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.657183:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.657186:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.657187:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.657188:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.657189:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.657190:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.657190:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.657191:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.657196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.657203:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.657206:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.657207:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.657207:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.657209:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.657212:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.657337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.657343:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.657345:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.657346:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.657348:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.657350:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.657351:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.657352:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.657354:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.657355:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.657356:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.657359:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.657361:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.657363:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.657363:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.657364:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.657366:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.657366:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.657367:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.657367:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.657372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.657380:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.657382:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.657382:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.657383:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.657384:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.657386:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.657402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.657410:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.657413:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.657413:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.657415:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.657418:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.657419:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.657420:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.657421:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.657422:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.657423:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.657426:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.657428:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.657430:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.657431:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.657432:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.657433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677448.657433:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.657434:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.657434:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.657435:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677448.657436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:25.2:1619677448.657439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.657442:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.657444:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.657444:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.657446:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.657446:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677448.657447:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:5.0:1619677448.657448:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.657449:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.657449:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:30.0:1619677448.657450:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677448.657450:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.657451:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.657452:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.657452:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:12.0:1619677448.657452:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.657453:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:30.0:1619677448.657454:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:20.0:1619677448.657454:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.657455:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:30.0:1619677448.657456:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.657456:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.657457:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.657458:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.657458:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.657459:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.657460:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677448.657462:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.657464:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.657467:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.657469:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.657470:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.657471:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.657472:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.657472:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.657473:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.657474:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.657474:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.657474:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.657476:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.657476:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:30.0:1619677448.657477:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.657477:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:30.0:1619677448.657478:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.657478:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677448.657480:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.657480:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.657480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.657483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.2:1619677448.657486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.657487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.657490:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.657491:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:4.2:1619677448.657491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:5.0:1619677448.657492:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.657494:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.657496:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.657496:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.657497:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.657497:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.657498:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.657498:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677448.657499:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.657499:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677448.657499:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677448.657500:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.657500:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:20.0:1619677448.657500:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:10.0:1619677448.657501:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.657501:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.657501:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.657502:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677448.657503:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.657504:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.657505:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.657506:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:26.0:1619677448.657508:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.657509:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.657511:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.657512:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.657513:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.657514:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.657517:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.657519:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.657522:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.657523:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.657523:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.657525:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.657525:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.657526:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.657527:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.657531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.657538:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.657541:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.657542:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.657543:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.657545:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.657548:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.657566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.657572:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.657575:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.657576:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.657578:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.657581:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.657583:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.657584:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.657586:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.657587:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.657588:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.657591:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.657593:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.657596:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.657597:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.657598:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.657600:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.657600:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.657601:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.657601:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.657605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.657612:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.657615:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:21.2:1619677448.657616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:13.0:1619677448.657616:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.657617:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.657619:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.657620:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.657622:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.657625:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.657626:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.657628:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.657630:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.657632:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.657633:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677448.657635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:2.0:1619677448.657635:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.657636:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.657637:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.657640:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.657642:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677448.657642:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:24.0:1619677448.657644:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.657645:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677448.657645:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.657646:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.657647:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:24.0:1619677448.657648:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.657649:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.657649:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:24.0:1619677448.657650:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.657650:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.657650:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:24.0:1619677448.657652:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.657653:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96712041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.657654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:24.0:1619677448.657655:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.657656:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.657657:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.657660:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.657660:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:24.0:1619677448.657662:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.657663:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.657664:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:24.0:1619677448.657664:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.657664:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.657665:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.657666:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:10.0:1619677448.657666:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.657668:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.657668:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.657669:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.657669:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677448.657669:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:8.2:1619677448.657674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.657682:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.657686:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.657687:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.657687:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.657689:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.657692:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.657711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:22.0:1619677448.657717:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677448.657720:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677448.657721:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677448.657723:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.657726:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.657727:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:22.0:1619677448.657729:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.657730:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.657731:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.657732:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677448.657735:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.657737:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677448.657740:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.657740:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:22.0:1619677448.657741:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677448.657743:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677448.657743:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677448.657745:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:22.0:1619677448.657746:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:22.0:1619677448.657746:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:9.2:1619677448.657749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677448.657751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (21)++
00000800:00000200:18.0:1619677448.657757:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677448.657757:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000800:00000200:6.0:1619677448.657758:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:6.0:1619677448.657759:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:18.0:1619677448.657760:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:6.0:1619677448.657760:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000400:00000200:18.0:1619677448.657761:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.657762:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.657763:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.657767:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.657768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.657775:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.657778:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.657778:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.657779:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.657780:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.657780:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.657822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.657829:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.657832:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.657833:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.657835:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.657836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:26.0:1619677448.657837:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.657839:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.657840:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.657841:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.657842:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.657843:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677448.657843:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.657846:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677448.657846:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.657848:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.657849:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.657850:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.657850:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.657851:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:31.0:1619677448.657852:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.657852:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:31.0:1619677448.657853:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:26.0:1619677448.657853:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:31.0:1619677448.657854:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.657854:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.657854:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:31.0:1619677448.657855:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.657855:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677448.657855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:31.0:1619677448.657856:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.657857:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:25.2:1619677448.657859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:31.0:1619677448.657860:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.657862:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.657863:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.657864:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.657864:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:31.0:1619677448.657865:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.657865:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:30.0:1619677448.657866:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.657867:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.657868:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:30.0:1619677448.657868:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.657869:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.657870:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:30.0:1619677448.657870:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:31.0:1619677448.657871:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:30.0:1619677448.657871:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96712037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.657872:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.657872:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.657872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:31.0:1619677448.657873:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.657873:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.657874:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.657875:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.657877:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677448.657877:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.657878:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.657878:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:31.0:1619677448.657879:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.657879:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.657880:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.657880:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:31.0:1619677448.657881:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:26.0:1619677448.657881:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.657881:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.657882:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.657882:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.657883:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.657883:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.657883:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:10.0:1619677448.657883:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.657884:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.657886:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.657886:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.657886:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:30.0:1619677448.657887:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.657888:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:4.2:1619677448.657891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.2:1619677448.657892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.657898:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.657899:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:26.0:1619677448.657900:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.657901:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677448.657902:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:26.0:1619677448.657903:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.657903:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.657905:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.657905:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.657906:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:5.0:1619677448.657906:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:26.0:1619677448.657908:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.657908:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:26.0:1619677448.657909:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.657910:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.657911:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.657914:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.657914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.657916:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.657919:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.657920:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.657920:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.657921:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.657922:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.657923:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.657924:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:5.0:1619677448.657924:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.657925:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:5.0:1619677448.657925:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:26.0:1619677448.657926:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.657926:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.657926:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:25.2:1619677448.657928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.657928:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.657931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.657936:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.657938:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.657939:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.657939:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.657940:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.657940:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.657941:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.657942:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.657942:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.657945:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.658050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.658055:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.658058:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.658058:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.658060:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.658063:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.658064:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.658065:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.658066:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.658067:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.658068:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.658072:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.658074:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.658076:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.658077:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.658077:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.658079:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.658080:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.658080:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.658081:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.658082:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677448.658085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.658087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.658092:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677448.658093:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677448.658094:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.658094:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.658095:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677448.658095:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.658096:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.658097:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.658099:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.658101:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.658149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.658155:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.658157:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.658159:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.658160:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.658162:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.658163:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.658165:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.658166:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.658167:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.658168:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.658171:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.658173:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.658176:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.658177:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.658177:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.658179:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.658180:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.658181:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.658182:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.658183:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.658185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.658188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.658191:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:31.0:1619677448.658192:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.658193:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.658193:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.658194:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677448.658195:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.658196:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.658197:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.658199:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.658201:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.658227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.658233:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.658236:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.658237:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.658238:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.658241:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.658243:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.658244:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.658245:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.658246:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.658247:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.658251:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.658253:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.658255:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.658256:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.658257:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.658258:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.658259:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.658259:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.658260:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.658264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.658272:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.658275:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.658276:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.658276:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.658277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:6.0:1619677448.658278:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.658278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677448.658280:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:2.0:1619677448.658282:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.658285:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:26.0:1619677448.658286:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.658286:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.658288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000400:00000200:2.0:1619677448.658288:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.658289:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.658290:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677448.658291:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.658292:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.658292:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677448.658298:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.658299:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:26.0:1619677448.658300:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.658300:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.658301:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.658302:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677448.658302:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677448.658302:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:6.0:1619677448.658303:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:26.0:1619677448.658304:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.658304:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.658305:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.658305:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:26.0:1619677448.658306:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.658306:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.658307:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.658308:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.658309:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.658312:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.658313:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.658313:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.658314:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.658316:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.658316:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.658316:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.658317:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.658318:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:26.0:1619677448.658319:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.658319:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.658320:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.658322:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.658322:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.658323:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.658323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.658324:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.658328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.658330:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.658333:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.658334:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.658335:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.658336:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.658337:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.658338:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:6.0:1619677448.658339:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.658340:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.658340:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.658342:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.658343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:25.2:1619677448.658345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.658348:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.658350:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.658350:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677448.658352:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.658352:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.658353:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.658353:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:6.0:1619677448.658354:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.658354:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:6.0:1619677448.658355:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677448.658355:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.658355:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.658356:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.658382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.658387:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.658389:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.658392:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.658394:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:5.0:1619677448.658394:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.658396:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:20.0:1619677448.658397:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.658397:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.658398:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.658400:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.658401:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.658402:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:5.0:1619677448.658404:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.658405:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:5.0:1619677448.658406:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.658408:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.658409:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677448.658409:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.658409:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:30.0:1619677448.658410:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.658410:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:30.0:1619677448.658412:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:5.0:1619677448.658412:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.658412:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677448.658413:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.658413:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.658413:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:30.0:1619677448.658415:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.658416:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.658417:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:8.2:1619677448.658417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:30.0:1619677448.658420:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.658422:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.658424:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.658425:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:20.0:1619677448.658425:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:30.0:1619677448.658426:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.658427:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677448.658427:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:30.0:1619677448.658428:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:20.0:1619677448.658428:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.658429:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.658429:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.658430:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.658430:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.658432:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:25.2:1619677448.658433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.658441:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.658443:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.658444:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.658446:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.658448:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.658449:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.658516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.658523:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.658525:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.658526:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.658528:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.658531:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.658532:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.658533:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.658535:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.658536:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.658537:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.658540:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.658541:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.658544:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.658545:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.658545:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.658547:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.658548:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.658549:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.658550:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.658551:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677448.658556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.658561:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.658563:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.658564:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.658565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.658565:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:4.2:1619677448.658570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677448.658572:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.658575:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.658576:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.658577:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677448.658577:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.658578:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.658579:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.658580:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.658580:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:10.0:1619677448.658582:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.658585:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.658587:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.658588:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.658589:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.658591:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.658591:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.658595:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.658595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.658596:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:21.2:1619677448.658599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.2:1619677448.658599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:10.0:1619677448.658600:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.658601:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.658602:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.658604:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.658604:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.658605:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.658605:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.658606:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.658607:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677448.658607:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:31.0:1619677448.658608:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677448.658608:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.658608:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.658610:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.658610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.658610:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:31.0:1619677448.658611:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677448.658611:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:26.0:1619677448.658612:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677448.658614:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.658614:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677448.658614:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.658614:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.658616:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.658617:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.658617:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.658617:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.658617:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.658618:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:1.0:1619677448.658618:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.658619:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:22.0:1619677448.658620:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:1.0:1619677448.658620:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96712025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.658621:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96712025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.658621:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.658621:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:26.0:1619677448.658623:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.658623:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.658624:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.658624:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.658624:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.658625:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.658625:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.658626:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.658626:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677448.658627:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:26.0:1619677448.658630:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.658631:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.658631:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677448.658632:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.658632:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.658633:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.658634:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.658635:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.658635:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:1.0:1619677448.658635:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.658636:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:31.0:1619677448.658637:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.658637:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.658638:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.658639:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:9.2:1619677448.658639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.658639:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.658640:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.658640:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:1.0:1619677448.658640:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.658641:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.658642:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.658642:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677448.658642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677448.658643:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:1.0:1619677448.658643:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677448.658644:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677448.658645:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:25.2:1619677448.658646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.658647:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.658649:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:10.0:1619677448.658649:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:25.2:1619677448.658650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:2.0:1619677448.658650:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.658651:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.658652:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.658652:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.658652:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.658653:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:18.0:1619677448.658653:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.658655:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.658655:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.658656:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:18.0:1619677448.658656:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:13.0:1619677448.658657:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.658658:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.658659:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.658660:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.658660:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.658662:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677448.658662:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.658665:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.658777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:5.2:1619677448.658777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:6.0:1619677448.658783:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.658784:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.658787:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677448.658787:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:6.0:1619677448.658787:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:1.0:1619677448.658788:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.658791:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.658791:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.658793:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.658794:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.658795:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677448.658796:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:6.0:1619677448.658796:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.658798:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.658798:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.658800:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.658800:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.658801:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677448.658802:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.658803:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:6.0:1619677448.658807:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.658808:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.658810:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677448.658811:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:6.0:1619677448.658813:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.658815:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.658815:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.658815:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:17.0:1619677448.658816:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.658817:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677448.658817:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.658818:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.658819:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.658819:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.658819:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677448.658820:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.658820:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677448.658821:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.658824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.658831:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.658833:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:25.2:1619677448.658834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000400:00000200:12.0:1619677448.658834:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.658836:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.658838:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.658839:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.658840:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.658843:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.658843:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.658844:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.658846:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.658849:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:25.2:1619677448.658856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000800:00000200:26.0:1619677448.658862:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.658864:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.658865:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.658865:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.658866:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.658867:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677448.658889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:4.2:1619677448.658892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.658897:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.658899:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.658900:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:10.0:1619677448.658902:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:31.0:1619677448.658903:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.658903:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.658904:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.658905:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.658905:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.658907:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.658907:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.658908:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.658909:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.658910:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.658910:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677448.658910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:31.0:1619677448.658911:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.658912:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.658912:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.658913:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.658915:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.658917:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.658917:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677448.658919:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.658919:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.658920:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.658921:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677448.658922:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.658922:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.658922:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.658923:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:8.0:1619677448.658923:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.658924:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.658925:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.658925:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.658925:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.658926:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.658926:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:10.0:1619677448.658926:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:8.0:1619677448.658926:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.658927:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:8.0:1619677448.658927:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:10.0:1619677448.658928:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:8.0:1619677448.658928:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96712019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.658929:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.658929:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.658930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:8.0:1619677448.658930:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.658931:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.658932:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:25.2:1619677448.658934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:8.0:1619677448.658935:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.658936:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677448.658937:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.658939:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:8.0:1619677448.658939:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677448.658940:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.658940:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.658940:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677448.658941:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:13.0:1619677448.658941:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.658942:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.658942:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:12.0:1619677448.658943:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677448.658944:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:13.0:1619677448.658944:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677448.658944:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:22.0:1619677448.658945:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.658945:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:6.0:1619677448.658945:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677448.658946:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677448.658947:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.658948:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677448.658950:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.658951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.2:1619677448.658954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.658956:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:24.0:1619677448.658958:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.658959:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.658959:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:24.0:1619677448.658960:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677448.658960:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.658961:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:12.0:1619677448.658961:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.658962:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.658962:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.658963:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.658964:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.658965:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677448.658965:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:12.0:1619677448.658966:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.658967:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.658991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.658994:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.658997:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.658999:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.659000:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.659001:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.659003:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.659004:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.659005:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.659006:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.659007:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.659010:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.659012:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.659014:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.659015:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.659016:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.659017:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.659018:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.659019:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.659020:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.659023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.659030:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.659033:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.659034:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.659034:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.659036:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.659038:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.659206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.659212:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.659214:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.659215:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.659217:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.659220:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.659221:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.659223:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.659225:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.659226:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.659227:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.659230:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.659231:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.659234:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.659235:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.659236:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.659238:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.659238:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.659240:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.659241:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.659241:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.659244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.659246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.659250:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:1.0:1619677448.659251:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677448.659253:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:1.0:1619677448.659253:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:24.0:1619677448.659254:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:1.0:1619677448.659254:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.659255:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.659255:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.659257:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.659260:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.659330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:4.2:1619677448.659335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677448.659336:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.659339:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.659340:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.659342:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.659343:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677448.659344:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.659346:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:2.0:1619677448.659346:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:10.0:1619677448.659347:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677448.659347:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.659349:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.659350:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.659350:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.659351:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.659353:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.659356:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.659358:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:2.0:1619677448.659358:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.659360:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.659361:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.659362:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.659362:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.659362:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:26.0:1619677448.659363:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.659364:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.659364:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.659365:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.659366:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000400:00000200:26.0:1619677448.659367:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.659367:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.659367:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:26.0:1619677448.659369:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:25.2:1619677448.659371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.659372:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.659373:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:21.2:1619677448.659373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677448.659374:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.659376:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.659377:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.659378:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.659378:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:18.0:1619677448.659379:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:10.0:1619677448.659379:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.659380:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.659381:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677448.659381:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:10.0:1619677448.659381:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.659382:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:25.2:1619677448.659382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.659382:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.659383:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:10.0:1619677448.659383:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.659386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.659389:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:22.0:1619677448.659390:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.659390:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.659392:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.659392:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677448.659393:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677448.659394:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.659394:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.659395:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.659397:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.659400:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.659445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.659450:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.659453:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.659455:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.659455:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.659458:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.659459:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.659461:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712014] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.659462:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.659464:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.659465:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.659468:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.659470:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.659472:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.659473:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.659474:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.659475:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.659476:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.659476:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.659477:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677448.659482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.659491:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.659493:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.659494:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.659494:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.659497:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.659499:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.659553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.659560:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.2:1619677448.659561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:26.0:1619677448.659563:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:20.0:1619677448.659564:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.659564:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.659566:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.659567:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:26.0:1619677448.659569:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.659569:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.659570:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:5.0:1619677448.659570:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677448.659572:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.659572:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.659573:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.659573:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.659574:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.659575:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.659576:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.659577:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.659578:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.659580:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.659582:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.659584:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.659585:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.659586:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.659588:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.659589:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.659589:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.659590:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.659590:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.659591:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.659591:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.659592:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677448.659592:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.659593:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677448.659593:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.659594:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:20.0:1619677448.659594:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677448.659595:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:20.0:1619677448.659595:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.659598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677448.659598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:25.2:1619677448.659599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.659602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.659607:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.659607:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677448.659608:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677448.659609:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.659610:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677448.659610:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:6.0:1619677448.659610:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.659611:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:18.0:1619677448.659611:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:6.0:1619677448.659611:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.659611:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.659612:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:22.0:1619677448.659612:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:10.0:1619677448.659613:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.659614:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.659614:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.659614:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.659615:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:10.0:1619677448.659616:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.659616:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.659618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.659618:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:10.0:1619677448.659619:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.659621:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.659622:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.659623:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.659624:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677448.659625:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:10.0:1619677448.659627:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.659628:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.659629:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.659629:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.659630:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.659630:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677448.659631:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677448.659632:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.659633:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.659634:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.659636:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.659636:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.659637:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.659638:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.659642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.659649:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.659652:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.659653:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.659654:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.659654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:18.0:1619677448.659656:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.659659:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677448.659660:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.659663:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.659664:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.659665:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.659668:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.659670:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.659671:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.659672:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.659673:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.659674:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.659678:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.659680:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.659682:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.659683:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.659684:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.659686:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.659686:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.659687:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.659687:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.659691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.659698:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.659700:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:8.0:1619677448.659701:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677448.659702:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.659704:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.659706:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677448.659734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.659740:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.659743:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.659743:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.659745:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.659748:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.659749:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.659751:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.659752:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.659753:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.659754:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.659758:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.659760:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.659763:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.659763:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.659764:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.659766:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.659766:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.659767:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.659767:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.659782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.659788:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.659791:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.659791:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.659792:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.659794:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.659797:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.659803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.659810:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.659813:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.659814:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.659815:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.659818:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.659819:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.659820:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.659821:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.659822:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.659823:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.659827:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.659829:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.659832:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.659832:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.659833:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.659835:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.659836:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.659836:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.659837:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.659841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.659848:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.659851:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.659852:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.659852:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.659854:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.659858:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.659861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.659868:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.659871:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.659872:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.659872:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.659873:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.659873:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.659940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.659948:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.659951:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.659954:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.659956:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.659956:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.659958:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.659959:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.659960:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.659962:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.659962:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.659966:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.659967:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.659971:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.659972:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.659972:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.659974:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.659975:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.659976:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.659978:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.659979:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.659980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.659984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.659989:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.659991:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:31.0:1619677448.659992:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:5.0:1619677448.659993:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.659994:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.659995:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.0:1619677448.659995:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.659995:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.659996:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.659998:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.660133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.660141:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.660143:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.660144:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.660146:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.660146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:8.0:1619677448.660148:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.660150:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.660151:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96712006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.660152:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.660153:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:8.0:1619677448.660153:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.660154:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.660156:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.660156:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:8.0:1619677448.660158:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.660159:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.660160:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:26.0:1619677448.660162:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.660163:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:8.0:1619677448.660163:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.660164:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:26.0:1619677448.660165:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96712005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.660165:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677448.660166:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.660167:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.660167:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.660167:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677448.660168:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:8.0:1619677448.660168:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.660168:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:26.0:1619677448.660170:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.660172:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.660173:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677448.660173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.660174:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.660175:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.660176:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.660176:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.660177:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.660178:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.660181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:1.0:1619677448.660181:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.660184:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.660185:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.660185:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.660187:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:1.0:1619677448.660187:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.660190:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.660190:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.660190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.660191:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.660192:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.660193:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:10.0:1619677448.660194:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:1.0:1619677448.660195:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.660197:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.660198:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.660198:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.660199:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.660200:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:25.2:1619677448.660202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.660208:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.660211:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.660211:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.660212:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.660213:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.660214:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.660384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.660391:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.660394:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.660396:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.660399:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.660399:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.660401:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.660402:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96712004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.660404:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.660405:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.660405:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.660409:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.660410:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.660413:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.660413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:31.0:1619677448.660414:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.660414:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.660416:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.660417:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.660418:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.660419:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:2.0:1619677448.660419:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677448.660420:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677448.660422:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:25.2:1619677448.660423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.660423:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:21.2:1619677448.660425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:2.0:1619677448.660425:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.660428:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.660429:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.660430:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.660431:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:2.0:1619677448.660431:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.660432:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.660433:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.660434:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677448.660434:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.660435:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.660436:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.660437:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:2.0:1619677448.660437:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.660438:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677448.660438:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.660439:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677448.660439:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.660441:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.660442:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677448.660442:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.660442:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.660444:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.660445:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.660446:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.660446:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.660450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.660458:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.660460:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:20.2:1619677448.660461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:18.0:1619677448.660461:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.660462:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.660463:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.660464:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677448.660464:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677448.660466:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.660468:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.660470:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.660471:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.660472:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.660474:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.660475:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.660476:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.660477:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.660480:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.660481:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.660484:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.660485:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.660486:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.660488:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.660488:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.660490:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.660491:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.660492:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.660494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.660496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677448.660502:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.660502:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677448.660505:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.660505:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:13.0:1619677448.660506:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.660506:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.660507:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:13.0:1619677448.660508:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.660509:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.660511:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.660655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:5.2:1619677448.660661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.660665:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677448.660668:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677448.660668:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.660670:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677448.660671:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.660672:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.660674:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.660675:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.660676:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677448.660677:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96712001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.660678:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.660679:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.660680:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677448.660682:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.660683:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96712000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.660684:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.660685:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.660685:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.660686:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:1.0:1619677448.660686:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.660689:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.660691:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.660693:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.660693:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.660694:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:1.0:1619677448.660694:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.660695:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.660695:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.660696:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.660697:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677448.660697:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677448.660697:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.660698:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677448.660698:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.660698:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.660699:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677448.660699:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:17.0:1619677448.660700:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.660703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.2:1619677448.660707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.660710:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.660712:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.660713:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.660714:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.660715:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:1.0:1619677448.660715:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.660716:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677448.660717:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.660718:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.660720:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.660721:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.660722:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.660971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.660976:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.660979:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.660980:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.660982:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.660984:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.660985:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.660986:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.660988:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.660989:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.660990:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.660993:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.660995:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.660996:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.660997:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.660998:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.660999:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.660999:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.661000:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.661000:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.661005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.661012:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.661014:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.661014:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.661015:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.661016:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.661018:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.661036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.661044:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.661047:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.661049:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.661051:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.661052:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.661054:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.661055:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711998] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.661056:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.661057:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.661058:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.661061:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.661063:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.661065:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.661066:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.661067:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.661068:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.661069:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.661070:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.661070:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.661074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.661081:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.661083:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.661084:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.661085:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.661087:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.661088:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.661090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:21.2:1619677448.661095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:13.0:1619677448.661097:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.661099:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.661100:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.661100:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.661101:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.661102:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:13.0:1619677448.661102:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.661105:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677448.661107:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.661109:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.661109:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677448.661111:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.661112:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.661114:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.661114:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.661115:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677448.661118:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.661120:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.661123:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.661123:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.661124:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.661126:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.661126:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677448.661127:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677448.661128:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677448.661132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.661139:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.661142:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.661143:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.661144:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.661146:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.661148:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.661152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.661158:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.661160:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.661161:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.661163:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.661165:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.661166:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.661167:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.661168:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.661169:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.661170:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.661174:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.661175:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.661177:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.661178:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.661179:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.661180:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.661181:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.661181:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.661182:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.661186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.661193:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:21.2:1619677448.661194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.661195:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.661196:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.661198:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.661198:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.661199:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.661199:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677448.661202:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.661202:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.661204:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.661205:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.661206:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677448.661207:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.661208:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.661209:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.661210:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677448.661213:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.661214:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.661216:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.661217:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677448.661217:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677448.661219:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.661219:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.661220:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.661220:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.661225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.661232:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.661235:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.661236:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.661236:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.661238:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:6.0:1619677448.661240:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.661241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.661244:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.661247:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:4.2:1619677448.661247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:20.0:1619677448.661250:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.661250:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.661253:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.661254:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:20.0:1619677448.661255:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.661256:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.661258:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:20.0:1619677448.661258:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.661258:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:20.0:1619677448.661259:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.661260:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677448.661262:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.661264:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.661265:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.661266:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:24.0:1619677448.661267:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.661268:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96711993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.661269:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677448.661270:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.661270:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:24.0:1619677448.661271:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.661271:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677448.661272:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.661273:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.661273:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.661274:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:24.0:1619677448.661275:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.661275:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:24.0:1619677448.661277:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.2:1619677448.661278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677448.661280:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.661281:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.661281:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.661283:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.661283:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.661284:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.661285:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677448.661285:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.661287:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.661288:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.661289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:12.0:1619677448.661290:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.661291:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.661293:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677448.661300:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.661303:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.661304:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.661305:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.661307:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.661310:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.661351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.661356:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.661359:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.661359:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.661362:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.661363:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.661365:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.661366:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.661367:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.661368:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.661369:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.661373:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.661375:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.661378:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.661379:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.661379:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.661382:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.661382:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.661383:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.661383:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.661388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.661395:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.661398:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.661399:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.661401:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.661402:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.661404:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.661496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.661504:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.661507:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.661508:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.661510:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.661511:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.661513:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.661514:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.661515:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.661516:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.661517:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.2:1619677448.661520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:10.0:1619677448.661521:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.661523:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.661527:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.661528:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.661529:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.661530:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.661530:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:26.0:1619677448.661532:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.661532:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.661533:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:26.0:1619677448.661534:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.661534:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.661534:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.661535:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.661536:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.661537:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.661538:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.661538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:26.0:1619677448.661539:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.661540:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.661543:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.661544:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:22.0:1619677448.661546:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.661547:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.661548:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:22.0:1619677448.661548:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.661549:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:6.0:1619677448.661549:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.661550:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:22.0:1619677448.661550:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.661551:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.661551:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.661552:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:22.0:1619677448.661552:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.661554:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.2:1619677448.661557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.661565:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.661568:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.661569:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.661571:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.2:1619677448.661571:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.2:1619677448.661575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:5.0:1619677448.661575:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.661578:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677448.661578:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.661580:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.661581:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.661582:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677448.661582:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.661582:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.661583:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.661584:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (22)--
00000400:00000200:10.0:1619677448.661585:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.661585:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:10.0:1619677448.661587:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.661589:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.661590:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.661591:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.661592:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.661593:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.661596:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.661597:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.661599:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.661600:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.661600:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.661602:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.661603:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.661604:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.661604:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.661607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.661614:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.661617:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.661617:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.661619:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.661621:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.661621:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.661624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.661632:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.661635:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.661637:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.661640:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.661640:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.661640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:30.0:1619677448.661641:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:31.0:1619677448.661642:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.661643:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.661643:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.661644:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.661645:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.661646:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.661648:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.661649:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.661651:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.661651:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.661652:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.661653:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.661653:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.661654:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.661654:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:31.0:1619677448.661655:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.661655:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.661656:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.661656:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:31.0:1619677448.661659:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.661660:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.661661:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.661661:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.661661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.661662:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677448.661663:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.661663:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.661664:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.661664:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.661666:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.661666:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.661667:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677448.661667:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:31.0:1619677448.661668:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.661669:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.661670:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.661670:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:25.2:1619677448.661671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:26.0:1619677448.661672:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.661675:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:30.0:1619677448.661678:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.661680:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.661681:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.661683:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.661684:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:21.2:1619677448.661684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:30.0:1619677448.661685:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:2.0:1619677448.661690:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.661693:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.661695:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.661696:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.661699:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.661700:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.661701:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.661702:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.661704:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.661705:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.661708:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.661710:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.661711:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.661712:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.661713:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.661714:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.661715:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.661715:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.661716:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.661720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.661728:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.661730:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.661731:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.661732:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.661733:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.661734:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.661740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.661747:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.661749:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.661750:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.661751:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.661751:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.661752:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.661833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.661839:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.661842:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.661842:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.661844:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.661847:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.661849:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.661850:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.661851:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.661852:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.661853:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.661856:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.661857:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.661859:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.661860:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.661861:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.661862:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.661862:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.661864:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.661865:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.661866:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.661868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.661871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.661876:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.661877:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.661877:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.661878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677448.661878:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.661879:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.661879:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.661879:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.661880:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.661881:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677448.661881:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.661881:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677448.661883:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.661886:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.661887:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.661888:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.661890:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.661891:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.661892:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.661893:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.661894:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.661898:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.661900:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.661903:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.661904:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.661904:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.661906:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.661907:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.661907:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.661908:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.661913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.661920:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.661923:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.661924:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.661925:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.661927:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.661928:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.662049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.662054:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.662057:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.662058:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.662060:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.662062:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.662064:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.662065:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.662067:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.662068:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.662069:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.662072:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.662074:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.662077:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.662078:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.662079:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.662081:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.662081:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.662082:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.662082:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.662087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.662093:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.662096:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.662097:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.662099:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.662100:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.662102:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.662103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.662110:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.662113:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.662115:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.662117:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.662117:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.662119:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677448.662120:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.662121:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.662122:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.662123:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677448.662127:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.662129:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677448.662131:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.662132:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:30.0:1619677448.662133:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:30.0:1619677448.662134:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:30.0:1619677448.662135:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.662136:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.662136:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.662140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.662143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:8.0:1619677448.662146:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.662148:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.662149:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.662149:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.662150:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000400:00000200:8.0:1619677448.662151:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.662153:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:8.0:1619677448.662154:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:12.0:1619677448.662155:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.662157:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677448.662158:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.662159:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:21.2:1619677448.662160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:12.0:1619677448.662161:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.662162:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.662163:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677448.662164:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.662166:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:12.0:1619677448.662167:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.662169:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:12.0:1619677448.662169:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.2:1619677448.662170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677448.662170:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.662173:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:12.0:1619677448.662173:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.662174:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:18.0:1619677448.662175:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.662175:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677448.662177:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:12.0:1619677448.662177:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.662178:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.662178:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677448.662178:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:10.0:1619677448.662178:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:18.0:1619677448.662179:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.662179:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000400:00000200:18.0:1619677448.662180:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.662181:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:18.0:1619677448.662181:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.662181:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:9.2:1619677448.662183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677448.662185:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.662187:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:10.0:1619677448.662188:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.662189:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.662189:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.662190:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96711979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.662191:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:10.0:1619677448.662192:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.662193:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.662193:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.662193:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.662194:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:2.0:1619677448.662194:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.662195:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:10.0:1619677448.662195:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.662197:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.662197:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:2.0:1619677448.662198:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.662199:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.662199:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677448.662200:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:10.0:1619677448.662200:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.662202:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:25.2:1619677448.662203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.662203:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.662204:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.662205:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.662206:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.662207:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.662208:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.662209:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.662209:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677448.662210:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.662212:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:25.2:1619677448.662213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677448.662213:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.662214:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.662216:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.662218:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:13.0:1619677448.662219:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.662222:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.662223:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.662224:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.662225:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.662227:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.662238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:20.2:1619677448.662243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:18.0:1619677448.662244:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.662247:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.662247:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.662249:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.662249:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:18.0:1619677448.662251:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.662252:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:18.0:1619677448.662253:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677448.662254:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.662254:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.662255:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711978] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.662256:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.662257:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.662257:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.662258:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.662259:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.662261:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.662263:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.662263:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.662264:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.662265:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.662266:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.662269:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.662269:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.662270:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.662271:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.662271:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.662272:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.662273:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.662274:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.662274:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677448.662274:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.662275:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.662275:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.662277:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:25.2:1619677448.662278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.662278:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.662279:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.662280:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:12.0:1619677448.662284:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677448.662287:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677448.662288:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.662290:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.662291:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.662292:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.662292:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.662298:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677448.662299:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.662300:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677448.662303:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.662304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.662312:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.662314:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.662315:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.662316:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.662316:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.662318:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.662460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677448.662461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:26.0:1619677448.662467:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677448.662469:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:26.0:1619677448.662470:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:24.0:1619677448.662471:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:10.0:1619677448.662471:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677448.662472:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:26.0:1619677448.662473:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.662475:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.662476:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.662477:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.662478:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.662479:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.662479:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.662481:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.662481:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.662482:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.662483:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.662484:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677448.662484:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.662485:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.662488:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.662490:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.662491:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677448.662492:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.662494:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.662495:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.662495:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677448.662495:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.662496:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.662496:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.662497:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.662498:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677448.662498:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.662499:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:24.0:1619677448.662499:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:26.0:1619677448.662500:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:24.0:1619677448.662500:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.662500:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:26.0:1619677448.662501:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.662503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:9.2:1619677448.662505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677448.662506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.662513:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.662513:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677448.662514:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.662516:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.662516:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677448.662516:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.662517:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.662517:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677448.662517:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.662518:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:6.0:1619677448.662518:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.662518:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.662520:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.662521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:2.0:1619677448.662521:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.662524:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.662525:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677448.662526:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.662529:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.662529:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.662530:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.662531:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.662531:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.662607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.662613:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.662616:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.662617:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.662619:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.662620:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.662622:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.662623:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.662624:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.662625:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.662626:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.662630:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.662632:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.662634:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.662635:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.662636:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.662638:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.662638:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.662639:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.662639:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.662644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.662651:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.662654:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.662655:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.662655:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.662657:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.662659:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.662701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.662709:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.662712:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.662714:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.662715:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.662718:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.662720:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.662721:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.662723:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.662724:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.662725:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.662728:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.662730:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.662733:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.662734:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.662734:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.662736:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.662736:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.662737:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.662737:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.662742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.662750:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.662752:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.662753:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.662753:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.662756:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.662758:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.662764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.662771:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.662774:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.662775:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.662775:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.662776:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.662776:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.662856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.2:1619677448.662861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:10.0:1619677448.662864:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.662867:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.662867:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:6.0:1619677448.662867:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.662869:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.662870:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677448.662870:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.662873:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.662874:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.662875:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.662876:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.662877:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.662877:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.662879:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.662880:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.662881:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.662882:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.662882:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.662883:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:6.0:1619677448.662884:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:10.0:1619677448.662889:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.662889:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.662892:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677448.662892:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.662894:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.662895:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.662896:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:6.0:1619677448.662896:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.662896:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.662897:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.662898:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.662898:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.662899:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677448.662899:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.662900:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:6.0:1619677448.662900:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.662901:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677448.662901:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.662902:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:25.2:1619677448.662903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:6.0:1619677448.662903:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.2:1619677448.662906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.2:1619677448.662908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677448.662911:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:25.2:1619677448.662911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:31.0:1619677448.662913:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.662914:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.662914:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:31.0:1619677448.662916:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.662916:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677448.662916:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.662917:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677448.662918:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.662918:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:22.0:1619677448.662918:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.662918:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:31.0:1619677448.662919:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:26.0:1619677448.662919:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.662920:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.662921:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:10.0:1619677448.662921:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.662922:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.662923:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.662926:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677448.662997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677448.663001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.663004:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.663007:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677448.663008:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677448.663009:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.663010:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:31.0:1619677448.663011:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677448.663011:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:30.0:1619677448.663012:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.663013:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.663013:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.663015:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.663015:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.663016:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.663017:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.663017:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.663018:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.663019:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.663019:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.663020:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.663021:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.663023:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.663025:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.663026:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.663028:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.663028:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.663029:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.663030:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.663030:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.663031:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.663031:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.663032:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.663032:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.663033:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:26.0:1619677448.663033:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:25.2:1619677448.663033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.663034:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.663035:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:26.0:1619677448.663035:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.663035:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.663036:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.663037:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:25.2:1619677448.663037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.663039:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.663040:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:30.0:1619677448.663041:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.116@o2ib7 : GET - for me
00000800:00000200:25.2:1619677448.663041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.663041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:30.0:1619677448.663043:0:19944:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.116@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000800:00000200:13.0:1619677448.663044:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:30.0:1619677448.663045:0:19944:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.116@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:30.0:1619677448.663046:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (21)++
00000800:00000200:13.0:1619677448.663046:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:30.0:1619677448.663047:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:13.0:1619677448.663047:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.663048:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.663049:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677448.663049:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:6.0:1619677448.663049:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:31.0:1619677448.663050:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:12.0:1619677448.663050:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.663051:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.663052:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.663052:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:30.0:1619677448.663053:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : REPLY try# 0
00000800:00000200:22.0:1619677448.663053:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.663053:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.663054:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:13.0:1619677448.663055:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:6.0:1619677448.663056:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.663057:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.663057:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.663058:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (22)++
00000800:00000200:6.0:1619677448.663058:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677448.663059:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.663060:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:25.2:1619677448.663065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.663071:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.663074:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.663074:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.663076:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: REPLY: OK
00000800:00000200:30.0:1619677448.663078:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.663079:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.663189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.663196:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.663198:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.663199:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.663201:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.663204:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.663206:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.663207:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.663209:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.663210:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.663211:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677448.663214:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.663215:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.663218:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.663219:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.663220:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.663221:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.663222:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.663222:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.663223:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.663227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.663235:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.663238:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.663238:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.663240:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.663242:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.663244:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.663401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.663407:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.663410:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.663410:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.663412:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.663414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:17.0:1619677448.663414:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.663416:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.663417:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.663419:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.663419:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.663420:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:17.0:1619677448.663421:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.663423:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.663425:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:17.0:1619677448.663425:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.663426:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.663428:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:5.0:1619677448.663429:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.663430:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.663431:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:5.0:1619677448.663431:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:17.0:1619677448.663432:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:5.0:1619677448.663432:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.663433:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.663434:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.663434:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677448.663434:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.663435:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:5.0:1619677448.663435:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.663436:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:25.2:1619677448.663440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:5.0:1619677448.663440:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.663441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677448.663442:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:21.2:1619677448.663444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:5.0:1619677448.663445:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.663446:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.663447:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.663449:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.663449:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.663450:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677448.663451:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.0:1619677448.663451:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677448.663452:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:5.0:1619677448.663452:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677448.663453:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.663453:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:22.0:1619677448.663454:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.663455:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.663455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.663455:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:22.0:1619677448.663456:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.663456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677448.663457:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.663457:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.663458:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.663459:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:22.0:1619677448.663459:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:30.0:1619677448.663460:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.663461:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:6.0:1619677448.663461:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.663462:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.663463:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.663463:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.663463:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:4.2:1619677448.663463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677448.663464:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000400:00000200:6.0:1619677448.663464:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677448.663465:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.663465:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.663466:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.663466:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:6.0:1619677448.663466:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.663467:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:18.0:1619677448.663467:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.663467:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677448.663468:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:6.0:1619677448.663468:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.663469:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.663469:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:18.0:1619677448.663470:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.663471:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:6.0:1619677448.663471:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.663472:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677448.663472:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.663473:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.663473:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:10.0:1619677448.663474:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.663475:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:30.0:1619677448.663477:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.663478:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.663478:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:10.0:1619677448.663478:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.663478:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.663479:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.663480:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.663480:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677448.663480:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.663481:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96711963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.663481:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:4.2:1619677448.663481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677448.663482:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.663482:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:10.0:1619677448.663483:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.663483:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.663484:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:10.0:1619677448.663485:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.663485:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:10.0:1619677448.663486:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:6.0:1619677448.663487:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.663487:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.2:1619677448.663488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.663488:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677448.663489:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.663489:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:10.0:1619677448.663491:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.663492:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.663493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677448.663494:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:10.0:1619677448.663494:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.663495:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96711962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.663495:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:10.0:1619677448.663495:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.663495:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:26.0:1619677448.663496:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.663497:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.663497:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:10.0:1619677448.663497:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.663498:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.663498:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.663498:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677448.663499:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.663500:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:1.0:1619677448.663500:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.663501:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:20.0:1619677448.663501:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.663501:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677448.663501:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:5.0:1619677448.663501:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.663502:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:26.0:1619677448.663503:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.663503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:1.0:1619677448.663503:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.663504:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:10.0:1619677448.663504:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:8.0:1619677448.663504:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.663504:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.663506:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.663506:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:10.0:1619677448.663506:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.663507:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:8.0:1619677448.663507:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.663507:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.663508:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.663509:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677448.663509:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:26.0:1619677448.663510:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.663510:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:10.0:1619677448.663510:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.663510:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96711961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.663510:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677448.663512:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.663512:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:8.2:1619677448.663512:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:26.0:1619677448.663513:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.663513:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.663514:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.663515:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:8.0:1619677448.663516:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.663518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:8.0:1619677448.663518:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.663518:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.663520:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:8.0:1619677448.663521:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.663522:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.663523:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.663523:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.663524:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:8.0:1619677448.663524:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.663525:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.663525:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.0:1619677448.663526:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.663527:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:8.0:1619677448.663527:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:31.0:1619677448.663528:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.663528:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677448.663529:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.663529:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.663530:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677448.663531:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.663532:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.0:1619677448.663532:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.663533:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.663533:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.663535:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:25.2:1619677448.663538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:4.2:1619677448.663539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.663542:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:30.0:1619677448.663545:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:24.0:1619677448.663545:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.663545:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.663546:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.663546:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.663548:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677448.663548:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677448.663548:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.663549:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:24.0:1619677448.663549:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677448.663550:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:8.0:1619677448.663550:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:30.0:1619677448.663551:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.663552:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:1.0:1619677448.663552:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.663759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.663766:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.663769:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.663771:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.663773:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.663774:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.663775:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.663777:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.663778:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.663779:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.663780:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.663783:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.663784:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.663787:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.663787:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.663788:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.663789:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.663790:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.663791:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.663791:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.663795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.663802:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.663804:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.663805:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.663806:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.663809:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.663809:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.2:1619677448.663889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.663894:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.663897:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.663898:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.663900:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.663902:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.663903:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.663905:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.663906:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.663907:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.663908:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.663911:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.663913:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.663916:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.663917:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.663917:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.663919:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.663920:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.663921:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.663921:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677448.663923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:25.2:1619677448.663926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.663931:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.663933:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:30.0:1619677448.663933:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:31.0:1619677448.663935:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.663935:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.663936:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.663937:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.663938:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.663939:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.663940:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.663942:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.663943:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.663944:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.663947:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.663947:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.663948:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.663948:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.663949:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.663951:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.663952:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.663952:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.663954:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.663954:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.663955:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.663956:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.663959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.663966:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.663968:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.663969:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.663970:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.663972:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.663973:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.663979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:12.0:1619677448.663985:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.663988:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.663988:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.663989:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.663990:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.663990:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.664010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.664018:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.664021:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:21.2:1619677448.664021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:10.0:1619677448.664021:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.664023:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.664025:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.664026:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677448.664027:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.664028:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.664029:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:10.0:1619677448.664030:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.664030:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677448.664031:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.664033:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.664033:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677448.664036:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.664037:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:10.0:1619677448.664037:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.664038:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:22.0:1619677448.664039:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.664040:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.664041:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.664042:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.664042:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.664043:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.664043:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.664045:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:22.0:1619677448.664046:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.664047:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.664047:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.664048:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.664048:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677448.664050:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.664051:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.664052:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:9.2:1619677448.664052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677448.664053:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.664054:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.664054:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.664055:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.664059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677448.664059:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.664062:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.664063:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.664063:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.664065:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:2.0:1619677448.664066:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.664068:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.664069:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.664069:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677448.664069:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.664070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:26.0:1619677448.664071:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.664075:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:18.0:1619677448.664077:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.664080:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.664081:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.664081:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.664082:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.664083:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.664306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677448.664313:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.0:1619677448.664316:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.664318:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677448.664318:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.664321:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.664321:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.664321:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.664322:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677448.664323:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.664324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:5.0:1619677448.664325:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677448.664325:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.664326:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.664327:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.664328:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.664329:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.664329:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:5.0:1619677448.664330:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.664331:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677448.664331:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.664332:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.664334:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.664334:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.664335:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.664335:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:1.0:1619677448.664335:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:5.0:1619677448.664337:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.664339:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.664339:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.664340:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:1.0:1619677448.664340:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.664341:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:10.0:1619677448.664342:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.664342:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.664343:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.664343:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:10.0:1619677448.664344:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:5.0:1619677448.664344:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:10.0:1619677448.664345:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96711953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.664345:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000400:00000200:10.0:1619677448.664346:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.664346:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677448.664346:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.664347:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.664347:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:10.0:1619677448.664348:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.664349:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.2:1619677448.664349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.664349:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.664349:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.664351:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:20.2:1619677448.664352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:10.0:1619677448.664352:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.664352:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.664352:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.664354:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:9.2:1619677448.664356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:10.0:1619677448.664357:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.664357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.664358:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.664358:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.664359:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.664360:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:10.0:1619677448.664360:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.664360:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677448.664361:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.664361:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.664361:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:24.0:1619677448.664362:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:5.0:1619677448.664362:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:20.0:1619677448.664363:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.664363:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:5.0:1619677448.664363:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.664364:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:24.0:1619677448.664365:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:20.0:1619677448.664365:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677448.664365:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:24.0:1619677448.664366:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.664366:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677448.664366:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:25.2:1619677448.664367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.664367:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677448.664368:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677448.664369:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.664369:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:24.0:1619677448.664372:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.664375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:6.0:1619677448.664375:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.664378:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.664379:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.664380:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.664381:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000400:00000200:6.0:1619677448.664382:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.664383:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.664384:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.664384:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.664385:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:6.0:1619677448.664385:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677448.664386:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.664401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.664407:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.664410:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.664411:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.664412:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.664416:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.664417:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:20.2:1619677448.664419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:2.0:1619677448.664419:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.664420:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.664421:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.664422:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.664425:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.664427:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.664428:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.664430:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:2.0:1619677448.664430:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.664431:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.664431:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:30.0:1619677448.664432:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.664433:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.664434:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.664434:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:31.0:1619677448.664435:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.664435:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.664435:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:30.0:1619677448.664436:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.664438:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.664439:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.664439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:30.0:1619677448.664440:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.664441:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.664444:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.664446:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.664446:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.664448:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:30.0:1619677448.664449:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.664449:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.664449:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:30.0:1619677448.664450:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:26.0:1619677448.664450:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.664451:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.664452:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677448.664452:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.664454:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.664455:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:26.0:1619677448.664455:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:30.0:1619677448.664456:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.664457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.664461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.664464:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.664467:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.664467:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.664469:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:12.0:1619677448.664469:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.664471:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:12.0:1619677448.664471:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.664472:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:12.0:1619677448.664472:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.664473:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.2:1619677448.664541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.664547:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.664549:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.664550:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.664552:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.664555:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.664556:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.664558:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.664559:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.664560:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.664561:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.664564:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.664566:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.664568:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.664568:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.664569:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.664571:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.664571:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.664572:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.664572:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.664577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.664584:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.664586:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.664586:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.664587:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.664588:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.664590:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.664616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.664623:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.2:1619677448.664623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:24.0:1619677448.664625:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.664626:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.664628:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.664630:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.664630:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:24.0:1619677448.664632:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:26.0:1619677448.664633:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:24.0:1619677448.664633:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711949] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.664633:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:24.0:1619677448.664635:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.664636:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.664636:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.664637:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:10.0:1619677448.664638:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.664640:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.664641:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.664641:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.664643:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:10.0:1619677448.664643:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.664644:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.664645:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.664646:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.664647:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.664648:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.664649:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:10.0:1619677448.664649:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.664650:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677448.664650:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.664651:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.664651:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677448.664653:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.664654:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.664655:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.2:1619677448.664655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677448.664657:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.664657:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.664658:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.664658:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.664663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.664663:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.664666:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.664666:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.664667:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.664669:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.664670:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677448.664671:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677448.664673:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.664674:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.664674:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.664676:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.664679:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.664710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.664718:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.664721:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.664723:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.664724:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.664725:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.664727:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677448.664728:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.664730:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.664731:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.664732:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.664735:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.664736:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.664739:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.664740:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.664740:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.664742:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.664743:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.664744:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.664744:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.664748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.664756:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.664759:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.664759:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.664760:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.664762:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.664765:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.664956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677448.664963:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.664966:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677448.664967:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677448.664969:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.664972:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.664973:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.664974:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.664976:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.664977:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.664977:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.664981:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.664982:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.664984:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.664985:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.664986:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.664987:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.664988:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.664988:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677448.664989:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677448.664993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.665000:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:4.2:1619677448.665002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.665003:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.665003:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.665005:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.665007:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.665008:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677448.665008:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.665011:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.665012:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.665013:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.665016:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.665017:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.665018:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711945] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.665020:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.665020:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.665021:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.665024:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.665026:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.665029:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.665029:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.665030:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.665031:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.665032:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.665033:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.665034:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.665035:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.665038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.665040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677448.665046:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677448.665047:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:5.0:1619677448.665048:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677448.665049:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:5.0:1619677448.665049:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.665050:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.665051:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:5.0:1619677448.665051:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.665052:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.2:1619677448.665054:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.0:1619677448.665058:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677448.665060:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.665063:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.665064:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.665066:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.665067:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.665069:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.665070:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.665071:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.665072:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.665073:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.665076:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.665078:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.665080:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.665081:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.665082:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.665083:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.665084:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.665085:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.665085:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.665089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.665096:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.665098:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.665099:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677448.665100:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.665101:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.665104:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.2:1619677448.665136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.665138:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.665141:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.665143:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.665144:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.665145:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.665146:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.665148:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.665149:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.665150:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.665151:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.665154:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.665156:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:20.0:1619677448.665160:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.665162:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.665163:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.665164:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.665164:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677448.665165:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.665166:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.665166:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.665167:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:20.0:1619677448.665167:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.665168:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:10.0:1619677448.665169:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.665172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:10.0:1619677448.665172:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.665173:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.665174:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.665176:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.665176:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.665178:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677448.665180:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:10.0:1619677448.665181:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.665182:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.665183:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.665183:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677448.665183:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.665185:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.665186:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.665187:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:10.0:1619677448.665187:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.665187:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.665189:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.665190:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.665191:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.665191:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:5.2:1619677448.665192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:21.2:1619677448.665193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:9.2:1619677448.665196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.665200:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:2.0:1619677448.665200:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.665203:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:24.0:1619677448.665204:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.665204:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:18.0:1619677448.665205:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.665207:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.665208:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.665209:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.665209:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.665210:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.665212:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.665214:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:18.0:1619677448.665214:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.665214:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.665215:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.665215:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.665216:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:2.0:1619677448.665216:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:24.0:1619677448.665217:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.665217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:24.0:1619677448.665218:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.665218:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.665219:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.665220:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.665221:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.665222:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:2.0:1619677448.665222:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.665225:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:24.0:1619677448.665226:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.665227:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.665228:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:24.0:1619677448.665229:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.665231:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:24.0:1619677448.665232:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.665232:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:5.0:1619677448.665232:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.665233:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677448.665233:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.665234:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:24.0:1619677448.665235:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.665235:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:5.0:1619677448.665235:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.665235:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:24.0:1619677448.665237:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000400:00000200:5.0:1619677448.665237:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:2.0:1619677448.665237:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.665237:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:24.0:1619677448.665238:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000400:00000200:5.0:1619677448.665238:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96711939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.665239:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:5.0:1619677448.665239:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.665239:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:25.2:1619677448.665240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:5.0:1619677448.665240:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.665240:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.665240:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:5.0:1619677448.665241:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:8.2:1619677448.665243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.665244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:5.2:1619677448.665245:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:13.0:1619677448.665248:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.0:1619677448.665250:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:13.0:1619677448.665250:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.665251:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.665251:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:5.0:1619677448.665251:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.665252:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:18.0:1619677448.665252:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:20.0:1619677448.665253:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.665253:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.665253:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:5.0:1619677448.665253:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.665254:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:12.0:1619677448.665254:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.665254:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.665255:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:20.0:1619677448.665255:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.665255:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:13.0:1619677448.665256:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677448.665256:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.665257:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677448.665257:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677448.665257:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.665258:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.665259:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677448.665259:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.665260:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.665261:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.665261:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.665265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.665272:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.665274:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.665275:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.665276:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.665278:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.665281:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.665284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.665289:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.665292:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.665293:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.665299:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.665300:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.665300:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.665349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.665358:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.665360:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.665363:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.665365:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.665365:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.665367:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.665368:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.665369:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.665371:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.665372:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.665375:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.665377:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.665379:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.665380:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.665380:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.665382:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.665382:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.665383:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.665383:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.665388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.665395:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.665397:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.665398:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.665399:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.665401:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.665402:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.665461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.665468:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.665471:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.665472:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.665473:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.665476:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.665477:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.665478:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.665480:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.665481:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.665481:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.665484:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.665486:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.665489:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.665490:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.665490:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.665492:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.665493:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.665493:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.665494:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.665498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.665506:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:21.2:1619677448.665508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677448.665510:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.665511:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.665512:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.665513:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:2.0:1619677448.665513:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.665514:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000400:00000200:18.0:1619677448.665517:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.665517:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.665518:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677448.665520:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.665522:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.665523:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.665524:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.665526:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.665526:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.665530:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.665531:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.665534:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.665535:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.665535:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.665537:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.665538:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.665538:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677448.665539:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677448.665544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677448.665551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677448.665552:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677448.665553:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677448.665554:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.665555:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.665556:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677448.665556:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.665558:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.665559:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.665559:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677448.665561:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:20.0:1619677448.665562:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.665564:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.665565:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.665566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000400:00000200:20.0:1619677448.665567:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.665568:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.665569:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.665572:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.665573:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.665574:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.665576:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.665577:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.665577:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.665578:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.665578:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.665578:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.665579:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.665579:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677448.665581:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.665582:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.665582:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.665583:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.665587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.665594:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.665597:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.665598:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.665600:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.665601:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.665602:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.665619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.665624:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.665627:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.665628:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.665630:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.665632:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.665634:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.665635:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.665636:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.665637:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.665639:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:20.2:1619677448.665641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:17.0:1619677448.665642:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.665643:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.665646:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.665647:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.665648:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.665649:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677448.665649:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.665650:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.665651:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.665651:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.665652:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.665654:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.665655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:30.0:1619677448.665656:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.665657:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.665658:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.665659:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.665661:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.665662:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.665662:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:31.0:1619677448.665663:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:24.0:1619677448.665665:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.665665:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:31.0:1619677448.665666:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.665666:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.665668:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:8.0:1619677448.665668:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.665670:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:31.0:1619677448.665671:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.665672:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.665673:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.665675:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.665675:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.665676:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.665677:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.665681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.665689:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.665691:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.665692:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.665694:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.665694:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.665697:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.665699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.665707:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.665710:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.665711:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.665711:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.665712:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.665713:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.665785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.665791:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.665794:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.665794:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.665797:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.665800:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.665801:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:20.2:1619677448.665802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:6.0:1619677448.665802:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.665802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:6.0:1619677448.665804:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.665805:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.665806:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.665808:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:6.0:1619677448.665809:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.665810:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:6.0:1619677448.665811:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.665811:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.665812:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:30.0:1619677448.665813:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:6.0:1619677448.665814:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.665815:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:2.0:1619677448.665815:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.665816:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:6.0:1619677448.665816:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.665817:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.665817:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.665818:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677448.665818:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.665819:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.665819:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:2.0:1619677448.665819:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677448.665820:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.665821:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.665822:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:8.2:1619677448.665823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:2.0:1619677448.665823:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.665824:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.665824:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.665825:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.665826:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.665826:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677448.665828:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.2:1619677448.665829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:2.0:1619677448.665830:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.665831:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:30.0:1619677448.665832:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.665833:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.665833:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.665833:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.665835:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 504 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:20.0:1619677448.665836:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.665836:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.665836:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.665837:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.665837:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.665838:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.665839:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.665839:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:18.0:1619677448.665839:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:30.0:1619677448.665840:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677448.665840:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.665841:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677448.665841:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:18.0:1619677448.665842:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.665842:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:30.0:1619677448.665843:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:30.0:1619677448.665843:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677448.665843:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:30.0:1619677448.665844:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:2.0:1619677448.665844:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:30.0:1619677448.665845:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:18.0:1619677448.665845:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.665846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:18.0:1619677448.665847:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.665848:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96711929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.665849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:18.0:1619677448.665850:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.665851:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.665852:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.665854:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:18.0:1619677448.665855:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.665856:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.665857:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677448.665858:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.665858:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.665858:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.665859:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.665860:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.665860:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.665861:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.665861:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.665862:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.665862:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677448.665863:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.665863:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677448.665864:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.665865:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:22.0:1619677448.665866:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:18.0:1619677448.665866:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.665867:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.665868:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.665870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:4.2:1619677448.665870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.2:1619677448.665874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:24.0:1619677448.665877:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677448.665877:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.665879:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:24.0:1619677448.665880:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:21.2:1619677448.665880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:13.0:1619677448.665880:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.665881:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.665881:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677448.665881:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.665882:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:13.0:1619677448.665883:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.665883:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:24.0:1619677448.665884:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.665884:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677448.665886:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.665886:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.665887:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:24.0:1619677448.665888:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:25.2:1619677448.665889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.665890:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:24.0:1619677448.665890:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.665892:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.665893:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.665893:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:24.0:1619677448.665893:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.665894:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:12.0:1619677448.665895:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.665896:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.665897:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.665898:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.665898:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.665898:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.665899:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.665899:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:31.0:1619677448.665900:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.665900:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677448.665900:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:31.0:1619677448.665901:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.665902:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.665902:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677448.665903:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.665904:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:31.0:1619677448.665905:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.665905:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.665907:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677448.665907:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.665907:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.665908:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.665908:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:31.0:1619677448.665909:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.665910:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.665911:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.665912:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.2:1619677448.665912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.665913:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.665914:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.665914:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.665918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.665920:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.665923:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.665924:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.665924:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.665925:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:13.0:1619677448.665926:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:17.0:1619677448.665927:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:13.0:1619677448.665927:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.665929:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.665930:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:12.0:1619677448.665930:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.665931:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.665983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.665991:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.665993:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.665994:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.665996:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.665999:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.666001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:10.0:1619677448.666001:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.666002:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.666003:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.666004:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.666005:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.666006:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:10.0:1619677448.666008:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.666009:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:10.0:1619677448.666010:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:5.0:1619677448.666011:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.666012:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677448.666013:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.666014:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.666014:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:5.0:1619677448.666014:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.666015:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677448.666016:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.666016:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677448.666016:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.666017:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.666018:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:5.0:1619677448.666018:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.666019:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.666020:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:25.2:1619677448.666022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:5.0:1619677448.666023:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.666025:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.666028:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.666029:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:6.0:1619677448.666030:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.0:1619677448.666030:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.666032:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:5.0:1619677448.666032:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.666032:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677448.666033:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:5.0:1619677448.666033:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:6.0:1619677448.666034:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.666034:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:6.0:1619677448.666036:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.666037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.666038:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.666042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677448.666044:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.666046:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.666047:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.666047:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:12.0:1619677448.666049:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.666050:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.666050:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.666051:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677448.666052:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677448.666053:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.666055:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.666057:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.666058:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.666059:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.666060:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.666061:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.666064:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.666066:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.666069:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.666070:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.666071:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.666072:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.666073:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.666074:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.666075:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.666076:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.666078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.666081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.666085:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677448.666086:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677448.666087:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.666088:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.666089:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.666089:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677448.666090:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.666090:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.666091:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.666092:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.666114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.666120:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.666122:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.666123:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.666125:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.666128:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.666130:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.666131:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711923] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.666132:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.666133:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.666134:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.666138:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.666139:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.666142:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.666142:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.666143:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.666145:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.666145:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.666146:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.666147:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.666147:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677448.666151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.666152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677448.666158:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677448.666158:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.666160:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.666160:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677448.666161:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677448.666161:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:22.0:1619677448.666162:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.666162:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.666164:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:25.2:1619677448.666166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.666170:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.666171:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.666174:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.666174:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.666175:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.666176:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.666176:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.666310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.666319:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.666322:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.666324:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.666326:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.666326:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.666328:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.666329:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.666331:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.666332:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.666333:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.666336:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.666338:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.666341:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.666342:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.666343:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.666345:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.666345:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.666347:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.666348:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.666349:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.666351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.666355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.666363:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.666363:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.666365:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677448.666366:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.666367:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.666367:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.666368:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.666368:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.666370:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.666372:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.666393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.666401:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.666404:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.666405:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.666407:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.666410:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.666410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:26.0:1619677448.666411:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.666413:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.666414:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.666415:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.666416:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:18.0:1619677448.666417:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:26.0:1619677448.666420:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.666420:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:26.0:1619677448.666421:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677448.666421:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.666423:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.666424:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.666424:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.666425:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.666426:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:18.0:1619677448.666426:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.666427:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:18.0:1619677448.666427:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:26.0:1619677448.666428:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:18.0:1619677448.666428:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.666429:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.666430:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.666430:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:18.0:1619677448.666431:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.666431:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:25.2:1619677448.666433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:18.0:1619677448.666435:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.666436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677448.666437:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.666439:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.666440:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:18.0:1619677448.666440:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.666441:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.666442:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:18.0:1619677448.666442:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:31.0:1619677448.666443:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.666443:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.666443:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:10.0:1619677448.666443:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:18.0:1619677448.666444:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:31.0:1619677448.666445:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.666445:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:30.0:1619677448.666446:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.666446:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.666447:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:10.0:1619677448.666448:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.2:1619677448.666448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.666456:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.666459:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.666460:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.666461:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.666462:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.666464:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.666481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.2:1619677448.666485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:20.2:1619677448.666486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:30.0:1619677448.666489:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.666492:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.0:1619677448.666492:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677448.666493:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677448.666494:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.666495:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:31.0:1619677448.666496:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677448.666496:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677448.666496:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:30.0:1619677448.666497:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.666498:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:8.0:1619677448.666499:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.666499:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.666500:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.666501:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.666502:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.666503:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.666503:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.666504:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.666504:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.666505:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.666506:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677448.666506:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.666507:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.666508:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.666509:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.666510:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.666510:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.666511:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677448.666512:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.666512:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.666516:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:8.0:1619677448.666516:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.666519:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.666519:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.666520:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.666520:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677448.666520:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.666521:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.666522:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.666523:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677448.666523:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.666523:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:30.0:1619677448.666524:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:8.0:1619677448.666524:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.666525:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.666525:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.666526:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677448.666526:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.666526:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:30.0:1619677448.666527:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677448.666527:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.2:1619677448.666528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:8.0:1619677448.666528:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.666528:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677448.666528:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.666529:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.666530:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.2:1619677448.666532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.666532:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:9.2:1619677448.666533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.666533:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677448.666535:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:20.2:1619677448.666537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677448.666537:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677448.666538:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.666540:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:12.0:1619677448.666540:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.666540:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677448.666541:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.666542:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677448.666542:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:20.0:1619677448.666543:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:13.0:1619677448.666543:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:31.0:1619677448.666544:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:20.0:1619677448.666544:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:13.0:1619677448.666544:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.666544:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.666544:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:31.0:1619677448.666545:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.666545:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677448.666545:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:31.0:1619677448.666546:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:13.0:1619677448.666546:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.666546:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.666548:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677448.666548:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.666550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:25.2:1619677448.666551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.666554:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.666555:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677448.666557:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.666557:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.666558:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.666559:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.666560:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:12.0:1619677448.666560:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.666560:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:12.0:1619677448.666561:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.666561:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.666562:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.666563:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.666643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.666649:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.666652:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.666653:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.666655:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.666658:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.666659:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.666660:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711916] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.666662:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.666663:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.666664:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.666667:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.666669:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.666672:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.666673:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.666674:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.666675:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.666676:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.666676:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.666677:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.666681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.666688:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.666690:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.666691:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.666693:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.666694:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.666696:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.666700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:13.0:1619677448.666707:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.666710:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.666711:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.666711:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.666712:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.666713:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.666778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.666786:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.666788:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.666789:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.666791:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.666793:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.666795:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.666796:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.666798:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.666798:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.666800:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.666803:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.666804:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:4.2:1619677448.666806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677448.666807:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.666808:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.666809:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.666810:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.666811:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.666812:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.666812:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:24.0:1619677448.666812:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.666815:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:20.2:1619677448.666815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:25.2:1619677448.666816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:8.0:1619677448.666816:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.666818:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.666821:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.666822:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:10.0:1619677448.666822:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:24.0:1619677448.666823:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96711914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.666824:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:24.0:1619677448.666825:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.666825:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.666826:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:24.0:1619677448.666826:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.666827:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:6.0:1619677448.666827:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.666828:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:10.0:1619677448.666828:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:6.0:1619677448.666828:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.666831:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.666831:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.666832:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.666833:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:24.0:1619677448.666834:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.666834:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.666835:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.666836:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.666837:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.666837:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:24.0:1619677448.666838:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.666838:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677448.666839:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677448.666840:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.666840:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.666841:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:10.0:1619677448.666841:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677448.666842:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677448.666843:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.666845:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.666846:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.666846:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.2:1619677448.666846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.666848:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.666849:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.666849:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.666850:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:17.0:1619677448.666854:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:8.2:1619677448.666854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677448.666856:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.666857:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.666858:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.666860:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.666862:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:17.0:1619677448.666862:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.0:1619677448.666864:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.666865:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.666867:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.666867:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.666870:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.666954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.666962:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.666964:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.666967:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.666968:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.666969:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.666971:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.666972:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.666973:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.666974:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.666975:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.666978:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.666980:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.666983:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.666984:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.666985:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.666987:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.666987:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.666988:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.666989:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.666993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.667000:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.667003:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.667004:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.667004:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.667006:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.667009:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.667117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.667123:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677448.667125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677448.667125:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.667126:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.667128:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.667131:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.667132:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:1.0:1619677448.667132:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.667134:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.667135:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:1.0:1619677448.667135:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.667136:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:1.0:1619677448.667137:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.667138:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.667140:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.667142:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.667142:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.667144:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:1.0:1619677448.667144:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.667145:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.667147:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.667147:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.667148:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.667148:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.667148:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:26.0:1619677448.667149:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.667150:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.667151:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.667151:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.667152:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677448.667153:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.667154:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:25.2:1619677448.667156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.667157:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.667158:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.667159:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.667160:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.667160:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.667161:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.667162:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.667162:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.667165:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:25.2:1619677448.667166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:12.0:1619677448.667166:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.667167:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.667170:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.667170:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:13.0:1619677448.667171:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.667174:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.667174:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.667176:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.667178:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.667178:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:21.2:1619677448.667188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.667194:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.667196:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.667197:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.667199:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.667201:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.667203:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.667204:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.667206:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.667207:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.667208:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.667211:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.667213:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.667216:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.667216:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.667217:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.667219:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.667219:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.667221:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.667222:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.667222:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677448.667225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.667228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.667233:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677448.667233:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677448.667235:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.667235:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.667236:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677448.667236:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.667237:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.667238:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.667239:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.667242:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.667246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:9.2:1619677448.667248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.667253:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.667255:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.667257:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.667258:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677448.667258:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.667259:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.667260:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.667260:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.667261:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.667261:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.667262:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:2.0:1619677448.667262:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.667264:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711908] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.667265:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.667267:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.667267:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.667271:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.667272:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.667274:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.667275:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.667275:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.667277:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:4.2:1619677448.667278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677448.667278:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.667278:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.667279:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.667283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.667285:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.667288:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.667289:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.667290:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.667291:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.667299:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:10.0:1619677448.667301:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.667302:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (22)--
00000400:00000200:10.0:1619677448.667302:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.667303:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.667305:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.667305:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.667306:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.667307:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:10.0:1619677448.667307:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.667308:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.667312:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.667314:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.667317:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.667318:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.667318:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.667320:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.667321:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.667322:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.667323:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.667324:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.667327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.667330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.667335:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.667337:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.667338:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.667339:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677448.667339:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.667339:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.667340:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.667341:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.667341:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.667344:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.667418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.667424:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.667427:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.667429:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677448.667430:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.667432:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.667433:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.667435:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.667436:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.667437:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.667438:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.667442:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.667444:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.667446:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.667447:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.667448:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.667449:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.667450:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.667451:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.667452:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.667453:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677448.667456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677448.667458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.667461:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.667463:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.667464:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.667464:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677448.667464:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.667466:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.667467:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.667468:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.667470:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677448.667472:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.667476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.667483:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.667485:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.667486:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.667487:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.667487:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.667488:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.667503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.667509:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.667512:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.667513:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.667514:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.667516:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.667518:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.667519:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.667521:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.667522:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.667523:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.667526:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.667528:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.667531:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.667532:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.667533:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.667535:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.667536:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.667536:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.667537:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.667541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.667549:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.667551:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.667552:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.667554:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.667555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:13.0:1619677448.667556:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.667557:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.667564:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.667567:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.667569:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.667572:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.667572:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.667573:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.667574:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.667576:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.667577:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.667578:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.667581:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.667583:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.667585:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.667586:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.667587:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.667588:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.667589:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.667590:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.667591:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.667594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.667600:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.667602:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.667603:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.667605:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.667606:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.667608:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.667764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677448.667767:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677448.667769:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677448.667771:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.667772:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677448.667774:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.667776:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677448.667777:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.667778:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.667779:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.667780:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677448.667783:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.667785:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.667788:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.667789:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.667789:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.2:1619677448.667790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677448.667791:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.667792:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677448.667792:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677448.667793:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:25.2:1619677448.667797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.667799:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.667802:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.667804:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:10.0:1619677448.667804:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.667805:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.667807:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.667807:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.667808:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.667809:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.667810:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.667812:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.667813:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.667814:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.667817:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.667818:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.667818:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.667819:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.667820:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.667823:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.667824:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.667824:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.667826:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.667826:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.667827:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.667827:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.667832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.667840:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.667843:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.667843:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.667844:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.667844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:22.0:1619677448.667846:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.667849:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:26.0:1619677448.667852:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:25.2:1619677448.667854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.667855:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.667856:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.667857:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.667860:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.667861:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.667862:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.667862:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:26.0:1619677448.667864:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.667864:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.667865:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.667865:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.667865:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.667865:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.667867:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.667867:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:26.0:1619677448.667869:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.667871:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.667873:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.667874:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.667874:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.667876:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.667877:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.667878:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.667878:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.667882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.2:1619677448.667883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677448.667889:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677448.667889:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.667892:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:17.0:1619677448.667892:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:22.0:1619677448.667893:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.667893:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677448.667893:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.667895:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.667896:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.667898:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.667899:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:17.0:1619677448.667900:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.667901:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.667902:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.667903:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.667904:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.667908:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.667910:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.667912:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.667913:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.667914:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.667915:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.667916:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.667917:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.667917:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.667921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:4.2:1619677448.667924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:6.0:1619677448.667928:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.0:1619677448.667931:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677448.667931:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.667932:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.667932:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.667934:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.667935:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.667936:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677448.667937:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:8.0:1619677448.667938:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.667941:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.667943:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.667944:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.667945:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.667946:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.667947:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.667951:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.667953:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.667956:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.667957:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.667957:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.667959:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.667960:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.667961:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.667962:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.667963:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.667965:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.667968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.667973:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.667975:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.667976:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.667976:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.667977:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.667978:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.667978:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677448.667979:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.667980:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677448.667982:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.668074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.668079:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.668081:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.668084:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.668084:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.668087:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.668088:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.668089:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.668091:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.668092:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.668093:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.668096:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.668098:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.668101:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.668102:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.668102:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.668105:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.668105:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.668106:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.668107:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.668111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.668119:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.668122:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.668123:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.668123:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.668125:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.668127:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.668139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.668145:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.668147:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.668148:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.668150:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.668152:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.668153:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.668155:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.668156:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.668157:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.668158:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.668161:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.668163:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.668165:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.668166:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.668166:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.668167:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.668168:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.668168:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.668169:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.668173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.668180:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.668182:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.668183:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.668184:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.668184:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.668186:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.668229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.668237:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.668240:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.668242:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.668243:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.668244:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.668246:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677448.668247:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.668249:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.668250:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.668251:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677448.668254:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.668256:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677448.668259:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.668259:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.668260:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.668261:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.668262:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.668263:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.668264:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.668267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.668274:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.668276:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.668277:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.668277:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.668279:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.668282:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.668302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.668309:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.668312:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.668312:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.668314:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.668317:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.668318:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.668320:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711895] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.668321:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.668322:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.668323:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.668326:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.668327:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.668330:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.668331:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.668331:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.668333:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.668333:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.668334:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.668335:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.668340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.668348:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.668351:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.668352:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.668352:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.668354:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.668357:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.668358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.668365:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.668367:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.668368:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.668368:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.668369:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:21.2:1619677448.668370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.668372:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677448.668373:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.668374:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.668375:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.668377:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.668379:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.668381:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.668382:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.668383:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.668384:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.668385:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.668388:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.668390:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.668392:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.668393:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.668394:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.668396:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.668396:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.668397:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.668397:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.668403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.668411:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.668413:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.668414:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.668414:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.668417:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.668419:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.668490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.668498:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.668501:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.668501:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.668504:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.668507:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.668508:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.668509:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.668510:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.668511:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.668512:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.668515:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.668517:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.668519:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.668519:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.668520:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.668521:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.668522:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.668522:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.668523:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.668527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.668533:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.668535:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.668535:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.668536:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.668537:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.668538:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.668564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.668572:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.668575:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.668577:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.668579:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.668580:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.668581:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.668582:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.668583:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.668584:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.668585:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.668588:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.668590:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.668593:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.668594:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.668594:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.668596:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.668596:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.668597:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.668598:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.668609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.668617:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.668619:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.668620:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.668622:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.668624:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.668625:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.668628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.668635:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.668638:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.668639:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.668640:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.668640:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.668641:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.668795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.668801:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.668804:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.668805:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.668807:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.668809:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.668810:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.668812:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.668813:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.668814:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.668815:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.668818:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.668820:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.668821:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.668822:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.668823:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.668825:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.668825:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.668826:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.668828:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.668828:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.668831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.668833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.668840:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.668840:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.668841:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.668841:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.668841:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.668842:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.668843:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.668843:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.668844:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677448.668845:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.668912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.668918:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.668921:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.668922:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.668924:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.668927:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.668928:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.668929:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.668931:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.668932:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.668933:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:21.2:1619677448.668935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:24.0:1619677448.668936:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.668938:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.668941:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.668941:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677448.668942:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.668942:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.668943:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:24.0:1619677448.668944:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.668944:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677448.668944:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677448.668945:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.668946:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677448.668946:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.668949:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.668950:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:8.2:1619677448.668950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:22.0:1619677448.668951:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.668952:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.668953:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.668954:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.668957:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.668957:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677448.668959:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677448.668960:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.668961:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.668962:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.668962:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.668963:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.668963:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:1.0:1619677448.668963:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.668965:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.668966:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.668966:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677448.668966:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677448.668967:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.668971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.668977:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.668980:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.668981:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.668981:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.668984:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.668986:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:25.2:1619677448.668991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000800:00000200:10.0:1619677448.668997:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.669000:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.669000:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.669001:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.669001:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.669002:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677448.669013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.669019:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.669022:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.669023:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.669024:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.669027:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.669029:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.669030:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.669031:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.669032:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.669033:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.669036:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.669038:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.669039:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.669040:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.669041:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.669042:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.669042:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.669043:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.669043:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.669048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.669055:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.669057:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.669057:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.669058:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.669059:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.669061:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.669066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.669073:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.669075:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.669076:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.669076:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.669077:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.669077:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.669156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.669164:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.669167:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.669169:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.669171:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.669173:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.669174:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677448.669175:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.669176:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.669178:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.669179:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677448.669182:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.669183:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677448.669186:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.669186:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.669187:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.669188:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.669189:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.669190:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.669192:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.669192:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677448.669194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:21.2:1619677448.669198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677448.669200:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.669203:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.669203:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.669204:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.669206:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677448.669206:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.669208:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677448.669208:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:30.0:1619677448.669209:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.669210:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677448.669390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.669399:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.669401:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.669403:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.669406:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.669406:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.669407:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.669408:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.669409:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.669410:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.669411:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.669414:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.669416:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.669420:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.669421:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.669421:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.669423:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.669424:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.669424:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.669425:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.669430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.669433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.669437:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.669439:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.669439:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.669439:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.669440:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.669441:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.669443:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.669444:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.669445:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:5.0:1619677448.669445:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.669446:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.669447:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.669448:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.669449:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.669450:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.669451:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.669455:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.669456:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.669459:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.669459:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.669460:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.669462:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.669462:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.669463:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.669464:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.669469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.669478:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.669481:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.669482:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.669482:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.669484:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.669486:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.669543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.669550:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.2:1619677448.669550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.669553:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.669554:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.669556:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.669557:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.669557:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.669559:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:17.0:1619677448.669559:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:26.0:1619677448.669560:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.669560:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.669562:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.669563:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.669563:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.669564:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677448.669565:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.669566:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.669568:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.669568:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.669569:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.669570:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.669571:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:17.0:1619677448.669571:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.669574:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.669575:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:17.0:1619677448.669575:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.669576:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.669577:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.669578:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.669578:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.669579:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.669579:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.669580:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677448.669580:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.669581:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.669582:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.669582:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.669583:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.669584:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.669584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.2:1619677448.669588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:18.0:1619677448.669593:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.669595:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:1.0:1619677448.669595:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:2.0:1619677448.669596:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.669597:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.669598:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.669598:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.669599:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.669599:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.669601:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.669603:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677448.669603:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.669606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.2:1619677448.669607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.669614:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.669615:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.669617:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.669618:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.669618:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.669619:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.669619:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.669620:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.669620:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:10.0:1619677448.669620:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.669623:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.669625:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.669626:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.669627:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.669628:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.669630:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.669633:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.669635:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.669637:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.669638:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.669639:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.669640:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.669641:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.669642:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.669643:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.669644:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.669646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.669649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.669654:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677448.669656:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:12.0:1619677448.669656:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.669657:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.669658:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:12.0:1619677448.669658:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.669659:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.669660:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677448.669660:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.669662:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.669665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.669671:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.669673:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.669674:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.669675:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.669675:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.669676:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.669852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.669859:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.669861:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.669862:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.669864:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.669867:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.669869:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.669870:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.669871:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.669872:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.669873:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.669876:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.669878:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.669880:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.669881:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.669882:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.669883:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.669884:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.669884:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.669885:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.669889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.669897:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.669900:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.669900:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.669901:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.669903:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.669905:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.670037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.670045:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.670048:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.670050:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.670051:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.670053:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.670054:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.670056:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.670057:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.670058:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.670059:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.670062:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.670064:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.670066:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.670067:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.670067:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.670069:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.670069:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.670070:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.670071:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677448.670072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:25.2:1619677448.670074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:17.0:1619677448.670077:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.670080:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.670080:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.670081:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:17.0:1619677448.670082:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.670083:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.670084:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:17.0:1619677448.670084:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.670084:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.670085:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.670087:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.670087:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.670088:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.670089:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.670090:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.670090:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:17.0:1619677448.670094:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.670095:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.670097:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.670098:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.670099:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.670100:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.670101:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.670102:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.670102:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.670106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.670110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:1.0:1619677448.670113:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.670115:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:2.0:1619677448.670116:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:1.0:1619677448.670116:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.670117:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.670118:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.670119:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.670119:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:1.0:1619677448.670120:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:2.0:1619677448.670122:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.670123:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.670124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000400:00000200:2.0:1619677448.670125:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.670126:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.670127:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.670128:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.670129:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.670132:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677448.670132:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.670134:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.670135:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.670136:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677448.670136:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.670136:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.670137:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.670137:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677448.670137:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.670138:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.670139:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.2:1619677448.670140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677448.670140:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.670141:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.670142:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.670143:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:9.2:1619677448.670145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.670148:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.2:1619677448.670148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:30.0:1619677448.670151:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.670153:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.670154:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677448.670155:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.670155:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.670155:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.670155:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:18.0:1619677448.670156:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.670156:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.670157:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:30.0:1619677448.670158:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:2.0:1619677448.670158:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:30.0:1619677448.670159:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.670159:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.670159:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677448.670160:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:30.0:1619677448.670161:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.670162:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.670163:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.670167:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.670168:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.670172:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.670173:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.670174:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.670175:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.670176:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.670177:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.670178:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.670182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.670190:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.670192:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.670193:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.670195:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.670196:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.670198:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.670199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.670206:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.670207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:5.0:1619677448.670209:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.670210:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.670211:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.670211:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.670212:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:26.0:1619677448.670215:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.670217:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.670218:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.670220:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.670222:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.670224:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.670225:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.670227:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.670228:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.670229:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.670232:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.670234:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.670236:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.670237:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.670238:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.670239:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.670240:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.670241:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677448.670242:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.670242:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677448.670245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677448.670248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:1.0:1619677448.670252:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:10.0:1619677448.670255:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:1.0:1619677448.670255:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.670256:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.670256:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.670257:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.670258:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:1.0:1619677448.670258:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:26.0:1619677448.670259:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:1.0:1619677448.670261:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.670277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.670282:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.670285:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.670285:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.670288:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.670290:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.670291:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.670293:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.670299:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.670300:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.670301:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.670305:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.670306:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.670309:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.670310:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.670310:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.670312:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.670313:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.670313:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.670314:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.670319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.670326:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.670329:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.670330:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.670330:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.670332:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.670332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:17.0:1619677448.670335:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677448.670341:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.670343:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.670344:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.670346:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.670349:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.670350:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.670352:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711874] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.670353:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.670354:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.670355:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.670358:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.670359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677448.670360:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.670362:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.670363:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.670364:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.670365:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.670365:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.670366:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.670366:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.670367:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.0:1619677448.670367:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.670369:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.670370:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.670371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677448.670371:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.670373:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.670374:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.670375:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.670376:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.670377:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.670379:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:4.2:1619677448.670379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:5.0:1619677448.670380:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.670382:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.670382:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:13.0:1619677448.670383:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.670384:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.670385:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.670386:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677448.670386:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.670387:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.670387:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:20.2:1619677448.670388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:13.0:1619677448.670388:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.670389:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.670389:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677448.670390:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.670390:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.670390:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.670391:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677448.670392:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.670392:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:5.0:1619677448.670392:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677448.670393:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.670394:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.670395:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.670395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:26.0:1619677448.670396:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.670397:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.670398:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.670399:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.670400:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.670401:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.670401:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:26.0:1619677448.670402:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.670402:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.670403:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.670404:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:12.0:1619677448.670404:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:20.0:1619677448.670405:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:12.0:1619677448.670405:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.670407:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.670408:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:26.0:1619677448.670410:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.670410:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.670410:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.670413:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.670414:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.670416:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.670417:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.670417:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.670417:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.670418:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.670418:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.670420:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.670420:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.670421:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.670421:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:26.0:1619677448.670422:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:20.0:1619677448.670422:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:26.0:1619677448.670423:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:20.0:1619677448.670423:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:26.0:1619677448.670424:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.670426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:9.2:1619677448.670426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677448.670429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.670435:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:10.0:1619677448.670436:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:2.0:1619677448.670436:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677448.670438:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.670439:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.670439:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.670439:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677448.670439:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:10.0:1619677448.670440:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:2.0:1619677448.670440:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.670441:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.670441:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677448.670442:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:2.0:1619677448.670443:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.670444:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:2.0:1619677448.670446:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.670477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.670482:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.670485:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.670486:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.670488:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.670490:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.670491:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.670492:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.670494:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.670495:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.670496:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.670499:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.670501:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.670503:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.670504:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.670504:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.670505:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.670506:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.670507:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.670507:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.670511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.670519:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.670521:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.670521:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.670522:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.670523:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.670524:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.670542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.670549:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.670552:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.670554:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.670556:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.670558:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.670559:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.670560:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711869] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.670562:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.670563:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.670564:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.670567:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.670569:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.670572:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.670572:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.670573:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.670574:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.670575:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.670576:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.670578:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.670578:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.670580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.670584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677448.670587:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.670589:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.670590:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.670591:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.670592:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:21.2:1619677448.670593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:13.0:1619677448.670593:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.670593:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.670594:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.670595:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.670596:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:18.0:1619677448.670599:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.670601:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.670602:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.670604:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.670606:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.670607:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.670609:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.670610:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.670611:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.670612:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.670615:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.670617:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.670619:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.670620:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.670621:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.670622:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.670623:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.670624:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.670624:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.670628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.670636:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.670639:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.670640:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.670640:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.670643:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677448.670645:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.670734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.670740:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.670742:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.670745:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.670745:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.670748:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.670749:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.670751:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.670752:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.670753:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.670754:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.670758:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.670759:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.670763:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.670764:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.670764:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.670766:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.670767:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.670768:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.670769:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.670770:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.670772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.670775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.670778:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677448.670779:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.670780:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.2:1619677448.670780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677448.670781:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677448.670781:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677448.670782:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.670782:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:31.0:1619677448.670784:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.670785:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.670785:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.670787:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:1.0:1619677448.670788:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:25.2:1619677448.670789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.670789:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.670791:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.670793:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.670794:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:30.0:1619677448.670795:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:1.0:1619677448.670796:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.670797:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.670798:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.670799:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.670799:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:1.0:1619677448.670799:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.670800:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.670801:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.670802:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:1.0:1619677448.670803:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.670804:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.670808:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.670808:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.670809:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.670811:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.670811:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.670813:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.670814:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.670814:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.670816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.670819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:4.2:1619677448.670821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:12.0:1619677448.670823:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677448.670825:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677448.670826:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:4.2:1619677448.670826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677448.670827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fdae00] (20)++
00000800:00000200:17.0:1619677448.670827:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.670827:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677448.670827:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.670828:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677448.670828:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:12.0:1619677448.670829:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.670830:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.670831:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677448.670831:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.670832:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677448.670833:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.670833:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fdae00] (21)++
00000400:00000200:8.0:1619677448.670833:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.670836:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:20.0:1619677448.670836:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.38@o2ib3
00000400:00000200:8.0:1619677448.670836:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.670837:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.670838:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (22)--
00000400:00000200:20.0:1619677448.670839:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.38@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.670839:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.670840:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.670841:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.670842:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.670842:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.670843:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.670844:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.670844:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677448.670844:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677448.670846:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677448.670846:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.670847:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.670848:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.670849:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.670850:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.670850:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.670851:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.670852:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677448.670852:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677448.670854:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.670857:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.38@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.670858:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.670860:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677448.670861:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.670862:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.0:1619677448.670862:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.670863:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.670863:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.670864:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.670864:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:26.0:1619677448.670865:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.670865:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.670865:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.670865:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677448.670866:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:8.0:1619677448.670866:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677448.670867:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:8.0:1619677448.670867:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:26.0:1619677448.670868:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.670868:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:26.0:1619677448.670869:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677448.670869:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fdae00] (21)++
00000800:00000200:26.0:1619677448.670870:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677448.670870:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fdae00] (22)--
00000800:00000200:5.2:1619677448.670870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:20.0:1619677448.670871:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fdae00] (21)--
00000800:00000200:8.2:1619677448.670871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:25.2:1619677448.670874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:9.2:1619677448.670874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:2.0:1619677448.670876:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.670878:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.670879:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677448.670879:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:10.0:1619677448.670880:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:2.0:1619677448.670881:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.670881:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677448.670882:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677448.670882:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:10.0:1619677448.670882:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:1.0:1619677448.670882:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.670883:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.670883:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.670884:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:2.0:1619677448.670884:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.670885:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.670885:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:1.0:1619677448.670885:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.670886:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.670886:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.670888:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.670888:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96711862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.670888:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677448.670889:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:2.0:1619677448.670889:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.670890:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.670891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000400:00000200:2.0:1619677448.670891:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.670892:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:2.0:1619677448.670895:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.670897:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.670897:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.670898:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.670899:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.670900:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.670900:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.670900:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.670901:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.670901:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.670902:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:2.0:1619677448.670902:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.670903:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.670904:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.670904:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.670908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.670915:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.670917:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.670918:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.670919:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.670920:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.670922:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.670928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.2:1619677448.670933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677448.670935:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.670937:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.670938:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.670938:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.670939:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.670939:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677448.670940:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.670943:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.670945:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.670947:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.670948:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.670949:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.670950:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.670952:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.670953:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.670954:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.670957:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.670959:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.670963:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.670963:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.670964:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.670966:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.670966:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.670967:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.670968:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.670972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.670980:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.670983:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.670984:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.670986:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.670986:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.670989:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.671025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.671030:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.671033:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.671034:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.671036:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.671038:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.671040:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.671041:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.671042:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.671043:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.671044:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.671048:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.671049:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.671052:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.671053:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.671054:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.671056:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.671056:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.671058:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.671059:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.671059:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:25.2:1619677448.671062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:21.2:1619677448.671064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677448.671069:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:6.0:1619677448.671069:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:30.0:1619677448.671072:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:6.0:1619677448.671072:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:30.0:1619677448.671073:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.671073:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:6.0:1619677448.671073:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.2:1619677448.671074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677448.671075:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.671076:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.671078:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677448.671079:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.671082:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.671083:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.671085:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.671087:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.671089:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.671090:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.671092:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.671093:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.671094:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.671097:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.671098:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.671101:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.671102:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.671103:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.671105:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.671105:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.671106:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.671106:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.671111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.671118:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.671121:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.671122:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.671123:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.671124:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.671126:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.671128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.671130:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.671133:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.671135:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.671136:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.671138:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.671139:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.671141:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.671142:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.671143:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.671144:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.671148:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.671150:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.671152:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.671153:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.671154:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.671156:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.671157:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.671158:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.671158:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.671162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.671168:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.671171:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.671172:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.671174:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.671175:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.671177:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.671269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.671275:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.671278:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.671279:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.671280:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.671282:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.671284:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.671285:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.671286:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.671287:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.671288:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.671291:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.671297:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.671301:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.671301:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.671302:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.671304:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.671304:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.671305:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.671305:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.671310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.671318:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.671321:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.671322:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.671323:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.671325:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.671326:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.671418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.671426:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.671429:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.671431:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.671434:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.671434:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.671435:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.671437:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.671438:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.671439:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.671440:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.671443:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.671445:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.671448:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.671449:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.671449:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.671451:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.671451:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.671452:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.671453:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.671456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:4.2:1619677448.671458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677448.671463:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.671465:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.671466:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.671466:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:12.0:1619677448.671467:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.671469:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.671469:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.671470:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:12.0:1619677448.671470:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677448.671471:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.671473:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.671474:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.671475:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.671477:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.671477:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.671478:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:25.2:1619677448.671480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:10.0:1619677448.671482:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.671483:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:13.0:1619677448.671486:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.671486:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.671487:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.2:1619677448.671488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677448.671488:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:13.0:1619677448.671489:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.671489:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:10.0:1619677448.671489:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:13.0:1619677448.671490:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.671490:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.671490:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.671490:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:13.0:1619677448.671491:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.671491:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.671494:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.2:1619677448.671496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.671496:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.671499:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.671500:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.671502:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:5.0:1619677448.671502:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.671503:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.671503:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677448.671504:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.671505:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.671506:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.671506:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.671508:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.671509:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677448.671512:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.671514:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.671515:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.671516:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677448.671516:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.671517:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.671518:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.671519:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.671520:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:4.2:1619677448.671520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:5.0:1619677448.671521:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.671521:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.671526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.671532:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677448.671533:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.0:1619677448.671534:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.671535:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677448.671536:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.671536:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.671538:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.671538:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.671539:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.671540:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:8.0:1619677448.671540:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.671542:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.671543:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.671544:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.671546:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.671547:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.671550:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.671552:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.2:1619677448.671554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.0:1619677448.671555:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.671555:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.671556:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.671558:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.671558:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.671559:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.671561:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:26.0:1619677448.671562:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677448.671562:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677448.671564:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:8.2:1619677448.671564:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.671565:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.671567:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.671567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677448.671570:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.671571:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.671572:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.671572:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:26.0:1619677448.671573:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.671574:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.671574:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.671574:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:26.0:1619677448.671575:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.671575:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.671576:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677448.671576:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.671577:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.671578:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:26.0:1619677448.671579:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.671579:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.671582:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.671582:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:26.0:1619677448.671585:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.671586:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.671587:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.671589:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.671589:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.671590:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.671590:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.671596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.671604:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.671606:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.671607:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.671608:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.671610:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.671613:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.671745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.671751:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.671754:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.671755:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.671757:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.671759:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.671761:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.671762:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.671764:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.671765:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.671766:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.671769:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.671771:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.671773:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.671774:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.671775:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.671776:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.671777:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.671777:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.671778:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.2:1619677448.671781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.2:1619677448.671783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.671789:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.671790:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.671792:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:17.0:1619677448.671793:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.671794:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.671795:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.671797:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.671797:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.671799:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.671800:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.671801:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.671802:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.671803:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.671804:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.671805:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.671806:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:17.0:1619677448.671807:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.671808:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.671809:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.671809:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.671810:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.671810:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.671811:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677448.671813:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.671814:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.671815:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.671817:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.671817:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.671818:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.671818:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.671823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.671829:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.671832:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.671833:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.671834:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.671836:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.671837:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.671840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:25.2:1619677448.671843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.671848:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.671849:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.671851:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:13.0:1619677448.671852:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.671852:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:30.0:1619677448.671853:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.671853:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:31.0:1619677448.671854:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677448.671854:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:30.0:1619677448.671855:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.671855:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:30.0:1619677448.671857:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.671858:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.671860:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.671861:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.671862:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.671865:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.671866:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.671870:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.671870:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.671871:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.671873:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.671873:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.671874:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.671875:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.671879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.671886:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.671890:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.671890:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.671891:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.671893:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.671896:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.671950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.671956:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.671959:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.671960:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.671962:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.671964:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.671966:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.671967:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.671968:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.671969:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.671970:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.671973:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.671975:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.671978:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.671979:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.671979:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.671981:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.671982:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.671983:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.671984:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.671984:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.671987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.671990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.671994:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:2.0:1619677448.671995:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:10.0:1619677448.671996:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.671997:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.671997:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.671997:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.671998:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.671999:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:10.0:1619677448.672000:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.672003:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.672076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.672083:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.672085:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.672086:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.672088:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.672091:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.672092:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.672093:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.672094:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.672095:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.672096:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.672099:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.672101:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.672104:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.672105:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.672105:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.672107:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.672107:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.672108:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.672109:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.672113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.672120:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.672123:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.672124:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.672125:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.672127:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.672130:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.672132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.672139:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.672141:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.672142:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.672142:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.672143:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.672144:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.672197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.672205:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.672208:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.672210:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.672211:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.672213:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.672214:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.672215:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.672217:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.672218:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.672219:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.672222:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.672222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:31.0:1619677448.672223:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.672225:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.672226:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.672227:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.672228:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.672228:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:31.0:1619677448.672229:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.672230:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:1.0:1619677448.672231:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:31.0:1619677448.672232:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.672232:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677448.672232:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677448.672234:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.672235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677448.672237:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.672238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677448.672238:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677448.672239:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.672241:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677448.672241:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.672242:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.672243:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:12.0:1619677448.672244:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:30.0:1619677448.672245:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:12.0:1619677448.672245:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.672246:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677448.672247:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.672248:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.672248:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677448.672248:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.672248:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.672249:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677448.672249:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.672251:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.672252:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:1.0:1619677448.672253:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.672255:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677448.672255:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677448.672256:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677448.672257:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677448.672258:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677448.672261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.672263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677448.672269:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:5.0:1619677448.672269:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677448.672271:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677448.672271:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677448.672272:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:5.0:1619677448.672272:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.672273:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.672274:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.672275:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.672277:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.672278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.672286:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.672289:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.672290:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.672290:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.672291:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.672292:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.672382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.672388:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.672391:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.672391:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.672393:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.672395:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.672397:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.672398:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.672399:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.672400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:1.0:1619677448.672400:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.672400:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.672403:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:1.0:1619677448.672403:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.672405:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.672406:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.672408:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:1.0:1619677448.672408:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.672409:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:1.0:1619677448.672409:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.672409:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:20.0:1619677448.672410:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.672411:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:20.0:1619677448.672412:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:1.0:1619677448.672412:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:20.0:1619677448.672413:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.672413:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.672413:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:20.0:1619677448.672414:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.672415:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.672416:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:25.2:1619677448.672418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:20.0:1619677448.672419:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.672421:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677448.672424:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.672425:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677448.672425:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:13.0:1619677448.672425:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.0:1619677448.672427:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677448.672428:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.672428:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:13.0:1619677448.672428:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:20.0:1619677448.672429:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:13.0:1619677448.672429:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.672431:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.672432:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:25.2:1619677448.672433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:13.0:1619677448.672433:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:30.0:1619677448.672441:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.672444:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.672445:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.672446:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677448.672448:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.672450:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.672452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.672459:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.672461:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.672462:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.672463:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.672464:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.672464:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.672491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.672498:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.672501:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.672501:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:21.2:1619677448.672502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:18.0:1619677448.672504:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.672506:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.672508:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.672509:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711842] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.672509:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:18.0:1619677448.672510:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.672511:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.672512:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677448.672513:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.672513:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:18.0:1619677448.672517:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.672517:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.672519:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:8.0:1619677448.672520:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.672522:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:18.0:1619677448.672523:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.672523:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:8.0:1619677448.672523:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96711841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.672524:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:8.0:1619677448.672525:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.672526:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:8.0:1619677448.672526:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.672527:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.672527:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:8.0:1619677448.672527:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.672528:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:8.0:1619677448.672530:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.672532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677448.672532:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.672535:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.672536:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.672537:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677448.672538:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677448.672538:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.672539:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.672539:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.672540:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677448.672541:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.672542:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.672543:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.672544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677448.672545:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.672546:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.672552:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.672555:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.672556:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.672556:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.672558:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.672561:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.672623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.672630:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:21.2:1619677448.672632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.672633:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.672635:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.672637:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.672637:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.672638:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:31.0:1619677448.672639:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:30.0:1619677448.672639:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.672641:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.672642:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.672643:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.672644:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:4.2:1619677448.672645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:30.0:1619677448.672647:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.672648:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.672650:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.672650:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.672651:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:24.0:1619677448.672652:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:30.0:1619677448.672653:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.672654:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.672655:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.672655:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:30.0:1619677448.672656:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.672656:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.672658:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.672659:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.672660:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:30.0:1619677448.672660:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.672661:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:24.0:1619677448.672661:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.672662:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.672662:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.672663:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:24.0:1619677448.672663:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:31.0:1619677448.672664:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.672664:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:24.0:1619677448.672664:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96711838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.672665:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.672666:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.672666:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.672666:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:24.0:1619677448.672666:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.672666:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.672668:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.672668:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:24.0:1619677448.672668:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.2:1619677448.672668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.672669:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.672670:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:24.0:1619677448.672672:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.672674:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.2:1619677448.672674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.672677:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:24.0:1619677448.672678:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.672678:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.672679:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (23)++
00000800:00000200:5.0:1619677448.672680:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:24.0:1619677448.672681:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (24)--
00000800:00000200:24.0:1619677448.672681:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:5.0:1619677448.672681:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.672682:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.672682:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677448.672682:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000400:00000200:5.0:1619677448.672682:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.672683:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:17.0:1619677448.672685:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000800:00000200:5.0:1619677448.672685:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:17.0:1619677448.672686:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.672686:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (24)++
00000800:00000200:1.0:1619677448.672687:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000400:00000200:17.0:1619677448.672688:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.672688:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.672689:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.672693:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.672693:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.672694:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.672716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.672723:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.672726:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.672727:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.672729:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.672731:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.672733:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.672734:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711837] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.672735:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.672736:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.672737:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.672740:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.672741:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.672744:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.672745:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.672745:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.672747:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.672748:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.672749:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.672750:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.672750:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.672754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:4.2:1619677448.672756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.672762:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.2:1619677448.672762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:6.0:1619677448.672765:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.672766:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.672766:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.672767:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.672768:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677448.672768:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.672769:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.672771:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.672771:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.672771:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:26.0:1619677448.672772:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.672773:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.672774:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.672775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000400:00000200:26.0:1619677448.672777:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.672779:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.672780:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.672781:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.672781:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000400:00000200:26.0:1619677448.672782:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.672783:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.672784:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.672784:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677448.672784:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000400:00000200:26.0:1619677448.672785:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.672785:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.672786:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:26.0:1619677448.672787:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.672789:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.672790:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.672790:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.672792:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.672792:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.672793:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.672794:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.672797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.672804:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.672806:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.672807:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.672808:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.672810:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.672811:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.672853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.672859:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677448.672860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:1.0:1619677448.672861:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.672862:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.672864:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.672865:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.672867:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:1.0:1619677448.672867:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.672868:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:6.0:1619677448.672869:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677448.672870:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.672871:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.672872:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.672873:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.672874:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.672874:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.672878:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.672880:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:1.0:1619677448.672880:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:22.0:1619677448.672882:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.672882:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.672883:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.672883:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:22.0:1619677448.672884:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.672884:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:22.0:1619677448.672885:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:1.0:1619677448.672886:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.672886:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.672887:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:22.0:1619677448.672889:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.672889:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.672889:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.672891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.672891:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.672893:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.672894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.672895:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.672895:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.672898:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.672899:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:17.0:1619677448.672899:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677448.672900:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.672900:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.672901:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677448.672901:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677448.672901:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.672902:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677448.672902:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677448.672902:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.672904:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.672904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:6.0:1619677448.672904:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.672907:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.672908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677448.672912:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.672915:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:22.0:1619677448.672916:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:18.0:1619677448.672916:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.672916:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.672918:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.672919:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.672920:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.672920:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.672921:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.672921:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:22.0:1619677448.672922:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.673018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.673026:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.673029:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.673031:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.673032:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.673034:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.673036:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677448.673037:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.673038:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.673039:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.673040:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.673044:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.673046:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:21.2:1619677448.673047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:31.0:1619677448.673048:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.673049:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.673050:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.673052:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.673052:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.673053:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.673054:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677448.673054:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.673057:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.2:1619677448.673057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:24.0:1619677448.673060:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.673060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:24.0:1619677448.673063:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.673063:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677448.673063:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.673065:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:1.0:1619677448.673065:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677448.673066:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:24.0:1619677448.673068:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96711832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.673068:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.673069:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.673069:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677448.673069:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:24.0:1619677448.673070:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.673070:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.673070:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.673072:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:20.0:1619677448.673072:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.673073:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677448.673073:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.673075:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96711831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.673076:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.673076:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.673076:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:20.0:1619677448.673077:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.673079:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:20.0:1619677448.673079:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.673082:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.673082:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.673083:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:20.0:1619677448.673083:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.673085:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.673085:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677448.673085:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.673086:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.673086:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677448.673087:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.673088:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.673089:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:25.2:1619677448.673090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.673091:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.673091:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.673092:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.673092:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.673097:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.2:1619677448.673097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:12.0:1619677448.673100:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.673100:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.673102:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.673103:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.673104:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677448.673105:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.673107:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.673108:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.673109:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.673110:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.673111:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.673144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.673151:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.673154:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.673154:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.673156:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.673159:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.673161:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.673162:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711830] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.673164:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.673165:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.673166:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.673169:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.673171:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.673174:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.673175:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.673175:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.673177:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.673178:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.673178:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.673179:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.673183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.673191:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.673194:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.673194:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.673195:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.673197:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.673200:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.673296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677448.673303:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.673306:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677448.673307:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677448.673309:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.673312:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.673313:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.673315:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.673316:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.673317:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.673318:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.673321:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.673322:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.673324:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.673325:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.673325:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.673327:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.673327:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.673328:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677448.673328:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677448.673333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.673341:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.673343:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:5.2:1619677448.673343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:30.0:1619677448.673344:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.673345:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.673348:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.673348:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.673350:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.673353:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.673354:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.673355:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.673358:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.673359:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.673360:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.673361:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.673362:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.673363:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.673366:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.673367:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.673369:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.673370:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.673370:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.673372:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.673372:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.673373:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.673374:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.673375:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.673377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.673380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677448.673385:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.673387:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:26.0:1619677448.673388:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:13.0:1619677448.673388:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.673389:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:13.0:1619677448.673389:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.673390:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.673391:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.673391:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.673392:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.673395:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.673396:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.673396:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.673397:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.673398:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.673484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.673490:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.673493:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.673494:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.673496:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.673498:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.673499:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.673501:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.673502:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.673503:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.673504:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.673508:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.673509:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.673512:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.673513:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.673514:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.673515:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.673516:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.673517:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.673517:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.673544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.673551:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.673554:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.673554:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.673555:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.673557:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.673561:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677448.673640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:4.2:1619677448.673640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.673648:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.673649:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.673650:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:30.0:1619677448.673651:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:26.0:1619677448.673651:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.673654:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.673655:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.673655:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.673656:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.673657:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.673658:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.673659:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.673660:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.673661:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.673661:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.673663:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.673663:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.673664:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.673664:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.673665:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.673668:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.673669:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.673670:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.673671:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.673674:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.673675:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.673675:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.673675:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.673676:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.673676:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.673677:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.673677:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.673678:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.673678:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.673679:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.673679:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.673679:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677448.673680:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.673683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.2:1619677448.673684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677448.673690:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677448.673692:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:22.0:1619677448.673693:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:20.0:1619677448.673694:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:6.0:1619677448.673694:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.673695:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.673695:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.673698:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.673698:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.673698:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:22.0:1619677448.673700:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677448.673701:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.673769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.673777:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.673780:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.673782:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.673783:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.673785:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.673786:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677448.673788:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.673789:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.673790:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.673791:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.673794:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.673796:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.673799:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.673800:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.673800:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.673802:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.673803:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.673804:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.673804:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677448.673804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.2:1619677448.673809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.673813:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.673816:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.673817:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:1.0:1619677448.673817:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:24.0:1619677448.673818:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.673820:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.673820:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.673821:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.673821:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.673822:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.673823:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96711823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.673824:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.673825:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.673826:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.673827:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.673827:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.673829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000400:00000200:24.0:1619677448.673832:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.673833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:24.0:1619677448.673834:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.673836:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.673836:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.673837:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.673838:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677448.673838:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677448.673839:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.673839:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.673840:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:17.0:1619677448.673840:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.673840:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.673841:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:17.0:1619677448.673841:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.673842:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:22.0:1619677448.673842:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:17.0:1619677448.673842:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:6.0:1619677448.673842:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:24.0:1619677448.673843:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:6.0:1619677448.673844:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.673845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:6.0:1619677448.673846:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.673848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000400:00000200:6.0:1619677448.673848:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.673849:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.673851:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:6.0:1619677448.673851:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.673852:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.673853:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:12.0:1619677448.673854:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.673854:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.673855:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:12.0:1619677448.673857:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.673857:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:6.0:1619677448.673857:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.673858:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.673858:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:12.0:1619677448.673859:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677448.673859:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:6.0:1619677448.673860:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.673862:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.673863:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.673864:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.673866:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.673866:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.673867:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.673868:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.673871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.673878:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.673880:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.673881:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.673882:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.673884:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.673887:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677448.673920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.673926:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.673928:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.673930:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:5.2:1619677448.673931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:2.0:1619677448.673931:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.673934:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.673935:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.673937:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.673938:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:2.0:1619677448.673938:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.673939:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.673941:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.673941:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:2.0:1619677448.673941:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.673944:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.673945:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000400:00000200:10.0:1619677448.673947:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.673948:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:2.0:1619677448.673948:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.673949:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.673950:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.673950:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:10.0:1619677448.673951:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.673951:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677448.673952:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.673952:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:10.0:1619677448.673953:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.673953:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.673954:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.673954:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:10.0:1619677448.673957:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.673958:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:9.2:1619677448.673958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.673961:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.673962:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.673962:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.673964:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.673964:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.673965:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677448.673966:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.673966:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:18.0:1619677448.673968:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:25.2:1619677448.673969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:18.0:1619677448.673969:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.673969:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.673971:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.673973:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677448.673976:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.673979:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.673980:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.673981:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.673983:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.673984:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.674090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.674097:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.674099:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.674100:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.674102:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.674104:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.674106:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.674107:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.674108:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.674109:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.674110:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.674114:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.674115:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.674118:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.674119:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.674120:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.674121:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.674122:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.674122:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.674122:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.674127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.674135:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.674137:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.674138:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.674139:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.674141:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.674143:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.674148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.674156:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.674158:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.674159:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.674159:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.674160:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.674161:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.674189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.674195:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.674197:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.674198:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.674200:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.674202:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.674203:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.674204:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.674205:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.674206:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.674207:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.674210:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.674212:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.674214:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.674215:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.674215:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.674217:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.674217:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.674218:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.674218:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.674222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.674229:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.674231:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.674231:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.674232:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.674233:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.674234:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.674390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.674397:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.674400:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.674401:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.674403:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.674406:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.674407:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.674408:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.674410:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.674411:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.674412:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.674415:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.674417:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.674420:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.674421:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.674422:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.674423:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.674424:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.674425:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.674425:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677448.674430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.674438:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.674441:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.674442:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.674443:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.674444:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.674447:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.674598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.674605:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.674608:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.674609:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.674611:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.674613:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.674615:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.674616:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.674617:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.674618:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.674619:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.674623:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.674624:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.674627:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.674628:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.674629:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.674631:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.674631:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.674631:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.674632:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.674636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.674644:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.674646:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.674647:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.674648:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.674650:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.674653:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.674666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.674673:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.674676:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.674676:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.674677:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.674678:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.674678:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.674749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.674757:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.674760:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.674761:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.674762:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.674765:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.674767:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.674768:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.674769:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.674770:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.674771:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.674774:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.674776:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.674778:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.674779:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.674779:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.674781:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.674782:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.674783:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.674783:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.674787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.674788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:12.0:1619677448.674794:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.674796:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:30.0:1619677448.674797:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:12.0:1619677448.674797:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.674798:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.674799:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:12.0:1619677448.674800:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:30.0:1619677448.674801:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.674801:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.674804:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.674804:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.674805:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.674807:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.674808:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.674809:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.674810:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.674812:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.674814:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.674816:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.674817:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.674818:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.674819:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.674820:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.674821:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.674822:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.674823:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.674824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.674828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.674831:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.674833:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.674834:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.674835:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.674836:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:12.0:1619677448.674837:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.674838:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.674839:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.674839:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.674840:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:21.2:1619677448.674886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.674892:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.674895:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.674895:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.674897:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.674899:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.674901:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.674902:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.674903:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.674904:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.674906:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.674908:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.674910:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.674912:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.674913:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.674914:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.674915:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.674916:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.674916:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.674917:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.674921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.674928:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.674930:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.674931:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.674932:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.674932:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.674934:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.675020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677448.675026:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.0:1619677448.675029:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.2:1619677448.675032:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:1.0:1619677448.675033:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.675035:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.675036:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677448.675036:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.675037:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.675038:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:5.0:1619677448.675039:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.675040:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.675041:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:5.0:1619677448.675041:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.675041:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.675042:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:1.0:1619677448.675042:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.675043:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.675044:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:18.0:1619677448.675045:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.675045:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.675045:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.675045:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.675047:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.675048:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.675048:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.675048:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.675048:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.675049:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.675050:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:18.0:1619677448.675051:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677448.675051:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:18.0:1619677448.675052:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.675054:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.675055:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.675055:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.675057:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.675058:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:2.0:1619677448.675059:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.675061:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96711809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.675062:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.675063:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.675063:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.675064:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.675065:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:2.0:1619677448.675065:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.675065:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.675066:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.675066:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.675067:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.675068:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.675068:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.675068:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.675068:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.675069:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (22)++
00000400:00000200:2.0:1619677448.675069:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.675069:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.675070:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677448.675071:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (23)--
00000800:00000200:2.0:1619677448.675071:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.675072:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677448.675073:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.675073:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:2.0:1619677448.675073:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.675074:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.675074:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:2.0:1619677448.675074:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.675075:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.2:1619677448.675075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (22)++
00000800:00000200:5.0:1619677448.675075:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677448.675075:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.675077:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.675077:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.675078:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)++
00000800:00000200:2.0:1619677448.675078:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)++
00000800:00000200:20.2:1619677448.675080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677448.675080:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (25)++
00000800:00000200:18.0:1619677448.675081:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (26)--
00000800:00000200:2.0:1619677448.675081:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (26)++
00000800:00000200:18.0:1619677448.675082:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (25)--
00000800:00000200:17.0:1619677448.675082:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (23)++
00000800:00000200:2.0:1619677448.675082:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (26)--
00000800:00000200:20.0:1619677448.675083:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:9.2:1619677448.675083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.675083:0:19932:0:(o2iblnd_cb.c:929:kiblnd_post_tx_locked()) 10.51.3.43@o2ib3(1): redundant or enough NOOP
00000800:00000200:18.0:1619677448.675084:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (24)--
00000800:00000200:17.0:1619677448.675084:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (24)--
00000800:00000200:8.2:1619677448.675084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.675084:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (24)--
00000800:00000200:2.0:1619677448.675084:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677448.675085:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:18.0:1619677448.675085:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677448.675085:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (24)++
00000800:00000200:20.0:1619677448.675086:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:17.0:1619677448.675086:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.675086:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (24)--
00000800:00000200:20.0:1619677448.675087:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677448.675087:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.675088:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:17.0:1619677448.675088:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.675088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677448.675089:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:18.0:1619677448.675089:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.675089:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.675090:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:17.0:1619677448.675090:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000400:00000200:20.0:1619677448.675091:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.675091:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.675092:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.675092:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.675093:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.0:1619677448.675093:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.675093:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000400:00000200:20.0:1619677448.675094:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.675094:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.675094:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.675095:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677448.675095:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677448.675097:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.675143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.675150:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.675152:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.675153:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.675155:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.675158:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.675160:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.675161:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.675162:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.675164:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.675165:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.675168:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.675170:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.675172:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.675173:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.675174:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.675176:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.675176:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.675177:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.675177:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.675182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.675190:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.675192:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.675193:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.675193:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.675195:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.675198:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.675357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.675365:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.675368:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.675368:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.675370:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.675373:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.675373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:10.0:1619677448.675374:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.675375:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.675377:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.675378:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.675379:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677448.675380:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677448.675383:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.675383:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.675384:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:10.0:1619677448.675385:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:8.0:1619677448.675386:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.675387:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.675388:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:8.0:1619677448.675388:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.675389:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.675390:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:8.0:1619677448.675390:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:10.0:1619677448.675391:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.675391:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:8.0:1619677448.675391:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96711806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.675392:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:8.0:1619677448.675392:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.675393:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.675394:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:25.2:1619677448.675397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:8.0:1619677448.675397:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.675398:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.675401:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.675402:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.675402:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:12.0:1619677448.675404:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:8.0:1619677448.675404:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.675405:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.675405:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:12.0:1619677448.675406:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.0:1619677448.675406:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:12.0:1619677448.675407:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.675408:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.675410:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.2:1619677448.675410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:13.0:1619677448.675411:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.675413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:25.2:1619677448.675416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.675418:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:5.0:1619677448.675418:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677448.675421:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:5.0:1619677448.675421:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:17.0:1619677448.675422:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.675422:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.675422:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.0:1619677448.675424:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.675424:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.675425:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.675425:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.675426:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:5.0:1619677448.675426:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.675427:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.2:1619677448.675427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:12.0:1619677448.675428:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677448.675428:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:12.0:1619677448.675429:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:5.0:1619677448.675429:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711805] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.675430:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:5.0:1619677448.675430:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.675431:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:5.0:1619677448.675431:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.675432:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.675434:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.675435:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.675435:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.675436:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.675436:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:5.0:1619677448.675436:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.675438:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.675440:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.675441:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.675442:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.675444:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.675444:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.675445:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.675446:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.675450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.675457:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.675460:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.675461:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.675461:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.675463:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.675465:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.675485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677448.675491:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677448.675494:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677448.675494:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000400:00000200:1.0:1619677448.675496:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677448.675499:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.675500:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:1.0:1619677448.675501:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.675503:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.675504:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.675504:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:1.0:1619677448.675507:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677448.675509:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:1.0:1619677448.675512:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.675513:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:1.0:1619677448.675513:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:1.0:1619677448.675515:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:1.0:1619677448.675516:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677448.675516:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677448.675517:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:25.2:1619677448.675521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.675528:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.675531:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.675532:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.675534:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677448.675536:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.675537:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.675547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.675555:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.675557:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.675560:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.675561:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.675562:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.675563:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.675565:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.675566:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.675567:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.675568:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.675571:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.675572:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.675575:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.675576:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.675576:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.675578:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.675578:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.675579:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.675580:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.675583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.675590:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.675592:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.675593:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.675595:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.675596:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.675598:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.675604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:12.0:1619677448.675611:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.675613:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.675614:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.675615:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.675615:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.675617:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.675698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.675704:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.675707:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.675708:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.675710:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.675712:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.675714:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.675715:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.675716:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.675717:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.675718:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.675721:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.675722:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.675724:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.675725:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.675726:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.675727:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.675728:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.675729:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.675730:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.675731:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.675733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.675736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.675739:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.675742:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.675743:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.675743:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677448.675744:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.675746:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.675746:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.675746:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.675746:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.675748:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.675749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.2:1619677448.675754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.675756:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.675759:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.675759:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.675760:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.675760:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.675760:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.675761:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677448.675763:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.675764:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.675765:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677448.675766:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.675767:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.675769:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.675770:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.675771:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.675772:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.675774:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.675775:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.675776:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.675779:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.675781:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.675781:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.675782:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.675784:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.675784:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.675785:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.675788:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.675789:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.675789:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:1.0:1619677448.675789:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.675790:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.675791:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.675792:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)++
00000800:00000200:1.0:1619677448.675793:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.675794:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (24)++
00000800:00000200:1.0:1619677448.675794:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.675795:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (25)--
00000800:00000200:1.0:1619677448.675795:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.675796:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (24)--
00000800:00000200:17.0:1619677448.675797:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.675797:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.675797:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.2:1619677448.675798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.675798:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.675801:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.675803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.2:1619677448.675803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.675804:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.675806:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.675806:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.675806:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.675806:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.675807:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.675808:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.675808:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.675809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677448.675809:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677448.675810:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:1.0:1619677448.675810:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677448.675812:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:24.0:1619677448.675813:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.675814:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.675814:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:8.0:1619677448.675814:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677448.675817:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.675818:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.675818:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:24.0:1619677448.675820:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:20.0:1619677448.675820:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.675822:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.675823:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.675824:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.675825:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.675826:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.675830:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.675832:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.675834:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.675835:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.675835:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.675837:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.675838:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.675838:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.675839:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.2:1619677448.675842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:25.2:1619677448.675843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.675849:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:22.0:1619677448.675850:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677448.675852:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:22.0:1619677448.675853:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:18.0:1619677448.675853:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:22.0:1619677448.675854:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.675854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:6.0:1619677448.675854:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:2.0:1619677448.675854:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.675856:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.675857:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.675858:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677448.675859:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:2.0:1619677448.675859:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:18.0:1619677448.675860:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677448.675860:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.675862:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.675863:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.675864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000400:00000200:2.0:1619677448.675864:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.675867:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.675869:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.675870:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:6.0:1619677448.675871:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.675871:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.675872:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.675872:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.675873:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677448.675874:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:18.0:1619677448.675874:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:6.0:1619677448.675874:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.675875:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.675875:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.675875:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:6.0:1619677448.675876:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.675876:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:18.0:1619677448.675877:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.675877:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.675878:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:9.2:1619677448.675881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677448.675882:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.675883:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.675884:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.675884:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.675885:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.675885:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677448.675886:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.675887:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000800:00000200:2.0:1619677448.675887:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.675888:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (24)--
00000800:00000200:2.0:1619677448.675888:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (24)++
00000800:00000200:18.0:1619677448.675889:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.675889:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000800:00000200:18.0:1619677448.675890:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:2.0:1619677448.675890:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.675891:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:2.0:1619677448.675891:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.675892:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.675893:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:18.0:1619677448.675893:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000800:00000200:2.0:1619677448.675893:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (24)--
00000400:00000200:18.0:1619677448.675894:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.675894:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.675896:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:21.2:1619677448.675897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.675899:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677448.675900:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677448.675901:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.675901:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.675903:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.675908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.2:1619677448.675915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:2.0:1619677448.675915:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.675918:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.675919:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.675919:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.675919:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.675920:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677448.675921:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.675924:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.2:1619677448.675925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:20.0:1619677448.675926:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.675927:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.675930:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:5.0:1619677448.675930:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.675932:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:22.0:1619677448.675933:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.675933:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:5.0:1619677448.675933:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.675935:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.675936:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.675937:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.675938:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.675939:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.675941:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.675942:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.675942:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.675944:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.675945:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.675945:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.675946:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.2:1619677448.675948:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:22.0:1619677448.675949:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.675951:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.675952:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.675953:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.675954:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.675954:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.675954:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.675955:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677448.675955:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.675956:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.675957:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.675957:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677448.675957:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:26.0:1619677448.675958:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:5.0:1619677448.675958:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.675958:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:22.0:1619677448.675959:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:10.0:1619677448.675959:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677448.675960:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.675960:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:26.0:1619677448.675961:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.675962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:26.0:1619677448.675963:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.675963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:26.0:1619677448.675965:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:21.2:1619677448.675965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:26.0:1619677448.675966:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96711794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.675967:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.675968:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.675969:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:26.0:1619677448.675969:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.675971:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677448.675971:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:30.0:1619677448.675972:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.675973:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.675973:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.675973:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:6.0:1619677448.675973:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.675973:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677448.675974:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.675974:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:26.0:1619677448.675975:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.675976:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.675976:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.675976:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.675977:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.675978:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.675979:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.675979:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.675979:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.675980:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:26.0:1619677448.675981:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.675981:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.2:1619677448.675981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.675982:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.675983:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.675984:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.675987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.675989:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677448.675992:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:13.0:1619677448.675993:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.675995:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.675995:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.675996:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:5.0:1619677448.675996:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000400:00000200:13.0:1619677448.675997:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.675997:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.675997:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.675998:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:13.0:1619677448.675998:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.675999:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.675999:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.675999:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.676000:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.676000:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.676059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1200] (20)++
00000800:00000200:20.2:1619677448.676063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677448.676065:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677448.676068:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:1.0:1619677448.676069:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:31.0:1619677448.676071:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.676071:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.676074:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:17.0:1619677448.676074:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.676076:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:31.0:1619677448.676077:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.676077:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:17.0:1619677448.676077:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.676079:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.676080:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.676080:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.676081:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:31.0:1619677448.676082:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.676083:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.676084:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.676085:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.676086:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.676087:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.676087:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 584 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677448.676090:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.676091:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:17.0:1619677448.676091:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:31.0:1619677448.676092:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.676093:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:31.0:1619677448.676094:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.676094:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677448.676094:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:17.0:1619677448.676095:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:31.0:1619677448.676097:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.676098:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.676098:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:8.2:1619677448.676099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:31.0:1619677448.676100:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.676101:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.676102:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.676103:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:4.2:1619677448.676106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.2:1619677448.676107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.676110:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.676112:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:10.0:1619677448.676113:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677448.676113:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.676114:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.676115:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:10.0:1619677448.676116:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.676117:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677448.676117:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.676117:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:20.0:1619677448.676118:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.676119:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:20.0:1619677448.676120:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.676120:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.676121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.0:1619677448.676121:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.676123:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:10.0:1619677448.676123:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.676124:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.676126:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.676127:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.676128:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.676129:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677448.676129:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.676131:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677448.676133:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.676134:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.676135:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677448.676135:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.676137:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.676138:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.676139:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:10.0:1619677448.676139:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:30.0:1619677448.676140:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.676140:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:30.0:1619677448.676141:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.676141:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:30.0:1619677448.676142:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.676142:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.676142:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:30.0:1619677448.676143:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.676143:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:30.0:1619677448.676147:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.676147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.676148:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.676150:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.676151:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.676152:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.676153:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.676154:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.676155:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677448.676155:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:30.0:1619677448.676157:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.676157:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677448.676158:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:25.2:1619677448.676159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:22.0:1619677448.676159:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.676159:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.676161:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.676163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:4.2:1619677448.676164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677448.676165:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:22.0:1619677448.676165:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:26.0:1619677448.676167:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.676168:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.676168:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.676170:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:26.0:1619677448.676171:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.676172:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677448.676172:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677448.676173:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:31.0:1619677448.676174:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.676174:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677448.676175:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.676175:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677448.676177:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.676179:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.676181:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.676182:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.676184:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.676184:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.676185:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.676188:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.676190:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.676193:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.676194:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.676194:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.676195:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.676196:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.676197:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.676199:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.676199:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.676202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.676205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.676210:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.676211:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.676212:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677448.676213:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677448.676213:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.676214:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677448.676214:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.676215:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.676216:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.676219:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.676234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.676240:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.676242:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.676244:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.676245:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.676247:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.676249:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.676250:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.676252:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.676252:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.676253:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.676257:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.676258:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.676262:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.676263:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.676263:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.676265:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.676266:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.676267:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.676267:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.676271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.676278:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.676280:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.676281:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.676283:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.676284:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.676286:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.676403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.676409:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.676411:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.676412:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.676414:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.676416:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.676418:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.676419:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.676420:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.676421:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.676423:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.676425:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.676427:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.676429:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.676430:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.676430:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.676432:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.676432:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.676432:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.676433:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.676451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.676458:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.676460:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.676461:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.676462:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.676463:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.676464:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.676472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.676479:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.676481:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.676482:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.676482:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.676483:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.676483:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.676500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb3800] (20)++
00000800:00000200:1.0:1619677448.676506:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:1.0:1619677448.676508:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.65@o2ib3
00000800:00000200:17.0:1619677448.676510:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (22)--
00000400:00000200:1.0:1619677448.676511:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.65@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.676513:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.676514:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:4.2:1619677448.676515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677448.676515:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.676517:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.676517:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.676518:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.676521:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.65@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.676523:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.676523:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:10.0:1619677448.676525:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.676526:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677448.676526:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.676527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:1.0:1619677448.676527:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.676528:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:10.0:1619677448.676529:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.676531:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.676531:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.676532:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb3800] (22)++
00000800:00000200:1.0:1619677448.676532:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb3800] (23)--
00000400:00000200:10.0:1619677448.676533:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:1.0:1619677448.676533:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (22)--
00000800:00000200:17.0:1619677448.676534:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)++
00000400:00000200:10.0:1619677448.676534:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.676535:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.676536:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.65@o2ib3
00000400:00000200:10.0:1619677448.676536:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.676536:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (22)--
00000800:00000200:9.2:1619677448.676537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:10.0:1619677448.676538:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.676539:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.65@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.676541:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.676542:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.676543:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:20.2:1619677448.676544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:17.0:1619677448.676544:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96711784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.676544:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:10.0:1619677448.676545:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.676546:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.676546:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.676547:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:17.0:1619677448.676547:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.676547:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.676548:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.676548:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.676549:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.676549:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:8.0:1619677448.676549:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.676551:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.676552:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.676552:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.65@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.676552:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.676552:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.676553:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.0:1619677448.676553:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:17.0:1619677448.676554:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.676555:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.676557:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.676557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.676557:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.676558:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.676559:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677448.676559:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:31.0:1619677448.676560:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.676561:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677448.676561:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.676561:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb3800] (21)++
00000400:00000200:31.0:1619677448.676562:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:30.0:1619677448.676562:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.676563:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96711783] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.676563:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.676563:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdbb3800] (22)++
00000800:00000200:17.0:1619677448.676564:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb3800] (23)--
00000800:00000200:17.0:1619677448.676564:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (22)--
00000400:00000200:31.0:1619677448.676565:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.676565:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.676566:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.676566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:31.0:1619677448.676567:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:5.2:1619677448.676569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb3800] (21)++
00000400:00000200:31.0:1619677448.676570:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.676571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:31.0:1619677448.676572:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.676573:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.676574:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.676574:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:13.0:1619677448.676574:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:31.0:1619677448.676575:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:1.0:1619677448.676575:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb3800] (22)++
00000800:00000200:31.0:1619677448.676576:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:13.0:1619677448.676576:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:31.0:1619677448.676577:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:13.0:1619677448.676577:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.676577:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdbb3800] (23)--
00000800:00000200:31.0:1619677448.676578:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.676578:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:1.0:1619677448.676578:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (22)--
00000800:00000200:31.0:1619677448.676579:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677448.676579:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)--
00000400:00000200:13.0:1619677448.676579:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.676580:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.676581:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:12.0:1619677448.676581:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.676582:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.676582:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677448.676583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:26.0:1619677448.676584:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.676586:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.676587:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:21.2:1619677448.676587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:26.0:1619677448.676588:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.676589:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:26.0:1619677448.676589:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.676590:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.676591:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.676592:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.676593:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.676593:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:30.0:1619677448.676595:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.676595:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.676596:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:22.0:1619677448.676596:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.676596:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:30.0:1619677448.676597:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:26.0:1619677448.676597:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.676598:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.676599:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.676600:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.676601:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.676601:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677448.676601:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.676602:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.676603:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.676603:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:22.0:1619677448.676603:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.676604:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.676604:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:6.0:1619677448.676604:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.676605:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.676607:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.676608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:6.0:1619677448.676608:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.676608:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.676610:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.676611:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96711780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.676612:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.676613:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.676614:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.676614:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677448.676614:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.676615:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:12.0:1619677448.676617:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.676618:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.676618:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.676619:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.676620:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:22.0:1619677448.676621:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.676621:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.676621:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.676622:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.676623:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:12.0:1619677448.676623:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.676623:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)++
00000800:00000200:6.0:1619677448.676624:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (24)--
00000800:00000200:6.0:1619677448.676624:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.676625:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.676627:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.676628:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:8.2:1619677448.676628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.676629:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.676631:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.676631:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.676632:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.676632:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677448.676635:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:25.2:1619677448.676636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.676638:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.676639:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.676639:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.676641:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.676643:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:1.0:1619677448.676644:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677448.676646:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.676647:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.676648:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.676649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb3800] (20)++
00000800:00000200:30.0:1619677448.676650:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.676652:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677448.676655:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:17.0:1619677448.676657:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.65@o2ib3
00000800:00000200:1.0:1619677448.676658:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (22)--
00000400:00000200:17.0:1619677448.676660:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.65@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.676663:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.676664:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:17.0:1619677448.676666:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.676667:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.676668:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.676669:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:17.0:1619677448.676672:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.65@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.676674:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:17.0:1619677448.676676:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.676677:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:17.0:1619677448.676678:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:17.0:1619677448.676680:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:17.0:1619677448.676680:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:17.0:1619677448.676681:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb3800] (22)--
00000800:00000200:17.0:1619677448.676681:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)--
00000800:00000200:9.2:1619677448.676686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.676694:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.676696:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.676697:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.676698:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.676699:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.676702:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.676771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.676778:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.676780:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.676781:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.676783:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.676786:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.676788:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.676789:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.676790:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.676791:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.676792:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.676796:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.676798:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.676801:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.676801:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.676802:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.676804:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.676804:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.676805:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.676806:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.676810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.676818:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.676821:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.676821:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.676822:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.676824:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.676827:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.676872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.676879:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.676882:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.676885:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.676886:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.676886:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.676888:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.676889:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711777] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.676890:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.676892:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.676892:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.676896:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.676897:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.676900:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.676900:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.676901:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.676902:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.676903:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.676904:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.676904:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.676908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.676914:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.676917:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.676917:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.676919:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.676920:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.676921:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.676931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:4.2:1619677448.676934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677448.676935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:12.0:1619677448.676937:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.676940:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.676940:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.676941:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:12.0:1619677448.676941:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.676941:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.676942:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.676943:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.676944:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.676944:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:10.0:1619677448.676944:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:1.0:1619677448.676945:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677448.676948:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.676949:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.676951:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.676952:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.676953:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.676953:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.676955:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.676955:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.676957:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.676957:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.676959:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.676959:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.676960:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.676961:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.676965:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.676967:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.676968:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.676970:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.676971:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.676971:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:17.0:1619677448.676972:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.676972:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.676973:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.676974:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677448.676974:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:10.0:1619677448.676974:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.676975:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:10.0:1619677448.676975:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.676976:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:10.0:1619677448.676976:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.676977:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.676977:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.676980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.676981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:25.2:1619677448.676984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.676984:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.676986:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:6.0:1619677448.676987:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:20.0:1619677448.676988:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.676989:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677448.676990:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677448.676991:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677448.676991:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:20.0:1619677448.676991:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.676991:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.676993:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677448.676994:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.676994:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.676994:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96711774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.676994:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.676996:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.676997:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.676997:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:20.0:1619677448.676997:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.676997:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:20.0:1619677448.676998:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.677000:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.677001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:20.0:1619677448.677002:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.677004:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.677007:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.677007:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.677008:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.677009:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.677010:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.677011:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.677011:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.677011:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.677012:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.677013:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.677013:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677448.677014:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.0:1619677448.677014:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.677015:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.677016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.677020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677448.677023:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.677025:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.677025:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:12.0:1619677448.677026:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.677028:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.677028:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.677028:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.677029:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.677030:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.677031:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.677052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.677057:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.677060:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.677062:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.677062:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677448.677065:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.677066:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.677068:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711773] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.677069:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.677070:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.677071:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.677074:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.677076:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.677079:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.677080:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.677081:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.677082:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.677083:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.677084:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.677085:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.677086:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.677088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.677091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.677094:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:13.0:1619677448.677094:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.677096:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.677096:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.677097:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:13.0:1619677448.677097:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.677098:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.677100:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677448.677101:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.677102:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.677198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.677205:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.677208:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.677210:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.677212:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.677212:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.677214:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677448.677216:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.677217:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.677218:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.677219:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:5.2:1619677448.677221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677448.677222:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.677224:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677448.677226:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.677227:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:30.0:1619677448.677227:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.677227:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.677229:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.677229:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:30.0:1619677448.677230:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.677230:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.677231:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.677231:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:1.0:1619677448.677233:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.677235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:1.0:1619677448.677235:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.677236:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.677238:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.677239:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.677240:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.677241:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:1.0:1619677448.677241:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677448.677244:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:1.0:1619677448.677244:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.677245:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.677245:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.677245:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:8.0:1619677448.677247:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.677249:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:1.0:1619677448.677249:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.677250:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.677250:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.677252:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.677253:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.677254:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.677254:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.677258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.677265:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.2:1619677448.677266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677448.677267:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.677268:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.677270:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.677271:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.677271:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677448.677273:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.677274:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.677275:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.677278:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.677279:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.677281:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.677282:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.677283:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.677284:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.677285:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.677288:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.677290:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.677294:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.677294:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.677302:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.677304:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.677305:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.677306:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.677306:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.677311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.677318:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.677321:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.677322:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.677324:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.677325:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.677326:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.677327:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.677327:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.677328:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.677328:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.677329:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.677403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.677410:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.677413:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.677414:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.677416:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.677418:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.677419:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.677420:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.677422:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.677423:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.677424:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.677428:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.677429:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.677431:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.677432:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.677433:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.677434:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.677435:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.677436:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.677437:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.677437:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.677440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.677443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.677448:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.677450:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:10.0:1619677448.677450:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677448.677451:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.677452:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.677452:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.677452:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.677453:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.677454:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.677456:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.677463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.677470:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.677473:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.677474:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.677476:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.677480:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.677481:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.677482:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.677484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:17.0:1619677448.677484:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.677485:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.677486:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.677489:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.677490:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677448.677491:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677448.677494:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:17.0:1619677448.677494:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.677494:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:17.0:1619677448.677495:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.677496:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:18.0:1619677448.677497:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.677498:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:18.0:1619677448.677499:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.677499:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677448.677500:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:17.0:1619677448.677500:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.677501:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.677501:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677448.677502:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.677503:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.677504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:18.0:1619677448.677504:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.677505:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:5.2:1619677448.677506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677448.677508:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.677510:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:22.0:1619677448.677512:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:18.0:1619677448.677512:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.677512:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:18.0:1619677448.677513:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677448.677514:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.677514:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677448.677515:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:18.0:1619677448.677515:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677448.677515:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677448.677516:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.677516:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:6.0:1619677448.677516:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.677516:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677448.677517:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.677517:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:22.0:1619677448.677518:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.677521:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.2:1619677448.677522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.677531:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.677533:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.677534:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.677536:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.677537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677448.677538:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677448.677538:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.677545:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.677548:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.677550:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.677552:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.677553:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.677554:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677448.677556:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.677557:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.677558:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.677559:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677448.677562:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.677564:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677448.677567:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.677567:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.677568:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.677570:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.677570:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.677571:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.677572:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.677575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.677582:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:4.2:1619677448.677583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.677584:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.677585:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.677587:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.677587:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.677588:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.677589:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677448.677590:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.677591:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.677592:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.677594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000400:00000200:8.0:1619677448.677595:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.677597:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.677598:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.677600:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.677601:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000400:00000200:8.0:1619677448.677601:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.677602:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677448.677602:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.677603:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.677604:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.677604:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.677605:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:8.0:1619677448.677606:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.677608:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.2:1619677448.677610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:8.0:1619677448.677611:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.677612:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.677613:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.677614:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.677615:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.677615:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.677615:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.0:1619677448.677615:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:5.0:1619677448.677618:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:8.2:1619677448.677620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:5.0:1619677448.677620:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.677621:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:5.0:1619677448.677624:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.677625:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677448.677626:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.677628:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:5.0:1619677448.677628:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.677629:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.677630:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.677631:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.677631:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.677632:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.677634:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.677634:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.677636:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.677638:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.0:1619677448.677639:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.677639:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.677640:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.677642:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.677642:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:5.0:1619677448.677643:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708600] (22)++
00000800:00000200:5.0:1619677448.677645:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (23)--
00000800:00000200:5.0:1619677448.677645:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:8.2:1619677448.677647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.677651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:5.2:1619677448.677657:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919065782400] (20)++
00000800:00000200:20.0:1619677448.677658:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.677659:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.677660:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.677661:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.677661:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (22)++
00000800:00000200:5.0:1619677448.677661:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708600] (23)--
00000800:00000200:5.0:1619677448.677662:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:2.0:1619677448.677662:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (21)++
00000800:00000200:20.0:1619677448.677663:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.677665:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:2.0:1619677448.677665:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:20.0:1619677448.677666:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:18.0:1619677448.677666:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000800:00000200:8.2:1619677448.677667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000400:00000200:2.0:1619677448.677668:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.677672:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.677673:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:5.0:1619677448.677674:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000400:00000200:2.0:1619677448.677674:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711763] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.677675:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.677677:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.677677:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.677678:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000400:00000200:2.0:1619677448.677678:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.677679:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.677679:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.677680:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:2.0:1619677448.677681:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.677683:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677448.677685:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.677686:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:2.0:1619677448.677687:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:2.0:1619677448.677688:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:2.0:1619677448.677689:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (21)++
00000800:00000200:2.0:1619677448.677690:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (22)--
00000800:00000200:2.0:1619677448.677690:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (21)--
00000800:00000200:8.2:1619677448.677694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:5.0:1619677448.677702:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:5.0:1619677448.677704:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.677705:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.677707:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.677708:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:5.0:1619677448.677709:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:5.2:1619677448.677847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.677853:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.677855:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.677856:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.677858:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.677859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:1.0:1619677448.677860:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.677861:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.677863:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.677864:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.677865:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.677866:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.677867:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.677870:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:1.0:1619677448.677870:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.677871:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.677873:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.677874:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.677875:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.677875:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.677876:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.677876:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:30.0:1619677448.677877:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.677878:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.677878:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.677878:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677448.677879:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.677879:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:30.0:1619677448.677880:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.677880:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677448.677881:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:25.2:1619677448.677884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:30.0:1619677448.677885:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.677887:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.677890:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.677890:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.677890:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:30.0:1619677448.677891:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.677893:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.677893:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.677893:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.677894:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.677895:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:13.0:1619677448.677895:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.677896:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.677897:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.677897:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677448.677898:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.677899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.677903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.677905:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.677908:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.677908:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.677910:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:12.0:1619677448.677910:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.677912:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:12.0:1619677448.677912:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.677913:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677448.677913:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.677914:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677448.677965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.677972:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.677975:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.677976:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.677978:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.677980:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.677982:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.677983:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711760] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.677984:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.677985:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.677986:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.677990:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.677992:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.677994:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.677995:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.677996:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.677998:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.677998:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.677999:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.677999:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.678004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.678011:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.2:1619677448.678012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:6.0:1619677448.678014:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.678015:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.678016:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.678018:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:6.0:1619677448.678018:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.678021:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:6.0:1619677448.678021:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677448.678022:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.678024:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.678026:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.678028:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.678029:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.678030:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.678031:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.678032:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.678036:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.678037:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.678040:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.678040:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.678041:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.678043:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.678043:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.678044:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.678044:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.678049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.678056:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.678059:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.678060:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.678060:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.678062:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.678064:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.678068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:31.0:1619677448.678076:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677448.678079:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:31.0:1619677448.678081:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.678083:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:31.0:1619677448.678084:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.678085:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.678086:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.678088:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.678089:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.678090:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.678093:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.678095:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.678099:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.678099:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.678100:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.678102:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.678102:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677448.678103:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:31.0:1619677448.678104:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:5.2:1619677448.678107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.2:1619677448.678108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.678114:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677448.678115:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:10.0:1619677448.678116:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.678117:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677448.678118:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.678119:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.678119:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.678120:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.678122:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.678122:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.678123:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:10.0:1619677448.678124:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.678125:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.678126:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.678127:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.678128:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.678131:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.678133:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.678135:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.678136:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.678137:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.678138:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.678139:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.678140:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.678140:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.678144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.678152:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.678155:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.678156:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.678156:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.678158:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.678161:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.678212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.2:1619677448.678218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677448.678218:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.678221:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.678222:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.678223:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.678225:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:1.0:1619677448.678226:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.678228:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677448.678229:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:1.0:1619677448.678229:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.678231:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.678232:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.678232:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.678232:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.678234:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677448.678235:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.678238:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.678240:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:1.0:1619677448.678240:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:30.0:1619677448.678242:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.678242:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.678243:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.678243:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:30.0:1619677448.678244:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.678244:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:30.0:1619677448.678245:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:1.0:1619677448.678245:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.678246:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.678246:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.678247:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677448.678249:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.678249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.678250:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:25.2:1619677448.678252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.678254:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.678255:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.678255:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.678257:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.678257:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.678258:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677448.678258:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.678259:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:31.0:1619677448.678260:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.678261:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.678262:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.678263:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.678263:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:8.2:1619677448.678263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.678264:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.678265:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.678265:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.678266:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96711754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.678267:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.678267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:31.0:1619677448.678268:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.678268:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.678269:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.678271:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:31.0:1619677448.678273:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.678273:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.678273:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.678274:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.678275:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.678275:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.678276:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.0:1619677448.678276:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.678276:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.678277:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.678278:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.678278:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.678278:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.678279:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.678279:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.678279:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:30.0:1619677448.678280:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:31.0:1619677448.678281:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.678281:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.678282:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.678283:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.678287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.678300:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.678303:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.678304:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:6.0:1619677448.678304:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.678306:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677448.678306:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.678307:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (23)++
00000800:00000200:21.2:1619677448.678307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:22.0:1619677448.678308:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (24)--
00000800:00000200:22.0:1619677448.678309:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.678310:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.678311:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:24.0:1619677448.678315:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.678319:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.678320:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.678322:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.678324:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.678326:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.678327:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711753] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.678328:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.678329:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.678330:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677448.678333:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.678335:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.678338:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.678338:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.678339:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.678341:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.678341:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.678342:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.678342:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.678346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.678355:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.678357:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.678358:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.678360:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.678361:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.678363:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.678400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.678408:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.678410:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.678411:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.678413:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.678417:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.678418:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.678419:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.678420:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.678421:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.678422:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.678425:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.678427:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.678429:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.678430:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.678430:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.678432:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.678433:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.678433:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.678434:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.678440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.678448:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.678450:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.678451:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.678452:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.678455:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.678456:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.678534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.678540:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.678543:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.678543:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.678545:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.678548:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.678550:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.678551:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.678552:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.678553:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.678554:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.678557:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.678559:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.678561:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.678562:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.678562:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.678564:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.678565:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.678565:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.678566:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.678570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.678578:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.678580:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.678582:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.678582:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.678584:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.678586:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.678637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.678643:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.678646:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.678647:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.678649:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.678652:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.678653:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.678654:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.678656:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.678657:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.678658:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.678661:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.678663:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.678666:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.678667:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.678668:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.678669:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.678670:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.678670:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.678671:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.678675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.678683:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.678685:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.678686:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.678688:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.678690:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.678691:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.678701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.678708:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.678711:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.678712:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.678714:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.678716:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.678717:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.678718:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.678719:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.678720:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.678721:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.678724:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.678726:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.678728:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.678728:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.678729:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.678730:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.678731:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.678731:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.678732:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.678736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.678742:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.678743:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.678744:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.678745:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.678746:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.678747:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.678941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.678949:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.678952:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.678954:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.678956:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.678956:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.678958:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677448.678959:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.678960:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.678961:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.678962:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677448.678965:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.678967:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.678970:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.678970:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.678971:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.678973:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.678973:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.678974:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.678975:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.678979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.678986:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.678989:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.678990:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.678990:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.678993:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.678995:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.678999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.679007:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.679009:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.679010:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.679010:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.679011:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.679012:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.679256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.679263:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.679265:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.679266:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.679268:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.679270:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.679271:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.679272:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711747] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.679274:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.679275:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.679276:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.679279:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.679280:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.679284:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.679284:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.679285:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.679286:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.679287:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.679288:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.679288:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.679293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.679302:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.679304:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.679305:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.679306:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.679308:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.679310:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.679343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.679350:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.679352:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.679353:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.679355:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.679358:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.679359:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.679360:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.679362:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.679363:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.679364:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.679367:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.679369:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.679372:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.679372:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.679373:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:4.2:1619677448.679375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:1.0:1619677448.679375:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.679375:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.679377:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.679378:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.679378:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.679382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.679383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.679385:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.679387:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:17.0:1619677448.679388:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677448.679389:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677448.679390:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:8.0:1619677448.679390:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.679391:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:17.0:1619677448.679391:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677448.679392:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.679392:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.679393:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.679394:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.679395:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.679396:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96711745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.679398:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.679399:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.679400:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.679404:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.679406:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.679407:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.679408:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.679409:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.0:1619677448.679409:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.679410:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.679411:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.679413:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.679413:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.679414:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.679415:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.679419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.679426:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.679429:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.679430:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.679430:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.679431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:1.0:1619677448.679432:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.679435:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:24.0:1619677448.679438:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.679441:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.679442:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.679443:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.679446:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.679448:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.679449:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.679449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:24.0:1619677448.679451:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.679452:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.679452:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.679455:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.679455:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677448.679456:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:6.0:1619677448.679458:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:24.0:1619677448.679459:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.679459:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:24.0:1619677448.679460:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.679460:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:6.0:1619677448.679461:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.679462:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.679462:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.679463:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:6.0:1619677448.679463:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.679464:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:21.2:1619677448.679464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:6.0:1619677448.679465:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.679466:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.679467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:6.0:1619677448.679467:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.679468:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.679470:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:18.0:1619677448.679471:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:6.0:1619677448.679473:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.679474:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:20.2:1619677448.679475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:13.0:1619677448.679475:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:6.0:1619677448.679475:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677448.679475:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.679478:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:13.0:1619677448.679478:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:6.0:1619677448.679478:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677448.679479:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.679479:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000400:00000200:18.0:1619677448.679480:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.679480:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:18.0:1619677448.679481:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:13.0:1619677448.679481:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.679482:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.679482:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.679483:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:18.0:1619677448.679483:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.679483:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677448.679483:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:18.0:1619677448.679484:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.679484:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677448.679484:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000400:00000200:18.0:1619677448.679485:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.679485:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677448.679487:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:18.0:1619677448.679487:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:12.0:1619677448.679487:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.679487:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677448.679488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:12.0:1619677448.679488:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.679489:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.679489:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:21.2:1619677448.679490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:12.0:1619677448.679491:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:18.0:1619677448.679492:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:26.0:1619677448.679493:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:22.0:1619677448.679495:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677448.679495:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.679495:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:26.0:1619677448.679496:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.679497:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:10.0:1619677448.679497:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:22.0:1619677448.679498:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677448.679498:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.679498:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.679499:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:22.0:1619677448.679499:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677448.679499:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.679499:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:26.0:1619677448.679500:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96711741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.679500:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.679501:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.679501:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:26.0:1619677448.679502:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.679502:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.679502:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:26.0:1619677448.679503:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:18.0:1619677448.679503:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.679507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.679507:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.679508:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.679508:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:26.0:1619677448.679510:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.679513:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.679514:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.679515:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:6.0:1619677448.679515:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.679516:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.679517:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.679517:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:6.0:1619677448.679517:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.679518:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:22.0:1619677448.679518:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.679518:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.679520:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677448.679523:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.2:1619677448.679533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.679542:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.679545:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.679546:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.679547:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677448.679548:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.2:1619677448.679550:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.0:1619677448.679555:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:10.0:1619677448.679558:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.679560:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.679560:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.679562:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.679563:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.679564:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.679566:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.679567:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.679568:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.679570:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.679571:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.679572:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.679573:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.679575:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.679576:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.679577:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.679578:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.679579:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.679579:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.679580:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.679583:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.679584:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:26.0:1619677448.679585:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.679585:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.679587:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.679588:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.679588:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)++
00000800:00000200:10.0:1619677448.679589:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (24)--
00000800:00000200:10.0:1619677448.679590:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.679591:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.679591:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.679592:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.679592:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.679594:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:9.2:1619677448.679594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.679595:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.679596:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.679597:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.679598:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.679600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677448.679601:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.2:1619677448.679603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.679603:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.679605:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.679605:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.679607:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.679608:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677448.679610:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:22.0:1619677448.679611:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:10.0:1619677448.679611:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:22.0:1619677448.679612:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.679612:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.679613:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.679614:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677448.679614:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.679614:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677448.679617:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.679619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:6.0:1619677448.679626:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.679629:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.679629:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.679629:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.679631:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.679631:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.679641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.679649:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.679653:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.679655:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.679657:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.679658:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.679659:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.679661:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.679662:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.679663:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.679664:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.679667:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.679669:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.679671:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.679672:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.679672:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.679674:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.679674:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.679675:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.679676:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.679679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.679686:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.679688:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.679689:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.679691:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.679692:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.679694:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.680033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.680038:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.680041:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.680042:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.680044:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.680046:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.680048:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.680049:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711737] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.680050:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.680051:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.680052:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.680055:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.680057:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.680059:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.680060:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.680060:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.680062:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.680062:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.680063:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.680063:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.680068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.680075:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.680077:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.680077:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.680078:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.680079:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.680081:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.680188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.680190:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.680193:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.680195:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.680196:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.680197:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.680198:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.680200:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.680201:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.680202:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.680203:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.680206:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.680207:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.680210:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.680211:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.680212:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.680214:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.680215:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.680216:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.680216:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.680220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.680227:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.680230:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.680231:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.680232:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.680234:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.680235:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.680277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.680285:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.680288:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.680290:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.680292:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.680292:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.680299:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.680301:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.680302:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.680303:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.680304:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.680308:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.680310:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.680312:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.680313:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.680313:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.680315:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.680315:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.680316:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.680317:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.2:1619677448.680319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:25.2:1619677448.680321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.680327:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.680328:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.680331:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.680331:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.680331:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.680331:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:31.0:1619677448.680334:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.680334:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.680335:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.680336:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:10.0:1619677448.680337:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.680339:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:25.2:1619677448.680340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000400:00000200:10.0:1619677448.680340:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.680341:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.680342:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.680343:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.680347:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000400:00000200:10.0:1619677448.680347:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.680348:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.680350:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.680350:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.680351:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:10.0:1619677448.680351:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.680352:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.680352:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677448.680353:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:10.0:1619677448.680353:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.680354:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.680355:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.680355:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.680356:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.680360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.680369:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.680371:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.680372:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.680373:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.680375:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.680378:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.680382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.680389:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.680392:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.680393:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.680395:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.680397:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.680398:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.680400:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.680401:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.680402:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.680403:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.680406:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.680408:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.680410:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.680411:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.680412:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.680414:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.680414:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.680415:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.680417:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.680417:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677448.680420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677448.680423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.680428:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.680429:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:18.0:1619677448.680431:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:10.0:1619677448.680431:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:18.0:1619677448.680432:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.680432:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677448.680432:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.680434:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.680434:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:18.0:1619677448.680438:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.680474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:5.2:1619677448.680479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.0:1619677448.680482:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.680485:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.680486:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677448.680486:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677448.680487:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.680489:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:8.0:1619677448.680490:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.680490:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677448.680492:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.680493:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.680493:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.680494:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.680494:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.680496:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.680496:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.680497:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:21.2:1619677448.680498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:17.0:1619677448.680498:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.680500:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.680501:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.680502:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.680505:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677448.680506:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.680508:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677448.680508:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.680511:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.680512:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:8.0:1619677448.680512:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.680513:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.680514:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.680515:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:17.0:1619677448.680515:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.680515:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:31.0:1619677448.680516:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96711730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.680516:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.680517:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.680517:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)++
00000400:00000200:31.0:1619677448.680518:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.680518:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:31.0:1619677448.680519:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.680519:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.680519:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (23)++
00000400:00000200:31.0:1619677448.680520:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:17.0:1619677448.680520:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.680521:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677448.680521:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677448.680522:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:31.0:1619677448.680523:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.680524:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.680525:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.680527:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.680527:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:31.0:1619677448.680528:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.680529:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:8.0:1619677448.680529:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.680529:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677448.680530:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.680531:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.680531:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000800:00000200:31.0:1619677448.680532:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.680532:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677448.680533:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000800:00000200:17.0:1619677448.680534:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (23)++
00000400:00000200:1.0:1619677448.680534:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.680535:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000800:00000200:4.2:1619677448.680535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.2:1619677448.680536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:17.0:1619677448.680536:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.680537:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.680538:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.680538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:1.0:1619677448.680539:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.680540:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677448.680540:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:26.0:1619677448.680542:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:24.0:1619677448.680542:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677448.680543:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:24.0:1619677448.680544:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:26.0:1619677448.680545:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:24.0:1619677448.680545:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:10.0:1619677448.680545:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.680546:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.680546:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677448.680546:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.680547:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:26.0:1619677448.680549:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.680550:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.680551:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.680552:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.680553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000400:00000200:2.0:1619677448.680553:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.680554:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.680555:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.680556:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.680557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000400:00000200:2.0:1619677448.680557:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.680559:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000400:00000200:2.0:1619677448.680561:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.680562:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677448.680563:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:10.0:1619677448.680563:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.680563:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.680565:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.680566:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.680566:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.680566:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.680567:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000400:00000200:6.0:1619677448.680567:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.680567:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.680568:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.680568:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.680569:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:6.0:1619677448.680569:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:2.0:1619677448.680569:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:6.0:1619677448.680570:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.680570:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:6.0:1619677448.680571:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.680572:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.680572:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.680573:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.680573:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:6.0:1619677448.680574:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.2:1619677448.680576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:6.0:1619677448.680576:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.680578:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.2:1619677448.680578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677448.680580:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.680580:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.680581:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.680583:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.680583:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:18.0:1619677448.680584:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:6.0:1619677448.680584:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677448.680585:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:5.0:1619677448.680586:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677448.680587:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.680587:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.680588:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.680589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.680589:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.680590:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.680591:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.680592:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.680594:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677448.680596:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.680598:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.680599:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.680600:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.680601:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.680602:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.680636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:21.2:1619677448.680640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:2.0:1619677448.680642:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.680645:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.680646:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:24.0:1619677448.680647:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:2.0:1619677448.680648:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.680650:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.680650:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:2.0:1619677448.680650:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.680652:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677448.680654:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.680654:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.680655:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.680656:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.680657:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.680657:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677448.680659:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.680661:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96711726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.680662:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.680662:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.680663:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.680664:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:24.0:1619677448.680665:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.680668:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677448.680669:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.680669:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.680669:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677448.680671:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.680671:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.680672:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.680672:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.680673:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:24.0:1619677448.680674:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677448.680674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:24.0:1619677448.680675:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.680676:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.680676:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:25.2:1619677448.680677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677448.680677:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.680678:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.680679:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.680679:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677448.680679:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.2:1619677448.680680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:20.0:1619677448.680681:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.680683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677448.680684:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.680684:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.680684:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.680685:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:6.0:1619677448.680685:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:20.0:1619677448.680687:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96711725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.680687:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.680688:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.680688:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.680689:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677448.680689:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.680690:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.680691:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.680691:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:13.0:1619677448.680691:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.680691:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:6.0:1619677448.680692:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.680694:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:13.0:1619677448.680694:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:20.0:1619677448.680695:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.680695:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.680695:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.680697:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.680698:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.680698:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.680700:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.680700:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:20.0:1619677448.680701:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.680701:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.680701:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:6.0:1619677448.680701:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.680702:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.680703:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.680703:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:6.0:1619677448.680703:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.680704:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.680705:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:6.0:1619677448.680705:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.680708:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.680709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.680710:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.680713:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.680714:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.680715:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.680717:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.680717:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.680718:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.680718:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.680719:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.680720:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.680720:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:22.0:1619677448.680721:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.680721:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.2:1619677448.680723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.680724:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.680728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.680731:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.680732:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.680733:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:5.0:1619677448.680733:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.680734:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.680735:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.680736:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.680737:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.680739:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677448.680741:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.680742:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.680743:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.680744:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:5.0:1619677448.680745:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.680746:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.680747:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:20.0:1619677448.680748:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.680748:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.680749:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.680750:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:5.0:1619677448.680750:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.680750:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.680751:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677448.680753:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.680754:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677448.680754:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677448.680756:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677448.680757:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.680757:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.680758:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677448.680762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.680768:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.680771:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.680772:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.680772:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.680774:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.680777:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:20.2:1619677448.680783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.680791:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.680793:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.680796:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.680797:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.680798:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.680799:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.680801:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711722] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.680802:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.680803:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.680804:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.680807:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.680809:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.680811:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.680812:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.680813:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.680814:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.680815:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.680815:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.680816:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.680820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.680826:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.680828:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.680829:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.680830:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.680832:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.680833:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.680981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.680986:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.680989:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.680991:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.680992:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.680994:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.680996:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.680997:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.680998:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.680999:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.681000:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.681004:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.681006:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.681009:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.681010:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.681011:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.681013:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.681013:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.681014:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.681014:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.681018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.681025:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.681027:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.681028:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.681030:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.681031:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.681032:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.681036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.681042:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.681045:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.681046:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.681047:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.681047:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.2:1619677448.681047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677448.681049:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.681053:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.681056:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.681056:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.681059:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.681060:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.681062:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.681063:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.681064:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.681065:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.681066:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.681070:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.681072:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.681075:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.681075:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.681076:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.681078:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.681078:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.681079:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.681079:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.681084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.2:1619677448.681089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:24.0:1619677448.681090:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.681093:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.681094:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.681094:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:26.0:1619677448.681096:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:24.0:1619677448.681096:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.681099:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:24.0:1619677448.681099:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:10.0:1619677448.681100:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.681102:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.681102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000400:00000200:26.0:1619677448.681104:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.681106:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.681107:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.681108:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.681108:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000400:00000200:26.0:1619677448.681109:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.681110:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:24.0:1619677448.681111:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.681111:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677448.681112:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.681113:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.681113:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:26.0:1619677448.681114:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.681116:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677448.681118:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.681119:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.681119:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.681121:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.681121:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.681122:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.681122:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.681126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.681133:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.681135:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.681136:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.681138:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.681140:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.681140:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:21.2:1619677448.681278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.681285:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.681287:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.681290:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.681291:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.681292:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.681298:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.681299:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.681300:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.681300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:30.0:1619677448.681301:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.681302:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.681303:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.681305:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:30.0:1619677448.681306:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.681308:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.681308:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.681309:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677448.681311:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.681311:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.681312:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.681312:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.681313:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677448.681313:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677448.681314:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.681315:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:20.0:1619677448.681315:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.681316:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.681317:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.681317:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677448.681317:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.681318:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:25.2:1619677448.681319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.681321:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.681323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.681323:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.2:1619677448.681325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:20.0:1619677448.681326:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.681327:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.681327:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.681328:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:20.0:1619677448.681330:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (25)--
00000800:00000200:20.0:1619677448.681331:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677448.681331:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:2.0:1619677448.681331:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677448.681332:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677448.681332:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:12.0:1619677448.681332:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.681333:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.681334:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:12.0:1619677448.681334:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.681334:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:31.0:1619677448.681335:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.681335:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677448.681335:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:13.0:1619677448.681335:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:31.0:1619677448.681337:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.681337:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.681337:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677448.681338:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.681338:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677448.681338:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.681339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:13.0:1619677448.681339:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.681340:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.681341:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:13.0:1619677448.681342:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677448.681343:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.681344:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:2.0:1619677448.681344:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.681345:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.681346:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.681347:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.681347:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.681348:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:2.0:1619677448.681348:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.681350:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.681352:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.681353:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.681353:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.681355:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.681355:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.681356:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.681356:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.681361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.681368:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.681370:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.681371:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.681371:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.681372:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.681374:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.681378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.681386:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.681388:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.681388:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.681389:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.681389:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.681390:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.681438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677448.681442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:10.0:1619677448.681446:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.681448:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.681449:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677448.681449:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677448.681451:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.681452:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:10.0:1619677448.681453:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.681453:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.681454:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.681455:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.681456:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.681457:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.681457:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.681458:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.681459:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.681460:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.681460:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:24.0:1619677448.681462:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.681463:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.681464:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.681464:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.681467:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:24.0:1619677448.681468:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.681470:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.681470:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.681471:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.681471:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:24.0:1619677448.681473:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.681473:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.681473:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677448.681474:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.681474:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.681474:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.681475:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:24.0:1619677448.681476:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.681476:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.681477:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.681477:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677448.681479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:8.2:1619677448.681482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677448.681487:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.681490:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:6.0:1619677448.681490:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.681491:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.681491:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.681493:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:6.0:1619677448.681493:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.681494:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.681494:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:6.0:1619677448.681496:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:17.0:1619677448.681497:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.681500:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.681510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.681515:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.681518:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.681519:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.681521:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.681523:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.681524:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.681526:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.681527:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.681528:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.681529:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.681533:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.681535:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.681538:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.681538:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.681539:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.681541:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.681542:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.681542:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.681543:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.681547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.681554:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.681556:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.681557:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.681559:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.681560:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.681562:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.681565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.681572:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.681574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.681575:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.681576:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.681576:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.681577:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.681578:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:31.0:1619677448.681579:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.681582:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.681584:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.681586:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.681586:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.681587:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.681589:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711712] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.681590:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.681591:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.681592:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.681595:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.681597:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.681599:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.681600:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.681600:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.681602:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.681602:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.681604:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.681605:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.681606:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.681607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.2:1619677448.681611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.681616:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.681617:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.681618:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.681619:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:30.0:1619677448.681619:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.681620:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.681621:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.681622:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.681623:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.681624:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.2:1619677448.681650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.2:1619677448.681651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:1.0:1619677448.681656:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.681658:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:1.0:1619677448.681659:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.681660:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677448.681660:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.681661:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:1.0:1619677448.681662:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.681664:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.681665:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.681666:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.681666:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.681668:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677448.681668:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.681670:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.681671:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.681672:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.681672:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.681673:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.681674:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.681675:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.681678:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.681680:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.681680:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.681683:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.681683:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.681684:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.681684:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.681686:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.681686:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.681686:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.681687:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.681688:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.681688:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.681689:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:8.0:1619677448.681690:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677448.681690:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.681692:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.681695:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:5.2:1619677448.681695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:9.2:1619677448.681696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.681696:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.681697:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.681698:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:17.0:1619677448.681701:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.681701:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.681702:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.681702:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:17.0:1619677448.681703:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.681703:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:21.2:1619677448.681704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:18.0:1619677448.681704:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677448.681704:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.681705:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.681705:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677448.681707:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:17.0:1619677448.681707:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677448.681707:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.681708:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.681710:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:18.0:1619677448.681710:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.681713:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.681713:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:18.0:1619677448.681713:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:24.0:1619677448.681715:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.681715:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.681723:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.681726:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.681727:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.681727:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.681728:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.681728:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.681791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.681796:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.681799:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.681801:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.681801:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.681804:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.681806:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.681807:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.681809:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.681810:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.681811:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.681815:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.681816:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.681819:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.681820:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.681821:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.681823:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.681823:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.681824:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.681825:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.681828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.681835:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.681838:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.681839:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.681841:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.681842:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.681844:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.681968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:26.0:1619677448.681975:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677448.681978:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:10.0:1619677448.681979:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:26.0:1619677448.681981:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.681984:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.681986:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677448.681987:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.681988:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.681989:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.681990:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677448.681994:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.681996:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677448.681998:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f90d380] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.681999:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e49d37c00] (20)++
00000800:00000200:26.0:1619677448.681999:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e49d37c00] (21)++
00000800:00000200:26.0:1619677448.682001:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e49d37c00] (22)--
00000800:00000200:26.0:1619677448.682001:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677448.682002:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75ca00] (22)++
00000800:00000200:26.0:1619677448.682004:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (23)--
00000800:00000200:26.0:1619677448.682004:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:24.2:1619677448.682007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (21)++
00000800:00000200:24.0:1619677448.682010:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)++
00000800:00000200:5.2:1619677448.682010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:24.0:1619677448.682012:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e49d37c00] (23)--
00000400:00000200:24.0:1619677448.682013:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.682013:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)--
00000400:00000200:24.0:1619677448.682015:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.682017:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)--
00000800:00000200:10.0:1619677448.682017:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)++
00000800:00000200:10.0:1619677448.682019:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75ca00] (23)--
00000800:00000200:10.0:1619677448.682019:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677448.682021:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:21.2:1619677448.682023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.682030:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.682033:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.682035:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.682037:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.682037:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.682037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:30.0:1619677448.682038:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677448.682039:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:30.0:1619677448.682040:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.682041:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.682042:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.682042:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:30.0:1619677448.682043:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.682044:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.682045:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.682047:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.682048:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.682049:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677448.682050:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.682051:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.682052:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.682053:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.682053:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:20.0:1619677448.682053:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.682054:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.682054:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.682055:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:21.2:1619677448.682055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:20.0:1619677448.682058:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.682059:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.682060:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.682060:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.682061:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.682061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.682061:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677448.682063:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.682063:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.682064:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.0:1619677448.682064:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.682064:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:20.0:1619677448.682066:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.682066:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.682067:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:12.0:1619677448.682067:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:6.0:1619677448.682067:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.682068:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:6.0:1619677448.682069:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.682070:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.682071:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.682071:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:25.2:1619677448.682072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:6.0:1619677448.682072:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96711705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.682073:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.682074:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:6.0:1619677448.682074:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.682075:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:6.0:1619677448.682075:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.682076:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.682078:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:6.0:1619677448.682080:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.682081:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.682082:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.682082:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.682083:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.682084:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.682085:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.682085:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.682086:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:6.0:1619677448.682086:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.682088:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.682088:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.682089:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.682089:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.682094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.682103:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.682105:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.682106:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.682107:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.682108:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.682110:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.682152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.682159:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.682161:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.682162:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.682164:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.682166:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.682168:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.682169:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.682171:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.682172:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.682173:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.682176:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.682178:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.682181:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.682182:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.682183:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.682184:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.682185:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.682185:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.682186:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.682190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.682198:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.682201:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.682202:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.682204:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.682204:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.682207:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.682210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.682218:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.682220:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.682221:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.682222:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.682222:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.682223:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.682412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.682420:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.682422:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.682423:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.682425:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.682428:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.682429:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.682431:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.682432:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.682433:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.682434:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.682437:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.682438:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.682441:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.682442:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.682443:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.682444:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.682445:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.682445:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.682445:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.682450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.682459:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.682462:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.682462:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.682463:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.682465:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.682468:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.682478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.682485:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.682488:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.682488:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.682489:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.682490:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.682490:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.682588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.682596:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.682598:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.682599:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.682601:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.682604:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.682605:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.682606:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711702] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.682608:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.682609:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.682610:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.682613:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.682614:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.682617:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.682618:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.682618:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.682620:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.682621:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.682621:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.682622:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677448.682623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.2:1619677448.682626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.682631:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.682633:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.682634:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.0:1619677448.682634:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:26.0:1619677448.682635:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.682636:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.682637:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.682638:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.682639:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677448.682640:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.682640:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.682641:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.682642:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.682643:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.682644:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.682646:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:26.0:1619677448.682648:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.682649:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.682652:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.682652:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.682653:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.682655:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.682655:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.682655:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.682656:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.682661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.682663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:13.0:1619677448.682669:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.682669:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:13.0:1619677448.682671:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.682672:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.682672:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.682673:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:13.0:1619677448.682674:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.682675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:12.0:1619677448.682676:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677448.682676:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.682677:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677448.682677:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.682679:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:25.2:1619677448.682680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:2.0:1619677448.682680:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.682682:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.682683:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.682684:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:2.0:1619677448.682684:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:12.0:1619677448.682686:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.682687:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:2.0:1619677448.682688:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.682689:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.682690:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.682690:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677448.682691:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.682691:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.682691:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.682691:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.682693:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.682693:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:2.0:1619677448.682694:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.682695:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:2.0:1619677448.682695:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.682695:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:30.0:1619677448.682696:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.682697:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:30.0:1619677448.682698:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.682698:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:30.0:1619677448.682699:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.682699:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000400:00000200:30.0:1619677448.682700:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:2.0:1619677448.682700:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.682701:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:30.0:1619677448.682703:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.682703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.682705:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.2:1619677448.682706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:30.0:1619677448.682708:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.682709:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.682709:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:30.0:1619677448.682710:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677448.682711:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:18.0:1619677448.682711:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:30.0:1619677448.682712:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.682712:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.682712:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:30.0:1619677448.682713:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.682713:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677448.682713:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:10.0:1619677448.682713:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.682714:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.682715:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:10.0:1619677448.682715:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.682717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.682718:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:17.0:1619677448.682725:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.682728:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.682728:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.682729:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.682731:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.682734:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.682751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.682756:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.682759:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.682760:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.682762:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.682765:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.682766:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677448.682768:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.682769:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.682770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:2.0:1619677448.682770:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.682771:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677448.682775:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.682777:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.682777:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677448.682778:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:18.0:1619677448.682781:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.682781:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.682782:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.682782:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677448.682783:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:18.0:1619677448.682784:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.682785:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.682785:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000400:00000200:18.0:1619677448.682787:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.682787:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.682788:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.682789:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.682790:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677448.682791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677448.682791:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:18.0:1619677448.682792:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.682792:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.682794:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.682797:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:18.0:1619677448.682797:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.682798:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.682799:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.682800:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:18.0:1619677448.682800:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.682800:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.682801:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.682801:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.682801:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.682802:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:26.0:1619677448.682803:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.682806:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:25.2:1619677448.682806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.682813:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.682815:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.682816:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.682817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:13.0:1619677448.682818:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.682819:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.682821:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.682823:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.682826:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.682827:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.682828:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.682830:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.682831:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.682833:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.682834:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.682835:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.682836:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.682839:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.682840:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.682844:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.682844:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.682845:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.682847:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.682847:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.682848:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.682849:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.682853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.682859:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.682862:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.682863:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.682864:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.682866:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.682867:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.682870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.682877:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.682880:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.682880:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.682882:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.682885:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.682886:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.682887:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.682888:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.682889:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.682890:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.682893:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.682895:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.682898:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.682898:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.682899:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.682901:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.682901:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.682902:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.682904:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.682904:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677448.682910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.682916:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.682918:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:25.2:1619677448.682918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.682919:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.682920:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.682926:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.682928:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.682929:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.682930:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.682932:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.682935:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.683003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677448.683010:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.683013:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677448.683014:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677448.683015:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.683018:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.683020:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677448.683021:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.683022:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.683023:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.683024:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.683027:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.683028:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.683031:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.683032:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:10.0:1619677448.683032:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.683034:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.683035:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.683035:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677448.683037:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677448.683037:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.2:1619677448.683040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:21.2:1619677448.683043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:20.0:1619677448.683048:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:26.0:1619677448.683050:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:20.0:1619677448.683051:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.683051:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.683052:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677448.683052:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677448.683053:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:20.0:1619677448.683053:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.683055:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.683056:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.683145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.683150:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.683153:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.683154:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.683156:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.683158:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.683159:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.683160:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.683162:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.683163:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.683164:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.683167:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.683169:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.683172:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.683172:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.683173:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.683175:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.683175:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.683177:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.683178:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.683178:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:25.2:1619677448.683181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:21.2:1619677448.683183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.683187:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:22.0:1619677448.683188:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:10.0:1619677448.683189:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.683190:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:22.0:1619677448.683190:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.683191:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677448.683191:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.683192:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677448.683193:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.683196:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:25.2:1619677448.683198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000800:00000200:26.0:1619677448.683204:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.683206:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.683207:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.683207:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.683209:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:4.2:1619677448.683209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.683213:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:26.0:1619677448.683214:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.683216:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677448.683218:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.683218:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.683221:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.683223:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:21.2:1619677448.683223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:26.0:1619677448.683224:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.683225:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.683226:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.683227:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.683230:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677448.683231:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.683233:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677448.683233:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:31.0:1619677448.683236:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.683236:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.683237:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.683237:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.683237:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:31.0:1619677448.683239:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.683239:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.683240:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.683240:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677448.683241:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:26.0:1619677448.683241:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:31.0:1619677448.683242:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.683243:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.683244:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.683245:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:25.2:1619677448.683245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:31.0:1619677448.683248:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.683249:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.683252:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.683253:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.683253:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.683253:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.683253:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.683254:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:30.0:1619677448.683255:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.683256:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.683256:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.683257:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.683257:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.683258:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677448.683258:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.683262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.683269:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.2:1619677448.683271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:6.0:1619677448.683272:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.683273:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.683273:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.683275:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.683278:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677448.683279:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.683282:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.683284:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.683285:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.683287:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.683287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000400:00000200:31.0:1619677448.683288:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.683290:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.683291:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.683292:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.683294:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:31.0:1619677448.683298:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:6.0:1619677448.683299:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:31.0:1619677448.683302:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.683302:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.683303:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.683304:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:22.0:1619677448.683304:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.683304:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677448.683307:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.683309:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.683309:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.683311:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.683312:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.683312:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.683313:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.683318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.683326:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.683329:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.683330:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.683331:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.683332:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.683334:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.683455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.683462:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:5.2:1619677448.683464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.683465:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677448.683467:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.683468:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.683469:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.683470:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.683470:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.683472:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.683472:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.683474:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.683475:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.683476:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.683477:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.683479:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.683480:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.683481:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.683482:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711688] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.683483:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.683483:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.683484:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.683485:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.683487:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.683489:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.683491:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.683491:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.683492:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.683492:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.683493:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.683493:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.683495:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.683495:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.683495:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.683495:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.683496:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (23)++
00000800:00000200:10.0:1619677448.683497:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677448.683498:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (24)--
00000800:00000200:26.0:1619677448.683498:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.683498:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.683499:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.683499:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677448.683501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:8.2:1619677448.683501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.683504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.2:1619677448.683509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:17.0:1619677448.683509:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.683512:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:10.0:1619677448.683512:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:17.0:1619677448.683513:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.683513:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.683514:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:17.0:1619677448.683515:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.683515:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.683516:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.683516:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.683517:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.683518:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:10.0:1619677448.683518:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.683519:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:26.0:1619677448.683520:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.683521:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.683521:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.683522:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.683522:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677448.683523:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:5.2:1619677448.683530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:4.2:1619677448.683533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677448.683536:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.683538:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.683540:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:10.0:1619677448.683541:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677448.683542:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.683543:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.683544:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677448.683544:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.683546:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.683548:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.683548:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.683549:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.683549:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677448.683550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:2.0:1619677448.683550:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.683551:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.683551:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.683553:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.683555:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.683556:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.683556:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.683557:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:24.0:1619677448.683558:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677448.683558:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677448.683561:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.683561:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677448.683561:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.683562:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.683562:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.683563:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677448.683564:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.683564:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.2:1619677448.683565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.683565:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.683565:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:24.0:1619677448.683566:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.683566:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.683567:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:24.0:1619677448.683568:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:10.0:1619677448.683568:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677448.683569:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96711685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.683569:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.683570:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677448.683570:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:25.2:1619677448.683571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677448.683571:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.683571:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.683571:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:24.0:1619677448.683572:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.683572:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.683573:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.683573:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677448.683573:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.683574:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677448.683576:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.683577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677448.683578:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677448.683578:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677448.683579:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.683581:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.683581:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:13.0:1619677448.683581:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:24.0:1619677448.683582:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:13.0:1619677448.683582:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.683582:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.683583:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.683584:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.683584:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677448.683584:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:24.0:1619677448.683585:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:13.0:1619677448.683585:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.683585:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:12.0:1619677448.683586:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:1.0:1619677448.683586:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96711684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.683587:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.683588:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.683588:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:1.0:1619677448.683588:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.683589:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.2:1619677448.683589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:6.0:1619677448.683589:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.683589:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:6.0:1619677448.683592:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.683595:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:1.0:1619677448.683595:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.683598:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.683598:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.683600:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.683601:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.683601:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.683602:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:17.0:1619677448.683603:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.683603:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.683605:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.683606:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.683606:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.683607:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.683608:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677448.683608:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.683611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.683618:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.683620:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.683621:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.683623:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.683624:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.683626:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.683628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:25.2:1619677448.683634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.683634:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.683637:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677448.683637:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677448.683639:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.683640:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:17.0:1619677448.683642:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.683643:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.683644:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:13.0:1619677448.683644:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.683644:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:17.0:1619677448.683645:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.683645:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:17.0:1619677448.683646:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.683646:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:17.0:1619677448.683647:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.683648:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677448.683650:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.683652:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.683654:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.683655:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:17.0:1619677448.683655:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.683657:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677448.683657:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.683659:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677448.683660:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677448.683660:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:25.2:1619677448.683663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.683664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677448.683670:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:30.0:1619677448.683672:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677448.683673:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.683674:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.683674:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:30.0:1619677448.683675:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.683677:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.683677:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.683679:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677448.683679:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:30.0:1619677448.683680:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.683682:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.683683:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711682] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.683685:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.683686:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.683687:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.683690:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.683692:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.683696:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.683696:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.683697:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.683699:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.683699:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.2:1619677448.683699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:30.0:1619677448.683701:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.683703:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.683703:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.683704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677448.683705:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677448.683707:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677448.683708:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:21.2:1619677448.683709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.683709:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:6.0:1619677448.683712:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.683715:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.683715:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.683716:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.683717:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:6.0:1619677448.683718:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.683719:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.683720:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.683721:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.683721:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.683967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.683974:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.683977:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.683978:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.2:1619677448.683978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677448.683980:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.683982:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.683983:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.683984:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.2:1619677448.683984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:26.0:1619677448.683985:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.683986:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.683986:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.683987:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.683988:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.683989:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677448.683989:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.683991:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.683991:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.683993:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:31.0:1619677448.683994:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:26.0:1619677448.683994:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.683995:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.683996:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:17.0:1619677448.683996:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.683997:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.683998:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:17.0:1619677448.683998:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.683999:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.683999:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677448.684000:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.684000:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.684000:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:31.0:1619677448.684002:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:26.0:1619677448.684002:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.684002:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.684003:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677448.684003:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.684004:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96711679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.684004:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:31.0:1619677448.684005:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.684005:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:31.0:1619677448.684007:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.684008:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:25.2:1619677448.684008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.684008:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.684010:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.684010:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:31.0:1619677448.684012:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.684012:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.684012:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.684013:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.684013:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.684014:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.684015:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:31.0:1619677448.684016:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.684017:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.684017:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677448.684018:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:9.2:1619677448.684018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.684019:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.684019:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:22.0:1619677448.684019:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.684019:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.684020:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.684021:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:22.0:1619677448.684021:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.684024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.684024:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:22.0:1619677448.684025:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677448.684027:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:25.2:1619677448.684028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677448.684028:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.684028:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.684030:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.684031:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.684033:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:8.0:1619677448.684033:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000400:00000200:12.0:1619677448.684034:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.684035:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.684035:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:12.0:1619677448.684037:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:6.0:1619677448.684038:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.684039:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:13.0:1619677448.684039:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677448.684039:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.684040:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.684041:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.684050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.2:1619677448.684056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:10.0:1619677448.684057:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.684060:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.684061:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677448.684062:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:10.0:1619677448.684063:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.684065:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:10.0:1619677448.684066:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.684066:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.684068:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677448.684068:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.684070:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.684071:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.684071:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.684072:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.684073:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.684073:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677448.684074:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.684076:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.684077:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.684078:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.684078:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.684081:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:18.0:1619677448.684083:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677448.684084:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.684085:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.684085:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.684085:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.684087:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.684087:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677448.684088:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.684088:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677448.684089:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.684089:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.684089:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:18.0:1619677448.684091:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.684091:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.684092:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677448.684093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677448.684093:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.684096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.684097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677448.684102:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.684104:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.684104:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:30.0:1619677448.684105:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:22.0:1619677448.684105:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.684105:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.684106:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.684107:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.684109:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:6.0:1619677448.684109:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:30.0:1619677448.684111:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.684111:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.684111:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.684112:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.684113:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.684113:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.684113:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677448.684114:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.684114:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:30.0:1619677448.684115:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677448.684115:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000400:00000200:30.0:1619677448.684116:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.684116:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.684116:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:30.0:1619677448.684118:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.684119:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.684121:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.684124:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.684126:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.684129:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.684130:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.684131:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.684132:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.684133:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.684134:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.684134:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.684138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.684146:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.684149:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.684150:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.684150:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.684152:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.684155:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.684162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.684168:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.684170:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.684171:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.684173:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.684176:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.684177:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.684178:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.684180:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.684181:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.684182:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.684185:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.684187:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.684190:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.684191:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.684192:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.684193:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.684194:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.684195:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.684195:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.684199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.684206:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.684208:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.684209:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.684211:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.684212:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.684214:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.684256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.684262:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.684264:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.684265:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.684267:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.684270:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.684272:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.684273:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.684275:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.684276:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.684277:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.684280:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.684282:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.684285:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.684286:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.684287:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.684288:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.684289:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.684290:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.684290:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.684296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.684303:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.684305:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.684306:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.684307:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.684309:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.684312:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.684323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.684330:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.684333:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.684334:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.684335:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.684338:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.684340:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.684341:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.684342:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.684343:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.684344:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.684348:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.684349:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.684353:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.684353:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.684354:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.684356:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.684356:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.684357:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.684358:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.684359:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.684362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.684364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.684369:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.684370:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.684371:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.684371:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677448.684372:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.684373:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.684374:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.684376:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.684378:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.684379:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.684379:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.684380:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.684380:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.684381:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.684381:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.684528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.684536:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.684539:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.684541:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.684544:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.684544:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.684546:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.684547:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.684548:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.684549:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.684550:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.684553:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.684555:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.684558:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.684559:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.684560:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.684561:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.684562:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.684563:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.684563:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.684568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.684577:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.684579:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.684581:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.684582:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.684583:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.684586:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.684616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.684622:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.684624:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.684625:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.684627:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.684630:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.684631:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.684633:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.684634:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.684635:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.684636:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.684640:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.684641:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.684644:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.684645:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.684646:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.684647:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.684648:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.684649:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.684649:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.684654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.684661:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.684664:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.684665:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.684665:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.684667:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.684670:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.684694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.684702:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.684705:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.684705:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.684707:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.684710:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.684711:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.684713:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.684714:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.684715:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.684716:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.684720:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.684722:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.684724:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.684725:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.684725:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.684727:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.684727:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.684728:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.684729:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.684733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.684739:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.684741:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.684742:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.684744:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.684745:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.684746:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.684902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.684908:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.2:1619677448.684909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:5.2:1619677448.684913:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677448.684914:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:18.0:1619677448.684915:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677448.684916:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:17.0:1619677448.684918:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677448.684918:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.684919:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.684919:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:17.0:1619677448.684921:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:5.0:1619677448.684921:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.684921:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677448.684922:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.684923:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.684924:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.684925:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.684925:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.684926:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.684927:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.684927:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.684927:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.684928:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.684929:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:5.0:1619677448.684929:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.684930:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.684931:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.684932:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.684932:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.684933:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.684934:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.684935:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:5.0:1619677448.684935:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.684938:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:18.0:1619677448.684940:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.684940:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.684942:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.684943:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.684943:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677448.684943:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.684944:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.684945:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:18.0:1619677448.684946:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.684946:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.684946:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:18.0:1619677448.684947:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.684947:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.684947:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:5.0:1619677448.684947:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677448.684948:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.684949:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677448.684950:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.684951:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677448.684951:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:9.2:1619677448.684951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.684952:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.684953:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:17.0:1619677448.684953:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.684954:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677448.684954:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.684956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677448.684958:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:25.2:1619677448.684960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:24.0:1619677448.684960:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.684962:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.684962:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:13.0:1619677448.684963:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:24.0:1619677448.684964:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.684965:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.684966:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.684967:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:22.0:1619677448.684968:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:13.0:1619677448.684968:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.684969:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.684970:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677448.684971:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.684972:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.684972:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:9.2:1619677448.684973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000400:00000200:22.0:1619677448.684974:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.684976:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677448.684979:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.684981:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.684982:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.684983:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.684984:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.684984:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.685104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:24.0:1619677448.685111:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677448.685114:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677448.685115:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677448.685116:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.685119:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.685121:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677448.685122:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.685123:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.685124:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.685125:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677448.685129:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.685130:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.2:1619677448.685131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.0:1619677448.685133:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.685133:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:24.0:1619677448.685134:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:24.0:1619677448.685136:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:24.0:1619677448.685136:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677448.685137:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:1.0:1619677448.685137:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.685138:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:17.0:1619677448.685140:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.685140:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:25.2:1619677448.685142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:1.0:1619677448.685143:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.685145:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.685146:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.685147:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.685149:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:1.0:1619677448.685149:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.685150:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.685151:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:1.0:1619677448.685151:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677448.685152:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.685155:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.685155:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:1.0:1619677448.685155:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.685158:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:1.0:1619677448.685158:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.685160:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677448.685161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:1.0:1619677448.685161:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.685161:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.685163:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.685163:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.685164:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.685165:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.685167:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:25.2:1619677448.685169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.685170:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.685171:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.685172:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.685173:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.685174:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:6.0:1619677448.685177:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.685179:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.685180:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.685180:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.685182:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.685184:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.685212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.685215:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.685217:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.685220:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.685221:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.685222:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.685224:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.685225:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.685227:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.685228:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.685229:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.685232:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.685234:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.685237:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.685238:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.685239:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.685241:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.685242:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.685243:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.685243:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.2:1619677448.685244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:25.2:1619677448.685247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.685251:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.685253:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677448.685254:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.685255:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:12.0:1619677448.685256:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:26.0:1619677448.685257:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677448.685257:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.685259:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.685259:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.685259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:13.0:1619677448.685260:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:26.0:1619677448.685261:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:12.0:1619677448.685261:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677448.685263:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.685264:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.685266:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.685267:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.685267:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.685268:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.685271:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.685272:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.685273:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.685274:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.685275:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.685276:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.685277:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.685278:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.685279:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.685281:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.685283:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.685283:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:10.0:1619677448.685283:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.685284:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.685285:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.685286:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.685287:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.685287:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.685288:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.685288:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.685289:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.685289:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:25.2:1619677448.685291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.685291:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.685292:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.685292:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:25.2:1619677448.685300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.685302:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.685302:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.685303:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.685306:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.685306:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.685308:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.685309:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:4.2:1619677448.685309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.685310:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.685310:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:22.0:1619677448.685311:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.685312:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.685313:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.685315:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.685316:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.685318:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677448.685318:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:26.0:1619677448.685319:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.685320:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:7.0:1619677448.685320:0:19929:0:(o2iblnd_cb.c:3440:kiblnd_check_conns()) conn[ffff91a7c1e0ea00] (20)++
00000800:00000200:7.0:1619677448.685325:0:19929:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c1e0ea00] (21)++
00000800:00000200:7.0:1619677448.685329:0:19929:0:(o2iblnd_cb.c:3475:kiblnd_check_conns()) conn[ffff91a7c1e0ea00] (22)--
00000800:00000200:5.2:1619677448.685373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.685379:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.685382:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.685383:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.685385:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.685388:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.685389:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.685390:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.685392:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.685393:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.685394:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.685397:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.685399:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.685401:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.685402:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.685403:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.685405:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.685405:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.685406:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.685407:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.685408:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.685412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.685413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.685419:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.685421:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.685421:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677448.685421:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677448.685422:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677448.685424:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.685425:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.685426:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.685427:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.685428:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.685429:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:20.0:1619677448.685430:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:5.0:1619677448.685430:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.685431:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.685432:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.685453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.685459:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677448.685460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677448.685462:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.685463:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.685465:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.685467:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.685468:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677448.685468:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.685470:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.685471:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:17.0:1619677448.685471:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.685472:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.685473:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.685474:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.685475:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.685477:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.685477:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.685478:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:31.0:1619677448.685479:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.685480:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.685481:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.685482:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.685482:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:31.0:1619677448.685483:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.685483:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:31.0:1619677448.685484:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:17.0:1619677448.685484:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.685485:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.685486:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:31.0:1619677448.685487:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.685487:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.685487:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.685489:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:25.2:1619677448.685491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.685492:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.685492:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.2:1619677448.685492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.685493:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.685495:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.685496:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.685497:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.685497:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677448.685498:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677448.685499:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.685500:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.685500:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.685501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:6.0:1619677448.685501:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:1.0:1619677448.685501:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.685502:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.685502:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.685504:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.685506:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:26.0:1619677448.685507:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.685509:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:25.2:1619677448.685509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000400:00000200:26.0:1619677448.685510:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.685510:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.685512:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.685515:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:22.0:1619677448.685516:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.685519:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.685519:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.685520:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.685521:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.685522:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.685574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:10.0:1619677448.685581:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677448.685584:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:26.0:1619677448.685585:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:10.0:1619677448.685587:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.685589:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.685590:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677448.685591:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.685592:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.685594:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.685594:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677448.685598:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.685600:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677448.685602:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f90d380] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.685603:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e49d37c00] (20)++
00000800:00000200:10.0:1619677448.685604:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e49d37c00] (21)++
00000800:00000200:10.0:1619677448.685605:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e49d37c00] (22)--
00000800:00000200:10.0:1619677448.685605:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677448.685606:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677448.685607:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:24.2:1619677448.685611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (21)++
00000800:00000200:24.0:1619677448.685614:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)++
00000800:00000200:24.0:1619677448.685617:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e49d37c00] (23)--
00000400:00000200:24.0:1619677448.685618:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.685618:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)--
00000400:00000200:24.0:1619677448.685620:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.685622:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)--
00000800:00000200:20.2:1619677448.685629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.685632:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.685634:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.685636:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.685637:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.685639:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.685640:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677448.685642:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.685644:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.685645:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.685646:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677448.685649:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.685651:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677448.685654:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.685655:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677448.685656:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.685658:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677448.685659:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.685660:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.685661:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.685662:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.685664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.2:1619677448.685667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677448.685671:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:5.0:1619677448.685672:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:30.0:1619677448.685674:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:5.0:1619677448.685674:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:30.0:1619677448.685675:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.685675:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.685676:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:30.0:1619677448.685677:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677448.685678:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.685680:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.685695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.685700:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.685703:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.685705:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.685706:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.685708:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.685709:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677448.685711:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.685712:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.685713:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.685714:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677448.685716:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.685718:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.685721:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.685722:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.685722:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.685724:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.685725:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.685725:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.685726:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.685730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.685738:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.685741:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.685742:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.685742:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.685744:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.685747:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.685800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.685807:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.685810:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.685810:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677448.685812:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.685815:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.685816:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.685818:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.685819:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.685820:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.685821:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.685824:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.685825:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.685828:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.685829:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.685830:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.685831:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.685832:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.685832:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.685833:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.685837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.685844:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.685847:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.685848:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.685849:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.685851:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.685852:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.685889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.685897:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.685899:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.685900:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.685902:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.685904:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.685906:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.685907:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.685908:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.685909:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.685910:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.685913:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.685914:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.685917:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.685918:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.685919:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.685920:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.685921:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.685921:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.685922:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.685926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.685934:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.685937:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.685938:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.685939:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.685940:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.685943:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.685967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.685972:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.685975:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.685977:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.685977:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.685979:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.685981:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.685982:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.685983:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.685984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:5.0:1619677448.685984:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.685985:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.685988:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.685990:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.685992:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.685994:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.685994:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.685995:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:5.0:1619677448.685995:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.685997:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.685997:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.685998:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677448.685998:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.685999:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.685999:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.686000:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000400:00000200:31.0:1619677448.686001:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:5.0:1619677448.686001:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677448.686002:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.686003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.686004:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.686005:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.686005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.686006:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:20.0:1619677448.686008:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:31.0:1619677448.686009:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.686010:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:13.0:1619677448.686010:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677448.686011:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677448.686011:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.686012:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.686012:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677448.686013:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677448.686013:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.686014:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.686014:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:13.0:1619677448.686015:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.686016:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.686016:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.686016:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.686018:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:13.0:1619677448.686018:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.686019:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.686020:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.686021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:25.2:1619677448.686025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.2:1619677448.686026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.686031:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:12.0:1619677448.686031:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.686032:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.686033:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.686034:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:30.0:1619677448.686034:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.686034:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.686035:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:12.0:1619677448.686035:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.686035:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677448.686036:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:12.0:1619677448.686036:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.686037:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:12.0:1619677448.686037:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.686038:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.686039:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:21.2:1619677448.686043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.686050:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.686052:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.2:1619677448.686052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:4.2:1619677448.686052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:31.0:1619677448.686054:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.686056:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.686057:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.686058:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677448.686058:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.686059:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.686059:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.686061:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.686061:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:10.0:1619677448.686061:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:1.0:1619677448.686061:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:31.0:1619677448.686062:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.686062:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.686063:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:31.0:1619677448.686064:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.686065:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.686066:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.686066:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.686067:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.686069:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.686069:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.686069:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.686070:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:31.0:1619677448.686071:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.686072:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.686073:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.686073:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:31.0:1619677448.686074:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:10.0:1619677448.686074:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.686074:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.686075:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.686076:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.686076:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.686076:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.686077:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677448.686077:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.686077:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.686078:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.686078:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:10.0:1619677448.686078:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:25.2:1619677448.686081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677448.686082:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.686083:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.686084:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:1.0:1619677448.686085:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.686086:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96711648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.686087:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:17.0:1619677448.686088:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.686088:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:17.0:1619677448.686089:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.686090:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:10.0:1619677448.686090:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.686091:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:13.0:1619677448.686091:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.686091:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.686092:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:13.0:1619677448.686093:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.686094:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.686094:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.686094:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.686094:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.686095:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.686095:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.2:1619677448.686095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677448.686095:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:13.0:1619677448.686096:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.686096:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.686098:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.686098:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677448.686098:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.686098:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)++
00000800:00000200:17.0:1619677448.686100:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.686100:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:9.2:1619677448.686100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.686100:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (24)--
00000800:00000200:1.0:1619677448.686100:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.686101:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.686101:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.686102:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.686103:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.686103:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.686104:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.686104:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.2:1619677448.686104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.686105:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.686105:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.686107:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677448.686107:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:17.0:1619677448.686108:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:10.0:1619677448.686108:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.686109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:21.2:1619677448.686109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:10.0:1619677448.686109:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:17.0:1619677448.686110:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:10.0:1619677448.686110:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96711647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.686110:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:17.0:1619677448.686111:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.686112:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:10.0:1619677448.686112:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.686112:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.686113:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.686113:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.686114:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:10.0:1619677448.686114:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.686115:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.686116:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:17.0:1619677448.686116:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.686117:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.686118:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:10.0:1619677448.686118:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.686120:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677448.686120:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.686121:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.686121:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.686122:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:18.0:1619677448.686123:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.686123:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.686123:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:10.0:1619677448.686124:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.686125:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:6.0:1619677448.686125:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:18.0:1619677448.686126:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.686126:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677448.686128:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:10.0:1619677448.686128:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.686128:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:18.0:1619677448.686129:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.686130:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:18.0:1619677448.686131:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.686131:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.686131:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.686132:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:18.0:1619677448.686133:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:9.2:1619677448.686134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677448.686137:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.686137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:5.2:1619677448.686138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677448.686139:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 504 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677448.686142:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.686143:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.686144:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:2.0:1619677448.686144:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.686145:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:24.0:1619677448.686145:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677448.686145:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:2.0:1619677448.686145:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.686147:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:2.0:1619677448.686147:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.686148:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.686148:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677448.686148:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:18.0:1619677448.686148:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:8.0:1619677448.686148:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677448.686149:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:26.0:1619677448.686150:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:24.0:1619677448.686150:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.686150:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.686151:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677448.686151:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:25.2:1619677448.686153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:24.0:1619677448.686153:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.686155:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.686156:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96711645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.686157:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.686158:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.686159:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:24.0:1619677448.686159:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:30.0:1619677448.686162:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:24.0:1619677448.686162:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.686163:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.686164:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:30.0:1619677448.686165:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.686166:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:24.0:1619677448.686167:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.686168:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:24.0:1619677448.686168:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.686169:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.686170:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.686171:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.686172:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.686174:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.686174:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.686176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:25.2:1619677448.686177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:4.2:1619677448.686180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677448.686183:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.2:1619677448.686185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:1.0:1619677448.686185:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:31.0:1619677448.686186:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.686187:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:8.0:1619677448.686187:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.686187:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:31.0:1619677448.686188:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.686188:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.686188:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.686189:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677448.686189:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.686190:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.0:1619677448.686190:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.686191:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677448.686191:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:1.0:1619677448.686191:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.686194:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.686194:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677448.686194:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:18.0:1619677448.686197:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.686199:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.686200:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:8.2:1619677448.686201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000400:00000200:18.0:1619677448.686202:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.686203:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.686204:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.686205:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.686208:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.686208:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:18.0:1619677448.686210:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.686211:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:18.0:1619677448.686212:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.686212:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.686212:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:18.0:1619677448.686213:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.686214:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.686214:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.686214:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677448.686215:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.686216:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.686216:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.686217:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.686221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.686227:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.686229:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.686230:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.686231:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.686232:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.686233:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.686238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.686245:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.686248:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.686248:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.686248:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.686249:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.686250:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.686287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.686290:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.686292:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.686298:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.686298:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.686301:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.686303:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.686304:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.686305:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.686307:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.686308:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.686311:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.686312:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.686315:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.686316:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.686317:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.686319:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.686319:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.686320:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.686321:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.686325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.686333:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.686335:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.686336:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.686338:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.686340:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.686341:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.686344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.686350:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.686353:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.686354:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.686356:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.686359:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.686360:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.686362:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.686363:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.686364:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.686365:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.686368:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.686370:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.686373:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.686374:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.686375:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.686376:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.686377:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.686378:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.686378:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.686382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.686390:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.686393:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.686394:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.686396:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.686398:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.686399:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.686486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.686493:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.686496:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.686497:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.686498:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.686500:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.686501:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.686503:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.686504:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.686505:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.686506:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.686509:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.686511:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.686512:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.686513:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.686514:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.686515:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.686516:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.686517:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.686518:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.686518:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677448.686521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.686524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.686529:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.686529:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.686530:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.686530:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.686530:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.686531:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.686532:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.686532:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.686533:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677448.686534:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.686674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.686679:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.686682:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.686684:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.686685:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.686688:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.686689:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.686690:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.686692:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.686693:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.686694:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.686696:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.686698:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.686701:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.686702:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.686703:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.686705:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.686705:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.686706:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.686707:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.686711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.686717:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.686720:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.686721:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.686723:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.686724:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.686726:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.686732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677448.686740:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677448.686743:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677448.686744:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.686746:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677448.686747:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.686748:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.686749:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.686751:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.686751:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.686752:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.686755:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.686757:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.686759:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.686760:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.686761:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.686762:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.686763:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677448.686764:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677448.686764:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:25.2:1619677448.686768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.686774:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.686777:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.686778:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.686779:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.686781:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.686782:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.686807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.686815:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.686818:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.686820:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.686821:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.686822:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.686823:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.686824:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.686825:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.686826:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.686827:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.686830:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.686831:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.686834:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.686834:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.686835:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.686836:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.686837:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:4.2:1619677448.686837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677448.686838:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.686838:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.686842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.686845:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.686847:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.686848:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677448.686848:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.686850:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:8.0:1619677448.686850:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.686851:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.686852:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.686853:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.686854:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:12.0:1619677448.686855:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:8.0:1619677448.686855:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.686856:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:8.0:1619677448.686856:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.686857:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.686858:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.686861:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.686862:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.686865:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.686866:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.686866:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.686868:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.686868:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.686869:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.686870:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.686874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.686883:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.686885:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.686886:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.686887:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.686889:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.686891:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.687128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.687134:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.687136:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.687137:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.687139:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.687142:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.687144:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.687145:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.687146:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.687147:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.687148:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.687151:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.687153:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.687155:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.687156:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.687157:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.687158:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.687159:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.687160:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.687160:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.687166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.687174:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.687176:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.687177:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.687178:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.687180:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.687182:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.687217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.687223:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.687225:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.687227:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.687228:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.687231:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.687232:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.687233:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.687234:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.687235:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.687236:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.687239:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.687241:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.687243:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.687244:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.687244:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.687246:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.687246:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.687247:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.687247:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.687252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.687258:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.687260:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.687261:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.687262:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.687262:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.687263:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.687540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.687547:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.687550:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.687551:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.687553:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.687556:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.687558:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.687559:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.687560:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.687561:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.687562:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.687565:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.687567:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.687570:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.687571:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.687571:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.687573:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.687573:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.687574:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.687574:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677448.687574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.2:1619677448.687579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.687583:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.687585:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.687587:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.687587:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:31.0:1619677448.687589:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.687589:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:30.0:1619677448.687590:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677448.687590:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:31.0:1619677448.687591:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677448.687591:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.687592:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.687593:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.687594:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.687595:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.687596:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.687596:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.687599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000400:00000200:31.0:1619677448.687600:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.687601:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.2:1619677448.687602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.687605:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.687605:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.687606:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677448.687606:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.687608:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.687608:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.687609:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.687609:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.687609:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.687610:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.687610:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677448.687610:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.687610:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.687611:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.687612:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.687612:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.687612:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.687614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.687615:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.687616:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.687617:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.687619:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.687620:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.687621:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.687622:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:30.0:1619677448.687624:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.687624:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677448.687625:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.687625:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.687627:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.687628:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.687629:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.687629:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.687629:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:21.2:1619677448.687630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:20.0:1619677448.687630:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677448.687631:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.687631:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.687633:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:4.2:1619677448.687633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.687635:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.687635:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677448.687635:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:25.2:1619677448.687637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.687638:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.687639:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.687641:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.687641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.687641:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677448.687643:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.687643:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.687644:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:12.0:1619677448.687644:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:22.0:1619677448.687645:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.687646:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.687647:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:10.0:1619677448.687647:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.687648:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:22.0:1619677448.687648:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.687648:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.687649:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677448.687650:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.687650:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.687651:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.687651:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677448.687651:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.687652:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.687652:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.687652:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:12.0:1619677448.687653:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677448.687653:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.687655:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.687656:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.687657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:22.0:1619677448.687657:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.687657:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.687658:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:22.0:1619677448.687660:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.687661:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.687662:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:10.0:1619677448.687662:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.687663:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:13.0:1619677448.687663:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.687664:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.687664:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:22.0:1619677448.687665:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:13.0:1619677448.687665:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.687666:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:13.0:1619677448.687666:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:22.0:1619677448.687667:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:13.0:1619677448.687667:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.687668:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.687668:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.687668:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:13.0:1619677448.687669:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.687669:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.2:1619677448.687669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.687671:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:21.2:1619677448.687672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.687672:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.687672:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.687673:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.687677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:6.0:1619677448.687678:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.687680:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.687680:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:5.0:1619677448.687680:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:6.0:1619677448.687681:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.0:1619677448.687682:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.687683:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.687685:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:5.0:1619677448.687685:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.687686:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.687687:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.687687:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:20.0:1619677448.687689:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.687689:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677448.687690:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.687692:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.687776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.687784:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.687787:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.687787:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.687789:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.687792:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.687793:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.687795:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711629] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.687796:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.687797:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.687798:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.687801:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.687803:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.687805:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.687806:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.687806:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.687807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.687808:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.687808:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.687809:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.687809:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.687813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.687813:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.687816:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.687816:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:12.0:1619677448.687819:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677448.687819:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.687821:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.687822:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:1.0:1619677448.687822:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:12.0:1619677448.687823:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.687823:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.687825:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.687826:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:1.0:1619677448.687826:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.687827:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677448.687827:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.687828:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.687831:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.687833:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.687836:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.687837:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.687838:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.687840:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.687840:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.687841:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.687842:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.687845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.687852:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.687855:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.687856:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.687857:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.687858:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.687860:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.687954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.687961:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.687963:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.687964:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:21.2:1619677448.687966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:8.0:1619677448.687966:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.687969:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.687970:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.687972:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.687972:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:8.0:1619677448.687973:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.687974:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.687975:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.687975:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.687976:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:8.0:1619677448.687979:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.687981:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.687982:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:8.0:1619677448.687983:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.687984:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:2.0:1619677448.687984:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.687985:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677448.687985:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:8.0:1619677448.687987:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.687987:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:2.0:1619677448.687987:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.687988:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000400:00000200:2.0:1619677448.687988:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.687989:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:2.0:1619677448.687989:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.687990:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:9.2:1619677448.687993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:2.0:1619677448.687993:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.687996:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.687999:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.688000:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.688001:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:2.0:1619677448.688001:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000800:00000200:2.0:1619677448.688003:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (25)--
00000800:00000200:18.0:1619677448.688004:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (25)--
00000400:00000200:18.0:1619677448.688004:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.688004:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.688005:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:18.0:1619677448.688006:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.688007:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.688007:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.688008:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:20.2:1619677448.688009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677448.688009:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000400:00000200:2.0:1619677448.688009:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.688010:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.688011:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.688011:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.688013:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.688014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.0:1619677448.688016:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.688019:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:18.0:1619677448.688020:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:20.0:1619677448.688021:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.688021:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.688022:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.688023:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.688024:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:5.0:1619677448.688024:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.688025:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.688026:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.688028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:5.0:1619677448.688028:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.688029:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.688030:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.688033:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.688035:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.688035:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.688037:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.688038:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677448.688039:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000800:00000200:2.0:1619677448.688039:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.688040:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:5.0:1619677448.688042:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (23)--
00000800:00000200:5.0:1619677448.688042:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.688042:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677448.688043:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.688044:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677448.688044:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.688044:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:9.2:1619677448.688047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.688055:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.688057:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.688058:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.688058:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.688060:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.688063:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.688101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.688109:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.688112:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.688114:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.688116:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.688116:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.688118:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.688119:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.688120:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.688121:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.688122:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.688125:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.688127:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.688130:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.688130:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.688131:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.688133:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.688133:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.688134:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.688135:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.688138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.688145:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.688147:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.688148:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.688149:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.688151:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.688152:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.688201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.688208:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.688211:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.688212:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.688213:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.688216:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.688217:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.688218:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.688220:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.688220:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.688221:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.688225:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.688226:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.688228:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.688229:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.688230:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.688231:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.688232:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.688232:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.688233:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.688237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.688244:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.688246:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.688247:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.688248:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.688250:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.688251:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.688261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.688268:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.688271:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.688272:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.688273:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.688276:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.688277:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.688279:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711622] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.688280:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.688281:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.688282:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.2:1619677448.688282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:24.0:1619677448.688285:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.688286:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.688288:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:24.0:1619677448.688289:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.688290:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.688290:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.688291:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:24.0:1619677448.688292:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.688292:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677448.688292:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.688299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677448.688299:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.688300:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677448.688300:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.688301:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677448.688302:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.688303:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.688305:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.688306:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.688307:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:2.0:1619677448.688307:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.688309:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:2.0:1619677448.688310:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.688310:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.688311:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.688313:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.688313:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.688316:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677448.688316:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.688317:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.688317:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.688319:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.688320:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.688320:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.688321:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.688326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.688326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:12.0:1619677448.688334:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677448.688335:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.688337:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677448.688338:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:12.0:1619677448.688338:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.688340:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677448.688340:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.688342:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:13.0:1619677448.688342:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.688342:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677448.688343:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.688344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:31.0:1619677448.688345:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.688346:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.688347:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.688348:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.688349:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.688350:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.688352:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.688353:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.688354:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:13.0:1619677448.688354:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.688354:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.688355:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.688356:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.688358:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.688358:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.688359:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.688361:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.688361:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.688362:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.688363:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.688367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.688375:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.688377:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.688378:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.688380:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.688382:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.688383:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.688385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.688392:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.688395:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.688396:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.688397:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.688397:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.688398:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.688478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.688486:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.688490:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.688491:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.688493:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.688495:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.688496:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.688497:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.688499:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.688500:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.688501:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.688504:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.688505:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.688507:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.688508:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.688508:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.688510:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.688510:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.688511:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.688511:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.688515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.688521:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.688523:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.688524:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.688524:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.688526:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.688527:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.688530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:25.2:1619677448.688533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:22.0:1619677448.688536:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:26.0:1619677448.688539:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:22.0:1619677448.688539:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677448.688540:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:26.0:1619677448.688542:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000400:00000200:22.0:1619677448.688542:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.688542:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.688543:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.688543:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.688544:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:22.0:1619677448.688545:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.688547:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:22.0:1619677448.688548:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.688549:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.688550:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.688551:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:22.0:1619677448.688555:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.688557:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.688560:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.688561:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:22.0:1619677448.688562:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677448.688564:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677448.688564:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677448.688565:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677448.688566:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:25.2:1619677448.688570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.688576:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.688579:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.688580:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.688582:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.688582:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.688585:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.688593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:12.0:1619677448.688599:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.688602:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.688603:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.688603:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.688604:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.688605:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.688638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.688645:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.688648:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.688649:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.688650:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.688652:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.688654:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.688655:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.688657:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.688657:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.688658:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.688662:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.688663:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.688666:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.688667:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.688668:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.688669:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.688670:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.688671:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.688672:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.688673:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.688675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.688678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677448.688683:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677448.688685:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:5.0:1619677448.688685:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677448.688687:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:5.0:1619677448.688687:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.688688:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:5.0:1619677448.688688:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.688689:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.688690:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.688691:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.688694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.688701:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.688703:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.688704:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.688705:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.688705:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.688706:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.688749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.688756:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.688759:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.688761:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.688762:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.688763:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.688765:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:30.0:1619677448.688766:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711616] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.688767:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.688768:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.688769:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:5.2:1619677448.688769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677448.688772:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.688774:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.688775:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.688777:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.688778:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:30.0:1619677448.688778:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.688778:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.688778:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:30.0:1619677448.688780:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:30.0:1619677448.688781:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677448.688781:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.688782:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.688783:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:1.0:1619677448.688784:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.688785:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:8.2:1619677448.688786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:1.0:1619677448.688787:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.688788:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.688789:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.688791:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.688794:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.688795:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.688796:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.688796:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.688797:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.688799:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.688800:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.688801:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.688802:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.688802:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.688803:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677448.688804:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.688805:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.688806:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.688807:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.688808:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.688810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.688813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677448.688817:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.688818:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677448.688819:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677448.688820:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:12.0:1619677448.688820:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.688821:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.688822:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677448.688822:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.688823:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.688825:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.688954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.688961:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.688964:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.688964:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.688966:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.688969:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.688971:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.688972:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.688973:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.688974:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.688975:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.688978:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.688980:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.688983:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.688983:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.688984:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.688986:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.688986:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.688987:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.688987:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.2:1619677448.688992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:21.2:1619677448.688997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:1.0:1619677448.688999:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:6.0:1619677448.689002:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677448.689002:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.689003:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.689003:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.689005:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.689006:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677448.689006:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.689008:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.689009:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:6.0:1619677448.689011:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.689012:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.689014:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.689016:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.689017:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.689018:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.689021:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.689023:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.689025:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.689026:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.689027:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.689028:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.689028:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.689029:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.689030:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:5.2:1619677448.689031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:8.2:1619677448.689034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.689037:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.689040:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:17.0:1619677448.689041:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677448.689041:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.689043:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.689043:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.689044:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:18.0:1619677448.689045:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.689045:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.689047:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.689047:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.689049:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.689050:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.689050:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:18.0:1619677448.689051:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.689052:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.689055:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.689057:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.689060:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.689061:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.689061:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.689063:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.689064:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.689065:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.689065:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.689069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.689077:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.689079:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.689080:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.689080:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.689082:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.689084:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.689133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677448.689135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.689142:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.689144:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.689144:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.689145:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:26.0:1619677448.689147:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.689148:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.689150:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.689151:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.689153:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.689154:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.689155:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.689156:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.689157:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.689159:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.689160:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.689161:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.689162:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.689163:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.689163:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.689164:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.689167:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.689168:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:10.0:1619677448.689168:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.689169:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.689169:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.2:1619677448.689170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:26.0:1619677448.689171:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.689172:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.689172:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.689173:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:20.0:1619677448.689173:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:10.0:1619677448.689173:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.689174:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.689174:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.689175:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677448.689175:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:26.0:1619677448.689176:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.689176:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:25.2:1619677448.689177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:20.0:1619677448.689177:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.689177:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.689178:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.689179:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:5.0:1619677448.689179:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.689180:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.689182:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:9.2:1619677448.689182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677448.689183:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96711609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.689184:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.689185:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:20.0:1619677448.689185:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.689186:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:22.0:1619677448.689187:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.689188:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.689189:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.689189:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.689190:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:22.0:1619677448.689191:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.689191:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.689192:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.689193:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:22.0:1619677448.689194:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677448.689194:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.689194:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.689195:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.689196:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:2.0:1619677448.689196:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:20.0:1619677448.689198:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.689199:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.689199:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:2.0:1619677448.689199:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677448.689200:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.689204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.689210:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.689213:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.689214:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.689214:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.689216:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.689219:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.689309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.689315:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.689318:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.689318:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.689320:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.689323:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.689324:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.689326:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.689327:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.689328:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.689329:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.689332:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.689334:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.689337:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.689338:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.689338:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.689340:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.689341:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.689342:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.689342:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.689347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.689355:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.689357:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.689358:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.689359:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.689361:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.689364:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.2:1619677448.689410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.689419:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.689421:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.689422:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.689424:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.689427:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.689428:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.689429:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.689431:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.689432:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.689433:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.689436:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.689437:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.689439:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.689440:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.689441:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.689442:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.689443:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.689444:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.689446:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.689446:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.689448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.2:1619677448.689451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.689456:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.689458:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.689459:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.689459:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:31.0:1619677448.689460:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.689461:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.689461:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.689463:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.689463:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677448.689464:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.689466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.689473:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.689475:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.689476:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.689477:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.689477:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.689478:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.689482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.689489:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.689492:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.689493:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.689495:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.689496:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.689498:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.689499:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.689500:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.689501:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.689503:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.689505:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:21.2:1619677448.689506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677448.689507:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.689508:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.689509:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.689510:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.689511:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.689512:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677448.689513:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.689513:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.689513:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:30.0:1619677448.689516:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:9.2:1619677448.689517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677448.689518:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.689520:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.689521:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.689523:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.689524:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.689525:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.689525:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:30.0:1619677448.689526:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.689527:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.689527:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.689528:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.689528:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.689529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:2.0:1619677448.689530:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000400:00000200:30.0:1619677448.689532:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.689532:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.689533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:30.0:1619677448.689534:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.689537:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.689537:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.689538:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.689539:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:30.0:1619677448.689539:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.689539:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.689541:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.689541:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.689541:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.689542:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.0:1619677448.689542:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:31.0:1619677448.689543:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.689543:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677448.689544:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.689544:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.689545:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:30.0:1619677448.689545:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.689546:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.689547:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.689547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:5.0:1619677448.689547:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.689548:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.689549:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.689550:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:5.0:1619677448.689550:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.689552:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.689553:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.689555:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.689555:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:5.0:1619677448.689555:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.689557:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677448.689558:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.689558:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.689559:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.689560:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.689561:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.689561:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.689562:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.689562:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677448.689562:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.689564:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:25.2:1619677448.689564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.689565:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.689565:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.689566:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.689566:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.689567:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:6.0:1619677448.689568:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.689569:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.689569:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.689569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677448.689569:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.689569:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.689569:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.689571:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.689571:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:22.0:1619677448.689572:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.689572:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.689572:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:6.0:1619677448.689573:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.689573:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.0:1619677448.689573:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.689574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.689575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:20.2:1619677448.689579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677448.689579:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.689581:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.689582:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.689583:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:30.0:1619677448.689584:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:13.0:1619677448.689584:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.689584:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.689585:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677448.689586:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.689586:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:12.0:1619677448.689586:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.689587:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.689587:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:13.0:1619677448.689587:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677448.689588:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677448.689588:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:12.0:1619677448.689588:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.689589:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677448.689590:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.689591:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.689685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.689693:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.689696:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.689696:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.689698:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.689701:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.689702:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.689704:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711602] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.689705:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.689706:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.689707:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.689710:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.689712:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.689715:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.689716:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.689717:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.689718:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.689719:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.689719:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.689720:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.689724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.2:1619677448.689724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.689730:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.689732:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:1.0:1619677448.689733:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.689734:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677448.689735:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.689736:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:1.0:1619677448.689736:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.689737:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.689738:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.689739:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.689740:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.689741:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.689742:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:1.0:1619677448.689743:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.689744:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.689745:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.689748:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.689750:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.689752:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.689752:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.689753:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.689755:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.689756:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.689756:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.689757:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.689761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.689769:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.689771:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.689772:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.689772:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.689774:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.689775:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.689779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.689786:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.689788:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.689789:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.689789:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.689790:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.689790:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.689883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.689888:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.689891:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.689892:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.689894:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.689896:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.689897:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.689898:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.689900:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.689901:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.689902:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.689906:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.689908:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.689911:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.689911:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.689912:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.689914:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.689914:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.689915:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.689915:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.689920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.689926:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.689929:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.689929:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.689930:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.689932:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.689935:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677448.690013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.2:1619677448.690016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.690021:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.690022:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.690024:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677448.690025:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.690025:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.690028:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.690028:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.690028:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.690030:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.690031:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.690032:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:30.0:1619677448.690033:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.690034:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.690035:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.690036:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.690037:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.690037:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.690038:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.690039:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677448.690040:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.690044:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.690045:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.690046:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677448.690048:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.690048:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.690049:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.690049:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677448.690050:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.690051:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.690051:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.690051:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.690052:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.690052:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.690052:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677448.690053:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.690054:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.690055:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.690057:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.690057:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677448.690057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:25.2:1619677448.690059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.690063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.690064:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:13.0:1619677448.690065:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.690065:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.690066:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.690067:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:13.0:1619677448.690068:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.690068:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.690068:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.690070:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:13.0:1619677448.690070:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.690070:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:12.0:1619677448.690071:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.690073:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:13.0:1619677448.690073:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.690074:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.690075:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.690077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.690084:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.690087:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.690088:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.690088:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.690089:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.690090:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.690114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.690121:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.690124:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.690124:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.690126:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.690129:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.690130:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.690132:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.690133:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.690134:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.690135:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.690139:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.690140:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.690143:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.690144:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.690145:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.690146:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.690147:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.690147:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.690148:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.690152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.690160:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.690163:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.690164:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.690164:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.690167:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.690169:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.690256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.690263:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.690266:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.690267:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.690268:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.690271:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.690273:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.690274:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.690275:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.690276:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.690277:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.690280:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.690281:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.690284:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.690285:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.690286:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.690287:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.690288:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.690289:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.690289:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.690294:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.690303:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.690306:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.690307:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.690307:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.690309:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.690313:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.690314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.2:1619677448.690321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.690321:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.690324:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.690325:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.690325:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.690326:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.690326:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677448.690327:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.690330:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.690330:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.690332:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.690333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:1.0:1619677448.690334:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.690335:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.690336:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.690338:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.690339:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:1.0:1619677448.690339:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.690340:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.690342:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.690342:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:1.0:1619677448.690344:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.690345:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.690346:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.690347:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.690348:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.690348:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:18.0:1619677448.690349:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:1.0:1619677448.690349:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:18.0:1619677448.690350:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.690350:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:18.0:1619677448.690351:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.690351:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677448.690352:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.690352:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:18.0:1619677448.690353:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.690353:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.690354:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677448.690357:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.690357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.690359:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:5.2:1619677448.690360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.690361:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.690362:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.690363:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.690364:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.690365:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677448.690365:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677448.690366:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:17.0:1619677448.690366:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.690366:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:18.0:1619677448.690367:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000400:00000200:17.0:1619677448.690367:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.690367:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.690367:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.690368:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677448.690369:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.690369:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.2:1619677448.690370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:1.0:1619677448.690370:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677448.690371:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.690373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677448.690378:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677448.690378:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.690380:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:20.0:1619677448.690381:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:2.0:1619677448.690381:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:20.0:1619677448.690382:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.690382:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:20.0:1619677448.690383:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.690385:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.690386:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.690452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.690459:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.690462:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.690463:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.690465:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.690468:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.690469:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.690470:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.690472:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.690473:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.690474:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.690477:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.690479:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.690481:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.690482:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.690483:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.690484:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.690485:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.690486:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.690487:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.690488:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.690490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.690493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.690499:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:26.0:1619677448.690500:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.690502:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677448.690502:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.690502:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.690503:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.690503:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.690505:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.690505:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.690508:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.690514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:5.2:1619677448.690517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.0:1619677448.690521:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.690522:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.690524:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.690525:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677448.690525:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.690526:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677448.690527:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.690528:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.690530:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.690531:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.690532:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:24.0:1619677448.690533:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.690533:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.690535:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.690535:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.690537:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.690537:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.690538:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.690538:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:24.0:1619677448.690539:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.690544:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.690545:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.690548:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.690548:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:24.0:1619677448.690551:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.690551:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.690552:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.690552:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:24.0:1619677448.690553:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.690553:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.690554:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.690555:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.690555:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.690555:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.690556:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.690556:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677448.690557:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:1.0:1619677448.690557:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677448.690559:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.690560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:9.2:1619677448.690561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:4.2:1619677448.690563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.690568:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677448.690569:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:17.0:1619677448.690570:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.690571:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.690571:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:1.0:1619677448.690571:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.690572:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:8.0:1619677448.690572:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.690573:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.690574:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.690574:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.690575:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.690575:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.690576:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677448.690576:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.0:1619677448.690580:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.2:1619677448.690611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.690619:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.690621:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.690623:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.690626:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.690626:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.690628:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.690629:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.690630:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.690631:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.690632:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.690636:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.690637:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.690640:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.690640:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.690641:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.690643:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.690643:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.690644:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.690644:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.690648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.690655:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.690658:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.690659:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.690660:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.690662:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.690662:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.690781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.690787:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.690789:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.690790:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.690792:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.690795:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.690796:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.690798:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.690799:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.690800:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.690801:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.690804:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.690806:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.690809:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.690810:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.690810:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.690812:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.690813:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.690813:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.690814:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.690818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.690826:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.690828:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.690829:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.690831:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.690833:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.690834:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.691000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.691006:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.691008:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.691010:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.691011:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.691013:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.691015:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.691016:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.691017:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.691018:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.691019:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.691023:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.691024:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.691027:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.691028:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.691029:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.691031:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.2:1619677448.691032:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.0:1619677448.691035:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.691036:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.691037:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677448.691038:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.691040:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.691041:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.691043:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.691046:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.691048:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.691049:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711587] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.691050:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.691051:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.691052:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:25.2:1619677448.691054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:17.0:1619677448.691056:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.691057:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.691060:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.691060:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:13.0:1619677448.691060:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.691061:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.691063:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.691063:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.691063:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677448.691064:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.691064:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677448.691064:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.691065:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.691067:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.691068:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.691069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:25.2:1619677448.691075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:22.0:1619677448.691076:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:5.2:1619677448.691079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677448.691080:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.691081:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.691081:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.691081:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.691083:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.691084:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.691085:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:12.0:1619677448.691085:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.691086:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.691086:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.691087:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.691088:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.691089:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.691091:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.691092:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.691095:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.691097:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.691098:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.691099:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.691100:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.691101:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.691105:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.691107:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.691110:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.691110:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.691111:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.691113:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.691113:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.691114:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.691114:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.691119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.691126:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.691129:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.691130:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.691130:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.691133:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.691135:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.691138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:22.0:1619677448.691145:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.691148:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.691149:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.691149:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.691150:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.691151:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.691161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.691168:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.691171:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.691173:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.691175:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.691175:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.691177:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.691178:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.691180:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.691181:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.691182:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.691184:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.691186:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.691188:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.691189:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.691189:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.691191:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.691192:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.691193:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.691193:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.691209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.691215:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.691218:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.691219:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.691220:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.691222:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.691222:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.691388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.691394:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.691398:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.691399:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.691401:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.691404:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.691405:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.691406:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.691408:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.691409:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.691410:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.691413:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.691415:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.691417:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.691418:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.691418:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.691420:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.691420:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.691421:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.691421:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.691428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.691435:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.691437:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.691438:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.691439:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.691440:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.691441:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.691504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.691513:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.691516:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.691518:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.691520:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.691520:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.691522:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.691523:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.691524:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.691525:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.691526:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.691530:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.691531:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.691533:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.691534:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.691535:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.2:1619677448.691535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:30.0:1619677448.691536:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.691537:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.691538:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.691538:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677448.691538:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.691540:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:25.2:1619677448.691542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000400:00000200:20.0:1619677448.691542:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.691544:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.691546:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.691547:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:31.0:1619677448.691548:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:20.0:1619677448.691549:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.691550:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.691551:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:20.0:1619677448.691551:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.691552:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.691552:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677448.691554:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.691555:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:20.0:1619677448.691556:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.691558:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.691559:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.0:1619677448.691561:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.691562:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.691563:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.691565:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.691566:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.691566:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.691567:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.691571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.691577:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.691580:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.691581:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.691583:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.691583:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.691586:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.691591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.691597:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.691600:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.691600:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.691602:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.691605:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.691607:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.691608:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.691610:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.691611:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.691612:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.691615:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.691617:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.691620:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.691620:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.691621:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.691623:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.691623:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.691624:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.691624:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.691630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.691638:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.691640:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.691641:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.691642:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.691643:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.691645:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.691704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.691712:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.691715:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.691716:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.691717:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.691721:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.691722:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.691723:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.691724:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.691725:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.691726:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.691729:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.691731:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.691733:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.691734:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.691734:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.691736:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.691736:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.691737:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.691737:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.691741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.2:1619677448.691742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.691748:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:1.0:1619677448.691748:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.691749:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.691750:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.691751:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:1.0:1619677448.691751:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.691752:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.691752:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.691754:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:1.0:1619677448.691754:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.691757:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.691759:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.691760:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.691761:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.691762:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.691763:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.691766:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.691768:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.691771:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.691772:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.691772:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:21.2:1619677448.691774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:1.0:1619677448.691774:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.691775:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.691776:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.691777:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.691778:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.691780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:24.0:1619677448.691781:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:5.2:1619677448.691782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.691784:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.691785:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.691786:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.691787:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:17.0:1619677448.691788:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:24.0:1619677448.691789:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.691790:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:17.0:1619677448.691790:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:31.0:1619677448.691791:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.691791:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:17.0:1619677448.691791:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677448.691792:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96711578] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.691792:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677448.691793:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.691794:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:24.0:1619677448.691794:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.691795:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.691796:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:24.0:1619677448.691796:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:25.2:1619677448.691798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:24.0:1619677448.691799:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.691801:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.691804:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:24.0:1619677448.691804:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.691805:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.691806:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.691807:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.691808:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:24.0:1619677448.691808:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.691808:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:30.0:1619677448.691809:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:24.0:1619677448.691809:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:30.0:1619677448.691810:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:24.0:1619677448.691810:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:31.0:1619677448.691811:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.2:1619677448.691813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.691821:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.691823:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.691824:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.691826:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.691827:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.691829:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.691933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.691938:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.691941:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.691943:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.691944:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.691946:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.691948:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.691949:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.691951:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.691952:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.691953:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.691956:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.691958:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.691961:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.691962:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.691963:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.691965:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.691966:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.691967:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.691967:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.691971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.691978:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.691981:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.691982:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.691983:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.691985:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.691986:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.692184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.692192:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:21.2:1619677448.692194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.692195:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.692197:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.692199:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.692199:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.692200:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:30.0:1619677448.692201:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:31.0:1619677448.692202:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.692202:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.692203:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.692204:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.692205:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.692208:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.692209:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.692210:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.692212:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.692212:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.692213:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.692214:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.692215:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.692216:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.692219:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.692220:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.692220:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:30.0:1619677448.692220:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.692221:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.692222:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.692223:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.692224:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.692224:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.692224:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.692224:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.692225:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.692226:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.692226:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.692227:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.692227:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.692228:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.692228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:25.2:1619677448.692231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:13.0:1619677448.692235:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:26.0:1619677448.692237:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:13.0:1619677448.692237:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.692238:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.692240:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:13.0:1619677448.692240:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.692241:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.692241:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:13.0:1619677448.692242:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:26.0:1619677448.692243:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.692243:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.692246:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.692397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.692399:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.692402:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.692404:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.692405:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.692407:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.692409:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.692410:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.692412:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.692413:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.692414:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.692417:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.692419:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.692422:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.692423:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.692423:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.692425:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.692426:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.692427:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.692428:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.692429:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.692432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.692434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677448.692439:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.692439:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677448.692442:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:5.0:1619677448.692442:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:13.0:1619677448.692443:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.692443:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.692444:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:13.0:1619677448.692445:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.692446:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.692448:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.692498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:4.2:1619677448.692498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677448.692505:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.692505:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.692508:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:10.0:1619677448.692508:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:6.0:1619677448.692508:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677448.692509:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.692511:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.692512:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.692513:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.692514:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.692515:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:22.0:1619677448.692516:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.692517:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711573] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.692517:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711573] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.692519:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.692519:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.692520:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.692521:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.692522:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.692522:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677448.692527:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.692528:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.692530:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.692531:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.692532:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.692533:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.692534:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.692534:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.692535:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.692536:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.692536:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.692536:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.692537:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.692538:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:10.0:1619677448.692538:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.692538:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.692539:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:21.2:1619677448.692539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:10.0:1619677448.692539:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.692540:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.692543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:25.2:1619677448.692544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.692547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.0:1619677448.692549:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:5.0:1619677448.692551:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:8.0:1619677448.692552:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.692553:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:6.0:1619677448.692553:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.692554:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:5.0:1619677448.692554:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:22.0:1619677448.692555:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:8.0:1619677448.692555:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.692555:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:5.0:1619677448.692555:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.692556:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.692556:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.692557:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.692557:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.692557:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.692558:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.692559:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:6.0:1619677448.692559:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.692559:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:8.0:1619677448.692560:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:25.2:1619677448.692562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:8.0:1619677448.692562:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96711571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.692563:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.692564:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.692565:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:6.0:1619677448.692565:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.692567:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:8.0:1619677448.692569:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.692569:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.692570:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.692570:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677448.692571:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:6.0:1619677448.692571:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.692572:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677448.692574:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.692574:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.692575:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.692576:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.692577:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.692578:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.692580:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.692580:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.2:1619677448.692582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:21.2:1619677448.692586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:12.0:1619677448.692589:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.692591:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:24.0:1619677448.692592:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:12.0:1619677448.692592:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.692593:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.692594:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.692595:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677448.692595:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.692596:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677448.692596:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.692599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:20.2:1619677448.692604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:13.0:1619677448.692606:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.692608:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.692609:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.692610:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.692610:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.692611:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:13.0:1619677448.692612:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.692614:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.692615:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.692617:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.692619:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.692621:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.692622:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.692623:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.692624:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.692625:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.692628:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.692630:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.692632:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.692633:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.692634:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.692636:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.692636:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.692637:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.692637:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.692641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.692649:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.692652:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.692652:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.692654:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.692655:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.692657:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.692693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.692700:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.692703:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.692704:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.692706:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.692708:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.692710:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.692711:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.692712:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.692713:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.692714:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.692717:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.692719:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.692721:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.692722:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.692722:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.692723:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.692724:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.692724:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.692725:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.692729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.692735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:12.0:1619677448.692735:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.692738:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.692738:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.692740:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.692741:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.692742:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:12.0:1619677448.692742:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677448.692745:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.692745:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.692747:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.692749:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.692750:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.692752:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.692753:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.692754:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.692755:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.692758:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.692760:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.692761:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.692762:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.692762:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.692763:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.692764:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.692765:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.692765:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:4.2:1619677448.692767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.2:1619677448.692768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:9.2:1619677448.692769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.692774:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677448.692776:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.692776:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:8.0:1619677448.692777:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.692778:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677448.692779:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.692779:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.692780:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:18.0:1619677448.692780:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:8.0:1619677448.692780:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.692782:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000400:00000200:31.0:1619677448.692783:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.692783:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.692783:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677448.692784:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:8.0:1619677448.692784:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.692785:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.692786:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.692787:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.692788:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:8.0:1619677448.692788:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.692790:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.692790:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.692792:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.692793:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.692794:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:8.0:1619677448.692794:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.692797:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.692797:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.692799:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.2:1619677448.692800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:8.0:1619677448.692800:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.692801:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.692801:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.692802:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.692802:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.692803:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:8.0:1619677448.692803:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.692804:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.692804:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677448.692805:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.692805:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677448.692805:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.0:1619677448.692805:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:31.0:1619677448.692806:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.692806:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677448.692808:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:8.2:1619677448.692809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:25.2:1619677448.692810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:5.0:1619677448.692810:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.692811:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.692813:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.692815:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:5.0:1619677448.692816:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96711565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.692817:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:17.0:1619677448.692817:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:5.0:1619677448.692817:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.692818:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.692819:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:5.0:1619677448.692819:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:26.0:1619677448.692820:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.692820:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:10.0:1619677448.692820:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:17.0:1619677448.692821:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.692821:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:26.0:1619677448.692824:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.692824:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.692825:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.692827:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.692828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:17.0:1619677448.692828:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677448.692828:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:5.0:1619677448.692831:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.692832:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677448.692833:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:10.0:1619677448.692834:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.692835:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677448.692835:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.692836:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:10.0:1619677448.692837:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:5.0:1619677448.692837:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:26.0:1619677448.692838:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.692838:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.692839:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.692839:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:9.2:1619677448.692841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.692847:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.692850:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.692850:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.692851:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.692853:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.692856:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.692908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.692915:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.692918:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.692919:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.692920:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.692923:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.692924:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.692925:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.692927:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.692928:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.692929:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.692932:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.692933:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.692935:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.692936:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.692937:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.692938:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.692938:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.692939:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.692939:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.692943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.692949:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.692950:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.692951:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.692952:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.692953:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.692954:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.692993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.693000:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.693003:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.693005:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.693007:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.693007:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.693008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:30.0:1619677448.693009:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.693010:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.693011:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.693012:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.693013:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.693014:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.693016:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:30.0:1619677448.693017:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.693017:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:30.0:1619677448.693020:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.693020:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.693022:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.693023:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:30.0:1619677448.693024:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.693025:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.693025:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:18.0:1619677448.693025:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.693026:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.693027:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:18.0:1619677448.693027:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.693028:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:18.0:1619677448.693028:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.693029:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.693031:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000400:00000200:18.0:1619677448.693031:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.693032:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:18.0:1619677448.693033:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.2:1619677448.693033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.693035:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.693035:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.693036:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677448.693038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677448.693038:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.693038:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.693039:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.693040:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.693043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.693046:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.693046:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.693048:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677448.693049:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:20.0:1619677448.693049:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.693050:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:2.0:1619677448.693050:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677448.693051:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.693051:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.693051:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677448.693052:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:5.0:1619677448.693053:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.693053:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.693054:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.693054:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:20.0:1619677448.693055:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.693055:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (24)--
00000800:00000200:5.0:1619677448.693055:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:2.0:1619677448.693055:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.693056:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677448.693057:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.693158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.693164:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.693166:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.693167:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.693169:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.693172:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.693173:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.693175:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.693176:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.693177:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.693178:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.693181:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.693183:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.693186:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.693187:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.693187:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.693189:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.693190:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.693190:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.693191:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.693195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.693202:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.693205:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.693205:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.693207:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.693208:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.693210:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.693252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:21.2:1619677448.693258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677448.693258:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.693261:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.693261:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.693264:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.693265:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677448.693265:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.693267:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677448.693268:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:18.0:1619677448.693268:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.693270:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.693271:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.693271:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677448.693272:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.693273:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.693274:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.693276:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.693276:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.693277:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.693278:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.693279:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.693280:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.693281:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.693282:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.693283:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.693283:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.693284:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.693285:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.693286:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.693286:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.693287:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.693287:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:31.0:1619677448.693290:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.693291:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:25.2:1619677448.693291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.693292:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677448.693299:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:31.0:1619677448.693300:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.693300:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:31.0:1619677448.693301:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677448.693302:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:31.0:1619677448.693303:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:12.0:1619677448.693303:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.693304:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:12.0:1619677448.693305:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.693307:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677448.693308:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677448.693309:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677448.693309:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.693309:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:21.2:1619677448.693310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.693311:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:13.0:1619677448.693312:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.693312:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677448.693312:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.693312:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.693313:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.693314:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.693318:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.693321:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.693321:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.693322:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677448.693352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.693358:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.693361:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.693362:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.693364:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.693367:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.693369:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677448.693370:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.693371:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.693372:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.693373:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.693377:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.693379:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.693381:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.693382:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.693383:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.693385:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.693385:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.693386:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.693386:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677448.693391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.693400:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.693402:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.693403:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.693405:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.693406:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.693407:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.693460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.693467:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.2:1619677448.693469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:26.0:1619677448.693470:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.693471:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.693473:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.693474:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.693475:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:26.0:1619677448.693476:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.693477:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.693478:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:26.0:1619677448.693479:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.693479:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:26.0:1619677448.693480:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.693481:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.693481:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.693484:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.693484:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.693485:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:26.0:1619677448.693486:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.693487:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.693488:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.693489:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.693490:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.693490:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.693491:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:22.0:1619677448.693491:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.693493:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.693493:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677448.693494:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.693495:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:5.2:1619677448.693495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.693496:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677448.693496:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.693498:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.693499:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:25.2:1619677448.693499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.693499:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.693499:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.693500:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.693502:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:22.0:1619677448.693502:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.693502:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:4.2:1619677448.693502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.693503:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.693503:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:26.0:1619677448.693504:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.693505:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677448.693506:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:10.0:1619677448.693506:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:26.0:1619677448.693507:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.693507:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.693507:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.693508:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.693508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.693509:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.693509:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.693509:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.693510:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96711555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.693511:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.693512:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.693513:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:26.0:1619677448.693513:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.693514:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:17.0:1619677448.693516:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:26.0:1619677448.693517:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.693518:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:30.0:1619677448.693519:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.693519:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:17.0:1619677448.693520:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.693520:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:26.0:1619677448.693522:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.693522:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.693523:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.693524:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:26.0:1619677448.693525:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:17.0:1619677448.693525:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677448.693526:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.693527:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.693528:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.693529:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.693531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.2:1619677448.693534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.2:1619677448.693536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.693537:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.693540:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.693541:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.693541:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:31.0:1619677448.693542:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.693543:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:30.0:1619677448.693544:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.693544:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.693544:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677448.693544:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.693545:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:1.0:1619677448.693547:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.693548:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.693548:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.693550:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.693550:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.693589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.693597:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.693600:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.693602:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.693604:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.693604:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.693606:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.693607:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.693608:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.693609:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.693610:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.693613:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.693615:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.693619:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.693619:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.693620:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.693622:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.693622:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.693623:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.693624:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.693628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.693635:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.693637:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.693638:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.693640:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.693641:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.693643:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.693777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.693782:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.693785:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.693787:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.693788:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.693790:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.693792:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.693793:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.693794:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.693795:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.693796:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.693800:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.693801:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.693804:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.693804:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.693805:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.693807:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.693808:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.693808:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.693809:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.693817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.693825:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.693828:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.693829:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.693829:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.2:1619677448.693830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000400:00000200:18.0:1619677448.693831:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.693832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:18.0:1619677448.693834:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677448.693836:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:8.0:1619677448.693838:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.693839:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677448.693839:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:8.0:1619677448.693841:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.693842:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.693842:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.693842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:8.0:1619677448.693844:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.693845:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.693846:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.693846:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.693848:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.693849:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:2.0:1619677448.693849:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:17.0:1619677448.693850:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.693850:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.693851:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.693852:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.693852:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.693853:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:8.0:1619677448.693853:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.693853:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.693853:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.693854:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:8.0:1619677448.693855:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.693856:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.693857:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.693860:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.693861:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.693862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677448.693863:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.693864:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.693865:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:17.0:1619677448.693866:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.693866:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.693867:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.693868:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677448.693868:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:8.0:1619677448.693868:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677448.693869:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.693869:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677448.693870:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:8.0:1619677448.693870:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.693870:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.693871:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:8.0:1619677448.693871:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677448.693872:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.0:1619677448.693872:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:31.0:1619677448.693874:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.693876:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677448.693876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677448.693877:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.693878:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.693879:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96711550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.693879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (21)++
00000400:00000200:31.0:1619677448.693880:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.693881:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.693882:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:9.2:1619677448.693884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.693885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:1.0:1619677448.693885:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:31.0:1619677448.693886:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.693886:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677448.693886:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.693887:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000400:00000200:1.0:1619677448.693887:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.693888:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.693888:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677448.693889:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.693889:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.693890:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.693890:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:31.0:1619677448.693891:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.693891:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677448.693892:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:18.0:1619677448.693892:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677448.693893:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.693893:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:31.0:1619677448.693894:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.693894:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.693895:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677448.693895:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:18.0:1619677448.693895:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.693896:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.693896:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:2.0:1619677448.693896:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:25.2:1619677448.693899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677448.693899:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.693899:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.693902:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:24.0:1619677448.693903:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.693904:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:12.0:1619677448.693905:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:24.0:1619677448.693906:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96711549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.693907:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.693908:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.693908:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:24.0:1619677448.693909:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677448.693909:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.693910:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.693912:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677448.693913:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.693913:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677448.693915:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.693918:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.693919:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.693919:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.693921:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.693922:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.693923:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.693923:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:4.2:1619677448.693928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.693935:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.693938:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:9.2:1619677448.693938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.693939:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.693940:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.693943:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.693944:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.693945:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.693945:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:10.0:1619677448.693947:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.693947:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.693948:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.693948:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.693949:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.693949:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.693952:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.693953:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.693953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.693954:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.693955:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.2:1619677448.693957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.693958:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.693959:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.693959:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.693960:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677448.693962:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:10.0:1619677448.693962:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677448.693962:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.693963:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677448.693964:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.693964:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.693965:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.693965:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.693966:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:17.0:1619677448.693966:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.693966:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.2:1619677448.693968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:17.0:1619677448.693969:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.693970:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.693971:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711547] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.693972:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.693972:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.693973:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.693974:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.693975:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.693975:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.693976:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:26.0:1619677448.693977:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.693978:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:26.0:1619677448.693979:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.693979:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.693979:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.693980:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677448.693981:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.693982:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.693982:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:26.0:1619677448.693984:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.693986:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.693987:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.693987:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:17.0:1619677448.693987:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.693988:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.693988:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.2:1619677448.693988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.693989:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:17.0:1619677448.693989:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.693990:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:17.0:1619677448.693990:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.693991:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:17.0:1619677448.693991:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.693991:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.693992:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.693993:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.693993:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.693995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.693996:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:25.2:1619677448.693998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.693998:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.693999:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.694000:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.694001:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.694002:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:13.0:1619677448.694002:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.2:1619677448.694004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:13.0:1619677448.694004:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:10.0:1619677448.694004:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:13.0:1619677448.694005:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.694006:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:13.0:1619677448.694007:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.694007:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.694008:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:12.0:1619677448.694008:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.694009:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.694009:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:13.0:1619677448.694010:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.694011:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.0:1619677448.694012:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.694014:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.694015:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.694017:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.694018:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.694020:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.694021:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.694022:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.694023:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.694027:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.694028:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.694031:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.694032:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.694032:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.694034:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.694035:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.694035:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.694036:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677448.694040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.694048:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.694050:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.694051:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.694051:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.694053:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.694056:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.694335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.694343:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.694346:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.694346:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.694348:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.694350:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.694351:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:21.2:1619677448.694351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:26.0:1619677448.694352:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.694353:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.694355:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.694356:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:6.0:1619677448.694357:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:26.0:1619677448.694358:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.694359:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:26.0:1619677448.694360:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:22.0:1619677448.694360:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:26.0:1619677448.694362:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.694363:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:6.0:1619677448.694363:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.694364:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.694365:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.694366:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.694366:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677448.694366:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.694367:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677448.694367:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.694368:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.694369:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.694370:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.694371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:6.0:1619677448.694371:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.694375:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.694377:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:6.0:1619677448.694377:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.694379:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:6.0:1619677448.694379:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.694380:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.694380:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.694380:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.694381:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:10.0:1619677448.694382:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.694383:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.694383:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.694384:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:6.0:1619677448.694384:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.694384:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.694389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.694395:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.694397:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.694398:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.694398:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.694400:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.694403:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.694411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.694418:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.694420:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.694421:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.694423:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.694425:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.694427:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.694428:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.694429:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.694430:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.694431:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.694434:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.694436:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.694438:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.694438:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.694439:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.694440:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.694441:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.694441:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.694442:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.694447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:21.2:1619677448.694452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:1.0:1619677448.694454:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.694456:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.694456:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.694457:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.694458:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.694459:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:1.0:1619677448.694460:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677448.694461:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.694462:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.694464:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.694466:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.694467:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.694469:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.694470:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.694471:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.694472:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.694475:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.694477:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.694479:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.694480:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.694480:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.694482:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.694482:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.694484:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.694485:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.694485:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.2:1619677448.694490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.694495:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.694497:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.694497:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677448.694499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.694502:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677448.694503:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.694504:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.694505:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.694506:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.694507:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.694508:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.694517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.694523:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.694525:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.694526:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.694528:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.694530:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.694531:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.694532:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.694533:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.694534:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.694535:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677448.694537:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.694539:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.694541:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.694541:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.694542:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.694543:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.694544:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.694545:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.694545:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.694549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.694556:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.694559:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.694560:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.694560:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.694562:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.694564:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.694702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677448.694708:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677448.694710:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677448.694711:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677448.694713:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.694714:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.694716:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677448.694717:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.694718:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.694719:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.694720:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677448.694723:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.694724:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677448.694727:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.694728:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:1.0:1619677448.694728:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.694730:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677448.694731:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677448.694731:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677448.694732:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677448.694736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.694743:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.694746:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.694746:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.694748:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.694749:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.694750:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.694782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.694790:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.694792:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.694793:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.694795:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.694797:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.694798:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.694800:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.694801:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.694802:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.694803:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.694806:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.694808:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.694811:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.694812:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.694812:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.694814:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.694815:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.694815:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.694816:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.694821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.694828:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.694831:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.694832:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.694832:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.694834:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.694837:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.694853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.694860:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.694863:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.694864:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.694866:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.694869:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.694870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:8.0:1619677448.694870:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.694871:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.694873:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.694874:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.694875:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.694876:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:8.0:1619677448.694878:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.694878:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:8.0:1619677448.694880:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.694881:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.694881:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.694883:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.694884:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:5.0:1619677448.694884:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.694885:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677448.694885:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:8.0:1619677448.694886:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.694887:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.694887:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:5.0:1619677448.694887:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.694888:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.2:1619677448.694889:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.2:1619677448.694892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677448.694892:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.694893:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.694894:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.694897:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677448.694897:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.694899:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:5.0:1619677448.694899:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.694901:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:17.0:1619677448.694902:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.694902:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.694902:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:1.0:1619677448.694902:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:5.0:1619677448.694903:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677448.694904:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.694905:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.694905:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.694905:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.694906:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.694906:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677448.694906:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.694907:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.694908:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96711535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.694909:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.694910:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.694911:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.694915:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.694917:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.694918:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.694919:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.694919:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.694920:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.694920:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677448.694921:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (22)++
00000800:00000200:9.2:1619677448.694922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.694923:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.694924:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.694925:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.694926:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.694927:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677448.694928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.2:1619677448.694929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.694929:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.2:1619677448.694932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677448.694932:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.694933:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.694933:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.694935:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.694936:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.694937:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677448.694937:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:26.0:1619677448.694938:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:17.0:1619677448.694938:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.694939:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:10.0:1619677448.694939:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677448.694939:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.694939:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.694940:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.694941:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:26.0:1619677448.694942:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.694942:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.694942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:1.0:1619677448.694942:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.694943:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:26.0:1619677448.694946:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.694946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000400:00000200:26.0:1619677448.694947:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.694948:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.694949:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.694950:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677448.694951:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.694951:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.694953:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.694953:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.694953:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.694954:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677448.694955:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.694956:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.694956:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677448.694956:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.694957:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.694957:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:26.0:1619677448.694958:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.694958:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.694959:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:26.0:1619677448.694960:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.694961:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.694962:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.694963:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.694964:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.694965:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.694965:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.694969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.694977:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.694979:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.694980:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.694981:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.694984:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.694984:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.2:1619677448.695121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.695124:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.695127:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.695129:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.695130:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.695132:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.695133:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677448.695135:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.695136:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.695137:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.695138:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.695141:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.695143:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.695145:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.695146:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.695147:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.695148:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.695149:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.695151:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.695152:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.695152:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677448.695154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677448.695157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:1.0:1619677448.695162:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677448.695163:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:1.0:1619677448.695164:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.695165:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.695165:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:1.0:1619677448.695165:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.695166:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.695167:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:1.0:1619677448.695167:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677448.695170:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.695230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.695235:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.695238:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.695239:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.695241:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.695243:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.695244:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.695245:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.695247:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.695248:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.695249:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.695252:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.695254:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.695256:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.695257:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.695258:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.695259:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.695260:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.695260:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.695261:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.695266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.695273:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.695276:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.695277:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.695277:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.695279:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.695282:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.695289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.695297:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.695299:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.695300:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.695301:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.695302:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.695302:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.695320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.695326:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.695329:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.695329:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.695331:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.695333:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.695335:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.695336:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.695337:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.695338:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.695339:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.695343:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.695344:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.695347:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.695348:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.695348:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.695351:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.695351:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.695352:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.695352:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.695357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.695364:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.695367:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.695368:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.695368:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.695370:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.695373:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.2:1619677448.695398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.695407:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.2:1619677448.695407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1200] (20)++
00000800:00000200:30.0:1619677448.695410:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.695412:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.695412:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:31.0:1619677448.695414:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.695414:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.695415:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000400:00000200:30.0:1619677448.695416:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:17.0:1619677448.695416:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:30.0:1619677448.695418:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677448.695418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677448.695419:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.695420:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.695421:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.695422:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.695422:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.695424:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:10.0:1619677448.695426:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677448.695426:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.695427:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.695427:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.695428:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.695429:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.695429:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.695430:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677448.695430:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:30.0:1619677448.695433:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.695433:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:10.0:1619677448.695433:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.695434:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:1.0:1619677448.695434:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.695436:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.695436:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677448.695436:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.695437:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:1.0:1619677448.695437:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:30.0:1619677448.695438:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:10.0:1619677448.695438:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.695439:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96711528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.695440:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677448.695441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:10.0:1619677448.695441:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.695441:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:1.0:1619677448.695441:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:10.0:1619677448.695442:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.695443:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.695443:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:1.0:1619677448.695443:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:1.0:1619677448.695444:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:1.0:1619677448.695444:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000400:00000200:10.0:1619677448.695447:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.695448:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:10.0:1619677448.695449:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.2:1619677448.695449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:12.0:1619677448.695451:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.695451:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.695451:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.695452:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:12.0:1619677448.695453:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.695453:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.695454:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:12.0:1619677448.695455:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.695455:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.695455:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:13.0:1619677448.695456:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.695456:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.695457:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:25.2:1619677448.695460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.695460:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000400:00000200:5.0:1619677448.695460:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.695462:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.695463:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.695464:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:5.0:1619677448.695464:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.695465:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.695466:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677448.695466:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.695467:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:22.0:1619677448.695468:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.695472:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.695472:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.695473:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.695475:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.695477:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.695480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.695488:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.695491:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.695493:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.695494:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.695495:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.695497:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.695498:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.695499:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.695500:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.695501:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.695505:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.695506:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.695509:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.695510:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.695511:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.695513:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.695513:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.695514:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.695515:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.695519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.695526:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.695530:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.695531:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.695531:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.695533:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.695536:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.695640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.695646:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.695649:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.695650:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.695652:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.695654:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.695655:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.695657:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.695658:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.695659:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.695660:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.695663:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.695665:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.695668:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.695669:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.695669:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.695671:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.695672:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.695673:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.695674:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.695674:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.695677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.2:1619677448.695679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.695685:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.695687:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.695687:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677448.695687:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:18.0:1619677448.695688:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677448.695689:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.695690:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.695692:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.695692:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.695694:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.695749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.695756:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.695759:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.695760:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.695762:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.695765:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.695767:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.695768:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.695769:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.695770:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.695771:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.695775:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.695777:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.695780:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.695780:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.695781:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.695783:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.695784:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.695785:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.695786:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.695787:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677448.695789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.695792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.695797:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.695799:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:10.0:1619677448.695800:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:18.0:1619677448.695801:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.695801:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.695802:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.695802:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:2.0:1619677448.695803:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.695804:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677448.695806:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.695859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:4.2:1619677448.695859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677448.695865:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.695867:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.695868:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:26.0:1619677448.695869:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:1.0:1619677448.695869:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.695870:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677448.695871:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.695873:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.695874:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.695875:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.695875:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677448.695877:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677448.695877:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.695879:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.695879:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.695880:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.695881:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.695881:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677448.695882:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.695883:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677448.695885:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.695887:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:26.0:1619677448.695888:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.695889:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.695890:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677448.695890:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.695890:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.695892:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.695892:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.695893:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.695893:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.695893:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.695894:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.695895:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.695896:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.695897:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.695898:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:8.2:1619677448.695898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.695899:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.695899:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.695902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.695905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.695905:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.695907:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.695908:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.695908:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.695910:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:1.0:1619677448.695910:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.695912:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:1.0:1619677448.695912:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:22.0:1619677448.695913:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.695914:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.695914:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677448.695914:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.695915:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.695916:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.695916:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.695919:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.696218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.696221:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.696223:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.696226:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.696226:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.696228:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.696230:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.696231:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.696232:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.696234:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.696234:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.696238:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.696241:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:4.2:1619677448.696241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677448.696244:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.696245:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.696246:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.696248:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.696248:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.696249:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.696249:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.696250:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677448.696251:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.696252:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.696254:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.696254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:26.0:1619677448.696257:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.696258:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.696259:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.696261:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:26.0:1619677448.696261:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.696261:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.696262:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.696264:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.696265:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.696265:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.696266:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.696267:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.696267:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.696270:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.696270:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.696271:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.696271:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.696271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:26.0:1619677448.696273:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.696273:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.696274:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.696274:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677448.696274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.696279:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:25.2:1619677448.696279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.696280:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.696282:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.696282:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.696283:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.696285:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.696286:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:31.0:1619677448.696287:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.696287:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.696288:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.696289:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.696289:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:31.0:1619677448.696290:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.696290:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:31.0:1619677448.696291:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:6.0:1619677448.696291:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.696297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:30.0:1619677448.696299:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.696301:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.696301:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:6.0:1619677448.696301:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.696302:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96711519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.696304:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.696304:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.696304:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:30.0:1619677448.696305:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.696305:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000400:00000200:30.0:1619677448.696306:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.696308:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.696308:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.696308:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.696309:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.696310:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.696310:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677448.696310:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000400:00000200:30.0:1619677448.696311:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.696311:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:31.0:1619677448.696312:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.696312:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.696312:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.696314:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.696314:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.696315:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.696315:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.696316:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.696316:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.696317:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.696318:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:25.2:1619677448.696318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.696319:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.696319:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.696320:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.696324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.696327:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.696329:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.696330:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.696331:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:12.0:1619677448.696331:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.696332:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.696333:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.696333:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.696334:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:13.0:1619677448.696335:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.696336:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.696336:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.696337:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.696337:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.696338:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.696339:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.696340:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.696345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.696351:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.696354:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.696355:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.696355:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.696356:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.696357:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.696365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.696372:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.696375:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.696376:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.696378:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.696380:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.696381:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677448.696382:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.696384:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677448.696384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:18.0:1619677448.696385:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.696386:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677448.696389:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.696391:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:8.0:1619677448.696392:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677448.696393:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.696394:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:8.0:1619677448.696394:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.696395:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677448.696395:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:18.0:1619677448.696396:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.696397:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:8.0:1619677448.696397:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.696398:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.696398:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:8.0:1619677448.696399:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.696401:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:25.2:1619677448.696402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:8.0:1619677448.696402:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96711517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.696403:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.696404:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.696405:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.696408:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.696410:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:8.0:1619677448.696410:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.696412:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.696413:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.696413:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.696414:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.696414:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.696414:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.696415:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:8.0:1619677448.696416:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.696417:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677448.696418:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677448.696418:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.696420:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.696420:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.696422:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.696426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.696430:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.696433:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.696433:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.696434:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.696434:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.696435:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.696435:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.696436:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677448.696436:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.696439:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.696464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.696469:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.696472:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.696472:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.696474:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.696475:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.696477:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.696478:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.696479:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.696480:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.696482:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.696485:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.696486:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.696489:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.696490:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.696490:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.696492:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.696492:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.696493:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.696493:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.696498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.696505:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.696508:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.696509:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.696509:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.696511:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.696513:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.696658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.696666:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.696669:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.696669:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.696671:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.696674:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.696676:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.696677:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.696678:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.696679:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.696680:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.696684:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.696686:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.696689:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.696689:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.696690:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.696692:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.696692:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.696693:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.696693:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.696698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.696705:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.696708:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.696709:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.696709:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.696711:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.696714:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.696753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.696759:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.696761:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.696764:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.696764:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.696767:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.696768:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.696770:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.696771:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.696772:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.696773:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.696776:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.696778:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.696781:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.696782:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.696782:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.696784:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.696785:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.696786:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.696787:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.696788:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.696790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.696793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.696796:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677448.696797:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.696798:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.696799:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677448.696799:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677448.696800:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:12.0:1619677448.696800:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.696802:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.696803:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.696805:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.696861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.2:1619677448.696863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.696869:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.696872:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.696872:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.696873:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:26.0:1619677448.696875:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.696875:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.696878:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.696879:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.696880:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.696881:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.696883:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.696883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:26.0:1619677448.696884:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.696885:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.696887:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.696888:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.696890:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.696890:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.696891:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677448.696891:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.696892:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.696892:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.696894:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:26.0:1619677448.696895:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.696896:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.696896:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:30.0:1619677448.696897:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677448.696897:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:10.0:1619677448.696897:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.696899:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.696899:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.696900:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:31.0:1619677448.696901:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:26.0:1619677448.696901:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000400:00000200:31.0:1619677448.696902:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96711511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.696902:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.696902:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.696903:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.696903:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.696904:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.696904:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.696904:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.696904:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677448.696905:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.696905:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.696906:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.696907:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.696908:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:31.0:1619677448.696909:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.696909:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:31.0:1619677448.696910:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677448.696910:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677448.696912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677448.696913:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.696914:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.696915:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.696915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:9.2:1619677448.696916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.696917:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.696917:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.696919:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.696920:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:17.0:1619677448.696920:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:31.0:1619677448.696921:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677448.696922:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:17.0:1619677448.696923:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:8.2:1619677448.696923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.696923:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.696924:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:17.0:1619677448.696924:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.696924:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.696925:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.696926:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:17.0:1619677448.696926:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.696926:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.696927:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.696928:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.696929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677448.696929:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:2.0:1619677448.696930:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.696933:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677448.696934:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677448.696936:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.696937:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.696938:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.696939:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:20.0:1619677448.696939:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.696940:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677448.696940:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.696941:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677448.696943:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.696954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.696959:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.696962:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.696963:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.696965:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.696967:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.696968:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.696970:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.696971:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.696972:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.696974:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.696977:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.696978:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.696981:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.696982:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.696982:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.696984:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.696985:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.696986:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.696987:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.696987:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677448.696990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.696993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.696998:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677448.696998:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677448.697000:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:18.0:1619677448.697000:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:22.0:1619677448.697001:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677448.697001:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.697002:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.697002:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:18.0:1619677448.697004:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.697006:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.697012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.697018:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.697021:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.697021:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.697023:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.697026:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.697027:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.697028:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.697030:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.697031:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.697032:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.697035:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.697037:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.697040:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.697041:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.697042:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.697043:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.697044:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.697045:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.697046:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.697046:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.697049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.2:1619677448.697051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.697055:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:17.0:1619677448.697057:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677448.697058:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.697059:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:17.0:1619677448.697059:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.697060:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677448.697060:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.697061:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677448.697062:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.697065:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.697121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.697128:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.697130:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.697131:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.697133:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.697136:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.697137:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.697138:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.697140:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.697141:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.697142:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677448.697145:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.697147:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.697150:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.697151:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.697152:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.697153:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.697154:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.697154:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.697155:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.697159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.697167:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.697170:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.697171:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.697173:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.697173:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.697176:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.697240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.697247:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.2:1619677448.697247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:30.0:1619677448.697250:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677448.697250:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677448.697252:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.697253:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:31.0:1619677448.697254:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.697255:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.697255:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.697256:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:5.0:1619677448.697256:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677448.697257:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.697258:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.697259:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.697260:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.697260:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.697261:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.697262:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677448.697264:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.697265:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.697266:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.697267:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.697270:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:20.0:1619677448.697270:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.697273:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.697273:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.697273:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.697274:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.697275:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.697275:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.697276:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.697276:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.697277:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677448.697277:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.697278:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.697279:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.697279:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.697279:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.697280:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.697281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.697281:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.697285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.697286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.697287:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.697289:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.697290:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.697290:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.697290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:22.0:1619677448.697293:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:10.0:1619677448.697293:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.697296:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:6.0:1619677448.697298:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677448.697299:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:25.2:1619677448.697300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:22.0:1619677448.697301:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.697301:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.697302:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:2.0:1619677448.697302:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:31.0:1619677448.697303:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677448.697303:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.697303:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677448.697304:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.697304:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.697305:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000400:00000200:2.0:1619677448.697305:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.697306:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:26.0:1619677448.697307:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.697308:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.697308:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.697309:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (21)++
00000400:00000200:2.0:1619677448.697309:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:10.0:1619677448.697310:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.697310:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:2.0:1619677448.697310:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.697312:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.697313:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.697314:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.697317:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.697319:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.697321:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.697322:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.697323:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.697324:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.697325:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.697326:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.697326:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.697341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.697348:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.697350:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.697351:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.697351:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.697353:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.697354:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.697390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.697397:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.697400:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.697400:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.697402:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.697405:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.697406:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.697408:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.697409:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.697410:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.697411:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.697415:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.697417:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.697419:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.697420:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.697421:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.697423:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.697423:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.697424:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.697424:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.697429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.697436:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.697438:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.697439:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.697441:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.697442:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.697443:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.697446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677448.697453:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.697455:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.697456:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.697457:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.697457:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.697458:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.697572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677448.697578:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677448.697581:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677448.697582:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677448.697584:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.697587:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.697589:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.697590:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.697591:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.697592:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.697593:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.697597:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.697598:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.697601:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.697602:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.697603:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.697605:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.697605:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677448.697606:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677448.697607:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:25.2:1619677448.697611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.697618:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.697621:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.697622:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.697623:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.697625:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.697626:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.697796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.697804:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.697807:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.697809:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.697810:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.697811:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.697813:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.697814:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.697815:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.697816:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.697817:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.697821:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.697823:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.697826:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.697827:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.697828:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.697829:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.697830:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.697831:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.697831:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.697836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.2:1619677448.697839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.697844:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.697846:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:17.0:1619677448.697847:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.697848:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.697849:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:21.2:1619677448.697850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:17.0:1619677448.697850:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.697851:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.697852:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:17.0:1619677448.697852:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.697853:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.697854:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.697855:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.697855:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677448.697856:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:17.0:1619677448.697857:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.697858:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.697858:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.697859:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:17.0:1619677448.697859:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.697861:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.697863:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.697863:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.697866:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.697866:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.697867:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.697869:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.697869:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.697870:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:6.0:1619677448.697870:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.697871:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:6.0:1619677448.697871:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.697873:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.697873:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.697874:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.697874:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.697874:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.697876:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:25.2:1619677448.697879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.697879:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.697880:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:6.0:1619677448.697881:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:6.0:1619677448.697883:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (24)--
00000800:00000200:6.0:1619677448.697883:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.697884:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677448.697885:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:6.0:1619677448.697885:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:30.0:1619677448.697888:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (24)--
00000400:00000200:30.0:1619677448.697889:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.697891:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.697892:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:31.0:1619677448.697893:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (24)--
00000400:00000200:31.0:1619677448.697893:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.697894:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.697895:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.697896:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.697898:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.697919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.697926:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.697929:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.697930:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.697931:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.697934:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.697935:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.697936:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.697938:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.697939:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.697940:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677448.697943:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.697945:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.697948:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.697949:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.697950:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.697952:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.697952:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.697953:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.697953:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.697957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.697965:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.697967:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.697968:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.697970:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.697971:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.697973:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.697978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.697985:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.697988:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.697989:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.697990:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.697990:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.697991:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.698143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.698150:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.698153:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.698154:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.698156:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.698158:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.698160:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.698161:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.698161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:10.0:1619677448.698162:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.698164:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.698165:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.698169:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:10.0:1619677448.698169:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.698170:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.698171:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.698172:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.698174:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.698175:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.698175:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.698175:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:26.0:1619677448.698176:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677448.698177:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:26.0:1619677448.698178:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711497] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.698179:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.698180:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.698180:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.698181:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.698182:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:25.2:1619677448.698183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:26.0:1619677448.698184:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.698184:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.698184:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.698185:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.698185:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.698188:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.698189:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.698190:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:26.0:1619677448.698190:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:4.2:1619677448.698190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.698191:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.698191:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.698192:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.698193:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.698193:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.698193:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:30.0:1619677448.698194:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.698195:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:31.0:1619677448.698196:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.698196:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.698196:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.698197:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.698197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.698197:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:12.0:1619677448.698203:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.698206:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.698207:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.698208:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.698210:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.698211:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.698272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.698279:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.698282:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.698284:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.698284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677448.698286:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.698287:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.698288:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.698289:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.698290:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.698291:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.698291:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:31.0:1619677448.698292:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677448.698294:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.698298:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677448.698302:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.698303:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.698305:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:24.0:1619677448.698306:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.698307:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.698308:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:24.0:1619677448.698308:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:31.0:1619677448.698309:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677448.698309:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96711495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.698310:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677448.698310:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.698311:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.698311:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:24.0:1619677448.698311:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.698312:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:24.0:1619677448.698312:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.698315:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.698316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677448.698317:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.698320:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.698321:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.698321:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.698323:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.698323:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677448.698323:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677448.698324:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.698325:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677448.698326:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.698327:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.698328:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.698329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677448.698331:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.698332:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.698335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.698337:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.698340:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.698342:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:13.0:1619677448.698342:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677448.698342:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.698344:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.698345:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.698346:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.698346:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.698347:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.698347:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:13.0:1619677448.698349:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677448.698350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:21.2:1619677448.698352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:17.0:1619677448.698357:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.698358:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677448.698359:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.698360:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.698360:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.698361:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:17.0:1619677448.698362:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.698362:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677448.698363:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.698364:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.698366:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.698367:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677448.698369:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.698370:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.698371:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.698372:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677448.698375:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.698377:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.698378:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.698379:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.698380:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.698381:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.698382:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.698383:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.698384:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.698384:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:9.2:1619677448.698388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.698390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.698396:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.698397:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.698397:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:18.0:1619677448.698398:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.698398:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.698399:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.698400:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.698400:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.698401:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677448.698402:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.698627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.698633:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.2:1619677448.698637:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:20.0:1619677448.698639:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.698640:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:2.0:1619677448.698642:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:5.0:1619677448.698643:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.698645:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.698645:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.698646:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:5.0:1619677448.698647:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677448.698649:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.698649:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.698650:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.698651:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.698652:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.698653:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677448.698653:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.698654:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.698656:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.698657:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.698658:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.698659:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.698660:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:2.0:1619677448.698662:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.698663:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.698664:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677448.698665:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:2.0:1619677448.698665:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.698667:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.698667:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.698668:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:2.0:1619677448.698668:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.698669:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:2.0:1619677448.698669:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.698669:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.698671:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.698671:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.698672:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:25.2:1619677448.698673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:2.0:1619677448.698673:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.698677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.698681:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.698683:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:12.0:1619677448.698683:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:30.0:1619677448.698684:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.698686:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.698686:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677448.698687:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:12.0:1619677448.698687:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.698689:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:12.0:1619677448.698689:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.698690:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:25.2:1619677448.698691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:12.0:1619677448.698692:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.698698:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.698700:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.698701:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.698702:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.698703:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.698704:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.698770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.2:1619677448.698774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:5.2:1619677448.698775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.698779:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677448.698781:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677448.698781:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.698782:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.698783:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677448.698785:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:17.0:1619677448.698785:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.698785:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:26.0:1619677448.698786:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.698786:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677448.698788:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.698789:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.698789:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.698790:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.698791:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:22.0:1619677448.698792:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.698792:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.698793:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.698794:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677448.698794:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.698795:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.698795:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.698796:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.698796:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.698797:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677448.698798:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.698798:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.698800:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.698801:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677448.698801:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.698804:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.698806:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:22.0:1619677448.698806:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.698808:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.2:1619677448.698809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:26.0:1619677448.698810:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.698810:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.698811:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:22.0:1619677448.698811:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.698812:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.698812:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:1.0:1619677448.698812:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.698813:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.698815:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677448.698815:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.698816:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677448.698816:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677448.698816:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.698816:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.698817:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677448.698817:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.0:1619677448.698817:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:1.0:1619677448.698817:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677448.698818:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677448.698818:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677448.698819:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:25.2:1619677448.698820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.698820:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.698821:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.2:1619677448.698821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.2:1619677448.698821:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:1.0:1619677448.698821:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.698821:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677448.698826:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:13.0:1619677448.698827:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000400:00000200:8.0:1619677448.698828:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.698829:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677448.698829:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:20.0:1619677448.698830:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:13.0:1619677448.698830:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.698830:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.698832:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:20.0:1619677448.698832:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:13.0:1619677448.698832:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.698832:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:20.0:1619677448.698833:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.698833:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677448.698834:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:8.0:1619677448.698834:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96711488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.698835:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.698835:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.698835:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.698835:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.698835:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.698836:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:5.0:1619677448.698836:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:13.0:1619677448.698837:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.698837:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.698837:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.698838:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.698838:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.698839:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:20.0:1619677448.698839:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677448.698840:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677448.698840:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000400:00000200:30.0:1619677448.698841:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.698842:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.698843:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.698843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:30.0:1619677448.698844:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.698846:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000400:00000200:8.0:1619677448.698847:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.698847:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677448.698848:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.698848:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.698849:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:8.0:1619677448.698849:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:5.0:1619677448.698849:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.698850:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.698850:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.698851:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.698851:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.0:1619677448.698852:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.0:1619677448.698852:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.698853:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.698853:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:5.0:1619677448.698853:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.698855:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.698855:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.698855:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.698855:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:30.0:1619677448.698856:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:5.0:1619677448.698856:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.698857:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000400:00000200:5.0:1619677448.698857:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677448.698858:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.698858:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:8.0:1619677448.698858:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:30.0:1619677448.698859:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:8.0:1619677448.698859:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:5.0:1619677448.698859:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96711486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.698860:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.698860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:5.0:1619677448.698860:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.698861:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.698862:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.2:1619677448.698864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.698865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000400:00000200:5.0:1619677448.698866:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.698867:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677448.698869:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.698870:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:24.0:1619677448.698872:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:20.0:1619677448.698872:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:13.0:1619677448.698872:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.698872:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.698873:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:13.0:1619677448.698873:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.698873:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677448.698873:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.698873:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:24.0:1619677448.698875:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.698875:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:20.0:1619677448.698875:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.698875:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677448.698876:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.698876:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677448.698876:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:24.0:1619677448.698877:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.0:1619677448.698877:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.698877:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677448.698879:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.698880:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:25.2:1619677448.698881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.698888:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.698890:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.698891:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.698893:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.698894:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.698895:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.698904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.698910:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.698912:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.698913:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.698914:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.698914:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.698915:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.699018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.699025:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.699027:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.699028:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.699030:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.699032:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.699034:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.699035:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.699037:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.699038:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.699039:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.699042:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.699043:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.699046:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.699047:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.699047:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.699049:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.699050:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.699050:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.699051:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.699057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.699065:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.699067:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.699068:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.699069:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.699070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:17.0:1619677448.699071:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.699075:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677448.699076:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.2:1619677448.699077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.699078:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677448.699080:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.699080:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.699081:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.699082:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.699083:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.699083:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.699084:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.699084:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.699085:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:1.0:1619677448.699085:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:17.0:1619677448.699086:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.699088:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.699089:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.699090:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.699093:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.699095:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.699098:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.699099:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.699100:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.699102:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.699102:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.699103:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.699103:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.699108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.699114:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.699117:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.699118:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.699120:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.699121:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.699122:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.699500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.699506:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.699509:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.699509:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.699511:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.699514:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.699515:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:21.2:1619677448.699517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:1.0:1619677448.699517:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.699518:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.699518:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.699519:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.699522:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.699523:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.699525:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.699526:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.699527:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.699528:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:1.0:1619677448.699528:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.699529:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:31.0:1619677448.699530:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.699530:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.699531:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.699532:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000400:00000200:31.0:1619677448.699533:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.699533:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677448.699533:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677448.699534:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.699536:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.699536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:31.0:1619677448.699537:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.699538:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.699538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.699539:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.699542:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.699543:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.699543:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677448.699544:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677448.699545:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:5.2:1619677448.699545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677448.699546:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.699546:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677448.699546:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:31.0:1619677448.699547:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.699547:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677448.699547:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.699548:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.699549:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.699549:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.699550:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.699551:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677448.699551:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.699552:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.699552:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.699552:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:25.2:1619677448.699555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.699555:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.699556:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.699557:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.699560:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.699561:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:12.0:1619677448.699562:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677448.699563:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.699564:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.699564:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.699564:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.699565:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677448.699566:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.699566:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.699568:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.699570:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.699572:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.699572:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.699574:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.699575:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.699575:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.699577:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.699577:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.699578:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.699579:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.699580:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.699582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.699585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677448.699588:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.699591:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.699591:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.699593:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:13.0:1619677448.699593:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.699594:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.699595:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.699595:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.699595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:13.0:1619677448.699596:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.699597:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:1.0:1619677448.699600:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677448.699603:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677448.699604:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677448.699606:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.699609:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.699611:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677448.699612:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.699613:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.699614:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.699615:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677448.699619:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.699621:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677448.699624:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.699624:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:1.0:1619677448.699625:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:1.0:1619677448.699627:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:1.0:1619677448.699627:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677448.699628:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677448.699628:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:25.2:1619677448.699633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.699639:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.699641:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.699642:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.699643:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.699645:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.699648:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.699668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.699675:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.699678:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.699679:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.699681:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.699683:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.699685:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.699686:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.699687:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.699688:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.699689:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.699692:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.699694:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.699697:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.699698:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.699698:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.699700:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.699701:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.699701:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.699702:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.699706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.699713:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.699716:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.699717:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.699717:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.699719:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.699722:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.699783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:20.2:1619677448.699785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677448.699789:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677448.699790:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677448.699792:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:5.0:1619677448.699793:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677448.699793:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677448.699796:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:18.0:1619677448.699796:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.699797:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.699799:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.699800:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.699801:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.699802:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.699803:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.699804:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.699804:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.699806:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.699806:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.699808:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.699808:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.699810:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.699812:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.699814:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.699815:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.699817:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.699817:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.699818:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.699819:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.699819:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.699820:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.699821:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.699821:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677448.699821:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.699822:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.699823:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677448.699823:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:21.2:1619677448.699824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:5.0:1619677448.699824:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.699825:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.699826:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.699827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:9.2:1619677448.699829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.699831:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.699834:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:1.0:1619677448.699834:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677448.699835:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677448.699836:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677448.699836:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:6.0:1619677448.699837:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.699837:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.699838:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.699838:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:6.0:1619677448.699839:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.699839:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.699840:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:6.0:1619677448.699840:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677448.699840:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.699842:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96711476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.699842:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.699843:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.699843:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:6.0:1619677448.699844:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.699845:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:2.0:1619677448.699845:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.699848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:6.0:1619677448.699849:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.699851:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.699854:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.699855:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.699855:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.699855:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.699857:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:18.0:1619677448.699858:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677448.699858:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.699858:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677448.699858:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.699859:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677448.699859:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:18.0:1619677448.699860:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.699861:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:25.2:1619677448.699863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.699869:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.699872:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.699873:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.699873:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.699875:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.699879:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:25.2:1619677448.699888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000800:00000200:10.0:1619677448.699894:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.699897:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.699897:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.699898:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.699898:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.699899:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677448.699917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:5.2:1619677448.699918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.699924:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.699924:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.699927:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:17.0:1619677448.699927:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:8.0:1619677448.699927:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:1.0:1619677448.699928:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.699931:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.699931:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.699934:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.699934:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.699936:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.699936:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.699937:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.699938:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.699939:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.699940:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.699941:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.699941:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.699942:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.699943:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.699947:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.699948:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.699950:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.699951:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.699952:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.699953:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.699954:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.699954:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.699954:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.699955:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.699955:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.699956:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.699956:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.699957:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677448.699957:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.699957:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.699958:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.699959:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.699961:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:25.2:1619677448.699962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.699968:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:12.0:1619677448.699969:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677448.699971:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.699972:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677448.699972:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:1.0:1619677448.699972:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.699973:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.699974:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.699976:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677448.699976:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.699976:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:12.0:1619677448.699978:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.699985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:13.0:1619677448.699991:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.699994:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.699995:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.699996:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.699996:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.699998:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.700252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.700259:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.700262:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.700262:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.700264:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.700267:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.700268:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.700269:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.700271:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.700272:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.700273:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.700276:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.700278:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.700281:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.700282:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.700282:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.700284:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.700284:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.700285:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.700285:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.700290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.700299:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.700302:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.700303:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.700304:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.700305:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.700307:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.700350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.700358:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.700361:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.700362:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.700364:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.700366:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.700367:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.700369:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.700370:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.700371:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.700372:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.700375:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.700376:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.700379:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.700380:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.700380:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.700382:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.700382:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.700384:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.700385:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.700385:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.700388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.700391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.700396:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677448.700398:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.700399:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.700400:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.700400:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:1.0:1619677448.700400:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.700401:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.700402:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.700402:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677448.700405:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.700407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.2:1619677448.700412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:1.0:1619677448.700414:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.700417:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.700418:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.700418:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.700419:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677448.700420:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:1.0:1619677448.700420:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:12.0:1619677448.700423:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677448.700425:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.700427:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677448.700428:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.700429:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:12.0:1619677448.700431:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.700432:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.700433:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677448.700434:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:12.0:1619677448.700437:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.700438:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.2:1619677448.700439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:12.0:1619677448.700442:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.700442:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:12.0:1619677448.700443:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677448.700445:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:12.0:1619677448.700445:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677448.700446:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:30.0:1619677448.700447:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.700447:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:30.0:1619677448.700450:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:8.2:1619677448.700451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.700452:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.700454:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.700454:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.700455:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.700456:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.700457:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.700458:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.700459:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:5.0:1619677448.700459:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:30.0:1619677448.700462:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.700462:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.700463:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.700464:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.700465:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.700465:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.700466:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.700467:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.700467:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677448.700468:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:30.0:1619677448.700469:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.700470:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.700471:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.700471:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.700476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.700483:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.700485:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.700486:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.700487:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.700489:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.700490:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.700498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.700501:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.700504:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.700506:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.700507:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.700509:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.700510:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.700511:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.700513:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.700514:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.700515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677448.700515:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.700518:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.700520:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.700522:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.700523:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.700523:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:31.0:1619677448.700524:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677448.700524:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.700526:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:31.0:1619677448.700527:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.700527:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.700528:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.700528:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:31.0:1619677448.700529:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.700529:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.700530:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677448.700531:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.700532:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.700532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677448.700533:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.700534:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.700535:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.2:1619677448.700535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.700539:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.700540:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.700540:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:2.0:1619677448.700541:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677448.700543:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.700543:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:2.0:1619677448.700543:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:31.0:1619677448.700544:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.700544:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.700544:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677448.700545:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.700545:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.700545:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.700546:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.700547:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:2.0:1619677448.700547:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.700548:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.700549:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677448.700549:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:25.2:1619677448.700553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.700561:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.700564:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.700564:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.700565:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.700567:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.700568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677448.700570:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677448.700574:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.700576:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.700577:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.700579:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.700581:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.700582:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.700584:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.700585:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.700586:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.700587:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:5.2:1619677448.700589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:17.0:1619677448.700590:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.700592:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.700594:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.700595:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677448.700595:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.700595:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.700596:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.700597:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.700598:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:17.0:1619677448.700598:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.700598:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677448.700599:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.700601:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.700602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:18.0:1619677448.700603:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.700604:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.700605:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.700607:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.700608:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.700609:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.700609:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.700611:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:18.0:1619677448.700612:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.700612:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.700613:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.700613:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.700614:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.700616:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.700616:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.700616:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677448.700617:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.700619:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.700619:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.700621:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.700622:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.700623:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.700625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.700627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677448.700632:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677448.700633:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:17.0:1619677448.700635:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:2.0:1619677448.700635:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:17.0:1619677448.700636:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.700636:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677448.700636:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.700637:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:17.0:1619677448.700638:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.700639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:17.0:1619677448.700640:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677448.700645:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.700648:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.700650:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.2:1619677448.700651:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:5.0:1619677448.700655:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.700657:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.700659:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:5.0:1619677448.700659:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.700660:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.700662:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:5.0:1619677448.700662:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.700663:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:5.0:1619677448.700663:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.700664:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.700665:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.700668:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.700668:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.700670:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:5.0:1619677448.700670:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.700671:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.700672:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.700673:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.700673:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:10.0:1619677448.700674:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.700674:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:10.0:1619677448.700675:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:5.0:1619677448.700676:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.700677:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:10.0:1619677448.700678:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.700678:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.700678:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:10.0:1619677448.700680:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:25.2:1619677448.700682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.700682:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.700683:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.700684:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.700685:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.700686:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.700686:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.700687:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677448.700689:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:25.2:1619677448.700691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.700692:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.700693:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.700694:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.700695:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:4.2:1619677448.700696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.700697:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:12.0:1619677448.700697:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:25.2:1619677448.700700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:12.0:1619677448.700700:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.700701:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.700702:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.700703:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677448.700704:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.700705:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:12.0:1619677448.700705:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.700706:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.700707:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:26.0:1619677448.700708:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.700709:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.700710:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:26.0:1619677448.700710:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.700711:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.700711:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.700712:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:26.0:1619677448.700712:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.700713:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.700714:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.700715:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.700716:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.700719:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.700720:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.700723:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.700724:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.700724:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.700726:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.700726:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.700727:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.700728:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.700732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.700739:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.700742:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.700743:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.700743:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.700745:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.700748:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.700751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.700758:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.700761:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.700762:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.700762:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.700763:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.700764:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.700885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.700893:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.700895:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.700897:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.700899:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.700900:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.700901:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.700903:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.700904:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.700905:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.700906:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.700909:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.700910:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.700912:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.700913:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.700914:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.700915:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.700916:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.700917:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.700919:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.700919:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.700920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.2:1619677448.700925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.700930:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.700931:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.700932:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.700933:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:30.0:1619677448.700933:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.700934:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.700935:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.700935:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.700936:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.700937:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.2:1619677448.701171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:21.2:1619677448.701173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677448.701178:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:31.0:1619677448.701180:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.701180:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.701181:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:31.0:1619677448.701182:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677448.701183:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.701184:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.701186:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.701187:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.701187:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.701188:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.701188:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:31.0:1619677448.701190:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.701190:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.701191:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.701191:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.701192:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.701192:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.701193:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.701193:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677448.701198:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.701198:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.701200:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.701201:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.701202:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.701203:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.701203:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.701203:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.701204:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:31.0:1619677448.701205:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.701205:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.701205:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677448.701206:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:10.0:1619677448.701207:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.701207:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:31.0:1619677448.701208:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:10.0:1619677448.701208:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:31.0:1619677448.701209:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677448.701209:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677448.701211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.2:1619677448.701213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:21.2:1619677448.701214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.701217:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.701219:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.701220:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.701220:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:30.0:1619677448.701221:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:13.0:1619677448.701222:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.701223:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.701223:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:30.0:1619677448.701224:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:17.0:1619677448.701224:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:13.0:1619677448.701224:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677448.701224:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.701225:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.701226:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:1.0:1619677448.701227:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.701230:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.701261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.701268:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.701270:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.701271:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.701273:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.701276:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.701278:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.701279:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.701280:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.701281:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.701282:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:21.2:1619677448.701283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:26.0:1619677448.701286:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.701287:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677448.701289:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:26.0:1619677448.701290:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.701291:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.701292:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.701292:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.701293:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:26.0:1619677448.701301:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.701301:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677448.701301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677448.701302:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.701303:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:18.0:1619677448.701303:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.701305:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.701306:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.701308:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.701309:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:18.0:1619677448.701309:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.701310:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.701311:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.701312:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.701312:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.701315:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.701315:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.701315:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:18.0:1619677448.701317:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.701318:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:18.0:1619677448.701320:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.701321:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.701321:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.701323:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.701323:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.701324:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.701325:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.701330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.2:1619677448.701333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:6.0:1619677448.701338:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677448.701339:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.701340:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:17.0:1619677448.701341:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:22.0:1619677448.701342:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.701342:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.701342:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.701345:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.701345:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:17.0:1619677448.701347:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.701347:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:17.0:1619677448.701349:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.701350:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.701351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:17.0:1619677448.701351:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.701352:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.701353:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.701356:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:17.0:1619677448.701357:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.701358:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:17.0:1619677448.701359:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.701359:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.701361:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.701362:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.701362:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.701363:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:22.0:1619677448.701364:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.701365:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.701365:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677448.701366:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:17.0:1619677448.701366:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677448.701367:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.701367:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677448.701368:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.701369:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.701370:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:25.2:1619677448.701371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:22.0:1619677448.701374:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.701377:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.701380:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.701380:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.701381:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.701381:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.701381:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.701382:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.701383:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.701384:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.701385:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.701386:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.701386:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.701388:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.701388:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.701388:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:9.2:1619677448.701390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:25.2:1619677448.701391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:21.2:1619677448.701394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677448.701399:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:6.0:1619677448.701400:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.701401:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.701401:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677448.701401:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:22.0:1619677448.701402:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.701402:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.701403:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:18.0:1619677448.701403:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.701403:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000400:00000200:2.0:1619677448.701403:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.701404:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:6.0:1619677448.701404:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.701405:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:2.0:1619677448.701405:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.701408:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.701477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.701485:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.701488:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.701490:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.701492:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.701492:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.701494:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.701495:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.701497:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.701498:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.701499:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.701502:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.701504:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.701507:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.701507:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.701508:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.701510:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.701510:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.701511:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.701512:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.701515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.701523:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.701525:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.701526:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.701527:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.701529:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.701530:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.701594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.701600:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.701603:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.701603:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.701605:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.701607:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.701609:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.701610:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.701611:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.701612:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.701613:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.701617:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.701619:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.701620:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.701621:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.701622:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.701623:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.701624:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.701624:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.701625:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.701629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.701636:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.2:1619677448.701638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677448.701638:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.701638:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.701640:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.701640:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.701641:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:10.0:1619677448.701646:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.701649:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.701649:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.701652:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.701654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:10.0:1619677448.701654:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.701656:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.701657:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.701659:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.701659:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:10.0:1619677448.701660:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.701661:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.701662:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.701664:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.701665:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.701666:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.701668:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.701668:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.701670:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.701671:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.701672:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.701673:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.701673:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677448.701673:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.701674:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.701675:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.701675:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:5.0:1619677448.701675:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.701676:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.701677:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:5.0:1619677448.701679:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.701681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.701681:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:4.2:1619677448.701682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.0:1619677448.701683:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.701684:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.701685:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.701687:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.701687:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.701688:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677448.701689:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677448.701689:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677448.701690:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.701692:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:18.0:1619677448.701692:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:25.2:1619677448.701693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:2.0:1619677448.701693:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.701694:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.701694:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.701696:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.701696:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.701699:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.701700:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:18.0:1619677448.701701:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677448.701703:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:26.0:1619677448.701703:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677448.701704:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.701704:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.701705:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.701706:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.701706:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:26.0:1619677448.701706:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.701708:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677448.701709:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:26.0:1619677448.701711:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.701713:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.701716:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.701717:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.701717:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.701719:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.701719:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.701721:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.701722:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.701723:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.701725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:4.2:1619677448.701728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.701733:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677448.701735:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.701736:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.701737:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.701737:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677448.701737:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.701738:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.701739:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.701739:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.701743:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.701769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.701774:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.701777:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.701778:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.701780:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.701782:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.701784:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.701785:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.701786:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.701787:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.701789:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.701792:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.701794:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.701796:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.701797:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.701798:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.701799:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.701800:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.701801:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.701802:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.701803:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.701805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.701808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.701813:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677448.701813:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.701815:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677448.701815:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.701816:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677448.701816:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677448.701817:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.701818:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.701819:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.701821:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.701966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.701974:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.701976:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.701978:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.701980:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.701981:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.701982:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.701984:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.701985:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.701986:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.701987:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.701990:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.701992:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.701994:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.701995:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.701996:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.701997:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.701998:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.701999:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.701999:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.702003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.702009:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.702012:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.702012:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.702014:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.702016:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.702017:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.702023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.702029:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.702032:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.702033:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.702033:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.702034:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.702034:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.702075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.702078:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.702080:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.702083:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.702083:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.702086:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.702087:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.702089:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.702090:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.702091:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.702092:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.702095:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.702097:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.702100:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.702101:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.702101:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.702103:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.702104:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.702105:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.702105:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.702109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.702116:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.702119:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.702120:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.702122:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.702123:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.702124:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.702165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.702170:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.702173:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.702174:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.702176:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.702178:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.702180:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.702181:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.702182:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.702183:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.702184:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.702187:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.702189:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.702191:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.702192:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.702192:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.702194:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.702194:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.702195:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.702195:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.702200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.702207:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.702210:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.702211:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.702211:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.702213:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.702215:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.702262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.702269:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.702272:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.702273:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.702275:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.702278:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.702279:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.702281:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.702282:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.702283:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.702284:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.702287:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.702289:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.702292:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.702293:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.702298:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.702299:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.702300:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.702301:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.702301:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.702306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.702314:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.702317:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.702318:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.702318:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.702320:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.702323:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.702419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.702428:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.702430:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.702432:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.702435:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.702435:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.702436:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.702437:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.702439:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.702440:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.702441:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.702444:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.702445:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.702448:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.702449:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.702450:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.702452:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.702452:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.702453:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.702453:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.702458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.702467:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.702469:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.702470:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.702472:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.702472:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.702476:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.702477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:20.0:1619677448.702484:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.702487:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.702488:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.702488:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.702489:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.702490:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.702493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:5.2:1619677448.702493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677448.702499:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.702501:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.702502:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.702503:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:10.0:1619677448.702503:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.702504:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:2.0:1619677448.702506:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.702507:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.702507:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.702509:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.702510:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.702511:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.702512:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677448.702512:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.702513:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.702514:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.702514:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.702517:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.702518:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.702518:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.702519:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:2.0:1619677448.702521:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.702523:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.702523:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.702524:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.702525:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.702526:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677448.702526:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.702527:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677448.702528:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.702528:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.702528:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:10.0:1619677448.702529:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.702529:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.702531:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.702531:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.702532:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677448.702532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.702533:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677448.702535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:25.2:1619677448.702537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:18.0:1619677448.702539:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677448.702540:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677448.702542:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.702542:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.702543:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677448.702543:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.702544:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:18.0:1619677448.702545:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.702546:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:20.0:1619677448.702546:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.702546:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.702547:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.702547:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.702548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:31.0:1619677448.702549:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.702549:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.702550:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.702551:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:5.0:1619677448.702552:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:9.2:1619677448.702553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:5.0:1619677448.702553:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.702554:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:5.0:1619677448.702554:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.702555:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.702557:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.702557:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:5.0:1619677448.702557:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.702560:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.702561:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.702562:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:22.0:1619677448.702563:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.702564:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.702564:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.702565:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.702565:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.702565:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000400:00000200:22.0:1619677448.702566:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.702566:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.702566:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000400:00000200:22.0:1619677448.702567:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:5.0:1619677448.702567:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.702567:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677448.702568:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:22.0:1619677448.702569:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.702569:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:22.0:1619677448.702570:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.702570:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:22.0:1619677448.702571:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.702572:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:25.2:1619677448.702574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:22.0:1619677448.702575:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.702578:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677448.702581:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.702582:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.702582:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.702582:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.702583:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.702584:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000400:00000200:6.0:1619677448.702584:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.702585:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.702586:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.702586:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.702587:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677448.702588:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.702589:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:9.2:1619677448.702590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.702598:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.702601:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.702602:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.702602:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.702604:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.702606:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.702614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.702621:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.702624:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.702625:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.702625:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.702626:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.702626:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.702744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.702751:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.702754:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.702755:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.702757:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.702759:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.702761:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.702762:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.702763:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.702764:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.702765:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.702769:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.702771:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.702774:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.702774:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.702775:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.702777:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.702777:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.702778:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.702778:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.702783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.702784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677448.702790:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.702791:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677448.702791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:22.0:1619677448.702793:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:17.0:1619677448.702794:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:6.0:1619677448.702794:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.702795:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.702795:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.702797:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:22.0:1619677448.702798:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.702798:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.702800:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:17.0:1619677448.702800:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.702800:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:22.0:1619677448.702801:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:17.0:1619677448.702802:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:25.2:1619677448.702803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000400:00000200:18.0:1619677448.702803:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.702803:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.702805:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.702806:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677448.702806:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.702807:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.702808:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.702808:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677448.702809:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.702810:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.702810:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:18.0:1619677448.702811:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677448.702813:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.702813:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.702814:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.702814:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.702815:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.702815:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:18.0:1619677448.702816:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.702817:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.702818:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.702820:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.702820:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.702821:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:17.0:1619677448.702821:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:18.0:1619677448.702822:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.702822:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677448.702823:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677448.702824:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:18.0:1619677448.702825:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677448.702825:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.702826:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677448.702826:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.702827:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677448.702827:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.702830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:9.2:1619677448.702830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.702837:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:12.0:1619677448.702838:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677448.702839:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.702839:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.702840:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:18.0:1619677448.702841:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:12.0:1619677448.702841:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.702842:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:12.0:1619677448.702843:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.702843:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677448.702845:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.702846:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.702850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:13.0:1619677448.702857:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.702860:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.702861:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.702861:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.702862:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.702863:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.703037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.703040:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.703043:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.703045:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.703046:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.703047:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.703049:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677448.703050:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.703052:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.703053:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.703054:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677448.703058:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.703060:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677448.703063:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.703063:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677448.703064:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.703066:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677448.703067:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.703067:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.703068:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.703072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.703079:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.703082:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.703083:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.703084:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.703086:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.703087:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.703095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.703102:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.703105:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.703105:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.703107:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.703110:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.703112:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:4.2:1619677448.703112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:10.0:1619677448.703113:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.703114:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.703115:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.703116:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677448.703120:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677448.703120:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.703122:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:10.0:1619677448.703122:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.703124:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.703124:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:10.0:1619677448.703125:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:24.0:1619677448.703126:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.703126:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.703127:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.703128:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.703128:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:24.0:1619677448.703129:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.703129:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:24.0:1619677448.703130:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.703131:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96711435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.703132:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.703133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:24.0:1619677448.703133:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.703134:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.703137:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.703138:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.703140:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:24.0:1619677448.703141:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.703142:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:24.0:1619677448.703142:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.703142:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677448.703143:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.703144:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.703144:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.703145:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.703145:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677448.703146:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:24.0:1619677448.703146:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:31.0:1619677448.703147:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:8.2:1619677448.703150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.703158:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.703160:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.703161:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.703162:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.703164:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.703166:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.703302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.703310:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.703312:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.703313:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.703315:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.703317:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.703318:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.703319:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.703321:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.703322:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.703323:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.703325:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.703327:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.703328:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.703329:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.703329:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.703330:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.703331:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.703332:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.703333:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.703334:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.703337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.2:1619677448.703340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.703345:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.703346:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.703347:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.703348:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:10.0:1619677448.703349:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.703350:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.703350:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.703351:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.703351:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.703351:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.703357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677448.703365:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677448.703367:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677448.703369:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.703372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:13.0:1619677448.703372:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677448.703372:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.703373:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:12.0:1619677448.703374:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.703376:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.703377:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.703378:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:12.0:1619677448.703378:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.703381:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:12.0:1619677448.703382:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.703382:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.703385:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.703385:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.703387:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.703388:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.703389:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:12.0:1619677448.703389:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:12.0:1619677448.703389:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:18.0:1619677448.703390:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.703391:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.703391:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:18.0:1619677448.703392:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.703392:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000400:00000200:18.0:1619677448.703393:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:12.0:1619677448.703394:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677448.703395:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677448.703396:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:18.0:1619677448.703397:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.703398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.703399:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.703402:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.703402:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.2:1619677448.703402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:18.0:1619677448.703403:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.703405:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.703405:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.703406:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.703407:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.703410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.703410:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:13.0:1619677448.703411:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:5.0:1619677448.703412:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:13.0:1619677448.703413:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000400:00000200:5.0:1619677448.703413:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.703414:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677448.703415:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000400:00000200:5.0:1619677448.703415:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.703416:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.703417:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:5.0:1619677448.703418:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:10.0:1619677448.703419:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.703420:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.703420:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.703423:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.703425:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.703468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.703474:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.703477:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.703477:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.703479:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.703482:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.703484:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.703485:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.703486:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.703487:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.703488:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.703492:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.703494:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.703497:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.703498:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.703498:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.703500:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.703501:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.703501:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.703502:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.703506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.703512:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:4.2:1619677448.703513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.703515:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.703516:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.703517:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:26.0:1619677448.703518:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.703519:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.703520:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.703522:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:26.0:1619677448.703523:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.703523:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.703524:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.703527:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.2:1619677448.703528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:10.0:1619677448.703528:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.703529:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.703530:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.703531:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.703534:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.703535:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.703536:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.703538:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:10.0:1619677448.703539:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.703539:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:30.0:1619677448.703540:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.703540:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.703542:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.703542:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.703543:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.703543:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.703543:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:30.0:1619677448.703544:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:10.0:1619677448.703544:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.703544:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:30.0:1619677448.703545:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.703547:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.703548:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.703548:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:25.2:1619677448.703548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.703550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677448.703552:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.703553:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.703555:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:30.0:1619677448.703556:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.703557:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:26.0:1619677448.703557:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:30.0:1619677448.703558:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.703558:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:30.0:1619677448.703559:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.703559:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677448.703559:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.703559:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.703560:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.703560:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.703561:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.703561:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.703561:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.703561:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.703564:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.2:1619677448.703565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:25.2:1619677448.703568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.703572:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:22.0:1619677448.703575:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.703575:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.703576:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.703578:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.703578:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.703578:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.703579:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.703579:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.703580:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.703581:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677448.703581:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.703589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.703594:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.703597:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.703598:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.703600:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.703603:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.703604:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.703605:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.703606:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.703608:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.703608:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.703612:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.703613:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.703615:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.703616:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.703616:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.703618:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.703618:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.703619:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.703620:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.703624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.703631:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.703634:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.703634:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.703635:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.703636:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.703638:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.703767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.703773:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.703776:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.703777:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.703779:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.703781:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.703782:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.703784:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.703785:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.703786:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.703787:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.703790:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.703792:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.703793:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.703794:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.703795:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.703796:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.703796:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.703797:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.703798:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.703802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.703809:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.703811:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.703811:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.703812:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.703813:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.703815:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.703931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.703938:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.703941:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.703942:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.703944:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.703947:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.703948:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.703950:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711426] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.703951:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.703952:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.703953:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.703956:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.703958:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.703960:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.703961:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.703962:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.703963:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.703964:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.703965:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.703965:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:4.2:1619677448.703966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:25.2:1619677448.703969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.2:1619677448.703973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.0:1619677448.703973:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677448.703975:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:8.0:1619677448.703975:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.703976:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677448.703978:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:8.0:1619677448.703978:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.703979:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.703979:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:4.2:1619677448.703979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:31.0:1619677448.703980:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.703980:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.703982:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:8.0:1619677448.703982:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:5.0:1619677448.703982:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:31.0:1619677448.703983:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:8.0:1619677448.703984:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.703985:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.703985:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.703986:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.703987:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:8.0:1619677448.703987:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.703987:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.703988:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:25.2:1619677448.703989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000400:00000200:5.0:1619677448.703989:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:26.0:1619677448.703990:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.703990:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:5.0:1619677448.703991:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.703993:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.703993:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.703994:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.703994:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.703995:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.703996:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000400:00000200:26.0:1619677448.703996:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.703997:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:26.0:1619677448.703998:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:30.0:1619677448.703999:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.704000:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.704000:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000400:00000200:26.0:1619677448.704000:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96711423] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.704000:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.704000:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.704001:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:26.0:1619677448.704001:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.704001:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.704002:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:26.0:1619677448.704002:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.704002:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:26.0:1619677448.704003:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:8.0:1619677448.704003:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.704003:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677448.704004:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.704004:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.704005:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.0:1619677448.704006:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.704007:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.704007:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.704008:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.2:1619677448.704009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.704010:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.0:1619677448.704010:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.704010:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.704011:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:26.0:1619677448.704013:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.704013:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:5.0:1619677448.704013:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.704013:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677448.704014:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:25.2:1619677448.704015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.704016:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.704016:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.704016:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:26.0:1619677448.704017:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.704017:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.2:1619677448.704018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677448.704019:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.704020:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.704020:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.704021:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:25.2:1619677448.704022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:17.0:1619677448.704022:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.704022:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.704023:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.704024:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.704025:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677448.704025:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:13.0:1619677448.704025:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.704026:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.704027:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.704028:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677448.704029:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:13.0:1619677448.704030:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.704032:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.704033:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.704033:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.704035:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.704038:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.704090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.704096:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.704099:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.704099:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.704102:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.704104:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.704105:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.704107:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.704108:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.704109:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.704110:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.704114:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.704115:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.704118:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.704119:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.704120:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.704122:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.704122:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.704123:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.704124:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.704129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.704136:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.704138:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.704139:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.704141:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.704142:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.704144:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.704276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.2:1619677448.704276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.704282:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.704284:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.704285:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.704285:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677448.704288:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:1.0:1619677448.704288:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.704290:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.704291:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.704291:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:1.0:1619677448.704292:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.704298:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.704299:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.704300:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.704301:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.704302:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.704303:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.704304:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.704304:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.704305:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.704306:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.704310:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.704311:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.704314:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.704315:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.704316:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.704316:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.704317:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.704319:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.704319:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.704319:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.704320:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.704320:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677448.704321:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.704322:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.704322:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.704323:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677448.704324:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.704325:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.704325:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.704325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.2:1619677448.704328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.704333:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:17.0:1619677448.704334:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677448.704335:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:17.0:1619677448.704336:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:13.0:1619677448.704336:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.704337:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677448.704337:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.704338:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.704339:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.704340:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677448.704341:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.704341:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677448.704341:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.704343:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.704345:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.704348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.704354:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.704357:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.704358:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.704358:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.704359:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.704361:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.704377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.704384:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.704386:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.704387:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.704389:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.704392:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.704393:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.704395:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.704396:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.704398:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.704399:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.704402:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.704404:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.704406:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.704407:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.704408:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.704410:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.704410:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.704411:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.704411:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.704416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.704424:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.704427:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.704427:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.704429:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.704430:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.704432:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.704435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.704442:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.704444:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.704445:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.704446:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.704446:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.704447:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.704457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677448.704463:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677448.704466:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677448.704466:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677448.704468:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.704471:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.704473:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677448.704474:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.704475:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.704476:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.704477:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677448.704480:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.704482:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677448.704485:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.704485:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:1.0:1619677448.704486:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:1.0:1619677448.704488:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677448.704488:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677448.704489:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677448.704489:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677448.704494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.704501:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.704503:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.704504:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.704506:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.704507:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.704509:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.2:1619677448.704559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.704567:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.704569:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.704572:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.704573:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.704574:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.704576:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.704577:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.704578:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.704579:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.704580:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.704584:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.704586:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.704589:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.704590:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.704590:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:4.2:1619677448.704591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677448.704592:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.704593:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.704594:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.704596:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.704596:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.704598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:24.0:1619677448.704599:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.704602:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:20.2:1619677448.704603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:8.0:1619677448.704604:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.704605:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.704608:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.704609:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.704610:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96711416] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.704610:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677448.704611:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:24.0:1619677448.704612:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.704612:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.704612:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:31.0:1619677448.704613:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:24.0:1619677448.704613:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677448.704613:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.704614:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.704615:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.704615:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.704615:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:24.0:1619677448.704618:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.704618:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:24.0:1619677448.704620:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.704623:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.704623:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.704624:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.704625:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.704626:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.704627:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.704627:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.704632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.704639:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.704642:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.704643:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.704643:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.704645:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.704648:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.704652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.704658:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.704661:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.704662:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.704662:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.704663:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.704663:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.704668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677448.704676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:10.0:1619677448.704676:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.704679:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.704679:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.704681:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.704682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.0:1619677448.704683:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:10.0:1619677448.704684:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.704685:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:8.0:1619677448.704686:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.704687:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:17.0:1619677448.704687:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.704687:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.704688:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.704689:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.704689:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.704690:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:10.0:1619677448.704690:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:1.0:1619677448.704691:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677448.704692:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.704693:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.704694:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.704694:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.704695:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96711414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.704696:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.704697:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:8.0:1619677448.704697:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.704698:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677448.704698:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677448.704698:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.704699:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.704699:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.704700:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:8.0:1619677448.704700:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677448.704701:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.704701:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:17.0:1619677448.704702:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.704702:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.704702:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677448.704703:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.704703:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:8.0:1619677448.704706:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.704707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:17.0:1619677448.704707:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.704708:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.704710:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677448.704711:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.704712:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.704713:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:17.0:1619677448.704713:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.704713:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677448.704714:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677448.704714:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.704715:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:17.0:1619677448.704715:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677448.704715:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:26.0:1619677448.704716:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.704716:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:8.0:1619677448.704716:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.704716:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:17.0:1619677448.704717:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.704717:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.704718:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.704718:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.704719:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.704720:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:8.2:1619677448.704720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:25.2:1619677448.704723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:25.2:1619677448.704728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:5.0:1619677448.704728:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:12.0:1619677448.704730:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.704730:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.704731:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.704732:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677448.704733:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:12.0:1619677448.704733:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.704733:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.704734:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000400:00000200:12.0:1619677448.704735:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.704736:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.704736:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:10.0:1619677448.704737:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.704738:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:12.0:1619677448.704738:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.704738:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.704739:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.704740:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.705040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.2:1619677448.705044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677448.705047:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.705050:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:1.0:1619677448.705050:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.705052:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.705053:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:30.0:1619677448.705054:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677448.705054:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677448.705055:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.705056:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677448.705056:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.705057:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.705058:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.705059:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.705060:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.705061:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677448.705061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:1.0:1619677448.705061:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711411] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.705062:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677448.705063:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.705064:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.705066:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.705067:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.705068:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.705069:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:1.0:1619677448.705070:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.705071:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:31.0:1619677448.705072:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.705072:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.705073:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.705073:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:1.0:1619677448.705073:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.705074:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.705075:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.705075:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.705076:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677448.705076:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.705077:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:26.0:1619677448.705077:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.705077:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:26.0:1619677448.705078:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677448.705078:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.705079:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:26.0:1619677448.705080:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96711410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.705080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:1.0:1619677448.705080:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.705080:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.705081:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677448.705081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:1.0:1619677448.705081:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677448.705082:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.705084:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:25.2:1619677448.705085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.705086:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:26.0:1619677448.705088:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.705088:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677448.705088:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.705089:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.705090:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.705090:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.705091:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:13.0:1619677448.705092:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.0:1619677448.705092:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677448.705094:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677448.705094:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.705094:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:10.0:1619677448.705094:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.705095:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:13.0:1619677448.705095:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.705095:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.705096:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677448.705097:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.705097:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.705098:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677448.705098:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:12.0:1619677448.705098:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.705099:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:24.0:1619677448.705099:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96711409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.705100:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:13.0:1619677448.705100:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.705101:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:24.0:1619677448.705101:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.705102:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.705103:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:25.2:1619677448.705104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677448.705106:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.705108:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.705110:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.705110:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677448.705111:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.705111:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677448.705111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:12.0:1619677448.705112:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:24.0:1619677448.705113:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.705113:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:12.0:1619677448.705113:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.705114:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:24.0:1619677448.705115:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:12.0:1619677448.705115:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.705116:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.705116:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677448.705116:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:13.0:1619677448.705117:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677448.705118:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.705118:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677448.705119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.705119:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.705122:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.705122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:20.0:1619677448.705124:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.705125:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.705127:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.705127:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:20.0:1619677448.705128:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.705129:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.705130:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.0:1619677448.705130:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:17.0:1619677448.705131:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.705131:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.705132:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.705132:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.705133:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:20.0:1619677448.705133:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.705133:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.705135:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.705137:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677448.705138:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.705139:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.705140:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.705142:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.705142:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.705143:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.705144:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.705148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.705154:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.705157:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.705158:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.705159:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.705160:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.705162:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.705470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.705477:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.705480:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.705481:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.705483:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.705485:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.705487:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.705488:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.705490:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.705491:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.705492:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.705495:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.705497:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.705500:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.705500:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.705501:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.705503:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.705503:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.705504:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.705504:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.705510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.705514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.705520:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.705521:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.705521:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.705522:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.705522:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.705525:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.705525:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.705528:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.705529:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.705530:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:1.0:1619677448.705530:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677448.705531:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.705532:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.705533:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677448.705533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000400:00000200:17.0:1619677448.705534:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.705536:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.705540:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.705541:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.705542:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.705543:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.705543:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.705544:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.705545:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.705546:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.705546:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677448.705547:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.705547:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.705549:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.705550:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.705551:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.705551:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.705556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.705558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.705564:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677448.705565:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:18.0:1619677448.705568:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:6.0:1619677448.705568:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677448.705568:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:22.0:1619677448.705569:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.705569:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.705571:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.705571:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.705572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677448.705573:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:18.0:1619677448.705575:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.705576:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677448.705577:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711405] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.705579:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.705580:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677448.705580:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.705581:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.705584:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:18.0:1619677448.705584:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.705587:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.705587:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.705587:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:30.0:1619677448.705590:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.705590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:18.0:1619677448.705590:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.705591:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.705591:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677448.705592:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:18.0:1619677448.705593:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.705593:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:30.0:1619677448.705594:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.705594:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:30.0:1619677448.705595:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.705595:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.0:1619677448.705595:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:30.0:1619677448.705596:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.705597:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.705598:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:8.2:1619677448.705599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.705601:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:30.0:1619677448.705602:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.705602:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677448.705604:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.705605:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.705606:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:1.0:1619677448.705606:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:5.0:1619677448.705607:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96711403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.705608:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.705609:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.705609:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677448.705609:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.705609:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.705610:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:5.0:1619677448.705610:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.705610:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.705611:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.705612:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:5.0:1619677448.705612:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.705612:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.705615:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677448.705615:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:25.2:1619677448.705616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.705617:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:5.0:1619677448.705617:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.705618:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.705619:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.0:1619677448.705620:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.705621:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.705621:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:12.0:1619677448.705623:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.705624:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:21.2:1619677448.705625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.705625:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:12.0:1619677448.705626:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.705626:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.705626:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000400:00000200:12.0:1619677448.705628:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.705628:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:25.2:1619677448.705629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.705629:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.2:1619677448.705630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:13.0:1619677448.705630:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.705631:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.705634:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.2:1619677448.705634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:31.0:1619677448.705635:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.705636:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677448.705636:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677448.705637:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:25.2:1619677448.705637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.705637:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:31.0:1619677448.705638:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677448.705639:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:22.0:1619677448.705639:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:20.0:1619677448.705639:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:6.0:1619677448.705639:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:31.0:1619677448.705640:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.705640:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.705640:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.705641:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:22.0:1619677448.705642:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.705643:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000400:00000200:22.0:1619677448.705644:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.705644:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.705646:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:22.0:1619677448.705646:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:13.0:1619677448.705646:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677448.705647:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.705648:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.705648:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:25.2:1619677448.705649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:22.0:1619677448.705649:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.705649:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:22.0:1619677448.705650:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.705650:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:22.0:1619677448.705651:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:21.2:1619677448.705653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:22.0:1619677448.705654:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.705656:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.705656:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.705658:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:31.0:1619677448.705659:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.705659:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.705660:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.705660:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:22.0:1619677448.705660:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.705660:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.705661:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:31.0:1619677448.705662:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.705662:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:22.0:1619677448.705662:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.705662:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677448.705662:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:22.0:1619677448.705664:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:2.0:1619677448.705664:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.705665:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.705666:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:2.0:1619677448.705666:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.705668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:2.0:1619677448.705668:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.705669:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.705670:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.705671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:2.0:1619677448.705671:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.705672:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.705675:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677448.705676:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000400:00000200:2.0:1619677448.705676:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.705678:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:6.0:1619677448.705678:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:2.0:1619677448.705678:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.705679:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.705679:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677448.705679:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.705680:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677448.705681:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.705682:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.705682:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.705683:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.705685:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:2.0:1619677448.705685:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.705685:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.705686:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.705687:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.705688:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.705690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.705693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:12.0:1619677448.705697:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.2:1619677448.705698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677448.705699:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:12.0:1619677448.705700:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677448.705701:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:12.0:1619677448.705701:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.705702:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.705702:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:12.0:1619677448.705702:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.705703:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.705705:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.705705:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.705708:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.705710:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.705712:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.705713:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.705714:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.705715:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.705716:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.705717:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.705718:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.705721:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.705723:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.705726:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.705726:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.705727:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.705729:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.705729:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.705730:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.705731:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.705734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:4.2:1619677448.705739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677448.705741:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.705743:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.705744:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.705746:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.705746:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677448.705747:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.705748:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:8.0:1619677448.705748:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.705749:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.705751:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.705754:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.705755:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.705757:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.705758:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.705759:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.705759:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.705762:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.705764:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.705766:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.705767:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.705768:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.705769:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.705770:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.705770:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.705771:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.705775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.705784:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.705786:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.705787:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.705787:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.705790:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.705792:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.705868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.705875:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.705877:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.705878:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.705878:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.705879:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.705880:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.705893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.705900:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.705903:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.705904:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677448.705905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:26.0:1619677448.705906:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.705909:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.705911:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.705912:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.705912:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:26.0:1619677448.705914:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.705914:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677448.705915:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.705916:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.705918:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.705919:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.705921:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.705921:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.705923:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.705924:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.705925:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.705926:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.705926:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.705929:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.705930:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:10.0:1619677448.705930:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.705931:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.705932:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.705933:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.705934:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.705935:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.705935:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.705936:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.705936:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.705937:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.705937:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:25.2:1619677448.705939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.705939:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.705940:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.705940:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.705941:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:13.0:1619677448.705945:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:9.2:1619677448.705945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677448.705947:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.705948:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.705950:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.705951:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.705953:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677448.705953:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.705956:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.705957:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.705957:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.705959:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.705963:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.706070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.706076:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.706079:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.706079:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.706081:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.706083:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.706085:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.706086:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.706087:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.706088:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.706090:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.706093:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.706094:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.706097:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.706098:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.706098:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.706100:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.706101:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.706102:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.706103:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.706104:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677448.706106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.2:1619677448.706109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677448.706113:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:17.0:1619677448.706114:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677448.706115:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:17.0:1619677448.706116:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.706117:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677448.706117:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:24.0:1619677448.706118:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.706118:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:24.0:1619677448.706119:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.706123:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:9.2:1619677448.706131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:8.0:1619677448.706138:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.706140:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677448.706141:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.706141:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.706142:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.706143:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677448.706225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.706232:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.706235:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.706237:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.706239:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.706239:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.706240:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.706242:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.706243:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.706244:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.706245:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.706248:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.706250:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.706253:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.706254:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.706254:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.706256:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.706257:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.706257:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.706258:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.706262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.706270:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.706273:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.706274:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.706274:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.706276:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.706279:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.706396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.706404:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.706406:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.706407:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.706409:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.706411:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.706413:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.706414:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.706415:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.706416:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.706417:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.706421:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.706422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:24.0:1619677448.706423:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.706426:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.706426:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.706427:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.706429:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.706429:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:18.0:1619677448.706429:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:24.0:1619677448.706430:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.706431:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:18.0:1619677448.706432:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.706433:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.706435:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.706436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:18.0:1619677448.706437:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.706439:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.706441:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677448.706442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:18.0:1619677448.706442:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.706444:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.706444:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:18.0:1619677448.706445:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.706445:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.706447:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.706448:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.706449:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.706450:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677448.706450:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.706452:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677448.706453:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:2.0:1619677448.706453:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.706454:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.706455:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.706455:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677448.706456:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.706456:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.706456:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.706457:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:31.0:1619677448.706458:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:18.0:1619677448.706458:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.706458:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:30.0:1619677448.706459:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.706459:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.706459:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677448.706460:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:30.0:1619677448.706461:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:18.0:1619677448.706461:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000400:00000200:30.0:1619677448.706462:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.706462:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:30.0:1619677448.706463:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.706464:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.706464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.706465:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:5.2:1619677448.706467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000400:00000200:30.0:1619677448.706469:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.706470:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.706472:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.706472:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677448.706472:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:30.0:1619677448.706473:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.706474:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.706474:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:2.0:1619677448.706474:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:30.0:1619677448.706475:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:20.0:1619677448.706475:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.706475:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:30.0:1619677448.706476:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677448.706476:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:30.0:1619677448.706477:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.706477:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.706477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.706478:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677448.706480:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:25.2:1619677448.706481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:20.0:1619677448.706481:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:17.0:1619677448.706483:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.2:1619677448.706486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677448.706486:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677448.706487:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:1.0:1619677448.706487:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.706489:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.706490:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.706491:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.706491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:31.0:1619677448.706492:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.706492:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.706493:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:31.0:1619677448.706494:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.706494:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:17.0:1619677448.706494:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.706496:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.706496:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.706496:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:17.0:1619677448.706498:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:26.0:1619677448.706499:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677448.706499:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:26.0:1619677448.706502:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:20.0:1619677448.706502:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:17.0:1619677448.706502:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.706502:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:5.0:1619677448.706502:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.706505:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:26.0:1619677448.706506:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.706506:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.706508:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.706508:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000400:00000200:5.0:1619677448.706508:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.706509:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.706509:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:5.0:1619677448.706509:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.706511:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:17.0:1619677448.706511:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.706511:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677448.706511:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.706512:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677448.706513:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.706513:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:5.0:1619677448.706513:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.706514:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.706515:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.706516:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677448.706517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:26.0:1619677448.706518:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.706518:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.706521:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.706522:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.706523:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:26.0:1619677448.706524:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.706524:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.706525:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.0:1619677448.706526:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:5.0:1619677448.706526:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.706527:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.706527:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.706527:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677448.706527:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.706528:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.706528:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.706528:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:8.0:1619677448.706529:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.706529:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677448.706530:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.706530:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677448.706531:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.706532:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677448.706532:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:26.0:1619677448.706533:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:25.2:1619677448.706533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.706534:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677448.706537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.706539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.706541:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.706544:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.706544:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.706545:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:12.0:1619677448.706546:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.706547:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:18.0:1619677448.706548:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:13.0:1619677448.706548:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:18.0:1619677448.706549:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.706549:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.706549:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:2.0:1619677448.706549:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.706550:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.706551:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:18.0:1619677448.706551:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.706555:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.706620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.706627:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.706629:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.706630:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.706632:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.706635:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.706636:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:4.2:1619677448.706636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677448.706637:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.706639:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.706639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:24.0:1619677448.706640:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.706641:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.706643:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:24.0:1619677448.706644:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.706645:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:2.0:1619677448.706645:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:26.0:1619677448.706646:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677448.706646:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:2.0:1619677448.706648:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:24.0:1619677448.706649:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.706649:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:10.0:1619677448.706649:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.706650:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.706650:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:2.0:1619677448.706651:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.706652:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.706652:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677448.706652:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.706653:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677448.706653:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.706654:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:10.0:1619677448.706654:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677448.706655:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.706656:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.706656:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.706658:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677448.706658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:2.0:1619677448.706659:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.706660:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.706661:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.706661:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.706662:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.706666:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.706666:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:10.0:1619677448.706667:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.706668:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.706669:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.706670:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.706670:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.706670:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.706670:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.706671:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.706671:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.706673:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.706673:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.706673:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:1.0:1619677448.706673:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.706674:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.706674:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.706674:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677448.706675:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.706676:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:1.0:1619677448.706676:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:10.0:1619677448.706677:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.706678:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.706679:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:9.2:1619677448.706679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.706680:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.706683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.706685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.706686:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.706688:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.706689:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.706690:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.706691:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.706692:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:20.0:1619677448.706692:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:18.0:1619677448.706693:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.0:1619677448.706694:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677448.706695:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:20.0:1619677448.706695:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.706696:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.706696:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:20.0:1619677448.706697:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.706698:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.706700:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.706842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.706849:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.706852:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.706854:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.706856:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.706857:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.706858:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.706860:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.706861:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.706862:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.706864:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.706867:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.706868:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.706870:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.706871:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.706872:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.706873:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.706874:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.706875:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.706875:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.706879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.706885:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.706888:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.706888:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.706890:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.706892:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.706893:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.706896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.706902:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.706905:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.706906:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.706907:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.706910:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.706912:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.706913:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.706914:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.706915:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.706916:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.706920:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.706921:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.706924:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.706925:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.706925:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.706927:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.706927:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.706928:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.706928:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677448.706930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:25.2:1619677448.706933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.0:1619677448.706936:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.706939:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.706941:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677448.706941:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:5.0:1619677448.706941:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.706944:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:5.0:1619677448.706944:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.706945:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.706945:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.706945:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677448.706947:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.706948:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.706949:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.706950:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:5.0:1619677448.706950:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.706951:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677448.706954:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.706956:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.706959:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.706959:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.706960:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.706962:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.706963:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.706963:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.706964:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.706968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.706976:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.706978:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.706979:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.706979:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.706981:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.706984:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.707024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:4.2:1619677448.707028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.707032:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.707035:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:26.0:1619677448.707036:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.707037:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.707038:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:31.0:1619677448.707039:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.707039:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.707039:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677448.707041:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.707042:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.707042:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.707044:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.707045:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.707045:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.707046:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.707046:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.707048:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.707050:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.707051:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.707051:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.707052:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.707053:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.707055:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.707055:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.707056:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.707057:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.707057:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.707058:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.707059:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.707060:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.707060:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.707060:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.707061:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.707061:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.707063:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.707064:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.707064:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.707064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.707065:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.707069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:12.0:1619677448.707070:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.707073:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.707073:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.707075:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.707077:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:13.0:1619677448.707077:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.707078:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:22.0:1619677448.707080:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.707081:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.707081:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.707083:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.707086:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.707109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677448.707112:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677448.707115:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677448.707117:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.707118:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677448.707119:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.707121:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:20.0:1619677448.707122:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.707124:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.707125:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.707126:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:20.0:1619677448.707129:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.707131:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677448.707134:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.707135:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:20.0:1619677448.707136:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.707138:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677448.707139:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677448.707140:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:20.0:1619677448.707141:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:20.0:1619677448.707142:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:25.2:1619677448.707144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.707147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:13.0:1619677448.707151:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677448.707152:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000800:00000200:13.0:1619677448.707154:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:5.0:1619677448.707154:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000400:00000200:13.0:1619677448.707155:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.707155:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:13.0:1619677448.707156:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.707156:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:12.0:1619677448.707157:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.707159:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.707167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:12.0:1619677448.707173:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:21.2:1619677448.707175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:12.0:1619677448.707176:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.707177:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.707178:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.707178:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.707180:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677448.707180:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.707183:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.707184:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.707186:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.707188:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.707189:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.707191:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.707192:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.707193:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.707194:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.707197:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.707199:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.707201:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.707202:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.707203:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.707204:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.707205:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.707206:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.707206:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.707211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.707218:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.707220:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.707222:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.707222:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.707224:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.707226:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.707357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.707364:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.707367:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.707368:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.707370:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.707373:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.707374:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.707376:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.707377:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.707378:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.707379:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.707382:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.707384:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.707386:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.707387:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.707387:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.707389:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.707389:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.707390:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.707390:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.707395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.707403:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.707405:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.707406:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.707408:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.707410:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.707411:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.707419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.707425:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.707427:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.707430:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.707430:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.707433:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.707434:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677448.707435:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.707437:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.707437:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.707438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:5.0:1619677448.707438:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:20.0:1619677448.707441:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:5.0:1619677448.707441:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.707442:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:5.0:1619677448.707444:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:20.0:1619677448.707445:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.707447:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.707447:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.707448:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677448.707448:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:20.0:1619677448.707449:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.707450:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.707450:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:20.0:1619677448.707452:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.707453:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.707454:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.707454:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.707455:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.707456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677448.707456:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:20.0:1619677448.707457:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.707457:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.707459:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.707461:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.707462:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677448.707463:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:20.0:1619677448.707463:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.707465:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:20.0:1619677448.707465:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:31.0:1619677448.707466:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.707466:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.707467:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:31.0:1619677448.707468:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.707468:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.707469:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677448.707470:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:25.2:1619677448.707471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.707473:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.707474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.707475:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.707477:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.707477:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.707477:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:31.0:1619677448.707479:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.707479:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677448.707481:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.0:1619677448.707481:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.707482:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.707483:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.2:1619677448.707560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.707568:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.707570:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.707571:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.707573:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.707576:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.707577:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.707579:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.707580:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.707581:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.707582:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.707585:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.707586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:26.0:1619677448.707587:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.707590:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.707590:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.707591:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.707593:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.707593:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.707594:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677448.707594:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:26.0:1619677448.707595:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677448.707596:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.707597:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.2:1619677448.707599:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:8.0:1619677448.707604:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.707606:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.707608:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:8.0:1619677448.707608:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.707609:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96711374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.707610:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.707611:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.707612:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.707612:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.707612:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.707612:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.707614:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.707617:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.707618:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.0:1619677448.707619:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.707622:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.707623:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.707623:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.707625:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.707626:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.707627:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.707629:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.707629:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677448.707631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.707634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:18.0:1619677448.707639:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677448.707641:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:18.0:1619677448.707642:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:24.0:1619677448.707643:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:18.0:1619677448.707643:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.707643:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.707644:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.707645:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:18.0:1619677448.707645:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.707649:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.707750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.707758:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.707761:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.707761:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.707764:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.707766:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.707768:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.707769:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.707771:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.707772:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.707773:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.707776:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.707777:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.707779:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.707780:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.707781:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.707782:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.707783:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.707783:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.707784:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.707788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.707795:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.707797:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.707798:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.707800:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.707801:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.707802:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.707862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677448.707868:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.0:1619677448.707871:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.707873:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.0:1619677448.707874:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.707876:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677448.707876:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.707877:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:5.0:1619677448.707877:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.707880:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.707880:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.707881:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.707882:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.707883:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.707883:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.707885:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.707886:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.707887:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.707888:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.707888:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.707889:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.707890:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.707892:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.707894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:2.0:1619677448.707894:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.707895:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.707897:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.707897:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.707899:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.707899:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.707899:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.707900:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:30.0:1619677448.707901:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.707901:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.707901:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.707902:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.707902:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.707903:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.707903:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677448.707903:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:30.0:1619677448.707904:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:2.0:1619677448.707904:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.707905:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:30.0:1619677448.707906:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.707908:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.707908:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.707908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:9.2:1619677448.707909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677448.707910:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.707911:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96711370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.707913:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.707914:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.707915:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.707916:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677448.707917:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:30.0:1619677448.707918:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.707918:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.707919:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.707919:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:30.0:1619677448.707920:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.707920:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.707920:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.707920:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:22.0:1619677448.707923:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.707923:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.707924:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.707925:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.707925:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.707926:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677448.707926:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677448.707927:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.707928:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.707929:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.707931:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.707931:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.707933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.707939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:9.2:1619677448.707941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.707945:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677448.707947:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.707947:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.707948:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.707948:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.707950:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.707950:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.707950:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.707951:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.707951:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.707952:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:18.0:1619677448.707952:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.707952:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:20.0:1619677448.707953:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677448.707953:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.707954:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.708078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.708084:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.708086:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.708087:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.708089:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.708091:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.708092:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.708093:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.708095:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.708096:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.708097:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.708100:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.708102:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.708105:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.708105:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.708106:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.708108:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.708108:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.708109:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.708109:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.708114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:4.2:1619677448.708118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677448.708121:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.708124:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.708125:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.708125:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:26.0:1619677448.708126:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:18.0:1619677448.708127:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.708129:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:18.0:1619677448.708129:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:10.0:1619677448.708129:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.708131:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.708134:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.708136:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.708137:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.708138:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.708139:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.708140:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.708143:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.708144:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.708147:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.708148:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.708149:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.708150:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.708151:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.708152:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.708153:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.708154:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.708156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.708159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.708164:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.708166:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.708167:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.708167:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.708168:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.708168:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.708169:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.708170:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.708170:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.708173:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.708182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.708189:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.708192:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.708193:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.708193:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.708194:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.708195:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.708364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.708370:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.708373:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.708375:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.708376:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.708377:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.708379:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:5.0:1619677448.708380:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.708381:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.708382:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.708383:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:5.0:1619677448.708387:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.708389:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:5.0:1619677448.708391:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.708392:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:5.0:1619677448.708393:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.0:1619677448.708394:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677448.708395:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.708396:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.708396:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.708401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.708410:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.708412:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.708413:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.708414:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.708416:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.708418:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.708508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.708516:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.708519:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.708520:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.708522:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.708525:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.708527:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.708528:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.708529:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.708530:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.708531:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.708535:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.708537:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.708540:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.708541:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.708541:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.708543:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.708543:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.708544:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.708544:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.708549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.708557:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.708560:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.708561:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.708561:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.708564:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.708566:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.708572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:6.0:1619677448.708579:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.708581:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.708582:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.708582:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.708583:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.708584:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.708619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.708625:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.708628:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.708629:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.708631:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.708633:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.708635:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.708636:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.708637:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.708638:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.708639:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.708642:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.708644:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.708646:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.708646:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.708647:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.708648:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.708649:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.708649:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.708650:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.708655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.708663:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.708664:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.708665:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.708666:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.708667:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.708668:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.708678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.708685:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.708688:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.708689:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.708691:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.708694:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.708695:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.708696:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.708698:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.708699:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.708700:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.708703:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.708705:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.708708:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.708709:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.708709:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.708711:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.708711:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.708712:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.708713:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.708717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.708724:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.708727:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.708728:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.708728:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.708730:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.708733:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.708784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.708788:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.708791:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.708792:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.708794:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.708796:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.708797:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.708798:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.708800:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.708800:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.708802:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677448.708805:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.708807:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.708809:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.708810:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:6.0:1619677448.708811:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:6.0:1619677448.708812:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.708813:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.708813:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.708813:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.708818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.708826:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.708828:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.708829:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.708830:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.708832:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.708833:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.708839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.708846:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.708849:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.708850:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.708851:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.708852:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.708853:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.708870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:21.2:1619677448.708876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.708881:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.708883:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.708885:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.708885:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.708886:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.708888:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.708890:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.708890:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.708891:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.708893:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.708894:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.708895:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.708897:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.708898:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.708898:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677448.708900:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.708900:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.708901:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.708902:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.708903:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.708906:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.708906:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.708907:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.708907:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.708908:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.708909:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.708910:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.708910:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.708911:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.708911:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.708912:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.708912:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.708912:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.708913:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677448.708913:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.708914:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.708914:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.708916:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.708916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:25.2:1619677448.708918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.708923:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:13.0:1619677448.708925:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.708925:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.708926:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.708928:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:13.0:1619677448.708928:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.708928:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.708928:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:13.0:1619677448.708930:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.708931:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677448.708932:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.708933:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.708942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.708949:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.708951:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.708952:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.708953:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.708953:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.708955:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.709114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.709121:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.709124:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.709125:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.709127:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.709129:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.709131:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.709132:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.709134:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.709134:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.709135:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.709139:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.709140:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.709143:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.709144:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.709144:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.709146:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.709146:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.709147:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.709147:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.709152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.709160:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.709162:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.709163:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.709164:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.709166:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.709168:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.709182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.709189:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.709192:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.709193:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.709193:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.709194:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.709195:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.709334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.709340:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.709342:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.709343:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.709345:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.709346:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.709347:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.709348:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.709350:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.709351:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.709352:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.709355:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.709356:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.709360:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.709360:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.709361:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.709363:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.709363:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.709365:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.709366:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.709366:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.709370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.709371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.709377:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677448.709377:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677448.709379:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.709380:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.709380:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:17.0:1619677448.709381:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677448.709381:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.709383:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.709385:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.709386:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.709438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677448.709444:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:20.2:1619677448.709447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:17.0:1619677448.709447:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677448.709447:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000400:00000200:1.0:1619677448.709449:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.709450:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.709452:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:1.0:1619677448.709452:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.709453:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.709455:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.709455:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:1.0:1619677448.709455:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.709457:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.709458:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.709458:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.709458:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.709460:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:21.2:1619677448.709461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:20.0:1619677448.709462:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.709462:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.709464:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.709464:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.709465:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 504 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.2:1619677448.709466:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677448.709467:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677448.709467:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.709468:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:1.0:1619677448.709468:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677448.709469:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:18.0:1619677448.709470:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.709470:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677448.709470:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677448.709471:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677448.709471:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:5.0:1619677448.709472:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.709472:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000400:00000200:20.0:1619677448.709474:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.709474:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:5.0:1619677448.709474:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:18.0:1619677448.709476:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.709476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.709477:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:5.0:1619677448.709477:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.709478:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:20.0:1619677448.709480:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.709480:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96711356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.709480:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.709481:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:20.0:1619677448.709482:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:18.0:1619677448.709482:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.709482:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677448.709484:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000400:00000200:18.0:1619677448.709484:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.709484:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96711355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.709484:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:18.0:1619677448.709485:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:5.0:1619677448.709486:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.709486:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:5.0:1619677448.709487:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.709487:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.709489:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.709489:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.709490:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.709490:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:9.2:1619677448.709490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677448.709491:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:18.0:1619677448.709491:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.709492:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677448.709492:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.709493:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:5.0:1619677448.709493:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.709494:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:18.0:1619677448.709496:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.709496:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677448.709497:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:18.0:1619677448.709497:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:18.0:1619677448.709498:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:24.0:1619677448.709499:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:5.0:1619677448.709499:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.709500:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:18.0:1619677448.709500:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.0:1619677448.709500:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677448.709500:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:24.0:1619677448.709501:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.709501:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.709502:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:24.0:1619677448.709503:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.709503:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.709503:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:5.0:1619677448.709503:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.709503:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.709504:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.709505:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.709506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:24.0:1619677448.709506:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.709509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:25.2:1619677448.709510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.709514:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:2.0:1619677448.709514:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:12.0:1619677448.709516:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:22.0:1619677448.709517:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:2.0:1619677448.709517:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:22.0:1619677448.709518:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.709518:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:12.0:1619677448.709519:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:6.0:1619677448.709519:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:21.2:1619677448.709520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:12.0:1619677448.709520:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.709520:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:22.0:1619677448.709521:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000400:00000200:12.0:1619677448.709522:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.709523:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:6.0:1619677448.709524:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.709525:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:22.0:1619677448.709526:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677448.709526:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:6.0:1619677448.709526:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.709529:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.709531:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.709532:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.709534:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.709535:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.709536:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.709537:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.709541:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.709542:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677448.709546:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.709546:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.709547:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:6.0:1619677448.709549:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.709550:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.709551:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.709552:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.709553:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:25.2:1619677448.709555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:21.2:1619677448.709558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:26.0:1619677448.709562:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:22.0:1619677448.709563:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:26.0:1619677448.709565:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:22.0:1619677448.709565:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:10.0:1619677448.709565:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.709566:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.709566:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.709567:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:26.0:1619677448.709568:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.709571:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.2:1619677448.709768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.709774:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.709777:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.709777:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.709780:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.709781:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.709783:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.709784:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.709785:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.709786:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.709787:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.709791:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.709792:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.709796:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.709797:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.709797:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.709799:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.709800:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.709801:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.709801:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.709805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.709812:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.709814:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.709815:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.709817:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.709818:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.709819:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.709886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.709893:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.709896:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.709897:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.709898:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.709901:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.709902:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.709904:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.709905:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.709906:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.709907:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.709910:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.709912:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.709915:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.709916:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.709917:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.709919:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.709919:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.709920:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.709920:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.709924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.709932:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.709935:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.709935:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.709936:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.709938:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.709940:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.709963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.709969:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.709972:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.709972:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.709974:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.709977:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.709979:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.709980:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.709981:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.709983:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.709983:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.709987:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.709989:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.709991:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.709992:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.709993:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.709995:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.709995:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.709996:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.709996:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.710000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.710007:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.710009:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.710010:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.710011:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.710013:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.710016:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.710036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.710042:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.710045:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.710046:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.710048:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.710050:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.710052:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.710053:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.710054:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.710055:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.710056:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.710060:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.710061:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.710064:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.710065:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.710066:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.710067:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.710068:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.710069:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.710069:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.710073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.710081:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.710083:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.710084:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.710086:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.710087:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.710089:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.710262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.710269:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.710272:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.710272:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677448.710274:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.710277:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.710278:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.710280:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.710281:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.710282:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.710283:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677448.710286:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.710288:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677448.710291:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.710292:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:10.0:1619677448.710292:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:10.0:1619677448.710299:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.710300:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.710301:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.710301:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.710306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.710315:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.710318:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.710318:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.710319:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.710321:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.710324:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.710336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.710344:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.710346:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.710347:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.710349:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.710352:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.710353:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.710354:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.710355:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.710357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:8.0:1619677448.710357:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.710357:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.710361:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.710363:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677448.710365:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677448.710365:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.710366:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.710367:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.710368:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.0:1619677448.710368:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.710369:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:30.0:1619677448.710370:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.710370:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.710371:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.710371:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:30.0:1619677448.710372:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.710373:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.710374:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:8.2:1619677448.710374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677448.710375:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.710376:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.710377:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677448.710377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:30.0:1619677448.710378:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.710381:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.710382:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:30.0:1619677448.710383:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677448.710384:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:30.0:1619677448.710385:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.710385:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.710385:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.710386:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.710386:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:1.0:1619677448.710386:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.710387:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:30.0:1619677448.710388:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.710388:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677448.710388:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677448.710388:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.710389:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:30.0:1619677448.710390:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:1.0:1619677448.710390:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677448.710392:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.710392:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.710394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.710398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.710402:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.710404:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.710405:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.710405:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.710406:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:26.0:1619677448.710407:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.710408:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.710409:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.710410:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.710410:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.710440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.710448:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.710451:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.710453:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.710455:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.710455:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.710456:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.710458:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.710459:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.710460:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.710461:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.710464:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.710466:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.710468:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.710469:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.710469:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.710471:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.710471:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.710472:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.710473:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.710476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.710484:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.710487:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.710487:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.710489:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.710491:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.710492:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.710503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:13.0:1619677448.710509:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.710511:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.710512:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.710513:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.710513:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.710514:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.710649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.2:1619677448.710655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.710657:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.710659:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.710660:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.710661:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.710661:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.710663:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:10.0:1619677448.710664:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.710664:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.710666:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.710666:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.710667:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.710668:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.710669:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.710669:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.710671:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.710671:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677448.710672:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.710674:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.710676:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.710677:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.710677:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.710680:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:17.0:1619677448.710681:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.710682:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.710683:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.710683:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:17.0:1619677448.710684:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.710684:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.710685:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.710685:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.710686:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677448.710687:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.710687:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.710688:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:25.2:1619677448.710690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:17.0:1619677448.710690:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.710690:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.710691:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.710691:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.710695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.710696:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.710698:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.710698:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.710699:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.710700:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.710702:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:26.0:1619677448.710703:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:13.0:1619677448.710705:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.710705:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.710707:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.710708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:12.0:1619677448.710709:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.710710:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.710714:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.710717:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.710717:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.710718:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.710719:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.710719:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.710848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.710854:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.710857:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.710858:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.710860:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.710863:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.710864:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.710866:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.710867:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.710868:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.710869:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.710873:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.710875:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.710877:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.710878:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.710879:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.710880:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.710881:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.710881:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.710882:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.710887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.710894:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.710896:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.710897:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.710899:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.710900:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.710902:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.710920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.710927:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.710930:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.710930:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.710933:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.710935:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.710937:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.710938:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.710939:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.710941:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.710941:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.710944:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.710946:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.710949:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.710949:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.710950:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.710952:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.710952:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.710953:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.710953:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.710957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.710963:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.710966:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.710967:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.710968:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.710971:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.710972:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.711071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.2:1619677448.711074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677448.711078:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.711080:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.711081:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:1.0:1619677448.711083:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:30.0:1619677448.711084:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.711084:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.711085:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.711086:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.711087:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.711088:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.711089:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.711089:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.711091:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.711091:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677448.711092:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.711092:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.711093:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.711094:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.711095:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.711096:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677448.711099:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.711101:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.711102:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.711104:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.711105:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.711106:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.711107:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.711107:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.711107:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:30.0:1619677448.711108:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.711108:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:30.0:1619677448.711109:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.711110:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.711110:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677448.711110:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.711110:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.711112:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:25.2:1619677448.711113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.711113:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.711113:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.711117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.2:1619677448.711118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:4.2:1619677448.711118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:13.0:1619677448.711120:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.711122:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.711123:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.711124:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677448.711125:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:13.0:1619677448.711125:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.711125:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.711126:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:12.0:1619677448.711126:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.711127:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.711127:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677448.711128:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:8.0:1619677448.711128:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.711129:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677448.711129:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.711129:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.711130:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.711131:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.711132:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677448.711133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677448.711134:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:8.0:1619677448.711134:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.711136:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:25.2:1619677448.711137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000400:00000200:8.0:1619677448.711137:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.711138:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.711139:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.711140:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:8.0:1619677448.711140:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.711143:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.711144:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677448.711144:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000400:00000200:8.0:1619677448.711145:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.711146:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.711147:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677448.711147:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.711148:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.711148:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.711148:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.711149:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000400:00000200:26.0:1619677448.711150:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:6.0:1619677448.711150:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:26.0:1619677448.711151:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.711151:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.711151:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:26.0:1619677448.711152:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.711152:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:26.0:1619677448.711153:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.711154:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.711154:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.711154:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.711155:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.711155:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:26.0:1619677448.711158:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.711159:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.711162:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.711163:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.711164:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.711166:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.711166:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.711167:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.711167:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677448.711170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:9.2:1619677448.711172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.711177:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:18.0:1619677448.711180:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677448.711180:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.711181:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.711181:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.711183:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.711183:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:1.0:1619677448.711183:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.711184:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.711186:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.711186:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677448.711189:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.711320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.711325:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.711328:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.711329:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.711330:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.711332:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.711334:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.711335:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.711337:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.711337:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.711339:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.711342:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.711344:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.711346:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.711347:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.711348:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.711349:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.711350:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.711351:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.711351:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.711356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.711364:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.711367:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.711368:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.711368:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.711370:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.711372:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.711387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.711395:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.711398:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.711399:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.711400:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.711403:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.711405:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.711406:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.711407:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.711408:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.711409:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.711412:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.711414:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.711415:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.711416:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.711417:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.711418:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.711418:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.711419:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.711421:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.711421:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.711424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.2:1619677448.711427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.711432:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.711433:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.711434:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.711435:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:26.0:1619677448.711436:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.711436:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.711437:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.711437:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.711438:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677448.711438:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.711550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.711555:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.711558:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.711558:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.711560:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.711563:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.711564:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.711565:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.711567:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.711568:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.711569:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.711572:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.711574:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.711577:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.711578:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.711578:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.711580:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.711581:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.711581:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.711582:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.711586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.711593:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.711595:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.711596:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.711596:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.711599:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.711602:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:21.2:1619677448.711623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.711629:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.711632:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.711632:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.711634:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.711637:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.711638:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.711639:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.711641:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.711642:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.711643:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.711646:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.711648:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.711649:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.711650:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.711650:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.711652:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.711652:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.711653:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.2:1619677448.711654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:18.0:1619677448.711654:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.711658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.711661:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.711663:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.711664:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:2.0:1619677448.711665:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:8.0:1619677448.711666:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.711667:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.711667:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.711669:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:8.0:1619677448.711669:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.711669:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000400:00000200:8.0:1619677448.711670:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:2.0:1619677448.711671:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:8.0:1619677448.711672:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.711673:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.711674:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.711675:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.711678:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.711680:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677448.711683:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.711684:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.711684:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.711686:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.711686:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.711687:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.711688:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.711691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:21.2:1619677448.711698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:12.0:1619677448.711699:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.711701:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.711702:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.711703:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.711704:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:12.0:1619677448.711705:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:22.0:1619677448.711706:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:13.0:1619677448.711707:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677448.711707:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.711709:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.711712:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.711714:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.711715:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.711717:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.711718:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.711719:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677448.711722:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.711724:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677448.711727:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.711727:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.711728:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.711730:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.711730:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.711731:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.711732:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.711736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.711741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677448.711744:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.711746:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.711747:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.711747:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.711749:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677448.711750:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.711751:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677448.711752:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:31.0:1619677448.711754:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.711754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000400:00000200:31.0:1619677448.711756:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.711756:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.711758:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.2:1619677448.711758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:31.0:1619677448.711759:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.711760:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.711761:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.711761:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677448.711762:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.711764:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677448.711764:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.711765:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.711765:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000400:00000200:31.0:1619677448.711766:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.711766:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.711767:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677448.711767:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677448.711769:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:5.0:1619677448.711769:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.711770:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677448.711772:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.711773:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:5.0:1619677448.711773:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.711774:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:5.0:1619677448.711774:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:31.0:1619677448.711775:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.711776:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:5.0:1619677448.711776:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.711777:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:5.0:1619677448.711777:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.711778:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:5.0:1619677448.711778:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.711779:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:25.2:1619677448.711781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:5.0:1619677448.711782:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.711784:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.711787:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.711787:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.711788:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.711789:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.711790:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.711790:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.711792:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677448.711792:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:22.0:1619677448.711793:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.711793:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.711793:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.711793:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.711796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:6.0:1619677448.711796:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.711798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.711799:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.0:1619677448.711801:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:30.0:1619677448.711802:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.711803:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.711804:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677448.711805:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677448.711805:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:30.0:1619677448.711806:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.711808:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.711809:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.711812:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.711814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677448.711821:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.711823:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.711824:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.711825:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.711826:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.711827:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.711827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.711829:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.711832:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.711834:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.711835:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:20.0:1619677448.711838:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.711838:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.711839:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677448.711840:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.711841:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:20.0:1619677448.711842:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.711843:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.711844:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:18.0:1619677448.711844:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.711844:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.711848:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.711850:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.711850:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.711852:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:18.0:1619677448.711852:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.711853:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.711855:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.711855:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.711856:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000400:00000200:18.0:1619677448.711856:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.711857:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:18.0:1619677448.711857:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.711859:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:18.0:1619677448.711861:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.711862:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.711862:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:18.0:1619677448.711862:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.711863:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.711864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:18.0:1619677448.711865:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.711866:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.711866:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.711868:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.711868:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.711870:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:30.0:1619677448.711871:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:18.0:1619677448.711871:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677448.711871:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:30.0:1619677448.711874:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:8.2:1619677448.711874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.711875:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.711876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000400:00000200:30.0:1619677448.711877:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.711878:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.711880:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:2.0:1619677448.711882:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:25.2:1619677448.711883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677448.711884:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:2.0:1619677448.711884:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.711885:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677448.711886:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:2.0:1619677448.711886:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:5.0:1619677448.711887:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.711889:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.711889:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.711890:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677448.711892:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:31.0:1619677448.711893:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.711894:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.711895:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.711896:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.711897:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.711944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.711950:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.711953:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.711953:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.711956:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.711958:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.711960:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.711961:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.711962:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.711963:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.711964:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.711967:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.711969:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.711972:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.711973:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.711973:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.711975:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.711975:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:4.2:1619677448.711975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677448.711976:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.711976:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.711981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.711982:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.711985:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:4.2:1619677448.711985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677448.711986:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.711988:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.711989:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:10.0:1619677448.711990:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.711991:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:24.0:1619677448.711992:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:22.0:1619677448.711992:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.711992:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:6.0:1619677448.711992:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:10.0:1619677448.711993:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711326] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.711995:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.711995:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.711996:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:10.0:1619677448.711996:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.711996:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677448.711998:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:10.0:1619677448.711998:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677448.711999:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.712001:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.712001:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677448.712003:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:10.0:1619677448.712003:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:24.0:1619677448.712004:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96711325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.712005:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.712006:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.712006:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677448.712007:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.712007:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.712008:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.712009:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:24.0:1619677448.712010:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.712010:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677448.712011:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.712011:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.712012:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.712012:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677448.712014:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677448.712015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677448.712015:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.712015:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.712017:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.712017:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.712018:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677448.712018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677448.712019:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677448.712022:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.2:1619677448.712023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.712025:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.712025:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.712026:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.712026:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.712027:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.712028:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.712028:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.712029:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.712031:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:1.0:1619677448.712031:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.712033:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.712034:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.712034:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.712037:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.712039:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.712064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.712071:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.712074:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.712076:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.712078:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.712079:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.712080:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.712082:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.712083:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.712084:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.712085:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.712088:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.712089:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.712092:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.712093:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.712094:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.712096:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.712096:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.712097:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.712098:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.712102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.712109:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.712111:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.712112:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.712114:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.712115:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.712117:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.712121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.712128:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.712130:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.712131:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.712132:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.712133:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.712134:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.712222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.712228:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.712231:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.712231:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.712233:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.712235:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.712237:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.712238:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.712239:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.712240:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.712242:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.712245:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.712246:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.712249:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.712250:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.712250:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.712252:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.712252:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.712253:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.712253:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.712258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.712265:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.712267:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.712268:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.712270:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.712271:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.712274:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.712337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.712345:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.712348:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.712349:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.712351:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.712353:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.712355:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.712356:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.712357:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.712358:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.712359:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.712363:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.712365:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.712368:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.712368:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.712369:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.712371:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.712371:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.712372:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.712372:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.712377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.712385:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.712388:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.712389:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.712389:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.712391:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.712394:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.712402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.712410:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.712413:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.712414:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.712415:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.712419:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.712420:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.712421:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.712423:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.712423:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.712424:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.712428:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.712429:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677448.712431:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.712432:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:10.0:1619677448.712432:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.712434:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:10.0:1619677448.712434:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.712435:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.712435:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.712439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.712447:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.712449:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.712450:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.712452:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.712454:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.712455:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:20.2:1619677448.712504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.712512:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.712514:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.712516:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.712518:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.712519:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.712520:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.712521:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.712522:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.712523:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.712524:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.712527:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.712528:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.712531:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.712531:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.712532:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.712533:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.712534:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.712535:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.712537:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.712537:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.712539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.712543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.712546:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.712548:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.712549:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.712551:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:12.0:1619677448.712551:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.712553:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:12.0:1619677448.712553:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.712554:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:12.0:1619677448.712554:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.712555:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.712558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:13.0:1619677448.712564:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.712567:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.712568:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.712568:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.712569:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.712570:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.712573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.712581:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.712584:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.712585:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.712586:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.712589:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.712591:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.712592:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.712593:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.712594:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.712595:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.712598:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.712600:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:4.2:1619677448.712600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677448.712603:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.712604:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.712604:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.712606:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.712606:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:26.0:1619677448.712607:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677448.712607:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.712607:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:26.0:1619677448.712609:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.712610:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.712612:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.712612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677448.712614:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.712615:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.712616:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.712617:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.712618:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.712619:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.712620:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:26.0:1619677448.712622:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.712623:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.712624:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.712624:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.712624:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.712626:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.712627:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.712627:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.712628:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.712630:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.712630:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677448.712630:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:26.0:1619677448.712631:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.712631:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.712631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:25.2:1619677448.712636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.712638:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.712641:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.712642:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.712642:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677448.712643:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677448.712643:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.712643:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677448.712646:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.712647:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.712648:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.712650:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.712652:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.712653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.712660:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.712663:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.712663:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.712664:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.712665:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.712665:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.712678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:5.2:1619677448.712684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.0:1619677448.712685:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.712688:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.712689:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.712690:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.712690:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:24.0:1619677448.712693:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.712693:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.712694:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677448.712695:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.712697:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.712697:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.712698:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.712699:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.712700:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.712700:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.712704:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.712705:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.712706:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.712707:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.712709:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.712709:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.712710:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.712710:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:1.0:1619677448.712710:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.712711:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.712712:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.712713:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.712713:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.712714:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.712714:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.712716:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.2:1619677448.712718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.712719:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.712719:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.712720:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.712722:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.712723:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.712724:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.712725:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.712726:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.712727:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.712727:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.712728:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.712728:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:8.2:1619677448.712729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:17.0:1619677448.712730:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.712731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.712733:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:17.0:1619677448.712735:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.712736:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.712736:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:17.0:1619677448.712737:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.712737:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:17.0:1619677448.712738:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.712738:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.712739:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.712740:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:1.0:1619677448.712740:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:21.2:1619677448.712756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.712761:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.712764:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.712765:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.712767:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.712769:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.712770:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.712771:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.712773:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.712774:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.712775:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.712778:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.712779:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.712782:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.712782:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.712783:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.712785:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.712786:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.712786:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.712787:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.712791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.712798:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.712800:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.712801:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.712802:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.712804:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.712806:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.712873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.712878:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.712881:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.712883:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.712884:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.712886:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.712888:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:5.0:1619677448.712889:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.712890:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.712891:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.712892:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:5.0:1619677448.712895:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.712897:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.712900:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.712901:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.712901:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.712903:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.712904:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.712904:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.712905:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.712909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.712916:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.712918:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.712919:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.712921:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.712922:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.712924:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.713005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.713012:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.713014:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.713015:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.713017:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.713020:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.713021:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.713022:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.713023:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.713024:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.713025:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.713028:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.713030:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.713032:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.713033:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.713034:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.713035:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.2:1619677448.713036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677448.713036:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.713037:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.713037:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.713042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.713044:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.713046:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.713048:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.713049:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:31.0:1619677448.713050:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.713050:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.713052:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:22.0:1619677448.713052:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.713052:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:30.0:1619677448.713053:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.713053:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.713055:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.713056:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.713056:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.713057:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.713059:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:30.0:1619677448.713060:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.713062:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.713066:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.713067:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.713067:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.713069:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.713070:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.713070:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.713071:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.713075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.713083:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.713085:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.713086:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.713088:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.713088:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.713091:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.713095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:5.0:1619677448.713102:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.713105:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.713106:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.713107:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:5.0:1619677448.713107:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.713108:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.713231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:31.0:1619677448.713239:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677448.713242:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:31.0:1619677448.713244:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.713246:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:31.0:1619677448.713247:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.713248:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.713249:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.713251:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.713252:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.713253:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677448.713256:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.713258:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.713261:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.713262:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.713262:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.713264:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.713265:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677448.713267:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871380600] (22)++
00000800:00000200:31.0:1619677448.713268:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (23)--
00000800:00000200:31.0:1619677448.713269:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:9.2:1619677448.713271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.2:1619677448.713274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (21)++
00000800:00000200:18.0:1619677448.713278:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.713281:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:30.0:1619677448.713282:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (22)++
00000400:00000200:18.0:1619677448.713282:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.713282:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.713284:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871380600] (23)--
00000400:00000200:18.0:1619677448.713284:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.713285:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677448.713286:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:18.0:1619677448.713287:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.713293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.713301:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.713304:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.713305:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:4.2:1619677448.713305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677448.713305:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.713307:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.713307:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:26.0:1619677448.713313:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.713316:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.713317:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.713319:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.713322:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.713324:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.713324:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.713326:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.713327:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.713327:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.713330:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.713332:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.713335:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.713336:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.713336:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.713338:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.713338:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.713339:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.713340:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.713341:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.713344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.713346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.713352:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.713353:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.713355:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677448.713355:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.713356:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.713357:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.713357:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.713357:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.713359:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.713362:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.713391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677448.713397:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.713400:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677448.713400:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677448.713403:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.713405:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.713406:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.713407:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711309] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.713409:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.713410:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.713411:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.713414:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.713416:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.713419:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.713419:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.713420:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.713422:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.713422:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.713423:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677448.713424:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677448.713425:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:25.2:1619677448.713428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.713430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:22.0:1619677448.713435:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:2.0:1619677448.713436:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:22.0:1619677448.713437:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677448.713437:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000400:00000200:22.0:1619677448.713438:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.713438:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677448.713439:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677448.713440:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:22.0:1619677448.713441:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.713443:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.713483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.713489:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.713492:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.713492:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.713494:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.713496:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.713498:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:4.2:1619677448.713498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:18.0:1619677448.713499:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.713500:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.713501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:18.0:1619677448.713501:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.713502:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.713505:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.713505:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677448.713506:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677448.713506:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677448.713508:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.713509:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677448.713509:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:18.0:1619677448.713509:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.713509:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.713509:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:6.0:1619677448.713509:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:18.0:1619677448.713511:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.713511:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.713512:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.713512:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:22.0:1619677448.713513:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.713513:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.713514:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.713516:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.713516:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.713517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:8.0:1619677448.713517:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.713518:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677448.713519:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677448.713520:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.713521:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96711307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.713523:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.713523:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.713524:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.713524:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:8.0:1619677448.713525:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.713526:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677448.713526:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.713526:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.713526:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.713527:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.713527:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.713529:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.713530:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.713532:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96711305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.713532:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.713533:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.713534:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.713535:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.713535:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.713535:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:22.0:1619677448.713536:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.0:1619677448.713536:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.713540:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.713540:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.713540:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:22.0:1619677448.713541:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.713541:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.713542:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:6.0:1619677448.713542:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.713544:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.0:1619677448.713544:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:6.0:1619677448.713544:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:8.0:1619677448.713545:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677448.713545:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)++
00000800:00000200:8.0:1619677448.713547:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677448.713547:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (24)--
00000800:00000200:6.0:1619677448.713547:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.713548:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.713548:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:8.0:1619677448.713548:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.713548:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677448.713549:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.713549:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677448.713550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:8.2:1619677448.713551:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677448.713552:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.713552:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.713553:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.713554:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:4.2:1619677448.713554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677448.713555:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:20.2:1619677448.713555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:25.2:1619677448.713558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:18.0:1619677448.713558:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:17.0:1619677448.713559:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:21.2:1619677448.713560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:18.0:1619677448.713560:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.713561:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.713561:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:2.0:1619677448.713561:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:24.0:1619677448.713562:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:5.0:1619677448.713562:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:17.0:1619677448.713563:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.713563:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.713564:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:18.0:1619677448.713564:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.713565:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:24.0:1619677448.713565:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.713565:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.713565:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:24.0:1619677448.713566:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677448.713566:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:6.0:1619677448.713566:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:30.0:1619677448.713567:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677448.713567:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677448.713568:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.713568:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000400:00000200:5.0:1619677448.713568:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.713569:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:6.0:1619677448.713569:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.713570:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:5.0:1619677448.713570:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.713571:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.713571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:31.0:1619677448.713572:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:5.0:1619677448.713573:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:30.0:1619677448.713574:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.713574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:5.0:1619677448.713574:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.713576:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.713578:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.713578:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:5.0:1619677448.713579:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.713581:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:1.0:1619677448.713581:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.713582:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.713582:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000400:00000200:5.0:1619677448.713583:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.713583:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.713584:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:1.0:1619677448.713584:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:24.0:1619677448.713585:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677448.713585:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.713587:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.713587:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.713588:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.713589:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:8.0:1619677448.713590:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.713590:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:8.0:1619677448.713591:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:5.0:1619677448.713591:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.713592:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:8.0:1619677448.713593:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96711303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.713593:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:8.0:1619677448.713594:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677448.713596:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:8.0:1619677448.713599:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.713601:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.713604:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:8.0:1619677448.713604:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.713606:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:8.0:1619677448.713606:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:17.0:1619677448.713607:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.713607:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.713609:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.713609:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.713610:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.713611:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.713612:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.0:1619677448.713614:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.713615:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.713615:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.713617:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.713617:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677448.713620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.713623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:2.0:1619677448.713628:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677448.713630:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:2.0:1619677448.713630:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.713631:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.713632:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.713632:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:2.0:1619677448.713632:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.713633:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:2.0:1619677448.713634:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.713636:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.714005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.714011:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.714013:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.714014:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.714016:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.714019:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.714021:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.714022:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.714023:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.714024:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.714025:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.714029:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.714030:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.714032:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.714033:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.714034:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.714036:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.714037:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.714037:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.714038:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.714042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.714045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.714053:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.714053:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.714054:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.714055:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.714055:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.714057:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.714058:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.714059:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:20.0:1619677448.714061:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.714061:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.0:1619677448.714062:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.714063:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677448.714064:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.714065:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.714066:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.714067:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677448.714071:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.714073:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677448.714075:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.714076:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.714077:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.714079:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:20.0:1619677448.714080:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.714081:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.714082:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.714083:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.714085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.2:1619677448.714088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:10.0:1619677448.714091:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:5.0:1619677448.714093:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:10.0:1619677448.714094:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.714095:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.714095:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.714095:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.714096:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:10.0:1619677448.714097:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.714097:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:5.2:1619677448.714099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.714101:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:1.0:1619677448.714105:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.714108:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.714108:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.714110:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.714112:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.714113:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.714114:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.714116:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.714117:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.714118:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.714120:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.714122:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.714124:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.714124:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.714125:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.714126:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.714127:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.714128:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.714128:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:21.2:1619677448.714130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.2:1619677448.714131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.2:1619677448.714133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.714138:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.714139:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.714140:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.714141:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677448.714142:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.714142:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.714143:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.714143:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.714144:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.714144:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:30.0:1619677448.714145:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677448.714146:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.714147:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.714147:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.714148:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:31.0:1619677448.714149:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.714149:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.714150:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.714151:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.714151:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:8.2:1619677448.714152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000400:00000200:31.0:1619677448.714153:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.714153:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.714154:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.714155:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.714156:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.714157:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677448.714158:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.714160:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.714161:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.714161:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.714163:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.714163:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.714163:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.714164:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.714164:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.714164:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.714165:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.714165:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.714165:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:31.0:1619677448.714167:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.714167:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.714167:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.714168:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.714168:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.714169:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.714169:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.714171:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.714171:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.714172:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.714172:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.714172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:9.2:1619677448.714177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677448.714180:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.714182:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.714183:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.714184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:13.0:1619677448.714184:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.714185:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:12.0:1619677448.714187:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.714188:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:13.0:1619677448.714188:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:18.0:1619677448.714189:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.714189:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.714189:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.714192:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.714192:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.714194:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.714195:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.714196:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:5.0:1619677448.714198:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.714199:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677448.714200:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.714202:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.714203:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.714204:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677448.714207:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.714209:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.714211:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.714212:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.714213:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.714214:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.714215:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.714216:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.714216:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.714220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.714228:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.714231:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.714231:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.714232:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.714234:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.714236:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.714596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.714604:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.714607:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.714607:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.714609:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.714612:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.714614:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.714615:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.714616:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.714618:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.714618:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.714621:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.714623:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.714625:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.714626:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.714627:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.714628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.714628:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.714629:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.714629:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.714631:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.714633:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.714634:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:25.2:1619677448.714635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:10.0:1619677448.714636:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.714637:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.714639:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.714641:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.714641:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.714642:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:10.0:1619677448.714643:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.714643:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.714644:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:10.0:1619677448.714644:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.714645:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.714647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:30.0:1619677448.714648:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.714649:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:10.0:1619677448.714649:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.714650:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677448.714651:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:25.2:1619677448.714653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:2.0:1619677448.714653:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:10.0:1619677448.714654:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.714654:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.714655:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:2.0:1619677448.714656:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.714657:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:10.0:1619677448.714657:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.714657:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.714658:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:31.0:1619677448.714659:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.714659:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:2.0:1619677448.714659:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677448.714660:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:2.0:1619677448.714661:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.714662:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677448.714662:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:31.0:1619677448.714663:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:8.2:1619677448.714663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.714664:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.714664:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:2.0:1619677448.714664:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.714665:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.714666:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.714666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:2.0:1619677448.714666:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.714667:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.714670:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.714672:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.714673:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677448.714674:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677448.714674:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.714675:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:2.0:1619677448.714675:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.714676:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:20.0:1619677448.714676:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:2.0:1619677448.714676:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.714677:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:20.0:1619677448.714677:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.714677:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.714678:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:20.0:1619677448.714679:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.714679:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.714679:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.0:1619677448.714681:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.714682:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:25.2:1619677448.714683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.714691:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.714694:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.714695:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.714695:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.714697:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677448.714699:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.714702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.714708:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.714711:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.714712:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.714712:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.714713:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.714714:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.714729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:6.0:1619677448.714733:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677448.714736:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677448.714737:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677448.714739:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.714741:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.714743:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:6.0:1619677448.714744:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.714745:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.714746:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.714747:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:6.0:1619677448.714750:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.714752:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677448.714755:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.714756:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:6.0:1619677448.714757:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:6.0:1619677448.714759:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:6.0:1619677448.714759:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677448.714759:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677448.714760:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:25.2:1619677448.714765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.714771:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:21.2:1619677448.714772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:30.0:1619677448.714774:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.714775:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.714777:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.714777:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677448.714778:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.714780:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:22.0:1619677448.714780:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.714781:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.714783:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.714786:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.714787:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.714788:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.714790:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.714791:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.714791:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.714794:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.714796:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.714798:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.714799:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.714800:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.714802:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.714802:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.714803:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.714803:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.2:1619677448.714805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:25.2:1619677448.714807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:5.0:1619677448.714810:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.714813:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:5.0:1619677448.714813:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.714815:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.714815:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.714816:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.714817:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.714817:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.714817:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.714819:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677448.714820:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.714820:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.714821:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.714823:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000400:00000200:5.0:1619677448.714823:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.714824:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677448.714827:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.714829:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.714831:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.714832:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.714833:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.714834:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.714835:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.714836:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.714838:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.714838:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677448.714840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677448.714843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.714846:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.714848:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:1.0:1619677448.714848:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:20.0:1619677448.714849:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.714850:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677448.714851:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.714852:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.714852:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.714854:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.714856:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.714879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.714884:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.714887:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.714888:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.714890:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.714891:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.714892:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.714894:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.714895:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.714896:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.714897:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.2:1619677448.714898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677448.714900:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000400:00000200:17.0:1619677448.714900:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.714902:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.714903:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677448.714905:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.714905:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.714906:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.714906:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677448.714907:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.714907:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.714909:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:17.0:1619677448.714909:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.714909:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.714910:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677448.714911:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.714911:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:20.0:1619677448.714912:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.714913:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.714914:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:25.2:1619677448.714915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.714917:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.714919:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:20.0:1619677448.714922:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.714922:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.714923:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:20.0:1619677448.714923:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:20.0:1619677448.714925:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:12.0:1619677448.714925:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677448.714926:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000400:00000200:12.0:1619677448.714926:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.714927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677448.714927:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677448.714927:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000400:00000200:12.0:1619677448.714928:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.714929:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.714930:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.714931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:31.0:1619677448.714934:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677448.714935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.714937:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677448.714937:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000400:00000200:31.0:1619677448.714939:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.714939:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.714940:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:31.0:1619677448.714941:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.714941:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677448.714941:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.714941:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:26.0:1619677448.714943:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.714943:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.714944:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:13.0:1619677448.714944:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:31.0:1619677448.714945:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.714945:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:31.0:1619677448.714946:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.714946:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:13.0:1619677448.714946:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:31.0:1619677448.714947:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.714947:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677448.714948:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.714951:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.714953:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.714956:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.714956:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:25.2:1619677448.714956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000800:00000200:31.0:1619677448.714957:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.714959:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.714959:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.714961:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.714962:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:10.0:1619677448.714962:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:31.0:1619677448.714963:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.714964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.714964:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.714965:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.714965:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.714966:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.714967:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677448.714969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.714970:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.714972:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.714973:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.714975:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.714975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677448.714976:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.714976:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.714977:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.714978:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.714979:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.714980:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677448.714983:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.714986:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.714987:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.714989:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.714992:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.714993:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.714994:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.714996:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.714997:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.714997:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.715001:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.715002:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.715005:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.715006:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.715006:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.715008:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.715009:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.715009:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.715010:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.715014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.715021:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.715024:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.715025:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.715025:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.715027:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.715030:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.715035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.715042:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.715044:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.715045:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.715045:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.715046:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.715047:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.715106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.715114:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.715117:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.715119:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.715121:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.715121:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.715123:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.715124:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.715126:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.715127:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.715128:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.715131:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.715132:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.715136:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.715136:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.715137:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.715139:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.715139:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.715140:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.715141:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.715145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.715153:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.715156:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.715156:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.715158:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.715159:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.715161:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.715166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.715172:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.715175:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.715175:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.715178:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.715180:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.715181:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.715183:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.715184:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.715185:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.715186:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.715189:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.715191:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.715193:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.715194:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.715195:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.715196:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.715196:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.715197:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.715197:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.715202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.715209:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.715211:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.715211:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.715212:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.715213:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.715214:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.715219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.715226:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.715229:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.715229:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.715229:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.715230:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.715230:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.715249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.715255:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.715257:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.715258:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.715260:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.715263:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.715264:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.715265:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711284] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.715266:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.715267:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.715268:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.715271:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.715273:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.715275:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677448.715276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677448.715276:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.715276:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.715278:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.715278:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.715279:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.715280:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.715284:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.2:1619677448.715284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:26.0:1619677448.715287:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.715288:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.715289:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.715291:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:26.0:1619677448.715292:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.715293:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.715298:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000400:00000200:26.0:1619677448.715300:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.715301:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.715301:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.715303:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.715303:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.715304:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.715304:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:26.0:1619677448.715305:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.715309:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.715311:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.715314:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.715315:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.715315:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.715317:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.715317:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.715318:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.715319:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.715323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.715331:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.715333:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.715334:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.715335:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.715337:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.715339:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.715364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.715367:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.715369:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.715372:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.715372:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.715375:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.715376:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.715377:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.715379:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.715380:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.715381:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.715384:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.715386:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.715388:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.715389:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.715390:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.715392:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.715392:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.715393:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.715394:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677448.715398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.715406:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.715409:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.715410:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.715410:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.715412:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.715414:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.715515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.715521:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.715523:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.715525:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.715526:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.715528:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.715529:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:5.0:1619677448.715531:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.715532:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.715533:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.715534:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:5.0:1619677448.715537:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.715539:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.715541:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.715542:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.715543:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.715544:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.715545:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.715546:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.715546:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.715551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.715558:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.715560:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.715561:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.715563:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.715564:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.715565:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.715574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.715580:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.715582:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.715583:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.715585:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.715587:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.715588:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.715589:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677448.715591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677448.715591:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.715592:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.715593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:1.0:1619677448.715593:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.715596:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.715598:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.715598:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.715601:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:1.0:1619677448.715601:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.715602:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.715602:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677448.715602:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.715603:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.715604:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.715605:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677448.715605:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.715605:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.715606:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:31.0:1619677448.715607:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.715607:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000400:00000200:10.0:1619677448.715608:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.715608:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.715609:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677448.715609:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677448.715610:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.715610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:10.0:1619677448.715610:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.715612:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.715612:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.715612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.715613:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.715613:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.715614:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.715614:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677448.715615:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.715616:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:12.0:1619677448.715617:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677448.715618:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677448.715619:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:17.0:1619677448.715620:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:31.0:1619677448.715621:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.715621:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677448.715621:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.715622:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:10.0:1619677448.715622:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.715623:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:12.0:1619677448.715623:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.715624:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.715625:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.715626:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.715626:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.715627:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.715627:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.715627:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.715628:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:31.0:1619677448.715629:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.715629:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.715630:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677448.715630:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.715630:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.715631:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.715631:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677448.715631:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.715632:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.715635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:25.2:1619677448.715638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.715641:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.715643:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.715644:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.715644:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.715645:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:26.0:1619677448.715646:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.715647:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.715648:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.715648:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:30.0:1619677448.715650:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.715652:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.715653:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.715942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.715949:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.715951:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.715952:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.715954:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.715956:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.715957:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.715959:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.715960:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.715961:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.715962:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.715965:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.715967:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.715971:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.715972:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.715972:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.715974:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.715974:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.715975:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.715976:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.2:1619677448.715976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:25.2:1619677448.715980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.715981:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.715984:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.715985:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.715986:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:1.0:1619677448.715987:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.715989:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.715990:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.715990:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.715991:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:13.0:1619677448.715992:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.715992:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:1.0:1619677448.715992:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711276] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.715994:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.715995:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:1.0:1619677448.715995:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.715996:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.715999:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.716001:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.716004:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.716004:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.716005:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.716007:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.716007:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.716008:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.716009:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.716013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.716019:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.716022:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.716022:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.716024:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.716025:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.716027:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.716071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.716076:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.716079:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.716080:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.716082:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.716085:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.716086:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.716088:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.716089:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.716090:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.716091:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.716094:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.716095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677448.716096:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.716099:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.716100:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.716101:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.716102:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677448.716103:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.716103:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.716104:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:26.0:1619677448.716105:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.716105:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:2.0:1619677448.716105:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:10.0:1619677448.716108:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.716109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.2:1619677448.716109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:10.0:1619677448.716110:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.716112:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.716114:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.716115:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.716115:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.716116:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:10.0:1619677448.716116:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.716117:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:26.0:1619677448.716118:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.716118:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.716119:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.716120:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.716120:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.716122:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.716123:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.716123:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.716125:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.716125:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:26.0:1619677448.716126:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:5.0:1619677448.716126:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.716127:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.716128:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.716128:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.716129:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.716129:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:5.0:1619677448.716129:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.716130:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.716130:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677448.716130:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.716132:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.716132:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.716133:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.716133:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.716133:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:5.0:1619677448.716133:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.716135:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:5.0:1619677448.716135:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.716136:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.716137:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.716138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.716138:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.716139:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.716140:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:5.2:1619677448.716142:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677448.716144:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:5.0:1619677448.716147:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (25)--
00000800:00000200:13.0:1619677448.716148:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:5.0:1619677448.716148:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.716148:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677448.716149:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:13.0:1619677448.716149:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.716149:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677448.716151:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:13.0:1619677448.716151:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.716151:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:26.0:1619677448.716152:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.716153:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.716153:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:25.2:1619677448.716154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.716160:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.716163:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.716163:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.716165:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.716166:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.716167:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.716168:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.716168:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.716169:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.716169:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.716170:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.716247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677448.716253:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677448.716256:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677448.716257:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677448.716259:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.716262:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.716263:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677448.716265:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.716266:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.716267:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.716268:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677448.716271:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.716273:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677448.716275:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.716276:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:17.0:1619677448.716277:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:17.0:1619677448.716278:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677448.716279:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677448.716280:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677448.716280:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677448.716285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.716292:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.716298:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.716299:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.716300:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:20.0:1619677448.716301:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.716303:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.716334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.716342:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.716345:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.716346:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.716348:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.716350:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.716350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1200] (20)++
00000400:00000200:10.0:1619677448.716352:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.716353:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.716354:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.716355:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.716356:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.716356:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:1.0:1619677448.716359:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:17.0:1619677448.716360:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:10.0:1619677448.716361:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.716362:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.716364:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:1.0:1619677448.716365:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.716366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:1.0:1619677448.716366:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:10.0:1619677448.716367:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.716368:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:1.0:1619677448.716368:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.716369:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677448.716369:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.716370:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.716371:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.716371:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677448.716371:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:10.0:1619677448.716372:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.716373:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677448.716374:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:1.0:1619677448.716374:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.716376:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:31.0:1619677448.716377:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:9.2:1619677448.716377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.716379:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.716380:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.716380:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:1.0:1619677448.716380:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:30.0:1619677448.716381:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:1.0:1619677448.716382:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:1.0:1619677448.716382:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000400:00000200:31.0:1619677448.716383:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.716384:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:1.0:1619677448.716384:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdbb1200] (22)++
00000800:00000200:1.0:1619677448.716385:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (23)--
00000400:00000200:31.0:1619677448.716386:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96711269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.716386:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677448.716386:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:31.0:1619677448.716387:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.716388:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.716389:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.716389:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:8.2:1619677448.716389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000400:00000200:18.0:1619677448.716390:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.716390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:2.0:1619677448.716390:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.716393:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.716394:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.716396:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.716396:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677448.716396:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)++
00000800:00000200:17.0:1619677448.716398:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdbb1200] (23)--
00000800:00000200:5.0:1619677448.716398:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:17.0:1619677448.716399:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:31.0:1619677448.716400:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.716400:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:17.0:1619677448.716400:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:5.0:1619677448.716400:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:31.0:1619677448.716401:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:5.0:1619677448.716401:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.716403:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.716403:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.716403:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.716403:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.716404:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.716405:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:5.0:1619677448.716406:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:8.2:1619677448.716409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.716416:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.716419:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.716420:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.716421:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.716422:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.716424:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.716454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.716462:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.716464:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.716467:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.716468:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.716469:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.716471:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.716472:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.716474:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.716475:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.716476:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.716479:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.716480:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.716482:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.716483:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.716484:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.716485:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.716486:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.716487:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.716487:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.716491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.716499:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.716501:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.716502:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.716503:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.716505:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.716506:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.716579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.716585:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.716588:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.716589:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.716590:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.716593:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.716594:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.716596:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.716597:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.716598:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.716599:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.716602:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.716604:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.716606:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.716606:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.716607:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.716608:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.716609:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.716609:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.716610:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.716615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.716622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677448.716624:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.716626:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.716627:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000400:00000200:1.0:1619677448.716627:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.716629:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:1.0:1619677448.716629:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.716632:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.716633:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.716634:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.716634:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677448.716635:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677448.716637:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:17.0:1619677448.716638:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.716639:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.716640:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.716641:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:17.0:1619677448.716645:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.716647:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:17.0:1619677448.716650:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f90d380] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.716650:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e49d37c00] (20)++
00000800:00000200:17.0:1619677448.716651:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e49d37c00] (21)++
00000800:00000200:17.0:1619677448.716653:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e49d37c00] (22)--
00000800:00000200:17.0:1619677448.716653:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677448.716654:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677448.716654:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:24.2:1619677448.716659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e49d37c00] (21)++
00000800:00000200:24.0:1619677448.716662:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)++
00000800:00000200:24.0:1619677448.716665:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e49d37c00] (23)--
00000400:00000200:24.0:1619677448.716666:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.716666:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (22)--
00000400:00000200:24.0:1619677448.716669:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.716671:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e49d37c00] (21)--
00000800:00000200:5.2:1619677448.716681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.716686:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.716689:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.716689:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.716692:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.716694:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.716696:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677448.716697:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.716699:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.716700:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.716701:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.716704:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.716706:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.716708:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.716709:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.716709:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:1.0:1619677448.716711:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.716711:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.716711:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.716712:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.716717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.716725:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.716727:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.716728:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.716728:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.716731:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.716733:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.716847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.716855:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.716857:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.716858:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.716861:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.716863:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.716865:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.716866:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.716868:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.716869:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.716870:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.716873:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.716875:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.716878:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.716879:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.716879:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.716881:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.716881:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.716882:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.716884:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.716884:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.716887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:4.2:1619677448.716890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.716894:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.716897:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:10.0:1619677448.716897:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.716897:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.716898:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.716899:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.716899:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.716900:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.716900:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.716904:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.716950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.716957:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.716960:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.716961:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.716963:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.716966:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.716967:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.716968:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.716970:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.716971:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.716972:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.716975:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.716976:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.2:1619677448.716977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:26.0:1619677448.716978:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.716979:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.716979:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.716981:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.716981:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.716982:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.716982:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.0:1619677448.716983:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.716985:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:25.2:1619677448.716987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:5.0:1619677448.716987:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.716988:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.716990:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.716991:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:12.0:1619677448.716993:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:5.0:1619677448.716993:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.716994:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.716995:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:5.0:1619677448.716995:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677448.716996:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.716996:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:12.0:1619677448.716999:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.717000:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:5.0:1619677448.717000:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.717001:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677448.717003:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.717006:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.717007:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.717007:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.717008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:5.0:1619677448.717009:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.717010:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.717010:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.717011:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.717015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:8.0:1619677448.717015:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.717018:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.717019:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.717021:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.717022:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:8.0:1619677448.717023:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.717024:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:31.0:1619677448.717025:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.717026:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.717026:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96711261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.717028:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.717028:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.717029:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:8.0:1619677448.717029:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.717030:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.717031:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:8.0:1619677448.717033:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.717035:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.717038:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.717039:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.717039:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.717041:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.717041:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.717042:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.717043:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.717047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.717054:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.717057:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.717058:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.717058:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.717061:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.717062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677448.717064:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:30.0:1619677448.717069:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.717072:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.717074:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.717076:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.717077:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.717078:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.717080:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.717081:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.717082:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.717083:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.717086:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.717088:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.717091:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.717092:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.717092:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:30.0:1619677448.717094:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.717095:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.717095:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.717096:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.717100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.717108:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.717111:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.717112:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.717112:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.717114:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.717118:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.717164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.717171:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.717173:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.717174:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.717176:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.717178:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.717179:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.717181:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.717182:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.717183:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.717184:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.717187:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.717189:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.717192:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.717192:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.717193:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.717195:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.717195:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.717196:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.717196:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.717201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.717209:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.717212:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.717212:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.717213:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.717215:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.717217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.717220:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.717222:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.717223:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.717225:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.717225:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.717225:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.717226:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.717263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.717270:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.717273:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.717273:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.717276:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.717278:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.717280:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677448.717281:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.717282:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.717282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:24.0:1619677448.717283:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.717284:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677448.717287:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.717288:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.717288:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.717290:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:24.0:1619677448.717291:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.717291:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677448.717292:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.717292:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.717294:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:24.0:1619677448.717299:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.717299:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:1.0:1619677448.717299:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.717300:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.717301:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:1.0:1619677448.717301:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.717302:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:31.0:1619677448.717303:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:1.0:1619677448.717304:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.717305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:1.0:1619677448.717305:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.717306:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:1.0:1619677448.717306:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.717308:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677448.717310:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.717311:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:1.0:1619677448.717312:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.717313:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.717313:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677448.717314:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:31.0:1619677448.717315:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:13.0:1619677448.717316:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.717316:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.717316:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.717317:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96711256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.717317:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:13.0:1619677448.717318:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.717318:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:31.0:1619677448.717319:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.717320:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.717320:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.717320:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.717320:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.717321:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:12.0:1619677448.717321:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.717321:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.717321:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677448.717324:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.717325:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:25.2:1619677448.717326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:12.0:1619677448.717326:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.717326:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:31.0:1619677448.717327:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:12.0:1619677448.717327:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.717328:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.717330:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.717331:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.717332:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:31.0:1619677448.717334:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.717334:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:22.0:1619677448.717334:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:31.0:1619677448.717336:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.717337:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:22.0:1619677448.717337:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:31.0:1619677448.717338:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:22.0:1619677448.717338:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.717338:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.717340:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.717340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:22.0:1619677448.717342:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:5.2:1619677448.717343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:20.2:1619677448.717345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677448.717349:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:30.0:1619677448.717352:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.717352:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:2.0:1619677448.717352:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.717353:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677448.717354:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:30.0:1619677448.717355:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:20.0:1619677448.717355:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.717355:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.717356:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.717357:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.717357:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.717357:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.717358:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:2.0:1619677448.717359:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:5.0:1619677448.717360:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.717360:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.717361:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.717361:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (23)++
00000800:00000200:20.0:1619677448.717362:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.717362:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000400:00000200:2.0:1619677448.717362:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.717363:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:2.0:1619677448.717363:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.717364:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.717367:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.717369:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.717371:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.717372:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.717372:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.717374:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.717375:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.717376:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.717377:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.717378:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.717380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.717383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.717388:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:17.0:1619677448.717388:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677448.717390:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:17.0:1619677448.717390:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677448.717391:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677448.717391:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.717392:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677448.717392:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.717393:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.717396:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.717517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.717525:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.717528:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.717529:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.717531:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.717534:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.717535:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.717537:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711254] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.717538:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.717539:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.717540:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.717544:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.717546:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.717548:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.717549:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.717549:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.717551:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.717551:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.717552:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.717553:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.717558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.717565:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.717568:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.717568:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.717570:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.717572:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.717573:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.717615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.717621:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.717623:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.717624:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.717626:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.717628:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.717630:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.717631:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.717632:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.717633:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.717634:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.717637:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.717639:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.717641:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.717642:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.717642:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:21.2:1619677448.717643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677448.717644:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.717644:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.717646:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.717647:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.717647:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.717650:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.2:1619677448.717650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677448.717653:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.2:1619677448.717653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.717655:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.717657:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.717658:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.717658:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.717659:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.717659:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:31.0:1619677448.717660:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.717660:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.717660:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:31.0:1619677448.717661:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.717661:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.717662:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.717662:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:31.0:1619677448.717663:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.717663:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677448.717664:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.717664:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:31.0:1619677448.717665:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.717668:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.717670:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.717673:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.717673:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.717674:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.717676:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.717676:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.717678:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.717679:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.717680:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.717681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.717686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.717688:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.717690:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.717691:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.717693:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:13.0:1619677448.717693:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.717694:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.717695:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:13.0:1619677448.717695:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.717696:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.717697:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677448.717802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.717809:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:21.2:1619677448.717811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:30.0:1619677448.717812:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.717814:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.717816:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.717816:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.717817:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:30.0:1619677448.717818:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.717819:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.717819:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.717820:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:30.0:1619677448.717821:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.717822:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.717823:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.717823:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.717825:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.717827:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.717827:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.717829:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.717830:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.717831:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.717832:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.717833:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.717833:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.717834:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.717834:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.717836:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.717836:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:2.0:1619677448.717836:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.717837:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.717838:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:2.0:1619677448.717838:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.717840:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677448.717841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677448.717841:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.717842:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.717843:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.717844:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.717844:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.717845:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677448.717848:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:9.2:1619677448.717849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677448.717850:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.717851:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.717853:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.717854:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.717855:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:18.0:1619677448.717857:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.717859:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.717860:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.717860:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.717862:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.717863:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.717863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.717870:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.717872:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.717873:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.717875:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.717878:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.717879:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.717881:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.717882:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.717883:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.717884:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.717888:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.717889:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.717892:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.717893:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.717894:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.717896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:10.0:1619677448.717896:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.717896:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.717897:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.717897:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.717901:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:8.2:1619677448.717902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:6.0:1619677448.717904:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.717905:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.717906:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.717909:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:6.0:1619677448.717909:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.717911:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.0:1619677448.717912:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.717912:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.717912:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.717913:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.717914:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.717915:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.717915:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:6.0:1619677448.717916:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677448.717918:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:6.0:1619677448.717920:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.717922:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.717924:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.717925:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.717925:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.717927:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.717928:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.717929:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.717929:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.717934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.717941:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.717944:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.717945:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.717945:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.717947:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.717949:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.717950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.2:1619677448.717955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.717959:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.717961:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.717961:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.717962:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.717962:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.717963:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.717964:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.717965:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:1.0:1619677448.717965:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.717966:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.717967:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.717968:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.717970:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711247] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.717971:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.717972:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.717973:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.717976:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.717978:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677448.717981:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.717982:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.717983:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.717985:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.717985:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.717985:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.717986:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.717990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.717997:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.718000:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.718000:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:8.0:1619677448.718001:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.718003:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.718006:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:4.2:1619677448.718119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.718125:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.718128:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.718128:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.718130:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.718132:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.718134:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.718135:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.718136:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.718138:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.718138:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.718142:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.718144:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.718147:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.718147:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.718148:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.718150:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.718150:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.718151:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.718151:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.718156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.718164:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.718166:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.718167:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.718168:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.718170:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.718173:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.718345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.718352:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.718355:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.718356:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.718357:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.718360:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.718362:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:20.2:1619677448.718362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:26.0:1619677448.718363:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711245] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.718364:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.718365:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.718366:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.718368:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:26.0:1619677448.718369:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.718370:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.718371:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.718373:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.718373:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:26.0:1619677448.718374:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.718374:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.718375:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:5.0:1619677448.718375:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.718377:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.718377:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677448.718377:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:26.0:1619677448.718379:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:5.0:1619677448.718379:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.718380:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.718380:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:5.0:1619677448.718380:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.718381:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.718382:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.2:1619677448.718383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.2:1619677448.718384:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:4.2:1619677448.718386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:5.0:1619677448.718389:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.718390:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.718391:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.718393:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.718394:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:17.0:1619677448.718394:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.718394:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:5.0:1619677448.718394:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.718394:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:20.0:1619677448.718395:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.718395:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.718396:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:5.0:1619677448.718396:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:20.0:1619677448.718397:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.718397:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.718397:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:1.0:1619677448.718397:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.718398:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.718399:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:1.0:1619677448.718399:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.718400:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:5.0:1619677448.718400:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000400:00000200:1.0:1619677448.718400:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:20.0:1619677448.718401:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.0:1619677448.718401:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:1.0:1619677448.718402:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.718403:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.718404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:1.0:1619677448.718404:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.718405:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677448.718408:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.718410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677448.718410:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677448.718412:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:1.0:1619677448.718413:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.718413:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.718414:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:31.0:1619677448.718415:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:5.0:1619677448.718415:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677448.718416:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.718416:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.718416:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.718417:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677448.718417:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.718417:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677448.718418:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: ACK: OK
00000800:00000200:30.0:1619677448.718419:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.718420:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.718420:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.718421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:31.0:1619677448.718422:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:5.0:1619677448.718424:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.718425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:5.0:1619677448.718425:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.718427:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.718428:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000400:00000200:5.0:1619677448.718428:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.718429:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.718430:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:5.0:1619677448.718430:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:24.0:1619677448.718431:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.718431:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.718432:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:8.0:1619677448.718434:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.718434:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.718435:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.718435:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.718436:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.718437:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:8.0:1619677448.718437:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.0:1619677448.718437:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.718438:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.0:1619677448.718439:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.718440:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:9.2:1619677448.718441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000800:00000200:5.0:1619677448.718441:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.718442:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.718443:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.718444:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.718446:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.718446:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:24.0:1619677448.718447:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:25.2:1619677448.718448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677448.718449:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677448.718450:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.718450:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:20.2:1619677448.718451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:24.0:1619677448.718452:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:24.0:1619677448.718452:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.0:1619677448.718454:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:22.0:1619677448.718456:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677448.718456:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.718457:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.718457:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677448.718459:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.718460:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.718460:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.718462:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.718464:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.718464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.718471:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.718474:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.718474:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677448.718476:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.718479:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.718480:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677448.718482:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.718483:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.718484:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.718485:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677448.718488:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.718490:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.718493:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.718494:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:8.0:1619677448.718494:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.0:1619677448.718496:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.718496:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.718497:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.718497:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.718501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:21.2:1619677448.718504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:20.0:1619677448.718509:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:6.0:1619677448.718509:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677448.718512:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:6.0:1619677448.718512:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.718513:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677448.718513:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.718515:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.718515:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.718516:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:6.0:1619677448.718517:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.718518:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:6.0:1619677448.718518:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.718520:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.718521:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.718522:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.718523:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.718526:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.718528:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.718531:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.718532:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.718532:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.718534:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.718534:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.718535:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.718536:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.718537:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:9.2:1619677448.718540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.718542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.718547:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677448.718547:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677448.718549:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.718550:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677448.718550:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:22.0:1619677448.718551:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:2.0:1619677448.718551:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.718552:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.718554:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.718556:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.718677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.718684:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.718687:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.718688:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.718690:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.718693:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.718694:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.718695:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.718697:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.718698:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.718699:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.718702:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.718704:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.718707:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.718707:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.718708:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.718710:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.718710:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.718712:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.718713:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.718713:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.718716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:4.2:1619677448.718719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.718724:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:8.0:1619677448.718726:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.718727:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.718727:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.718728:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.718728:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.718729:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.718730:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.718730:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677448.718733:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.718815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.718822:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.718825:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.718827:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.718829:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.718829:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.718831:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.718832:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.718833:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.718834:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.718835:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.718838:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.718840:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.718843:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.718844:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.718844:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.718846:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.718847:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.718848:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.718848:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.718852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.718860:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.718863:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.718864:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.718866:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.718866:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.718869:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.718870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.718878:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.718881:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.718882:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.718882:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.718883:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.718884:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.718948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.718954:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.718957:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.718957:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.718959:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.718962:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.718964:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677448.718965:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.718966:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.718967:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677448.718968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:1.0:1619677448.718968:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677448.718972:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.718973:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:1.0:1619677448.718973:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.718976:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:1.0:1619677448.718976:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.718977:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:1.0:1619677448.718977:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:1.0:1619677448.718978:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:6.0:1619677448.718979:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.718980:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.718980:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.718981:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.718981:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677448.718982:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.718983:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.718984:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.718986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:6.0:1619677448.718986:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.718987:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.718988:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.718991:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.718993:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:6.0:1619677448.718993:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:31.0:1619677448.718996:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:6.0:1619677448.718996:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.718997:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.718997:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.718997:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:31.0:1619677448.718998:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.718999:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.718999:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677448.719000:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.719000:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.719001:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:31.0:1619677448.719002:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.719005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.719011:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.719014:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.719014:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.719015:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.719017:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.719020:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677448.719039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.719046:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.719048:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677448.719049:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677448.719051:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.719053:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.719055:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.719056:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.719057:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.719058:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.719059:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.719062:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.719064:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.719067:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.719068:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.719069:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.719070:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.719071:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.719071:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.719072:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.719076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.719083:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.719086:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.719087:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.719088:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.719089:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.719091:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.719094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:5.0:1619677448.719101:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.719104:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:5.0:1619677448.719105:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.719105:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.719106:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.719107:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:21.2:1619677448.719169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.719176:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.719179:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.719181:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.719183:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.719184:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.719185:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.719186:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.719187:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.719188:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.719189:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:4.2:1619677448.719191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:30.0:1619677448.719192:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.719194:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.719196:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.719197:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.719197:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.719198:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.719199:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.719199:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.719201:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.719201:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677448.719201:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.719202:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.719203:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.719204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:26.0:1619677448.719206:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.719207:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.719208:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.719209:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.719210:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.719211:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:13.0:1619677448.719212:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:26.0:1619677448.719214:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.719214:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.719215:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.719216:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:13.0:1619677448.719217:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.719219:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.719219:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.719220:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.719220:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:12.0:1619677448.719220:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.719222:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.719222:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.719223:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.719224:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.719226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.2:1619677448.719228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:13.0:1619677448.719232:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.719234:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.719235:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.719235:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.719236:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.719236:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:13.0:1619677448.719237:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.719238:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.719239:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.719241:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.719241:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.719244:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:4.2:1619677448.719271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.719278:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.719281:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.719281:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.719283:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.719286:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.719287:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.719288:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.719290:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.719291:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.719292:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.719299:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.719300:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.719303:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.719304:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.719304:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.719306:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.719306:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.719307:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.719307:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.719312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.719320:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.719323:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.719324:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.719324:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.719326:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.719329:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.719333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.719340:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.719343:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.719344:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.719344:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.719345:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.719345:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.719417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.719420:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.2:1619677448.719422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:20.0:1619677448.719422:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.719425:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.719426:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.719427:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.719428:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:20.0:1619677448.719429:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.719430:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.719431:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:20.0:1619677448.719432:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.719432:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:20.0:1619677448.719433:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.719435:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.719435:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677448.719437:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.719438:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.719439:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.719440:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.719441:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:22.0:1619677448.719442:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.719443:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.719444:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:20.0:1619677448.719444:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.719445:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.719446:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677448.719447:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.719448:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.719448:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677448.719449:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677448.719449:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.719450:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677448.719452:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.719453:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.719453:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:25.2:1619677448.719454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.719455:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.719455:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.719456:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.719457:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.719461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:13.0:1619677448.719462:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.719465:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.719466:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.719468:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.719469:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:12.0:1619677448.719469:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.719470:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677448.719472:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.719473:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.719473:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.719475:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.719477:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.719479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.719487:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.719490:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.719491:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.719491:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.719492:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.719492:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.2:1619677448.719898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.719904:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.719907:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.719908:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.719910:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.719912:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.719914:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.719915:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.719916:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.719917:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.719918:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.719922:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.719923:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.719925:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.719926:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.719927:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.719928:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.719929:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.719930:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.719931:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.719931:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.719934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.719937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.719943:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.719944:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.719944:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.719944:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.719945:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.719945:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.719946:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.719946:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.719947:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.719948:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.720086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.720094:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:4.2:1619677448.720095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.720097:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.720100:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.720102:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.720102:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.720102:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.720104:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.720105:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.720105:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.720106:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677448.720107:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.720108:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.720108:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.720109:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.720111:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.720113:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.720113:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.720114:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.720115:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:10.0:1619677448.720116:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.720117:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.720117:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.720117:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.720118:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.720119:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.720120:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.720121:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:10.0:1619677448.720121:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.720122:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.720122:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:10.0:1619677448.720122:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.720125:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677448.720126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.720126:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:21.2:1619677448.720127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:10.0:1619677448.720127:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.720128:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.720129:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.720129:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.720130:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:4.2:1619677448.720130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677448.720132:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:18.0:1619677448.720133:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:25.2:1619677448.720134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.720135:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.720135:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.720136:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.720136:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:8.0:1619677448.720137:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:30.0:1619677448.720138:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.720139:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:18.0:1619677448.720139:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.720139:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:30.0:1619677448.720140:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:24.0:1619677448.720140:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:18.0:1619677448.720141:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.720141:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:8.0:1619677448.720142:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.720143:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.720144:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.720144:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.720145:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:8.0:1619677448.720145:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.720146:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.720146:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.720147:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.720147:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.720148:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96711225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.720149:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:6.0:1619677448.720149:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:8.0:1619677448.720150:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.720151:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.720152:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:18.0:1619677448.720153:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.720153:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.720156:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:8.0:1619677448.720157:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.720159:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.720159:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.720160:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.720160:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.720162:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677448.720162:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.720163:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:8.0:1619677448.720163:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.720164:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:8.0:1619677448.720164:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.720165:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:8.0:1619677448.720165:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.720166:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:8.0:1619677448.720166:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.720167:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.720167:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677448.720168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.720171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:8.2:1619677448.720171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677448.720175:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.720177:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:12.0:1619677448.720178:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:2.0:1619677448.720178:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:17.0:1619677448.720179:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:12.0:1619677448.720179:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.720180:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:12.0:1619677448.720181:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.720181:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677448.720182:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:13.0:1619677448.720182:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:17.0:1619677448.720183:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.720183:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677448.720184:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:17.0:1619677448.720185:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.720189:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.720266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.720271:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.720274:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.720274:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.720276:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.720279:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.720280:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.720281:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.720283:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.720284:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.720285:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.720289:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.720291:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.720293:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.720294:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.720299:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.720301:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.720302:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.720303:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.720303:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.720308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.720316:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.720319:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.720319:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.720321:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.720322:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.720323:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.720446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.720449:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.720452:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.720454:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.720454:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.720457:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.720459:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.720460:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.720461:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.720462:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.720463:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.720467:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.720468:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.720471:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.720472:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.720473:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.720475:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.720475:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.2:1619677448.720475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677448.720476:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.720476:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.720481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.720483:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.720486:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.720487:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.720489:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.720489:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.720491:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:26.0:1619677448.720492:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.720492:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.720494:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:13.0:1619677448.720494:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.720495:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.720496:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:26.0:1619677448.720497:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.720497:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.720497:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677448.720499:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.720502:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677448.720503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:26.0:1619677448.720504:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.720505:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.720506:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.720507:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.720508:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.0:1619677448.720508:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:26.0:1619677448.720509:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.720510:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.720510:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.0:1619677448.720511:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.720513:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.720514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.720514:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.720516:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.720518:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.720519:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.720520:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:5.0:1619677448.720520:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.720521:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.720522:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:5.0:1619677448.720522:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:10.0:1619677448.720523:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.720524:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.720525:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.720527:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:5.0:1619677448.720527:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.720529:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.720531:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.720532:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.720533:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.720535:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.720535:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.720536:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.720537:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677448.720541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.720549:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.720551:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.720552:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.720552:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.720554:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.720557:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.720559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.720566:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.720569:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.720569:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.720570:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.720571:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.720571:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.720699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.720705:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.720707:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.720708:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.720710:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.720711:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.720712:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677448.720713:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.720715:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.720716:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.720717:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.720720:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.720722:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:21.2:1619677448.720723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677448.720725:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.720726:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.720727:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.720728:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.720729:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.720729:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.720730:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.720731:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.720733:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:25.2:1619677448.720734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:31.0:1619677448.720736:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.720738:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.720738:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.720739:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.720741:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.720741:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:31.0:1619677448.720742:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.720743:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.720743:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:31.0:1619677448.720744:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:12.0:1619677448.720744:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.720746:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.720747:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:31.0:1619677448.720748:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.720750:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.720751:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:31.0:1619677448.720752:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.720753:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.720754:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.720755:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.720756:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.720757:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.720757:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.720761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.720767:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.720769:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.720770:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.720771:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.720773:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.720774:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.720780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.720786:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.720789:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.720789:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.720790:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.720791:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.720791:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.720881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.720888:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.720891:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.720892:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.720894:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.720897:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.720898:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.720900:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.720901:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.720902:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.720903:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.720906:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.720908:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.720911:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.720912:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.720913:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.720914:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.720915:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.720916:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.720916:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:25.2:1619677448.720920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.720927:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.720930:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.720930:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.720932:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.720934:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.720935:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.721021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.721027:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.721029:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.721031:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.721032:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.721035:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.721036:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677448.721037:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.721039:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.721039:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.721040:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.721043:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.721044:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.721046:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.721047:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.721047:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.721049:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.721049:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.721049:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.721051:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677448.721051:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677448.721054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:25.2:1619677448.721055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.721056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.0:1619677448.721060:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.721062:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:18.0:1619677448.721062:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:6.0:1619677448.721062:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:20.0:1619677448.721064:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.721064:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.721064:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.721065:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677448.721065:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677448.721065:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677448.721066:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.721066:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.721067:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.721069:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:6.0:1619677448.721069:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:20.0:1619677448.721070:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.721071:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:20.0:1619677448.721072:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.721072:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.721074:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.721077:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.721079:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.721082:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.721082:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.721083:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.721085:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.721086:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.721086:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.721087:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.721091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.721097:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.721100:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.721101:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.721103:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.721104:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.721105:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.721298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.721306:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.721308:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.721309:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.721311:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.721313:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.721314:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.721316:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.721317:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.721318:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.721319:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.721323:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.721324:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.721328:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.721328:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.721329:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.721331:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.721331:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.721332:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.721332:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.721337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.721345:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:21.2:1619677448.721345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677448.721348:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.721349:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.721349:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.721352:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.721353:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677448.721355:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:30.0:1619677448.721356:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.721358:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.721360:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.721361:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.2:1619677448.721361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677448.721362:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.721363:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.721365:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.721365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:30.0:1619677448.721366:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.721367:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.721367:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677448.721370:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.721370:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.721371:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.721372:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.721373:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.721374:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.721375:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:30.0:1619677448.721375:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.721375:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.721376:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.721377:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.721377:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:17.0:1619677448.721377:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.721378:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677448.721378:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677448.721379:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.721379:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.721379:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:17.0:1619677448.721379:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.721381:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.721382:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677448.721382:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.721383:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.721383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:17.0:1619677448.721383:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.721384:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677448.721385:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.721386:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.721388:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677448.721388:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.721390:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:4.2:1619677448.721390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:17.0:1619677448.721391:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:31.0:1619677448.721392:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.721392:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.721393:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.721395:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.721395:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677448.721395:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.721396:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.721396:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.721397:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677448.721397:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.721398:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.721398:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:12.0:1619677448.721398:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.721399:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.721399:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677448.721399:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.721399:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677448.721400:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:17.0:1619677448.721400:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677448.721401:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.721402:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:8.0:1619677448.721402:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677448.721403:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:24.0:1619677448.721403:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.721404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.721405:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.721405:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:24.0:1619677448.721405:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.721408:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.721409:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96711211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.721410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:24.0:1619677448.721411:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677448.721411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:24.0:1619677448.721412:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.721414:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.721417:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:24.0:1619677448.721418:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.721419:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:5.0:1619677448.721419:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677448.721420:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:24.0:1619677448.721420:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.721420:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.721421:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:8.0:1619677448.721422:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.721422:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.721423:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:24.0:1619677448.721423:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.721424:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:24.0:1619677448.721424:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:20.0:1619677448.721424:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.721424:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.721425:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:24.0:1619677448.721425:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.721425:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.721427:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.721427:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.0:1619677448.721427:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:24.0:1619677448.721428:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.721428:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.721428:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.721429:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677448.721429:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.721430:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.721430:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.721431:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:8.2:1619677448.721433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.721440:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.721443:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.721444:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.721444:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.721447:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.721449:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.721491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.721497:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.721500:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.721501:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.721502:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.721506:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.721507:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.721508:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.721509:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.721510:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.721511:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.721514:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.721516:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.721519:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.721520:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.721520:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.721522:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.721523:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.721523:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.721524:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.721528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.721534:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.721537:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.721538:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.721538:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.721541:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.721544:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.721631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.721634:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.721636:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.721639:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.721639:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.721641:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.721643:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.721644:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.721645:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.721647:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.721647:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.721651:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.721652:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.721655:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.721656:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.721657:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.721657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:20.0:1619677448.721658:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.721659:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.721660:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.721661:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.721662:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677448.721663:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:25.2:1619677448.721664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.721666:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:20.2:1619677448.721667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.721667:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.721669:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.721671:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:18.0:1619677448.721672:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.721672:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:18.0:1619677448.721673:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:13.0:1619677448.721673:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.721674:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.721675:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.721675:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.721675:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.721676:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677448.721677:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677448.721677:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.721677:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:18.0:1619677448.721678:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.721679:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.721680:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:18.0:1619677448.721683:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.721684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.721685:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.721688:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.721689:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.721690:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.721691:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:18.0:1619677448.721693:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:18.0:1619677448.721693:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.721694:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.721694:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677448.721694:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.721695:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:12.0:1619677448.721696:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.721696:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.721698:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.721700:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.721701:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.721703:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.721704:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.721705:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.721800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.721807:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.721810:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.721811:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.721812:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.721815:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.721817:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.721818:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.721819:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.721820:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.721821:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.721824:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.721825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.721826:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.721829:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.721830:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.721831:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.721833:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.721834:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.721834:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.721835:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.721835:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.721836:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.721837:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.721838:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.721839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:10.0:1619677448.721840:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.721842:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.721843:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.721844:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.721844:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.721845:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.721846:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:10.0:1619677448.721848:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.721849:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.721849:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.721850:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.721850:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.721852:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.721853:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.721853:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:6.0:1619677448.721853:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.721855:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.721856:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677448.721856:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:10.0:1619677448.721858:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.721859:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:25.2:1619677448.721860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.721860:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.721865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677448.721866:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.721869:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.721870:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.721872:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:13.0:1619677448.721872:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.721873:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.721874:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:13.0:1619677448.721874:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.721875:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.721876:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.2:1619677448.721924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.721929:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.721932:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.721933:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.721935:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.721937:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.721939:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.721940:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.721941:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.721942:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.721943:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.721947:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.721949:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.721951:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.721952:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.721953:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.721954:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.721955:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.721956:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.721956:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.721960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.2:1619677448.721966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:24.0:1619677448.721967:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.721970:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.721970:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.721971:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.721973:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.721974:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:24.0:1619677448.721975:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:30.0:1619677448.721977:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.721979:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.721980:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.721981:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.721983:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.721984:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.721986:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.721987:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.721988:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.721991:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.721993:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.721996:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.721997:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.721998:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.721999:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.722000:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.722001:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.722001:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.722005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.722013:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.722016:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.722016:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.722018:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.722019:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.722021:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.722092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.722100:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.722103:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.722105:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.722107:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.722108:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.722110:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.722111:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.722112:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.722113:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.722114:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.722117:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.722119:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677448.722121:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.722122:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.722122:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.722124:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.722124:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.722126:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.722127:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.722128:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.722129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.722134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.722135:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.722138:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.722139:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.722139:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.2:1619677448.722140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:26.0:1619677448.722141:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.722142:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:26.0:1619677448.722143:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:30.0:1619677448.722144:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.722144:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.722145:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677448.722145:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.722148:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.722149:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.722151:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.722154:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.722155:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.722156:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.722158:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.722159:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.722160:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.722163:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.722165:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.722167:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.722168:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.722169:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.722170:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.722170:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.722171:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.722172:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.722176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.722184:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.722186:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.722187:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.722187:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.722189:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.722192:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.722228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.722235:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.722238:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.722239:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.722241:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.722243:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.722245:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.722246:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.722247:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.722248:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.722249:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.722252:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.722254:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.2:1619677448.722254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.722257:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.722258:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.722259:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.722261:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.722262:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.722262:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.722263:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.722264:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.722265:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.722265:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.722266:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.722266:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.722267:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.722267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:26.0:1619677448.722269:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.722270:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.722271:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677448.722271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:26.0:1619677448.722272:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.722273:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677448.722274:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:26.0:1619677448.722277:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.722277:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.722278:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.722278:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:26.0:1619677448.722279:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.722279:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.722280:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677448.722281:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.722281:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.722282:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.722283:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.722283:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:26.0:1619677448.722284:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.722284:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.722286:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.722287:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.722288:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.722288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.722289:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.722293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.722297:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.722300:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.722301:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.722302:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.722302:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:6.0:1619677448.722302:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.722303:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:18.0:1619677448.722305:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.722306:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.722306:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:21.2:1619677448.722307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:18.0:1619677448.722308:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.722311:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.0:1619677448.722315:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.722318:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.722319:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.2:1619677448.722321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:8.0:1619677448.722321:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.722324:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:9.2:1619677448.722324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000400:00000200:8.0:1619677448.722324:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.722325:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:20.0:1619677448.722326:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:8.0:1619677448.722327:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.722329:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.722329:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.722329:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:8.0:1619677448.722330:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.722331:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.722331:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677448.722332:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.722334:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:2.0:1619677448.722334:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.722335:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.722335:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:8.0:1619677448.722336:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.722336:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.722336:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000400:00000200:20.0:1619677448.722337:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.722337:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:20.0:1619677448.722338:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.722338:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.722339:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.0:1619677448.722341:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.722342:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.722342:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677448.722343:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.722344:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.722345:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.0:1619677448.722345:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.722346:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.722346:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677448.722348:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.722349:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.722350:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:9.2:1619677448.722350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.722352:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.722352:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.722353:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.722353:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.722358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:18.0:1619677448.722358:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.722360:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.722361:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.722362:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.722364:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.722365:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:18.0:1619677448.722366:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677448.722368:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.722369:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.722370:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.722372:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.722373:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.722381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.722389:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.722392:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.722394:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.722395:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.722396:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.722398:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.722399:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.722400:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.722401:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.722402:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.722405:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.722407:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.722410:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.722410:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.722411:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.722413:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.722413:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.722414:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.722415:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.722418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.722426:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.722428:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.722429:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.722431:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.722433:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.722434:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.722640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677448.722645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.722647:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.722650:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.722651:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.722651:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.722653:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.722654:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.722654:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.722655:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.722656:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.722657:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.722658:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.722659:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.722659:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.722660:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.722661:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.722662:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.722662:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677448.722664:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.722665:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.722666:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.722667:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.722670:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:17.0:1619677448.722671:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.722673:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.722673:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.722674:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.722675:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.722676:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.722676:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:17.0:1619677448.722677:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.722677:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.722677:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677448.722678:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.722678:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.2:1619677448.722679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:17.0:1619677448.722680:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.722680:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:25.2:1619677448.722682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.722682:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.722682:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.722683:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.722685:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:25.2:1619677448.722686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.722688:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.722690:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677448.722690:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.2:1619677448.722691:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677448.722693:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:12.0:1619677448.722693:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:1.0:1619677448.722693:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:22.0:1619677448.722694:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.722694:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.722695:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:12.0:1619677448.722696:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677448.722696:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:22.0:1619677448.722697:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:12.0:1619677448.722697:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.722697:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:5.0:1619677448.722698:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:1.0:1619677448.722698:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677448.722699:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:12.0:1619677448.722699:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.722700:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:26.0:1619677448.722701:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:13.0:1619677448.722701:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:5.0:1619677448.722701:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.722702:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.722702:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:5.0:1619677448.722702:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:25.2:1619677448.722704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:5.0:1619677448.722704:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.722705:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.722706:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.722707:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.722707:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.722708:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.722709:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.722710:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.722711:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.722711:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:26.0:1619677448.722713:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.722714:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.722714:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.722715:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.722715:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.722716:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.722718:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:5.0:1619677448.722719:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000400:00000200:26.0:1619677448.722720:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.722722:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.722723:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.722726:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.722726:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.722727:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:5.0:1619677448.722727:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.722728:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677448.722728:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.722730:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677448.722730:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.722731:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677448.722731:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:26.0:1619677448.722732:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.722733:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.0:1619677448.722733:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.722734:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:25.2:1619677448.722735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677448.722735:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.722739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.2:1619677448.722740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:12.0:1619677448.722742:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.0:1619677448.722743:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:12.0:1619677448.722744:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:20.0:1619677448.722745:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:12.0:1619677448.722745:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.722745:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:20.0:1619677448.722746:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:12.0:1619677448.722746:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.722747:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:10.0:1619677448.722747:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.722748:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:13.0:1619677448.722748:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:10.0:1619677448.722748:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.722749:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:10.0:1619677448.722750:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:10.0:1619677448.722754:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.722756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.722762:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.722765:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.722765:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.722766:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.722767:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.722767:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.722827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.722834:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.722837:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.722839:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.722841:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.722841:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.722843:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.722844:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.722845:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.722846:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.722847:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.722850:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.722852:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.722854:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.722855:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.722856:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.722857:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.722858:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.722859:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.722859:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.722863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.722869:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.722871:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.722872:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.722873:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.722875:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.722876:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.722883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.722889:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.722892:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.722893:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.722894:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.722896:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.722898:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.722899:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.722900:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.722901:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.722903:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.722906:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.722908:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.722910:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.722911:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.722911:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.722913:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.722914:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.722915:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.722916:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.722916:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.722919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.722921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.722927:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:17.0:1619677448.722927:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677448.722929:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:17.0:1619677448.722929:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677448.722930:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677448.722930:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.722930:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.722931:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:17.0:1619677448.722932:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.722935:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.723003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677448.723011:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677448.723014:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677448.723017:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.723018:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677448.723019:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.723021:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677448.723022:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.723023:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.723024:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.723025:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.2:1619677448.723028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:31.0:1619677448.723029:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.723031:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:21.2:1619677448.723031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:20.0:1619677448.723031:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.723034:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.723034:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:4.2:1619677448.723034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.723035:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:31.0:1619677448.723036:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.723036:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:20.0:1619677448.723037:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.723037:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677448.723038:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.723039:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:2.0:1619677448.723039:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:31.0:1619677448.723040:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677448.723040:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000400:00000200:20.0:1619677448.723040:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.723040:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:20.0:1619677448.723042:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:10.0:1619677448.723042:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677448.723042:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677448.723043:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677448.723044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.723045:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677448.723045:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.723045:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:2.0:1619677448.723045:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.723046:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.723046:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.723047:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677448.723048:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.723049:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.723050:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.723051:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.723052:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.723053:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:10.0:1619677448.723053:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:18.0:1619677448.723054:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:10.0:1619677448.723054:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.723054:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.723056:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.723057:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.723058:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.723058:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.723060:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.723060:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:5.2:1619677448.723060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:2.0:1619677448.723060:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.723063:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677448.723064:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.723065:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.723066:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677448.723066:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.723067:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.723067:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.723067:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.723068:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.723068:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:1.0:1619677448.723068:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.0:1619677448.723069:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:17.0:1619677448.723069:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.723069:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677448.723070:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.723071:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677448.723071:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.723071:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:1.0:1619677448.723071:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.723072:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677448.723072:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.723072:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.0:1619677448.723073:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:10.0:1619677448.723073:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:2.0:1619677448.723073:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:10.0:1619677448.723074:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:2.0:1619677448.723074:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:1.0:1619677448.723074:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.723075:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:1.0:1619677448.723075:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:10.0:1619677448.723076:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677448.723076:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96711186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.723077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.723077:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.723077:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.723077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:1.0:1619677448.723078:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.723079:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.723080:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:25.2:1619677448.723082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:1.0:1619677448.723083:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.723085:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:1.0:1619677448.723085:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.723087:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677448.723088:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:6.0:1619677448.723088:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:1.0:1619677448.723088:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.723089:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.723089:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:1.0:1619677448.723089:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.723090:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (23)++
00000400:00000200:6.0:1619677448.723090:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.723090:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (23)++
00000400:00000200:18.0:1619677448.723091:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:6.0:1619677448.723092:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:1.0:1619677448.723092:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (25)--
00000800:00000200:22.0:1619677448.723093:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000800:00000200:18.0:1619677448.723093:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:1.0:1619677448.723093:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.723093:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677448.723094:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.723094:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.723095:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.723096:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.723097:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677448.723098:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (23)++
00000800:00000200:22.0:1619677448.723099:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (24)--
00000800:00000200:6.0:1619677448.723099:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (24)--
00000800:00000200:22.0:1619677448.723100:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.723100:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.723101:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.723103:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.723105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:22.0:1619677448.723112:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.723115:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.723115:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.723116:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.723117:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.723118:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.723139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.723147:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.723150:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.723152:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.723153:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.723155:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.723156:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.723158:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.723159:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.723160:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.723161:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.723164:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.723166:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.723169:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.723170:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.723170:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.723172:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.723173:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.723173:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.723174:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.723178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.723186:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.723188:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.723189:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.723191:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.723191:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.723194:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.723196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.723203:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.723206:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.723207:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.723208:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.723208:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.723209:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.723299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.723305:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.723308:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.723309:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.723311:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.723313:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.723314:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.723316:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.723317:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.723318:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.723319:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.723322:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.723323:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.723325:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.723326:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.723327:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.723328:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.723329:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.723329:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.723330:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677448.723335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.723342:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.723344:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.723345:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.723345:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.723346:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.723348:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.723451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.723456:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.723459:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.723460:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.723462:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.723464:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.723465:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.723467:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.723468:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.723469:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.723470:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677448.723474:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.723476:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677448.723478:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.723479:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.723479:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.723481:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:6.0:1619677448.723482:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.723483:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.723484:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.723484:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:8.2:1619677448.723488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:21.2:1619677448.723490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.723494:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.723496:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:17.0:1619677448.723496:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:22.0:1619677448.723497:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.723498:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677448.723498:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.723500:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.723500:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.723501:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.723503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.723506:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.723508:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.723510:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.723511:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.723511:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.723512:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.723513:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.723532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.723540:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.723543:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.723543:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.723546:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.723549:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.723550:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.723552:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.723553:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.723554:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.723555:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.723558:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.723560:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.723563:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.723564:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.723564:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.723566:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.723566:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.723568:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677448.723569:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.723569:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.2:1619677448.723570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.2:1619677448.723572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.723575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.723577:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.723579:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.723581:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677448.723582:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:1.0:1619677448.723582:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.723583:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.723584:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.723584:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677448.723584:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.723585:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.723586:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.723586:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.723586:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.723587:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.723588:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.723589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:1.0:1619677448.723589:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.723590:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.723593:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.723595:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.723596:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:26.0:1619677448.723597:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.723597:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.723597:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.723598:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677448.723598:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.723599:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.723600:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:1.0:1619677448.723600:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.723601:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677448.723601:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.723602:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.723602:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677448.723603:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.723606:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.723606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:26.0:1619677448.723608:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677448.723609:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.723610:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.723611:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.723612:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:17.0:1619677448.723614:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:26.0:1619677448.723616:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.723616:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.723617:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.723618:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:1.0:1619677448.723618:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.723619:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.723620:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.723621:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:26.0:1619677448.723621:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:17.0:1619677448.723621:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:26.0:1619677448.723623:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:26.0:1619677448.723624:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.723624:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.723625:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.723629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.723636:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.723639:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.723640:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.723641:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.723644:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.723645:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.723650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.723656:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.723659:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.723659:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.723660:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:31.0:1619677448.723660:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.723661:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:4.2:1619677448.723670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.723677:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.723679:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.723680:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.723682:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.723684:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.723686:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.723687:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.723688:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.723689:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.723690:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.723693:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.723694:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.723697:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.723698:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.723698:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.723700:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.723701:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.723702:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.723702:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.723706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.723714:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.723716:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.723717:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.723718:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.723720:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.723722:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.723726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.723733:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.723736:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.723737:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.723738:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.723741:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.723743:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.723744:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.723745:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.723746:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.723747:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.723750:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.723752:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.723754:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.723754:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.723755:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.723756:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.723757:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.723758:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.723758:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677448.723762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.723770:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.723772:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.723773:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.723774:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.723777:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.723778:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.723782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:12.0:1619677448.723788:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.723791:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.723792:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.723792:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.723793:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.723794:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.723803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677448.723809:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677448.723812:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677448.723812:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:1.0:1619677448.723815:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.723817:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.723818:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.723820:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.723821:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.723822:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.723823:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.723826:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.723828:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.723831:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.723832:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.723833:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.723835:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.723835:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677448.723836:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677448.723836:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:25.2:1619677448.723840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.723847:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.723850:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.723851:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.723852:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.723853:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.723855:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.723953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.723959:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.723962:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.723962:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:21.2:1619677448.723963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:17.0:1619677448.723964:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.723967:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.723968:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.723970:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.723971:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677448.723971:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.723972:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.723973:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:17.0:1619677448.723973:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:30.0:1619677448.723977:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.723978:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.723978:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.723980:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.723981:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.723982:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.723982:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.723982:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677448.723983:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:17.0:1619677448.723983:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.723984:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.723984:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677448.723985:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.723985:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677448.723986:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.723987:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.723988:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:21.2:1619677448.723989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:8.2:1619677448.723990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677448.723991:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.723993:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677448.723995:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:31.0:1619677448.723996:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.723996:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.723997:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.723997:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:1.0:1619677448.723997:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:6.0:1619677448.723998:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:31.0:1619677448.723999:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.723999:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677448.723999:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:31.0:1619677448.724000:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677448.724000:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.724000:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.724001:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677448.724001:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.724002:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.724003:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.724004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677448.724004:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:1.0:1619677448.724004:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:22.0:1619677448.724005:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.724007:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.724008:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.724009:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:12.0:1619677448.724011:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:22.0:1619677448.724012:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.724013:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:22.0:1619677448.724014:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:12.0:1619677448.724014:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.724016:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.724017:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677448.724018:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.724018:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:22.0:1619677448.724019:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.724019:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.724021:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.724022:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.724023:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.724023:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.724027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.724033:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.724036:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.724037:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.724037:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.724039:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.724042:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677448.724221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.724224:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.724227:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.724229:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.724230:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.724231:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.724233:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.724234:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.724236:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.724237:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.724238:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.724241:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.724243:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.724246:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.724247:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.724247:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.724249:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.724250:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.724251:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.724251:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.724255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.724262:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.724264:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.724265:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.724267:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.724268:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.724270:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.724349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.724355:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.724357:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.724358:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.724360:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.724361:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.724363:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677448.724364:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.724365:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.724366:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.724367:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677448.724370:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.724372:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677448.724375:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.724376:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:17.0:1619677448.724376:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.724378:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.724378:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.724380:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.724381:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.724381:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.724385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.2:1619677448.724387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.724392:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:1.0:1619677448.724393:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.724394:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.724395:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.724396:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:1.0:1619677448.724396:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677448.724397:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.724399:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.724400:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.724402:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.724454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.724460:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.724463:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.724464:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.724466:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.724469:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.724470:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.724472:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.724473:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.724474:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.724475:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.724479:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.724480:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.724484:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.724484:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.724485:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.724487:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.724488:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.724488:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.724489:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:21.2:1619677448.724491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:25.2:1619677448.724493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:2.0:1619677448.724497:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:31.0:1619677448.724500:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:2.0:1619677448.724500:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.724501:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:31.0:1619677448.724503:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:2.0:1619677448.724503:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.724504:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.724504:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.724506:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.724506:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:30.0:1619677448.724507:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:2.0:1619677448.724508:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.724509:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:2.0:1619677448.724509:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.724510:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.724511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:2.0:1619677448.724511:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:4.2:1619677448.724513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:2.0:1619677448.724515:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.724516:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.724518:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:24.0:1619677448.724520:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677448.724520:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.724520:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.724521:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:2.0:1619677448.724521:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.724522:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.724523:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.724523:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:24.0:1619677448.724523:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:2.0:1619677448.724523:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.0:1619677448.724524:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677448.724524:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:30.0:1619677448.724525:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:2.0:1619677448.724525:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.724525:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:24.0:1619677448.724526:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.724529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677448.724529:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.724531:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.724532:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96711169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.724533:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.724534:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.724535:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:12.0:1619677448.724536:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:24.0:1619677448.724539:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.724539:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.724540:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.724541:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:12.0:1619677448.724542:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.724544:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.724544:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.724544:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:24.0:1619677448.724545:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.724545:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.724547:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:25.2:1619677448.724548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:24.0:1619677448.724548:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.724548:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.724549:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.724553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:13.0:1619677448.724554:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.724556:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:21.2:1619677448.724557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:13.0:1619677448.724557:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.724558:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.724558:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.724559:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677448.724561:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:18.0:1619677448.724563:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:17.0:1619677448.724564:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.724565:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.724565:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:18.0:1619677448.724566:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677448.724566:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:17.0:1619677448.724567:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.724569:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.724570:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:18.0:1619677448.724571:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.724572:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677448.724574:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.724575:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.724576:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.724577:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.724581:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.724582:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.724586:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.724586:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.724587:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.724589:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.724590:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677448.724590:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677448.724591:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:25.2:1619677448.724595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.724602:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.724604:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.724605:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.724606:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.724608:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.724611:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.724646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.724653:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.724655:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.724656:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.724658:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.724660:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.724662:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.724663:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.724664:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.724665:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.724666:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.724669:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.724671:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.724674:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.724675:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.724675:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.724677:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.724678:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.724679:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.724680:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.724680:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677448.724683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.724686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.0:1619677448.724692:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677448.724693:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:5.0:1619677448.724694:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677448.724695:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.724695:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:5.0:1619677448.724695:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.724697:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677448.724697:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.724697:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.724700:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.724781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.724786:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.724789:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.724790:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.2:1619677448.724792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:1.0:1619677448.724792:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.724794:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.724796:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.724797:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711166] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.724798:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.724799:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.724800:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:1.0:1619677448.724800:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.724803:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:1.0:1619677448.724804:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.724806:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.724806:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677448.724807:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.724808:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.724808:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.724809:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:31.0:1619677448.724810:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:1.0:1619677448.724810:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677448.724811:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.724811:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:31.0:1619677448.724812:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.724812:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.724812:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677448.724813:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.724813:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677448.724814:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.724817:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.724817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:31.0:1619677448.724819:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.724821:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.724822:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.724823:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.724824:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:17.0:1619677448.724824:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:31.0:1619677448.724825:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677448.724825:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:31.0:1619677448.724826:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:17.0:1619677448.724826:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.724827:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677448.724828:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000400:00000200:17.0:1619677448.724828:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.724829:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.724830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:17.0:1619677448.724831:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.724834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.724839:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:8.2:1619677448.724839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:30.0:1619677448.724840:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.724841:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.724842:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.724843:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:30.0:1619677448.724843:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.724844:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.724845:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.724846:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:1.0:1619677448.724846:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677448.724847:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:17.0:1619677448.724849:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.724849:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.724850:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.724851:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.724852:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.724911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.724916:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.724919:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.724919:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.724921:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.724924:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.724926:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677448.724927:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.724928:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.724929:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.724930:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677448.724934:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.724936:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.724939:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.724940:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:22.0:1619677448.724940:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:22.0:1619677448.724942:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.724942:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.724943:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.724944:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.724948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.724954:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.724957:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.724957:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.724958:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.724960:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.724963:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:20.2:1619677448.725063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.725066:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.725069:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.725071:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.725071:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.725073:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.725075:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.725076:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.725078:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.725078:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.725080:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.725083:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.725085:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.725087:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.725088:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.725088:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.725090:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.725091:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.725092:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.725093:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.725094:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:9.2:1619677448.725096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677448.725098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.725103:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:5.0:1619677448.725104:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.725106:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:2.0:1619677448.725106:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.725108:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.725108:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:2.0:1619677448.725108:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.725109:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:2.0:1619677448.725110:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.725113:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.725115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.725122:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.725125:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.725126:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.725126:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.725127:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.725128:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.725174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.725182:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.725185:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.725185:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.725187:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.725190:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.725192:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.725193:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.725195:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.725196:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.725197:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.725200:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.725201:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.725204:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.725205:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.725206:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.725208:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.725208:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.725209:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.725209:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677448.725212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:25.2:1619677448.725214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.725217:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.725220:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.725221:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:6.0:1619677448.725221:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:2.0:1619677448.725223:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.725224:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.725225:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.725225:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.725225:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.725227:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:6.0:1619677448.725228:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.725229:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.725230:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677448.725232:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:2.0:1619677448.725232:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.725233:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.725236:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.725238:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.725241:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.725242:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:2.0:1619677448.725242:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677448.725244:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:2.0:1619677448.725244:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.725246:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.725247:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.725248:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.2:1619677448.725249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:25.2:1619677448.725250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:21.2:1619677448.725253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:31.0:1619677448.725256:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.725257:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:18.0:1619677448.725258:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:31.0:1619677448.725259:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:13.0:1619677448.725259:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:18.0:1619677448.725260:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:13.0:1619677448.725260:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.725261:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.725261:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.725262:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:13.0:1619677448.725262:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.725263:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677448.725263:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:31.0:1619677448.725264:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.725265:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:31.0:1619677448.725266:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677448.725267:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.725268:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.725270:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.725270:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.725274:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.725275:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.725279:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.725279:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.725280:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:31.0:1619677448.725282:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.725282:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.725283:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.725284:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.725288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.725295:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.725301:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.725302:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.725303:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.725305:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.725308:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.725324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.725330:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.725333:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.725333:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.725335:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.725337:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.725338:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.725339:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.725340:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.725341:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.725342:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.725345:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.725347:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.725350:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.725351:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.725351:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.725353:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.725354:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.725354:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.725355:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.725370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.725378:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.725380:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.725381:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.725383:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.725385:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.725386:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.725493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.725501:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.725504:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.725504:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677448.725507:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.725509:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.725511:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.725512:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.725513:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.725515:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.725515:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.725518:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.725520:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.725523:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.725524:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.725524:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.725526:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.725526:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.725527:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.725528:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.725532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.725540:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.725542:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.725543:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.725543:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.725546:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.725548:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.725640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.725646:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.725649:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.725650:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.725652:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.725655:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.725656:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:20.2:1619677448.725656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:22.0:1619677448.725658:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.725659:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.725660:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.725661:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.725664:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:22.0:1619677448.725665:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.725666:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.725667:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:22.0:1619677448.725669:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.725670:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.725670:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.725671:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:22.0:1619677448.725671:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.725672:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000400:00000200:30.0:1619677448.725673:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.725673:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.725674:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000400:00000200:30.0:1619677448.725675:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:22.0:1619677448.725675:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.725675:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:30.0:1619677448.725676:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711156] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.725677:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677448.725678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:30.0:1619677448.725679:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.725680:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:21.2:1619677448.725680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:30.0:1619677448.725683:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.725685:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:6.0:1619677448.725685:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677448.725686:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:30.0:1619677448.725687:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.725687:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:30.0:1619677448.725688:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:6.0:1619677448.725688:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677448.725689:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677448.725689:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:2.0:1619677448.725689:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:30.0:1619677448.725690:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:2.0:1619677448.725690:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.725691:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677448.725691:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.725692:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.725692:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:2.0:1619677448.725693:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.725696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677448.725696:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:13.0:1619677448.725702:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.725705:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.725705:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.725707:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.725708:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.725709:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.725744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.725752:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.725755:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.725756:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.725758:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.725761:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.725762:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.725763:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.725765:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.725766:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.725766:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.725769:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.725770:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.725772:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.725773:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.725773:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.725775:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.725775:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.725776:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.725777:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.725778:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.725780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.2:1619677448.725783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:4.2:1619677448.725788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.725791:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.725792:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.725793:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.725793:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.725794:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:26.0:1619677448.725795:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.725795:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.725796:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.725796:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.725797:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.725797:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.725799:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:10.0:1619677448.725799:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.725800:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.2:1619677448.725801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:10.0:1619677448.725801:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.725803:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:10.0:1619677448.725803:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.725804:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.725805:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.725806:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:10.0:1619677448.725806:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.725807:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.725809:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.725810:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.725812:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.725813:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.725814:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:10.0:1619677448.725814:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:20.0:1619677448.725815:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.725816:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.725817:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.725817:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:20.0:1619677448.725818:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.725818:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:20.0:1619677448.725819:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.725820:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.725820:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.725821:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.725821:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:20.0:1619677448.725823:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.725825:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.2:1619677448.725826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.725827:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.725828:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.725828:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.725830:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.725831:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.725832:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.725832:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677448.725833:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.725836:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:9.2:1619677448.725836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:22.0:1619677448.725837:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.725837:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.725839:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.725842:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.725843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.725844:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.725846:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.725847:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.725848:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.725850:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.725851:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.725852:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:6.0:1619677448.725853:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.725855:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.725855:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.725856:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.725856:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.725897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.725902:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.725905:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.725907:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.725907:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677448.725910:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.725912:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.725913:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.725914:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.725915:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.725916:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.725920:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.725922:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.725924:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.725925:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.725925:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.725927:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:5.0:1619677448.725928:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.725929:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.725929:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677448.725933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.725941:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.725943:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.725944:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.725944:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.725946:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677448.725948:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.726056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.726062:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.726064:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.726065:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.726067:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.726070:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.726071:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677448.726073:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.726074:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.726075:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.726076:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:21.2:1619677448.726077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:5.2:1619677448.726077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:2.0:1619677448.726080:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.726082:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.726083:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677448.726083:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.726085:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:17.0:1619677448.726086:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:2.0:1619677448.726086:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.726087:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677448.726088:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:18.0:1619677448.726089:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:2.0:1619677448.726089:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:17.0:1619677448.726090:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.726091:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.726091:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:2.0:1619677448.726091:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.726092:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.726093:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677448.726093:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.726093:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:18.0:1619677448.726094:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.726094:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:17.0:1619677448.726095:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.726096:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677448.726097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:18.0:1619677448.726097:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.726098:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.726098:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.726099:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677448.726099:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.726100:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.726103:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:18.0:1619677448.726104:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000400:00000200:17.0:1619677448.726105:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.726106:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:18.0:1619677448.726107:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:12.0:1619677448.726107:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.726108:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.726109:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.726109:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:12.0:1619677448.726109:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.726110:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677448.726110:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.726111:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677448.726111:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.726112:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677448.726112:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.726112:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677448.726113:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.726113:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677448.726113:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.726115:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.726115:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.726116:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.726117:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.726117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:25.2:1619677448.726121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.726124:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.726127:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.726127:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.726128:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677448.726128:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:2.0:1619677448.726129:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:13.0:1619677448.726130:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.726131:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.726132:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:13.0:1619677448.726133:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.726134:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.726135:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.726179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.726186:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.726189:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.726191:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.726193:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.726193:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.726195:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.726196:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.726198:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.726199:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.726200:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.726203:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.726205:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.726207:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.726208:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.726209:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.726210:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.726211:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.726212:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.726214:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.726214:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677448.726216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:21.2:1619677448.726220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.726222:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.726224:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.726225:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.726226:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.726228:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.726228:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.726229:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.726230:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.726230:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.726231:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.726233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.726239:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.726242:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.726242:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.726243:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.726244:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.726245:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.726386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.726394:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.726397:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.726398:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.726400:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.726403:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.726404:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.726405:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.726406:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.726407:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.726408:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.726411:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.726413:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.726415:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.726416:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.726417:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.726418:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.726419:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.726419:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.726420:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.726425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.726433:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.726436:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.726437:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.726438:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.726440:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.726442:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.726450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.726453:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.726456:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.726458:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.726459:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.726461:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.726462:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.726464:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.726465:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.726466:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.726467:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.726470:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.726471:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.726474:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.726474:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.726475:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.726477:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.726478:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.726479:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.726479:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.726483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.726491:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.726494:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.726494:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.726495:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.726497:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.726499:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.726530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.726538:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:5.2:1619677448.726540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:30.0:1619677448.726541:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.726544:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.726545:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677448.726546:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.726546:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.726548:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:18.0:1619677448.726548:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.726548:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:30.0:1619677448.726549:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.726551:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.726552:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.726553:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.726553:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.726554:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.726555:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.726557:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.726558:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.726559:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.726560:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.726561:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.726563:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:2.0:1619677448.726565:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.726566:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.726567:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.726567:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.726568:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.726570:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:2.0:1619677448.726570:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.726571:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677448.726571:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.726572:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:2.0:1619677448.726572:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.726573:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:2.0:1619677448.726574:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.726574:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.726575:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.726576:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.726577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:8.2:1619677448.726580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.726585:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.726587:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.726587:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.726588:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.726589:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.726589:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.726590:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.726592:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.726593:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.726594:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.726595:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:20.0:1619677448.726596:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.726774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.726780:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.726783:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.726784:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.726786:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.726789:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.726791:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:4.2:1619677448.726792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:1.0:1619677448.726792:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.726793:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.726795:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.726795:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677448.726799:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.726800:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.726801:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677448.726802:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.726804:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677448.726804:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.726805:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:24.0:1619677448.726806:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.726806:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:1.0:1619677448.726807:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:24.0:1619677448.726808:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.726808:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.726809:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:24.0:1619677448.726810:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:1.0:1619677448.726810:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000400:00000200:24.0:1619677448.726811:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96711142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.726811:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677448.726813:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.726814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:24.0:1619677448.726814:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.726815:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.2:1619677448.726815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:24.0:1619677448.726818:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.726819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:24.0:1619677448.726820:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.726820:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677448.726821:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:17.0:1619677448.726822:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:24.0:1619677448.726823:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.726823:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.726823:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677448.726824:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677448.726824:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:1.0:1619677448.726824:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677448.726825:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.726826:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.726826:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:12.0:1619677448.726826:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.726826:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677448.726827:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.726828:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677448.726828:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.726829:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.726829:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.726830:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.726832:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.726832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:10.0:1619677448.726835:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.726836:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.726837:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.726839:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:10.0:1619677448.726839:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.726839:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.726840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:10.0:1619677448.726840:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.726842:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.726843:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.726843:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.726843:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.726845:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.726845:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677448.726847:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.726848:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:10.0:1619677448.726848:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.726849:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:13.0:1619677448.726850:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.726850:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.2:1619677448.726850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:13.0:1619677448.726851:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.726851:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.726852:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.726852:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.726852:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.726853:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:13.0:1619677448.726854:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:10.0:1619677448.726854:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:1.0:1619677448.726857:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:9.2:1619677448.726858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.726860:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.726861:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.726861:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.726862:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.726863:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:18.0:1619677448.726865:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.726868:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.726869:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.726869:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.726871:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.726873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.726876:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.726877:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.726879:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.726880:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.726880:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.726881:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.726882:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.726961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.726969:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.726972:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.726975:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.726976:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.726978:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.726978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:31.0:1619677448.726980:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.726981:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.726982:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.726983:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.726984:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.726985:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677448.726987:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.726987:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.726988:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.726989:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.726990:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.726993:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.726993:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.726994:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.726994:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:26.0:1619677448.726994:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.726995:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.726996:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.726996:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.726997:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677448.726997:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.726997:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.726998:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:26.0:1619677448.726998:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.727001:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.727002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.727003:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.727006:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.727007:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.727007:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:21.2:1619677448.727007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677448.727009:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.727009:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.727010:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.727011:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:20.2:1619677448.727011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:26.0:1619677448.727012:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.0:1619677448.727012:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.727014:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.727014:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677448.727015:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677448.727015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:5.0:1619677448.727015:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.727016:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.727017:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.727017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.727018:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:5.0:1619677448.727019:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.727021:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677448.727021:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.727021:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:30.0:1619677448.727022:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.727022:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.727022:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.727023:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.727025:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.727025:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:10.0:1619677448.727025:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.727025:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:30.0:1619677448.727026:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:22.0:1619677448.727026:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.727026:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.727027:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.727027:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:30.0:1619677448.727028:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.727028:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677448.727029:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.727029:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.727029:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.727030:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.727030:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.727031:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677448.727032:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.727032:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:30.0:1619677448.727033:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.727037:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.727038:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.727039:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.727040:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.727041:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.727042:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:30.0:1619677448.727043:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.727043:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.727044:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.727045:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.727045:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677448.727046:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.727046:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677448.727046:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.727047:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.727047:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.727047:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677448.727048:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.727051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:9.2:1619677448.727052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677448.727059:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:2.0:1619677448.727060:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:12.0:1619677448.727061:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.727062:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.727063:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.727063:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.727064:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.727064:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.727065:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.727066:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677448.727066:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.727068:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.727099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.727105:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.727107:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.727108:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.727110:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.727111:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.727113:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.727114:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.727116:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.727117:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.727118:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.727121:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.727123:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.727126:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.727127:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.727127:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.727129:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.727129:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.727130:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.727131:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.727135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.727142:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.727145:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:21.2:1619677448.727146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:5.0:1619677448.727146:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.727148:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.727148:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.727151:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:2.0:1619677448.727152:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.727154:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.727155:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.727157:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.727159:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.727161:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.727162:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.727164:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.727164:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.727165:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.727169:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.727170:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:5.2:1619677448.727173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677448.727173:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.727173:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.727174:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.727176:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.727176:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.727177:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.727177:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:26.0:1619677448.727181:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:25.2:1619677448.727182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.727183:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677448.727186:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.727186:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.727188:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000400:00000200:26.0:1619677448.727190:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.727190:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.727191:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.727192:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.727193:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.727193:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:26.0:1619677448.727194:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.727195:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.727197:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.727200:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.727200:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.727201:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:26.0:1619677448.727202:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.727205:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.727206:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.727207:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.727208:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.727209:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.727209:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.727210:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.727214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.727220:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.727222:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.727223:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.727225:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.727226:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.727227:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.727353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.727361:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.727364:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.727366:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.727368:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.727369:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.727370:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677448.727371:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.727372:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.727373:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.727374:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.727377:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.727379:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.727382:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.727383:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.727383:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.727385:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.727386:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.727387:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.727389:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.727389:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:4.2:1619677448.727391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.2:1619677448.727392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.727397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:10.0:1619677448.727400:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.727403:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:30.0:1619677448.727404:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:26.0:1619677448.727404:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677448.727404:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:10.0:1619677448.727405:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.727406:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677448.727407:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.727408:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.727408:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.727408:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.727409:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:20.0:1619677448.727410:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.727410:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:5.0:1619677448.727410:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000400:00000200:10.0:1619677448.727411:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.727412:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.727413:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (24)--
00000400:00000200:10.0:1619677448.727413:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.727413:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)++
00000800:00000200:5.0:1619677448.727413:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (24)--
00000400:00000200:10.0:1619677448.727414:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.727414:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.727414:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.727415:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:10.0:1619677448.727418:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.727420:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.727423:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.727423:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.727424:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.727426:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.727426:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.727427:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.727427:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.727432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.727440:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.727443:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.727444:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.727445:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.727447:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.727449:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.727504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.727510:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.727513:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.727513:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.727515:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.727518:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.727520:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.727521:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.727522:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.727523:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.727524:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.727528:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.727529:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.727532:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.727533:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.727534:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.727536:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.727536:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.727536:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.727537:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.727541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.727548:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.727551:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.727552:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.727552:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.727554:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.727557:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.2:1619677448.727562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.727569:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.727572:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.727573:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.727575:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.727578:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.727579:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.727580:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677448.727581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:26.0:1619677448.727582:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.727583:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.727583:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.727587:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.727587:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:26.0:1619677448.727588:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:18.0:1619677448.727590:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.727591:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:26.0:1619677448.727592:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.727593:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:18.0:1619677448.727593:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.727594:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.727595:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.2:1619677448.727595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:26.0:1619677448.727596:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.727596:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:18.0:1619677448.727596:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.727597:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:18.0:1619677448.727597:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.2:1619677448.727598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:18.0:1619677448.727598:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.727600:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.727601:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.727601:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.727601:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:8.2:1619677448.727602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.0:1619677448.727603:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.727603:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.727604:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.727607:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.727607:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.727608:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.727610:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.727610:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.727610:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:5.0:1619677448.727611:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.727613:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96711128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.727613:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:18.0:1619677448.727614:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.727614:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:5.0:1619677448.727614:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.727614:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.727615:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.727616:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:5.0:1619677448.727616:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.727617:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.727617:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.727618:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.727618:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000400:00000200:20.0:1619677448.727619:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.727619:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.727619:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:20.0:1619677448.727621:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:1.0:1619677448.727621:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:20.0:1619677448.727622:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96711127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.727622:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:25.2:1619677448.727624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.727624:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.727625:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.727625:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.727626:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:5.0:1619677448.727629:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.727630:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.727630:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.727631:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:20.0:1619677448.727632:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677448.727632:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)++
00000800:00000200:20.0:1619677448.727634:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.727634:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (25)--
00000800:00000200:20.0:1619677448.727635:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:12.0:1619677448.727635:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:5.0:1619677448.727635:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.727635:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.727636:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000400:00000200:12.0:1619677448.727636:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.727636:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.727637:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677448.727637:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677448.727638:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)++
00000400:00000200:12.0:1619677448.727638:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.727638:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.727639:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677448.727640:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:20.0:1619677448.727641:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.727641:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677448.727641:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:12.0:1619677448.727641:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:13.0:1619677448.727642:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.727642:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677448.727643:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:9.2:1619677448.727643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.2:1619677448.727646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677448.727646:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.727651:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:2.0:1619677448.727651:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.727653:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:5.0:1619677448.727654:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:18.0:1619677448.727655:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:5.0:1619677448.727655:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:2.0:1619677448.727655:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.727656:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:2.0:1619677448.727657:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.727660:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:21.2:1619677448.727812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:20.2:1619677448.727813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:18.0:1619677448.727818:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677448.727819:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:20.2:1619677448.727821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677448.727821:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.727821:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:17.0:1619677448.727822:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677448.727822:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:18.0:1619677448.727824:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.727826:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.727827:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.727828:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:30.0:1619677448.727829:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:17.0:1619677448.727829:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.727830:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.727830:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:30.0:1619677448.727832:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:18.0:1619677448.727832:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.727832:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.727833:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.727834:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677448.727834:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.727835:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.727836:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:17.0:1619677448.727836:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.727837:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.727840:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.727840:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.727841:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.727842:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.727843:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96711124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.727843:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.727844:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.727845:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.727845:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.727846:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.727846:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.727847:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.727847:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.727848:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.727848:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:18.0:1619677448.727849:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.727849:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677448.727849:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:18.0:1619677448.727850:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:30.0:1619677448.727851:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.727851:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677448.727851:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:17.0:1619677448.727851:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677448.727853:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:30.0:1619677448.727854:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.727854:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:9.2:1619677448.727854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.727855:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:30.0:1619677448.727856:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.727857:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.2:1619677448.727857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.727857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.727858:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.727859:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.727860:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.727861:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.727861:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.2:1619677448.727861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:2.0:1619677448.727861:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.727863:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:10.0:1619677448.727864:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677448.727864:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.727865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677448.727865:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.727865:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.2:1619677448.727866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:2.0:1619677448.727866:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.727866:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:10.0:1619677448.727867:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:5.0:1619677448.727868:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.727868:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.727868:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:2.0:1619677448.727869:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677448.727869:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:26.0:1619677448.727870:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:5.0:1619677448.727870:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.727870:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.0:1619677448.727871:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.727872:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:12.0:1619677448.727872:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:10.0:1619677448.727872:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.727873:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:5.0:1619677448.727873:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:12.0:1619677448.727874:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.727875:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.727876:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.727876:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.727877:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.727878:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:13.0:1619677448.727879:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.727879:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:10.0:1619677448.727879:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677448.727880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000400:00000200:10.0:1619677448.727881:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.727882:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.727883:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.727884:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.727886:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677448.727887:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.727887:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000400:00000200:10.0:1619677448.727887:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.727889:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.727889:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:21.2:1619677448.727890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677448.727890:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677448.727891:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.727891:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677448.727891:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.727892:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:20.0:1619677448.727892:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.727893:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:10.0:1619677448.727895:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.727896:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.727896:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.727897:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.727898:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677448.727898:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.727899:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.727900:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.727901:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:10.0:1619677448.727901:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.727902:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.727902:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.727903:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:26.0:1619677448.727903:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.727903:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:31.0:1619677448.727904:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.727905:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.727905:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:9.2:1619677448.727905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:31.0:1619677448.727906:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.727906:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:31.0:1619677448.727907:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:26.0:1619677448.727907:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.727907:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:31.0:1619677448.727908:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96711121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.727910:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.727910:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.727911:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.2:1619677448.727911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:31.0:1619677448.727914:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.727914:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677448.727916:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.727916:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.727917:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.727918:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.727919:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:31.0:1619677448.727920:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.727920:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:18.0:1619677448.727920:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.727921:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.727921:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:31.0:1619677448.727922:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:20.0:1619677448.727922:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.727923:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.727923:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677448.727924:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677448.727924:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.727924:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.727925:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677448.727927:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:25.2:1619677448.727929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.727937:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.727940:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.727940:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.727941:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.727943:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.727946:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.727949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.727956:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.727959:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.727959:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.727960:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.727961:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.727961:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.727966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.727974:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.727977:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.727977:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.727980:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.727983:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.727984:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.727986:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.727988:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.727989:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.727990:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.727992:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.727994:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.727996:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.727997:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.727998:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.727999:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.728000:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.728001:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.728003:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.728003:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.728005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.2:1619677448.728008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677448.728012:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.728015:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.728015:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.728016:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:13.0:1619677448.728017:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.728018:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.728018:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:12.0:1619677448.728018:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.728019:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.728020:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.2:1619677448.728109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.728115:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.728118:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.728118:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.728121:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.728123:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.728124:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.728125:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.728127:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.728128:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.728129:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.728131:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.728133:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.728136:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.728137:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.728138:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.728139:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.728140:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.728141:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.728141:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.728145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.728152:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.728155:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.728156:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.728158:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.728159:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.728161:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.728164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:13.0:1619677448.728170:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.728173:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.728174:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.728174:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.728175:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.728177:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.728361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677448.728365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677448.728367:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.728370:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.728370:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.728372:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.728373:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:1.0:1619677448.728375:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.728376:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.728376:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:1.0:1619677448.728377:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.728378:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.728379:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.728380:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.728381:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.728381:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.728382:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677448.728383:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.728385:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.728386:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.728387:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.728388:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.728388:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:10.0:1619677448.728389:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.728391:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.728391:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:1.0:1619677448.728392:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000400:00000200:10.0:1619677448.728393:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.728394:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.728394:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.728395:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677448.728395:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.728396:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.728397:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.728398:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.728399:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.728400:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:9.2:1619677448.728400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:10.0:1619677448.728401:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.728402:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.728402:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.2:1619677448.728402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:25.2:1619677448.728406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.728408:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:2.0:1619677448.728408:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:24.0:1619677448.728411:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:2.0:1619677448.728411:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:24.0:1619677448.728412:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.728412:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.0:1619677448.728412:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:12.0:1619677448.728414:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:2.0:1619677448.728414:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.728415:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.728416:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:4.2:1619677448.728416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:2.0:1619677448.728416:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.728417:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.728418:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677448.728419:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.728420:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.728421:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:2.0:1619677448.728421:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.728422:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:2.0:1619677448.728422:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.728423:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:2.0:1619677448.728423:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.728424:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.728427:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.728427:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677448.728427:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.728430:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:24.0:1619677448.728431:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.728432:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.728433:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:24.0:1619677448.728434:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.728434:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:24.0:1619677448.728435:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.728436:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96711115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.728436:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.728436:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.728437:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:24.0:1619677448.728438:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.728438:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000400:00000200:24.0:1619677448.728439:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.728439:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:24.0:1619677448.728440:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.2:1619677448.728442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:24.0:1619677448.728444:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.728444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:24.0:1619677448.728446:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.728449:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.728449:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.728450:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677448.728450:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:24.0:1619677448.728451:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (24)++
00000800:00000200:18.0:1619677448.728452:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:24.0:1619677448.728453:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (25)--
00000800:00000200:24.0:1619677448.728453:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677448.728453:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.728454:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677448.728454:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (24)--
00000800:00000200:24.0:1619677448.728455:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:17.0:1619677448.728455:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.728455:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677448.728456:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.728457:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.728457:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.728459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.728462:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:4.2:1619677448.728462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.728464:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.728466:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.728466:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.728467:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.728468:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.728469:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677448.728470:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.0:1619677448.728471:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.728472:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.728473:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.2:1619677448.728492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677448.728499:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.728503:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677448.728503:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677448.728505:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.728508:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.728509:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.728510:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.728512:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.728513:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.728514:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.728517:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.728519:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.728522:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.728523:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.728523:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.728525:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.728525:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.728526:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677448.728526:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677448.728531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.728539:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.728542:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.728542:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.728543:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.728545:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.728547:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.728552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.728559:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.728562:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.728563:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.728565:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.728567:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.728568:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.728570:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.728571:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.728572:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.728573:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.728577:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.728579:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.728581:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.728582:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.728583:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.728585:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.728585:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.728586:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.728587:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.728591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.728599:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.728601:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.728602:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.728604:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.728606:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.728607:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.728680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.2:1619677448.728681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677448.728686:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.728689:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.728689:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.728690:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.728691:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.728692:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.728694:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.728694:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.728695:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:30.0:1619677448.728696:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.728697:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.728697:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711112] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.728699:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677448.728699:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.728700:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.728700:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.728701:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677448.728702:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.728703:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.728705:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677448.728705:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.728708:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:31.0:1619677448.728709:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.728711:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.728712:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677448.728712:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.728712:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:21.2:1619677448.728713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677448.728714:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.728714:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:31.0:1619677448.728715:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:31.0:1619677448.728715:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:17.0:1619677448.728715:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.728716:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677448.728717:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:17.0:1619677448.728717:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:31.0:1619677448.728718:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.728718:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677448.728719:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.728719:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.728720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.728721:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:5.2:1619677448.728722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677448.728724:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000800:00000200:25.2:1619677448.728724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:31.0:1619677448.728726:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.728726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:22.0:1619677448.728727:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:30.0:1619677448.728728:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:1.0:1619677448.728728:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:31.0:1619677448.728729:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.728730:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:22.0:1619677448.728730:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:10.0:1619677448.728730:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:1.0:1619677448.728730:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:31.0:1619677448.728731:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96711110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.728731:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.728731:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:1.0:1619677448.728731:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.728732:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:2.0:1619677448.728732:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:1.0:1619677448.728732:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677448.728733:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.728733:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.728733:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.728734:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.728735:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.728735:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:31.0:1619677448.728736:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.728736:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:10.0:1619677448.728736:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.728737:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:2.0:1619677448.728738:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.728740:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:31.0:1619677448.728741:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.728743:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.2:1619677448.728743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000400:00000200:2.0:1619677448.728744:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.728745:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:31.0:1619677448.728747:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.728747:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.728748:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:2.0:1619677448.728748:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.728749:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.728749:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000400:00000200:2.0:1619677448.728749:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.728750:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.728751:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:31.0:1619677448.728752:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:26.0:1619677448.728752:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.728752:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.728753:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:26.0:1619677448.728753:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.728754:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.728754:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:2.0:1619677448.728754:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.728755:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:31.0:1619677448.728755:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:2.0:1619677448.728756:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:9.2:1619677448.728757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.728759:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.728760:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)++
00000800:00000200:21.2:1619677448.728761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:2.0:1619677448.728761:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (23)++
00000800:00000200:2.0:1619677448.728763:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (24)--
00000800:00000200:2.0:1619677448.728764:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.728766:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000800:00000200:2.0:1619677448.728766:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.728767:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000800:00000200:2.0:1619677448.728767:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000400:00000200:18.0:1619677448.728768:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.728768:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:30.0:1619677448.728769:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:2.0:1619677448.728769:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (23)++
00000400:00000200:18.0:1619677448.728770:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.728770:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (24)--
00000400:00000200:2.0:1619677448.728770:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.728771:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:30.0:1619677448.728772:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:2.0:1619677448.728772:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.728773:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:21.2:1619677448.728773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.728774:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.728775:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.728775:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.728776:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.728777:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.728777:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.728777:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.728777:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.728778:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.728779:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.728779:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.728779:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677448.728898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.728904:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.728907:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.728909:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.728910:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.728912:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.728913:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.728914:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.728916:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.728916:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.728918:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.728921:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.728923:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.0:1619677448.728927:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.728928:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.728930:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.728931:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:5.0:1619677448.728931:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.728932:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.728932:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.728933:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:17.0:1619677448.728934:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.728934:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.728935:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.728935:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677448.728937:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.728938:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:25.2:1619677448.728939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:17.0:1619677448.728939:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.728941:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.728942:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.728943:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.728946:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.728946:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.728948:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677448.728949:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.728949:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.728950:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.728951:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.728951:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:13.0:1619677448.728951:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.728952:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.728953:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.728953:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.728954:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.728954:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.728954:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.728959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:8.2:1619677448.728959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677448.728965:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.728966:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:12.0:1619677448.728968:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.728968:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:12.0:1619677448.728969:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.728969:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:1.0:1619677448.728969:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.728970:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:12.0:1619677448.728970:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:1.0:1619677448.728971:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.728972:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:1.0:1619677448.728973:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.729067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.729074:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.729077:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.729077:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.729079:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.729082:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.729083:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.729085:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.729086:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.729087:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.729088:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.729091:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.729093:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.729095:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.729095:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.729096:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.729097:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.729098:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.729098:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.729099:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.729103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.729109:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.729111:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.729111:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.729112:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.729113:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.729114:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:21.2:1619677448.729178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.729183:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.729186:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.729187:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.729189:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.729192:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.729193:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.729194:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.729196:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.729197:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.729198:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.729201:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.729203:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.729206:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.729207:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.729207:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.729209:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.729210:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.729210:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.729211:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.729215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.729222:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.729225:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.729226:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.729227:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.729229:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.729230:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.729265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:5.2:1619677448.729271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.729272:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.729275:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.729276:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.729277:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.729277:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.729279:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:10.0:1619677448.729279:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.729280:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.729281:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.729282:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.729283:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.729284:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.729284:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.729286:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.729286:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.729287:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.729288:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.729289:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.729291:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.729292:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.729300:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.729301:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.729304:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.729304:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.729307:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.729308:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.729308:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.729309:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.729309:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.729310:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.729310:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.729311:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.729311:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.729312:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.729312:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677448.729313:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.729313:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.729313:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.729317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:25.2:1619677448.729321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677448.729325:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.729328:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.729329:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.729329:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:6.0:1619677448.729329:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.729331:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.729331:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.729332:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.729334:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:13.0:1619677448.729334:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.729336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:12.0:1619677448.729336:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.729337:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677448.729344:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.729347:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.729347:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.729348:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.729349:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.729349:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.729501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.729509:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.729513:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.729515:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.729517:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.729517:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.729519:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.729520:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.729522:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.729523:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.729524:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.729527:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.729528:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.729531:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.729532:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.729532:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.729532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677448.729534:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.729535:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.729536:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677448.729536:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677448.729538:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.729538:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.729538:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:25.2:1619677448.729539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.729540:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.729542:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.729543:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.729544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677448.729545:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.729546:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.729546:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:20.0:1619677448.729547:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.729548:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.729548:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:20.0:1619677448.729549:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:12.0:1619677448.729549:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.729551:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.729552:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:13.0:1619677448.729552:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:20.0:1619677448.729553:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.729554:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:30.0:1619677448.729555:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.729555:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.729555:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.729556:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677448.729558:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.729559:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.729559:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.0:1619677448.729561:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.729562:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.729562:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.729563:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.729567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.729574:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.729577:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.729578:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.729578:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:4.2:1619677448.729578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:22.0:1619677448.729580:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.729582:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:8.0:1619677448.729585:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.729588:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.729589:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.729591:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.729594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:8.0:1619677448.729594:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.729595:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.729596:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.729598:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.729599:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.729600:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:18.0:1619677448.729601:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:8.0:1619677448.729603:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.729604:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:8.0:1619677448.729605:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:2.0:1619677448.729605:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.729607:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.729608:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.729608:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.729609:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:18.0:1619677448.729610:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:8.0:1619677448.729610:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:18.0:1619677448.729611:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.729612:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.729612:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:18.0:1619677448.729613:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.729613:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.729613:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.729614:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677448.729615:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.729618:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:8.2:1619677448.729618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:18.0:1619677448.729620:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.729622:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.729623:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.729623:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:18.0:1619677448.729625:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677448.729625:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:18.0:1619677448.729626:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.729626:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.729627:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677448.729628:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.729630:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.729630:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.729631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:1.0:1619677448.729632:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.729634:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:6.0:1619677448.729638:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.2:1619677448.729639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:22.0:1619677448.729641:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.729641:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.729642:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.729644:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677448.729646:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:17.0:1619677448.729647:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.729650:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.729650:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.729650:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.729652:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.729652:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.729796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.729802:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.729804:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.729805:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.729807:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.729810:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.729811:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677448.729812:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.729814:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.729815:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.729816:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677448.729819:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.729820:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.729823:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.729824:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.729825:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:22.0:1619677448.729826:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:22.0:1619677448.729827:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.729828:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.729828:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:25.2:1619677448.729832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.729839:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.729842:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.729843:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.729845:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.729846:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.729848:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.729942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.729949:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.729951:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.729952:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.729954:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.729957:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.729959:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.729960:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.729961:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.729962:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.729963:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.729966:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.729968:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.729971:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.729972:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.729973:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.729974:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.729975:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.729976:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.729977:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.729977:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.729980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:4.2:1619677448.729983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.729987:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:10.0:1619677448.729990:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.729990:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.729991:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.729991:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.729992:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.729993:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.729993:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.729994:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677448.729997:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.730095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677448.730099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.0:1619677448.730101:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.730104:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.730106:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.730107:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.730107:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.730108:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.730110:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:5.0:1619677448.730110:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:5.0:1619677448.730112:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.730113:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.730113:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.730114:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.730115:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.730115:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.730115:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.730117:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.730119:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.730120:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.730120:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.730121:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.730122:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.730123:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:5.0:1619677448.730125:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.730126:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.730126:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.730126:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.730128:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677448.730128:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.730128:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.730129:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.730130:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677448.730131:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.730132:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.730132:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.730134:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.730134:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:25.2:1619677448.730134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.730135:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.730136:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.730139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:22.0:1619677448.730141:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.730144:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.730144:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.730145:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.730146:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:22.0:1619677448.730147:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.730148:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.730149:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.730149:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:30.0:1619677448.730151:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.730152:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.730153:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.730213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.730219:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.730222:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.730222:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.730225:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.730228:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.730229:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.730231:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.730232:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.730233:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.730234:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.730237:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.730239:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.730242:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.730243:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.730244:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.730246:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.730246:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.730247:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.730247:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.730252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.730259:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.730261:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.730262:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.730264:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.730265:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.730267:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.730430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.730433:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.730435:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.730438:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.730438:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.730440:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.730442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:20.0:1619677448.730442:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.730443:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.730444:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.730446:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.730447:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.730448:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:20.0:1619677448.730450:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.730451:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.730451:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:20.0:1619677448.730452:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.730453:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.730456:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.730456:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:6.0:1619677448.730456:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.730457:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:6.0:1619677448.730457:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:20.0:1619677448.730459:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:6.0:1619677448.730459:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.730460:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.730460:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000400:00000200:6.0:1619677448.730460:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.730461:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:6.0:1619677448.730461:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.730462:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.730465:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.730466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.730467:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.730470:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.730470:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.730471:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.730473:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:6.0:1619677448.730473:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.730473:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.730474:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.730475:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.730476:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:31.0:1619677448.730477:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.730477:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.730479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:31.0:1619677448.730480:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.730481:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:21.2:1619677448.730481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677448.730482:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.730483:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.730483:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)++
00000800:00000200:31.0:1619677448.730484:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.730484:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.730485:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:22.0:1619677448.730486:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:2.0:1619677448.730487:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:22.0:1619677448.730488:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.730489:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:2.0:1619677448.730489:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:6.0:1619677448.730490:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:2.0:1619677448.730490:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.730491:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.730492:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.730495:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.730498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:18.0:1619677448.730505:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.730508:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.730509:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.730509:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.730510:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.730510:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.730624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.730632:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.730634:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.730635:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.730637:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.730640:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.730642:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.730643:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.730644:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.730645:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.730646:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.730650:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.730652:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.730655:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.730655:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.730656:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.730658:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.730658:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.730659:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.730659:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.730664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.730671:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.730674:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.730675:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.730675:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.730678:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.730680:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.730720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.730727:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.730730:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.730731:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.730733:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.730736:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.730737:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.730738:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.730739:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.730740:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.730741:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.730745:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.730746:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.730749:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.730750:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.730750:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.730752:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.730752:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.730753:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.730753:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.730758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.730766:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.730769:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.730770:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.730770:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.730772:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.730775:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.730948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:21.2:1619677448.730950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.730955:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.730956:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.730957:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.730958:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677448.730960:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.730961:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.730962:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.730963:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.730963:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677448.730965:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711089] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.730966:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.730968:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.730969:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677448.730972:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000400:00000200:18.0:1619677448.730974:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:2.0:1619677448.730974:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:18.0:1619677448.730975:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.730977:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.730978:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.730979:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.730981:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.730982:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000400:00000200:18.0:1619677448.730983:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.730983:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.730985:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.730985:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.730985:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:20.2:1619677448.730986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677448.730987:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (23)++
00000800:00000200:2.0:1619677448.730988:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (24)--
00000800:00000200:18.0:1619677448.730989:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.730989:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.730990:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:9.2:1619677448.730990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.730990:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.730991:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.730991:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.2:1619677448.730993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.730993:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.730993:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.730993:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.730993:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:31.0:1619677448.730994:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677448.730994:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.730994:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.730995:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:2.0:1619677448.730996:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:31.0:1619677448.730997:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:18.0:1619677448.730997:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.730997:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:18.0:1619677448.730998:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:31.0:1619677448.730999:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.730999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.730999:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677448.731000:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:31.0:1619677448.731001:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.731001:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:18.0:1619677448.731001:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:31.0:1619677448.731003:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.731004:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.731005:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.731006:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.731006:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:31.0:1619677448.731007:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.731009:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:31.0:1619677448.731010:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677448.731010:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.731011:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.731012:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.731013:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.731014:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.731015:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.731016:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.731017:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.731018:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.731019:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.731020:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.731020:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.731024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.731030:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.731032:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.731033:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.731035:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.731036:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.731037:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.731131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:5.2:1619677448.731136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.731137:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.731140:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.731141:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677448.731142:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.731143:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.731144:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.731145:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.731146:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.731147:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.731148:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.731149:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.731150:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.731150:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.731152:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.731152:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677448.731154:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677448.731154:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.731155:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.731157:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.731158:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677448.731161:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.731163:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:1.0:1619677448.731163:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.731165:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.731166:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.731167:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.731168:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677448.731168:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.731169:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.731170:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.731170:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677448.731170:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:10.0:1619677448.731171:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.731172:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677448.731172:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.731172:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.731174:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.731175:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.731175:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.731176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:25.2:1619677448.731180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.731180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677448.731183:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:17.0:1619677448.731186:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677448.731186:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:6.0:1619677448.731186:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.731187:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.731187:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.731188:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.731188:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.731189:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.731189:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.731190:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.731190:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:12.0:1619677448.731192:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.731192:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.731193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:13.0:1619677448.731193:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:25.2:1619677448.731195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:12.0:1619677448.731195:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.731200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:22.0:1619677448.731201:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677448.731203:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677448.731204:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.731205:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677448.731206:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.731206:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.731207:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.731208:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.731209:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.731209:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.731209:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.731209:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:13.0:1619677448.731210:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.731210:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:6.0:1619677448.731210:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:22.0:1619677448.731211:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:13.0:1619677448.731211:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.731212:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:22.0:1619677448.731213:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.731214:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.731215:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.731216:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677448.731220:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.731222:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677448.731224:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.731225:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:22.0:1619677448.731226:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:22.0:1619677448.731227:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:22.0:1619677448.731228:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677448.731229:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677448.731230:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:8.2:1619677448.731233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.731241:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.731243:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.731244:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.731244:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.731247:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.731249:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.731379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.731387:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.731390:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.731392:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.731394:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.731394:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.731396:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.731397:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.731398:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.731400:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.731401:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.731404:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.731406:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.731409:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.731409:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.731410:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.731412:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.731412:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.731413:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.731414:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.731418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.731426:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.731428:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.731429:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.731430:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.731432:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.731433:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.731481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.731489:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.731492:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.731493:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.731495:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.731498:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.731499:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.731500:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.731501:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.731502:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.731503:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.731506:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.731508:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.731510:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.731510:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.731511:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.731512:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:5.2:1619677448.731512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:26.0:1619677448.731513:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.731513:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.731514:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:4.2:1619677448.731516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677448.731518:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:25.2:1619677448.731519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.731521:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.731522:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.731524:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.731525:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.731526:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.731527:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.731527:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:2.0:1619677448.731527:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.731528:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.731529:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.731530:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:10.0:1619677448.731530:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.731530:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.731532:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:2.0:1619677448.731532:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.731533:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.731533:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.731533:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.731534:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:2.0:1619677448.731534:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677448.731535:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.731537:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.731538:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.731538:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.731539:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.731540:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.731540:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.731543:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.731544:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.731544:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:10.0:1619677448.731545:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.731546:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:10.0:1619677448.731547:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677448.731547:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.731547:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.731548:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:10.0:1619677448.731550:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.731551:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.731551:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.731553:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.2:1619677448.731553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:10.0:1619677448.731554:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.731554:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.731555:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.731559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.731561:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.731564:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.731565:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.731566:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.731567:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:6.0:1619677448.731567:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.731570:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.0:1619677448.731570:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677448.731571:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.731572:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.731574:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.731577:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.731634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.731642:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.731645:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.731648:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.731649:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.731650:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.731651:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.731653:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.731654:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.731655:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.731656:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.731659:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.731661:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.731663:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.731664:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.731664:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.731666:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.731666:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.731668:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.731669:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.731670:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.731671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.731676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.731678:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.731680:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.731681:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.731682:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.731683:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:12.0:1619677448.731684:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.731685:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.731685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677448.731686:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.731687:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.731687:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.0:1619677448.731692:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.2:1619677448.731693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.731695:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.731696:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.731698:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.731700:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:8.0:1619677448.731700:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.731702:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677448.731702:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:12.0:1619677448.731703:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:8.0:1619677448.731703:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.731704:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.731705:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:8.0:1619677448.731705:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.731705:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.731706:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:8.0:1619677448.731706:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.731710:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.731711:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.731714:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.731715:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.731715:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.731717:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.731717:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.731718:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.731718:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.731723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.731730:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.731733:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.731734:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.731734:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.731736:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.731739:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.731739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.731746:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.731749:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.731750:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.731750:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.731751:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.731752:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.731764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.731767:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.731769:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.731771:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.731772:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.731774:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.731776:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.731778:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.731779:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.731780:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.731781:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.731784:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.731786:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.731790:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.731790:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.731791:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.731793:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:20.0:1619677448.731794:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.731795:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.731796:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.731797:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.731799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.731802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677448.731805:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.731807:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:13.0:1619677448.731808:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.731809:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.731809:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.731810:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:13.0:1619677448.731811:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.731811:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.731812:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.731813:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.731851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.731857:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.731860:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.731860:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.731863:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.731864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:22.0:1619677448.731865:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.731866:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.731868:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.731869:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677448.731870:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.731871:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.731872:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677448.731872:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.731873:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677448.731876:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.731876:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.731879:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:17.0:1619677448.731879:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.731880:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:22.0:1619677448.731881:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.731882:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:17.0:1619677448.731882:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.731883:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:17.0:1619677448.731883:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.731884:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.731885:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.731885:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.731885:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:17.0:1619677448.731885:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:22.0:1619677448.731886:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:17.0:1619677448.731888:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.731890:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.2:1619677448.731891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:17.0:1619677448.731894:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.731894:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.731895:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.731897:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.731897:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.731898:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.731898:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:17.0:1619677448.731899:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.0:1619677448.731901:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:25.2:1619677448.731902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:5.0:1619677448.731902:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.731904:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.731904:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.731906:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:12.0:1619677448.731909:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:4.2:1619677448.731909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:12.0:1619677448.731912:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.731913:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.731915:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.731916:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677448.731916:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.731917:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.731919:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.731920:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.731922:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.731925:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.731926:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.731927:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.731929:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.731930:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.731931:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.731934:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.731936:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.731939:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.731940:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.731940:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.731942:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.731943:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.731944:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.731945:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.731945:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.731948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:4.2:1619677448.731951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677448.731956:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.731958:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.731958:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.731959:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.731960:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677448.731960:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.731961:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.731962:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.731962:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.731965:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.732014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.732022:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.732025:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.732027:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.732029:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.732029:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.732031:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.732032:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.732034:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.732035:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.732036:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.732039:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.732040:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.732043:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.732044:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.732045:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.732047:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.732047:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.732048:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.732049:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.732052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.0:1619677448.732060:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.732062:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.732063:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.732065:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.732066:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.732068:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.732077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.732084:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.732087:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.732087:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.732090:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.732092:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.732093:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677448.732095:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.732096:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.732097:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.732098:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677448.732101:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.732103:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677448.732106:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.732106:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:18.0:1619677448.732107:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:18.0:1619677448.732109:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:18.0:1619677448.732109:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.732110:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.732110:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.732115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.732122:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.732125:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.732126:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.732127:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.732128:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.732130:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.732186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677448.732192:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.732194:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.732195:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.732197:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.732199:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.732201:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677448.732202:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.732203:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.732204:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.732205:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677448.732208:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.732209:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.732212:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.732213:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.732213:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.732215:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:2.0:1619677448.732215:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.732216:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677448.732216:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.732221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.732227:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.732230:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.732230:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.732231:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.732233:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.732236:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.732250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.732257:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.732259:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.732260:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.732262:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.732264:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.732266:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.732267:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.732268:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.732269:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.732270:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.732274:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.732275:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.732279:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.732280:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.732280:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.732282:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.732282:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.732283:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.732283:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.2:1619677448.732288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:8.2:1619677448.732288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.732298:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677448.732298:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677448.732300:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:20.0:1619677448.732301:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:10.0:1619677448.732301:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.732302:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.732302:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.732304:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.732305:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:26.0:1619677448.732306:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.732307:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:26.0:1619677448.732309:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.732310:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.732311:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.732313:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.732313:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.732317:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.732318:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.732321:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.732322:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.732322:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.732324:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.732324:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.732326:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677448.732327:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.732328:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677448.732331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677448.732333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:1.0:1619677448.732339:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:10.0:1619677448.732341:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:1.0:1619677448.732341:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.732342:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.732343:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.732343:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:1.0:1619677448.732343:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.732345:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:1.0:1619677448.732345:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.732348:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.732372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.732381:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.732384:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.732386:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.732388:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.732389:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.732390:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.732392:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.732393:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.732394:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.732395:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.732397:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.732399:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.732401:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.732402:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.732402:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.732404:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.732405:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.732406:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.732406:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.732410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.732419:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.732421:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.732422:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.732423:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.732425:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.732426:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.732452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.732458:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.732461:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:5.2:1619677448.732461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677448.732461:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.732463:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.732465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:17.0:1619677448.732466:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.732467:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:26.0:1619677448.732468:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677448.732469:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.732470:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.732470:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.732471:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:17.0:1619677448.732471:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.732472:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.732472:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.0:1619677448.732473:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:26.0:1619677448.732475:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.732476:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.732477:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.732478:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.732478:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677448.732480:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:17.0:1619677448.732480:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:5.0:1619677448.732480:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.732481:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.732482:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.732483:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.732484:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677448.732485:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.732485:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:5.0:1619677448.732485:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.732486:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.732486:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:5.0:1619677448.732486:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.732488:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.732488:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.732488:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677448.732488:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:17.0:1619677448.732489:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.732489:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677448.732493:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.732493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:5.0:1619677448.732494:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.732496:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.732496:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.732499:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.732499:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.732500:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:13.0:1619677448.732500:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677448.732500:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.732501:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.732501:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677448.732502:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:26.0:1619677448.732503:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:13.0:1619677448.732503:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.732503:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.732504:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.0:1619677448.732504:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677448.732505:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:13.0:1619677448.732505:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.732505:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677448.732506:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:12.0:1619677448.732506:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677448.732506:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677448.732507:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:13.0:1619677448.732507:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.2:1619677448.732508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:9.2:1619677448.732509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.732512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677448.732516:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:18.0:1619677448.732517:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677448.732518:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:18.0:1619677448.732520:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:17.0:1619677448.732520:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:10.0:1619677448.732520:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:1.0:1619677448.732520:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.732521:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.732521:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.732522:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:1.0:1619677448.732522:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.732523:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.732524:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:18.0:1619677448.732524:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.732525:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:18.0:1619677448.732527:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:8.2:1619677448.732527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.732533:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.732536:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.732537:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.732537:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.732539:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.732539:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.732577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.732585:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.732588:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.732590:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.732591:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.732592:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.732594:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677448.732595:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.732597:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.732598:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.732599:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677448.732602:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.732604:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677448.732607:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.732607:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.732608:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:30.0:1619677448.732609:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:30.0:1619677448.732610:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.732611:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.732612:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.732615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.732621:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.732623:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.732624:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.732624:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.732626:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.732629:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.732632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.732638:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.732641:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.732641:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.732641:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.732642:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.732643:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:4.2:1619677448.732704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.732710:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.732713:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.732714:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.732715:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.732718:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.732719:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.732721:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.732722:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.732723:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.732724:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.732727:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.732729:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.732731:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.732732:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.732733:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.732734:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.732735:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.732735:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.732735:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.732740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.732747:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.732750:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.732750:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.732752:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.732754:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.732755:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.732810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.732817:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.732820:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.732821:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.732823:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.732826:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.732826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:24.0:1619677448.732827:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.732828:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.732829:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.732830:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.732831:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.732834:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:24.0:1619677448.732834:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.732835:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.732836:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.732837:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.732839:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677448.732839:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.732839:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.732840:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:26.0:1619677448.732841:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.732842:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.732842:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:26.0:1619677448.732843:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:24.0:1619677448.732843:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000400:00000200:26.0:1619677448.732844:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.732844:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000400:00000200:26.0:1619677448.732845:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677448.732845:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:26.0:1619677448.732846:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.732847:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.2:1619677448.732848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:26.0:1619677448.732850:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677448.732850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:26.0:1619677448.732852:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.732854:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.732855:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.732855:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677448.732856:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.732857:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677448.732858:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.732858:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:20.0:1619677448.732858:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.732858:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:26.0:1619677448.732859:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.732860:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:20.0:1619677448.732860:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.732860:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.732861:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:5.0:1619677448.732861:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.732863:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.0:1619677448.732863:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.732864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.732867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:8.2:1619677448.732873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000800:00000200:2.0:1619677448.732873:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.732875:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.732875:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.732876:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.732876:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.732878:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:18.0:1619677448.732879:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.732881:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000400:00000200:18.0:1619677448.732882:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.732882:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.732883:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:2.0:1619677448.732883:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677448.732884:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677448.732884:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.732885:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.732885:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.732885:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.732886:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.732887:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:18.0:1619677448.732887:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.732888:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.732889:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.732892:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.732894:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.732896:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.732897:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.732897:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.732898:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.732899:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.732900:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.732900:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.732905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.732912:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.732914:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.732914:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.732915:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.732916:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.732918:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.732921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.732928:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.732931:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.732931:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.732932:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.732932:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.732933:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.732938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.732946:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.732949:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.732951:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.732952:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.732954:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.732955:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.732956:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.732958:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.732958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677448.732959:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.732960:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.732963:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.732963:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677448.732965:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677448.732966:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.732967:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677448.732969:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.732969:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.732970:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.732970:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.732972:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:1.0:1619677448.732972:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.732973:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:1.0:1619677448.732973:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:30.0:1619677448.732974:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.732974:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:1.0:1619677448.732974:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.732976:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.732977:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.732978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:1.0:1619677448.732978:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677448.732981:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.732983:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.732985:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.732986:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:1.0:1619677448.732986:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:1.0:1619677448.732987:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:5.0:1619677448.732988:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:1.0:1619677448.732988:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:5.0:1619677448.732989:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.732989:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.732989:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.732990:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677448.732991:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.732991:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.732994:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:25.2:1619677448.732995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:8.2:1619677448.732995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.733002:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:20.0:1619677448.733003:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.733005:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.733006:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.733006:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:6.0:1619677448.733006:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:20.0:1619677448.733007:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.733007:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000400:00000200:22.0:1619677448.733008:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.733008:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.733009:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:22.0:1619677448.733011:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.733093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.733100:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.733103:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.733104:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.733106:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.733108:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.733110:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.733111:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.733112:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.733113:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.733115:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.733118:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.733119:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.733121:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.733122:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.733123:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.733124:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.733125:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.733125:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.733126:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.733130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.733137:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.733139:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.733140:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.733142:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.733143:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.733144:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.733180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.733183:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.733185:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.733187:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.733188:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.733190:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.733191:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.733192:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.733194:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.733195:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.733196:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.733200:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.733201:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.733204:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.733205:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.733206:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.733208:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.733208:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.733209:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.733210:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.733213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.733220:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.733223:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.733224:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.733226:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.733226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677448.733227:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.733229:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.733234:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.733236:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.733237:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.733239:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.733241:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.733243:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.733244:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.733245:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.733246:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.733247:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.733251:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.733252:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.733255:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.733256:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.733256:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.733258:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.733258:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.733259:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.733260:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677448.733260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:25.2:1619677448.733264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.733267:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.733269:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.733271:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.733271:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.733273:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:31.0:1619677448.733274:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.733274:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:12.0:1619677448.733274:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.733276:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677448.733276:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.733277:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.733277:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:31.0:1619677448.733278:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677448.733278:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:31.0:1619677448.733279:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.733280:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.733283:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.733283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.733285:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.733288:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.733289:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:31.0:1619677448.733289:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677448.733289:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.733291:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.733291:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.733292:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677448.733292:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.733293:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677448.733298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:13.0:1619677448.733299:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.733300:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.733300:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.733301:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:17.0:1619677448.733307:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.733309:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.733310:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.733311:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.733313:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.733316:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.733335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.733340:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.733343:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.733343:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.733345:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.733348:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.733349:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.733350:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.733351:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.733352:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.733353:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.733356:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.733358:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.733360:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.733361:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.733361:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.733363:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.733363:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.733365:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.733366:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.733366:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.733369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.733372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.733378:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.733379:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.733379:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.733379:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.733379:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.733380:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.733381:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.733381:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.733382:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677448.733383:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.733502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.733508:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.733511:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677448.733513:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.733513:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:5.0:1619677448.733516:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.733517:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.733518:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.733520:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.733521:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.733522:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.733525:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.733526:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.733530:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.733530:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.733531:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.733533:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.733533:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.733534:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.733536:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.733536:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677448.733539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.733541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677448.733544:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.733546:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:12.0:1619677448.733546:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.733547:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.733548:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:12.0:1619677448.733548:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.733549:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.733551:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677448.733552:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.733554:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.733596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:5.2:1619677448.733600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:24.0:1619677448.733604:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.733606:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.733607:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:10.0:1619677448.733608:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:24.0:1619677448.733609:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.733610:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.733611:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:24.0:1619677448.733612:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.733613:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.733613:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.733614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:24.0:1619677448.733615:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.733616:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.733616:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.733617:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.733617:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.733619:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.733619:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.733620:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:10.0:1619677448.733621:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.733621:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.733622:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.733623:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677448.733623:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.733624:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.733627:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:6.0:1619677448.733627:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.733629:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.733629:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.733630:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.733630:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:24.0:1619677448.733631:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.733631:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.733631:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:6.0:1619677448.733632:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96711050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.733633:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.733633:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:6.0:1619677448.733633:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677448.733633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1200] (20)++
00000800:00000200:24.0:1619677448.733634:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:10.0:1619677448.733634:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677448.733634:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.733635:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:10.0:1619677448.733635:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000400:00000200:6.0:1619677448.733635:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677448.733636:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.733638:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:1.0:1619677448.733638:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:8.2:1619677448.733639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:6.0:1619677448.733639:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.733640:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.733641:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.733641:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677448.733641:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677448.733641:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:17.0:1619677448.733642:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:9.2:1619677448.733644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:6.0:1619677448.733644:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.733644:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.733645:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.733645:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:20.0:1619677448.733646:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:1.0:1619677448.733646:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.733647:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:6.0:1619677448.733648:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:1.0:1619677448.733648:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:20.0:1619677448.733649:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.733649:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.733649:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.733649:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:1.0:1619677448.733649:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96711049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.733650:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.733651:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.733651:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:1.0:1619677448.733651:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.733652:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:1.0:1619677448.733652:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:25.2:1619677448.733653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677448.733654:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.733655:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.733655:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.733656:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.733657:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.733658:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:1.0:1619677448.733658:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 600 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677448.733659:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:2.0:1619677448.733661:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677448.733661:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.733662:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:1.0:1619677448.733662:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000400:00000200:26.0:1619677448.733663:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.733663:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:1.0:1619677448.733663:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000400:00000200:26.0:1619677448.733665:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.733665:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:1.0:1619677448.733665:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:1.0:1619677448.733666:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:1.0:1619677448.733666:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:26.0:1619677448.733669:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:8.2:1619677448.733671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:25.2:1619677448.733673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000800:00000200:5.0:1619677448.733678:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:10.0:1619677448.733679:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:5.0:1619677448.733681:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:26.0:1619677448.733682:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.733682:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000400:00000200:5.0:1619677448.733682:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.733683:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000400:00000200:5.0:1619677448.733683:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.733684:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:10.0:1619677448.733684:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.733685:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:5.0:1619677448.733686:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:21.2:1619677448.733732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.733741:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.733743:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.733746:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.733746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677448.733747:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.733748:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.733750:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.733751:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.733752:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677448.733753:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.733754:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.733754:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:30.0:1619677448.733755:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.733755:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.733758:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.733760:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677448.733761:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.733762:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:17.0:1619677448.733763:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.733764:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.733765:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.733766:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.733766:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:17.0:1619677448.733766:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.733767:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:17.0:1619677448.733767:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:30.0:1619677448.733769:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.733769:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.733771:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:17.0:1619677448.733771:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.733772:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.733773:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677448.733773:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:25.2:1619677448.733775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.733775:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.733776:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.733777:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:21.2:1619677448.733778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677448.733778:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.733779:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.733779:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.733780:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.733782:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:25.2:1619677448.733784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.733785:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:31.0:1619677448.733786:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:22.0:1619677448.733786:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.733788:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:22.0:1619677448.733788:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.733788:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.733789:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677448.733789:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:31.0:1619677448.733790:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677448.733791:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:6.0:1619677448.733791:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.733792:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677448.733793:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.733793:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.733795:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.733803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000800:00000200:22.0:1619677448.733810:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.733813:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.733814:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.733814:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.733815:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:4.2:1619677448.733815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677448.733816:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.0:1619677448.733821:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.733824:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.733825:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.733827:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.733830:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.733832:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.733833:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.733834:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.733835:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.733836:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.733840:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.733841:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.733844:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.733845:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.733846:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.733847:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.733848:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.733849:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.733850:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.733850:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677448.733853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.733856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.733861:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:24.0:1619677448.733863:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677448.733863:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.733864:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.733864:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.733865:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677448.733866:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.733867:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.733867:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.733870:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.733956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.733959:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.733961:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.733963:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.733964:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.733966:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.733967:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677448.733969:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.733970:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.733971:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.733972:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677448.733976:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.733978:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677448.733980:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.733981:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:20.0:1619677448.733982:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.0:1619677448.733984:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.733984:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.733985:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.733986:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.733990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.733998:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.734000:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.734001:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.734001:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.734004:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.734006:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.734049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.734055:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.734058:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.734058:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.734060:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.734062:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.734064:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.734065:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.734066:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.734067:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.734068:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677448.734071:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.734073:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.734076:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.734077:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.734077:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.734079:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.734080:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.734080:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.734081:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.734085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.734092:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.734095:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.734096:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.734097:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.734099:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.734100:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.734349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.734357:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.734359:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.734360:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.734362:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.734365:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.734366:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.734368:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.734369:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.734370:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.734371:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.734374:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.734376:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.734378:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.734379:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.734380:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.734381:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.734382:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.734383:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.734384:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.734384:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.734387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.734390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.734395:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.734397:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.734398:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.734398:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:12.0:1619677448.734399:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.734399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.734400:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677448.734401:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:12.0:1619677448.734402:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.734402:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677448.734403:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.734405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.734405:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.734408:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.734409:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.2:1619677448.734410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:1.0:1619677448.734411:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.734412:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677448.734413:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.734414:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.734415:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:5.0:1619677448.734415:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677448.734415:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:12.0:1619677448.734416:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:1.0:1619677448.734416:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.734417:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.734418:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.734418:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:1.0:1619677448.734418:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.734419:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.734420:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:20.0:1619677448.734421:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.734421:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.734424:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.734424:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.734426:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:1.0:1619677448.734426:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:5.0:1619677448.734428:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.734428:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.734429:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000400:00000200:5.0:1619677448.734430:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.734430:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.734430:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.734431:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.734431:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677448.734432:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.734432:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.734433:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:5.0:1619677448.734435:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.734437:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.2:1619677448.734438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.734439:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.734440:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.734440:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:5.0:1619677448.734442:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.734443:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.734444:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:17.0:1619677448.734445:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:5.0:1619677448.734445:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.734446:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677448.734447:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:25.2:1619677448.734448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:17.0:1619677448.734448:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.734449:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:20.2:1619677448.734450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:17.0:1619677448.734450:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.734452:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.0:1619677448.734453:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.734455:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.734456:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677448.734456:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:20.0:1619677448.734457:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677448.734458:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.734459:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.734460:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.734461:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.734463:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.734467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:22.0:1619677448.734474:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.734477:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.734477:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.734478:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.734479:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.734480:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.734512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.734520:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.734523:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.734525:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.734528:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.734529:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.734530:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.734532:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.734533:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.734534:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.734535:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:5.2:1619677448.734537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:31.0:1619677448.734538:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.734540:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.734543:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.734543:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.734543:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677448.734544:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.734546:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.734546:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.734546:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:31.0:1619677448.734547:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677448.734547:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677448.734548:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:2.0:1619677448.734549:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.734551:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677448.734552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:2.0:1619677448.734553:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.734555:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.734556:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.734558:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.734559:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.734560:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.734562:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.734563:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.734563:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.734565:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.734565:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.2:1619677448.734567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708000] (20)++
00000800:00000200:18.0:1619677448.734569:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.734569:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.734570:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708000] (21)++
00000800:00000200:18.0:1619677448.734570:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677448.734570:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.734570:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.734572:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.4.67@o2ib3
00000800:00000200:2.0:1619677448.734572:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.734573:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.734574:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000400:00000200:20.0:1619677448.734575:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.67@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.734575:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:5.0:1619677448.734576:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (22)--
00000800:00000200:2.0:1619677448.734576:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.0:1619677448.734577:0:19934:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 4872
00000800:00000200:8.2:1619677448.734578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.734579:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708000] (21)++
00000800:00000200:20.0:1619677448.734581:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708000] (22)++
00000800:00000200:20.0:1619677448.734581:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708000] (23)--
00000800:00000200:5.2:1619677448.734581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.0:1619677448.734582:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (22)--
00000800:00000200:20.2:1619677448.734586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708000] (21)++
00000800:00000200:1.0:1619677448.734586:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677448.734587:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:1.0:1619677448.734588:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677448.734589:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000400:00000200:1.0:1619677448.734589:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.734590:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677448.734590:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.734591:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:1.0:1619677448.734591:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.734592:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708000] (22)++
00000800:00000200:5.0:1619677448.734594:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (23)--
00000800:00000200:1.0:1619677448.734594:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677448.734595:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708000] (22)++
00000800:00000200:20.0:1619677448.734597:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (23)--
00000800:00000200:5.0:1619677448.734597:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.4.67@o2ib3
00000800:00000200:5.0:1619677448.734598:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708000] (22)--
00000400:00000200:5.0:1619677448.734599:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.734601:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:5.0:1619677448.734603:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.734605:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.734606:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.734607:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:5.0:1619677448.734610:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.67@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.734612:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 4872 bytes in 256 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:5.0:1619677448.734613:0:19935:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 4872
00000800:00000200:5.0:1619677448.734616:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.734617:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677448.734617:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.0:1619677448.734619:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677448.734620:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708000] (21)++
00000800:00000200:5.0:1619677448.734621:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708000] (22)--
00000800:00000200:5.0:1619677448.734621:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (21)--
00000800:00000200:9.2:1619677448.734625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.734632:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.734634:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.734635:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:9.2:1619677448.734646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.734652:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.734655:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[1] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677448.734655:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.734657:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.734659:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.734659:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:9.2:1619677448.734667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.2:1619677448.734673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708000] (20)++
00000800:00000200:8.0:1619677448.734673:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.734675:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.734676:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677448.734676:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708000] (21)++
00000400:00000200:8.0:1619677448.734676:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.734678:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.4.67@o2ib3
00000400:00000200:8.0:1619677448.734679:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.734680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:20.0:1619677448.734681:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.67@o2ib3 : REPLY - routed
00000800:00000200:8.0:1619677448.734682:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.0:1619677448.734683:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (22)--
00000800:00000200:20.0:1619677448.734685:0:19934:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 40528
00000800:00000200:20.0:1619677448.734686:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708000] (21)++
00000800:00000200:18.0:1619677448.734686:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.0:1619677448.734688:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708000] (22)++
00000800:00000200:20.0:1619677448.734688:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708000] (23)--
00000800:00000200:20.0:1619677448.734689:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (22)--
00000800:00000200:18.0:1619677448.734689:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.734689:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.734692:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:20.2:1619677448.734694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708000] (21)++
00000400:00000200:18.0:1619677448.734694:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.734695:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.734696:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.734697:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.734699:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.734699:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708000] (22)++
00000400:00000200:18.0:1619677448.734700:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.734701:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (23)--
00000400:00000200:18.0:1619677448.734702:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.734702:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (22)--
00000800:00000200:18.0:1619677448.734704:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.2:1619677448.734705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708000] (21)++
00000800:00000200:18.0:1619677448.734705:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.734706:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.734707:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.734708:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.734709:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.734710:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.734710:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.0:1619677448.734711:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708000] (22)++
00000800:00000200:5.0:1619677448.734713:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.4.67@o2ib3
00000800:00000200:9.2:1619677448.734714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.734714:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708000] (23)--
00000400:00000200:5.0:1619677448.734715:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.734716:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (22)--
00000400:00000200:5.0:1619677448.734717:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:5.0:1619677448.734718:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.734719:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.734720:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.734721:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:2.0:1619677448.734721:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.734723:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.734724:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.734725:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:5.0:1619677448.734725:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.67@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : REPLY try# 0
00000400:00000200:2.0:1619677448.734726:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:5.0:1619677448.734727:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 40528 bytes in 256 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:2.0:1619677448.734728:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.0:1619677448.734729:0:19935:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 40528
00000800:00000200:5.0:1619677448.734732:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.734732:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:5.0:1619677448.734733:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:5.0:1619677448.734735:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:5.0:1619677448.734736:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708000] (21)++
00000800:00000200:5.0:1619677448.734736:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708000] (22)--
00000800:00000200:5.0:1619677448.734737:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (21)--
00000800:00000200:9.2:1619677448.734741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:24.0:1619677448.734747:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:24.0:1619677448.734750:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.734750:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:20.2:1619677448.734759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708000] (20)++
00000800:00000200:20.0:1619677448.734762:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708000] (21)++
00000800:00000200:9.2:1619677448.734763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:20.0:1619677448.734765:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.4.67@o2ib3
00000400:00000200:20.0:1619677448.734767:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.67@o2ib3 : REPLY - routed
00000800:00000200:5.0:1619677448.734767:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (22)--
00000800:00000200:20.0:1619677448.734769:0:19934:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 40530
00000800:00000200:8.0:1619677448.734769:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:20.0:1619677448.734770:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708000] (21)++
00000800:00000200:20.0:1619677448.734771:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708000] (22)++
00000800:00000200:24.0:1619677448.734772:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (23)--
00000800:00000200:20.0:1619677448.734772:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708000] (23)--
00000800:00000200:8.0:1619677448.734772:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[2] from 10.0.10.109@o2ib7
00000800:00000200:20.0:1619677448.734773:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (22)--
00000800:00000200:8.0:1619677448.734774:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.734776:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (23)--
00000800:00000200:20.2:1619677448.734777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708000] (21)++
00000800:00000200:8.0:1619677448.734777:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:5.0:1619677448.734782:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708000] (22)++
00000800:00000200:5.0:1619677448.734784:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (23)--
00000800:00000200:5.0:1619677448.734785:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (22)--
00000800:00000200:20.2:1619677448.734789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708000] (21)++
00000800:00000200:9.2:1619677448.734793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.0:1619677448.734794:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708000] (22)++
00000800:00000200:5.0:1619677448.734796:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.4.67@o2ib3
00000800:00000200:5.0:1619677448.734797:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708000] (23)--
00000400:00000200:5.0:1619677448.734798:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.734799:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:20.0:1619677448.734799:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (22)--
00000400:00000200:5.0:1619677448.734799:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:5.0:1619677448.734801:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.734802:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:5.0:1619677448.734802:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.734803:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.734803:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:5.0:1619677448.734803:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.734804:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677448.734806:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: REPLY: OK
00000400:00000200:5.0:1619677448.734808:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.67@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : REPLY try# 0
00000800:00000200:5.0:1619677448.734810:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 40530 bytes in 256 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.734812:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:5.0:1619677448.734813:0:19935:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 40530
00000800:00000200:5.0:1619677448.734816:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677448.734817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:5.0:1619677448.734817:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677448.734818:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.0:1619677448.734819:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:5.0:1619677448.734820:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708000] (21)++
00000800:00000200:5.0:1619677448.734821:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708000] (22)--
00000800:00000200:6.0:1619677448.734822:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.0:1619677448.734822:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (21)--
00000800:00000200:9.2:1619677448.734825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:6.0:1619677448.734825:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.734826:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.734828:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.734830:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.734831:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:6.0:1619677448.734832:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.734833:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.734834:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.734834:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:6.0:1619677448.734835:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.734836:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.734837:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.2:1619677448.734839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708000] (20)++
00000400:00000200:6.0:1619677448.734840:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677448.734841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.0:1619677448.734842:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708000] (21)++
00000800:00000200:6.0:1619677448.734842:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.734844:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.4.67@o2ib3
00000800:00000200:6.0:1619677448.734845:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.734846:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.67@o2ib3 : REPLY - routed
00000800:00000200:6.0:1619677448.734846:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:24.0:1619677448.734847:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:6.0:1619677448.734847:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.734847:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (22)--
00000800:00000200:6.0:1619677448.734849:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.734849:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:24.0:1619677448.734850:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[1] from 10.0.10.101@o2ib7
00000800:00000200:20.0:1619677448.734850:0:19934:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 40529
00000800:00000200:6.0:1619677448.734850:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.734850:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677448.734851:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708000] (21)++
00000800:00000200:8.0:1619677448.734851:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.734853:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:20.0:1619677448.734853:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708000] (22)++
00000800:00000200:20.0:1619677448.734854:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708000] (23)--
00000800:00000200:20.0:1619677448.734854:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (22)--
00000800:00000200:25.2:1619677448.734855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:24.0:1619677448.734855:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.734856:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.2:1619677448.734858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708000] (21)++
00000800:00000200:30.0:1619677448.734862:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.734865:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:5.2:1619677448.734865:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677448.734866:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.734867:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708000] (22)++
00000400:00000200:30.0:1619677448.734868:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.734868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.734869:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.0:1619677448.734869:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (23)--
00000800:00000200:5.0:1619677448.734870:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (22)--
00000800:00000200:30.0:1619677448.734871:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677448.734871:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.2:1619677448.734873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708000] (21)++
00000800:00000200:17.0:1619677448.734873:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:25.2:1619677448.734874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:8.0:1619677448.734874:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:1.0:1619677448.734874:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.734876:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.734876:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.734877:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.734877:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.734878:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.734878:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708000] (22)++
00000800:00000200:31.0:1619677448.734880:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:17.0:1619677448.734880:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677448.734881:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.734881:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: REPLY: OK
00000800:00000200:5.0:1619677448.734881:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.4.67@o2ib3
00000800:00000200:5.0:1619677448.734882:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708000] (23)--
00000400:00000200:17.0:1619677448.734883:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.734883:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.734884:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.734884:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (22)--
00000400:00000200:17.0:1619677448.734884:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.734884:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:31.0:1619677448.734885:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.734885:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:17.0:1619677448.734886:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:5.0:1619677448.734886:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:31.0:1619677448.734887:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.734887:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:5.0:1619677448.734888:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.734890:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.734890:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.734891:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.734892:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.734893:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:5.0:1619677448.734896:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.67@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : REPLY try# 0
00000800:00000200:17.0:1619677448.734897:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.734898:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.734898:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677448.734898:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 40529 bytes in 256 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.734900:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.734900:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.734900:0:19935:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 40529
00000800:00000200:17.0:1619677448.734902:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:5.0:1619677448.734902:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.734903:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:5.0:1619677448.734903:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.734904:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.734904:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677448.734905:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:25.2:1619677448.734906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:5.0:1619677448.734906:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708000] (21)++
00000800:00000200:5.0:1619677448.734907:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708000] (22)--
00000800:00000200:5.2:1619677448.734908:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.0:1619677448.734911:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (21)--
00000800:00000200:13.0:1619677448.734913:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:1.0:1619677448.734914:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677448.734915:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (24)--
00000400:00000200:13.0:1619677448.734916:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.734916:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.734917:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.734917:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677448.734918:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.734919:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:12.0:1619677448.734920:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:13.0:1619677448.734921:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:20.2:1619677448.734922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708000] (20)++
00000800:00000200:12.0:1619677448.734922:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677448.734925:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708000] (21)++
00000800:00000200:20.0:1619677448.734928:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.4.67@o2ib3
00000400:00000200:20.0:1619677448.734930:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.67@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.734931:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (22)--
00000800:00000200:25.2:1619677448.734932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.734934:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.0:1619677448.734938:0:19934:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 8056
00000800:00000200:13.0:1619677448.734939:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.0:1619677448.734940:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708000] (21)++
00000800:00000200:20.0:1619677448.734941:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708000] (22)++
00000800:00000200:20.0:1619677448.734941:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708000] (23)--
00000800:00000200:31.0:1619677448.734942:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.734942:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (22)--
00000800:00000200:13.0:1619677448.734942:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[2] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.734944:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:31.0:1619677448.734945:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:12.0:1619677448.734945:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:20.2:1619677448.734946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708000] (21)++
00000400:00000200:31.0:1619677448.734947:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.734947:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.734948:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.734949:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.734950:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.734952:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:25.2:1619677448.734952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000800:00000200:5.0:1619677448.734952:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708000] (22)++
00000400:00000200:31.0:1619677448.734953:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.734954:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.734954:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (23)--
00000800:00000200:5.0:1619677448.734954:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708000] (22)++
00000400:00000200:31.0:1619677448.734955:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.734956:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.4.67@o2ib3
00000400:00000200:31.0:1619677448.734957:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:20.0:1619677448.734957:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (23)--
00000800:00000200:5.0:1619677448.734957:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708000] (23)--
00000800:00000200:26.0:1619677448.734958:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000400:00000200:5.0:1619677448.734958:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.734959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:5.0:1619677448.734959:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:26.0:1619677448.734960:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.0.10.106@o2ib7
00000400:00000200:31.0:1619677448.734961:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.734961:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:5.0:1619677448.734961:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96711030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.734963:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.734964:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.734965:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:26.0:1619677448.734965:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.106@o2ib7 : GET - for me
00000400:00000200:5.0:1619677448.734965:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.734966:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:31.0:1619677448.734968:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.734968:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.734969:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000400:00000200:26.0:1619677448.734969:0:19941:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.106@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000800:00000200:12.0:1619677448.734969:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:5.0:1619677448.734969:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.67@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.734970:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000400:00000200:12.0:1619677448.734970:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.734971:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677448.734971:0:19941:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.106@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:31.0:1619677448.734972:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:5.0:1619677448.734972:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 8056 bytes in 256 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.734973:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (21)++
00000800:00000200:13.0:1619677448.734973:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:12.0:1619677448.734973:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: REPLY: OK
00000800:00000200:31.0:1619677448.734974:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:26.0:1619677448.734974:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:5.0:1619677448.734974:0:19935:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 8056
00000800:00000200:31.0:1619677448.734975:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677448.734975:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:25.2:1619677448.734976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:12.0:1619677448.734976:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677448.734976:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.734977:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:5.0:1619677448.734978:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:26.0:1619677448.734980:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : REPLY try# 0
00000800:00000200:20.2:1619677448.734980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.0:1619677448.734980:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677448.734980:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708000] (21)++
00000800:00000200:5.0:1619677448.734981:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708000] (22)--
00000800:00000200:5.0:1619677448.734981:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (21)--
00000800:00000200:26.0:1619677448.734982:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.734984:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.734984:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (21)++
00000800:00000200:30.0:1619677448.734985:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:26.0:1619677448.734985:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (22)++
00000800:00000200:25.2:1619677448.734985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.734986:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (23)--
00000800:00000200:30.0:1619677448.734987:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000800:00000200:26.0:1619677448.734987:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:30.0:1619677448.734988:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.734989:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677448.734989:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:30.0:1619677448.734989:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.734990:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.734992:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.734992:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:25.2:1619677448.734992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:13.0:1619677448.734992:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:31.0:1619677448.734993:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:13.0:1619677448.734994:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.734995:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.734998:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:25.2:1619677448.734999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.735000:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.735000:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.735001:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.735003:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: REPLY: OK
00000800:00000200:10.0:1619677448.735004:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:13.0:1619677448.735006:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.735008:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[1] from 10.0.10.103@o2ib7
00000800:00000200:20.2:1619677448.735010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708000] (20)++
00000800:00000200:13.0:1619677448.735010:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)++
00000800:00000200:13.0:1619677448.735011:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (24)--
00000800:00000200:20.0:1619677448.735012:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708000] (21)++
00000800:00000200:13.0:1619677448.735012:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.735013:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:20.0:1619677448.735015:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.4.67@o2ib3
00000400:00000200:20.0:1619677448.735017:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.67@o2ib3 : REPLY - routed
00000800:00000200:5.0:1619677448.735018:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (22)--
00000800:00000200:25.2:1619677448.735020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.0:1619677448.735020:0:19934:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 40530
00000800:00000200:20.0:1619677448.735022:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708000] (21)++
00000800:00000200:20.0:1619677448.735023:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708000] (22)++
00000800:00000200:20.0:1619677448.735024:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708000] (23)--
00000800:00000200:20.0:1619677448.735024:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (22)--
00000800:00000200:13.0:1619677448.735027:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.2:1619677448.735029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708000] (21)++
00000800:00000200:13.0:1619677448.735029:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.735030:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.735032:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.735034:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677448.735034:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708000] (22)++
00000800:00000200:13.0:1619677448.735035:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.735036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.735037:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (23)--
00000800:00000200:5.0:1619677448.735038:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (22)--
00000800:00000200:5.0:1619677448.735039:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.735041:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.735041:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.735043:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.735045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708000] (21)++
00000400:00000200:5.0:1619677448.735046:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.735047:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708000] (22)++
00000400:00000200:5.0:1619677448.735047:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677448.735049:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.4.67@o2ib3
00000400:00000200:5.0:1619677448.735049:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.735050:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708000] (23)--
00000400:00000200:20.0:1619677448.735051:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.735051:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.735052:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.735052:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677448.735055:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.735056:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.735057:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677448.735058:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.735059:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.735060:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.735060:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.735063:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677448.735064:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.67@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : REPLY try# 0
00000800:00000200:5.0:1619677448.735064:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.735065:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.735066:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 40530 bytes in 256 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.735066:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.735067:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.735068:0:19934:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 40530
00000800:00000200:5.0:1619677448.735068:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.735068:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.0:1619677448.735070:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (22)--
00000800:00000200:21.2:1619677448.735071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:20.0:1619677448.735071:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.735072:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:25.2:1619677448.735073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.735073:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.735075:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:20.0:1619677448.735076:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708000] (21)++
00000800:00000200:20.0:1619677448.735077:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708000] (22)--
00000800:00000200:20.0:1619677448.735078:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (21)--
00000800:00000200:24.0:1619677448.735079:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:12.0:1619677448.735079:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:20.2:1619677448.735080:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708000] (20)++
00000800:00000200:24.0:1619677448.735082:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:12.0:1619677448.735082:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (24)--
00000400:00000200:12.0:1619677448.735083:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.735083:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.735085:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677448.735085:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.735085:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708000] (21)++
00000800:00000200:13.0:1619677448.735086:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:13.0:1619677448.735087:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:5.0:1619677448.735087:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.4.67@o2ib3
00000400:00000200:24.0:1619677448.735088:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.735088:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.735088:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:24.0:1619677448.735090:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:20.0:1619677448.735090:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (22)--
00000400:00000200:5.0:1619677448.735091:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.67@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.735092:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96711027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.735093:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.735094:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.735094:0:19935:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 6784
00000400:00000200:24.0:1619677448.735095:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.735095:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708000] (21)++
00000800:00000200:25.2:1619677448.735096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.735097:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708000] (22)++
00000800:00000200:5.0:1619677448.735097:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708000] (23)--
00000800:00000200:5.0:1619677448.735098:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (22)--
00000400:00000200:24.0:1619677448.735099:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.735101:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.2:1619677448.735102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708000] (21)++
00000800:00000200:13.0:1619677448.735102:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:24.0:1619677448.735104:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.735104:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708000] (22)++
00000800:00000200:24.0:1619677448.735105:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.735105:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:13.0:1619677448.735105:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[1] from 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677448.735107:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:20.0:1619677448.735107:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (23)--
00000800:00000200:20.0:1619677448.735107:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (22)--
00000800:00000200:13.0:1619677448.735107:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)++
00000800:00000200:24.0:1619677448.735108:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.735109:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.735109:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677448.735109:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (24)--
00000800:00000200:13.0:1619677448.735110:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.735110:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:9.2:1619677448.735113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:20.2:1619677448.735115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708000] (21)++
00000800:00000200:20.0:1619677448.735118:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708000] (22)++
00000800:00000200:20.2:1619677448.735120:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.735122:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.4.67@o2ib3
00000800:00000200:18.0:1619677448.735122:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:25.2:1619677448.735123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.0:1619677448.735123:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708000] (23)--
00000800:00000200:5.0:1619677448.735123:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (22)--
00000400:00000200:20.0:1619677448.735124:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.735124:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:5.0:1619677448.735124:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:20.0:1619677448.735125:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677448.735125:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.735125:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.735126:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.735127:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96711026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.735128:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.735129:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.735130:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.735130:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:9.2:1619677448.735130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:20.0:1619677448.735131:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:12.0:1619677448.735132:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.735133:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.735134:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.735135:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000400:00000200:5.0:1619677448.735135:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677448.735136:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.67@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.735136:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.735136:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: REPLY: OK
00000800:00000200:18.0:1619677448.735138:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.735138:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.735139:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 6784 bytes in 256 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677448.735139:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.735139:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.735140:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:12.0:1619677448.735140:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677448.735141:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:5.0:1619677448.735141:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.735142:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677448.735143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:5.0:1619677448.735144:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.735145:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.735146:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.2:1619677448.735147:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708000] (21)++
00000800:00000200:20.0:1619677448.735149:0:19934:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 6784
00000800:00000200:8.0:1619677448.735150:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:5.0:1619677448.735150:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677448.735152:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.735152:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.735153:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677448.735153:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677448.735153:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:20.0:1619677448.735154:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:5.0:1619677448.735155:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.735156:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.735156:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.735156:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:5.0:1619677448.735156:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.735157:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708000] (22)++
00000800:00000200:20.0:1619677448.735158:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708000] (23)--
00000800:00000200:20.0:1619677448.735158:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (22)--
00000800:00000200:5.0:1619677448.735158:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:8.0:1619677448.735159:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.735159:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677448.735160:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677448.735160:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708000] (21)++
00000800:00000200:5.0:1619677448.735160:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:9.2:1619677448.735161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:8.0:1619677448.735161:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:5.0:1619677448.735161:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:5.0:1619677448.735162:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (22)--
00000800:00000200:20.0:1619677448.735163:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.4.67@o2ib3
00000400:00000200:20.0:1619677448.735165:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.67@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.735165:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.735168:0:19934:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 8024
00000800:00000200:20.0:1619677448.735168:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708000] (21)++
00000800:00000200:24.0:1619677448.735169:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:20.0:1619677448.735169:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708000] (22)++
00000800:00000200:20.0:1619677448.735170:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708000] (23)--
00000400:00000200:8.0:1619677448.735170:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96711024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.735171:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:20.0:1619677448.735171:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (22)--
00000400:00000200:8.0:1619677448.735171:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.735172:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.735173:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.735174:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.735174:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.2:1619677448.735175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708000] (21)++
00000800:00000200:9.2:1619677448.735176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.735176:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.735177:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:8.0:1619677448.735177:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677448.735177:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.735180:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708000] (22)++
00000400:00000200:1.0:1619677448.735180:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:8.0:1619677448.735181:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.735182:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (23)--
00000800:00000200:24.0:1619677448.735183:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:5.0:1619677448.735183:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (22)--
00000800:00000200:1.0:1619677448.735183:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:24.0:1619677448.735185:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[1] from 10.0.10.101@o2ib7
00000800:00000200:8.2:1619677448.735185:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.735187:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (23)++
00000800:00000200:24.0:1619677448.735188:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (24)--
00000800:00000200:24.0:1619677448.735189:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.735190:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.2:1619677448.735190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708000] (21)++
00000800:00000200:8.0:1619677448.735191:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.735193:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.735193:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.735193:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (22)++
00000800:00000200:8.0:1619677448.735195:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (24)--
00000800:00000200:5.0:1619677448.735195:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708000] (22)++
00000800:00000200:8.0:1619677448.735196:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677448.735197:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:9.2:1619677448.735197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.735197:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677448.735198:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (23)++
00000800:00000200:5.0:1619677448.735198:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.4.67@o2ib3
00000800:00000200:1.0:1619677448.735198:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.735199:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:5.0:1619677448.735199:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708000] (23)--
00000400:00000200:5.0:1619677448.735199:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.735200:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (22)--
00000800:00000200:17.0:1619677448.735200:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.735200:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:5.0:1619677448.735201:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:8.2:1619677448.735202:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:5.0:1619677448.735202:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96711023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.735204:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.735205:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677448.735206:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000400:00000200:5.0:1619677448.735206:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677448.735209:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:5.0:1619677448.735209:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.67@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.735210:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:8.0:1619677448.735210:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.735210:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677448.735211:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 8024 bytes in 256 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:8.0:1619677448.735212:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.735212:0:19935:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 8024
00000800:00000200:1.0:1619677448.735212:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.735214:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.735214:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.735215:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.0:1619677448.735215:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.735216:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:5.0:1619677448.735216:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:1.0:1619677448.735216:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.735218:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.735218:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677448.735219:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708000] (21)++
00000800:00000200:5.0:1619677448.735220:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708000] (22)--
00000800:00000200:5.0:1619677448.735221:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (21)--
00000800:00000200:9.2:1619677448.735224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.735230:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.735233:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.735234:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:9.2:1619677448.735240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:20.2:1619677448.735243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708000] (20)++
00000800:00000200:20.0:1619677448.735245:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708000] (21)++
00000800:00000200:8.0:1619677448.735246:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:20.0:1619677448.735248:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.67@o2ib3
00000800:00000200:24.0:1619677448.735249:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.735249:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[1] from 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677448.735251:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.67@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.735251:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:5.0:1619677448.735251:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (22)--
00000800:00000200:8.0:1619677448.735253:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (23)--
00000800:00000200:8.0:1619677448.735253:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:20.0:1619677448.735254:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.735255:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677448.735257:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.735258:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.735259:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.735260:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:9.2:1619677448.735262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:20.0:1619677448.735263:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.67@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.735265:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 1432 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677448.735267:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.735268:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:20.0:1619677448.735268:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677448.735269:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:24.0:1619677448.735270:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.735271:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.735271:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677448.735271:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708000] (21)++
00000800:00000200:8.0:1619677448.735271:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:20.0:1619677448.735272:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708000] (22)--
00000800:00000200:20.0:1619677448.735273:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708000] (21)--
00000400:00000200:24.0:1619677448.735274:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.735277:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.735277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.735284:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.735287:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.735288:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.735289:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.735290:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.735298:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.735439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.735447:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.2:1619677448.735448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:26.0:1619677448.735450:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.735450:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.735452:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.735454:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:26.0:1619677448.735455:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.735456:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677448.735457:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:26.0:1619677448.735458:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.735458:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:26.0:1619677448.735459:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.735460:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.735461:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.735462:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677448.735464:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.735466:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.735469:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:22.0:1619677448.735470:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677448.735471:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.735472:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.735473:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.735473:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:22.0:1619677448.735473:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.735474:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.735475:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.735475:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677448.735475:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.735476:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.735478:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.735478:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.735478:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.735480:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.2:1619677448.735482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.735482:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.735483:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.735484:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:4.2:1619677448.735484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.735486:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.735487:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.735488:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.735489:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.735490:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.735490:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.735491:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.735492:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.735492:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677448.735493:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:8.2:1619677448.735493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:6.0:1619677448.735493:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.735494:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677448.735494:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.2:1619677448.735495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:6.0:1619677448.735495:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.735499:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:6.0:1619677448.735500:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.735501:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.735501:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:17.0:1619677448.735502:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:6.0:1619677448.735502:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:17.0:1619677448.735505:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.735506:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.735506:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.735508:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.735510:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.735628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.735634:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.735637:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.735637:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.735640:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.735642:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.735643:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.735645:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.735646:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.735647:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.735648:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.735652:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.735653:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.735657:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.735658:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.735658:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.735660:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.735661:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.735661:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.735662:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.735666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.735674:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.735677:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.735678:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.735679:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.735681:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.735682:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.735807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.735814:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.735817:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.735818:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.735820:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.735822:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.735824:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.735825:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.735826:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.735828:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.735829:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.735832:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.735834:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.735837:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.735838:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.735838:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.735840:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.735841:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.735841:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.735842:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.735846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.735854:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.735857:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.735857:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.735858:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.735860:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.735863:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.735880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677448.735887:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.735889:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.735890:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677448.735892:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.735895:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.735896:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.735897:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.735898:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.735899:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.735900:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.735903:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.735904:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.735907:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.735908:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.735908:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.735910:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.735911:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.735911:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.735912:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.735916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.735924:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.735927:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.735928:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.735928:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.735931:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.735933:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.735963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1200] (20)++
00000800:00000200:1.0:1619677448.735968:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:1.0:1619677448.735971:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:17.0:1619677448.735972:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:1.0:1619677448.735974:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.735976:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.735978:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677448.735979:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.735980:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.735981:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.735982:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677448.735985:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.735987:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677448.735989:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.735990:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:1.0:1619677448.735990:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:1.0:1619677448.735992:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:1.0:1619677448.735992:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:1.0:1619677448.735993:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:1.0:1619677448.735993:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:25.2:1619677448.735998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.736005:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.736008:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.736009:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.736009:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.736011:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.736014:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.736163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.736171:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.736174:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.736177:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.736178:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.736179:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.736180:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.736182:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.736183:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.736184:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.736185:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.736188:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.736190:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.736193:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.736194:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.736194:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.736196:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.736196:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.736198:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.736198:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.736201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.736208:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:20.2:1619677448.736210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:12.0:1619677448.736210:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.736211:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.736212:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.736214:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.736215:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.0:1619677448.736216:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.736219:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.736221:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.736221:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.736224:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.736226:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.736227:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711014] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.736228:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.736229:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.736230:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:5.0:1619677448.736234:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.736235:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.736239:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.736240:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.736241:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.736242:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.736243:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.736244:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.736244:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.736248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.736255:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.736257:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.736258:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.736260:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.736261:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.736263:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.736355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.736361:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.736364:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.736364:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.736367:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.736369:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.736370:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677448.736371:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.736373:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.736374:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.736374:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677448.736377:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.736379:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677448.736381:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.736382:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:6.0:1619677448.736383:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:6.0:1619677448.736384:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:6.0:1619677448.736385:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.736385:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.736386:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:9.2:1619677448.736391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.736399:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.736402:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:18.0:1619677448.736403:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:2.0:1619677448.736403:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.736405:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.736406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:5.2:1619677448.736410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677448.736410:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:9.2:1619677448.736411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.736413:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.736415:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.736415:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.736416:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:17.0:1619677448.736416:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.736417:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.736418:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.736418:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:18.0:1619677448.736419:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:17.0:1619677448.736419:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:18.0:1619677448.736420:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.736420:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.736420:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677448.736421:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000400:00000200:2.0:1619677448.736422:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677448.736423:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.736424:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.736425:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.736425:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.736426:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.736427:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677448.736428:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677448.736429:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.736431:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.736432:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.736432:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:17.0:1619677448.736433:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.736434:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:17.0:1619677448.736437:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.736437:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.736438:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:2.0:1619677448.736438:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.736439:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.736440:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:2.0:1619677448.736440:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.736441:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.736441:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677448.736442:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.736443:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.736444:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:25.2:1619677448.736446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:17.0:1619677448.736446:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.736447:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.736447:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.736448:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.736452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677448.736453:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:4.2:1619677448.736455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677448.736456:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.736456:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.736457:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.736459:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:12.0:1619677448.736459:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:22.0:1619677448.736461:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:12.0:1619677448.736462:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.736463:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.736463:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:12.0:1619677448.736464:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.736466:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.736466:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.736467:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:12.0:1619677448.736467:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:10.0:1619677448.736469:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.736472:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.736474:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.736475:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.736476:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.736477:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.736478:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.736482:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.736484:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.736487:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.736488:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.736489:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.736490:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.736491:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.736491:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.736492:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.736496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.736504:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.736506:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.736507:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.736507:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.736509:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.736512:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.736592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.736599:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.736602:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.736603:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.736604:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.736607:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.736609:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677448.736610:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.736612:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.736613:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.736614:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677448.736617:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.736619:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677448.736621:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.736622:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:26.0:1619677448.736623:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:26.0:1619677448.736624:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:26.0:1619677448.736624:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.736625:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.736626:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677448.736631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.736639:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:4.2:1619677448.736640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:17.0:1619677448.736642:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.736642:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.736643:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.736645:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.736647:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677448.736648:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.0:1619677448.736650:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.736651:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.736653:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.736655:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.736657:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.736658:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96711008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.736659:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.736660:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.736661:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677448.736664:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.736665:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677448.736668:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.736669:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.736669:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.736671:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.736671:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.736672:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.736672:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.736677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.736684:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.736687:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.736688:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.736688:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.736690:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.736692:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.736694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.736702:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.736704:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.736705:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.736705:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.736706:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.736707:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.736714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:5.2:1619677448.736718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919065782400] (20)++
00000800:00000200:18.0:1619677448.736722:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.736724:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.736725:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (21)++
00000800:00000200:2.0:1619677448.736726:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000400:00000200:18.0:1619677448.736727:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.736728:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.736731:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.736731:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.736733:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.736734:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96711007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.736735:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.736737:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.736738:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.2:1619677448.736740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:2.0:1619677448.736740:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677448.736741:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.736742:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.736743:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.736744:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.736744:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.736745:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:10.0:1619677448.736747:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677448.736749:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677448.736749:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.736750:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.736750:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.736751:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677448.736751:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.736752:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677448.736753:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.736753:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.736754:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677448.736755:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.736755:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.736756:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:10.0:1619677448.736756:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.736756:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:2.0:1619677448.736756:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:18.0:1619677448.736757:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:10.0:1619677448.736757:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:18.0:1619677448.736758:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000800:00000200:2.0:1619677448.736758:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:25.2:1619677448.736759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:10.0:1619677448.736759:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96711005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.736759:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (21)++
00000400:00000200:10.0:1619677448.736761:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677448.736761:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919065782400] (22)++
00000400:00000200:10.0:1619677448.736762:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.736762:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (23)--
00000800:00000200:2.0:1619677448.736762:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000400:00000200:10.0:1619677448.736763:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:8.2:1619677448.736764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:13.0:1619677448.736765:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677448.736766:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.736767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919065782400] (21)++
00000800:00000200:13.0:1619677448.736768:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.736768:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.736768:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:13.0:1619677448.736770:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.736770:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.736771:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.736771:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:20.0:1619677448.736772:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:10.0:1619677448.736772:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:18.0:1619677448.736773:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (22)++
00000800:00000200:13.0:1619677448.736773:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.736773:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:20.0:1619677448.736774:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:10.0:1619677448.736774:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.2:1619677448.736774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:10.0:1619677448.736775:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.736775:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:20.0:1619677448.736776:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.736776:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919065782400] (23)--
00000800:00000200:18.0:1619677448.736776:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000800:00000200:25.2:1619677448.736777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:20.0:1619677448.736777:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.736778:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (21)--
00000800:00000200:2.0:1619677448.736778:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:25.2:1619677448.736780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:18.0:1619677448.736780:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.0:1619677448.736780:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.736781:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000800:00000200:2.0:1619677448.736781:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:12.0:1619677448.736783:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:2.0:1619677448.736783:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.736786:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677448.736786:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.736787:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.736787:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:2.0:1619677448.736787:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:31.0:1619677448.736788:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:12.0:1619677448.736788:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677448.736788:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.736790:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:2.0:1619677448.736790:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.736791:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:2.0:1619677448.736791:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.736792:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.736792:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677448.736794:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.736795:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000400:00000200:2.0:1619677448.736796:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.736797:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:2.0:1619677448.736798:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:25.2:1619677448.736799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000800:00000200:2.0:1619677448.736801:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.736802:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.736803:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.736804:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.736805:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.736805:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:30.0:1619677448.736806:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000800:00000200:2.0:1619677448.736806:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:30.0:1619677448.736809:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.736810:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:8.2:1619677448.736810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.736811:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.736811:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.736812:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.0:1619677448.736818:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.736821:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.736822:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.736824:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.736824:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.736826:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:20.2:1619677448.736845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.736853:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.736856:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.736859:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.736860:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.736861:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.736862:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677448.736864:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.736865:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.736866:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.736867:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677448.736870:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.736870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.736872:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677448.736875:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.736876:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:31.0:1619677448.736877:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:26.0:1619677448.736877:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.736879:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:31.0:1619677448.736879:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.736879:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:31.0:1619677448.736880:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.736880:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.736881:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:26.0:1619677448.736882:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.736884:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.736885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:26.0:1619677448.736886:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.736887:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96711002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.736888:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.736889:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.736890:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.736892:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000400:00000200:26.0:1619677448.736894:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.736895:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677448.736896:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.736896:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.736897:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.736898:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.736898:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.736898:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.736899:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.736900:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:26.0:1619677448.736901:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.736901:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.736902:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.736903:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.736906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.736913:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.736915:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.736916:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.736918:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.736919:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.736921:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.736966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.736973:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.736975:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.736977:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.736979:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.736980:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.736981:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.736983:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96711001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.736984:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.736985:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.736986:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.736988:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.736990:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.736993:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.736993:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.2:1619677448.736993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677448.736994:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.736996:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.736996:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.736997:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.736997:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.0:1619677448.736998:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:25.2:1619677448.737001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.737001:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.737003:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.737003:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.737006:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.737007:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:5.0:1619677448.737007:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.737009:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96711000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.737010:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.737010:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.737010:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.737011:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.737012:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:13.0:1619677448.737013:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.737013:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:5.0:1619677448.737016:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.737018:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.737019:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.0:1619677448.737021:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.737022:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.737022:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.737024:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.737025:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.737026:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.737027:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.737030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.737037:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.737039:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.737040:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.737042:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.737043:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.737044:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.737152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677448.737159:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677448.737162:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677448.737163:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677448.737165:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677448.737167:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.737169:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677448.737170:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96710999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.737171:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.737172:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.737173:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677448.737176:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677448.737178:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677448.737181:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.737181:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:24.0:1619677448.737182:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:24.0:1619677448.737184:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:24.0:1619677448.737184:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677448.737185:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677448.737185:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:25.2:1619677448.737190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.737197:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.737200:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.737200:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.737201:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.737203:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.737206:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.737207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.737214:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.737217:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677448.737218:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.737218:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.737219:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.737220:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.737240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.737248:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.737251:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.737253:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.737254:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.737255:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.737257:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.737258:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710998] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.737259:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.737260:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.737261:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.737264:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.737266:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.737269:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.737270:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.737271:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.737272:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.737273:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.737274:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.737274:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.737279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.737286:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.737288:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.737289:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.737291:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.737292:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.737298:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.737317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.737325:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.737328:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.737329:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.737331:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.737333:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.737334:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677448.737335:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.737337:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.737337:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.737338:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.737342:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.737343:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677448.737345:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.737346:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:10.0:1619677448.737347:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.737348:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:10.0:1619677448.737348:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.737349:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.737349:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.737354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.737362:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.737364:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.737365:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.737367:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.737369:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.737370:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.737390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.737398:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.737400:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.737401:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.2:1619677448.737401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:26.0:1619677448.737402:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.737404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677448.737405:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.737406:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.737407:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.737408:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:26.0:1619677448.737409:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.737409:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.737410:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.737411:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:18.0:1619677448.737411:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.737411:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677448.737413:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.737414:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677448.737415:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.737418:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.737418:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.737418:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.737419:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.737421:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96710995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.737421:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.737422:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:18.0:1619677448.737423:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.737423:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:26.0:1619677448.737424:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.737424:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:18.0:1619677448.737424:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.737424:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96710994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.737425:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:18.0:1619677448.737425:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.737426:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.737426:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677448.737426:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.737427:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:17.0:1619677448.737427:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.737428:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:17.0:1619677448.737428:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677448.737429:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.737431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.737432:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:17.0:1619677448.737433:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.737434:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.737435:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.737435:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:17.0:1619677448.737435:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.737437:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.737437:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:10.0:1619677448.737437:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:18.0:1619677448.737438:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677448.737438:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.737439:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677448.737439:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.737439:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:17.0:1619677448.737440:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:10.0:1619677448.737440:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.737441:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:17.0:1619677448.737442:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.737442:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677448.737442:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.737443:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.737443:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.737443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.737444:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:25.2:1619677448.737448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677448.737450:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.737452:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.737453:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.737454:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:13.0:1619677448.737455:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:2.0:1619677448.737455:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.737456:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:13.0:1619677448.737458:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.737459:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.737461:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.737462:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.737463:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.737467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:12.0:1619677448.737473:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.737476:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.737477:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.737478:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.737478:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.737480:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.737512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.737518:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.737521:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.737521:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.737523:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.737526:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.737527:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677448.737529:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.737530:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.737531:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.737532:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677448.737536:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.737537:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677448.737540:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.737541:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:22.0:1619677448.737542:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:22.0:1619677448.737543:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:22.0:1619677448.737544:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.737545:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.737545:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:8.2:1619677448.737550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.737558:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.737560:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.737561:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.737563:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.737564:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.737565:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.737592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.737599:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.737602:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.737602:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677448.737605:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.737608:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.737609:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:21.2:1619677448.737611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:8.0:1619677448.737611:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96710992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.737612:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.737613:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.737614:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.737617:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.737618:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677448.737619:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.737621:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.0:1619677448.737622:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.737623:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.737623:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.737624:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:30.0:1619677448.737625:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677448.737625:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:31.0:1619677448.737626:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.737626:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:31.0:1619677448.737627:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:8.0:1619677448.737627:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:31.0:1619677448.737628:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96710991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.737628:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.737629:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:31.0:1619677448.737630:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.737630:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.737631:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.2:1619677448.737631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677448.737634:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677448.737634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677448.737636:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.737638:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.737639:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.737639:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.737639:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:31.0:1619677448.737641:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.737641:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677448.737641:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:31.0:1619677448.737642:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677448.737642:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.737642:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:31.0:1619677448.737643:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677448.737643:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677448.737643:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.737644:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.737645:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677448.737645:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.737646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.737648:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:13.0:1619677448.737653:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.737655:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.737656:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.737658:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.737659:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.737660:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677448.737661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.737669:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.737672:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.737672:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.737673:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.737674:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.737675:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.737720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.737728:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.737731:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.737734:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.737735:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.737736:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.737738:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.737739:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.737740:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.737741:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.737742:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.737746:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.737747:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.737751:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.737751:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.737752:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.737754:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.737754:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.737756:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.737757:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.737758:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:8.2:1619677448.737759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.737766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:20.0:1619677448.737771:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677448.737774:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:20.0:1619677448.737774:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.737775:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.737776:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.737777:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:20.0:1619677448.737777:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.737777:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.737778:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:4.2:1619677448.737778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.2:1619677448.737780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.737783:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:26.0:1619677448.737785:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.737785:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.737787:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.737788:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:20.0:1619677448.737788:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.737788:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.737789:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.737789:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:10.0:1619677448.737789:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.737791:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.737792:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.737793:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.737795:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.737796:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.737797:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.737798:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677448.737801:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.737802:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677448.737804:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.737805:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.737806:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.737807:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.737807:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.737809:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.737810:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.737810:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.737813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:4.2:1619677448.737816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.737819:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.737822:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.737822:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.737823:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677448.737824:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.737825:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.737825:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.737826:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.737827:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:10.0:1619677448.737827:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677448.737844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.737850:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.737853:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.737854:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.2:1619677448.737855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:1.0:1619677448.737855:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.737858:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.737860:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677448.737860:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677448.737861:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.737862:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.737863:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:1.0:1619677448.737863:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.737864:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:20.0:1619677448.737865:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.737865:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.737868:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.737868:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.737870:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:1.0:1619677448.737870:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:5.0:1619677448.737871:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96710987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.737873:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.737873:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.737874:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677448.737874:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000400:00000200:5.0:1619677448.737875:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677448.737875:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:1.0:1619677448.737877:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:1.0:1619677448.737877:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:5.0:1619677448.737879:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.737879:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.737880:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.737880:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.737881:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:9.2:1619677448.737883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:5.0:1619677448.737884:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.737885:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.2:1619677448.737886:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677448.737889:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:5.0:1619677448.737889:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.737890:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.737891:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:5.0:1619677448.737891:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:24.0:1619677448.737892:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.737892:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:5.0:1619677448.737892:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:17.0:1619677448.737893:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:8.0:1619677448.737893:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.737893:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:17.0:1619677448.737894:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.0:1619677448.737894:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:8.0:1619677448.737895:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.737896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:17.0:1619677448.737896:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677448.737899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677448.737899:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.0:1619677448.737902:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:30.0:1619677448.737904:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:20.0:1619677448.737904:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.737905:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677448.737906:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:20.0:1619677448.737906:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:30.0:1619677448.737908:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.737909:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.737910:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.737913:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:5.2:1619677448.737924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:25.2:1619677448.737926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.737931:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677448.737932:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:26.0:1619677448.737934:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.737934:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677448.737935:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.737936:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:21.2:1619677448.737936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677448.737937:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:26.0:1619677448.737937:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.737938:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:26.0:1619677448.737939:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.737940:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000400:00000200:26.0:1619677448.737940:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:30.0:1619677448.737941:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000400:00000200:26.0:1619677448.737941:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.737943:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.737944:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:26.0:1619677448.737944:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.737945:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.737946:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.737948:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:26.0:1619677448.737948:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.737949:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.737950:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:26.0:1619677448.737950:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:30.0:1619677448.737952:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96710985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.737952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:30.0:1619677448.737953:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.737953:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677448.737954:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.737954:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.737954:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:30.0:1619677448.737955:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.737956:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.737956:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.737957:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.737957:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:30.0:1619677448.737958:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.737959:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677448.737959:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:25.2:1619677448.737961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:24.0:1619677448.737961:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.737962:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:30.0:1619677448.737963:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.737964:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.737964:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:24.0:1619677448.737964:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.737966:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.737967:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677448.737967:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:24.0:1619677448.737967:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.737968:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000400:00000200:24.0:1619677448.737968:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:12.0:1619677448.737968:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:24.0:1619677448.737969:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96710984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.737970:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:24.0:1619677448.737971:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.737971:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.737972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:24.0:1619677448.737972:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677448.737972:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.737973:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.737974:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:24.0:1619677448.737977:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.737978:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:24.0:1619677448.737979:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:6.0:1619677448.737979:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:24.0:1619677448.737981:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.737982:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:24.0:1619677448.737982:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677448.737982:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.737983:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:24.0:1619677448.737984:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:24.0:1619677448.737984:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000400:00000200:6.0:1619677448.737984:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.737985:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000400:00000200:6.0:1619677448.737986:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.737987:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.737987:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.2:1619677448.737989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:6.0:1619677448.737989:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.737993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:13.0:1619677448.737996:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.737998:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.737999:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.737999:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:13.0:1619677448.738001:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.738001:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:12.0:1619677448.738002:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.738002:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677448.738003:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:8.0:1619677448.738004:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:5.2:1619677448.738110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.738116:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:4.2:1619677448.738119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677448.738119:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.738120:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.738122:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.738125:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677448.738125:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.738126:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:10.0:1619677448.738128:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:1.0:1619677448.738128:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.738129:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677448.738130:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.738131:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.738132:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.738132:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:10.0:1619677448.738134:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.738136:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677448.738136:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.738137:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96710982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.738139:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.738139:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:10.0:1619677448.738140:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.738141:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.738142:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.738143:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.738143:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:10.0:1619677448.738145:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.738145:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.738146:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.738146:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677448.738147:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677448.738147:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.738150:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.738151:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:9.2:1619677448.738151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:10.0:1619677448.738152:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.738154:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.738154:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.738155:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.738155:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:24.0:1619677448.738157:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.738160:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:9.2:1619677448.738160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:24.0:1619677448.738161:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.738161:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.738163:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.738165:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:18.0:1619677448.738167:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.738170:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.738171:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.738171:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.738173:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.738177:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.738190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.738197:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.738200:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:20.2:1619677448.738201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677448.738201:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.738202:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.738205:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.738206:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677448.738208:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677448.738208:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.738209:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.738210:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.738211:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:26.0:1619677448.738211:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.738215:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.738215:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677448.738216:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000400:00000200:31.0:1619677448.738218:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.738218:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.738220:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:26.0:1619677448.738220:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.738221:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96710980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.738221:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.738221:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.738222:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.738223:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.738223:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.738223:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:31.0:1619677448.738224:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.738224:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.738225:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:31.0:1619677448.738227:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.738228:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:25.2:1619677448.738228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.738231:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.738232:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.738233:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.738234:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.738235:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.738235:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677448.738236:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.738236:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:12.0:1619677448.738237:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.738238:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.738239:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677448.738241:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:8.2:1619677448.738241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:12.0:1619677448.738242:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677448.738249:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:25.2:1619677448.738251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.738251:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.738252:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.738253:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.738255:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.738257:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:13.0:1619677448.738257:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.738260:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.738261:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.738262:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.738262:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.738263:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.738320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677448.738328:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.738330:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677448.738333:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.738335:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.738335:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.738337:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677448.738338:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.738339:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.738340:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.738341:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677448.738344:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.738346:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.738349:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.738349:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:30.0:1619677448.738350:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:5.2:1619677448.738350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:30.0:1619677448.738352:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.738352:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.738354:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677448.738355:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.738356:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677448.738356:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:9.2:1619677448.738358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.738359:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:21.2:1619677448.738361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.738362:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.738362:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.738364:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000400:00000200:2.0:1619677448.738365:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.738366:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:2.0:1619677448.738366:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.738367:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.738368:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96710978] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.738369:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:18.0:1619677448.738369:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.738369:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.738371:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.738372:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:2.0:1619677448.738372:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677448.738373:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.738374:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:2.0:1619677448.738375:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.738377:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.738379:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.738380:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.738381:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:2.0:1619677448.738381:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.738382:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.738383:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.738384:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.738385:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.738386:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.738389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.738396:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.738397:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.738398:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.738399:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.738399:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.738401:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.738408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.738415:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.738417:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.738418:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.738418:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.738419:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.738419:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.738452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.738458:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.738461:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.738461:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.738464:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.738466:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.738468:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677448.738469:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.738471:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.738472:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.738473:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677448.738476:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.738478:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.738479:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.738480:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.738481:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.738482:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.738483:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.738483:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.738484:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.738489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.738497:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.738498:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.738499:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.738500:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.738501:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.738502:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:4.2:1619677448.738557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.738565:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.738568:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.738568:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677448.738570:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.738573:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.738575:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.738576:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.738577:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.738578:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.738579:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.738583:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.738585:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.738587:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.738588:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.738589:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.738591:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.738592:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.738592:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.738593:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677448.738597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.738605:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.738608:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.738608:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.738609:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.738611:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.738614:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.738688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.738696:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.738699:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.738699:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.738701:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.738704:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.738705:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.738707:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.738708:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.738709:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.738710:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.2:1619677448.738710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:26.0:1619677448.738712:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.738714:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.738716:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.738716:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:5.0:1619677448.738716:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:26.0:1619677448.738717:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.738718:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.738719:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.738720:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:5.2:1619677448.738720:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677448.738721:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:20.0:1619677448.738721:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:26.0:1619677448.738722:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:5.0:1619677448.738722:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:25.2:1619677448.738724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:5.0:1619677448.738724:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.738726:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677448.738727:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677448.738728:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.738729:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.738731:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.738731:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.738732:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.738732:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:5.0:1619677448.738733:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:12.0:1619677448.738734:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:10.0:1619677448.738734:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:5.0:1619677448.738734:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96710974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.738735:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:12.0:1619677448.738735:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.738736:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96710974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.738736:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:5.0:1619677448.738736:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.738737:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:17.0:1619677448.738738:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.738738:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.738738:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:5.0:1619677448.738738:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.738739:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.738739:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:5.0:1619677448.738739:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:17.0:1619677448.738740:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:12.0:1619677448.738740:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:5.0:1619677448.738744:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.738746:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:17.0:1619677448.738748:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.738749:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.738750:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:17.0:1619677448.738751:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:5.0:1619677448.738751:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:17.0:1619677448.738753:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.738753:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.738753:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677448.738754:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.738754:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:17.0:1619677448.738755:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:5.0:1619677448.738755:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677448.738757:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:17.0:1619677448.738757:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.738758:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.738758:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.738759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:25.2:1619677448.738762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.738766:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.738768:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.738769:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.738769:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.738770:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.738771:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.738771:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.738773:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.738774:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.738775:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:30.0:1619677448.738776:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.738777:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:4.2:1619677448.738864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.738871:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.738873:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.738874:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.738876:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.738878:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.738880:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.738881:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.738882:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.738883:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.738884:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.738888:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.738890:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.738893:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.738893:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.738894:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:26.0:1619677448.738896:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.738896:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.738897:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.738897:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.738902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.738910:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.738912:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.738913:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.738914:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.738916:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.738919:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:21.2:1619677448.738991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.2:1619677448.738995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:30.0:1619677448.738998:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.739001:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.0:1619677448.739002:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:30.0:1619677448.739003:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.739005:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677448.739005:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:30.0:1619677448.739006:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.739006:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:30.0:1619677448.739007:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677448.739008:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.739009:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.739010:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.739011:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.739011:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.739012:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677448.739013:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677448.739014:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96710970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.739016:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.739017:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.739017:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.739018:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.739019:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.739021:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677448.739021:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.739022:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.739022:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677448.739023:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.739024:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.739024:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.739025:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677448.739025:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.739026:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677448.739026:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:8.0:1619677448.739026:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:8.0:1619677448.739028:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:8.0:1619677448.739028:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:25.2:1619677448.739029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:8.0:1619677448.739029:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677448.739029:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:25.2:1619677448.739034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.739038:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.739039:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.739040:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.739041:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.739041:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.739042:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.739043:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:12.0:1619677448.739043:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.739044:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:12.0:1619677448.739044:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.739046:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.739046:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.739053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:13.0:1619677448.739059:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.739061:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677448.739062:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.739063:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.739063:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.739065:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.739137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.739142:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.2:1619677448.739143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:22.0:1619677448.739145:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.739145:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:20.0:1619677448.739146:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:6.0:1619677448.739148:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.739149:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:6.0:1619677448.739150:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.739151:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.739152:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:5.0:1619677448.739152:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:6.0:1619677448.739153:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.739154:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.739154:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.739155:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.739156:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.739157:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.739157:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.739159:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.739160:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.739161:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677448.739161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:20.0:1619677448.739162:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.739165:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.739166:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.739167:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.739168:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.739168:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.739169:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.739169:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:1.0:1619677448.739169:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.739170:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.739171:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.739172:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.739172:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:20.0:1619677448.739173:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.739173:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:6.0:1619677448.739173:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.739173:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:1.0:1619677448.739173:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.739175:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:1.0:1619677448.739175:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.739176:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:25.2:1619677448.739177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.739177:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.739177:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677448.739177:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.739178:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96710967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.739179:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.739180:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.739181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:1.0:1619677448.739181:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.739184:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:1.0:1619677448.739185:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.739186:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.739187:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:12.0:1619677448.739187:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:31.0:1619677448.739188:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.739188:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.739189:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.739189:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677448.739190:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.739190:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:30.0:1619677448.739191:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:12.0:1619677448.739191:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.739191:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.739191:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:12.0:1619677448.739192:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.739192:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677448.739193:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.739194:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:1.0:1619677448.739194:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.739194:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677448.739196:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:8.2:1619677448.739197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.739200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.739206:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.739206:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.739207:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.739207:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:17.0:1619677448.739208:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.739208:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.739209:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.739210:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.739210:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677448.739211:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:4.2:1619677448.739370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.739377:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.739380:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.739381:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.739383:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.739386:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.739387:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677448.739389:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.739390:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.739391:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.739392:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677448.739395:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.739397:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677448.739400:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.739401:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:10.0:1619677448.739402:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:10.0:1619677448.739404:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.739404:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.739405:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.739406:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.739407:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677448.739411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677448.739413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.739418:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.739420:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:2.0:1619677448.739421:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:26.0:1619677448.739422:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:18.0:1619677448.739422:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.739422:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.739423:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.739424:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:2.0:1619677448.739424:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.739428:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.739683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:22.0:1619677448.739689:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677448.739691:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677448.739692:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677448.739694:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.739698:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.739699:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.739700:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677448.739701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000400:00000200:22.0:1619677448.739702:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.739703:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.739704:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677448.739707:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.739708:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677448.739709:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:22.0:1619677448.739711:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.739712:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000800:00000200:22.0:1619677448.739712:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.739712:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.2:1619677448.739713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:30.0:1619677448.739714:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.739714:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.739714:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677448.739715:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:31.0:1619677448.739716:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677448.739716:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.739716:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000400:00000200:30.0:1619677448.739718:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:5.0:1619677448.739718:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677448.739719:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96710964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.739721:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.739721:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:30.0:1619677448.739722:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.739724:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:20.0:1619677448.739724:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.739724:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.739727:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.739727:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.739729:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:9.2:1619677448.739729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:5.0:1619677448.739729:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.739730:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96710963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.739732:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:5.0:1619677448.739732:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.739733:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:30.0:1619677448.739734:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000400:00000200:5.0:1619677448.739734:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.739735:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000400:00000200:5.0:1619677448.739735:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.739736:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:18.0:1619677448.739736:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:30.0:1619677448.739737:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677448.739738:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000400:00000200:5.0:1619677448.739738:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.739739:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.739740:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.739740:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.739740:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:9.2:1619677448.739741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000400:00000200:18.0:1619677448.739742:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.739744:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677448.739744:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.739744:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.739745:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.739747:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:24.0:1619677448.739748:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:5.0:1619677448.739748:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.739749:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.739750:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:24.0:1619677448.739751:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.739751:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.739752:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:25.2:1619677448.739753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:24.0:1619677448.739754:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:21.2:1619677448.739754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:24.0:1619677448.739756:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:13.0:1619677448.739761:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:31.0:1619677448.739762:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677448.739763:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:31.0:1619677448.739764:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:13.0:1619677448.739764:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.739766:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.739767:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.739768:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.739769:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677448.739769:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677448.739770:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.739771:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.739772:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.739773:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.739775:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.739776:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.739779:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.739781:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.739783:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.739784:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.739785:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.739786:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.739787:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.739788:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.739788:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.739791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.739798:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.739800:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.739801:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.739802:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.739804:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.739805:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.739810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:13.0:1619677448.739816:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.739819:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677448.739819:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.739820:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:13.0:1619677448.739821:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.739822:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.739915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.739921:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.739923:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.739924:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.739926:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.739928:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.739930:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677448.739931:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.739933:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.739934:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.739935:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677448.739938:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.739939:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677448.739943:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.739943:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:2.0:1619677448.739944:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:2.0:1619677448.739946:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:2.0:1619677448.739946:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.739947:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.739947:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677448.739951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.739958:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.739961:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000800:00000200:10.0:1619677448.739961:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.739962:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.739964:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.739967:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677448.740027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.740032:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.740035:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.740035:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.740038:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.740040:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.740041:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.740043:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.740044:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.740045:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.740046:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.740050:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.740052:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.740055:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.740056:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.740056:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.740058:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.740058:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.740059:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.740060:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.740061:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:25.2:1619677448.740064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:21.2:1619677448.740066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677448.740071:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677448.740071:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.740073:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.740073:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:30.0:1619677448.740074:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:22.0:1619677448.740074:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:30.0:1619677448.740075:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.740077:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.740078:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.2:1619677448.740078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677448.740080:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:26.0:1619677448.740085:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.740088:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.740089:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.740091:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.740093:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.740095:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.740096:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.740098:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.740098:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.740100:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.740103:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.740105:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.740107:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.740108:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.740109:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.740110:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.740111:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.740111:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.740112:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.740116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.740122:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.740124:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.740125:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.740127:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.740128:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.740130:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.740133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677448.740140:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677448.740143:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677448.740144:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677448.740146:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.740148:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.740150:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:8.0:1619677448.740151:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96710958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.740152:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.740153:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.740154:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677448.740158:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.740159:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:8.0:1619677448.740162:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.740162:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:8.0:1619677448.740163:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:8.0:1619677448.740164:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:8.0:1619677448.740165:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677448.740166:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:8.0:1619677448.740167:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:8.0:1619677448.740167:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:25.2:1619677448.740169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:4.2:1619677448.740173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (21)++
00000800:00000200:20.2:1619677448.740175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677448.740176:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:20.0:1619677448.740178:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.740178:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.740179:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.740179:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.740180:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000400:00000200:10.0:1619677448.740180:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.740181:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:24.0:1619677448.740182:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:24.0:1619677448.740183:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:20.0:1619677448.740183:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.740183:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:8.0:1619677448.740184:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:5.2:1619677448.740184:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:20.0:1619677448.740186:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.740187:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.740188:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.740188:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.740189:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.740190:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.740191:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677448.740191:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.740194:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.740194:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:20.0:1619677448.740196:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.740197:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.740199:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:18.0:1619677448.740200:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.740201:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.740202:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:18.0:1619677448.740202:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.0:1619677448.740203:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:18.0:1619677448.740203:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96710956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.740204:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:18.0:1619677448.740204:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.740205:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:18.0:1619677448.740205:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.740206:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.740206:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677448.740206:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:25.2:1619677448.740210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:18.0:1619677448.740210:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.740211:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677448.740214:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.740215:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:18.0:1619677448.740215:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:18.0:1619677448.740217:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:18.0:1619677448.740217:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677448.740217:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:18.0:1619677448.740219:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.740220:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:6.0:1619677448.740220:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.740221:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.740221:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:6.0:1619677448.740221:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.740223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:6.0:1619677448.740223:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.740225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:6.0:1619677448.740226:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:2.0:1619677448.740231:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:1.0:1619677448.740232:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677448.740233:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.740234:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677448.740234:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:2.0:1619677448.740235:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:1.0:1619677448.740235:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.740236:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.740237:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.740240:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.740249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.740257:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.740260:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.740262:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.740263:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.740264:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.740266:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677448.740267:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.740268:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.740269:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.740270:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677448.740274:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.740275:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677448.740278:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.740278:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.740279:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.740280:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.740281:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.740282:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.740283:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.2:1619677448.740283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:25.2:1619677448.740286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677448.740288:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.740291:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:30.0:1619677448.740292:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:20.0:1619677448.740293:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.740297:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.740300:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (23)--
00000400:00000200:5.0:1619677448.740300:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.740301:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.740302:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.740302:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.740303:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96710954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.740304:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.740304:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.740305:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.740306:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:5.0:1619677448.740306:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.740310:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.740312:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.740314:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.740315:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.740316:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.740318:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.740319:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.740319:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.740320:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677448.740325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.740334:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.740336:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.740337:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.740337:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.740339:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.2:1619677448.740339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677448.740342:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:17.0:1619677448.740345:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.740348:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.740348:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.740350:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.740352:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.740354:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677448.740355:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.740356:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.740357:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.740358:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677448.740361:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.740363:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.740365:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.740366:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.740366:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.740368:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.740368:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.740369:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.740369:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.740374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.740382:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.740384:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.740385:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.740385:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:22.0:1619677448.740387:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.740390:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.740545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.740553:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.740555:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.740556:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.740558:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.740560:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.740562:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.740563:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.740565:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.740566:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.740567:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677448.740570:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.740572:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677448.740575:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.740575:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.740576:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.740578:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.740578:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.740579:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.740579:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.740584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.740591:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.740594:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.740594:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.740595:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.740597:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.740600:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:4.2:1619677448.740803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.740811:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.740814:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.740815:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.740816:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.740818:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.740820:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677448.740821:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96710951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.740823:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.740824:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.740825:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677448.740828:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.740829:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677448.740832:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.740833:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.740833:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:24.0:1619677448.740835:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.740835:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.740836:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.740836:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.740841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.740849:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.740851:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.740852:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.740852:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.740854:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.740857:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.740948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.740953:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.740956:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.740957:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.740958:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.740961:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.740962:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.740963:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.740965:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.740965:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.740967:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.740970:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.740972:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.2:1619677448.740974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677448.740974:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.740975:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.740976:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677448.740976:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.740978:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.740978:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677448.740979:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:17.0:1619677448.740979:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.740979:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:20.0:1619677448.740981:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.740982:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.740983:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.740984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.740985:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677448.740986:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96710949] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.740988:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.740989:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.740990:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:12.0:1619677448.740991:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:20.0:1619677448.740993:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.740994:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:20.0:1619677448.740995:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:12.0:1619677448.740995:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.740997:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.740998:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.740998:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:20.0:1619677448.740999:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:13.0:1619677448.740999:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.741000:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.0:1619677448.741001:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:20.0:1619677448.741002:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.741003:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.741004:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.741005:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.741010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:20.2:1619677448.741010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.741015:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:22.0:1619677448.741018:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:5.0:1619677448.741018:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.741018:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.741019:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677448.741020:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.741021:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.741021:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.741023:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.741025:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.741037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:6.0:1619677448.741044:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.741047:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.741047:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.741048:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.741049:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.741049:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.741257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677448.741265:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677448.741268:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677448.741269:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677448.741271:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677448.741273:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.741274:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677448.741276:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96710948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.741277:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.741278:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.741279:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677448.741282:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677448.741284:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677448.741287:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.741288:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:8.0:1619677448.741288:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677448.741290:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:8.0:1619677448.741290:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677448.741292:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:25.2:1619677448.741297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677448.741299:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:5.2:1619677448.741299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (23)++
00000800:00000200:8.0:1619677448.741303:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (23)--
00000800:00000200:22.0:1619677448.741304:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:8.0:1619677448.741305:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677448.741306:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:22.0:1619677448.741307:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:8.0:1619677448.741307:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677448.741308:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:6.0:1619677448.741308:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:22.0:1619677448.741309:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.741311:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677448.741314:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:20.2:1619677448.741328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.741337:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.741340:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.741342:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.741344:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.741345:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.741346:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.741348:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.741349:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.741350:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.741351:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.741354:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.741356:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.741358:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.741359:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.741360:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.741361:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.741362:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.741362:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.741363:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.741367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.741374:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.741377:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.741377:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.741379:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.741381:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.741382:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:4.2:1619677448.741409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.741416:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.741419:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.741420:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.741421:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.741424:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.741425:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.741427:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.741428:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.741429:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.741430:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.741433:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.741435:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.741439:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.741439:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.741440:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.741442:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.741442:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.741443:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.741443:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.741447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.741455:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.741458:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.741458:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.741459:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.741461:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.741464:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.741474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.741479:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.741482:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.741483:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.741485:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.741487:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.741489:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:5.2:1619677448.741489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:22.0:1619677448.741490:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710945] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.741492:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.741493:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677448.741494:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.741495:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677448.741497:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.741498:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:22.0:1619677448.741499:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.741499:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.741501:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.741502:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.741503:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.741503:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:1.0:1619677448.741504:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.741505:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.741505:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:22.0:1619677448.741506:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.741506:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.741507:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:1.0:1619677448.741507:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96710944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.741508:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.741509:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.741510:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:8.2:1619677448.741512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:1.0:1619677448.741513:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.741516:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.741519:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.741520:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:17.0:1619677448.741521:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.741521:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.741522:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:1.0:1619677448.741522:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:17.0:1619677448.741523:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.741523:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.741524:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:21.2:1619677448.741525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000400:00000200:17.0:1619677448.741525:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.741525:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677448.741526:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.741528:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:25.2:1619677448.741529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:24.0:1619677448.741533:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.741536:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.741536:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:6.0:1619677448.741536:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:24.0:1619677448.741539:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.741539:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.741540:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.741540:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.741541:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.741542:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:24.0:1619677448.741543:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677448.741545:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96710943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.741545:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.741546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000400:00000200:24.0:1619677448.741546:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.741547:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.741548:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677448.741551:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.741553:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677448.741553:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.741556:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:24.0:1619677448.741557:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.741557:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:24.0:1619677448.741558:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.741558:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:22.0:1619677448.741558:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (21)++
00000800:00000200:22.0:1619677448.741559:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.741559:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:24.0:1619677448.741560:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.741561:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.741562:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.741563:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677448.741567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.741574:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.741577:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.741578:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.741578:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.741580:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.741583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.741587:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.741588:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.741591:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677448.741591:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.741592:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.741592:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (23)--
00000800:00000200:2.0:1619677448.741593:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:20.2:1619677448.741643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.741649:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.741651:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.741653:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.741654:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.741656:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.741657:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:5.0:1619677448.741659:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.741660:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.741661:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.741662:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:5.0:1619677448.741665:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.741666:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.0:1619677448.741669:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.741669:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:5.0:1619677448.741670:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.0:1619677448.741672:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:5.0:1619677448.741672:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.741673:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.741674:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677448.741678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:20.2:1619677448.741683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.0:1619677448.741686:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:1.0:1619677448.741686:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:20.0:1619677448.741689:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:1.0:1619677448.741689:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.741690:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.741690:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.741692:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.741692:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:1.0:1619677448.741692:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.741694:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.741695:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000400:00000200:20.0:1619677448.741696:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:20.0:1619677448.741698:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.741699:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.741700:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.741701:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677448.741704:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.741705:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.741707:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.741708:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.741708:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.741710:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.741711:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677448.741711:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.0:1619677448.741712:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677448.741716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.741724:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.741727:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.741727:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.741729:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.741730:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.741731:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.741817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.741825:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.741828:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.741828:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.741831:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.741833:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.741835:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.741836:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.741837:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.741838:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.741839:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.741842:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.741844:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.741845:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.741846:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.741846:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.741848:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.741848:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.741849:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.741850:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.741854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.741860:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.741862:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.741862:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.741863:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.741864:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.741865:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:20.2:1619677448.741980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.741988:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.741990:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.741992:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.741994:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.741994:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.741996:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.741997:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.741998:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.741999:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.742000:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.742003:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.742005:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:4.2:1619677448.742005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677448.742008:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.742008:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.742009:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.742011:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.742011:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:26.0:1619677448.742012:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.742013:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:26.0:1619677448.742014:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:30.0:1619677448.742015:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.742015:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:10.0:1619677448.742015:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.742017:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677448.742017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:26.0:1619677448.742018:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.742020:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.742021:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96710938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.742022:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.742023:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677448.742023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677448.742024:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.742027:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.742029:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.0:1619677448.742029:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:31.0:1619677448.742031:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:26.0:1619677448.742031:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.742031:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:26.0:1619677448.742032:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:20.0:1619677448.742032:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.742033:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:26.0:1619677448.742033:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:31.0:1619677448.742034:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677448.742034:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000400:00000200:20.0:1619677448.742034:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.742034:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.742035:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:26.0:1619677448.742035:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.742035:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.742036:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677448.742037:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:25.2:1619677448.742040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:6.0:1619677448.742048:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.742051:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.742052:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.742052:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.742054:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.742057:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:4.2:1619677448.742070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.742077:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.742079:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.742080:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.742082:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677448.742083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:8.0:1619677448.742085:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.742086:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.742088:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96710937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.742089:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677448.742089:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.742090:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.742091:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.742092:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.742092:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677448.742095:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.742096:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.742098:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.742099:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677448.742100:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.742100:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.742100:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:17.0:1619677448.742102:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:8.0:1619677448.742102:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.742102:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:5.2:1619677448.742102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:17.0:1619677448.742103:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96710936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.742103:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677448.742104:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.742104:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677448.742105:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.742106:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:18.0:1619677448.742108:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:8.2:1619677448.742108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:17.0:1619677448.742110:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.742111:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:17.0:1619677448.742112:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.742112:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.742114:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.742116:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.742117:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.742117:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000400:00000200:18.0:1619677448.742118:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:17.0:1619677448.742118:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:1.0:1619677448.742118:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:18.0:1619677448.742119:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96710935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.742119:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:1.0:1619677448.742119:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.742120:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.742121:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.742121:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:18.0:1619677448.742122:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.742122:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:1.0:1619677448.742122:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.742124:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.742125:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.742125:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:18.0:1619677448.742126:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.742126:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.742126:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:25.2:1619677448.742127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677448.742128:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.2:1619677448.742130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:18.0:1619677448.742131:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.742132:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.742132:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.742133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.742134:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.742134:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:18.0:1619677448.742135:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.742135:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.742136:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677448.742136:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.742137:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.742139:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000400:00000200:13.0:1619677448.742139:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.742140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.742140:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.742141:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:12.0:1619677448.742141:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.742141:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.742142:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:13.0:1619677448.742142:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:1.0:1619677448.742142:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.742143:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:12.0:1619677448.742143:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.742144:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.742144:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:17.0:1619677448.742145:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677448.742146:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.742147:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.742149:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.742150:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.742152:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.742185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:21.2:1619677448.742190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677448.742191:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677448.742193:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677448.742195:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677448.742196:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.742198:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:2.0:1619677448.742199:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.742200:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677448.742201:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:2.0:1619677448.742201:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.742203:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.742204:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.742204:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.742205:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:2.0:1619677448.742205:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.742207:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.742209:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.742210:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.742211:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96710933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.742211:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.742212:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.742214:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.742215:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.742215:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.742216:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.742216:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.742218:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:31.0:1619677448.742219:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.742220:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:31.0:1619677448.742221:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.742222:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:25.2:1619677448.742223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:2.0:1619677448.742223:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:31.0:1619677448.742224:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.742224:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:31.0:1619677448.742225:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.742226:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.742227:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.742228:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.2:1619677448.742228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:31.0:1619677448.742229:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.742229:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677448.742231:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.742233:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:9.2:1619677448.742233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000400:00000200:13.0:1619677448.742234:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.742236:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.742237:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.742238:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:13.0:1619677448.742238:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677448.742239:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677448.742240:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:25.2:1619677448.742242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.742242:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.742242:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677448.742244:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.742245:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.742245:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.742247:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.742249:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.742250:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:12.0:1619677448.742251:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.742252:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.742253:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.742253:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.742255:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:21.2:1619677448.742270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:22.0:1619677448.742275:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.742278:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:6.0:1619677448.742279:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.742281:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677448.742282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:22.0:1619677448.742283:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.742284:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.2:1619677448.742285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:22.0:1619677448.742286:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.742287:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.742288:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.742288:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:22.0:1619677448.742289:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:5.0:1619677448.742290:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.742293:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677448.742293:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.742299:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:22.0:1619677448.742299:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:5.0:1619677448.742299:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.742301:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:5.0:1619677448.742301:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.742302:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:5.0:1619677448.742302:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:22.0:1619677448.742304:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.742304:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:5.0:1619677448.742304:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96710931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.742305:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:5.0:1619677448.742305:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.742306:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.742307:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:10.0:1619677448.742307:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.742307:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.742307:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.742308:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.742308:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000400:00000200:10.0:1619677448.742308:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:22.0:1619677448.742309:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:10.0:1619677448.742310:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96710930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.742312:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.742312:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.742313:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677448.742314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:10.0:1619677448.742315:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.742315:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.742318:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.742319:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.742320:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:5.0:1619677448.742320:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.742321:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.742322:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.742322:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:5.0:1619677448.742323:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.742323:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677448.742324:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.742324:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:10.0:1619677448.742325:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:1.0:1619677448.742325:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:10.0:1619677448.742326:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000400:00000200:1.0:1619677448.742326:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.742327:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:10.0:1619677448.742327:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.742328:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:25.2:1619677448.742329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:10.0:1619677448.742329:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.742329:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:1.0:1619677448.742329:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.742331:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.742333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:9.2:1619677448.742334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.742337:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.742340:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:17.0:1619677448.742340:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677448.742341:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.742342:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000400:00000200:30.0:1619677448.742343:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.742343:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.742344:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:1.0:1619677448.742344:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.742345:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677448.742345:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:30.0:1619677448.742346:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:17.0:1619677448.742346:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000400:00000200:2.0:1619677448.742346:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.742347:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:17.0:1619677448.742347:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:2.0:1619677448.742348:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.742349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:2.0:1619677448.742352:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:31.0:1619677448.742356:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.742359:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677448.742360:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.742361:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:31.0:1619677448.742362:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.742363:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.742523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677448.742530:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.742532:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:21.2:1619677448.742533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:31.0:1619677448.742535:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.742536:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.742537:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677448.742538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:31.0:1619677448.742539:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677448.742540:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.742541:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677448.742541:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:31.0:1619677448.742543:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677448.742543:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:31.0:1619677448.742544:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:2.0:1619677448.742544:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.742546:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:18.0:1619677448.742547:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:31.0:1619677448.742550:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.742550:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.742551:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.742553:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:18.0:1619677448.742553:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677448.742554:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96710928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.742556:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677448.742556:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.742557:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:18.0:1619677448.742557:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.742558:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:18.0:1619677448.742558:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.742559:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:2.0:1619677448.742559:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.742560:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.742560:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.742561:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:2.0:1619677448.742561:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677448.742562:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96710927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.742563:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:5.2:1619677448.742564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:2.0:1619677448.742564:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.742565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:18.0:1619677448.742565:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:2.0:1619677448.742565:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.742566:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677448.742569:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.742571:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.742572:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677448.742572:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677448.742573:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:2.0:1619677448.742573:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.742573:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.742574:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:18.0:1619677448.742574:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:2.0:1619677448.742574:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.742575:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:13.0:1619677448.742575:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:18.0:1619677448.742576:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:13.0:1619677448.742576:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.742576:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:2.0:1619677448.742576:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.742577:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677448.742577:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)++
00000400:00000200:13.0:1619677448.742578:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.742578:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000400:00000200:26.0:1619677448.742579:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.742579:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (23)++
00000800:00000200:2.0:1619677448.742579:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.742580:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (24)--
00000800:00000200:12.0:1619677448.742580:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:4.2:1619677448.742580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677448.742581:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:13.0:1619677448.742581:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677448.742581:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:26.0:1619677448.742582:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.742582:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:26.0:1619677448.742583:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:25.2:1619677448.742583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:9.2:1619677448.742584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:26.0:1619677448.742585:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96710926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.742586:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.742587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:26.0:1619677448.742588:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.742588:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:26.0:1619677448.742589:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.742589:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:6.0:1619677448.742591:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000400:00000200:26.0:1619677448.742592:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.742593:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:26.0:1619677448.742594:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677448.742594:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:2.0:1619677448.742594:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.742595:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:10.0:1619677448.742595:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.742595:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.742595:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.742596:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.742596:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:18.0:1619677448.742596:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:26.0:1619677448.742597:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:18.0:1619677448.742597:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:26.0:1619677448.742598:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:18.0:1619677448.742598:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:10.0:1619677448.742598:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.742598:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.742599:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:18.0:1619677448.742599:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:6.0:1619677448.742599:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:10.0:1619677448.742600:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:2.0:1619677448.742600:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:10.0:1619677448.742602:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96710925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.742602:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:10.0:1619677448.742603:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.742604:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.742605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000400:00000200:10.0:1619677448.742605:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677448.742606:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.742607:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.742608:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677448.742609:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.742610:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.742611:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:26.0:1619677448.742611:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.742611:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.742612:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.742614:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.742615:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.742616:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.742616:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.742617:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:10.0:1619677448.742617:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.742619:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.742620:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.742621:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.742622:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.742622:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677448.742625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:4.2:1619677448.742628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.742633:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:26.0:1619677448.742635:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677448.742635:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.742636:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.742636:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.742637:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.742638:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.742639:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.742639:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.742641:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.2:1619677448.742684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677448.742687:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677448.742689:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677448.742691:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.742692:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677448.742694:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.742696:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:20.0:1619677448.742698:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.742699:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.742700:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.742701:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:20.0:1619677448.742704:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.742706:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:20.0:1619677448.742709:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.742709:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:20.0:1619677448.742710:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:20.0:1619677448.742712:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:20.0:1619677448.742712:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677448.742713:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677448.742714:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:25.2:1619677448.742717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.742723:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.742726:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000800:00000200:26.0:1619677448.742727:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000400:00000200:10.0:1619677448.742727:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.742729:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.742732:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:25.2:1619677448.742738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (20)++
00000800:00000200:26.0:1619677448.742744:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.742746:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677448.742747:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)++
00000800:00000200:10.0:1619677448.742747:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.742749:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:26.0:1619677448.742749:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:4.2:1619677448.742778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677448.742785:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.2:1619677448.742786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:24.0:1619677448.742787:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677448.742788:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677448.742790:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.742792:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:24.0:1619677448.742793:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.742795:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:18.0:1619677448.742795:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.742795:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:24.0:1619677448.742798:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96710923] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.742798:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.742799:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.742800:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.742801:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:18.0:1619677448.742801:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.742802:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.742804:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96710922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677448.742806:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677448.742806:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.742807:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.742809:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:18.0:1619677448.742809:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.742812:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.742813:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.742813:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:24.0:1619677448.742814:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:18.0:1619677448.742814:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677448.742816:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:24.0:1619677448.742816:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677448.742816:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.742816:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:24.0:1619677448.742817:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677448.742817:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:24.0:1619677448.742818:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677448.742818:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.742819:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.742820:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.742820:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677448.742822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:9.2:1619677448.742825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:1.0:1619677448.742829:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:2.0:1619677448.742832:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:1.0:1619677448.742832:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.742833:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.742833:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.742834:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.742835:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.742836:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:1.0:1619677448.742836:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.742837:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.742839:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:1.0:1619677448.742839:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:21.2:1619677448.742927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.742932:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.742935:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.742936:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.742938:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677448.742939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:6.0:1619677448.742940:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.742942:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677448.742943:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.742944:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.742945:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.742946:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677448.742946:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.742949:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:6.0:1619677448.742950:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.742952:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677448.742952:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677448.742953:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677448.742954:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.742955:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.742955:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000400:00000200:30.0:1619677448.742956:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:6.0:1619677448.742956:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000400:00000200:30.0:1619677448.742957:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96710920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.742958:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000400:00000200:30.0:1619677448.742959:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.742959:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.742959:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.742959:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.742960:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:30.0:1619677448.742961:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.742964:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677448.742964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.742966:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.742968:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.742969:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:30.0:1619677448.742970:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:30.0:1619677448.742971:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:30.0:1619677448.742972:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:5.0:1619677448.742972:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:30.0:1619677448.742974:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:5.0:1619677448.742974:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000800:00000200:30.0:1619677448.742975:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:5.0:1619677448.742975:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.742976:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677448.742977:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.742977:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:25.2:1619677448.742978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.742979:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:21.2:1619677448.742982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.742985:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.742987:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:8.2:1619677448.742987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000400:00000200:12.0:1619677448.742988:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.742990:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:12.0:1619677448.742990:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.742991:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.742992:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:12.0:1619677448.742992:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.742993:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.742994:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677448.742994:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.742997:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.742998:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.742998:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.742999:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.743001:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.743006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.743013:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.743016:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.743017:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.743019:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.743021:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.743023:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.743024:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.743025:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.743026:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.743027:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.743030:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.743032:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.743033:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.743034:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.743035:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.743036:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.743036:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.743037:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.743037:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677448.743038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:25.2:1619677448.743041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:17.0:1619677448.743044:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677448.743047:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:17.0:1619677448.743047:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.743047:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677448.743049:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:17.0:1619677448.743049:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.743050:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.743051:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.743052:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.743052:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.743053:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:26.0:1619677448.743054:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:17.0:1619677448.743054:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.743056:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.743057:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.743058:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.2:1619677448.743061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:17.0:1619677448.743061:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.743063:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.2:1619677448.743064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:20.0:1619677448.743064:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677448.743065:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.743065:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:17.0:1619677448.743066:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.0:1619677448.743067:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:17.0:1619677448.743068:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.743068:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:20.0:1619677448.743069:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.743069:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.743070:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677448.743070:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.0:1619677448.743070:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.743072:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.743073:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:20.0:1619677448.743073:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:10.0:1619677448.743073:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:8.2:1619677448.743073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677448.743074:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.743074:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:20.0:1619677448.743075:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96710917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677448.743075:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:20.0:1619677448.743076:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.743077:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.743078:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:1.0:1619677448.743080:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:20.0:1619677448.743081:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.743083:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:1.0:1619677448.743083:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:1.0:1619677448.743083:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.743084:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.743086:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677448.743086:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.743087:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:20.0:1619677448.743088:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:1.0:1619677448.743088:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.0:1619677448.743089:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:20.0:1619677448.743090:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.743091:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.743092:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.743096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:20.2:1619677448.743103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:18.0:1619677448.743103:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.743106:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.743107:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.743107:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.743109:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.743109:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.2:1619677448.743111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677448.743111:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677448.743111:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.743113:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677448.743115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (20)++
00000400:00000200:5.0:1619677448.743116:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.743117:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677448.743118:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:5.0:1619677448.743118:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:5.0:1619677448.743119:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710916] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.743121:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:5.0:1619677448.743121:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.743122:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677448.743122:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000400:00000200:5.0:1619677448.743123:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.743125:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.743125:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:18.0:1619677448.743125:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.743125:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.743126:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000400:00000200:5.0:1619677448.743127:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.743127:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000400:00000200:31.0:1619677448.743128:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.743128:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:5.0:1619677448.743129:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:31.0:1619677448.743130:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:5.0:1619677448.743131:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.743132:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96710915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.0:1619677448.743132:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:31.0:1619677448.743133:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.0:1619677448.743133:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000400:00000200:31.0:1619677448.743134:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.0:1619677448.743134:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:31.0:1619677448.743135:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:5.0:1619677448.743135:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:31.0:1619677448.743138:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.743140:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.2:1619677448.743140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.0:1619677448.743140:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:5.0:1619677448.743141:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:5.0:1619677448.743142:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677448.743143:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.743144:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.743145:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.743146:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:31.0:1619677448.743147:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:20.2:1619677448.743147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:31.0:1619677448.743148:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:17.0:1619677448.743148:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:31.0:1619677448.743149:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:20.0:1619677448.743149:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:17.0:1619677448.743150:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.743151:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.743151:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:20.0:1619677448.743152:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:8.2:1619677448.743152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000400:00000200:17.0:1619677448.743153:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.743154:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.743156:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677448.743156:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.0:1619677448.743157:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.743160:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.743161:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.743162:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.743163:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.743165:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.743203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.743210:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.743213:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.743214:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.743216:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.743217:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.743219:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.743220:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.743221:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.743222:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.743223:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.743226:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.743228:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.743231:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.743232:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.743233:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.743234:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.743235:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.743236:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.743237:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.743238:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677448.743241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.2:1619677448.743243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677448.743248:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.743251:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:10.0:1619677448.743251:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:18.0:1619677448.743252:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.743252:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:2.0:1619677448.743252:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.743253:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.743254:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:18.0:1619677448.743255:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.743258:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.743260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.743267:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.743270:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.743271:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.743271:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.743272:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.743272:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.743336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.743341:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.743344:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.743345:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.743347:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.743348:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.743349:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.743350:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.743351:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.743352:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.743353:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.743356:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.743358:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.743360:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.743360:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.743361:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.743363:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.743363:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.743364:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677448.743364:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:9.2:1619677448.743369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.743376:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.743378:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.743378:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.743380:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.743380:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.743381:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.743434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677448.743440:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.743442:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677448.743443:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.743444:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.743446:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.743447:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.743448:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.743450:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.743450:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.743451:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.743454:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.743456:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.743459:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.743459:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.743460:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.743462:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:18.0:1619677448.743462:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.743463:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.743463:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677448.743468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.743476:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.743478:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.743479:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.743481:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.743483:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.743484:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.743579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.743586:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.743589:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.743590:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.743592:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.743595:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.743596:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.743598:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.743599:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.743600:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.743601:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.743604:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.743605:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.743607:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.743608:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.743608:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.743610:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.743610:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.743611:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.743611:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.743615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.743621:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.743623:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.743623:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.743624:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.743625:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.743626:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:5.2:1619677448.743634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.743641:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.743644:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.743644:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.743646:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.743648:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.743649:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.743650:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.743652:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.743653:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.743654:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.743657:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.743659:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.743660:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.743661:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.743662:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.743663:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.743664:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.743665:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.743665:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677448.743669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:8.2:1619677448.743672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:30.0:1619677448.743677:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677448.743679:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:30.0:1619677448.743680:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:17.0:1619677448.743681:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.743681:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.743682:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.743682:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.743683:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.743684:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.743685:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.743685:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:30.0:1619677448.743686:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677448.743687:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.743689:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.743690:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.743691:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677448.743695:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.743697:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677448.743700:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.743701:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.743702:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:30.0:1619677448.743704:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.743704:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.743705:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.743706:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.743710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.2:1619677448.743713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.743719:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.743721:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.743721:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677448.743722:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:20.0:1619677448.743723:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.743723:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677448.743725:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.743726:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.743728:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:5.0:1619677448.743729:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710908] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.743731:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.743731:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.743733:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677448.743734:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:20.0:1619677448.743735:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.743736:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000400:00000200:5.0:1619677448.743736:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.743738:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:5.0:1619677448.743741:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.743742:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:5.0:1619677448.743743:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.743745:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.743746:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.743747:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.0:1619677448.743748:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.743749:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.743750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:20.2:1619677448.743753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.743756:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:12.0:1619677448.743757:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:20.0:1619677448.743758:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.743758:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.743759:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.2:1619677448.743760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:12.0:1619677448.743760:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.743761:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.743762:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.743763:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.743765:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:31.0:1619677448.743768:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.743771:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.743773:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.743775:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.743776:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.743778:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677448.743779:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.743781:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.743782:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.743783:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.743786:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.743788:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677448.743790:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.743791:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:31.0:1619677448.743792:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:31.0:1619677448.743793:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.743794:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.743795:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.743796:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677448.743797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:25.2:1619677448.743799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.743804:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677448.743805:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:26.0:1619677448.743807:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:13.0:1619677448.743807:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.743808:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.743808:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.743810:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677448.743810:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.743811:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.743812:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:26.0:1619677448.743813:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.743814:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.743815:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.743817:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.743817:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.743817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:26.0:1619677448.743818:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.743821:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.743823:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677448.743823:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.743826:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.743826:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.743827:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.743827:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:12.0:1619677448.743827:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.743827:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.743828:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.743829:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.743829:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.743829:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:26.0:1619677448.743830:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.743831:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.743835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.743843:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.743846:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.743847:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.743847:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.743849:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.743853:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.744042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.744049:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.744052:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.744053:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.744055:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.744057:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.744059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:8.0:1619677448.744059:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.744060:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96710905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.744062:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:8.0:1619677448.744062:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.744063:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.744064:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:4.2:1619677448.744064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677448.744066:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:8.0:1619677448.744068:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.744068:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677448.744069:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:8.0:1619677448.744070:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677448.744071:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677448.744072:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.744073:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.744074:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677448.744074:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:10.0:1619677448.744074:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:8.0:1619677448.744074:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000400:00000200:20.0:1619677448.744075:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96710904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677448.744075:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.744076:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:20.0:1619677448.744077:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677448.744077:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677448.744078:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.744078:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:20.0:1619677448.744079:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.744079:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:8.0:1619677448.744079:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.744080:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:10.0:1619677448.744081:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.744082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:20.0:1619677448.744085:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:4.2:1619677448.744085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677448.744087:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:20.0:1619677448.744088:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.744088:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96710903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.744090:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.744090:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:20.0:1619677448.744091:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.744091:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677448.744092:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:20.0:1619677448.744092:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:10.0:1619677448.744092:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677448.744093:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:1.0:1619677448.744093:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:24.0:1619677448.744094:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677448.744094:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.744094:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.744095:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677448.744095:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:24.0:1619677448.744096:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677448.744096:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:10.0:1619677448.744096:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.744097:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677448.744097:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:1.0:1619677448.744097:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.744098:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677448.744099:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:8.2:1619677448.744100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:25.2:1619677448.744101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.744101:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.744102:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.744103:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.744105:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.744106:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.2:1619677448.744106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677448.744107:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.744108:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.744108:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.744110:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:17.0:1619677448.744110:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000800:00000200:25.2:1619677448.744111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.744111:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.744112:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.744113:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.744113:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677448.744114:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:17.0:1619677448.744114:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (22)++
00000800:00000200:1.0:1619677448.744114:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.744115:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.744116:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:26.0:1619677448.744117:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:17.0:1619677448.744117:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:1.0:1619677448.744117:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677448.744118:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.744118:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.744120:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:1.0:1619677448.744121:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.744122:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96710902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677448.744123:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.744124:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.744124:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:1.0:1619677448.744124:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.744124:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.744126:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:22.0:1619677448.744127:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.744130:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:1.0:1619677448.744130:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.744132:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.744135:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.744135:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.744136:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.744138:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.744138:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.744139:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.744140:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.744154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.744160:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.744163:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.744164:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.744165:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.744167:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.744170:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.744177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677448.744183:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.744185:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677448.744186:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.744186:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.744187:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.744188:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.744285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.744291:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.744294:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.744299:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:6.0:1619677448.744302:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.744304:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.744305:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677448.744307:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.744308:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.744309:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.744310:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677448.744314:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.744316:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677448.744319:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.744320:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:6.0:1619677448.744320:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.744322:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.744323:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.744324:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:6.0:1619677448.744325:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.744326:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:25.2:1619677448.744329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:21.2:1619677448.744332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000800:00000200:30.0:1619677448.744337:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:22.0:1619677448.744338:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.744339:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:30.0:1619677448.744340:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:22.0:1619677448.744340:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:30.0:1619677448.744341:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.744341:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:30.0:1619677448.744343:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.744344:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.744346:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.744380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.744387:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.744389:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677448.744392:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.744393:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.744394:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.744396:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677448.744397:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.744398:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.744399:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.744400:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677448.744403:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.744405:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677448.744409:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.744409:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:31.0:1619677448.744410:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:31.0:1619677448.744412:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:31.0:1619677448.744412:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.744413:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.744414:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.744418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.744425:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.744428:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.744429:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.744430:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.744432:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:4.2:1619677448.744434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677448.744435:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:25.2:1619677448.744436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.744441:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.744443:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.744445:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.744445:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:22.0:1619677448.744446:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.744447:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.744448:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000400:00000200:10.0:1619677448.744448:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677448.744449:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:22.0:1619677448.744449:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:10.0:1619677448.744451:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.744452:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.744454:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.744455:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.744456:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.744457:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.744460:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.744462:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.744465:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.744466:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.744466:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.744468:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.744468:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.744470:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677448.744471:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677448.744471:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.2:1619677448.744472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:25.2:1619677448.744474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000800:00000200:4.2:1619677448.744477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677448.744478:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.744481:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:6.0:1619677448.744481:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:1.0:1619677448.744482:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.744483:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.744484:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677448.744484:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.744485:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000400:00000200:6.0:1619677448.744485:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.744486:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.744486:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677448.744486:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.744487:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.744488:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:17.0:1619677448.744488:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.744489:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96710898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.744490:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:17.0:1619677448.744491:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.744491:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677448.744493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000400:00000200:17.0:1619677448.744493:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677448.744496:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.744498:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.744499:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.744500:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.744500:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.744501:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.744502:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:22.0:1619677448.744503:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:17.0:1619677448.744503:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.744504:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.744504:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677448.744504:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.744505:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:17.0:1619677448.744505:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677448.744506:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:17.0:1619677448.744506:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677448.744509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:5.2:1619677448.744512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.744518:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.744519:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.744519:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.744520:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.744520:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.744521:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:1.0:1619677448.744521:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.744522:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.744523:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.744524:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.744623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.744631:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.744633:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.744634:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.744637:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.744640:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.744641:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.744642:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.744644:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.744645:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.744646:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.744649:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.744651:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.744653:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.744654:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.744655:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.744657:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.744657:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.744658:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.744658:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.744662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.744670:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.744672:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.744673:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.744674:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.744676:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.744678:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:25.2:1619677448.744681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:12.0:1619677448.744687:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.744690:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.744690:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.744691:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.744692:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.744693:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.744718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.744725:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.744728:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.744729:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.744731:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677448.744733:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.744734:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.744736:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.744737:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.744738:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.744739:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.744742:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677448.744744:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.744745:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.744746:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.744746:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.744748:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.744748:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.744749:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.744750:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.744754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.744761:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.744764:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.744764:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.744765:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.744766:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.744768:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.744771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.744776:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.744779:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.744779:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.744781:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.744783:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.744784:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677448.744786:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710895] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.744787:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.744788:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.744789:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677448.744792:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.744794:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677448.744797:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.744798:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:17.0:1619677448.744799:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:17.0:1619677448.744800:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:17.0:1619677448.744801:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.744802:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.744802:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.744806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000800:00000200:8.0:1619677448.744813:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:8.0:1619677448.744815:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000800:00000200:24.0:1619677448.744816:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:8.0:1619677448.744817:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.744819:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.744822:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:20.2:1619677448.744904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.744910:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.744912:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.744914:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.744915:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.744918:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.744919:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.744920:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.744922:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.744923:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.744923:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.744927:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.744928:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.744931:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.744932:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.744933:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.744934:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.744935:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.744936:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.744937:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.744940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.744948:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.744950:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.744951:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.744951:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.744953:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.744956:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.744958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:9.2:1619677448.744961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:30.0:1619677448.744966:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677448.744967:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:30.0:1619677448.744969:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:2.0:1619677448.744970:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677448.744971:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.744971:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.744971:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:31.0:1619677448.744972:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677448.744972:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.744973:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000400:00000200:30.0:1619677448.744974:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.744976:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677448.744977:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.744978:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.744979:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.744980:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.744983:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.744985:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.744988:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.744989:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:30.0:1619677448.744990:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.744991:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:30.0:1619677448.744992:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.744993:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.744994:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.744998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.745005:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.745008:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.745008:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000400:00000200:6.0:1619677448.745009:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.745011:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.745014:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.745084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.745090:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.745093:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.745094:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.2:1619677448.745095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:1.0:1619677448.745096:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.745098:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.745099:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.745101:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.745102:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.745103:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:1.0:1619677448.745103:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.745104:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:31.0:1619677448.745106:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:1.0:1619677448.745108:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.745109:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.745109:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:1.0:1619677448.745110:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:31.0:1619677448.745112:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.745112:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.745112:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.745113:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677448.745114:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:1.0:1619677448.745114:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000400:00000200:31.0:1619677448.745115:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96710891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677448.745115:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677448.745116:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.745116:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.745116:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677448.745117:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.745118:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:8.2:1619677448.745120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000400:00000200:31.0:1619677448.745122:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.745124:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:4.2:1619677448.745125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677448.745126:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.745127:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:31.0:1619677448.745127:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:17.0:1619677448.745128:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:31.0:1619677448.745129:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.745129:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677448.745130:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:31.0:1619677448.745131:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000400:00000200:17.0:1619677448.745131:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.745131:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:31.0:1619677448.745132:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:26.0:1619677448.745132:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677448.745133:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:17.0:1619677448.745133:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.745134:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:25.2:1619677448.745134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:17.0:1619677448.745135:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:10.0:1619677448.745135:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.2:1619677448.745136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:26.0:1619677448.745137:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.745139:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.745139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677448.745141:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:22.0:1619677448.745141:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:26.0:1619677448.745142:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96710890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.745143:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000400:00000200:26.0:1619677448.745143:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.745144:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:30.0:1619677448.745145:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:26.0:1619677448.745145:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.745145:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:30.0:1619677448.745146:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.745147:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000400:00000200:26.0:1619677448.745147:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677448.745148:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000400:00000200:22.0:1619677448.745148:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.745149:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.745149:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.745150:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.745151:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:26.0:1619677448.745151:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.745152:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:26.0:1619677448.745152:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:22.0:1619677448.745154:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.745155:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.745156:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000400:00000200:22.0:1619677448.745156:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:26.0:1619677448.745157:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:22.0:1619677448.745158:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96710889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.745159:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.745159:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677448.745159:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.745160:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677448.745160:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.745161:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677448.745161:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:25.2:1619677448.745165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:22.0:1619677448.745165:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.745166:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677448.745170:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.745171:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:22.0:1619677448.745172:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.745173:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.745174:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:6.0:1619677448.745174:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:22.0:1619677448.745175:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000400:00000200:6.0:1619677448.745175:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.745176:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.745177:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000400:00000200:6.0:1619677448.745177:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.745178:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:25.2:1619677448.745180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.745180:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:31.0:1619677448.745187:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.745190:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.745191:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.745192:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.745194:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.745195:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.2:1619677448.745357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677448.745364:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.745367:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677448.745367:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677448.745369:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.745372:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.745374:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677448.745375:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.745376:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.745377:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.745378:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677448.745382:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.745384:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677448.745387:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.745388:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:10.0:1619677448.745389:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:10.0:1619677448.745390:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:10.0:1619677448.745391:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677448.745392:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677448.745393:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677448.745394:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677448.745397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:20.2:1619677448.745401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.745408:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677448.745409:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:26.0:1619677448.745410:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677448.745410:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.745411:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:20.0:1619677448.745411:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:20.0:1619677448.745412:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.745414:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.745415:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:20.0:1619677448.745417:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.745511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.745517:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.745520:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.745520:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.745523:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.745524:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.745525:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677448.745527:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.745528:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.745529:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.745530:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677448.745533:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.745535:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677448.745536:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.745537:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.745538:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.745539:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.745539:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.745540:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.745541:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.745545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.745552:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:17.0:1619677448.745554:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:17.0:1619677448.745554:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.745555:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:17.0:1619677448.745556:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.745557:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:8.2:1619677448.745563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (20)++
00000800:00000200:1.0:1619677448.745570:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.745573:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.745574:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.745574:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.745574:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd27e00] (23)--
00000800:00000200:1.0:1619677448.745575:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:20.2:1619677448.745594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.0:1619677448.745599:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.745602:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:5.0:1619677448.745604:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.745605:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:5.0:1619677448.745607:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.745608:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:5.0:1619677448.745609:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.745610:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.745611:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.745612:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.745615:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.745617:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.745619:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.745620:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.745620:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.745622:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.745623:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.745624:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.745624:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677448.745629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.745638:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.745641:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.745642:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.745642:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.745644:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.745646:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.745664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677448.745667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677448.745669:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.745672:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.745672:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677448.745674:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.745675:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:2.0:1619677448.745676:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.745677:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:10.0:1619677448.745678:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.745679:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:2.0:1619677448.745679:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.745681:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.745682:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.745682:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.745683:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:10.0:1619677448.745685:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.745686:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.745687:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677448.745689:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96710884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677448.745689:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:10.0:1619677448.745690:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.745691:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.745692:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:2.0:1619677448.745692:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.745693:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.745693:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.745695:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000400:00000200:10.0:1619677448.745696:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.745696:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.745696:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.745697:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:10.0:1619677448.745698:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677448.745698:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:10.0:1619677448.745700:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.745701:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:8.2:1619677448.745701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:10.0:1619677448.745702:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:10.0:1619677448.745703:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.2:1619677448.745703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:10.0:1619677448.745704:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.745704:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.745705:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677448.745708:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:25.2:1619677448.745709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:18.0:1619677448.745709:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677448.745711:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:17.0:1619677448.745711:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677448.745712:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677448.745712:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.745712:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.745713:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:17.0:1619677448.745714:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.745716:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:17.0:1619677448.745716:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:21.2:1619677448.745717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:30.0:1619677448.745718:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.745719:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.745721:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.745723:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.745724:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:24.0:1619677448.745724:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.745727:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677448.745728:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.2:1619677448.745729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (20)++
00000400:00000200:24.0:1619677448.745729:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.745732:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.745733:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677448.745734:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96710883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.745735:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)++
00000400:00000200:24.0:1619677448.745736:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.745737:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677448.745738:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677448.745738:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.745739:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.745739:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71b400] (23)--
00000800:00000200:30.0:1619677448.745740:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.745741:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000400:00000200:24.0:1619677448.745741:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.745743:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677448.745746:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.745747:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:24.0:1619677448.745747:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:24.0:1619677448.745749:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:24.0:1619677448.745749:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677448.745750:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677448.745751:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677448.745755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:4.2:1619677448.745757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.745764:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677448.745764:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:20.0:1619677448.745766:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:26.0:1619677448.745767:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:20.0:1619677448.745767:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.745768:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677448.745769:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.745770:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:26.0:1619677448.745771:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.745773:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000400:00000200:26.0:1619677448.745774:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677448.745774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (20)++
00000400:00000200:26.0:1619677448.745775:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.745776:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.745778:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.745779:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.745780:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.745781:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (21)++
00000400:00000200:26.0:1619677448.745783:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.745784:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:26.0:1619677448.745785:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.0:1619677448.745785:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.745786:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197849c5400] (23)--
00000800:00000200:5.0:1619677448.745786:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:26.0:1619677448.745787:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.745787:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:26.0:1619677448.745788:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.745789:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:26.0:1619677448.745790:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.745791:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.745791:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.745792:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.745796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.745804:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.745807:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.745808:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.745809:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.745811:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.745813:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.746037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677448.746044:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.746047:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677448.746048:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677448.746050:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.746053:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.746054:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.746055:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.746056:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.746057:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.746058:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.746061:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.746062:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.746065:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.746065:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.746066:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.746067:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.746068:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677448.746069:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677448.746069:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677448.746073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.746080:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.746083:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.746083:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.746085:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677448.746087:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.746088:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:5.2:1619677448.746100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677448.746108:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677448.746110:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677448.746111:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677448.746113:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677448.746116:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.746117:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677448.746118:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96710880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.746120:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.746120:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.746121:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677448.746124:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677448.746125:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677448.746127:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.746127:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:8.0:1619677448.746128:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.746129:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:8.0:1619677448.746129:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677448.746130:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677448.746131:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677448.746135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.746141:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.746142:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.746143:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.746144:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.746145:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677448.746146:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.2:1619677448.746159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.746165:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.746167:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.746168:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.746170:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.746172:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.746173:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.746174:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.746176:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.746177:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.746178:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.746181:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.746183:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.746186:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.746187:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.746188:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.746189:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.746190:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.746191:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.746192:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.746193:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.746195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.746197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677448.746202:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:17.0:1619677448.746203:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.746205:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:13.0:1619677448.746205:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:17.0:1619677448.746206:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677448.746206:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.746206:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677448.746207:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.746208:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.746210:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:5.2:1619677448.746277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677448.746285:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.746287:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677448.746288:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677448.746290:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.746292:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.746299:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.746301:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.746302:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.746303:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.746304:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677448.746307:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.746309:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677448.746312:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.746312:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:26.0:1619677448.746313:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:26.0:1619677448.746315:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:26.0:1619677448.746315:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.746317:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677448.746318:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677448.746318:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677448.746321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.746324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.746328:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.746331:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:12.0:1619677448.746332:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.746332:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:12.0:1619677448.746333:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.746334:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:12.0:1619677448.746335:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.746335:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:13.0:1619677448.746336:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:10.0:1619677448.746336:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.2:1619677448.746344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677448.746350:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.746353:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677448.746354:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677448.746356:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.746359:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.746360:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677448.746362:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.746363:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.746364:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.746365:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677448.746368:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.746370:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677448.746373:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.746374:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:17.0:1619677448.746375:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000800:00000200:17.0:1619677448.746377:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:17.0:1619677448.746377:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677448.746378:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677448.746378:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:25.2:1619677448.746383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:26.0:1619677448.746389:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.746392:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.746393:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.746393:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (23)--
00000400:00000200:26.0:1619677448.746395:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.746398:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:4.2:1619677448.746443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.746450:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.746453:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.746454:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.746455:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.746458:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.746459:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677448.746460:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.746461:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.746462:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.746463:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677448.746466:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.746468:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.746471:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.746471:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:10.0:1619677448.746472:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.746474:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:10.0:1619677448.746474:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.746475:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.746475:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.746480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:6.0:1619677448.746488:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.746490:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:22.0:1619677448.746491:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000400:00000200:6.0:1619677448.746492:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.746494:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.746496:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:21.2:1619677448.746499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:20.2:1619677448.746503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677448.746505:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.746508:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:5.0:1619677448.746508:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677448.746509:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677448.746511:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.746511:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:18.0:1619677448.746513:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.746513:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.746514:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:18.0:1619677448.746515:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677448.746516:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.746516:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.746518:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.746518:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677448.746520:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.746520:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.746521:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:5.0:1619677448.746522:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.746523:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.746524:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677448.746526:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.746528:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:5.0:1619677448.746530:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.746532:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.746532:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:18.0:1619677448.746533:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677448.746533:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677448.746535:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:5.0:1619677448.746535:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.746536:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:5.0:1619677448.746536:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:18.0:1619677448.746537:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677448.746537:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.0:1619677448.746537:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.746539:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:5.0:1619677448.746539:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.0:1619677448.746540:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.746541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:5.0:1619677448.746541:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.746544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.746549:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.746550:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:13.0:1619677448.746551:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:12.0:1619677448.746551:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677448.746552:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.746553:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.746553:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:13.0:1619677448.746555:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.746556:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.746557:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.746558:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:13.0:1619677448.746559:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:20.2:1619677448.746640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.746648:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.746651:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.746653:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677448.746655:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677448.746655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677448.746656:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.746657:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.746658:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.746659:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.746660:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.746661:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:8.0:1619677448.746662:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:30.0:1619677448.746665:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.746665:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.746666:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677448.746667:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:8.0:1619677448.746669:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.746671:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.746671:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000400:00000200:8.0:1619677448.746671:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.746672:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000400:00000200:8.0:1619677448.746673:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:30.0:1619677448.746674:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000400:00000200:8.0:1619677448.746674:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96710872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.746675:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000400:00000200:8.0:1619677448.746675:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677448.746676:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.746676:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000400:00000200:8.0:1619677448.746676:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.746677:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.2:1619677448.746680:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000400:00000200:8.0:1619677448.746686:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.746687:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677448.746688:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:8.0:1619677448.746690:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.746691:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000800:00000200:8.0:1619677448.746691:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.746691:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000400:00000200:20.0:1619677448.746692:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.746693:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.746693:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.746693:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.746694:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:5.0:1619677448.746694:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000800:00000200:8.0:1619677448.746695:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677448.746697:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:8.2:1619677448.746699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.746707:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:21.2:1619677448.746708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:17.0:1619677448.746710:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000800:00000200:1.0:1619677448.746710:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.746711:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.746713:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.746715:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:1.0:1619677448.746716:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:31.0:1619677448.746718:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677448.746721:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.746722:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677448.746724:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.746726:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677448.746727:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.746728:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.746729:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.746730:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677448.746734:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.746735:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677448.746738:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.746738:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:31.0:1619677448.746739:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:31.0:1619677448.746740:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:31.0:1619677448.746741:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677448.746742:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677448.746742:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677448.746746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.746752:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.746755:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.746756:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.746756:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.746758:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.746760:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:4.2:1619677448.747093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677448.747096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:26.0:1619677448.747100:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.747103:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:30.0:1619677448.747104:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:10.0:1619677448.747104:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.747105:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677448.747106:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677448.747108:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.747108:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.747109:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:31.0:1619677448.747110:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:26.0:1619677448.747110:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.747111:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.747112:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677448.747112:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.747114:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710869] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.747114:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.747115:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.747115:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677448.747116:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.747118:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677448.747119:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677448.747121:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.747121:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677448.747123:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.747124:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.747124:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.747125:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:30.0:1619677448.747126:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.747127:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:30.0:1619677448.747127:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:26.0:1619677448.747127:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.747127:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.747128:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:30.0:1619677448.747129:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:30.0:1619677448.747129:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:26.0:1619677448.747129:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:30.0:1619677448.747130:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:26.0:1619677448.747130:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677448.747131:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677448.747133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.2:1619677448.747135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:4.2:1619677448.747135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677448.747140:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:22.0:1619677448.747143:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:10.0:1619677448.747143:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:22.0:1619677448.747144:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.747144:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:5.0:1619677448.747144:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:10.0:1619677448.747145:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677448.747146:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.747146:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.747147:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.0:1619677448.747147:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.747148:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.747149:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000400:00000200:5.0:1619677448.747149:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.747150:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:25.2:1619677448.747151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (20)++
00000800:00000200:5.0:1619677448.747152:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:6.0:1619677448.747158:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (21)++
00000800:00000200:22.0:1619677448.747161:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.747161:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677448.747162:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1200] (22)++
00000800:00000200:6.0:1619677448.747163:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1200] (22)--
00000800:00000200:6.0:1619677448.747163:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:5.2:1619677448.747197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.747202:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677448.747203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:17.0:1619677448.747205:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.747206:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.747208:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677448.747209:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:21.2:1619677448.747211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:17.0:1619677448.747211:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.747211:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:17.0:1619677448.747212:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:18.0:1619677448.747213:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:17.0:1619677448.747214:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.747215:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.747216:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.747217:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677448.747218:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677448.747218:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677448.747218:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.747220:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:31.0:1619677448.747221:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:2.0:1619677448.747222:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96710867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.747224:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677448.747224:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677448.747224:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.747225:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.747225:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.747226:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:17.0:1619677448.747228:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.747229:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.747230:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677448.747231:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:17.0:1619677448.747231:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.747232:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000400:00000200:31.0:1619677448.747233:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96710866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.747233:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:2.0:1619677448.747233:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.747234:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.747235:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.747235:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:17.0:1619677448.747235:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.747236:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.747236:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677448.747236:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677448.747237:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.747237:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:2.0:1619677448.747237:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:5.2:1619677448.747238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.747239:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:31.0:1619677448.747240:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.747240:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:31.0:1619677448.747242:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.747242:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677448.747243:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677448.747244:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:31.0:1619677448.747245:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677448.747245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.747245:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677448.747246:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:31.0:1619677448.747247:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (23)++
00000800:00000200:18.0:1619677448.747247:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.747247:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:21.2:1619677448.747248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:31.0:1619677448.747249:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (24)--
00000800:00000200:31.0:1619677448.747249:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677448.747250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:2.0:1619677448.747250:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.747251:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:26.0:1619677448.747251:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)++
00000800:00000200:31.0:1619677448.747252:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:2.0:1619677448.747252:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.747253:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677448.747254:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (24)--
00000800:00000200:18.0:1619677448.747254:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:10.0:1619677448.747254:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (24)++
00000400:00000200:2.0:1619677448.747254:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677448.747255:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.747256:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:12.0:1619677448.747256:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:10.0:1619677448.747256:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (24)--
00000800:00000200:18.0:1619677448.747257:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:10.0:1619677448.747257:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.747257:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96710865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.747258:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.747258:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:21.2:1619677448.747259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.747259:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:2.0:1619677448.747259:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677448.747260:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.747260:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:2.0:1619677448.747261:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.747262:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:2.0:1619677448.747262:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:12.0:1619677448.747263:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.747264:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (23)--
00000800:00000200:10.0:1619677448.747264:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.747265:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:30.0:1619677448.747266:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:2.0:1619677448.747266:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.747267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000800:00000200:30.0:1619677448.747268:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:2.0:1619677448.747268:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677448.747269:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677448.747270:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677448.747270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:12.0:1619677448.747270:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:2.0:1619677448.747272:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.747273:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000800:00000200:2.0:1619677448.747273:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:2.0:1619677448.747273:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.747275:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:2.0:1619677448.747275:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.747276:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:2.0:1619677448.747276:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.747276:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:26.0:1619677448.747277:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.747277:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:2.0:1619677448.747277:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:22.0:1619677448.747278:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:10.0:1619677448.747279:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.747279:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:6.0:1619677448.747279:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:22.0:1619677448.747280:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.747281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677448.747283:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677448.747285:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677448.747286:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96710864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.747288:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.747288:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677448.747288:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:22.0:1619677448.747290:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:13.0:1619677448.747290:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.747291:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.747294:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:13.0:1619677448.747299:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:22.0:1619677448.747300:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.747302:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677448.747303:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:25.2:1619677448.747304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:22.0:1619677448.747305:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677448.747306:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:22.0:1619677448.747306:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:22.0:1619677448.747308:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:22.0:1619677448.747309:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.747310:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:22.0:1619677448.747310:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:12.0:1619677448.747311:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.747314:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677448.747315:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:9.2:1619677448.747315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:12.0:1619677448.747316:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:12.0:1619677448.747316:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.747318:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:18.0:1619677448.747323:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.747326:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:2.0:1619677448.747326:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.747327:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.747329:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.747332:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.747345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677448.747348:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.747351:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.747353:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.747354:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.747356:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.747358:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677448.747359:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.747360:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.747362:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.747363:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.747366:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.747368:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.747371:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.747372:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.747372:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.747374:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.747375:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.747376:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677448.747377:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677448.747378:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677448.747380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.2:1619677448.747383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677448.747388:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:5.0:1619677448.747388:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:30.0:1619677448.747390:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:30.0:1619677448.747391:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.747391:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:5.0:1619677448.747392:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.0:1619677448.747392:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:30.0:1619677448.747394:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.747395:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.747396:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.747471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677448.747478:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.747481:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677448.747482:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677448.747484:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.747487:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.747488:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677448.747489:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.747491:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.747492:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.747493:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677448.747496:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.747498:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677448.747500:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.747501:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:26.0:1619677448.747502:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.747504:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.747504:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.747505:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677448.747505:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677448.747520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:13.0:1619677448.747526:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.747529:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.747530:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.747531:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.747533:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.747534:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.747542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:25.2:1619677448.747547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.747549:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.747552:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677448.747554:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677448.747554:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.747556:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677448.747557:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.747557:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.747557:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:31.0:1619677448.747558:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:12.0:1619677448.747558:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.747558:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:31.0:1619677448.747559:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677448.747559:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000400:00000200:31.0:1619677448.747560:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.747561:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.747562:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677448.747565:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.747566:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677448.747569:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.747569:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:31.0:1619677448.747570:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:31.0:1619677448.747571:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.747572:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677448.747573:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.747573:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.747577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.747583:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.747585:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.747586:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.747587:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.747590:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.747591:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:20.2:1619677448.747755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:5.0:1619677448.747760:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.747763:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:5.0:1619677448.747765:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677448.747766:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.747768:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.747770:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:5.0:1619677448.747771:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:5.0:1619677448.747772:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.747773:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.747775:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:5.0:1619677448.747778:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:5.0:1619677448.747780:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:5.0:1619677448.747783:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.747783:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:5.0:1619677448.747784:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:5.0:1619677448.747786:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:5.0:1619677448.747786:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.747787:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.747788:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677448.747792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.747799:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:30.0:1619677448.747801:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:30.0:1619677448.747802:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.747804:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677448.747805:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.747807:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:5.2:1619677448.747864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.747872:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.747875:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.747875:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677448.747877:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.747879:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.747881:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677448.747882:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.747883:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.747884:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.747885:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677448.747888:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.747889:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677448.747891:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.747892:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:10.0:1619677448.747892:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:10.0:1619677448.747893:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:10.0:1619677448.747894:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.747895:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.747895:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677448.747897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:25.2:1619677448.747899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:20.0:1619677448.747899:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.747902:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677448.747904:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.747905:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:5.0:1619677448.747905:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677448.747907:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:26.0:1619677448.747907:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.747907:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677448.747908:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:10.0:1619677448.747908:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:26.0:1619677448.747910:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:20.0:1619677448.747910:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96710858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.747911:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.747912:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:20.0:1619677448.747912:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.747913:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677448.747917:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677448.747919:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677448.747922:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677448.747923:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:20.0:1619677448.747923:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:20.0:1619677448.747925:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.747926:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.747927:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.747927:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677448.747931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.747938:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.747941:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.747942:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.747943:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.747945:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:31.0:1619677448.747946:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:25.2:1619677448.747950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:30.0:1619677448.747956:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:30.0:1619677448.747959:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677448.747960:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:30.0:1619677448.747961:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71bc00] (23)--
00000800:00000200:30.0:1619677448.747961:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:30.0:1619677448.747963:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:21.2:1619677448.747983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677448.747990:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.747993:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.2:1619677448.747994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:31.0:1619677448.747995:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.747997:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.747997:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677448.747999:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677448.748000:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677448.748000:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:31.0:1619677448.748001:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.748003:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677448.748003:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.748003:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:31.0:1619677448.748004:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:6.0:1619677448.748006:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677448.748009:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677448.748009:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.748011:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:6.0:1619677448.748011:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677448.748012:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96710856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677448.748013:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.748014:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:6.0:1619677448.748014:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677448.748015:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:6.0:1619677448.748015:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.748016:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677448.748017:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:31.0:1619677448.748017:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.748018:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.748018:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:6.0:1619677448.748020:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.748022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:6.0:1619677448.748022:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:5.2:1619677448.748022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:6.0:1619677448.748024:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.748025:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:6.0:1619677448.748026:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:6.0:1619677448.748027:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:10.0:1619677448.748028:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:6.0:1619677448.748028:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.748029:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.748029:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677448.748029:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677448.748030:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000800:00000200:26.0:1619677448.748031:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.748031:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.748031:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.748033:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677448.748034:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:8.2:1619677448.748034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000400:00000200:1.0:1619677448.748035:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.748036:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000400:00000200:1.0:1619677448.748039:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.748041:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:17.0:1619677448.748043:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000400:00000200:1.0:1619677448.748043:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96710855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.748044:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677448.748045:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.748045:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.748046:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.748047:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:1.0:1619677448.748047:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.748051:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.748053:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.748054:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.748055:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677448.748055:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.748056:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.748057:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.748058:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.748059:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.748060:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.748060:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677448.748064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:17.0:1619677448.748070:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:17.0:1619677448.748072:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.748073:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.748074:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.748074:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.748076:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:20.2:1619677448.748109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:30.0:1619677448.748117:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.748120:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.748123:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.748124:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.748125:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.748126:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.748128:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.748129:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.748130:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.748131:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:4.2:1619677448.748132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:30.0:1619677448.748134:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.748136:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.748138:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.748139:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.748139:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:26.0:1619677448.748140:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677448.748141:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.748141:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.748142:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:26.0:1619677448.748142:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:30.0:1619677448.748143:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:10.0:1619677448.748143:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.748145:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.748146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000400:00000200:26.0:1619677448.748147:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.748148:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677448.748149:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96710853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.748150:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.748151:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.748152:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:12.0:1619677448.748153:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000400:00000200:26.0:1619677448.748155:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677448.748155:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.748156:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.748157:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:12.0:1619677448.748158:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.748159:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.748159:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:26.0:1619677448.748160:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000800:00000200:26.0:1619677448.748160:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:13.0:1619677448.748160:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:26.0:1619677448.748162:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:26.0:1619677448.748162:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.748163:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.748164:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677448.748168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:31.0:1619677448.748174:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000800:00000200:31.0:1619677448.748177:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:31.0:1619677448.748177:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.748179:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.748181:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:30.0:1619677448.748181:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:5.2:1619677448.748236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.748242:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.748245:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.748246:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.748247:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.748250:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.748252:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677448.748253:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.748254:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.748256:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.748257:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677448.748260:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.748261:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677448.748264:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.748265:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:2.0:1619677448.748265:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:2.0:1619677448.748267:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:2.0:1619677448.748267:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.748268:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677448.748268:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:8.2:1619677448.748273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.748281:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.748284:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.748284:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:5.0:1619677448.748286:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.748287:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.748289:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:8.2:1619677448.748293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (20)++
00000800:00000200:20.0:1619677448.748301:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)++
00000800:00000200:20.0:1619677448.748304:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677448.748305:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197927b5c00] (22)++
00000800:00000200:20.0:1619677448.748306:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197927b5c00] (23)--
00000800:00000200:20.0:1619677448.748307:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000800:00000200:5.0:1619677448.748308:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:4.2:1619677448.748313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677448.748320:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.748323:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677448.748324:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677448.748326:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.748329:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.748330:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677448.748331:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96710851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.748333:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.748334:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.748335:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677448.748338:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.748340:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677448.748343:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.748343:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:8.0:1619677448.748344:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:8.0:1619677448.748346:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:8.0:1619677448.748346:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677448.748347:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677448.748347:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.748352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.748360:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.748362:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000800:00000200:17.0:1619677448.748363:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.748364:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.748366:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.748368:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.748458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.748464:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.748466:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:21.2:1619677448.748467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:1.0:1619677448.748467:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.748469:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.748471:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.748472:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:22.0:1619677448.748473:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000400:00000200:17.0:1619677448.748473:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.748475:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677448.748476:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000400:00000200:17.0:1619677448.748476:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.748476:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:17.0:1619677448.748477:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:22.0:1619677448.748479:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.748481:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.748481:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677448.748483:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:17.0:1619677448.748483:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:22.0:1619677448.748484:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96710849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677448.748485:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677448.748487:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.748487:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.748488:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.748488:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:17.0:1619677448.748488:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:17.0:1619677448.748490:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000400:00000200:22.0:1619677448.748491:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.748492:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.748493:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677448.748493:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.748494:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677448.748496:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677448.748497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:22.0:1619677448.748497:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:22.0:1619677448.748497:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:22.0:1619677448.748499:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:22.0:1619677448.748499:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.748500:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f7d3000] (22)++
00000800:00000200:22.0:1619677448.748501:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (23)--
00000800:00000200:22.0:1619677448.748502:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:12.0:1619677448.748504:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:8.2:1619677448.748505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:12.0:1619677448.748507:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.748508:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677448.748510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (21)++
00000400:00000200:12.0:1619677448.748510:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:20.2:1619677448.748512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:13.0:1619677448.748512:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:1.0:1619677448.748512:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:12.0:1619677448.748513:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:6.0:1619677448.748515:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)++
00000800:00000200:1.0:1619677448.748515:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:17.0:1619677448.748516:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:1.0:1619677448.748516:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.748517:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f7d3000] (23)--
00000800:00000200:6.0:1619677448.748518:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:1.0:1619677448.748518:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.748519:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:1.0:1619677448.748520:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:30.0:1619677448.748521:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:8.2:1619677448.748522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (20)++
00000800:00000200:30.0:1619677448.748524:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:30.0:1619677448.748526:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.748528:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:30.0:1619677448.748528:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.748529:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677448.748530:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677448.748531:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677448.748532:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677448.748533:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677448.748533:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:30.0:1619677448.748534:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677448.748534:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd24c00] (21)++
00000400:00000200:30.0:1619677448.748535:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677448.748535:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd24c00] (22)--
00000800:00000200:17.0:1619677448.748535:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:30.0:1619677448.748539:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.748540:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677448.748543:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.748544:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:30.0:1619677448.748545:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:30.0:1619677448.748546:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:30.0:1619677448.748547:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.748548:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677448.748550:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (23)--
00000800:00000200:30.0:1619677448.748550:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:25.2:1619677448.748551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:20.2:1619677448.748556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:13.0:1619677448.748558:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:13.0:1619677448.748561:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:13.0:1619677448.748562:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.748563:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:31.0:1619677448.748564:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:13.0:1619677448.748565:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:12.0:1619677448.748565:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:31.0:1619677448.748566:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677448.748567:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677448.748568:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.748572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:12.0:1619677448.748579:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.748581:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677448.748582:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.748582:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d8a00] (23)--
00000800:00000200:12.0:1619677448.748583:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:12.0:1619677448.748584:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.748585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677448.748592:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.748595:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677448.748595:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677448.748597:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.748600:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.748601:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677448.748602:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.748604:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.748605:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.748606:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677448.748608:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.748610:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.748612:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.748613:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.748613:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.748615:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.748615:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677448.748616:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677448.748617:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.748621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.748628:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.748630:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.748631:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.748632:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.748633:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.748635:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.748798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677448.748806:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.748809:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677448.748810:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677448.748812:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.748814:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.748816:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677448.748817:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.748818:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.748819:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.748820:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677448.748824:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.748826:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.2:1619677448.748827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677448.748829:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.748829:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:20.0:1619677448.748830:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677448.748830:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:20.0:1619677448.748832:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:10.0:1619677448.748832:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:10.0:1619677448.748832:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.748833:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.748833:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:20.0:1619677448.748834:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:5.0:1619677448.748835:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677448.748837:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677448.748839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:20.0:1619677448.748839:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677448.748840:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96710845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677448.748841:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677448.748842:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677448.748843:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:5.2:1619677448.748846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:22.0:1619677448.748847:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000400:00000200:20.0:1619677448.748847:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677448.748849:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:20.0:1619677448.748849:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:22.0:1619677448.748850:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.748850:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:20.0:1619677448.748852:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.748853:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.748853:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:20.0:1619677448.748854:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:12.0:1619677448.748854:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:20.0:1619677448.748855:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:22.0:1619677448.748856:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:20.0:1619677448.748856:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677448.748856:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677448.748856:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:20.0:1619677448.748857:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:12.0:1619677448.748858:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677448.748860:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:25.2:1619677448.748861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:12.0:1619677448.748861:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.748863:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:12.0:1619677448.748864:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96710844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677448.748866:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677448.748867:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:12.0:1619677448.748867:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677448.748868:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:13.0:1619677448.748869:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.748869:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.748871:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:12.0:1619677448.748872:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677448.748874:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:12.0:1619677448.748874:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:5.2:1619677448.748874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:13.0:1619677448.748875:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:12.0:1619677448.748877:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82800] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677448.748878:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46983800] (20)++
00000800:00000200:12.0:1619677448.748878:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46983800] (21)++
00000800:00000200:12.0:1619677448.748880:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46983800] (22)--
00000800:00000200:12.0:1619677448.748881:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677448.748881:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:26.0:1619677448.748882:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677448.748882:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:26.0:1619677448.748884:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:20.2:1619677448.748884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:10.0:1619677448.748885:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677448.748886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (21)++
00000400:00000200:26.0:1619677448.748887:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.748890:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.748891:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:31.0:1619677448.748892:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000400:00000200:26.0:1619677448.748892:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96710843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677448.748892:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (22)++
00000800:00000200:31.0:1619677448.748894:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:26.0:1619677448.748894:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.748894:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.748895:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:24.0:1619677448.748895:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46983800] (23)--
00000400:00000200:24.0:1619677448.748896:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.748896:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000400:00000200:31.0:1619677448.748897:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677448.748898:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.748899:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.748899:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:26.0:1619677448.748899:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677448.748900:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000400:00000200:31.0:1619677448.748901:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:26.0:1619677448.748901:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:31.0:1619677448.748902:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96710842] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.748903:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677448.748903:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.748903:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000400:00000200:31.0:1619677448.748904:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677448.748904:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000400:00000200:31.0:1619677448.748905:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677448.748905:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.748906:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.748906:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677448.748907:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677448.748907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46983800] (20)++
00000400:00000200:31.0:1619677448.748908:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.748910:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:25.2:1619677448.748911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:31.0:1619677448.748913:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978679ed00] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.748913:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46983800] (21)++
00000800:00000200:31.0:1619677448.748914:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197927b5c00] (20)++
00000800:00000200:31.0:1619677448.748915:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197927b5c00] (21)++
00000800:00000200:31.0:1619677448.748916:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197927b5c00] (22)--
00000800:00000200:8.0:1619677448.748916:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:31.0:1619677448.748917:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:24.0:1619677448.748917:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (22)--
00000800:00000200:10.0:1619677448.748917:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:8.0:1619677448.748917:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46983800] (22)++
00000800:00000200:31.0:1619677448.748918:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:8.0:1619677448.748918:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46983800] (22)--
00000800:00000200:8.0:1619677448.748918:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46983800] (21)--
00000800:00000200:31.0:1619677448.748919:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:10.0:1619677448.748919:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.748920:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.748921:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000400:00000200:10.0:1619677448.748922:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.748923:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:8.2:1619677448.748923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197927b5c00] (21)++
00000800:00000200:5.0:1619677448.748930:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)++
00000800:00000200:5.0:1619677448.748933:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197927b5c00] (23)--
00000400:00000200:5.0:1619677448.748934:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.748936:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (22)--
00000400:00000200:5.0:1619677448.748936:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.748938:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197927b5c00] (21)--
00000800:00000200:5.2:1619677448.749025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.749030:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.749033:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.749033:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.749036:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.749038:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.749039:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677448.749040:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.749042:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.749043:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.749044:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677448.749047:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.749049:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677448.749050:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.749051:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:1.0:1619677448.749052:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:1.0:1619677448.749053:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:1.0:1619677448.749054:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.749055:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.749056:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.749057:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:21.2:1619677448.749059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:8.2:1619677448.749059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.749062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677448.749064:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.749067:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:17.0:1619677448.749068:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:2.0:1619677448.749068:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:17.0:1619677448.749069:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000400:00000200:17.0:1619677448.749069:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677448.749069:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:18.0:1619677448.749070:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.749070:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677448.749070:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:21.2:1619677448.749072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:18.0:1619677448.749072:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.749072:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.749072:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:18.0:1619677448.749073:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:17.0:1619677448.749073:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:1.0:1619677448.749073:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:18.0:1619677448.749074:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.749076:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677448.749077:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677448.749078:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677448.749079:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677448.749081:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.749082:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:18.0:1619677448.749083:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:30.0:1619677448.749085:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677448.749085:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.749085:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:31.0:1619677448.749086:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677448.749086:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.749087:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000400:00000200:30.0:1619677448.749088:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.749088:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677448.749089:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:30.0:1619677448.749090:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677448.749091:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96710839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677448.749091:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:30.0:1619677448.749092:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.749093:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677448.749093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000400:00000200:30.0:1619677448.749094:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677448.749098:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.749100:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677448.749100:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:30.0:1619677448.749102:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.749102:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:30.0:1619677448.749103:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000400:00000200:2.0:1619677448.749103:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.749104:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:18.0:1619677448.749104:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:30.0:1619677448.749105:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:2.0:1619677448.749105:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.749106:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677448.749107:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677448.749107:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:30.0:1619677448.749108:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677448.749111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.749117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1200] (20)++
00000800:00000200:13.0:1619677448.749118:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.749120:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.749121:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.749123:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000400:00000200:13.0:1619677448.749123:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.749124:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.749125:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:17.0:1619677448.749126:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:1.0:1619677448.749126:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:17.0:1619677448.749128:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.749130:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.749132:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:17.0:1619677448.749133:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.749134:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.749136:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.749136:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:17.0:1619677448.749140:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.749142:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677448.749145:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3b00] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.749145:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919784128a00] (20)++
00000800:00000200:17.0:1619677448.749146:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919784128a00] (21)++
00000800:00000200:17.0:1619677448.749148:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919784128a00] (22)--
00000800:00000200:17.0:1619677448.749148:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677448.749149:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:17.0:1619677448.749149:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:8.2:1619677448.749154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919784128a00] (21)++
00000800:00000200:20.2:1619677448.749160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677448.749165:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.749167:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919784128a00] (23)--
00000800:00000200:5.0:1619677448.749167:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:20.0:1619677448.749168:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.0:1619677448.749168:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677448.749169:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:5.0:1619677448.749171:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677448.749172:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000800:00000200:20.0:1619677448.749173:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919784128a00] (21)++
00000800:00000200:20.0:1619677448.749174:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000400:00000200:5.0:1619677448.749174:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.749175:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919784128a00] (22)++
00000800:00000200:20.0:1619677448.749176:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919784128a00] (23)--
00000800:00000200:20.0:1619677448.749176:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (22)--
00000400:00000200:5.0:1619677448.749176:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:20.0:1619677448.749177:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:5.0:1619677448.749177:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710837] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.749178:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919784128a00] (21)--
00000400:00000200:5.0:1619677448.749179:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:5.0:1619677448.749179:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:5.0:1619677448.749180:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:5.0:1619677448.749184:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:5.0:1619677448.749185:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:5.0:1619677448.749188:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:5.0:1619677448.749189:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:5.0:1619677448.749189:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:5.0:1619677448.749191:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:5.0:1619677448.749192:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:5.0:1619677448.749192:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:5.0:1619677448.749193:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677448.749197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.749205:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:18.0:1619677448.749208:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.749209:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.749209:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.749212:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.749214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.749217:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.749219:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:18.0:1619677448.749221:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.749222:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.749222:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.749223:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.749223:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:4.2:1619677448.749247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.749254:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.749257:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.749258:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.749260:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.749261:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.749262:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677448.749264:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.749265:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677448.749265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000400:00000200:26.0:1619677448.749266:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.749267:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677448.749270:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.749271:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677448.749271:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:22.0:1619677448.749274:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.749274:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:26.0:1619677448.749275:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.749276:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:26.0:1619677448.749276:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000400:00000200:6.0:1619677448.749277:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.749278:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:26.0:1619677448.749278:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.749279:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677448.749279:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.749280:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677448.749280:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677448.749281:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96710835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677448.749282:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677448.749284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:6.0:1619677448.749284:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677448.749284:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677448.749289:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.749291:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677448.749293:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:22.0:1619677448.749298:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000800:00000200:6.0:1619677448.749299:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3aedb1c80] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677448.749300:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.749300:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aef85400] (20)++
00000800:00000200:6.0:1619677448.749301:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aef85400] (21)++
00000400:00000200:22.0:1619677448.749302:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677448.749303:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aef85400] (22)--
00000800:00000200:22.0:1619677448.749304:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.749304:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677448.749305:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.749305:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:22.0:1619677448.749306:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:25.2:1619677448.749310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aef85400] (21)++
00000800:00000200:10.0:1619677448.749317:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)++
00000800:00000200:26.0:1619677448.749320:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (22)--
00000800:00000200:10.0:1619677448.749320:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aef85400] (23)--
00000400:00000200:10.0:1619677448.749321:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.749323:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677448.749326:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aef85400] (21)--
00000800:00000200:21.2:1619677448.749401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677448.749407:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.749410:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677448.749412:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677448.749413:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677448.749415:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.749417:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677448.749418:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.749419:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.749420:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.749421:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677448.749424:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677448.749426:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677448.749427:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.749428:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:2.0:1619677448.749429:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.749430:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:2.0:1619677448.749431:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677448.749432:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677448.749432:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677448.749437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.749444:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:18.0:1619677448.749446:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.749446:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.749447:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000400:00000200:18.0:1619677448.749448:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677448.749450:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:20.2:1619677448.749463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677448.749470:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.749474:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677448.749475:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677448.749476:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677448.749480:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.749481:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.749483:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.749484:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.749485:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.749486:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.749489:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677448.749490:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.749493:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.749494:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.749495:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.749497:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.749497:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677448.749498:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677448.749499:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677448.749503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.749510:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.749513:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.749514:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:2.0:1619677448.749514:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.749516:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677448.749518:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.749548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.749554:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.749558:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677448.749559:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677448.749561:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.749563:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.749565:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677448.749566:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.749567:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.749568:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.749569:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677448.749573:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.749574:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677448.749577:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.749578:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:1.0:1619677448.749579:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:1.0:1619677448.749581:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:1.0:1619677448.749581:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.749582:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.749582:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.749586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.749594:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.749598:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000400:00000200:24.0:1619677448.749599:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677448.749599:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000400:00000200:24.0:1619677448.749601:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677448.749605:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:5.2:1619677448.749625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677448.749632:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.749635:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.749636:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.749638:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677448.749641:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.749642:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677448.749643:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.749645:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.749646:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.749646:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677448.749649:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.749651:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677448.749653:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.749654:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (20)++
00000800:00000200:10.0:1619677448.749655:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:10.0:1619677448.749656:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:10.0:1619677448.749657:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.749657:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.749658:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677448.749662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d8a00] (21)++
00000800:00000200:12.0:1619677448.749670:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)++
00000800:00000200:12.0:1619677448.749673:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d8a00] (23)--
00000400:00000200:12.0:1619677448.749673:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.749675:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.749677:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (22)--
00000800:00000200:13.0:1619677448.749678:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d8a00] (21)--
00000800:00000200:21.2:1619677448.749724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.2:1619677448.749724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677448.749731:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:31.0:1619677448.749734:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.749734:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677448.749734:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677448.749736:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.749737:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:18.0:1619677448.749738:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.749739:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677448.749740:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:30.0:1619677448.749741:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677448.749741:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710830] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677448.749742:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677448.749743:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.749744:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677448.749744:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.749746:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677448.749746:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677448.749748:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.749749:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677448.749750:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677448.749750:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677448.749753:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:31.0:1619677448.749755:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.749757:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.749758:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677448.749759:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:18.0:1619677448.749760:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:31.0:1619677448.749761:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.749762:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.749762:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.749762:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:18.0:1619677448.749762:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.749763:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:31.0:1619677448.749764:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.749764:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.749764:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:31.0:1619677448.749765:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677448.749766:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677448.749767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:9.2:1619677448.749770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:2.0:1619677448.749776:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.749777:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:2.0:1619677448.749778:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677448.749779:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:4.2:1619677448.749780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677448.749780:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677448.749781:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000400:00000200:18.0:1619677448.749781:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.749782:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000400:00000200:18.0:1619677448.749783:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.749783:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:2.0:1619677448.749784:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.749786:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:26.0:1619677448.749787:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.749790:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.749791:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.749793:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.749795:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.749796:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677448.749798:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.749799:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.749800:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.749801:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677448.749804:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.749806:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677448.749809:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.749810:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:26.0:1619677448.749810:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:26.0:1619677448.749813:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:26.0:1619677448.749813:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.749814:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677448.749815:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677448.749816:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677448.749818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:4.2:1619677448.749821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677448.749826:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:10.0:1619677448.749828:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:2.0:1619677448.749828:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000800:00000200:18.0:1619677448.749829:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:10.0:1619677448.749830:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:2.0:1619677448.749830:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.749831:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.749832:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:2.0:1619677448.749832:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677448.749835:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:5.2:1619677448.749963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.749969:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.749972:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.749972:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.749974:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.749977:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.749979:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677448.749980:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.749981:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.749982:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.749983:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677448.749987:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.749988:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677448.749992:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e7ac82880] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.749992:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e46980c00] (20)++
00000800:00000200:17.0:1619677448.749993:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e46980c00] (21)++
00000800:00000200:17.0:1619677448.749995:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e46980c00] (22)--
00000800:00000200:17.0:1619677448.749995:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.749996:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677448.749996:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677448.750000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (21)++
00000800:00000200:8.0:1619677448.750007:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (22)++
00000800:00000200:8.0:1619677448.750010:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e46980c00] (23)--
00000800:00000200:24.0:1619677448.750011:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (23)--
00000400:00000200:8.0:1619677448.750011:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.750013:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.750015:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:9.2:1619677448.750022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e46980c00] (20)++
00000800:00000200:24.0:1619677448.750028:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e46980c00] (21)++
00000800:00000200:24.0:1619677448.750031:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677448.750031:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.750032:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e46980c00] (22)++
00000800:00000200:24.0:1619677448.750033:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e46980c00] (22)--
00000800:00000200:24.0:1619677448.750033:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e46980c00] (21)--
00000800:00000200:21.2:1619677448.750095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677448.750102:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.750104:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677448.750105:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677448.750107:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677448.750109:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677448.750110:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677448.750111:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96710826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.750113:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.750114:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.750115:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677448.750117:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677448.750119:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677448.750122:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fe00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677448.750123:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2984200] (20)++
00000800:00000200:8.0:1619677448.750124:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2984200] (21)++
00000800:00000200:8.0:1619677448.750126:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2984200] (22)--
00000800:00000200:8.0:1619677448.750126:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677448.750127:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677448.750129:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677448.750129:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677448.750132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (21)++
00000800:00000200:21.2:1619677448.750135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:18.0:1619677448.750139:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (22)++
00000800:00000200:24.0:1619677448.750141:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:18.0:1619677448.750142:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2984200] (23)--
00000400:00000200:18.0:1619677448.750143:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.750143:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:24.0:1619677448.750144:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677448.750144:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:18.0:1619677448.750145:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.0:1619677448.750145:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:18.0:1619677448.750148:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:9.2:1619677448.750150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2984200] (20)++
00000800:00000200:2.0:1619677448.750157:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2984200] (21)++
00000800:00000200:2.0:1619677448.750159:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677448.750160:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.750160:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2984200] (22)++
00000800:00000200:2.0:1619677448.750161:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2984200] (22)--
00000800:00000200:2.0:1619677448.750162:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2984200] (21)--
00000800:00000200:21.2:1619677448.750205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677448.750213:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.750215:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677448.750216:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677448.750218:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.750221:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.750222:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677448.750223:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.750224:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.750225:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.750226:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677448.750229:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.750231:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.750232:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0494600] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.750233:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aea38600] (20)++
00000800:00000200:26.0:1619677448.750234:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aea38600] (21)++
00000800:00000200:26.0:1619677448.750235:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.750236:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677448.750237:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677448.750238:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677448.750238:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:25.2:1619677448.750241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (21)++
00000800:00000200:21.2:1619677448.750245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:4.2:1619677448.750246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:10.0:1619677448.750250:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.750252:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aea38600] (23)--
00000400:00000200:10.0:1619677448.750252:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.750253:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677448.750254:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:10.0:1619677448.750254:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.750255:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677448.750256:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:24.0:1619677448.750256:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:10.0:1619677448.750256:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.750257:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:8.0:1619677448.750257:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:10.0:1619677448.750258:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:24.0:1619677448.750259:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.750260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aea38600] (20)++
00000400:00000200:24.0:1619677448.750262:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677448.750263:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677448.750264:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96710824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.750266:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)++
00000400:00000200:24.0:1619677448.750266:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677448.750266:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677448.750267:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.750268:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677448.750269:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)++
00000800:00000200:10.0:1619677448.750269:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (22)--
00000800:00000200:26.0:1619677448.750270:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aea38600] (22)--
00000400:00000200:24.0:1619677448.750270:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.750271:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aea38600] (21)--
00000800:00000200:24.0:1619677448.750272:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677448.750275:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677448.750276:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:24.0:1619677448.750276:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:24.0:1619677448.750278:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:24.0:1619677448.750278:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677448.750279:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677448.750280:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677448.750284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:20.2:1619677448.750287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677448.750292:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:30.0:1619677448.750295:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677448.750298:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (23)--
00000800:00000200:30.0:1619677448.750301:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000800:00000200:1.0:1619677448.750301:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (22)--
00000800:00000200:31.0:1619677448.750302:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:1.0:1619677448.750302:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.750304:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.750305:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677448.750307:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000400:00000200:30.0:1619677448.750308:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677448.750309:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677448.750310:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.750312:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.750312:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.750314:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677448.750317:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.750319:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677448.750321:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056b100] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.750322:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71b400] (20)++
00000800:00000200:30.0:1619677448.750322:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71b400] (21)++
00000800:00000200:30.0:1619677448.750324:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71b400] (22)--
00000800:00000200:30.0:1619677448.750324:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.750325:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.750326:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:25.2:1619677448.750330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71b400] (21)++
00000800:00000200:31.0:1619677448.750337:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)++
00000800:00000200:31.0:1619677448.750339:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71b400] (23)--
00000400:00000200:31.0:1619677448.750340:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677448.750342:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677448.750343:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (22)--
00000800:00000200:31.0:1619677448.750345:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71b400] (21)--
00000800:00000200:21.2:1619677448.750457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f7d3000] (20)++
00000800:00000200:6.0:1619677448.750463:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:5.2:1619677448.750464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:6.0:1619677448.750467:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677448.750468:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677448.750469:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.750472:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:6.0:1619677448.750473:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677448.750475:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:10.0:1619677448.750476:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:6.0:1619677448.750476:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677448.750477:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677448.750478:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677448.750479:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.750480:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677448.750480:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.750482:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.750484:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:6.0:1619677448.750484:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677448.750485:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96710821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.750486:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.750487:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677448.750487:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677448.750488:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.2:1619677448.750490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:6.0:1619677448.750490:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abd84f8780] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677448.750491:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (20)++
00000400:00000200:10.0:1619677448.750492:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677448.750492:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71bc00] (21)++
00000800:00000200:6.0:1619677448.750493:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71bc00] (22)--
00000800:00000200:20.0:1619677448.750494:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677448.750494:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:10.0:1619677448.750495:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677448.750495:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677448.750495:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677448.750497:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:10.0:1619677448.750497:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.750498:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:25.2:1619677448.750499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71bc00] (21)++
00000400:00000200:20.0:1619677448.750499:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677448.750499:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.0:1619677448.750500:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677448.750501:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.750501:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:20.0:1619677448.750502:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677448.750502:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677448.750503:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000400:00000200:20.0:1619677448.750504:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:4.2:1619677448.750504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:25.2:1619677448.750506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:20.0:1619677448.750506:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96710820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677448.750507:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)++
00000400:00000200:20.0:1619677448.750507:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677448.750507:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:20.0:1619677448.750508:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677448.750508:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.2:1619677448.750509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677448.750510:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71bc00] (23)--
00000400:00000200:20.0:1619677448.750510:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.750510:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:30.0:1619677448.750511:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.750512:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677448.750513:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (22)--
00000400:00000200:30.0:1619677448.750513:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677448.750514:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:12.0:1619677448.750514:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000400:00000200:10.0:1619677448.750514:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.750515:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:20.0:1619677448.750515:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.750516:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:12.0:1619677448.750516:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:26.0:1619677448.750517:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:12.0:1619677448.750517:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677448.750517:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.750518:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71bc00] (21)--
00000800:00000200:20.0:1619677448.750518:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:12.0:1619677448.750519:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:20.0:1619677448.750520:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677448.750520:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:20.0:1619677448.750521:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000400:00000200:10.0:1619677448.750521:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96710819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677448.750522:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:13.0:1619677448.750522:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:10.0:1619677448.750522:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677448.750523:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:10.0:1619677448.750523:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677448.750524:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:10.0:1619677448.750524:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677448.750525:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677448.750525:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677448.750526:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:10.0:1619677448.750528:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677448.750530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:10.0:1619677448.750530:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677448.750533:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780400] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.750534:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1800] (20)++
00000800:00000200:10.0:1619677448.750534:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1800] (21)++
00000800:00000200:10.0:1619677448.750536:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1800] (22)--
00000800:00000200:22.0:1619677448.750537:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.750537:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677448.750537:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677448.750538:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677448.750540:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:5.2:1619677448.750540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:22.0:1619677448.750542:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.750542:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (22)--
00000800:00000200:25.2:1619677448.750543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (21)++
00000400:00000200:22.0:1619677448.750544:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.750546:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677448.750547:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:22.0:1619677448.750549:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (22)++
00000800:00000200:17.0:1619677448.750549:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.750550:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677448.750551:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1800] (23)--
00000400:00000200:22.0:1619677448.750551:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677448.750551:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (23)--
00000400:00000200:17.0:1619677448.750552:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677448.750553:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:17.0:1619677448.750555:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677448.750556:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000400:00000200:17.0:1619677448.750556:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677448.750557:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.750559:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.750560:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.750561:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:25.2:1619677448.750564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1800] (20)++
00000400:00000200:17.0:1619677448.750564:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.750566:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677448.750567:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fc2778e80] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.750568:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd27e00] (20)++
00000800:00000200:17.0:1619677448.750569:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd27e00] (21)++
00000800:00000200:17.0:1619677448.750570:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd27e00] (22)--
00000800:00000200:17.0:1619677448.750570:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.750571:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677448.750571:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1800] (21)++
00000800:00000200:17.0:1619677448.750572:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677448.750574:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677448.750575:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.750575:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b884e1800] (22)++
00000800:00000200:6.0:1619677448.750576:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b884e1800] (22)--
00000800:00000200:6.0:1619677448.750576:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1800] (21)--
00000800:00000200:8.2:1619677448.750577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd27e00] (21)++
00000800:00000200:1.0:1619677448.750584:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd27e00] (22)++
00000800:00000200:1.0:1619677448.750586:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd27e00] (23)--
00000400:00000200:1.0:1619677448.750587:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677448.750588:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (22)--
00000400:00000200:1.0:1619677448.750589:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677448.750591:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd27e00] (21)--
00000800:00000200:5.2:1619677448.750844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677448.750849:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.750852:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677448.750852:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677448.750854:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677448.750855:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677448.750857:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677448.750858:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677448.750859:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677448.750860:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677448.750861:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677448.750864:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677448.750865:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677448.750868:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677448.750869:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:17.0:1619677448.750869:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:17.0:1619677448.750871:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:17.0:1619677448.750872:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.750873:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677448.750874:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677448.750875:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677448.750877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:5.2:1619677448.750880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.750885:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677448.750886:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:21.2:1619677448.750887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677448.750887:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677448.750888:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:1.0:1619677448.750888:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677448.750893:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.750894:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677448.750895:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677448.750895:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:31.0:1619677448.750897:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:13.0:1619677448.750897:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:31.0:1619677448.750899:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677448.750900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.750901:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677448.750901:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677448.750902:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:31.0:1619677448.750903:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677448.750904:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677448.750904:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:31.0:1619677448.750905:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677448.750906:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677448.750906:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.750907:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677448.750908:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:13.0:1619677448.750908:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.750909:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.750910:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:31.0:1619677448.750911:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677448.750913:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677448.750916:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1080] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677448.750917:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2987800] (20)++
00000800:00000200:31.0:1619677448.750917:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2987800] (21)++
00000800:00000200:31.0:1619677448.750919:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2987800] (22)--
00000800:00000200:31.0:1619677448.750919:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677448.750920:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677448.750922:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677448.750923:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677448.750925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:21.2:1619677448.750928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677448.750934:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (22)++
00000800:00000200:30.0:1619677448.750936:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:18.0:1619677448.750937:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2987800] (23)--
00000800:00000200:30.0:1619677448.750938:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677448.750939:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677448.750939:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.750939:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:30.0:1619677448.750940:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:18.0:1619677448.750941:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:9.2:1619677448.750944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.750947:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.750949:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2987800] (21)++
00000800:00000200:18.0:1619677448.750951:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677448.750952:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2987800] (22)++
00000800:00000200:2.0:1619677448.750952:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.750953:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2987800] (22)--
00000800:00000200:18.0:1619677448.750954:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2987800] (21)--
00000800:00000200:4.2:1619677448.751045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677448.751052:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.751055:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.751056:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677448.751058:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.751061:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.751062:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677448.751063:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.751065:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.751066:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.751066:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.2:1619677448.751066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:26.0:1619677448.751070:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.751072:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677448.751074:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677448.751075:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.751076:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677448.751077:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.751077:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000400:00000200:10.0:1619677448.751078:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677448.751080:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000400:00000200:10.0:1619677448.751080:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677448.751081:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:10.0:1619677448.751081:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:26.0:1619677448.751082:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677448.751082:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677448.751083:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677448.751083:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96710814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677448.751084:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677448.751085:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677448.751086:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:25.2:1619677448.751087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000400:00000200:10.0:1619677448.751089:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677448.751090:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677448.751092:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677448.751093:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:10.0:1619677448.751093:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:22.0:1619677448.751094:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b884e1200] (22)++
00000800:00000200:10.0:1619677448.751094:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:10.0:1619677448.751095:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677448.751095:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677448.751096:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677448.751097:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b884e1200] (23)--
00000800:00000200:6.0:1619677448.751097:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (23)--
00000400:00000200:22.0:1619677448.751098:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677448.751099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:25.2:1619677448.751100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000400:00000200:22.0:1619677448.751100:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677448.751103:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b884e1200] (21)--
00000800:00000200:30.0:1619677448.751107:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677448.751107:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:12.0:1619677448.751109:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000800:00000200:30.0:1619677448.751110:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:12.0:1619677448.751110:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677448.751112:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:30.0:1619677448.751113:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677448.751114:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:12.0:1619677448.751115:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000400:00000200:30.0:1619677448.751116:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677448.751116:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000400:00000200:30.0:1619677448.751117:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677448.751119:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677448.751120:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677448.751121:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677448.751122:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677448.751125:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677448.751127:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677448.751130:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9b80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677448.751131:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197849c5400] (20)++
00000800:00000200:30.0:1619677448.751132:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197849c5400] (21)++
00000800:00000200:30.0:1619677448.751133:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197849c5400] (22)--
00000800:00000200:30.0:1619677448.751134:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677448.751135:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000800:00000200:30.0:1619677448.751135:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:8.2:1619677448.751140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197849c5400] (21)++
00000800:00000200:5.0:1619677448.751149:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197849c5400] (22)++
00000800:00000200:5.0:1619677448.751151:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197849c5400] (23)--
00000400:00000200:5.0:1619677448.751152:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677448.751154:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (22)--
00000400:00000200:5.0:1619677448.751154:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.0:1619677448.751157:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197849c5400] (21)--
00000800:00000200:5.2:1619677448.751195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677448.751200:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.751203:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677448.751204:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677448.751206:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677448.751208:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677448.751210:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677448.751211:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677448.751212:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677448.751213:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677448.751214:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677448.751218:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677448.751219:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677448.751221:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677448.751222:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:2.0:1619677448.751223:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:2.0:1619677448.751224:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:2.0:1619677448.751225:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677448.751226:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677448.751227:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677448.751228:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:8.2:1619677448.751231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918febd24c00] (21)++
00000800:00000200:5.2:1619677448.751233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677448.751238:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:17.0:1619677448.751238:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd24c00] (22)++
00000800:00000200:18.0:1619677448.751240:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:17.0:1619677448.751240:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd24c00] (23)--
00000800:00000200:18.0:1619677448.751241:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677448.751241:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677448.751242:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677448.751242:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (22)--
00000400:00000200:17.0:1619677448.751244:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677448.751246:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd24c00] (21)--
00000800:00000200:5.2:1619677448.751252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677448.751257:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677448.751260:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.751260:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677448.751262:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677448.751264:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677448.751265:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677448.751267:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96710811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677448.751268:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677448.751269:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677448.751270:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677448.751272:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677448.751274:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677448.751277:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537280] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677448.751278:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (20)++
00000800:00000200:1.0:1619677448.751279:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:1.0:1619677448.751280:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:1.0:1619677448.751281:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677448.751281:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677448.751282:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677448.751286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9200] (21)++
00000800:00000200:13.0:1619677448.751300:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)++
00000800:00000200:13.0:1619677448.751304:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9200] (23)--
00000400:00000200:13.0:1619677448.751306:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677448.751308:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677448.751308:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (22)--
00000800:00000200:13.0:1619677448.751312:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9200] (21)--
00000800:00000200:4.2:1619677448.751396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677448.751400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677448.751404:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.751407:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677448.751408:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677448.751408:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677448.751410:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677448.751411:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:26.0:1619677448.751412:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677448.751413:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:26.0:1619677448.751414:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677448.751414:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677448.751415:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96710810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677448.751417:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677448.751417:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677448.751418:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677448.751419:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677448.751419:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677448.751420:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96710809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677448.751422:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677448.751423:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677448.751423:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677448.751424:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.751425:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:8.0:1619677448.751427:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677448.751428:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b904fe400] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.751429:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b884e1200] (20)++
00000800:00000200:26.0:1619677448.751429:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677448.751429:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677448.751431:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b884e1200] (22)--
00000800:00000200:26.0:1619677448.751431:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677448.751432:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677448.751432:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918febc9f080] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677448.751433:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.0:1619677448.751433:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd24c00] (20)++
00000800:00000200:8.0:1619677448.751433:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd24c00] (21)++
00000800:00000200:8.0:1619677448.751435:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd24c00] (22)--
00000800:00000200:8.0:1619677448.751435:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.2:1619677448.751437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b884e1200] (21)++
00000800:00000200:8.0:1619677448.751437:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677448.751438:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677448.751438:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000100:8.0:1619677451.585582:0:13335:0:(o2iblnd_cb.c:2085:kiblnd_close_conn_locked()) Closing conn to 10.0.10.52@o2ib7: error 0(waiting)
00000400:00000100:5.0:1619677451.585900:0:19935:0:(lib-msg.c:698:lnet_attempt_msg_resend()) msg 10.51.5.10@o2ib3->10.0.10.52@o2ib7 exceeded retry count 0
00000800:00000100:8.0:1619677451.718082:0:13335:0:(o2iblnd_cb.c:2085:kiblnd_close_conn_locked()) Closing conn to 10.0.10.103@o2ib7: error 0(waiting)
00000400:00000100:12.0:1619677451.718386:0:19943:0:(lib-msg.c:698:lnet_attempt_msg_resend()) msg 10.51.14.10@o2ib3->10.0.10.103@o2ib7 exceeded retry count 0
00000800:00000100:8.0:1619677451.719057:0:13335:0:(o2iblnd_cb.c:2085:kiblnd_close_conn_locked()) Closing conn to 10.0.10.102@o2ib7: error 0(waiting)
00000400:00000100:18.0:1619677451.720026:0:19933:0:(lib-msg.c:698:lnet_attempt_msg_resend()) msg 10.51.14.10@o2ib3->10.0.10.102@o2ib7 exceeded retry count 0
00000800:00020000:7.0:1619677453.685327:0:19929:0:(o2iblnd_cb.c:3359:kiblnd_check_txs_locked()) Timed out tx: active_txs, 0 seconds
00000800:00020000:7.0:1619677453.685332:0:19929:0:(o2iblnd_cb.c:3434:kiblnd_check_conns()) Timed out RDMA with 10.51.3.10@o2ib3 (106): c: 8, oc: 0, rc: 8
00000800:00000100:7.0:1619677453.685335:0:19929:0:(o2iblnd_cb.c:2085:kiblnd_close_conn_locked()) Closing conn to 10.51.3.10@o2ib3: error -110(waiting)
00000800:00020000:7.0:1619677455.685344:0:19929:0:(o2iblnd_cb.c:3359:kiblnd_check_txs_locked()) Timed out tx: active_txs, 0 seconds
00000800:00020000:7.0:1619677455.685346:0:19929:0:(o2iblnd_cb.c:3434:kiblnd_check_conns()) Timed out RDMA with 10.51.1.14@o2ib3 (106): c: 8, oc: 0, rc: 8
00000800:00000100:7.0:1619677455.685347:0:19929:0:(o2iblnd_cb.c:2085:kiblnd_close_conn_locked()) Closing conn to 10.51.1.14@o2ib3: error -110(waiting)
Debug log: 583978 lines, 583978 kept, 0 dropped, 0 bad.
